From df0090e32a5262897fe35271788da98016abc5f1 Mon Sep 17 00:00:00 2001 From: Arceliar Date: Sat, 3 Aug 2019 21:46:18 -0500 Subject: [PATCH 01/51] Add per-session read/write workers, work in progress, they still unfortunately need to take a mutex for safety --- src/yggdrasil/conn.go | 59 ++++-------------------- src/yggdrasil/dialer.go | 1 + src/yggdrasil/router.go | 2 +- src/yggdrasil/session.go | 98 +++++++++++++++++++++++++++++++++++++++- 4 files changed, 106 insertions(+), 54 deletions(-) diff --git a/src/yggdrasil/conn.go b/src/yggdrasil/conn.go index d1cb7609..e47ea2fb 100644 --- a/src/yggdrasil/conn.go +++ b/src/yggdrasil/conn.go @@ -137,7 +137,6 @@ func (c *Conn) Read(b []byte) (int, error) { sinfo := c.session cancel := c.getDeadlineCancellation(&c.readDeadline) defer cancel.Cancel(nil) - var bs []byte for { // Wait for some traffic to come through from the session select { @@ -147,54 +146,25 @@ func (c *Conn) Read(b []byte) (int, error) { } else { return 0, ConnError{errors.New("session closed"), false, false, true, 0} } - case p, ok := <-sinfo.recv: - // If the session is closed then do nothing - if !ok { - return 0, ConnError{errors.New("session closed"), false, false, true, 0} - } + case bs := <-sinfo.recv: var err error - sessionFunc := func() { - defer util.PutBytes(p.Payload) - // If the nonce is bad then drop the packet and return an error - if !sinfo.nonceIsOK(&p.Nonce) { - err = ConnError{errors.New("packet dropped due to invalid nonce"), false, true, false, 0} - return - } - // Decrypt the packet - var isOK bool - bs, isOK = crypto.BoxOpen(&sinfo.sharedSesKey, p.Payload, &p.Nonce) - // Check if we were unable to decrypt the packet for some reason and - // return an error if we couldn't - if !isOK { - err = ConnError{errors.New("packet dropped due to decryption failure"), false, true, false, 0} - return - } - // Update the session - sinfo.updateNonce(&p.Nonce) - sinfo.time = time.Now() - sinfo.bytesRecvd += uint64(len(bs)) - } - sinfo.doFunc(sessionFunc) - // Something went wrong in the session worker so abort - if err != nil { - if ce, ok := err.(*ConnError); ok && ce.Temporary() { - continue - } - return 0, err + n := len(bs) + if len(bs) > len(b) { + n = len(b) + err = ConnError{errors.New("read buffer too small for entire packet"), false, true, false, 0} } // Copy results to the output slice and clean up copy(b, bs) util.PutBytes(bs) // If we've reached this point then everything went to plan, return the // number of bytes we populated back into the given slice - return len(bs), nil + return n, err } } } func (c *Conn) Write(b []byte) (bytesWritten int, err error) { sinfo := c.session - var packet []byte written := len(b) sessionFunc := func() { // Does the packet exceed the permitted size for the session? @@ -202,18 +172,6 @@ func (c *Conn) Write(b []byte) (bytesWritten int, err error) { written, err = 0, ConnError{errors.New("packet too big"), true, false, false, int(sinfo.getMTU())} return } - // Encrypt the packet - payload, nonce := crypto.BoxSeal(&sinfo.sharedSesKey, b, &sinfo.myNonce) - defer util.PutBytes(payload) - // Construct the wire packet to send to the router - p := wire_trafficPacket{ - Coords: sinfo.coords, - Handle: sinfo.theirHandle, - Nonce: *nonce, - Payload: payload, - } - packet = p.encode() - sinfo.bytesSent += uint64(len(b)) // The rest of this work is session keep-alive traffic doSearch := func() { routerWork := func() { @@ -244,11 +202,10 @@ func (c *Conn) Write(b []byte) (bytesWritten int, err error) { } } sinfo.doFunc(sessionFunc) - // Give the packet to the router if written > 0 { - sinfo.core.router.out(packet) + bs := append(util.GetBytes(), b...) + sinfo.send <- bs } - // Finally return the number of bytes we wrote return written, err } diff --git a/src/yggdrasil/dialer.go b/src/yggdrasil/dialer.go index 6b24cfb4..6ce2e8ac 100644 --- a/src/yggdrasil/dialer.go +++ b/src/yggdrasil/dialer.go @@ -69,6 +69,7 @@ func (d *Dialer) DialByNodeIDandMask(nodeID, nodeMask *crypto.NodeID) (*Conn, er defer t.Stop() select { case <-conn.session.init: + conn.session.startWorkers(conn.cancel) return conn, nil case <-t.C: conn.Close() diff --git a/src/yggdrasil/router.go b/src/yggdrasil/router.go index c5e1dde0..77012863 100644 --- a/src/yggdrasil/router.go +++ b/src/yggdrasil/router.go @@ -166,7 +166,7 @@ func (r *router) handleTraffic(packet []byte) { return } select { - case sinfo.recv <- &p: // FIXME ideally this should be front drop + case sinfo.fromRouter <- &p: // FIXME ideally this should be front drop default: util.PutBytes(p.Payload) } diff --git a/src/yggdrasil/session.go b/src/yggdrasil/session.go index eca3bb00..0552ec1b 100644 --- a/src/yggdrasil/session.go +++ b/src/yggdrasil/session.go @@ -6,11 +6,13 @@ package yggdrasil import ( "bytes" + "errors" "sync" "time" "github.com/yggdrasil-network/yggdrasil-go/src/address" "github.com/yggdrasil-network/yggdrasil-go/src/crypto" + "github.com/yggdrasil-network/yggdrasil-go/src/util" ) // All the information we know about an active session. @@ -44,8 +46,11 @@ type sessionInfo struct { tstamp int64 // ATOMIC - tstamp from their last session ping, replay attack mitigation bytesSent uint64 // Bytes of real traffic sent in this session bytesRecvd uint64 // Bytes of real traffic received in this session - recv chan *wire_trafficPacket // Received packets go here, picked up by the associated Conn + fromRouter chan *wire_trafficPacket // Received packets go here, picked up by the associated Conn init chan struct{} // Closed when the first session pong arrives, used to signal that the session is ready for initial use + cancel util.Cancellation // Used to terminate workers + recv chan []byte + send chan []byte } func (sinfo *sessionInfo) doFunc(f func()) { @@ -222,7 +227,9 @@ func (ss *sessions) createSession(theirPermKey *crypto.BoxPubKey) *sessionInfo { sinfo.myHandle = *crypto.NewHandle() sinfo.theirAddr = *address.AddrForNodeID(crypto.GetNodeID(&sinfo.theirPermPub)) sinfo.theirSubnet = *address.SubnetForNodeID(crypto.GetNodeID(&sinfo.theirPermPub)) - sinfo.recv = make(chan *wire_trafficPacket, 32) + sinfo.fromRouter = make(chan *wire_trafficPacket, 32) + sinfo.recv = make(chan []byte, 32) + sinfo.send = make(chan []byte, 32) ss.sinfos[sinfo.myHandle] = &sinfo ss.byTheirPerm[sinfo.theirPermPub] = &sinfo.myHandle return &sinfo @@ -355,6 +362,7 @@ func (ss *sessions) handlePing(ping *sessionPing) { for i := range conn.nodeMask { conn.nodeMask[i] = 0xFF } + conn.session.startWorkers(conn.cancel) ss.listener.conn <- conn } ss.listenerMutex.Unlock() @@ -418,3 +426,89 @@ func (ss *sessions) reset() { }) } } + +//////////////////////////////////////////////////////////////////////////////// +//////////////////////////// Worker Functions Below //////////////////////////// +//////////////////////////////////////////////////////////////////////////////// + +func (sinfo *sessionInfo) startWorkers(cancel util.Cancellation) { + sinfo.cancel = cancel + go sinfo.recvWorker() + go sinfo.sendWorker() +} + +func (sinfo *sessionInfo) recvWorker() { + // TODO move theirNonce etc into a struct that gets stored here, passed in over a channel + // Since there's no reason for anywhere else in the session code to need to *read* it... + // Only needs to be updated from the outside if a ping resets it... + // That would get rid of the need to take a mutex for the sessionFunc + for { + select { + case <-sinfo.cancel.Finished(): + return + case p := <-sinfo.fromRouter: + var bs []byte + var err error + sessionFunc := func() { + defer util.PutBytes(p.Payload) + // If the nonce is bad then drop the packet and return an error + if !sinfo.nonceIsOK(&p.Nonce) { + err = ConnError{errors.New("packet dropped due to invalid nonce"), false, true, false, 0} + return + } + // Decrypt the packet + var isOK bool + bs, isOK = crypto.BoxOpen(&sinfo.sharedSesKey, p.Payload, &p.Nonce) + // Check if we were unable to decrypt the packet for some reason and + // return an error if we couldn't + if !isOK { + err = ConnError{errors.New("packet dropped due to decryption failure"), false, true, false, 0} + return + } + // Update the session + sinfo.updateNonce(&p.Nonce) + sinfo.time = time.Now() + sinfo.bytesRecvd += uint64(len(bs)) + } + sinfo.doFunc(sessionFunc) + if len(bs) > 0 { + if err != nil { + // Bad packet, drop it + util.PutBytes(bs) + } else { + // Pass the packet to the buffer for Conn.Read + sinfo.recv <- bs + } + } + } + } +} + +func (sinfo *sessionInfo) sendWorker() { + // TODO move info that this worker needs here, send updates via a channel + // Otherwise we need to take a mutex to avoid races with update() + for { + select { + case <-sinfo.cancel.Finished(): + return + case bs := <-sinfo.send: + // TODO + var packet []byte + sessionFunc := func() { + sinfo.bytesSent += uint64(len(bs)) + payload, nonce := crypto.BoxSeal(&sinfo.sharedSesKey, bs, &sinfo.myNonce) + defer util.PutBytes(payload) + // Construct the wire packet to send to the router + p := wire_trafficPacket{ + Coords: sinfo.coords, + Handle: sinfo.theirHandle, + Nonce: *nonce, + Payload: payload, + } + packet = p.encode() + } + sinfo.doFunc(sessionFunc) + sinfo.core.router.out(packet) + } + } +} From 5dfc71e1ee26cca6343efaa4eeaa0dafb09b8c1e Mon Sep 17 00:00:00 2001 From: Arceliar Date: Sat, 3 Aug 2019 22:00:47 -0500 Subject: [PATCH 02/51] put bytes back when done --- src/yggdrasil/session.go | 1 + 1 file changed, 1 insertion(+) diff --git a/src/yggdrasil/session.go b/src/yggdrasil/session.go index 0552ec1b..f60e81a2 100644 --- a/src/yggdrasil/session.go +++ b/src/yggdrasil/session.go @@ -495,6 +495,7 @@ func (sinfo *sessionInfo) sendWorker() { // TODO var packet []byte sessionFunc := func() { + defer util.PutBytes(bs) sinfo.bytesSent += uint64(len(bs)) payload, nonce := crypto.BoxSeal(&sinfo.sharedSesKey, bs, &sinfo.myNonce) defer util.PutBytes(payload) From 72ed541bf3cec1f8e0b9a23bcbd845b8f479749e Mon Sep 17 00:00:00 2001 From: Arceliar Date: Sat, 3 Aug 2019 22:07:38 -0500 Subject: [PATCH 03/51] a little cleanup to Conn functions --- src/yggdrasil/conn.go | 36 +++++++++++++++++++----------------- 1 file changed, 19 insertions(+), 17 deletions(-) diff --git a/src/yggdrasil/conn.go b/src/yggdrasil/conn.go index e47ea2fb..f72bdd2f 100644 --- a/src/yggdrasil/conn.go +++ b/src/yggdrasil/conn.go @@ -82,7 +82,7 @@ func (c *Conn) String() string { return fmt.Sprintf("conn=%p", c) } -// This should never be called from the router goroutine +// This should never be called from the router goroutine, used in the dial functions func (c *Conn) search() error { var sinfo *searchInfo var isIn bool @@ -122,6 +122,23 @@ func (c *Conn) search() error { return nil } +// Used in session keep-alive traffic in Conn.Write +func (c *Conn) doSearch() { + routerWork := func() { + // Check to see if there is a search already matching the destination + sinfo, isIn := c.core.searches.searches[*c.nodeID] + if !isIn { + // Nothing was found, so create a new search + searchCompleted := func(sinfo *sessionInfo, e error) {} + sinfo = c.core.searches.newIterSearch(c.nodeID, c.nodeMask, searchCompleted) + c.core.log.Debugf("%s DHT search started: %p", c.String(), sinfo) + } + // Continue the search + sinfo.continueSearch() + } + go func() { c.core.router.admin <- routerWork }() +} + func (c *Conn) getDeadlineCancellation(value *atomic.Value) util.Cancellation { if deadline, ok := value.Load().(time.Time); ok { // A deadline is set, so return a Cancellation that uses it @@ -173,26 +190,11 @@ func (c *Conn) Write(b []byte) (bytesWritten int, err error) { return } // The rest of this work is session keep-alive traffic - doSearch := func() { - routerWork := func() { - // Check to see if there is a search already matching the destination - sinfo, isIn := c.core.searches.searches[*c.nodeID] - if !isIn { - // Nothing was found, so create a new search - searchCompleted := func(sinfo *sessionInfo, e error) {} - sinfo = c.core.searches.newIterSearch(c.nodeID, c.nodeMask, searchCompleted) - c.core.log.Debugf("%s DHT search started: %p", c.String(), sinfo) - } - // Continue the search - sinfo.continueSearch() - } - go func() { c.core.router.admin <- routerWork }() - } switch { case time.Since(sinfo.time) > 6*time.Second: if sinfo.time.Before(sinfo.pingTime) && time.Since(sinfo.pingTime) > 6*time.Second { // TODO double check that the above condition is correct - doSearch() + c.doSearch() } else { sinfo.core.sessions.ping(sinfo) } From 099bd3ae1e00d14344964ce8199654331a89339e Mon Sep 17 00:00:00 2001 From: Arceliar Date: Sat, 3 Aug 2019 22:35:10 -0500 Subject: [PATCH 04/51] reduce part of sendWorker that needs to keep a mutex --- src/yggdrasil/session.go | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/src/yggdrasil/session.go b/src/yggdrasil/session.go index f60e81a2..993dc52b 100644 --- a/src/yggdrasil/session.go +++ b/src/yggdrasil/session.go @@ -492,23 +492,27 @@ func (sinfo *sessionInfo) sendWorker() { case <-sinfo.cancel.Finished(): return case bs := <-sinfo.send: - // TODO - var packet []byte + var p wire_trafficPacket + var k crypto.BoxSharedKey sessionFunc := func() { - defer util.PutBytes(bs) sinfo.bytesSent += uint64(len(bs)) - payload, nonce := crypto.BoxSeal(&sinfo.sharedSesKey, bs, &sinfo.myNonce) - defer util.PutBytes(payload) - // Construct the wire packet to send to the router - p := wire_trafficPacket{ - Coords: sinfo.coords, - Handle: sinfo.theirHandle, - Nonce: *nonce, - Payload: payload, + p = wire_trafficPacket{ + Coords: append([]byte(nil), sinfo.coords...), + Handle: sinfo.theirHandle, + Nonce: sinfo.myNonce, } - packet = p.encode() + sinfo.myNonce.Increment() + k = sinfo.sharedSesKey } + // Get the mutex-protected info needed to encrypt the packet sinfo.doFunc(sessionFunc) + // Encrypt the packet + p.Payload, _ = crypto.BoxSeal(&k, bs, &p.Nonce) + packet := p.encode() + // Cleanup + util.PutBytes(bs) + util.PutBytes(p.Payload) + // Send the packet sinfo.core.router.out(packet) } } From b9987b4fdc92d3d19eaa9fec83a88681b2ef359b Mon Sep 17 00:00:00 2001 From: Arceliar Date: Sat, 3 Aug 2019 22:47:10 -0500 Subject: [PATCH 05/51] reduce time spent with a mutex held in sessionInfo.recvWorker --- src/yggdrasil/session.go | 42 +++++++++++++++++++++++----------------- 1 file changed, 24 insertions(+), 18 deletions(-) diff --git a/src/yggdrasil/session.go b/src/yggdrasil/session.go index 993dc52b..a48f44b1 100644 --- a/src/yggdrasil/session.go +++ b/src/yggdrasil/session.go @@ -449,36 +449,42 @@ func (sinfo *sessionInfo) recvWorker() { case p := <-sinfo.fromRouter: var bs []byte var err error + var k crypto.BoxSharedKey sessionFunc := func() { - defer util.PutBytes(p.Payload) - // If the nonce is bad then drop the packet and return an error if !sinfo.nonceIsOK(&p.Nonce) { err = ConnError{errors.New("packet dropped due to invalid nonce"), false, true, false, 0} return } - // Decrypt the packet - var isOK bool - bs, isOK = crypto.BoxOpen(&sinfo.sharedSesKey, p.Payload, &p.Nonce) - // Check if we were unable to decrypt the packet for some reason and - // return an error if we couldn't - if !isOK { - err = ConnError{errors.New("packet dropped due to decryption failure"), false, true, false, 0} + k = sinfo.sharedSesKey + } + sinfo.doFunc(sessionFunc) + if err != nil { + util.PutBytes(p.Payload) + continue + } + var isOK bool + bs, isOK = crypto.BoxOpen(&k, p.Payload, &p.Nonce) + if !isOK { + util.PutBytes(bs) + continue + } + sessionFunc = func() { + if k != sinfo.sharedSesKey || !sinfo.nonceIsOK(&p.Nonce) { + // The session updated in the mean time, so return an error + err = ConnError{errors.New("session updated during crypto operation"), false, true, false, 0} return } - // Update the session sinfo.updateNonce(&p.Nonce) sinfo.time = time.Now() sinfo.bytesRecvd += uint64(len(bs)) } sinfo.doFunc(sessionFunc) - if len(bs) > 0 { - if err != nil { - // Bad packet, drop it - util.PutBytes(bs) - } else { - // Pass the packet to the buffer for Conn.Read - sinfo.recv <- bs - } + if err != nil { + // Not sure what else to do with this packet, I guess just drop it + util.PutBytes(bs) + } else { + // Pass the packet to the buffer for Conn.Read + sinfo.recv <- bs } } } From 7a9ad0c8ccbe0cdd4210dada7b52c54c04428714 Mon Sep 17 00:00:00 2001 From: Arceliar Date: Sat, 3 Aug 2019 23:10:37 -0500 Subject: [PATCH 06/51] add workerpool to util --- src/util/workerpool.go | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 src/util/workerpool.go diff --git a/src/util/workerpool.go b/src/util/workerpool.go new file mode 100644 index 00000000..fd37f397 --- /dev/null +++ b/src/util/workerpool.go @@ -0,0 +1,29 @@ +package util + +import "runtime" + +var workerPool chan func() + +func init() { + maxProcs := runtime.GOMAXPROCS(0) + if maxProcs < 1 { + maxProcs = 1 + } + workerPool = make(chan func(), maxProcs) + for idx := 0; idx < maxProcs; idx++ { + go func() { + for f := range workerPool { + f() + } + }() + } +} + +// WorkerGo submits a job to a pool of GOMAXPROCS worker goroutines. +// This is meant for short non-blocking functions f() where you could just go f(), +// but you want some kind of backpressure to prevent spawning endless goroutines. +// WorkerGo returns as soon as the function is queued to run, not when it finishes. +// In Yggdrasil, these workers are used for certain cryptographic operations. +func WorkerGo(f func()) { + workerPool <- f +} From befd1b43a03cc9fdb8d53ef60e413fce4ccb0b33 Mon Sep 17 00:00:00 2001 From: Arceliar Date: Sat, 3 Aug 2019 23:14:51 -0500 Subject: [PATCH 07/51] refactor session worker code slightly --- src/yggdrasil/session.go | 128 ++++++++++++++++++++------------------- 1 file changed, 67 insertions(+), 61 deletions(-) diff --git a/src/yggdrasil/session.go b/src/yggdrasil/session.go index a48f44b1..d39f129d 100644 --- a/src/yggdrasil/session.go +++ b/src/yggdrasil/session.go @@ -442,50 +442,53 @@ func (sinfo *sessionInfo) recvWorker() { // Since there's no reason for anywhere else in the session code to need to *read* it... // Only needs to be updated from the outside if a ping resets it... // That would get rid of the need to take a mutex for the sessionFunc + doRecv := func(p *wire_trafficPacket) { + var bs []byte + var err error + var k crypto.BoxSharedKey + sessionFunc := func() { + if !sinfo.nonceIsOK(&p.Nonce) { + err = ConnError{errors.New("packet dropped due to invalid nonce"), false, true, false, 0} + return + } + k = sinfo.sharedSesKey + } + sinfo.doFunc(sessionFunc) + if err != nil { + util.PutBytes(p.Payload) + return + } + var isOK bool + bs, isOK = crypto.BoxOpen(&k, p.Payload, &p.Nonce) + if !isOK { + util.PutBytes(bs) + return + } + sessionFunc = func() { + if k != sinfo.sharedSesKey || !sinfo.nonceIsOK(&p.Nonce) { + // The session updated in the mean time, so return an error + err = ConnError{errors.New("session updated during crypto operation"), false, true, false, 0} + return + } + sinfo.updateNonce(&p.Nonce) + sinfo.time = time.Now() + sinfo.bytesRecvd += uint64(len(bs)) + } + sinfo.doFunc(sessionFunc) + if err != nil { + // Not sure what else to do with this packet, I guess just drop it + util.PutBytes(bs) + } else { + // Pass the packet to the buffer for Conn.Read + sinfo.recv <- bs + } + } for { select { case <-sinfo.cancel.Finished(): return case p := <-sinfo.fromRouter: - var bs []byte - var err error - var k crypto.BoxSharedKey - sessionFunc := func() { - if !sinfo.nonceIsOK(&p.Nonce) { - err = ConnError{errors.New("packet dropped due to invalid nonce"), false, true, false, 0} - return - } - k = sinfo.sharedSesKey - } - sinfo.doFunc(sessionFunc) - if err != nil { - util.PutBytes(p.Payload) - continue - } - var isOK bool - bs, isOK = crypto.BoxOpen(&k, p.Payload, &p.Nonce) - if !isOK { - util.PutBytes(bs) - continue - } - sessionFunc = func() { - if k != sinfo.sharedSesKey || !sinfo.nonceIsOK(&p.Nonce) { - // The session updated in the mean time, so return an error - err = ConnError{errors.New("session updated during crypto operation"), false, true, false, 0} - return - } - sinfo.updateNonce(&p.Nonce) - sinfo.time = time.Now() - sinfo.bytesRecvd += uint64(len(bs)) - } - sinfo.doFunc(sessionFunc) - if err != nil { - // Not sure what else to do with this packet, I guess just drop it - util.PutBytes(bs) - } else { - // Pass the packet to the buffer for Conn.Read - sinfo.recv <- bs - } + doRecv(p) } } } @@ -493,33 +496,36 @@ func (sinfo *sessionInfo) recvWorker() { func (sinfo *sessionInfo) sendWorker() { // TODO move info that this worker needs here, send updates via a channel // Otherwise we need to take a mutex to avoid races with update() + doSend := func(bs []byte) { + var p wire_trafficPacket + var k crypto.BoxSharedKey + sessionFunc := func() { + sinfo.bytesSent += uint64(len(bs)) + p = wire_trafficPacket{ + Coords: append([]byte(nil), sinfo.coords...), + Handle: sinfo.theirHandle, + Nonce: sinfo.myNonce, + } + sinfo.myNonce.Increment() + k = sinfo.sharedSesKey + } + // Get the mutex-protected info needed to encrypt the packet + sinfo.doFunc(sessionFunc) + // Encrypt the packet + p.Payload, _ = crypto.BoxSeal(&k, bs, &p.Nonce) + packet := p.encode() + // Cleanup + util.PutBytes(bs) + util.PutBytes(p.Payload) + // Send the packet + sinfo.core.router.out(packet) + } for { select { case <-sinfo.cancel.Finished(): return case bs := <-sinfo.send: - var p wire_trafficPacket - var k crypto.BoxSharedKey - sessionFunc := func() { - sinfo.bytesSent += uint64(len(bs)) - p = wire_trafficPacket{ - Coords: append([]byte(nil), sinfo.coords...), - Handle: sinfo.theirHandle, - Nonce: sinfo.myNonce, - } - sinfo.myNonce.Increment() - k = sinfo.sharedSesKey - } - // Get the mutex-protected info needed to encrypt the packet - sinfo.doFunc(sessionFunc) - // Encrypt the packet - p.Payload, _ = crypto.BoxSeal(&k, bs, &p.Nonce) - packet := p.encode() - // Cleanup - util.PutBytes(bs) - util.PutBytes(p.Payload) - // Send the packet - sinfo.core.router.out(packet) + doSend(bs) } } } From 00e9c3dbd9ba9246b598ca3b8f340f9d6c909e51 Mon Sep 17 00:00:00 2001 From: Arceliar Date: Sat, 3 Aug 2019 23:27:52 -0500 Subject: [PATCH 08/51] do session crypto work using the worker pool --- src/yggdrasil/session.go | 98 ++++++++++++++++++++++++++++------------ 1 file changed, 69 insertions(+), 29 deletions(-) diff --git a/src/yggdrasil/session.go b/src/yggdrasil/session.go index d39f129d..ea0b1a12 100644 --- a/src/yggdrasil/session.go +++ b/src/yggdrasil/session.go @@ -442,6 +442,7 @@ func (sinfo *sessionInfo) recvWorker() { // Since there's no reason for anywhere else in the session code to need to *read* it... // Only needs to be updated from the outside if a ping resets it... // That would get rid of the need to take a mutex for the sessionFunc + var callbacks []chan func() doRecv := func(p *wire_trafficPacket) { var bs []byte var err error @@ -459,31 +460,51 @@ func (sinfo *sessionInfo) recvWorker() { return } var isOK bool - bs, isOK = crypto.BoxOpen(&k, p.Payload, &p.Nonce) - if !isOK { - util.PutBytes(bs) - return - } - sessionFunc = func() { - if k != sinfo.sharedSesKey || !sinfo.nonceIsOK(&p.Nonce) { - // The session updated in the mean time, so return an error - err = ConnError{errors.New("session updated during crypto operation"), false, true, false, 0} - return + ch := make(chan func(), 1) + poolFunc := func() { + bs, isOK = crypto.BoxOpen(&k, p.Payload, &p.Nonce) + callback := func() { + if !isOK { + util.PutBytes(bs) + return + } + sessionFunc = func() { + if k != sinfo.sharedSesKey || !sinfo.nonceIsOK(&p.Nonce) { + // The session updated in the mean time, so return an error + err = ConnError{errors.New("session updated during crypto operation"), false, true, false, 0} + return + } + sinfo.updateNonce(&p.Nonce) + sinfo.time = time.Now() + sinfo.bytesRecvd += uint64(len(bs)) + } + sinfo.doFunc(sessionFunc) + if err != nil { + // Not sure what else to do with this packet, I guess just drop it + util.PutBytes(bs) + } else { + // Pass the packet to the buffer for Conn.Read + sinfo.recv <- bs + } } - sinfo.updateNonce(&p.Nonce) - sinfo.time = time.Now() - sinfo.bytesRecvd += uint64(len(bs)) - } - sinfo.doFunc(sessionFunc) - if err != nil { - // Not sure what else to do with this packet, I guess just drop it - util.PutBytes(bs) - } else { - // Pass the packet to the buffer for Conn.Read - sinfo.recv <- bs + ch <- callback } + // Send to the worker and wait for it to finish + util.WorkerGo(poolFunc) + callbacks = append(callbacks, ch) } for { + for len(callbacks) > 0 { + select { + case f := <-callbacks[0]: + callbacks = callbacks[1:] + f() + case <-sinfo.cancel.Finished(): + return + case p := <-sinfo.fromRouter: + doRecv(p) + } + } select { case <-sinfo.cancel.Finished(): return @@ -496,6 +517,7 @@ func (sinfo *sessionInfo) recvWorker() { func (sinfo *sessionInfo) sendWorker() { // TODO move info that this worker needs here, send updates via a channel // Otherwise we need to take a mutex to avoid races with update() + var callbacks []chan func() doSend := func(bs []byte) { var p wire_trafficPacket var k crypto.BoxSharedKey @@ -511,16 +533,34 @@ func (sinfo *sessionInfo) sendWorker() { } // Get the mutex-protected info needed to encrypt the packet sinfo.doFunc(sessionFunc) - // Encrypt the packet - p.Payload, _ = crypto.BoxSeal(&k, bs, &p.Nonce) - packet := p.encode() - // Cleanup - util.PutBytes(bs) - util.PutBytes(p.Payload) - // Send the packet - sinfo.core.router.out(packet) + ch := make(chan func(), 1) + poolFunc := func() { + // Encrypt the packet + p.Payload, _ = crypto.BoxSeal(&k, bs, &p.Nonce) + packet := p.encode() + // Cleanup + util.PutBytes(bs) + util.PutBytes(p.Payload) + // The callback will send the packet + callback := func() { sinfo.core.router.out(packet) } + ch <- callback + } + // Send to the worker and wait for it to finish + util.WorkerGo(poolFunc) + callbacks = append(callbacks, ch) } for { + for len(callbacks) > 0 { + select { + case f := <-callbacks[0]: + callbacks = callbacks[1:] + f() + case <-sinfo.cancel.Finished(): + return + case bs := <-sinfo.send: + doSend(bs) + } + } select { case <-sinfo.cancel.Finished(): return From cbbb61b01978c811ef9c10f9598bee1c23343029 Mon Sep 17 00:00:00 2001 From: Arceliar Date: Sun, 4 Aug 2019 00:00:41 -0500 Subject: [PATCH 09/51] fix another drain on the bytestore --- src/yggdrasil/session.go | 1 + 1 file changed, 1 insertion(+) diff --git a/src/yggdrasil/session.go b/src/yggdrasil/session.go index ea0b1a12..161d8eda 100644 --- a/src/yggdrasil/session.go +++ b/src/yggdrasil/session.go @@ -463,6 +463,7 @@ func (sinfo *sessionInfo) recvWorker() { ch := make(chan func(), 1) poolFunc := func() { bs, isOK = crypto.BoxOpen(&k, p.Payload, &p.Nonce) + util.PutBytes(p.Payload) callback := func() { if !isOK { util.PutBytes(bs) From 144c823beea1163d5e91ce6c9ee97daf7e2b4382 Mon Sep 17 00:00:00 2001 From: Arceliar Date: Sun, 4 Aug 2019 00:28:13 -0500 Subject: [PATCH 10/51] just use a sync.Pool as the bytestore to not overcomplicate things, the allocations from interface{} casting don't seem to actually hurt in practice right now --- src/util/util.go | 17 +++-------------- 1 file changed, 3 insertions(+), 14 deletions(-) diff --git a/src/util/util.go b/src/util/util.go index 4596474e..7f03ad05 100644 --- a/src/util/util.go +++ b/src/util/util.go @@ -22,27 +22,16 @@ func UnlockThread() { } // This is used to buffer recently used slices of bytes, to prevent allocations in the hot loops. -var byteStoreMutex sync.Mutex -var byteStore [][]byte +var byteStore = sync.Pool{New: func() interface{} { return []byte(nil) }} // Gets an empty slice from the byte store. func GetBytes() []byte { - byteStoreMutex.Lock() - defer byteStoreMutex.Unlock() - if len(byteStore) > 0 { - var bs []byte - bs, byteStore = byteStore[len(byteStore)-1][:0], byteStore[:len(byteStore)-1] - return bs - } else { - return nil - } + return byteStore.Get().([]byte)[:0] } // Puts a slice in the store. func PutBytes(bs []byte) { - byteStoreMutex.Lock() - defer byteStoreMutex.Unlock() - byteStore = append(byteStore, bs) + byteStore.Put(bs) } // This is a workaround to go's broken timer implementation From 6da5802ae56b0d348661763c5dc7b091d77b72d1 Mon Sep 17 00:00:00 2001 From: Arceliar Date: Sun, 4 Aug 2019 02:08:47 -0500 Subject: [PATCH 11/51] don't block forever in Write if the session is cancelled, cleanup Conn.Read slightly --- src/yggdrasil/conn.go | 54 +++++++++++++++++++++++++------------------ 1 file changed, 31 insertions(+), 23 deletions(-) diff --git a/src/yggdrasil/conn.go b/src/yggdrasil/conn.go index f72bdd2f..d00db5c5 100644 --- a/src/yggdrasil/conn.go +++ b/src/yggdrasil/conn.go @@ -154,29 +154,27 @@ func (c *Conn) Read(b []byte) (int, error) { sinfo := c.session cancel := c.getDeadlineCancellation(&c.readDeadline) defer cancel.Cancel(nil) - for { - // Wait for some traffic to come through from the session - select { - case <-cancel.Finished(): - if cancel.Error() == util.CancellationTimeoutError { - return 0, ConnError{errors.New("read timeout"), true, false, false, 0} - } else { - return 0, ConnError{errors.New("session closed"), false, false, true, 0} - } - case bs := <-sinfo.recv: - var err error - n := len(bs) - if len(bs) > len(b) { - n = len(b) - err = ConnError{errors.New("read buffer too small for entire packet"), false, true, false, 0} - } - // Copy results to the output slice and clean up - copy(b, bs) - util.PutBytes(bs) - // If we've reached this point then everything went to plan, return the - // number of bytes we populated back into the given slice - return n, err + // Wait for some traffic to come through from the session + select { + case <-cancel.Finished(): + if cancel.Error() == util.CancellationTimeoutError { + return 0, ConnError{errors.New("read timeout"), true, false, false, 0} + } else { + return 0, ConnError{errors.New("session closed"), false, false, true, 0} } + case bs := <-sinfo.recv: + var err error + n := len(bs) + if len(bs) > len(b) { + n = len(b) + err = ConnError{errors.New("read buffer too small for entire packet"), false, true, false, 0} + } + // Copy results to the output slice and clean up + copy(b, bs) + util.PutBytes(bs) + // If we've reached this point then everything went to plan, return the + // number of bytes we populated back into the given slice + return n, err } } @@ -206,7 +204,17 @@ func (c *Conn) Write(b []byte) (bytesWritten int, err error) { sinfo.doFunc(sessionFunc) if written > 0 { bs := append(util.GetBytes(), b...) - sinfo.send <- bs + cancel := c.getDeadlineCancellation(&c.writeDeadline) + defer cancel.Cancel(nil) + select { + case <-cancel.Finished(): + if cancel.Error() == util.CancellationTimeoutError { + return 0, ConnError{errors.New("write timeout"), true, false, false, 0} + } else { + return 0, ConnError{errors.New("session closed"), false, false, true, 0} + } + case sinfo.send <- bs: + } } return written, err } From 7bf5884ac1af2df80df503ccecd3fca629065218 Mon Sep 17 00:00:00 2001 From: Arceliar Date: Sun, 4 Aug 2019 02:14:45 -0500 Subject: [PATCH 12/51] remove some lossy channel sends that should be safe to allow to block --- src/tuntap/conn.go | 6 +----- src/tuntap/iface.go | 13 +++---------- 2 files changed, 4 insertions(+), 15 deletions(-) diff --git a/src/tuntap/conn.go b/src/tuntap/conn.go index 1d47b378..0bb4efdc 100644 --- a/src/tuntap/conn.go +++ b/src/tuntap/conn.go @@ -72,11 +72,7 @@ func (s *tunConn) reader() (err error) { } } else if n > 0 { bs := append(util.GetBytes(), b[:n]...) - select { - case s.tun.send <- bs: - default: - util.PutBytes(bs) - } + s.tun.send <- bs s.stillAlive() } } diff --git a/src/tuntap/iface.go b/src/tuntap/iface.go index a95dfae4..1cee9b45 100644 --- a/src/tuntap/iface.go +++ b/src/tuntap/iface.go @@ -260,11 +260,8 @@ func (tun *TunAdapter) readerPacketHandler(ch chan []byte) { tun.mutex.Unlock() if tc != nil { for _, packet := range packets { - select { - case tc.send <- packet: - default: - util.PutBytes(packet) - } + p := packet // Possibly required because of how range + tc.send <- p } } }() @@ -274,11 +271,7 @@ func (tun *TunAdapter) readerPacketHandler(ch chan []byte) { } // If we have a connection now, try writing to it if isIn && session != nil { - select { - case session.send <- bs: - default: - util.PutBytes(bs) - } + session.send <- bs } } } From 1e6a6d216022f30af4b589a2432af4b09e311f22 Mon Sep 17 00:00:00 2001 From: Arceliar Date: Sun, 4 Aug 2019 02:21:41 -0500 Subject: [PATCH 13/51] use session.cancel in the router to make blocking safe, reduce size of fromRouter buffer so the drops in the switch are closer to the intended front-drop behavior --- src/yggdrasil/router.go | 4 ++-- src/yggdrasil/session.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/yggdrasil/router.go b/src/yggdrasil/router.go index 77012863..4eaa56d8 100644 --- a/src/yggdrasil/router.go +++ b/src/yggdrasil/router.go @@ -166,8 +166,8 @@ func (r *router) handleTraffic(packet []byte) { return } select { - case sinfo.fromRouter <- &p: // FIXME ideally this should be front drop - default: + case sinfo.fromRouter <- &p: + case <-sinfo.cancel.Finished(): util.PutBytes(p.Payload) } } diff --git a/src/yggdrasil/session.go b/src/yggdrasil/session.go index 161d8eda..c17fb045 100644 --- a/src/yggdrasil/session.go +++ b/src/yggdrasil/session.go @@ -227,7 +227,7 @@ func (ss *sessions) createSession(theirPermKey *crypto.BoxPubKey) *sessionInfo { sinfo.myHandle = *crypto.NewHandle() sinfo.theirAddr = *address.AddrForNodeID(crypto.GetNodeID(&sinfo.theirPermPub)) sinfo.theirSubnet = *address.SubnetForNodeID(crypto.GetNodeID(&sinfo.theirPermPub)) - sinfo.fromRouter = make(chan *wire_trafficPacket, 32) + sinfo.fromRouter = make(chan *wire_trafficPacket, 1) sinfo.recv = make(chan []byte, 32) sinfo.send = make(chan []byte, 32) ss.sinfos[sinfo.myHandle] = &sinfo From f52955ee0fd17f5adb7960f9602e1e498c056c9c Mon Sep 17 00:00:00 2001 From: Arceliar Date: Sun, 4 Aug 2019 14:18:59 -0500 Subject: [PATCH 14/51] WARNING: CRYPTO DISABLED while speeding up stream writeMsg --- src/crypto/crypto.go | 2 ++ src/util/util.go | 9 +++++++++ src/yggdrasil/router.go | 1 - src/yggdrasil/stream.go | 27 ++++++++++++++++----------- src/yggdrasil/switch.go | 2 +- 5 files changed, 28 insertions(+), 13 deletions(-) diff --git a/src/crypto/crypto.go b/src/crypto/crypto.go index 75736ba7..14b12186 100644 --- a/src/crypto/crypto.go +++ b/src/crypto/crypto.go @@ -172,6 +172,7 @@ func BoxOpen(shared *BoxSharedKey, boxed []byte, nonce *BoxNonce) ([]byte, bool) { out := util.GetBytes() + return append(out, boxed...), true s := (*[BoxSharedKeyLen]byte)(shared) n := (*[BoxNonceLen]byte)(nonce) unboxed, success := box.OpenAfterPrecomputation(out, boxed, n, s) @@ -184,6 +185,7 @@ func BoxSeal(shared *BoxSharedKey, unboxed []byte, nonce *BoxNonce) ([]byte, *Bo } nonce.Increment() out := util.GetBytes() + return append(out, unboxed...), nonce s := (*[BoxSharedKeyLen]byte)(shared) n := (*[BoxNonceLen]byte)(nonce) boxed := box.SealAfterPrecomputation(out, unboxed, n, s) diff --git a/src/util/util.go b/src/util/util.go index 7f03ad05..b5e6ccc6 100644 --- a/src/util/util.go +++ b/src/util/util.go @@ -34,6 +34,15 @@ func PutBytes(bs []byte) { byteStore.Put(bs) } +// Gets a slice of the appropriate length, reusing existing slice capacity when possible +func ResizeBytes(bs []byte, length int) []byte { + if cap(bs) >= length { + return bs[:length] + } else { + return make([]byte, length) + } +} + // This is a workaround to go's broken timer implementation func TimerStop(t *time.Timer) bool { stopped := t.Stop() diff --git a/src/yggdrasil/router.go b/src/yggdrasil/router.go index 4eaa56d8..2df7684f 100644 --- a/src/yggdrasil/router.go +++ b/src/yggdrasil/router.go @@ -127,7 +127,6 @@ func (r *router) mainLoop() { r.core.switchTable.doMaintenance() r.core.dht.doMaintenance() r.core.sessions.cleanup() - util.GetBytes() // To slowly drain things } case f := <-r.admin: f() diff --git a/src/yggdrasil/stream.go b/src/yggdrasil/stream.go index 4d73844f..30dd9244 100644 --- a/src/yggdrasil/stream.go +++ b/src/yggdrasil/stream.go @@ -4,6 +4,7 @@ import ( "errors" "fmt" "io" + "net" "github.com/yggdrasil-network/yggdrasil-go/src/util" ) @@ -12,10 +13,11 @@ import ( var _ = linkInterfaceMsgIO(&stream{}) type stream struct { - rwc io.ReadWriteCloser - inputBuffer []byte // Incoming packet stream - frag [2 * streamMsgSize]byte // Temporary data read off the underlying rwc, on its way to the inputBuffer - outputBuffer [2 * streamMsgSize]byte // Temporary data about to be written to the rwc + rwc io.ReadWriteCloser + inputBuffer []byte // Incoming packet stream + frag [2 * streamMsgSize]byte // Temporary data read off the underlying rwc, on its way to the inputBuffer + //outputBuffer [2 * streamMsgSize]byte // Temporary data about to be written to the rwc + outputBuffer net.Buffers } func (s *stream) close() error { @@ -35,14 +37,17 @@ func (s *stream) init(rwc io.ReadWriteCloser) { // writeMsg writes a message with stream padding, and is *not* thread safe. func (s *stream) writeMsg(bs []byte) (int, error) { buf := s.outputBuffer[:0] - buf = append(buf, streamMsg[:]...) - buf = wire_put_uint64(uint64(len(bs)), buf) - padLen := len(buf) - buf = append(buf, bs...) + buf = append(buf, streamMsg[:]) + l := wire_put_uint64(uint64(len(bs)), util.GetBytes()) + defer util.PutBytes(l) + buf = append(buf, l) + padLen := len(buf[0]) + len(buf[1]) + buf = append(buf, bs) + totalLen := padLen + len(bs) var bn int - for bn < len(buf) { - n, err := s.rwc.Write(buf[bn:]) - bn += n + for bn < totalLen { + n, err := buf.WriteTo(s.rwc) + bn += int(n) if err != nil { l := bn - padLen if l < 0 { diff --git a/src/yggdrasil/switch.go b/src/yggdrasil/switch.go index 1bc40501..0e11593f 100644 --- a/src/yggdrasil/switch.go +++ b/src/yggdrasil/switch.go @@ -820,7 +820,7 @@ func (t *switchTable) doWorker() { select { case bs := <-t.toRouter: buf = append(buf, bs) - for len(buf) > 32 { + for len(buf) > 32768 { // FIXME realistically don't drop anything, just for testing util.PutBytes(buf[0]) buf = buf[1:] } From 75b931f37ea562198fbc372dd9a44789036ddaec Mon Sep 17 00:00:00 2001 From: Arceliar Date: Sun, 4 Aug 2019 14:50:19 -0500 Subject: [PATCH 15/51] eliminate some more copying between slices --- src/tuntap/conn.go | 7 ++++--- src/tuntap/iface.go | 14 +++++++++----- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/src/tuntap/conn.go b/src/tuntap/conn.go index 0bb4efdc..1881bdea 100644 --- a/src/tuntap/conn.go +++ b/src/tuntap/conn.go @@ -54,13 +54,13 @@ func (s *tunConn) reader() (err error) { s.tun.log.Debugln("Starting conn reader for", s.conn.String()) defer s.tun.log.Debugln("Stopping conn reader for", s.conn.String()) var n int - b := make([]byte, 65535) for { select { case <-s.stop: return nil default: } + b := util.ResizeBytes(util.GetBytes(), 65535) if n, err = s.conn.Read(b); err != nil { if e, eok := err.(yggdrasil.ConnError); eok && !e.Temporary() { if e.Closed() { @@ -71,9 +71,10 @@ func (s *tunConn) reader() (err error) { return e } } else if n > 0 { - bs := append(util.GetBytes(), b[:n]...) - s.tun.send <- bs + s.tun.send <- b[:n] s.stillAlive() + } else { + util.PutBytes(b) } } } diff --git a/src/tuntap/iface.go b/src/tuntap/iface.go index 1cee9b45..670f7829 100644 --- a/src/tuntap/iface.go +++ b/src/tuntap/iface.go @@ -139,8 +139,10 @@ func (tun *TunAdapter) readerPacketHandler(ch chan []byte) { continue } } - // Shift forward to avoid leaking bytes off the front of the slide when we eventually store it - bs = append(recvd[:0], bs...) + if offset != 0 { + // Shift forward to avoid leaking bytes off the front of the slice when we eventually store it + bs = append(recvd[:0], bs...) + } // From the IP header, work out what our source and destination addresses // and node IDs are. We will need these in order to work out where to send // the packet @@ -277,11 +279,12 @@ func (tun *TunAdapter) readerPacketHandler(ch chan []byte) { } func (tun *TunAdapter) reader() error { - recvd := make([]byte, 65535+tun_ETHER_HEADER_LENGTH) toWorker := make(chan []byte, 32) defer close(toWorker) go tun.readerPacketHandler(toWorker) for { + // Get a slice to store the packet in + recvd := util.ResizeBytes(util.GetBytes(), 65535+tun_ETHER_HEADER_LENGTH) // Wait for a packet to be delivered to us through the TUN/TAP adapter n, err := tun.iface.Read(recvd) if err != nil { @@ -291,9 +294,10 @@ func (tun *TunAdapter) reader() error { panic(err) } if n == 0 { + util.PutBytes(recvd) continue } - bs := append(util.GetBytes(), recvd[:n]...) - toWorker <- bs + // Send the packet to the worker + toWorker <- recvd[:n] } } From 0ba8c6a34f3bfe5eb919e5fd1fe5a0d138c45b6d Mon Sep 17 00:00:00 2001 From: Arceliar Date: Sun, 4 Aug 2019 15:21:04 -0500 Subject: [PATCH 16/51] have the stream code use bufio instead of copying manually to an input buffer, slightly reduces total uses of memmove --- src/crypto/crypto.go | 4 +-- src/yggdrasil/stream.go | 75 +++++++++++++++-------------------------- 2 files changed, 30 insertions(+), 49 deletions(-) diff --git a/src/crypto/crypto.go b/src/crypto/crypto.go index 14b12186..44d2d8e8 100644 --- a/src/crypto/crypto.go +++ b/src/crypto/crypto.go @@ -172,7 +172,7 @@ func BoxOpen(shared *BoxSharedKey, boxed []byte, nonce *BoxNonce) ([]byte, bool) { out := util.GetBytes() - return append(out, boxed...), true + return append(out, boxed...), true //FIXME disabled crypto for benchmarking s := (*[BoxSharedKeyLen]byte)(shared) n := (*[BoxNonceLen]byte)(nonce) unboxed, success := box.OpenAfterPrecomputation(out, boxed, n, s) @@ -185,7 +185,7 @@ func BoxSeal(shared *BoxSharedKey, unboxed []byte, nonce *BoxNonce) ([]byte, *Bo } nonce.Increment() out := util.GetBytes() - return append(out, unboxed...), nonce + return append(out, unboxed...), nonce // FIXME disabled crypto for benchmarking s := (*[BoxSharedKeyLen]byte)(shared) n := (*[BoxNonceLen]byte)(nonce) boxed := box.SealAfterPrecomputation(out, unboxed, n, s) diff --git a/src/yggdrasil/stream.go b/src/yggdrasil/stream.go index 30dd9244..011943f5 100644 --- a/src/yggdrasil/stream.go +++ b/src/yggdrasil/stream.go @@ -1,6 +1,7 @@ package yggdrasil import ( + "bufio" "errors" "fmt" "io" @@ -13,10 +14,8 @@ import ( var _ = linkInterfaceMsgIO(&stream{}) type stream struct { - rwc io.ReadWriteCloser - inputBuffer []byte // Incoming packet stream - frag [2 * streamMsgSize]byte // Temporary data read off the underlying rwc, on its way to the inputBuffer - //outputBuffer [2 * streamMsgSize]byte // Temporary data about to be written to the rwc + rwc io.ReadWriteCloser + inputBuffer *bufio.Reader outputBuffer net.Buffers } @@ -32,6 +31,7 @@ func (s *stream) init(rwc io.ReadWriteCloser) { // TODO have this also do the metadata handshake and create the peer struct s.rwc = rwc // TODO call something to do the metadata exchange + s.inputBuffer = bufio.NewReaderSize(s.rwc, 2*streamMsgSize) } // writeMsg writes a message with stream padding, and is *not* thread safe. @@ -62,26 +62,11 @@ func (s *stream) writeMsg(bs []byte) (int, error) { // readMsg reads a message from the stream, accounting for stream padding, and is *not* thread safe. func (s *stream) readMsg() ([]byte, error) { for { - buf := s.inputBuffer - msg, ok, err := stream_chopMsg(&buf) - switch { - case err != nil: - // Something in the stream format is corrupt + bs, err := s.readMsgFromBuffer() + if err != nil { return nil, fmt.Errorf("message error: %v", err) - case ok: - // Copy the packet into bs, shift the buffer, and return - msg = append(util.GetBytes(), msg...) - s.inputBuffer = append(s.inputBuffer[:0], buf...) - return msg, nil - default: - // Wait for the underlying reader to return enough info for us to proceed - n, err := s.rwc.Read(s.frag[:]) - if n > 0 { - s.inputBuffer = append(s.inputBuffer, s.frag[:n]...) - } else if err != nil { - return nil, err - } } + return bs, err } } @@ -113,34 +98,30 @@ func (s *stream) _recvMetaBytes() ([]byte, error) { return metaBytes, nil } -// This takes a pointer to a slice as an argument. It checks if there's a -// complete message and, if so, slices out those parts and returns the message, -// true, and nil. If there's no error, but also no complete message, it returns -// nil, false, and nil. If there's an error, it returns nil, false, and the -// error, which the reader then handles (currently, by returning from the -// reader, which causes the connection to close). -func stream_chopMsg(bs *[]byte) ([]byte, bool, error) { - // Returns msg, ok, err - if len(*bs) < len(streamMsg) { - return nil, false, nil +// Reads bytes from the underlying rwc and returns 1 full message +func (s *stream) readMsgFromBuffer() ([]byte, error) { + pad := streamMsg // Copy + _, err := io.ReadFull(s.inputBuffer, pad[:]) + if err != nil { + return nil, err + } else if pad != streamMsg { + return nil, errors.New("bad message") } - for idx := range streamMsg { - if (*bs)[idx] != streamMsg[idx] { - return nil, false, errors.New("bad message") + lenSlice := make([]byte, 0, 10) + // FIXME this nextByte stuff depends on wire.go format, kind of ugly to have it here + nextByte := byte(0xff) + for nextByte > 127 { + nextByte, err = s.inputBuffer.ReadByte() + if err != nil { + return nil, err } + lenSlice = append(lenSlice, nextByte) } - msgLen, msgLenLen := wire_decode_uint64((*bs)[len(streamMsg):]) + msgLen, _ := wire_decode_uint64(lenSlice) if msgLen > streamMsgSize { - return nil, false, errors.New("oversized message") + return nil, errors.New("oversized message") } - msgBegin := len(streamMsg) + msgLenLen - msgEnd := msgBegin + int(msgLen) - if msgLenLen == 0 || len(*bs) < msgEnd { - // We don't have the full message - // Need to buffer this and wait for the rest to come in - return nil, false, nil - } - msg := (*bs)[msgBegin:msgEnd] - (*bs) = (*bs)[msgEnd:] - return msg, true, nil + msg := util.ResizeBytes(util.GetBytes(), int(msgLen)) + _, err = io.ReadFull(s.inputBuffer, msg) + return msg, err } From 07f14f92ed4f79a062946bda6e858164437f16cf Mon Sep 17 00:00:00 2001 From: Arceliar Date: Sun, 4 Aug 2019 15:25:14 -0500 Subject: [PATCH 17/51] disable crypto and switch buffer changes from testing --- src/crypto/crypto.go | 4 ++-- src/yggdrasil/switch.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/crypto/crypto.go b/src/crypto/crypto.go index 44d2d8e8..e85493f0 100644 --- a/src/crypto/crypto.go +++ b/src/crypto/crypto.go @@ -172,7 +172,7 @@ func BoxOpen(shared *BoxSharedKey, boxed []byte, nonce *BoxNonce) ([]byte, bool) { out := util.GetBytes() - return append(out, boxed...), true //FIXME disabled crypto for benchmarking + //return append(out, boxed...), true //FIXME disabled crypto for benchmarking s := (*[BoxSharedKeyLen]byte)(shared) n := (*[BoxNonceLen]byte)(nonce) unboxed, success := box.OpenAfterPrecomputation(out, boxed, n, s) @@ -185,7 +185,7 @@ func BoxSeal(shared *BoxSharedKey, unboxed []byte, nonce *BoxNonce) ([]byte, *Bo } nonce.Increment() out := util.GetBytes() - return append(out, unboxed...), nonce // FIXME disabled crypto for benchmarking + //return append(out, unboxed...), nonce // FIXME disabled crypto for benchmarking s := (*[BoxSharedKeyLen]byte)(shared) n := (*[BoxNonceLen]byte)(nonce) boxed := box.SealAfterPrecomputation(out, unboxed, n, s) diff --git a/src/yggdrasil/switch.go b/src/yggdrasil/switch.go index 0e11593f..1bc40501 100644 --- a/src/yggdrasil/switch.go +++ b/src/yggdrasil/switch.go @@ -820,7 +820,7 @@ func (t *switchTable) doWorker() { select { case bs := <-t.toRouter: buf = append(buf, bs) - for len(buf) > 32768 { // FIXME realistically don't drop anything, just for testing + for len(buf) > 32 { util.PutBytes(buf[0]) buf = buf[1:] } From 5d5486049b6d6ea9b9103bd0a86dece142618290 Mon Sep 17 00:00:00 2001 From: Arceliar Date: Sun, 4 Aug 2019 15:53:34 -0500 Subject: [PATCH 18/51] add Conn.ReadNoCopy and Conn.WriteNoCopy that transfer ownership of a slice instead of copying, have Read and Write use the NoCopy versions under the hood and just manage copying as needed --- src/yggdrasil/conn.go | 85 ++++++++++++++++++++++++++----------------- 1 file changed, 51 insertions(+), 34 deletions(-) diff --git a/src/yggdrasil/conn.go b/src/yggdrasil/conn.go index d00db5c5..1a05bd83 100644 --- a/src/yggdrasil/conn.go +++ b/src/yggdrasil/conn.go @@ -149,73 +149,90 @@ func (c *Conn) getDeadlineCancellation(value *atomic.Value) util.Cancellation { } } -func (c *Conn) Read(b []byte) (int, error) { - // Take a copy of the session object - sinfo := c.session +// Used internally by Read, the caller is responsible for util.PutBytes when they're done. +func (c *Conn) ReadNoCopy() ([]byte, error) { cancel := c.getDeadlineCancellation(&c.readDeadline) defer cancel.Cancel(nil) // Wait for some traffic to come through from the session select { case <-cancel.Finished(): if cancel.Error() == util.CancellationTimeoutError { - return 0, ConnError{errors.New("read timeout"), true, false, false, 0} + return nil, ConnError{errors.New("read timeout"), true, false, false, 0} } else { - return 0, ConnError{errors.New("session closed"), false, false, true, 0} + return nil, ConnError{errors.New("session closed"), false, false, true, 0} } - case bs := <-sinfo.recv: - var err error - n := len(bs) - if len(bs) > len(b) { - n = len(b) - err = ConnError{errors.New("read buffer too small for entire packet"), false, true, false, 0} - } - // Copy results to the output slice and clean up - copy(b, bs) - util.PutBytes(bs) - // If we've reached this point then everything went to plan, return the - // number of bytes we populated back into the given slice - return n, err + case bs := <-c.session.recv: + return bs, nil } } -func (c *Conn) Write(b []byte) (bytesWritten int, err error) { - sinfo := c.session - written := len(b) +// Implements net.Conn.Read +func (c *Conn) Read(b []byte) (int, error) { + bs, err := c.ReadNoCopy() + if err != nil { + return 0, err + } + n := len(bs) + if len(bs) > len(b) { + n = len(b) + err = ConnError{errors.New("read buffer too small for entire packet"), false, true, false, 0} + } + // Copy results to the output slice and clean up + copy(b, bs) + util.PutBytes(bs) + // Return the number of bytes copied to the slice, along with any error + return n, err +} + +// Used internally by Write, the caller must not reuse the argument bytes when no error occurs +func (c *Conn) WriteNoCopy(bs []byte) error { + var err error sessionFunc := func() { // Does the packet exceed the permitted size for the session? - if uint16(len(b)) > sinfo.getMTU() { - written, err = 0, ConnError{errors.New("packet too big"), true, false, false, int(sinfo.getMTU())} + if uint16(len(bs)) > c.session.getMTU() { + err = ConnError{errors.New("packet too big"), true, false, false, int(c.session.getMTU())} return } // The rest of this work is session keep-alive traffic switch { - case time.Since(sinfo.time) > 6*time.Second: - if sinfo.time.Before(sinfo.pingTime) && time.Since(sinfo.pingTime) > 6*time.Second { + case time.Since(c.session.time) > 6*time.Second: + if c.session.time.Before(c.session.pingTime) && time.Since(c.session.pingTime) > 6*time.Second { // TODO double check that the above condition is correct c.doSearch() } else { - sinfo.core.sessions.ping(sinfo) + c.core.sessions.ping(c.session) } - case sinfo.reset && sinfo.pingTime.Before(sinfo.time): - sinfo.core.sessions.ping(sinfo) + case c.session.reset && c.session.pingTime.Before(c.session.time): + c.core.sessions.ping(c.session) default: // Don't do anything, to keep traffic throttled } } - sinfo.doFunc(sessionFunc) - if written > 0 { - bs := append(util.GetBytes(), b...) + c.session.doFunc(sessionFunc) + if err == nil { cancel := c.getDeadlineCancellation(&c.writeDeadline) defer cancel.Cancel(nil) select { case <-cancel.Finished(): if cancel.Error() == util.CancellationTimeoutError { - return 0, ConnError{errors.New("write timeout"), true, false, false, 0} + err = ConnError{errors.New("write timeout"), true, false, false, 0} } else { - return 0, ConnError{errors.New("session closed"), false, false, true, 0} + err = ConnError{errors.New("session closed"), false, false, true, 0} } - case sinfo.send <- bs: + case c.session.send <- bs: } } + return err +} + +// Implements net.Conn.Write +func (c *Conn) Write(b []byte) (int, error) { + written := len(b) + bs := append(util.GetBytes(), b...) + err := c.WriteNoCopy(bs) + if err != nil { + util.PutBytes(bs) + written = 0 + } return written, err } From 6803f209b0a353195a4645f2b008ea9881aa36da Mon Sep 17 00:00:00 2001 From: Arceliar Date: Sun, 4 Aug 2019 15:59:51 -0500 Subject: [PATCH 19/51] have tuntap code use Conn.ReadNoCopy and Conn.WriteNoCopy to avoid copying between slices --- src/tuntap/conn.go | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/src/tuntap/conn.go b/src/tuntap/conn.go index 1881bdea..ab3179bf 100644 --- a/src/tuntap/conn.go +++ b/src/tuntap/conn.go @@ -53,15 +53,14 @@ func (s *tunConn) reader() (err error) { } s.tun.log.Debugln("Starting conn reader for", s.conn.String()) defer s.tun.log.Debugln("Stopping conn reader for", s.conn.String()) - var n int for { select { case <-s.stop: return nil default: } - b := util.ResizeBytes(util.GetBytes(), 65535) - if n, err = s.conn.Read(b); err != nil { + var bs []byte + if bs, err = s.conn.ReadNoCopy(); err != nil { if e, eok := err.(yggdrasil.ConnError); eok && !e.Temporary() { if e.Closed() { s.tun.log.Debugln(s.conn.String(), "TUN/TAP conn read debug:", err) @@ -70,11 +69,11 @@ func (s *tunConn) reader() (err error) { } return e } - } else if n > 0 { - s.tun.send <- b[:n] + } else if len(bs) > 0 { + s.tun.send <- bs s.stillAlive() } else { - util.PutBytes(b) + util.PutBytes(bs) } } } @@ -93,12 +92,12 @@ func (s *tunConn) writer() error { select { case <-s.stop: return nil - case b, ok := <-s.send: + case bs, ok := <-s.send: if !ok { return errors.New("send closed") } // TODO write timeout and close - if _, err := s.conn.Write(b); err != nil { + if err := s.conn.WriteNoCopy(bs); err != nil { if e, eok := err.(yggdrasil.ConnError); !eok { if e.Closed() { s.tun.log.Debugln(s.conn.String(), "TUN/TAP generic write debug:", err) @@ -109,9 +108,9 @@ func (s *tunConn) writer() error { // TODO: This currently isn't aware of IPv4 for CKR ptb := &icmp.PacketTooBig{ MTU: int(e.PacketMaximumSize()), - Data: b[:900], + Data: bs[:900], } - if packet, err := CreateICMPv6(b[8:24], b[24:40], ipv6.ICMPTypePacketTooBig, 0, ptb); err == nil { + if packet, err := CreateICMPv6(bs[8:24], bs[24:40], ipv6.ICMPTypePacketTooBig, 0, ptb); err == nil { s.tun.send <- packet } } else { @@ -124,7 +123,6 @@ func (s *tunConn) writer() error { } else { s.stillAlive() } - util.PutBytes(b) } } } From c55d7b4705e61ba8f78b893e72a3f1ee31b618d0 Mon Sep 17 00:00:00 2001 From: Arceliar Date: Sun, 4 Aug 2019 16:16:49 -0500 Subject: [PATCH 20/51] have the switch queue drop packts to ourself when the total size of all packets is at least queueTotalMaxSize, instead of an arbitrary unconfigurable packet count --- src/yggdrasil/switch.go | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/yggdrasil/switch.go b/src/yggdrasil/switch.go index 1bc40501..b53229cc 100644 --- a/src/yggdrasil/switch.go +++ b/src/yggdrasil/switch.go @@ -814,17 +814,23 @@ func (t *switchTable) doWorker() { go func() { // Keep taking packets from the idle worker and sending them to the above whenever it's idle, keeping anything extra in a (fifo, head-drop) buffer var buf [][]byte + var size int for { - buf = append(buf, <-t.toRouter) + bs := <-t.toRouter + size += len(bs) + buf = append(buf, bs) for len(buf) > 0 { select { case bs := <-t.toRouter: + size += len(bs) buf = append(buf, bs) - for len(buf) > 32 { + for size > int(t.queueTotalMaxSize) { + size -= len(buf[0]) util.PutBytes(buf[0]) buf = buf[1:] } case sendingToRouter <- buf[0]: + size -= len(buf[0]) buf = buf[1:] } } From 979c3d4c07250afbda4dc3faa38f788e31300282 Mon Sep 17 00:00:00 2001 From: Arceliar Date: Sun, 4 Aug 2019 16:29:58 -0500 Subject: [PATCH 21/51] move some potentially blocking operations out of session pool workers, minor cleanup --- src/crypto/crypto.go | 2 -- src/yggdrasil/session.go | 13 ++++++++----- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/src/crypto/crypto.go b/src/crypto/crypto.go index e85493f0..75736ba7 100644 --- a/src/crypto/crypto.go +++ b/src/crypto/crypto.go @@ -172,7 +172,6 @@ func BoxOpen(shared *BoxSharedKey, boxed []byte, nonce *BoxNonce) ([]byte, bool) { out := util.GetBytes() - //return append(out, boxed...), true //FIXME disabled crypto for benchmarking s := (*[BoxSharedKeyLen]byte)(shared) n := (*[BoxNonceLen]byte)(nonce) unboxed, success := box.OpenAfterPrecomputation(out, boxed, n, s) @@ -185,7 +184,6 @@ func BoxSeal(shared *BoxSharedKey, unboxed []byte, nonce *BoxNonce) ([]byte, *Bo } nonce.Increment() out := util.GetBytes() - //return append(out, unboxed...), nonce // FIXME disabled crypto for benchmarking s := (*[BoxSharedKeyLen]byte)(shared) n := (*[BoxNonceLen]byte)(nonce) boxed := box.SealAfterPrecomputation(out, unboxed, n, s) diff --git a/src/yggdrasil/session.go b/src/yggdrasil/session.go index c17fb045..c39f60de 100644 --- a/src/yggdrasil/session.go +++ b/src/yggdrasil/session.go @@ -463,8 +463,8 @@ func (sinfo *sessionInfo) recvWorker() { ch := make(chan func(), 1) poolFunc := func() { bs, isOK = crypto.BoxOpen(&k, p.Payload, &p.Nonce) - util.PutBytes(p.Payload) callback := func() { + util.PutBytes(p.Payload) if !isOK { util.PutBytes(bs) return @@ -539,11 +539,14 @@ func (sinfo *sessionInfo) sendWorker() { // Encrypt the packet p.Payload, _ = crypto.BoxSeal(&k, bs, &p.Nonce) packet := p.encode() - // Cleanup - util.PutBytes(bs) - util.PutBytes(p.Payload) // The callback will send the packet - callback := func() { sinfo.core.router.out(packet) } + callback := func() { + // Cleanup + util.PutBytes(bs) + util.PutBytes(p.Payload) + // Send the packet + sinfo.core.router.out(packet) + } ch <- callback } // Send to the worker and wait for it to finish From 37533f157d6ee118ae087cb6a28b7869704eda21 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Mon, 5 Aug 2019 00:30:12 +0100 Subject: [PATCH 22/51] Make some API changes (currently broken) --- src/admin/admin.go | 42 +++++++++++++++------- src/util/util.go | 23 ++++++++++-- src/yggdrasil/api.go | 79 ++++++++--------------------------------- src/yggdrasil/switch.go | 21 +++++++++++ 4 files changed, 84 insertions(+), 81 deletions(-) diff --git a/src/admin/admin.go b/src/admin/admin.go index 2b73764c..db4b64b0 100644 --- a/src/admin/admin.go +++ b/src/admin/admin.go @@ -17,6 +17,7 @@ import ( "github.com/yggdrasil-network/yggdrasil-go/src/address" "github.com/yggdrasil-network/yggdrasil-go/src/config" "github.com/yggdrasil-network/yggdrasil-go/src/crypto" + "github.com/yggdrasil-network/yggdrasil-go/src/util" "github.com/yggdrasil-network/yggdrasil-go/src/yggdrasil" ) @@ -243,31 +244,46 @@ func (a *AdminSocket) Init(c *yggdrasil.Core, state *config.NodeState, log *log. } }) a.AddHandler("dhtPing", []string{"box_pub_key", "coords", "[target]"}, func(in Info) (Info, error) { + var reserr error + var result yggdrasil.DHTRes if in["target"] == nil { in["target"] = "none" } - result, err := a.core.DHTPing(in["box_pub_key"].(string), in["coords"].(string), in["target"].(string)) - if err == nil { - infos := make(map[string]map[string]string, len(result.Infos)) - for _, dinfo := range result.Infos { - info := map[string]string{ - "box_pub_key": hex.EncodeToString(dinfo.PublicKey[:]), - "coords": fmt.Sprintf("%v", dinfo.Coords), - } - addr := net.IP(address.AddrForNodeID(crypto.GetNodeID(&dinfo.PublicKey))[:]).String() - infos[addr] = info + coords := util.DecodeCoordString(in["coords"].(string)) + var boxPubKey crypto.BoxPubKey + if b, err := hex.DecodeString(in["box_pub_key"].(string)); err == nil { + copy(boxPubKey[:], b[:]) + if n, err := hex.DecodeString(in["target"].(string)); err == nil { + var targetNodeID crypto.NodeID + copy(targetNodeID[:], n[:]) + result, reserr = a.core.DHTPing(boxPubKey, coords, &targetNodeID) + } else { + result, reserr = a.core.DHTPing(boxPubKey, coords, nil) } - return Info{"nodes": infos}, nil } else { return Info{}, err } + if reserr != nil { + return Info{}, reserr + } + infos := make(map[string]map[string]string, len(result.Infos)) + for _, dinfo := range result.Infos { + info := map[string]string{ + "box_pub_key": hex.EncodeToString(dinfo.PublicKey[:]), + "coords": fmt.Sprintf("%v", dinfo.Coords), + } + addr := net.IP(address.AddrForNodeID(crypto.GetNodeID(&dinfo.PublicKey))[:]).String() + infos[addr] = info + } + return Info{"nodes": infos}, nil }) a.AddHandler("getNodeInfo", []string{"[box_pub_key]", "[coords]", "[nocache]"}, func(in Info) (Info, error) { var nocache bool if in["nocache"] != nil { nocache = in["nocache"].(string) == "true" } - var box_pub_key, coords string + var box_pub_key string + var coords []uint64 if in["box_pub_key"] == nil && in["coords"] == nil { nodeinfo := a.core.MyNodeInfo() var jsoninfo interface{} @@ -280,7 +296,7 @@ func (a *AdminSocket) Init(c *yggdrasil.Core, state *config.NodeState, log *log. return Info{}, errors.New("Expecting both box_pub_key and coords") } else { box_pub_key = in["box_pub_key"].(string) - coords = in["coords"].(string) + coords = util.DecodeCoordString(in["coords"].(string)) } result, err := a.core.GetNodeInfo(box_pub_key, coords, nocache) if err == nil { diff --git a/src/util/util.go b/src/util/util.go index 4596474e..5be97603 100644 --- a/src/util/util.go +++ b/src/util/util.go @@ -2,9 +2,13 @@ package util // These are misc. utility functions that didn't really fit anywhere else -import "runtime" -import "sync" -import "time" +import ( + "runtime" + "strconv" + "strings" + "sync" + "time" +) // A wrapper around runtime.Gosched() so it doesn't need to be imported elsewhere. func Yield() { @@ -91,3 +95,16 @@ func Difference(a, b []string) []string { } return ab } + +// DecodeCoordString decodes a string representing coordinates in [1 2 3] format +// and returns a []byte. +func DecodeCoordString(in string) (out []uint64) { + s := strings.Trim(in, "[]") + t := strings.Split(s, " ") + for _, a := range t { + if u, err := strconv.ParseUint(a, 0, 64); err == nil { + out = append(out, u) + } + } + return out +} diff --git a/src/yggdrasil/api.go b/src/yggdrasil/api.go index 014c370f..f1cfc18e 100644 --- a/src/yggdrasil/api.go +++ b/src/yggdrasil/api.go @@ -6,8 +6,6 @@ import ( "fmt" "net" "sort" - "strconv" - "strings" "sync/atomic" "time" @@ -46,14 +44,14 @@ type SwitchPeer struct { // DHT searches. type DHTEntry struct { PublicKey crypto.BoxPubKey - Coords []byte + Coords []uint64 LastSeen time.Duration } // DHTRes represents a DHT response, as returned by DHTPing. type DHTRes struct { PublicKey crypto.BoxPubKey // key of the sender - Coords []byte // coords of the sender + Coords []uint64 // coords of the sender Dest crypto.NodeID // the destination node ID Infos []DHTEntry // response } @@ -166,7 +164,7 @@ func (c *Core) GetDHT() []DHTEntry { }) for _, v := range dhtentry { info := DHTEntry{ - Coords: append([]byte{}, v.coords...), + Coords: append([]uint64{}, coordsBytestoUint64s(v.coords)...), LastSeen: now.Sub(v.recv), } copy(info.PublicKey[:], v.key[:]) @@ -346,30 +344,7 @@ func (c *Core) SetNodeInfo(nodeinfo interface{}, nodeinfoprivacy bool) { // key and coordinates specified. The third parameter specifies whether a cached // result is acceptable - this results in less traffic being generated than is // necessary when, e.g. crawling the network. -func (c *Core) GetNodeInfo(keyString, coordString string, nocache bool) (NodeInfoPayload, error) { - var key crypto.BoxPubKey - if keyBytes, err := hex.DecodeString(keyString); err != nil { - return NodeInfoPayload{}, err - } else { - copy(key[:], keyBytes) - } - if !nocache { - if response, err := c.router.nodeinfo.getCachedNodeInfo(key); err == nil { - return response, nil - } - } - var coords []byte - for _, cstr := range strings.Split(strings.Trim(coordString, "[]"), " ") { - if cstr == "" { - // Special case, happens if trimmed is the empty string, e.g. this is the root - continue - } - if u64, err := strconv.ParseUint(cstr, 10, 8); err != nil { - return NodeInfoPayload{}, err - } else { - coords = append(coords, uint8(u64)) - } - } +func (c *Core) GetNodeInfo(key crypto.BoxPubKey, coords []byte, nocache bool) (NodeInfoPayload, error) { response := make(chan *NodeInfoPayload, 1) sendNodeInfoRequest := func() { c.router.nodeinfo.addCallback(key, func(nodeinfo *NodeInfoPayload) { @@ -389,7 +364,7 @@ func (c *Core) GetNodeInfo(keyString, coordString string, nocache bool) (NodeInf for res := range response { return *res, nil } - return NodeInfoPayload{}, fmt.Errorf("getNodeInfo timeout: %s", keyString) + return NodeInfoPayload{}, fmt.Errorf("getNodeInfo timeout: %s", hex.EncodeToString(key[:])) } // SetSessionGatekeeper allows you to configure a handler function for deciding @@ -477,64 +452,38 @@ func (c *Core) RemoveAllowedEncryptionPublicKey(bstr string) (err error) { // DHTPing sends a DHT ping to the node with the provided key and coords, // optionally looking up the specified target NodeID. -func (c *Core) DHTPing(keyString, coordString, targetString string) (DHTRes, error) { - var key crypto.BoxPubKey - if keyBytes, err := hex.DecodeString(keyString); err != nil { - return DHTRes{}, err - } else { - copy(key[:], keyBytes) - } - var coords []byte - for _, cstr := range strings.Split(strings.Trim(coordString, "[]"), " ") { - if cstr == "" { - // Special case, happens if trimmed is the empty string, e.g. this is the root - continue - } - if u64, err := strconv.ParseUint(cstr, 10, 8); err != nil { - return DHTRes{}, err - } else { - coords = append(coords, uint8(u64)) - } - } +func (c *Core) DHTPing(key crypto.BoxPubKey, coords []uint64, target *crypto.NodeID) (DHTRes, error) { resCh := make(chan *dhtRes, 1) info := dhtInfo{ key: key, - coords: coords, + coords: coordsUint64stoBytes(coords), } - target := *info.getNodeID() - if targetString == "none" { - // Leave the default target in place - } else if targetBytes, err := hex.DecodeString(targetString); err != nil { - return DHTRes{}, err - } else if len(targetBytes) != len(target) { - return DHTRes{}, errors.New("Incorrect target NodeID length") - } else { - var target crypto.NodeID - copy(target[:], targetBytes) + if target == nil { + target = info.getNodeID() } - rq := dhtReqKey{info.key, target} + rq := dhtReqKey{info.key, *target} sendPing := func() { c.dht.addCallback(&rq, func(res *dhtRes) { resCh <- res }) - c.dht.ping(&info, &target) + c.dht.ping(&info, &rq.dest) } c.router.doAdmin(sendPing) // TODO: do something better than the below... res := <-resCh if res != nil { r := DHTRes{ - Coords: append([]byte{}, res.Coords...), + Coords: append([]uint64{}, coordsBytestoUint64s(res.Coords)...), } copy(r.PublicKey[:], res.Key[:]) for _, i := range res.Infos { e := DHTEntry{ - Coords: append([]byte{}, i.coords...), + Coords: append([]uint64{}, coordsBytestoUint64s(i.coords)...), } copy(e.PublicKey[:], i.key[:]) r.Infos = append(r.Infos, e) } return r, nil } - return DHTRes{}, fmt.Errorf("DHT ping timeout: %s", keyString) + return DHTRes{}, fmt.Errorf("DHT ping timeout: %s", hex.EncodeToString(key[:])) } diff --git a/src/yggdrasil/switch.go b/src/yggdrasil/switch.go index 1bc40501..34b57aa6 100644 --- a/src/yggdrasil/switch.go +++ b/src/yggdrasil/switch.go @@ -137,6 +137,27 @@ type peerInfo struct { // This is just a uint64 with a named type for clarity reasons. type switchPort uint64 +func coordsUint64stoBytes(in []uint64) (out []byte) { + for _, coord := range in { + c := wire_encode_uint64(coord) + out = append(out, c...) + } + return out +} + +func coordsBytestoUint64s(in []byte) (out []uint64) { + offset := 0 + for { + coord, length := wire_decode_uint64(in[offset:]) + if length == 0 { + break + } + out = append(out, coord) + offset += length + } + return out +} + // This is the subset of the information about a peer needed to make routing decisions, and it stored separately in an atomically accessed table, which gets hammered in the "hot loop" of the routing logic (see: peer.handleTraffic in peers.go). type tableElem struct { port switchPort From 3a2ae9d902bdcdffe5a6567ed0abe990fc8ba09e Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Mon, 5 Aug 2019 10:17:19 +0100 Subject: [PATCH 23/51] Update API to represent coords as []uint64 --- src/.DS_Store | Bin 0 -> 6148 bytes src/admin/admin.go | 10 +++++++--- src/util/util.go | 2 +- src/yggdrasil/api.go | 28 ++++++++++++++-------------- src/yggdrasil/switch.go | 21 --------------------- src/yggdrasil/wire.go | 23 +++++++++++++++++++++++ 6 files changed, 45 insertions(+), 39 deletions(-) create mode 100644 src/.DS_Store diff --git a/src/.DS_Store b/src/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..5008ddfcf53c02e82d7eee2e57c38e5672ef89f6 GIT binary patch literal 6148 zcmeH~Jr2S!425mzP>H1@V-^m;4Wg<&0T*E43hX&L&p$$qDprKhvt+--jT7}7np#A3 zem<@ulZcFPQ@L2!n>{z**++&mCkOWA81W14cNZlEfg7;MkzE(HCqgga^y>{tEnwC%0;vJ&^%eQ zLs35+`xjp>T0 Date: Mon, 5 Aug 2019 10:21:40 +0100 Subject: [PATCH 24/51] Return box_pub_key as hex string in JSON (replaces #481) --- src/admin/admin.go | 143 ++------------------------------------------- 1 file changed, 4 insertions(+), 139 deletions(-) diff --git a/src/admin/admin.go b/src/admin/admin.go index 59d0f954..9789bca2 100644 --- a/src/admin/admin.go +++ b/src/admin/admin.go @@ -79,11 +79,6 @@ func (a *AdminSocket) Init(c *yggdrasil.Core, state *config.NodeState, log *log. } return Info{"list": handlers}, nil }) - /* - a.AddHandler("dot", []string{}, func(in Info) (Info, error) { - return Info{"dot": string(a.getResponse_dot())}, nil - }) - */ a.AddHandler("getSelf", []string{}, func(in Info) (Info, error) { ip := c.Address().String() subnet := c.Subnet() @@ -111,7 +106,7 @@ func (a *AdminSocket) Init(c *yggdrasil.Core, state *config.NodeState, log *log. "bytes_recvd": p.BytesRecvd, "proto": p.Protocol, "endpoint": p.Endpoint, - "box_pub_key": p.PublicKey, + "box_pub_key": hex.EncodeToString(p.PublicKey[:]), } } return Info{"peers": peers}, nil @@ -129,7 +124,7 @@ func (a *AdminSocket) Init(c *yggdrasil.Core, state *config.NodeState, log *log. "bytes_recvd": s.BytesRecvd, "proto": s.Protocol, "endpoint": s.Endpoint, - "box_pub_key": s.PublicKey, + "box_pub_key": hex.EncodeToString(s.PublicKey[:]), } } return Info{"switchpeers": switchpeers}, nil @@ -148,7 +143,7 @@ func (a *AdminSocket) Init(c *yggdrasil.Core, state *config.NodeState, log *log. dht[so] = Info{ "coords": fmt.Sprintf("%v", d.Coords), "last_seen": d.LastSeen.Seconds(), - "box_pub_key": d.PublicKey, + "box_pub_key": hex.EncodeToString(d.PublicKey[:]), } } return Info{"dht": dht}, nil @@ -165,7 +160,7 @@ func (a *AdminSocket) Init(c *yggdrasil.Core, state *config.NodeState, log *log. "mtu": s.MTU, "uptime": s.Uptime.Seconds(), "was_mtu_fixed": s.WasMTUFixed, - "box_pub_key": s.PublicKey, + "box_pub_key": hex.EncodeToString(s.PublicKey[:]), } } return Info{"sessions": sessions}, nil @@ -492,133 +487,3 @@ func (a *AdminSocket) handleRequest(conn net.Conn) { } } } - -// getResponse_dot returns a response for a graphviz dot formatted -// representation of the known parts of the network. This is color-coded and -// labeled, and includes the self node, switch peers, nodes known to the DHT, -// and nodes with open sessions. The graph is structured as a tree with directed -// links leading away from the root. -/* -func (a *AdminSocket) getResponse_dot() []byte { - //self := a.getData_getSelf() - peers := a.core.GetSwitchPeers() - dht := a.core.GetDHT() - sessions := a.core.GetSessions() - // Start building a tree from all known nodes - type nodeInfo struct { - name string - key string - parent string - port uint64 - options string - } - infos := make(map[string]nodeInfo) - // Get coords as a slice of strings, FIXME? this looks very fragile - coordSlice := func(coords string) []string { - tmp := strings.Replace(coords, "[", "", -1) - tmp = strings.Replace(tmp, "]", "", -1) - return strings.Split(tmp, " ") - } - // First fill the tree with all known nodes, no parents - addInfo := func(nodes []admin_nodeInfo, options string, tag string) { - for _, node := range nodes { - n := node.asMap() - info := nodeInfo{ - key: n["coords"].(string), - options: options, - } - if len(tag) > 0 { - info.name = fmt.Sprintf("%s\n%s", n["ip"].(string), tag) - } else { - info.name = n["ip"].(string) - } - coordsSplit := coordSlice(info.key) - if len(coordsSplit) != 0 { - portStr := coordsSplit[len(coordsSplit)-1] - portUint, err := strconv.ParseUint(portStr, 10, 64) - if err == nil { - info.port = portUint - } - } - infos[info.key] = info - } - } - addInfo(dht, "fillcolor=\"#ffffff\" style=filled fontname=\"sans serif\"", "Known in DHT") // white - addInfo(sessions, "fillcolor=\"#acf3fd\" style=filled fontname=\"sans serif\"", "Open session") // blue - addInfo(peers, "fillcolor=\"#ffffb5\" style=filled fontname=\"sans serif\"", "Connected peer") // yellow - addInfo(append([]admin_nodeInfo(nil), *self), "fillcolor=\"#a5ff8a\" style=filled fontname=\"sans serif\"", "This node") // green - // Now go through and create placeholders for any missing nodes - for _, info := range infos { - // This is ugly string manipulation - coordsSplit := coordSlice(info.key) - for idx := range coordsSplit { - key := fmt.Sprintf("[%v]", strings.Join(coordsSplit[:idx], " ")) - newInfo, isIn := infos[key] - if isIn { - continue - } - newInfo.name = "?" - newInfo.key = key - newInfo.options = "fontname=\"sans serif\" style=dashed color=\"#999999\" fontcolor=\"#999999\"" - - coordsSplit := coordSlice(newInfo.key) - if len(coordsSplit) != 0 { - portStr := coordsSplit[len(coordsSplit)-1] - portUint, err := strconv.ParseUint(portStr, 10, 64) - if err == nil { - newInfo.port = portUint - } - } - - infos[key] = newInfo - } - } - // Now go through and attach parents - for _, info := range infos { - pSplit := coordSlice(info.key) - if len(pSplit) > 0 { - pSplit = pSplit[:len(pSplit)-1] - } - info.parent = fmt.Sprintf("[%v]", strings.Join(pSplit, " ")) - infos[info.key] = info - } - // Finally, get a sorted list of keys, which we use to organize the output - var keys []string - for _, info := range infos { - keys = append(keys, info.key) - } - // sort - sort.SliceStable(keys, func(i, j int) bool { - return keys[i] < keys[j] - }) - sort.SliceStable(keys, func(i, j int) bool { - return infos[keys[i]].port < infos[keys[j]].port - }) - // Now print it all out - var out []byte - put := func(s string) { - out = append(out, []byte(s)...) - } - put("digraph {\n") - // First set the labels - for _, key := range keys { - info := infos[key] - put(fmt.Sprintf("\"%v\" [ label = \"%v\" %v ];\n", info.key, info.name, info.options)) - } - // Then print the tree structure - for _, key := range keys { - info := infos[key] - if info.key == info.parent { - continue - } // happens for the root, skip it - port := fmt.Sprint(info.port) - style := "fontname=\"sans serif\"" - if infos[info.parent].name == "?" || infos[info.key].name == "?" { - style = "fontname=\"sans serif\" style=dashed color=\"#999999\" fontcolor=\"#999999\"" - } - put(fmt.Sprintf(" \"%+v\" -> \"%+v\" [ label = \"%v\" %s ];\n", info.parent, info.key, port, style)) - } - put("}\n") - return out -} -*/ From 84a4f54217e9ad34ca4a5beab8fb0c9b7b84c370 Mon Sep 17 00:00:00 2001 From: Arceliar Date: Mon, 5 Aug 2019 18:49:15 -0500 Subject: [PATCH 25/51] temporary fix to nil pointer, better to make sure it's never nil --- src/yggdrasil/router.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/yggdrasil/router.go b/src/yggdrasil/router.go index 2df7684f..161ae60a 100644 --- a/src/yggdrasil/router.go +++ b/src/yggdrasil/router.go @@ -161,7 +161,9 @@ func (r *router) handleTraffic(packet []byte) { return } sinfo, isIn := r.core.sessions.getSessionForHandle(&p.Handle) - if !isIn { + if !isIn || sinfo.cancel == nil { + // FIXME make sure sinfo.cancel can never be nil + util.PutBytes(p.Payload) return } select { From 8a8514981769eacff68f226150cbcd6a3fa9ff34 Mon Sep 17 00:00:00 2001 From: Arceliar Date: Mon, 5 Aug 2019 18:50:08 -0500 Subject: [PATCH 26/51] remove src/.DS_Store --- src/.DS_Store | Bin 6148 -> 0 bytes 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 src/.DS_Store diff --git a/src/.DS_Store b/src/.DS_Store deleted file mode 100644 index 5008ddfcf53c02e82d7eee2e57c38e5672ef89f6..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6148 zcmeH~Jr2S!425mzP>H1@V-^m;4Wg<&0T*E43hX&L&p$$qDprKhvt+--jT7}7np#A3 zem<@ulZcFPQ@L2!n>{z**++&mCkOWA81W14cNZlEfg7;MkzE(HCqgga^y>{tEnwC%0;vJ&^%eQ zLs35+`xjp>T0 Date: Mon, 5 Aug 2019 19:11:28 -0500 Subject: [PATCH 27/51] have createSession fill the sessionInfo.cancel field, have Conn use Conn.session.cancel instead of storing its own cancellation, this should prevent any of these things from being both nil and reachable at the same time --- src/yggdrasil/conn.go | 13 +++++-------- src/yggdrasil/dialer.go | 2 +- src/yggdrasil/router.go | 3 +-- src/yggdrasil/session.go | 11 ++++++++--- 4 files changed, 15 insertions(+), 14 deletions(-) diff --git a/src/yggdrasil/conn.go b/src/yggdrasil/conn.go index 1a05bd83..2452a3d6 100644 --- a/src/yggdrasil/conn.go +++ b/src/yggdrasil/conn.go @@ -57,7 +57,6 @@ type Conn struct { core *Core readDeadline atomic.Value // time.Time // TODO timer writeDeadline atomic.Value // time.Time // TODO timer - cancel util.Cancellation mutex sync.RWMutex // protects the below nodeID *crypto.NodeID nodeMask *crypto.NodeID @@ -71,7 +70,6 @@ func newConn(core *Core, nodeID *crypto.NodeID, nodeMask *crypto.NodeID, session nodeID: nodeID, nodeMask: nodeMask, session: session, - cancel: util.NewCancellation(), } return &conn } @@ -142,10 +140,10 @@ func (c *Conn) doSearch() { func (c *Conn) getDeadlineCancellation(value *atomic.Value) util.Cancellation { if deadline, ok := value.Load().(time.Time); ok { // A deadline is set, so return a Cancellation that uses it - return util.CancellationWithDeadline(c.cancel, deadline) + return util.CancellationWithDeadline(c.session.cancel, deadline) } else { // No cancellation was set, so return a child cancellation with no timeout - return util.CancellationChild(c.cancel) + return util.CancellationChild(c.session.cancel) } } @@ -241,10 +239,9 @@ func (c *Conn) Close() (err error) { defer c.mutex.Unlock() if c.session != nil { // Close the session, if it hasn't been closed already - c.core.router.doAdmin(c.session.close) - } - if e := c.cancel.Cancel(errors.New("connection closed")); e != nil { - err = ConnError{errors.New("close failed, session already closed"), false, false, true, 0} + if e := c.session.cancel.Cancel(errors.New("connection closed")); e != nil { + err = ConnError{errors.New("close failed, session already closed"), false, false, true, 0} + } } return } diff --git a/src/yggdrasil/dialer.go b/src/yggdrasil/dialer.go index 6ce2e8ac..db5d5a4d 100644 --- a/src/yggdrasil/dialer.go +++ b/src/yggdrasil/dialer.go @@ -69,7 +69,7 @@ func (d *Dialer) DialByNodeIDandMask(nodeID, nodeMask *crypto.NodeID) (*Conn, er defer t.Stop() select { case <-conn.session.init: - conn.session.startWorkers(conn.cancel) + conn.session.startWorkers() return conn, nil case <-t.C: conn.Close() diff --git a/src/yggdrasil/router.go b/src/yggdrasil/router.go index 161ae60a..a11f6ae5 100644 --- a/src/yggdrasil/router.go +++ b/src/yggdrasil/router.go @@ -161,8 +161,7 @@ func (r *router) handleTraffic(packet []byte) { return } sinfo, isIn := r.core.sessions.getSessionForHandle(&p.Handle) - if !isIn || sinfo.cancel == nil { - // FIXME make sure sinfo.cancel can never be nil + if !isIn { util.PutBytes(p.Payload) return } diff --git a/src/yggdrasil/session.go b/src/yggdrasil/session.go index c39f60de..f9c38faa 100644 --- a/src/yggdrasil/session.go +++ b/src/yggdrasil/session.go @@ -208,6 +208,7 @@ func (ss *sessions) createSession(theirPermKey *crypto.BoxPubKey) *sessionInfo { sinfo.pingTime = now sinfo.pingSend = now sinfo.init = make(chan struct{}) + sinfo.cancel = util.NewCancellation() higher := false for idx := range ss.core.boxPub { if ss.core.boxPub[idx] > sinfo.theirPermPub[idx] { @@ -232,6 +233,11 @@ func (ss *sessions) createSession(theirPermKey *crypto.BoxPubKey) *sessionInfo { sinfo.send = make(chan []byte, 32) ss.sinfos[sinfo.myHandle] = &sinfo ss.byTheirPerm[sinfo.theirPermPub] = &sinfo.myHandle + go func() { + // Run cleanup when the session is canceled + <-sinfo.cancel.Finished() + sinfo.core.router.doAdmin(sinfo.close) + }() return &sinfo } @@ -362,7 +368,7 @@ func (ss *sessions) handlePing(ping *sessionPing) { for i := range conn.nodeMask { conn.nodeMask[i] = 0xFF } - conn.session.startWorkers(conn.cancel) + conn.session.startWorkers() ss.listener.conn <- conn } ss.listenerMutex.Unlock() @@ -431,8 +437,7 @@ func (ss *sessions) reset() { //////////////////////////// Worker Functions Below //////////////////////////// //////////////////////////////////////////////////////////////////////////////// -func (sinfo *sessionInfo) startWorkers(cancel util.Cancellation) { - sinfo.cancel = cancel +func (sinfo *sessionInfo) startWorkers() { go sinfo.recvWorker() go sinfo.sendWorker() } From 790524bd1c6afa4a4a64f4551ec019d8e77ff86a Mon Sep 17 00:00:00 2001 From: Arceliar Date: Tue, 6 Aug 2019 19:25:55 -0500 Subject: [PATCH 28/51] copy/paste old flowkey logic into a util function, add a struct of key and packet, make WriteNoCopy accept this instead of a slice --- src/tuntap/conn.go | 7 ++- src/util/util.go | 38 ++++++++++++++++ src/yggdrasil/conn.go | 12 ++--- src/yggdrasil/router.go | 2 +- src/yggdrasil/session.go | 98 ++++++++++++++++++++++------------------ 5 files changed, 105 insertions(+), 52 deletions(-) diff --git a/src/tuntap/conn.go b/src/tuntap/conn.go index ab3179bf..61cdb2b4 100644 --- a/src/tuntap/conn.go +++ b/src/tuntap/conn.go @@ -96,8 +96,11 @@ func (s *tunConn) writer() error { if !ok { return errors.New("send closed") } - // TODO write timeout and close - if err := s.conn.WriteNoCopy(bs); err != nil { + msg := yggdrasil.FlowKeyMessage{ + FlowKey: util.GetFlowKey(bs), + Message: bs, + } + if err := s.conn.WriteNoCopy(msg); err != nil { if e, eok := err.(yggdrasil.ConnError); !eok { if e.Closed() { s.tun.log.Debugln(s.conn.String(), "TUN/TAP generic write debug:", err) diff --git a/src/util/util.go b/src/util/util.go index 1158156c..a588a35c 100644 --- a/src/util/util.go +++ b/src/util/util.go @@ -106,3 +106,41 @@ func DecodeCoordString(in string) (out []uint64) { } return out } + +// GetFlowLabel takes an IP packet as an argument and returns some information about the traffic flow. +// For IPv4 packets, this is derived from the source and destination protocol and port numbers. +// For IPv6 packets, this is derived from the FlowLabel field of the packet if this was set, otherwise it's handled like IPv4. +// The FlowKey is then used internally by Yggdrasil for congestion control. +func GetFlowKey(bs []byte) uint64 { + // Work out the flowkey - this is used to determine which switch queue + // traffic will be pushed to in the event of congestion + var flowkey uint64 + // Get the IP protocol version from the packet + switch bs[0] & 0xf0 { + case 0x40: // IPv4 packet + // Check the packet meets minimum UDP packet length + if len(bs) >= 24 { + // Is the protocol TCP, UDP or SCTP? + if bs[9] == 0x06 || bs[9] == 0x11 || bs[9] == 0x84 { + ihl := bs[0] & 0x0f * 4 // Header length + flowkey = uint64(bs[9])<<32 /* proto */ | + uint64(bs[ihl+0])<<24 | uint64(bs[ihl+1])<<16 /* sport */ | + uint64(bs[ihl+2])<<8 | uint64(bs[ihl+3]) /* dport */ + } + } + case 0x60: // IPv6 packet + // Check if the flowlabel was specified in the packet header + flowkey = uint64(bs[1]&0x0f)<<16 | uint64(bs[2])<<8 | uint64(bs[3]) + // If the flowlabel isn't present, make protokey from proto | sport | dport + // if the packet meets minimum UDP packet length + if flowkey == 0 && len(bs) >= 48 { + // Is the protocol TCP, UDP or SCTP? + if bs[6] == 0x06 || bs[6] == 0x11 || bs[6] == 0x84 { + flowkey = uint64(bs[6])<<32 /* proto */ | + uint64(bs[40])<<24 | uint64(bs[41])<<16 /* sport */ | + uint64(bs[42])<<8 | uint64(bs[43]) /* dport */ + } + } + } + return flowkey +} diff --git a/src/yggdrasil/conn.go b/src/yggdrasil/conn.go index 2452a3d6..20db931d 100644 --- a/src/yggdrasil/conn.go +++ b/src/yggdrasil/conn.go @@ -183,11 +183,11 @@ func (c *Conn) Read(b []byte) (int, error) { } // Used internally by Write, the caller must not reuse the argument bytes when no error occurs -func (c *Conn) WriteNoCopy(bs []byte) error { +func (c *Conn) WriteNoCopy(msg FlowKeyMessage) error { var err error sessionFunc := func() { // Does the packet exceed the permitted size for the session? - if uint16(len(bs)) > c.session.getMTU() { + if uint16(len(msg.Message)) > c.session.getMTU() { err = ConnError{errors.New("packet too big"), true, false, false, int(c.session.getMTU())} return } @@ -216,7 +216,7 @@ func (c *Conn) WriteNoCopy(bs []byte) error { } else { err = ConnError{errors.New("session closed"), false, false, true, 0} } - case c.session.send <- bs: + case c.session.send <- msg: } } return err @@ -225,10 +225,10 @@ func (c *Conn) WriteNoCopy(bs []byte) error { // Implements net.Conn.Write func (c *Conn) Write(b []byte) (int, error) { written := len(b) - bs := append(util.GetBytes(), b...) - err := c.WriteNoCopy(bs) + msg := FlowKeyMessage{Message: append(util.GetBytes(), b...)} + err := c.WriteNoCopy(msg) if err != nil { - util.PutBytes(bs) + util.PutBytes(msg.Message) written = 0 } return written, err diff --git a/src/yggdrasil/router.go b/src/yggdrasil/router.go index a11f6ae5..7e2a325a 100644 --- a/src/yggdrasil/router.go +++ b/src/yggdrasil/router.go @@ -166,7 +166,7 @@ func (r *router) handleTraffic(packet []byte) { return } select { - case sinfo.fromRouter <- &p: + case sinfo.fromRouter <- p: case <-sinfo.cancel.Finished(): util.PutBytes(p.Payload) } diff --git a/src/yggdrasil/session.go b/src/yggdrasil/session.go index f9c38faa..517947e8 100644 --- a/src/yggdrasil/session.go +++ b/src/yggdrasil/session.go @@ -18,39 +18,39 @@ import ( // All the information we know about an active session. // This includes coords, permanent and ephemeral keys, handles and nonces, various sorts of timing information for timeout and maintenance, and some metadata for the admin API. type sessionInfo struct { - mutex sync.Mutex // Protects all of the below, use it any time you read/chance the contents of a session - core *Core // - reconfigure chan chan error // - theirAddr address.Address // - theirSubnet address.Subnet // - theirPermPub crypto.BoxPubKey // - theirSesPub crypto.BoxPubKey // - mySesPub crypto.BoxPubKey // - mySesPriv crypto.BoxPrivKey // - sharedSesKey crypto.BoxSharedKey // derived from session keys - theirHandle crypto.Handle // - myHandle crypto.Handle // - theirNonce crypto.BoxNonce // - theirNonceMask uint64 // - myNonce crypto.BoxNonce // - theirMTU uint16 // - myMTU uint16 // - wasMTUFixed bool // Was the MTU fixed by a receive error? - timeOpened time.Time // Time the sessino was opened - time time.Time // Time we last received a packet - mtuTime time.Time // time myMTU was last changed - pingTime time.Time // time the first ping was sent since the last received packet - pingSend time.Time // time the last ping was sent - coords []byte // coords of destination - reset bool // reset if coords change - tstamp int64 // ATOMIC - tstamp from their last session ping, replay attack mitigation - bytesSent uint64 // Bytes of real traffic sent in this session - bytesRecvd uint64 // Bytes of real traffic received in this session - fromRouter chan *wire_trafficPacket // Received packets go here, picked up by the associated Conn - init chan struct{} // Closed when the first session pong arrives, used to signal that the session is ready for initial use - cancel util.Cancellation // Used to terminate workers - recv chan []byte - send chan []byte + mutex sync.Mutex // Protects all of the below, use it any time you read/chance the contents of a session + core *Core // + reconfigure chan chan error // + theirAddr address.Address // + theirSubnet address.Subnet // + theirPermPub crypto.BoxPubKey // + theirSesPub crypto.BoxPubKey // + mySesPub crypto.BoxPubKey // + mySesPriv crypto.BoxPrivKey // + sharedSesKey crypto.BoxSharedKey // derived from session keys + theirHandle crypto.Handle // + myHandle crypto.Handle // + theirNonce crypto.BoxNonce // + theirNonceMask uint64 // + myNonce crypto.BoxNonce // + theirMTU uint16 // + myMTU uint16 // + wasMTUFixed bool // Was the MTU fixed by a receive error? + timeOpened time.Time // Time the sessino was opened + time time.Time // Time we last received a packet + mtuTime time.Time // time myMTU was last changed + pingTime time.Time // time the first ping was sent since the last received packet + pingSend time.Time // time the last ping was sent + coords []byte // coords of destination + reset bool // reset if coords change + tstamp int64 // ATOMIC - tstamp from their last session ping, replay attack mitigation + bytesSent uint64 // Bytes of real traffic sent in this session + bytesRecvd uint64 // Bytes of real traffic received in this session + init chan struct{} // Closed when the first session pong arrives, used to signal that the session is ready for initial use + cancel util.Cancellation // Used to terminate workers + fromRouter chan wire_trafficPacket // Received packets go here, to be decrypted by the session + recv chan []byte // Decrypted packets go here, picked up by the associated Conn + send chan FlowKeyMessage // Packets with optional flow key go here, to be encrypted and sent } func (sinfo *sessionInfo) doFunc(f func()) { @@ -228,9 +228,9 @@ func (ss *sessions) createSession(theirPermKey *crypto.BoxPubKey) *sessionInfo { sinfo.myHandle = *crypto.NewHandle() sinfo.theirAddr = *address.AddrForNodeID(crypto.GetNodeID(&sinfo.theirPermPub)) sinfo.theirSubnet = *address.SubnetForNodeID(crypto.GetNodeID(&sinfo.theirPermPub)) - sinfo.fromRouter = make(chan *wire_trafficPacket, 1) + sinfo.fromRouter = make(chan wire_trafficPacket, 1) sinfo.recv = make(chan []byte, 32) - sinfo.send = make(chan []byte, 32) + sinfo.send = make(chan FlowKeyMessage, 32) ss.sinfos[sinfo.myHandle] = &sinfo ss.byTheirPerm[sinfo.theirPermPub] = &sinfo.myHandle go func() { @@ -442,13 +442,18 @@ func (sinfo *sessionInfo) startWorkers() { go sinfo.sendWorker() } +type FlowKeyMessage struct { + FlowKey uint64 + Message []byte +} + func (sinfo *sessionInfo) recvWorker() { // TODO move theirNonce etc into a struct that gets stored here, passed in over a channel // Since there's no reason for anywhere else in the session code to need to *read* it... // Only needs to be updated from the outside if a ping resets it... // That would get rid of the need to take a mutex for the sessionFunc var callbacks []chan func() - doRecv := func(p *wire_trafficPacket) { + doRecv := func(p wire_trafficPacket) { var bs []byte var err error var k crypto.BoxSharedKey @@ -524,16 +529,22 @@ func (sinfo *sessionInfo) sendWorker() { // TODO move info that this worker needs here, send updates via a channel // Otherwise we need to take a mutex to avoid races with update() var callbacks []chan func() - doSend := func(bs []byte) { + doSend := func(msg FlowKeyMessage) { var p wire_trafficPacket var k crypto.BoxSharedKey sessionFunc := func() { - sinfo.bytesSent += uint64(len(bs)) + sinfo.bytesSent += uint64(len(msg.Message)) p = wire_trafficPacket{ Coords: append([]byte(nil), sinfo.coords...), Handle: sinfo.theirHandle, Nonce: sinfo.myNonce, } + if msg.FlowKey != 0 { + // Helps ensure that traffic from this flow ends up in a separate queue from other flows + // The zero padding relies on the fact that the self-peer is always on port 0 + p.Coords = append(p.Coords, 0) + p.Coords = wire_put_uint64(msg.FlowKey, p.Coords) + } sinfo.myNonce.Increment() k = sinfo.sharedSesKey } @@ -542,12 +553,13 @@ func (sinfo *sessionInfo) sendWorker() { ch := make(chan func(), 1) poolFunc := func() { // Encrypt the packet - p.Payload, _ = crypto.BoxSeal(&k, bs, &p.Nonce) - packet := p.encode() + p.Payload, _ = crypto.BoxSeal(&k, msg.Message, &p.Nonce) // The callback will send the packet callback := func() { + // Encoding may block on a util.GetBytes(), so kept out of the worker pool + packet := p.encode() // Cleanup - util.PutBytes(bs) + util.PutBytes(msg.Message) util.PutBytes(p.Payload) // Send the packet sinfo.core.router.out(packet) @@ -566,8 +578,8 @@ func (sinfo *sessionInfo) sendWorker() { f() case <-sinfo.cancel.Finished(): return - case bs := <-sinfo.send: - doSend(bs) + case msg := <-sinfo.send: + doSend(msg) } } select { From d795ab1b650e5437ba3b7ca91dbccfcf1b6e00f5 Mon Sep 17 00:00:00 2001 From: Arceliar Date: Tue, 6 Aug 2019 20:51:38 -0500 Subject: [PATCH 29/51] minor allocation fix --- src/yggdrasil/stream.go | 1 + 1 file changed, 1 insertion(+) diff --git a/src/yggdrasil/stream.go b/src/yggdrasil/stream.go index 011943f5..56d4754a 100644 --- a/src/yggdrasil/stream.go +++ b/src/yggdrasil/stream.go @@ -44,6 +44,7 @@ func (s *stream) writeMsg(bs []byte) (int, error) { padLen := len(buf[0]) + len(buf[1]) buf = append(buf, bs) totalLen := padLen + len(bs) + s.outputBuffer = buf[:0] // So we can reuse the same underlying array later var bn int for bn < totalLen { n, err := buf.WriteTo(s.rwc) From bbb35d72094ddb912c1ff5a660bf9fe36aee34b9 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Wed, 7 Aug 2019 10:52:19 +0100 Subject: [PATCH 30/51] Transform Listen statement to new format if needed --- cmd/yggdrasil/main.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/cmd/yggdrasil/main.go b/cmd/yggdrasil/main.go index 129b01d5..ee8ebe8e 100644 --- a/cmd/yggdrasil/main.go +++ b/cmd/yggdrasil/main.go @@ -74,6 +74,12 @@ func readConfig(useconf *bool, useconffile *string, normaliseconf *bool) *config if err := hjson.Unmarshal(conf, &dat); err != nil { panic(err) } + // Check for fields that have changed type recently, e.g. the Listen config + // option is now a []string rather than a string + if listen, ok := dat["Listen"].(string); ok { + dat["Listen"] = []string{listen} + } + // Sanitise the config confJson, err := json.Marshal(dat) if err != nil { panic(err) From 9ab08446ff07e77ba25941c0be55aebe2bc372c0 Mon Sep 17 00:00:00 2001 From: Arceliar Date: Wed, 7 Aug 2019 17:40:50 -0500 Subject: [PATCH 31/51] make sure the sessionInfo.recvWorker doesn't block if sinfo.recv somehow fills --- src/yggdrasil/session.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/yggdrasil/session.go b/src/yggdrasil/session.go index 517947e8..d8b7e9b3 100644 --- a/src/yggdrasil/session.go +++ b/src/yggdrasil/session.go @@ -495,7 +495,11 @@ func (sinfo *sessionInfo) recvWorker() { util.PutBytes(bs) } else { // Pass the packet to the buffer for Conn.Read - sinfo.recv <- bs + select { + case <-sinfo.cancel.Finished(): + util.PutBytes(bs) + case sinfo.recv <- bs: + } } } ch <- callback From 5e81a0c42167f4ff882e89fb2d0f5b17433c3d3f Mon Sep 17 00:00:00 2001 From: Arceliar Date: Wed, 7 Aug 2019 18:08:31 -0500 Subject: [PATCH 32/51] Use a separate buffer per session for incoming packets, so 1 session that floods won't block other sessions --- src/yggdrasil/session.go | 30 ++++++++++++++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/src/yggdrasil/session.go b/src/yggdrasil/session.go index d8b7e9b3..da66ad59 100644 --- a/src/yggdrasil/session.go +++ b/src/yggdrasil/session.go @@ -508,6 +508,32 @@ func (sinfo *sessionInfo) recvWorker() { util.WorkerGo(poolFunc) callbacks = append(callbacks, ch) } + fromHelper := make(chan wire_trafficPacket, 1) + go func() { + var buf []wire_trafficPacket + for { + for len(buf) > 0 { + select { + case <-sinfo.cancel.Finished(): + return + case p := <-sinfo.fromRouter: + buf = append(buf, p) + for len(buf) > 64 { // Based on nonce window size + util.PutBytes(buf[0].Payload) + buf = buf[1:] + } + case fromHelper <- buf[0]: + buf = buf[1:] + } + } + select { + case <-sinfo.cancel.Finished(): + return + case p := <-sinfo.fromRouter: + buf = append(buf, p) + } + } + }() for { for len(callbacks) > 0 { select { @@ -516,14 +542,14 @@ func (sinfo *sessionInfo) recvWorker() { f() case <-sinfo.cancel.Finished(): return - case p := <-sinfo.fromRouter: + case p := <-fromHelper: doRecv(p) } } select { case <-sinfo.cancel.Finished(): return - case p := <-sinfo.fromRouter: + case p := <-fromHelper: doRecv(p) } } From 589ad638eab4a38060f39061c00ad81be7cc2fbe Mon Sep 17 00:00:00 2001 From: Slex Date: Sun, 11 Aug 2019 00:31:22 +0300 Subject: [PATCH 33/51] Implement feature from https://github.com/yggdrasil-network/yggdrasil-go/issues/488 --- build | 2 +- cmd/yggdrasil/main.go | 11 ++++++----- cmd/yggdrasilctl/main.go | 9 +++++++++ src/admin/admin.go | 5 +++-- src/version/version.go | 22 ++++++++++++++++++++++ src/yggdrasil/api.go | 18 ------------------ src/yggdrasil/core.go | 8 +++----- src/yggdrasil/nodeinfo.go | 5 +++-- 8 files changed, 47 insertions(+), 33 deletions(-) create mode 100644 src/version/version.go diff --git a/build b/build index 80a971f9..c1e5f863 100755 --- a/build +++ b/build @@ -2,7 +2,7 @@ set -ef -PKGSRC=${PKGSRC:-github.com/yggdrasil-network/yggdrasil-go/src/yggdrasil} +PKGSRC=${PKGSRC:-github.com/yggdrasil-network/yggdrasil-go/src/version} PKGNAME=${PKGNAME:-$(sh contrib/semver/name.sh)} PKGVER=${PKGVER:-$(sh contrib/semver/version.sh --bare)} diff --git a/cmd/yggdrasil/main.go b/cmd/yggdrasil/main.go index ee8ebe8e..36494c40 100644 --- a/cmd/yggdrasil/main.go +++ b/cmd/yggdrasil/main.go @@ -25,6 +25,7 @@ import ( "github.com/yggdrasil-network/yggdrasil-go/src/crypto" "github.com/yggdrasil-network/yggdrasil-go/src/multicast" "github.com/yggdrasil-network/yggdrasil-go/src/tuntap" + "github.com/yggdrasil-network/yggdrasil-go/src/version" "github.com/yggdrasil-network/yggdrasil-go/src/yggdrasil" ) @@ -119,7 +120,7 @@ func main() { normaliseconf := flag.Bool("normaliseconf", false, "use in combination with either -useconf or -useconffile, outputs your configuration normalised") confjson := flag.Bool("json", false, "print configuration from -genconf or -normaliseconf as JSON instead of HJSON") autoconf := flag.Bool("autoconf", false, "automatic mode (dynamic IP, peer with IPv6 neighbors)") - version := flag.Bool("version", false, "prints the version of this build") + ver := flag.Bool("version", false, "prints the version of this build") logging := flag.String("logging", "info,warn,error", "comma-separated list of logging levels to enable") logto := flag.String("logto", "stdout", "file path to log to, \"syslog\" or \"stdout\"") flag.Parse() @@ -127,9 +128,9 @@ func main() { var cfg *config.NodeConfig var err error switch { - case *version: - fmt.Println("Build name:", yggdrasil.BuildName()) - fmt.Println("Build version:", yggdrasil.BuildVersion()) + case *ver: + fmt.Println("Build name:", version.BuildName()) + fmt.Println("Build version:", version.BuildVersion()) os.Exit(0) case *autoconf: // Use an autoconf-generated config, this will give us random keys and @@ -174,7 +175,7 @@ func main() { case "stdout": logger = log.New(os.Stdout, "", log.Flags()) case "syslog": - if syslogger, err := gsyslog.NewLogger(gsyslog.LOG_NOTICE, "DAEMON", yggdrasil.BuildName()); err == nil { + if syslogger, err := gsyslog.NewLogger(gsyslog.LOG_NOTICE, "DAEMON", version.BuildName()); err == nil { logger = log.New(syslogger, "", log.Flags()) } default: diff --git a/cmd/yggdrasilctl/main.go b/cmd/yggdrasilctl/main.go index 51f4fa51..4cc9745d 100644 --- a/cmd/yggdrasilctl/main.go +++ b/cmd/yggdrasilctl/main.go @@ -19,6 +19,7 @@ import ( "github.com/hjson/hjson-go" "github.com/yggdrasil-network/yggdrasil-go/src/defaults" + "github.com/yggdrasil-network/yggdrasil-go/src/version" ) type admin_info map[string]interface{} @@ -53,9 +54,17 @@ func main() { server := flag.String("endpoint", endpoint, "Admin socket endpoint") injson := flag.Bool("json", false, "Output in JSON format (as opposed to pretty-print)") verbose := flag.Bool("v", false, "Verbose output (includes public keys)") + ver := flag.Bool("version", false, "Prints the version of this build") flag.Parse() args := flag.Args() + if *ver { + fmt.Println(os.Args[0], "build name:", version.BuildName()) + fmt.Println(os.Args[0], "version:", version.BuildVersion()) + fmt.Println("\nFor get yggdrasil version use\n - ", os.Args[0], "getSelf") + os.Exit(0) + } + if len(args) == 0 { flag.Usage() return diff --git a/src/admin/admin.go b/src/admin/admin.go index 9789bca2..c7fc151d 100644 --- a/src/admin/admin.go +++ b/src/admin/admin.go @@ -18,6 +18,7 @@ import ( "github.com/yggdrasil-network/yggdrasil-go/src/config" "github.com/yggdrasil-network/yggdrasil-go/src/crypto" "github.com/yggdrasil-network/yggdrasil-go/src/util" + "github.com/yggdrasil-network/yggdrasil-go/src/version" "github.com/yggdrasil-network/yggdrasil-go/src/yggdrasil" ) @@ -86,8 +87,8 @@ func (a *AdminSocket) Init(c *yggdrasil.Core, state *config.NodeState, log *log. "self": Info{ ip: Info{ "box_pub_key": c.EncryptionPublicKey(), - "build_name": yggdrasil.BuildName(), - "build_version": yggdrasil.BuildVersion(), + "build_name": version.BuildName(), + "build_version": version.BuildVersion(), "coords": fmt.Sprintf("%v", c.Coords()), "subnet": subnet.String(), }, diff --git a/src/version/version.go b/src/version/version.go new file mode 100644 index 00000000..4cc7a8f2 --- /dev/null +++ b/src/version/version.go @@ -0,0 +1,22 @@ +package version + +var buildName string +var buildVersion string + +// BuildName gets the current build name. This is usually injected if built +// from git, or returns "unknown" otherwise. +func BuildName() string { + if buildName == "" { + return "yggdrasilctl" + } + return buildName +} + +// BuildVersion gets the current build version. This is usually injected if +// built from git, or returns "unknown" otherwise. +func BuildVersion() string { + if buildVersion == "" { + return "unknown" + } + return buildVersion +} diff --git a/src/yggdrasil/api.go b/src/yggdrasil/api.go index 98f90130..57ee5c65 100644 --- a/src/yggdrasil/api.go +++ b/src/yggdrasil/api.go @@ -241,24 +241,6 @@ func (c *Core) GetSessions() []Session { return sessions } -// BuildName gets the current build name. This is usually injected if built -// from git, or returns "unknown" otherwise. -func BuildName() string { - if buildName == "" { - return "yggdrasil" - } - return buildName -} - -// BuildVersion gets the current build version. This is usually injected if -// built from git, or returns "unknown" otherwise. -func BuildVersion() string { - if buildVersion == "" { - return "unknown" - } - return buildVersion -} - // ConnListen returns a listener for Yggdrasil session connections. func (c *Core) ConnListen() (*Listener, error) { c.sessions.listenerMutex.Lock() diff --git a/src/yggdrasil/core.go b/src/yggdrasil/core.go index 2aa78347..0921ab9f 100644 --- a/src/yggdrasil/core.go +++ b/src/yggdrasil/core.go @@ -10,11 +10,9 @@ import ( "github.com/yggdrasil-network/yggdrasil-go/src/config" "github.com/yggdrasil-network/yggdrasil-go/src/crypto" + "github.com/yggdrasil-network/yggdrasil-go/src/version" ) -var buildName string -var buildVersion string - // The Core object represents the Yggdrasil node. You should create a Core // object for each Yggdrasil node you plan to run. type Core struct { @@ -164,10 +162,10 @@ func (c *Core) Start(nc *config.NodeConfig, log *log.Logger) (*config.NodeState, Previous: *nc, } - if name := BuildName(); name != "unknown" { + if name := version.BuildName(); name != "unknown" { c.log.Infoln("Build name:", name) } - if version := BuildVersion(); version != "unknown" { + if version := version.BuildVersion(); version != "unknown" { c.log.Infoln("Build version:", version) } diff --git a/src/yggdrasil/nodeinfo.go b/src/yggdrasil/nodeinfo.go index f1c7ed07..73d4e115 100644 --- a/src/yggdrasil/nodeinfo.go +++ b/src/yggdrasil/nodeinfo.go @@ -9,6 +9,7 @@ import ( "time" "github.com/yggdrasil-network/yggdrasil-go/src/crypto" + "github.com/yggdrasil-network/yggdrasil-go/src/version" ) type nodeinfo struct { @@ -99,8 +100,8 @@ func (m *nodeinfo) setNodeInfo(given interface{}, privacy bool) error { m.myNodeInfoMutex.Lock() defer m.myNodeInfoMutex.Unlock() defaults := map[string]interface{}{ - "buildname": BuildName(), - "buildversion": BuildVersion(), + "buildname": version.BuildName(), + "buildversion": version.BuildVersion(), "buildplatform": runtime.GOOS, "buildarch": runtime.GOARCH, } From 7a28eb787ee860e1555af52673b046ccb4b0d56c Mon Sep 17 00:00:00 2001 From: Arceliar Date: Sun, 11 Aug 2019 13:00:19 -0500 Subject: [PATCH 34/51] try to fix a few edge cases with searches that could lead them to ending without the callback being run or without cleaning up the old search info --- src/yggdrasil/conn.go | 4 ++-- src/yggdrasil/search.go | 14 ++------------ 2 files changed, 4 insertions(+), 14 deletions(-) diff --git a/src/yggdrasil/conn.go b/src/yggdrasil/conn.go index 20db931d..134f3cd2 100644 --- a/src/yggdrasil/conn.go +++ b/src/yggdrasil/conn.go @@ -130,9 +130,9 @@ func (c *Conn) doSearch() { searchCompleted := func(sinfo *sessionInfo, e error) {} sinfo = c.core.searches.newIterSearch(c.nodeID, c.nodeMask, searchCompleted) c.core.log.Debugf("%s DHT search started: %p", c.String(), sinfo) + // Start the search + sinfo.continueSearch() } - // Continue the search - sinfo.continueSearch() } go func() { c.core.router.admin <- routerWork }() } diff --git a/src/yggdrasil/search.go b/src/yggdrasil/search.go index 4c31fd6b..676ac4f9 100644 --- a/src/yggdrasil/search.go +++ b/src/yggdrasil/search.go @@ -36,7 +36,6 @@ type searchInfo struct { core *Core dest crypto.NodeID mask crypto.NodeID - time time.Time toVisit []*dhtInfo visited map[crypto.NodeID]bool callback func(*sessionInfo, error) @@ -65,17 +64,10 @@ func (s *searches) init(core *Core) { // Creates a new search info, adds it to the searches struct, and returns a pointer to the info. func (s *searches) createSearch(dest *crypto.NodeID, mask *crypto.NodeID, callback func(*sessionInfo, error)) *searchInfo { - now := time.Now() - //for dest, sinfo := range s.searches { - // if now.Sub(sinfo.time) > time.Minute { - // delete(s.searches, dest) - // } - //} info := searchInfo{ core: s.core, dest: *dest, mask: *mask, - time: now.Add(-time.Second), callback: callback, } s.searches[*dest] = &info @@ -154,10 +146,6 @@ func (sinfo *searchInfo) doSearchStep() { // If we've recenty sent a ping for this search, do nothing. // Otherwise, doSearchStep and schedule another continueSearch to happen after search_RETRY_TIME. func (sinfo *searchInfo) continueSearch() { - if time.Since(sinfo.time) < search_RETRY_TIME { - return - } - sinfo.time = time.Now() sinfo.doSearchStep() // In case the search dies, try to spawn another thread later // Note that this will spawn multiple parallel searches as time passes @@ -209,6 +197,8 @@ func (sinfo *searchInfo) checkDHTRes(res *dhtRes) bool { if sess == nil { // nil if the DHT search finished but the session wasn't allowed sinfo.callback(nil, errors.New("session not allowed")) + // Cleanup + delete(sinfo.core.searches.searches, res.Dest) return true } _, isIn := sinfo.core.sessions.getByTheirPerm(&res.Key) From 277da1fe60fb6e32bfdbe572ff23f3fc585dbd99 Mon Sep 17 00:00:00 2001 From: Arceliar Date: Sun, 11 Aug 2019 13:11:14 -0500 Subject: [PATCH 35/51] make sure searches don't end if try to continue (in parallel) with nowhere left to send, but we just sent a search and are still waiting for a response --- src/yggdrasil/search.go | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/yggdrasil/search.go b/src/yggdrasil/search.go index 676ac4f9..b970fe55 100644 --- a/src/yggdrasil/search.go +++ b/src/yggdrasil/search.go @@ -36,6 +36,7 @@ type searchInfo struct { core *Core dest crypto.NodeID mask crypto.NodeID + time time.Time toVisit []*dhtInfo visited map[crypto.NodeID]bool callback func(*sessionInfo, error) @@ -68,6 +69,7 @@ func (s *searches) createSearch(dest *crypto.NodeID, mask *crypto.NodeID, callba core: s.core, dest: *dest, mask: *mask, + time: time.Now(), callback: callback, } s.searches[*dest] = &info @@ -130,9 +132,11 @@ func (sinfo *searchInfo) addToSearch(res *dhtRes) { // Otherwise, it pops the closest node to the destination (in keyspace) off of the toVisit list and sends a dht ping. func (sinfo *searchInfo) doSearchStep() { if len(sinfo.toVisit) == 0 { - // Dead end, do cleanup - delete(sinfo.core.searches.searches, sinfo.dest) - sinfo.callback(nil, errors.New("search reached dead end")) + if time.Since(sinfo.time) > search_RETRY_TIME { + // Dead end and no response in too long, do cleanup + delete(sinfo.core.searches.searches, sinfo.dest) + sinfo.callback(nil, errors.New("search reached dead end")) + } return } // Send to the next search target @@ -141,6 +145,7 @@ func (sinfo *searchInfo) doSearchStep() { rq := dhtReqKey{next.key, sinfo.dest} sinfo.core.dht.addCallback(&rq, sinfo.handleDHTRes) sinfo.core.dht.ping(next, &sinfo.dest) + sinfo.time = time.Now() } // If we've recenty sent a ping for this search, do nothing. From 70a118ae98164e93ad122d187a18f1a0d0403b90 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Mon, 12 Aug 2019 11:41:29 +0100 Subject: [PATCH 36/51] Update go.mod/go.sum --- go.mod | 6 +++--- go.sum | 4 ++++ 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 84025df3..c7773cc4 100644 --- a/go.mod +++ b/go.mod @@ -8,10 +8,10 @@ require ( github.com/kardianos/minwinsvc v0.0.0-20151122163309-cad6b2b879b0 github.com/mitchellh/mapstructure v1.1.2 github.com/songgao/packets v0.0.0-20160404182456-549a10cd4091 - github.com/yggdrasil-network/water v0.0.0-20190725123504-a16161896c34 + github.com/yggdrasil-network/water v0.0.0-20190812103929-c83fe40250f8 golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4 golang.org/x/net v0.0.0-20190724013045-ca1201d0de80 - golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3 + golang.org/x/sys v0.0.0-20190812073006-9eafafc0a87e golang.org/x/text v0.3.2 - golang.org/x/tools v0.0.0-20190802003818-e9bb7d36c060 // indirect + golang.org/x/tools v0.0.0-20190809145639-6d4652c779c4 // indirect ) diff --git a/go.sum b/go.sum index 81d337e1..0369a785 100644 --- a/go.sum +++ b/go.sum @@ -22,6 +22,7 @@ github.com/yggdrasil-network/water v0.0.0-20190725073841-250edb919f8a h1:mQ0mPD+ github.com/yggdrasil-network/water v0.0.0-20190725073841-250edb919f8a/go.mod h1:R0SBCsugm+Sf1katgTb2t7GXMm+nRIv43tM4VDZbaOs= github.com/yggdrasil-network/water v0.0.0-20190725123504-a16161896c34 h1:Qh5FE+Q5iGqpmR/FPMYHuoZLN921au/nxAlmKe+Hdbo= github.com/yggdrasil-network/water v0.0.0-20190725123504-a16161896c34/go.mod h1:R0SBCsugm+Sf1katgTb2t7GXMm+nRIv43tM4VDZbaOs= +github.com/yggdrasil-network/water v0.0.0-20190812103929-c83fe40250f8/go.mod h1:R0SBCsugm+Sf1katgTb2t7GXMm+nRIv43tM4VDZbaOs= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9 h1:mKdxBk7AujPs8kU4m80U72y/zjbZ3UcXC7dClwKbUI0= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= @@ -44,6 +45,7 @@ golang.org/x/sys v0.0.0-20190712062909-fae7ac547cb7/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3 h1:4y9KwBHBgBNwDbtu44R5o1fdOCQUEXhbk/P4A9WmJq0= golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190812073006-9eafafc0a87e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= @@ -53,3 +55,5 @@ golang.org/x/tools v0.0.0-20190719005602-e377ae9d6386/go.mod h1:jcCCGcm9btYwXyDq golang.org/x/tools v0.0.0-20190724185037-8aa4eac1a7c1/go.mod h1:jcCCGcm9btYwXyDqrUWc6MKQKKGJCWEQ3AfLSRIbEuI= golang.org/x/tools v0.0.0-20190729092621-ff9f1409240a/go.mod h1:jcCCGcm9btYwXyDqrUWc6MKQKKGJCWEQ3AfLSRIbEuI= golang.org/x/tools v0.0.0-20190802003818-e9bb7d36c060/go.mod h1:jcCCGcm9btYwXyDqrUWc6MKQKKGJCWEQ3AfLSRIbEuI= +golang.org/x/tools v0.0.0-20190809145639-6d4652c779c4/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= From c15976e4dc50e1a2c8308513bcbc426f98995e03 Mon Sep 17 00:00:00 2001 From: Arceliar Date: Mon, 12 Aug 2019 18:08:02 -0500 Subject: [PATCH 37/51] go.sum --- go.sum | 2 ++ 1 file changed, 2 insertions(+) diff --git a/go.sum b/go.sum index 0369a785..c786f5ea 100644 --- a/go.sum +++ b/go.sum @@ -22,6 +22,7 @@ github.com/yggdrasil-network/water v0.0.0-20190725073841-250edb919f8a h1:mQ0mPD+ github.com/yggdrasil-network/water v0.0.0-20190725073841-250edb919f8a/go.mod h1:R0SBCsugm+Sf1katgTb2t7GXMm+nRIv43tM4VDZbaOs= github.com/yggdrasil-network/water v0.0.0-20190725123504-a16161896c34 h1:Qh5FE+Q5iGqpmR/FPMYHuoZLN921au/nxAlmKe+Hdbo= github.com/yggdrasil-network/water v0.0.0-20190725123504-a16161896c34/go.mod h1:R0SBCsugm+Sf1katgTb2t7GXMm+nRIv43tM4VDZbaOs= +github.com/yggdrasil-network/water v0.0.0-20190812103929-c83fe40250f8 h1:YY9Pg2BEp0jeUVU60svTOaDr+fs1ySC9RbdC1Qc6wOw= github.com/yggdrasil-network/water v0.0.0-20190812103929-c83fe40250f8/go.mod h1:R0SBCsugm+Sf1katgTb2t7GXMm+nRIv43tM4VDZbaOs= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9 h1:mKdxBk7AujPs8kU4m80U72y/zjbZ3UcXC7dClwKbUI0= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -45,6 +46,7 @@ golang.org/x/sys v0.0.0-20190712062909-fae7ac547cb7/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3 h1:4y9KwBHBgBNwDbtu44R5o1fdOCQUEXhbk/P4A9WmJq0= golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190812073006-9eafafc0a87e h1:TsjK5I7fXk8f2FQrgu6NS7i5Qih3knl2FL1htyguLRE= golang.org/x/sys v0.0.0-20190812073006-9eafafc0a87e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= From b2cb1d965cd3951a224397a7a1d421c411e6fc8a Mon Sep 17 00:00:00 2001 From: Arceliar Date: Mon, 12 Aug 2019 18:22:30 -0500 Subject: [PATCH 38/51] avoid leaking sessions when no listener exists, or blocking if it's busy --- src/yggdrasil/session.go | 50 ++++++++++++++++++++-------------------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/src/yggdrasil/session.go b/src/yggdrasil/session.go index da66ad59..179273e6 100644 --- a/src/yggdrasil/session.go +++ b/src/yggdrasil/session.go @@ -348,40 +348,40 @@ func (ss *sessions) sendPingPong(sinfo *sessionInfo, isPong bool) { func (ss *sessions) handlePing(ping *sessionPing) { // Get the corresponding session (or create a new session) sinfo, isIn := ss.getByTheirPerm(&ping.SendPermPub) - // Check if the session is allowed - // TODO: this check may need to be moved - if !isIn && !ss.isSessionAllowed(&ping.SendPermPub, false) { - return - } - // Create the session if it doesn't already exist - if !isIn { - ss.createSession(&ping.SendPermPub) - sinfo, isIn = ss.getByTheirPerm(&ping.SendPermPub) - if !isIn { - panic("This should not happen") - } + switch { + case isIn: // Session already exists + case !ss.isSessionAllowed(&ping.SendPermPub, false): // Session is not allowed + case ping.IsPong: // This is a response, not an initial ping, so ignore it. + default: ss.listenerMutex.Lock() - // Check and see if there's a Listener waiting to accept connections - // TODO: this should not block if nothing is accepting - if !ping.IsPong && ss.listener != nil { + if ss.listener != nil { + // This is a ping from an allowed node for which no session exists, and we have a listener ready to handle sessions. + // We need to create a session and pass it to the listener. + sinfo = ss.createSession(&ping.SendPermPub) + if s, _ := ss.getByTheirPerm(&ping.SendPermPub); s != sinfo { + panic("This should not happen") + } conn := newConn(ss.core, crypto.GetNodeID(&sinfo.theirPermPub), &crypto.NodeID{}, sinfo) for i := range conn.nodeMask { conn.nodeMask[i] = 0xFF } conn.session.startWorkers() - ss.listener.conn <- conn + c := ss.listener.conn + go func() { c <- conn }() } ss.listenerMutex.Unlock() } - sinfo.doFunc(func() { - // Update the session - if !sinfo.update(ping) { /*panic("Should not happen in testing")*/ - return - } - if !ping.IsPong { - ss.sendPingPong(sinfo, true) - } - }) + if sinfo != nil { + sinfo.doFunc(func() { + // Update the session + if !sinfo.update(ping) { /*panic("Should not happen in testing")*/ + return + } + if !ping.IsPong { + ss.sendPingPong(sinfo, true) + } + }) + } } // Get the MTU of the session. From 46c5df1c239187db35965037c59fba1582750422 Mon Sep 17 00:00:00 2001 From: Arceliar Date: Tue, 13 Aug 2019 18:49:49 -0500 Subject: [PATCH 39/51] when we abandon a link because we already have a connection to that peer, only wait for the connection to close if it's an *outgoing* link, otherwise incomming connection attempts can cause us to leak links --- src/yggdrasil/link.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/yggdrasil/link.go b/src/yggdrasil/link.go index de90fd94..eca96ebd 100644 --- a/src/yggdrasil/link.go +++ b/src/yggdrasil/link.go @@ -179,7 +179,10 @@ func (intf *linkInterface) handler() error { // That lets them do things like close connections on its own, avoid printing a connection message in the first place, etc. intf.link.core.log.Debugln("DEBUG: found existing interface for", intf.name) intf.msgIO.close() - <-oldIntf.closed + if !intf.incoming { + // Block outgoing connection attempts until the existing connection closes + <-oldIntf.closed + } return nil } else { intf.closed = make(chan struct{}) From 4702da2bcbb7b50d1fb1bb1535045a0df6908a44 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Wed, 14 Aug 2019 19:32:40 +0100 Subject: [PATCH 40/51] Use new netlink library (fixes #493) --- go.mod | 2 ++ go.sum | 4 ++++ src/tuntap/tun_linux.go | 30 ++++++------------------------ 3 files changed, 12 insertions(+), 24 deletions(-) diff --git a/go.mod b/go.mod index c7773cc4..be3c3025 100644 --- a/go.mod +++ b/go.mod @@ -8,6 +8,8 @@ require ( github.com/kardianos/minwinsvc v0.0.0-20151122163309-cad6b2b879b0 github.com/mitchellh/mapstructure v1.1.2 github.com/songgao/packets v0.0.0-20160404182456-549a10cd4091 + github.com/vishvananda/netlink v1.0.0 + github.com/vishvananda/netns v0.0.0-20190625233234-7109fa855b0f // indirect github.com/yggdrasil-network/water v0.0.0-20190812103929-c83fe40250f8 golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4 golang.org/x/net v0.0.0-20190724013045-ca1201d0de80 diff --git a/go.sum b/go.sum index c786f5ea..2fde9549 100644 --- a/go.sum +++ b/go.sum @@ -12,6 +12,10 @@ github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQz github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/songgao/packets v0.0.0-20160404182456-549a10cd4091 h1:1zN6ImoqhSJhN8hGXFaJlSC8msLmIbX8bFqOfWLKw0w= github.com/songgao/packets v0.0.0-20160404182456-549a10cd4091/go.mod h1:N20Z5Y8oye9a7HmytmZ+tr8Q2vlP0tAHP13kTHzwvQY= +github.com/vishvananda/netlink v1.0.0 h1:bqNY2lgheFIu1meHUFSH3d7vG93AFyqg3oGbJCOJgSM= +github.com/vishvananda/netlink v1.0.0/go.mod h1:+SR5DhBJrl6ZM7CoCKvpw5BKroDKQ+PJqOg65H/2ktk= +github.com/vishvananda/netns v0.0.0-20190625233234-7109fa855b0f h1:nBX3nTcmxEtHSERBJaIo1Qa26VwRaopnZmfDQUXsF4I= +github.com/vishvananda/netns v0.0.0-20190625233234-7109fa855b0f/go.mod h1:ZjcWmFBXmLKZu9Nxj3WKYEafiSqer2rnvPr0en9UNpI= github.com/yggdrasil-network/water v0.0.0-20180615095340-f732c88f34ae h1:MYCANF1kehCG6x6G+/9txLfq6n3lS5Vp0Mxn1hdiBAc= github.com/yggdrasil-network/water v0.0.0-20180615095340-f732c88f34ae/go.mod h1:R0SBCsugm+Sf1katgTb2t7GXMm+nRIv43tM4VDZbaOs= github.com/yggdrasil-network/water v0.0.0-20190719211521-a76871ea954b/go.mod h1:R0SBCsugm+Sf1katgTb2t7GXMm+nRIv43tM4VDZbaOs= diff --git a/src/tuntap/tun_linux.go b/src/tuntap/tun_linux.go index c9c03c09..764e56b9 100644 --- a/src/tuntap/tun_linux.go +++ b/src/tuntap/tun_linux.go @@ -5,11 +5,7 @@ package tuntap // The linux platform specific tun parts import ( - "errors" - "fmt" - "net" - - "github.com/docker/libcontainer/netlink" + "github.com/vishvananda/netlink" water "github.com/yggdrasil-network/water" ) @@ -51,35 +47,21 @@ func (tun *TunAdapter) setup(ifname string, iftapmode bool, addr string, mtu int // to exist on the system, but this will fail if Netlink is not present in the // kernel (it nearly always is). func (tun *TunAdapter) setupAddress(addr string) error { - // Set address - var netIF *net.Interface - ifces, err := net.Interfaces() + nladdr, err := netlink.ParseAddr(addr) if err != nil { return err } - for _, ifce := range ifces { - if ifce.Name == tun.iface.Name() { - var newIF = ifce - netIF = &newIF // Don't point inside ifces, it's apparently unsafe?... - } - } - if netIF == nil { - return errors.New(fmt.Sprintf("Failed to find interface: %s", tun.iface.Name())) - } - ip, ipNet, err := net.ParseCIDR(addr) + nlintf, err := netlink.LinkByName(tun.iface.Name()) if err != nil { return err } - err = netlink.NetworkLinkAddIp(netIF, ip, ipNet) - if err != nil { + if err := netlink.AddrAdd(nlintf, nladdr); err != nil { return err } - err = netlink.NetworkSetMTU(netIF, tun.mtu) - if err != nil { + if err := netlink.LinkSetMTU(nlintf, tun.mtu); err != nil { return err } - netlink.NetworkLinkUp(netIF) - if err != nil { + if err := netlink.LinkSetUp(nlintf); err != nil { return err } return nil From 02bfe283991b1eed20dbebca52e9582a3b11a378 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Wed, 14 Aug 2019 20:09:02 +0100 Subject: [PATCH 41/51] Minor tweaks --- cmd/yggdrasil/main.go | 4 ++-- cmd/yggdrasilctl/main.go | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/cmd/yggdrasil/main.go b/cmd/yggdrasil/main.go index 36494c40..15d7d226 100644 --- a/cmd/yggdrasil/main.go +++ b/cmd/yggdrasil/main.go @@ -120,7 +120,7 @@ func main() { normaliseconf := flag.Bool("normaliseconf", false, "use in combination with either -useconf or -useconffile, outputs your configuration normalised") confjson := flag.Bool("json", false, "print configuration from -genconf or -normaliseconf as JSON instead of HJSON") autoconf := flag.Bool("autoconf", false, "automatic mode (dynamic IP, peer with IPv6 neighbors)") - ver := flag.Bool("version", false, "prints the version of this build") + ver := flag.Bool("version", false, "prints the version of this build") logging := flag.String("logging", "info,warn,error", "comma-separated list of logging levels to enable") logto := flag.String("logto", "stdout", "file path to log to, \"syslog\" or \"stdout\"") flag.Parse() @@ -131,7 +131,7 @@ func main() { case *ver: fmt.Println("Build name:", version.BuildName()) fmt.Println("Build version:", version.BuildVersion()) - os.Exit(0) + return case *autoconf: // Use an autoconf-generated config, this will give us random keys and // port numbers, and will use an automatically selected TUN/TAP interface. diff --git a/cmd/yggdrasilctl/main.go b/cmd/yggdrasilctl/main.go index 4cc9745d..94d90842 100644 --- a/cmd/yggdrasilctl/main.go +++ b/cmd/yggdrasilctl/main.go @@ -59,10 +59,10 @@ func main() { args := flag.Args() if *ver { - fmt.Println(os.Args[0], "build name:", version.BuildName()) - fmt.Println(os.Args[0], "version:", version.BuildVersion()) - fmt.Println("\nFor get yggdrasil version use\n - ", os.Args[0], "getSelf") - os.Exit(0) + fmt.Println("Build name:", version.BuildName()) + fmt.Println("Build version:", version.BuildVersion()) + fmt.Println("To get the version number of the running Yggdrasil node, run", os.Args[0], "getSelf") + return } if len(args) == 0 { From 2abb71682fdcf956e5cfd39628b384530da37508 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Wed, 14 Aug 2019 22:21:30 +0100 Subject: [PATCH 42/51] Update changelog, readme, go.mod/go.sum --- CHANGELOG.md | 20 +++++++++++++++++++- README.md | 2 +- go.mod | 6 +++--- go.sum | 3 +++ 4 files changed, 26 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ee50dcc4..c2032806 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,24 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - in case of vulnerabilities. --> +## [0.3.7] - 2019-08-14 +### Fixed +- A number of significant performance regressions introduced in version 0.3.6 + have been fixed, resulting in better performance +- Flow labels are now used to prioritise traffic flows again correctly +- The `Listen` statement, when configured as a string rather than an array, + will now be parsed correctly +- The admin socket now returns `coords` as a correct array of unsigned 64-bit + integers, rather than the internal representation +- The admin socket now returns `box_pub_key` in string format again +- Sessions no longer leak/block when no listener (e.g. TUN/TAP) is configured +- Incoming session connections no longer block when a session already exists, + which reduces in less leaked goroutines +- Flooded sessions will no longer block other sessions +- Searches are now cleaned up properly and a couple of edge-cases with duplicate + searches have been fixed +- A number of minor allocation and pointer fixes + ## [0.3.6] - 2019-08-03 ### Added - Yggdrasil now has a public API with interfaces such as `yggdrasil.ConnDialer`, `yggdrasil.ConnListener` and `yggdrasil.Conn` for using Yggdrasil as a transport directly within applications @@ -53,7 +71,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Session MTUs are now always calculated correctly, in some cases they were incorrectly defaulting to 1280 before - Multiple searches now don't take place for a single connection - Concurrency bugs fixed -- Fixed a number of bugs in the ICMPv6 neighbor solicitation in the TUN/TAP code +- Fixed a number of bugs in the ICMPv6 neighbor solicitation in the TUN/TAP code - A case where peers weren't always added correctly if one or more peers were unreachable has been fixed - Searches which include the local node are now handled correctly - Lots of small bug tweaks and clean-ups throughout the codebase diff --git a/README.md b/README.md index 3f07a2b2..ec7fe74c 100644 --- a/README.md +++ b/README.md @@ -49,7 +49,7 @@ You may also find other platform-specific wrappers, scripts or tools in the If you want to build from source, as opposed to installing one of the pre-built packages: -1. Install [Go](https://golang.org) (requires Go 1.11 or later) +1. Install [Go](https://golang.org) (requires Go 1.12 or later) 2. Clone this repository 2. Run `./build` diff --git a/go.mod b/go.mod index be3c3025..5654dff8 100644 --- a/go.mod +++ b/go.mod @@ -12,8 +12,8 @@ require ( github.com/vishvananda/netns v0.0.0-20190625233234-7109fa855b0f // indirect github.com/yggdrasil-network/water v0.0.0-20190812103929-c83fe40250f8 golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4 - golang.org/x/net v0.0.0-20190724013045-ca1201d0de80 - golang.org/x/sys v0.0.0-20190812073006-9eafafc0a87e + golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7 + golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a golang.org/x/text v0.3.2 - golang.org/x/tools v0.0.0-20190809145639-6d4652c779c4 // indirect + golang.org/x/tools v0.0.0-20190814171936-5b18234b3ae0 // indirect ) diff --git a/go.sum b/go.sum index 2fde9549..5c5e6de4 100644 --- a/go.sum +++ b/go.sum @@ -40,6 +40,7 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190724013045-ca1201d0de80 h1:Ao/3l156eZf2AW5wK8a7/smtodRU+gha3+BeqJ69lRk= golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20181206074257-70b957f3b65e h1:njOxP/wVblhCLIUhjHXf6X+dzTt5OQ3vMQo9mkOIKIo= golang.org/x/sys v0.0.0-20181206074257-70b957f3b65e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -52,6 +53,7 @@ golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3 h1:4y9KwBHBgBNwDbtu44R5o1fdO golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190812073006-9eafafc0a87e h1:TsjK5I7fXk8f2FQrgu6NS7i5Qih3knl2FL1htyguLRE= golang.org/x/sys v0.0.0-20190812073006-9eafafc0a87e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= @@ -62,4 +64,5 @@ golang.org/x/tools v0.0.0-20190724185037-8aa4eac1a7c1/go.mod h1:jcCCGcm9btYwXyDq golang.org/x/tools v0.0.0-20190729092621-ff9f1409240a/go.mod h1:jcCCGcm9btYwXyDqrUWc6MKQKKGJCWEQ3AfLSRIbEuI= golang.org/x/tools v0.0.0-20190802003818-e9bb7d36c060/go.mod h1:jcCCGcm9btYwXyDqrUWc6MKQKKGJCWEQ3AfLSRIbEuI= golang.org/x/tools v0.0.0-20190809145639-6d4652c779c4/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190814171936-5b18234b3ae0/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= From 1a2b7a8b60985ed82681f5faefd51932ededde0b Mon Sep 17 00:00:00 2001 From: Arceliar Date: Wed, 14 Aug 2019 17:57:36 -0500 Subject: [PATCH 43/51] test a change to how switch hops are selected when multiple links are idle --- src/yggdrasil/switch.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/yggdrasil/switch.go b/src/yggdrasil/switch.go index b53229cc..adc49abe 100644 --- a/src/yggdrasil/switch.go +++ b/src/yggdrasil/switch.go @@ -676,7 +676,8 @@ func (t *switchTable) handleIn(packet []byte, idle map[switchPort]time.Time) boo update = true case cinfo.dist > bestDist: //nothing - case thisTime.Before(bestTime): + case thisTime.After(bestTime): + // Pick the one that was used most recently -- at least this should pick the same link consistently in low-traffic scenarios update = true default: //nothing From 382c2e65462e7c7ad0660239b4645e3827453bfa Mon Sep 17 00:00:00 2001 From: Arceliar Date: Wed, 14 Aug 2019 18:14:24 -0500 Subject: [PATCH 44/51] even more go.sum --- go.sum | 2 ++ 1 file changed, 2 insertions(+) diff --git a/go.sum b/go.sum index 5c5e6de4..62a38033 100644 --- a/go.sum +++ b/go.sum @@ -40,6 +40,7 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190724013045-ca1201d0de80 h1:Ao/3l156eZf2AW5wK8a7/smtodRU+gha3+BeqJ69lRk= golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7 h1:fHDIZ2oxGnUZRN6WgWFCbYBjH9uqVPRCUVUDhs0wnbA= golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20181206074257-70b957f3b65e h1:njOxP/wVblhCLIUhjHXf6X+dzTt5OQ3vMQo9mkOIKIo= @@ -53,6 +54,7 @@ golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3 h1:4y9KwBHBgBNwDbtu44R5o1fdO golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190812073006-9eafafc0a87e h1:TsjK5I7fXk8f2FQrgu6NS7i5Qih3knl2FL1htyguLRE= golang.org/x/sys v0.0.0-20190812073006-9eafafc0a87e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a h1:aYOabOQFp6Vj6W1F80affTUvO9UxmJRx8K0gsfABByQ= golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= From 5b054766a2d893ce0ca01673b528970753e90b2e Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Thu, 15 Aug 2019 10:54:04 +0100 Subject: [PATCH 45/51] Update comments in handleIn, add switch_getFlowLabelFromCoords helper (in case it is useful if we try to consider flowlabels in multi-link scenarios) --- src/yggdrasil/switch.go | 33 +++++++++++++++++++++++++-------- 1 file changed, 25 insertions(+), 8 deletions(-) diff --git a/src/yggdrasil/switch.go b/src/yggdrasil/switch.go index adc49abe..d092625b 100644 --- a/src/yggdrasil/switch.go +++ b/src/yggdrasil/switch.go @@ -630,6 +630,16 @@ func switch_getPacketStreamID(packet []byte) string { return string(switch_getPacketCoords(packet)) } +// Returns the flowlabel from a given set of coords +func switch_getFlowLabelFromCoords(in []byte) []byte { + for i, v := range in { + if v == 0 { + return in[i+1:] + } + } + return []byte{} +} + // Find the best port for a given set of coords func (t *switchTable) bestPortForCoords(coords []byte) switchPort { table := t.getTable() @@ -667,20 +677,28 @@ func (t *switchTable) handleIn(packet []byte, idle map[switchPort]time.Time) boo var update bool switch { case to == nil: - //nothing + // no port was found, ignore it case !isIdle: - //nothing + // the port is busy, ignore it case best == nil: + // this is the first idle port we've found, so select it until we find a + // better candidate port to use instead update = true case cinfo.dist < bestDist: + // the port takes a shorter path/is more direct than our current + // candidate, so select that instead update = true case cinfo.dist > bestDist: - //nothing + // the port takes a longer path/is less direct than our current candidate, + // ignore it case thisTime.After(bestTime): - // Pick the one that was used most recently -- at least this should pick the same link consistently in low-traffic scenarios + // all else equal, this port was used more recently than our current + // candidate, so choose that instead. this should mean that, in low + // traffic scenarios, we consistently pick the same link which helps with + // packet ordering update = true default: - //nothing + // the search for a port has finished } if update { best = to @@ -693,10 +711,9 @@ func (t *switchTable) handleIn(packet []byte, idle map[switchPort]time.Time) boo delete(idle, best.port) best.sendPacket(packet) return true - } else { - // Didn't find anyone idle to send it to - return false } + // Didn't find anyone idle to send it to + return false } // Info about a buffered packet From ae0fe93de5c7ba994fdbab4b1a30a0f3973dd131 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Thu, 15 Aug 2019 12:54:04 +0100 Subject: [PATCH 46/51] Update changelog --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c2032806..a3d7903c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,6 +30,9 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - A number of significant performance regressions introduced in version 0.3.6 have been fixed, resulting in better performance - Flow labels are now used to prioritise traffic flows again correctly +- In low-traffic scenarios where there are multiple peerings between a pair of + nodes, Yggdrasil now prefers the most active peering instead of the least + active, helping to reduce packet reordering - The `Listen` statement, when configured as a string rather than an array, will now be parsed correctly - The admin socket now returns `coords` as a correct array of unsigned 64-bit From fdac8932a867a33c56242c34e67b9bd55bccdb2b Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Thu, 15 Aug 2019 13:11:54 +0100 Subject: [PATCH 47/51] Update changelog --- CHANGELOG.md | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a3d7903c..0057a005 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,23 +27,16 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ## [0.3.7] - 2019-08-14 ### Fixed -- A number of significant performance regressions introduced in version 0.3.6 - have been fixed, resulting in better performance +- A number of significant performance regressions introduced in version 0.3.6 have been fixed, resulting in better performance - Flow labels are now used to prioritise traffic flows again correctly -- In low-traffic scenarios where there are multiple peerings between a pair of - nodes, Yggdrasil now prefers the most active peering instead of the least - active, helping to reduce packet reordering -- The `Listen` statement, when configured as a string rather than an array, - will now be parsed correctly -- The admin socket now returns `coords` as a correct array of unsigned 64-bit - integers, rather than the internal representation +- In low-traffic scenarios where there are multiple peerings between a pair of nodes, Yggdrasil now prefers the most active peering instead of the least active, helping to reduce packet reordering +- The `Listen` statement, when configured as a string rather than an array, will now be parsed correctly +- The admin socket now returns `coords` as a correct array of unsigned 64-bit integers, rather than the internal representation - The admin socket now returns `box_pub_key` in string format again - Sessions no longer leak/block when no listener (e.g. TUN/TAP) is configured -- Incoming session connections no longer block when a session already exists, - which reduces in less leaked goroutines +- Incoming session connections no longer block when a session already exists, which reduces in less leaked goroutines - Flooded sessions will no longer block other sessions -- Searches are now cleaned up properly and a couple of edge-cases with duplicate - searches have been fixed +- Searches are now cleaned up properly and a couple of edge-cases with duplicate searches have been fixed - A number of minor allocation and pointer fixes ## [0.3.6] - 2019-08-03 From 03b8af9f1a1c178ec46d7cf7693371a322a0acae Mon Sep 17 00:00:00 2001 From: Arceliar Date: Fri, 16 Aug 2019 18:37:16 -0500 Subject: [PATCH 48/51] keep track of recent nonces with a heap and a map instead of a fixed-size bitmask --- src/yggdrasil/session.go | 131 +++++++++++++++++++++++++-------------- 1 file changed, 83 insertions(+), 48 deletions(-) diff --git a/src/yggdrasil/session.go b/src/yggdrasil/session.go index 179273e6..b675b469 100644 --- a/src/yggdrasil/session.go +++ b/src/yggdrasil/session.go @@ -6,6 +6,7 @@ package yggdrasil import ( "bytes" + "container/heap" "errors" "sync" "time" @@ -15,42 +16,62 @@ import ( "github.com/yggdrasil-network/yggdrasil-go/src/util" ) +// Duration that we keep track of old nonces per session, to allow some out-of-order packet delivery +const nonceWindow = time.Second + +// A heap of nonces, used with a map[nonce]time to allow out-of-order packets a little time to arrive without rejecting them +// Less is backwards so the oldest node is the highest priority for Pop +type nonceHeap []crypto.BoxNonce + +func (h nonceHeap) Len() int { return len(h) } +func (h nonceHeap) Less(i, j int) bool { return h[i].Minus(&h[j]) > 0 } +func (h nonceHeap) Swap(i, j int) { h[i], h[j] = h[j], h[i] } +func (h *nonceHeap) Push(x interface{}) { *h = append(*h, x.(crypto.BoxNonce)) } +func (h *nonceHeap) Pop() interface{} { + l := len(*h) + var n crypto.BoxNonce + n, *h = (*h)[l-1], (*h)[:l-1] + return n +} +func (h nonceHeap) peek() *crypto.BoxNonce { return &h[len(h)-1] } + // All the information we know about an active session. // This includes coords, permanent and ephemeral keys, handles and nonces, various sorts of timing information for timeout and maintenance, and some metadata for the admin API. type sessionInfo struct { - mutex sync.Mutex // Protects all of the below, use it any time you read/chance the contents of a session - core *Core // - reconfigure chan chan error // - theirAddr address.Address // - theirSubnet address.Subnet // - theirPermPub crypto.BoxPubKey // - theirSesPub crypto.BoxPubKey // - mySesPub crypto.BoxPubKey // - mySesPriv crypto.BoxPrivKey // - sharedSesKey crypto.BoxSharedKey // derived from session keys - theirHandle crypto.Handle // - myHandle crypto.Handle // - theirNonce crypto.BoxNonce // - theirNonceMask uint64 // - myNonce crypto.BoxNonce // - theirMTU uint16 // - myMTU uint16 // - wasMTUFixed bool // Was the MTU fixed by a receive error? - timeOpened time.Time // Time the sessino was opened - time time.Time // Time we last received a packet - mtuTime time.Time // time myMTU was last changed - pingTime time.Time // time the first ping was sent since the last received packet - pingSend time.Time // time the last ping was sent - coords []byte // coords of destination - reset bool // reset if coords change - tstamp int64 // ATOMIC - tstamp from their last session ping, replay attack mitigation - bytesSent uint64 // Bytes of real traffic sent in this session - bytesRecvd uint64 // Bytes of real traffic received in this session - init chan struct{} // Closed when the first session pong arrives, used to signal that the session is ready for initial use - cancel util.Cancellation // Used to terminate workers - fromRouter chan wire_trafficPacket // Received packets go here, to be decrypted by the session - recv chan []byte // Decrypted packets go here, picked up by the associated Conn - send chan FlowKeyMessage // Packets with optional flow key go here, to be encrypted and sent + mutex sync.Mutex // Protects all of the below, use it any time you read/chance the contents of a session + core *Core // + reconfigure chan chan error // + theirAddr address.Address // + theirSubnet address.Subnet // + theirPermPub crypto.BoxPubKey // + theirSesPub crypto.BoxPubKey // + mySesPub crypto.BoxPubKey // + mySesPriv crypto.BoxPrivKey // + sharedSesKey crypto.BoxSharedKey // derived from session keys + theirHandle crypto.Handle // + myHandle crypto.Handle // + theirNonce crypto.BoxNonce // + theirNonceHeap nonceHeap // priority queue to keep track of the lowest nonce we recently accepted + theirNonceMap map[crypto.BoxNonce]time.Time // time we added each nonce to the heap + myNonce crypto.BoxNonce // + theirMTU uint16 // + myMTU uint16 // + wasMTUFixed bool // Was the MTU fixed by a receive error? + timeOpened time.Time // Time the sessino was opened + time time.Time // Time we last received a packet + mtuTime time.Time // time myMTU was last changed + pingTime time.Time // time the first ping was sent since the last received packet + pingSend time.Time // time the last ping was sent + coords []byte // coords of destination + reset bool // reset if coords change + tstamp int64 // ATOMIC - tstamp from their last session ping, replay attack mitigation + bytesSent uint64 // Bytes of real traffic sent in this session + bytesRecvd uint64 // Bytes of real traffic received in this session + init chan struct{} // Closed when the first session pong arrives, used to signal that the session is ready for initial use + cancel util.Cancellation // Used to terminate workers + fromRouter chan wire_trafficPacket // Received packets go here, to be decrypted by the session + recv chan []byte // Decrypted packets go here, picked up by the associated Conn + send chan FlowKeyMessage // Packets with optional flow key go here, to be encrypted and sent } func (sinfo *sessionInfo) doFunc(f func()) { @@ -87,7 +108,8 @@ func (s *sessionInfo) update(p *sessionPing) bool { s.theirHandle = p.Handle s.sharedSesKey = *crypto.GetSharedKey(&s.mySesPriv, &s.theirSesPub) s.theirNonce = crypto.BoxNonce{} - s.theirNonceMask = 0 + s.theirNonceHeap = nil + s.theirNonceMap = make(map[crypto.BoxNonce]time.Time) } if p.MTU >= 1280 || p.MTU == 0 { s.theirMTU = p.MTU @@ -400,27 +422,40 @@ func (sinfo *sessionInfo) getMTU() uint16 { // Checks if a packet's nonce is recent enough to fall within the window of allowed packets, and not already received. func (sinfo *sessionInfo) nonceIsOK(theirNonce *crypto.BoxNonce) bool { // The bitmask is to allow for some non-duplicate out-of-order packets - diff := theirNonce.Minus(&sinfo.theirNonce) - if diff > 0 { + if theirNonce.Minus(&sinfo.theirNonce) > 0 { + // This is newer than the newest nonce we've seen return true } - return ^sinfo.theirNonceMask&(0x01< 0 { + if theirNonce.Minus(sinfo.theirNonceHeap.peek()) > 0 { + if _, isIn := sinfo.theirNonceMap[*theirNonce]; !isIn { + // This nonce is recent enough that we keep track of older nonces, but it's not one we've seen yet + return true + } + } + } + return false } // Updates the nonce mask by (possibly) shifting the bitmask and setting the bit corresponding to this nonce to 1, and then updating the most recent nonce func (sinfo *sessionInfo) updateNonce(theirNonce *crypto.BoxNonce) { - // Shift nonce mask if needed - // Set bit - diff := theirNonce.Minus(&sinfo.theirNonce) - if diff > 0 { - // This nonce is newer, so shift the window before setting the bit, and update theirNonce in the session info. - sinfo.theirNonceMask <<= uint64(diff) - sinfo.theirNonceMask &= 0x01 - sinfo.theirNonce = *theirNonce - } else { - // This nonce is older, so set the bit but do not shift the window. - sinfo.theirNonceMask &= 0x01 << uint64(-diff) + // Start with some cleanup + for len(sinfo.theirNonceHeap) > 64 { + if time.Since(sinfo.theirNonceMap[*sinfo.theirNonceHeap.peek()]) < nonceWindow { + // This nonce is still fairly new, so keep it around + break + } + // TODO? reallocate the map in some cases, to free unused map space? + delete(sinfo.theirNonceMap, *sinfo.theirNonceHeap.peek()) + heap.Pop(&sinfo.theirNonceHeap) } + if theirNonce.Minus(&sinfo.theirNonce) > 0 { + // This nonce is the newest we've seen, so make a note of that + sinfo.theirNonce = *theirNonce + } + // Add it to the heap/map so we know not to allow it again + heap.Push(&sinfo.theirNonceHeap, *theirNonce) + sinfo.theirNonceMap[*theirNonce] = time.Now() } // Resets all sessions to an uninitialized state. From fd5f3ca7649c9167290c76c3562c314f42121c98 Mon Sep 17 00:00:00 2001 From: Arceliar Date: Fri, 16 Aug 2019 23:07:40 -0500 Subject: [PATCH 49/51] fix heap pop order --- src/yggdrasil/session.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/yggdrasil/session.go b/src/yggdrasil/session.go index b675b469..57522209 100644 --- a/src/yggdrasil/session.go +++ b/src/yggdrasil/session.go @@ -20,11 +20,10 @@ import ( const nonceWindow = time.Second // A heap of nonces, used with a map[nonce]time to allow out-of-order packets a little time to arrive without rejecting them -// Less is backwards so the oldest node is the highest priority for Pop type nonceHeap []crypto.BoxNonce func (h nonceHeap) Len() int { return len(h) } -func (h nonceHeap) Less(i, j int) bool { return h[i].Minus(&h[j]) > 0 } +func (h nonceHeap) Less(i, j int) bool { return h[i].Minus(&h[j]) < 0 } func (h nonceHeap) Swap(i, j int) { h[i], h[j] = h[j], h[i] } func (h *nonceHeap) Push(x interface{}) { *h = append(*h, x.(crypto.BoxNonce)) } func (h *nonceHeap) Pop() interface{} { From 57e7acdda806399484f480a8d818efaa993a560b Mon Sep 17 00:00:00 2001 From: Arceliar Date: Sat, 17 Aug 2019 12:42:17 -0500 Subject: [PATCH 50/51] Update CHANGELOG.md --- CHANGELOG.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0057a005..9cea99c1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,6 +26,10 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. --> ## [0.3.7] - 2019-08-14 +### Changed +- The switch should now forward packets along a single path more consistently in cases where congestion is low and multiple equal-length paths exist, which should improve stability and result in fewer out-of-order packets +- Sessions should now be more tolerant of out-of-order packets, by replacing a bitmask with a variable sized heap+map structure to track recently received nonces, which should reduce the number of packets dropped due to reordering when multiple paths are used or multiple independent flows are transmitted through the same session + ### Fixed - A number of significant performance regressions introduced in version 0.3.6 have been fixed, resulting in better performance - Flow labels are now used to prioritise traffic flows again correctly @@ -34,7 +38,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - The admin socket now returns `coords` as a correct array of unsigned 64-bit integers, rather than the internal representation - The admin socket now returns `box_pub_key` in string format again - Sessions no longer leak/block when no listener (e.g. TUN/TAP) is configured -- Incoming session connections no longer block when a session already exists, which reduces in less leaked goroutines +- Incoming session connections no longer block when a session already exists, which results in less leaked goroutines - Flooded sessions will no longer block other sessions - Searches are now cleaned up properly and a couple of edge-cases with duplicate searches have been fixed - A number of minor allocation and pointer fixes From 039dd98f0de08170856295a3152c5ad93aab59e2 Mon Sep 17 00:00:00 2001 From: Arceliar Date: Sat, 17 Aug 2019 12:46:34 -0500 Subject: [PATCH 51/51] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9cea99c1..844409f5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,6 +29,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ### Changed - The switch should now forward packets along a single path more consistently in cases where congestion is low and multiple equal-length paths exist, which should improve stability and result in fewer out-of-order packets - Sessions should now be more tolerant of out-of-order packets, by replacing a bitmask with a variable sized heap+map structure to track recently received nonces, which should reduce the number of packets dropped due to reordering when multiple paths are used or multiple independent flows are transmitted through the same session +- The admin socket can no longer return a dotfile representation of the known parts of the network, this could be rebuilt by clients using information from `getSwitchPeers`,`getDHT` and `getSessions` ### Fixed - A number of significant performance regressions introduced in version 0.3.6 have been fixed, resulting in better performance