From 38e77041612c99253fb5a6e91958cab90f109d7d Mon Sep 17 00:00:00 2001 From: Arceliar Date: Sun, 27 May 2018 13:37:35 -0500 Subject: [PATCH 01/42] use backpressure instead of estimated bandwidth, sorted by uptime to break ties --- src/yggdrasil/peer.go | 21 +++++---------------- src/yggdrasil/switch.go | 30 +++++++++++++++--------------- src/yggdrasil/tcp.go | 22 +++++----------------- src/yggdrasil/udp.go | 5 ++--- 4 files changed, 27 insertions(+), 51 deletions(-) diff --git a/src/yggdrasil/peer.go b/src/yggdrasil/peer.go index 4ce1a780..ed73ee44 100644 --- a/src/yggdrasil/peer.go +++ b/src/yggdrasil/peer.go @@ -25,7 +25,6 @@ package yggdrasil import "time" import "sync" import "sync/atomic" -import "math" //import "fmt" @@ -86,7 +85,7 @@ func (ps *peers) putPorts(ports map[switchPort]*peer) { type peer struct { // Rolling approximation of bandwidth, in bps, used by switch, updated by packet sends // use get/update methods only! (atomic accessors as float64) - bandwidth uint64 + queueSize int64 bytesSent uint64 // To track bandwidth usage for getPeers bytesRecvd uint64 // To track bandwidth usage for getPeers // BUG: sync/atomic, 32 bit platforms need the above to be the first element @@ -116,22 +115,12 @@ type peer struct { const peer_Throttle = 1 -func (p *peer) getBandwidth() float64 { - bits := atomic.LoadUint64(&p.bandwidth) - return math.Float64frombits(bits) +func (p *peer) getQueueSize() int64 { + return atomic.LoadInt64(&p.queueSize) } -func (p *peer) updateBandwidth(bytes int, duration time.Duration) { - if p == nil { - return - } - for ok := false; !ok; { - oldBits := atomic.LoadUint64(&p.bandwidth) - oldBandwidth := math.Float64frombits(oldBits) - bandwidth := oldBandwidth*7/8 + float64(bytes)/duration.Seconds() - bits := math.Float64bits(bandwidth) - ok = atomic.CompareAndSwapUint64(&p.bandwidth, oldBits, bits) - } +func (p *peer) updateQueueSize(delta int64) { + atomic.AddInt64(&p.queueSize, delta) } func (ps *peers) newPeer(box *boxPubKey, diff --git a/src/yggdrasil/switch.go b/src/yggdrasil/switch.go index a005b106..bbc28392 100644 --- a/src/yggdrasil/switch.go +++ b/src/yggdrasil/switch.go @@ -12,6 +12,7 @@ package yggdrasil // A little annoying to do with constant changes from bandwidth estimates import "time" +import "sort" import "sync" import "sync/atomic" @@ -397,37 +398,36 @@ func (t *switchTable) updateTable() { port: pinfo.port, }) } + sort.SliceStable(newTable.elems, func(i, j int) bool { + return t.data.peers[newTable.elems[i].port].firstSeen.Before(t.data.peers[newTable.elems[j].port].firstSeen) + }) t.table.Store(newTable) } func (t *switchTable) lookup(dest []byte, ttl uint64) (switchPort, uint64) { t.updater.Load().(*sync.Once).Do(t.updateTable) table := t.table.Load().(lookupTable) - ports := t.core.peers.getPorts() - getBandwidth := func(port switchPort) float64 { - var bandwidth float64 - if p, isIn := ports[port]; isIn { - bandwidth = p.getBandwidth() - } - return bandwidth - } - var best switchPort myDist := table.self.dist(dest) //getDist(table.self.coords) if !(uint64(myDist) < ttl) { return 0, 0 } - // score is in units of bandwidth / distance - bestScore := float64(-1) + // cost is in units of (expected distance) + (expected queue size), where expected distance is used as an approximation of the minimum backpressure gradient needed for packets to flow + ports := t.core.peers.getPorts() + var best switchPort + bestCost := int64(^uint64(0) >> 1) for _, info := range table.elems { dist := info.locator.dist(dest) //getDist(info.locator.coords) if !(dist < myDist) { continue } - score := getBandwidth(info.port) - score /= float64(1 + dist) - if score > bestScore { + p, isIn := ports[info.port] + if !isIn { + continue + } + cost := int64(dist) + p.getQueueSize() + if cost < bestCost { best = info.port - bestScore = score + bestCost = cost } } //t.core.log.Println("DEBUG: sending to", best, "bandwidth", getBandwidth(best)) diff --git a/src/yggdrasil/tcp.go b/src/yggdrasil/tcp.go index 869b6afd..f02aff5b 100644 --- a/src/yggdrasil/tcp.go +++ b/src/yggdrasil/tcp.go @@ -190,26 +190,12 @@ func (iface *tcpInterface) handler(sock net.Conn, incoming bool) { buf := bufio.NewWriterSize(sock, tcp_msgSize) send := func(msg []byte) { msgLen := wire_encode_uint64(uint64(len(msg))) - before := buf.Buffered() - start := time.Now() buf.Write(tcp_msg[:]) buf.Write(msgLen) buf.Write(msg) - timed := time.Since(start) - after := buf.Buffered() - written := (before + len(tcp_msg) + len(msgLen) + len(msg)) - after - if written > 0 { - p.updateBandwidth(written, timed) - } + p.updateQueueSize(-1) util_putBytes(msg) } - flush := func() { - size := buf.Buffered() - start := time.Now() - buf.Flush() - timed := time.Since(start) - p.updateBandwidth(size, timed) - } go func() { var stack [][]byte put := func(msg []byte) { @@ -217,6 +203,7 @@ func (iface *tcpInterface) handler(sock net.Conn, incoming bool) { for len(stack) > 32 { util_putBytes(stack[0]) stack = stack[1:] + p.updateQueueSize(-1) } } for msg := range out { @@ -226,7 +213,7 @@ func (iface *tcpInterface) handler(sock net.Conn, incoming bool) { select { case msg, ok := <-out: if !ok { - flush() + buf.Flush() return } put(msg) @@ -236,13 +223,14 @@ func (iface *tcpInterface) handler(sock net.Conn, incoming bool) { send(msg) } } - flush() + buf.Flush() } }() p.out = func(msg []byte) { defer func() { recover() }() select { case out <- msg: + p.updateQueueSize(1) default: util_putBytes(msg) } diff --git a/src/yggdrasil/udp.go b/src/yggdrasil/udp.go index 02fb9d6d..68f53a8e 100644 --- a/src/yggdrasil/udp.go +++ b/src/yggdrasil/udp.go @@ -265,6 +265,7 @@ func (iface *udpInterface) handleKeys(msg []byte, addr connAddr) { defer func() { recover() }() select { case conn.out <- msg: + conn.peer.updateQueueSize(1) default: util_putBytes(msg) } @@ -282,16 +283,14 @@ func (iface *udpInterface) handleKeys(msg []byte, addr connAddr) { if len(chunks) > 255 { continue } - start := time.Now() for idx, bs := range chunks { nChunks, nChunk, count := uint8(len(chunks)), uint8(idx)+1, conn.countOut out = udp_encode(out[:0], nChunks, nChunk, count, bs) //iface.core.log.Println("DEBUG out:", nChunks, nChunk, count, len(bs)) iface.sock.WriteToUDP(out, udpAddr) } - timed := time.Since(start) conn.countOut += 1 - conn.peer.updateBandwidth(len(msg), timed) + conn.peer.updateQueueSize(-1) util_putBytes(msg) } }() From 1b898926108f715eecb2afe028d2a872408afe49 Mon Sep 17 00:00:00 2001 From: Arceliar Date: Wed, 6 Jun 2018 16:40:35 -0500 Subject: [PATCH 02/42] remove UDP, to be replaced with a better implementation later --- src/yggdrasil/admin.go | 16 +- src/yggdrasil/core.go | 6 - src/yggdrasil/debug.go | 2 + src/yggdrasil/udp.go | 394 ----------------------------------------- 4 files changed, 6 insertions(+), 412 deletions(-) delete mode 100644 src/yggdrasil/udp.go diff --git a/src/yggdrasil/admin.go b/src/yggdrasil/admin.go index fe9c05e9..1abf7a27 100644 --- a/src/yggdrasil/admin.go +++ b/src/yggdrasil/admin.go @@ -361,8 +361,6 @@ func (a *admin) addPeer(addr string) error { switch strings.ToLower(u.Scheme) { case "tcp": a.core.tcp.connect(u.Host) - case "udp": - a.core.udp.connect(u.Host) case "socks": a.core.tcp.connectSOCKS(u.Host, u.Path[1:]) default: @@ -371,17 +369,11 @@ func (a *admin) addPeer(addr string) error { } else { // no url scheme provided addr = strings.ToLower(addr) - if strings.HasPrefix(addr, "udp:") { - a.core.udp.connect(addr[4:]) - return nil - } else { - if strings.HasPrefix(addr, "tcp:") { - addr = addr[4:] - } - a.core.tcp.connect(addr) - return nil + if strings.HasPrefix(addr, "tcp:") { + addr = addr[4:] } - return errors.New("invalid peer: " + addr) + a.core.tcp.connect(addr) + return nil } return nil } diff --git a/src/yggdrasil/core.go b/src/yggdrasil/core.go index ff9c3602..b94d1544 100644 --- a/src/yggdrasil/core.go +++ b/src/yggdrasil/core.go @@ -27,7 +27,6 @@ type Core struct { searches searches multicast multicast tcp tcpInterface - udp udpInterface log *log.Logger ifceExpr []*regexp.Regexp // the zone of link-local IPv6 peers must match this } @@ -99,11 +98,6 @@ func (c *Core) Start(nc *config.NodeConfig, log *log.Logger) error { return err } - if err := c.udp.init(c, nc.Listen); err != nil { - c.log.Println("Failed to start UDP interface") - return err - } - if err := c.router.start(); err != nil { c.log.Println("Failed to start router") return err diff --git a/src/yggdrasil/debug.go b/src/yggdrasil/debug.go index a987fbdf..cd42560c 100644 --- a/src/yggdrasil/debug.go +++ b/src/yggdrasil/debug.go @@ -317,6 +317,7 @@ func (c *Core) DEBUG_init(bpub []byte, //////////////////////////////////////////////////////////////////////////////// +/* func (c *Core) DEBUG_setupAndStartGlobalUDPInterface(addrport string) { if err := c.udp.init(c, addrport); err != nil { c.log.Println("Failed to start UDP interface:", err) @@ -342,6 +343,7 @@ func (c *Core) DEBUG_maybeSendUDPKeys(saddr string) { c.udp.sendKeys(addr) } } +*/ //////////////////////////////////////////////////////////////////////////////// diff --git a/src/yggdrasil/udp.go b/src/yggdrasil/udp.go deleted file mode 100644 index 0be507f3..00000000 --- a/src/yggdrasil/udp.go +++ /dev/null @@ -1,394 +0,0 @@ -package yggdrasil - -// This communicates with peers via UDP -// It's not as well tested or debugged as the TCP transport -// It's intended to use UDP, so debugging/optimzing this is a high priority -// TODO? use golang.org/x/net/ipv6.PacketConn's ReadBatch and WriteBatch? -// To send all chunks of a message / recv all available chunks in one syscall -// That might be faster on supported platforms, but it needs investigation -// Chunks are currently murged, but outgoing messages aren't chunked -// This is just to support chunking in the future, if it's needed and debugged -// Basically, right now we might send UDP packets that are too large - -// TODO remove old/unused code and better document live code - -import "net" -import "time" -import "sync" -import "fmt" - -type udpInterface struct { - core *Core - sock *net.UDPConn // Or more general PacketConn? - mutex sync.RWMutex // each conn has an owner goroutine - conns map[connAddr]*connInfo -} - -type connAddr struct { - ip [16]byte - port int - zone string -} - -func (c *connAddr) fromUDPAddr(u *net.UDPAddr) { - copy(c.ip[:], u.IP.To16()) - c.port = u.Port - c.zone = u.Zone -} - -func (c *connAddr) toUDPAddr() *net.UDPAddr { - var u net.UDPAddr - u.IP = make([]byte, 16) - copy(u.IP, c.ip[:]) - u.Port = c.port - u.Zone = c.zone - return &u -} - -type connInfo struct { - name string - addr connAddr - peer *peer - linkIn chan []byte - keysIn chan *udpKeys - closeIn chan *udpKeys - timeout int // count of how many heartbeats have been missed - in func([]byte) - out chan []byte - countIn uint8 - countOut uint8 - chunkSize uint16 -} - -type udpKeys struct { - box boxPubKey - sig sigPubKey -} - -func (iface *udpInterface) getAddr() *net.UDPAddr { - return iface.sock.LocalAddr().(*net.UDPAddr) -} - -func (iface *udpInterface) connect(saddr string) { - udpAddr, err := net.ResolveUDPAddr("udp", saddr) - if err != nil { - panic(err) - } - var addr connAddr - addr.fromUDPAddr(udpAddr) - iface.mutex.RLock() - _, isIn := iface.conns[addr] - iface.mutex.RUnlock() - if !isIn { - iface.sendKeys(addr) - } -} - -func (iface *udpInterface) init(core *Core, addr string) (err error) { - iface.core = core - udpAddr, err := net.ResolveUDPAddr("udp", addr) - if err != nil { - return - } - iface.sock, err = net.ListenUDP("udp", udpAddr) - if err != nil { - return - } - iface.conns = make(map[connAddr]*connInfo) - go iface.reader() - return -} - -func (iface *udpInterface) sendKeys(addr connAddr) { - udpAddr := addr.toUDPAddr() - msg := []byte{} - msg = udp_encode(msg, 0, 0, 0, nil) - msg = append(msg, iface.core.boxPub[:]...) - msg = append(msg, iface.core.sigPub[:]...) - iface.sock.WriteToUDP(msg, udpAddr) -} - -func (iface *udpInterface) sendClose(addr connAddr) { - udpAddr := addr.toUDPAddr() - msg := []byte{} - msg = udp_encode(msg, 0, 1, 0, nil) - msg = append(msg, iface.core.boxPub[:]...) - msg = append(msg, iface.core.sigPub[:]...) - iface.sock.WriteToUDP(msg, udpAddr) -} - -func udp_isKeys(msg []byte) bool { - keyLen := 3 + boxPubKeyLen + sigPubKeyLen - return len(msg) == keyLen && msg[0] == 0x00 && msg[1] == 0x00 -} - -func udp_isClose(msg []byte) bool { - keyLen := 3 + boxPubKeyLen + sigPubKeyLen - return len(msg) == keyLen && msg[0] == 0x00 && msg[1] == 0x01 -} - -func (iface *udpInterface) startConn(info *connInfo) { - ticker := time.NewTicker(6 * time.Second) - defer ticker.Stop() - defer func() { - // Cleanup - iface.mutex.Lock() - delete(iface.conns, info.addr) - iface.mutex.Unlock() - iface.core.peers.removePeer(info.peer.port) - close(info.linkIn) - close(info.keysIn) - close(info.closeIn) - close(info.out) - iface.core.log.Println("Removing peer:", info.name) - }() - for { - select { - case ks := <-info.closeIn: - { - if ks.box == info.peer.box && ks.sig == info.peer.sig { - // TODO? secure this somehow - // Maybe add a signature and sequence number (timestamp) to close and keys? - return - } - } - case ks := <-info.keysIn: - { - // FIXME? need signatures/sequence-numbers or something - // Spoofers could lock out a peer with fake/bad keys - if ks.box == info.peer.box && ks.sig == info.peer.sig { - info.timeout = 0 - } - } - case <-ticker.C: - { - if info.timeout > 10 { - return - } - info.timeout++ - iface.sendKeys(info.addr) - } - } - } -} - -func (iface *udpInterface) handleClose(msg []byte, addr connAddr) { - //defer util_putBytes(msg) - var ks udpKeys - _, _, _, bs := udp_decode(msg) - switch { - case !wire_chop_slice(ks.box[:], &bs): - return - case !wire_chop_slice(ks.sig[:], &bs): - return - } - if ks.box == iface.core.boxPub { - return - } - if ks.sig == iface.core.sigPub { - return - } - iface.mutex.RLock() - conn, isIn := iface.conns[addr] - iface.mutex.RUnlock() - if !isIn { - return - } - func() { - defer func() { recover() }() - select { - case conn.closeIn <- &ks: - default: - } - }() -} - -func (iface *udpInterface) handleKeys(msg []byte, addr connAddr) { - //defer util_putBytes(msg) - var ks udpKeys - _, _, _, bs := udp_decode(msg) - switch { - case !wire_chop_slice(ks.box[:], &bs): - return - case !wire_chop_slice(ks.sig[:], &bs): - return - } - if ks.box == iface.core.boxPub { - return - } - if ks.sig == iface.core.sigPub { - return - } - iface.mutex.RLock() - conn, isIn := iface.conns[addr] - iface.mutex.RUnlock() - if !isIn { - udpAddr := addr.toUDPAddr() - // Check if we're authorized to connect to this key / IP - // TODO monitor and always allow outgoing connections - if !iface.core.peers.isAllowedEncryptionPublicKey(&ks.box) { - // Allow unauthorized peers if they're link-local - if !udpAddr.IP.IsLinkLocalUnicast() { - return - } - } - themNodeID := getNodeID(&ks.box) - themAddr := address_addrForNodeID(themNodeID) - themAddrString := net.IP(themAddr[:]).String() - themString := fmt.Sprintf("%s@%s", themAddrString, udpAddr.String()) - conn = &connInfo{ - name: themString, - addr: connAddr(addr), - peer: iface.core.peers.newPeer(&ks.box, &ks.sig), - linkIn: make(chan []byte, 1), - keysIn: make(chan *udpKeys, 1), - closeIn: make(chan *udpKeys, 1), - out: make(chan []byte, 32), - chunkSize: 576 - 60 - 8 - 3, // max safe - max ip - udp header - chunk overhead - } - if udpAddr.IP.IsLinkLocalUnicast() { - ifce, err := net.InterfaceByName(udpAddr.Zone) - if ifce != nil && err == nil { - conn.chunkSize = uint16(ifce.MTU) - 60 - 8 - 3 - } - } - var inChunks uint8 - var inBuf []byte - conn.in = func(bs []byte) { - //defer util_putBytes(bs) - chunks, chunk, count, payload := udp_decode(bs) - if count != conn.countIn { - if len(inBuf) > 0 { - // Something went wrong - // Forward whatever we have - // Maybe the destination can do something about it - msg := append(util_getBytes(), inBuf...) - conn.peer.handlePacket(msg, conn.linkIn) - } - inChunks = 0 - inBuf = inBuf[:0] - conn.countIn = count - } - if chunk <= chunks && chunk == inChunks+1 { - inChunks += 1 - inBuf = append(inBuf, payload...) - if chunks != chunk { - return - } - msg := append(util_getBytes(), inBuf...) - conn.peer.handlePacket(msg, conn.linkIn) - inBuf = inBuf[:0] - } - } - conn.peer.out = func(msg []byte) { - defer func() { recover() }() - select { - case conn.out <- msg: - default: - util_putBytes(msg) - } - } - go func() { - var out []byte - var chunks [][]byte - for msg := range conn.out { - chunks = chunks[:0] - bs := msg - for len(bs) > int(conn.chunkSize) { - chunks, bs = append(chunks, bs[:conn.chunkSize]), bs[conn.chunkSize:] - } - chunks = append(chunks, bs) - if len(chunks) > 255 { - continue - } - start := time.Now() - for idx, bs := range chunks { - nChunks, nChunk, count := uint8(len(chunks)), uint8(idx)+1, conn.countOut - out = udp_encode(out[:0], nChunks, nChunk, count, bs) - //iface.core.log.Println("DEBUG out:", nChunks, nChunk, count, len(bs)) - iface.sock.WriteToUDP(out, udpAddr) - } - timed := time.Since(start) - conn.countOut += 1 - conn.peer.updateBandwidth(len(msg), timed) - util_putBytes(msg) - } - }() - //*/ - conn.peer.close = func() { iface.sendClose(conn.addr) } - iface.mutex.Lock() - iface.conns[addr] = conn - iface.mutex.Unlock() - iface.core.log.Println("Adding peer:", conn.name) - go iface.startConn(conn) - go conn.peer.linkLoop(conn.linkIn) - iface.sendKeys(conn.addr) - } - func() { - defer func() { recover() }() - select { - case conn.keysIn <- &ks: - default: - } - }() -} - -func (iface *udpInterface) handlePacket(msg []byte, addr connAddr) { - iface.mutex.RLock() - if conn, isIn := iface.conns[addr]; isIn { - conn.in(msg) - } - iface.mutex.RUnlock() -} - -func (iface *udpInterface) reader() { - iface.core.log.Println("Listening for UDP on:", iface.sock.LocalAddr().String()) - bs := make([]byte, 65536) // This needs to be large enough for everything... - for { - n, udpAddr, err := iface.sock.ReadFromUDP(bs) - //iface.core.log.Println("DEBUG: read:", bs[0], bs[1], bs[2], n) - if err != nil { - panic(err) - break - } - msg := bs[:n] - var addr connAddr - addr.fromUDPAddr(udpAddr) - switch { - case udp_isKeys(msg): - var them address - copy(them[:], udpAddr.IP.To16()) - if them.isValid() { - continue - } - if udpAddr.IP.IsLinkLocalUnicast() { - if len(iface.core.ifceExpr) == 0 { - break - } - for _, expr := range iface.core.ifceExpr { - if expr.MatchString(udpAddr.Zone) { - iface.handleKeys(msg, addr) - break - } - } - } - case udp_isClose(msg): - iface.handleClose(msg, addr) - default: - iface.handlePacket(msg, addr) - } - } -} - -//////////////////////////////////////////////////////////////////////////////// - -func udp_decode(bs []byte) (chunks, chunk, count uint8, payload []byte) { - if len(bs) >= 3 { - chunks, chunk, count, payload = bs[0], bs[1], bs[2], bs[3:] - } - return -} - -func udp_encode(out []byte, chunks, chunk, count uint8, payload []byte) []byte { - return append(append(out, chunks, chunk, count), payload...) -} From 240841eb38eb026395f249f71db88fbef04a6599 Mon Sep 17 00:00:00 2001 From: Arceliar Date: Wed, 6 Jun 2018 16:46:15 -0500 Subject: [PATCH 03/42] remove unused recursive search packets --- build | 2 +- src/yggdrasil/router.go | 20 ------- src/yggdrasil/search.go | 123 ---------------------------------------- src/yggdrasil/wire.go | 57 ------------------- 4 files changed, 1 insertion(+), 201 deletions(-) diff --git a/build b/build index 8d34668a..d7e31991 100755 --- a/build +++ b/build @@ -5,7 +5,7 @@ go get -d -v go get -d -v yggdrasil for file in *.go ; do echo "Building: $file" - go build -v $file + go build $@ $file #go build -ldflags="-s -w" -v $file #upx --brute ${file/.go/} done diff --git a/src/yggdrasil/router.go b/src/yggdrasil/router.go index 7934b78a..b848a792 100644 --- a/src/yggdrasil/router.go +++ b/src/yggdrasil/router.go @@ -310,10 +310,6 @@ func (r *router) handleProto(packet []byte) { r.handleDHTReq(bs, &p.FromKey) case wire_DHTLookupResponse: r.handleDHTRes(bs, &p.FromKey) - case wire_SearchRequest: - r.handleSearchReq(bs) - case wire_SearchResponse: - r.handleSearchRes(bs) default: /*panic("Should not happen in testing") ;*/ return } @@ -350,22 +346,6 @@ func (r *router) handleDHTRes(bs []byte, fromKey *boxPubKey) { r.core.dht.handleRes(&res) } -func (r *router) handleSearchReq(bs []byte) { - req := searchReq{} - if !req.decode(bs) { - return - } - r.core.searches.handleSearchReq(&req) -} - -func (r *router) handleSearchRes(bs []byte) { - res := searchRes{} - if !res.decode(bs) { - return - } - r.core.searches.handleSearchRes(&res) -} - func (r *router) doAdmin(f func()) { // Pass this a function that needs to be run by the router's main goroutine // It will pass the function to the router and wait for the router to finish diff --git a/src/yggdrasil/search.go b/src/yggdrasil/search.go index 2aa3a858..593e7a23 100644 --- a/src/yggdrasil/search.go +++ b/src/yggdrasil/search.go @@ -170,126 +170,3 @@ func (s *searches) checkDHTRes(info *searchInfo, res *dhtRes) bool { return true } -//////////////////////////////////////////////////////////////////////////////// - -type searchReq struct { - key boxPubKey // Who I am - coords []byte // Where I am - dest NodeID // Who I'm trying to connect to -} - -type searchRes struct { - key boxPubKey // Who I am - coords []byte // Where I am - dest NodeID // Who I was asked about -} - -func (s *searches) sendSearch(info *searchInfo) { - now := time.Now() - if now.Sub(info.time) < time.Second { - return - } - loc := s.core.switchTable.getLocator() - coords := loc.getCoords() - req := searchReq{ - key: s.core.boxPub, - coords: coords, - dest: info.dest, - } - info.time = time.Now() - s.handleSearchReq(&req) -} - -func (s *searches) handleSearchReq(req *searchReq) { - lookup := s.core.dht.lookup(&req.dest, false) - sent := false - //fmt.Println("DEBUG len:", len(lookup)) - for _, info := range lookup { - //fmt.Println("DEBUG lup:", info.getNodeID()) - if dht_firstCloserThanThird(info.getNodeID(), - &req.dest, - &s.core.dht.nodeID) { - s.forwardSearch(req, info) - sent = true - break - } - } - if !sent { - s.sendSearchRes(req) - } -} - -func (s *searches) forwardSearch(req *searchReq, next *dhtInfo) { - //fmt.Println("DEBUG fwd:", req.dest, next.getNodeID()) - bs := req.encode() - shared := s.core.sessions.getSharedKey(&s.core.boxPriv, &next.key) - payload, nonce := boxSeal(shared, bs, nil) - p := wire_protoTrafficPacket{ - TTL: ^uint64(0), - Coords: next.coords, - ToKey: next.key, - FromKey: s.core.boxPub, - Nonce: *nonce, - Payload: payload, - } - packet := p.encode() - s.core.router.out(packet) -} - -func (s *searches) sendSearchRes(req *searchReq) { - //fmt.Println("DEBUG res:", req.dest, s.core.dht.nodeID) - loc := s.core.switchTable.getLocator() - coords := loc.getCoords() - res := searchRes{ - key: s.core.boxPub, - coords: coords, - dest: req.dest, - } - bs := res.encode() - shared := s.core.sessions.getSharedKey(&s.core.boxPriv, &req.key) - payload, nonce := boxSeal(shared, bs, nil) - p := wire_protoTrafficPacket{ - TTL: ^uint64(0), - Coords: req.coords, - ToKey: req.key, - FromKey: s.core.boxPub, - Nonce: *nonce, - Payload: payload, - } - packet := p.encode() - s.core.router.out(packet) -} - -func (s *searches) handleSearchRes(res *searchRes) { - info, isIn := s.searches[res.dest] - if !isIn { - return - } - them := getNodeID(&res.key) - var destMasked NodeID - var themMasked NodeID - for idx := 0; idx < NodeIDLen; idx++ { - destMasked[idx] = info.dest[idx] & info.mask[idx] - themMasked[idx] = them[idx] & info.mask[idx] - } - //fmt.Println("DEBUG search res1:", themMasked, destMasked) - //fmt.Println("DEBUG search res2:", *them, *info.dest, *info.mask) - if themMasked != destMasked { - return - } - // They match, so create a session and send a sessionRequest - sinfo, isIn := s.core.sessions.getByTheirPerm(&res.key) - if !isIn { - sinfo = s.core.sessions.createSession(&res.key) - _, isIn := s.core.sessions.getByTheirPerm(&res.key) - if !isIn { - panic("This should never happen") - } - } - // FIXME (!) replay attacks could mess with coords? Give it a handle (tstamp)? - sinfo.coords = res.coords - sinfo.packet = info.packet - s.core.sessions.ping(sinfo) - // Cleanup - delete(s.searches, res.dest) -} diff --git a/src/yggdrasil/wire.go b/src/yggdrasil/wire.go index 71aa0b22..512a75c1 100644 --- a/src/yggdrasil/wire.go +++ b/src/yggdrasil/wire.go @@ -19,8 +19,6 @@ const ( wire_SessionPong // inside protocol traffic header wire_DHTLookupRequest // inside protocol traffic header wire_DHTLookupResponse // inside protocol traffic header - wire_SearchRequest // inside protocol traffic header - wire_SearchResponse // inside protocol traffic header ) // Encode uint64 using a variable length scheme @@ -514,58 +512,3 @@ func (r *dhtRes) decode(bs []byte) bool { return true } -//////////////////////////////////////////////////////////////////////////////// - -func (r *searchReq) encode() []byte { - coords := wire_encode_coords(r.coords) - bs := wire_encode_uint64(wire_SearchRequest) - bs = append(bs, r.key[:]...) - bs = append(bs, coords...) - bs = append(bs, r.dest[:]...) - return bs -} - -func (r *searchReq) decode(bs []byte) bool { - var pType uint64 - switch { - case !wire_chop_uint64(&pType, &bs): - return false - case pType != wire_SearchRequest: - return false - case !wire_chop_slice(r.key[:], &bs): - return false - case !wire_chop_coords(&r.coords, &bs): - return false - case !wire_chop_slice(r.dest[:], &bs): - return false - default: - return true - } -} - -func (r *searchRes) encode() []byte { - coords := wire_encode_coords(r.coords) - bs := wire_encode_uint64(wire_SearchResponse) - bs = append(bs, r.key[:]...) - bs = append(bs, coords...) - bs = append(bs, r.dest[:]...) - return bs -} - -func (r *searchRes) decode(bs []byte) bool { - var pType uint64 - switch { - case !wire_chop_uint64(&pType, &bs): - return false - case pType != wire_SearchResponse: - return false - case !wire_chop_slice(r.key[:], &bs): - return false - case !wire_chop_coords(&r.coords, &bs): - return false - case !wire_chop_slice(r.dest[:], &bs): - return false - default: - return true - } -} From da928af361f1e8eed970621356b3ac0d859cc830 Mon Sep 17 00:00:00 2001 From: Arceliar Date: Wed, 6 Jun 2018 16:49:23 -0500 Subject: [PATCH 04/42] fix sim and run gofmt --- misc/sim/treesim.go | 12 ++++++------ src/yggdrasil/search.go | 1 - src/yggdrasil/wire.go | 1 - 3 files changed, 6 insertions(+), 8 deletions(-) diff --git a/misc/sim/treesim.go b/misc/sim/treesim.go index 8ea06596..d8859261 100644 --- a/misc/sim/treesim.go +++ b/misc/sim/treesim.go @@ -379,12 +379,12 @@ func dumpDHTSize(store map[[32]byte]*Node) { fmt.Printf("DHT min %d / avg %f / max %d\n", min, avg, max) } -func (n *Node) startUDP(listen string) { - n.core.DEBUG_setupAndStartGlobalUDPInterface(listen) +func (n *Node) startTCP(listen string) { + n.core.DEBUG_setupAndStartGlobalTCPInterface(listen) } -func (n *Node) connectUDP(remoteAddr string) { - n.core.DEBUG_maybeSendUDPKeys(remoteAddr) +func (n *Node) connectTCP(remoteAddr string) { + n.core.AddPeer(remoteAddr) } //////////////////////////////////////////////////////////////////////////////// @@ -440,8 +440,8 @@ func main() { if false { // This connects the sim to the local network for _, node := range kstore { - node.startUDP("localhost:0") - node.connectUDP("localhost:12345") + node.startTCP("localhost:0") + node.connectTCP("localhost:12345") break // just 1 } for _, node := range kstore { diff --git a/src/yggdrasil/search.go b/src/yggdrasil/search.go index 593e7a23..4b9cd03a 100644 --- a/src/yggdrasil/search.go +++ b/src/yggdrasil/search.go @@ -169,4 +169,3 @@ func (s *searches) checkDHTRes(info *searchInfo, res *dhtRes) bool { delete(s.searches, res.Dest) return true } - diff --git a/src/yggdrasil/wire.go b/src/yggdrasil/wire.go index 512a75c1..6b592e5b 100644 --- a/src/yggdrasil/wire.go +++ b/src/yggdrasil/wire.go @@ -511,4 +511,3 @@ func (r *dhtRes) decode(bs []byte) bool { } return true } - From fad6f6b50e4d1b63a8695a82855ea3c646e7e44d Mon Sep 17 00:00:00 2001 From: Arceliar Date: Wed, 6 Jun 2018 16:57:36 -0500 Subject: [PATCH 05/42] remove udp.go --- src/yggdrasil/udp.go | 374 ------------------------------------------- 1 file changed, 374 deletions(-) delete mode 100644 src/yggdrasil/udp.go diff --git a/src/yggdrasil/udp.go b/src/yggdrasil/udp.go deleted file mode 100644 index 68f53a8e..00000000 --- a/src/yggdrasil/udp.go +++ /dev/null @@ -1,374 +0,0 @@ -package yggdrasil - -// This communicates with peers via UDP -// It's not as well tested or debugged as the TCP transport -// It's intended to use UDP, so debugging/optimzing this is a high priority -// TODO? use golang.org/x/net/ipv6.PacketConn's ReadBatch and WriteBatch? -// To send all chunks of a message / recv all available chunks in one syscall -// That might be faster on supported platforms, but it needs investigation -// Chunks are currently murged, but outgoing messages aren't chunked -// This is just to support chunking in the future, if it's needed and debugged -// Basically, right now we might send UDP packets that are too large - -// TODO remove old/unused code and better document live code - -import "net" -import "time" -import "sync" -import "fmt" - -type udpInterface struct { - core *Core - sock *net.UDPConn // Or more general PacketConn? - mutex sync.RWMutex // each conn has an owner goroutine - conns map[connAddr]*connInfo -} - -type connAddr struct { - ip [16]byte - port int - zone string -} - -func (c *connAddr) fromUDPAddr(u *net.UDPAddr) { - copy(c.ip[:], u.IP.To16()) - c.port = u.Port - c.zone = u.Zone -} - -func (c *connAddr) toUDPAddr() *net.UDPAddr { - var u net.UDPAddr - u.IP = make([]byte, 16) - copy(u.IP, c.ip[:]) - u.Port = c.port - u.Zone = c.zone - return &u -} - -type connInfo struct { - name string - addr connAddr - peer *peer - linkIn chan []byte - keysIn chan *udpKeys - closeIn chan *udpKeys - timeout int // count of how many heartbeats have been missed - in func([]byte) - out chan []byte - countIn uint8 - countOut uint8 - chunkSize uint16 -} - -type udpKeys struct { - box boxPubKey - sig sigPubKey -} - -func (iface *udpInterface) init(core *Core, addr string) (err error) { - iface.core = core - udpAddr, err := net.ResolveUDPAddr("udp", addr) - if err != nil { - return - } - iface.sock, err = net.ListenUDP("udp", udpAddr) - if err != nil { - return - } - iface.conns = make(map[connAddr]*connInfo) - go iface.reader() - return -} - -func (iface *udpInterface) sendKeys(addr connAddr) { - udpAddr := addr.toUDPAddr() - msg := []byte{} - msg = udp_encode(msg, 0, 0, 0, nil) - msg = append(msg, iface.core.boxPub[:]...) - msg = append(msg, iface.core.sigPub[:]...) - iface.sock.WriteToUDP(msg, udpAddr) -} - -func (iface *udpInterface) sendClose(addr connAddr) { - udpAddr := addr.toUDPAddr() - msg := []byte{} - msg = udp_encode(msg, 0, 1, 0, nil) - msg = append(msg, iface.core.boxPub[:]...) - msg = append(msg, iface.core.sigPub[:]...) - iface.sock.WriteToUDP(msg, udpAddr) -} - -func udp_isKeys(msg []byte) bool { - keyLen := 3 + boxPubKeyLen + sigPubKeyLen - return len(msg) == keyLen && msg[0] == 0x00 && msg[1] == 0x00 -} - -func udp_isClose(msg []byte) bool { - keyLen := 3 + boxPubKeyLen + sigPubKeyLen - return len(msg) == keyLen && msg[0] == 0x00 && msg[1] == 0x01 -} - -func (iface *udpInterface) startConn(info *connInfo) { - ticker := time.NewTicker(6 * time.Second) - defer ticker.Stop() - defer func() { - // Cleanup - iface.mutex.Lock() - delete(iface.conns, info.addr) - iface.mutex.Unlock() - iface.core.peers.removePeer(info.peer.port) - close(info.linkIn) - close(info.keysIn) - close(info.closeIn) - close(info.out) - iface.core.log.Println("Removing peer:", info.name) - }() - for { - select { - case ks := <-info.closeIn: - { - if ks.box == info.peer.box && ks.sig == info.peer.sig { - // TODO? secure this somehow - // Maybe add a signature and sequence number (timestamp) to close and keys? - return - } - } - case ks := <-info.keysIn: - { - // FIXME? need signatures/sequence-numbers or something - // Spoofers could lock out a peer with fake/bad keys - if ks.box == info.peer.box && ks.sig == info.peer.sig { - info.timeout = 0 - } - } - case <-ticker.C: - { - if info.timeout > 10 { - return - } - info.timeout++ - iface.sendKeys(info.addr) - } - } - } -} - -func (iface *udpInterface) handleClose(msg []byte, addr connAddr) { - //defer util_putBytes(msg) - var ks udpKeys - _, _, _, bs := udp_decode(msg) - switch { - case !wire_chop_slice(ks.box[:], &bs): - return - case !wire_chop_slice(ks.sig[:], &bs): - return - } - if ks.box == iface.core.boxPub { - return - } - if ks.sig == iface.core.sigPub { - return - } - iface.mutex.RLock() - conn, isIn := iface.conns[addr] - iface.mutex.RUnlock() - if !isIn { - return - } - func() { - defer func() { recover() }() - select { - case conn.closeIn <- &ks: - default: - } - }() -} - -func (iface *udpInterface) handleKeys(msg []byte, addr connAddr) { - //defer util_putBytes(msg) - var ks udpKeys - _, _, _, bs := udp_decode(msg) - switch { - case !wire_chop_slice(ks.box[:], &bs): - return - case !wire_chop_slice(ks.sig[:], &bs): - return - } - if ks.box == iface.core.boxPub { - return - } - if ks.sig == iface.core.sigPub { - return - } - iface.mutex.RLock() - conn, isIn := iface.conns[addr] - iface.mutex.RUnlock() - if !isIn { - udpAddr := addr.toUDPAddr() - // Check if we're authorized to connect to this key / IP - // TODO monitor and always allow outgoing connections - if !iface.core.peers.isAllowedEncryptionPublicKey(&ks.box) { - // Allow unauthorized peers if they're link-local - if !udpAddr.IP.IsLinkLocalUnicast() { - return - } - } - themNodeID := getNodeID(&ks.box) - themAddr := address_addrForNodeID(themNodeID) - themAddrString := net.IP(themAddr[:]).String() - themString := fmt.Sprintf("%s@%s", themAddrString, udpAddr.String()) - conn = &connInfo{ - name: themString, - addr: connAddr(addr), - peer: iface.core.peers.newPeer(&ks.box, &ks.sig), - linkIn: make(chan []byte, 1), - keysIn: make(chan *udpKeys, 1), - closeIn: make(chan *udpKeys, 1), - out: make(chan []byte, 32), - chunkSize: 576 - 60 - 8 - 3, // max safe - max ip - udp header - chunk overhead - } - if udpAddr.IP.IsLinkLocalUnicast() { - ifce, err := net.InterfaceByName(udpAddr.Zone) - if ifce != nil && err == nil { - conn.chunkSize = uint16(ifce.MTU) - 60 - 8 - 3 - } - } - var inChunks uint8 - var inBuf []byte - conn.in = func(bs []byte) { - //defer util_putBytes(bs) - chunks, chunk, count, payload := udp_decode(bs) - if count != conn.countIn { - if len(inBuf) > 0 { - // Something went wrong - // Forward whatever we have - // Maybe the destination can do something about it - msg := append(util_getBytes(), inBuf...) - conn.peer.handlePacket(msg, conn.linkIn) - } - inChunks = 0 - inBuf = inBuf[:0] - conn.countIn = count - } - if chunk <= chunks && chunk == inChunks+1 { - inChunks += 1 - inBuf = append(inBuf, payload...) - if chunks != chunk { - return - } - msg := append(util_getBytes(), inBuf...) - conn.peer.handlePacket(msg, conn.linkIn) - inBuf = inBuf[:0] - } - } - conn.peer.out = func(msg []byte) { - defer func() { recover() }() - select { - case conn.out <- msg: - conn.peer.updateQueueSize(1) - default: - util_putBytes(msg) - } - } - go func() { - var out []byte - var chunks [][]byte - for msg := range conn.out { - chunks = chunks[:0] - bs := msg - for len(bs) > int(conn.chunkSize) { - chunks, bs = append(chunks, bs[:conn.chunkSize]), bs[conn.chunkSize:] - } - chunks = append(chunks, bs) - if len(chunks) > 255 { - continue - } - for idx, bs := range chunks { - nChunks, nChunk, count := uint8(len(chunks)), uint8(idx)+1, conn.countOut - out = udp_encode(out[:0], nChunks, nChunk, count, bs) - //iface.core.log.Println("DEBUG out:", nChunks, nChunk, count, len(bs)) - iface.sock.WriteToUDP(out, udpAddr) - } - conn.countOut += 1 - conn.peer.updateQueueSize(-1) - util_putBytes(msg) - } - }() - //*/ - conn.peer.close = func() { iface.sendClose(conn.addr) } - iface.mutex.Lock() - iface.conns[addr] = conn - iface.mutex.Unlock() - iface.core.log.Println("Adding peer:", conn.name) - go iface.startConn(conn) - go conn.peer.linkLoop(conn.linkIn) - iface.sendKeys(conn.addr) - } - func() { - defer func() { recover() }() - select { - case conn.keysIn <- &ks: - default: - } - }() -} - -func (iface *udpInterface) handlePacket(msg []byte, addr connAddr) { - iface.mutex.RLock() - if conn, isIn := iface.conns[addr]; isIn { - conn.in(msg) - } - iface.mutex.RUnlock() -} - -func (iface *udpInterface) reader() { - iface.core.log.Println("Listening for UDP on:", iface.sock.LocalAddr().String()) - bs := make([]byte, 65536) // This needs to be large enough for everything... - for { - n, udpAddr, err := iface.sock.ReadFromUDP(bs) - //iface.core.log.Println("DEBUG: read:", bs[0], bs[1], bs[2], n) - if err != nil { - panic(err) - break - } - msg := bs[:n] - var addr connAddr - addr.fromUDPAddr(udpAddr) - switch { - case udp_isKeys(msg): - var them address - copy(them[:], udpAddr.IP.To16()) - if them.isValid() { - continue - } - if udpAddr.IP.IsLinkLocalUnicast() { - if len(iface.core.ifceExpr) == 0 { - break - } - for _, expr := range iface.core.ifceExpr { - if expr.MatchString(udpAddr.Zone) { - iface.handleKeys(msg, addr) - break - } - } - } - case udp_isClose(msg): - iface.handleClose(msg, addr) - default: - iface.handlePacket(msg, addr) - } - } -} - -//////////////////////////////////////////////////////////////////////////////// - -func udp_decode(bs []byte) (chunks, chunk, count uint8, payload []byte) { - if len(bs) >= 3 { - chunks, chunk, count, payload = bs[0], bs[1], bs[2], bs[3:] - } - return -} - -func udp_encode(out []byte, chunks, chunk, count uint8, payload []byte) []byte { - return append(append(out, chunks, chunk, count), payload...) -} From 690d29435d7a4c88e413a1d8e470cccf062c9468 Mon Sep 17 00:00:00 2001 From: Arceliar Date: Wed, 6 Jun 2018 17:44:10 -0500 Subject: [PATCH 06/42] adjust link packet logic so they bypass the lifo stack and are delivered first --- src/yggdrasil/debug.go | 12 ++++++------ src/yggdrasil/peer.go | 21 +++++++++------------ src/yggdrasil/router.go | 2 +- src/yggdrasil/tcp.go | 30 ++++++++++++++++++++---------- 4 files changed, 36 insertions(+), 29 deletions(-) diff --git a/src/yggdrasil/debug.go b/src/yggdrasil/debug.go index cd42560c..599ee90c 100644 --- a/src/yggdrasil/debug.go +++ b/src/yggdrasil/debug.go @@ -453,16 +453,16 @@ func (c *Core) DEBUG_addAllowedEncryptionPublicKey(boxStr string) { func DEBUG_simLinkPeers(p, q *peer) { // Sets q.out() to point to p and starts p.linkLoop() - plinkIn := make(chan []byte, 1) - qlinkIn := make(chan []byte, 1) + p.linkIn, q.linkIn = make(chan []byte, 32), make(chan []byte, 32) + p.linkOut, q.linkOut = q.linkIn, p.linkIn p.out = func(bs []byte) { - go q.handlePacket(bs, qlinkIn) + go q.handlePacket(bs) } q.out = func(bs []byte) { - go p.handlePacket(bs, plinkIn) + go p.handlePacket(bs) } - go p.linkLoop(plinkIn) - go q.linkLoop(qlinkIn) + go p.linkLoop() + go q.linkLoop() } func (c *Core) DEBUG_simFixMTU() { diff --git a/src/yggdrasil/peer.go b/src/yggdrasil/peer.go index fa1a2789..0113470e 100644 --- a/src/yggdrasil/peer.go +++ b/src/yggdrasil/peer.go @@ -111,6 +111,9 @@ type peer struct { close func() // To allow the peer to call close if idle for too long lastAnc time.Time + // used for protocol traffic (to bypass queues) + linkIn (chan []byte) // handlePacket sends, linkLoop recvs + linkOut (chan []byte) } const peer_Throttle = 1 @@ -123,8 +126,7 @@ func (p *peer) updateQueueSize(delta int64) { atomic.AddInt64(&p.queueSize, delta) } -func (ps *peers) newPeer(box *boxPubKey, - sig *sigPubKey) *peer { +func (ps *peers) newPeer(box *boxPubKey, sig *sigPubKey) *peer { now := time.Now() p := peer{box: *box, sig: *sig, @@ -170,14 +172,14 @@ func (ps *peers) removePeer(port switchPort) { } } -func (p *peer) linkLoop(in <-chan []byte) { +func (p *peer) linkLoop() { ticker := time.NewTicker(time.Second) defer ticker.Stop() var counter uint8 var lastRSeq uint64 for { select { - case packet, ok := <-in: + case packet, ok := <-p.linkIn: if !ok { return } @@ -214,7 +216,7 @@ func (p *peer) linkLoop(in <-chan []byte) { } } -func (p *peer) handlePacket(packet []byte, linkIn chan<- []byte) { +func (p *peer) handlePacket(packet []byte) { // TODO See comment in sendPacket about atomics technically being done wrong atomic.AddUint64(&p.bytesRecvd, uint64(len(packet))) pType, pTypeLen := wire_decode_uint64(packet) @@ -227,12 +229,7 @@ func (p *peer) handlePacket(packet []byte, linkIn chan<- []byte) { case wire_ProtocolTraffic: p.handleTraffic(packet, pTypeLen) case wire_LinkProtocolTraffic: - { - select { - case linkIn <- packet: - default: - } - } + p.linkIn <- packet default: /*panic(pType) ;*/ return } @@ -284,7 +281,7 @@ func (p *peer) sendLinkPacket(packet []byte) { Payload: bs, } packet = linkPacket.encode() - p.sendPacket(packet) + p.linkOut <- packet } func (p *peer) handleLinkTraffic(bs []byte) { diff --git a/src/yggdrasil/router.go b/src/yggdrasil/router.go index b848a792..a8797d5f 100644 --- a/src/yggdrasil/router.go +++ b/src/yggdrasil/router.go @@ -55,7 +55,7 @@ func (r *router) init(core *Core) { } } r.in = in - r.out = func(packet []byte) { p.handlePacket(packet, nil) } // The caller is responsible for go-ing if it needs to not block + r.out = func(packet []byte) { p.handlePacket(packet) } // The caller is responsible for go-ing if it needs to not block recv := make(chan []byte, 32) send := make(chan []byte, 32) r.recv = recv diff --git a/src/yggdrasil/tcp.go b/src/yggdrasil/tcp.go index acde0344..e21522b8 100644 --- a/src/yggdrasil/tcp.go +++ b/src/yggdrasil/tcp.go @@ -208,10 +208,11 @@ func (iface *tcpInterface) handler(sock net.Conn, incoming bool) { }() // Note that multiple connections to the same node are allowed // E.g. over different interfaces - linkIn := make(chan []byte, 1) - p := iface.core.peers.newPeer(&info.box, &info.sig) //, in, out) + p := iface.core.peers.newPeer(&info.box, &info.sig) + p.linkIn = make(chan []byte, 1) + p.linkOut = make(chan []byte, 1) in := func(bs []byte) { - p.handlePacket(bs, linkIn) + p.handlePacket(bs) } out := make(chan []byte, 32) // TODO? what size makes sense defer close(out) @@ -221,10 +222,10 @@ func (iface *tcpInterface) handler(sock net.Conn, incoming bool) { buf.Write(tcp_msg[:]) buf.Write(msgLen) buf.Write(msg) - p.updateQueueSize(-1) util_putBytes(msg) } go func() { + defer buf.Flush() var stack [][]byte put := func(msg []byte) { stack = append(stack, msg) @@ -234,14 +235,22 @@ func (iface *tcpInterface) handler(sock net.Conn, incoming bool) { p.updateQueueSize(-1) } } - for msg := range out { - put(msg) + for { + select { + case msg := <-p.linkOut: + send(msg) + case msg, ok := <-out: + if !ok { + return + } + put(msg) + } for len(stack) > 0 { - // Keep trying to fill the stack (LIFO order) while sending select { + case msg := <-p.linkOut: + send(msg) case msg, ok := <-out: if !ok { - buf.Flush() return } put(msg) @@ -249,6 +258,7 @@ func (iface *tcpInterface) handler(sock net.Conn, incoming bool) { msg := stack[len(stack)-1] stack = stack[:len(stack)-1] send(msg) + p.updateQueueSize(-1) } } buf.Flush() @@ -265,11 +275,11 @@ func (iface *tcpInterface) handler(sock net.Conn, incoming bool) { } p.close = func() { sock.Close() } setNoDelay(sock, true) - go p.linkLoop(linkIn) + go p.linkLoop() defer func() { // Put all of our cleanup here... p.core.peers.removePeer(p.port) - close(linkIn) + close(p.linkIn) }() them, _, _ := net.SplitHostPort(sock.RemoteAddr().String()) themNodeID := getNodeID(&info.box) From 6811759fc9f3fd5e103546c3d6505c410503602c Mon Sep 17 00:00:00 2001 From: Arceliar Date: Wed, 6 Jun 2018 21:11:10 -0500 Subject: [PATCH 07/42] add new switchMsg struct to replace old anc/hop/res approach --- src/yggdrasil/peer.go | 90 ++++++++++++++++++++++++++++++++++++++----- src/yggdrasil/wire.go | 56 +++++++++++++++++++++++++++ 2 files changed, 137 insertions(+), 9 deletions(-) diff --git a/src/yggdrasil/peer.go b/src/yggdrasil/peer.go index 0113470e..1656d392 100644 --- a/src/yggdrasil/peer.go +++ b/src/yggdrasil/peer.go @@ -205,11 +205,13 @@ func (p *peer) linkLoop() { case counter%4 == 0: update = true } - if update { + if true || update { + // TODO change update logic, the new switchMsg works differently if p.msgAnc != nil { lastRSeq = p.msgAnc.Seq } - p.sendSwitchAnnounce() + p.sendSwitchMsg() + //p.sendSwitchAnnounce() } counter = (counter + 1) % 4 } @@ -236,10 +238,11 @@ func (p *peer) handlePacket(packet []byte) { } func (p *peer) handleTraffic(packet []byte, pTypeLen int) { - if p.port != 0 && p.msgAnc == nil { - // Drop traffic until the peer manages to send us at least one anc - return - } + //if p.port != 0 && p.msgAnc == nil { + // // Drop traffic until the peer manages to send us at least one anc + // // TODO? equivalent for new switch format? + // return + //} ttl, ttlLen := wire_decode_uint64(packet[pTypeLen:]) ttlBegin := pTypeLen ttlEnd := pTypeLen + ttlLen @@ -298,6 +301,8 @@ func (p *peer) handleLinkTraffic(bs []byte) { return } switch pType { + case wire_SwitchMsg: + p.handleSwitchMsg(payload) case wire_SwitchAnnounce: p.handleSwitchAnnounce(payload) case wire_SwitchHopRequest: @@ -467,12 +472,79 @@ func (p *peer) processSwitchMessage() { if len(coords) == 0 { return } - // Reuse locator, set the coords to the peer's coords, to use in dht - msg.locator.coords = coords[:len(coords)-1] // Pass a mesage to the dht informing it that this peer (still) exists + l := msg.locator + l.coords = l.coords[:len(l.coords)-1] dinfo := dhtInfo{ key: p.box, - coords: msg.locator.getCoords(), + coords: l.getCoords(), + } + p.core.dht.peers <- &dinfo + p.core.log.Println("DEBUG: peers<-&dhtInfo", dinfo, p.box, msg) +} + +func (p *peer) sendSwitchMsg() { + info, sigs := p.core.switchTable.createMessage(p.port) + var msg switchMsg + msg.Root, msg.TStamp = info.locator.root, info.locator.tstamp + for idx, sig := range sigs { + hop := switchMsgHop{ + Port: info.locator.coords[idx], + Next: sig.next, + Sig: sig.sig, + } + msg.Hops = append(msg.Hops, hop) + } + msg.Hops = append(msg.Hops, switchMsgHop{ + Port: p.port, + Next: p.sig, + Sig: *sign(&p.core.sigPriv, getBytesForSig(&p.sig, &info.locator)), + }) + packet := msg.encode() + var test switchMsg + test.decode(packet) + //p.core.log.Println("Encoded msg:", msg, "; bytes:", packet) + p.sendLinkPacket(packet) +} + +func (p *peer) handleSwitchMsg(packet []byte) { + var msg switchMsg + msg.decode(packet) + //p.core.log.Println("Decoded msg:", msg, "; bytes:", packet) + if len(msg.Hops) < 1 { + p.throttle++ + panic("FIXME testing") + return + } + var info switchMessage + var sigs []sigInfo + info.locator.root = msg.Root + info.locator.tstamp = msg.TStamp + thisHopKey := &msg.Root + for _, hop := range msg.Hops { + var sig sigInfo + sig.next = hop.Next + sig.sig = hop.Sig + sigs = append(sigs, sig) + info.locator.coords = append(info.locator.coords, hop.Port) + // TODO check signatures + bs := getBytesForSig(&hop.Next, &info.locator) + if !p.core.sigs.check(thisHopKey, &hop.Sig, bs) { + //p.throttle++ + //panic("FIXME testing") + //return + } + thisHopKey = &hop.Next + } + info.from = p.sig + info.seq = uint64(time.Now().Unix()) + p.core.switchTable.handleMessage(&info, p.port, sigs) + // Pass a mesage to the dht informing it that this peer (still) exists + l := info.locator + l.coords = l.coords[:len(l.coords)-1] + dinfo := dhtInfo{ + key: p.box, + coords: l.getCoords(), } p.core.dht.peers <- &dinfo } diff --git a/src/yggdrasil/wire.go b/src/yggdrasil/wire.go index 6b592e5b..bd298de9 100644 --- a/src/yggdrasil/wire.go +++ b/src/yggdrasil/wire.go @@ -12,6 +12,7 @@ const ( wire_Traffic = iota // data being routed somewhere, handle for crypto wire_ProtocolTraffic // protocol traffic, pub keys for crypto wire_LinkProtocolTraffic // link proto traffic, pub keys for crypto + wire_SwitchMsg // inside link protocol traffic header wire_SwitchAnnounce // inside protocol traffic header wire_SwitchHopRequest // inside protocol traffic header wire_SwitchHop // inside protocol traffic header @@ -117,6 +118,61 @@ func wire_decode_coords(packet []byte) ([]byte, int) { //////////////////////////////////////////////////////////////////////////////// +type switchMsg struct { + Root sigPubKey + TStamp int64 + Hops []switchMsgHop +} + +type switchMsgHop struct { + Port switchPort + Next sigPubKey + Sig sigBytes +} + +func (m *switchMsg) encode() []byte { + bs := wire_encode_uint64(wire_SwitchMsg) + bs = append(bs, m.Root[:]...) + bs = append(bs, wire_encode_uint64(wire_intToUint(m.TStamp))...) + for _, hop := range m.Hops { + bs = append(bs, wire_encode_uint64(uint64(hop.Port))...) + bs = append(bs, hop.Next[:]...) + bs = append(bs, hop.Sig[:]...) + } + return bs +} + +func (m *switchMsg) decode(bs []byte) bool { + var pType uint64 + var tstamp uint64 + switch { + case !wire_chop_uint64(&pType, &bs): + return false + case pType != wire_SwitchMsg: + return false + case !wire_chop_slice(m.Root[:], &bs): + return false + case !wire_chop_uint64(&tstamp, &bs): + return false + } + m.TStamp = wire_intFromUint(tstamp) + for len(bs) > 0 { + var hop switchMsgHop + switch { + case !wire_chop_uint64((*uint64)(&hop.Port), &bs): + return false + case !wire_chop_slice(hop.Next[:], &bs): + return false + case !wire_chop_slice(hop.Sig[:], &bs): + return false + } + m.Hops = append(m.Hops, hop) + } + return true +} + +//////////////////////////////////////////////////////////////////////////////// + // Announces that we can send parts of a Message with a particular seq type msgAnnounce struct { Root sigPubKey From 5fb33da3a2521319ff6a81c3f3c55d8141003b25 Mon Sep 17 00:00:00 2001 From: Arceliar Date: Wed, 6 Jun 2018 21:18:21 -0500 Subject: [PATCH 08/42] remove old switch anc/hop/res code --- src/yggdrasil/peer.go | 241 ++---------------------------------------- src/yggdrasil/wire.go | 133 +---------------------- 2 files changed, 7 insertions(+), 367 deletions(-) diff --git a/src/yggdrasil/peer.go b/src/yggdrasil/peer.go index 1656d392..7ddf4bbb 100644 --- a/src/yggdrasil/peer.go +++ b/src/yggdrasil/peer.go @@ -96,13 +96,9 @@ type peer struct { //in <-chan []byte //out chan<- []byte //in func([]byte) - out func([]byte) - core *Core - port switchPort - msgAnc *msgAnnounce - msgHops []*msgHop - myMsg *switchMessage - mySigs []sigInfo + out func([]byte) + core *Core + port switchPort // This is used to limit how often we perform expensive operations // Specifically, processing switch messages, signing, and verifying sigs // Resets at the start of each tick @@ -175,8 +171,6 @@ func (ps *peers) removePeer(port switchPort) { func (p *peer) linkLoop() { ticker := time.NewTicker(time.Second) defer ticker.Stop() - var counter uint8 - var lastRSeq uint64 for { select { case packet, ok := <-p.linkIn: @@ -193,27 +187,8 @@ func (p *peer) linkLoop() { if p.port == 0 { continue } // Don't send announces on selfInterface - p.myMsg, p.mySigs = p.core.switchTable.createMessage(p.port) - var update bool - switch { - case p.msgAnc == nil: - update = true - case lastRSeq != p.msgAnc.Seq: - update = true - case p.msgAnc.Rseq != p.myMsg.seq: - update = true - case counter%4 == 0: - update = true - } - if true || update { - // TODO change update logic, the new switchMsg works differently - if p.msgAnc != nil { - lastRSeq = p.msgAnc.Seq - } - p.sendSwitchMsg() - //p.sendSwitchAnnounce() - } - counter = (counter + 1) % 4 + // TODO change update logic, the new switchMsg works differently, we only need to send if something changes + p.sendSwitchMsg() } } } @@ -303,186 +278,10 @@ func (p *peer) handleLinkTraffic(bs []byte) { switch pType { case wire_SwitchMsg: p.handleSwitchMsg(payload) - case wire_SwitchAnnounce: - p.handleSwitchAnnounce(payload) - case wire_SwitchHopRequest: - p.handleSwitchHopRequest(payload) - case wire_SwitchHop: - p.handleSwitchHop(payload) + default: // TODO?... } } -func (p *peer) handleSwitchAnnounce(packet []byte) { - //p.core.log.Println("DEBUG: handleSwitchAnnounce") - anc := msgAnnounce{} - //err := wire_decode_struct(packet, &anc) - //if err != nil { return } - if !anc.decode(packet) { - return - } - //if p.msgAnc != nil && anc.Seq != p.msgAnc.Seq { p.msgHops = nil } - if p.msgAnc == nil || - anc.Root != p.msgAnc.Root || - anc.Tstamp != p.msgAnc.Tstamp || - anc.Seq != p.msgAnc.Seq { - p.msgHops = nil - } - p.msgAnc = &anc - p.processSwitchMessage() - p.lastAnc = time.Now() -} - -func (p *peer) requestHop(hop uint64) { - //p.core.log.Println("DEBUG requestHop") - req := msgHopReq{} - req.Root = p.msgAnc.Root - req.Tstamp = p.msgAnc.Tstamp - req.Seq = p.msgAnc.Seq - req.Hop = hop - packet := req.encode() - p.sendLinkPacket(packet) -} - -func (p *peer) handleSwitchHopRequest(packet []byte) { - //p.core.log.Println("DEBUG: handleSwitchHopRequest") - if p.throttle > peer_Throttle { - return - } - if p.myMsg == nil { - return - } - req := msgHopReq{} - if !req.decode(packet) { - return - } - if req.Root != p.myMsg.locator.root { - return - } - if req.Tstamp != p.myMsg.locator.tstamp { - return - } - if req.Seq != p.myMsg.seq { - return - } - if uint64(len(p.myMsg.locator.coords)) <= req.Hop { - return - } - res := msgHop{} - res.Root = p.myMsg.locator.root - res.Tstamp = p.myMsg.locator.tstamp - res.Seq = p.myMsg.seq - res.Hop = req.Hop - res.Port = p.myMsg.locator.coords[res.Hop] - sinfo := p.getSig(res.Hop) - //p.core.log.Println("DEBUG sig:", sinfo) - res.Next = sinfo.next - res.Sig = sinfo.sig - packet = res.encode() - p.sendLinkPacket(packet) -} - -func (p *peer) handleSwitchHop(packet []byte) { - //p.core.log.Println("DEBUG: handleSwitchHop") - if p.throttle > peer_Throttle { - return - } - if p.msgAnc == nil { - return - } - res := msgHop{} - if !res.decode(packet) { - return - } - if res.Root != p.msgAnc.Root { - return - } - if res.Tstamp != p.msgAnc.Tstamp { - return - } - if res.Seq != p.msgAnc.Seq { - return - } - if res.Hop != uint64(len(p.msgHops)) { - return - } // always process in order - loc := switchLocator{coords: make([]switchPort, 0, len(p.msgHops)+1)} - loc.root = res.Root - loc.tstamp = res.Tstamp - for _, hop := range p.msgHops { - loc.coords = append(loc.coords, hop.Port) - } - loc.coords = append(loc.coords, res.Port) - thisHopKey := &res.Root - if res.Hop != 0 { - thisHopKey = &p.msgHops[res.Hop-1].Next - } - bs := getBytesForSig(&res.Next, &loc) - if p.core.sigs.check(thisHopKey, &res.Sig, bs) { - p.msgHops = append(p.msgHops, &res) - p.processSwitchMessage() - } else { - p.throttle++ - } -} - -func (p *peer) processSwitchMessage() { - //p.core.log.Println("DEBUG: processSwitchMessage") - if p.throttle > peer_Throttle { - return - } - if p.msgAnc == nil { - return - } - if uint64(len(p.msgHops)) < p.msgAnc.Len { - p.requestHop(uint64(len(p.msgHops))) - return - } - p.throttle++ - if p.msgAnc.Len != uint64(len(p.msgHops)) { - return - } - msg := switchMessage{} - coords := make([]switchPort, 0, len(p.msgHops)) - sigs := make([]sigInfo, 0, len(p.msgHops)) - for idx, hop := range p.msgHops { - // Consistency checks, should be redundant (already checked these...) - if hop.Root != p.msgAnc.Root { - return - } - if hop.Tstamp != p.msgAnc.Tstamp { - return - } - if hop.Seq != p.msgAnc.Seq { - return - } - if hop.Hop != uint64(idx) { - return - } - coords = append(coords, hop.Port) - sigs = append(sigs, sigInfo{next: hop.Next, sig: hop.Sig}) - } - msg.from = p.sig - msg.locator.root = p.msgAnc.Root - msg.locator.tstamp = p.msgAnc.Tstamp - msg.locator.coords = coords - msg.seq = p.msgAnc.Seq - //msg.RSeq = p.msgAnc.RSeq - //msg.Degree = p.msgAnc.Deg - p.core.switchTable.handleMessage(&msg, p.port, sigs) - if len(coords) == 0 { - return - } - // Pass a mesage to the dht informing it that this peer (still) exists - l := msg.locator - l.coords = l.coords[:len(l.coords)-1] - dinfo := dhtInfo{ - key: p.box, - coords: l.getCoords(), - } - p.core.dht.peers <- &dinfo - p.core.log.Println("DEBUG: peers<-&dhtInfo", dinfo, p.box, msg) -} - func (p *peer) sendSwitchMsg() { info, sigs := p.core.switchTable.createMessage(p.port) var msg switchMsg @@ -549,34 +348,6 @@ func (p *peer) handleSwitchMsg(packet []byte) { p.core.dht.peers <- &dinfo } -func (p *peer) sendSwitchAnnounce() { - anc := msgAnnounce{} - anc.Root = p.myMsg.locator.root - anc.Tstamp = p.myMsg.locator.tstamp - anc.Seq = p.myMsg.seq - anc.Len = uint64(len(p.myMsg.locator.coords)) - //anc.Deg = p.myMsg.Degree - if p.msgAnc != nil { - anc.Rseq = p.msgAnc.Seq - } - packet := anc.encode() - p.sendLinkPacket(packet) -} - -func (p *peer) getSig(hop uint64) sigInfo { - //p.core.log.Println("DEBUG getSig:", len(p.mySigs), hop) - if hop < uint64(len(p.mySigs)) { - return p.mySigs[hop] - } - bs := getBytesForSig(&p.sig, &p.myMsg.locator) - sig := sigInfo{} - sig.next = p.sig - sig.sig = *sign(&p.core.sigPriv, bs) - p.mySigs = append(p.mySigs, sig) - //p.core.log.Println("DEBUG sig bs:", bs) - return sig -} - func getBytesForSig(next *sigPubKey, loc *switchLocator) []byte { //bs, err := wire_encode_locator(loc) //if err != nil { panic(err) } diff --git a/src/yggdrasil/wire.go b/src/yggdrasil/wire.go index bd298de9..9344d902 100644 --- a/src/yggdrasil/wire.go +++ b/src/yggdrasil/wire.go @@ -13,9 +13,6 @@ const ( wire_ProtocolTraffic // protocol traffic, pub keys for crypto wire_LinkProtocolTraffic // link proto traffic, pub keys for crypto wire_SwitchMsg // inside link protocol traffic header - wire_SwitchAnnounce // inside protocol traffic header - wire_SwitchHopRequest // inside protocol traffic header - wire_SwitchHop // inside protocol traffic header wire_SessionPing // inside protocol traffic header wire_SessionPong // inside protocol traffic header wire_DHTLookupRequest // inside protocol traffic header @@ -173,136 +170,8 @@ func (m *switchMsg) decode(bs []byte) bool { //////////////////////////////////////////////////////////////////////////////// -// Announces that we can send parts of a Message with a particular seq -type msgAnnounce struct { - Root sigPubKey - Tstamp int64 - Seq uint64 - Len uint64 - //Deg uint64 - Rseq uint64 -} - -func (m *msgAnnounce) encode() []byte { - bs := wire_encode_uint64(wire_SwitchAnnounce) - bs = append(bs, m.Root[:]...) - bs = append(bs, wire_encode_uint64(wire_intToUint(m.Tstamp))...) - bs = append(bs, wire_encode_uint64(m.Seq)...) - bs = append(bs, wire_encode_uint64(m.Len)...) - bs = append(bs, wire_encode_uint64(m.Rseq)...) - return bs -} - -func (m *msgAnnounce) decode(bs []byte) bool { - var pType uint64 - var tstamp uint64 - switch { - case !wire_chop_uint64(&pType, &bs): - return false - case pType != wire_SwitchAnnounce: - return false - case !wire_chop_slice(m.Root[:], &bs): - return false - case !wire_chop_uint64(&tstamp, &bs): - return false - case !wire_chop_uint64(&m.Seq, &bs): - return false - case !wire_chop_uint64(&m.Len, &bs): - return false - case !wire_chop_uint64(&m.Rseq, &bs): - return false - } - m.Tstamp = wire_intFromUint(tstamp) - return true -} - -type msgHopReq struct { - Root sigPubKey - Tstamp int64 - Seq uint64 - Hop uint64 -} - -func (m *msgHopReq) encode() []byte { - bs := wire_encode_uint64(wire_SwitchHopRequest) - bs = append(bs, m.Root[:]...) - bs = append(bs, wire_encode_uint64(wire_intToUint(m.Tstamp))...) - bs = append(bs, wire_encode_uint64(m.Seq)...) - bs = append(bs, wire_encode_uint64(m.Hop)...) - return bs -} - -func (m *msgHopReq) decode(bs []byte) bool { - var pType uint64 - var tstamp uint64 - switch { - case !wire_chop_uint64(&pType, &bs): - return false - case pType != wire_SwitchHopRequest: - return false - case !wire_chop_slice(m.Root[:], &bs): - return false - case !wire_chop_uint64(&tstamp, &bs): - return false - case !wire_chop_uint64(&m.Seq, &bs): - return false - case !wire_chop_uint64(&m.Hop, &bs): - return false - } - m.Tstamp = wire_intFromUint(tstamp) - return true -} - -type msgHop struct { - Root sigPubKey - Tstamp int64 - Seq uint64 - Hop uint64 - Port switchPort - Next sigPubKey - Sig sigBytes -} - -func (m *msgHop) encode() []byte { - bs := wire_encode_uint64(wire_SwitchHop) - bs = append(bs, m.Root[:]...) - bs = append(bs, wire_encode_uint64(wire_intToUint(m.Tstamp))...) - bs = append(bs, wire_encode_uint64(m.Seq)...) - bs = append(bs, wire_encode_uint64(m.Hop)...) - bs = append(bs, wire_encode_uint64(uint64(m.Port))...) - bs = append(bs, m.Next[:]...) - bs = append(bs, m.Sig[:]...) - return bs -} - -func (m *msgHop) decode(bs []byte) bool { - var pType uint64 - var tstamp uint64 - switch { - case !wire_chop_uint64(&pType, &bs): - return false - case pType != wire_SwitchHop: - return false - case !wire_chop_slice(m.Root[:], &bs): - return false - case !wire_chop_uint64(&tstamp, &bs): - return false - case !wire_chop_uint64(&m.Seq, &bs): - return false - case !wire_chop_uint64(&m.Hop, &bs): - return false - case !wire_chop_uint64((*uint64)(&m.Port), &bs): - return false - case !wire_chop_slice(m.Next[:], &bs): - return false - case !wire_chop_slice(m.Sig[:], &bs): - return false - } - m.Tstamp = wire_intFromUint(tstamp) - return true -} - // Format used to check signatures only, so no need to also support decoding +// TODO something else for signatures func wire_encode_locator(loc *switchLocator) []byte { coords := wire_encode_coords(loc.getCoords()) var bs []byte From 1e7d34492d3cc923ea250cacd71f6dcf4d3f0fbe Mon Sep 17 00:00:00 2001 From: Arceliar Date: Wed, 6 Jun 2018 22:39:22 -0500 Subject: [PATCH 09/42] fix signature checks and add some TODO reminder comments --- src/yggdrasil/peer.go | 32 +++++++++++++++----------------- 1 file changed, 15 insertions(+), 17 deletions(-) diff --git a/src/yggdrasil/peer.go b/src/yggdrasil/peer.go index 7ddf4bbb..059ec739 100644 --- a/src/yggdrasil/peer.go +++ b/src/yggdrasil/peer.go @@ -100,13 +100,11 @@ type peer struct { core *Core port switchPort // This is used to limit how often we perform expensive operations - // Specifically, processing switch messages, signing, and verifying sigs - // Resets at the start of each tick - throttle uint8 + throttle uint8 // TODO apply this sanely // Called when a peer is removed, to close the underlying connection, or via admin api close func() // To allow the peer to call close if idle for too long - lastAnc time.Time + lastAnc time.Time // TODO? rename and use this // used for protocol traffic (to bypass queues) linkIn (chan []byte) // handlePacket sends, linkLoop recvs linkOut (chan []byte) @@ -149,7 +147,6 @@ func (ps *peers) newPeer(box *boxPubKey, sig *sigPubKey) *peer { } func (ps *peers) removePeer(port switchPort) { - // TODO? store linkIn in the peer struct, close it here? (once) if port == 0 { return } // Can't remove self peer @@ -181,6 +178,7 @@ func (p *peer) linkLoop() { case <-ticker.C: if time.Since(p.lastAnc) > 16*time.Second && p.close != nil { // Seems to have timed out, try to trigger a close + // FIXME this depends on lastAnc or something equivalent being updated p.close() } p.throttle = 0 @@ -215,7 +213,7 @@ func (p *peer) handlePacket(packet []byte) { func (p *peer) handleTraffic(packet []byte, pTypeLen int) { //if p.port != 0 && p.msgAnc == nil { // // Drop traffic until the peer manages to send us at least one anc - // // TODO? equivalent for new switch format? + // // TODO equivalent for new switch format, maybe add some bool flag? // return //} ttl, ttlLen := wire_decode_uint64(packet[pTypeLen:]) @@ -294,14 +292,13 @@ func (p *peer) sendSwitchMsg() { } msg.Hops = append(msg.Hops, hop) } + bs := getBytesForSig(&p.sig, &info.locator) msg.Hops = append(msg.Hops, switchMsgHop{ Port: p.port, Next: p.sig, - Sig: *sign(&p.core.sigPriv, getBytesForSig(&p.sig, &info.locator)), + Sig: *sign(&p.core.sigPriv, bs), }) packet := msg.encode() - var test switchMsg - test.decode(packet) //p.core.log.Println("Encoded msg:", msg, "; bytes:", packet) p.sendLinkPacket(packet) } @@ -319,21 +316,22 @@ func (p *peer) handleSwitchMsg(packet []byte) { var sigs []sigInfo info.locator.root = msg.Root info.locator.tstamp = msg.TStamp - thisHopKey := &msg.Root + prevKey := msg.Root for _, hop := range msg.Hops { + // Build locator and signatures var sig sigInfo sig.next = hop.Next sig.sig = hop.Sig sigs = append(sigs, sig) info.locator.coords = append(info.locator.coords, hop.Port) - // TODO check signatures - bs := getBytesForSig(&hop.Next, &info.locator) - if !p.core.sigs.check(thisHopKey, &hop.Sig, bs) { - //p.throttle++ - //panic("FIXME testing") - //return + // Check signature + bs := getBytesForSig(&sig.next, &info.locator) + if !p.core.sigs.check(&prevKey, &sig.sig, bs) { + p.throttle++ + panic("FIXME testing") + return } - thisHopKey = &hop.Next + prevKey = sig.next } info.from = p.sig info.seq = uint64(time.Now().Unix()) From 5dc0cb5544744e5cc68a05ca96dbe9a07b4d6f01 Mon Sep 17 00:00:00 2001 From: Arceliar Date: Wed, 6 Jun 2018 23:00:17 -0500 Subject: [PATCH 10/42] move wire_encode_locator logic into getBytesForSig, since that's the only place it's used --- src/yggdrasil/peer.go | 8 +++----- src/yggdrasil/wire.go | 11 ----------- 2 files changed, 3 insertions(+), 16 deletions(-) diff --git a/src/yggdrasil/peer.go b/src/yggdrasil/peer.go index 059ec739..47f29f81 100644 --- a/src/yggdrasil/peer.go +++ b/src/yggdrasil/peer.go @@ -347,11 +347,9 @@ func (p *peer) handleSwitchMsg(packet []byte) { } func getBytesForSig(next *sigPubKey, loc *switchLocator) []byte { - //bs, err := wire_encode_locator(loc) - //if err != nil { panic(err) } bs := append([]byte(nil), next[:]...) - bs = append(bs, wire_encode_locator(loc)...) - //bs := wire_encode_locator(loc) - //bs = append(next[:], bs...) + bs = append(bs, loc.root[:]...) + bs = append(bs, wire_encode_uint64(wire_intToUint(loc.tstamp))...) + bs = append(bs, wire_encode_coords(loc.getCoords())...) return bs } diff --git a/src/yggdrasil/wire.go b/src/yggdrasil/wire.go index 9344d902..be2fb4e6 100644 --- a/src/yggdrasil/wire.go +++ b/src/yggdrasil/wire.go @@ -170,17 +170,6 @@ func (m *switchMsg) decode(bs []byte) bool { //////////////////////////////////////////////////////////////////////////////// -// Format used to check signatures only, so no need to also support decoding -// TODO something else for signatures -func wire_encode_locator(loc *switchLocator) []byte { - coords := wire_encode_coords(loc.getCoords()) - var bs []byte - bs = append(bs, loc.root[:]...) - bs = append(bs, wire_encode_uint64(wire_intToUint(loc.tstamp))...) - bs = append(bs, coords...) - return bs -} - func wire_chop_slice(toSlice []byte, fromSlice *[]byte) bool { if len(*fromSlice) < len(toSlice) { return false From 3b783fbf974b1a5fc6a7c62242da105aa5137411 Mon Sep 17 00:00:00 2001 From: Arceliar Date: Wed, 6 Jun 2018 23:10:33 -0500 Subject: [PATCH 11/42] move periodic switch maintenance into the router instead of its own goroutine --- src/yggdrasil/core.go | 5 ----- src/yggdrasil/debug.go | 4 ---- src/yggdrasil/router.go | 1 + src/yggdrasil/switch.go | 15 +-------------- 4 files changed, 2 insertions(+), 23 deletions(-) diff --git a/src/yggdrasil/core.go b/src/yggdrasil/core.go index b94d1544..3b5fcc1a 100644 --- a/src/yggdrasil/core.go +++ b/src/yggdrasil/core.go @@ -103,11 +103,6 @@ func (c *Core) Start(nc *config.NodeConfig, log *log.Logger) error { return err } - if err := c.switchTable.start(); err != nil { - c.log.Println("Failed to start switch table ticker") - return err - } - if err := c.admin.start(); err != nil { c.log.Println("Failed to start admin socket") return err diff --git a/src/yggdrasil/debug.go b/src/yggdrasil/debug.go index 599ee90c..940db794 100644 --- a/src/yggdrasil/debug.go +++ b/src/yggdrasil/debug.go @@ -36,7 +36,6 @@ func (c *Core) Init() { spub, spriv := newSigKeys() c.init(bpub, bpriv, spub, spriv) c.router.start() - c.switchTable.start() } //////////////////////////////////////////////////////////////////////////////// @@ -310,9 +309,6 @@ func (c *Core) DEBUG_init(bpub []byte, panic(err) } - if err := c.switchTable.start(); err != nil { - panic(err) - } } //////////////////////////////////////////////////////////////////////////////// diff --git a/src/yggdrasil/router.go b/src/yggdrasil/router.go index a8797d5f..872a4734 100644 --- a/src/yggdrasil/router.go +++ b/src/yggdrasil/router.go @@ -91,6 +91,7 @@ func (r *router) mainLoop() { case <-ticker.C: { // Any periodic maintenance stuff goes here + r.core.switchTable.doMaintenance() r.core.dht.doMaintenance() util_getBytes() // To slowly drain things } diff --git a/src/yggdrasil/switch.go b/src/yggdrasil/switch.go index 8731e504..03921de2 100644 --- a/src/yggdrasil/switch.go +++ b/src/yggdrasil/switch.go @@ -170,26 +170,13 @@ func (t *switchTable) init(core *Core, key sigPubKey) { t.drop = make(map[sigPubKey]int64) } -func (t *switchTable) start() error { - doTicker := func() { - ticker := time.NewTicker(time.Second) - defer ticker.Stop() - for { - <-ticker.C - t.Tick() - } - } - go doTicker() - return nil -} - func (t *switchTable) getLocator() switchLocator { t.mutex.RLock() defer t.mutex.RUnlock() return t.data.locator.clone() } -func (t *switchTable) Tick() { +func (t *switchTable) doMaintenance() { // Periodic maintenance work to keep things internally consistent t.mutex.Lock() // Write lock defer t.mutex.Unlock() // Release lock when we're done From 85afe187ff79eb3dececbffb5075b664aa193408 Mon Sep 17 00:00:00 2001 From: Arceliar Date: Wed, 6 Jun 2018 23:23:16 -0500 Subject: [PATCH 12/42] remove peer timeout logic from the switch, so switch peer entrires are only removed when the peer struct is removed --- src/yggdrasil/peer.go | 10 ++++++++-- src/yggdrasil/switch.go | 17 +++-------------- src/yggdrasil/tcp.go | 1 - 3 files changed, 11 insertions(+), 17 deletions(-) diff --git a/src/yggdrasil/peer.go b/src/yggdrasil/peer.go index 47f29f81..15371e3e 100644 --- a/src/yggdrasil/peer.go +++ b/src/yggdrasil/peer.go @@ -150,6 +150,9 @@ func (ps *peers) removePeer(port switchPort) { if port == 0 { return } // Can't remove self peer + ps.core.router.doAdmin(func() { + ps.core.switchTable.removePeer(port) + }) ps.mutex.Lock() oldPorts := ps.getPorts() p, isIn := oldPorts[port] @@ -160,8 +163,11 @@ func (ps *peers) removePeer(port switchPort) { delete(newPorts, port) ps.putPorts(newPorts) ps.mutex.Unlock() - if isIn && p.close != nil { - p.close() + if isIn { + if p.close != nil { + p.close() + } + close(p.linkIn) } } diff --git a/src/yggdrasil/switch.go b/src/yggdrasil/switch.go index 03921de2..d765f594 100644 --- a/src/yggdrasil/switch.go +++ b/src/yggdrasil/switch.go @@ -181,7 +181,6 @@ func (t *switchTable) doMaintenance() { t.mutex.Lock() // Write lock defer t.mutex.Unlock() // Release lock when we're done t.cleanRoot() - t.cleanPeers() t.cleanDropped() } @@ -227,19 +226,9 @@ func (t *switchTable) cleanRoot() { } } -func (t *switchTable) cleanPeers() { - now := time.Now() - changed := false - for idx, info := range t.data.peers { - if info.port != switchPort(0) && now.Sub(info.time) > 6*time.Second /*switch_timeout*/ { - //fmt.Println("peer timed out", t.key, info.locator) - delete(t.data.peers, idx) - changed = true - } - } - if changed { - t.updater.Store(&sync.Once{}) - } +func (t *switchTable) removePeer(port switchPort) { + delete(t.data.peers, port) + t.updater.Store(&sync.Once{}) } func (t *switchTable) cleanDropped() { diff --git a/src/yggdrasil/tcp.go b/src/yggdrasil/tcp.go index e21522b8..75be31b2 100644 --- a/src/yggdrasil/tcp.go +++ b/src/yggdrasil/tcp.go @@ -279,7 +279,6 @@ func (iface *tcpInterface) handler(sock net.Conn, incoming bool) { defer func() { // Put all of our cleanup here... p.core.peers.removePeer(p.port) - close(p.linkIn) }() them, _, _ := net.SplitHostPort(sock.RemoteAddr().String()) themNodeID := getNodeID(&info.box) From ecf37cae8a1548ef2ebcb40dd53f0de2b51feeb1 Mon Sep 17 00:00:00 2001 From: Arceliar Date: Thu, 7 Jun 2018 00:16:47 -0500 Subject: [PATCH 13/42] make the switch react to peer coord changes immediately, and send out updates immediately --- src/yggdrasil/peer.go | 57 ++++++++++++++++++++++++++++++----------- src/yggdrasil/switch.go | 4 +++ 2 files changed, 46 insertions(+), 15 deletions(-) diff --git a/src/yggdrasil/peer.go b/src/yggdrasil/peer.go index 15371e3e..0efcbe3f 100644 --- a/src/yggdrasil/peer.go +++ b/src/yggdrasil/peer.go @@ -108,6 +108,8 @@ type peer struct { // used for protocol traffic (to bypass queues) linkIn (chan []byte) // handlePacket sends, linkLoop recvs linkOut (chan []byte) + lastMsg []byte // last switchMsg accepted + doSend (chan struct{}) // tell the linkLoop to send a switchMsg } const peer_Throttle = 1 @@ -127,6 +129,7 @@ func (ps *peers) newPeer(box *boxPubKey, sig *sigPubKey) *peer { shared: *getSharedKey(&ps.core.boxPriv, box), lastAnc: now, firstSeen: now, + doSend: make(chan struct{}, 1), core: ps.core} ps.mutex.Lock() defer ps.mutex.Unlock() @@ -171,8 +174,33 @@ func (ps *peers) removePeer(port switchPort) { } } +func (ps *peers) sendSwitchMsgs() { + ports := ps.getPorts() + for _, p := range ports { + if p.port == 0 { + continue + } + select { + case p.doSend <- struct{}{}: + default: + } + } +} + +func (ps *peers) fixSwitchAfterPeerDisconnect() { + // TODO something better, this is very wasteful + ports := ps.getPorts() + for _, p := range ports { + if p.lastMsg == nil { + continue + } + p.handleSwitchMsg(p.lastMsg) + } +} + func (p *peer) linkLoop() { - ticker := time.NewTicker(time.Second) + go func() { p.doSend <- struct{}{} }() + ticker := time.NewTicker(10 * time.Second) defer ticker.Stop() for { select { @@ -182,16 +210,15 @@ func (p *peer) linkLoop() { } p.handleLinkTraffic(packet) case <-ticker.C: - if time.Since(p.lastAnc) > 16*time.Second && p.close != nil { - // Seems to have timed out, try to trigger a close - // FIXME this depends on lastAnc or something equivalent being updated - p.close() - } p.throttle = 0 - if p.port == 0 { - continue - } // Don't send announces on selfInterface - // TODO change update logic, the new switchMsg works differently, we only need to send if something changes + if p.lastMsg != nil { + // TODO? remove ticker completely + // p.throttle isn't useful anymore (if they send a wrong message, remove peer instead) + // the handleMessage below is just for debugging, but it *shouldn't* be needed now that things react to state changes instantly + // The one case where it's maybe useful is if you get messages faster than the switch throttle, but that should fix itself after the next periodic update or timeout + p.handleSwitchMsg(p.lastMsg) + } + case <-p.doSend: p.sendSwitchMsg() } } @@ -217,11 +244,10 @@ func (p *peer) handlePacket(packet []byte) { } func (p *peer) handleTraffic(packet []byte, pTypeLen int) { - //if p.port != 0 && p.msgAnc == nil { - // // Drop traffic until the peer manages to send us at least one anc - // // TODO equivalent for new switch format, maybe add some bool flag? - // return - //} + if p.port != 0 && p.lastMsg == nil { + // Drop traffic until the peer manages to send us at least one good switchMsg + return + } ttl, ttlLen := wire_decode_uint64(packet[pTypeLen:]) ttlBegin := pTypeLen ttlEnd := pTypeLen + ttlLen @@ -350,6 +376,7 @@ func (p *peer) handleSwitchMsg(packet []byte) { coords: l.getCoords(), } p.core.dht.peers <- &dinfo + p.lastMsg = packet } func getBytesForSig(next *sigPubKey, loc *switchLocator) []byte { diff --git a/src/yggdrasil/switch.go b/src/yggdrasil/switch.go index d765f594..a9a0fcd6 100644 --- a/src/yggdrasil/switch.go +++ b/src/yggdrasil/switch.go @@ -223,12 +223,14 @@ func (t *switchTable) cleanRoot() { } t.data.locator = switchLocator{root: t.key, tstamp: now.Unix()} t.data.sigs = nil + t.core.peers.sendSwitchMsgs() } } func (t *switchTable) removePeer(port switchPort) { delete(t.data.peers, port) t.updater.Store(&sync.Once{}) + t.core.peers.fixSwitchAfterPeerDisconnect() } func (t *switchTable) cleanDropped() { @@ -250,6 +252,7 @@ func (t *switchTable) createMessage(port switchPort) (*switchMessage, []sigInfo) } func (t *switchTable) handleMessage(msg *switchMessage, fromPort switchPort, sigs []sigInfo) { + // TODO directly use a switchMsg instead of switchMessage + sigs t.mutex.Lock() defer t.mutex.Unlock() now := time.Now() @@ -344,6 +347,7 @@ func (t *switchTable) handleMessage(msg *switchMessage, fromPort switchPort, sig t.parent = sender.port t.data.sigs = sigs //t.core.log.Println("Switch update:", msg.Locator.Root, msg.Locator.Tstamp, msg.Locator.Coords) + t.core.peers.sendSwitchMsgs() } if doUpdate { t.updater.Store(&sync.Once{}) From deb755e3e99768de613cb441bd18053378783b30 Mon Sep 17 00:00:00 2001 From: Arceliar Date: Thu, 7 Jun 2018 00:49:06 -0500 Subject: [PATCH 14/42] remove peer.linkIn channel and related logic --- src/yggdrasil/debug.go | 13 +++++++++++-- src/yggdrasil/peer.go | 30 +++++------------------------- src/yggdrasil/tcp.go | 1 - 3 files changed, 16 insertions(+), 28 deletions(-) diff --git a/src/yggdrasil/debug.go b/src/yggdrasil/debug.go index 940db794..f0808826 100644 --- a/src/yggdrasil/debug.go +++ b/src/yggdrasil/debug.go @@ -449,8 +449,17 @@ func (c *Core) DEBUG_addAllowedEncryptionPublicKey(boxStr string) { func DEBUG_simLinkPeers(p, q *peer) { // Sets q.out() to point to p and starts p.linkLoop() - p.linkIn, q.linkIn = make(chan []byte, 32), make(chan []byte, 32) - p.linkOut, q.linkOut = q.linkIn, p.linkIn + p.linkOut, q.linkOut = make(chan []byte, 1), make(chan []byte, 1) + go func() { + for bs := range p.linkOut { + q.handlePacket(bs) + } + }() + go func() { + for bs := range q.linkOut { + p.handlePacket(bs) + } + }() p.out = func(bs []byte) { go q.handlePacket(bs) } diff --git a/src/yggdrasil/peer.go b/src/yggdrasil/peer.go index 0efcbe3f..ce5cd0c2 100644 --- a/src/yggdrasil/peer.go +++ b/src/yggdrasil/peer.go @@ -106,7 +106,6 @@ type peer struct { // To allow the peer to call close if idle for too long lastAnc time.Time // TODO? rename and use this // used for protocol traffic (to bypass queues) - linkIn (chan []byte) // handlePacket sends, linkLoop recvs linkOut (chan []byte) lastMsg []byte // last switchMsg accepted doSend (chan struct{}) // tell the linkLoop to send a switchMsg @@ -170,7 +169,7 @@ func (ps *peers) removePeer(port switchPort) { if p.close != nil { p.close() } - close(p.linkIn) + close(p.doSend) } } @@ -200,27 +199,8 @@ func (ps *peers) fixSwitchAfterPeerDisconnect() { func (p *peer) linkLoop() { go func() { p.doSend <- struct{}{} }() - ticker := time.NewTicker(10 * time.Second) - defer ticker.Stop() - for { - select { - case packet, ok := <-p.linkIn: - if !ok { - return - } - p.handleLinkTraffic(packet) - case <-ticker.C: - p.throttle = 0 - if p.lastMsg != nil { - // TODO? remove ticker completely - // p.throttle isn't useful anymore (if they send a wrong message, remove peer instead) - // the handleMessage below is just for debugging, but it *shouldn't* be needed now that things react to state changes instantly - // The one case where it's maybe useful is if you get messages faster than the switch throttle, but that should fix itself after the next periodic update or timeout - p.handleSwitchMsg(p.lastMsg) - } - case <-p.doSend: - p.sendSwitchMsg() - } + for range p.doSend { + p.sendSwitchMsg() } } @@ -237,8 +217,8 @@ func (p *peer) handlePacket(packet []byte) { case wire_ProtocolTraffic: p.handleTraffic(packet, pTypeLen) case wire_LinkProtocolTraffic: - p.linkIn <- packet - default: /*panic(pType) ;*/ + p.handleLinkTraffic(packet) + default: return } } diff --git a/src/yggdrasil/tcp.go b/src/yggdrasil/tcp.go index 75be31b2..90fb80b0 100644 --- a/src/yggdrasil/tcp.go +++ b/src/yggdrasil/tcp.go @@ -209,7 +209,6 @@ func (iface *tcpInterface) handler(sock net.Conn, incoming bool) { // Note that multiple connections to the same node are allowed // E.g. over different interfaces p := iface.core.peers.newPeer(&info.box, &info.sig) - p.linkIn = make(chan []byte, 1) p.linkOut = make(chan []byte, 1) in := func(bs []byte) { p.handlePacket(bs) From 3dab94be9f4cc500652fb0f9f97914da7be21794 Mon Sep 17 00:00:00 2001 From: Arceliar Date: Thu, 7 Jun 2018 10:58:24 -0500 Subject: [PATCH 15/42] keep dht peers alive --- src/yggdrasil/dht.go | 2 +- src/yggdrasil/peer.go | 33 +++++++++++++++++---------------- src/yggdrasil/router.go | 1 + src/yggdrasil/switch.go | 3 ++- 4 files changed, 21 insertions(+), 18 deletions(-) diff --git a/src/yggdrasil/dht.go b/src/yggdrasil/dht.go index e59017a4..3c9f61c9 100644 --- a/src/yggdrasil/dht.go +++ b/src/yggdrasil/dht.go @@ -81,7 +81,7 @@ type dht struct { func (t *dht) init(c *Core) { t.core = c t.nodeID = *t.core.GetNodeID() - t.peers = make(chan *dhtInfo, 1) + t.peers = make(chan *dhtInfo, 1024) t.reqs = make(map[boxPubKey]map[NodeID]time.Time) } diff --git a/src/yggdrasil/peer.go b/src/yggdrasil/peer.go index ce5cd0c2..00f8a9c1 100644 --- a/src/yggdrasil/peer.go +++ b/src/yggdrasil/peer.go @@ -107,8 +107,8 @@ type peer struct { lastAnc time.Time // TODO? rename and use this // used for protocol traffic (to bypass queues) linkOut (chan []byte) - lastMsg []byte // last switchMsg accepted doSend (chan struct{}) // tell the linkLoop to send a switchMsg + dinfo *dhtInfo // used to keep the DHT working } const peer_Throttle = 1 @@ -186,21 +186,22 @@ func (ps *peers) sendSwitchMsgs() { } } -func (ps *peers) fixSwitchAfterPeerDisconnect() { - // TODO something better, this is very wasteful - ports := ps.getPorts() - for _, p := range ports { - if p.lastMsg == nil { - continue - } - p.handleSwitchMsg(p.lastMsg) - } -} - func (p *peer) linkLoop() { go func() { p.doSend <- struct{}{} }() - for range p.doSend { - p.sendSwitchMsg() + tick := time.NewTicker(time.Second) + defer tick.Stop() + for { + select { + case _, ok := <-p.doSend: + if !ok { + return + } + p.sendSwitchMsg() + case _ = <-tick.C: + if p.dinfo != nil { + p.core.dht.peers <- p.dinfo + } + } } } @@ -224,7 +225,7 @@ func (p *peer) handlePacket(packet []byte) { } func (p *peer) handleTraffic(packet []byte, pTypeLen int) { - if p.port != 0 && p.lastMsg == nil { + if p.port != 0 && p.dinfo == nil { // Drop traffic until the peer manages to send us at least one good switchMsg return } @@ -356,7 +357,7 @@ func (p *peer) handleSwitchMsg(packet []byte) { coords: l.getCoords(), } p.core.dht.peers <- &dinfo - p.lastMsg = packet + p.dinfo = &dinfo } func getBytesForSig(next *sigPubKey, loc *switchLocator) []byte { diff --git a/src/yggdrasil/router.go b/src/yggdrasil/router.go index 872a4734..3246f63e 100644 --- a/src/yggdrasil/router.go +++ b/src/yggdrasil/router.go @@ -93,6 +93,7 @@ func (r *router) mainLoop() { // Any periodic maintenance stuff goes here r.core.switchTable.doMaintenance() r.core.dht.doMaintenance() + //r.core.peers.fixSwitchAfterPeerDisconnect() // FIXME makes sure dht peers get added quickly util_getBytes() // To slowly drain things } case f := <-r.admin: diff --git a/src/yggdrasil/switch.go b/src/yggdrasil/switch.go index a9a0fcd6..fec47af1 100644 --- a/src/yggdrasil/switch.go +++ b/src/yggdrasil/switch.go @@ -230,7 +230,7 @@ func (t *switchTable) cleanRoot() { func (t *switchTable) removePeer(port switchPort) { delete(t.data.peers, port) t.updater.Store(&sync.Once{}) - t.core.peers.fixSwitchAfterPeerDisconnect() + // TODO if parent, find a new peer to use as parent instead } func (t *switchTable) cleanDropped() { @@ -287,6 +287,7 @@ func (t *switchTable) handleMessage(msg *switchMessage, fromPort switchPort, sig doUpdate := false if !equiv(&msg.locator, &oldSender.locator) { doUpdate = true + //sender.firstSeen = now // TODO? uncomment to prevent flapping? } t.data.peers[fromPort] = sender updateRoot := false From 00e4da28c74155c3b8310a0a1d702b1aea974b99 Mon Sep 17 00:00:00 2001 From: Arceliar Date: Thu, 7 Jun 2018 13:56:11 -0500 Subject: [PATCH 16/42] use/store switchMsg in the switch instead of going through the old switchMessage --- src/yggdrasil/peer.go | 58 +++++++++++++++++------------------------ src/yggdrasil/switch.go | 54 ++++++++++++++++++++++++++++++++++++-- src/yggdrasil/wire.go | 12 --------- 3 files changed, 76 insertions(+), 48 deletions(-) diff --git a/src/yggdrasil/peer.go b/src/yggdrasil/peer.go index 00f8a9c1..042702e5 100644 --- a/src/yggdrasil/peer.go +++ b/src/yggdrasil/peer.go @@ -294,18 +294,11 @@ func (p *peer) handleLinkTraffic(bs []byte) { } func (p *peer) sendSwitchMsg() { - info, sigs := p.core.switchTable.createMessage(p.port) - var msg switchMsg - msg.Root, msg.TStamp = info.locator.root, info.locator.tstamp - for idx, sig := range sigs { - hop := switchMsgHop{ - Port: info.locator.coords[idx], - Next: sig.next, - Sig: sig.sig, - } - msg.Hops = append(msg.Hops, hop) + msg := p.core.switchTable.getMsg() + if msg == nil { + return } - bs := getBytesForSig(&p.sig, &info.locator) + bs := getBytesForSig(&p.sig, msg) msg.Hops = append(msg.Hops, switchMsgHop{ Port: p.port, Next: p.sig, @@ -313,6 +306,7 @@ func (p *peer) sendSwitchMsg() { }) packet := msg.encode() //p.core.log.Println("Encoded msg:", msg, "; bytes:", packet) + //fmt.Println("Encoded msg:", msg, "; bytes:", packet) p.sendLinkPacket(packet) } @@ -326,44 +320,40 @@ func (p *peer) handleSwitchMsg(packet []byte) { return } var info switchMessage - var sigs []sigInfo - info.locator.root = msg.Root - info.locator.tstamp = msg.TStamp + var loc switchLocator prevKey := msg.Root - for _, hop := range msg.Hops { - // Build locator and signatures - var sig sigInfo - sig.next = hop.Next - sig.sig = hop.Sig - sigs = append(sigs, sig) - info.locator.coords = append(info.locator.coords, hop.Port) - // Check signature - bs := getBytesForSig(&sig.next, &info.locator) - if !p.core.sigs.check(&prevKey, &sig.sig, bs) { + for idx, hop := range msg.Hops { + // Check signatures and collect coords for dht + sigMsg := msg + sigMsg.Hops = msg.Hops[:idx] + loc.coords = append(loc.coords, hop.Port) + bs := getBytesForSig(&hop.Next, &sigMsg) + if !p.core.sigs.check(&prevKey, &hop.Sig, bs) { p.throttle++ panic("FIXME testing") return } - prevKey = sig.next + prevKey = hop.Next } - info.from = p.sig - info.seq = uint64(time.Now().Unix()) - p.core.switchTable.handleMessage(&info, p.port, sigs) + p.core.switchTable.handleMsg(&msg, &info, p.port) // Pass a mesage to the dht informing it that this peer (still) exists - l := info.locator - l.coords = l.coords[:len(l.coords)-1] + loc.coords = loc.coords[:len(loc.coords)-1] dinfo := dhtInfo{ key: p.box, - coords: l.getCoords(), + coords: loc.getCoords(), } p.core.dht.peers <- &dinfo p.dinfo = &dinfo } -func getBytesForSig(next *sigPubKey, loc *switchLocator) []byte { +func getBytesForSig(next *sigPubKey, msg *switchMsg) []byte { + var loc switchLocator + for _, hop := range msg.Hops { + loc.coords = append(loc.coords, hop.Port) + } bs := append([]byte(nil), next[:]...) - bs = append(bs, loc.root[:]...) - bs = append(bs, wire_encode_uint64(wire_intToUint(loc.tstamp))...) + bs = append(bs, msg.Root[:]...) + bs = append(bs, wire_encode_uint64(wire_intToUint(msg.TStamp))...) bs = append(bs, wire_encode_coords(loc.getCoords())...) return bs } diff --git a/src/yggdrasil/switch.go b/src/yggdrasil/switch.go index fec47af1..09083262 100644 --- a/src/yggdrasil/switch.go +++ b/src/yggdrasil/switch.go @@ -118,6 +118,7 @@ type peerInfo struct { firstSeen time.Time port switchPort // Interface number of this peer seq uint64 // Seq number we last saw this peer advertise + smsg switchMsg // The wire switchMsg used } type switchMessage struct { @@ -144,6 +145,7 @@ type switchData struct { seq uint64 // Sequence number, reported to peers, so they know about changes peers map[switchPort]peerInfo sigs []sigInfo + msg *switchMsg } type switchTable struct { @@ -251,11 +253,58 @@ func (t *switchTable) createMessage(port switchPort) (*switchMessage, []sigInfo) return &msg, t.data.sigs } -func (t *switchTable) handleMessage(msg *switchMessage, fromPort switchPort, sigs []sigInfo) { +type switchMsg struct { + Root sigPubKey + TStamp int64 + Hops []switchMsgHop +} + +type switchMsgHop struct { + Port switchPort + Next sigPubKey + Sig sigBytes +} + +func (t *switchTable) getMsg() *switchMsg { + t.mutex.RLock() + defer t.mutex.RUnlock() + if t.parent == 0 { + return &switchMsg{Root: t.key, TStamp: t.data.locator.tstamp} + } else if parent, isIn := t.data.peers[t.parent]; isIn { + msg := parent.smsg + msg.Hops = append([]switchMsgHop(nil), msg.Hops...) + return &msg + } else { + return nil + } +} + +func (t *switchTable) handleMsg(smsg *switchMsg, xmsg *switchMessage, fromPort switchPort) { // TODO directly use a switchMsg instead of switchMessage + sigs t.mutex.Lock() defer t.mutex.Unlock() now := time.Now() + + //* + var msg switchMessage + var sigs []sigInfo + msg.locator.root = smsg.Root + msg.locator.tstamp = smsg.TStamp + msg.from = smsg.Root + prevKey := msg.from + for _, hop := range smsg.Hops { + // Build locator and signatures + var sig sigInfo + sig.next = hop.Next + sig.sig = hop.Sig + sigs = append(sigs, sig) + msg.locator.coords = append(msg.locator.coords, hop.Port) + msg.from = prevKey + prevKey = hop.Next + } + msg.seq = uint64(now.Unix()) + //*/ + if len(msg.locator.coords) == 0 { return } // Should always have >=1 links @@ -269,7 +318,8 @@ func (t *switchTable) handleMessage(msg *switchMessage, fromPort switchPort, sig time: now, firstSeen: oldSender.firstSeen, port: fromPort, - seq: msg.seq} + seq: msg.seq, + smsg: *smsg} equiv := func(x *switchLocator, y *switchLocator) bool { if x.root != y.root { return false diff --git a/src/yggdrasil/wire.go b/src/yggdrasil/wire.go index be2fb4e6..305e5fca 100644 --- a/src/yggdrasil/wire.go +++ b/src/yggdrasil/wire.go @@ -115,18 +115,6 @@ func wire_decode_coords(packet []byte) ([]byte, int) { //////////////////////////////////////////////////////////////////////////////// -type switchMsg struct { - Root sigPubKey - TStamp int64 - Hops []switchMsgHop -} - -type switchMsgHop struct { - Port switchPort - Next sigPubKey - Sig sigBytes -} - func (m *switchMsg) encode() []byte { bs := wire_encode_uint64(wire_SwitchMsg) bs = append(bs, m.Root[:]...) From f8ba80e7d89b5dabb3eb965d3e84cd482c3e9b5b Mon Sep 17 00:00:00 2001 From: Arceliar Date: Thu, 7 Jun 2018 14:13:31 -0500 Subject: [PATCH 17/42] remove old switchMessage and clean up related code --- src/yggdrasil/peer.go | 3 +- src/yggdrasil/switch.go | 95 ++++++++++++++--------------------------- 2 files changed, 32 insertions(+), 66 deletions(-) diff --git a/src/yggdrasil/peer.go b/src/yggdrasil/peer.go index 042702e5..f9aca08d 100644 --- a/src/yggdrasil/peer.go +++ b/src/yggdrasil/peer.go @@ -319,7 +319,6 @@ func (p *peer) handleSwitchMsg(packet []byte) { panic("FIXME testing") return } - var info switchMessage var loc switchLocator prevKey := msg.Root for idx, hop := range msg.Hops { @@ -335,7 +334,7 @@ func (p *peer) handleSwitchMsg(packet []byte) { } prevKey = hop.Next } - p.core.switchTable.handleMsg(&msg, &info, p.port) + p.core.switchTable.handleMsg(&msg, p.port) // Pass a mesage to the dht informing it that this peer (still) exists loc.coords = loc.coords[:len(loc.coords)-1] dinfo := dhtInfo{ diff --git a/src/yggdrasil/switch.go b/src/yggdrasil/switch.go index 09083262..aa9e3d30 100644 --- a/src/yggdrasil/switch.go +++ b/src/yggdrasil/switch.go @@ -113,18 +113,10 @@ type peerInfo struct { key sigPubKey // ID of this peer locator switchLocator // Should be able to respond with signatures upon request degree uint64 // Self-reported degree - coords []switchPort // Coords of this peer (taken from coords of the sent locator) time time.Time // Time this node was last seen firstSeen time.Time port switchPort // Interface number of this peer - seq uint64 // Seq number we last saw this peer advertise - smsg switchMsg // The wire switchMsg used -} - -type switchMessage struct { - from sigPubKey // key of the sender - locator switchLocator // Locator advertised for the receiver, not the sender's loc! - seq uint64 + msg switchMsg // The wire switchMsg used } type switchPort uint64 @@ -144,7 +136,6 @@ type switchData struct { locator switchLocator seq uint64 // Sequence number, reported to peers, so they know about changes peers map[switchPort]peerInfo - sigs []sigInfo msg *switchMsg } @@ -224,7 +215,6 @@ func (t *switchTable) cleanRoot() { } } t.data.locator = switchLocator{root: t.key, tstamp: now.Unix()} - t.data.sigs = nil t.core.peers.sendSwitchMsgs() } } @@ -244,15 +234,6 @@ func (t *switchTable) cleanDropped() { } } -func (t *switchTable) createMessage(port switchPort) (*switchMessage, []sigInfo) { - t.mutex.RLock() - defer t.mutex.RUnlock() - msg := switchMessage{from: t.key, locator: t.data.locator.clone()} - msg.locator.coords = append(msg.locator.coords, port) - msg.seq = t.data.seq - return &msg, t.data.sigs -} - type switchMsg struct { Root sigPubKey TStamp int64 @@ -271,7 +252,7 @@ func (t *switchTable) getMsg() *switchMsg { if t.parent == 0 { return &switchMsg{Root: t.key, TStamp: t.data.locator.tstamp} } else if parent, isIn := t.data.peers[t.parent]; isIn { - msg := parent.smsg + msg := parent.msg msg.Hops = append([]switchMsgHop(nil), msg.Hops...) return &msg } else { @@ -279,47 +260,34 @@ func (t *switchTable) getMsg() *switchMsg { } } -func (t *switchTable) handleMsg(smsg *switchMsg, xmsg *switchMessage, fromPort switchPort) { +func (t *switchTable) handleMsg(msg *switchMsg, fromPort switchPort) { // TODO directly use a switchMsg instead of switchMessage + sigs t.mutex.Lock() defer t.mutex.Unlock() now := time.Now() - - //* - var msg switchMessage - var sigs []sigInfo - msg.locator.root = smsg.Root - msg.locator.tstamp = smsg.TStamp - msg.from = smsg.Root - prevKey := msg.from - for _, hop := range smsg.Hops { + // Set up the sender peerInfo + var sender peerInfo + sender.locator.root = msg.Root + sender.locator.tstamp = msg.TStamp + prevKey := msg.Root + for _, hop := range msg.Hops { // Build locator and signatures var sig sigInfo sig.next = hop.Next sig.sig = hop.Sig - sigs = append(sigs, sig) - msg.locator.coords = append(msg.locator.coords, hop.Port) - msg.from = prevKey + sender.locator.coords = append(sender.locator.coords, hop.Port) + sender.key = prevKey prevKey = hop.Next } - msg.seq = uint64(now.Unix()) - //*/ - - if len(msg.locator.coords) == 0 { - return - } // Should always have >=1 links + sender.msg = *msg oldSender, isIn := t.data.peers[fromPort] if !isIn { oldSender.firstSeen = now } - sender := peerInfo{key: msg.from, - locator: msg.locator, - coords: msg.locator.coords[:len(msg.locator.coords)-1], - time: now, - firstSeen: oldSender.firstSeen, - port: fromPort, - seq: msg.seq, - smsg: *smsg} + sender.firstSeen = oldSender.firstSeen + sender.port = fromPort + sender.time = now + // Decide what to do equiv := func(x *switchLocator, y *switchLocator) bool { if x.root != y.root { return false @@ -335,7 +303,7 @@ func (t *switchTable) handleMsg(smsg *switchMsg, xmsg *switchMessage, fromPort s return true } doUpdate := false - if !equiv(&msg.locator, &oldSender.locator) { + if !equiv(&sender.locator, &oldSender.locator) { doUpdate = true //sender.firstSeen = now // TODO? uncomment to prevent flapping? } @@ -344,12 +312,12 @@ func (t *switchTable) handleMsg(smsg *switchMsg, xmsg *switchMessage, fromPort s oldParent, isIn := t.data.peers[t.parent] noParent := !isIn noLoop := func() bool { - for idx := 0; idx < len(sigs)-1; idx++ { - if sigs[idx].next == t.core.sigPub { + for idx := 0; idx < len(msg.Hops)-1; idx++ { + if msg.Hops[idx].Next == t.core.sigPub { return false } } - if msg.locator.root == t.core.sigPub { + if sender.locator.root == t.core.sigPub { return false } return true @@ -358,30 +326,30 @@ func (t *switchTable) handleMsg(smsg *switchMsg, xmsg *switchMessage, fromPort s pTime := oldParent.time.Sub(oldParent.firstSeen) + switch_timeout // Really want to compare sLen/sTime and pLen/pTime // Cross multiplied to avoid divide-by-zero - cost := len(msg.locator.coords) * int(pTime.Seconds()) + cost := len(sender.locator.coords) * int(pTime.Seconds()) pCost := len(t.data.locator.coords) * int(sTime.Seconds()) - dropTstamp, isIn := t.drop[msg.locator.root] + dropTstamp, isIn := t.drop[sender.locator.root] // Here be dragons switch { case !noLoop: // do nothing - case isIn && dropTstamp >= msg.locator.tstamp: // do nothing - case firstIsBetter(&msg.locator.root, &t.data.locator.root): + case isIn && dropTstamp >= sender.locator.tstamp: // do nothing + case firstIsBetter(&sender.locator.root, &t.data.locator.root): updateRoot = true - case t.data.locator.root != msg.locator.root: // do nothing - case t.data.locator.tstamp > msg.locator.tstamp: // do nothing + case t.data.locator.root != sender.locator.root: // do nothing + case t.data.locator.tstamp > sender.locator.tstamp: // do nothing case noParent: updateRoot = true case cost < pCost: updateRoot = true case sender.port != t.parent: // do nothing - case !equiv(&msg.locator, &t.data.locator): + case !equiv(&sender.locator, &t.data.locator): updateRoot = true case now.Sub(t.time) < switch_throttle: // do nothing - case msg.locator.tstamp > t.data.locator.tstamp: + case sender.locator.tstamp > t.data.locator.tstamp: updateRoot = true } if updateRoot { - if !equiv(&msg.locator, &t.data.locator) { + if !equiv(&sender.locator, &t.data.locator) { doUpdate = true t.data.seq++ select { @@ -391,12 +359,11 @@ func (t *switchTable) handleMsg(smsg *switchMsg, xmsg *switchMessage, fromPort s //t.core.log.Println("Switch update:", msg.locator.root, msg.locator.tstamp, msg.locator.coords) //fmt.Println("Switch update:", msg.Locator.Root, msg.Locator.Tstamp, msg.Locator.Coords) } - if t.data.locator.tstamp != msg.locator.tstamp { + if t.data.locator.tstamp != sender.locator.tstamp { t.time = now } - t.data.locator = msg.locator + t.data.locator = sender.locator t.parent = sender.port - t.data.sigs = sigs //t.core.log.Println("Switch update:", msg.Locator.Root, msg.Locator.Tstamp, msg.Locator.Coords) t.core.peers.sendSwitchMsgs() } From d46888214780baee629043b2b17e4abe6c707dd9 Mon Sep 17 00:00:00 2001 From: Arceliar Date: Thu, 7 Jun 2018 14:24:02 -0500 Subject: [PATCH 18/42] cleanup --- src/yggdrasil/peer.go | 49 +++++++++-------------------------------- src/yggdrasil/switch.go | 2 +- 2 files changed, 12 insertions(+), 39 deletions(-) diff --git a/src/yggdrasil/peer.go b/src/yggdrasil/peer.go index f9aca08d..1834cfd2 100644 --- a/src/yggdrasil/peer.go +++ b/src/yggdrasil/peer.go @@ -11,17 +11,6 @@ package yggdrasil // It needs to ignore messages with a lower seq // Probably best to start setting seq to a timestamp in that case... -// FIXME (!?) if it takes too long to communicate all the msgHops, then things hit a horizon -// That could happen with a peer over a high-latency link, with many msgHops -// Possible workarounds: -// 1. Pre-emptively send all hops when one is requested, or after any change -// Maybe requires changing how the throttle works and msgHops are saved -// In case some arrive out of order or are dropped -// This is relatively easy to implement, but could be wasteful -// 2. Save your old locator, sigs, etc, so you can respond to older ancs -// And finish requesting an old anc before updating to a new one -// But that may lead to other issues if not done carefully... - import "time" import "sync" import "sync/atomic" @@ -83,36 +72,23 @@ func (ps *peers) putPorts(ports map[switchPort]*peer) { } type peer struct { - // Rolling approximation of bandwidth, in bps, used by switch, updated by packet sends - // use get/update methods only! (atomic accessors as float64) - queueSize int64 + queueSize int64 // used to track local backpressure bytesSent uint64 // To track bandwidth usage for getPeers bytesRecvd uint64 // To track bandwidth usage for getPeers // BUG: sync/atomic, 32 bit platforms need the above to be the first element - firstSeen time.Time // To track uptime for getPeers + core *Core + port switchPort box boxPubKey sig sigPubKey shared boxSharedKey - //in <-chan []byte - //out chan<- []byte - //in func([]byte) - out func([]byte) - core *Core - port switchPort - // This is used to limit how often we perform expensive operations - throttle uint8 // TODO apply this sanely - // Called when a peer is removed, to close the underlying connection, or via admin api - close func() - // To allow the peer to call close if idle for too long - lastAnc time.Time // TODO? rename and use this - // used for protocol traffic (to bypass queues) - linkOut (chan []byte) - doSend (chan struct{}) // tell the linkLoop to send a switchMsg - dinfo *dhtInfo // used to keep the DHT working + firstSeen time.Time // To track uptime for getPeers + linkOut (chan []byte) // used for protocol traffic (to bypass queues) + doSend (chan struct{}) // tell the linkLoop to send a switchMsg + dinfo *dhtInfo // used to keep the DHT working + out func([]byte) // Set up by whatever created the peers struct, used to send packets to other nodes + close func() // Called when a peer is removed, to close the underlying connection, or via admin api } -const peer_Throttle = 1 - func (p *peer) getQueueSize() int64 { return atomic.LoadInt64(&p.queueSize) } @@ -126,7 +102,6 @@ func (ps *peers) newPeer(box *boxPubKey, sig *sigPubKey) *peer { p := peer{box: *box, sig: *sig, shared: *getSharedKey(&ps.core.boxPriv, box), - lastAnc: now, firstSeen: now, doSend: make(chan struct{}, 1), core: ps.core} @@ -315,9 +290,8 @@ func (p *peer) handleSwitchMsg(packet []byte) { msg.decode(packet) //p.core.log.Println("Decoded msg:", msg, "; bytes:", packet) if len(msg.Hops) < 1 { - p.throttle++ panic("FIXME testing") - return + p.core.peers.removePeer(p.port) } var loc switchLocator prevKey := msg.Root @@ -328,9 +302,8 @@ func (p *peer) handleSwitchMsg(packet []byte) { loc.coords = append(loc.coords, hop.Port) bs := getBytesForSig(&hop.Next, &sigMsg) if !p.core.sigs.check(&prevKey, &hop.Sig, bs) { - p.throttle++ panic("FIXME testing") - return + p.core.peers.removePeer(p.port) } prevKey = hop.Next } diff --git a/src/yggdrasil/switch.go b/src/yggdrasil/switch.go index aa9e3d30..67d9d8fd 100644 --- a/src/yggdrasil/switch.go +++ b/src/yggdrasil/switch.go @@ -9,7 +9,7 @@ package yggdrasil // TODO document/comment everything in a lot more detail // TODO? use a pre-computed lookup table (python version had this) -// A little annoying to do with constant changes from bandwidth estimates +// A little annoying to do with constant changes from backpressure import "time" import "sort" From c1f8baf9b5f6ecff858374b74f70a49047ae6bd8 Mon Sep 17 00:00:00 2001 From: Arceliar Date: Thu, 7 Jun 2018 14:39:43 -0500 Subject: [PATCH 19/42] update dht.reset() to possibly play better with coord changes --- src/yggdrasil/dht.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/yggdrasil/dht.go b/src/yggdrasil/dht.go index 3c9f61c9..e77ab90d 100644 --- a/src/yggdrasil/dht.go +++ b/src/yggdrasil/dht.go @@ -520,9 +520,15 @@ func dht_firstCloserThanThird(first *NodeID, func (t *dht) reset() { // This is mostly so bootstrapping will reset to resend coords into the network + t.offset = 0 + t.rumorMill = nil // reset mill for _, b := range t.buckets_hidden { b.peers = b.peers[:0] + for _, info := range b.other { + // Add other nodes to the rumor mill so they'll be pinged soon + // This will hopefully tell them our coords and re-learn theirs quickly if they haven't changed + t.addToMill(info, info.getNodeID()) + } b.other = b.other[:0] } - t.offset = 0 } From 63feed8dc32a499aa316dcf404a33edab3433962 Mon Sep 17 00:00:00 2001 From: Arceliar Date: Thu, 7 Jun 2018 15:04:17 -0500 Subject: [PATCH 20/42] adjust tcp timeout and add shadow queues to track dropped packets --- src/yggdrasil/tcp.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/yggdrasil/tcp.go b/src/yggdrasil/tcp.go index 90fb80b0..6564dfda 100644 --- a/src/yggdrasil/tcp.go +++ b/src/yggdrasil/tcp.go @@ -225,16 +225,20 @@ func (iface *tcpInterface) handler(sock net.Conn, incoming bool) { } go func() { defer buf.Flush() + var shadow uint64 var stack [][]byte put := func(msg []byte) { stack = append(stack, msg) for len(stack) > 32 { util_putBytes(stack[0]) stack = stack[1:] - p.updateQueueSize(-1) + shadow++ } } for { + for ; shadow > 0; shadow-- { + p.updateQueueSize(-1) + } select { case msg := <-p.linkOut: send(msg) @@ -294,7 +298,7 @@ func (iface *tcpInterface) reader(sock net.Conn, in func([]byte)) { bs := make([]byte, 2*tcp_msgSize) frag := bs[:0] for { - timeout := time.Now().Add(6 * time.Second) + timeout := time.Now().Add(2 * time.Minute) sock.SetReadDeadline(timeout) n, err := sock.Read(bs[len(frag):]) if err != nil || n == 0 { From bcfeb2291518cd966dbbf345ea1188a1735cf034 Mon Sep 17 00:00:00 2001 From: Arceliar Date: Thu, 7 Jun 2018 16:49:51 -0500 Subject: [PATCH 21/42] more tcp debugging --- src/yggdrasil/router.go | 2 +- src/yggdrasil/switch.go | 10 +++++++++- src/yggdrasil/tcp.go | 31 +++++++++++++++++++------------ 3 files changed, 29 insertions(+), 14 deletions(-) diff --git a/src/yggdrasil/router.go b/src/yggdrasil/router.go index 3246f63e..ddb48486 100644 --- a/src/yggdrasil/router.go +++ b/src/yggdrasil/router.go @@ -93,7 +93,7 @@ func (r *router) mainLoop() { // Any periodic maintenance stuff goes here r.core.switchTable.doMaintenance() r.core.dht.doMaintenance() - //r.core.peers.fixSwitchAfterPeerDisconnect() // FIXME makes sure dht peers get added quickly + //r.core.peers.sendSwitchMsgs() // FIXME debugging util_getBytes() // To slowly drain things } case f := <-r.admin: diff --git a/src/yggdrasil/switch.go b/src/yggdrasil/switch.go index 67d9d8fd..de79f4a3 100644 --- a/src/yggdrasil/switch.go +++ b/src/yggdrasil/switch.go @@ -223,6 +223,9 @@ func (t *switchTable) removePeer(port switchPort) { delete(t.data.peers, port) t.updater.Store(&sync.Once{}) // TODO if parent, find a new peer to use as parent instead + for _, info := range t.data.peers { + t.unlockedHandleMsg(&info.msg, info.port) + } } func (t *switchTable) cleanDropped() { @@ -261,9 +264,13 @@ func (t *switchTable) getMsg() *switchMsg { } func (t *switchTable) handleMsg(msg *switchMsg, fromPort switchPort) { - // TODO directly use a switchMsg instead of switchMessage + sigs t.mutex.Lock() defer t.mutex.Unlock() + t.unlockedHandleMsg(msg, fromPort) +} + +func (t *switchTable) unlockedHandleMsg(msg *switchMsg, fromPort switchPort) { + // TODO directly use a switchMsg instead of switchMessage + sigs now := time.Now() // Set up the sender peerInfo var sender peerInfo @@ -433,6 +440,7 @@ func (t *switchTable) lookup(dest []byte, ttl uint64) (switchPort, uint64) { } } //t.core.log.Println("DEBUG: sending to", best, "bandwidth", getBandwidth(best)) + //t.core.log.Println("DEBUG: sending to", best, "cost", bestCost) return best, uint64(myDist) } diff --git a/src/yggdrasil/tcp.go b/src/yggdrasil/tcp.go index 6564dfda..e7b682c9 100644 --- a/src/yggdrasil/tcp.go +++ b/src/yggdrasil/tcp.go @@ -15,7 +15,6 @@ import "time" import "errors" import "sync" import "fmt" -import "bufio" import "golang.org/x/net/proxy" const tcp_msgSize = 2048 + 65535 // TODO figure out what makes sense @@ -215,16 +214,7 @@ func (iface *tcpInterface) handler(sock net.Conn, incoming bool) { } out := make(chan []byte, 32) // TODO? what size makes sense defer close(out) - buf := bufio.NewWriterSize(sock, tcp_msgSize) - send := func(msg []byte) { - msgLen := wire_encode_uint64(uint64(len(msg))) - buf.Write(tcp_msg[:]) - buf.Write(msgLen) - buf.Write(msg) - util_putBytes(msg) - } go func() { - defer buf.Flush() var shadow uint64 var stack [][]byte put := func(msg []byte) { @@ -235,11 +225,29 @@ func (iface *tcpInterface) handler(sock net.Conn, incoming bool) { shadow++ } } + send := func(msg []byte) { + msgLen := wire_encode_uint64(uint64(len(msg))) + buf := net.Buffers{tcp_msg[:], msgLen, msg} + buf.WriteTo(sock) + util_putBytes(msg) + } + timerInterval := 4 * time.Second + timer := time.NewTimer(timerInterval) + defer timer.Stop() for { for ; shadow > 0; shadow-- { p.updateQueueSize(-1) } + timer.Stop() select { + case <-timer.C: + default: + } + timer.Reset(timerInterval) + select { + case _ = <-timer.C: + //iface.core.log.Println("DEBUG: sending keep-alive:", sock.RemoteAddr().String()) + send(nil) // TCP keep-alive traffic case msg := <-p.linkOut: send(msg) case msg, ok := <-out: @@ -264,7 +272,6 @@ func (iface *tcpInterface) handler(sock net.Conn, incoming bool) { p.updateQueueSize(-1) } } - buf.Flush() } }() p.out = func(msg []byte) { @@ -298,7 +305,7 @@ func (iface *tcpInterface) reader(sock net.Conn, in func([]byte)) { bs := make([]byte, 2*tcp_msgSize) frag := bs[:0] for { - timeout := time.Now().Add(2 * time.Minute) + timeout := time.Now().Add(6 * time.Second) sock.SetReadDeadline(timeout) n, err := sock.Read(bs[len(frag):]) if err != nil || n == 0 { From ec1c173ca55a134646750404ba81ce560d6824ff Mon Sep 17 00:00:00 2001 From: Arceliar Date: Thu, 7 Jun 2018 16:53:39 -0500 Subject: [PATCH 22/42] it helps to check that messages decoded correctly --- src/yggdrasil/peer.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/yggdrasil/peer.go b/src/yggdrasil/peer.go index 1834cfd2..27be44b3 100644 --- a/src/yggdrasil/peer.go +++ b/src/yggdrasil/peer.go @@ -287,7 +287,9 @@ func (p *peer) sendSwitchMsg() { func (p *peer) handleSwitchMsg(packet []byte) { var msg switchMsg - msg.decode(packet) + if !msg.decode(packet) { + return + } //p.core.log.Println("Decoded msg:", msg, "; bytes:", packet) if len(msg.Hops) < 1 { panic("FIXME testing") From fe12e1509ade0a3a9f78eb4dadb26765d6b87721 Mon Sep 17 00:00:00 2001 From: Arceliar Date: Thu, 7 Jun 2018 17:55:43 -0500 Subject: [PATCH 23/42] add a throttle to nodes in the dht. the throttle is gradually increased each time the node is pinged. it determines the minimum amount of time to wait between using the node in a bootstrapping search --- src/yggdrasil/dht.go | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/src/yggdrasil/dht.go b/src/yggdrasil/dht.go index e77ab90d..95f6bb1e 100644 --- a/src/yggdrasil/dht.go +++ b/src/yggdrasil/dht.go @@ -36,6 +36,7 @@ type dhtInfo struct { send time.Time // When we last sent a message recv time.Time // When we last received a message pings int // Decide when to drop + throttle uint8 // Number of seconds to wait before pinging a node to bootstrap buckets, gradually increases up to 1 minute } func (info *dhtInfo) getNodeID() *NodeID { @@ -116,10 +117,11 @@ func (t *dht) handleRes(res *dhtRes) { return } rinfo := dhtInfo{ - key: res.Key, - coords: res.Coords, - send: time.Now(), // Technically wrong but should be OK... - recv: time.Now(), + key: res.Key, + coords: res.Coords, + send: time.Now(), // Technically wrong but should be OK... + recv: time.Now(), + throttle: 1, } // If they're already in the table, then keep the correct send time bidx, isOK := t.getBucketIndex(rinfo.getNodeID()) @@ -130,11 +132,13 @@ func (t *dht) handleRes(res *dhtRes) { for _, oldinfo := range b.peers { if oldinfo.key == rinfo.key { rinfo.send = oldinfo.send + rinfo.throttle += oldinfo.throttle } } for _, oldinfo := range b.other { if oldinfo.key == rinfo.key { rinfo.send = oldinfo.send + rinfo.throttle += oldinfo.throttle } } // Insert into table @@ -231,6 +235,9 @@ func (t *dht) insert(info *dhtInfo, isPeer bool) { // This speeds up bootstrapping info.recv = info.recv.Add(-time.Hour) } + if isPeer || info.throttle > 60 { + info.throttle = 60 + } // First drop any existing entry from the bucket b.drop(&info.key) // Now add to the *end* of the bucket @@ -460,7 +467,7 @@ func (t *dht) doMaintenance() { } target := t.getTarget(t.offset) for _, info := range t.lookup(target, true) { - if time.Since(info.recv) > time.Minute { + if time.Since(info.recv) > time.Duration(info.throttle)*time.Second { t.addToMill(info, target) t.offset++ break From 84c13fac90df2018e449e05f6f6e514ff676b6af Mon Sep 17 00:00:00 2001 From: Arceliar Date: Thu, 7 Jun 2018 20:18:13 -0500 Subject: [PATCH 24/42] don't use TTL --- misc/sim/treesim.go | 4 ++++ src/yggdrasil/peer.go | 12 ++---------- src/yggdrasil/switch.go | 5 ++++- 3 files changed, 10 insertions(+), 11 deletions(-) diff --git a/misc/sim/treesim.go b/misc/sim/treesim.go index d8859261..91b8e0ef 100644 --- a/misc/sim/treesim.go +++ b/misc/sim/treesim.go @@ -207,6 +207,10 @@ func testPaths(store map[[32]byte]*Node) bool { // This is sufficient to check for routing loops or blackholes //break } + if here == next { + fmt.Println("Drop2:", source.index, here.index, dest.index, oldTTL) + return false + } here = next } } diff --git a/src/yggdrasil/peer.go b/src/yggdrasil/peer.go index 27be44b3..ee25b072 100644 --- a/src/yggdrasil/peer.go +++ b/src/yggdrasil/peer.go @@ -204,15 +204,14 @@ func (p *peer) handleTraffic(packet []byte, pTypeLen int) { // Drop traffic until the peer manages to send us at least one good switchMsg return } - ttl, ttlLen := wire_decode_uint64(packet[pTypeLen:]) - ttlBegin := pTypeLen + _, ttlLen := wire_decode_uint64(packet[pTypeLen:]) ttlEnd := pTypeLen + ttlLen coords, coordLen := wire_decode_coords(packet[ttlEnd:]) coordEnd := ttlEnd + coordLen if coordEnd == len(packet) { return } // No payload - toPort, newTTL := p.core.switchTable.lookup(coords, ttl) + toPort, _ := p.core.switchTable.lookup(coords, 0) if toPort == p.port { return } @@ -220,13 +219,6 @@ func (p *peer) handleTraffic(packet []byte, pTypeLen int) { if to == nil { return } - // This mutates the packet in-place if the length of the TTL changes! - ttlSlice := wire_encode_uint64(newTTL) - newTTLLen := len(ttlSlice) - shift := ttlLen - newTTLLen - copy(packet[shift:], packet[:pTypeLen]) - copy(packet[ttlBegin+shift:], ttlSlice) - packet = packet[shift:] to.sendPacket(packet) } diff --git a/src/yggdrasil/switch.go b/src/yggdrasil/switch.go index de79f4a3..be3027aa 100644 --- a/src/yggdrasil/switch.go +++ b/src/yggdrasil/switch.go @@ -418,6 +418,9 @@ func (t *switchTable) lookup(dest []byte, ttl uint64) (switchPort, uint64) { table := t.table.Load().(lookupTable) myDist := table.self.dist(dest) //getDist(table.self.coords) if !(uint64(myDist) < ttl) { + //return 0, 0 + } + if myDist == 0 { return 0, 0 } // cost is in units of (expected distance) + (expected queue size), where expected distance is used as an approximation of the minimum backpressure gradient needed for packets to flow @@ -441,7 +444,7 @@ func (t *switchTable) lookup(dest []byte, ttl uint64) (switchPort, uint64) { } //t.core.log.Println("DEBUG: sending to", best, "bandwidth", getBandwidth(best)) //t.core.log.Println("DEBUG: sending to", best, "cost", bestCost) - return best, uint64(myDist) + return best, ttl //uint64(myDist) } //////////////////////////////////////////////////////////////////////////////// From bced15b1389a648dc4694f953304103bf5dc8ad9 Mon Sep 17 00:00:00 2001 From: Arceliar Date: Thu, 7 Jun 2018 20:29:22 -0500 Subject: [PATCH 25/42] remove TTL from traffic packets --- misc/sim/treesim.go | 12 +++--------- src/yggdrasil/debug.go | 4 ++-- src/yggdrasil/dht.go | 2 -- src/yggdrasil/peer.go | 9 +++------ src/yggdrasil/session.go | 2 -- src/yggdrasil/switch.go | 14 +++++--------- src/yggdrasil/wire.go | 8 -------- 7 files changed, 13 insertions(+), 38 deletions(-) diff --git a/misc/sim/treesim.go b/misc/sim/treesim.go index 91b8e0ef..83548e81 100644 --- a/misc/sim/treesim.go +++ b/misc/sim/treesim.go @@ -160,17 +160,11 @@ func testPaths(store map[[32]byte]*Node) bool { ttl := ^uint64(0) oldTTL := ttl for here := source; here != dest; { - if ttl == 0 { - fmt.Println("Drop:", source.index, here.index, dest.index, oldTTL) - return false - } temp++ if temp > 4096 { panic("Loop?") } - oldTTL = ttl - nextPort, newTTL := here.core.DEBUG_switchLookup(coords, ttl) - ttl = newTTL + nextPort := here.core.DEBUG_switchLookup(coords) // First check if "here" is accepting packets from the previous node // TODO explain how this works ports := here.core.DEBUG_getPeers().DEBUG_getPorts() @@ -208,7 +202,7 @@ func testPaths(store map[[32]byte]*Node) bool { //break } if here == next { - fmt.Println("Drop2:", source.index, here.index, dest.index, oldTTL) + fmt.Println("Drop:", source.index, here.index, dest.index, oldTTL) return false } here = next @@ -231,7 +225,7 @@ func stressTest(store map[[32]byte]*Node) { start := time.Now() for _, source := range store { for _, coords := range dests { - source.core.DEBUG_switchLookup(coords, ^uint64(0)) + source.core.DEBUG_switchLookup(coords) lookups++ } } diff --git a/src/yggdrasil/debug.go b/src/yggdrasil/debug.go index f0808826..6b8211cb 100644 --- a/src/yggdrasil/debug.go +++ b/src/yggdrasil/debug.go @@ -126,8 +126,8 @@ func (l *switchLocator) DEBUG_getCoords() []byte { return l.getCoords() } -func (c *Core) DEBUG_switchLookup(dest []byte, ttl uint64) (switchPort, uint64) { - return c.switchTable.lookup(dest, ttl) +func (c *Core) DEBUG_switchLookup(dest []byte) switchPort { + return c.switchTable.lookup(dest) } /* diff --git a/src/yggdrasil/dht.go b/src/yggdrasil/dht.go index 95f6bb1e..33933914 100644 --- a/src/yggdrasil/dht.go +++ b/src/yggdrasil/dht.go @@ -326,7 +326,6 @@ func (t *dht) sendReq(req *dhtReq, dest *dhtInfo) { shared := t.core.sessions.getSharedKey(&t.core.boxPriv, &dest.key) payload, nonce := boxSeal(shared, bs, nil) p := wire_protoTrafficPacket{ - TTL: ^uint64(0), Coords: dest.coords, ToKey: dest.key, FromKey: t.core.boxPub, @@ -352,7 +351,6 @@ func (t *dht) sendRes(res *dhtRes, req *dhtReq) { shared := t.core.sessions.getSharedKey(&t.core.boxPriv, &req.Key) payload, nonce := boxSeal(shared, bs, nil) p := wire_protoTrafficPacket{ - TTL: ^uint64(0), Coords: req.Coords, ToKey: req.Key, FromKey: t.core.boxPub, diff --git a/src/yggdrasil/peer.go b/src/yggdrasil/peer.go index ee25b072..497024ca 100644 --- a/src/yggdrasil/peer.go +++ b/src/yggdrasil/peer.go @@ -204,14 +204,11 @@ func (p *peer) handleTraffic(packet []byte, pTypeLen int) { // Drop traffic until the peer manages to send us at least one good switchMsg return } - _, ttlLen := wire_decode_uint64(packet[pTypeLen:]) - ttlEnd := pTypeLen + ttlLen - coords, coordLen := wire_decode_coords(packet[ttlEnd:]) - coordEnd := ttlEnd + coordLen - if coordEnd == len(packet) { + coords, coordLen := wire_decode_coords(packet[pTypeLen:]) + if coordLen >= len(packet) { return } // No payload - toPort, _ := p.core.switchTable.lookup(coords, 0) + toPort := p.core.switchTable.lookup(coords) if toPort == p.port { return } diff --git a/src/yggdrasil/session.go b/src/yggdrasil/session.go index 8bbbc33c..33111424 100644 --- a/src/yggdrasil/session.go +++ b/src/yggdrasil/session.go @@ -255,7 +255,6 @@ func (ss *sessions) sendPingPong(sinfo *sessionInfo, isPong bool) { shared := ss.getSharedKey(&ss.core.boxPriv, &sinfo.theirPermPub) payload, nonce := boxSeal(shared, bs, nil) p := wire_protoTrafficPacket{ - TTL: ^uint64(0), Coords: sinfo.coords, ToKey: sinfo.theirPermPub, FromKey: ss.core.boxPub, @@ -383,7 +382,6 @@ func (sinfo *sessionInfo) doSend(bs []byte) { payload, nonce := boxSeal(&sinfo.sharedSesKey, bs, &sinfo.myNonce) defer util_putBytes(payload) p := wire_trafficPacket{ - TTL: ^uint64(0), Coords: sinfo.coords, Handle: sinfo.theirHandle, Nonce: *nonce, diff --git a/src/yggdrasil/switch.go b/src/yggdrasil/switch.go index be3027aa..4db4c67f 100644 --- a/src/yggdrasil/switch.go +++ b/src/yggdrasil/switch.go @@ -413,22 +413,19 @@ func (t *switchTable) updateTable() { t.table.Store(newTable) } -func (t *switchTable) lookup(dest []byte, ttl uint64) (switchPort, uint64) { +func (t *switchTable) lookup(dest []byte) switchPort { t.updater.Load().(*sync.Once).Do(t.updateTable) table := t.table.Load().(lookupTable) - myDist := table.self.dist(dest) //getDist(table.self.coords) - if !(uint64(myDist) < ttl) { - //return 0, 0 - } + myDist := table.self.dist(dest) if myDist == 0 { - return 0, 0 + return 0 } // cost is in units of (expected distance) + (expected queue size), where expected distance is used as an approximation of the minimum backpressure gradient needed for packets to flow ports := t.core.peers.getPorts() var best switchPort bestCost := int64(^uint64(0) >> 1) for _, info := range table.elems { - dist := info.locator.dist(dest) //getDist(info.locator.coords) + dist := info.locator.dist(dest) if !(dist < myDist) { continue } @@ -442,9 +439,8 @@ func (t *switchTable) lookup(dest []byte, ttl uint64) (switchPort, uint64) { bestCost = cost } } - //t.core.log.Println("DEBUG: sending to", best, "bandwidth", getBandwidth(best)) //t.core.log.Println("DEBUG: sending to", best, "cost", bestCost) - return best, ttl //uint64(myDist) + return best } //////////////////////////////////////////////////////////////////////////////// diff --git a/src/yggdrasil/wire.go b/src/yggdrasil/wire.go index 305e5fca..3b43143b 100644 --- a/src/yggdrasil/wire.go +++ b/src/yggdrasil/wire.go @@ -192,7 +192,6 @@ func wire_chop_uint64(toUInt64 *uint64, fromSlice *[]byte) bool { // Wire traffic packets type wire_trafficPacket struct { - TTL uint64 Coords []byte Handle handle Nonce boxNonce @@ -203,7 +202,6 @@ type wire_trafficPacket struct { func (p *wire_trafficPacket) encode() []byte { bs := util_getBytes() bs = wire_put_uint64(wire_Traffic, bs) - bs = wire_put_uint64(p.TTL, bs) bs = wire_put_coords(p.Coords, bs) bs = append(bs, p.Handle[:]...) bs = append(bs, p.Nonce[:]...) @@ -219,8 +217,6 @@ func (p *wire_trafficPacket) decode(bs []byte) bool { return false case pType != wire_Traffic: return false - case !wire_chop_uint64(&p.TTL, &bs): - return false case !wire_chop_coords(&p.Coords, &bs): return false case !wire_chop_slice(p.Handle[:], &bs): @@ -233,7 +229,6 @@ func (p *wire_trafficPacket) decode(bs []byte) bool { } type wire_protoTrafficPacket struct { - TTL uint64 Coords []byte ToKey boxPubKey FromKey boxPubKey @@ -244,7 +239,6 @@ type wire_protoTrafficPacket struct { func (p *wire_protoTrafficPacket) encode() []byte { coords := wire_encode_coords(p.Coords) bs := wire_encode_uint64(wire_ProtocolTraffic) - bs = append(bs, wire_encode_uint64(p.TTL)...) bs = append(bs, coords...) bs = append(bs, p.ToKey[:]...) bs = append(bs, p.FromKey[:]...) @@ -260,8 +254,6 @@ func (p *wire_protoTrafficPacket) decode(bs []byte) bool { return false case pType != wire_ProtocolTraffic: return false - case !wire_chop_uint64(&p.TTL, &bs): - return false case !wire_chop_coords(&p.Coords, &bs): return false case !wire_chop_slice(p.ToKey[:], &bs): From 6bdc9a7eb634036f9a8883f797aea5fab1e3636d Mon Sep 17 00:00:00 2001 From: Arceliar Date: Thu, 7 Jun 2018 21:06:30 -0500 Subject: [PATCH 26/42] fix the sim, part of it bypasses queues so it's expected to see loops in those cases while things are in the middle of updating --- misc/sim/treesim.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/misc/sim/treesim.go b/misc/sim/treesim.go index 83548e81..0316b8fd 100644 --- a/misc/sim/treesim.go +++ b/misc/sim/treesim.go @@ -162,7 +162,9 @@ func testPaths(store map[[32]byte]*Node) bool { for here := source; here != dest; { temp++ if temp > 4096 { - panic("Loop?") + fmt.Println("Loop?") + time.Sleep(time.Second) + return false } nextPort := here.core.DEBUG_switchLookup(coords) // First check if "here" is accepting packets from the previous node @@ -195,7 +197,7 @@ func testPaths(store map[[32]byte]*Node) bool { source.index, source.core.DEBUG_getLocator(), here.index, here.core.DEBUG_getLocator(), dest.index, dest.core.DEBUG_getLocator()) - here.core.DEBUG_getSwitchTable().DEBUG_dumpTable() + //here.core.DEBUG_getSwitchTable().DEBUG_dumpTable() } if here != source { // This is sufficient to check for routing loops or blackholes From ea1d21f7e55c51dc2b0553b89a8c5fe1dcafed67 Mon Sep 17 00:00:00 2001 From: Arceliar Date: Thu, 7 Jun 2018 21:28:08 -0500 Subject: [PATCH 27/42] don't change dhtInfo.pings when sending a search, to prevent multiple different searches from evicting a node --- src/yggdrasil/search.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/yggdrasil/search.go b/src/yggdrasil/search.go index 4b9cd03a..d9dc76e3 100644 --- a/src/yggdrasil/search.go +++ b/src/yggdrasil/search.go @@ -107,7 +107,10 @@ func (s *searches) doSearchStep(sinfo *searchInfo) { // Send to the next search target var next *dhtInfo next, sinfo.toVisit = sinfo.toVisit[0], sinfo.toVisit[1:] + var oldPings int + oldPings, next.pings = next.pings, 0 s.core.dht.ping(next, &sinfo.dest) + next.pings = oldPings // Don't evict a node for searching with it too much sinfo.visited[*next.getNodeID()] = true } } From 495891d9e82a2ba2ffe4716a7ebd96c5ec7c35b3 Mon Sep 17 00:00:00 2001 From: Arceliar Date: Thu, 7 Jun 2018 22:32:01 -0500 Subject: [PATCH 28/42] remove testing panics --- src/yggdrasil/peer.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/yggdrasil/peer.go b/src/yggdrasil/peer.go index 497024ca..51175ef3 100644 --- a/src/yggdrasil/peer.go +++ b/src/yggdrasil/peer.go @@ -281,7 +281,6 @@ func (p *peer) handleSwitchMsg(packet []byte) { } //p.core.log.Println("Decoded msg:", msg, "; bytes:", packet) if len(msg.Hops) < 1 { - panic("FIXME testing") p.core.peers.removePeer(p.port) } var loc switchLocator @@ -293,7 +292,6 @@ func (p *peer) handleSwitchMsg(packet []byte) { loc.coords = append(loc.coords, hop.Port) bs := getBytesForSig(&hop.Next, &sigMsg) if !p.core.sigs.check(&prevKey, &hop.Sig, bs) { - panic("FIXME testing") p.core.peers.removePeer(p.port) } prevKey = hop.Next From 1dcc60f0543ad492afcb23d1812af78f486a4bf5 Mon Sep 17 00:00:00 2001 From: Arceliar Date: Fri, 8 Jun 2018 17:33:16 -0500 Subject: [PATCH 29/42] check root before accepting that a message is good and unblocking a new peer --- src/yggdrasil/peer.go | 7 +++++++ src/yggdrasil/switch.go | 21 +++++++++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/src/yggdrasil/peer.go b/src/yggdrasil/peer.go index 51175ef3..a67e3f65 100644 --- a/src/yggdrasil/peer.go +++ b/src/yggdrasil/peer.go @@ -297,6 +297,13 @@ func (p *peer) handleSwitchMsg(packet []byte) { prevKey = hop.Next } p.core.switchTable.handleMsg(&msg, p.port) + if !p.core.switchTable.checkRoot(&msg) { + // Bad switch message + // Stop forwarding traffic from it + // Stop refreshing it in the DHT + p.dinfo = nil + return + } // Pass a mesage to the dht informing it that this peer (still) exists loc.coords = loc.coords[:len(loc.coords)-1] dinfo := dhtInfo{ diff --git a/src/yggdrasil/switch.go b/src/yggdrasil/switch.go index 4db4c67f..6c42f456 100644 --- a/src/yggdrasil/switch.go +++ b/src/yggdrasil/switch.go @@ -263,6 +263,27 @@ func (t *switchTable) getMsg() *switchMsg { } } +func (t *switchTable) checkRoot(msg *switchMsg) bool { + // returns false if it's a dropped root, not a better root, or has an older timestamp + // returns true otherwise + // used elsewhere to keep inserting peers into the dht only if root info is OK + t.mutex.RLock() + defer t.mutex.RUnlock() + dropTstamp, isIn := t.drop[msg.Root] + switch { + case isIn && dropTstamp >= msg.TStamp: + return false + case firstIsBetter(&msg.Root, &t.data.locator.root): + return true + case t.data.locator.root != msg.Root: + return false + case t.data.locator.tstamp > msg.TStamp: + return false + default: + return true + } +} + func (t *switchTable) handleMsg(msg *switchMsg, fromPort switchPort) { t.mutex.Lock() defer t.mutex.Unlock() From e5eb6de1f6762909121e203fd8d88bb3a6194a34 Mon Sep 17 00:00:00 2001 From: Arceliar Date: Fri, 8 Jun 2018 18:42:56 -0500 Subject: [PATCH 30/42] add inner crypto to linkProtoTraffic, using ephemeral keys, to prevent replay attacks from spoofing peer connections --- misc/sim/treesim.go | 6 +++-- src/yggdrasil/debug.go | 9 ++++--- src/yggdrasil/peer.go | 54 ++++++++++++++++++++++++++--------------- src/yggdrasil/router.go | 4 +-- src/yggdrasil/tcp.go | 15 ++++++++---- 5 files changed, 57 insertions(+), 31 deletions(-) diff --git a/misc/sim/treesim.go b/misc/sim/treesim.go index 0316b8fd..c5c67e79 100644 --- a/misc/sim/treesim.go +++ b/misc/sim/treesim.go @@ -54,10 +54,12 @@ func linkNodes(m, n *Node) { // Create peers // Buffering reduces packet loss in the sim // This slightly speeds up testing (fewer delays before retrying a ping) + pLinkPub, pLinkPriv := m.core.DEBUG_newBoxKeys() + qLinkPub, qLinkPriv := m.core.DEBUG_newBoxKeys() p := m.core.DEBUG_getPeers().DEBUG_newPeer(n.core.DEBUG_getEncryptionPublicKey(), - n.core.DEBUG_getSigningPublicKey()) + n.core.DEBUG_getSigningPublicKey(), *m.core.DEBUG_getSharedKey(pLinkPriv, qLinkPub)) q := n.core.DEBUG_getPeers().DEBUG_newPeer(m.core.DEBUG_getEncryptionPublicKey(), - m.core.DEBUG_getSigningPublicKey()) + m.core.DEBUG_getSigningPublicKey(), *n.core.DEBUG_getSharedKey(qLinkPriv, pLinkPub)) DEBUG_simLinkPeers(p, q) return } diff --git a/src/yggdrasil/debug.go b/src/yggdrasil/debug.go index 6b8211cb..61b4c2f1 100644 --- a/src/yggdrasil/debug.go +++ b/src/yggdrasil/debug.go @@ -64,11 +64,10 @@ func (c *Core) DEBUG_getPeers() *peers { return &c.peers } -func (ps *peers) DEBUG_newPeer(box boxPubKey, - sig sigPubKey) *peer { +func (ps *peers) DEBUG_newPeer(box boxPubKey, sig sigPubKey, link boxSharedKey) *peer { //in <-chan []byte, //out chan<- []byte) *peer { - return ps.newPeer(&box, &sig) //, in, out) + return ps.newPeer(&box, &sig, &link) //, in, out) } /* @@ -275,6 +274,10 @@ func (c *Core) DEBUG_newBoxKeys() (*boxPubKey, *boxPrivKey) { return newBoxKeys() } +func (c *Core) DEBUG_getSharedKey(myPrivKey *boxPrivKey, othersPubKey *boxPubKey) *boxSharedKey { + return getSharedKey(myPrivKey, othersPubKey) +} + func (c *Core) DEBUG_newSigKeys() (*sigPubKey, *sigPrivKey) { return newSigKeys() } diff --git a/src/yggdrasil/peer.go b/src/yggdrasil/peer.go index a67e3f65..2a322cf7 100644 --- a/src/yggdrasil/peer.go +++ b/src/yggdrasil/peer.go @@ -76,17 +76,18 @@ type peer struct { bytesSent uint64 // To track bandwidth usage for getPeers bytesRecvd uint64 // To track bandwidth usage for getPeers // BUG: sync/atomic, 32 bit platforms need the above to be the first element - core *Core - port switchPort - box boxPubKey - sig sigPubKey - shared boxSharedKey - firstSeen time.Time // To track uptime for getPeers - linkOut (chan []byte) // used for protocol traffic (to bypass queues) - doSend (chan struct{}) // tell the linkLoop to send a switchMsg - dinfo *dhtInfo // used to keep the DHT working - out func([]byte) // Set up by whatever created the peers struct, used to send packets to other nodes - close func() // Called when a peer is removed, to close the underlying connection, or via admin api + core *Core + port switchPort + box boxPubKey + sig sigPubKey + shared boxSharedKey + linkShared boxSharedKey + firstSeen time.Time // To track uptime for getPeers + linkOut (chan []byte) // used for protocol traffic (to bypass queues) + doSend (chan struct{}) // tell the linkLoop to send a switchMsg + dinfo *dhtInfo // used to keep the DHT working + out func([]byte) // Set up by whatever created the peers struct, used to send packets to other nodes + close func() // Called when a peer is removed, to close the underlying connection, or via admin api } func (p *peer) getQueueSize() int64 { @@ -97,14 +98,15 @@ func (p *peer) updateQueueSize(delta int64) { atomic.AddInt64(&p.queueSize, delta) } -func (ps *peers) newPeer(box *boxPubKey, sig *sigPubKey) *peer { +func (ps *peers) newPeer(box *boxPubKey, sig *sigPubKey, linkShared *boxSharedKey) *peer { now := time.Now() p := peer{box: *box, - sig: *sig, - shared: *getSharedKey(&ps.core.boxPriv, box), - firstSeen: now, - doSend: make(chan struct{}, 1), - core: ps.core} + sig: *sig, + shared: *getSharedKey(&ps.core.boxPriv, box), + linkShared: *linkShared, + firstSeen: now, + doSend: make(chan struct{}, 1), + core: ps.core} ps.mutex.Lock() defer ps.mutex.Unlock() oldPorts := ps.getPorts() @@ -228,7 +230,13 @@ func (p *peer) sendPacket(packet []byte) { } func (p *peer) sendLinkPacket(packet []byte) { - bs, nonce := boxSeal(&p.shared, packet, nil) + innerPayload, innerNonce := boxSeal(&p.linkShared, packet, nil) + innerLinkPacket := wire_linkProtoTrafficPacket{ + Nonce: *innerNonce, + Payload: innerPayload, + } + outerPayload := innerLinkPacket.encode() + bs, nonce := boxSeal(&p.shared, outerPayload, nil) linkPacket := wire_linkProtoTrafficPacket{ Nonce: *nonce, Payload: bs, @@ -242,7 +250,15 @@ func (p *peer) handleLinkTraffic(bs []byte) { if !packet.decode(bs) { return } - payload, isOK := boxOpen(&p.shared, packet.Payload, &packet.Nonce) + outerPayload, isOK := boxOpen(&p.shared, packet.Payload, &packet.Nonce) + if !isOK { + return + } + innerPacket := wire_linkProtoTrafficPacket{} + if !innerPacket.decode(outerPayload) { + return + } + payload, isOK := boxOpen(&p.linkShared, innerPacket.Payload, &innerPacket.Nonce) if !isOK { return } diff --git a/src/yggdrasil/router.go b/src/yggdrasil/router.go index ddb48486..9e48668e 100644 --- a/src/yggdrasil/router.go +++ b/src/yggdrasil/router.go @@ -43,8 +43,8 @@ type router struct { func (r *router) init(core *Core) { r.core = core r.addr = *address_addrForNodeID(&r.core.dht.nodeID) - in := make(chan []byte, 32) // TODO something better than this... - p := r.core.peers.newPeer(&r.core.boxPub, &r.core.sigPub) //, out, in) + in := make(chan []byte, 32) // TODO something better than this... + p := r.core.peers.newPeer(&r.core.boxPub, &r.core.sigPub, &boxSharedKey{}) p.out = func(packet []byte) { // This is to make very sure it never blocks select { diff --git a/src/yggdrasil/tcp.go b/src/yggdrasil/tcp.go index e7b682c9..c5f7927b 100644 --- a/src/yggdrasil/tcp.go +++ b/src/yggdrasil/tcp.go @@ -141,10 +141,12 @@ func (iface *tcpInterface) call(saddr string) { func (iface *tcpInterface) handler(sock net.Conn, incoming bool) { defer sock.Close() // Get our keys + myLinkPub, myLinkPriv := newBoxKeys() // ephemeral link keys keys := []byte{} keys = append(keys, tcp_key[:]...) keys = append(keys, iface.core.boxPub[:]...) keys = append(keys, iface.core.sigPub[:]...) + keys = append(keys, myLinkPub[:]...) _, err := sock.Write(keys) if err != nil { return @@ -158,8 +160,9 @@ func (iface *tcpInterface) handler(sock net.Conn, incoming bool) { if n < len(keys) { /*panic("Partial key packet?") ;*/ return } - info := tcpInfo{} - if !tcp_chop_keys(&info.box, &info.sig, &keys) { /*panic("Invalid key packet?") ;*/ + info := tcpInfo{} // used as a map key, so don't include ephemeral link eys + var theirLinkPub boxPubKey + if !tcp_chop_keys(&info.box, &info.sig, &theirLinkPub, &keys) { /*panic("Invalid key packet?") ;*/ return } // Quit the parent call if this is a connection to ourself @@ -207,7 +210,7 @@ func (iface *tcpInterface) handler(sock net.Conn, incoming bool) { }() // Note that multiple connections to the same node are allowed // E.g. over different interfaces - p := iface.core.peers.newPeer(&info.box, &info.sig) + p := iface.core.peers.newPeer(&info.box, &info.sig, getSharedKey(myLinkPriv, &theirLinkPub)) p.linkOut = make(chan []byte, 1) in := func(bs []byte) { p.handlePacket(bs) @@ -336,10 +339,10 @@ func (iface *tcpInterface) reader(sock net.Conn, in func([]byte)) { var tcp_key = [...]byte{'k', 'e', 'y', 's'} var tcp_msg = [...]byte{0xde, 0xad, 0xb1, 0x75} // "dead bits" -func tcp_chop_keys(box *boxPubKey, sig *sigPubKey, bs *[]byte) bool { +func tcp_chop_keys(box *boxPubKey, sig *sigPubKey, link *boxPubKey, bs *[]byte) bool { // This one is pretty simple: we know how long the message should be // So don't call this with a message that's too short - if len(*bs) < len(tcp_key)+len(*box)+len(*sig) { + if len(*bs) < len(tcp_key)+2*len(*box)+len(*sig) { return false } for idx := range tcp_key { @@ -352,6 +355,8 @@ func tcp_chop_keys(box *boxPubKey, sig *sigPubKey, bs *[]byte) bool { (*bs) = (*bs)[len(box):] copy(sig[:], *bs) (*bs) = (*bs)[len(sig):] + copy(link[:], *bs) + (*bs) = (*bs)[len(sig):] return true } From f5c850f0982eee4881def95578d0b2b052d2c2ad Mon Sep 17 00:00:00 2001 From: Arceliar Date: Sat, 9 Jun 2018 16:36:13 -0500 Subject: [PATCH 31/42] better way to do wire signed ints (no negative zero, remove conditionals) --- src/yggdrasil/wire.go | 19 +++++-------------- 1 file changed, 5 insertions(+), 14 deletions(-) diff --git a/src/yggdrasil/wire.go b/src/yggdrasil/wire.go index 3b43143b..23e9ab3d 100644 --- a/src/yggdrasil/wire.go +++ b/src/yggdrasil/wire.go @@ -66,23 +66,14 @@ func wire_decode_uint64(bs []byte) (uint64, int) { } func wire_intToUint(i int64) uint64 { - var u uint64 - if i < 0 { - u = uint64(-i) << 1 - u |= 0x01 // sign bit - } else { - u = uint64(i) << 1 - } - return u + // Non-negative integers mapped to even integers: 0 -> 0, 1 -> 2, etc. + // Negative integres mapped to odd integes: -1 -> 1, -2 -> 3, etc. + // This means the least significant bit is a sign bit. + return ((uint64(-(i+1))<<1)|0x01)*(uint64(i)>>63) + (uint64(i)<<1)*(^uint64(i)>>63) } func wire_intFromUint(u uint64) int64 { - var i int64 - i = int64(u >> 1) - if u&0x01 != 0 { - i *= -1 - } - return i + return int64(u&0x01)*(-int64(u>>1)-1) + int64(^u&0x01)*int64(u>>1) } //////////////////////////////////////////////////////////////////////////////// From 87330995167e7f79b28d9110f6921d6cb003d8b1 Mon Sep 17 00:00:00 2001 From: Arceliar Date: Sat, 9 Jun 2018 17:46:19 -0500 Subject: [PATCH 32/42] add version metadata to key exchange at the start of connections --- src/yggdrasil/tcp.go | 42 ++++++++++++++++++++++++++++++------------ 1 file changed, 30 insertions(+), 12 deletions(-) diff --git a/src/yggdrasil/tcp.go b/src/yggdrasil/tcp.go index c5f7927b..459b51b1 100644 --- a/src/yggdrasil/tcp.go +++ b/src/yggdrasil/tcp.go @@ -10,6 +10,10 @@ package yggdrasil // Could be used to DoS (connect, give someone else's keys, spew garbage) // I guess the "peer" part should watch for link packets, disconnect? +// TCP connections start with a metadata exchange. +// It involves exchanging version numbers and crypto keys +// See version.go for version metadata format + import "net" import "time" import "errors" @@ -142,29 +146,43 @@ func (iface *tcpInterface) handler(sock net.Conn, incoming bool) { defer sock.Close() // Get our keys myLinkPub, myLinkPriv := newBoxKeys() // ephemeral link keys - keys := []byte{} - keys = append(keys, tcp_key[:]...) - keys = append(keys, iface.core.boxPub[:]...) - keys = append(keys, iface.core.sigPub[:]...) - keys = append(keys, myLinkPub[:]...) - _, err := sock.Write(keys) + meta := version_getBaseMetadata() + meta.box = iface.core.boxPub + meta.sig = iface.core.sigPub + meta.link = *myLinkPub + metaBytes := meta.encode() + _, err := sock.Write(metaBytes) if err != nil { return } timeout := time.Now().Add(6 * time.Second) sock.SetReadDeadline(timeout) - n, err := sock.Read(keys) + n, err := sock.Read(metaBytes) if err != nil { return } - if n < len(keys) { /*panic("Partial key packet?") ;*/ + if n != version_getMetaLength() { return } - info := tcpInfo{} // used as a map key, so don't include ephemeral link eys - var theirLinkPub boxPubKey - if !tcp_chop_keys(&info.box, &info.sig, &theirLinkPub, &keys) { /*panic("Invalid key packet?") ;*/ + meta = version_metadata{} // Reset to zero value + if !meta.decode(metaBytes) { return } + if !meta.check() { + base := version_getBaseMetadata() + if meta.meta == base.meta { + if meta.ver > base.ver { + iface.core.log.Println("Failed to connect to node:", sock.RemoteAddr().String(), "version:", meta.ver) + } else if meta.ver == base.ver && meta.minorVer > base.minorVer { + iface.core.log.Println("Failed to connect to node:", sock.RemoteAddr().String(), "version:", fmt.Sprintf("%d.%d", meta.ver, meta.minorVer)) + } + } + return + } + info := tcpInfo{ // used as a map key, so don't include ephemeral link key + box: meta.box, + sig: meta.sig, + } // Quit the parent call if this is a connection to ourself equiv := func(k1, k2 []byte) bool { for idx := range k1 { @@ -210,7 +228,7 @@ func (iface *tcpInterface) handler(sock net.Conn, incoming bool) { }() // Note that multiple connections to the same node are allowed // E.g. over different interfaces - p := iface.core.peers.newPeer(&info.box, &info.sig, getSharedKey(myLinkPriv, &theirLinkPub)) + p := iface.core.peers.newPeer(&info.box, &info.sig, getSharedKey(myLinkPriv, &meta.link)) p.linkOut = make(chan []byte, 1) in := func(bs []byte) { p.handlePacket(bs) From 076350f963a6abb854b804ac7b8e8bdd48db311b Mon Sep 17 00:00:00 2001 From: Arceliar Date: Sat, 9 Jun 2018 17:49:02 -0500 Subject: [PATCH 33/42] remove old tcp key exchange code --- src/yggdrasil/tcp.go | 22 ---------------------- 1 file changed, 22 deletions(-) diff --git a/src/yggdrasil/tcp.go b/src/yggdrasil/tcp.go index 459b51b1..98918676 100644 --- a/src/yggdrasil/tcp.go +++ b/src/yggdrasil/tcp.go @@ -354,30 +354,8 @@ func (iface *tcpInterface) reader(sock net.Conn, in func([]byte)) { //////////////////////////////////////////////////////////////////////////////// // Magic bytes to check -var tcp_key = [...]byte{'k', 'e', 'y', 's'} var tcp_msg = [...]byte{0xde, 0xad, 0xb1, 0x75} // "dead bits" -func tcp_chop_keys(box *boxPubKey, sig *sigPubKey, link *boxPubKey, bs *[]byte) bool { - // This one is pretty simple: we know how long the message should be - // So don't call this with a message that's too short - if len(*bs) < len(tcp_key)+2*len(*box)+len(*sig) { - return false - } - for idx := range tcp_key { - if (*bs)[idx] != tcp_key[idx] { - return false - } - } - (*bs) = (*bs)[len(tcp_key):] - copy(box[:], *bs) - (*bs) = (*bs)[len(box):] - copy(sig[:], *bs) - (*bs) = (*bs)[len(sig):] - copy(link[:], *bs) - (*bs) = (*bs)[len(sig):] - return true -} - func tcp_chop_msg(bs *[]byte) ([]byte, bool, error) { // Returns msg, ok, err if len(*bs) < len(tcp_msg) { From 72cca4ea4330eb17673e6cc02658e80d470e8b4a Mon Sep 17 00:00:00 2001 From: Arceliar Date: Sat, 9 Jun 2018 18:38:30 -0500 Subject: [PATCH 34/42] version check/warning adjustments --- src/yggdrasil/tcp.go | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/src/yggdrasil/tcp.go b/src/yggdrasil/tcp.go index 98918676..e325fcfe 100644 --- a/src/yggdrasil/tcp.go +++ b/src/yggdrasil/tcp.go @@ -157,18 +157,14 @@ func (iface *tcpInterface) handler(sock net.Conn, incoming bool) { } timeout := time.Now().Add(6 * time.Second) sock.SetReadDeadline(timeout) - n, err := sock.Read(metaBytes) + _, err = sock.Read(metaBytes) if err != nil { return } - if n != version_getMetaLength() { - return - } meta = version_metadata{} // Reset to zero value - if !meta.decode(metaBytes) { - return - } - if !meta.check() { + if !meta.decode(metaBytes) || !meta.check() { + // Failed to decode and check the metadata + // If it's a version mismatch issue, then print an error message base := version_getBaseMetadata() if meta.meta == base.meta { if meta.ver > base.ver { @@ -177,6 +173,7 @@ func (iface *tcpInterface) handler(sock net.Conn, incoming bool) { iface.core.log.Println("Failed to connect to node:", sock.RemoteAddr().String(), "version:", fmt.Sprintf("%d.%d", meta.ver, meta.minorVer)) } } + // TODO? Block forever to prevent future connection attempts? suppress future messages about the same node? return } info := tcpInfo{ // used as a map key, so don't include ephemeral link key From 038a51fd134db28a53064ffda87e8fc09415d3f8 Mon Sep 17 00:00:00 2001 From: Arceliar Date: Sat, 9 Jun 2018 18:44:59 -0500 Subject: [PATCH 35/42] it helps to add new files --- src/yggdrasil/version.go | 70 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) create mode 100644 src/yggdrasil/version.go diff --git a/src/yggdrasil/version.go b/src/yggdrasil/version.go new file mode 100644 index 00000000..40a5b965 --- /dev/null +++ b/src/yggdrasil/version.go @@ -0,0 +1,70 @@ +package yggdrasil + +// This file contains the version metadata struct +// Used in the inital connection setup and key exchange +// Some of this could arguably go in wire.go instead + +type version_metadata struct { + meta [4]byte + ver uint64 // 1 byte in this version + // Everything after this point potentially depends on the version number, and is subject to change in future versions + minorVer uint64 // 1 byte in this version + box boxPubKey + sig sigPubKey + link boxPubKey +} + +func version_getBaseMetadata() version_metadata { + return version_metadata{ + meta: [4]byte{'m', 'e', 't', 'a'}, + ver: 0, + minorVer: 2, + } +} + +func version_getMetaLength() (mlen int) { + mlen += 4 // meta + mlen += 1 // ver + mlen += 1 // minorVer + mlen += boxPubKeyLen // box + mlen += sigPubKeyLen // sig + mlen += boxPubKeyLen // link + return +} + +func (m *version_metadata) encode() []byte { + bs := make([]byte, 0, version_getMetaLength()) + bs = append(bs, m.meta[:]...) + bs = append(bs, wire_encode_uint64(m.ver)...) + bs = append(bs, wire_encode_uint64(m.minorVer)...) + bs = append(bs, m.box[:]...) + bs = append(bs, m.sig[:]...) + bs = append(bs, m.link[:]...) + if len(bs) != version_getMetaLength() { + panic("Inconsistent metadata length") + } + return bs +} + +func (m *version_metadata) decode(bs []byte) bool { + switch { + case !wire_chop_slice(m.meta[:], &bs): + return false + case !wire_chop_uint64(&m.ver, &bs): + return false + case !wire_chop_uint64(&m.minorVer, &bs): + return false + case !wire_chop_slice(m.box[:], &bs): + return false + case !wire_chop_slice(m.sig[:], &bs): + return false + case !wire_chop_slice(m.link[:], &bs): + return false + } + return true +} + +func (m *version_metadata) check() bool { + base := version_getBaseMetadata() + return base.meta == m.meta && base.ver == m.ver && base.minorVer == m.minorVer +} From 56802d569ebae6f19e326a7a7c96137a8d827521 Mon Sep 17 00:00:00 2001 From: Arceliar Date: Sun, 10 Jun 2018 18:03:28 -0500 Subject: [PATCH 36/42] minor documentation updates, code comments, and a couple of bugfixes that I noticed when going through the code to comment it --- README.md | 11 +++--- VERSION | 2 +- src/yggdrasil/address.go | 32 +++++++++++++++-- src/yggdrasil/admin.go | 27 +++++++++++++- src/yggdrasil/dht.go | 72 ++++++++++++++++++++++++++++++++++--- src/yggdrasil/peer.go | 50 +++++++++++++++++++------- src/yggdrasil/router.go | 30 ++++++++++++++-- src/yggdrasil/search.go | 25 +++++++++++++ src/yggdrasil/session.go | 55 ++++++++++++++++++++++++---- src/yggdrasil/signature.go | 27 ++++++++++---- src/yggdrasil/switch.go | 73 +++++++++++++++++++++++++++----------- src/yggdrasil/tcp.go | 39 +++++++++++++++++--- src/yggdrasil/util.go | 8 +++++ src/yggdrasil/version.go | 12 +++++-- src/yggdrasil/wire.go | 58 +++++++++++++++++++++--------- 15 files changed, 436 insertions(+), 85 deletions(-) diff --git a/README.md b/README.md index e4f517a4..e44024eb 100644 --- a/README.md +++ b/README.md @@ -11,12 +11,12 @@ It's named Yggdrasil after the world tree from Norse mythology, because that see For a longer, rambling version of this readme with more information, see: [doc](doc/README.md). A very early incomplete draft of a [whitepaper](doc/Whitepaper.md) describing the protocol is also available. -This is a toy / proof-of-principle, so it's not even alpha quality software--any nontrivial update is likely to break backwards compatibility with no possibility for a clean upgrade path. +This is a toy / proof-of-principle, and considered alpha quality by the developers. It's not expected to be feature complete, and future updates may not be backwards compatible, though it should warn you if it sees a connection attempt with a node running a newer version. You're encouraged to play with it, but it is strongly advised not to use it for anything mission critical. ## Building -1. Install Go (tested on 1.9+, [godeb](https://github.com/niemeyer/godeb) is recommended). +1. Install Go (tested on 1.9+, [godeb](https://github.com/niemeyer/godeb) is recommended for debian-based linux distributions). 2. Clone this repository. 2. `./build` @@ -44,10 +44,9 @@ In practice, you probably want to run this instead: This keeps a persistent set of keys (and by extension, IP address) and gives you the option of editing the configuration file. If you want to use it as an overlay network on top of e.g. the internet, then you can do so by adding the remote devices domain/address and port (as a string, e.g. `"1.2.3.4:5678"`) to the list of `Peers` in the configuration file. -You can control whether or not it peers over TCP or UDP by adding `tcp://` or `udp://` to the start of the string, i.e. `"udp://1.2.3.4:5678"`. -It is also possible to route outgoing TCP connections through a socks proxy using the syntax: `"socks://socksHost:socksPort/destHost:destPort"`. -It is currently configured to accept incoming TCP and UDP connections. -In the interest of testing the TCP machinery, it's set to create TCP connections for auto-peering (over link-local IPv6), and to use TCP by default if no transport is specified for a manually configured peer. +By default, it peers over TCP (which can be forced with `"tcp://1.2.3.4:5678"` syntax), but it's also possible to connect over a socks proxy (`"socks://socksHost:socksPort/1.2.3.4:5678"`). +The socks proxy approach is useful for e.g. [peering over tor hidden services](https://github.com/yggdrasil-network/public-peers/blob/master/other/tor.md). +UDP support was removed as part of v0.2, and may be replaced by a better implementation at a later date. ### Platforms diff --git a/VERSION b/VERSION index 49d59571..3b04cfb6 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.1 +0.2 diff --git a/src/yggdrasil/address.go b/src/yggdrasil/address.go index b5ac73d2..e0baf9ca 100644 --- a/src/yggdrasil/address.go +++ b/src/yggdrasil/address.go @@ -1,10 +1,17 @@ package yggdrasil -type address [16]byte // IPv6 address within the network -type subnet [8]byte // It's a /64 +// address represents an IPv6 address in the yggdrasil address range. +type address [16]byte -var address_prefix = [...]byte{0xfd} // For node addresses + local subnets +// subnet represents an IPv6 /64 subnet in the yggdrasil subnet range. +type subnet [8]byte +// address_prefix is the prefix used for all addresses and subnets in the network. +// The current implementation requires this to be a multiple of 8 bits. +// Nodes that configure this differently will be unable to communicate with eachother, though routing and the DHT machinery *should* still work. +var address_prefix = [...]byte{0xfd} + +// isValid returns true if an address falls within the range used by nodes in the network. func (a *address) isValid() bool { for idx := range address_prefix { if (*a)[idx] != address_prefix[idx] { @@ -14,6 +21,7 @@ func (a *address) isValid() bool { return (*a)[len(address_prefix)]&0x80 == 0 } +// isValid returns true if a prefix falls within the range usable by the network. func (s *subnet) isValid() bool { for idx := range address_prefix { if (*s)[idx] != address_prefix[idx] { @@ -23,6 +31,11 @@ func (s *subnet) isValid() bool { return (*s)[len(address_prefix)]&0x80 != 0 } +// address_addrForNodeID takes a *NodeID as an argument and returns an *address. +// This address begins with the address prefix. +// The next bit is 0 for an address, and 1 for a subnet. +// The following 7 bits are set to the number of leading 1 bits in the NodeID. +// The NodeID, excluding the leading 1 bits and the first leading 1 bit, is truncated to the appropriate length and makes up the remainder of the address. func address_addrForNodeID(nid *NodeID) *address { // 128 bit address // Begins with prefix @@ -59,6 +72,11 @@ func address_addrForNodeID(nid *NodeID) *address { return &addr } +// address_subnetForNodeID takes a *NodeID as an argument and returns a *subnet. +// This subnet begins with the address prefix. +// The next bit is 0 for an address, and 1 for a subnet. +// The following 7 bits are set to the number of leading 1 bits in the NodeID. +// The NodeID, excluding the leading 1 bits and the first leading 1 bit, is truncated to the appropriate length and makes up the remainder of the subnet. func address_subnetForNodeID(nid *NodeID) *subnet { // Exactly as the address version, with two exceptions: // 1) The first bit after the fixed prefix is a 1 instead of a 0 @@ -70,6 +88,10 @@ func address_subnetForNodeID(nid *NodeID) *subnet { return &snet } +// getNodeIDandMask returns two *NodeID. +// The first is a NodeID with all the bits known from the address set to their correct values. +// The second is a bitmask with 1 bit set for each bit that was known from the address. +// This is used to look up NodeIDs in the DHT and tell if they match an address. func (a *address) getNodeIDandMask() (*NodeID, *NodeID) { // Mask is a bitmask to mark the bits visible from the address // This means truncated leading 1s, first leading 0, and visible part of addr @@ -95,6 +117,10 @@ func (a *address) getNodeIDandMask() (*NodeID, *NodeID) { return &nid, &mask } +// getNodeIDandMask returns two *NodeID. +// The first is a NodeID with all the bits known from the address set to their correct values. +// The second is a bitmask with 1 bit set for each bit that was known from the subnet. +// This is used to look up NodeIDs in the DHT and tell if they match a subnet. func (s *subnet) getNodeIDandMask() (*NodeID, *NodeID) { // As with the address version, but visible parts of the subnet prefix instead var nid NodeID diff --git a/src/yggdrasil/admin.go b/src/yggdrasil/admin.go index 01ed93a5..b32e20de 100644 --- a/src/yggdrasil/admin.go +++ b/src/yggdrasil/admin.go @@ -29,17 +29,21 @@ type admin_handlerInfo struct { handler func(admin_info) (admin_info, error) // First is input map, second is output } -// Maps things like "IP", "port", "bucket", or "coords" onto strings +// admin_pair maps things like "IP", "port", "bucket", or "coords" onto values. type admin_pair struct { key string val interface{} } + +// admin_nodeInfo represents the information we know about a node for an admin response. type admin_nodeInfo []admin_pair +// addHandler is called for each admin function to add the handler and help documentation to the API. func (a *admin) addHandler(name string, args []string, handler func(admin_info) (admin_info, error)) { a.handlers = append(a.handlers, admin_handlerInfo{name, args, handler}) } +// init runs the initial admin setup. func (a *admin) init(c *Core, listenaddr string) { a.core = c a.listenaddr = listenaddr @@ -215,11 +219,13 @@ func (a *admin) init(c *Core, listenaddr string) { }) } +// start runs the admin API socket to listen for / respond to admin API calls. func (a *admin) start() error { go a.listen() return nil } +// listen is run by start and manages API connections. func (a *admin) listen() { l, err := net.Listen("tcp", a.listenaddr) if err != nil { @@ -236,6 +242,7 @@ func (a *admin) listen() { } } +// handleRequest calls the request handler for each request sent to the admin API. func (a *admin) handleRequest(conn net.Conn) { decoder := json.NewDecoder(conn) encoder := json.NewEncoder(conn) @@ -328,6 +335,7 @@ func (a *admin) handleRequest(conn net.Conn) { } } +// asMap converts an admin_nodeInfo into a map of key/value pairs. func (n *admin_nodeInfo) asMap() map[string]interface{} { m := make(map[string]interface{}, len(*n)) for _, p := range *n { @@ -336,6 +344,7 @@ func (n *admin_nodeInfo) asMap() map[string]interface{} { return m } +// toString creates a printable string representation of an admin_nodeInfo. func (n *admin_nodeInfo) toString() string { // TODO return something nicer looking than this var out []string @@ -346,6 +355,7 @@ func (n *admin_nodeInfo) toString() string { return fmt.Sprint(*n) } +// printInfos returns a newline separated list of strings from admin_nodeInfos, e.g. a printable string of info about all peers. func (a *admin) printInfos(infos []admin_nodeInfo) string { var out []string for _, info := range infos { @@ -355,6 +365,7 @@ func (a *admin) printInfos(infos []admin_nodeInfo) string { return strings.Join(out, "\n") } +// addPeer triggers a connection attempt to a node. func (a *admin) addPeer(addr string) error { u, err := url.Parse(addr) if err == nil { @@ -378,6 +389,7 @@ func (a *admin) addPeer(addr string) error { return nil } +// removePeer disconnects an existing node (given by the node's port number). func (a *admin) removePeer(p string) error { iport, err := strconv.Atoi(p) if err != nil { @@ -387,6 +399,7 @@ func (a *admin) removePeer(p string) error { return nil } +// startTunWithMTU creates the tun/tap device, sets its address, and sets the MTU to the provided value. func (a *admin) startTunWithMTU(ifname string, iftapmode bool, ifmtu int) error { // Close the TUN first if open _ = a.core.tun.close() @@ -415,6 +428,7 @@ func (a *admin) startTunWithMTU(ifname string, iftapmode bool, ifmtu int) error return nil } +// getData_getSelf returns the self node's info for admin responses. func (a *admin) getData_getSelf() *admin_nodeInfo { table := a.core.switchTable.table.Load().(lookupTable) coords := table.self.getCoords() @@ -426,6 +440,7 @@ func (a *admin) getData_getSelf() *admin_nodeInfo { return &self } +// getData_getPeers returns info from Core.peers for an admin response. func (a *admin) getData_getPeers() []admin_nodeInfo { ports := a.core.peers.ports.Load().(map[switchPort]*peer) var peerInfos []admin_nodeInfo @@ -449,6 +464,7 @@ func (a *admin) getData_getPeers() []admin_nodeInfo { return peerInfos } +// getData_getSwitchPeers returns info from Core.switchTable for an admin response. func (a *admin) getData_getSwitchPeers() []admin_nodeInfo { var peerInfos []admin_nodeInfo table := a.core.switchTable.table.Load().(lookupTable) @@ -470,6 +486,7 @@ func (a *admin) getData_getSwitchPeers() []admin_nodeInfo { return peerInfos } +// getData_getDHT returns info from Core.dht for an admin response. func (a *admin) getData_getDHT() []admin_nodeInfo { var infos []admin_nodeInfo now := time.Now() @@ -497,6 +514,7 @@ func (a *admin) getData_getDHT() []admin_nodeInfo { return infos } +// getData_getSessions returns info from Core.sessions for an admin response. func (a *admin) getData_getSessions() []admin_nodeInfo { var infos []admin_nodeInfo getSessions := func() { @@ -517,6 +535,7 @@ func (a *admin) getData_getSessions() []admin_nodeInfo { return infos } +// getAllowedEncryptionPublicKeys returns the public keys permitted for incoming peer connections. func (a *admin) getAllowedEncryptionPublicKeys() []string { pubs := a.core.peers.getAllowedEncryptionPublicKeys() var out []string @@ -526,6 +545,7 @@ func (a *admin) getAllowedEncryptionPublicKeys() []string { return out } +// addAllowedEncryptionPublicKey whitelists a key for incoming peer connections. func (a *admin) addAllowedEncryptionPublicKey(bstr string) (err error) { boxBytes, err := hex.DecodeString(bstr) if err == nil { @@ -536,6 +556,8 @@ func (a *admin) addAllowedEncryptionPublicKey(bstr string) (err error) { return } +// removeAllowedEncryptionPublicKey removes a key from the whitelist for incoming peer connections. +// If none are set, an empty list permits all incoming connections. func (a *admin) removeAllowedEncryptionPublicKey(bstr string) (err error) { boxBytes, err := hex.DecodeString(bstr) if err == nil { @@ -546,6 +568,9 @@ func (a *admin) removeAllowedEncryptionPublicKey(bstr string) (err error) { return } +// getResponse_dot returns a response for a graphviz dot formatted representation of the known parts of the network. +// This is color-coded and labeled, and includes the self node, switch peers, nodes known to the DHT, and nodes with open sessions. +// The graph is structured as a tree with directed links leading away from the root. func (a *admin) getResponse_dot() []byte { self := a.getData_getSelf() peers := a.getData_getSwitchPeers() diff --git a/src/yggdrasil/dht.go b/src/yggdrasil/dht.go index 33933914..3d341e71 100644 --- a/src/yggdrasil/dht.go +++ b/src/yggdrasil/dht.go @@ -23,12 +23,20 @@ import "time" //import "fmt" -// Maximum size for buckets and lookups -// Exception for buckets if the next one is non-full -const dht_bucket_number = 8 * NodeIDLen // This shouldn't be changed -const dht_bucket_size = 2 // This should be at least 2 -const dht_lookup_size = 16 // This should be at least 1, below 2 is impractical +// Number of DHT buckets, equal to the number of bits in a NodeID. +// Note that, in practice, nearly all of these will be empty. +const dht_bucket_number = 8 * NodeIDLen +// Number of nodes to keep in each DHT bucket. +// Additional entries may be kept for peers, for bootstrapping reasons, if they don't already have an entry in the bucket. +const dht_bucket_size = 2 + +// Number of responses to include in a lookup. +// If extras are given, they will be truncated from the response handler to prevent abuse. +const dht_lookup_size = 16 + +// dhtInfo represents everything we know about a node in the DHT. +// This includes its key, a cache of it's NodeID, coords, and timing/ping related info for deciding who/when to ping nodes for maintenance. type dhtInfo struct { nodeID_hidden *NodeID key boxPubKey @@ -39,6 +47,7 @@ type dhtInfo struct { throttle uint8 // Number of seconds to wait before pinging a node to bootstrap buckets, gradually increases up to 1 minute } +// Returns the *NodeID associated with dhtInfo.key, calculating it on the fly the first time or from a cache all subsequent times. func (info *dhtInfo) getNodeID() *NodeID { if info.nodeID_hidden == nil { info.nodeID_hidden = getNodeID(&info.key) @@ -46,17 +55,23 @@ func (info *dhtInfo) getNodeID() *NodeID { return info.nodeID_hidden } +// The nodes we known in a bucket (a region of keyspace with a matching prefix of some length). type bucket struct { peers []*dhtInfo other []*dhtInfo } +// Request for a node to do a lookup. +// Includes our key and coords so they can send a response back, and the destination NodeID we want to ask about. type dhtReq struct { Key boxPubKey // Key of whoever asked Coords []byte // Coords of whoever asked Dest NodeID // NodeID they're asking about } +// Response to a DHT lookup. +// Includes the key and coords of the node that's responding, and the destination they were asked about. +// The main part is Infos []*dhtInfo, the lookup response. type dhtRes struct { Key boxPubKey // key to respond to Coords []byte // coords to respond to @@ -64,11 +79,16 @@ type dhtRes struct { Infos []*dhtInfo // response } +// Information about a node, either taken from our table or from a lookup response. +// Used to schedule pings at a later time (they're throttled to 1/second for background maintenance traffic). type dht_rumor struct { info *dhtInfo target *NodeID } +// The main DHT struct. +// Includes a slice of buckets, to organize known nodes based on their region of keyspace. +// Also includes information about outstanding DHT requests and the rumor mill of nodes to ping at some point. type dht struct { core *Core nodeID NodeID @@ -79,6 +99,7 @@ type dht struct { rumorMill []dht_rumor } +// Initializes the DHT. func (t *dht) init(c *Core) { t.core = c t.nodeID = *t.core.GetNodeID() @@ -86,6 +107,8 @@ func (t *dht) init(c *Core) { t.reqs = make(map[boxPubKey]map[NodeID]time.Time) } +// Reads a request, performs a lookup, and responds. +// If the node that sent the request isn't in our DHT, but should be, then we add them. func (t *dht) handleReq(req *dhtReq) { // Send them what they asked for loc := t.core.switchTable.getLocator() @@ -106,6 +129,8 @@ func (t *dht) handleReq(req *dhtReq) { //if req.dest != t.nodeID { t.ping(&info, info.getNodeID()) } // Or spam... } +// Reads a lookup response, checks that we had sent a matching request, and processes the response info. +// This mainly consists of updating the node we asked in our DHT (they responded, so we know they're still alive), and adding the response info to the rumor mill. func (t *dht) handleRes(res *dhtRes) { t.core.searches.handleDHTRes(res) reqs, isIn := t.reqs[res.Key] @@ -157,6 +182,7 @@ func (t *dht) handleRes(res *dhtRes) { } } +// Does a DHT lookup and returns the results, sorted in ascending order of distance from the destination. func (t *dht) lookup(nodeID *NodeID, allowCloser bool) []*dhtInfo { // FIXME this allocates a bunch, sorts, and keeps the part it likes // It would be better to only track the part it likes to begin with @@ -192,14 +218,18 @@ func (t *dht) lookup(nodeID *NodeID, allowCloser bool) []*dhtInfo { return res } +// Gets the bucket for a specified matching prefix length. func (t *dht) getBucket(bidx int) *bucket { return &t.buckets_hidden[bidx] } +// Lists the number of buckets. func (t *dht) nBuckets() int { return len(t.buckets_hidden) } +// Inserts a node into the DHT if they meet certain requirements. +// In particular, they must either be a peer that's not already in the DHT, or else be someone we should insert into the DHT (see: shouldInsert). func (t *dht) insertIfNew(info *dhtInfo, isPeer bool) { //fmt.Println("DEBUG: dht insertIfNew:", info.getNodeID(), info.coords) // Insert if no "other" entry already exists @@ -219,6 +249,7 @@ func (t *dht) insertIfNew(info *dhtInfo, isPeer bool) { } } +// Adds a node to the DHT, possibly removing another node in the process. func (t *dht) insert(info *dhtInfo, isPeer bool) { //fmt.Println("DEBUG: dht insert:", info.getNodeID(), info.coords) // First update the time on this info @@ -253,6 +284,7 @@ func (t *dht) insert(info *dhtInfo, isPeer bool) { } } +// Gets the bucket index for the bucket where we would put the given NodeID. func (t *dht) getBucketIndex(nodeID *NodeID) (int, bool) { for bidx := 0; bidx < t.nBuckets(); bidx++ { them := nodeID[bidx/8] & (0x80 >> byte(bidx%8)) @@ -264,6 +296,8 @@ func (t *dht) getBucketIndex(nodeID *NodeID) (int, bool) { return t.nBuckets(), false } +// Helper called by containsPeer, containsOther, and contains. +// Returns true if a node with the same ID *and coords* is already in the given part of the bucket. func dht_bucket_check(newInfo *dhtInfo, infos []*dhtInfo) bool { // Compares if key and coords match if newInfo == nil { @@ -293,18 +327,22 @@ func dht_bucket_check(newInfo *dhtInfo, infos []*dhtInfo) bool { return false } +// Calls bucket_check over the bucket's peers infos. func (b *bucket) containsPeer(info *dhtInfo) bool { return dht_bucket_check(info, b.peers) } +// Calls bucket_check over the bucket's other info. func (b *bucket) containsOther(info *dhtInfo) bool { return dht_bucket_check(info, b.other) } +// returns containsPeer || containsOther func (b *bucket) contains(info *dhtInfo) bool { return b.containsPeer(info) || b.containsOther(info) } +// Removes a node with the corresponding key, if any, from a bucket. func (b *bucket) drop(key *boxPubKey) { clean := func(infos []*dhtInfo) []*dhtInfo { cleaned := infos[:0] @@ -320,6 +358,7 @@ func (b *bucket) drop(key *boxPubKey) { b.other = clean(b.other) } +// Sends a lookup request to the specified node. func (t *dht) sendReq(req *dhtReq, dest *dhtInfo) { // Send a dhtReq to the node in dhtInfo bs := req.encode() @@ -345,6 +384,7 @@ func (t *dht) sendReq(req *dhtReq, dest *dhtInfo) { reqsToDest[req.Dest] = time.Now() } +// Sends a lookup response to the specified node. func (t *dht) sendRes(res *dhtRes, req *dhtReq) { // Send a reply for a dhtReq bs := res.encode() @@ -361,10 +401,14 @@ func (t *dht) sendRes(res *dhtRes, req *dhtReq) { t.core.router.out(packet) } +// Returns true of a bucket contains no peers and no other nodes. func (b *bucket) isEmpty() bool { return len(b.peers)+len(b.other) == 0 } +// Gets the next node that should be pinged from the bucket. +// There's a cooldown of 6 seconds between ping attempts for each node, to give them time to respond. +// It returns the least recently pinged node, subject to that send cooldown. func (b *bucket) nextToPing() *dhtInfo { // Check the nodes in the bucket // Return whichever one responded least recently @@ -387,12 +431,16 @@ func (b *bucket) nextToPing() *dhtInfo { return toPing } +// Returns a useful target address to ask about for pings. +// Equal to the our node's ID, except for exactly 1 bit at the bucket index. func (t *dht) getTarget(bidx int) *NodeID { targetID := t.nodeID targetID[bidx/8] ^= 0x80 >> byte(bidx%8) return &targetID } +// Sends a ping to a node, or removes the node if it has failed to respond to too many pings. +// If target is nil, we will ask the node about our own NodeID. func (t *dht) ping(info *dhtInfo, target *NodeID) { if info.pings > 2 { bidx, isOK := t.getBucketIndex(info.getNodeID()) @@ -418,6 +466,8 @@ func (t *dht) ping(info *dhtInfo, target *NodeID) { t.sendReq(&req, info) } +// Adds a node info and target to the rumor mill. +// The node will be asked about the target at a later point, if doing so would still be useful at the time. func (t *dht) addToMill(info *dhtInfo, target *NodeID) { rumor := dht_rumor{ info: info, @@ -426,6 +476,11 @@ func (t *dht) addToMill(info *dhtInfo, target *NodeID) { t.rumorMill = append(t.rumorMill, rumor) } +// Regular periodic maintenance. +// If the mill is empty, it adds two pings to the rumor mill. +// The first is to the node that responded least recently, provided that it's been at least 1 minute, to make sure we eventually detect and remove unresponsive nodes. +// The second is used for bootstrapping, and attempts to fill some bucket, iterating over buckets and resetting after it hits the last non-empty one. +// If the mill is not empty, it pops nodes from the mill until it finds one that would be useful to ping (see: shouldInsert), and then pings it. func (t *dht) doMaintenance() { // First clean up reqs for key, reqs := range t.reqs { @@ -489,6 +544,8 @@ func (t *dht) doMaintenance() { } } +// Returns true if it would be worth pinging the specified node. +// This requires that the bucket doesn't already contain the node, and that either the bucket isn't full yet or the node is closer to us in keyspace than some other node in that bucket. func (t *dht) shouldInsert(info *dhtInfo) bool { bidx, isOK := t.getBucketIndex(info.getNodeID()) if !isOK { @@ -509,6 +566,7 @@ func (t *dht) shouldInsert(info *dhtInfo) bool { return false } +// Returns true if the keyspace distance between the first and second node is smaller than the keyspace distance between the second and third node. func dht_firstCloserThanThird(first *NodeID, second *NodeID, third *NodeID) bool { @@ -523,6 +581,10 @@ func dht_firstCloserThanThird(first *NodeID, return false } +// Resets the DHT in response to coord changes. +// This empties all buckets, resets the bootstrapping cycle to 0, and empties the rumor mill. +// It adds all old "other" node info to the rumor mill, so they'll be pinged quickly. +// If those nodes haven't also changed coords, then this is a relatively quick way to notify those nodes of our new coords and re-add them to our own DHT if they respond. func (t *dht) reset() { // This is mostly so bootstrapping will reset to resend coords into the network t.offset = 0 diff --git a/src/yggdrasil/peer.go b/src/yggdrasil/peer.go index 2a322cf7..5e522f36 100644 --- a/src/yggdrasil/peer.go +++ b/src/yggdrasil/peer.go @@ -4,19 +4,16 @@ package yggdrasil // Commented code should be removed // Live code should be better commented -// FIXME (!) this part may be at least sligtly vulnerable to replay attacks -// The switch message part should catch / drop old tstamps -// So the damage is limited -// But you could still mess up msgAnc / msgHops and break some things there -// It needs to ignore messages with a lower seq -// Probably best to start setting seq to a timestamp in that case... - import "time" import "sync" import "sync/atomic" //import "fmt" +// The peers struct represents peers with an active connection. +// Incomping packets are passed to the corresponding peer, which handles them somehow. +// In most cases, this involves passing the packet to the handler for outgoing traffic to another peer. +// In other cases, it's link protocol traffic used to build the spanning tree, in which case this checks signatures and passes the message along to the switch. type peers struct { core *Core mutex sync.Mutex // Synchronize writes to atomic @@ -26,6 +23,7 @@ type peers struct { allowedEncryptionPublicKeys map[boxPubKey]struct{} } +// Initializes the peers struct. func (ps *peers) init(c *Core) { ps.mutex.Lock() defer ps.mutex.Unlock() @@ -34,6 +32,7 @@ func (ps *peers) init(c *Core) { ps.allowedEncryptionPublicKeys = make(map[boxPubKey]struct{}) } +// Returns true if an incoming peer connection to a key is allowed, either because the key is in the whitelist or because the whitelist is empty. func (ps *peers) isAllowedEncryptionPublicKey(box *boxPubKey) bool { ps.authMutex.RLock() defer ps.authMutex.RUnlock() @@ -41,18 +40,21 @@ func (ps *peers) isAllowedEncryptionPublicKey(box *boxPubKey) bool { return isIn || len(ps.allowedEncryptionPublicKeys) == 0 } +// Adds a key to the whitelist. func (ps *peers) addAllowedEncryptionPublicKey(box *boxPubKey) { ps.authMutex.Lock() defer ps.authMutex.Unlock() ps.allowedEncryptionPublicKeys[*box] = struct{}{} } +// Removes a key from the whitelist. func (ps *peers) removeAllowedEncryptionPublicKey(box *boxPubKey) { ps.authMutex.Lock() defer ps.authMutex.Unlock() delete(ps.allowedEncryptionPublicKeys, *box) } +// Gets the whitelist of allowed keys for incoming connections. func (ps *peers) getAllowedEncryptionPublicKeys() []boxPubKey { ps.authMutex.RLock() defer ps.authMutex.RUnlock() @@ -63,14 +65,17 @@ func (ps *peers) getAllowedEncryptionPublicKeys() []boxPubKey { return keys } +// Atomically gets a map[switchPort]*peer of known peers. func (ps *peers) getPorts() map[switchPort]*peer { return ps.ports.Load().(map[switchPort]*peer) } +// Stores a map[switchPort]*peer (note that you should take a mutex before store operations to avoid conflicts with other nodes attempting to read/change/store at the same time). func (ps *peers) putPorts(ports map[switchPort]*peer) { ps.ports.Store(ports) } +// Information known about a peer, including thier box/sig keys, precomputed shared keys (static and ephemeral), a handler for their outgoing traffic, and queue sizes for local backpressure. type peer struct { queueSize int64 // used to track local backpressure bytesSent uint64 // To track bandwidth usage for getPeers @@ -90,14 +95,17 @@ type peer struct { close func() // Called when a peer is removed, to close the underlying connection, or via admin api } +// Size of the queue of packets to be sent to the node. func (p *peer) getQueueSize() int64 { return atomic.LoadInt64(&p.queueSize) } +// Used to increment or decrement the queue. func (p *peer) updateQueueSize(delta int64) { atomic.AddInt64(&p.queueSize, delta) } +// Creates a new peer with the specified box, sig, and linkShared keys, using the lowest unocupied port number. func (ps *peers) newPeer(box *boxPubKey, sig *sigPubKey, linkShared *boxSharedKey) *peer { now := time.Now() p := peer{box: *box, @@ -125,12 +133,13 @@ func (ps *peers) newPeer(box *boxPubKey, sig *sigPubKey, linkShared *boxSharedKe return &p } +// Removes a peer for a given port, if one exists. func (ps *peers) removePeer(port switchPort) { if port == 0 { return } // Can't remove self peer ps.core.router.doAdmin(func() { - ps.core.switchTable.removePeer(port) + ps.core.switchTable.unlockedRemovePeer(port) }) ps.mutex.Lock() oldPorts := ps.getPorts() @@ -150,6 +159,8 @@ func (ps *peers) removePeer(port switchPort) { } } +// If called, sends a notification to each peer that they should send a new switch message. +// Mainly called by the switch after an update. func (ps *peers) sendSwitchMsgs() { ports := ps.getPorts() for _, p := range ports { @@ -163,6 +174,8 @@ func (ps *peers) sendSwitchMsgs() { } } +// This must be launched in a separate goroutine by whatever sets up the peer struct. +// It handles link protocol traffic. func (p *peer) linkLoop() { go func() { p.doSend <- struct{}{} }() tick := time.NewTicker(time.Second) @@ -182,6 +195,8 @@ func (p *peer) linkLoop() { } } +// Called to handle incoming packets. +// Passes the packet to a handler for that packet type. func (p *peer) handlePacket(packet []byte) { // TODO See comment in sendPacket about atomics technically being done wrong atomic.AddUint64(&p.bytesRecvd, uint64(len(packet))) @@ -197,10 +212,12 @@ func (p *peer) handlePacket(packet []byte) { case wire_LinkProtocolTraffic: p.handleLinkTraffic(packet) default: - return + util_putBytes(packet) } } +// Called to handle traffic or protocolTraffic packets. +// In either case, this reads from the coords of the packet header, does a switch lookup, and forwards to the next node. func (p *peer) handleTraffic(packet []byte, pTypeLen int) { if p.port != 0 && p.dinfo == nil { // Drop traffic until the peer manages to send us at least one good switchMsg @@ -221,14 +238,15 @@ func (p *peer) handleTraffic(packet []byte, pTypeLen int) { to.sendPacket(packet) } +// This just calls p.out(packet) for now. func (p *peer) sendPacket(packet []byte) { // Is there ever a case where something more complicated is needed? // What if p.out blocks? p.out(packet) - // TODO this should really happen at the interface, to account for LIFO packet drops and additional per-packet/per-message overhead, but this should be pretty close... better to move it to the tcp/udp stuff *after* rewriting both to give a common interface - atomic.AddUint64(&p.bytesSent, uint64(len(packet))) } +// This wraps the packet in the inner (ephemeral) and outer (permanent) crypto layers. +// It sends it to p.linkOut, which bypasses the usual packet queues. func (p *peer) sendLinkPacket(packet []byte) { innerPayload, innerNonce := boxSeal(&p.linkShared, packet, nil) innerLinkPacket := wire_linkProtoTrafficPacket{ @@ -245,6 +263,8 @@ func (p *peer) sendLinkPacket(packet []byte) { p.linkOut <- packet } +// Decrypts the outer (permanent) and inner (ephemeral) crypto layers on link traffic. +// Identifies the link traffic type and calls the appropriate handler. func (p *peer) handleLinkTraffic(bs []byte) { packet := wire_linkProtoTrafficPacket{} if !packet.decode(bs) { @@ -269,10 +289,12 @@ func (p *peer) handleLinkTraffic(bs []byte) { switch pType { case wire_SwitchMsg: p.handleSwitchMsg(payload) - default: // TODO?... + default: + util_putBytes(bs) } } +// Gets a switchMsg from the switch, adds signed next-hop info for this peer, and sends it to them. func (p *peer) sendSwitchMsg() { msg := p.core.switchTable.getMsg() if msg == nil { @@ -290,6 +312,8 @@ func (p *peer) sendSwitchMsg() { p.sendLinkPacket(packet) } +// Handles a switchMsg from the peer, checking signatures and passing good messages to the switch. +// Also creates a dhtInfo struct and arranges for it to be added to the dht (this is how dht bootstrapping begins). func (p *peer) handleSwitchMsg(packet []byte) { var msg switchMsg if !msg.decode(packet) { @@ -330,6 +354,8 @@ func (p *peer) handleSwitchMsg(packet []byte) { p.dinfo = &dinfo } +// This generates the bytes that we sign or check the signature of for a switchMsg. +// It begins with the next node's key, followed by the root and the timetsamp, followed by coords being advertised to the next node. func getBytesForSig(next *sigPubKey, msg *switchMsg) []byte { var loc switchLocator for _, hop := range msg.Hops { diff --git a/src/yggdrasil/router.go b/src/yggdrasil/router.go index 9e48668e..4f15dea3 100644 --- a/src/yggdrasil/router.go +++ b/src/yggdrasil/router.go @@ -29,6 +29,8 @@ import "golang.org/x/net/ipv6" //import "fmt" //import "net" +// The router struct has channels to/from the tun/tap device and a self peer (0), which is how messages are passed between this node and the peers/switch layer. +// The router's mainLoop goroutine is responsible for managing all information related to the dht, searches, and crypto sessions. type router struct { core *Core addr address @@ -40,6 +42,7 @@ type router struct { admin chan func() // pass a lambda for the admin socket to query stuff } +// Initializes the router struct, which includes setting up channels to/from the tun/tap. func (r *router) init(core *Core) { r.core = core r.addr = *address_addrForNodeID(&r.core.dht.nodeID) @@ -67,12 +70,17 @@ func (r *router) init(core *Core) { // go r.mainLoop() } +// Starts the mainLoop goroutine. func (r *router) start() error { r.core.log.Println("Starting router") go r.mainLoop() return nil } +// Takes traffic from the tun/tap and passes it to router.send, or from r.in and handles incoming traffic. +// Also adds new peer info to the DHT. +// Also resets the DHT and sesssions in the event of a coord change. +// Also does periodic maintenance stuff. func (r *router) mainLoop() { ticker := time.NewTicker(time.Second) defer ticker.Stop() @@ -102,6 +110,11 @@ func (r *router) mainLoop() { } } +// Checks a packet's to/from address to make sure it's in the allowed range. +// If a session to the destination exists, gets the session and passes the packet to it. +// If no session exists, it triggers (or continues) a search. +// If the session hasn't responded recently, it triggers a ping or search to keep things alive or deal with broken coords *relatively* quickly. +// It also deals with oversized packets if there are MTU issues by calling into icmpv6.go to spoof PacketTooBig traffic, or DestinationUnreachable if the other side has their tun/tap disabled. func (r *router) sendPacket(bs []byte) { if len(bs) < 40 { panic("Tried to send a packet shorter than a header...") @@ -226,6 +239,8 @@ func (r *router) sendPacket(bs []byte) { } } +// Called for incoming traffic by the session worker for that connection. +// Checks that the IP address is correct (matches the session) and passes the packet to the tun/tap. func (r *router) recvPacket(bs []byte, theirAddr *address, theirSubnet *subnet) { // Note: called directly by the session worker, not the router goroutine //fmt.Println("Recv packet") @@ -248,6 +263,7 @@ func (r *router) recvPacket(bs []byte, theirAddr *address, theirSubnet *subnet) r.recv <- bs } +// Checks incoming traffic type and passes it to the appropriate handler. func (r *router) handleIn(packet []byte) { pType, pTypeLen := wire_decode_uint64(packet) if pTypeLen == 0 { @@ -262,6 +278,8 @@ func (r *router) handleIn(packet []byte) { } } +// Handles incoming traffic, i.e. encapuslated ordinary IPv6 packets. +// Passes them to the crypto session worker to be decrypted and sent to the tun/tap. func (r *router) handleTraffic(packet []byte) { defer util_putBytes(packet) p := wire_trafficPacket{} @@ -276,6 +294,7 @@ func (r *router) handleTraffic(packet []byte) { sinfo.recv <- &p } +// Handles protocol traffic by decrypting it, checking its type, and passing it to the appropriate handler for that traffic type. func (r *router) handleProto(packet []byte) { // First parse the packet p := wire_protoTrafficPacket{} @@ -312,11 +331,12 @@ func (r *router) handleProto(packet []byte) { r.handleDHTReq(bs, &p.FromKey) case wire_DHTLookupResponse: r.handleDHTRes(bs, &p.FromKey) - default: /*panic("Should not happen in testing") ;*/ - return + default: + util_putBytes(packet) } } +// Decodes session pings from wire format and passes them to sessions.handlePing where they either create or update a session. func (r *router) handlePing(bs []byte, fromKey *boxPubKey) { ping := sessionPing{} if !ping.decode(bs) { @@ -326,10 +346,12 @@ func (r *router) handlePing(bs []byte, fromKey *boxPubKey) { r.core.sessions.handlePing(&ping) } +// Handles session pongs (which are really pings with an extra flag to prevent acknowledgement). func (r *router) handlePong(bs []byte, fromKey *boxPubKey) { r.handlePing(bs, fromKey) } +// Decodes dht requests and passes them to dht.handleReq to trigger a lookup/response. func (r *router) handleDHTReq(bs []byte, fromKey *boxPubKey) { req := dhtReq{} if !req.decode(bs) { @@ -339,6 +361,7 @@ func (r *router) handleDHTReq(bs []byte, fromKey *boxPubKey) { r.core.dht.handleReq(&req) } +// Decodes dht responses and passes them to dht.handleRes to update the DHT table and further pass them to the search code (if applicable). func (r *router) handleDHTRes(bs []byte, fromKey *boxPubKey) { res := dhtRes{} if !res.decode(bs) { @@ -348,6 +371,9 @@ func (r *router) handleDHTRes(bs []byte, fromKey *boxPubKey) { r.core.dht.handleRes(&res) } +// Passed a function to call. +// This will send the function to r.admin and block until it finishes. +// It's used by the admin socket to ask the router mainLoop goroutine about information in the session or dht structs, which cannot be read safely from outside that goroutine. func (r *router) doAdmin(f func()) { // Pass this a function that needs to be run by the router's main goroutine // It will pass the function to the router and wait for the router to finish diff --git a/src/yggdrasil/search.go b/src/yggdrasil/search.go index d9dc76e3..772b3848 100644 --- a/src/yggdrasil/search.go +++ b/src/yggdrasil/search.go @@ -16,9 +16,16 @@ import "time" //import "fmt" +// This defines the maximum number of dhtInfo that we keep track of for nodes to query in an ongoing search. const search_MAX_SEARCH_SIZE = 16 + +// This defines the time after which we send a new search packet. +// Search packets are sent automatically immediately after a response is received. +// So this allows for timeouts and for long searches to become increasingly parallel. const search_RETRY_TIME = time.Second +// Information about an ongoing search. +// Includes the targed NodeID, the bitmask to match it to an IP, and the list of nodes to visit / already visited. type searchInfo struct { dest NodeID mask NodeID @@ -28,16 +35,19 @@ type searchInfo struct { visited map[NodeID]bool } +// This stores a map of active searches. type searches struct { core *Core searches map[NodeID]*searchInfo } +// Intializes the searches struct. func (s *searches) init(core *Core) { s.core = core s.searches = make(map[NodeID]*searchInfo) } +// Creates a new search info, adds it to the searches struct, and returns a pointer to the info. func (s *searches) createSearch(dest *NodeID, mask *NodeID) *searchInfo { now := time.Now() for dest, sinfo := range s.searches { @@ -56,6 +66,9 @@ func (s *searches) createSearch(dest *NodeID, mask *NodeID) *searchInfo { //////////////////////////////////////////////////////////////////////////////// +// Checks if there's an ongoing search relaed to a dhtRes. +// If there is, it adds the response info to the search and triggers a new search step. +// If there's no ongoing search, or we if the dhtRes finished the search (it was from the target node), then don't do anything more. func (s *searches) handleDHTRes(res *dhtRes) { sinfo, isIn := s.searches[res.Dest] if !isIn || s.checkDHTRes(sinfo, res) { @@ -68,6 +81,10 @@ func (s *searches) handleDHTRes(res *dhtRes) { } } +// Adds the information from a dhtRes to an ongoing search. +// Info about a node that has already been visited is not re-added to the search. +// Duplicate information about nodes toVisit is deduplicated (the newest information is kept). +// The toVisit list is sorted in ascending order of keyspace distance from the destination. func (s *searches) addToSearch(sinfo *searchInfo, res *dhtRes) { // Add responses to toVisit if closer to dest than the res node from := dhtInfo{key: res.Key, coords: res.Coords} @@ -98,6 +115,8 @@ func (s *searches) addToSearch(sinfo *searchInfo, res *dhtRes) { } } +// If there are no nodes left toVisit, then this cleans up the search. +// Otherwise, it pops the closest node to the destination (in keyspace) off of the toVisit list and sends a dht ping. func (s *searches) doSearchStep(sinfo *searchInfo) { if len(sinfo.toVisit) == 0 { // Dead end, do cleanup @@ -115,6 +134,8 @@ func (s *searches) doSearchStep(sinfo *searchInfo) { } } +// If we've recenty sent a ping for this search, do nothing. +// Otherwise, doSearchStep and schedule another continueSearch to happen after search_RETRY_TIME. func (s *searches) continueSearch(sinfo *searchInfo) { if time.Since(sinfo.time) < search_RETRY_TIME { return @@ -137,6 +158,7 @@ func (s *searches) continueSearch(sinfo *searchInfo) { }() } +// Calls create search, and initializes the iterative search parts of the struct before returning it. func (s *searches) newIterSearch(dest *NodeID, mask *NodeID) *searchInfo { sinfo := s.createSearch(dest, mask) sinfo.toVisit = s.core.dht.lookup(dest, true) @@ -144,6 +166,9 @@ func (s *searches) newIterSearch(dest *NodeID, mask *NodeID) *searchInfo { return sinfo } +// Checks if a dhtRes is good (called by handleDHTRes). +// If the response is from the target, get/create a session, trigger a session ping, and return true. +// Otherwise return false. func (s *searches) checkDHTRes(info *searchInfo, res *dhtRes) bool { them := getNodeID(&res.Key) var destMasked NodeID diff --git a/src/yggdrasil/session.go b/src/yggdrasil/session.go index 33111424..01bfb959 100644 --- a/src/yggdrasil/session.go +++ b/src/yggdrasil/session.go @@ -6,6 +6,8 @@ package yggdrasil import "time" +// All the information we know about an active session. +// This includes coords, permanent and ephemeral keys, handles and nonces, various sorts of timing information for timeout and maintenance, and some metadata for the admin API. type sessionInfo struct { core *Core theirAddr address @@ -37,6 +39,7 @@ type sessionInfo struct { bytesRecvd uint64 // Bytes of real traffic received in this session } +// Represents a session ping/pong packet, andincludes information like public keys, a session handle, coords, a timestamp to prevent replays, and the tun/tap MTU. type sessionPing struct { SendPermPub boxPubKey // Sender's permanent key Handle handle // Random number to ID session @@ -47,7 +50,8 @@ type sessionPing struct { MTU uint16 } -// Returns true if the session was updated, false otherwise +// Updates session info in response to a ping, after checking that the ping is OK. +// Returns true if the session was updated, or false otherwise. func (s *sessionInfo) update(p *sessionPing) bool { if !(p.Tstamp > s.tstamp) { // To protect against replay attacks @@ -76,10 +80,14 @@ func (s *sessionInfo) update(p *sessionPing) bool { return true } +// Returns true if the session has been idle for longer than the allowed timeout. func (s *sessionInfo) timedout() bool { return time.Since(s.time) > time.Minute } +// Struct of all active sessions. +// Sessions are indexed by handle. +// Additionally, stores maps of address/subnet onto keys, and keys onto handles. type sessions struct { core *Core // Maps known permanent keys to their shared key, used by DHT a lot @@ -94,6 +102,7 @@ type sessions struct { subnetToPerm map[subnet]*boxPubKey } +// Initializes the session struct. func (ss *sessions) init(core *Core) { ss.core = core ss.permShared = make(map[boxPubKey]*boxSharedKey) @@ -104,6 +113,7 @@ func (ss *sessions) init(core *Core) { ss.subnetToPerm = make(map[subnet]*boxPubKey) } +// Gets the session corresponding to a given handle. func (ss *sessions) getSessionForHandle(handle *handle) (*sessionInfo, bool) { sinfo, isIn := ss.sinfos[*handle] if isIn && sinfo.timedout() { @@ -113,6 +123,7 @@ func (ss *sessions) getSessionForHandle(handle *handle) (*sessionInfo, bool) { return sinfo, isIn } +// Gets a session corresponding to an ephemeral session key used by this node. func (ss *sessions) getByMySes(key *boxPubKey) (*sessionInfo, bool) { h, isIn := ss.byMySes[*key] if !isIn { @@ -122,6 +133,7 @@ func (ss *sessions) getByMySes(key *boxPubKey) (*sessionInfo, bool) { return sinfo, isIn } +// Gets a session corresponding to a permanent key used by the remote node. func (ss *sessions) getByTheirPerm(key *boxPubKey) (*sessionInfo, bool) { h, isIn := ss.byTheirPerm[*key] if !isIn { @@ -131,6 +143,7 @@ func (ss *sessions) getByTheirPerm(key *boxPubKey) (*sessionInfo, bool) { return sinfo, isIn } +// Gets a session corresponding to an IPv6 address used by the remote node. func (ss *sessions) getByTheirAddr(addr *address) (*sessionInfo, bool) { p, isIn := ss.addrToPerm[*addr] if !isIn { @@ -140,6 +153,7 @@ func (ss *sessions) getByTheirAddr(addr *address) (*sessionInfo, bool) { return sinfo, isIn } +// Gets a session corresponding to an IPv6 /64 subnet used by the remote node/network. func (ss *sessions) getByTheirSubnet(snet *subnet) (*sessionInfo, bool) { p, isIn := ss.subnetToPerm[*snet] if !isIn { @@ -149,6 +163,8 @@ func (ss *sessions) getByTheirSubnet(snet *subnet) (*sessionInfo, bool) { return sinfo, isIn } +// Creates a new session and lazily cleans up old/timedout existing sessions. +// This includse initializing session info to sane defaults (e.g. lowest supported MTU). func (ss *sessions) createSession(theirPermKey *boxPubKey) *sessionInfo { sinfo := sessionInfo{} sinfo.core = ss.core @@ -201,6 +217,7 @@ func (ss *sessions) createSession(theirPermKey *boxPubKey) *sessionInfo { return &sinfo } +// Closes a session, removing it from sessions maps and killing the worker goroutine. func (sinfo *sessionInfo) close() { delete(sinfo.core.sessions.sinfos, sinfo.myHandle) delete(sinfo.core.sessions.byMySes, sinfo.mySesPub) @@ -211,6 +228,7 @@ func (sinfo *sessionInfo) close() { close(sinfo.recv) } +// Returns a session ping appropriate for the given session info. func (ss *sessions) getPing(sinfo *sessionInfo) sessionPing { loc := ss.core.switchTable.getLocator() coords := loc.getCoords() @@ -226,6 +244,9 @@ func (ss *sessions) getPing(sinfo *sessionInfo) sessionPing { return ref } +// Gets the shared key for a pair of box keys. +// Used to cache recently used shared keys for protocol traffic. +// This comes up with dht req/res and session ping/pong traffic. func (ss *sessions) getSharedKey(myPriv *boxPrivKey, theirPub *boxPubKey) *boxSharedKey { if skey, isIn := ss.permShared[*theirPub]; isIn { @@ -244,10 +265,13 @@ func (ss *sessions) getSharedKey(myPriv *boxPrivKey, return ss.permShared[*theirPub] } +// Sends a session ping by calling sendPingPong in ping mode. func (ss *sessions) ping(sinfo *sessionInfo) { ss.sendPingPong(sinfo, false) } +// Calls getPing, sets the appropriate ping/pong flag, encodes to wire format, and send it. +// Updates the time the last ping was sent in the session info. func (ss *sessions) sendPingPong(sinfo *sessionInfo, isPong bool) { ping := ss.getPing(sinfo) ping.IsPong = isPong @@ -268,6 +292,8 @@ func (ss *sessions) sendPingPong(sinfo *sessionInfo, isPong bool) { } } +// Handles a session ping, creating a session if needed and calling update, then possibly responding with a pong if the ping was in ping mode and the update was successful. +// If the session has a packet cached (common when first setting up a session), it will be sent. func (ss *sessions) handlePing(ping *sessionPing) { // Get the corresponding session (or create a new session) sinfo, isIn := ss.getByTheirPerm(&ping.SendPermPub) @@ -296,6 +322,9 @@ func (ss *sessions) handlePing(ping *sessionPing) { } } +// Used to subtract one nonce from another, staying in the range +- 64. +// This is used by the nonce progression machinery to advance the bitmask of recently received packets (indexed by nonce), or to check the appropriate bit of the bitmask. +// It's basically part of the machinery that prevents replays and duplicate packets. func (n *boxNonce) minus(m *boxNonce) int64 { diff := int64(0) for idx := range n { @@ -311,6 +340,9 @@ func (n *boxNonce) minus(m *boxNonce) int64 { return diff } +// Get the MTU of the session. +// Will be equal to the smaller of this node's MTU or the remote node's MTU. +// If sending over links with a maximum message size (this was a thing with the old UDP code), it could be further lowered, to a minimum of 1280. func (sinfo *sessionInfo) getMTU() uint16 { if sinfo.theirMTU == 0 || sinfo.myMTU == 0 { return 0 @@ -321,6 +353,7 @@ func (sinfo *sessionInfo) getMTU() uint16 { return sinfo.myMTU } +// Checks if a packet's nonce is recent enough to fall within the window of allowed packets, and not already received. func (sinfo *sessionInfo) nonceIsOK(theirNonce *boxNonce) bool { // The bitmask is to allow for some non-duplicate out-of-order packets diff := theirNonce.minus(&sinfo.theirNonce) @@ -330,19 +363,24 @@ func (sinfo *sessionInfo) nonceIsOK(theirNonce *boxNonce) bool { return ^sinfo.nonceMask&(0x01< 0 { + // This nonce is newer, so shift the window before setting the bit, and update theirNonce in the session info. sinfo.nonceMask <<= uint64(diff) sinfo.nonceMask &= 0x01 + sinfo.theirNonce = *theirNonce } else { + // This nonce is older, so set the bit but do not shift the window. sinfo.nonceMask &= 0x01 << uint64(-diff) } - sinfo.theirNonce = *theirNonce } +// Resets all sessions to an uninitialized state. +// Called after coord changes, so attemtps to use a session will trigger a new ping and notify the remote end of the coord change. func (ss *sessions) resetInits() { for _, sinfo := range ss.sinfos { sinfo.init = false @@ -351,10 +389,9 @@ func (ss *sessions) resetInits() { //////////////////////////////////////////////////////////////////////////////// -// This is for a per-session worker -// It handles calling the relatively expensive crypto operations -// It's also responsible for keeping nonces consistent - +// This is for a per-session worker. +// It handles calling the relatively expensive crypto operations. +// It's also responsible for checking nonces and dropping out-of-date/duplicate packets, or else calling the function to update nonces if the packet is OK. func (sinfo *sessionInfo) doWorker() { for { select { @@ -374,6 +411,7 @@ func (sinfo *sessionInfo) doWorker() { } } +// This encrypts a packet, creates a trafficPacket struct, encodes it, and sends it to router.out to pass it to the switch layer. func (sinfo *sessionInfo) doSend(bs []byte) { defer util_putBytes(bs) if !sinfo.init { @@ -392,6 +430,11 @@ func (sinfo *sessionInfo) doSend(bs []byte) { sinfo.core.router.out(packet) } +// This takes a trafficPacket and checks the nonce. +// If the nonce is OK, it decrypts the packet. +// If the decrypted packet is OK, it calls router.recvPacket to pass the packet to the tun/tap. +// If a packet does not decrypt successfully, it assumes the packet was truncated, and updates the MTU accordingly. +// TODO? remove the MTU updating part? That should never happen with TCP peers, and the old UDP code that caused it was removed (and if replaced, should be replaced with something that can reliably send messages with an arbitrary size). func (sinfo *sessionInfo) doRecv(p *wire_trafficPacket) { defer util_putBytes(p.Payload) payloadSize := uint16(len(p.Payload)) diff --git a/src/yggdrasil/signature.go b/src/yggdrasil/signature.go index b146b9f9..f7b88ae9 100644 --- a/src/yggdrasil/signature.go +++ b/src/yggdrasil/signature.go @@ -6,40 +6,52 @@ package yggdrasil import "sync" import "time" +// This keeps track of what signatures have already been checked. +// It's used to skip expensive crypto operations, given that many signatures are likely to be the same for the average node's peers. type sigManager struct { mutex sync.RWMutex checked map[sigBytes]knownSig lastCleaned time.Time } +// Represents a known signature. +// Includes the key, the signature bytes, the bytes that were signed, and the time it was last used. type knownSig struct { + key sigPubKey + sig sigBytes bs []byte time time.Time } +// Initializes the signature manager. func (m *sigManager) init() { m.checked = make(map[sigBytes]knownSig) } +// Checks if a key and signature match the supplied bytes. +// If the same key/sig/bytes have been checked before, it returns true from the cached results. +// If not, it checks the key, updates it in the cache if successful, and returns the checked results. func (m *sigManager) check(key *sigPubKey, sig *sigBytes, bs []byte) bool { - if m.isChecked(sig, bs) { + if m.isChecked(key, sig, bs) { return true } verified := verify(key, bs, sig) if verified { - m.putChecked(sig, bs) + m.putChecked(key, sig, bs) } return verified } -func (m *sigManager) isChecked(sig *sigBytes, bs []byte) bool { +// Checks the cache to see if this key/sig/bytes combination has already been verified. +// Returns true if it finds a match. +func (m *sigManager) isChecked(key *sigPubKey, sig *sigBytes, bs []byte) bool { m.mutex.RLock() defer m.mutex.RUnlock() k, isIn := m.checked[*sig] if !isIn { return false } - if len(bs) != len(k.bs) { + if k.key != *key || k.sig != *sig || len(bs) != len(k.bs) { return false } for idx := 0; idx < len(bs); idx++ { @@ -51,7 +63,10 @@ func (m *sigManager) isChecked(sig *sigBytes, bs []byte) bool { return true } -func (m *sigManager) putChecked(newsig *sigBytes, bs []byte) { +// Puts a new result into the cache. +// This result is then used by isChecked to skip the expensive crypto verification if it's needed again. +// This is useful because, for nodes with multiple peers, there is often a lot of overlap between the signatures provided by each peer. +func (m *sigManager) putChecked(key *sigPubKey, newsig *sigBytes, bs []byte) { m.mutex.Lock() defer m.mutex.Unlock() now := time.Now() @@ -64,6 +79,6 @@ func (m *sigManager) putChecked(newsig *sigBytes, bs []byte) { } m.lastCleaned = now } - k := knownSig{bs: bs, time: now} + k := knownSig{key: *key, sig: *newsig, bs: bs, time: now} m.checked[*newsig] = k } diff --git a/src/yggdrasil/switch.go b/src/yggdrasil/switch.go index 6c42f456..a819594b 100644 --- a/src/yggdrasil/switch.go +++ b/src/yggdrasil/switch.go @@ -22,15 +22,17 @@ const switch_timeout = time.Minute const switch_updateInterval = switch_timeout / 2 const switch_throttle = switch_updateInterval / 2 -// You should be able to provide crypto signatures for this -// 1 signature per coord, from the *sender* to that coord -// E.g. A->B->C has sigA(A->B) and sigB(A->B->C) +// The switch locator represents the topology and network state dependent info about a node, minus the signatures that go with it. +// Nodes will pick the best root they see, provided that the root continues to push out updates with new timestamps. +// The coords represent a path from the root to a node. +// This path is generally part of a spanning tree, except possibly the last hop (it can loop when sending coords to your parent, but they see this and know not to use a looping path). type switchLocator struct { root sigPubKey tstamp int64 coords []switchPort } +// Returns true if the first sigPubKey has a higher TreeID. func firstIsBetter(first, second *sigPubKey) bool { // Higher TreeID is better ftid := getTreeID(first) @@ -45,6 +47,7 @@ func firstIsBetter(first, second *sigPubKey) bool { return false } +// Returns a copy of the locator which can safely be mutated. func (l *switchLocator) clone() switchLocator { // Used to create a deep copy for use in messages // Copy required because we need to mutate coords before sending @@ -55,6 +58,7 @@ func (l *switchLocator) clone() switchLocator { return loc } +// Gets the distance a locator is from the provided destination coords, with the coords provided in []byte format (used to compress integers sent over the wire). func (l *switchLocator) dist(dest []byte) int { // Returns distance (on the tree) from these coords offset := 0 @@ -85,6 +89,7 @@ func (l *switchLocator) dist(dest []byte) int { return dist } +// Gets coords in wire encoded format, with *no* length prefix. func (l *switchLocator) getCoords() []byte { bs := make([]byte, 0, len(l.coords)) for _, coord := range l.coords { @@ -94,6 +99,8 @@ func (l *switchLocator) getCoords() []byte { return bs } +// Returns true if the this locator represents an ancestor of the locator given as an argument. +// Ancestor means that it's the parent node, or the parent of parent, and so on... func (x *switchLocator) isAncestorOf(y *switchLocator) bool { if x.root != y.root { return false @@ -109,6 +116,7 @@ func (x *switchLocator) isAncestorOf(y *switchLocator) bool { return true } +// Information about a peer, used by the switch to build the tree and eventually make routing decisions. type peerInfo struct { key sigPubKey // ID of this peer locator switchLocator // Should be able to respond with signatures upon request @@ -119,17 +127,23 @@ type peerInfo struct { msg switchMsg // The wire switchMsg used } +// This is just a uint64 with a named type for clarity reasons. type switchPort uint64 + +// This is the subset of the information about a peer needed to make routing decisions, and it stored separately in an atomically accessed table, which gets hammered in the "hot loop" of the routing logic (see: peer.handleTraffic in peers.go). type tableElem struct { port switchPort locator switchLocator } +// This is the subset of the information about all peers needed to make routing decisions, and it stored separately in an atomically accessed table, which gets hammered in the "hot loop" of the routing logic (see: peer.handleTraffic in peers.go). type lookupTable struct { self switchLocator elems []tableElem } +// This is switch information which is mutable and needs to be modified by other goroutines, but is not accessed atomically. +// Use the switchTable functions to access it safely using the RWMutex for synchronization. type switchData struct { // All data that's mutable and used by exported Table methods // To be read/written with atomic.Value Store/Load calls @@ -139,6 +153,7 @@ type switchData struct { msg *switchMsg } +// All the information stored by the switch. type switchTable struct { core *Core key sigPubKey // Our own key @@ -151,6 +166,7 @@ type switchTable struct { table atomic.Value //lookupTable } +// Initializes the switchTable struct. func (t *switchTable) init(core *Core, key sigPubKey) { now := time.Now() t.core = core @@ -163,12 +179,14 @@ func (t *switchTable) init(core *Core, key sigPubKey) { t.drop = make(map[sigPubKey]int64) } +// Safely gets a copy of this node's locator. func (t *switchTable) getLocator() switchLocator { t.mutex.RLock() defer t.mutex.RUnlock() return t.data.locator.clone() } +// Regular maintenance to possibly timeout/reset the root and similar. func (t *switchTable) doMaintenance() { // Periodic maintenance work to keep things internally consistent t.mutex.Lock() // Write lock @@ -177,6 +195,7 @@ func (t *switchTable) doMaintenance() { t.cleanDropped() } +// Updates the root periodically if it is ourself, or promotes ourself to root if we're better than the current root or if the current root has timed out. func (t *switchTable) cleanRoot() { // TODO rethink how this is done?... // Get rid of the root if it looks like its timed out @@ -219,15 +238,23 @@ func (t *switchTable) cleanRoot() { } } -func (t *switchTable) removePeer(port switchPort) { +// Removes a peer. +// Must be called by the router mainLoop goroutine, e.g. call router.doAdmin with a lambda that calls this. +// If the removed peer was this node's parent, it immediately tries to find a new parent. +func (t *switchTable) unlockedRemovePeer(port switchPort) { delete(t.data.peers, port) t.updater.Store(&sync.Once{}) - // TODO if parent, find a new peer to use as parent instead + if port != t.parent { + return + } for _, info := range t.data.peers { t.unlockedHandleMsg(&info.msg, info.port) } } +// Dropped is a list of roots that are better than the current root, but stopped sending new timestamps. +// If we switch to a new root, and that root is better than an old root that previously timed out, then we can clean up the old dropped root infos. +// This function is called periodically to do that cleanup. func (t *switchTable) cleanDropped() { // TODO? only call this after root changes, not periodically for root := range t.drop { @@ -237,18 +264,23 @@ func (t *switchTable) cleanDropped() { } } +// A switchMsg contains the root node's sig key, timestamp, and signed per-hop information about a path from the root node to some other node in the network. +// This is exchanged with peers to construct the spanning tree. +// A subset of this information, excluding the signatures, is used to construct locators that are used elsewhere in the code. type switchMsg struct { Root sigPubKey TStamp int64 Hops []switchMsgHop } +// This represents the signed information about the path leading from the root the Next node, via the Port specified here. type switchMsgHop struct { Port switchPort Next sigPubKey Sig sigBytes } +// This returns a *switchMsg to a copy of this node's current switchMsg, which can safely have additional information appended to Hops and sent to a peer. func (t *switchTable) getMsg() *switchMsg { t.mutex.RLock() defer t.mutex.RUnlock() @@ -263,6 +295,8 @@ func (t *switchTable) getMsg() *switchMsg { } } +// This function checks that the root information in a switchMsg is OK. +// In particular, that the root is better, or else the same as the current root but with a good timestamp, and that this root+timestamp haven't been dropped due to timeout. func (t *switchTable) checkRoot(msg *switchMsg) bool { // returns false if it's a dropped root, not a better root, or has an older timestamp // returns true otherwise @@ -284,12 +318,18 @@ func (t *switchTable) checkRoot(msg *switchMsg) bool { } } +// This is a mutexed wrapper to unlockedHandleMsg, and is called by the peer structs in peers.go to pass a switchMsg for that peer into the switch. func (t *switchTable) handleMsg(msg *switchMsg, fromPort switchPort) { t.mutex.Lock() defer t.mutex.Unlock() t.unlockedHandleMsg(msg, fromPort) } +// This updates the switch with information about a peer. +// Then the tricky part, it decides if it should update our own locator as a result. +// That happens if this node is already our parent, or is advertising a better root, or is advertising a better path to the same root, etc... +// There are a lot of very delicate order sensitive checks here, so its' best to just read the code if you need to understand what it's doing. +// It's very important to not change the order of the statements in the case function unless you're absolutely sure that it's safe, including safe if used along side nodes that used the previous order. func (t *switchTable) unlockedHandleMsg(msg *switchMsg, fromPort switchPort) { // TODO directly use a switchMsg instead of switchMessage + sigs now := time.Now() @@ -299,10 +339,7 @@ func (t *switchTable) unlockedHandleMsg(msg *switchMsg, fromPort switchPort) { sender.locator.tstamp = msg.TStamp prevKey := msg.Root for _, hop := range msg.Hops { - // Build locator and signatures - var sig sigInfo - sig.next = hop.Next - sig.sig = hop.Sig + // Build locator sender.locator.coords = append(sender.locator.coords, hop.Port) sender.key = prevKey prevKey = hop.Next @@ -401,6 +438,7 @@ func (t *switchTable) unlockedHandleMsg(msg *switchMsg, fromPort switchPort) { return } +// This is called via a sync.Once to update the atomically readable subset of switch information that gets used for routing decisions. func (t *switchTable) updateTable() { // WARNING this should only be called from within t.data.updater.Do() // It relies on the sync.Once for synchronization with messages and lookups @@ -434,6 +472,12 @@ func (t *switchTable) updateTable() { t.table.Store(newTable) } +// This does the switch layer lookups that decide how to route traffic. +// Traffic uses greedy routing in a metric space, where the metric distance between nodes is equal to the distance between them on the tree. +// Traffic must be routed to a node that is closer to the destination via the metric space distance. +// In the event that two nodes are equally close, it gets routed to the one with the longest uptime (due to the order that things are iterated over). +// The size of the outgoing packet queue is added to a node's tree distance when the cost of forwarding to a node, subject to the constraint that the real tree distance puts them closer to the destination than ourself. +// Doing so adds a limited form of backpressure routing, based on local information, which allows us to forward traffic around *local* bottlenecks, provided that another greedy path exists. func (t *switchTable) lookup(dest []byte) switchPort { t.updater.Load().(*sync.Once).Do(t.updateTable) table := t.table.Load().(lookupTable) @@ -463,14 +507,3 @@ func (t *switchTable) lookup(dest []byte) switchPort { //t.core.log.Println("DEBUG: sending to", best, "cost", bestCost) return best } - -//////////////////////////////////////////////////////////////////////////////// - -//Signature stuff - -type sigInfo struct { - next sigPubKey - sig sigBytes -} - -//////////////////////////////////////////////////////////////////////////////// diff --git a/src/yggdrasil/tcp.go b/src/yggdrasil/tcp.go index e325fcfe..75a5618e 100644 --- a/src/yggdrasil/tcp.go +++ b/src/yggdrasil/tcp.go @@ -18,12 +18,13 @@ import "net" import "time" import "errors" import "sync" +import "sync/atomic" import "fmt" import "golang.org/x/net/proxy" const tcp_msgSize = 2048 + 65535 // TODO figure out what makes sense -// wrapper function for non tcp/ip connections +// Wrapper function for non tcp/ip connections. func setNoDelay(c net.Conn, delay bool) { tcp, ok := c.(*net.TCPConn) if ok { @@ -31,6 +32,7 @@ func setNoDelay(c net.Conn, delay bool) { } } +// The TCP listener and information about active TCP connections, to avoid duplication. type tcpInterface struct { core *Core serv net.Listener @@ -39,6 +41,8 @@ type tcpInterface struct { conns map[tcpInfo](chan struct{}) } +// This is used as the key to a map that tracks existing connections, to prevent multiple connections to the same keys and local/remote address pair from occuring. +// Different address combinations are allowed, so multi-homing is still technically possible (but not necessarily advisable). type tcpInfo struct { box boxPubKey sig sigPubKey @@ -46,15 +50,21 @@ type tcpInfo struct { remoteAddr string } +// Returns the address of the listener. func (iface *tcpInterface) getAddr() *net.TCPAddr { return iface.serv.Addr().(*net.TCPAddr) } +// Attempts to initiate a connection to the provided address. func (iface *tcpInterface) connect(addr string) { iface.call(addr) } +// Attempst to initiate a connection to the provided address, viathe provided socks proxy address. func (iface *tcpInterface) connectSOCKS(socksaddr, peeraddr string) { + // TODO make sure this doesn't keep attempting/killing connections when one is already active. + // I think some of the interaction between this and callWithConn needs work, so the dial isn't even attempted if there's already an outgoing call to peeraddr. + // Or maybe only if there's already an outgoing call to peeraddr via this socksaddr? go func() { dialer, err := proxy.SOCKS5("tcp", socksaddr, nil, proxy.Direct) if err == nil { @@ -72,6 +82,7 @@ func (iface *tcpInterface) connectSOCKS(socksaddr, peeraddr string) { }() } +// Initializes the struct. func (iface *tcpInterface) init(core *Core, addr string) (err error) { iface.core = core @@ -85,6 +96,7 @@ func (iface *tcpInterface) init(core *Core, addr string) (err error) { return err } +// Runs the listener, which spawns off goroutines for incoming connections. func (iface *tcpInterface) listener() { defer iface.serv.Close() iface.core.log.Println("Listening for TCP on:", iface.serv.Addr().String()) @@ -97,6 +109,7 @@ func (iface *tcpInterface) listener() { } } +// Called by connectSOCKS, it's like call but with the connection already established. func (iface *tcpInterface) callWithConn(conn net.Conn) { go func() { raddr := conn.RemoteAddr().String() @@ -117,6 +130,11 @@ func (iface *tcpInterface) callWithConn(conn net.Conn) { }() } +// Checks if a connection already exists. +// If not, it adds it to the list of active outgoing calls (to block future attempts) and dials the address. +// If the dial is successful, it launches the handler. +// When finished, it removes the outgoing call, so reconnection attempts can be made later. +// This all happens in a separate goroutine that it spawns. func (iface *tcpInterface) call(saddr string) { go func() { quit := false @@ -142,6 +160,8 @@ func (iface *tcpInterface) call(saddr string) { }() } +// This exchanges/checks connection metadata, sets up the peer struct, sets up the writer goroutine, and then runs the reader within the current goroutine. +// It defers a bunch of cleanup stuff to tear down all of these things when the reader exists (e.g. due to a closed connection or a timeout). func (iface *tcpInterface) handler(sock net.Conn, incoming bool) { defer sock.Close() // Get our keys @@ -233,7 +253,7 @@ func (iface *tcpInterface) handler(sock net.Conn, incoming bool) { out := make(chan []byte, 32) // TODO? what size makes sense defer close(out) go func() { - var shadow uint64 + var shadow int64 var stack [][]byte put := func(msg []byte) { stack = append(stack, msg) @@ -247,14 +267,16 @@ func (iface *tcpInterface) handler(sock net.Conn, incoming bool) { msgLen := wire_encode_uint64(uint64(len(msg))) buf := net.Buffers{tcp_msg[:], msgLen, msg} buf.WriteTo(sock) + atomic.AddUint64(&p.bytesSent, uint64(len(tcp_msg)+len(msgLen)+len(msg))) util_putBytes(msg) } timerInterval := 4 * time.Second timer := time.NewTimer(timerInterval) defer timer.Stop() for { - for ; shadow > 0; shadow-- { - p.updateQueueSize(-1) + if shadow != 0 { + p.updateQueueSize(-shadow) + shadow = 0 } timer.Stop() select { @@ -319,6 +341,9 @@ func (iface *tcpInterface) handler(sock net.Conn, incoming bool) { return } +// This reads from the socket into a []byte buffer for incomping messages. +// It copies completed messages out of the cache into a new slice, and passes them to the peer struct via the provided `in func([]byte)` argument. +// Then it shifts the incomplete fragments of data forward so future reads won't overwrite it. func (iface *tcpInterface) reader(sock net.Conn, in func([]byte)) { bs := make([]byte, 2*tcp_msgSize) frag := bs[:0] @@ -350,9 +375,13 @@ func (iface *tcpInterface) reader(sock net.Conn, in func([]byte)) { //////////////////////////////////////////////////////////////////////////////// -// Magic bytes to check +// These are 4 bytes of padding used to catch if something went horribly wrong with the tcp connection. var tcp_msg = [...]byte{0xde, 0xad, 0xb1, 0x75} // "dead bits" +// This takes a pointer to a slice as an argument. +// It checks if there's a complete message and, if so, slices out those parts and returns the message, true, and nil. +// If there's no error, but also no complete message, it returns nil, false, and nil. +// If there's an error, it returns nil, false, and the error, which the reader then handles (currently, by returning from the reader, which causes the connection to close). func tcp_chop_msg(bs *[]byte) ([]byte, bool, error) { // Returns msg, ok, err if len(*bs) < len(tcp_msg) { diff --git a/src/yggdrasil/util.go b/src/yggdrasil/util.go index 908b1ae9..fec5747c 100644 --- a/src/yggdrasil/util.go +++ b/src/yggdrasil/util.go @@ -6,14 +6,17 @@ import "runtime" //import "sync" +// A wrapper around runtime.Gosched() so it doesn't need to be imported elsewhere. func util_yield() { runtime.Gosched() } +// A wrapper around runtime.LockOSThread() so it doesn't need to be imported elsewhere. func util_lockthread() { runtime.LockOSThread() } +// A wrapper around runtime.UnlockOSThread() so it doesn't need to be imported elsewhere. func util_unlockthread() { runtime.UnlockOSThread() } @@ -32,14 +35,18 @@ func util_putBytes(bs []byte) { } */ +// This is used to buffer recently used slices of bytes, to prevent allocations in the hot loops. +// It's used like a sync.Pool, but with a fixed size and typechecked without type casts to/from interface{} (which were making the profiles look ugly). var byteStore chan []byte +// Initializes the byteStore func util_initByteStore() { if byteStore == nil { byteStore = make(chan []byte, 32) } } +// Gets an empty slice from the byte store, if one is available, or else returns a new nil slice. func util_getBytes() []byte { select { case bs := <-byteStore: @@ -49,6 +56,7 @@ func util_getBytes() []byte { } } +// Puts a slice in the store, if there's room, or else returns and lets the slice get collected. func util_putBytes(bs []byte) { select { case byteStore <- bs: diff --git a/src/yggdrasil/version.go b/src/yggdrasil/version.go index 40a5b965..75b08a6f 100644 --- a/src/yggdrasil/version.go +++ b/src/yggdrasil/version.go @@ -4,6 +4,9 @@ package yggdrasil // Used in the inital connection setup and key exchange // Some of this could arguably go in wire.go instead +// This is the version-specific metadata exchanged at the start of a connection. +// It must always beign with the 4 bytes "meta" and a wire formatted uint64 major version number. +// The current version also includes a minor version number, and the box/sig/link keys that need to be exchanged to open an connection. type version_metadata struct { meta [4]byte ver uint64 // 1 byte in this version @@ -14,6 +17,7 @@ type version_metadata struct { link boxPubKey } +// Gets a base metadata with no keys set, but with the correct version numbers. func version_getBaseMetadata() version_metadata { return version_metadata{ meta: [4]byte{'m', 'e', 't', 'a'}, @@ -22,16 +26,18 @@ func version_getBaseMetadata() version_metadata { } } +// Gest the length of the metadata for this version, used to know how many bytes to read from the start of a connection. func version_getMetaLength() (mlen int) { mlen += 4 // meta - mlen += 1 // ver - mlen += 1 // minorVer + mlen += 1 // ver, as long as it's < 127, which it is in this version + mlen += 1 // minorVer, as long as it's < 127, which it is in this version mlen += boxPubKeyLen // box mlen += sigPubKeyLen // sig mlen += boxPubKeyLen // link return } +// Encodes version metadata into its wire format. func (m *version_metadata) encode() []byte { bs := make([]byte, 0, version_getMetaLength()) bs = append(bs, m.meta[:]...) @@ -46,6 +52,7 @@ func (m *version_metadata) encode() []byte { return bs } +// Decodes version metadata from its wire format into the struct. func (m *version_metadata) decode(bs []byte) bool { switch { case !wire_chop_slice(m.meta[:], &bs): @@ -64,6 +71,7 @@ func (m *version_metadata) decode(bs []byte) bool { return true } +// Checks that the "meta" bytes and the version numbers are the expected values. func (m *version_metadata) check() bool { base := version_getBaseMetadata() return base.meta == m.meta && base.ver == m.ver && base.minorVer == m.minorVer diff --git a/src/yggdrasil/wire.go b/src/yggdrasil/wire.go index 23e9ab3d..e92b4fcf 100644 --- a/src/yggdrasil/wire.go +++ b/src/yggdrasil/wire.go @@ -5,9 +5,8 @@ package yggdrasil // TODO clean up unused/commented code, and add better comments to whatever is left -// Packet types, as an Encode_uint64 at the start of each packet -// TODO? make things still work after reordering (after things stabilize more?) -// Type safety would also be nice, `type wire_type uint64`, rewrite as needed? +// Packet types, as wire_encode_uint64(type) at the start of each packet + const ( wire_Traffic = iota // data being routed somewhere, handle for crypto wire_ProtocolTraffic // protocol traffic, pub keys for crypto @@ -19,13 +18,13 @@ const ( wire_DHTLookupResponse // inside protocol traffic header ) -// Encode uint64 using a variable length scheme -// Similar to binary.Uvarint, but big-endian +// Calls wire_put_uint64 on a nil slice. func wire_encode_uint64(elem uint64) []byte { return wire_put_uint64(elem, nil) } -// Occasionally useful for appending to an existing slice (if there's room) +// Encode uint64 using a variable length scheme. +// Similar to binary.Uvarint, but big-endian. func wire_put_uint64(elem uint64, out []byte) []byte { bs := make([]byte, 0, 10) bs = append(bs, byte(elem&0x7f)) @@ -41,6 +40,7 @@ func wire_put_uint64(elem uint64, out []byte) []byte { return append(out, bs...) } +// Returns the length of a wire encoded uint64 of this value. func wire_uint64_len(elem uint64) int { l := 1 for e := elem >> 7; e > 0; e >>= 7 { @@ -49,8 +49,8 @@ func wire_uint64_len(elem uint64) int { return l } -// Decode uint64 from a []byte slice -// Returns the decoded uint64 and the number of bytes used +// Decode uint64 from a []byte slice. +// Returns the decoded uint64 and the number of bytes used. func wire_decode_uint64(bs []byte) (uint64, int) { length := 0 elem := uint64(0) @@ -65,20 +65,22 @@ func wire_decode_uint64(bs []byte) (uint64, int) { return elem, length } +// Converts an int64 into uint64 so it can be written to the wire. +// Non-negative integers are mapped to even integers: 0 -> 0, 1 -> 2, etc. +// Negative integres are mapped to odd integes: -1 -> 1, -2 -> 3, etc. +// This means the least significant bit is a sign bit. func wire_intToUint(i int64) uint64 { - // Non-negative integers mapped to even integers: 0 -> 0, 1 -> 2, etc. - // Negative integres mapped to odd integes: -1 -> 1, -2 -> 3, etc. - // This means the least significant bit is a sign bit. return ((uint64(-(i+1))<<1)|0x01)*(uint64(i)>>63) + (uint64(i)<<1)*(^uint64(i)>>63) } +// Converts uint64 back to int64, genreally when being read from the wire. func wire_intFromUint(u uint64) int64 { return int64(u&0x01)*(-int64(u>>1)-1) + int64(^u&0x01)*int64(u>>1) } //////////////////////////////////////////////////////////////////////////////// -// Takes coords, returns coords prefixed with encoded coord length +// Takes coords, returns coords prefixed with encoded coord length. func wire_encode_coords(coords []byte) []byte { coordLen := wire_encode_uint64(uint64(len(coords))) bs := make([]byte, 0, len(coordLen)+len(coords)) @@ -87,14 +89,17 @@ func wire_encode_coords(coords []byte) []byte { return bs } +// Puts a length prefix and the coords into bs, returns the wire formatted coords. +// Useful in hot loops where we don't want to allocate and we know the rest of the later parts of the slice are safe to overwrite. func wire_put_coords(coords []byte, bs []byte) []byte { bs = wire_put_uint64(uint64(len(coords)), bs) bs = append(bs, coords...) return bs } -// Takes a packet that begins with coords (starting with coord length) -// Returns a slice of coords and the number of bytes read +// Takes a slice that begins with coords (starting with coord length). +// Returns a slice of coords and the number of bytes read. +// Used as part of various decode() functions for structs. func wire_decode_coords(packet []byte) ([]byte, int) { coordLen, coordBegin := wire_decode_uint64(packet) coordEnd := coordBegin + int(coordLen) @@ -106,6 +111,7 @@ func wire_decode_coords(packet []byte) ([]byte, int) { //////////////////////////////////////////////////////////////////////////////// +// Encodes a swtichMsg into its wire format. func (m *switchMsg) encode() []byte { bs := wire_encode_uint64(wire_SwitchMsg) bs = append(bs, m.Root[:]...) @@ -118,6 +124,7 @@ func (m *switchMsg) encode() []byte { return bs } +// Decodes a wire formatted switchMsg into the struct, returns true if successful. func (m *switchMsg) decode(bs []byte) bool { var pType uint64 var tstamp uint64 @@ -149,6 +156,7 @@ func (m *switchMsg) decode(bs []byte) bool { //////////////////////////////////////////////////////////////////////////////// +// A utility function used to copy bytes into a slice and advance the beginning of the source slice, returns true if successful. func wire_chop_slice(toSlice []byte, fromSlice *[]byte) bool { if len(*fromSlice) < len(toSlice) { return false @@ -158,6 +166,7 @@ func wire_chop_slice(toSlice []byte, fromSlice *[]byte) bool { return true } +// A utility function to extract coords from a slice and advance the source slices, returning true if successful. func wire_chop_coords(toCoords *[]byte, fromSlice *[]byte) bool { coords, coordLen := wire_decode_coords(*fromSlice) if coordLen == 0 { @@ -168,6 +177,7 @@ func wire_chop_coords(toCoords *[]byte, fromSlice *[]byte) bool { return true } +// A utility function to extract a wire encoded uint64 into the provided pointer while advancing the start of the source slice, returning true if successful. func wire_chop_uint64(toUInt64 *uint64, fromSlice *[]byte) bool { dec, decLen := wire_decode_uint64(*fromSlice) if decLen == 0 { @@ -182,6 +192,7 @@ func wire_chop_uint64(toUInt64 *uint64, fromSlice *[]byte) bool { // Wire traffic packets +// The wire format for ordinary IPv6 traffic encapsulated by the network. type wire_trafficPacket struct { Coords []byte Handle handle @@ -189,7 +200,7 @@ type wire_trafficPacket struct { Payload []byte } -// This is basically MarshalBinary, but decode doesn't allow that... +// Encodes a wire_trafficPacket into its wire format. func (p *wire_trafficPacket) encode() []byte { bs := util_getBytes() bs = wire_put_uint64(wire_Traffic, bs) @@ -200,7 +211,7 @@ func (p *wire_trafficPacket) encode() []byte { return bs } -// Not just UnmarshalBinary becuase the original slice isn't always copied from +// Decodes an encoded wire_trafficPacket into the struct, returning true if successful. func (p *wire_trafficPacket) decode(bs []byte) bool { var pType uint64 switch { @@ -219,6 +230,7 @@ func (p *wire_trafficPacket) decode(bs []byte) bool { return true } +// The wire format for protocol traffic, such as dht req/res or session ping/pong packets. type wire_protoTrafficPacket struct { Coords []byte ToKey boxPubKey @@ -227,6 +239,7 @@ type wire_protoTrafficPacket struct { Payload []byte } +// Encodes a wire_protoTrafficPacket into its wire format. func (p *wire_protoTrafficPacket) encode() []byte { coords := wire_encode_coords(p.Coords) bs := wire_encode_uint64(wire_ProtocolTraffic) @@ -238,6 +251,7 @@ func (p *wire_protoTrafficPacket) encode() []byte { return bs } +// Decodes an encoded wire_protoTrafficPacket into the struct, returning true if successful. func (p *wire_protoTrafficPacket) decode(bs []byte) bool { var pType uint64 switch { @@ -258,11 +272,16 @@ func (p *wire_protoTrafficPacket) decode(bs []byte) bool { return true } +// The wire format for link protocol traffic, namely switchMsg. +// There's really two layers of this, with the outer layer using permanent keys, and the inner layer using ephemeral keys. +// The keys themselves are exchanged as part of the connection setup, and then omitted from the packets. +// The two layer logic is handled in peers.go, but it's kind of ugly. type wire_linkProtoTrafficPacket struct { Nonce boxNonce Payload []byte } +// Encodes a wire_linkProtoTrafficPacket into its wire format. func (p *wire_linkProtoTrafficPacket) encode() []byte { bs := wire_encode_uint64(wire_LinkProtocolTraffic) bs = append(bs, p.Nonce[:]...) @@ -270,6 +289,7 @@ func (p *wire_linkProtoTrafficPacket) encode() []byte { return bs } +// Decodes an encoded wire_linkProtoTrafficPacket into the struct, returning true if successful. func (p *wire_linkProtoTrafficPacket) decode(bs []byte) bool { var pType uint64 switch { @@ -286,6 +306,7 @@ func (p *wire_linkProtoTrafficPacket) decode(bs []byte) bool { //////////////////////////////////////////////////////////////////////////////// +// Encodes a sessionPing into its wire format. func (p *sessionPing) encode() []byte { var pTypeVal uint64 if p.IsPong { @@ -304,6 +325,7 @@ func (p *sessionPing) encode() []byte { return bs } +// Decodes an encoded sessionPing into the struct, returning true if successful. func (p *sessionPing) decode(bs []byte) bool { var pType uint64 var tstamp uint64 @@ -335,6 +357,7 @@ func (p *sessionPing) decode(bs []byte) bool { //////////////////////////////////////////////////////////////////////////////// +// Encodes a dhtReq into its wire format. func (r *dhtReq) encode() []byte { coords := wire_encode_coords(r.Coords) bs := wire_encode_uint64(wire_DHTLookupRequest) @@ -343,6 +366,7 @@ func (r *dhtReq) encode() []byte { return bs } +// Decodes an encoded dhtReq into the struct, returning true if successful. func (r *dhtReq) decode(bs []byte) bool { var pType uint64 switch { @@ -359,6 +383,7 @@ func (r *dhtReq) decode(bs []byte) bool { } } +// Encodes a dhtRes into its wire format. func (r *dhtRes) encode() []byte { coords := wire_encode_coords(r.Coords) bs := wire_encode_uint64(wire_DHTLookupResponse) @@ -372,6 +397,7 @@ func (r *dhtRes) encode() []byte { return bs } +// Decodes an encoded dhtRes into the struct, returning true if successful. func (r *dhtRes) decode(bs []byte) bool { var pType uint64 switch { From 359af66d0d1ff6bf39a2233003785ae7f8f5d422 Mon Sep 17 00:00:00 2001 From: Arceliar Date: Tue, 12 Jun 2018 03:16:10 -0500 Subject: [PATCH 37/42] exponential dht throttle backoff, and make it based on when packets were sent as part of bootstrapping/maintenance, not when arbitrary packets were received --- src/yggdrasil/dht.go | 64 ++++++++++++++++++++++++++++++-------------- 1 file changed, 44 insertions(+), 20 deletions(-) diff --git a/src/yggdrasil/dht.go b/src/yggdrasil/dht.go index 3d341e71..dcd9a93d 100644 --- a/src/yggdrasil/dht.go +++ b/src/yggdrasil/dht.go @@ -41,10 +41,11 @@ type dhtInfo struct { nodeID_hidden *NodeID key boxPubKey coords []byte - send time.Time // When we last sent a message - recv time.Time // When we last received a message - pings int // Decide when to drop - throttle uint8 // Number of seconds to wait before pinging a node to bootstrap buckets, gradually increases up to 1 minute + send time.Time // When we last sent a message + recv time.Time // When we last received a message + pings int // Decide when to drop + throttle time.Duration // Time to wait before pinging a node to bootstrap buckets, increases exponentially from 1 second to 1 minute + bootstrapSend time.Time // The time checked/updated as part of throttle checks } // Returns the *NodeID associated with dhtInfo.key, calculating it on the fly the first time or from a cache all subsequent times. @@ -141,12 +142,14 @@ func (t *dht) handleRes(res *dhtRes) { if !isIn { return } + now := time.Now() rinfo := dhtInfo{ - key: res.Key, - coords: res.Coords, - send: time.Now(), // Technically wrong but should be OK... - recv: time.Now(), - throttle: 1, + key: res.Key, + coords: res.Coords, + send: now, // Technically wrong but should be OK... + recv: now, + throttle: time.Second, + bootstrapSend: now, } // If they're already in the table, then keep the correct send time bidx, isOK := t.getBucketIndex(rinfo.getNodeID()) @@ -157,13 +160,15 @@ func (t *dht) handleRes(res *dhtRes) { for _, oldinfo := range b.peers { if oldinfo.key == rinfo.key { rinfo.send = oldinfo.send - rinfo.throttle += oldinfo.throttle + rinfo.throttle = oldinfo.throttle + rinfo.bootstrapSend = oldinfo.bootstrapSend } } for _, oldinfo := range b.other { if oldinfo.key == rinfo.key { rinfo.send = oldinfo.send - rinfo.throttle += oldinfo.throttle + rinfo.throttle = oldinfo.throttle + rinfo.bootstrapSend = oldinfo.bootstrapSend } } // Insert into table @@ -266,8 +271,8 @@ func (t *dht) insert(info *dhtInfo, isPeer bool) { // This speeds up bootstrapping info.recv = info.recv.Add(-time.Hour) } - if isPeer || info.throttle > 60 { - info.throttle = 60 + if isPeer || info.throttle > time.Minute { + info.throttle = time.Minute } // First drop any existing entry from the bucket b.drop(&info.key) @@ -512,20 +517,39 @@ func (t *dht) doMaintenance() { } } if oldest != nil && time.Since(oldest.recv) > time.Minute { + // Ping the oldest node in the DHT, but don't ping nodes that have been checked within the last minute t.addToMill(oldest, nil) - } // if the DHT isn't empty + } // Refresh buckets if t.offset > last { t.offset = 0 } target := t.getTarget(t.offset) - for _, info := range t.lookup(target, true) { - if time.Since(info.recv) > time.Duration(info.throttle)*time.Second { - t.addToMill(info, target) - t.offset++ - break + func() { + closer := t.lookup(target, false) + for _, info := range closer { + // Throttled ping of a node that's closer to the destination + if time.Since(info.recv) > info.throttle { + t.addToMill(info, target) + t.offset++ + info.bootstrapSend = time.Now() + info.throttle *= 2 + if info.throttle > time.Minute { + info.throttle = time.Minute + } + return + } } - } + if len(closer) == 0 { + // If we don't know of anyone closer at all, then there's a hole in our dht + // Ping the closest node we know and ignore the throttle, to try to fill it + for _, info := range t.lookup(target, true) { + t.addToMill(info, target) + t.offset++ + return + } + } + }() //t.offset++ } for len(t.rumorMill) > 0 { From 24228bd381a669823c5717d20f8295ed6a552200 Mon Sep 17 00:00:00 2001 From: Arceliar Date: Tue, 12 Jun 2018 14:00:04 -0500 Subject: [PATCH 38/42] disable unused configuration Net field and clean up comments to remove mention of UDP where no longer applicable --- src/yggdrasil/config/config.go | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/src/yggdrasil/config/config.go b/src/yggdrasil/config/config.go index e8af6275..2b8fbe3b 100644 --- a/src/yggdrasil/config/config.go +++ b/src/yggdrasil/config/config.go @@ -2,19 +2,19 @@ package config // NodeConfig defines all configuration values needed to run a signle yggdrasil node type NodeConfig struct { - Listen string `comment:"Listen address for peer connections. Default is to listen for all\nUDP and TCP connections over IPv4 and IPv6."` - AdminListen string `comment:"Listen address for admin connections Default is to listen for local\nconnections only on TCP port 9001."` - Peers []string `comment:"List of connection strings for static peers in URI format, i.e.\ntcp://a.b.c.d:e, udp://a.b.c.d:e, socks://a.b.c.d:e/f.g.h.i:j etc."` - AllowedEncryptionPublicKeys []string `comment:"List of peer encryption public keys to allow incoming/outgoing UDP\nor incoming TCP connections from. If left empty/undefined then all\nconnections will be allowed by default."` - EncryptionPublicKey string `comment:"Your public encryption key. Your peers may ask you for this to put\ninto their AllowedEncryptionPublicKeys configuration."` - EncryptionPrivateKey string `comment:"Your private encryption key. DO NOT share this with anyone!"` - SigningPublicKey string `comment:"Your public signing key. You should not ordinarily need to share\nthis with anyone."` - SigningPrivateKey string `comment:"Your private signing key. DO NOT share this with anyone!"` - MulticastInterfaces []string `comment:"Regular expressions for which interfaces multicast peer discovery\nshould be enabled on. If none specified, multicast peer discovery is\ndisabled. The default value is .* which uses all interfaces."` - IfName string `comment:"Local network interface name for TUN/TAP adapter, or \"auto\" to select\nan interface automatically, or \"none\" to run without TUN/TAP."` - IfTAPMode bool `comment:"Set local network interface to TAP mode rather than TUN mode if\nsupported by your platform - option will be ignored if not."` - IfMTU int `comment:"Maximux Transmission Unit (MTU) size for your local TUN/TAP interface.\nDefault is the largest supported size for your platform. The lowest\npossible value is 1280."` - Net NetConfig `comment:"Extended options for connecting to peers over other networks."` + Listen string `comment:"Listen address for peer connections. Default is to listen for all\nTCP connections over IPv4 and IPv6 with a random port."` + AdminListen string `comment:"Listen address for admin connections Default is to listen for local\nconnections only on TCP port 9001."` + Peers []string `comment:"List of connection strings for static peers in URI format, i.e.\ntcp://a.b.c.d:e or socks://a.b.c.d:e/f.g.h.i:j"` + AllowedEncryptionPublicKeys []string `comment:"List of peer encryption public keys to allow or incoming TCP\nconnections from. If left empty/undefined then all connections\nwill be allowed by default."` + EncryptionPublicKey string `comment:"Your public encryption key. Your peers may ask you for this to put\ninto their AllowedEncryptionPublicKeys configuration."` + EncryptionPrivateKey string `comment:"Your private encryption key. DO NOT share this with anyone!"` + SigningPublicKey string `comment:"Your public signing key. You should not ordinarily need to share\nthis with anyone."` + SigningPrivateKey string `comment:"Your private signing key. DO NOT share this with anyone!"` + MulticastInterfaces []string `comment:"Regular expressions for which interfaces multicast peer discovery\nshould be enabled on. If none specified, multicast peer discovery is\ndisabled. The default value is .* which uses all interfaces."` + IfName string `comment:"Local network interface name for TUN/TAP adapter, or \"auto\" to select\nan interface automatically, or \"none\" to run without TUN/TAP."` + IfTAPMode bool `comment:"Set local network interface to TAP mode rather than TUN mode if\nsupported by your platform - option will be ignored if not."` + IfMTU int `comment:"Maximux Transmission Unit (MTU) size for your local TUN/TAP interface.\nDefault is the largest supported size for your platform. The lowest\npossible value is 1280."` + //Net NetConfig `comment:"Extended options for connecting to peers over other networks."` } // NetConfig defines network/proxy related configuration values From 8e2c2aa9772406934049cb9b7f62ce97c09e7e12 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Tue, 12 Jun 2018 22:45:53 +0100 Subject: [PATCH 39/42] Document ICMPv6 and TUN/TAP --- src/yggdrasil/icmpv6.go | 38 +++++++++++++++++++++++++++++++++--- src/yggdrasil/tun.go | 19 ++++++++++++++++++ src/yggdrasil/tun_bsd.go | 5 +++++ src/yggdrasil/tun_darwin.go | 5 +++++ src/yggdrasil/tun_freebsd.go | 2 ++ src/yggdrasil/tun_linux.go | 8 +++++++- src/yggdrasil/tun_netbsd.go | 2 ++ src/yggdrasil/tun_openbsd.go | 2 ++ src/yggdrasil/tun_other.go | 6 ++++++ src/yggdrasil/tun_windows.go | 7 +++++++ 10 files changed, 90 insertions(+), 4 deletions(-) diff --git a/src/yggdrasil/icmpv6.go b/src/yggdrasil/icmpv6.go index fa14b2ac..f3b00ffc 100644 --- a/src/yggdrasil/icmpv6.go +++ b/src/yggdrasil/icmpv6.go @@ -1,8 +1,13 @@ package yggdrasil -// The NDP functions are needed when you are running with a -// TAP adapter - as the operating system expects neighbor solicitations -// for on-link traffic, this goroutine provides them +// The ICMPv6 module implements functions to easily create ICMPv6 +// packets. These functions, when mixed with the built-in Go IPv6 +// and ICMP libraries, can be used to send control messages back +// to the host. Examples include: +// - NDP messages, when running in TAP mode +// - Packet Too Big messages, when packets exceed the session MTU +// - Destination Unreachable messages, when a session prohibits +// incoming traffic import "net" import "golang.org/x/net/ipv6" @@ -39,6 +44,9 @@ func ipv6Header_Marshal(h *ipv6.Header) ([]byte, error) { return b, nil } +// Initialises the ICMPv6 module by assigning our link-local IPv6 address and +// our MAC address. ICMPv6 messages will always appear to originate from these +// addresses. func (i *icmpv6) init(t *tunDevice) { i.tun = t @@ -50,6 +58,10 @@ func (i *icmpv6) init(t *tunDevice) { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0xFE} } +// Parses an incoming ICMPv6 packet. The packet provided may be either an +// ethernet frame containing an IP packet, or the IP packet alone. This is +// determined by whether the TUN/TAP adapter is running in TUN (layer 3) or +// TAP (layer 2) mode. func (i *icmpv6) parse_packet(datain []byte) { var response []byte var err error @@ -69,6 +81,10 @@ func (i *icmpv6) parse_packet(datain []byte) { i.tun.iface.Write(response) } +// Unwraps the ethernet headers of an incoming ICMPv6 packet and hands off +// the IP packet to the parse_packet_tun function for further processing. +// A response buffer is also created for the response message, also complete +// with ethernet headers. func (i *icmpv6) parse_packet_tap(datain []byte) ([]byte, error) { // Store the peer MAC address copy(i.peermac[:6], datain[6:12]) @@ -97,6 +113,10 @@ func (i *icmpv6) parse_packet_tap(datain []byte) ([]byte, error) { return dataout, nil } +// Unwraps the IP headers of an incoming IPv6 packet and performs various +// sanity checks on the packet - i.e. is the packet an ICMPv6 packet, does the +// ICMPv6 message match a known expected type. The relevant handler function +// is then called and a response packet may be returned. func (i *icmpv6) parse_packet_tun(datain []byte) ([]byte, error) { // Parse the IPv6 packet headers ipv6Header, err := ipv6.ParseHeader(datain[:ipv6.HeaderLen]) @@ -149,6 +169,9 @@ func (i *icmpv6) parse_packet_tun(datain []byte) ([]byte, error) { return nil, errors.New("ICMPv6 type not matched") } +// Creates an ICMPv6 packet based on the given icmp.MessageBody and other +// parameters, complete with ethernet and IP headers, which can be written +// directly to a TAP adapter. func (i *icmpv6) create_icmpv6_tap(dstmac macAddress, dst net.IP, src net.IP, mtype ipv6.ICMPType, mcode int, mbody icmp.MessageBody) ([]byte, error) { // Pass through to create_icmpv6_tun ipv6packet, err := i.create_icmpv6_tun(dst, src, mtype, mcode, mbody) @@ -169,6 +192,10 @@ func (i *icmpv6) create_icmpv6_tap(dstmac macAddress, dst net.IP, src net.IP, mt return dataout, nil } +// Creates an ICMPv6 packet based on the given icmp.MessageBody and other +// parameters, complete with IP headers only, which can be written directly to +// a TUN adapter, or called directly by the create_icmpv6_tap function when +// generating a message for TAP adapters. func (i *icmpv6) create_icmpv6_tun(dst net.IP, src net.IP, mtype ipv6.ICMPType, mcode int, mbody icmp.MessageBody) ([]byte, error) { // Create the ICMPv6 message icmpMessage := icmp.Message{ @@ -208,6 +235,11 @@ func (i *icmpv6) create_icmpv6_tun(dst net.IP, src net.IP, mtype ipv6.ICMPType, return responsePacket, nil } +// Generates a response to an NDP discovery packet. This is effectively called +// when the host operating system generates an NDP request for any address in +// the fd00::/8 range, so that the operating system knows to route that traffic +// to the Yggdrasil TAP adapter. +// TODO: Make this respect the value of address_prefix in address.go func (i *icmpv6) handle_ndp(in []byte) ([]byte, error) { // Ignore NDP requests for anything outside of fd00::/8 if in[8] != 0xFD { diff --git a/src/yggdrasil/tun.go b/src/yggdrasil/tun.go index ca94713a..b29aabef 100644 --- a/src/yggdrasil/tun.go +++ b/src/yggdrasil/tun.go @@ -8,6 +8,7 @@ import "github.com/yggdrasil-network/water" const tun_IPv6_HEADER_LENGTH = 40 const tun_ETHER_HEADER_LENGTH = 14 +// Represents a running TUN/TAP interface. type tunDevice struct { core *Core icmpv6 icmpv6 @@ -17,6 +18,9 @@ type tunDevice struct { iface *water.Interface } +// Defines which parameters are expected by default for a TUN/TAP adapter on a +// specific platform. These values are populated in the relevant tun_*.go for +// the platform being targeted. They must be set. type tunDefaultParameters struct { maximumIfMTU int defaultIfMTU int @@ -24,6 +28,8 @@ type tunDefaultParameters struct { defaultIfTAPMode bool } +// Gets the maximum supported MTU for the platform based on the defaults in +// getDefaults(). func getSupportedMTU(mtu int) int { if mtu > getDefaults().maximumIfMTU { return getDefaults().maximumIfMTU @@ -31,11 +37,14 @@ func getSupportedMTU(mtu int) int { return mtu } +// Initialises the TUN/TAP adapter. func (tun *tunDevice) init(core *Core) { tun.core = core tun.icmpv6.init(tun) } +// Starts the setup process for the TUN/TAP adapter, and if successful, starts +// the read/write goroutines to handle packets on that interface. func (tun *tunDevice) start(ifname string, iftapmode bool, addr string, mtu int) error { if ifname == "none" { return nil @@ -48,6 +57,9 @@ func (tun *tunDevice) start(ifname string, iftapmode bool, addr string, mtu int) return nil } +// Writes a packet to the TUN/TAP adapter. If the adapter is running in TAP +// mode then additional ethernet encapsulation is added for the benefit of the +// host operating system. func (tun *tunDevice) write() error { for { data := <-tun.recv @@ -75,6 +87,10 @@ func (tun *tunDevice) write() error { } } +// Reads any packets that are waiting on the TUN/TAP adapter. If the adapter +// is running in TAP mode then the ethernet headers will automatically be +// processed and stripped if necessary. If an ICMPv6 packet is found, then +// the relevant helper functions in icmpv6.go are called. func (tun *tunDevice) read() error { mtu := tun.mtu if tun.iface.IsTAP() { @@ -109,6 +125,9 @@ func (tun *tunDevice) read() error { } } +// Closes the TUN/TAP adapter. This is only usually called when the Yggdrasil +// process stops. Typically this operation will happen quickly, but on macOS +// it can block until a read operation is completed. func (tun *tunDevice) close() error { if tun.iface == nil { return nil diff --git a/src/yggdrasil/tun_bsd.go b/src/yggdrasil/tun_bsd.go index 721b6778..9455d5d2 100644 --- a/src/yggdrasil/tun_bsd.go +++ b/src/yggdrasil/tun_bsd.go @@ -70,6 +70,11 @@ type in6_ifreq_lifetime struct { ifru_addrlifetime in6_addrlifetime } +// Sets the IPv6 address of the utun adapter. On all BSD platforms (FreeBSD, +// OpenBSD, NetBSD) an attempt is made to set the adapter properties by using +// a system socket and making syscalls to the kernel. This is not refined though +// and often doesn't work (if at all), therefore if a call fails, it resorts +// to calling "ifconfig" instead. func (tun *tunDevice) setup(ifname string, iftapmode bool, addr string, mtu int) error { var config water.Config if ifname[:4] == "auto" { diff --git a/src/yggdrasil/tun_darwin.go b/src/yggdrasil/tun_darwin.go index 6096d6ac..4211fe7d 100644 --- a/src/yggdrasil/tun_darwin.go +++ b/src/yggdrasil/tun_darwin.go @@ -10,6 +10,8 @@ import "golang.org/x/sys/unix" import water "github.com/yggdrasil-network/water" +// Sane defaults for the Darwin/macOS platform. The "default" options may be +// may be replaced by the running configuration. func getDefaults() tunDefaultParameters { return tunDefaultParameters{ maximumIfMTU: 65535, @@ -19,6 +21,7 @@ func getDefaults() tunDefaultParameters { } } +// Configures the "utun" adapter with the correct IPv6 address and MTU. func (tun *tunDevice) setup(ifname string, iftapmode bool, addr string, mtu int) error { if iftapmode { tun.core.log.Printf("TAP mode is not supported on this platform, defaulting to TUN") @@ -65,6 +68,8 @@ type ifreq struct { ifru_mtu uint32 } +// Sets the IPv6 address of the utun adapter. On Darwin/macOS this is done using +// a system socket and making direct syscalls to the kernel. func (tun *tunDevice) setupAddress(addr string) error { var fd int var err error diff --git a/src/yggdrasil/tun_freebsd.go b/src/yggdrasil/tun_freebsd.go index dd9a49bd..4cfdcee1 100644 --- a/src/yggdrasil/tun_freebsd.go +++ b/src/yggdrasil/tun_freebsd.go @@ -1,5 +1,7 @@ package yggdrasil +// Sane defaults for the FreeBSD platform. The "default" options may be +// may be replaced by the running configuration. func getDefaults() tunDefaultParameters { return tunDefaultParameters{ maximumIfMTU: 32767, diff --git a/src/yggdrasil/tun_linux.go b/src/yggdrasil/tun_linux.go index 42ff4a4a..d038d4e8 100644 --- a/src/yggdrasil/tun_linux.go +++ b/src/yggdrasil/tun_linux.go @@ -1,7 +1,6 @@ package yggdrasil // The linux platform specific tun parts -// It depends on iproute2 being installed to set things on the tun device import "errors" import "fmt" @@ -11,6 +10,8 @@ import water "github.com/yggdrasil-network/water" import "github.com/docker/libcontainer/netlink" +// Sane defaults for the Linux platform. The "default" options may be +// may be replaced by the running configuration. func getDefaults() tunDefaultParameters { return tunDefaultParameters{ maximumIfMTU: 65535, @@ -20,6 +21,7 @@ func getDefaults() tunDefaultParameters { } } +// Configures the TAP adapter with the correct IPv6 address and MTU. func (tun *tunDevice) setup(ifname string, iftapmode bool, addr string, mtu int) error { var config water.Config if iftapmode { @@ -39,6 +41,10 @@ func (tun *tunDevice) setup(ifname string, iftapmode bool, addr string, mtu int) return tun.setupAddress(addr) } +// Configures the TAP adapter with the correct IPv6 address and MTU. Netlink +// is used to do this, so there is not a hard requirement on "ip" or "ifconfig" +// to exist on the system, but this will fail if Netlink is not present in the +// kernel (it nearly always is). func (tun *tunDevice) setupAddress(addr string) error { // Set address var netIF *net.Interface diff --git a/src/yggdrasil/tun_netbsd.go b/src/yggdrasil/tun_netbsd.go index f6d34ee3..d3e93c4d 100644 --- a/src/yggdrasil/tun_netbsd.go +++ b/src/yggdrasil/tun_netbsd.go @@ -1,5 +1,7 @@ package yggdrasil +// Sane defaults for the NetBSD platform. The "default" options may be +// may be replaced by the running configuration. func getDefaults() tunDefaultParameters { return tunDefaultParameters{ maximumIfMTU: 9000, diff --git a/src/yggdrasil/tun_openbsd.go b/src/yggdrasil/tun_openbsd.go index bc75f0c6..c96c8658 100644 --- a/src/yggdrasil/tun_openbsd.go +++ b/src/yggdrasil/tun_openbsd.go @@ -1,5 +1,7 @@ package yggdrasil +// Sane defaults for the OpenBSD platform. The "default" options may be +// may be replaced by the running configuration. func getDefaults() tunDefaultParameters { return tunDefaultParameters{ maximumIfMTU: 16384, diff --git a/src/yggdrasil/tun_other.go b/src/yggdrasil/tun_other.go index 339faba4..7bc7100c 100644 --- a/src/yggdrasil/tun_other.go +++ b/src/yggdrasil/tun_other.go @@ -7,6 +7,8 @@ import water "github.com/yggdrasil-network/water" // This is to catch unsupported platforms // If your platform supports tun devices, you could try configuring it manually +// These are sane defaults for any platform that has not been matched by one of +// the other tun_*.go files. func getDefaults() tunDefaultParameters { return tunDefaultParameters{ maximumIfMTU: 65535, @@ -16,6 +18,8 @@ func getDefaults() tunDefaultParameters { } } +// Creates the TUN/TAP adapter, if supported by the Water library. Note that +// no guarantees are made at this point on an unsupported platform. func (tun *tunDevice) setup(ifname string, iftapmode bool, addr string, mtu int) error { var config water.Config if iftapmode { @@ -32,6 +36,8 @@ func (tun *tunDevice) setup(ifname string, iftapmode bool, addr string, mtu int) return tun.setupAddress(addr) } +// We don't know how to set the IPv6 address on an unknown platform, therefore +// write about it to stdout and don't try to do anything further. func (tun *tunDevice) setupAddress(addr string) error { tun.core.log.Println("Platform not supported, you must set the address of", tun.iface.Name(), "to", addr) return nil diff --git a/src/yggdrasil/tun_windows.go b/src/yggdrasil/tun_windows.go index d4a15f93..644cfdb4 100644 --- a/src/yggdrasil/tun_windows.go +++ b/src/yggdrasil/tun_windows.go @@ -7,6 +7,8 @@ import "fmt" // This is to catch Windows platforms +// Sane defaults for the Windows platform. The "default" options may be +// may be replaced by the running configuration. func getDefaults() tunDefaultParameters { return tunDefaultParameters{ maximumIfMTU: 65535, @@ -16,6 +18,9 @@ func getDefaults() tunDefaultParameters { } } +// Configures the TAP adapter with the correct IPv6 address and MTU. On Windows +// we don't make use of a direct operating system API to do this - we instead +// delegate the hard work to "netsh". func (tun *tunDevice) setup(ifname string, iftapmode bool, addr string, mtu int) error { if !iftapmode { tun.core.log.Printf("TUN mode is not supported on this platform, defaulting to TAP") @@ -63,6 +68,7 @@ func (tun *tunDevice) setup(ifname string, iftapmode bool, addr string, mtu int) return tun.setupAddress(addr) } +// Sets the MTU of the TAP adapter. func (tun *tunDevice) setupMTU(mtu int) error { // Set MTU cmd := exec.Command("netsh", "interface", "ipv6", "set", "subinterface", @@ -79,6 +85,7 @@ func (tun *tunDevice) setupMTU(mtu int) error { return nil } +// Sets the IPv6 address of the TAP adapter. func (tun *tunDevice) setupAddress(addr string) error { // Set address cmd := exec.Command("netsh", "interface", "ipv6", "add", "address", From b006748da441807c03ed0ddb486f464779640ffa Mon Sep 17 00:00:00 2001 From: Arceliar Date: Tue, 12 Jun 2018 17:50:08 -0500 Subject: [PATCH 40/42] code cleanup --- src/yggdrasil/admin.go | 29 +++++++++++++++-------------- src/yggdrasil/core.go | 17 ++++++++++------- src/yggdrasil/crypto.go | 13 +++++++------ src/yggdrasil/dht.go | 10 ++++------ src/yggdrasil/icmpv6.go | 23 ++++++++++++++++------- src/yggdrasil/multicast.go | 23 +++++++---------------- src/yggdrasil/peer.go | 22 +++++++++------------- src/yggdrasil/release.go | 6 ++++-- src/yggdrasil/router.go | 17 ++++++----------- src/yggdrasil/search.go | 8 ++++---- src/yggdrasil/session.go | 2 -- src/yggdrasil/signature.go | 6 ++++-- src/yggdrasil/switch.go | 22 ++++++---------------- src/yggdrasil/tcp.go | 22 +++++++++++----------- src/yggdrasil/tun.go | 8 +++++--- src/yggdrasil/tun_bsd.go | 20 +++++++++++--------- src/yggdrasil/tun_darwin.go | 17 ++++++++++------- src/yggdrasil/tun_linux.go | 12 +++++++----- src/yggdrasil/tun_windows.go | 11 +++++++---- src/yggdrasil/util.go | 16 ---------------- 20 files changed, 143 insertions(+), 161 deletions(-) diff --git a/src/yggdrasil/admin.go b/src/yggdrasil/admin.go index b32e20de..f09835f6 100644 --- a/src/yggdrasil/admin.go +++ b/src/yggdrasil/admin.go @@ -1,17 +1,19 @@ package yggdrasil -import "net" -import "os" -import "encoding/hex" -import "encoding/json" -import "errors" -import "fmt" -import "net/url" -import "sort" -import "strings" -import "strconv" -import "sync/atomic" -import "time" +import ( + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "net" + "net/url" + "os" + "sort" + "strconv" + "strings" + "sync/atomic" + "time" +) // TODO: Add authentication @@ -324,7 +326,6 @@ func (a *admin) handleRequest(conn net.Conn) { // Send the response back if err := encoder.Encode(&send); err != nil { - // fmt.Println("Admin socket JSON encode error:", err) return } @@ -640,7 +641,7 @@ func (a *admin) getResponse_dot() []byte { for _, info := range infos { keys = append(keys, info.key) } - // TODO sort + // sort less := func(i, j int) bool { return keys[i] < keys[j] } diff --git a/src/yggdrasil/core.go b/src/yggdrasil/core.go index 3b5fcc1a..e1da6fee 100644 --- a/src/yggdrasil/core.go +++ b/src/yggdrasil/core.go @@ -1,12 +1,15 @@ package yggdrasil -import "io/ioutil" -import "log" -import "regexp" -import "net" -import "fmt" -import "encoding/hex" -import "yggdrasil/config" +import ( + "encoding/hex" + "fmt" + "io/ioutil" + "log" + "net" + "regexp" + + "yggdrasil/config" +) // The Core object represents the Yggdrasil node. You should create a Core // object for each Yggdrasil node you plan to run. diff --git a/src/yggdrasil/crypto.go b/src/yggdrasil/crypto.go index 405c60d7..efca41d4 100644 --- a/src/yggdrasil/crypto.go +++ b/src/yggdrasil/crypto.go @@ -10,10 +10,13 @@ It also defines NodeID and TreeID as hashes of keys, and wraps hash functions */ -import "crypto/rand" -import "crypto/sha512" -import "golang.org/x/crypto/ed25519" -import "golang.org/x/crypto/nacl/box" +import ( + "crypto/rand" + "crypto/sha512" + + "golang.org/x/crypto/ed25519" + "golang.org/x/crypto/nacl/box" +) //////////////////////////////////////////////////////////////////////////////// @@ -121,7 +124,6 @@ func boxOpen(shared *boxSharedKey, boxed []byte, nonce *boxNonce) ([]byte, bool) { out := util_getBytes() - //return append(out, boxed...), true // XXX HACK to test without encryption s := (*[boxSharedKeyLen]byte)(shared) n := (*[boxNonceLen]byte)(nonce) unboxed, success := box.OpenAfterPrecomputation(out, boxed, n, s) @@ -134,7 +136,6 @@ func boxSeal(shared *boxSharedKey, unboxed []byte, nonce *boxNonce) ([]byte, *bo } nonce.update() out := util_getBytes() - //return append(out, unboxed...), nonce // XXX HACK to test without encryption s := (*[boxSharedKeyLen]byte)(shared) n := (*[boxNonceLen]byte)(nonce) boxed := box.SealAfterPrecomputation(out, unboxed, n, s) diff --git a/src/yggdrasil/dht.go b/src/yggdrasil/dht.go index dcd9a93d..4e53be31 100644 --- a/src/yggdrasil/dht.go +++ b/src/yggdrasil/dht.go @@ -18,10 +18,10 @@ Slight changes *do* make it blackhole hard, bootstrapping isn't an easy problem */ -import "sort" -import "time" - -//import "fmt" +import ( + "sort" + "time" +) // Number of DHT buckets, equal to the number of bits in a NodeID. // Note that, in practice, nearly all of these will be empty. @@ -236,7 +236,6 @@ func (t *dht) nBuckets() int { // Inserts a node into the DHT if they meet certain requirements. // In particular, they must either be a peer that's not already in the DHT, or else be someone we should insert into the DHT (see: shouldInsert). func (t *dht) insertIfNew(info *dhtInfo, isPeer bool) { - //fmt.Println("DEBUG: dht insertIfNew:", info.getNodeID(), info.coords) // Insert if no "other" entry already exists nodeID := info.getNodeID() bidx, isOK := t.getBucketIndex(nodeID) @@ -256,7 +255,6 @@ func (t *dht) insertIfNew(info *dhtInfo, isPeer bool) { // Adds a node to the DHT, possibly removing another node in the process. func (t *dht) insert(info *dhtInfo, isPeer bool) { - //fmt.Println("DEBUG: dht insert:", info.getNodeID(), info.coords) // First update the time on this info info.recv = time.Now() // Get the bucket for this node diff --git a/src/yggdrasil/icmpv6.go b/src/yggdrasil/icmpv6.go index f3b00ffc..1eb1c67c 100644 --- a/src/yggdrasil/icmpv6.go +++ b/src/yggdrasil/icmpv6.go @@ -9,11 +9,14 @@ package yggdrasil // - Destination Unreachable messages, when a session prohibits // incoming traffic -import "net" -import "golang.org/x/net/ipv6" -import "golang.org/x/net/icmp" -import "encoding/binary" -import "errors" +import ( + "encoding/binary" + "errors" + "net" + + "golang.org/x/net/icmp" + "golang.org/x/net/ipv6" +) type macAddress [6]byte @@ -239,10 +242,16 @@ func (i *icmpv6) create_icmpv6_tun(dst net.IP, src net.IP, mtype ipv6.ICMPType, // when the host operating system generates an NDP request for any address in // the fd00::/8 range, so that the operating system knows to route that traffic // to the Yggdrasil TAP adapter. -// TODO: Make this respect the value of address_prefix in address.go func (i *icmpv6) handle_ndp(in []byte) ([]byte, error) { // Ignore NDP requests for anything outside of fd00::/8 - if in[8] != 0xFD { + var source address + copy(source[:], in[8:]) + var snet subnet + copy(snet[:], in[8:]) + switch { + case source.isValid(): + case snet.isValid(): + default: return nil, errors.New("Not an NDP for fd00::/8") } diff --git a/src/yggdrasil/multicast.go b/src/yggdrasil/multicast.go index 8211e422..d9d0ccc4 100644 --- a/src/yggdrasil/multicast.go +++ b/src/yggdrasil/multicast.go @@ -1,10 +1,12 @@ package yggdrasil -import "net" -import "time" -import "fmt" +import ( + "fmt" + "net" + "time" -import "golang.org/x/net/ipv6" + "golang.org/x/net/ipv6" +) type multicast struct { core *Core @@ -37,11 +39,9 @@ func (m *multicast) start() error { if err != nil { return err } - //defer conn.Close() // Let it close on its own when the application exits m.sock = ipv6.NewPacketConn(conn) if err = m.sock.SetControlMessage(ipv6.FlagDst, true); err != nil { // Windows can't set this flag, so we need to handle it in other ways - //panic(err) } go m.listen() @@ -95,8 +95,6 @@ func (m *multicast) announce() { for { for _, iface := range m.interfaces() { m.sock.JoinGroup(&iface, groupAddr) - //err := n.sock.JoinGroup(&iface, groupAddr) - //if err != nil { panic(err) } addrs, err := iface.Addrs() if err != nil { panic(err) @@ -133,8 +131,6 @@ func (m *multicast) listen() { if err != nil { panic(err) } - //if rcm == nil { continue } // wat - //fmt.Println("DEBUG:", "packet from:", fromAddr.String()) if rcm != nil { // Windows can't set the flag needed to return a non-nil value here // So only make these checks if we get something useful back @@ -149,19 +145,14 @@ func (m *multicast) listen() { anAddr := string(bs[:nBytes]) addr, err := net.ResolveTCPAddr("tcp6", anAddr) if err != nil { - panic(err) continue - } // Panic for testing, remove later + } from := fromAddr.(*net.UDPAddr) - //fmt.Println("DEBUG:", "heard:", addr.IP.String(), "from:", from.IP.String()) if addr.IP.String() != from.IP.String() { continue } addr.Zone = from.Zone saddr := addr.String() - //if _, isIn := n.peers[saddr]; isIn { continue } - //n.peers[saddr] = struct{}{} m.core.tcp.connect(saddr) - //fmt.Println("DEBUG:", "added multicast peer:", saddr) } } diff --git a/src/yggdrasil/peer.go b/src/yggdrasil/peer.go index 5e522f36..caedeb70 100644 --- a/src/yggdrasil/peer.go +++ b/src/yggdrasil/peer.go @@ -4,21 +4,20 @@ package yggdrasil // Commented code should be removed // Live code should be better commented -import "time" -import "sync" -import "sync/atomic" - -//import "fmt" +import ( + "sync" + "sync/atomic" + "time" +) // The peers struct represents peers with an active connection. // Incomping packets are passed to the corresponding peer, which handles them somehow. // In most cases, this involves passing the packet to the handler for outgoing traffic to another peer. // In other cases, it's link protocol traffic used to build the spanning tree, in which case this checks signatures and passes the message along to the switch. type peers struct { - core *Core - mutex sync.Mutex // Synchronize writes to atomic - ports atomic.Value //map[Port]*peer, use CoW semantics - //ports map[Port]*peer + core *Core + mutex sync.Mutex // Synchronize writes to atomic + ports atomic.Value //map[Port]*peer, use CoW semantics authMutex sync.RWMutex allowedEncryptionPublicKeys map[boxPubKey]struct{} } @@ -198,7 +197,7 @@ func (p *peer) linkLoop() { // Called to handle incoming packets. // Passes the packet to a handler for that packet type. func (p *peer) handlePacket(packet []byte) { - // TODO See comment in sendPacket about atomics technically being done wrong + // FIXME this is off by stream padding and msg length overhead, should be done in tcp.go atomic.AddUint64(&p.bytesRecvd, uint64(len(packet))) pType, pTypeLen := wire_decode_uint64(packet) if pTypeLen == 0 { @@ -307,8 +306,6 @@ func (p *peer) sendSwitchMsg() { Sig: *sign(&p.core.sigPriv, bs), }) packet := msg.encode() - //p.core.log.Println("Encoded msg:", msg, "; bytes:", packet) - //fmt.Println("Encoded msg:", msg, "; bytes:", packet) p.sendLinkPacket(packet) } @@ -319,7 +316,6 @@ func (p *peer) handleSwitchMsg(packet []byte) { if !msg.decode(packet) { return } - //p.core.log.Println("Decoded msg:", msg, "; bytes:", packet) if len(msg.Hops) < 1 { p.core.peers.removePeer(p.port) } diff --git a/src/yggdrasil/release.go b/src/yggdrasil/release.go index 1b5e9caf..3b136904 100644 --- a/src/yggdrasil/release.go +++ b/src/yggdrasil/release.go @@ -2,8 +2,10 @@ package yggdrasil -import "errors" -import "log" +import ( + "errors" + "log" +) // Starts the function profiler. This is only supported when built with // '-tags build'. diff --git a/src/yggdrasil/router.go b/src/yggdrasil/router.go index 4f15dea3..5a6eb455 100644 --- a/src/yggdrasil/router.go +++ b/src/yggdrasil/router.go @@ -22,12 +22,12 @@ package yggdrasil // The packet is passed to the session, which decrypts it, router.recvPacket // The router then runs some sanity checks before passing it to the tun -import "time" -import "golang.org/x/net/icmp" -import "golang.org/x/net/ipv6" +import ( + "time" -//import "fmt" -//import "net" + "golang.org/x/net/icmp" + "golang.org/x/net/ipv6" +) // The router struct has channels to/from the tun/tap device and a self peer (0), which is how messages are passed between this node and the peers/switch layer. // The router's mainLoop goroutine is responsible for managing all information related to the dht, searches, and crypto sessions. @@ -101,7 +101,6 @@ func (r *router) mainLoop() { // Any periodic maintenance stuff goes here r.core.switchTable.doMaintenance() r.core.dht.doMaintenance() - //r.core.peers.sendSwitchMsgs() // FIXME debugging util_getBytes() // To slowly drain things } case f := <-r.admin: @@ -243,7 +242,6 @@ func (r *router) sendPacket(bs []byte) { // Checks that the IP address is correct (matches the session) and passes the packet to the tun/tap. func (r *router) recvPacket(bs []byte, theirAddr *address, theirSubnet *subnet) { // Note: called directly by the session worker, not the router goroutine - //fmt.Println("Recv packet") if len(bs) < 24 { util_putBytes(bs) return @@ -274,7 +272,7 @@ func (r *router) handleIn(packet []byte) { r.handleTraffic(packet) case wire_ProtocolTraffic: r.handleProto(packet) - default: /*panic("Should not happen in testing") ;*/ + default: } } @@ -290,7 +288,6 @@ func (r *router) handleTraffic(packet []byte) { if !isIn { return } - //go func () { sinfo.recv<-&p }() sinfo.recv <- &p } @@ -303,7 +300,6 @@ func (r *router) handleProto(packet []byte) { } // Now try to open the payload var sharedKey *boxSharedKey - //var theirPermPub *boxPubKey if p.ToKey == r.core.boxPub { // Try to open using our permanent key sharedKey = r.core.sessions.getSharedKey(&r.core.boxPriv, &p.FromKey) @@ -321,7 +317,6 @@ func (r *router) handleProto(packet []byte) { if bsTypeLen == 0 { return } - //fmt.Println("RECV bytes:", bs) switch bsType { case wire_SessionPing: r.handlePing(bs, &p.FromKey) diff --git a/src/yggdrasil/search.go b/src/yggdrasil/search.go index 772b3848..2928faa3 100644 --- a/src/yggdrasil/search.go +++ b/src/yggdrasil/search.go @@ -11,10 +11,10 @@ package yggdrasil // A new search packet is sent immediately after receiving a response // A new search packet is sent periodically, once per second, in case a packet was dropped (this slowly causes the search to become parallel if the search doesn't timeout but also doesn't finish within 1 second for whatever reason) -import "sort" -import "time" - -//import "fmt" +import ( + "sort" + "time" +) // This defines the maximum number of dhtInfo that we keep track of for nodes to query in an ongoing search. const search_MAX_SEARCH_SIZE = 16 diff --git a/src/yggdrasil/session.go b/src/yggdrasil/session.go index 01bfb959..2580f45d 100644 --- a/src/yggdrasil/session.go +++ b/src/yggdrasil/session.go @@ -456,7 +456,6 @@ func (sinfo *sessionInfo) doRecv(p *wire_trafficPacket) { } if newMTU < sinfo.myMTU { sinfo.myMTU = newMTU - //sinfo.core.log.Println("DEBUG set MTU to:", sinfo.myMTU) sinfo.core.sessions.sendPingPong(sinfo, false) sinfo.mtuTime = time.Now() sinfo.wasMTUFixed = true @@ -470,7 +469,6 @@ func (sinfo *sessionInfo) doRecv(p *wire_trafficPacket) { if time.Since(sinfo.mtuTime) > time.Minute { sinfo.myMTU = uint16(sinfo.core.tun.mtu) sinfo.mtuTime = time.Now() - //sinfo.core.log.Println("DEBUG: Reset MTU to:", sinfo.myMTU) } } go func() { sinfo.core.router.admin <- fixSessionMTU }() diff --git a/src/yggdrasil/signature.go b/src/yggdrasil/signature.go index f7b88ae9..374183a0 100644 --- a/src/yggdrasil/signature.go +++ b/src/yggdrasil/signature.go @@ -3,8 +3,10 @@ package yggdrasil // This is where we record which signatures we've previously checked // It's so we can avoid needlessly checking them again -import "sync" -import "time" +import ( + "sync" + "time" +) // This keeps track of what signatures have already been checked. // It's used to skip expensive crypto operations, given that many signatures are likely to be the same for the average node's peers. diff --git a/src/yggdrasil/switch.go b/src/yggdrasil/switch.go index a819594b..9c0e3f17 100644 --- a/src/yggdrasil/switch.go +++ b/src/yggdrasil/switch.go @@ -11,12 +11,12 @@ package yggdrasil // TODO? use a pre-computed lookup table (python version had this) // A little annoying to do with constant changes from backpressure -import "time" -import "sort" -import "sync" -import "sync/atomic" - -//import "fmt" +import ( + "sort" + "sync" + "sync/atomic" + "time" +) const switch_timeout = time.Minute const switch_updateInterval = switch_timeout / 2 @@ -201,25 +201,19 @@ func (t *switchTable) cleanRoot() { // Get rid of the root if it looks like its timed out now := time.Now() doUpdate := false - //fmt.Println("DEBUG clean root:", now.Sub(t.time)) if now.Sub(t.time) > switch_timeout { - //fmt.Println("root timed out", t.data.locator) dropped := t.data.peers[t.parent] dropped.time = t.time t.drop[t.data.locator.root] = t.data.locator.tstamp doUpdate = true - //t.core.log.Println("DEBUG: switch root timeout", len(t.drop)) } // Or, if we're better than our root, root ourself if firstIsBetter(&t.key, &t.data.locator.root) { - //fmt.Println("root is worse than us", t.data.locator.Root) doUpdate = true - //t.core.log.Println("DEBUG: switch root replace with self", t.data.locator.Root) } // Or, if we are the root, possibly update our timestamp if t.data.locator.root == t.key && now.Sub(t.time) > switch_updateInterval { - //fmt.Println("root is self and old, updating", t.data.locator.Root) doUpdate = true } if doUpdate { @@ -421,15 +415,12 @@ func (t *switchTable) unlockedHandleMsg(msg *switchMsg, fromPort switchPort) { case t.core.router.reset <- struct{}{}: default: } - //t.core.log.Println("Switch update:", msg.locator.root, msg.locator.tstamp, msg.locator.coords) - //fmt.Println("Switch update:", msg.Locator.Root, msg.Locator.Tstamp, msg.Locator.Coords) } if t.data.locator.tstamp != sender.locator.tstamp { t.time = now } t.data.locator = sender.locator t.parent = sender.port - //t.core.log.Println("Switch update:", msg.Locator.Root, msg.Locator.Tstamp, msg.Locator.Coords) t.core.peers.sendSwitchMsgs() } if doUpdate { @@ -504,6 +495,5 @@ func (t *switchTable) lookup(dest []byte) switchPort { bestCost = cost } } - //t.core.log.Println("DEBUG: sending to", best, "cost", bestCost) return best } diff --git a/src/yggdrasil/tcp.go b/src/yggdrasil/tcp.go index 75a5618e..9590f221 100644 --- a/src/yggdrasil/tcp.go +++ b/src/yggdrasil/tcp.go @@ -14,13 +14,16 @@ package yggdrasil // It involves exchanging version numbers and crypto keys // See version.go for version metadata format -import "net" -import "time" -import "errors" -import "sync" -import "sync/atomic" -import "fmt" -import "golang.org/x/net/proxy" +import ( + "errors" + "fmt" + "net" + "sync" + "sync/atomic" + "time" + + "golang.org/x/net/proxy" +) const tcp_msgSize = 2048 + 65535 // TODO figure out what makes sense @@ -211,7 +214,7 @@ func (iface *tcpInterface) handler(sock net.Conn, incoming bool) { } if equiv(info.box[:], iface.core.boxPub[:]) { return - } // testing + } if equiv(info.sig[:], iface.core.sigPub[:]) { return } @@ -286,7 +289,6 @@ func (iface *tcpInterface) handler(sock net.Conn, incoming bool) { timer.Reset(timerInterval) select { case _ = <-timer.C: - //iface.core.log.Println("DEBUG: sending keep-alive:", sock.RemoteAddr().String()) send(nil) // TCP keep-alive traffic case msg := <-p.linkOut: send(msg) @@ -352,14 +354,12 @@ func (iface *tcpInterface) reader(sock net.Conn, in func([]byte)) { sock.SetReadDeadline(timeout) n, err := sock.Read(bs[len(frag):]) if err != nil || n == 0 { - // iface.core.log.Println(err) break } frag = bs[:len(frag)+n] for { msg, ok, err := tcp_chop_msg(&frag) if err != nil { - // iface.core.log.Println(err) return } if !ok { diff --git a/src/yggdrasil/tun.go b/src/yggdrasil/tun.go index b29aabef..94adbcb4 100644 --- a/src/yggdrasil/tun.go +++ b/src/yggdrasil/tun.go @@ -2,8 +2,10 @@ package yggdrasil // This manages the tun driver to send/recv packets to/from applications -import "github.com/songgao/packets/ethernet" -import "github.com/yggdrasil-network/water" +import ( + "github.com/songgao/packets/ethernet" + "github.com/yggdrasil-network/water" +) const tun_IPv6_HEADER_LENGTH = 40 const tun_ETHER_HEADER_LENGTH = 14 @@ -127,7 +129,7 @@ func (tun *tunDevice) read() error { // Closes the TUN/TAP adapter. This is only usually called when the Yggdrasil // process stops. Typically this operation will happen quickly, but on macOS -// it can block until a read operation is completed. +// it can block until a read operation is completed. func (tun *tunDevice) close() error { if tun.iface == nil { return nil diff --git a/src/yggdrasil/tun_bsd.go b/src/yggdrasil/tun_bsd.go index 9455d5d2..ca5eaea8 100644 --- a/src/yggdrasil/tun_bsd.go +++ b/src/yggdrasil/tun_bsd.go @@ -2,16 +2,18 @@ package yggdrasil -import "unsafe" -import "syscall" -import "strings" -import "strconv" -import "encoding/binary" -import "os/exec" +import ( + "encoding/binary" + "os/exec" + "strconv" + "strings" + "syscall" + "unsafe" -import "golang.org/x/sys/unix" + "golang.org/x/sys/unix" -import "github.com/yggdrasil-network/water" + "github.com/yggdrasil-network/water" +) const SIOCSIFADDR_IN6 = (0x80000000) | ((288 & 0x1fff) << 16) | uint32(byte('i'))<<8 | 12 @@ -74,7 +76,7 @@ type in6_ifreq_lifetime struct { // OpenBSD, NetBSD) an attempt is made to set the adapter properties by using // a system socket and making syscalls to the kernel. This is not refined though // and often doesn't work (if at all), therefore if a call fails, it resorts -// to calling "ifconfig" instead. +// to calling "ifconfig" instead. func (tun *tunDevice) setup(ifname string, iftapmode bool, addr string, mtu int) error { var config water.Config if ifname[:4] == "auto" { diff --git a/src/yggdrasil/tun_darwin.go b/src/yggdrasil/tun_darwin.go index 4211fe7d..bdfda387 100644 --- a/src/yggdrasil/tun_darwin.go +++ b/src/yggdrasil/tun_darwin.go @@ -2,13 +2,16 @@ package yggdrasil // The darwin platform specific tun parts -import "unsafe" -import "strings" -import "strconv" -import "encoding/binary" -import "golang.org/x/sys/unix" +import ( + "encoding/binary" + "strconv" + "strings" + "unsafe" -import water "github.com/yggdrasil-network/water" + "golang.org/x/sys/unix" + + water "github.com/yggdrasil-network/water" +) // Sane defaults for the Darwin/macOS platform. The "default" options may be // may be replaced by the running configuration. @@ -69,7 +72,7 @@ type ifreq struct { } // Sets the IPv6 address of the utun adapter. On Darwin/macOS this is done using -// a system socket and making direct syscalls to the kernel. +// a system socket and making direct syscalls to the kernel. func (tun *tunDevice) setupAddress(addr string) error { var fd int var err error diff --git a/src/yggdrasil/tun_linux.go b/src/yggdrasil/tun_linux.go index d038d4e8..977d73d0 100644 --- a/src/yggdrasil/tun_linux.go +++ b/src/yggdrasil/tun_linux.go @@ -2,13 +2,15 @@ package yggdrasil // The linux platform specific tun parts -import "errors" -import "fmt" -import "net" +import ( + "errors" + "fmt" + "net" -import water "github.com/yggdrasil-network/water" + "github.com/docker/libcontainer/netlink" -import "github.com/docker/libcontainer/netlink" + water "github.com/yggdrasil-network/water" +) // Sane defaults for the Linux platform. The "default" options may be // may be replaced by the running configuration. diff --git a/src/yggdrasil/tun_windows.go b/src/yggdrasil/tun_windows.go index 644cfdb4..131bb344 100644 --- a/src/yggdrasil/tun_windows.go +++ b/src/yggdrasil/tun_windows.go @@ -1,9 +1,12 @@ package yggdrasil -import water "github.com/yggdrasil-network/water" -import "os/exec" -import "strings" -import "fmt" +import ( + "fmt" + "os/exec" + "strings" + + water "github.com/yggdrasil-network/water" +) // This is to catch Windows platforms diff --git a/src/yggdrasil/util.go b/src/yggdrasil/util.go index fec5747c..c9897eda 100644 --- a/src/yggdrasil/util.go +++ b/src/yggdrasil/util.go @@ -4,8 +4,6 @@ package yggdrasil import "runtime" -//import "sync" - // A wrapper around runtime.Gosched() so it doesn't need to be imported elsewhere. func util_yield() { runtime.Gosched() @@ -21,20 +19,6 @@ func util_unlockthread() { runtime.UnlockOSThread() } -/* Used previously, but removed because casting to an interface{} allocates... -var byteStore sync.Pool = sync.Pool{ - New: func () interface{} { return []byte(nil) }, -} - -func util_getBytes() []byte { - return byteStore.Get().([]byte)[:0] -} - -func util_putBytes(bs []byte) { - byteStore.Put(bs) // This is the part that allocates -} -*/ - // This is used to buffer recently used slices of bytes, to prevent allocations in the hot loops. // It's used like a sync.Pool, but with a fixed size and typechecked without type casts to/from interface{} (which were making the profiles look ugly). var byteStore chan []byte From 98ee657447e5c73920b2c1375b928174d2320c50 Mon Sep 17 00:00:00 2001 From: Arceliar Date: Tue, 12 Jun 2018 19:31:36 -0500 Subject: [PATCH 41/42] Update README.md Replace some links to the old documentation with links to the .io site --- README.md | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index e44024eb..122fd224 100644 --- a/README.md +++ b/README.md @@ -8,8 +8,7 @@ This is a toy implementation of an encrypted IPv6 network, with many good ideas stolen from [cjdns](https://github.com/cjdelisle/cjdns), which was written to test a particular routing scheme that was cobbled together one random afternoon. It's notably not a shortest path routing scheme, with the goal of scalable name-independent routing on dynamic networks with an internet-like topology. It's named Yggdrasil after the world tree from Norse mythology, because that seemed like the obvious name given how it works. -For a longer, rambling version of this readme with more information, see: [doc](doc/README.md). -A very early incomplete draft of a [whitepaper](doc/Whitepaper.md) describing the protocol is also available. +More information is available at . This is a toy / proof-of-principle, and considered alpha quality by the developers. It's not expected to be feature complete, and future updates may not be backwards compatible, though it should warn you if it sees a connection attempt with a node running a newer version. You're encouraged to play with it, but it is strongly advised not to use it for anything mission critical. @@ -129,11 +128,11 @@ interface eth0 }; ``` -This is enough to give unsupported devices on the LAN access to the yggdrasil network, with a few security and performance cautions outlined in the [doc](doc/README.md) file. +This is enough to give unsupported devices on the LAN access to the yggdrasil network. See the [configuration](https://yggdrasil-network.github.io/configuration.html) page for more info. ## How does it work? -I'd rather not try to explain in the readme, but it is described further in the [doc](doc/README.md) file or the very draft of a [whitepaper](doc/Whitepaper.md), so you can check there if you're interested. +I'd rather not try to explain in the readme, but it is described further on the [about](https://yggdrasil-network.github.io/about.html) page, so you can check there if you're interested. Be warned that it's still not a very good explanation, but it at least gives a high-level overview and links to some relevant work by other people. ## Obligatory performance propaganda From cd514799dad434a4c8c2154d45efc5aa552c1c6a Mon Sep 17 00:00:00 2001 From: Arceliar Date: Wed, 13 Jun 2018 00:24:12 -0500 Subject: [PATCH 42/42] recover if p.doSend is closed due to a race between peers.doSendSwitchMsgs and peers.removePeer --- src/yggdrasil/peer.go | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/src/yggdrasil/peer.go b/src/yggdrasil/peer.go index caedeb70..ad99750a 100644 --- a/src/yggdrasil/peer.go +++ b/src/yggdrasil/peer.go @@ -166,17 +166,24 @@ func (ps *peers) sendSwitchMsgs() { if p.port == 0 { continue } - select { - case p.doSend <- struct{}{}: - default: - } + p.doSendSwitchMsgs() + } +} + +// If called, sends a notification to the peer's linkLoop to trigger a switchMsg send. +// Mainly called by sendSwitchMsgs or during linkLoop startup. +func (p *peer) doSendSwitchMsgs() { + defer func() { recover() }() // In case there's a race with close(p.doSend) + select { + case p.doSend <- struct{}{}: + default: } } // This must be launched in a separate goroutine by whatever sets up the peer struct. // It handles link protocol traffic. func (p *peer) linkLoop() { - go func() { p.doSend <- struct{}{} }() + go p.doSendSwitchMsgs() tick := time.NewTicker(time.Second) defer tick.Stop() for {