wgengine/magicsock: prune key.Public-keyed on peer removals

Fixes #215

wgengine/userspace.go

@@ -327,10 +327,12 @@ func (e *userspaceEngine) Reconfig(cfg *wgcfg.Config, dnsDomains []string) error
 	e.wgLock.Lock()
 	defer e.wgLock.Unlock()
 
+	peerSet := make(map[key.Public]struct{}, len(cfg.Peers))
 	e.mu.Lock()
 	e.peerSequence = e.peerSequence[:0]
 	for _, p := range cfg.Peers {
 		e.peerSequence = append(e.peerSequence, p.PublicKey)
+		peerSet[key.Public(p.PublicKey)] = struct{}{}
 	}
 	e.mu.Unlock()
 
@@ -362,6 +364,8 @@ func (e *userspaceEngine) Reconfig(cfg *wgcfg.Config, dnsDomains []string) error
 		return err
 	}
 
+	e.magicConn.UpdatePeers(peerSet)
+
 	// TODO(apenwarr): only handling the first local address.
 	//   Currently we never use more than one anyway.
 	var cidr wgcfg.CIDR