From ba870cc5132a339b71f53edb45f865cf38381cc4 Mon Sep 17 00:00:00 2001 From: Percy Wegmann Date: Mon, 9 Dec 2024 13:33:31 -0600 Subject: [PATCH] quic-go experimentation Signed-off-by: Percy Wegmann --- derp/derp_client.go | 10 +++- derp/derp_server.go | 10 ---- derp/derp_test.go | 112 ++++++++++++++++++++++++++++++++------------ go.mod | 2 +- go.sum | 2 + 5 files changed, 93 insertions(+), 43 deletions(-) diff --git a/derp/derp_client.go b/derp/derp_client.go index 7a646fa51..7a5afc62c 100644 --- a/derp/derp_client.go +++ b/derp/derp_client.go @@ -216,24 +216,32 @@ func (c *Client) send(dstKey key.NodePublic, pkt []byte) (ret error) { return fmt.Errorf("packet too big: %d", len(pkt)) } + fmt.Println("ZZZZ acquiring write lock") c.wmu.Lock() defer c.wmu.Unlock() + fmt.Println("ZZZZ acquired write lock") if c.rate != nil { pktLen := frameHeaderLen + key.NodePublicRawLen + len(pkt) if !c.rate.AllowN(c.clock.Now(), pktLen) { return nil // drop } } + fmt.Println("ZZZZ writing frame header") if err := writeFrameHeader(c.bw, frameSendPacket, uint32(key.NodePublicRawLen+len(pkt))); err != nil { return err } + fmt.Println("ZZZZ writing destination key") if _, err := c.bw.Write(dstKey.AppendTo(nil)); err != nil { return err } + fmt.Println("ZZZZ writing packet") if _, err := c.bw.Write(pkt); err != nil { return err } - return c.bw.Flush() + fmt.Println("ZZZZ flushing buffer") + err := c.bw.Flush() + fmt.Println("ZZZZ flushed buffer") + return err } func (c *Client) ForwardPacket(srcKey, dstKey key.NodePublic, pkt []byte) (err error) { diff --git a/derp/derp_server.go b/derp/derp_server.go index ca74b9c0e..0e146d56a 100644 --- a/derp/derp_server.go +++ b/derp/derp_server.go @@ -891,10 +891,6 @@ func (s *Server) debugLogf(format string, v ...any) { // run serves the client until there's an error. // If the client hangs up or the server is closed, run returns nil, otherwise run returns an error. func (c *sclient) run(ctx context.Context) error { - fmt.Println("ZZZZ Client Running") - defer func() { - fmt.Println("ZZZZ Client Stopped") - }() // Launch sender, but don't return from run until sender goroutine is done. var grp errgroup.Group sendCtx, cancelSender := context.WithCancel(ctx) @@ -916,9 +912,7 @@ func (c *sclient) run(ctx context.Context) error { c.startStatsLoop(sendCtx) for { - fmt.Println("ZZZZ reading header") ft, fl, err := readFrameHeader(c.br) - fmt.Println("ZZZZ read header") c.debugLogf("read frame type %d len %d err %v", ft, fl, err) if err != nil { if errors.Is(err, io.EOF) { @@ -1691,7 +1685,6 @@ func (c *sclient) sendLoop(ctx context.Context) error { inBatch := -1 // for bufferedWriteFrames for { if werr != nil { - fmt.Printf("ZZZZ send loop ending with werr: %s\n", werr) return werr } inBatch++ @@ -1699,7 +1692,6 @@ func (c *sclient) sendLoop(ctx context.Context) error { // does as many non-flushing writes as possible. select { case <-ctx.Done(): - fmt.Println("ZZZZ send loop context done") return nil case msg := <-c.peerGone: werr = c.sendPeerGone(msg.peer, msg.reason) @@ -1725,7 +1717,6 @@ func (c *sclient) sendLoop(ctx context.Context) error { // Flush any writes from the 3 sends above, or from // the blocking loop below. if werr = c.bw.Flush(); werr != nil { - fmt.Printf("ZZZZ flush failed: %s\n", werr) return werr } if inBatch != 0 { // the first loop will almost always hit default & be size zero @@ -1737,7 +1728,6 @@ func (c *sclient) sendLoop(ctx context.Context) error { // Then a blocking select with same: select { case <-ctx.Done(): - fmt.Println("ZZZZ send loop context done") return nil case msg := <-c.peerGone: werr = c.sendPeerGone(msg.peer, msg.reason) diff --git a/derp/derp_test.go b/derp/derp_test.go index f50db5d73..3a7509669 100644 --- a/derp/derp_test.go +++ b/derp/derp_test.go @@ -8,22 +8,20 @@ import ( "bytes" "context" "crypto/rand" - "crypto/rsa" "crypto/tls" "crypto/x509" "encoding/asn1" "encoding/json" - "encoding/pem" "errors" "expvar" "fmt" "io" "log" - "math/big" "net" "os" "reflect" "strconv" + "strings" "sync" "testing" "time" @@ -33,6 +31,7 @@ import ( "golang.org/x/time/rate" "tailscale.com/disco" "tailscale.com/net/memnet" + "tailscale.com/syncs" "tailscale.com/tstest" "tailscale.com/types/key" "tailscale.com/types/logger" @@ -1427,8 +1426,11 @@ func BenchmarkSendRecvDERP(b *testing.B) { b.SetBytes(int64(len(msg))) b.ReportAllocs() + inFlight := syncs.NewSemaphore(28) + go func() { for { + inFlight.Acquire() if err := client.Send(clientKey, msg); err != nil { connIn.Close() connOut.Close() @@ -1442,6 +1444,7 @@ func BenchmarkSendRecvDERP(b *testing.B) { if _, err := client.Recv(); err != nil { b.Fatal(err) } + inFlight.Release() } } @@ -1459,7 +1462,9 @@ func BenchmarkSendRecvQUIC(b *testing.B) { k := key.NewNode() clientKey := k.Public() - ln, err := quic.ListenAddr("127.0.0.1:0", generateTLSConfig(), nil) + qcfg := &quic.Config{} + + ln, err := quic.ListenAddr("127.0.0.1:0", generateTLSConfig(), qcfg) if err != nil { b.Fatal(err) } @@ -1474,11 +1479,12 @@ func BenchmarkSendRecvQUIC(b *testing.B) { b.Fatal(err) } defer qconnIn.CloseWithError(0, "") - connIn, err := qconnIn.AcceptStream(context.Background()) + _connIn, err := qconnIn.AcceptStream(context.Background()) if err != nil { b.Fatal(err) } - defer connIn.Close() + defer _connIn.Close() + connIn := &connWithAddr{_connIn, "server", qconnIn.LocalAddr()} // read and discard initial byte if _, err := connIn.Read(make([]byte, 1)); err != nil { @@ -1487,28 +1493,29 @@ func BenchmarkSendRecvQUIC(b *testing.B) { brwServer := bufio.NewReadWriter(bufio.NewReader(connIn), bufio.NewWriter(connIn)) - s.Accept(ctx, &connWithAddr{connIn, qconnIn.LocalAddr()}, brwServer, "test-client") + s.Accept(ctx, connIn, brwServer, "test-client") }() tlsConf := &tls.Config{ InsecureSkipVerify: true, // NextProtos: []string{"quic-echo-example"}, } - qconnOut, err := quic.DialAddr(context.Background(), ln.Addr().String(), tlsConf, nil) + qconnOut, err := quic.DialAddr(context.Background(), ln.Addr().String(), tlsConf, qcfg) if err != nil { b.Fatal(err) } defer qconnOut.CloseWithError(0, "") - connOut, err := qconnOut.OpenStream() + _connOut, err := qconnOut.OpenStream() if err != nil { b.Fatal(err) } - defer connOut.Close() + defer _connOut.Close() + connOut := &connWithAddr{_connOut, "client", qconnOut.LocalAddr()} connOut.Write([]byte{0}) brw := bufio.NewReadWriter(bufio.NewReader(connOut), bufio.NewWriter(connOut)) - client, err := NewClient(k, &connWithAddr{connOut, qconnOut.LocalAddr()}, brw, logger.Discard) + client, err := NewClient(k, connOut, brw, logger.Discard) if err != nil { b.Fatalf("client: %v", err) } @@ -1518,8 +1525,11 @@ func BenchmarkSendRecvQUIC(b *testing.B) { b.SetBytes(int64(len(msg))) b.ReportAllocs() + // inFlight := syncs.NewSemaphore(28) + go func() { for { + // inFlight.Acquire() if err := client.Send(clientKey, msg); err != nil { fmt.Println(err) connOut.Close() @@ -1533,17 +1543,19 @@ func BenchmarkSendRecvQUIC(b *testing.B) { if _, err := client.Recv(); err != nil { b.Fatal(err) } + // inFlight.Release() } } - // for _, size := range []int{10, 100, 1000, 10000} { - for _, size := range []int{10} { + for _, size := range []int{10, 100, 1000, 10000} { + // for _, size := range []int{10} { b.Run(fmt.Sprintf("msgsize=%d", size), func(b *testing.B) { benchmarkSendRecvSize(b, size) }) } } type connWithAddr struct { quic.Stream + label string localAddr net.Addr } @@ -1551,6 +1563,13 @@ func (c *connWithAddr) LocalAddr() net.Addr { return c.localAddr } +func (c *connWithAddr) Write(b []byte) (int, error) { + fmt.Printf("ZZZZ Writing length %d\n", len(b)) + n, err := c.Stream.Write(b) + fmt.Printf("ZZZZ Wrote %d with error %v\n", n, err) + return n, err +} + // func BenchmarkSendRecvPlain(b *testing.B) { // benchmarkSendRecvSize := func(b *testing.B, packetSize int) { // ln, err := net.Listen("tcp", "127.0.0.1:0") @@ -1767,24 +1786,55 @@ func TestServerRepliesToPing(t *testing.T) { // Setup a bare-bones TLS config for the server func generateTLSConfig() *tls.Config { - key, err := rsa.GenerateKey(rand.Reader, 1024) - if err != nil { - panic(err) - } - template := x509.Certificate{SerialNumber: big.NewInt(1)} - certDER, err := x509.CreateCertificate(rand.Reader, &template, &template, &key.PublicKey, key) - if err != nil { - panic(err) - } - keyPEM := pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(key)}) - certPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: certDER}) - - tlsCert, err := tls.X509KeyPair(certPEM, keyPEM) - if err != nil { - panic(err) - } return &tls.Config{ - Certificates: []tls.Certificate{tlsCert}, - // NextProtos: []string{"quic-echo-example"}, + Certificates: []tls.Certificate{testCert}, + InsecureSkipVerify: true, + CipherSuites: []uint16{ + tls.TLS_AES_128_GCM_SHA256, + tls.TLS_AES_256_GCM_SHA384, + tls.TLS_CHACHA20_POLY1305_SHA256, + }, + MinVersion: tls.VersionTLS13, + // Default key exchange mechanisms as of Go 1.23 minus X25519Kyber768Draft00, + // which bloats the client hello enough to spill into a second datagram. + // Tests were written with the assuption each flight in the handshake + // fits in one datagram, and it's simpler to keep that property. + CurvePreferences: []tls.CurveID{ + tls.X25519, tls.CurveP256, tls.CurveP384, tls.CurveP521, + }, } } + +var testCert = func() tls.Certificate { + cert, err := tls.X509KeyPair(localhostCert, localhostKey) + if err != nil { + panic(err) + } + return cert +}() + +// localhostCert is a PEM-encoded TLS cert with SAN IPs +// "127.0.0.1" and "[::1]", expiring at Jan 29 16:00:00 2084 GMT. +// generated from src/crypto/tls: +// go run generate_cert.go --ecdsa-curve P256 --host 127.0.0.1,::1,example.com --ca --start-date "Jan 1 00:00:00 1970" --duration=1000000h +var localhostCert = []byte(`-----BEGIN CERTIFICATE----- +MIIBrDCCAVKgAwIBAgIPCvPhO+Hfv+NW76kWxULUMAoGCCqGSM49BAMCMBIxEDAO +BgNVBAoTB0FjbWUgQ28wIBcNNzAwMTAxMDAwMDAwWhgPMjA4NDAxMjkxNjAwMDBa +MBIxEDAOBgNVBAoTB0FjbWUgQ28wWTATBgcqhkjOPQIBBggqhkjOPQMBBwNCAARh +WRF8p8X9scgW7JjqAwI9nYV8jtkdhqAXG9gyEgnaFNN5Ze9l3Tp1R9yCDBMNsGms +PyfMPe5Jrha/LmjgR1G9o4GIMIGFMA4GA1UdDwEB/wQEAwIChDATBgNVHSUEDDAK +BggrBgEFBQcDATAPBgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBSOJri/wLQxq6oC +Y6ZImms/STbTljAuBgNVHREEJzAlggtleGFtcGxlLmNvbYcEfwAAAYcQAAAAAAAA +AAAAAAAAAAAAATAKBggqhkjOPQQDAgNIADBFAiBUguxsW6TGhixBAdORmVNnkx40 +HjkKwncMSDbUaeL9jQIhAJwQ8zV9JpQvYpsiDuMmqCuW35XXil3cQ6Drz82c+fvE +-----END CERTIFICATE-----`) + +// localhostKey is the private key for localhostCert. +var localhostKey = []byte(testingKey(`-----BEGIN TESTING KEY----- +MIGHAgEAMBMGByqGSM49AgEGCCqGSM49AwEHBG0wawIBAQQgY1B1eL/Bbwf/MDcs +rnvvWhFNr1aGmJJR59PdCN9lVVqhRANCAARhWRF8p8X9scgW7JjqAwI9nYV8jtkd +hqAXG9gyEgnaFNN5Ze9l3Tp1R9yCDBMNsGmsPyfMPe5Jrha/LmjgR1G9 +-----END TESTING KEY-----`)) + +// testingKey helps keep security scanners from getting excited about a private key in this file. +func testingKey(s string) string { return strings.ReplaceAll(s, "TESTING KEY", "PRIVATE KEY") } diff --git a/go.mod b/go.mod index 9299bf5b8..6e4b223b9 100644 --- a/go.mod +++ b/go.mod @@ -157,7 +157,7 @@ require ( github.com/macabu/inamedparam v0.1.3 // indirect github.com/moby/docker-image-spec v1.3.1 // indirect github.com/onsi/ginkgo/v2 v2.17.1 // indirect - github.com/quic-go/quic-go v0.48.2 // indirect + github.com/quic-go/quic-go v0.48.2-0.20241205065829-2dca400b5c16 // indirect github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 // indirect github.com/xen0n/gosmopolitan v1.2.2 // indirect github.com/ykadowak/zerologlint v0.1.5 // indirect diff --git a/go.sum b/go.sum index 68c56422e..b5ddb5270 100644 --- a/go.sum +++ b/go.sum @@ -821,6 +821,8 @@ github.com/quasilyte/regex/syntax v0.0.0-20210819130434-b3f0c404a727 h1:TCg2WBOl github.com/quasilyte/regex/syntax v0.0.0-20210819130434-b3f0c404a727/go.mod h1:rlzQ04UMyJXu/aOvhd8qT+hvDrFpiwqp8MRXDY9szc0= github.com/quasilyte/stdinfo v0.0.0-20220114132959-f7386bf02567 h1:M8mH9eK4OUR4lu7Gd+PU1fV2/qnDNfzT635KRSObncs= github.com/quasilyte/stdinfo v0.0.0-20220114132959-f7386bf02567/go.mod h1:DWNGW8A4Y+GyBgPuaQJuWiy0XYftx4Xm/y5Jqk9I6VQ= +github.com/quic-go/quic-go v0.48.2-0.20241205065829-2dca400b5c16 h1:qKr8kL9UtS7OMpCRvR+o/ixevCsHq7GYBsvhU1d78eU= +github.com/quic-go/quic-go v0.48.2-0.20241205065829-2dca400b5c16/go.mod h1:9RyLbf3jjSZB+/l5DgQ4KFq/fguTLs6WAJaK4mfDJw8= github.com/quic-go/quic-go v0.48.2 h1:wsKXZPeGWpMpCGSWqOcqpW2wZYic/8T3aqiOID0/KWE= github.com/quic-go/quic-go v0.48.2/go.mod h1:yBgs3rWBOADpga7F+jJsb6Ybg1LSYiQvwWlLX+/6HMs= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=