Update dependencies

This commit is contained in:
Alexander Neumann
2017-09-13 14:09:48 +02:00
parent f3b49987f8
commit fda563d606
926 changed files with 189726 additions and 98666 deletions

View File

@@ -45,6 +45,7 @@ var (
flagNextProto = flag.String("nextproto", "h2,h2-14", "Comma-separated list of NPN/ALPN protocol names to negotiate.")
flagInsecure = flag.Bool("insecure", false, "Whether to skip TLS cert validation")
flagSettings = flag.String("settings", "empty", "comma-separated list of KEY=value settings for the initial SETTINGS frame. The magic value 'empty' sends an empty initial settings frame, and the magic value 'omit' causes no initial settings frame to be sent.")
flagDial = flag.String("dial", "", "optional ip:port to dial, to connect to a host:port but use a different SNI name (including a SNI name without DNS)")
)
type command struct {
@@ -147,11 +148,14 @@ func (app *h2i) Main() error {
InsecureSkipVerify: *flagInsecure,
}
hostAndPort := withPort(app.host)
hostAndPort := *flagDial
if hostAndPort == "" {
hostAndPort = withPort(app.host)
}
log.Printf("Connecting to %s ...", hostAndPort)
tc, err := tls.Dial("tcp", hostAndPort, cfg)
if err != nil {
return fmt.Errorf("Error dialing %s: %v", withPort(app.host), err)
return fmt.Errorf("Error dialing %s: %v", hostAndPort, err)
}
log.Printf("Connected to %v", tc.RemoteAddr())
defer tc.Close()

View File

@@ -18,6 +18,7 @@ import (
"io/ioutil"
"log"
"math"
mathrand "math/rand"
"net"
"net/http"
"sort"
@@ -86,7 +87,7 @@ type Transport struct {
// MaxHeaderListSize is the http2 SETTINGS_MAX_HEADER_LIST_SIZE to
// send in the initial settings frame. It is how many bytes
// of response headers are allow. Unlike the http2 spec, zero here
// of response headers are allowed. Unlike the http2 spec, zero here
// means to use a default limit (currently 10MB). If you actually
// want to advertise an ulimited value to the peer, Transport
// interprets the highest possible value here (0xffffffff or 1<<32-1)
@@ -164,15 +165,17 @@ type ClientConn struct {
goAwayDebug string // goAway frame's debug data, retained as a string
streams map[uint32]*clientStream // client-initiated
nextStreamID uint32
pendingRequests int // requests blocked and waiting to be sent because len(streams) == maxConcurrentStreams
pings map[[8]byte]chan struct{} // in flight ping data to notification channel
bw *bufio.Writer
br *bufio.Reader
fr *Framer
lastActive time.Time
// Settings from peer: (also guarded by mu)
maxFrameSize uint32
maxConcurrentStreams uint32
initialWindowSize uint32
maxFrameSize uint32
maxConcurrentStreams uint32
peerMaxHeaderListSize uint64
initialWindowSize uint32
hbuf bytes.Buffer // HPACK encoder writes into this
henc *hpack.Encoder
@@ -216,35 +219,45 @@ type clientStream struct {
resTrailer *http.Header // client's Response.Trailer
}
// awaitRequestCancel runs in its own goroutine and waits for the user
// to cancel a RoundTrip request, its context to expire, or for the
// request to be done (any way it might be removed from the cc.streams
// map: peer reset, successful completion, TCP connection breakage,
// etc)
func (cs *clientStream) awaitRequestCancel(req *http.Request) {
// awaitRequestCancel waits for the user to cancel a request or for the done
// channel to be signaled. A non-nil error is returned only if the request was
// canceled.
func awaitRequestCancel(req *http.Request, done <-chan struct{}) error {
ctx := reqContext(req)
if req.Cancel == nil && ctx.Done() == nil {
return
return nil
}
select {
case <-req.Cancel:
cs.cancelStream()
cs.bufPipe.CloseWithError(errRequestCanceled)
return errRequestCanceled
case <-ctx.Done():
return ctx.Err()
case <-done:
return nil
}
}
// awaitRequestCancel waits for the user to cancel a request, its context to
// expire, or for the request to be done (any way it might be removed from the
// cc.streams map: peer reset, successful completion, TCP connection breakage,
// etc). If the request is canceled, then cs will be canceled and closed.
func (cs *clientStream) awaitRequestCancel(req *http.Request) {
if err := awaitRequestCancel(req, cs.done); err != nil {
cs.cancelStream()
cs.bufPipe.CloseWithError(ctx.Err())
case <-cs.done:
cs.bufPipe.CloseWithError(err)
}
}
func (cs *clientStream) cancelStream() {
cs.cc.mu.Lock()
cc := cs.cc
cc.mu.Lock()
didReset := cs.didReset
cs.didReset = true
cs.cc.mu.Unlock()
cc.mu.Unlock()
if !didReset {
cs.cc.writeStreamReset(cs.ID, ErrCodeCancel, nil)
cc.writeStreamReset(cs.ID, ErrCodeCancel, nil)
cc.forgetStreamID(cs.ID)
}
}
@@ -329,7 +342,7 @@ func (t *Transport) RoundTripOpt(req *http.Request, opt RoundTripOpt) (*http.Res
}
addr := authorityAddr(req.URL.Scheme, req.URL.Host)
for {
for retry := 0; ; retry++ {
cc, err := t.connPool().GetClientConn(req, addr)
if err != nil {
t.vlogf("http2: Transport failed to get client conn for %s: %v", addr, err)
@@ -337,9 +350,25 @@ func (t *Transport) RoundTripOpt(req *http.Request, opt RoundTripOpt) (*http.Res
}
traceGotConn(req, cc)
res, err := cc.RoundTrip(req)
if err != nil {
if req, err = shouldRetryRequest(req, err); err == nil {
continue
if err != nil && retry <= 6 {
afterBodyWrite := false
if e, ok := err.(afterReqBodyWriteError); ok {
err = e
afterBodyWrite = true
}
if req, err = shouldRetryRequest(req, err, afterBodyWrite); err == nil {
// After the first retry, do exponential backoff with 10% jitter.
if retry == 0 {
continue
}
backoff := float64(uint(1) << (uint(retry) - 1))
backoff += backoff * (0.1 * mathrand.Float64())
select {
case <-time.After(time.Second * time.Duration(backoff)):
continue
case <-reqContext(req).Done():
return nil, reqContext(req).Err()
}
}
}
if err != nil {
@@ -360,43 +389,60 @@ func (t *Transport) CloseIdleConnections() {
}
var (
errClientConnClosed = errors.New("http2: client conn is closed")
errClientConnUnusable = errors.New("http2: client conn not usable")
errClientConnGotGoAway = errors.New("http2: Transport received Server's graceful shutdown GOAWAY")
errClientConnGotGoAwayAfterSomeReqBody = errors.New("http2: Transport received Server's graceful shutdown GOAWAY; some request body already written")
errClientConnClosed = errors.New("http2: client conn is closed")
errClientConnUnusable = errors.New("http2: client conn not usable")
errClientConnGotGoAway = errors.New("http2: Transport received Server's graceful shutdown GOAWAY")
)
// afterReqBodyWriteError is a wrapper around errors returned by ClientConn.RoundTrip.
// It is used to signal that err happened after part of Request.Body was sent to the server.
type afterReqBodyWriteError struct {
err error
}
func (e afterReqBodyWriteError) Error() string {
return e.err.Error() + "; some request body already written"
}
// shouldRetryRequest is called by RoundTrip when a request fails to get
// response headers. It is always called with a non-nil error.
// It returns either a request to retry (either the same request, or a
// modified clone), or an error if the request can't be replayed.
func shouldRetryRequest(req *http.Request, err error) (*http.Request, error) {
switch err {
default:
func shouldRetryRequest(req *http.Request, err error, afterBodyWrite bool) (*http.Request, error) {
if !canRetryError(err) {
return nil, err
case errClientConnUnusable, errClientConnGotGoAway:
return req, nil
case errClientConnGotGoAwayAfterSomeReqBody:
// If the Body is nil (or http.NoBody), it's safe to reuse
// this request and its Body.
if req.Body == nil || reqBodyIsNoBody(req.Body) {
return req, nil
}
// Otherwise we depend on the Request having its GetBody
// func defined.
getBody := reqGetBody(req) // Go 1.8: getBody = req.GetBody
if getBody == nil {
return nil, errors.New("http2: Transport: peer server initiated graceful shutdown after some of Request.Body was written; define Request.GetBody to avoid this error")
}
body, err := getBody()
if err != nil {
return nil, err
}
newReq := *req
newReq.Body = body
return &newReq, nil
}
if !afterBodyWrite {
return req, nil
}
// If the Body is nil (or http.NoBody), it's safe to reuse
// this request and its Body.
if req.Body == nil || reqBodyIsNoBody(req.Body) {
return req, nil
}
// Otherwise we depend on the Request having its GetBody
// func defined.
getBody := reqGetBody(req) // Go 1.8: getBody = req.GetBody
if getBody == nil {
return nil, fmt.Errorf("http2: Transport: cannot retry err [%v] after Request.Body was written; define Request.GetBody to avoid this error", err)
}
body, err := getBody()
if err != nil {
return nil, err
}
newReq := *req
newReq.Body = body
return &newReq, nil
}
func canRetryError(err error) bool {
if err == errClientConnUnusable || err == errClientConnGotGoAway {
return true
}
if se, ok := err.(StreamError); ok {
return se.Code == ErrCodeRefusedStream
}
return false
}
func (t *Transport) dialClientConn(addr string, singleUse bool) (*ClientConn, error) {
@@ -474,17 +520,18 @@ func (t *Transport) NewClientConn(c net.Conn) (*ClientConn, error) {
func (t *Transport) newClientConn(c net.Conn, singleUse bool) (*ClientConn, error) {
cc := &ClientConn{
t: t,
tconn: c,
readerDone: make(chan struct{}),
nextStreamID: 1,
maxFrameSize: 16 << 10, // spec default
initialWindowSize: 65535, // spec default
maxConcurrentStreams: 1000, // "infinite", per spec. 1000 seems good enough.
streams: make(map[uint32]*clientStream),
singleUse: singleUse,
wantSettingsAck: true,
pings: make(map[[8]byte]chan struct{}),
t: t,
tconn: c,
readerDone: make(chan struct{}),
nextStreamID: 1,
maxFrameSize: 16 << 10, // spec default
initialWindowSize: 65535, // spec default
maxConcurrentStreams: 1000, // "infinite", per spec. 1000 seems good enough.
peerMaxHeaderListSize: 0xffffffffffffffff, // "infinite", per spec. Use 2^64-1 instead.
streams: make(map[uint32]*clientStream),
singleUse: singleUse,
wantSettingsAck: true,
pings: make(map[[8]byte]chan struct{}),
}
if d := t.idleConnTimeout(); d != 0 {
cc.idleTimeout = d
@@ -560,6 +607,8 @@ func (cc *ClientConn) setGoAway(f *GoAwayFrame) {
}
}
// CanTakeNewRequest reports whether the connection can take a new request,
// meaning it has not been closed or received or sent a GOAWAY.
func (cc *ClientConn) CanTakeNewRequest() bool {
cc.mu.Lock()
defer cc.mu.Unlock()
@@ -571,8 +620,7 @@ func (cc *ClientConn) canTakeNewRequestLocked() bool {
return false
}
return cc.goAway == nil && !cc.closed &&
int64(len(cc.streams)+1) < int64(cc.maxConcurrentStreams) &&
cc.nextStreamID < math.MaxInt32
int64(cc.nextStreamID)+int64(cc.pendingRequests) < math.MaxInt32
}
// onIdleTimeout is called from a time.AfterFunc goroutine. It will
@@ -718,10 +766,9 @@ func (cc *ClientConn) RoundTrip(req *http.Request) (*http.Response, error) {
hasTrailers := trailers != ""
cc.mu.Lock()
cc.lastActive = time.Now()
if cc.closed || !cc.canTakeNewRequestLocked() {
if err := cc.awaitOpenSlotForRequest(req); err != nil {
cc.mu.Unlock()
return nil, errClientConnUnusable
return nil, err
}
body := req.Body
@@ -816,14 +863,13 @@ func (cc *ClientConn) RoundTrip(req *http.Request) (*http.Response, error) {
cs.abortRequestBodyWrite(errStopReqBodyWrite)
}
if re.err != nil {
if re.err == errClientConnGotGoAway {
cc.mu.Lock()
if cs.startedWrite {
re.err = errClientConnGotGoAwayAfterSomeReqBody
}
cc.mu.Unlock()
}
cc.mu.Lock()
afterBodyWrite := cs.startedWrite
cc.mu.Unlock()
cc.forgetStreamID(cs.ID)
if afterBodyWrite {
return nil, afterReqBodyWriteError{re.err}
}
return nil, re.err
}
res.Request = req
@@ -836,31 +882,31 @@ func (cc *ClientConn) RoundTrip(req *http.Request) (*http.Response, error) {
case re := <-readLoopResCh:
return handleReadLoopResponse(re)
case <-respHeaderTimer:
cc.forgetStreamID(cs.ID)
if !hasBody || bodyWritten {
cc.writeStreamReset(cs.ID, ErrCodeCancel, nil)
} else {
bodyWriter.cancel()
cs.abortRequestBodyWrite(errStopReqBodyWriteAndCancel)
}
cc.forgetStreamID(cs.ID)
return nil, errTimeout
case <-ctx.Done():
cc.forgetStreamID(cs.ID)
if !hasBody || bodyWritten {
cc.writeStreamReset(cs.ID, ErrCodeCancel, nil)
} else {
bodyWriter.cancel()
cs.abortRequestBodyWrite(errStopReqBodyWriteAndCancel)
}
cc.forgetStreamID(cs.ID)
return nil, ctx.Err()
case <-req.Cancel:
cc.forgetStreamID(cs.ID)
if !hasBody || bodyWritten {
cc.writeStreamReset(cs.ID, ErrCodeCancel, nil)
} else {
bodyWriter.cancel()
cs.abortRequestBodyWrite(errStopReqBodyWriteAndCancel)
}
cc.forgetStreamID(cs.ID)
return nil, errRequestCanceled
case <-cs.peerReset:
// processResetStream already removed the
@@ -887,6 +933,45 @@ func (cc *ClientConn) RoundTrip(req *http.Request) (*http.Response, error) {
}
}
// awaitOpenSlotForRequest waits until len(streams) < maxConcurrentStreams.
// Must hold cc.mu.
func (cc *ClientConn) awaitOpenSlotForRequest(req *http.Request) error {
var waitingForConn chan struct{}
var waitingForConnErr error // guarded by cc.mu
for {
cc.lastActive = time.Now()
if cc.closed || !cc.canTakeNewRequestLocked() {
return errClientConnUnusable
}
if int64(len(cc.streams))+1 <= int64(cc.maxConcurrentStreams) {
if waitingForConn != nil {
close(waitingForConn)
}
return nil
}
// Unfortunately, we cannot wait on a condition variable and channel at
// the same time, so instead, we spin up a goroutine to check if the
// request is canceled while we wait for a slot to open in the connection.
if waitingForConn == nil {
waitingForConn = make(chan struct{})
go func() {
if err := awaitRequestCancel(req, waitingForConn); err != nil {
cc.mu.Lock()
waitingForConnErr = err
cc.cond.Broadcast()
cc.mu.Unlock()
}
}()
}
cc.pendingRequests++
cc.cond.Wait()
cc.pendingRequests--
if waitingForConnErr != nil {
return waitingForConnErr
}
}
}
// requires cc.wmu be held
func (cc *ClientConn) writeHeaders(streamID uint32, endStream bool, hdrs []byte) error {
first := true // first frame written (HEADERS is first, then CONTINUATION)
@@ -1002,8 +1087,13 @@ func (cs *clientStream) writeRequestBody(body io.Reader, bodyCloser io.Closer) (
var trls []byte
if hasTrailers {
cc.mu.Lock()
defer cc.mu.Unlock()
trls = cc.encodeTrailers(req)
trls, err = cc.encodeTrailers(req)
cc.mu.Unlock()
if err != nil {
cc.writeStreamReset(cs.ID, ErrCodeInternal, err)
cc.forgetStreamID(cs.ID)
return err
}
}
cc.wmu.Lock()
@@ -1106,62 +1196,86 @@ func (cc *ClientConn) encodeHeaders(req *http.Request, addGzipHeader bool, trail
}
}
// 8.1.2.3 Request Pseudo-Header Fields
// The :path pseudo-header field includes the path and query parts of the
// target URI (the path-absolute production and optionally a '?' character
// followed by the query production (see Sections 3.3 and 3.4 of
// [RFC3986]).
cc.writeHeader(":authority", host)
cc.writeHeader(":method", req.Method)
if req.Method != "CONNECT" {
cc.writeHeader(":path", path)
cc.writeHeader(":scheme", req.URL.Scheme)
}
if trailers != "" {
cc.writeHeader("trailer", trailers)
enumerateHeaders := func(f func(name, value string)) {
// 8.1.2.3 Request Pseudo-Header Fields
// The :path pseudo-header field includes the path and query parts of the
// target URI (the path-absolute production and optionally a '?' character
// followed by the query production (see Sections 3.3 and 3.4 of
// [RFC3986]).
f(":authority", host)
f(":method", req.Method)
if req.Method != "CONNECT" {
f(":path", path)
f(":scheme", req.URL.Scheme)
}
if trailers != "" {
f("trailer", trailers)
}
var didUA bool
for k, vv := range req.Header {
if strings.EqualFold(k, "host") || strings.EqualFold(k, "content-length") {
// Host is :authority, already sent.
// Content-Length is automatic, set below.
continue
} else if strings.EqualFold(k, "connection") || strings.EqualFold(k, "proxy-connection") ||
strings.EqualFold(k, "transfer-encoding") || strings.EqualFold(k, "upgrade") ||
strings.EqualFold(k, "keep-alive") {
// Per 8.1.2.2 Connection-Specific Header
// Fields, don't send connection-specific
// fields. We have already checked if any
// are error-worthy so just ignore the rest.
continue
} else if strings.EqualFold(k, "user-agent") {
// Match Go's http1 behavior: at most one
// User-Agent. If set to nil or empty string,
// then omit it. Otherwise if not mentioned,
// include the default (below).
didUA = true
if len(vv) < 1 {
continue
}
vv = vv[:1]
if vv[0] == "" {
continue
}
}
for _, v := range vv {
f(k, v)
}
}
if shouldSendReqContentLength(req.Method, contentLength) {
f("content-length", strconv.FormatInt(contentLength, 10))
}
if addGzipHeader {
f("accept-encoding", "gzip")
}
if !didUA {
f("user-agent", defaultUserAgent)
}
}
var didUA bool
for k, vv := range req.Header {
lowKey := strings.ToLower(k)
switch lowKey {
case "host", "content-length":
// Host is :authority, already sent.
// Content-Length is automatic, set below.
continue
case "connection", "proxy-connection", "transfer-encoding", "upgrade", "keep-alive":
// Per 8.1.2.2 Connection-Specific Header
// Fields, don't send connection-specific
// fields. We have already checked if any
// are error-worthy so just ignore the rest.
continue
case "user-agent":
// Match Go's http1 behavior: at most one
// User-Agent. If set to nil or empty string,
// then omit it. Otherwise if not mentioned,
// include the default (below).
didUA = true
if len(vv) < 1 {
continue
}
vv = vv[:1]
if vv[0] == "" {
continue
}
}
for _, v := range vv {
cc.writeHeader(lowKey, v)
}
}
if shouldSendReqContentLength(req.Method, contentLength) {
cc.writeHeader("content-length", strconv.FormatInt(contentLength, 10))
}
if addGzipHeader {
cc.writeHeader("accept-encoding", "gzip")
}
if !didUA {
cc.writeHeader("user-agent", defaultUserAgent)
// Do a first pass over the headers counting bytes to ensure
// we don't exceed cc.peerMaxHeaderListSize. This is done as a
// separate pass before encoding the headers to prevent
// modifying the hpack state.
hlSize := uint64(0)
enumerateHeaders(func(name, value string) {
hf := hpack.HeaderField{Name: name, Value: value}
hlSize += uint64(hf.Size())
})
if hlSize > cc.peerMaxHeaderListSize {
return nil, errRequestHeaderListSize
}
// Header list size is ok. Write the headers.
enumerateHeaders(func(name, value string) {
cc.writeHeader(strings.ToLower(name), value)
})
return cc.hbuf.Bytes(), nil
}
@@ -1188,17 +1302,29 @@ func shouldSendReqContentLength(method string, contentLength int64) bool {
}
// requires cc.mu be held.
func (cc *ClientConn) encodeTrailers(req *http.Request) []byte {
func (cc *ClientConn) encodeTrailers(req *http.Request) ([]byte, error) {
cc.hbuf.Reset()
hlSize := uint64(0)
for k, vv := range req.Trailer {
// Transfer-Encoding, etc.. have already been filter at the
for _, v := range vv {
hf := hpack.HeaderField{Name: k, Value: v}
hlSize += uint64(hf.Size())
}
}
if hlSize > cc.peerMaxHeaderListSize {
return nil, errRequestHeaderListSize
}
for k, vv := range req.Trailer {
// Transfer-Encoding, etc.. have already been filtered at the
// start of RoundTrip
lowKey := strings.ToLower(k)
for _, v := range vv {
cc.writeHeader(lowKey, v)
}
}
return cc.hbuf.Bytes()
return cc.hbuf.Bytes(), nil
}
func (cc *ClientConn) writeHeader(name, value string) {
@@ -1246,7 +1372,9 @@ func (cc *ClientConn) streamByID(id uint32, andRemove bool) *clientStream {
cc.idleTimer.Reset(cc.idleTimeout)
}
close(cs.done)
cc.cond.Broadcast() // wake up checkResetOrDone via clientStream.awaitFlowControl
// Wake up checkResetOrDone via clientStream.awaitFlowControl and
// wake up RoundTrip if there is a pending request.
cc.cond.Broadcast()
}
return cs
}
@@ -1345,8 +1473,9 @@ func (rl *clientConnReadLoop) run() error {
cc.vlogf("http2: Transport readFrame error on conn %p: (%T) %v", cc, err, err)
}
if se, ok := err.(StreamError); ok {
if cs := cc.streamByID(se.StreamID, true /*ended; remove it*/); cs != nil {
if cs := cc.streamByID(se.StreamID, false); cs != nil {
cs.cc.writeStreamReset(cs.ID, se.Code, err)
cs.cc.forgetStreamID(cs.ID)
if se.Cause == nil {
se.Cause = cc.fr.errDetail
}
@@ -1668,6 +1797,7 @@ func (b transportResponseBody) Close() error {
}
cs.bufPipe.BreakWithError(errClosedResponseBody)
cc.forgetStreamID(cs.ID)
return nil
}
@@ -1702,6 +1832,14 @@ func (rl *clientConnReadLoop) processData(f *DataFrame) error {
}
return nil
}
if !cs.firstByte {
cc.logf("protocol error: received DATA before a HEADERS frame")
rl.endStreamError(cs, StreamError{
StreamID: f.StreamID,
Code: ErrCodeProtocol,
})
return nil
}
if f.Length > 0 {
// Check connection-level flow control.
cc.mu.Lock()
@@ -1816,6 +1954,8 @@ func (rl *clientConnReadLoop) processSettings(f *SettingsFrame) error {
cc.maxFrameSize = s.Val
case SettingMaxConcurrentStreams:
cc.maxConcurrentStreams = s.Val
case SettingMaxHeaderListSize:
cc.peerMaxHeaderListSize = uint64(s.Val)
case SettingInitialWindowSize:
// Values above the maximum flow-control
// window size of 2^31-1 MUST be treated as a
@@ -1982,6 +2122,7 @@ func (cc *ClientConn) writeStreamReset(streamID uint32, code ErrCode, err error)
var (
errResponseHeaderListSize = errors.New("http2: response header list larger than advertised limit")
errRequestHeaderListSize = errors.New("http2: request header list larger than peer's advertised limit")
errPseudoTrailers = errors.New("http2: invalid pseudo header in trailers")
)

View File

@@ -16,6 +16,7 @@ import (
"math/rand"
"net"
"net/http"
"net/http/httptest"
"net/url"
"os"
"reflect"
@@ -685,7 +686,7 @@ func newLocalListener(t *testing.T) net.Listener {
return ln
}
func (ct *clientTester) greet() {
func (ct *clientTester) greet(settings ...Setting) {
buf := make([]byte, len(ClientPreface))
_, err := io.ReadFull(ct.sc, buf)
if err != nil {
@@ -699,7 +700,7 @@ func (ct *clientTester) greet() {
ct.t.Fatalf("Wanted client settings frame; got %v", f)
_ = sf // stash it away?
}
if err := ct.fr.WriteSettings(); err != nil {
if err := ct.fr.WriteSettings(settings...); err != nil {
ct.t.Fatal(err)
}
if err := ct.fr.WriteSettingsAck(); err != nil {
@@ -1370,6 +1371,269 @@ func testInvalidTrailer(t *testing.T, trailers headerType, wantErr error, writeT
ct.run()
}
// headerListSize returns the HTTP2 header list size of h.
// http://httpwg.org/specs/rfc7540.html#SETTINGS_MAX_HEADER_LIST_SIZE
// http://httpwg.org/specs/rfc7540.html#MaxHeaderBlock
func headerListSize(h http.Header) (size uint32) {
for k, vv := range h {
for _, v := range vv {
hf := hpack.HeaderField{Name: k, Value: v}
size += hf.Size()
}
}
return size
}
// padHeaders adds data to an http.Header until headerListSize(h) ==
// limit. Due to the way header list sizes are calculated, padHeaders
// cannot add fewer than len("Pad-Headers") + 32 bytes to h, and will
// call t.Fatal if asked to do so. PadHeaders first reserves enough
// space for an empty "Pad-Headers" key, then adds as many copies of
// filler as possible. Any remaining bytes necessary to push the
// header list size up to limit are added to h["Pad-Headers"].
func padHeaders(t *testing.T, h http.Header, limit uint64, filler string) {
if limit > 0xffffffff {
t.Fatalf("padHeaders: refusing to pad to more than 2^32-1 bytes. limit = %v", limit)
}
hf := hpack.HeaderField{Name: "Pad-Headers", Value: ""}
minPadding := uint64(hf.Size())
size := uint64(headerListSize(h))
minlimit := size + minPadding
if limit < minlimit {
t.Fatalf("padHeaders: limit %v < %v", limit, minlimit)
}
// Use a fixed-width format for name so that fieldSize
// remains constant.
nameFmt := "Pad-Headers-%06d"
hf = hpack.HeaderField{Name: fmt.Sprintf(nameFmt, 1), Value: filler}
fieldSize := uint64(hf.Size())
// Add as many complete filler values as possible, leaving
// room for at least one empty "Pad-Headers" key.
limit = limit - minPadding
for i := 0; size+fieldSize < limit; i++ {
name := fmt.Sprintf(nameFmt, i)
h.Add(name, filler)
size += fieldSize
}
// Add enough bytes to reach limit.
remain := limit - size
lastValue := strings.Repeat("*", int(remain))
h.Add("Pad-Headers", lastValue)
}
func TestPadHeaders(t *testing.T) {
check := func(h http.Header, limit uint32, fillerLen int) {
if h == nil {
h = make(http.Header)
}
filler := strings.Repeat("f", fillerLen)
padHeaders(t, h, uint64(limit), filler)
gotSize := headerListSize(h)
if gotSize != limit {
t.Errorf("Got size = %v; want %v", gotSize, limit)
}
}
// Try all possible combinations for small fillerLen and limit.
hf := hpack.HeaderField{Name: "Pad-Headers", Value: ""}
minLimit := hf.Size()
for limit := minLimit; limit <= 128; limit++ {
for fillerLen := 0; uint32(fillerLen) <= limit; fillerLen++ {
check(nil, limit, fillerLen)
}
}
// Try a few tests with larger limits, plus cumulative
// tests. Since these tests are cumulative, tests[i+1].limit
// must be >= tests[i].limit + minLimit. See the comment on
// padHeaders for more info on why the limit arg has this
// restriction.
tests := []struct {
fillerLen int
limit uint32
}{
{
fillerLen: 64,
limit: 1024,
},
{
fillerLen: 1024,
limit: 1286,
},
{
fillerLen: 256,
limit: 2048,
},
{
fillerLen: 1024,
limit: 10 * 1024,
},
{
fillerLen: 1023,
limit: 11 * 1024,
},
}
h := make(http.Header)
for _, tc := range tests {
check(nil, tc.limit, tc.fillerLen)
check(h, tc.limit, tc.fillerLen)
}
}
func TestTransportChecksRequestHeaderListSize(t *testing.T) {
st := newServerTester(t,
func(w http.ResponseWriter, r *http.Request) {
// Consume body & force client to send
// trailers before writing response.
// ioutil.ReadAll returns non-nil err for
// requests that attempt to send greater than
// maxHeaderListSize bytes of trailers, since
// those requests generate a stream reset.
ioutil.ReadAll(r.Body)
r.Body.Close()
},
func(ts *httptest.Server) {
ts.Config.MaxHeaderBytes = 16 << 10
},
optOnlyServer,
optQuiet,
)
defer st.Close()
tr := &Transport{TLSClientConfig: tlsConfigInsecure}
defer tr.CloseIdleConnections()
checkRoundTrip := func(req *http.Request, wantErr error, desc string) {
res, err := tr.RoundTrip(req)
if err != wantErr {
if res != nil {
res.Body.Close()
}
t.Errorf("%v: RoundTrip err = %v; want %v", desc, err, wantErr)
return
}
if err == nil {
if res == nil {
t.Errorf("%v: response nil; want non-nil.", desc)
return
}
defer res.Body.Close()
if res.StatusCode != http.StatusOK {
t.Errorf("%v: response status = %v; want %v", desc, res.StatusCode, http.StatusOK)
}
return
}
if res != nil {
t.Errorf("%v: RoundTrip err = %v but response non-nil", desc, err)
}
}
headerListSizeForRequest := func(req *http.Request) (size uint64) {
contentLen := actualContentLength(req)
trailers, err := commaSeparatedTrailers(req)
if err != nil {
t.Fatalf("headerListSizeForRequest: %v", err)
}
cc := &ClientConn{peerMaxHeaderListSize: 0xffffffffffffffff}
cc.henc = hpack.NewEncoder(&cc.hbuf)
cc.mu.Lock()
hdrs, err := cc.encodeHeaders(req, true, trailers, contentLen)
cc.mu.Unlock()
if err != nil {
t.Fatalf("headerListSizeForRequest: %v", err)
}
hpackDec := hpack.NewDecoder(initialHeaderTableSize, func(hf hpack.HeaderField) {
size += uint64(hf.Size())
})
if len(hdrs) > 0 {
if _, err := hpackDec.Write(hdrs); err != nil {
t.Fatalf("headerListSizeForRequest: %v", err)
}
}
return size
}
// Create a new Request for each test, rather than reusing the
// same Request, to avoid a race when modifying req.Headers.
// See https://github.com/golang/go/issues/21316
newRequest := func() *http.Request {
// Body must be non-nil to enable writing trailers.
body := strings.NewReader("hello")
req, err := http.NewRequest("POST", st.ts.URL, body)
if err != nil {
t.Fatalf("newRequest: NewRequest: %v", err)
}
return req
}
// Make an arbitrary request to ensure we get the server's
// settings frame and initialize peerMaxHeaderListSize.
req := newRequest()
checkRoundTrip(req, nil, "Initial request")
// Get the ClientConn associated with the request and validate
// peerMaxHeaderListSize.
addr := authorityAddr(req.URL.Scheme, req.URL.Host)
cc, err := tr.connPool().GetClientConn(req, addr)
if err != nil {
t.Fatalf("GetClientConn: %v", err)
}
cc.mu.Lock()
peerSize := cc.peerMaxHeaderListSize
cc.mu.Unlock()
st.scMu.Lock()
wantSize := uint64(st.sc.maxHeaderListSize())
st.scMu.Unlock()
if peerSize != wantSize {
t.Errorf("peerMaxHeaderListSize = %v; want %v", peerSize, wantSize)
}
// Sanity check peerSize. (*serverConn) maxHeaderListSize adds
// 320 bytes of padding.
wantHeaderBytes := uint64(st.ts.Config.MaxHeaderBytes) + 320
if peerSize != wantHeaderBytes {
t.Errorf("peerMaxHeaderListSize = %v; want %v.", peerSize, wantHeaderBytes)
}
// Pad headers & trailers, but stay under peerSize.
req = newRequest()
req.Header = make(http.Header)
req.Trailer = make(http.Header)
filler := strings.Repeat("*", 1024)
padHeaders(t, req.Trailer, peerSize, filler)
// cc.encodeHeaders adds some default headers to the request,
// so we need to leave room for those.
defaultBytes := headerListSizeForRequest(req)
padHeaders(t, req.Header, peerSize-defaultBytes, filler)
checkRoundTrip(req, nil, "Headers & Trailers under limit")
// Add enough header bytes to push us over peerSize.
req = newRequest()
req.Header = make(http.Header)
padHeaders(t, req.Header, peerSize, filler)
checkRoundTrip(req, errRequestHeaderListSize, "Headers over limit")
// Push trailers over the limit.
req = newRequest()
req.Trailer = make(http.Header)
padHeaders(t, req.Trailer, peerSize+1, filler)
checkRoundTrip(req, errRequestHeaderListSize, "Trailers over limit")
// Send headers with a single large value.
req = newRequest()
filler = strings.Repeat("*", int(peerSize))
req.Header = make(http.Header)
req.Header.Set("Big", filler)
checkRoundTrip(req, errRequestHeaderListSize, "Single large header")
// Send trailers with a single large value.
req = newRequest()
req.Trailer = make(http.Header)
req.Trailer.Set("Big", filler)
checkRoundTrip(req, errRequestHeaderListSize, "Single large trailer")
}
func TestTransportChecksResponseHeaderListSize(t *testing.T) {
ct := newClientTester(t)
ct.client = func() error {
@@ -2662,7 +2926,7 @@ func TestTransportRequestPathPseudo(t *testing.T) {
},
}
for i, tt := range tests {
cc := &ClientConn{}
cc := &ClientConn{peerMaxHeaderListSize: 0xffffffffffffffff}
cc.henc = hpack.NewEncoder(&cc.hbuf)
cc.mu.Lock()
hdrs, err := cc.encodeHeaders(tt.req, false, "", -1)
@@ -2926,6 +3190,339 @@ func TestTransportRetryAfterGOAWAY(t *testing.T) {
}
}
func TestTransportRetryAfterRefusedStream(t *testing.T) {
clientDone := make(chan struct{})
ct := newClientTester(t)
ct.client = func() error {
defer ct.cc.(*net.TCPConn).CloseWrite()
defer close(clientDone)
req, _ := http.NewRequest("GET", "https://dummy.tld/", nil)
resp, err := ct.tr.RoundTrip(req)
if err != nil {
return fmt.Errorf("RoundTrip: %v", err)
}
resp.Body.Close()
if resp.StatusCode != 204 {
return fmt.Errorf("Status = %v; want 204", resp.StatusCode)
}
return nil
}
ct.server = func() error {
ct.greet()
var buf bytes.Buffer
enc := hpack.NewEncoder(&buf)
nreq := 0
for {
f, err := ct.fr.ReadFrame()
if err != nil {
select {
case <-clientDone:
// If the client's done, it
// will have reported any
// errors on its side.
return nil
default:
return err
}
}
switch f := f.(type) {
case *WindowUpdateFrame, *SettingsFrame:
case *HeadersFrame:
if !f.HeadersEnded() {
return fmt.Errorf("headers should have END_HEADERS be ended: %v", f)
}
nreq++
if nreq == 1 {
ct.fr.WriteRSTStream(f.StreamID, ErrCodeRefusedStream)
} else {
enc.WriteField(hpack.HeaderField{Name: ":status", Value: "204"})
ct.fr.WriteHeaders(HeadersFrameParam{
StreamID: f.StreamID,
EndHeaders: true,
EndStream: true,
BlockFragment: buf.Bytes(),
})
}
default:
return fmt.Errorf("Unexpected client frame %v", f)
}
}
}
ct.run()
}
func TestTransportRetryHasLimit(t *testing.T) {
// Skip in short mode because the total expected delay is 1s+2s+4s+8s+16s=29s.
if testing.Short() {
t.Skip("skipping long test in short mode")
}
clientDone := make(chan struct{})
ct := newClientTester(t)
ct.client = func() error {
defer ct.cc.(*net.TCPConn).CloseWrite()
defer close(clientDone)
req, _ := http.NewRequest("GET", "https://dummy.tld/", nil)
resp, err := ct.tr.RoundTrip(req)
if err == nil {
return fmt.Errorf("RoundTrip expected error, got response: %+v", resp)
}
t.Logf("expected error, got: %v", err)
return nil
}
ct.server = func() error {
ct.greet()
for {
f, err := ct.fr.ReadFrame()
if err != nil {
select {
case <-clientDone:
// If the client's done, it
// will have reported any
// errors on its side.
return nil
default:
return err
}
}
switch f := f.(type) {
case *WindowUpdateFrame, *SettingsFrame:
case *HeadersFrame:
if !f.HeadersEnded() {
return fmt.Errorf("headers should have END_HEADERS be ended: %v", f)
}
ct.fr.WriteRSTStream(f.StreamID, ErrCodeRefusedStream)
default:
return fmt.Errorf("Unexpected client frame %v", f)
}
}
}
ct.run()
}
func TestTransportResponseDataBeforeHeaders(t *testing.T) {
ct := newClientTester(t)
ct.client = func() error {
defer ct.cc.(*net.TCPConn).CloseWrite()
req := httptest.NewRequest("GET", "https://dummy.tld/", nil)
// First request is normal to ensure the check is per stream and not per connection.
_, err := ct.tr.RoundTrip(req)
if err != nil {
return fmt.Errorf("RoundTrip expected no error, got: %v", err)
}
// Second request returns a DATA frame with no HEADERS.
resp, err := ct.tr.RoundTrip(req)
if err == nil {
return fmt.Errorf("RoundTrip expected error, got response: %+v", resp)
}
if err, ok := err.(StreamError); !ok || err.Code != ErrCodeProtocol {
return fmt.Errorf("expected stream PROTOCOL_ERROR, got: %v", err)
}
return nil
}
ct.server = func() error {
ct.greet()
for {
f, err := ct.fr.ReadFrame()
if err == io.EOF {
return nil
} else if err != nil {
return err
}
switch f := f.(type) {
case *WindowUpdateFrame, *SettingsFrame:
case *HeadersFrame:
switch f.StreamID {
case 1:
// Send a valid response to first request.
var buf bytes.Buffer
enc := hpack.NewEncoder(&buf)
enc.WriteField(hpack.HeaderField{Name: ":status", Value: "200"})
ct.fr.WriteHeaders(HeadersFrameParam{
StreamID: f.StreamID,
EndHeaders: true,
EndStream: true,
BlockFragment: buf.Bytes(),
})
case 3:
ct.fr.WriteData(f.StreamID, true, []byte("payload"))
}
default:
return fmt.Errorf("Unexpected client frame %v", f)
}
}
}
ct.run()
}
func TestTransportRequestsStallAtServerLimit(t *testing.T) {
const maxConcurrent = 2
greet := make(chan struct{}) // server sends initial SETTINGS frame
gotRequest := make(chan struct{}) // server received a request
clientDone := make(chan struct{})
// Collect errors from goroutines.
var wg sync.WaitGroup
errs := make(chan error, 100)
defer func() {
wg.Wait()
close(errs)
for err := range errs {
t.Error(err)
}
}()
// We will send maxConcurrent+2 requests. This checker goroutine waits for the
// following stages:
// 1. The first maxConcurrent requests are received by the server.
// 2. The client will cancel the next request
// 3. The server is unblocked so it can service the first maxConcurrent requests
// 4. The client will send the final request
wg.Add(1)
unblockClient := make(chan struct{})
clientRequestCancelled := make(chan struct{})
unblockServer := make(chan struct{})
go func() {
defer wg.Done()
// Stage 1.
for k := 0; k < maxConcurrent; k++ {
<-gotRequest
}
// Stage 2.
close(unblockClient)
<-clientRequestCancelled
// Stage 3: give some time for the final RoundTrip call to be scheduled and
// verify that the final request is not sent.
time.Sleep(50 * time.Millisecond)
select {
case <-gotRequest:
errs <- errors.New("last request did not stall")
close(unblockServer)
return
default:
}
close(unblockServer)
// Stage 4.
<-gotRequest
}()
ct := newClientTester(t)
ct.client = func() error {
var wg sync.WaitGroup
defer func() {
wg.Wait()
close(clientDone)
ct.cc.(*net.TCPConn).CloseWrite()
}()
for k := 0; k < maxConcurrent+2; k++ {
wg.Add(1)
go func(k int) {
defer wg.Done()
// Don't send the second request until after receiving SETTINGS from the server
// to avoid a race where we use the default SettingMaxConcurrentStreams, which
// is much larger than maxConcurrent. We have to send the first request before
// waiting because the first request triggers the dial and greet.
if k > 0 {
<-greet
}
// Block until maxConcurrent requests are sent before sending any more.
if k >= maxConcurrent {
<-unblockClient
}
req, _ := http.NewRequest("GET", fmt.Sprintf("https://dummy.tld/%d", k), nil)
if k == maxConcurrent {
// This request will be canceled.
cancel := make(chan struct{})
req.Cancel = cancel
close(cancel)
_, err := ct.tr.RoundTrip(req)
close(clientRequestCancelled)
if err == nil {
errs <- fmt.Errorf("RoundTrip(%d) should have failed due to cancel", k)
return
}
} else {
resp, err := ct.tr.RoundTrip(req)
if err != nil {
errs <- fmt.Errorf("RoundTrip(%d): %v", k, err)
return
}
ioutil.ReadAll(resp.Body)
resp.Body.Close()
if resp.StatusCode != 204 {
errs <- fmt.Errorf("Status = %v; want 204", resp.StatusCode)
return
}
}
}(k)
}
return nil
}
ct.server = func() error {
var wg sync.WaitGroup
defer wg.Wait()
ct.greet(Setting{SettingMaxConcurrentStreams, maxConcurrent})
// Server write loop.
var buf bytes.Buffer
enc := hpack.NewEncoder(&buf)
writeResp := make(chan uint32, maxConcurrent+1)
wg.Add(1)
go func() {
defer wg.Done()
<-unblockServer
for id := range writeResp {
buf.Reset()
enc.WriteField(hpack.HeaderField{Name: ":status", Value: "204"})
ct.fr.WriteHeaders(HeadersFrameParam{
StreamID: id,
EndHeaders: true,
EndStream: true,
BlockFragment: buf.Bytes(),
})
}
}()
// Server read loop.
var nreq int
for {
f, err := ct.fr.ReadFrame()
if err != nil {
select {
case <-clientDone:
// If the client's done, it will have reported any errors on its side.
return nil
default:
return err
}
}
switch f := f.(type) {
case *WindowUpdateFrame:
case *SettingsFrame:
// Wait for the client SETTINGS ack until ending the greet.
close(greet)
case *HeadersFrame:
if !f.HeadersEnded() {
return fmt.Errorf("headers should have END_HEADERS be ended: %v", f)
}
gotRequest <- struct{}{}
nreq++
writeResp <- f.StreamID
if nreq == maxConcurrent+1 {
close(writeResp)
}
default:
return fmt.Errorf("Unexpected client frame %v", f)
}
}
}
ct.run()
}
func TestAuthorityAddr(t *testing.T) {
tests := []struct {
scheme, authority string
@@ -3039,3 +3636,51 @@ func TestTransportNoBodyMeansNoDATA(t *testing.T) {
}
ct.run()
}
func benchSimpleRoundTrip(b *testing.B, nHeaders int) {
defer disableGoroutineTracking()()
b.ReportAllocs()
st := newServerTester(b,
func(w http.ResponseWriter, r *http.Request) {
},
optOnlyServer,
optQuiet,
)
defer st.Close()
tr := &Transport{TLSClientConfig: tlsConfigInsecure}
defer tr.CloseIdleConnections()
req, err := http.NewRequest("GET", st.ts.URL, nil)
if err != nil {
b.Fatal(err)
}
for i := 0; i < nHeaders; i++ {
name := fmt.Sprint("A-", i)
req.Header.Set(name, "*")
}
b.ResetTimer()
for i := 0; i < b.N; i++ {
res, err := tr.RoundTrip(req)
if err != nil {
if res != nil {
res.Body.Close()
}
b.Fatalf("RoundTrip err = %v; want nil", err)
}
res.Body.Close()
if res.StatusCode != http.StatusOK {
b.Fatalf("Response code = %v; want %v", res.StatusCode, http.StatusOK)
}
}
}
func BenchmarkClientRequestHeaders(b *testing.B) {
b.Run(" 0 Headers", func(b *testing.B) { benchSimpleRoundTrip(b, 0) })
b.Run(" 10 Headers", func(b *testing.B) { benchSimpleRoundTrip(b, 10) })
b.Run(" 100 Headers", func(b *testing.B) { benchSimpleRoundTrip(b, 100) })
b.Run("1000 Headers", func(b *testing.B) { benchSimpleRoundTrip(b, 1000) })
}