Source file src/net/http/internal/chunked.go

     1  // Copyright 2009 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  // The wire protocol for HTTP's "chunked" Transfer-Encoding.
     6  
     7  // Package internal contains HTTP internals shared by net/http and
     8  // net/http/httputil.
     9  package internal
    10  
    11  import (
    12  	"bufio"
    13  	"bytes"
    14  	"errors"
    15  	"fmt"
    16  	"io"
    17  )
    18  
    19  const maxLineLength = 4096 // assumed <= bufio.defaultBufSize
    20  
    21  var ErrLineTooLong = errors.New("header line too long")
    22  
    23  // NewChunkedReader returns a new chunkedReader that translates the data read from r
    24  // out of HTTP "chunked" format before returning it.
    25  // The chunkedReader returns io.EOF when the final 0-length chunk is read.
    26  //
    27  // NewChunkedReader is not needed by normal applications. The http package
    28  // automatically decodes chunking when reading response bodies.
    29  func NewChunkedReader(r io.Reader) io.Reader {
    30  	br, ok := r.(*bufio.Reader)
    31  	if !ok {
    32  		br = bufio.NewReader(r)
    33  	}
    34  	return &chunkedReader{r: br}
    35  }
    36  
    37  type chunkedReader struct {
    38  	r        *bufio.Reader
    39  	n        uint64 // unread bytes in chunk
    40  	err      error
    41  	buf      [2]byte
    42  	checkEnd bool // whether need to check for \r\n chunk footer
    43  }
    44  
    45  func (cr *chunkedReader) beginChunk() {
    46  	// chunk-size CRLF
    47  	var line []byte
    48  	line, cr.err = readChunkLine(cr.r)
    49  	if cr.err != nil {
    50  		return
    51  	}
    52  	cr.n, cr.err = parseHexUint(line)
    53  	if cr.err != nil {
    54  		return
    55  	}
    56  	if cr.n == 0 {
    57  		cr.err = io.EOF
    58  	}
    59  }
    60  
    61  func (cr *chunkedReader) chunkHeaderAvailable() bool {
    62  	n := cr.r.Buffered()
    63  	if n > 0 {
    64  		peek, _ := cr.r.Peek(n)
    65  		return bytes.IndexByte(peek, '\n') >= 0
    66  	}
    67  	return false
    68  }
    69  
    70  func (cr *chunkedReader) Read(b []uint8) (n int, err error) {
    71  	for cr.err == nil {
    72  		if cr.checkEnd {
    73  			if n > 0 && cr.r.Buffered() < 2 {
    74  				// We have some data. Return early (per the io.Reader
    75  				// contract) instead of potentially blocking while
    76  				// reading more.
    77  				break
    78  			}
    79  			if _, cr.err = io.ReadFull(cr.r, cr.buf[:2]); cr.err == nil {
    80  				if string(cr.buf[:]) != "\r\n" {
    81  					cr.err = errors.New("malformed chunked encoding")
    82  					break
    83  				}
    84  			} else {
    85  				if cr.err == io.EOF {
    86  					cr.err = io.ErrUnexpectedEOF
    87  				}
    88  				break
    89  			}
    90  			cr.checkEnd = false
    91  		}
    92  		if cr.n == 0 {
    93  			if n > 0 && !cr.chunkHeaderAvailable() {
    94  				// We've read enough. Don't potentially block
    95  				// reading a new chunk header.
    96  				break
    97  			}
    98  			cr.beginChunk()
    99  			continue
   100  		}
   101  		if len(b) == 0 {
   102  			break
   103  		}
   104  		rbuf := b
   105  		if uint64(len(rbuf)) > cr.n {
   106  			rbuf = rbuf[:cr.n]
   107  		}
   108  		var n0 int
   109  		n0, cr.err = cr.r.Read(rbuf)
   110  		n += n0
   111  		b = b[n0:]
   112  		cr.n -= uint64(n0)
   113  		// If we're at the end of a chunk, read the next two
   114  		// bytes to verify they are "\r\n".
   115  		if cr.n == 0 && cr.err == nil {
   116  			cr.checkEnd = true
   117  		} else if cr.err == io.EOF {
   118  			cr.err = io.ErrUnexpectedEOF
   119  		}
   120  	}
   121  	return n, cr.err
   122  }
   123  
   124  // Read a line of bytes (up to \n) from b.
   125  // Give up if the line exceeds maxLineLength.
   126  // The returned bytes are owned by the bufio.Reader
   127  // so they are only valid until the next bufio read.
   128  func readChunkLine(b *bufio.Reader) ([]byte, error) {
   129  	p, err := b.ReadSlice('\n')
   130  	if err != nil {
   131  		// We always know when EOF is coming.
   132  		// If the caller asked for a line, there should be a line.
   133  		if err == io.EOF {
   134  			err = io.ErrUnexpectedEOF
   135  		} else if err == bufio.ErrBufferFull {
   136  			err = ErrLineTooLong
   137  		}
   138  		return nil, err
   139  	}
   140  	if len(p) >= maxLineLength {
   141  		return nil, ErrLineTooLong
   142  	}
   143  	p = trimTrailingWhitespace(p)
   144  	p, err = removeChunkExtension(p)
   145  	if err != nil {
   146  		return nil, err
   147  	}
   148  	return p, nil
   149  }
   150  
   151  func trimTrailingWhitespace(b []byte) []byte {
   152  	for len(b) > 0 && isASCIISpace(b[len(b)-1]) {
   153  		b = b[:len(b)-1]
   154  	}
   155  	return b
   156  }
   157  
   158  func isASCIISpace(b byte) bool {
   159  	return b == ' ' || b == '\t' || b == '\n' || b == '\r'
   160  }
   161  
   162  var semi = []byte(";")
   163  
   164  // removeChunkExtension removes any chunk-extension from p.
   165  // For example,
   166  //     "0" => "0"
   167  //     "0;token" => "0"
   168  //     "0;token=val" => "0"
   169  //     `0;token="quoted string"` => "0"
   170  func removeChunkExtension(p []byte) ([]byte, error) {
   171  	p, _, _ = bytes.Cut(p, semi)
   172  	// TODO: care about exact syntax of chunk extensions? We're
   173  	// ignoring and stripping them anyway. For now just never
   174  	// return an error.
   175  	return p, nil
   176  }
   177  
   178  // NewChunkedWriter returns a new chunkedWriter that translates writes into HTTP
   179  // "chunked" format before writing them to w. Closing the returned chunkedWriter
   180  // sends the final 0-length chunk that marks the end of the stream but does
   181  // not send the final CRLF that appears after trailers; trailers and the last
   182  // CRLF must be written separately.
   183  //
   184  // NewChunkedWriter is not needed by normal applications. The http
   185  // package adds chunking automatically if handlers don't set a
   186  // Content-Length header. Using newChunkedWriter inside a handler
   187  // would result in double chunking or chunking with a Content-Length
   188  // length, both of which are wrong.
   189  func NewChunkedWriter(w io.Writer) io.WriteCloser {
   190  	return &chunkedWriter{w}
   191  }
   192  
   193  // Writing to chunkedWriter translates to writing in HTTP chunked Transfer
   194  // Encoding wire format to the underlying Wire chunkedWriter.
   195  type chunkedWriter struct {
   196  	Wire io.Writer
   197  }
   198  
   199  // Write the contents of data as one chunk to Wire.
   200  // NOTE: Note that the corresponding chunk-writing procedure in Conn.Write has
   201  // a bug since it does not check for success of io.WriteString
   202  func (cw *chunkedWriter) Write(data []byte) (n int, err error) {
   203  
   204  	// Don't send 0-length data. It looks like EOF for chunked encoding.
   205  	if len(data) == 0 {
   206  		return 0, nil
   207  	}
   208  
   209  	if _, err = fmt.Fprintf(cw.Wire, "%x\r\n", len(data)); err != nil {
   210  		return 0, err
   211  	}
   212  	if n, err = cw.Wire.Write(data); err != nil {
   213  		return
   214  	}
   215  	if n != len(data) {
   216  		err = io.ErrShortWrite
   217  		return
   218  	}
   219  	if _, err = io.WriteString(cw.Wire, "\r\n"); err != nil {
   220  		return
   221  	}
   222  	if bw, ok := cw.Wire.(*FlushAfterChunkWriter); ok {
   223  		err = bw.Flush()
   224  	}
   225  	return
   226  }
   227  
   228  func (cw *chunkedWriter) Close() error {
   229  	_, err := io.WriteString(cw.Wire, "0\r\n")
   230  	return err
   231  }
   232  
   233  // FlushAfterChunkWriter signals from the caller of NewChunkedWriter
   234  // that each chunk should be followed by a flush. It is used by the
   235  // http.Transport code to keep the buffering behavior for headers and
   236  // trailers, but flush out chunks aggressively in the middle for
   237  // request bodies which may be generated slowly. See Issue 6574.
   238  type FlushAfterChunkWriter struct {
   239  	*bufio.Writer
   240  }
   241  
   242  func parseHexUint(v []byte) (n uint64, err error) {
   243  	for i, b := range v {
   244  		switch {
   245  		case '0' <= b && b <= '9':
   246  			b = b - '0'
   247  		case 'a' <= b && b <= 'f':
   248  			b = b - 'a' + 10
   249  		case 'A' <= b && b <= 'F':
   250  			b = b - 'A' + 10
   251  		default:
   252  			return 0, errors.New("invalid byte in chunk length")
   253  		}
   254  		if i == 16 {
   255  			return 0, errors.New("http chunk length too large")
   256  		}
   257  		n <<= 4
   258  		n |= uint64(b)
   259  	}
   260  	return
   261  }
   262  

View as plain text