Source file src/pkg/net/http/transfer.go

     1	// Copyright 2009 The Go Authors. All rights reserved.
     2	// Use of this source code is governed by a BSD-style
     3	// license that can be found in the LICENSE file.
     4	
     5	package http
     6	
     7	import (
     8		"bufio"
     9		"bytes"
    10		"errors"
    11		"fmt"
    12		"io"
    13		"io/ioutil"
    14		"net/textproto"
    15		"strconv"
    16		"strings"
    17	)
    18	
    19	// transferWriter inspects the fields of a user-supplied Request or Response,
    20	// sanitizes them without changing the user object and provides methods for
    21	// writing the respective header, body and trailer in wire format.
    22	type transferWriter struct {
    23		Method           string
    24		Body             io.Reader
    25		BodyCloser       io.Closer
    26		ResponseToHEAD   bool
    27		ContentLength    int64 // -1 means unknown, 0 means exactly none
    28		Close            bool
    29		TransferEncoding []string
    30		Trailer          Header
    31	}
    32	
    33	func newTransferWriter(r interface{}) (t *transferWriter, err error) {
    34		t = &transferWriter{}
    35	
    36		// Extract relevant fields
    37		atLeastHTTP11 := false
    38		switch rr := r.(type) {
    39		case *Request:
    40			if rr.ContentLength != 0 && rr.Body == nil {
    41				return nil, fmt.Errorf("http: Request.ContentLength=%d with nil Body", rr.ContentLength)
    42			}
    43			t.Method = rr.Method
    44			t.Body = rr.Body
    45			t.BodyCloser = rr.Body
    46			t.ContentLength = rr.ContentLength
    47			t.Close = rr.Close
    48			t.TransferEncoding = rr.TransferEncoding
    49			t.Trailer = rr.Trailer
    50			atLeastHTTP11 = rr.ProtoAtLeast(1, 1)
    51			if t.Body != nil && len(t.TransferEncoding) == 0 && atLeastHTTP11 {
    52				if t.ContentLength == 0 {
    53					// Test to see if it's actually zero or just unset.
    54					var buf [1]byte
    55					n, _ := io.ReadFull(t.Body, buf[:])
    56					if n == 1 {
    57						// Oh, guess there is data in this Body Reader after all.
    58						// The ContentLength field just wasn't set.
    59						// Stich the Body back together again, re-attaching our
    60						// consumed byte.
    61						t.ContentLength = -1
    62						t.Body = io.MultiReader(bytes.NewBuffer(buf[:]), t.Body)
    63					} else {
    64						// Body is actually empty.
    65						t.Body = nil
    66						t.BodyCloser = nil
    67					}
    68				}
    69				if t.ContentLength < 0 {
    70					t.TransferEncoding = []string{"chunked"}
    71				}
    72			}
    73		case *Response:
    74			t.Method = rr.Request.Method
    75			t.Body = rr.Body
    76			t.BodyCloser = rr.Body
    77			t.ContentLength = rr.ContentLength
    78			t.Close = rr.Close
    79			t.TransferEncoding = rr.TransferEncoding
    80			t.Trailer = rr.Trailer
    81			atLeastHTTP11 = rr.ProtoAtLeast(1, 1)
    82			t.ResponseToHEAD = noBodyExpected(rr.Request.Method)
    83		}
    84	
    85		// Sanitize Body,ContentLength,TransferEncoding
    86		if t.ResponseToHEAD {
    87			t.Body = nil
    88			t.TransferEncoding = nil
    89			// ContentLength is expected to hold Content-Length
    90			if t.ContentLength < 0 {
    91				return nil, ErrMissingContentLength
    92			}
    93		} else {
    94			if !atLeastHTTP11 || t.Body == nil {
    95				t.TransferEncoding = nil
    96			}
    97			if chunked(t.TransferEncoding) {
    98				t.ContentLength = -1
    99			} else if t.Body == nil { // no chunking, no body
   100				t.ContentLength = 0
   101			}
   102		}
   103	
   104		// Sanitize Trailer
   105		if !chunked(t.TransferEncoding) {
   106			t.Trailer = nil
   107		}
   108	
   109		return t, nil
   110	}
   111	
   112	func noBodyExpected(requestMethod string) bool {
   113		return requestMethod == "HEAD"
   114	}
   115	
   116	func (t *transferWriter) shouldSendContentLength() bool {
   117		if chunked(t.TransferEncoding) {
   118			return false
   119		}
   120		if t.ContentLength > 0 {
   121			return true
   122		}
   123		if t.ResponseToHEAD {
   124			return true
   125		}
   126		// Many servers expect a Content-Length for these methods
   127		if t.Method == "POST" || t.Method == "PUT" {
   128			return true
   129		}
   130		if t.ContentLength == 0 && isIdentity(t.TransferEncoding) {
   131			return true
   132		}
   133	
   134		return false
   135	}
   136	
   137	func (t *transferWriter) WriteHeader(w io.Writer) (err error) {
   138		if t.Close {
   139			_, err = io.WriteString(w, "Connection: close\r\n")
   140			if err != nil {
   141				return
   142			}
   143		}
   144	
   145		// Write Content-Length and/or Transfer-Encoding whose values are a
   146		// function of the sanitized field triple (Body, ContentLength,
   147		// TransferEncoding)
   148		if t.shouldSendContentLength() {
   149			io.WriteString(w, "Content-Length: ")
   150			_, err = io.WriteString(w, strconv.FormatInt(t.ContentLength, 10)+"\r\n")
   151			if err != nil {
   152				return
   153			}
   154		} else if chunked(t.TransferEncoding) {
   155			_, err = io.WriteString(w, "Transfer-Encoding: chunked\r\n")
   156			if err != nil {
   157				return
   158			}
   159		}
   160	
   161		// Write Trailer header
   162		if t.Trailer != nil {
   163			// TODO: At some point, there should be a generic mechanism for
   164			// writing long headers, using HTTP line splitting
   165			io.WriteString(w, "Trailer: ")
   166			needComma := false
   167			for k := range t.Trailer {
   168				k = CanonicalHeaderKey(k)
   169				switch k {
   170				case "Transfer-Encoding", "Trailer", "Content-Length":
   171					return &badStringError{"invalid Trailer key", k}
   172				}
   173				if needComma {
   174					io.WriteString(w, ",")
   175				}
   176				io.WriteString(w, k)
   177				needComma = true
   178			}
   179			_, err = io.WriteString(w, "\r\n")
   180		}
   181	
   182		return
   183	}
   184	
   185	func (t *transferWriter) WriteBody(w io.Writer) (err error) {
   186		var ncopy int64
   187	
   188		// Write body
   189		if t.Body != nil {
   190			if chunked(t.TransferEncoding) {
   191				cw := newChunkedWriter(w)
   192				_, err = io.Copy(cw, t.Body)
   193				if err == nil {
   194					err = cw.Close()
   195				}
   196			} else if t.ContentLength == -1 {
   197				ncopy, err = io.Copy(w, t.Body)
   198			} else {
   199				ncopy, err = io.Copy(w, io.LimitReader(t.Body, t.ContentLength))
   200				nextra, err := io.Copy(ioutil.Discard, t.Body)
   201				if err != nil {
   202					return err
   203				}
   204				ncopy += nextra
   205			}
   206			if err != nil {
   207				return err
   208			}
   209			if err = t.BodyCloser.Close(); err != nil {
   210				return err
   211			}
   212		}
   213	
   214		if t.ContentLength != -1 && t.ContentLength != ncopy {
   215			return fmt.Errorf("http: Request.ContentLength=%d with Body length %d",
   216				t.ContentLength, ncopy)
   217		}
   218	
   219		// TODO(petar): Place trailer writer code here.
   220		if chunked(t.TransferEncoding) {
   221			// Last chunk, empty trailer
   222			_, err = io.WriteString(w, "\r\n")
   223		}
   224	
   225		return
   226	}
   227	
   228	type transferReader struct {
   229		// Input
   230		Header        Header
   231		StatusCode    int
   232		RequestMethod string
   233		ProtoMajor    int
   234		ProtoMinor    int
   235		// Output
   236		Body             io.ReadCloser
   237		ContentLength    int64
   238		TransferEncoding []string
   239		Close            bool
   240		Trailer          Header
   241	}
   242	
   243	// bodyAllowedForStatus returns whether a given response status code
   244	// permits a body.  See RFC2616, section 4.4.
   245	func bodyAllowedForStatus(status int) bool {
   246		switch {
   247		case status >= 100 && status <= 199:
   248			return false
   249		case status == 204:
   250			return false
   251		case status == 304:
   252			return false
   253		}
   254		return true
   255	}
   256	
   257	// msg is *Request or *Response.
   258	func readTransfer(msg interface{}, r *bufio.Reader) (err error) {
   259		t := &transferReader{}
   260	
   261		// Unify input
   262		isResponse := false
   263		switch rr := msg.(type) {
   264		case *Response:
   265			t.Header = rr.Header
   266			t.StatusCode = rr.StatusCode
   267			t.RequestMethod = rr.Request.Method
   268			t.ProtoMajor = rr.ProtoMajor
   269			t.ProtoMinor = rr.ProtoMinor
   270			t.Close = shouldClose(t.ProtoMajor, t.ProtoMinor, t.Header)
   271			isResponse = true
   272		case *Request:
   273			t.Header = rr.Header
   274			t.ProtoMajor = rr.ProtoMajor
   275			t.ProtoMinor = rr.ProtoMinor
   276			// Transfer semantics for Requests are exactly like those for
   277			// Responses with status code 200, responding to a GET method
   278			t.StatusCode = 200
   279			t.RequestMethod = "GET"
   280		default:
   281			panic("unexpected type")
   282		}
   283	
   284		// Default to HTTP/1.1
   285		if t.ProtoMajor == 0 && t.ProtoMinor == 0 {
   286			t.ProtoMajor, t.ProtoMinor = 1, 1
   287		}
   288	
   289		// Transfer encoding, content length
   290		t.TransferEncoding, err = fixTransferEncoding(t.RequestMethod, t.Header)
   291		if err != nil {
   292			return err
   293		}
   294	
   295		t.ContentLength, err = fixLength(isResponse, t.StatusCode, t.RequestMethod, t.Header, t.TransferEncoding)
   296		if err != nil {
   297			return err
   298		}
   299	
   300		// Trailer
   301		t.Trailer, err = fixTrailer(t.Header, t.TransferEncoding)
   302		if err != nil {
   303			return err
   304		}
   305	
   306		// If there is no Content-Length or chunked Transfer-Encoding on a *Response
   307		// and the status is not 1xx, 204 or 304, then the body is unbounded.
   308		// See RFC2616, section 4.4.
   309		switch msg.(type) {
   310		case *Response:
   311			if t.ContentLength == -1 &&
   312				!chunked(t.TransferEncoding) &&
   313				bodyAllowedForStatus(t.StatusCode) {
   314				// Unbounded body.
   315				t.Close = true
   316			}
   317		}
   318	
   319		// Prepare body reader.  ContentLength < 0 means chunked encoding
   320		// or close connection when finished, since multipart is not supported yet
   321		switch {
   322		case chunked(t.TransferEncoding):
   323			t.Body = &body{Reader: newChunkedReader(r), hdr: msg, r: r, closing: t.Close}
   324		case t.ContentLength >= 0:
   325			// TODO: limit the Content-Length. This is an easy DoS vector.
   326			t.Body = &body{Reader: io.LimitReader(r, t.ContentLength), closing: t.Close}
   327		default:
   328			// t.ContentLength < 0, i.e. "Content-Length" not mentioned in header
   329			if t.Close {
   330				// Close semantics (i.e. HTTP/1.0)
   331				t.Body = &body{Reader: r, closing: t.Close}
   332			} else {
   333				// Persistent connection (i.e. HTTP/1.1)
   334				t.Body = &body{Reader: io.LimitReader(r, 0), closing: t.Close}
   335			}
   336		}
   337	
   338		// Unify output
   339		switch rr := msg.(type) {
   340		case *Request:
   341			rr.Body = t.Body
   342			rr.ContentLength = t.ContentLength
   343			rr.TransferEncoding = t.TransferEncoding
   344			rr.Close = t.Close
   345			rr.Trailer = t.Trailer
   346		case *Response:
   347			rr.Body = t.Body
   348			rr.ContentLength = t.ContentLength
   349			rr.TransferEncoding = t.TransferEncoding
   350			rr.Close = t.Close
   351			rr.Trailer = t.Trailer
   352		}
   353	
   354		return nil
   355	}
   356	
   357	// Checks whether chunked is part of the encodings stack
   358	func chunked(te []string) bool { return len(te) > 0 && te[0] == "chunked" }
   359	
   360	// Checks whether the encoding is explicitly "identity".
   361	func isIdentity(te []string) bool { return len(te) == 1 && te[0] == "identity" }
   362	
   363	// Sanitize transfer encoding
   364	func fixTransferEncoding(requestMethod string, header Header) ([]string, error) {
   365		raw, present := header["Transfer-Encoding"]
   366		if !present {
   367			return nil, nil
   368		}
   369	
   370		delete(header, "Transfer-Encoding")
   371	
   372		// Head responses have no bodies, so the transfer encoding
   373		// should be ignored.
   374		if requestMethod == "HEAD" {
   375			return nil, nil
   376		}
   377	
   378		encodings := strings.Split(raw[0], ",")
   379		te := make([]string, 0, len(encodings))
   380		// TODO: Even though we only support "identity" and "chunked"
   381		// encodings, the loop below is designed with foresight. One
   382		// invariant that must be maintained is that, if present,
   383		// chunked encoding must always come first.
   384		for _, encoding := range encodings {
   385			encoding = strings.ToLower(strings.TrimSpace(encoding))
   386			// "identity" encoding is not recorded
   387			if encoding == "identity" {
   388				break
   389			}
   390			if encoding != "chunked" {
   391				return nil, &badStringError{"unsupported transfer encoding", encoding}
   392			}
   393			te = te[0 : len(te)+1]
   394			te[len(te)-1] = encoding
   395		}
   396		if len(te) > 1 {
   397			return nil, &badStringError{"too many transfer encodings", strings.Join(te, ",")}
   398		}
   399		if len(te) > 0 {
   400			// Chunked encoding trumps Content-Length. See RFC 2616
   401			// Section 4.4. Currently len(te) > 0 implies chunked
   402			// encoding.
   403			delete(header, "Content-Length")
   404			return te, nil
   405		}
   406	
   407		return nil, nil
   408	}
   409	
   410	// Determine the expected body length, using RFC 2616 Section 4.4. This
   411	// function is not a method, because ultimately it should be shared by
   412	// ReadResponse and ReadRequest.
   413	func fixLength(isResponse bool, status int, requestMethod string, header Header, te []string) (int64, error) {
   414	
   415		// Logic based on response type or status
   416		if noBodyExpected(requestMethod) {
   417			return 0, nil
   418		}
   419		if status/100 == 1 {
   420			return 0, nil
   421		}
   422		switch status {
   423		case 204, 304:
   424			return 0, nil
   425		}
   426	
   427		// Logic based on Transfer-Encoding
   428		if chunked(te) {
   429			return -1, nil
   430		}
   431	
   432		// Logic based on Content-Length
   433		cl := strings.TrimSpace(header.Get("Content-Length"))
   434		if cl != "" {
   435			n, err := strconv.ParseInt(cl, 10, 64)
   436			if err != nil || n < 0 {
   437				return -1, &badStringError{"bad Content-Length", cl}
   438			}
   439			return n, nil
   440		} else {
   441			header.Del("Content-Length")
   442		}
   443	
   444		if !isResponse && requestMethod == "GET" {
   445			// RFC 2616 doesn't explicitly permit nor forbid an
   446			// entity-body on a GET request so we permit one if
   447			// declared, but we default to 0 here (not -1 below)
   448			// if there's no mention of a body.
   449			return 0, nil
   450		}
   451	
   452		// Logic based on media type. The purpose of the following code is just
   453		// to detect whether the unsupported "multipart/byteranges" is being
   454		// used. A proper Content-Type parser is needed in the future.
   455		if strings.Contains(strings.ToLower(header.Get("Content-Type")), "multipart/byteranges") {
   456			return -1, ErrNotSupported
   457		}
   458	
   459		// Body-EOF logic based on other methods (like closing, or chunked coding)
   460		return -1, nil
   461	}
   462	
   463	// Determine whether to hang up after sending a request and body, or
   464	// receiving a response and body
   465	// 'header' is the request headers
   466	func shouldClose(major, minor int, header Header) bool {
   467		if major < 1 {
   468			return true
   469		} else if major == 1 && minor == 0 {
   470			if !strings.Contains(strings.ToLower(header.Get("Connection")), "keep-alive") {
   471				return true
   472			}
   473			return false
   474		} else {
   475			// TODO: Should split on commas, toss surrounding white space,
   476			// and check each field.
   477			if strings.ToLower(header.Get("Connection")) == "close" {
   478				header.Del("Connection")
   479				return true
   480			}
   481		}
   482		return false
   483	}
   484	
   485	// Parse the trailer header
   486	func fixTrailer(header Header, te []string) (Header, error) {
   487		raw := header.Get("Trailer")
   488		if raw == "" {
   489			return nil, nil
   490		}
   491	
   492		header.Del("Trailer")
   493		trailer := make(Header)
   494		keys := strings.Split(raw, ",")
   495		for _, key := range keys {
   496			key = CanonicalHeaderKey(strings.TrimSpace(key))
   497			switch key {
   498			case "Transfer-Encoding", "Trailer", "Content-Length":
   499				return nil, &badStringError{"bad trailer key", key}
   500			}
   501			trailer.Del(key)
   502		}
   503		if len(trailer) == 0 {
   504			return nil, nil
   505		}
   506		if !chunked(te) {
   507			// Trailer and no chunking
   508			return nil, ErrUnexpectedTrailer
   509		}
   510		return trailer, nil
   511	}
   512	
   513	// body turns a Reader into a ReadCloser.
   514	// Close ensures that the body has been fully read
   515	// and then reads the trailer if necessary.
   516	type body struct {
   517		io.Reader
   518		hdr     interface{}   // non-nil (Response or Request) value means read trailer
   519		r       *bufio.Reader // underlying wire-format reader for the trailer
   520		closing bool          // is the connection to be closed after reading body?
   521		closed  bool
   522	
   523		res *response // response writer for server requests, else nil
   524	}
   525	
   526	// ErrBodyReadAfterClose is returned when reading a Request Body after
   527	// the body has been closed. This typically happens when the body is
   528	// read after an HTTP Handler calls WriteHeader or Write on its
   529	// ResponseWriter.
   530	var ErrBodyReadAfterClose = errors.New("http: invalid Read on closed request Body")
   531	
   532	func (b *body) Read(p []byte) (n int, err error) {
   533		if b.closed {
   534			return 0, ErrBodyReadAfterClose
   535		}
   536		n, err = b.Reader.Read(p)
   537	
   538		// Read the final trailer once we hit EOF.
   539		if err == io.EOF && b.hdr != nil {
   540			if e := b.readTrailer(); e != nil {
   541				err = e
   542			}
   543			b.hdr = nil
   544		}
   545		return n, err
   546	}
   547	
   548	var (
   549		singleCRLF = []byte("\r\n")
   550		doubleCRLF = []byte("\r\n\r\n")
   551	)
   552	
   553	func seeUpcomingDoubleCRLF(r *bufio.Reader) bool {
   554		for peekSize := 4; ; peekSize++ {
   555			// This loop stops when Peek returns an error,
   556			// which it does when r's buffer has been filled.
   557			buf, err := r.Peek(peekSize)
   558			if bytes.HasSuffix(buf, doubleCRLF) {
   559				return true
   560			}
   561			if err != nil {
   562				break
   563			}
   564		}
   565		return false
   566	}
   567	
   568	func (b *body) readTrailer() error {
   569		// The common case, since nobody uses trailers.
   570		buf, _ := b.r.Peek(2)
   571		if bytes.Equal(buf, singleCRLF) {
   572			b.r.ReadByte()
   573			b.r.ReadByte()
   574			return nil
   575		}
   576	
   577		// Make sure there's a header terminator coming up, to prevent
   578		// a DoS with an unbounded size Trailer.  It's not easy to
   579		// slip in a LimitReader here, as textproto.NewReader requires
   580		// a concrete *bufio.Reader.  Also, we can't get all the way
   581		// back up to our conn's LimitedReader that *might* be backing
   582		// this bufio.Reader.  Instead, a hack: we iteratively Peek up
   583		// to the bufio.Reader's max size, looking for a double CRLF.
   584		// This limits the trailer to the underlying buffer size, typically 4kB.
   585		if !seeUpcomingDoubleCRLF(b.r) {
   586			return errors.New("http: suspiciously long trailer after chunked body")
   587		}
   588	
   589		hdr, err := textproto.NewReader(b.r).ReadMIMEHeader()
   590		if err != nil {
   591			return err
   592		}
   593		switch rr := b.hdr.(type) {
   594		case *Request:
   595			rr.Trailer = Header(hdr)
   596		case *Response:
   597			rr.Trailer = Header(hdr)
   598		}
   599		return nil
   600	}
   601	
   602	func (b *body) Close() error {
   603		if b.closed {
   604			return nil
   605		}
   606		defer func() {
   607			b.closed = true
   608		}()
   609		if b.hdr == nil && b.closing {
   610			// no trailer and closing the connection next.
   611			// no point in reading to EOF.
   612			return nil
   613		}
   614	
   615		// In a server request, don't continue reading from the client
   616		// if we've already hit the maximum body size set by the
   617		// handler. If this is set, that also means the TCP connection
   618		// is about to be closed, so getting to the next HTTP request
   619		// in the stream is not necessary.
   620		if b.res != nil && b.res.requestBodyLimitHit {
   621			return nil
   622		}
   623	
   624		// Fully consume the body, which will also lead to us reading
   625		// the trailer headers after the body, if present.
   626		if _, err := io.Copy(ioutil.Discard, b); err != nil {
   627			return err
   628		}
   629		return nil
   630	}