dump_test.go

     1  // Copyright 2011 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package httputil
     6  
     7  import (
     8  	"bufio"
     9  	"bytes"
    10  	"context"
    11  	"fmt"
    12  	"io"
    13  	"math/rand"
    14  	"net/http"
    15  	"net/url"
    16  	"runtime"
    17  	"runtime/pprof"
    18  	"strings"
    19  	"testing"
    20  	"time"
    21  )
    22  
    23  type eofReader struct{}
    24  
    25  func (n eofReader) Close() error { return nil }
    26  
    27  func (n eofReader) Read([]byte) (int, error) { return 0, io.EOF }
    28  
    29  type dumpTest struct {
    30  	// Either Req or GetReq can be set/nil but not both.
    31  	Req    *http.Request
    32  	GetReq func() *http.Request
    33  
    34  	Body any // optional []byte or func() io.ReadCloser to populate Req.Body
    35  
    36  	WantDump    string
    37  	WantDumpOut string
    38  	MustError   bool // if true, the test is expected to throw an error
    39  	NoBody      bool // if true, set DumpRequest{,Out} body to false
    40  }
    41  
    42  var dumpTests = []dumpTest{
    43  	// HTTP/1.1 => chunked coding; body; empty trailer
    44  	{
    45  		Req: &http.Request{
    46  			Method: "GET",
    47  			URL: &url.URL{
    48  				Scheme: "http",
    49  				Host:   "www.google.com",
    50  				Path:   "/search",
    51  			},
    52  			ProtoMajor:       1,
    53  			ProtoMinor:       1,
    54  			TransferEncoding: []string{"chunked"},
    55  		},
    56  
    57  		Body: []byte("abcdef"),
    58  
    59  		WantDump: "GET /search HTTP/1.1\r\n" +
    60  			"Host: www.google.com\r\n" +
    61  			"Transfer-Encoding: chunked\r\n\r\n" +
    62  			chunk("abcdef") + chunk(""),
    63  	},
    64  
    65  	// Verify that DumpRequest preserves the HTTP version number, doesn't add a Host,
    66  	// and doesn't add a User-Agent.
    67  	{
    68  		Req: &http.Request{
    69  			Method:     "GET",
    70  			URL:        mustParseURL("/foo"),
    71  			ProtoMajor: 1,
    72  			ProtoMinor: 0,
    73  			Header: http.Header{
    74  				"X-Foo": []string{"X-Bar"},
    75  			},
    76  		},
    77  
    78  		WantDump: "GET /foo HTTP/1.0\r\n" +
    79  			"X-Foo: X-Bar\r\n\r\n",
    80  	},
    81  
    82  	{
    83  		Req: mustNewRequest("GET", "http://example.com/foo", nil),
    84  
    85  		WantDumpOut: "GET /foo HTTP/1.1\r\n" +
    86  			"Host: example.com\r\n" +
    87  			"User-Agent: Go-http-client/1.1\r\n" +
    88  			"Accept-Encoding: gzip\r\n\r\n",
    89  	},
    90  
    91  	// Test that an https URL doesn't try to do an SSL negotiation
    92  	// with a bytes.Buffer and hang with all goroutines not
    93  	// runnable.
    94  	{
    95  		Req: mustNewRequest("GET", "https://example.com/foo", nil),
    96  		WantDumpOut: "GET /foo HTTP/1.1\r\n" +
    97  			"Host: example.com\r\n" +
    98  			"User-Agent: Go-http-client/1.1\r\n" +
    99  			"Accept-Encoding: gzip\r\n\r\n",
   100  	},
   101  
   102  	// Request with Body, but Dump requested without it.
   103  	{
   104  		Req: &http.Request{
   105  			Method: "POST",
   106  			URL: &url.URL{
   107  				Scheme: "http",
   108  				Host:   "post.tld",
   109  				Path:   "/",
   110  			},
   111  			ContentLength: 6,
   112  			ProtoMajor:    1,
   113  			ProtoMinor:    1,
   114  		},
   115  
   116  		Body: []byte("abcdef"),
   117  
   118  		WantDumpOut: "POST / HTTP/1.1\r\n" +
   119  			"Host: post.tld\r\n" +
   120  			"User-Agent: Go-http-client/1.1\r\n" +
   121  			"Content-Length: 6\r\n" +
   122  			"Accept-Encoding: gzip\r\n\r\n",
   123  
   124  		NoBody: true,
   125  	},
   126  
   127  	// Request with Body > 8196 (default buffer size)
   128  	{
   129  		Req: &http.Request{
   130  			Method: "POST",
   131  			URL: &url.URL{
   132  				Scheme: "http",
   133  				Host:   "post.tld",
   134  				Path:   "/",
   135  			},
   136  			Header: http.Header{
   137  				"Content-Length": []string{"8193"},
   138  			},
   139  
   140  			ContentLength: 8193,
   141  			ProtoMajor:    1,
   142  			ProtoMinor:    1,
   143  		},
   144  
   145  		Body: bytes.Repeat([]byte("a"), 8193),
   146  
   147  		WantDumpOut: "POST / HTTP/1.1\r\n" +
   148  			"Host: post.tld\r\n" +
   149  			"User-Agent: Go-http-client/1.1\r\n" +
   150  			"Content-Length: 8193\r\n" +
   151  			"Accept-Encoding: gzip\r\n\r\n" +
   152  			strings.Repeat("a", 8193),
   153  		WantDump: "POST / HTTP/1.1\r\n" +
   154  			"Host: post.tld\r\n" +
   155  			"Content-Length: 8193\r\n\r\n" +
   156  			strings.Repeat("a", 8193),
   157  	},
   158  
   159  	{
   160  		GetReq: func() *http.Request {
   161  			return mustReadRequest("GET http://foo.com/ HTTP/1.1\r\n" +
   162  				"User-Agent: blah\r\n\r\n")
   163  		},
   164  		NoBody: true,
   165  		WantDump: "GET http://foo.com/ HTTP/1.1\r\n" +
   166  			"User-Agent: blah\r\n\r\n",
   167  	},
   168  
   169  	// Issue #7215. DumpRequest should return the "Content-Length" when set
   170  	{
   171  		GetReq: func() *http.Request {
   172  			return mustReadRequest("POST /v2/api/?login HTTP/1.1\r\n" +
   173  				"Host: passport.myhost.com\r\n" +
   174  				"Content-Length: 3\r\n" +
   175  				"\r\nkey1=name1&key2=name2")
   176  		},
   177  		WantDump: "POST /v2/api/?login HTTP/1.1\r\n" +
   178  			"Host: passport.myhost.com\r\n" +
   179  			"Content-Length: 3\r\n" +
   180  			"\r\nkey",
   181  	},
   182  	// Issue #7215. DumpRequest should return the "Content-Length" in ReadRequest
   183  	{
   184  		GetReq: func() *http.Request {
   185  			return mustReadRequest("POST /v2/api/?login HTTP/1.1\r\n" +
   186  				"Host: passport.myhost.com\r\n" +
   187  				"Content-Length: 0\r\n" +
   188  				"\r\nkey1=name1&key2=name2")
   189  		},
   190  		WantDump: "POST /v2/api/?login HTTP/1.1\r\n" +
   191  			"Host: passport.myhost.com\r\n" +
   192  			"Content-Length: 0\r\n\r\n",
   193  	},
   194  
   195  	// Issue #7215. DumpRequest should not return the "Content-Length" if unset
   196  	{
   197  		GetReq: func() *http.Request {
   198  			return mustReadRequest("POST /v2/api/?login HTTP/1.1\r\n" +
   199  				"Host: passport.myhost.com\r\n" +
   200  				"\r\nkey1=name1&key2=name2")
   201  		},
   202  		WantDump: "POST /v2/api/?login HTTP/1.1\r\n" +
   203  			"Host: passport.myhost.com\r\n\r\n",
   204  	},
   205  
   206  	// Issue 18506: make drainBody recognize NoBody. Otherwise
   207  	// this was turning into a chunked request.
   208  	{
   209  		Req: mustNewRequest("POST", "http://example.com/foo", http.NoBody),
   210  		WantDumpOut: "POST /foo HTTP/1.1\r\n" +
   211  			"Host: example.com\r\n" +
   212  			"User-Agent: Go-http-client/1.1\r\n" +
   213  			"Content-Length: 0\r\n" +
   214  			"Accept-Encoding: gzip\r\n\r\n",
   215  	},
   216  
   217  	// Issue 34504: a non-nil Body without ContentLength set should be chunked
   218  	{
   219  		Req: &http.Request{
   220  			Method: "PUT",
   221  			URL: &url.URL{
   222  				Scheme: "http",
   223  				Host:   "post.tld",
   224  				Path:   "/test",
   225  			},
   226  			ContentLength: 0,
   227  			Proto:         "HTTP/1.1",
   228  			ProtoMajor:    1,
   229  			ProtoMinor:    1,
   230  			Body:          &eofReader{},
   231  		},
   232  		NoBody: true,
   233  		WantDumpOut: "PUT /test HTTP/1.1\r\n" +
   234  			"Host: post.tld\r\n" +
   235  			"User-Agent: Go-http-client/1.1\r\n" +
   236  			"Transfer-Encoding: chunked\r\n" +
   237  			"Accept-Encoding: gzip\r\n\r\n",
   238  	},
   239  }
   240  
   241  func TestDumpRequest(t *testing.T) {
   242  	// Make a copy of dumpTests and add 10 new cases with an empty URL
   243  	// to test that no goroutines are leaked. See golang.org/issue/32571.
   244  	// 10 seems to be a decent number which always triggers the failure.
   245  	dumpTests := dumpTests[:]
   246  	for i := 0; i < 10; i++ {
   247  		dumpTests = append(dumpTests, dumpTest{
   248  			Req:       mustNewRequest("GET", "", nil),
   249  			MustError: true,
   250  		})
   251  	}
   252  	numg0 := runtime.NumGoroutine()
   253  	for i, tt := range dumpTests {
   254  		if tt.Req != nil && tt.GetReq != nil || tt.Req == nil && tt.GetReq == nil {
   255  			t.Errorf("#%d: either .Req(%p) or .GetReq(%p) can be set/nil but not both", i, tt.Req, tt.GetReq)
   256  			continue
   257  		}
   258  
   259  		freshReq := func(ti dumpTest) *http.Request {
   260  			req := ti.Req
   261  			if req == nil {
   262  				req = ti.GetReq()
   263  			}
   264  
   265  			if req.Header == nil {
   266  				req.Header = make(http.Header)
   267  			}
   268  
   269  			if ti.Body == nil {
   270  				return req
   271  			}
   272  			switch b := ti.Body.(type) {
   273  			case []byte:
   274  				req.Body = io.NopCloser(bytes.NewReader(b))
   275  			case func() io.ReadCloser:
   276  				req.Body = b()
   277  			default:
   278  				t.Fatalf("Test %d: unsupported Body of %T", i, ti.Body)
   279  			}
   280  			return req
   281  		}
   282  
   283  		if tt.WantDump != "" {
   284  			req := freshReq(tt)
   285  			dump, err := DumpRequest(req, !tt.NoBody)
   286  			if err != nil {
   287  				t.Errorf("DumpRequest #%d: %s\nWantDump:\n%s", i, err, tt.WantDump)
   288  				continue
   289  			}
   290  			if string(dump) != tt.WantDump {
   291  				t.Errorf("DumpRequest %d, expecting:\n%s\nGot:\n%s\n", i, tt.WantDump, string(dump))
   292  				continue
   293  			}
   294  		}
   295  
   296  		if tt.MustError {
   297  			req := freshReq(tt)
   298  			_, err := DumpRequestOut(req, !tt.NoBody)
   299  			if err == nil {
   300  				t.Errorf("DumpRequestOut #%d: expected an error, got nil", i)
   301  			}
   302  			continue
   303  		}
   304  
   305  		if tt.WantDumpOut != "" {
   306  			req := freshReq(tt)
   307  			dump, err := DumpRequestOut(req, !tt.NoBody)
   308  			if err != nil {
   309  				t.Errorf("DumpRequestOut #%d: %s", i, err)
   310  				continue
   311  			}
   312  			if string(dump) != tt.WantDumpOut {
   313  				t.Errorf("DumpRequestOut %d, expecting:\n%s\nGot:\n%s\n", i, tt.WantDumpOut, string(dump))
   314  				continue
   315  			}
   316  		}
   317  	}
   318  
   319  	// Validate we haven't leaked any goroutines.
   320  	var dg int
   321  	dl := deadline(t, 5*time.Second, time.Second)
   322  	for time.Now().Before(dl) {
   323  		if dg = runtime.NumGoroutine() - numg0; dg <= 4 {
   324  			// No unexpected goroutines.
   325  			return
   326  		}
   327  
   328  		// Allow goroutines to schedule and die off.
   329  		runtime.Gosched()
   330  	}
   331  
   332  	buf := make([]byte, 4096)
   333  	buf = buf[:runtime.Stack(buf, true)]
   334  	t.Errorf("Unexpectedly large number of new goroutines: %d new: %s", dg, buf)
   335  }
   336  
   337  // deadline returns the time which is needed before t.Deadline()
   338  // if one is configured and it is s greater than needed in the future,
   339  // otherwise defaultDelay from the current time.
   340  func deadline(t *testing.T, defaultDelay, needed time.Duration) time.Time {
   341  	if dl, ok := t.Deadline(); ok {
   342  		if dl = dl.Add(-needed); dl.After(time.Now()) {
   343  			// Allow an arbitrarily long delay.
   344  			return dl
   345  		}
   346  	}
   347  
   348  	// No deadline configured or its closer than needed from now
   349  	// so just use the default.
   350  	return time.Now().Add(defaultDelay)
   351  }
   352  
   353  func chunk(s string) string {
   354  	return fmt.Sprintf("%x\r\n%s\r\n", len(s), s)
   355  }
   356  
   357  func mustParseURL(s string) *url.URL {
   358  	u, err := url.Parse(s)
   359  	if err != nil {
   360  		panic(fmt.Sprintf("Error parsing URL %q: %v", s, err))
   361  	}
   362  	return u
   363  }
   364  
   365  func mustNewRequest(method, url string, body io.Reader) *http.Request {
   366  	req, err := http.NewRequest(method, url, body)
   367  	if err != nil {
   368  		panic(fmt.Sprintf("NewRequest(%q, %q, %p) err = %v", method, url, body, err))
   369  	}
   370  	return req
   371  }
   372  
   373  func mustReadRequest(s string) *http.Request {
   374  	req, err := http.ReadRequest(bufio.NewReader(strings.NewReader(s)))
   375  	if err != nil {
   376  		panic(err)
   377  	}
   378  	return req
   379  }
   380  
   381  var dumpResTests = []struct {
   382  	res  *http.Response
   383  	body bool
   384  	want string
   385  }{
   386  	{
   387  		res: &http.Response{
   388  			Status:        "200 OK",
   389  			StatusCode:    200,
   390  			Proto:         "HTTP/1.1",
   391  			ProtoMajor:    1,
   392  			ProtoMinor:    1,
   393  			ContentLength: 50,
   394  			Header: http.Header{
   395  				"Foo": []string{"Bar"},
   396  			},
   397  			Body: io.NopCloser(strings.NewReader("foo")), // shouldn't be used
   398  		},
   399  		body: false, // to verify we see 50, not empty or 3.
   400  		want: `HTTP/1.1 200 OK
   401  Content-Length: 50
   402  Foo: Bar`,
   403  	},
   404  
   405  	{
   406  		res: &http.Response{
   407  			Status:        "200 OK",
   408  			StatusCode:    200,
   409  			Proto:         "HTTP/1.1",
   410  			ProtoMajor:    1,
   411  			ProtoMinor:    1,
   412  			ContentLength: 3,
   413  			Body:          io.NopCloser(strings.NewReader("foo")),
   414  		},
   415  		body: true,
   416  		want: `HTTP/1.1 200 OK
   417  Content-Length: 3
   418  
   419  foo`,
   420  	},
   421  
   422  	{
   423  		res: &http.Response{
   424  			Status:           "200 OK",
   425  			StatusCode:       200,
   426  			Proto:            "HTTP/1.1",
   427  			ProtoMajor:       1,
   428  			ProtoMinor:       1,
   429  			ContentLength:    -1,
   430  			Body:             io.NopCloser(strings.NewReader("foo")),
   431  			TransferEncoding: []string{"chunked"},
   432  		},
   433  		body: true,
   434  		want: `HTTP/1.1 200 OK
   435  Transfer-Encoding: chunked
   436  
   437  3
   438  foo
   439  0`,
   440  	},
   441  	{
   442  		res: &http.Response{
   443  			Status:        "200 OK",
   444  			StatusCode:    200,
   445  			Proto:         "HTTP/1.1",
   446  			ProtoMajor:    1,
   447  			ProtoMinor:    1,
   448  			ContentLength: 0,
   449  			Header: http.Header{
   450  				// To verify if headers are not filtered out.
   451  				"Foo1": []string{"Bar1"},
   452  				"Foo2": []string{"Bar2"},
   453  			},
   454  			Body: nil,
   455  		},
   456  		body: false, // to verify we see 0, not empty.
   457  		want: `HTTP/1.1 200 OK
   458  Foo1: Bar1
   459  Foo2: Bar2
   460  Content-Length: 0`,
   461  	},
   462  }
   463  
   464  func TestDumpResponse(t *testing.T) {
   465  	for i, tt := range dumpResTests {
   466  		gotb, err := DumpResponse(tt.res, tt.body)
   467  		if err != nil {
   468  			t.Errorf("%d. DumpResponse = %v", i, err)
   469  			continue
   470  		}
   471  		got := string(gotb)
   472  		got = strings.TrimSpace(got)
   473  		got = strings.ReplaceAll(got, "\r", "")
   474  
   475  		if got != tt.want {
   476  			t.Errorf("%d.\nDumpResponse got:\n%s\n\nWant:\n%s\n", i, got, tt.want)
   477  		}
   478  	}
   479  }
   480  
   481  // Issue 38352: Check for deadlock on canceled requests.
   482  func TestDumpRequestOutIssue38352(t *testing.T) {
   483  	if testing.Short() {
   484  		return
   485  	}
   486  	t.Parallel()
   487  
   488  	timeout := 10 * time.Second
   489  	if deadline, ok := t.Deadline(); ok {
   490  		timeout = time.Until(deadline)
   491  		timeout -= time.Second * 2 // Leave 2 seconds to report failures.
   492  	}
   493  	for i := 0; i < 1000; i++ {
   494  		delay := time.Duration(rand.Intn(5)) * time.Millisecond
   495  		ctx, cancel := context.WithTimeout(context.Background(), delay)
   496  		defer cancel()
   497  
   498  		r := bytes.NewBuffer(make([]byte, 10000))
   499  		req, err := http.NewRequestWithContext(ctx, http.MethodPost, "http://example.com", r)
   500  		if err != nil {
   501  			t.Fatal(err)
   502  		}
   503  
   504  		out := make(chan error)
   505  		go func() {
   506  			_, err = DumpRequestOut(req, true)
   507  			out <- err
   508  		}()
   509  
   510  		select {
   511  		case <-out:
   512  		case <-time.After(timeout):
   513  			b := &bytes.Buffer{}
   514  			fmt.Fprintf(b, "deadlock detected on iteration %d after %s with delay: %v\n", i, timeout, delay)
   515  			pprof.Lookup("goroutine").WriteTo(b, 1)
   516  			t.Fatal(b.String())
   517  		}
   518  	}
   519  }
   520
View as plain text