netramesh

1
// Copyright 2009 The Go Authors. All rights reserved.
2
// Use of this source code is governed by a BSD-style
3
// license that can be found in the LICENSE file.
4

5
// HTTP client. See RFC 7230 through 7235.
6
//
7
// This is the high-level Client interface.
8
// The low-level implementation is in transport.go.
9

10
package http
11

12
import (
13
	"crypto/tls"
14
	"encoding/base64"
15
	"errors"
16
	"fmt"
17
	"io"
18
	"io/ioutil"
19
	"log"
20
	"net/url"
21
	"sort"
22
	"strings"
23
	"sync"
24
	"time"
25
)
26

27
// A Client is an HTTP client. Its zero value (DefaultClient) is a
28
// usable client that uses DefaultTransport.
29
//
30
// The Client's Transport typically has internal state (cached TCP
31
// connections), so Clients should be reused instead of created as
32
// needed. Clients are safe for concurrent use by multiple goroutines.
33
//
34
// A Client is higher-level than a RoundTripper (such as Transport)
35
// and additionally handles HTTP details such as cookies and
36
// redirects.
37
//
38
// When following redirects, the Client will forward all headers set on the
39
// initial Request except:
40
//
41
// • when forwarding sensitive headers like "Authorization",
42
// "WWW-Authenticate", and "Cookie" to untrusted targets.
43
// These headers will be ignored when following a redirect to a domain
44
// that is not a subdomain match or exact match of the initial domain.
45
// For example, a redirect from "foo.com" to either "foo.com" or "sub.foo.com"
46
// will forward the sensitive headers, but a redirect to "bar.com" will not.
47
//
48
// • when forwarding the "Cookie" header with a non-nil cookie Jar.
49
// Since each redirect may mutate the state of the cookie jar,
50
// a redirect may possibly alter a cookie set in the initial request.
51
// When forwarding the "Cookie" header, any mutated cookies will be omitted,
52
// with the expectation that the Jar will insert those mutated cookies
53
// with the updated values (assuming the origin matches).
54
// If Jar is nil, the initial cookies are forwarded without change.
55
//
56
type Client struct {
57
	// Transport specifies the mechanism by which individual
58
	// HTTP requests are made.
59
	// If nil, DefaultTransport is used.
60
	Transport RoundTripper
61

62
	// CheckRedirect specifies the policy for handling redirects.
63
	// If CheckRedirect is not nil, the client calls it before
64
	// following an HTTP redirect. The arguments req and via are
65
	// the upcoming request and the requests made already, oldest
66
	// first. If CheckRedirect returns an error, the Client's Get
67
	// method returns both the previous Response (with its Body
68
	// closed) and CheckRedirect's error (wrapped in a url.Error)
69
	// instead of issuing the Request req.
70
	// As a special case, if CheckRedirect returns ErrUseLastResponse,
71
	// then the most recent response is returned with its body
72
	// unclosed, along with a nil error.
73
	//
74
	// If CheckRedirect is nil, the Client uses its default policy,
75
	// which is to stop after 10 consecutive requests.
76
	CheckRedirect func(req *Request, via []*Request) error
77

78
	// Jar specifies the cookie jar.
79
	//
80
	// The Jar is used to insert relevant cookies into every
81
	// outbound Request and is updated with the cookie values
82
	// of every inbound Response. The Jar is consulted for every
83
	// redirect that the Client follows.
84
	//
85
	// If Jar is nil, cookies are only sent if they are explicitly
86
	// set on the Request.
87
	Jar CookieJar
88

89
	// Timeout specifies a time limit for requests made by this
90
	// Client. The timeout includes connection time, any
91
	// redirects, and reading the response body. The timer remains
92
	// running after Get, Head, Post, or Do return and will
93
	// interrupt reading of the Response.Body.
94
	//
95
	// A Timeout of zero means no timeout.
96
	//
97
	// The Client cancels requests to the underlying Transport
98
	// as if the Request's Context ended.
99
	//
100
	// For compatibility, the Client will also use the deprecated
101
	// CancelRequest method on Transport if found. New
102
	// RoundTripper implementations should use the Request's Context
103
	// for cancelation instead of implementing CancelRequest.
104
	Timeout time.Duration
105
}
106

107
// DefaultClient is the default Client and is used by Get, Head, and Post.
108
var DefaultClient = &Client{}
109

110
// RoundTripper is an interface representing the ability to execute a
111
// single HTTP transaction, obtaining the Response for a given Request.
112
//
113
// A RoundTripper must be safe for concurrent use by multiple
114
// goroutines.
115
type RoundTripper interface {
116
	// RoundTrip executes a single HTTP transaction, returning
117
	// a Response for the provided Request.
118
	//
119
	// RoundTrip should not attempt to interpret the response. In
120
	// particular, RoundTrip must return err == nil if it obtained
121
	// a response, regardless of the response's HTTP status code.
122
	// A non-nil err should be reserved for failure to obtain a
123
	// response. Similarly, RoundTrip should not attempt to
124
	// handle higher-level protocol details such as redirects,
125
	// authentication, or cookies.
126
	//
127
	// RoundTrip should not modify the request, except for
128
	// consuming and closing the Request's Body. RoundTrip may
129
	// read fields of the request in a separate goroutine. Callers
130
	// should not mutate or reuse the request until the Response's
131
	// Body has been closed.
132
	//
133
	// RoundTrip must always close the body, including on errors,
134
	// but depending on the implementation may do so in a separate
135
	// goroutine even after RoundTrip returns. This means that
136
	// callers wanting to reuse the body for subsequent requests
137
	// must arrange to wait for the Close call before doing so.
138
	//
139
	// The Request's URL and Header fields must be initialized.
140
	RoundTrip(*Request) (*Response, error)
141
}
142

143
// refererForURL returns a referer without any authentication info or
144
// an empty string if lastReq scheme is https and newReq scheme is http.
145
func refererForURL(lastReq, newReq *url.URL) string {
146
	// https://tools.ietf.org/html/rfc7231#section-5.5.2
147
	//   "Clients SHOULD NOT include a Referer header field in a
148
	//    (non-secure) HTTP request if the referring page was
149
	//    transferred with a secure protocol."
150
	if lastReq.Scheme == "https" && newReq.Scheme == "http" {
151
		return ""
152
	}
153
	referer := lastReq.String()
154
	if lastReq.User != nil {
155
		// This is not very efficient, but is the best we can
156
		// do without:
157
		// - introducing a new method on URL
158
		// - creating a race condition
159
		// - copying the URL struct manually, which would cause
160
		//   maintenance problems down the line
161
		auth := lastReq.User.String() + "@"
162
		referer = strings.Replace(referer, auth, "", 1)
163
	}
164
	return referer
165
}
166

167
// didTimeout is non-nil only if err != nil.
168
func (c *Client) send(req *Request, deadline time.Time) (resp *Response, didTimeout func() bool, err error) {
169
	if c.Jar != nil {
170
		for _, cookie := range c.Jar.Cookies(req.URL) {
171
			req.AddCookie(cookie)
172
		}
173
	}
174
	resp, didTimeout, err = send(req, c.transport(), deadline)
175
	if err != nil {
176
		return nil, didTimeout, err
177
	}
178
	if c.Jar != nil {
179
		if rc := resp.Cookies(); len(rc) > 0 {
180
			c.Jar.SetCookies(req.URL, rc)
181
		}
182
	}
183
	return resp, nil, nil
184
}
185

186
func (c *Client) deadline() time.Time {
187
	if c.Timeout > 0 {
188
		return time.Now().Add(c.Timeout)
189
	}
190
	return time.Time{}
191
}
192

193
func (c *Client) transport() RoundTripper {
194
	if c.Transport != nil {
195
		return c.Transport
196
	}
197
	return DefaultTransport
198
}
199

200
// send issues an HTTP request.
201
// Caller should close resp.Body when done reading from it.
202
func send(ireq *Request, rt RoundTripper, deadline time.Time) (resp *Response, didTimeout func() bool, err error) {
203
	req := ireq // req is either the original request, or a modified fork
204

205
	if rt == nil {
206
		req.closeBody()
207
		return nil, alwaysFalse, errors.New("http: no Client.Transport or DefaultTransport")
208
	}
209

210
	if req.URL == nil {
211
		req.closeBody()
212
		return nil, alwaysFalse, errors.New("http: nil Request.URL")
213
	}
214

215
	if req.RequestURI != "" {
216
		req.closeBody()
217
		return nil, alwaysFalse, errors.New("http: Request.RequestURI can't be set in client requests.")
218
	}
219

220
	// forkReq forks req into a shallow clone of ireq the first
221
	// time it's called.
222
	forkReq := func() {
223
		if ireq == req {
224
			req = new(Request)
225
			*req = *ireq // shallow clone
226
		}
227
	}
228

229
	// Most the callers of send (Get, Post, et al) don't need
230
	// Headers, leaving it uninitialized. We guarantee to the
231
	// Transport that this has been initialized, though.
232
	if req.Header == nil {
233
		forkReq()
234
		req.Header = make(Header)
235
	}
236

237
	if u := req.URL.User; u != nil && req.Header.Get("Authorization") == "" {
238
		username := u.Username()
239
		password, _ := u.Password()
240
		forkReq()
241
		req.Header = cloneHeader(ireq.Header)
242
		req.Header.Set("Authorization", "Basic "+basicAuth(username, password))
243
	}
244

245
	if !deadline.IsZero() {
246
		forkReq()
247
	}
248
	stopTimer, didTimeout := setRequestCancel(req, rt, deadline)
249

250
	resp, err = rt.RoundTrip(req)
251
	if err != nil {
252
		stopTimer()
253
		if resp != nil {
254
			log.Printf("RoundTripper returned a response & error; ignoring response")
255
		}
256
		if tlsErr, ok := err.(tls.RecordHeaderError); ok {
257
			// If we get a bad TLS record header, check to see if the
258
			// response looks like HTTP and give a more helpful error.
259
			// See golang.org/issue/11111.
260
			if string(tlsErr.RecordHeader[:]) == "HTTP/" {
261
				err = errors.New("http: server gave HTTP response to HTTPS client")
262
			}
263
		}
264
		return nil, didTimeout, err
265
	}
266
	if !deadline.IsZero() {
267
		resp.Body = &cancelTimerBody{
268
			stop:          stopTimer,
269
			rc:            resp.Body,
270
			reqDidTimeout: didTimeout,
271
		}
272
	}
273
	return resp, nil, nil
274
}
275

276
// setRequestCancel sets the Cancel field of req, if deadline is
277
// non-zero. The RoundTripper's type is used to determine whether the legacy
278
// CancelRequest behavior should be used.
279
//
280
// As background, there are three ways to cancel a request:
281
// First was Transport.CancelRequest. (deprecated)
282
// Second was Request.Cancel (this mechanism).
283
// Third was Request.Context.
284
func setRequestCancel(req *Request, rt RoundTripper, deadline time.Time) (stopTimer func(), didTimeout func() bool) {
285
	if deadline.IsZero() {
286
		return nop, alwaysFalse
287
	}
288

289
	initialReqCancel := req.Cancel // the user's original Request.Cancel, if any
290

291
	cancel := make(chan struct{})
292
	req.Cancel = cancel
293

294
	doCancel := func() {
295
		// The newer way (the second way in the func comment):
296
		close(cancel)
297

298
		// The legacy compatibility way, used only
299
		// for RoundTripper implementations written
300
		// before Go 1.5 or Go 1.6.
301
		type canceler interface {
302
			CancelRequest(*Request)
303
		}
304
		switch v := rt.(type) {
305
		case *Transport, *http2Transport:
306
			// Do nothing. The net/http package's transports
307
			// support the new Request.Cancel channel
308
		case canceler:
309
			v.CancelRequest(req)
310
		}
311
	}
312

313
	stopTimerCh := make(chan struct{})
314
	var once sync.Once
315
	stopTimer = func() { once.Do(func() { close(stopTimerCh) }) }
316

317
	timer := time.NewTimer(time.Until(deadline))
318
	var timedOut atomicBool
319

320
	go func() {
321
		select {
322
		case <-initialReqCancel:
323
			doCancel()
324
			timer.Stop()
325
		case <-timer.C:
326
			timedOut.setTrue()
327
			doCancel()
328
		case <-stopTimerCh:
329
			timer.Stop()
330
		}
331
	}()
332

333
	return stopTimer, timedOut.isSet
334
}
335

336
// See 2 (end of page 4) https://www.ietf.org/rfc/rfc2617.txt
337
// "To receive authorization, the client sends the userid and password,
338
// separated by a single colon (":") character, within a base64
339
// encoded string in the credentials."
340
// It is not meant to be urlencoded.
341
func basicAuth(username, password string) string {
342
	auth := username + ":" + password
343
	return base64.StdEncoding.EncodeToString([]byte(auth))
344
}
345

346
// Get issues a GET to the specified URL. If the response is one of
347
// the following redirect codes, Get follows the redirect, up to a
348
// maximum of 10 redirects:
349
//
350
//    301 (Moved Permanently)
351
//    302 (Found)
352
//    303 (See Other)
353
//    307 (Temporary Redirect)
354
//    308 (Permanent Redirect)
355
//
356
// An error is returned if there were too many redirects or if there
357
// was an HTTP protocol error. A non-2xx response doesn't cause an
358
// error. Any returned error will be of type *url.Error. The url.Error
359
// value's Timeout method will report true if request timed out or was
360
// canceled.
361
//
362
// When err is nil, resp always contains a non-nil resp.Body.
363
// Caller should close resp.Body when done reading from it.
364
//
365
// Get is a wrapper around DefaultClient.Get.
366
//
367
// To make a request with custom headers, use NewRequest and
368
// DefaultClient.Do.
369
func Get(url string) (resp *Response, err error) {
370
	return DefaultClient.Get(url)
371
}
372

373
// Get issues a GET to the specified URL. If the response is one of the
374
// following redirect codes, Get follows the redirect after calling the
375
// Client's CheckRedirect function:
376
//
377
//    301 (Moved Permanently)
378
//    302 (Found)
379
//    303 (See Other)
380
//    307 (Temporary Redirect)
381
//    308 (Permanent Redirect)
382
//
383
// An error is returned if the Client's CheckRedirect function fails
384
// or if there was an HTTP protocol error. A non-2xx response doesn't
385
// cause an error. Any returned error will be of type *url.Error. The
386
// url.Error value's Timeout method will report true if request timed
387
// out or was canceled.
388
//
389
// When err is nil, resp always contains a non-nil resp.Body.
390
// Caller should close resp.Body when done reading from it.
391
//
392
// To make a request with custom headers, use NewRequest and Client.Do.
393
func (c *Client) Get(url string) (resp *Response, err error) {
394
	req, err := NewRequest("GET", url, nil)
395
	if err != nil {
396
		return nil, err
397
	}
398
	return c.Do(req)
399
}
400

401
func alwaysFalse() bool { return false }
402

403
// ErrUseLastResponse can be returned by Client.CheckRedirect hooks to
404
// control how redirects are processed. If returned, the next request
405
// is not sent and the most recent response is returned with its body
406
// unclosed.
407
var ErrUseLastResponse = errors.New("net/http: use last response")
408

409
// checkRedirect calls either the user's configured CheckRedirect
410
// function, or the default.
411
func (c *Client) checkRedirect(req *Request, via []*Request) error {
412
	fn := c.CheckRedirect
413
	if fn == nil {
414
		fn = defaultCheckRedirect
415
	}
416
	return fn(req, via)
417
}
418

419
// redirectBehavior describes what should happen when the
420
// client encounters a 3xx status code from the server
421
func redirectBehavior(reqMethod string, resp *Response, ireq *Request) (redirectMethod string, shouldRedirect, includeBody bool) {
422
	switch resp.StatusCode {
423
	case 301, 302, 303:
424
		redirectMethod = reqMethod
425
		shouldRedirect = true
426
		includeBody = false
427

428
		// RFC 2616 allowed automatic redirection only with GET and
429
		// HEAD requests. RFC 7231 lifts this restriction, but we still
430
		// restrict other methods to GET to maintain compatibility.
431
		// See Issue 18570.
432
		if reqMethod != "GET" && reqMethod != "HEAD" {
433
			redirectMethod = "GET"
434
		}
435
	case 307, 308:
436
		redirectMethod = reqMethod
437
		shouldRedirect = true
438
		includeBody = true
439

440
		// Treat 307 and 308 specially, since they're new in
441
		// Go 1.8, and they also require re-sending the request body.
442
		if resp.Header.Get("Location") == "" {
443
			// 308s have been observed in the wild being served
444
			// without Location headers. Since Go 1.7 and earlier
445
			// didn't follow these codes, just stop here instead
446
			// of returning an error.
447
			// See Issue 17773.
448
			shouldRedirect = false
449
			break
450
		}
451
		if ireq.GetBody == nil && ireq.outgoingLength() != 0 {
452
			// We had a request body, and 307/308 require
453
			// re-sending it, but GetBody is not defined. So just
454
			// return this response to the user instead of an
455
			// error, like we did in Go 1.7 and earlier.
456
			shouldRedirect = false
457
		}
458
	}
459
	return redirectMethod, shouldRedirect, includeBody
460
}
461

462
// urlErrorOp returns the (*url.Error).Op value to use for the
463
// provided (*Request).Method value.
464
func urlErrorOp(method string) string {
465
	if method == "" {
466
		return "Get"
467
	}
468
	return method[:1] + strings.ToLower(method[1:])
469
}
470

471
// Do sends an HTTP request and returns an HTTP response, following
472
// policy (such as redirects, cookies, auth) as configured on the
473
// client.
474
//
475
// An error is returned if caused by client policy (such as
476
// CheckRedirect), or failure to speak HTTP (such as a network
477
// connectivity problem). A non-2xx status code doesn't cause an
478
// error.
479
//
480
// If the returned error is nil, the Response will contain a non-nil
481
// Body which the user is expected to close. If the Body is not
482
// closed, the Client's underlying RoundTripper (typically Transport)
483
// may not be able to re-use a persistent TCP connection to the server
484
// for a subsequent "keep-alive" request.
485
//
486
// The request Body, if non-nil, will be closed by the underlying
487
// Transport, even on errors.
488
//
489
// On error, any Response can be ignored. A non-nil Response with a
490
// non-nil error only occurs when CheckRedirect fails, and even then
491
// the returned Response.Body is already closed.
492
//
493
// Generally Get, Post, or PostForm will be used instead of Do.
494
//
495
// If the server replies with a redirect, the Client first uses the
496
// CheckRedirect function to determine whether the redirect should be
497
// followed. If permitted, a 301, 302, or 303 redirect causes
498
// subsequent requests to use HTTP method GET
499
// (or HEAD if the original request was HEAD), with no body.
500
// A 307 or 308 redirect preserves the original HTTP method and body,
501
// provided that the Request.GetBody function is defined.
502
// The NewRequest function automatically sets GetBody for common
503
// standard library body types.
504
//
505
// Any returned error will be of type *url.Error. The url.Error
506
// value's Timeout method will report true if request timed out or was
507
// canceled.
508
func (c *Client) Do(req *Request) (*Response, error) {
509
	return c.do(req)
510
}
511

512
var testHookClientDoResult func(retres *Response, reterr error)
513

514
func (c *Client) do(req *Request) (retres *Response, reterr error) {
515
	if testHookClientDoResult != nil {
516
		defer func() { testHookClientDoResult(retres, reterr) }()
517
	}
518
	if req.URL == nil {
519
		req.closeBody()
520
		return nil, &url.Error{
521
			Op:  urlErrorOp(req.Method),
522
			Err: errors.New("http: nil Request.URL"),
523
		}
524
	}
525

526
	var (
527
		deadline      = c.deadline()
528
		reqs          []*Request
529
		resp          *Response
530
		copyHeaders   = c.makeHeadersCopier(req)
531
		reqBodyClosed = false // have we closed the current req.Body?
532

533
		// Redirect behavior:
534
		redirectMethod string
535
		includeBody    bool
536
	)
537
	uerr := func(err error) error {
538
		// the body may have been closed already by c.send()
539
		if !reqBodyClosed {
540
			req.closeBody()
541
		}
542
		var urlStr string
543
		if resp != nil && resp.Request != nil {
544
			urlStr = stripPassword(resp.Request.URL)
545
		} else {
546
			urlStr = stripPassword(req.URL)
547
		}
548
		return &url.Error{
549
			Op:  urlErrorOp(reqs[0].Method),
550
			URL: urlStr,
551
			Err: err,
552
		}
553
	}
554
	for {
555
		// For all but the first request, create the next
556
		// request hop and replace req.
557
		if len(reqs) > 0 {
558
			loc := resp.Header.Get("Location")
559
			if loc == "" {
560
				resp.closeBody()
561
				return nil, uerr(fmt.Errorf("%d response missing Location header", resp.StatusCode))
562
			}
563
			u, err := req.URL.Parse(loc)
564
			if err != nil {
565
				resp.closeBody()
566
				return nil, uerr(fmt.Errorf("failed to parse Location header %q: %v", loc, err))
567
			}
568
			host := ""
569
			if req.Host != "" && req.Host != req.URL.Host {
570
				// If the caller specified a custom Host header and the
571
				// redirect location is relative, preserve the Host header
572
				// through the redirect. See issue #22233.
573
				if u, _ := url.Parse(loc); u != nil && !u.IsAbs() {
574
					host = req.Host
575
				}
576
			}
577
			ireq := reqs[0]
578
			req = &Request{
579
				Method:   redirectMethod,
580
				Response: resp,
581
				URL:      u,
582
				Header:   make(Header),
583
				Host:     host,
584
				Cancel:   ireq.Cancel,
585
				ctx:      ireq.ctx,
586
			}
587
			if includeBody && ireq.GetBody != nil {
588
				req.Body, err = ireq.GetBody()
589
				if err != nil {
590
					resp.closeBody()
591
					return nil, uerr(err)
592
				}
593
				req.ContentLength = ireq.ContentLength
594
			}
595

596
			// Copy original headers before setting the Referer,
597
			// in case the user set Referer on their first request.
598
			// If they really want to override, they can do it in
599
			// their CheckRedirect func.
600
			copyHeaders(req)
601

602
			// Add the Referer header from the most recent
603
			// request URL to the new one, if it's not https->http:
604
			if ref := refererForURL(reqs[len(reqs)-1].URL, req.URL); ref != "" {
605
				req.Header.Set("Referer", ref)
606
			}
607
			err = c.checkRedirect(req, reqs)
608

609
			// Sentinel error to let users select the
610
			// previous response, without closing its
611
			// body. See Issue 10069.
612
			if err == ErrUseLastResponse {
613
				return resp, nil
614
			}
615

616
			// Close the previous response's body. But
617
			// read at least some of the body so if it's
618
			// small the underlying TCP connection will be
619
			// re-used. No need to check for errors: if it
620
			// fails, the Transport won't reuse it anyway.
621
			const maxBodySlurpSize = 2 << 10
622
			if resp.ContentLength == -1 || resp.ContentLength <= maxBodySlurpSize {
623
				io.CopyN(ioutil.Discard, resp.Body, maxBodySlurpSize)
624
			}
625
			resp.Body.Close()
626

627
			if err != nil {
628
				// Special case for Go 1 compatibility: return both the response
629
				// and an error if the CheckRedirect function failed.
630
				// See https://golang.org/issue/3795
631
				// The resp.Body has already been closed.
632
				ue := uerr(err)
633
				ue.(*url.Error).URL = loc
634
				return resp, ue
635
			}
636
		}
637

638
		reqs = append(reqs, req)
639
		var err error
640
		var didTimeout func() bool
641
		if resp, didTimeout, err = c.send(req, deadline); err != nil {
642
			// c.send() always closes req.Body
643
			reqBodyClosed = true
644
			if !deadline.IsZero() && didTimeout() {
645
				err = &httpError{
646
					// TODO: early in cycle: s/Client.Timeout exceeded/timeout or context cancelation/
647
					err:     err.Error() + " (Client.Timeout exceeded while awaiting headers)",
648
					timeout: true,
649
				}
650
			}
651
			return nil, uerr(err)
652
		}
653

654
		var shouldRedirect bool
655
		redirectMethod, shouldRedirect, includeBody = redirectBehavior(req.Method, resp, reqs[0])
656
		if !shouldRedirect {
657
			return resp, nil
658
		}
659

660
		req.closeBody()
661
	}
662
}
663

664
// makeHeadersCopier makes a function that copies headers from the
665
// initial Request, ireq. For every redirect, this function must be called
666
// so that it can copy headers into the upcoming Request.
667
func (c *Client) makeHeadersCopier(ireq *Request) func(*Request) {
668
	// The headers to copy are from the very initial request.
669
	// We use a closured callback to keep a reference to these original headers.
670
	var (
671
		ireqhdr  = ireq.Header.clone()
672
		icookies map[string][]*Cookie
673
	)
674
	if c.Jar != nil && ireq.Header.Get("Cookie") != "" {
675
		icookies = make(map[string][]*Cookie)
676
		for _, c := range ireq.Cookies() {
677
			icookies[c.Name] = append(icookies[c.Name], c)
678
		}
679
	}
680

681
	preq := ireq // The previous request
682
	return func(req *Request) {
683
		// If Jar is present and there was some initial cookies provided
684
		// via the request header, then we may need to alter the initial
685
		// cookies as we follow redirects since each redirect may end up
686
		// modifying a pre-existing cookie.
687
		//
688
		// Since cookies already set in the request header do not contain
689
		// information about the original domain and path, the logic below
690
		// assumes any new set cookies override the original cookie
691
		// regardless of domain or path.
692
		//
693
		// See https://golang.org/issue/17494
694
		if c.Jar != nil && icookies != nil {
695
			var changed bool
696
			resp := req.Response // The response that caused the upcoming redirect
697
			for _, c := range resp.Cookies() {
698
				if _, ok := icookies[c.Name]; ok {
699
					delete(icookies, c.Name)
700
					changed = true
701
				}
702
			}
703
			if changed {
704
				ireqhdr.Del("Cookie")
705
				var ss []string
706
				for _, cs := range icookies {
707
					for _, c := range cs {
708
						ss = append(ss, c.Name+"="+c.Value)
709
					}
710
				}
711
				sort.Strings(ss) // Ensure deterministic headers
712
				ireqhdr.Set("Cookie", strings.Join(ss, "; "))
713
			}
714
		}
715

716
		// Copy the initial request's Header values
717
		// (at least the safe ones).
718
		for k, vv := range ireqhdr {
719
			if shouldCopyHeaderOnRedirect(k, preq.URL, req.URL) {
720
				req.Header[k] = vv
721
			}
722
		}
723

724
		preq = req // Update previous Request with the current request
725
	}
726
}
727

728
func defaultCheckRedirect(req *Request, via []*Request) error {
729
	if len(via) >= 10 {
730
		return errors.New("stopped after 10 redirects")
731
	}
732
	return nil
733
}
734

735
// Post issues a POST to the specified URL.
736
//
737
// Caller should close resp.Body when done reading from it.
738
//
739
// If the provided body is an io.Closer, it is closed after the
740
// request.
741
//
742
// Post is a wrapper around DefaultClient.Post.
743
//
744
// To set custom headers, use NewRequest and DefaultClient.Do.
745
//
746
// See the Client.Do method documentation for details on how redirects
747
// are handled.
748
func Post(url, contentType string, body io.Reader) (resp *Response, err error) {
749
	return DefaultClient.Post(url, contentType, body)
750
}
751

752
// Post issues a POST to the specified URL.
753
//
754
// Caller should close resp.Body when done reading from it.
755
//
756
// If the provided body is an io.Closer, it is closed after the
757
// request.
758
//
759
// To set custom headers, use NewRequest and Client.Do.
760
//
761
// See the Client.Do method documentation for details on how redirects
762
// are handled.
763
func (c *Client) Post(url, contentType string, body io.Reader) (resp *Response, err error) {
764
	req, err := NewRequest("POST", url, body)
765
	if err != nil {
766
		return nil, err
767
	}
768
	req.Header.Set("Content-Type", contentType)
769
	return c.Do(req)
770
}
771

772
// PostForm issues a POST to the specified URL, with data's keys and
773
// values URL-encoded as the request body.
774
//
775
// The Content-Type header is set to application/x-www-form-urlencoded.
776
// To set other headers, use NewRequest and DefaultClient.Do.
777
//
778
// When err is nil, resp always contains a non-nil resp.Body.
779
// Caller should close resp.Body when done reading from it.
780
//
781
// PostForm is a wrapper around DefaultClient.PostForm.
782
//
783
// See the Client.Do method documentation for details on how redirects
784
// are handled.
785
func PostForm(url string, data url.Values) (resp *Response, err error) {
786
	return DefaultClient.PostForm(url, data)
787
}
788

789
// PostForm issues a POST to the specified URL,
790
// with data's keys and values URL-encoded as the request body.
791
//
792
// The Content-Type header is set to application/x-www-form-urlencoded.
793
// To set other headers, use NewRequest and Client.Do.
794
//
795
// When err is nil, resp always contains a non-nil resp.Body.
796
// Caller should close resp.Body when done reading from it.
797
//
798
// See the Client.Do method documentation for details on how redirects
799
// are handled.
800
func (c *Client) PostForm(url string, data url.Values) (resp *Response, err error) {
801
	return c.Post(url, "application/x-www-form-urlencoded", strings.NewReader(data.Encode()))
802
}
803

804
// Head issues a HEAD to the specified URL. If the response is one of
805
// the following redirect codes, Head follows the redirect, up to a
806
// maximum of 10 redirects:
807
//
808
//    301 (Moved Permanently)
809
//    302 (Found)
810
//    303 (See Other)
811
//    307 (Temporary Redirect)
812
//    308 (Permanent Redirect)
813
//
814
// Head is a wrapper around DefaultClient.Head
815
func Head(url string) (resp *Response, err error) {
816
	return DefaultClient.Head(url)
817
}
818

819
// Head issues a HEAD to the specified URL. If the response is one of the
820
// following redirect codes, Head follows the redirect after calling the
821
// Client's CheckRedirect function:
822
//
823
//    301 (Moved Permanently)
824
//    302 (Found)
825
//    303 (See Other)
826
//    307 (Temporary Redirect)
827
//    308 (Permanent Redirect)
828
func (c *Client) Head(url string) (resp *Response, err error) {
829
	req, err := NewRequest("HEAD", url, nil)
830
	if err != nil {
831
		return nil, err
832
	}
833
	return c.Do(req)
834
}
835

836
// cancelTimerBody is an io.ReadCloser that wraps rc with two features:
837
// 1) on Read error or close, the stop func is called.
838
// 2) On Read failure, if reqDidTimeout is true, the error is wrapped and
839
//    marked as net.Error that hit its timeout.
840
type cancelTimerBody struct {
841
	stop          func() // stops the time.Timer waiting to cancel the request
842
	rc            io.ReadCloser
843
	reqDidTimeout func() bool
844
}
845

846
func (b *cancelTimerBody) Read(p []byte) (n int, err error) {
847
	n, err = b.rc.Read(p)
848
	if err == nil {
849
		return n, nil
850
	}
851
	b.stop()
852
	if err == io.EOF {
853
		return n, err
854
	}
855
	if b.reqDidTimeout() {
856
		err = &httpError{
857
			// TODO: early in cycle: s/Client.Timeout exceeded/timeout or context cancelation/
858
			err:     err.Error() + " (Client.Timeout exceeded while reading body)",
859
			timeout: true,
860
		}
861
	}
862
	return n, err
863
}
864

865
func (b *cancelTimerBody) Close() error {
866
	err := b.rc.Close()
867
	b.stop()
868
	return err
869
}
870

871
func shouldCopyHeaderOnRedirect(headerKey string, initial, dest *url.URL) bool {
872
	switch CanonicalHeaderKey(headerKey) {
873
	case "Authorization", "Www-Authenticate", "Cookie", "Cookie2":
874
		// Permit sending auth/cookie headers from "foo.com"
875
		// to "sub.foo.com".
876

877
		// Note that we don't send all cookies to subdomains
878
		// automatically. This function is only used for
879
		// Cookies set explicitly on the initial outgoing
880
		// client request. Cookies automatically added via the
881
		// CookieJar mechanism continue to follow each
882
		// cookie's scope as set by Set-Cookie. But for
883
		// outgoing requests with the Cookie header set
884
		// directly, we don't know their scope, so we assume
885
		// it's for *.domain.com.
886

887
		ihost := canonicalAddr(initial)
888
		dhost := canonicalAddr(dest)
889
		return isDomainOrSubdomain(dhost, ihost)
890
	}
891
	// All other headers are copied:
892
	return true
893
}
894

895
// isDomainOrSubdomain reports whether sub is a subdomain (or exact
896
// match) of the parent domain.
897
//
898
// Both domains must already be in canonical form.
899
func isDomainOrSubdomain(sub, parent string) bool {
900
	if sub == parent {
901
		return true
902
	}
903
	// If sub is "foo.example.com" and parent is "example.com",
904
	// that means sub must end in "."+parent.
905
	// Do it without allocating.
906
	if !strings.HasSuffix(sub, parent) {
907
		return false
908
	}
909
	return sub[len(sub)-len(parent)-1] == '.'
910
}
911

912
func stripPassword(u *url.URL) string {
913
	pass, passSet := u.User.Password()
914
	if passSet {
915
		return strings.Replace(u.String(), pass+"@", "***@", 1)
916
	}
917

918
	return u.String()
919
}
920