urllib3

1
from __future__ import annotations
2

3
import http.client as httplib
4
import ssl
5
import typing
6
from http.client import HTTPException
7
from queue import Empty
8
from socket import error as SocketError
9
from ssl import SSLError as BaseSSLError
10
from test import SHORT_TIMEOUT
11
from unittest.mock import Mock, patch
12

13
import pytest
14

15
from dummyserver.socketserver import DEFAULT_CA
16
from urllib3 import Retry
17
from urllib3.connection import HTTPConnection
18
from urllib3.connectionpool import (
19
    HTTPConnectionPool,
20
    HTTPSConnectionPool,
21
    _url_from_pool,
22
    connection_from_url,
23
)
24
from urllib3.exceptions import (
25
    ClosedPoolError,
26
    EmptyPoolError,
27
    FullPoolError,
28
    HostChangedError,
29
    LocationValueError,
30
    MaxRetryError,
31
    ProtocolError,
32
    ReadTimeoutError,
33
    SSLError,
34
    TimeoutError,
35
)
36
from urllib3.response import HTTPResponse
37
from urllib3.util.ssl_match_hostname import CertificateError
38
from urllib3.util.timeout import _DEFAULT_TIMEOUT, Timeout
39

40
from .test_response import MockChunkedEncodingResponse, MockSock
41

42

43
class HTTPUnixConnection(HTTPConnection):
44
    def __init__(self, host: str, timeout: int = 60, **kwargs: typing.Any) -> None:
45
        super().__init__("localhost")
46
        self.unix_socket = host
47
        self.timeout = timeout
48
        self.sock = None
49

50

51
class HTTPUnixConnectionPool(HTTPConnectionPool):
52
    scheme = "http+unix"
53
    ConnectionCls = HTTPUnixConnection
54

55

56
class TestConnectionPool:
57
    """
58
    Tests in this suite should exercise the ConnectionPool functionality
59
    without actually making any network requests or connections.
60
    """
61

62
    @pytest.mark.parametrize(
63
        "a, b",
64
        [
65
            ("http://google.com/", "/"),
66
            ("http://google.com/", "http://google.com/"),
67
            ("http://google.com/", "http://google.com"),
68
            ("http://google.com/", "http://google.com/abra/cadabra"),
69
            ("http://google.com:42/", "http://google.com:42/abracadabra"),
70
            # Test comparison using default ports
71
            ("http://google.com:80/", "http://google.com/abracadabra"),
72
            ("http://google.com/", "http://google.com:80/abracadabra"),
73
            ("https://google.com:443/", "https://google.com/abracadabra"),
74
            ("https://google.com/", "https://google.com:443/abracadabra"),
75
            (
76
                "http://[2607:f8b0:4005:805::200e%25eth0]/",
77
                "http://[2607:f8b0:4005:805::200e%eth0]/",
78
            ),
79
            (
80
                "https://[2607:f8b0:4005:805::200e%25eth0]:443/",
81
                "https://[2607:f8b0:4005:805::200e%eth0]:443/",
82
            ),
83
            ("http://[::1]/", "http://[::1]"),
84
            (
85
                "http://[2001:558:fc00:200:f816:3eff:fef9:b954%lo]/",
86
                "http://[2001:558:fc00:200:f816:3eff:fef9:b954%25lo]",
87
            ),
88
        ],
89
    )
90
    def test_same_host(self, a: str, b: str) -> None:
91
        with connection_from_url(a) as c:
92
            assert c.is_same_host(b)
93

94
    @pytest.mark.parametrize(
95
        "a, b",
96
        [
97
            ("https://google.com/", "http://google.com/"),
98
            ("http://google.com/", "https://google.com/"),
99
            ("http://yahoo.com/", "http://google.com/"),
100
            ("http://google.com:42", "https://google.com/abracadabra"),
101
            ("http://google.com", "https://google.net/"),
102
            # Test comparison with default ports
103
            ("http://google.com:42", "http://google.com"),
104
            ("https://google.com:42", "https://google.com"),
105
            ("http://google.com:443", "http://google.com"),
106
            ("https://google.com:80", "https://google.com"),
107
            ("http://google.com:443", "https://google.com"),
108
            ("https://google.com:80", "http://google.com"),
109
            ("https://google.com:443", "http://google.com"),
110
            ("http://google.com:80", "https://google.com"),
111
            # Zone identifiers are unique connection end points and should
112
            # never be equivalent.
113
            ("http://[dead::beef]", "https://[dead::beef%en5]/"),
114
        ],
115
    )
116
    def test_not_same_host(self, a: str, b: str) -> None:
117
        with connection_from_url(a) as c:
118
            assert not c.is_same_host(b)
119

120
        with connection_from_url(b) as c:
121
            assert not c.is_same_host(a)
122

123
    @pytest.mark.parametrize(
124
        "a, b",
125
        [
126
            ("google.com", "/"),
127
            ("google.com", "http://google.com/"),
128
            ("google.com", "http://google.com"),
129
            ("google.com", "http://google.com/abra/cadabra"),
130
            # Test comparison using default ports
131
            ("google.com", "http://google.com:80/abracadabra"),
132
        ],
133
    )
134
    def test_same_host_no_port_http(self, a: str, b: str) -> None:
135
        # This test was introduced in #801 to deal with the fact that urllib3
136
        # never initializes ConnectionPool objects with port=None.
137
        with HTTPConnectionPool(a) as c:
138
            assert c.is_same_host(b)
139

140
    @pytest.mark.parametrize(
141
        "a, b",
142
        [
143
            ("google.com", "/"),
144
            ("google.com", "https://google.com/"),
145
            ("google.com", "https://google.com"),
146
            ("google.com", "https://google.com/abra/cadabra"),
147
            # Test comparison using default ports
148
            ("google.com", "https://google.com:443/abracadabra"),
149
        ],
150
    )
151
    def test_same_host_no_port_https(self, a: str, b: str) -> None:
152
        # This test was introduced in #801 to deal with the fact that urllib3
153
        # never initializes ConnectionPool objects with port=None.
154
        with HTTPSConnectionPool(a) as c:
155
            assert c.is_same_host(b)
156

157
    @pytest.mark.parametrize(
158
        "a, b",
159
        [
160
            ("google.com", "https://google.com/"),
161
            ("yahoo.com", "http://google.com/"),
162
            ("google.com", "https://google.net/"),
163
            ("google.com", "http://google.com./"),
164
        ],
165
    )
166
    def test_not_same_host_no_port_http(self, a: str, b: str) -> None:
167
        with HTTPConnectionPool(a) as c:
168
            assert not c.is_same_host(b)
169

170
        with HTTPConnectionPool(b) as c:
171
            assert not c.is_same_host(a)
172

173
    @pytest.mark.parametrize(
174
        "a, b",
175
        [
176
            ("google.com", "http://google.com/"),
177
            ("yahoo.com", "https://google.com/"),
178
            ("google.com", "https://google.net/"),
179
            ("google.com", "https://google.com./"),
180
        ],
181
    )
182
    def test_not_same_host_no_port_https(self, a: str, b: str) -> None:
183
        with HTTPSConnectionPool(a) as c:
184
            assert not c.is_same_host(b)
185

186
        with HTTPSConnectionPool(b) as c:
187
            assert not c.is_same_host(a)
188

189
    @pytest.mark.parametrize(
190
        "a, b",
191
        [
192
            ("%2Fvar%2Frun%2Fdocker.sock", "http+unix://%2Fvar%2Frun%2Fdocker.sock"),
193
            ("%2Fvar%2Frun%2Fdocker.sock", "http+unix://%2Fvar%2Frun%2Fdocker.sock/"),
194
            (
195
                "%2Fvar%2Frun%2Fdocker.sock",
196
                "http+unix://%2Fvar%2Frun%2Fdocker.sock/abracadabra",
197
            ),
198
            ("%2Ftmp%2FTEST.sock", "http+unix://%2Ftmp%2FTEST.sock"),
199
            ("%2Ftmp%2FTEST.sock", "http+unix://%2Ftmp%2FTEST.sock/"),
200
            ("%2Ftmp%2FTEST.sock", "http+unix://%2Ftmp%2FTEST.sock/abracadabra"),
201
        ],
202
    )
203
    def test_same_host_custom_protocol(self, a: str, b: str) -> None:
204
        with HTTPUnixConnectionPool(a) as c:
205
            assert c.is_same_host(b)
206

207
    @pytest.mark.parametrize(
208
        "a, b",
209
        [
210
            ("%2Ftmp%2Ftest.sock", "http+unix://%2Ftmp%2FTEST.sock"),
211
            ("%2Ftmp%2Ftest.sock", "http+unix://%2Ftmp%2FTEST.sock/"),
212
            ("%2Ftmp%2Ftest.sock", "http+unix://%2Ftmp%2FTEST.sock/abracadabra"),
213
            ("%2Fvar%2Frun%2Fdocker.sock", "http+unix://%2Ftmp%2FTEST.sock"),
214
        ],
215
    )
216
    def test_not_same_host_custom_protocol(self, a: str, b: str) -> None:
217
        with HTTPUnixConnectionPool(a) as c:
218
            assert not c.is_same_host(b)
219

220
    def test_max_connections(self) -> None:
221
        with HTTPConnectionPool(host="localhost", maxsize=1, block=True) as pool:
222
            pool._get_conn(timeout=SHORT_TIMEOUT)
223

224
            with pytest.raises(EmptyPoolError):
225
                pool._get_conn(timeout=SHORT_TIMEOUT)
226

227
            with pytest.raises(EmptyPoolError):
228
                pool.request("GET", "/", pool_timeout=SHORT_TIMEOUT)
229

230
            assert pool.num_connections == 1
231

232
    def test_put_conn_when_pool_is_full_nonblocking(
233
        self, caplog: pytest.LogCaptureFixture
234
    ) -> None:
235
        """
236
        If maxsize = n and we _put_conn n + 1 conns, the n + 1th conn will
237
        get closed and will not get added to the pool.
238
        """
239
        with HTTPConnectionPool(host="localhost", maxsize=1, block=False) as pool:
240
            conn1 = pool._get_conn()
241
            # pool.pool is empty because we popped the one None that pool.pool was initialized with
242
            # but this pool._get_conn call will not raise EmptyPoolError because block is False
243
            conn2 = pool._get_conn()
244

245
            with patch.object(conn1, "close") as conn1_close:
246
                with patch.object(conn2, "close") as conn2_close:
247
                    pool._put_conn(conn1)
248
                    pool._put_conn(conn2)
249

250
            assert conn1_close.called is False
251
            assert conn2_close.called is True
252

253
            assert conn1 == pool._get_conn()
254
            assert conn2 != pool._get_conn()
255

256
            assert pool.num_connections == 3
257
            assert "Connection pool is full, discarding connection" in caplog.text
258
            assert "Connection pool size: 1" in caplog.text
259

260
    def test_put_conn_when_pool_is_full_blocking(self) -> None:
261
        """
262
        If maxsize = n and we _put_conn n + 1 conns, the n + 1th conn will
263
        cause a FullPoolError.
264
        """
265
        with HTTPConnectionPool(host="localhost", maxsize=1, block=True) as pool:
266
            conn1 = pool._get_conn()
267
            conn2 = pool._new_conn()
268

269
            with patch.object(conn1, "close") as conn1_close:
270
                with patch.object(conn2, "close") as conn2_close:
271
                    pool._put_conn(conn1)
272
                    with pytest.raises(FullPoolError):
273
                        pool._put_conn(conn2)
274

275
            assert conn1_close.called is False
276
            assert conn2_close.called is True
277

278
            assert conn1 == pool._get_conn()
279

280
    def test_put_conn_closed_pool(self) -> None:
281
        with HTTPConnectionPool(host="localhost", maxsize=1, block=True) as pool:
282
            conn1 = pool._get_conn()
283
            with patch.object(conn1, "close") as conn1_close:
284
                pool.close()
285

286
                assert pool.pool is None
287

288
                # Accessing pool.pool will raise AttributeError, which will get
289
                # caught and will close conn1
290
                pool._put_conn(conn1)
291

292
            assert conn1_close.called is True
293

294
    def test_exception_str(self) -> None:
295
        assert (
296
            str(EmptyPoolError(HTTPConnectionPool(host="localhost"), "Test."))
297
            == "HTTPConnectionPool(host='localhost', port=None): Test."
298
        )
299

300
    def test_retry_exception_str(self) -> None:
301
        assert (
302
            str(MaxRetryError(HTTPConnectionPool(host="localhost"), "Test.", None))
303
            == "HTTPConnectionPool(host='localhost', port=None): "
304
            "Max retries exceeded with url: Test. (Caused by None)"
305
        )
306

307
        err = SocketError("Test")
308

309
        # using err.__class__ here, as socket.error is an alias for OSError
310
        # since Py3.3 and gets printed as this
311
        assert (
312
            str(MaxRetryError(HTTPConnectionPool(host="localhost"), "Test.", err))
313
            == "HTTPConnectionPool(host='localhost', port=None): "
314
            "Max retries exceeded with url: Test. "
315
            "(Caused by %r)" % err
316
        )
317

318
    def test_pool_size(self) -> None:
319
        POOL_SIZE = 1
320
        with HTTPConnectionPool(
321
            host="localhost", maxsize=POOL_SIZE, block=True
322
        ) as pool:
323

324
            def _test(
325
                exception: type[BaseException],
326
                expect: type[BaseException],
327
                reason: type[BaseException] | None = None,
328
            ) -> None:
329
                with patch.object(pool, "_make_request", side_effect=exception()):
330
                    with pytest.raises(expect) as excinfo:
331
                        pool.request("GET", "/")
332
                if reason is not None:
333
                    assert isinstance(excinfo.value.reason, reason)  # type: ignore[attr-defined]
334
                assert pool.pool is not None
335
                assert pool.pool.qsize() == POOL_SIZE
336

337
            # Make sure that all of the exceptions return the connection
338
            # to the pool
339
            _test(BaseSSLError, MaxRetryError, SSLError)
340
            _test(CertificateError, MaxRetryError, SSLError)
341

342
            # The pool should never be empty, and with these two exceptions
343
            # being raised, a retry will be triggered, but that retry will
344
            # fail, eventually raising MaxRetryError, not EmptyPoolError
345
            # See: https://github.com/urllib3/urllib3/issues/76
346
            with patch.object(pool, "_make_request", side_effect=HTTPException()):
347
                with pytest.raises(MaxRetryError):
348
                    pool.request("GET", "/", retries=1, pool_timeout=SHORT_TIMEOUT)
349
            assert pool.pool is not None
350
            assert pool.pool.qsize() == POOL_SIZE
351

352
    def test_empty_does_not_put_conn(self) -> None:
353
        """Do not put None back in the pool if the pool was empty"""
354

355
        with HTTPConnectionPool(host="localhost", maxsize=1, block=True) as pool:
356
            with patch.object(
357
                pool, "_get_conn", side_effect=EmptyPoolError(pool, "Pool is empty")
358
            ):
359
                with patch.object(
360
                    pool,
361
                    "_put_conn",
362
                    side_effect=AssertionError("Unexpected _put_conn"),
363
                ):
364
                    with pytest.raises(EmptyPoolError):
365
                        pool.request("GET", "/")
366

367
    def test_assert_same_host(self) -> None:
368
        with connection_from_url("http://google.com:80") as c:
369
            with pytest.raises(HostChangedError):
370
                c.request("GET", "http://yahoo.com:80", assert_same_host=True)
371

372
    def test_pool_close(self) -> None:
373
        pool = connection_from_url("http://google.com:80")
374

375
        # Populate with some connections
376
        conn1 = pool._get_conn()
377
        conn2 = pool._get_conn()
378
        conn3 = pool._get_conn()
379
        pool._put_conn(conn1)
380
        pool._put_conn(conn2)
381

382
        old_pool_queue = pool.pool
383

384
        pool.close()
385
        assert pool.pool is None
386

387
        with pytest.raises(ClosedPoolError):
388
            pool._get_conn()
389

390
        pool._put_conn(conn3)
391

392
        with pytest.raises(ClosedPoolError):
393
            pool._get_conn()
394

395
        with pytest.raises(Empty):
396
            assert old_pool_queue is not None
397
            old_pool_queue.get(block=False)
398

399
    def test_pool_close_twice(self) -> None:
400
        pool = connection_from_url("http://google.com:80")
401

402
        # Populate with some connections
403
        conn1 = pool._get_conn()
404
        conn2 = pool._get_conn()
405
        pool._put_conn(conn1)
406
        pool._put_conn(conn2)
407

408
        pool.close()
409
        assert pool.pool is None
410

411
        try:
412
            pool.close()
413
        except AttributeError:
414
            pytest.fail("Pool of the ConnectionPool is None and has no attribute get.")
415

416
    def test_pool_timeouts(self) -> None:
417
        with HTTPConnectionPool(host="localhost") as pool:
418
            conn = pool._new_conn()
419
            assert conn.__class__ == HTTPConnection
420
            assert pool.timeout.__class__ == Timeout
421
            assert pool.timeout._read == _DEFAULT_TIMEOUT
422
            assert pool.timeout._connect == _DEFAULT_TIMEOUT
423
            assert pool.timeout.total is None
424

425
            pool = HTTPConnectionPool(host="localhost", timeout=SHORT_TIMEOUT)
426
            assert pool.timeout._read == SHORT_TIMEOUT
427
            assert pool.timeout._connect == SHORT_TIMEOUT
428
            assert pool.timeout.total is None
429

430
    def test_no_host(self) -> None:
431
        with pytest.raises(LocationValueError):
432
            HTTPConnectionPool(None)  # type: ignore[arg-type]
433

434
    def test_contextmanager(self) -> None:
435
        with connection_from_url("http://google.com:80") as pool:
436
            # Populate with some connections
437
            conn1 = pool._get_conn()
438
            conn2 = pool._get_conn()
439
            conn3 = pool._get_conn()
440
            pool._put_conn(conn1)
441
            pool._put_conn(conn2)
442

443
            old_pool_queue = pool.pool
444

445
        assert pool.pool is None
446
        with pytest.raises(ClosedPoolError):
447
            pool._get_conn()
448

449
        pool._put_conn(conn3)
450
        with pytest.raises(ClosedPoolError):
451
            pool._get_conn()
452
        with pytest.raises(Empty):
453
            assert old_pool_queue is not None
454
            old_pool_queue.get(block=False)
455

456
    def test_url_from_pool(self) -> None:
457
        with connection_from_url("http://google.com:80") as pool:
458
            path = "path?query=foo"
459
            assert f"http://google.com:80/{path}" == _url_from_pool(pool, path)
460

461
    def test_ca_certs_default_cert_required(self) -> None:
462
        with connection_from_url("https://google.com:80", ca_certs=DEFAULT_CA) as pool:
463
            conn = pool._get_conn()
464
            assert conn.cert_reqs == ssl.CERT_REQUIRED  # type: ignore[attr-defined]
465

466
    def test_cleanup_on_extreme_connection_error(self) -> None:
467
        """
468
        This test validates that we clean up properly even on exceptions that
469
        we'd not otherwise catch, i.e. those that inherit from BaseException
470
        like KeyboardInterrupt or gevent.Timeout. See #805 for more details.
471
        """
472

473
        class RealBad(BaseException):
474
            pass
475

476
        def kaboom(*args: typing.Any, **kwargs: typing.Any) -> None:
477
            raise RealBad()
478

479
        with connection_from_url("http://localhost:80") as c:
480
            with patch.object(c, "_make_request", kaboom):
481
                assert c.pool is not None
482
                initial_pool_size = c.pool.qsize()
483

484
                try:
485
                    # We need to release_conn this way or we'd put it away
486
                    # regardless.
487
                    c.urlopen("GET", "/", release_conn=False)
488
                except RealBad:
489
                    pass
490

491
            new_pool_size = c.pool.qsize()
492
            assert initial_pool_size == new_pool_size
493

494
    def test_release_conn_param_is_respected_after_http_error_retry(self) -> None:
495
        """For successful ```urlopen(release_conn=False)```,
496
        the connection isn't released, even after a retry.
497

498
        This is a regression test for issue #651 [1], where the connection
499
        would be released if the initial request failed, even if a retry
500
        succeeded.
501

502
        [1] <https://github.com/urllib3/urllib3/issues/651>
503
        """
504

505
        class _raise_once_make_request_function:
506
            """Callable that can mimic `_make_request()`.
507

508
            Raises the given exception on its first call, but returns a
509
            successful response on subsequent calls.
510
            """
511

512
            def __init__(
513
                self, ex: type[BaseException], pool: HTTPConnectionPool
514
            ) -> None:
515
                super().__init__()
516
                self._ex: type[BaseException] | None = ex
517
                self._pool = pool
518

519
            def __call__(
520
                self,
521
                conn: HTTPConnection,
522
                method: str,
523
                url: str,
524
                *args: typing.Any,
525
                retries: Retry,
526
                **kwargs: typing.Any,
527
            ) -> HTTPResponse:
528
                if self._ex:
529
                    ex, self._ex = self._ex, None
530
                    raise ex()
531
                httplib_response = httplib.HTTPResponse(MockSock)  # type: ignore[arg-type]
532
                httplib_response.fp = MockChunkedEncodingResponse([b"f", b"o", b"o"])  # type: ignore[assignment]
533
                httplib_response.headers = httplib_response.msg = httplib.HTTPMessage()
534

535
                response_conn: HTTPConnection | None = kwargs.get("response_conn")
536

537
                response = HTTPResponse(
538
                    body=httplib_response,
539
                    headers=httplib_response.headers,  # type: ignore[arg-type]
540
                    status=httplib_response.status,
541
                    version=httplib_response.version,
542
                    reason=httplib_response.reason,
543
                    original_response=httplib_response,
544
                    retries=retries,
545
                    request_method=method,
546
                    request_url=url,
547
                    preload_content=False,
548
                    connection=response_conn,
549
                    pool=self._pool,
550
                )
551
                return response
552

553
        def _test(exception: type[BaseException]) -> None:
554
            with HTTPConnectionPool(host="localhost", maxsize=1, block=True) as pool:
555
                # Verify that the request succeeds after two attempts, and that the
556
                # connection is left on the response object, instead of being
557
                # released back into the pool.
558
                with patch.object(
559
                    pool,
560
                    "_make_request",
561
                    _raise_once_make_request_function(exception, pool),
562
                ):
563
                    response = pool.urlopen(
564
                        "GET",
565
                        "/",
566
                        retries=1,
567
                        release_conn=False,
568
                        preload_content=False,
569
                        chunked=True,
570
                    )
571
                assert pool.pool is not None
572
                assert pool.pool.qsize() == 0
573
                assert pool.num_connections == 2
574
                assert response.connection is not None
575

576
                response.release_conn()
577
                assert pool.pool.qsize() == 1
578
                assert response.connection is None
579

580
        # Run the test case for all the retriable exceptions.
581
        _test(TimeoutError)
582
        _test(HTTPException)
583
        _test(SocketError)
584
        _test(ProtocolError)
585

586
    def test_read_timeout_0_does_not_raise_bad_status_line_error(self) -> None:
587
        with HTTPConnectionPool(host="localhost", maxsize=1) as pool:
588
            conn = Mock(spec=HTTPConnection)
589
            # Needed to tell the pool that the connection is alive.
590
            conn.is_closed = False
591
            with patch.object(Timeout, "read_timeout", 0):
592
                timeout = Timeout(1, 1, 1)
593
                with pytest.raises(ReadTimeoutError):
594
                    pool._make_request(conn, "", "", timeout=timeout)
595