githubmirror/scrapy

conftest.py
138 строк4 KB
Rui Xi
Cherry-pick: Mitigate brotli and deflate decompression bombs DoS (#7134)
17 ноя 2025, 14:25
17 ноя 2025, 14:25c44b8df
from pathlib import Path

import pytest
from twisted.web.http import H2_ENABLED

from scrapy.utils.reactor import install_reactor
from tests.keys import generate_keys


def _py_files(folder):
    return (str(p) for p in Path(folder).rglob("*.py"))


collect_ignore = [
    # not a test, but looks like a test
    "scrapy/utils/testproc.py",
    "scrapy/utils/testsite.py",
    "tests/ftpserver.py",
    "tests/mockserver.py",
    "tests/pipelines.py",
    "tests/spiders.py",
    # contains scripts to be run by tests/test_crawler.py::CrawlerProcessSubprocess
    *_py_files("tests/CrawlerProcess"),
    # contains scripts to be run by tests/test_crawler.py::CrawlerRunnerSubprocess
    *_py_files("tests/CrawlerRunner"),
]

base_dir = Path(__file__).parent
ignore_file_path = base_dir / "tests" / "ignores.txt"
with ignore_file_path.open(encoding="utf-8") as reader:
    for line in reader:
        file_path = line.strip()
        if file_path and file_path[0] != "#":
            collect_ignore.append(file_path)

if not H2_ENABLED:
    collect_ignore.extend(
        (
            "scrapy/core/downloader/handlers/http2.py",
            *_py_files("scrapy/core/http2"),
        )
    )


@pytest.fixture
def chdir(tmpdir):
    """Change to pytest-provided temporary directory"""
    tmpdir.chdir()


def pytest_addoption(parser):
    parser.addoption(
        "--reactor",
        default="asyncio",
        choices=["default", "asyncio"],
    )


@pytest.fixture(scope="class")
def reactor_pytest(request):
    if not request.cls:
        # doctests
        return None
    request.cls.reactor_pytest = request.config.getoption("--reactor")
    return request.cls.reactor_pytest


@pytest.fixture(autouse=True)
def only_asyncio(request, reactor_pytest):
    if request.node.get_closest_marker("only_asyncio") and reactor_pytest == "default":
        pytest.skip("This test is only run without --reactor=default")


@pytest.fixture(autouse=True)
def only_not_asyncio(request, reactor_pytest):
    if (
        request.node.get_closest_marker("only_not_asyncio")
        and reactor_pytest != "default"
    ):
        pytest.skip("This test is only run with --reactor=default")


@pytest.fixture(autouse=True)
def requires_uvloop(request):
    if not request.node.get_closest_marker("requires_uvloop"):
        return
    try:
        import uvloop

        del uvloop
    except ImportError:
        pytest.skip("uvloop is not installed")


@pytest.fixture(autouse=True)
def requires_botocore(request):
    if not request.node.get_closest_marker("requires_botocore"):
        return
    try:
        import botocore

        del botocore
    except ImportError:
        pytest.skip("botocore is not installed")


@pytest.fixture(autouse=True)
def requires_boto3(request):
    if not request.node.get_closest_marker("requires_boto3"):
        return
    try:
        import boto3

        del boto3
    except ImportError:
        pytest.skip("boto3 is not installed")


@pytest.fixture(autouse=True)
def requires_mitmproxy(request):
    if not request.node.get_closest_marker("requires_mitmproxy"):
        return
    try:
        import mitmproxy  # noqa: F401
    except ImportError:
        pytest.skip("mitmproxy is not installed")


def pytest_configure(config):
    if config.getoption("--reactor") != "default":
        install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor")
    else:
        # install the reactor explicitly
        from twisted.internet import reactor  # noqa: F401


# Generate localhost certificate files, needed by some tests
generate_keys()