tox.ini - master - githubmirror/scrapy

tox.ini
383 строки10 KB
Adrian
Upgrade twinecheck (#7979)
11 авг 2026, 19:23
11 авг 2026, 19:2374f062f
# Tox (https://tox.readthedocs.io/) is a tool for running tests
# in multiple virtualenvs. This configuration file will run the
# test suite on all supported python versions. To use it, "pip install tox"
# and then run "tox" from this directory.

[tox]
requires =
    sphinx-scrapy[tox] @ git+https://github.com/scrapy/sphinx-scrapy.git@0.8.11
    tox-uv
envlist =
    pre-commit
    pylint
    mypy
    mypy-tests
    twinecheck
    docs
    docs-tests
    docs-links
    docs-coverage
    min
    min-extra-deps
    min-default-reactor
    min-no-reactor
    min-botocore
    min-pypy3
    py{310,311,312,313,314}
    extra-deps
    default-reactor
    no-reactor
    no-reactor-extra-deps
    botocore
    pypy3
    pypy3-extra-deps
    benchmark
minversion = 1.7.0

[test-requirements]
deps =
    attrs
    coverage >= 7.10.6
    httpx2
    pexpect >= 4.8.0
    pyftpdlib >= 2.0.1
    pygments
    pytest
    pytest-cov >= 7.0.0
    pytest-timeout
    pytest-xdist
    sybil >= 1.3.0  # https://github.com/cjw296/sybil/issues/20#issuecomment-605433422
    pytest-twisted >= 1.14.3

[testenv]
deps =
    {[test-requirements]deps}
    pytest >= 8.4.1  # https://github.com/pytest-dev/pytest/pull/13502
passenv =
    PYTHONTRACEMALLOC
    PYTEST_ADDOPTS
    MITMDUMP
    AWS_ACCESS_KEY_ID
    AWS_SECRET_ACCESS_KEY
    AWS_SESSION_TOKEN
#allow tox virtualenv to upgrade pip/wheel/setuptools
download = true
commands =
    pytest {posargs:--cov-config=pyproject.toml --cov=scrapy --cov-report= --cov-report=term-missing --cov-report=xml --junitxml=testenv.junit.xml -o junit_family=legacy --durations=10 scrapy tests --doctest-modules}

[testenv:mypy]
basepython = python3.10
deps =
    mypy==2.1.0
    typing-extensions==4.16.0
    Pillow==12.3.0
    Protego==0.6.2
    Twisted==26.4.0
    attrs==26.1.0
    boto3-stubs[s3]==1.43.41
    botocore-stubs==1.43.14
    h2==4.3.0
    httpx2==2.7.0
    itemadapter==0.13.1
    ptpython==3.0.32
    # newer ones require newer Python
    ipython==8.39.0
    pyOpenSSL==26.3.0
    pytest==9.1.1
    socksio==1.0.0
    types-Pygments==2.20.0.20260518
    types-defusedxml==0.7.0.20260504
    types-lxml==2026.2.16
    types-pexpect==4.9.0.20260518
    uvloop==0.22.1
    w3lib==2.4.1
    zstandard==0.25.0
commands =
    mypy {posargs:scrapy tests}

[testenv:mypy-tests]
basepython = python3.10
deps =
    {[test-requirements]deps}
    {[testenv:mypy]deps}
    pytest-mypy-testing==0.2.0
commands =
    pytest {posargs:tests_typing}

[testenv:pre-commit]
basepython = python3
deps =
    pre-commit
commands =
    pre-commit run {posargs:--all-files}

[testenv:pylint]
# Some checks are Python-version-dependent, so pin the version used in CI.
basepython = python3.14
deps =
    {[testenv:extra-deps]deps}
    pylint==4.0.6
    pylint-per-file-ignores # https://github.com/pylint-dev/pylint/issues/3767#issuecomment-1319916278
commands =
    pylint {posargs:conftest.py docs extras scrapy tests}

[testenv:twinecheck]
basepython = python3
deps =
    twine==7.0.0
    build==1.5.0
commands =
    python -m build --sdist
    twine check dist/*

[min]
basepython = python3.10
deps =
    # pytest 8.4.1 adds support for Twisted 25.5.0 but drops support for Twisted < 24.10.0
    pytest==8.4.0
    Protego==0.1.15
    Twisted==21.7.0
    brotli==1.2.0; implementation_name != "pypy"
    brotlicffi==1.2.0.0; implementation_name == "pypy"
    cryptography==37.0.0
    cssselect==0.9.1
    httpx2==2.0.0
    itemadapter==0.1.0
    lxml==4.6.4
    parsel==1.5.0
    pyOpenSSL==22.0.0
    queuelib==1.6.1
    service_identity==23.1.0
    w3lib==1.17.0
    zope.interface==5.1.0
    {[test-requirements]deps}
setenv =
    _SCRAPY_MIN=true
commands =
    pytest {posargs:--cov-config=pyproject.toml --cov=scrapy --cov-report=xml --cov-report= --junitxml=min.junit.xml -o junit_family=legacy --durations=10 scrapy tests}

[testenv:min]
basepython = {[min]basepython}
deps =
    {[min]deps}
    PyDispatcher==2.0.5
setenv =
    {[min]setenv}
commands = {[min]commands}

[testenv:extra-deps]
basepython = python3
deps =
    {[testenv]deps}
    Pillow
    Twisted[http2]
    boto3
    bpython  # optional for shell wrapper tests
    google-cloud-storage
    httpx2[http2,socks]
    ipython
    ptpython  # optional for shell wrapper tests
    robotexclusionrulesparser
    uvloop; platform_system != "Windows" and implementation_name != "pypy"
    zstandard; implementation_name != "pypy"  # optional for HTTP compress downloader middleware tests

[testenv:min-extra-deps]
basepython = {[min]basepython}
deps =
    {[min]deps}
    Pillow==8.3.2
    Twisted[http2]==21.7.0
    boto3==1.20.0
    bpython==0.7.1
    google-cloud-storage==1.29.0
    httpx2[http2,socks]==2.0.0
    ipython==8.15.0
    ptpython==3.0.23
    robotexclusionrulesparser==1.6.2
    uvloop==0.16.0; platform_system != "Windows" and implementation_name != "pypy"
    zstandard==0.16.0; implementation_name != "pypy"
setenv =
    {[min]setenv}
commands = {[min]commands}

[testenv:vcs-deps]
basepython = python3
deps =
    {[testenv:extra-deps]deps}
    uv
# Dependencies cap each other at their latest release, so their development
# branches usually cannot be resolved together: pyOpenSSL, for one, requires a
# cryptography older than the one cryptography itself is heading towards.
# --no-deps skips resolution entirely, replacing only these distributions and
# leaving the rest of the environment as the install above resolved it.
#
# Pillow and uvloop build from source, and need the libjpeg headers and
# autotools respectively. robotexclusionrulesparser has no public repository,
# and PyDispatcher has seen no commit since 2023-10-23, so they stay at their
# latest release.
commands_pre =
    uv pip install --python {envpython} --no-deps --reinstall \
        git+https://github.com/twisted/twisted \
        git+https://github.com/python-pillow/Pillow \
        git+https://github.com/MagicStack/uvloop \
        git+https://github.com/pyca/cryptography \
        git+https://github.com/scrapy/cssselect \
        git+https://github.com/tiran/defusedxml \
        git+https://github.com/scrapy/itemadapter \
        git+https://github.com/scrapy/itemloaders \
        git+https://github.com/lxml/lxml \
        git+https://github.com/pypa/packaging \
        git+https://github.com/scrapy/parsel \
        git+https://github.com/scrapy/protego \
        git+https://github.com/pyca/pyopenssl \
        git+https://github.com/scrapy/queuelib \
        git+https://github.com/pyca/service-identity \
        git+https://github.com/john-kurkowski/tldextract \
        git+https://github.com/scrapy/w3lib \
        git+https://github.com/zopefoundation/zope.interface \
        git+https://github.com/boto/boto3 \
        git+https://github.com/bpython/bpython \
        git+https://github.com/google/brotli \
        git+https://github.com/python-hyper/brotlicffi \
        git+https://github.com/googleapis/python-storage \
        git+https://github.com/pydantic/httpx2\#subdirectory=src/httpx2 \
        git+https://github.com/ipython/ipython \
        git+https://github.com/prompt-toolkit/ptpython \
        git+https://github.com/indygreg/python-zstandard

[testenv:default-reactor]
commands =
    {[testenv]commands} --reactor=default

[testenv:min-default-reactor]
basepython = {[min]basepython}
deps = {[testenv:min]deps}
commands = {[min]commands} --reactor=default
setenv =
    {[min]setenv}

[testenv:no-reactor]
deps =
    {[testenv]deps}
    pytest-asyncio
commands =
    {[testenv]commands} -p no:twisted --reactor=none

[testenv:no-reactor-extra-deps]
deps =
    {[testenv:extra-deps]deps}
    pytest-asyncio
commands =
    {[testenv]commands} -p no:twisted --reactor=none

[testenv:min-no-reactor]
basepython = {[min]basepython}
deps =
    {[testenv:min]deps}
    pytest-asyncio
commands = {[min]commands} -p no:twisted --reactor=none
setenv =
    {[min]setenv}

[testenv:pypy3]
basepython = pypy3
commands =
    ; not enabling coverage as it significantly increases the run time
    pytest {posargs:--durations=10 scrapy tests}

[testenv:pypy3-extra-deps]
basepython = pypy3
deps =
    {[testenv:extra-deps]deps}

[testenv:min-pypy3]
basepython = pypy3.11
deps =
    PyPyDispatcher==2.1.0
    {[test-requirements]deps}
    pytest==8.4.0
    Protego==0.1.15
    Twisted==21.7.0
    brotlicffi==1.2.0.0
    cryptography==44.0.2
    cssselect==0.9.1
    itemadapter==0.1.0
    lxml==5.3.2
    parsel==1.5.0
    pyOpenSSL==24.3.0
    queuelib==1.6.1
    service_identity==23.1.0
    # w3lib 1.17 fails to import on PyPy 3.11 because its encoding regex uses
    # an inline flag placement that Python 3.11 treats as an error: global
    # flags not at the start of the expression. w3lib 1.18 stopped encoding []
    # in URLs until 2.1.0 brought that behavior back. Tests for verbatim_url
    # rely on that encoding.
    w3lib==2.1.0
    zope.interface==5.1.0
commands =
    ; disabling coverage
    pytest {posargs:--durations=10 scrapy tests}
setenv =
    {[min]setenv}

[testenv:docs-tests]
changedir = docs
deps =
    {[test-requirements]deps}
    -rdocs/requirements.txt
    form2request
commands =
    pytest

[testenv:docs-coverage]
changedir = docs
deps =
    -rdocs/requirements.txt
commands =
    sphinx-build -b coverage . {envtmpdir}/coverage

[testenv:docs-links]
changedir = docs
deps =
    -rdocs/requirements.txt
commands =
    sphinx-build -W -b linkcheck . {envtmpdir}/linkcheck


# Run S3 tests with botocore installed but without boto3.

[testenv:botocore]
deps =
    {[testenv]deps}
    botocore>=1.13.45
commands =
    pytest {posargs:--cov-config=pyproject.toml --cov=scrapy --cov-report=xml --cov-report= tests --junitxml=botocore.junit.xml -o junit_family=legacy} -m requires_botocore

[testenv:min-botocore]
basepython = {[min]basepython}
deps =
    {[min]deps}
    botocore==1.13.45
    # botocore 1.13.45 requires urllib3>=1.20,<1.26; requests 2.33.0 requires urllib3>=1.26,<3
    requests<2.33.0
setenv =
    {[min]setenv}
commands =
    pytest {posargs:--cov-config=pyproject.toml --cov=scrapy --cov-report=xml --cov-report= tests --junitxml=min-botocore.junit.xml -o junit_family=legacy} -m requires_botocore


# CPU benchmarks, tracked on CodSpeed.
#
# pytest-twisted is left out on purpose: benchmarked code must be callable
# synchronously, so tests/benchmarks drives the reactor itself.

[testenv:benchmark]
basepython = python3.14
deps =
    pytest >= 8.4.1
    pytest-codspeed
passenv =
    *codspeed*
    *ci*
commands =
    pytest {posargs:tests/benchmarks} --codspeed --codspeed-mode=simulation