/
githubmirror
/
scrapy
Обзор
Документация
Войти
/
githubmirror
/
scrapy
Код
Запросы
0
Пакеты
0
Релизы
0
Аналитика
Безопасность
master
pyproject.toml
506 строк
14 KB
Adrian
Address warnings from the VCS workflow (#7966)
10 авг 2026, 11:46
Не верифицирован
10 авг 2026, 11:46
c7ee394
Код
Авторство
О чём код?
[build-system] requires = ["hatchling>=1.27.0"] build-backend = "hatchling.build" [project] name = "Scrapy" dynamic = ["version"] description = "A high-level Web Crawling and Web Scraping framework" dependencies = [ "Twisted>=21.7.0", "cryptography>=37.0.0", "cssselect>=0.9.1", "defusedxml>=0.7.1", "itemadapter>=0.1.0", "itemloaders>=1.0.1", "lxml>=4.6.4", "packaging", "parsel>=1.5.0", "protego>=0.1.15", "pyOpenSSL>=22.0.0", "queuelib>=1.6.1", "service_identity>=23.1.0", "tldextract", "w3lib>=1.17.0", "zope.interface>=5.1.0", # Platform-specific dependencies 'PyDispatcher>=2.0.5; platform_python_implementation == "CPython"', 'PyPyDispatcher>=2.1.0; platform_python_implementation == "PyPy"', 'brotli>=1.2.0; implementation_name != "pypy"', 'brotlicffi>=1.2.0.0; implementation_name == "pypy"', ] classifiers = [ "Development Status :: 5 - Production/Stable", "Environment :: Console", "Framework :: Scrapy", "Intended Audience :: Developers", "Operating System :: OS Independent", "Programming Language :: Python", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", "Programming Language :: Python :: 3.14", "Programming Language :: Python :: Implementation :: CPython", "Programming Language :: Python :: Implementation :: PyPy", "Topic :: Internet :: WWW/HTTP", "Topic :: Software Development :: Libraries :: Application Frameworks", "Topic :: Software Development :: Libraries :: Python Modules", ] license = "BSD-3-Clause" license-files = ["LICENSE", "AUTHORS"] readme = "README.rst" requires-python = ">=3.10" authors = [{ name = "Scrapy developers", email = "pablo@pablohoffman.com" }] maintainers = [{ name = "Pablo Hoffman", email = "pablo@pablohoffman.com" }] [project.urls] Homepage = "https://scrapy.org/" Documentation = "https://docs.scrapy.org/" Source = "https://github.com/scrapy/scrapy" Tracker = "https://github.com/scrapy/scrapy/issues" "Release notes" = "https://docs.scrapy.org/en/latest/news.html" [project.optional-dependencies] bpython = ["bpython>=0.7.1"] gcs = ["google-cloud-storage>=1.29.0"] httpx = ["httpx2[http2,socks]>=2.0.0"] images = ["Pillow>=8.3.2"] ipython = ["ipython>=8.15.0"] ptpython = ["ptpython>=3.0.23"] robotparser = ["robotexclusionrulesparser>=1.6.2"] s3 = ["boto3>=1.20.0"] twisted-http2 = ["Twisted[http2]>=21.7.0"] uvloop = [ "uvloop>=0.16.0; platform_system != 'Windows' and implementation_name != 'pypy'", ] zstd = ["zstandard>=0.16.0; implementation_name != 'pypy'"] [project.scripts] scrapy = "scrapy.cmdline:execute" [tool.hatch.build.targets.sdist] include = [ "/docs", "/extras", "/scrapy", "/tests", "/tests_typing", "/CODE_OF_CONDUCT.md", "/CONTRIBUTING.md", "/INSTALL.md", "/NEWS", "/SECURITY.md", "/codecov.yml", "/conftest.py", "/tox.ini", ] [tool.hatch.version] path = "scrapy/VERSION" pattern = "^(?P<version>.+)$" [tool.mypy] strict = true extra_checks = false # weird addErrback() errors untyped_calls_exclude = [ "twisted", ] [[tool.mypy.overrides]] module = "tests.*" allow_untyped_defs = true allow_incomplete_defs = true # 59 errors # TODO [[tool.mypy.overrides]] module = [ "tests.spiders", "tests.test_closespider", "tests.test_cmdline", "tests.test_contracts", "tests.test_downloaderslotssettings", "tests.test_dupefilters", "tests.test_engine_loop", "tests.test_extension_statsmailer", "tests.test_extension_throttle", "tests.test_linkextractors", "tests.test_logformatter", "tests.test_mail", "tests.test_pqueues", "tests.test_scheduler_base", "tests.test_settings", "tests.test_spider", "tests.test_spider_crawl", "tests.test_spidermiddleware_output_chain", "tests.test_spidermiddleware_process_start", "tests.test_spider_sitemap", "tests.test_squeues", "tests.test_squeues_request", "tests.test_stats", "tests.utils.bases.spider", ] check_untyped_defs = false # Interface classes are hard to support [[tool.mypy.overrides]] module = "twisted.internet.interfaces" follow_imports = "skip" [[tool.mypy.overrides]] module = "scrapy.interfaces" ignore_errors = true [[tool.mypy.overrides]] module = "twisted.internet.reactor" follow_imports = "skip" # just for twisted.version [[tool.mypy.overrides]] module = "twisted" implicit_reexport = true # TODO [[tool.mypy.overrides]] module = "scrapy.settings.default_settings" ignore_errors = true # usually no type hints [[tool.mypy.overrides]] module = [ "bpython", "brotli", "brotlicffi", "google.*", "pydispatch.*", "pyftpdlib.*", "pytest_twisted", "robotexclusionrulesparser", "zope.interface.*", ] ignore_missing_imports = true [tool.bumpversion] current_version = "2.17.0" commit = true tag = true tag_name = "{new_version}" [[tool.bumpversion.files]] filename = "docs/news.rst" search = "\\(unreleased\\)$" replace = "({now:%Y-%m-%d})" regex = true [[tool.bumpversion.files]] filename = "scrapy/VERSION" [[tool.bumpversion.files]] filename = "SECURITY.md" parse = """(?P<major>0|[1-9]\\d*)\\.(?P<minor>0|[1-9]\\d*)""" serialize = ["{major}.{minor}"] [tool.coverage.run] # sysmon, default on 3.14, is too slow: https://github.com/coveragepy/coveragepy/issues/2172 core = "ctrace" branch = true include = ["scrapy/*"] omit = ["tests/*"] disable_warnings = ["include-ignored"] patch = [ "subprocess", ] [tool.coverage.paths] source = [ "scrapy", ".tox/**/site-packages/scrapy" ] [tool.coverage.report] exclude_also = [ "@(abc\\.)?abstractmethod", '\A(?s:.*# pragma: no file cover.*)\Z', ] [tool.pylint.MASTER] persistent = "no" jobs = 1 # >1 hides results extension-pkg-allow-list=[ "lxml", ] load-plugins = ["pylint_per_file_ignores"] [tool.pylint."MESSAGES CONTROL"] enable = [ "useless-suppression", ] # Make INFO checks like useless-suppression also cause pylint to return a # non-zero exit code. fail-on = "I" disable = [ # Ones we want to ignore "attribute-defined-outside-init", "broad-exception-caught", "consider-using-with", "cyclic-import", "disallowed-name", "duplicate-code", # https://github.com/pylint-dev/pylint/issues/214 "fixme", "inherit-non-class", # false positives with create_deprecated_class() "invalid-name", "invalid-overridden-method", "isinstance-second-argument-not-valid-type", # false positives with create_deprecated_class() "line-too-long", "logging-format-interpolation", "logging-fstring-interpolation", "logging-not-lazy", "missing-docstring", "no-member", "no-value-for-parameter", # https://github.com/pylint-dev/pylint/issues/3268 "not-callable", "protected-access", "redefined-outer-name", "too-few-public-methods", "too-many-ancestors", "too-many-arguments", "too-many-function-args", "too-many-instance-attributes", "too-many-lines", "too-many-locals", "too-many-positional-arguments", "too-many-public-methods", "too-many-return-statements", "undefined-variable", "unused-argument", "unused-variable", "use-implicit-booleaness-not-comparison", "useless-import-alias", # used as a hint to mypy "useless-return", # https://github.com/pylint-dev/pylint/issues/6530 "wrong-import-position", # Ones that are implemented in ruff and need to be disabled for some lines (listed here to avoid two disable comments) "bare-except", "eval-used", "global-statement", "import-outside-toplevel", "import-self", "inconsistent-return-statements", "redefined-builtin", "too-many-branches", "unused-import", # Ones that we may want to address (fix, ignore per-line or move to "don't want to fix") "arguments-differ", "keyword-arg-before-vararg", ] # requires `pylint_per_file_ignores` plugin per-file-ignores = [ # Extended list of ones that we may want to address, only for tests "./tests/*:abstract-method,arguments-renamed,dangerous-default-value,pointless-statement,raise-missing-from,unnecessary-dunder-call,used-before-assignment", ] [tool.pytest.ini_options] addopts = [ "--reactor=asyncio", ] xfail_strict = true python_files = ["test_*.py", "test_*/__init__.py"] markers = [ "only_asyncio: marks tests that require the asyncio loop to be used", "only_not_asyncio: marks tests that require the asyncio loop to not be used", "requires_reactor: marks tests that require a reactor", "requires_uvloop: marks tests as only enabled when uvloop is known to be working", "requires_botocore: marks tests that need botocore (but not boto3)", "requires_boto3: marks tests that need botocore and boto3", "requires_mitmproxy: marks tests that need a mitmdump executable", "requires_internet: marks tests that need real Internet access", ] filterwarnings = [ "ignore::DeprecationWarning:twisted.web.static", # Jobs that do not report coverage disable it with --no-cov, which pytest-cov # warns about because the coverage options below stay in place. "ignore::pytest_cov.CovDisabledWarning", # Twisted doesn't close failed sockets after CannotListenError: https://github.com/twisted/twisted/issues/6108 "ignore:Exception ignored in. <socket\\.socket.*laddr=..0\\.0\\.0\\.0., 0.:pytest.PytestUnraisableExceptionWarning", ] [tool.ruff.lint] extend-select = [ # flake8-builtins "A", # flake8-async "ASYNC", # flake8-bugbear "B", # flake8-comprehensions "C4", # flake8-commas "COM", # pydocstyle "D", # flake8-future-annotations "FA", # flynt "FLY", # refurb "FURB", # isort "I", # flake8-implicit-str-concat "ISC", # flake8-logging "LOG", # Perflint "PERF", # pygrep-hooks "PGH", # flake8-pie "PIE", # pylint "PL", # flake8-pytest-style "PT", # flake8-use-pathlib "PTH", # flake8-pyi "PYI", # flake8-quotes "Q", # flake8-return "RET", # flake8-raise "RSE", # Ruff-specific rules "RUF", # flake8-bandit "S", # flake8-simplify "SIM", # flake8-slots "SLOT", # flake8-debugger "T10", # flake8-type-checking "TC", # flake8-tidy-imports "TID", # pyupgrade "UP", # pycodestyle warnings "W", # flake8-2020 "YTT", ] ignore = [ # Ones we want to ignore # Trailing comma missing "COM812", # Missing docstring in public module "D100", # Missing docstring in public class "D101", # Missing docstring in public method "D102", # Missing docstring in public function "D103", # Missing docstring in public package "D104", # Missing docstring in magic method "D105", # Missing docstring in public nested class "D106", # Missing docstring in __init__ "D107", # One-line docstring should fit on one line with quotes "D200", # No blank lines allowed after function docstring "D202", # 1 blank line required between summary line and description "D205", # Multi-line docstring closing quotes should be on a separate line "D209", # First line should end with a period "D400", # First line should be in imperative mood; try rephrasing "D401", # First line should not be the function's "signature" "D402", # First word of the first line should be properly capitalized "D403", # `try`-`except` within a loop incurs performance overhead "PERF203", # Too many return statements "PLR0911", # Too many arguments in function definition "PLR0913", # Magic value used in comparison "PLR2004", # String contains ambiguous {}. "RUF001", # Docstring contains ambiguous {}. "RUF002", # Comment contains ambiguous {}. "RUF003", # Use of `assert` detected; needed for mypy "S101", # FTP-related functions are being called; https://github.com/scrapy/scrapy/issues/4180 "S321", # Use a context manager for opening files "SIM115", # Yoda condition detected "SIM300", ] [tool.ruff.lint.flake8-tidy-imports] banned-module-level-imports = [ "twisted.internet.reactor", # indirectly imports twisted.conch.insults.helper which imports twisted.internet.reactor "twisted.conch.manhole", # directly imports twisted.internet.reactor "twisted.protocols.ftp", ] [tool.ruff.lint.isort] split-on-trailing-comma = false [tool.ruff.lint.per-file-ignores] # Circular import workarounds "scrapy/linkextractors/__init__.py" = ["E402"] "scrapy/spiders/__init__.py" = ["E402"] "tests/**" = [ # Skip bandit and allow blocking file I/O in tests "ASYNC240", "S", # Ones that we may want to address (fix, ignore per-line or move to "don't want to fix") # Assigning to `os.environ` doesn't clear the environment. "B003", # Do not use mutable data structures for argument defaults. "B006", # Found useless expression. "B018", # No explicit stacklevel argument found. "B028", # Within an `except` clause, raise exceptions with `raise ... from` "B904", # `for` loop variable overwritten by assignment target "PLW2901", # Mutable class attributes should be annotated with `typing.ClassVar` "RUF012", # Use capitalized environment variable "SIM112", ] # Issues pending a review: "docs/conf.py" = ["E402"] "scrapy/utils/url.py" = ["F403", "F405"] "tests/test_loader.py" = ["E741"] [tool.ruff.lint.pydocstyle] convention = "pep257" [tool.sphinx-scrapy] python-version = "3.14" # Keep in sync with .github/workflows/checks.yml.