tests/benchmarks/test_crawl.py - master - githubmirror/scrapy

tests/benchmarks/test_crawl.py
270 строк9 KB
Adrian
Benchmark item processing and item concurrency (#7954)
09 авг 2026, 12:01
09 авг 2026, 12:011e92635
from __future__ import annotations

import asyncio
from collections import Counter
from typing import TYPE_CHECKING, Any
from urllib.parse import urlencode

import pytest

from scrapy import Field, Item, Request, Spider
from scrapy.linkextractors import LinkExtractor
from tests.benchmarks import NullDownloadHandler, crawl

if TYPE_CHECKING:
    from collections.abc import AsyncIterator

    from pytest_codspeed import BenchmarkFixture  # type: ignore[import-not-found]

    from scrapy.crawler import Crawler
    from scrapy.http import Response
    from tests.mockserver.http import MockServer

pytest.importorskip("pytest_codspeed", reason="Benchmarks require pytest-codspeed")

PAGES = 100
LINKS_PER_PAGE = 5

# Requests per crawl of the benchmarks that use NullDownloadHandler. The broad
# crawl scenarios split them differently between hostnames and pages per
# hostname.
REQUESTS = 200
BROAD_DEEP_PAGES = 10

# Requests per crawl and delay of the benchmarks that wait, where wall time,
# unlike in the other benchmarks, is a function of the delay.
DELAYED_REQUESTS = 50
DELAY = 0.005

# Requests per crawl and items per response of the benchmarks that measure item
# processing, which reaches fewer pages than the other benchmarks because every
# page costs it several items.
ITEM_REQUESTS = 20
ITEMS_PER_RESPONSE = 100

# Item concurrency limits of the benchmarks that measure item processing. The
# high limit is above the number of items that a response yields in any of
# them.
HIGH_CONCURRENT_ITEMS = 1000
DELAYED_CONCURRENT_ITEMS = 50

NULL_SETTINGS: dict[str, Any] = {
    "DOWNLOAD_HANDLERS": {"http": NullDownloadHandler},
    "LOG_ENABLED": False,
}


class _Page(Item):
    url = Field()
    anchors = Field()


class _FollowSpider(Spider):
    name = "benchmark"
    url: str
    link_extractor = LinkExtractor()

    async def start(self) -> AsyncIterator[Any]:
        yield Request(self.url, dont_filter=True)

    def parse(self, response: Response) -> Any:
        yield _Page(
            url=response.url,
            anchors=response.css("a::text").getall(),
        )
        for link in self.link_extractor.extract_links(response):  # type: ignore[arg-type]
            yield Request(link.url)


class _TreeSpider(Spider):
    """Crawl *pages* pages on each of *domains* hostnames, yielding *items*
    items from every page.

    Pages are numbered from 1, and page *n* links to pages *2n* and *2n+1*, so
    that requests also reach the scheduler from callbacks, and not only from
    :meth:`~scrapy.Spider.start`.
    """

    name = "benchmark-tree"
    domains: int = 1
    pages: int = 1
    items: int = 0

    async def start(self) -> AsyncIterator[Any]:
        for domain in range(self.domains):
            yield Request(f"http://d{domain}.example.com/1")

    def parse(self, response: Response) -> Any:
        page = int(response.url.rpartition("/")[2])
        for child in (page * 2, page * 2 + 1):
            if child <= self.pages:
                yield Request(response.urljoin(f"/{child}"))
        for _ in range(self.items):
            yield _Page(url=response.url)


class _Pipeline:
    def process_item(self, item: Any) -> Any:
        return item


class _DelayedPipeline:
    """Item pipeline that waits, so that the item concurrency limit applies.

    The peak number of items of a same response in progress is tracked in the
    ``benchmark/peak_items`` stat. Items are counted per response because the
    limit is per response, and the items of a response are processed while
    later responses are already being downloaded.
    """

    def __init__(self, crawler: Crawler):
        self._crawler = crawler
        self._active: Counter[str] = Counter()

    @classmethod
    def from_crawler(cls, crawler: Crawler) -> _DelayedPipeline:
        return cls(crawler)

    async def process_item(self, item: Any) -> Any:
        url = item["url"]
        self._active[url] += 1
        assert self._crawler.stats
        self._crawler.stats.max_value("benchmark/peak_items", self._active[url])
        try:
            await asyncio.sleep(DELAY)
            return item
        finally:
            self._active[url] -= 1


def _crawl_tree(
    settings: dict[str, Any], *, domains: int, pages: int, items: int = 0
) -> Crawler:
    crawler = crawl(
        _TreeSpider,
        {**NULL_SETTINGS, **settings},
        domains=domains,
        pages=pages,
        items=items,
    )
    assert crawler.stats
    assert crawler.stats.get_value("downloader/response_count") == domains * pages
    assert crawler.stats.get_value("item_scraped_count", 0) == domains * pages * items
    return crawler


def test_overhead_http(benchmark: BenchmarkFixture, mockserver: MockServer) -> None:
    """Per-request overhead of a crawl over HTTP.

    The pages are small on purpose, so that the cost of parsing them stays
    negligible next to the cost of moving requests and responses through the
    engine, the middlewares and the download handler.
    """
    query = urlencode({"total": PAGES, "show": LINKS_PER_PAGE, "order": "desc"})
    url = mockserver.url(f"/follow?{query}")
    settings = {"ITEM_PIPELINES": {_Pipeline: 100}, "LOG_ENABLED": False}

    def run() -> None:
        crawler = crawl(_FollowSpider, settings, url=url)
        assert crawler.stats
        assert crawler.stats.get_value("item_scraped_count") == PAGES + 1

    benchmark(run)


def test_overhead_engine(benchmark: BenchmarkFixture) -> None:
    """Per-request overhead of a crawl of a single hostname without any I/O."""

    def run() -> None:
        crawler = _crawl_tree({}, domains=1, pages=REQUESTS)
        assert crawler.stats
        assert crawler.stats.get_value("benchmark/peak_concurrency") > 1

    benchmark(run)


@pytest.mark.parametrize(
    ("domains", "pages"),
    [
        pytest.param(REQUESTS, 1, id="shallow"),
        pytest.param(REQUESTS // BROAD_DEEP_PAGES, BROAD_DEEP_PAGES, id="deep"),
    ],
)
def test_overhead_broad(benchmark: BenchmarkFixture, domains: int, pages: int) -> None:
    """Per-request overhead of a broad crawl.

    The shallow scenario, which reaches a single page of every hostname, pays
    the cost of tracking a hostname for the first time on every request, and
    gets its requests from :meth:`~scrapy.Spider.start`. The deep scenario,
    which reaches the same number of pages spread over fewer hostnames,
    amortizes that cost, and instead keeps several requests per hostname
    waiting in the scheduler.
    """
    benchmark(lambda: _crawl_tree({}, domains=domains, pages=pages))


def test_overhead_concurrency(benchmark: BenchmarkFixture) -> None:
    """Overhead of a crawl limited to 1 request at a time on a single hostname."""
    settings = {"CONCURRENT_REQUESTS_PER_DOMAIN": 1}
    benchmark(lambda: _crawl_tree(settings, domains=1, pages=REQUESTS))


def test_overhead_delay(benchmark: BenchmarkFixture) -> None:
    """Overhead of a crawl where every request waits for a download delay.

    The delay is not randomized, so that wall time, and hence the number of
    reactor iterations that the crawl needs, does not change between runs.
    """
    settings = {"DOWNLOAD_DELAY": DELAY, "RANDOMIZE_DOWNLOAD_DELAY": False}
    benchmark(lambda: _crawl_tree(settings, domains=1, pages=DELAYED_REQUESTS))


@pytest.mark.parametrize(
    ("items", "settings"),
    [
        pytest.param(1, {}, id="single"),
        pytest.param(ITEMS_PER_RESPONSE, {}, id="many"),
        pytest.param(
            1,
            {"CONCURRENT_ITEMS": HIGH_CONCURRENT_ITEMS},
            id="high-limit",
        ),
    ],
)
def test_overhead_items(
    benchmark: BenchmarkFixture, items: int, settings: dict[str, Any]
) -> None:
    """Overhead of sending the items of a callback through the item pipeline.

    The single and many scenarios, which use the default
    :setting:`CONCURRENT_ITEMS` value, measure how that overhead grows with the
    number of items that a response yields. The high-limit scenario instead
    raises :setting:`CONCURRENT_ITEMS` well above that number.
    """
    benchmark(
        lambda: _crawl_tree(settings, domains=1, pages=ITEM_REQUESTS, items=items)
    )


def test_overhead_item_concurrency(benchmark: BenchmarkFixture) -> None:
    """Overhead of a crawl where item processing waits.

    Every response yields more items than :setting:`CONCURRENT_ITEMS` allows in
    parallel, so that the item pipeline gets them in several batches, and wall
    time, unlike in most of the other benchmarks, is a function of the delay.
    """
    settings = {
        "CONCURRENT_ITEMS": DELAYED_CONCURRENT_ITEMS,
        "ITEM_PIPELINES": {_DelayedPipeline: 100},
    }

    def run() -> None:
        crawler = _crawl_tree(
            settings, domains=1, pages=ITEM_REQUESTS, items=ITEMS_PER_RESPONSE
        )
        assert crawler.stats
        assert (
            crawler.stats.get_value("benchmark/peak_items") == DELAYED_CONCURRENT_ITEMS
        )

    benchmark(run)