tests/test_utils_sitemap.py - master - githubmirror/scrapy

tests/test_utils_sitemap.py
326 строк9 KB
Andrey Rakhmatullin
Next refactoring pass of test_utils_*. (#7797)
28 июл 2026, 15:24
28 июл 2026, 15:24e7d8b34
from __future__ import annotations

import pytest

from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.utils.sitemap import Sitemap, sitemap_urls_from_robots


def test_sitemap():
    s = Sitemap(
        b"""<?xml version="1.0" encoding="UTF-8"?>
<urlset xmlns="http://www.google.com/schemas/sitemap/0.84">
<url>
<loc>http://www.example.com/</loc>
<lastmod>2009-08-16</lastmod>
<changefreq>daily</changefreq>
<priority>1</priority>
</url>
<url>
<loc>http://www.example.com/Special-Offers.html</loc>
<lastmod>2009-08-16</lastmod>
<changefreq>weekly</changefreq>
<priority>0.8</priority>
</url>
</urlset>"""
    )
    assert s.type == "urlset"
    assert list(s) == [
        {
            "priority": "1",
            "loc": "http://www.example.com/",
            "lastmod": "2009-08-16",
            "changefreq": "daily",
        },
        {
            "priority": "0.8",
            "loc": "http://www.example.com/Special-Offers.html",
            "lastmod": "2009-08-16",
            "changefreq": "weekly",
        },
    ]


def test_sitemap_index():
    s = Sitemap(
        b"""<?xml version="1.0" encoding="UTF-8"?>
<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
<sitemap>
  <loc>http://www.example.com/sitemap1.xml.gz</loc>
  <lastmod>2004-10-01T18:23:17+00:00</lastmod>
</sitemap>
<sitemap>
  <loc>http://www.example.com/sitemap2.xml.gz</loc>
  <lastmod>2005-01-01</lastmod>
</sitemap>
</sitemapindex>"""
    )
    assert s.type == "sitemapindex"
    assert list(s) == [
        {
            "loc": "http://www.example.com/sitemap1.xml.gz",
            "lastmod": "2004-10-01T18:23:17+00:00",
        },
        {
            "loc": "http://www.example.com/sitemap2.xml.gz",
            "lastmod": "2005-01-01",
        },
    ]


def test_sitemap_strip():
    """Assert we can deal with trailing spaces inside <loc> tags - we've
    seen those
    """
    s = Sitemap(
        b"""<?xml version="1.0" encoding="UTF-8"?>
<urlset xmlns="http://www.google.com/schemas/sitemap/0.84">
<url>
<loc> http://www.example.com/</loc>
<lastmod>2009-08-16</lastmod>
<changefreq>daily</changefreq>
<priority>1</priority>
</url>
<url>
<loc> http://www.example.com/2</loc>
<lastmod />
</url>
</urlset>
"""
    )
    assert list(s) == [
        {
            "priority": "1",
            "loc": "http://www.example.com/",
            "lastmod": "2009-08-16",
            "changefreq": "daily",
        },
        {"loc": "http://www.example.com/2", "lastmod": ""},
    ]


def test_sitemap_wrong_ns():
    """We have seen sitemaps with wrongs ns. Presumably, Google still works
    with these, though is not 100% confirmed"""
    s = Sitemap(
        b"""<?xml version="1.0" encoding="UTF-8"?>
<urlset xmlns="http://www.google.com/schemas/sitemap/0.84">
<url xmlns="">
<loc> http://www.example.com/</loc>
<lastmod>2009-08-16</lastmod>
<changefreq>daily</changefreq>
<priority>1</priority>
</url>
<url xmlns="">
<loc> http://www.example.com/2</loc>
<lastmod />
</url>
</urlset>
"""
    )
    assert list(s) == [
        {
            "priority": "1",
            "loc": "http://www.example.com/",
            "lastmod": "2009-08-16",
            "changefreq": "daily",
        },
        {"loc": "http://www.example.com/2", "lastmod": ""},
    ]


def test_sitemap_wrong_ns2():
    """We have seen sitemaps with wrongs ns. Presumably, Google still works
    with these, though is not 100% confirmed"""
    s = Sitemap(
        b"""<?xml version="1.0" encoding="UTF-8"?>
<urlset>
<url xmlns="">
<loc> http://www.example.com/</loc>
<lastmod>2009-08-16</lastmod>
<changefreq>daily</changefreq>
<priority>1</priority>
</url>
<url xmlns="">
<loc> http://www.example.com/2</loc>
<lastmod />
</url>
</urlset>
"""
    )
    assert s.type == "urlset"
    assert list(s) == [
        {
            "priority": "1",
            "loc": "http://www.example.com/",
            "lastmod": "2009-08-16",
            "changefreq": "daily",
        },
        {"loc": "http://www.example.com/2", "lastmod": ""},
    ]


def test_sitemap_urls_from_robots():
    robots = b"""User-agent: *
Disallow: /aff/
Disallow: /wl/

# Search and shopping refining
Disallow: /s*/*facet
Disallow: /s*/*tags

# Sitemap files
Sitemap: http://example.com/sitemap.xml
Sitemap: http://example.com/sitemap-product-index.xml
Sitemap: HTTP://example.com/sitemap-uppercase.xml
Sitemap: /sitemap-relative-url.xml

# Forums
Disallow: /forum/search/
Disallow: /forum/active/
"""
    assert list(sitemap_urls_from_robots(robots, base_url="http://example.com")) == [
        "http://example.com/sitemap.xml",
        "http://example.com/sitemap-product-index.xml",
        "http://example.com/sitemap-uppercase.xml",
        "http://example.com/sitemap-relative-url.xml",
    ]


def test_sitemap_urls_from_robots_str_compat():
    robots = """User-agent: *
Disallow: /aff/
Disallow: /wl/

# Search and shopping refining
Disallow: /s*/*facet
Disallow: /s*/*tags

# Sitemap files
Sitemap: http://example.com/sitemap.xml
Sitemap: http://example.com/sitemap-product-index.xml
Sitemap: HTTP://example.com/sitemap-uppercase.xml
Sitemap: /sitemap-relative-url.xml

# Forums
Disallow: /forum/search/
Disallow: /forum/active/
"""

    with pytest.warns(
        ScrapyDeprecationWarning,
        match="Passing `str` type as `robots_text` is deprecated",
    ):
        assert list(
            sitemap_urls_from_robots(robots, base_url="http://example.com")
        ) == [
            "http://example.com/sitemap.xml",
            "http://example.com/sitemap-product-index.xml",
            "http://example.com/sitemap-uppercase.xml",
            "http://example.com/sitemap-relative-url.xml",
        ]


def test_sitemap_blanklines():
    """Assert we can deal with starting blank lines before <xml> tag"""
    s = Sitemap(
        b"""
<?xml version="1.0" encoding="UTF-8"?>
<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">

<!-- cache: cached = yes name = sitemap_jspCache key = sitemap -->
<sitemap>
<loc>http://www.example.com/sitemap1.xml</loc>
<lastmod>2013-07-15</lastmod>
</sitemap>

<sitemap>
<loc>http://www.example.com/sitemap2.xml</loc>
<lastmod>2013-07-15</lastmod>
</sitemap>

<sitemap>
<loc>http://www.example.com/sitemap3.xml</loc>
<lastmod>2013-07-15</lastmod>
</sitemap>

<!-- end cache -->
</sitemapindex>
"""
    )
    assert list(s) == [
        {"lastmod": "2013-07-15", "loc": "http://www.example.com/sitemap1.xml"},
        {"lastmod": "2013-07-15", "loc": "http://www.example.com/sitemap2.xml"},
        {"lastmod": "2013-07-15", "loc": "http://www.example.com/sitemap3.xml"},
    ]


def test_comment():
    s = Sitemap(
        b"""<?xml version="1.0" encoding="UTF-8"?>
<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"
    xmlns:xhtml="http://www.w3.org/1999/xhtml">
    <url>
        <loc>http://www.example.com/</loc>
        <!-- this is a comment on which the parser might raise an exception if implemented incorrectly -->
    </url>
</urlset>"""
    )
    assert list(s) == [{"loc": "http://www.example.com/"}]


def test_alternate():
    s = Sitemap(
        b"""<?xml version="1.0" encoding="UTF-8"?>
<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"
    xmlns:xhtml="http://www.w3.org/1999/xhtml">
    <url>
        <loc>http://www.example.com/english/</loc>
        <xhtml:link rel="alternate" hreflang="de"
            href="http://www.example.com/deutsch/"/>
        <xhtml:link rel="alternate" hreflang="de-ch"
            href="http://www.example.com/schweiz-deutsch/"/>
        <xhtml:link rel="alternate" hreflang="en"
            href="http://www.example.com/english/"/>
        <xhtml:link rel="alternate" hreflang="en"/><!-- wrong tag without href -->
    </url>
</urlset>"""
    )
    assert list(s) == [
        {
            "loc": "http://www.example.com/english/",
            "alternate": [
                "http://www.example.com/deutsch/",
                "http://www.example.com/schweiz-deutsch/",
                "http://www.example.com/english/",
            ],
        }
    ]


def test_xml_entity_expansion():
    s = Sitemap(
        b"""<?xml version="1.0" encoding="utf-8"?>
      <!DOCTYPE foo [
      <!ELEMENT foo ANY >
      <!ENTITY xxe SYSTEM "file:///etc/passwd" >
      ]>
      <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
        <url>
          <loc>http://127.0.0.1:8000/&xxe;</loc>
        </url>
      </urlset>
    """
    )
    assert list(s) == [{"loc": "http://127.0.0.1:8000/"}]


def test_sitemap_non_string_tag():
    """With recover=True and resolve_entities=False, libxml2 >= 2.14.6 (used
    by lxml >= 6.1.1) preserves undeclared entity reference nodes whose
    .tag is a non-string ``Cython function`` object instead of a ``str``.
    _get_tag_name must handle this gracefully instead of raising
    AttributeError.
    """
    results = list(Sitemap(b"<url>&k;"))
    assert results == []