From 25628c47764501f9d292f96ab5153ea92150a00f Mon Sep 17 00:00:00 2001 From: "Aryan Singh K." <70511529+aryansk@users.noreply.github.com> Date: Sat, 15 Aug 2026 13:03:24 +0530 Subject: [PATCH 1/3] fix(sitemap): let from_xml_string opt into enqueue host filtering MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Sitemap.from_xml_string hardcoded a raw SitemapSource with no url, so host filtering (default `same-hostname`) was silently skipped whenever raw XML was parsed — even when the caller knew where the sitemap came from. Accept optional `sitemap_url` and `parse_sitemap_options` and forward them so the source carries an origin and the same enqueue strategy rules apply as for URL-loaded sitemaps (gh #2118). 🤖 Generated with Codebuff Co-Authored-By: Codebuff --- src/crawlee/_utils/sitemap.py | 13 +++++++++++-- tests/unit/_utils/test_sitemap.py | 31 +++++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+), 2 deletions(-) diff --git a/src/crawlee/_utils/sitemap.py b/src/crawlee/_utils/sitemap.py index d110f0225c..d34cd04581 100644 --- a/src/crawlee/_utils/sitemap.py +++ b/src/crawlee/_utils/sitemap.py @@ -496,8 +496,17 @@ async def load( ) @classmethod - async def from_xml_string(cls, content: str) -> Sitemap: - return await cls.parse([SitemapSource(type='raw', content=content)]) + async def from_xml_string( + cls, + content: str, + *, + sitemap_url: str | None = None, + parse_sitemap_options: ParseSitemapOptions | None = None, + ) -> Sitemap: + source: SitemapSource = {'type': 'raw', 'content': content} + if sitemap_url is not None: + source['url'] = sitemap_url + return await cls.parse([source], parse_sitemap_options=parse_sitemap_options) @classmethod async def parse( diff --git a/tests/unit/_utils/test_sitemap.py b/tests/unit/_utils/test_sitemap.py index e1030844ab..5dcc0fa6ab 100644 --- a/tests/unit/_utils/test_sitemap.py +++ b/tests/unit/_utils/test_sitemap.py @@ -321,6 +321,37 @@ async def test_sitemap_from_string() -> None: assert set(sitemap.urls) == get_basic_results() +async def test_sitemap_from_string_keeps_same_host_with_sitemap_url() -> None: + """URLs on the sitemap's own host survive the default `same-hostname` filter.""" + sitemap = await Sitemap.from_xml_string( + get_basic_sitemap(), + sitemap_url=f'{DEFAULT_URL}sitemap.xml', + ) + + assert set(sitemap.urls) == get_basic_results() + + +async def test_sitemap_from_string_filters_cross_host_with_sitemap_url() -> None: + """`from_xml_string` opts into host filtering when `sitemap_url` is given.""" + sitemap = await Sitemap.from_xml_string( + get_basic_sitemap(url='https://other.com/'), + sitemap_url=f'{DEFAULT_URL}sitemap.xml', + ) + + assert sitemap.urls == [] + + +async def test_sitemap_from_string_allows_cross_host_with_strategy_all() -> None: + """`enqueue_strategy='all'` disables host filtering for raw string sitemaps too.""" + sitemap = await Sitemap.from_xml_string( + get_basic_sitemap(url='https://other.com/'), + sitemap_url=f'{DEFAULT_URL}sitemap.xml', + parse_sitemap_options={'enqueue_strategy': 'all'}, + ) + + assert set(sitemap.urls) == get_basic_results('https://other.com/') + + async def test_malformed_sitemap_keeps_urls() -> None: """A parse error must not discard the URLs collected before it.""" malformed = ( From 6b2c6a00eec3f95a526b42de4120f4e7fcb0a4d7 Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Tue, 18 Aug 2026 10:01:05 +0200 Subject: [PATCH 2/3] fix(sitemap): report the provided URL as origin_sitemap_url for raw sources --- src/crawlee/_utils/sitemap.py | 17 +++++++++++++---- tests/unit/_utils/test_sitemap.py | 18 ++++++++++++++++++ 2 files changed, 31 insertions(+), 4 deletions(-) diff --git a/src/crawlee/_utils/sitemap.py b/src/crawlee/_utils/sitemap.py index d34cd04581..74469f67cb 100644 --- a/src/crawlee/_utils/sitemap.py +++ b/src/crawlee/_utils/sitemap.py @@ -234,10 +234,10 @@ def _get_parser(content_type: str = '', url: str | None = None) -> _XmlSitemapPa def _get_origin_url(source: SitemapSource) -> str: """Determine the origin URL for a sitemap source.""" - if source['type'] == 'url' and 'url' in source: - return source['url'] + if url := source.get('url'): + return url if source['type'] == 'raw' and 'content' in source: - # For raw content sources, create a consistent identifier + # Raw sources without a known URL get a consistent content-derived identifier. return f'raw://{sha256(source["content"].encode()).hexdigest()}' return '' @@ -503,7 +503,16 @@ async def from_xml_string( sitemap_url: str | None = None, parse_sitemap_options: ParseSitemapOptions | None = None, ) -> Sitemap: - source: SitemapSource = {'type': 'raw', 'content': content} + """Parse a sitemap from a raw XML string. + + Args: + content: The sitemap XML content. + sitemap_url: URL the content was retrieved from. Providing it enables the same URL filtering as for + URL-loaded sitemaps: entries are kept only if they match the enqueue strategy (`same-hostname` by + default) relative to this URL. Without it, no filtering is applied. + parse_sitemap_options: Options for parsing, see `ParseSitemapOptions`. + """ + source = SitemapSource(type='raw', content=content) if sitemap_url is not None: source['url'] = sitemap_url return await cls.parse([source], parse_sitemap_options=parse_sitemap_options) diff --git a/tests/unit/_utils/test_sitemap.py b/tests/unit/_utils/test_sitemap.py index 5dcc0fa6ab..f82a6af31a 100644 --- a/tests/unit/_utils/test_sitemap.py +++ b/tests/unit/_utils/test_sitemap.py @@ -321,6 +321,13 @@ async def test_sitemap_from_string() -> None: assert set(sitemap.urls) == get_basic_results() +async def test_sitemap_from_string_keeps_cross_host_without_sitemap_url() -> None: + """Without `sitemap_url` there is no origin to filter against, so all URLs are kept.""" + sitemap = await Sitemap.from_xml_string(get_basic_sitemap(url='https://other.com/')) + + assert set(sitemap.urls) == get_basic_results('https://other.com/') + + async def test_sitemap_from_string_keeps_same_host_with_sitemap_url() -> None: """URLs on the sitemap's own host survive the default `same-hostname` filter.""" sitemap = await Sitemap.from_xml_string( @@ -352,6 +359,17 @@ async def test_sitemap_from_string_allows_cross_host_with_strategy_all() -> None assert set(sitemap.urls) == get_basic_results('https://other.com/') +async def test_raw_source_with_url_uses_it_as_origin() -> None: + """A raw source with a known URL reports it as `origin_sitemap_url` instead of a `raw://` identifier.""" + sitemap_url = f'{DEFAULT_URL}sitemap.xml' + items = [ + item async for item in parse_sitemap([{'type': 'raw', 'content': get_basic_sitemap(), 'url': sitemap_url}]) + ] + + assert len(items) == 5 + assert all(item.origin_sitemap_url == sitemap_url for item in items) + + async def test_malformed_sitemap_keeps_urls() -> None: """A parse error must not discard the URLs collected before it.""" malformed = ( From c07c3a3c450524cacfa836ca682be3c7e7cc1ff4 Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Tue, 18 Aug 2026 10:11:16 +0200 Subject: [PATCH 3/3] test: cover the raw:// content-hash origin fallback for raw sitemap sources --- tests/unit/_utils/test_sitemap.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tests/unit/_utils/test_sitemap.py b/tests/unit/_utils/test_sitemap.py index f82a6af31a..ffe38efadb 100644 --- a/tests/unit/_utils/test_sitemap.py +++ b/tests/unit/_utils/test_sitemap.py @@ -370,6 +370,17 @@ async def test_raw_source_with_url_uses_it_as_origin() -> None: assert all(item.origin_sitemap_url == sitemap_url for item in items) +async def test_raw_source_without_url_uses_content_hash_as_origin() -> None: + """A raw source without a URL reports a consistent content-derived `raw://` identifier as `origin_sitemap_url`.""" + items = [item async for item in parse_sitemap([{'type': 'raw', 'content': get_basic_sitemap()}])] + + assert len(items) == 5 + origin = items[0].origin_sitemap_url + assert origin is not None + assert origin.startswith('raw://') + assert all(item.origin_sitemap_url == origin for item in items) + + async def test_malformed_sitemap_keeps_urls() -> None: """A parse error must not discard the URLs collected before it.""" malformed = (