diff --git a/src/crawlee/crawlers/_parsel/_parsel_parser.py b/src/crawlee/crawlers/_parsel/_parsel_parser.py index afb5dfdef1..07f57e2148 100644 --- a/src/crawlee/crawlers/_parsel/_parsel_parser.py +++ b/src/crawlee/crawlers/_parsel/_parsel_parser.py @@ -50,7 +50,11 @@ def find_links(self, parsed_content: Selector, selector: str, attribute: str) -> @staticmethod def _parse_body(body: bytes, content_type: str | None) -> Selector: + media_type = (content_type or '').split(';')[0].strip().lower() + # Other responses keep Parsel's own detection of JSON, XML and HTML. + selector_type = 'html' if media_type in {'text/html', 'application/xhtml+xml'} else None + encoding = get_declared_html_encoding(body, content_type) if encoding is None: - return Selector(body=body) - return Selector(text=decode_html_body(body, encoding)) + return Selector(body=body, type=selector_type) + return Selector(text=decode_html_body(body, encoding), type=selector_type) diff --git a/tests/unit/crawlers/_parsel/test_parsel_crawler.py b/tests/unit/crawlers/_parsel/test_parsel_crawler.py index 84e95a7f69..01111f1a2c 100644 --- a/tests/unit/crawlers/_parsel/test_parsel_crawler.py +++ b/tests/unit/crawlers/_parsel/test_parsel_crawler.py @@ -548,3 +548,24 @@ async def test_parse_decodes_page_encoding(texts: list[str], body: bytes, conten selector = await ParselParser().parse(response) assert selector.css('p::text').getall() == texts + + +@pytest.mark.parametrize( + ('content_type', 'expected_type'), + [ + pytest.param('text/html', 'html', id='html'), + pytest.param('application/xhtml+xml; charset=utf-8', 'html', id='xhtml'), + pytest.param('application/xml', 'xml', id='xml'), + ], +) +async def test_parse_xml_declaration(content_type: str, expected_type: str) -> None: + """A page starting with an XML declaration is read as HTML unless the response is XML.""" + body = ( + b'' + b'Next' + ) + response = mock.Mock(headers=HttpHeaders({'Content-Type': content_type}), read=mock.AsyncMock(return_value=body)) + + selector = await ParselParser().parse(response) + + assert selector.type == expected_type