fix(crawlers): Parse HTML responses starting with an XML declaration as HTML in ParselCrawler

src/crawlee/crawlers/_parsel/_parsel_parser.py

@@ -50,7 +50,11 @@ def find_links(self, parsed_content: Selector, selector: str, attribute: str) ->
 
     @staticmethod
     def _parse_body(body: bytes, content_type: str | None) -> Selector:
+        media_type = (content_type or '').split(';')[0].strip().lower()
+        # Other responses keep Parsel's own detection of JSON, XML and HTML.
+        selector_type = 'html' if media_type in {'text/html', 'application/xhtml+xml'} else None
+
         encoding = get_declared_html_encoding(body, content_type)
         if encoding is None:
-            return Selector(body=body)
-        return Selector(text=decode_html_body(body, encoding))
+            return Selector(body=body, type=selector_type)
+        return Selector(text=decode_html_body(body, encoding), type=selector_type)

tests/unit/crawlers/_parsel/test_parsel_crawler.py

@@ -548,3 +548,24 @@ async def test_parse_decodes_page_encoding(texts: list[str], body: bytes, conten
     selector = await ParselParser().parse(response)
 
     assert selector.css('p::text').getall() == texts
+
+
+@pytest.mark.parametrize(
+    ('content_type', 'expected_type'),
+    [
+        pytest.param('text/html', 'html', id='html'),
+        pytest.param('application/xhtml+xml; charset=utf-8', 'html', id='xhtml'),
+        pytest.param('application/xml', 'xml', id='xml'),
+    ],
+)
+async def test_parse_xml_declaration(content_type: str, expected_type: str) -> None:
+    """A page starting with an XML declaration is read as HTML unless the response is XML."""
+    body = (
+        b'<?xml version="1.0" encoding="UTF-8"?>'
+        b'<html xmlns="http://www.w3.org/1999/xhtml"><body><a href="/next">Next</a></body></html>'
+    )
+    response = mock.Mock(headers=HttpHeaders({'Content-Type': content_type}), read=mock.AsyncMock(return_value=body))
+
+    selector = await ParselParser().parse(response)
+
+    assert selector.type == expected_type