fix(crawlers): Resolve relative and invalid base URLs when extracting links

src/crawlee/crawlers/_abstract_http/_abstract_http_crawler.py

@@ -13,7 +13,7 @@
 from crawlee._request import Request, RequestOptions, RequestState
 from crawlee._utils.docs import docs_group
 from crawlee._utils.time import SharedTimeout
-from crawlee._utils.urls import to_absolute_url_iterator
+from crawlee._utils.urls import convert_to_absolute_url, is_url_absolute, to_absolute_url_iterator
 from crawlee.crawlers._basic import BasicCrawler, BasicCrawlerOptions, ContextPipeline
 from crawlee.errors import RequestThrottledError, SessionError
 from crawlee.statistics import StatisticsState
@@ -218,13 +218,19 @@ async def extract_links(
                 self._parser.find_links(parsed_content, selector=selector, attribute=attribute)
             )
 
-            # Get base URL from <base> tag if present
+            # A relative `<base href>` is resolved against the page URL, and one that does not resolve to an absolute
+            # URL (malformed, `mailto:`, ...) is ignored.
+            base_url = context.request.loaded_url or context.request.url
             extracted_base_urls = list(self._parser.find_links(parsed_content, 'base[href]', 'href'))
-            base_url: str = (
-                str(extracted_base_urls[0])
-                if extracted_base_urls
-                else context.request.loaded_url or context.request.url
-            )
+            if extracted_base_urls:
+                try:
+                    resolved_base_url = convert_to_absolute_url(base_url, extracted_base_urls[0])
+                except ValueError:
+                    resolved_base_url = ''
+                if is_url_absolute(resolved_base_url):
+                    base_url = resolved_base_url
+                else:
+                    context.log.debug(f'Ignoring invalid base URL "{extracted_base_urls[0]}", using the page URL.')
             links_iterator = to_absolute_url_iterator(base_url, links_iterator, logger=context.log)
 
             if robots_txt_file:

src/crawlee/crawlers/_playwright/_playwright_crawler.py

@@ -18,7 +18,7 @@
 from crawlee._utils.docs import docs_group
 from crawlee._utils.robots import RobotsTxtFile
 from crawlee._utils.time import SharedTimeout
-from crawlee._utils.urls import to_absolute_url_iterator
+from crawlee._utils.urls import is_url_absolute, to_absolute_url_iterator
 from crawlee.browsers import BrowserPool
 from crawlee.crawlers._basic import BasicCrawler, BasicCrawlerOptions, ContextPipeline
 from crawlee.errors import RequestThrottledError, SessionError
@@ -445,13 +445,17 @@ async def extract_links(
             strategy = kwargs.get('strategy', 'same-hostname')
 
             elements = await context.page.query_selector_all(selector)
+            # Browsers ignore the whitespace around a URL, as `find_links` of the HTTP parsers does.
             links_iterator: Iterator[str] = iter(
-                [url for element in elements if (url := await element.get_attribute(attribute)) is not None]
+                [url.strip() for element in elements if (url := await element.get_attribute(attribute)) is not None]
             )
 
-            # Get base URL from <base> tag if present
+            # The browser resolves `<base href>` itself. Chromium reports an invalid one as `about:blank` and a
+            # `mailto:`-like one as-is, so the page URL is used whenever the base is not an absolute URL.
             extracted_base_url = await context.page.evaluate('document.baseURI')
-            base_url: str = extracted_base_url or context.request.loaded_url or context.request.url
+            base_url = context.request.loaded_url or context.request.url
+            if is_url_absolute(extracted_base_url):
+                base_url = extracted_base_url
 
             links_iterator = to_absolute_url_iterator(base_url, links_iterator, logger=context.log)
 

tests/unit/crawlers/_beautifulsoup/test_beautifulsoup_crawler.py

@@ -289,6 +289,31 @@ async def request_handler(context: BeautifulSoupCrawlingContext) -> None:
     assert extracted_links[0] == str(server_url / 'page_1')
 
 
+@pytest.mark.parametrize(
+    ('base', 'expected_path'),
+    [
+        pytest.param('{origin}/abs/', '/abs/page', id='absolute-base'),
+        pytest.param('/sub/', '/sub/page', id='relative-base'),
+        pytest.param('http://[bad', '/page', id='invalid-base'),
+        pytest.param('mailto:a@b.c', '/page', id='non-hierarchical-base'),
+    ],
+)
+async def test_extract_links_base_href(server_url: URL, http_client: HttpClient, base: str, expected_path: str) -> None:
+    """Links are resolved against `<base href>`, a relative one against the page URL, and an invalid one is ignored."""
+    crawler = BeautifulSoupCrawler(http_client=http_client)
+    handler = mock.AsyncMock()
+
+    @crawler.router.default_handler
+    async def request_handler(context: BeautifulSoupCrawlingContext) -> None:
+        links = await context.extract_links()
+        await handler([request.url for request in links])
+
+    page = f'<head><base href="{base.format(origin=server_url.origin())}"></head><body><a href="page">Page</a></body>'
+    await crawler.run([str((server_url / 'echo_content').with_query(content=page))])
+
+    handler.assert_awaited_once_with([str(server_url.with_path(expected_path))])
+
+
 async def test_extract_non_href_links(server_url: URL, http_client: HttpClient) -> None:
     crawler = BeautifulSoupCrawler(http_client=http_client)
     extracted_links: list[str] = []

tests/unit/crawlers/_parsel/test_parsel_crawler.py

@@ -375,6 +375,31 @@ async def request_handler(context: ParselCrawlingContext) -> None:
     assert extracted_links[0] == str(server_url / 'page_1')
 
 
+@pytest.mark.parametrize(
+    ('base', 'expected_path'),
+    [
+        pytest.param('{origin}/abs/', '/abs/page', id='absolute-base'),
+        pytest.param('/sub/', '/sub/page', id='relative-base'),
+        pytest.param('http://[bad', '/page', id='invalid-base'),
+        pytest.param('mailto:a@b.c', '/page', id='non-hierarchical-base'),
+    ],
+)
+async def test_extract_links_base_href(server_url: URL, http_client: HttpClient, base: str, expected_path: str) -> None:
+    """Links are resolved against `<base href>`, a relative one against the page URL, and an invalid one is ignored."""
+    crawler = ParselCrawler(http_client=http_client)
+    handler = mock.AsyncMock()
+
+    @crawler.router.default_handler
+    async def request_handler(context: ParselCrawlingContext) -> None:
+        links = await context.extract_links()
+        await handler([request.url for request in links])
+
+    page = f'<head><base href="{base.format(origin=server_url.origin())}"></head><body><a href="page">Page</a></body>'
+    await crawler.run([str((server_url / 'echo_content').with_query(content=page))])
+
+    handler.assert_awaited_once_with([str(server_url.with_path(expected_path))])
+
+
 async def test_extract_non_href_links(server_url: URL, http_client: HttpClient) -> None:
     crawler = ParselCrawler(http_client=http_client)
     extracted_links: list[str] = []

tests/unit/crawlers/_playwright/test_playwright_crawler.py

@@ -934,6 +934,47 @@ async def request_handler(context: PlaywrightCrawlingContext) -> None:
     assert extracted_links[0] == str(server_url / 'page_1')
 
 
+@pytest.mark.parametrize(
+    ('base', 'expected_path'),
+    [
+        pytest.param('{origin}/abs/', '/abs/page', id='absolute-base'),
+        pytest.param('/sub/', '/sub/page', id='relative-base'),
+        pytest.param('http://[bad', '/page', id='invalid-base'),
+        pytest.param('mailto:a@b.c', '/page', id='non-hierarchical-base'),
+    ],
+)
+async def test_extract_links_base_href(server_url: URL, base: str, expected_path: str) -> None:
+    """Links are resolved against `<base href>`, a relative one against the page URL, and an invalid one is ignored."""
+    crawler = PlaywrightCrawler()
+    handler = mock.AsyncMock()
+
+    @crawler.router.default_handler
+    async def request_handler(context: PlaywrightCrawlingContext) -> None:
+        links = await context.extract_links()
+        await handler([request.url for request in links])
+
+    page = f'<head><base href="{base.format(origin=server_url.origin())}"></head><body><a href="page">Page</a></body>'
+    await crawler.run([str((server_url / 'echo_content').with_query(content=page))])
+
+    handler.assert_awaited_once_with([str(server_url.with_path(expected_path))])
+
+
+async def test_extract_links_whitespace(server_url: URL) -> None:
+    """The whitespace around a link is ignored."""
+    crawler = PlaywrightCrawler()
+    handler = mock.AsyncMock()
+
+    @crawler.router.default_handler
+    async def request_handler(context: PlaywrightCrawlingContext) -> None:
+        links = await context.extract_links()
+        await handler([request.url for request in links])
+
+    page = '<body><a href="  /page  ">Page</a></body>'
+    await crawler.run([str((server_url / 'echo_content').with_query(content=page))])
+
+    handler.assert_awaited_once_with([str(server_url / 'page')])
+
+
 async def test_extract_non_href_links(server_url: URL) -> None:
     crawler = PlaywrightCrawler()
     extracted_links: list[str] = []