diff --git a/src/crawlee/crawlers/_abstract_http/_abstract_http_crawler.py b/src/crawlee/crawlers/_abstract_http/_abstract_http_crawler.py
index ed4d79b143..01d298b710 100644
--- a/src/crawlee/crawlers/_abstract_http/_abstract_http_crawler.py
+++ b/src/crawlee/crawlers/_abstract_http/_abstract_http_crawler.py
@@ -13,7 +13,7 @@
from crawlee._request import Request, RequestOptions, RequestState
from crawlee._utils.docs import docs_group
from crawlee._utils.time import SharedTimeout
-from crawlee._utils.urls import to_absolute_url_iterator
+from crawlee._utils.urls import convert_to_absolute_url, is_url_absolute, to_absolute_url_iterator
from crawlee.crawlers._basic import BasicCrawler, BasicCrawlerOptions, ContextPipeline
from crawlee.errors import RequestThrottledError, SessionError
from crawlee.statistics import StatisticsState
@@ -218,13 +218,19 @@ async def extract_links(
self._parser.find_links(parsed_content, selector=selector, attribute=attribute)
)
- # Get base URL from tag if present
+ # A relative `` is resolved against the page URL, and one that does not resolve to an absolute
+ # URL (malformed, `mailto:`, ...) is ignored.
+ base_url = context.request.loaded_url or context.request.url
extracted_base_urls = list(self._parser.find_links(parsed_content, 'base[href]', 'href'))
- base_url: str = (
- str(extracted_base_urls[0])
- if extracted_base_urls
- else context.request.loaded_url or context.request.url
- )
+ if extracted_base_urls:
+ try:
+ resolved_base_url = convert_to_absolute_url(base_url, extracted_base_urls[0])
+ except ValueError:
+ resolved_base_url = ''
+ if is_url_absolute(resolved_base_url):
+ base_url = resolved_base_url
+ else:
+ context.log.debug(f'Ignoring invalid base URL "{extracted_base_urls[0]}", using the page URL.')
links_iterator = to_absolute_url_iterator(base_url, links_iterator, logger=context.log)
if robots_txt_file:
diff --git a/src/crawlee/crawlers/_playwright/_playwright_crawler.py b/src/crawlee/crawlers/_playwright/_playwright_crawler.py
index bd7cdbedc4..399455d723 100644
--- a/src/crawlee/crawlers/_playwright/_playwright_crawler.py
+++ b/src/crawlee/crawlers/_playwright/_playwright_crawler.py
@@ -18,7 +18,7 @@
from crawlee._utils.docs import docs_group
from crawlee._utils.robots import RobotsTxtFile
from crawlee._utils.time import SharedTimeout
-from crawlee._utils.urls import to_absolute_url_iterator
+from crawlee._utils.urls import is_url_absolute, to_absolute_url_iterator
from crawlee.browsers import BrowserPool
from crawlee.crawlers._basic import BasicCrawler, BasicCrawlerOptions, ContextPipeline
from crawlee.errors import RequestThrottledError, SessionError
@@ -445,13 +445,17 @@ async def extract_links(
strategy = kwargs.get('strategy', 'same-hostname')
elements = await context.page.query_selector_all(selector)
+ # Browsers ignore the whitespace around a URL, as `find_links` of the HTTP parsers does.
links_iterator: Iterator[str] = iter(
- [url for element in elements if (url := await element.get_attribute(attribute)) is not None]
+ [url.strip() for element in elements if (url := await element.get_attribute(attribute)) is not None]
)
- # Get base URL from tag if present
+ # The browser resolves `` itself. Chromium reports an invalid one as `about:blank` and a
+ # `mailto:`-like one as-is, so the page URL is used whenever the base is not an absolute URL.
extracted_base_url = await context.page.evaluate('document.baseURI')
- base_url: str = extracted_base_url or context.request.loaded_url or context.request.url
+ base_url = context.request.loaded_url or context.request.url
+ if is_url_absolute(extracted_base_url):
+ base_url = extracted_base_url
links_iterator = to_absolute_url_iterator(base_url, links_iterator, logger=context.log)
diff --git a/tests/unit/crawlers/_beautifulsoup/test_beautifulsoup_crawler.py b/tests/unit/crawlers/_beautifulsoup/test_beautifulsoup_crawler.py
index e13a5a48a0..f3bac73206 100644
--- a/tests/unit/crawlers/_beautifulsoup/test_beautifulsoup_crawler.py
+++ b/tests/unit/crawlers/_beautifulsoup/test_beautifulsoup_crawler.py
@@ -289,6 +289,31 @@ async def request_handler(context: BeautifulSoupCrawlingContext) -> None:
assert extracted_links[0] == str(server_url / 'page_1')
+@pytest.mark.parametrize(
+ ('base', 'expected_path'),
+ [
+ pytest.param('{origin}/abs/', '/abs/page', id='absolute-base'),
+ pytest.param('/sub/', '/sub/page', id='relative-base'),
+ pytest.param('http://[bad', '/page', id='invalid-base'),
+ pytest.param('mailto:a@b.c', '/page', id='non-hierarchical-base'),
+ ],
+)
+async def test_extract_links_base_href(server_url: URL, http_client: HttpClient, base: str, expected_path: str) -> None:
+ """Links are resolved against ``, a relative one against the page URL, and an invalid one is ignored."""
+ crawler = BeautifulSoupCrawler(http_client=http_client)
+ handler = mock.AsyncMock()
+
+ @crawler.router.default_handler
+ async def request_handler(context: BeautifulSoupCrawlingContext) -> None:
+ links = await context.extract_links()
+ await handler([request.url for request in links])
+
+ page = f'
Page'
+ await crawler.run([str((server_url / 'echo_content').with_query(content=page))])
+
+ handler.assert_awaited_once_with([str(server_url.with_path(expected_path))])
+
+
async def test_extract_non_href_links(server_url: URL, http_client: HttpClient) -> None:
crawler = BeautifulSoupCrawler(http_client=http_client)
extracted_links: list[str] = []
diff --git a/tests/unit/crawlers/_parsel/test_parsel_crawler.py b/tests/unit/crawlers/_parsel/test_parsel_crawler.py
index 84e95a7f69..084c540760 100644
--- a/tests/unit/crawlers/_parsel/test_parsel_crawler.py
+++ b/tests/unit/crawlers/_parsel/test_parsel_crawler.py
@@ -375,6 +375,31 @@ async def request_handler(context: ParselCrawlingContext) -> None:
assert extracted_links[0] == str(server_url / 'page_1')
+@pytest.mark.parametrize(
+ ('base', 'expected_path'),
+ [
+ pytest.param('{origin}/abs/', '/abs/page', id='absolute-base'),
+ pytest.param('/sub/', '/sub/page', id='relative-base'),
+ pytest.param('http://[bad', '/page', id='invalid-base'),
+ pytest.param('mailto:a@b.c', '/page', id='non-hierarchical-base'),
+ ],
+)
+async def test_extract_links_base_href(server_url: URL, http_client: HttpClient, base: str, expected_path: str) -> None:
+ """Links are resolved against ``, a relative one against the page URL, and an invalid one is ignored."""
+ crawler = ParselCrawler(http_client=http_client)
+ handler = mock.AsyncMock()
+
+ @crawler.router.default_handler
+ async def request_handler(context: ParselCrawlingContext) -> None:
+ links = await context.extract_links()
+ await handler([request.url for request in links])
+
+ page = f'Page'
+ await crawler.run([str((server_url / 'echo_content').with_query(content=page))])
+
+ handler.assert_awaited_once_with([str(server_url.with_path(expected_path))])
+
+
async def test_extract_non_href_links(server_url: URL, http_client: HttpClient) -> None:
crawler = ParselCrawler(http_client=http_client)
extracted_links: list[str] = []
diff --git a/tests/unit/crawlers/_playwright/test_playwright_crawler.py b/tests/unit/crawlers/_playwright/test_playwright_crawler.py
index 7323811ccc..444fc3d8f7 100644
--- a/tests/unit/crawlers/_playwright/test_playwright_crawler.py
+++ b/tests/unit/crawlers/_playwright/test_playwright_crawler.py
@@ -934,6 +934,47 @@ async def request_handler(context: PlaywrightCrawlingContext) -> None:
assert extracted_links[0] == str(server_url / 'page_1')
+@pytest.mark.parametrize(
+ ('base', 'expected_path'),
+ [
+ pytest.param('{origin}/abs/', '/abs/page', id='absolute-base'),
+ pytest.param('/sub/', '/sub/page', id='relative-base'),
+ pytest.param('http://[bad', '/page', id='invalid-base'),
+ pytest.param('mailto:a@b.c', '/page', id='non-hierarchical-base'),
+ ],
+)
+async def test_extract_links_base_href(server_url: URL, base: str, expected_path: str) -> None:
+ """Links are resolved against ``, a relative one against the page URL, and an invalid one is ignored."""
+ crawler = PlaywrightCrawler()
+ handler = mock.AsyncMock()
+
+ @crawler.router.default_handler
+ async def request_handler(context: PlaywrightCrawlingContext) -> None:
+ links = await context.extract_links()
+ await handler([request.url for request in links])
+
+ page = f'Page'
+ await crawler.run([str((server_url / 'echo_content').with_query(content=page))])
+
+ handler.assert_awaited_once_with([str(server_url.with_path(expected_path))])
+
+
+async def test_extract_links_whitespace(server_url: URL) -> None:
+ """The whitespace around a link is ignored."""
+ crawler = PlaywrightCrawler()
+ handler = mock.AsyncMock()
+
+ @crawler.router.default_handler
+ async def request_handler(context: PlaywrightCrawlingContext) -> None:
+ links = await context.extract_links()
+ await handler([request.url for request in links])
+
+ page = 'Page'
+ await crawler.run([str((server_url / 'echo_content').with_query(content=page))])
+
+ handler.assert_awaited_once_with([str(server_url / 'page')])
+
+
async def test_extract_non_href_links(server_url: URL) -> None:
crawler = PlaywrightCrawler()
extracted_links: list[str] = []