diff --git a/src/crawlee/crawlers/_abstract_http/_abstract_http_crawler.py b/src/crawlee/crawlers/_abstract_http/_abstract_http_crawler.py index ed4d79b143..01d298b710 100644 --- a/src/crawlee/crawlers/_abstract_http/_abstract_http_crawler.py +++ b/src/crawlee/crawlers/_abstract_http/_abstract_http_crawler.py @@ -13,7 +13,7 @@ from crawlee._request import Request, RequestOptions, RequestState from crawlee._utils.docs import docs_group from crawlee._utils.time import SharedTimeout -from crawlee._utils.urls import to_absolute_url_iterator +from crawlee._utils.urls import convert_to_absolute_url, is_url_absolute, to_absolute_url_iterator from crawlee.crawlers._basic import BasicCrawler, BasicCrawlerOptions, ContextPipeline from crawlee.errors import RequestThrottledError, SessionError from crawlee.statistics import StatisticsState @@ -218,13 +218,19 @@ async def extract_links( self._parser.find_links(parsed_content, selector=selector, attribute=attribute) ) - # Get base URL from tag if present + # A relative `` is resolved against the page URL, and one that does not resolve to an absolute + # URL (malformed, `mailto:`, ...) is ignored. + base_url = context.request.loaded_url or context.request.url extracted_base_urls = list(self._parser.find_links(parsed_content, 'base[href]', 'href')) - base_url: str = ( - str(extracted_base_urls[0]) - if extracted_base_urls - else context.request.loaded_url or context.request.url - ) + if extracted_base_urls: + try: + resolved_base_url = convert_to_absolute_url(base_url, extracted_base_urls[0]) + except ValueError: + resolved_base_url = '' + if is_url_absolute(resolved_base_url): + base_url = resolved_base_url + else: + context.log.debug(f'Ignoring invalid base URL "{extracted_base_urls[0]}", using the page URL.') links_iterator = to_absolute_url_iterator(base_url, links_iterator, logger=context.log) if robots_txt_file: diff --git a/src/crawlee/crawlers/_playwright/_playwright_crawler.py b/src/crawlee/crawlers/_playwright/_playwright_crawler.py index bd7cdbedc4..399455d723 100644 --- a/src/crawlee/crawlers/_playwright/_playwright_crawler.py +++ b/src/crawlee/crawlers/_playwright/_playwright_crawler.py @@ -18,7 +18,7 @@ from crawlee._utils.docs import docs_group from crawlee._utils.robots import RobotsTxtFile from crawlee._utils.time import SharedTimeout -from crawlee._utils.urls import to_absolute_url_iterator +from crawlee._utils.urls import is_url_absolute, to_absolute_url_iterator from crawlee.browsers import BrowserPool from crawlee.crawlers._basic import BasicCrawler, BasicCrawlerOptions, ContextPipeline from crawlee.errors import RequestThrottledError, SessionError @@ -445,13 +445,17 @@ async def extract_links( strategy = kwargs.get('strategy', 'same-hostname') elements = await context.page.query_selector_all(selector) + # Browsers ignore the whitespace around a URL, as `find_links` of the HTTP parsers does. links_iterator: Iterator[str] = iter( - [url for element in elements if (url := await element.get_attribute(attribute)) is not None] + [url.strip() for element in elements if (url := await element.get_attribute(attribute)) is not None] ) - # Get base URL from tag if present + # The browser resolves `` itself. Chromium reports an invalid one as `about:blank` and a + # `mailto:`-like one as-is, so the page URL is used whenever the base is not an absolute URL. extracted_base_url = await context.page.evaluate('document.baseURI') - base_url: str = extracted_base_url or context.request.loaded_url or context.request.url + base_url = context.request.loaded_url or context.request.url + if is_url_absolute(extracted_base_url): + base_url = extracted_base_url links_iterator = to_absolute_url_iterator(base_url, links_iterator, logger=context.log) diff --git a/tests/unit/crawlers/_beautifulsoup/test_beautifulsoup_crawler.py b/tests/unit/crawlers/_beautifulsoup/test_beautifulsoup_crawler.py index e13a5a48a0..f3bac73206 100644 --- a/tests/unit/crawlers/_beautifulsoup/test_beautifulsoup_crawler.py +++ b/tests/unit/crawlers/_beautifulsoup/test_beautifulsoup_crawler.py @@ -289,6 +289,31 @@ async def request_handler(context: BeautifulSoupCrawlingContext) -> None: assert extracted_links[0] == str(server_url / 'page_1') +@pytest.mark.parametrize( + ('base', 'expected_path'), + [ + pytest.param('{origin}/abs/', '/abs/page', id='absolute-base'), + pytest.param('/sub/', '/sub/page', id='relative-base'), + pytest.param('http://[bad', '/page', id='invalid-base'), + pytest.param('mailto:a@b.c', '/page', id='non-hierarchical-base'), + ], +) +async def test_extract_links_base_href(server_url: URL, http_client: HttpClient, base: str, expected_path: str) -> None: + """Links are resolved against ``, a relative one against the page URL, and an invalid one is ignored.""" + crawler = BeautifulSoupCrawler(http_client=http_client) + handler = mock.AsyncMock() + + @crawler.router.default_handler + async def request_handler(context: BeautifulSoupCrawlingContext) -> None: + links = await context.extract_links() + await handler([request.url for request in links]) + + page = f'Page' + await crawler.run([str((server_url / 'echo_content').with_query(content=page))]) + + handler.assert_awaited_once_with([str(server_url.with_path(expected_path))]) + + async def test_extract_non_href_links(server_url: URL, http_client: HttpClient) -> None: crawler = BeautifulSoupCrawler(http_client=http_client) extracted_links: list[str] = [] diff --git a/tests/unit/crawlers/_parsel/test_parsel_crawler.py b/tests/unit/crawlers/_parsel/test_parsel_crawler.py index 84e95a7f69..084c540760 100644 --- a/tests/unit/crawlers/_parsel/test_parsel_crawler.py +++ b/tests/unit/crawlers/_parsel/test_parsel_crawler.py @@ -375,6 +375,31 @@ async def request_handler(context: ParselCrawlingContext) -> None: assert extracted_links[0] == str(server_url / 'page_1') +@pytest.mark.parametrize( + ('base', 'expected_path'), + [ + pytest.param('{origin}/abs/', '/abs/page', id='absolute-base'), + pytest.param('/sub/', '/sub/page', id='relative-base'), + pytest.param('http://[bad', '/page', id='invalid-base'), + pytest.param('mailto:a@b.c', '/page', id='non-hierarchical-base'), + ], +) +async def test_extract_links_base_href(server_url: URL, http_client: HttpClient, base: str, expected_path: str) -> None: + """Links are resolved against ``, a relative one against the page URL, and an invalid one is ignored.""" + crawler = ParselCrawler(http_client=http_client) + handler = mock.AsyncMock() + + @crawler.router.default_handler + async def request_handler(context: ParselCrawlingContext) -> None: + links = await context.extract_links() + await handler([request.url for request in links]) + + page = f'Page' + await crawler.run([str((server_url / 'echo_content').with_query(content=page))]) + + handler.assert_awaited_once_with([str(server_url.with_path(expected_path))]) + + async def test_extract_non_href_links(server_url: URL, http_client: HttpClient) -> None: crawler = ParselCrawler(http_client=http_client) extracted_links: list[str] = [] diff --git a/tests/unit/crawlers/_playwright/test_playwright_crawler.py b/tests/unit/crawlers/_playwright/test_playwright_crawler.py index 7323811ccc..444fc3d8f7 100644 --- a/tests/unit/crawlers/_playwright/test_playwright_crawler.py +++ b/tests/unit/crawlers/_playwright/test_playwright_crawler.py @@ -934,6 +934,47 @@ async def request_handler(context: PlaywrightCrawlingContext) -> None: assert extracted_links[0] == str(server_url / 'page_1') +@pytest.mark.parametrize( + ('base', 'expected_path'), + [ + pytest.param('{origin}/abs/', '/abs/page', id='absolute-base'), + pytest.param('/sub/', '/sub/page', id='relative-base'), + pytest.param('http://[bad', '/page', id='invalid-base'), + pytest.param('mailto:a@b.c', '/page', id='non-hierarchical-base'), + ], +) +async def test_extract_links_base_href(server_url: URL, base: str, expected_path: str) -> None: + """Links are resolved against ``, a relative one against the page URL, and an invalid one is ignored.""" + crawler = PlaywrightCrawler() + handler = mock.AsyncMock() + + @crawler.router.default_handler + async def request_handler(context: PlaywrightCrawlingContext) -> None: + links = await context.extract_links() + await handler([request.url for request in links]) + + page = f'Page' + await crawler.run([str((server_url / 'echo_content').with_query(content=page))]) + + handler.assert_awaited_once_with([str(server_url.with_path(expected_path))]) + + +async def test_extract_links_whitespace(server_url: URL) -> None: + """The whitespace around a link is ignored.""" + crawler = PlaywrightCrawler() + handler = mock.AsyncMock() + + @crawler.router.default_handler + async def request_handler(context: PlaywrightCrawlingContext) -> None: + links = await context.extract_links() + await handler([request.url for request in links]) + + page = 'Page' + await crawler.run([str((server_url / 'echo_content').with_query(content=page))]) + + handler.assert_awaited_once_with([str(server_url / 'page')]) + + async def test_extract_non_href_links(server_url: URL) -> None: crawler = PlaywrightCrawler() extracted_links: list[str] = []