From 1bf24e73b65d98641308f15fdd668b7d9db04f47 Mon Sep 17 00:00:00 2001 From: sean0407 <22557931+sean0407@users.noreply.github.com> Date: Sun, 11 Oct 2026 06:34:42 +0800 Subject: [PATCH 1/5] fix: preserve query strings containing slashes --- .../filtering/basic/BasicURLNormalizer.java | 7 +++++++ .../filtering/BasicURLNormalizerTest.java | 18 ++++++++++++++++++ 2 files changed, 25 insertions(+) diff --git a/core/src/main/java/org/apache/stormcrawler/filtering/basic/BasicURLNormalizer.java b/core/src/main/java/org/apache/stormcrawler/filtering/basic/BasicURLNormalizer.java index af4eb5a2a..39d9aac3e 100644 --- a/core/src/main/java/org/apache/stormcrawler/filtering/basic/BasicURLNormalizer.java +++ b/core/src/main/java/org/apache/stormcrawler/filtering/basic/BasicURLNormalizer.java @@ -343,6 +343,13 @@ public int compare(NameValuePair p1, NameValuePair p2) { * @return corrected url */ private String unmangleQueryString(String urlToFilter) { + int fragmentStart = urlToFilter.indexOf('#'); + String urlBeforeFragment = + fragmentStart == -1 ? urlToFilter : urlToFilter.substring(0, fragmentStart); + if (urlBeforeFragment.indexOf('?') != -1) { + return urlToFilter; + } + String[] pathElements = urlToFilter.split("/"); final String lastPathElement = pathElements[pathElements.length - 1]; int firstAmp = lastPathElement.indexOf('&'); diff --git a/core/src/test/java/org/apache/stormcrawler/filtering/BasicURLNormalizerTest.java b/core/src/test/java/org/apache/stormcrawler/filtering/BasicURLNormalizerTest.java index 1791bbcc3..9936936bd 100644 --- a/core/src/test/java/org/apache/stormcrawler/filtering/BasicURLNormalizerTest.java +++ b/core/src/test/java/org/apache/stormcrawler/filtering/BasicURLNormalizerTest.java @@ -243,6 +243,24 @@ void testFixMangledQueryString() throws MalformedURLException { assertEquals(expectedResult, normalizedUrl, "Failed to filter query string"); } + @Test + void testDoesNotMangleQueryWhenValueContainsSlashes() throws MalformedURLException { + URLFilter urlFilter = createFilter(new ObjectNode(JsonNodeFactory.instance)); + URL testSourceUrl = URLUtil.toURL("http://a.example/"); + String inputUrl = + "http://a.example/article.pl?sid=01/10/23/1816257&mode=thread&tid=107"; + assertEquals( + inputUrl, + urlFilter.filter(testSourceUrl, new Metadata(), inputUrl), + "An existing query must not be unmangled because a value contains slashes"); + + inputUrl = "http://a.example/p?u=http://b.example/x&z=1"; + assertEquals( + inputUrl, + urlFilter.filter(testSourceUrl, new Metadata(), inputUrl), + "An embedded URL in a query value must remain intact"); + } + @Test void testProperURLEncodingWithoutQueryParameter() throws MalformedURLException { URLFilter urlFilter = createFilter(queryParamsToFilter); From a712459b7832fe571fcf7aea579803c00036666d Mon Sep 17 00:00:00 2001 From: Sean Li <22557931+sean0407@users.noreply.github.com> Date: Wed, 21 Oct 2026 10:28:29 +0800 Subject: [PATCH 2/5] fix: preserve query fragment during repair --- .../filtering/basic/BasicURLNormalizer.java | 18 ++++++------- .../filtering/BasicURLNormalizerTest.java | 25 +++++++++++++++++-- 2 files changed, 32 insertions(+), 11 deletions(-) diff --git a/core/src/main/java/org/apache/stormcrawler/filtering/basic/BasicURLNormalizer.java b/core/src/main/java/org/apache/stormcrawler/filtering/basic/BasicURLNormalizer.java index 39d9aac3e..0035f8745 100644 --- a/core/src/main/java/org/apache/stormcrawler/filtering/basic/BasicURLNormalizer.java +++ b/core/src/main/java/org/apache/stormcrawler/filtering/basic/BasicURLNormalizer.java @@ -336,30 +336,30 @@ public int compare(NameValuePair p1, NameValuePair p2) { }; /** - * A common error to find is a query string that starts with an & instead of a ? This will fix - * that error. So http://foo.com&a=b will be changed to http://foo.com?a=b. + * Repairs a query string that starts with an & instead of a ?. The query check ignores the + * fragment, and the repair only checks the last path segment so an & in an earlier path segment + * is not mistaken for a query separator. * * @param urlToFilter the url to filter * @return corrected url */ private String unmangleQueryString(String urlToFilter) { int fragmentStart = urlToFilter.indexOf('#'); - String urlBeforeFragment = - fragmentStart == -1 ? urlToFilter : urlToFilter.substring(0, fragmentStart); - if (urlBeforeFragment.indexOf('?') != -1) { + String fragment = fragmentStart == -1 ? "" : urlToFilter.substring(fragmentStart); + String url = fragmentStart == -1 ? urlToFilter : urlToFilter.substring(0, fragmentStart); + if (url.indexOf('?') != -1) { return urlToFilter; } - String[] pathElements = urlToFilter.split("/"); + String[] pathElements = url.split("/"); final String lastPathElement = pathElements[pathElements.length - 1]; int firstAmp = lastPathElement.indexOf('&'); if (firstAmp == -1) { return urlToFilter; } - int firstQuestionMark = lastPathElement.indexOf('?'); - if (firstQuestionMark == -1 && lastPathElement.indexOf("=") > 0) { + if (lastPathElement.indexOf("=") > 0) { pathElements[pathElements.length - 1] = lastPathElement.replaceFirst("&", "?"); - return String.join("/", pathElements); + return String.join("/", pathElements) + fragment; } return urlToFilter; } diff --git a/core/src/test/java/org/apache/stormcrawler/filtering/BasicURLNormalizerTest.java b/core/src/test/java/org/apache/stormcrawler/filtering/BasicURLNormalizerTest.java index 9936936bd..30449d5fc 100644 --- a/core/src/test/java/org/apache/stormcrawler/filtering/BasicURLNormalizerTest.java +++ b/core/src/test/java/org/apache/stormcrawler/filtering/BasicURLNormalizerTest.java @@ -247,8 +247,7 @@ void testFixMangledQueryString() throws MalformedURLException { void testDoesNotMangleQueryWhenValueContainsSlashes() throws MalformedURLException { URLFilter urlFilter = createFilter(new ObjectNode(JsonNodeFactory.instance)); URL testSourceUrl = URLUtil.toURL("http://a.example/"); - String inputUrl = - "http://a.example/article.pl?sid=01/10/23/1816257&mode=thread&tid=107"; + String inputUrl = "http://a.example/article.pl?sid=01/10/23/1816257&mode=thread&tid=107"; assertEquals( inputUrl, urlFilter.filter(testSourceUrl, new Metadata(), inputUrl), @@ -261,6 +260,28 @@ void testDoesNotMangleQueryWhenValueContainsSlashes() throws MalformedURLExcepti "An embedded URL in a query value must remain intact"); } + @Test + void testUnmanglesQueryBeforeFragmentQuestionMark() throws MalformedURLException { + URLFilter urlFilter = createFilter(false, false); + URL testSourceUrl = URLUtil.toURL("http://a.example/"); + String inputUrl = "http://a.example/p&a=b#frag?x"; + assertEquals( + "http://a.example/p?a=b#frag?x", + urlFilter.filter(testSourceUrl, new Metadata(), inputUrl), + "A question mark in the fragment must not suppress query repair"); + } + + @Test + void testRemovesQueryElementWhenValueContainsSlashes() throws MalformedURLException { + URLFilter urlFilter = createFilter(List.of("utm_source")); + URL testSourceUrl = URLUtil.toURL("http://a.example/"); + String inputUrl = "http://a.example/login/?next=/a/b/&utm_source=ref&z=1"; + assertEquals( + "http://a.example/login/?next=%2Fa%2Fb%2F&z=1", + urlFilter.filter(testSourceUrl, new Metadata(), inputUrl), + "Query elements following a slash-containing value must remain removable"); + } + @Test void testProperURLEncodingWithoutQueryParameter() throws MalformedURLException { URLFilter urlFilter = createFilter(queryParamsToFilter); From c27a9d56b704204885ef0ee51a15160e822dc597 Mon Sep 17 00:00:00 2001 From: Sean Li <22557931+sean0407@users.noreply.github.com> Date: Sat, 24 Oct 2026 19:24:02 +0800 Subject: [PATCH 3/5] fix: preserve trailing slash during query repair --- .../filtering/basic/BasicURLNormalizer.java | 2 +- .../filtering/BasicURLNormalizerTest.java | 22 +++++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/org/apache/stormcrawler/filtering/basic/BasicURLNormalizer.java b/core/src/main/java/org/apache/stormcrawler/filtering/basic/BasicURLNormalizer.java index 0035f8745..05e5508d8 100644 --- a/core/src/main/java/org/apache/stormcrawler/filtering/basic/BasicURLNormalizer.java +++ b/core/src/main/java/org/apache/stormcrawler/filtering/basic/BasicURLNormalizer.java @@ -351,7 +351,7 @@ private String unmangleQueryString(String urlToFilter) { return urlToFilter; } - String[] pathElements = url.split("/"); + String[] pathElements = url.split("/", -1); final String lastPathElement = pathElements[pathElements.length - 1]; int firstAmp = lastPathElement.indexOf('&'); if (firstAmp == -1) { diff --git a/core/src/test/java/org/apache/stormcrawler/filtering/BasicURLNormalizerTest.java b/core/src/test/java/org/apache/stormcrawler/filtering/BasicURLNormalizerTest.java index 30449d5fc..423896e16 100644 --- a/core/src/test/java/org/apache/stormcrawler/filtering/BasicURLNormalizerTest.java +++ b/core/src/test/java/org/apache/stormcrawler/filtering/BasicURLNormalizerTest.java @@ -271,6 +271,28 @@ void testUnmanglesQueryBeforeFragmentQuestionMark() throws MalformedURLException "A question mark in the fragment must not suppress query repair"); } + @Test + void testDoesNotDropTrailingSlashWhenUnmanglingWithFragment() throws MalformedURLException { + URLFilter urlFilter = createFilter(false, false); + URL testSourceUrl = URLUtil.toURL("http://a.example/"); + String inputUrl = "http://a.example/x&a=b/#frag"; + assertEquals( + inputUrl, + urlFilter.filter(testSourceUrl, new Metadata(), inputUrl), + "A trailing slash before a fragment must not be mistaken for a query"); + } + + @Test + void testDoesNotDropTrailingSlashWhenUnmanglingByDefault() throws MalformedURLException { + URLFilter urlFilter = createFilter(new ObjectNode(JsonNodeFactory.instance)); + URL testSourceUrl = URLUtil.toURL("http://a.example/"); + String inputUrl = "http://a.example/x&a=b/"; + assertEquals( + inputUrl, + urlFilter.filter(testSourceUrl, new Metadata(), inputUrl), + "A trailing slash must not be removed during default normalization"); + } + @Test void testRemovesQueryElementWhenValueContainsSlashes() throws MalformedURLException { URLFilter urlFilter = createFilter(List.of("utm_source")); From b956b447e4f9e30dee9aa857516a5229b9645a5a Mon Sep 17 00:00:00 2001 From: Sean Date: Sun, 25 Oct 2026 07:23:47 +0800 Subject: [PATCH 4/5] Improve unmangled query repair checks --- .../filtering/basic/BasicURLNormalizer.java | 14 +++-- .../filtering/BasicURLNormalizerTest.java | 56 ++++++++++++------- 2 files changed, 43 insertions(+), 27 deletions(-) diff --git a/core/src/main/java/org/apache/stormcrawler/filtering/basic/BasicURLNormalizer.java b/core/src/main/java/org/apache/stormcrawler/filtering/basic/BasicURLNormalizer.java index 05e5508d8..2105e5234 100644 --- a/core/src/main/java/org/apache/stormcrawler/filtering/basic/BasicURLNormalizer.java +++ b/core/src/main/java/org/apache/stormcrawler/filtering/basic/BasicURLNormalizer.java @@ -344,6 +344,10 @@ public int compare(NameValuePair p1, NameValuePair p2) { * @return corrected url */ private String unmangleQueryString(String urlToFilter) { + if (urlToFilter.indexOf('&') == -1) { + return urlToFilter; + } + int fragmentStart = urlToFilter.indexOf('#'); String fragment = fragmentStart == -1 ? "" : urlToFilter.substring(fragmentStart); String url = fragmentStart == -1 ? urlToFilter : urlToFilter.substring(0, fragmentStart); @@ -351,15 +355,13 @@ private String unmangleQueryString(String urlToFilter) { return urlToFilter; } - String[] pathElements = url.split("/", -1); - final String lastPathElement = pathElements[pathElements.length - 1]; - int firstAmp = lastPathElement.indexOf('&'); + int lastSlash = url.lastIndexOf('/'); + int firstAmp = url.indexOf('&', lastSlash + 1); if (firstAmp == -1) { return urlToFilter; } - if (lastPathElement.indexOf("=") > 0) { - pathElements[pathElements.length - 1] = lastPathElement.replaceFirst("&", "?"); - return String.join("/", pathElements) + fragment; + if (url.indexOf('=', firstAmp + 1) > firstAmp) { + return url.substring(0, firstAmp) + '?' + url.substring(firstAmp + 1) + fragment; } return urlToFilter; } diff --git a/core/src/test/java/org/apache/stormcrawler/filtering/BasicURLNormalizerTest.java b/core/src/test/java/org/apache/stormcrawler/filtering/BasicURLNormalizerTest.java index 423896e16..95b05a60f 100644 --- a/core/src/test/java/org/apache/stormcrawler/filtering/BasicURLNormalizerTest.java +++ b/core/src/test/java/org/apache/stormcrawler/filtering/BasicURLNormalizerTest.java @@ -35,6 +35,8 @@ import org.apache.stormcrawler.util.URLUtil; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; /** * Utility class which encapsulates the filtering of URLs based on the hostname or domain of the @@ -243,21 +245,20 @@ void testFixMangledQueryString() throws MalformedURLException { assertEquals(expectedResult, normalizedUrl, "Failed to filter query string"); } - @Test - void testDoesNotMangleQueryWhenValueContainsSlashes() throws MalformedURLException { + @ParameterizedTest + @CsvSource({ + "http://a.example/article.pl?sid=01/10/23/1816257&mode=thread&tid=107", + "http://a.example/p?u=http://b.example/x&z=1", + "http://a.example/login/?next=/a/b/&utm_source=ref&z=1" + }) + void testDoesNotMangleQueryWhenValueContainsSlashes(String inputUrl) + throws MalformedURLException { URLFilter urlFilter = createFilter(new ObjectNode(JsonNodeFactory.instance)); URL testSourceUrl = URLUtil.toURL("http://a.example/"); - String inputUrl = "http://a.example/article.pl?sid=01/10/23/1816257&mode=thread&tid=107"; assertEquals( inputUrl, urlFilter.filter(testSourceUrl, new Metadata(), inputUrl), "An existing query must not be unmangled because a value contains slashes"); - - inputUrl = "http://a.example/p?u=http://b.example/x&z=1"; - assertEquals( - inputUrl, - urlFilter.filter(testSourceUrl, new Metadata(), inputUrl), - "An embedded URL in a query value must remain intact"); } @Test @@ -271,26 +272,39 @@ void testUnmanglesQueryBeforeFragmentQuestionMark() throws MalformedURLException "A question mark in the fragment must not suppress query repair"); } - @Test - void testDoesNotDropTrailingSlashWhenUnmanglingWithFragment() throws MalformedURLException { - URLFilter urlFilter = createFilter(false, false); + @ParameterizedTest + @CsvSource({ + "false, http://a.example/x&a=b/#frag", + "true, http://a.example/x&a=b/" + }) + void testDoesNotDropTrailingSlashWhenUnmangling(boolean removeAnchor, String inputUrl) + throws MalformedURLException { + URLFilter urlFilter = createFilter(removeAnchor, true, List.of()); URL testSourceUrl = URLUtil.toURL("http://a.example/"); - String inputUrl = "http://a.example/x&a=b/#frag"; assertEquals( inputUrl, urlFilter.filter(testSourceUrl, new Metadata(), inputUrl), - "A trailing slash before a fragment must not be mistaken for a query"); + "The URL must remain unchanged when the last path segment has no query parameters"); + } + + @ParameterizedTest + @CsvSource({ + "http://a.example/p&a=b, http://a.example/p?a=b", + "http://a.example/p&a=b&c=d, http://a.example/p?a=b&c=d" + }) + void testRepairsMangledQueryString(String inputUrl, String expectedUrl) + throws MalformedURLException { + URLFilter urlFilter = createFilter(false, true, List.of()); + URL testSourceUrl = URLUtil.toURL("http://a.example/"); + assertEquals(expectedUrl, urlFilter.filter(testSourceUrl, new Metadata(), inputUrl)); } @Test - void testDoesNotDropTrailingSlashWhenUnmanglingByDefault() throws MalformedURLException { - URLFilter urlFilter = createFilter(new ObjectNode(JsonNodeFactory.instance)); + void testDoesNotRewriteAmpersandInFragment() throws MalformedURLException { + URLFilter urlFilter = createFilter(false, true, List.of()); URL testSourceUrl = URLUtil.toURL("http://a.example/"); - String inputUrl = "http://a.example/x&a=b/"; - assertEquals( - inputUrl, - urlFilter.filter(testSourceUrl, new Metadata(), inputUrl), - "A trailing slash must not be removed during default normalization"); + String inputUrl = "http://a.example/p#a&b=c"; + assertEquals(inputUrl, urlFilter.filter(testSourceUrl, new Metadata(), inputUrl)); } @Test From d7d226b2681a8307eeac013a65e1f728bd18de42 Mon Sep 17 00:00:00 2001 From: Richard Zowalla Date: Sun, 25 Oct 2026 19:43:29 +0100 Subject: [PATCH 5/5] Format BasicURLNormalizerTest --- .../stormcrawler/filtering/BasicURLNormalizerTest.java | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/core/src/test/java/org/apache/stormcrawler/filtering/BasicURLNormalizerTest.java b/core/src/test/java/org/apache/stormcrawler/filtering/BasicURLNormalizerTest.java index 95b05a60f..712521a85 100644 --- a/core/src/test/java/org/apache/stormcrawler/filtering/BasicURLNormalizerTest.java +++ b/core/src/test/java/org/apache/stormcrawler/filtering/BasicURLNormalizerTest.java @@ -273,10 +273,7 @@ void testUnmanglesQueryBeforeFragmentQuestionMark() throws MalformedURLException } @ParameterizedTest - @CsvSource({ - "false, http://a.example/x&a=b/#frag", - "true, http://a.example/x&a=b/" - }) + @CsvSource({"false, http://a.example/x&a=b/#frag", "true, http://a.example/x&a=b/"}) void testDoesNotDropTrailingSlashWhenUnmangling(boolean removeAnchor, String inputUrl) throws MalformedURLException { URLFilter urlFilter = createFilter(removeAnchor, true, List.of());