From e3e5bcff2be0a23d1075f84a95f4148f74ed09a3 Mon Sep 17 00:00:00 2001 From: Nico Scandolo Date: Tue, 20 Oct 2026 14:57:13 -0300 Subject: [PATCH 1/4] #2250 CharsetIdentification: read the content type under the protocol prefix --- .../stormcrawler/bolt/JSoupParserBolt.java | 10 ++++- .../util/CharsetIdentification.java | 33 ++++++++++++++-- .../bolt/JSoupParserBoltTest.java | 38 ++++++++++++++++++- 3 files changed, 74 insertions(+), 7 deletions(-) diff --git a/core/src/main/java/org/apache/stormcrawler/bolt/JSoupParserBolt.java b/core/src/main/java/org/apache/stormcrawler/bolt/JSoupParserBolt.java index dc8c32c92..c128082a6 100644 --- a/core/src/main/java/org/apache/stormcrawler/bolt/JSoupParserBolt.java +++ b/core/src/main/java/org/apache/stormcrawler/bolt/JSoupParserBolt.java @@ -286,11 +286,17 @@ public void execute(Tuple tuple) { if (fastCharsetDetection) { charset = CharsetIdentification.getCharsetFast( - metadata, content, maxLengthCharsetDetection); + metadata, + content, + maxLengthCharsetDetection, + protocolMetadataPrefix); } else { charset = CharsetIdentification.getCharset( - metadata, content, maxLengthCharsetDetection); + metadata, + content, + maxLengthCharsetDetection, + protocolMetadataPrefix); } LOG.debug( diff --git a/core/src/main/java/org/apache/stormcrawler/util/CharsetIdentification.java b/core/src/main/java/org/apache/stormcrawler/util/CharsetIdentification.java index 9fc778dd3..846971164 100644 --- a/core/src/main/java/org/apache/stormcrawler/util/CharsetIdentification.java +++ b/core/src/main/java/org/apache/stormcrawler/util/CharsetIdentification.java @@ -61,6 +61,18 @@ public class CharsetIdentification { */ public static String getCharsetFast( final Metadata metadata, final byte[] content, final int maxLengthCharsetDetection) { + return getCharsetFast(metadata, content, maxLengthCharsetDetection, ""); + } + + /** + * Same as {@link #getCharsetFast(Metadata, byte[], int)}, reading the HTTP Content-Type under + * the prefix the fetcher stores the response headers with (protocol.md.prefix). + */ + public static String getCharsetFast( + final Metadata metadata, + final byte[] content, + final int maxLengthCharsetDetection, + final String protocolMetadataPrefix) { // let's look at the BOM first String charset = getCharsetFromBOM(content); @@ -69,7 +81,7 @@ public static String getCharsetFast( } // then look at what we get from HTTP headers - charset = getCharsetFromHTTP(metadata); + charset = getCharsetFromHTTP(metadata, protocolMetadataPrefix); if (charset != null) { return charset; } @@ -97,6 +109,18 @@ public static String getCharsetFast( */ public static String getCharset( Metadata metadata, byte[] content, int maxLengthCharsetDetection) { + return getCharset(metadata, content, maxLengthCharsetDetection, ""); + } + + /** + * Same as {@link #getCharset(Metadata, byte[], int)}, reading the HTTP Content-Type under the + * prefix the fetcher stores the response headers with (protocol.md.prefix). + */ + public static String getCharset( + Metadata metadata, + byte[] content, + int maxLengthCharsetDetection, + String protocolMetadataPrefix) { // let's look at the BOM first String BOMCharset = getCharsetFromBOM(content); @@ -105,7 +129,7 @@ public static String getCharset( } // then look at what we get from HTTP headers and HTML content - String httpCharset = getCharsetFromHTTP(metadata); + String httpCharset = getCharsetFromHTTP(metadata, protocolMetadataPrefix); String htmlCharset = getCharsetFromMeta(content, maxLengthCharsetDetection); // both exist and agree @@ -133,8 +157,9 @@ public static String getCharset( } /** Returns the charset declared by the server if any. */ - private static String getCharsetFromHTTP(Metadata metadata) { - return getCharsetFromContentType(metadata.getFirstValue(HttpHeaders.CONTENT_TYPE)); + private static String getCharsetFromHTTP(Metadata metadata, String protocolMetadataPrefix) { + return getCharsetFromContentType( + metadata.getFirstValue(HttpHeaders.CONTENT_TYPE, protocolMetadataPrefix)); } /** Detects any BOMs and returns the corresponding charset. */ diff --git a/core/src/test/java/org/apache/stormcrawler/bolt/JSoupParserBoltTest.java b/core/src/test/java/org/apache/stormcrawler/bolt/JSoupParserBoltTest.java index ba28bc98c..1b41d12ad 100644 --- a/core/src/test/java/org/apache/stormcrawler/bolt/JSoupParserBoltTest.java +++ b/core/src/test/java/org/apache/stormcrawler/bolt/JSoupParserBoltTest.java @@ -22,6 +22,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import org.apache.http.HttpHeaders; import org.apache.storm.task.OutputCollector; import org.apache.stormcrawler.Constants; import org.apache.stormcrawler.Metadata; @@ -129,7 +130,10 @@ void charsetDetectionFailureIsReportedAsParseError() throws IOException { .when( () -> CharsetIdentification.getCharset( - Mockito.any(), Mockito.any(), Mockito.anyInt())) + Mockito.any(), + Mockito.any(), + Mockito.anyInt(), + Mockito.any())) .thenThrow(new StackOverflowError()); parse("https://stormcrawler.apache.org", "stormcrawler.apache.org.html"); } @@ -365,4 +369,36 @@ void testBaseHrefOutlinkResolution() throws IOException { "https://www.example.be/fr/rayons/stockage-archivage-de-donnees/218725", statusTuples.get(0).get(0)); } + + /** + * The fetcher stores the response headers under protocol.md.prefix, so the charset declared in + * the Content-Type has to be read with that prefix. Without it the header is never seen, the + * header and the meta charset can never agree, and ICU guesses a UTF-8 page whose detection + * window is plain ASCII as ISO-8859-1, see issue #2250. + */ + @Test + void prefixedContentTypeCharsetIsUsed() throws IOException { + Map parserConfig = new HashMap<>(); + parserConfig.put("protocol.md.prefix", "protocol."); + parserConfig.put("detect.charset.maxlength", 2048); + bolt.prepare( + parserConfig, TestUtil.getMockedTopologyContext(), new OutputCollector(output)); + // a meta charset followed by an ASCII head longer than the detection window, like most real + // pages + String head = + ""; + String text = "What\u2019s new: caf\u00e9 imotine\u2122"; + byte[] content = + ("" + head + "

" + text + "

") + .getBytes(StandardCharsets.UTF_8); + Metadata metadata = new Metadata(); + metadata.setValue("protocol." + HttpHeaders.CONTENT_TYPE, "text/html; charset=UTF-8"); + parse("http://www.example.com/page.html", content, metadata); + + Assertions.assertEquals(1, output.getEmitted().size()); + String parsedText = (String) output.getEmitted().get(0).get(3); + Assertions.assertTrue(parsedText.contains(text), parsedText); + } } From 6e8deda8b007e8be41a8c7ef8db1d6d8db4354b8 Mon Sep 17 00:00:00 2001 From: Nico Scandolo Date: Wed, 21 Oct 2026 15:30:29 -0300 Subject: [PATCH 2/4] #2250 Review: deprecate the unprefixed overloads, fix JsRenderingDetector, meta charset as the hint, more tests --- .../util/CharsetIdentification.java | 29 ++++++++---- .../bolt/JSoupParserBoltTest.java | 17 ++++--- .../util/CharsetIdentificationTest.java | 44 +++++++++++++++++++ .../parsefilter/JsRenderingDetector.java | 8 +++- .../parsefilter/JsRenderingDetectorTest.java | 22 ++++++++++ 5 files changed, 105 insertions(+), 15 deletions(-) diff --git a/core/src/main/java/org/apache/stormcrawler/util/CharsetIdentification.java b/core/src/main/java/org/apache/stormcrawler/util/CharsetIdentification.java index 846971164..4f34deceb 100644 --- a/core/src/main/java/org/apache/stormcrawler/util/CharsetIdentification.java +++ b/core/src/main/java/org/apache/stormcrawler/util/CharsetIdentification.java @@ -58,7 +58,10 @@ public class CharsetIdentification { * less reliable than getCharset. * * @since 1.18 + * @deprecated reads the HTTP Content-Type without a prefix, while fetched metadata carries the + * one set by protocol.md.prefix: use {@link #getCharsetFast(Metadata, byte[], int, String)} */ + @Deprecated public static String getCharsetFast( final Metadata metadata, final byte[] content, final int maxLengthCharsetDetection) { return getCharsetFast(metadata, content, maxLengthCharsetDetection, ""); @@ -67,6 +70,10 @@ public static String getCharsetFast( /** * Same as {@link #getCharsetFast(Metadata, byte[], int)}, reading the HTTP Content-Type under * the prefix the fetcher stores the response headers with (protocol.md.prefix). + * + * @param protocolMetadataPrefix prefix of the response headers in the metadata, including its + * separator (e.g. {@code "protocol."}); empty if they are not prefixed + * @since 4.0 */ public static String getCharsetFast( final Metadata metadata, @@ -104,9 +111,13 @@ public static String getCharsetFast( /** * Identifies the charset of a document based on the following logic: guess from the * ByteOrderMark - else if the same charset is specified in the http headers and the html - * metadata then use it - otherwise use ICU's charset detector to make an educated guess and if - * that fails too returns UTF-8. + * metadata then use it - otherwise use ICU's charset detector to make an educated guess, with + * the html metadata or else the http headers as a hint, and if that fails too returns UTF-8. + * + * @deprecated reads the HTTP Content-Type without a prefix, while fetched metadata carries the + * one set by protocol.md.prefix: use {@link #getCharset(Metadata, byte[], int, String)} */ + @Deprecated public static String getCharset( Metadata metadata, byte[] content, int maxLengthCharsetDetection) { return getCharset(metadata, content, maxLengthCharsetDetection, ""); @@ -115,6 +126,10 @@ public static String getCharset( /** * Same as {@link #getCharset(Metadata, byte[], int)}, reading the HTTP Content-Type under the * prefix the fetcher stores the response headers with (protocol.md.prefix). + * + * @param protocolMetadataPrefix prefix of the response headers in the metadata, including its + * separator (e.g. {@code "protocol."}); empty if they are not prefixed + * @since 4.0 */ public static String getCharset( Metadata metadata, @@ -139,13 +154,9 @@ public static String getCharset( return httpCharset; } - // let's guess from the text - using a hint or not - String hintCharset = null; - if (httpCharset != null && htmlCharset == null) { - hintCharset = httpCharset; - } else if (httpCharset == null && htmlCharset != null) { - hintCharset = htmlCharset; - } + // let's guess from the text, with a hint if any: when the header and the meta disagree + // the meta is the hint, as it was while the header could not be read (#2250) + String hintCharset = htmlCharset != null ? htmlCharset : httpCharset; String textCharset = getCharsetFromText(content, hintCharset, maxLengthCharsetDetection); if (textCharset != null) { diff --git a/core/src/test/java/org/apache/stormcrawler/bolt/JSoupParserBoltTest.java b/core/src/test/java/org/apache/stormcrawler/bolt/JSoupParserBoltTest.java index 1b41d12ad..c63c3cd17 100644 --- a/core/src/test/java/org/apache/stormcrawler/bolt/JSoupParserBoltTest.java +++ b/core/src/test/java/org/apache/stormcrawler/bolt/JSoupParserBoltTest.java @@ -34,6 +34,8 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; import org.mockito.MockedStatic; import org.mockito.Mockito; @@ -376,17 +378,22 @@ void testBaseHrefOutlinkResolution() throws IOException { * header and the meta charset can never agree, and ICU guesses a UTF-8 page whose detection * window is plain ASCII as ISO-8859-1, see issue #2250. */ - @Test - void prefixedContentTypeCharsetIsUsed() throws IOException { + @ParameterizedTest(name = "detect.charset.fast={0}") + @ValueSource(booleans = {false, true}) + void prefixedContentTypeCharsetIsUsed(boolean fast) throws IOException { Map parserConfig = new HashMap<>(); parserConfig.put("protocol.md.prefix", "protocol."); parserConfig.put("detect.charset.maxlength", 2048); + parserConfig.put("detect.charset.fast", fast); bolt.prepare( parserConfig, TestUtil.getMockedTopologyContext(), new OutputCollector(output)); - // a meta charset followed by an ASCII head longer than the detection window, like most real - // pages + // an ASCII head longer than the detection window, like most real pages. The default + // detection uses the header when it agrees with the meta charset; the fast one returns the + // header first, so it is tested without a meta charset, which it would use otherwise + String meta = fast ? "" : ""; String head = - ""; String text = "What\u2019s new: caf\u00e9 imotine\u2122"; diff --git a/core/src/test/java/org/apache/stormcrawler/util/CharsetIdentificationTest.java b/core/src/test/java/org/apache/stormcrawler/util/CharsetIdentificationTest.java index 0a22c7a5d..547f81bc2 100644 --- a/core/src/test/java/org/apache/stormcrawler/util/CharsetIdentificationTest.java +++ b/core/src/test/java/org/apache/stormcrawler/util/CharsetIdentificationTest.java @@ -20,6 +20,7 @@ import java.nio.charset.StandardCharsets; import java.util.Arrays; import java.util.concurrent.atomic.AtomicReference; +import org.apache.http.HttpHeaders; import org.apache.stormcrawler.Metadata; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -142,4 +143,47 @@ void lookAheadForTheClosingQuoteIsBounded() { Assertions.assertNotEquals( DECLARED, beyond, "a declaration closing beyond the look-ahead is not used"); } + + /** The prefix the fetcher stores the response headers with, see protocol.md.prefix. */ + private static final String PREFIX = "protocol."; + + /** The detection window of these tests: the page is plain ASCII within it. */ + private static final int WINDOW = 2048; + + /** + * A UTF-8 page, optionally with a meta charset, whose first non-ASCII byte is past the window. + */ + private static byte[] utf8PageWithAsciiWindow(boolean withMeta) { + String meta = withMeta ? "" : ""; + String head = + ""; + return ("" + meta + head + "

caf\u00e9

") + .getBytes(StandardCharsets.UTF_8); + } + + private static Metadata prefixedUtf8ContentType() { + Metadata metadata = new Metadata(); + metadata.setValue(PREFIX + HttpHeaders.CONTENT_TYPE, "text/html; charset=UTF-8"); + return metadata; + } + + /** The fast detection returns the charset of the prefixed Content-Type, see #2250. */ + @Test + void fastDetectionReadsThePrefixedContentType() { + String charset = + CharsetIdentification.getCharsetFast( + prefixedUtf8ContentType(), utf8PageWithAsciiWindow(false), WINDOW, PREFIX); + + Assertions.assertEquals("UTF-8", charset); + } + + /** The prefixed Content-Type agrees with the meta charset, so it is used, see #2250. */ + @Test + void detectionReadsThePrefixedContentType() { + String charset = + CharsetIdentification.getCharset( + prefixedUtf8ContentType(), utf8PageWithAsciiWindow(true), WINDOW, PREFIX); + + Assertions.assertEquals("UTF-8", charset); + } } diff --git a/external/playwright/src/main/java/org/apache/stormcrawler/protocol/playwright/parsefilter/JsRenderingDetector.java b/external/playwright/src/main/java/org/apache/stormcrawler/protocol/playwright/parsefilter/JsRenderingDetector.java index 451d75b0c..c0d1f889d 100644 --- a/external/playwright/src/main/java/org/apache/stormcrawler/protocol/playwright/parsefilter/JsRenderingDetector.java +++ b/external/playwright/src/main/java/org/apache/stormcrawler/protocol/playwright/parsefilter/JsRenderingDetector.java @@ -27,8 +27,10 @@ import org.apache.stormcrawler.parse.ParseData; import org.apache.stormcrawler.parse.ParseFilter; import org.apache.stormcrawler.parse.ParseResult; +import org.apache.stormcrawler.protocol.ProtocolResponse; import org.apache.stormcrawler.protocol.playwright.HttpProtocol; import org.apache.stormcrawler.util.CharsetIdentification; +import org.apache.stormcrawler.util.ConfUtils; import org.jetbrains.annotations.NotNull; import org.slf4j.LoggerFactory; import org.w3c.dom.DocumentFragment; @@ -138,6 +140,7 @@ public class JsRenderingDetector extends ParseFilter { private List emptyRootIds = DEFAULT_EMPTY_ROOT_IDS; private List requiredMessages = List.of(); private String skipIfMetadataPresent = HttpProtocol.MD_KEY_END; + private String protocolMetadataPrefix = ""; private boolean recordReason = true; private final List emptyRootPatterns = new ArrayList<>(); @@ -145,6 +148,8 @@ public class JsRenderingDetector extends ParseFilter { @Override public void configure( @NotNull final Map stormConf, @NotNull final JsonNode params) { + protocolMetadataPrefix = + ConfUtils.getString(stormConf, ProtocolResponse.PROTOCOL_MD_PREFIX_PARAM, ""); if (params != null && !params.isMissingNode() && !params.isNull()) { if (params.has("metadataKey")) { this.metadataKey = params.get("metadataKey").asText(); @@ -223,7 +228,8 @@ public void filter( private String detectReason(final String url, final byte[] content, final ParseResult parse) { final Metadata md = parse.getOrCreate(url).getMetadata(); - final String charsetName = CharsetIdentification.getCharsetFast(md, content, -1); + final String charsetName = + CharsetIdentification.getCharsetFast(md, content, -1, protocolMetadataPrefix); java.nio.charset.Charset cs; try { cs = java.nio.charset.Charset.forName(charsetName); diff --git a/external/playwright/src/test/java/org/apache/stormcrawler/protocol/playwright/parsefilter/JsRenderingDetectorTest.java b/external/playwright/src/test/java/org/apache/stormcrawler/protocol/playwright/parsefilter/JsRenderingDetectorTest.java index df854d7c6..96b5eafcc 100644 --- a/external/playwright/src/test/java/org/apache/stormcrawler/protocol/playwright/parsefilter/JsRenderingDetectorTest.java +++ b/external/playwright/src/test/java/org/apache/stormcrawler/protocol/playwright/parsefilter/JsRenderingDetectorTest.java @@ -18,8 +18,10 @@ package org.apache.stormcrawler.protocol.playwright.parsefilter; import com.fasterxml.jackson.databind.ObjectMapper; +import java.nio.charset.Charset; import java.nio.charset.StandardCharsets; import java.util.Map; +import org.apache.stormcrawler.Metadata; import org.apache.stormcrawler.parse.ParseResult; import org.apache.stormcrawler.protocol.playwright.HttpProtocol; import org.junit.jupiter.api.Assertions; @@ -197,4 +199,24 @@ void requiredMessageNoHitDoesNotFlag() throws Exception { + "

xyz"); Assertions.assertNull(p.getOrCreate("u").getMetadata().getFirstValue("fetch.with")); } + + /** + * The page is decoded with the charset of the Content-Type stored under protocol.md.prefix, see + * #2250. ISO-8859-15 is not one ICU detects, so without the header the euro sign is read as + * another character and the required message is not found. + */ + @Test + void decodesWithThePrefixedContentTypeCharset() throws Exception { + final String message = "JavaScript required to pay 5 \u20ac"; + final JsRenderingDetector d = new JsRenderingDetector(); + d.configure( + Map.of("protocol.md.prefix", "protocol."), + MAPPER.readTree("{\"requiredMessages\":[\"" + message + "\"]}")); + final ParseResult parse = new ParseResult(); + final Metadata md = parse.getOrCreate("u").getMetadata(); + md.setValue("protocol.Content-Type", "text/html; charset=ISO-8859-15"); + final String html = "

" + message + "

"; + d.filter("u", html.getBytes(Charset.forName("ISO-8859-15")), null, parse); + Assertions.assertEquals("playwright", md.getFirstValue("fetch.with")); + } } From 652a1aa934a5fc04507948e6d0a5660927528904 Mon Sep 17 00:00:00 2001 From: Nico Scandolo Date: Wed, 21 Oct 2026 18:01:50 -0300 Subject: [PATCH 3/4] #2250 Remove the unprefixed getCharset and getCharsetFast overloads --- .../util/CharsetIdentification.java | 30 +------------------ .../util/CharsetIdentificationTest.java | 11 ++++--- 2 files changed, 8 insertions(+), 33 deletions(-) diff --git a/core/src/main/java/org/apache/stormcrawler/util/CharsetIdentification.java b/core/src/main/java/org/apache/stormcrawler/util/CharsetIdentification.java index 4f34deceb..5954ad463 100644 --- a/core/src/main/java/org/apache/stormcrawler/util/CharsetIdentification.java +++ b/core/src/main/java/org/apache/stormcrawler/util/CharsetIdentification.java @@ -57,20 +57,6 @@ public class CharsetIdentification { * guess and if that fails too returns UTF-8. This approach is expected to be faster but maybe * less reliable than getCharset. * - * @since 1.18 - * @deprecated reads the HTTP Content-Type without a prefix, while fetched metadata carries the - * one set by protocol.md.prefix: use {@link #getCharsetFast(Metadata, byte[], int, String)} - */ - @Deprecated - public static String getCharsetFast( - final Metadata metadata, final byte[] content, final int maxLengthCharsetDetection) { - return getCharsetFast(metadata, content, maxLengthCharsetDetection, ""); - } - - /** - * Same as {@link #getCharsetFast(Metadata, byte[], int)}, reading the HTTP Content-Type under - * the prefix the fetcher stores the response headers with (protocol.md.prefix). - * * @param protocolMetadataPrefix prefix of the response headers in the metadata, including its * separator (e.g. {@code "protocol."}); empty if they are not prefixed * @since 4.0 @@ -114,19 +100,6 @@ public static String getCharsetFast( * metadata then use it - otherwise use ICU's charset detector to make an educated guess, with * the html metadata or else the http headers as a hint, and if that fails too returns UTF-8. * - * @deprecated reads the HTTP Content-Type without a prefix, while fetched metadata carries the - * one set by protocol.md.prefix: use {@link #getCharset(Metadata, byte[], int, String)} - */ - @Deprecated - public static String getCharset( - Metadata metadata, byte[] content, int maxLengthCharsetDetection) { - return getCharset(metadata, content, maxLengthCharsetDetection, ""); - } - - /** - * Same as {@link #getCharset(Metadata, byte[], int)}, reading the HTTP Content-Type under the - * prefix the fetcher stores the response headers with (protocol.md.prefix). - * * @param protocolMetadataPrefix prefix of the response headers in the metadata, including its * separator (e.g. {@code "protocol."}); empty if they are not prefixed * @since 4.0 @@ -154,8 +127,7 @@ public static String getCharset( return httpCharset; } - // let's guess from the text, with a hint if any: when the header and the meta disagree - // the meta is the hint, as it was while the header could not be read (#2250) + // let's guess from the text - using a hint or not String hintCharset = htmlCharset != null ? htmlCharset : httpCharset; String textCharset = getCharsetFromText(content, hintCharset, maxLengthCharsetDetection); diff --git a/core/src/test/java/org/apache/stormcrawler/util/CharsetIdentificationTest.java b/core/src/test/java/org/apache/stormcrawler/util/CharsetIdentificationTest.java index 547f81bc2..fd2f56463 100644 --- a/core/src/test/java/org/apache/stormcrawler/util/CharsetIdentificationTest.java +++ b/core/src/test/java/org/apache/stormcrawler/util/CharsetIdentificationTest.java @@ -57,7 +57,7 @@ private static Throwable detectOnSmallStack(byte[] content, int maxlength) () -> { try { CharsetIdentification.getCharset( - new Metadata(), content, maxlength); + new Metadata(), content, maxlength, ""); } catch (Throwable t) { thrown.set(t); } @@ -107,7 +107,7 @@ void metaCharsetCutByTheDetectionWindowIsStillRead() { page.append("").append(BODY); byte[] content = page.toString().getBytes(StandardCharsets.US_ASCII); - String charset = CharsetIdentification.getCharsetFast(new Metadata(), content, cut); + String charset = CharsetIdentification.getCharsetFast(new Metadata(), content, cut, ""); Assertions.assertEquals(DECLARED, charset); } @@ -134,10 +134,13 @@ private static byte[] metaCharsetClosingAfterTheWindow(int quoteOffset) { void lookAheadForTheClosingQuoteIsBounded() { String within = CharsetIdentification.getCharsetFast( - new Metadata(), metaCharsetClosingAfterTheWindow(LOOKAHEAD - 1), MAXLENGTH); + new Metadata(), + metaCharsetClosingAfterTheWindow(LOOKAHEAD - 1), + MAXLENGTH, + ""); String beyond = CharsetIdentification.getCharsetFast( - new Metadata(), metaCharsetClosingAfterTheWindow(LOOKAHEAD), MAXLENGTH); + new Metadata(), metaCharsetClosingAfterTheWindow(LOOKAHEAD), MAXLENGTH, ""); Assertions.assertEquals(DECLARED, within); Assertions.assertNotEquals( From eca1b7681a25f9da7ce76ef214f24eb6e3b66047 Mon Sep 17 00:00:00 2001 From: Nico Scandolo Date: Thu, 22 Oct 2026 09:33:07 -0300 Subject: [PATCH 4/4] #2250 Use literal characters in the new tests (checkstyle) --- .../java/org/apache/stormcrawler/bolt/JSoupParserBoltTest.java | 2 +- .../org/apache/stormcrawler/util/CharsetIdentificationTest.java | 2 +- .../playwright/parsefilter/JsRenderingDetectorTest.java | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/core/src/test/java/org/apache/stormcrawler/bolt/JSoupParserBoltTest.java b/core/src/test/java/org/apache/stormcrawler/bolt/JSoupParserBoltTest.java index c63c3cd17..ebd913398 100644 --- a/core/src/test/java/org/apache/stormcrawler/bolt/JSoupParserBoltTest.java +++ b/core/src/test/java/org/apache/stormcrawler/bolt/JSoupParserBoltTest.java @@ -396,7 +396,7 @@ void prefixedContentTypeCharsetIsUsed(boolean fast) throws IOException { + ""; - String text = "What\u2019s new: caf\u00e9 imotine\u2122"; + String text = "What’s new: café imotine™"; byte[] content = ("" + head + "

" + text + "

") .getBytes(StandardCharsets.UTF_8); diff --git a/core/src/test/java/org/apache/stormcrawler/util/CharsetIdentificationTest.java b/core/src/test/java/org/apache/stormcrawler/util/CharsetIdentificationTest.java index fd2f56463..73e23666a 100644 --- a/core/src/test/java/org/apache/stormcrawler/util/CharsetIdentificationTest.java +++ b/core/src/test/java/org/apache/stormcrawler/util/CharsetIdentificationTest.java @@ -160,7 +160,7 @@ private static byte[] utf8PageWithAsciiWindow(boolean withMeta) { String meta = withMeta ? "" : ""; String head = ""; - return ("" + meta + head + "

caf\u00e9

") + return ("" + meta + head + "

café

") .getBytes(StandardCharsets.UTF_8); } diff --git a/external/playwright/src/test/java/org/apache/stormcrawler/protocol/playwright/parsefilter/JsRenderingDetectorTest.java b/external/playwright/src/test/java/org/apache/stormcrawler/protocol/playwright/parsefilter/JsRenderingDetectorTest.java index 96b5eafcc..937c51328 100644 --- a/external/playwright/src/test/java/org/apache/stormcrawler/protocol/playwright/parsefilter/JsRenderingDetectorTest.java +++ b/external/playwright/src/test/java/org/apache/stormcrawler/protocol/playwright/parsefilter/JsRenderingDetectorTest.java @@ -207,7 +207,7 @@ void requiredMessageNoHitDoesNotFlag() throws Exception { */ @Test void decodesWithThePrefixedContentTypeCharset() throws Exception { - final String message = "JavaScript required to pay 5 \u20ac"; + final String message = "JavaScript required to pay 5 €"; final JsRenderingDetector d = new JsRenderingDetector(); d.configure( Map.of("protocol.md.prefix", "protocol."),