diff --git a/core/src/main/java/org/apache/stormcrawler/bolt/JSoupParserBolt.java b/core/src/main/java/org/apache/stormcrawler/bolt/JSoupParserBolt.java index dc8c32c92..c128082a6 100644 --- a/core/src/main/java/org/apache/stormcrawler/bolt/JSoupParserBolt.java +++ b/core/src/main/java/org/apache/stormcrawler/bolt/JSoupParserBolt.java @@ -286,11 +286,17 @@ public void execute(Tuple tuple) { if (fastCharsetDetection) { charset = CharsetIdentification.getCharsetFast( - metadata, content, maxLengthCharsetDetection); + metadata, + content, + maxLengthCharsetDetection, + protocolMetadataPrefix); } else { charset = CharsetIdentification.getCharset( - metadata, content, maxLengthCharsetDetection); + metadata, + content, + maxLengthCharsetDetection, + protocolMetadataPrefix); } LOG.debug( diff --git a/core/src/main/java/org/apache/stormcrawler/util/CharsetIdentification.java b/core/src/main/java/org/apache/stormcrawler/util/CharsetIdentification.java index 9fc778dd3..5954ad463 100644 --- a/core/src/main/java/org/apache/stormcrawler/util/CharsetIdentification.java +++ b/core/src/main/java/org/apache/stormcrawler/util/CharsetIdentification.java @@ -57,10 +57,15 @@ public class CharsetIdentification { * guess and if that fails too returns UTF-8. This approach is expected to be faster but maybe * less reliable than getCharset. * - * @since 1.18 + * @param protocolMetadataPrefix prefix of the response headers in the metadata, including its + * separator (e.g. {@code "protocol."}); empty if they are not prefixed + * @since 4.0 */ public static String getCharsetFast( - final Metadata metadata, final byte[] content, final int maxLengthCharsetDetection) { + final Metadata metadata, + final byte[] content, + final int maxLengthCharsetDetection, + final String protocolMetadataPrefix) { // let's look at the BOM first String charset = getCharsetFromBOM(content); @@ -69,7 +74,7 @@ public static String getCharsetFast( } // then look at what we get from HTTP headers - charset = getCharsetFromHTTP(metadata); + charset = getCharsetFromHTTP(metadata, protocolMetadataPrefix); if (charset != null) { return charset; } @@ -92,11 +97,18 @@ public static String getCharsetFast( /** * Identifies the charset of a document based on the following logic: guess from the * ByteOrderMark - else if the same charset is specified in the http headers and the html - * metadata then use it - otherwise use ICU's charset detector to make an educated guess and if - * that fails too returns UTF-8. + * metadata then use it - otherwise use ICU's charset detector to make an educated guess, with + * the html metadata or else the http headers as a hint, and if that fails too returns UTF-8. + * + * @param protocolMetadataPrefix prefix of the response headers in the metadata, including its + * separator (e.g. {@code "protocol."}); empty if they are not prefixed + * @since 4.0 */ public static String getCharset( - Metadata metadata, byte[] content, int maxLengthCharsetDetection) { + Metadata metadata, + byte[] content, + int maxLengthCharsetDetection, + String protocolMetadataPrefix) { // let's look at the BOM first String BOMCharset = getCharsetFromBOM(content); @@ -105,7 +117,7 @@ public static String getCharset( } // then look at what we get from HTTP headers and HTML content - String httpCharset = getCharsetFromHTTP(metadata); + String httpCharset = getCharsetFromHTTP(metadata, protocolMetadataPrefix); String htmlCharset = getCharsetFromMeta(content, maxLengthCharsetDetection); // both exist and agree @@ -116,12 +128,7 @@ public static String getCharset( } // let's guess from the text - using a hint or not - String hintCharset = null; - if (httpCharset != null && htmlCharset == null) { - hintCharset = httpCharset; - } else if (httpCharset == null && htmlCharset != null) { - hintCharset = htmlCharset; - } + String hintCharset = htmlCharset != null ? htmlCharset : httpCharset; String textCharset = getCharsetFromText(content, hintCharset, maxLengthCharsetDetection); if (textCharset != null) { @@ -133,8 +140,9 @@ public static String getCharset( } /** Returns the charset declared by the server if any. */ - private static String getCharsetFromHTTP(Metadata metadata) { - return getCharsetFromContentType(metadata.getFirstValue(HttpHeaders.CONTENT_TYPE)); + private static String getCharsetFromHTTP(Metadata metadata, String protocolMetadataPrefix) { + return getCharsetFromContentType( + metadata.getFirstValue(HttpHeaders.CONTENT_TYPE, protocolMetadataPrefix)); } /** Detects any BOMs and returns the corresponding charset. */ diff --git a/core/src/test/java/org/apache/stormcrawler/bolt/JSoupParserBoltTest.java b/core/src/test/java/org/apache/stormcrawler/bolt/JSoupParserBoltTest.java index ba28bc98c..ebd913398 100644 --- a/core/src/test/java/org/apache/stormcrawler/bolt/JSoupParserBoltTest.java +++ b/core/src/test/java/org/apache/stormcrawler/bolt/JSoupParserBoltTest.java @@ -22,6 +22,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import org.apache.http.HttpHeaders; import org.apache.storm.task.OutputCollector; import org.apache.stormcrawler.Constants; import org.apache.stormcrawler.Metadata; @@ -33,6 +34,8 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; import org.mockito.MockedStatic; import org.mockito.Mockito; @@ -129,7 +132,10 @@ void charsetDetectionFailureIsReportedAsParseError() throws IOException { .when( () -> CharsetIdentification.getCharset( - Mockito.any(), Mockito.any(), Mockito.anyInt())) + Mockito.any(), + Mockito.any(), + Mockito.anyInt(), + Mockito.any())) .thenThrow(new StackOverflowError()); parse("https://stormcrawler.apache.org", "stormcrawler.apache.org.html"); } @@ -365,4 +371,41 @@ void testBaseHrefOutlinkResolution() throws IOException { "https://www.example.be/fr/rayons/stockage-archivage-de-donnees/218725", statusTuples.get(0).get(0)); } + + /** + * The fetcher stores the response headers under protocol.md.prefix, so the charset declared in + * the Content-Type has to be read with that prefix. Without it the header is never seen, the + * header and the meta charset can never agree, and ICU guesses a UTF-8 page whose detection + * window is plain ASCII as ISO-8859-1, see issue #2250. + */ + @ParameterizedTest(name = "detect.charset.fast={0}") + @ValueSource(booleans = {false, true}) + void prefixedContentTypeCharsetIsUsed(boolean fast) throws IOException { + Map parserConfig = new HashMap<>(); + parserConfig.put("protocol.md.prefix", "protocol."); + parserConfig.put("detect.charset.maxlength", 2048); + parserConfig.put("detect.charset.fast", fast); + bolt.prepare( + parserConfig, TestUtil.getMockedTopologyContext(), new OutputCollector(output)); + // an ASCII head longer than the detection window, like most real pages. The default + // detection uses the header when it agrees with the meta charset; the fast one returns the + // header first, so it is tested without a meta charset, which it would use otherwise + String meta = fast ? "" : ""; + String head = + meta + + ""; + String text = "What’s new: café imotine™"; + byte[] content = + ("" + head + "

" + text + "

") + .getBytes(StandardCharsets.UTF_8); + Metadata metadata = new Metadata(); + metadata.setValue("protocol." + HttpHeaders.CONTENT_TYPE, "text/html; charset=UTF-8"); + parse("http://www.example.com/page.html", content, metadata); + + Assertions.assertEquals(1, output.getEmitted().size()); + String parsedText = (String) output.getEmitted().get(0).get(3); + Assertions.assertTrue(parsedText.contains(text), parsedText); + } } diff --git a/core/src/test/java/org/apache/stormcrawler/util/CharsetIdentificationTest.java b/core/src/test/java/org/apache/stormcrawler/util/CharsetIdentificationTest.java index 0a22c7a5d..73e23666a 100644 --- a/core/src/test/java/org/apache/stormcrawler/util/CharsetIdentificationTest.java +++ b/core/src/test/java/org/apache/stormcrawler/util/CharsetIdentificationTest.java @@ -20,6 +20,7 @@ import java.nio.charset.StandardCharsets; import java.util.Arrays; import java.util.concurrent.atomic.AtomicReference; +import org.apache.http.HttpHeaders; import org.apache.stormcrawler.Metadata; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -56,7 +57,7 @@ private static Throwable detectOnSmallStack(byte[] content, int maxlength) () -> { try { CharsetIdentification.getCharset( - new Metadata(), content, maxlength); + new Metadata(), content, maxlength, ""); } catch (Throwable t) { thrown.set(t); } @@ -106,7 +107,7 @@ void metaCharsetCutByTheDetectionWindowIsStillRead() { page.append("").append(BODY); byte[] content = page.toString().getBytes(StandardCharsets.US_ASCII); - String charset = CharsetIdentification.getCharsetFast(new Metadata(), content, cut); + String charset = CharsetIdentification.getCharsetFast(new Metadata(), content, cut, ""); Assertions.assertEquals(DECLARED, charset); } @@ -133,13 +134,59 @@ private static byte[] metaCharsetClosingAfterTheWindow(int quoteOffset) { void lookAheadForTheClosingQuoteIsBounded() { String within = CharsetIdentification.getCharsetFast( - new Metadata(), metaCharsetClosingAfterTheWindow(LOOKAHEAD - 1), MAXLENGTH); + new Metadata(), + metaCharsetClosingAfterTheWindow(LOOKAHEAD - 1), + MAXLENGTH, + ""); String beyond = CharsetIdentification.getCharsetFast( - new Metadata(), metaCharsetClosingAfterTheWindow(LOOKAHEAD), MAXLENGTH); + new Metadata(), metaCharsetClosingAfterTheWindow(LOOKAHEAD), MAXLENGTH, ""); Assertions.assertEquals(DECLARED, within); Assertions.assertNotEquals( DECLARED, beyond, "a declaration closing beyond the look-ahead is not used"); } + + /** The prefix the fetcher stores the response headers with, see protocol.md.prefix. */ + private static final String PREFIX = "protocol."; + + /** The detection window of these tests: the page is plain ASCII within it. */ + private static final int WINDOW = 2048; + + /** + * A UTF-8 page, optionally with a meta charset, whose first non-ASCII byte is past the window. + */ + private static byte[] utf8PageWithAsciiWindow(boolean withMeta) { + String meta = withMeta ? "" : ""; + String head = + ""; + return ("" + meta + head + "

café

") + .getBytes(StandardCharsets.UTF_8); + } + + private static Metadata prefixedUtf8ContentType() { + Metadata metadata = new Metadata(); + metadata.setValue(PREFIX + HttpHeaders.CONTENT_TYPE, "text/html; charset=UTF-8"); + return metadata; + } + + /** The fast detection returns the charset of the prefixed Content-Type, see #2250. */ + @Test + void fastDetectionReadsThePrefixedContentType() { + String charset = + CharsetIdentification.getCharsetFast( + prefixedUtf8ContentType(), utf8PageWithAsciiWindow(false), WINDOW, PREFIX); + + Assertions.assertEquals("UTF-8", charset); + } + + /** The prefixed Content-Type agrees with the meta charset, so it is used, see #2250. */ + @Test + void detectionReadsThePrefixedContentType() { + String charset = + CharsetIdentification.getCharset( + prefixedUtf8ContentType(), utf8PageWithAsciiWindow(true), WINDOW, PREFIX); + + Assertions.assertEquals("UTF-8", charset); + } } diff --git a/external/playwright/src/main/java/org/apache/stormcrawler/protocol/playwright/parsefilter/JsRenderingDetector.java b/external/playwright/src/main/java/org/apache/stormcrawler/protocol/playwright/parsefilter/JsRenderingDetector.java index 451d75b0c..c0d1f889d 100644 --- a/external/playwright/src/main/java/org/apache/stormcrawler/protocol/playwright/parsefilter/JsRenderingDetector.java +++ b/external/playwright/src/main/java/org/apache/stormcrawler/protocol/playwright/parsefilter/JsRenderingDetector.java @@ -27,8 +27,10 @@ import org.apache.stormcrawler.parse.ParseData; import org.apache.stormcrawler.parse.ParseFilter; import org.apache.stormcrawler.parse.ParseResult; +import org.apache.stormcrawler.protocol.ProtocolResponse; import org.apache.stormcrawler.protocol.playwright.HttpProtocol; import org.apache.stormcrawler.util.CharsetIdentification; +import org.apache.stormcrawler.util.ConfUtils; import org.jetbrains.annotations.NotNull; import org.slf4j.LoggerFactory; import org.w3c.dom.DocumentFragment; @@ -138,6 +140,7 @@ public class JsRenderingDetector extends ParseFilter { private List emptyRootIds = DEFAULT_EMPTY_ROOT_IDS; private List requiredMessages = List.of(); private String skipIfMetadataPresent = HttpProtocol.MD_KEY_END; + private String protocolMetadataPrefix = ""; private boolean recordReason = true; private final List emptyRootPatterns = new ArrayList<>(); @@ -145,6 +148,8 @@ public class JsRenderingDetector extends ParseFilter { @Override public void configure( @NotNull final Map stormConf, @NotNull final JsonNode params) { + protocolMetadataPrefix = + ConfUtils.getString(stormConf, ProtocolResponse.PROTOCOL_MD_PREFIX_PARAM, ""); if (params != null && !params.isMissingNode() && !params.isNull()) { if (params.has("metadataKey")) { this.metadataKey = params.get("metadataKey").asText(); @@ -223,7 +228,8 @@ public void filter( private String detectReason(final String url, final byte[] content, final ParseResult parse) { final Metadata md = parse.getOrCreate(url).getMetadata(); - final String charsetName = CharsetIdentification.getCharsetFast(md, content, -1); + final String charsetName = + CharsetIdentification.getCharsetFast(md, content, -1, protocolMetadataPrefix); java.nio.charset.Charset cs; try { cs = java.nio.charset.Charset.forName(charsetName); diff --git a/external/playwright/src/test/java/org/apache/stormcrawler/protocol/playwright/parsefilter/JsRenderingDetectorTest.java b/external/playwright/src/test/java/org/apache/stormcrawler/protocol/playwright/parsefilter/JsRenderingDetectorTest.java index df854d7c6..937c51328 100644 --- a/external/playwright/src/test/java/org/apache/stormcrawler/protocol/playwright/parsefilter/JsRenderingDetectorTest.java +++ b/external/playwright/src/test/java/org/apache/stormcrawler/protocol/playwright/parsefilter/JsRenderingDetectorTest.java @@ -18,8 +18,10 @@ package org.apache.stormcrawler.protocol.playwright.parsefilter; import com.fasterxml.jackson.databind.ObjectMapper; +import java.nio.charset.Charset; import java.nio.charset.StandardCharsets; import java.util.Map; +import org.apache.stormcrawler.Metadata; import org.apache.stormcrawler.parse.ParseResult; import org.apache.stormcrawler.protocol.playwright.HttpProtocol; import org.junit.jupiter.api.Assertions; @@ -197,4 +199,24 @@ void requiredMessageNoHitDoesNotFlag() throws Exception { + "

xyz"); Assertions.assertNull(p.getOrCreate("u").getMetadata().getFirstValue("fetch.with")); } + + /** + * The page is decoded with the charset of the Content-Type stored under protocol.md.prefix, see + * #2250. ISO-8859-15 is not one ICU detects, so without the header the euro sign is read as + * another character and the required message is not found. + */ + @Test + void decodesWithThePrefixedContentTypeCharset() throws Exception { + final String message = "JavaScript required to pay 5 €"; + final JsRenderingDetector d = new JsRenderingDetector(); + d.configure( + Map.of("protocol.md.prefix", "protocol."), + MAPPER.readTree("{\"requiredMessages\":[\"" + message + "\"]}")); + final ParseResult parse = new ParseResult(); + final Metadata md = parse.getOrCreate("u").getMetadata(); + md.setValue("protocol.Content-Type", "text/html; charset=ISO-8859-15"); + final String html = "

" + message + "

"; + d.filter("u", html.getBytes(Charset.forName("ISO-8859-15")), null, parse); + Assertions.assertEquals("playwright", md.getFirstValue("fetch.with")); + } }