diff --git a/core/src/main/java/org/apache/stormcrawler/jsoup/LinkParseFilter.java b/core/src/main/java/org/apache/stormcrawler/jsoup/LinkParseFilter.java index 92a114f07..91d7fb299 100644 --- a/core/src/main/java/org/apache/stormcrawler/jsoup/LinkParseFilter.java +++ b/core/src/main/java/org/apache/stormcrawler/jsoup/LinkParseFilter.java @@ -39,7 +39,7 @@ * *
{@code
  * {
- *   "class": "org.apache.stormcrawler.parse.filter.LinkParseFilter",
+ *   "class": "org.apache.stormcrawler.jsoup.LinkParseFilter",
  *   "name": "LinkParseFilter",
  *   "params": {
  *     "pattern": "//IMG/@src",
diff --git a/core/src/main/java/org/apache/stormcrawler/parse/filter/LDJsonParseFilter.java b/core/src/main/java/org/apache/stormcrawler/parse/filter/LDJsonParseFilter.java
deleted file mode 100644
index 8a91c6aa8..000000000
--- a/core/src/main/java/org/apache/stormcrawler/parse/filter/LDJsonParseFilter.java
+++ /dev/null
@@ -1,123 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to you under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.stormcrawler.parse.filter;
-
-import com.fasterxml.jackson.core.JsonPointer;
-import com.fasterxml.jackson.databind.JsonNode;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-import javax.xml.xpath.XPath;
-import javax.xml.xpath.XPathConstants;
-import javax.xml.xpath.XPathExpression;
-import javax.xml.xpath.XPathFactory;
-import org.apache.stormcrawler.Metadata;
-import org.apache.stormcrawler.parse.ParseData;
-import org.apache.stormcrawler.parse.ParseFilter;
-import org.apache.stormcrawler.parse.ParseResult;
-import org.jetbrains.annotations.NotNull;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.w3c.dom.DocumentFragment;
-import org.w3c.dom.Node;
-
-/** Extracts data from JSON-LD representation (https://json-ld.org/). */
-public class LDJsonParseFilter extends ParseFilter {
-
-    public static final Logger LOG = LoggerFactory.getLogger(LDJsonParseFilter.class);
-
-    private static XPathFactory factory = XPathFactory.newInstance();
-    private static XPath xpath = factory.newXPath();
-
-    private static ObjectMapper mapper = new ObjectMapper();
-
-    private List expressions = new LinkedList<>();
-
-    @Override
-    public void filter(String url, byte[] content, DocumentFragment doc, ParseResult parse) {
-        if (doc == null) {
-            return;
-        }
-        try {
-            JsonNode json = filterJson(doc);
-            if (json == null) {
-                return;
-            }
-
-            ParseData parseData = parse.getOrCreate(url);
-            Metadata metadata = parseData.getMetadata();
-
-            // extract patterns and store as metadata
-            for (LabelledJsonPointer expression : expressions) {
-                JsonNode match = json.at(expression.pointer);
-                if (match.isMissingNode()) {
-                    continue;
-                }
-                metadata.addValue(expression.label, match.asText());
-            }
-
-        } catch (Exception e) {
-            LOG.error("Exception caught when extracting json", e);
-        }
-    }
-
-    public static JsonNode filterJson(DocumentFragment doc) throws Exception {
-        XPathExpression expressionJobPosting =
-                xpath.compile("//SCRIPT[@type=\"application/ld+json\"]");
-        Node scriptNode = (Node) expressionJobPosting.evaluate(doc, XPathConstants.NODE);
-        if (scriptNode == null) {
-            return null;
-        }
-        return mapper.readValue(scriptNode.getTextContent(), JsonNode.class);
-    }
-
-    public void configure(@NotNull Map stormConf, @NotNull JsonNode filterParams) {
-        java.util.Iterator> iter = filterParams.fields();
-        while (iter.hasNext()) {
-            Entry entry = iter.next();
-            String key = entry.getKey();
-            JsonNode node = entry.getValue();
-            LabelledJsonPointer labelP =
-                    new LabelledJsonPointer(key, JsonPointer.valueOf(node.asText()));
-            expressions.add(labelP);
-        }
-    }
-
-    class LabelledJsonPointer {
-
-        String label;
-        JsonPointer pointer;
-
-        public LabelledJsonPointer(String label, JsonPointer pointer) {
-            this.label = label;
-            this.pointer = pointer;
-        }
-
-        @Override
-        public String toString() {
-            return label + " => " + pointer.toString();
-        }
-    }
-
-    @Override
-    public boolean needsDOM() {
-        return true;
-    }
-}
diff --git a/core/src/main/java/org/apache/stormcrawler/parse/filter/LinkParseFilter.java b/core/src/main/java/org/apache/stormcrawler/parse/filter/LinkParseFilter.java
deleted file mode 100644
index 7468a55c9..000000000
--- a/core/src/main/java/org/apache/stormcrawler/parse/filter/LinkParseFilter.java
+++ /dev/null
@@ -1,131 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to you under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.stormcrawler.parse.filter;
-
-import com.fasterxml.jackson.databind.JsonNode;
-import java.net.MalformedURLException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import org.apache.stormcrawler.Metadata;
-import org.apache.stormcrawler.filtering.URLFilters;
-import org.apache.stormcrawler.parse.Outlink;
-import org.apache.stormcrawler.parse.ParseData;
-import org.apache.stormcrawler.parse.ParseResult;
-import org.apache.stormcrawler.util.MetadataTransfer;
-import org.apache.stormcrawler.util.URLUtil;
-import org.jetbrains.annotations.NotNull;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.w3c.dom.DocumentFragment;
-
-/**
- * ParseFilter to extract additional links with Xpath can be configured with e.g.
- *
- * 
{@code
- * {
- *   "class": "org.apache.stormcrawler.parse.filter.LinkParseFilter",
- *   "name": "LinkParseFilter",
- *   "params": {
- *     "pattern": "//IMG/@src",
- *     "pattern2": "//VIDEO/SOURCE/@src"
- *   }
- * }
- *
- * }
- */ -public class LinkParseFilter extends XPathFilter { - - private static final Logger LOG = LoggerFactory.getLogger(LinkParseFilter.class); - - private MetadataTransfer metadataTransfer; - - private URLFilters urlFilters; - - @Override - public void filter(String url, byte[] content, DocumentFragment doc, ParseResult parse) { - - ParseData parseData = parse.getOrCreate(url); - Metadata metadata = parseData.getMetadata(); - - Map dedup = new HashMap<>(); - - for (Outlink o : parse.getOutlinks()) { - dedup.put(o.getTargetURL(), o); - } - - java.net.URL sourceUrl; - try { - sourceUrl = URLUtil.toURL(url); - } catch (MalformedURLException e1) { - // we would have known by now as previous components check whether - // the URL is valid - LOG.error("Malformed URL on {}", url); - return; - } - - // applies the XPATH expression in the order in which they are produced - for (List leList : expressions.values()) { - for (LabelledExpression le : leList) { - try { - List values = le.evaluate(doc); - if (values == null || values.isEmpty()) { - continue; - } - for (String target : values) { - // resolve URL - target = URLUtil.resolveUrl(sourceUrl, target).toExternalForm(); - - // apply filtering - target = urlFilters.filter(sourceUrl, metadata, target); - if (target == null) { - continue; - } - - // check whether we already have this link - if (dedup.containsKey(target)) { - continue; - } - - // create outlink - Outlink ol = new Outlink(target); - - // get the metadata for the outlink from the parent one - Metadata metadataOutlink = - metadataTransfer.getMetaForOutlink(target, url, metadata); - - ol.setMetadata(metadataOutlink); - dedup.put(ol.getTargetURL(), ol); - } - } catch (Exception e) { - LOG.error("Error evaluating {}: {}", le.key, e); - } - } - } - - parse.setOutlinks(new ArrayList<>(dedup.values())); - } - - @Override - public void configure(@NotNull Map stormConf, @NotNull JsonNode filterParams) { - super.configure(stormConf, filterParams); - this.metadataTransfer = MetadataTransfer.getInstance(stormConf); - this.urlFilters = URLFilters.fromConf(stormConf); - } -} diff --git a/core/src/main/java/org/apache/stormcrawler/util/RobotsTags.java b/core/src/main/java/org/apache/stormcrawler/util/RobotsTags.java index 31c7238c1..767820164 100644 --- a/core/src/main/java/org/apache/stormcrawler/util/RobotsTags.java +++ b/core/src/main/java/org/apache/stormcrawler/util/RobotsTags.java @@ -17,16 +17,7 @@ package org.apache.stormcrawler.util; -import javax.xml.xpath.XPath; -import javax.xml.xpath.XPathConstants; -import javax.xml.xpath.XPathExpression; -import javax.xml.xpath.XPathExpressionException; -import javax.xml.xpath.XPathFactory; import org.apache.stormcrawler.Metadata; -import org.w3c.dom.DocumentFragment; -import org.w3c.dom.NamedNodeMap; -import org.w3c.dom.Node; -import org.w3c.dom.NodeList; /** * Normalises the robots instructions provided by the HTML meta tags or the HTTP X-Robots-Tag @@ -52,17 +43,6 @@ public class RobotsTags { private boolean noCache = false; - private static final XPathExpression expression; - - static { - XPath xpath = XPathFactory.newInstance().newXPath(); - try { - expression = xpath.compile("/HTML/*/META"); - } catch (XPathExpressionException e) { - throw new RuntimeException(e); - } - } - /** Get the values from the fetch metadata. */ public RobotsTags(Metadata metadata, String protocolMDprefix) { // HTTP headers @@ -80,46 +60,6 @@ public RobotsTags(Metadata metadata, String protocolMDprefix) { public RobotsTags() {} - // set the values based on the meta tags - // HTML tags - // - // called by the parser bolts - public void extractMetaTags(DocumentFragment doc) throws XPathExpressionException { - NodeList nodes = (NodeList) expression.evaluate(doc, XPathConstants.NODESET); - if (nodes == null) { - return; - } - int numNodes = nodes.getLength(); - for (int i = 0; i < numNodes; i++) { - Node n = (Node) nodes.item(i); - // iterate on the attributes - // and check that it has name=robots and content - // whatever the case is - boolean isRobots = false; - String content = null; - NamedNodeMap attrs = n.getAttributes(); - for (int att = 0; att < attrs.getLength(); att++) { - Node keyval = attrs.item(att); - if ("name".equalsIgnoreCase(keyval.getNodeName()) - && "robots".equalsIgnoreCase(keyval.getNodeValue())) { - isRobots = true; - continue; - } - if ("content".equalsIgnoreCase(keyval.getNodeName())) { - content = keyval.getNodeValue(); - continue; - } - } - - if (isRobots && content != null) { - // got a value - split it - String[] vals = content.split(" *, *"); - parseValues(vals); - return; - } - } - } - /** Extracts meta tags based on the value of the content attribute. */ public void extractMetaTags(String content) { if (content == null) { diff --git a/core/src/test/java/org/apache/stormcrawler/parse/filter/XPathFilterTest.java b/core/src/test/java/org/apache/stormcrawler/parse/filter/XPathFilterTest.java index c4a525bc6..1ffbfae3f 100644 --- a/core/src/test/java/org/apache/stormcrawler/parse/filter/XPathFilterTest.java +++ b/core/src/test/java/org/apache/stormcrawler/parse/filter/XPathFilterTest.java @@ -64,16 +64,4 @@ void testScriptExtraction() throws IOException { Assertions.assertEquals("", scripts[0].trim()); Assertions.assertTrue(scripts[1].contains("_paq")); } - - @Test - void testLDJsonExtraction() throws IOException { - prepareParserBolt("test.parsefilters.json"); - parse("https://stormcrawler.apache.org", "stormcrawler.apache.org.html"); - Assertions.assertEquals(1, output.getEmitted().size()); - List parsedTuple = output.getEmitted().get(0); - Metadata metadata = (Metadata) parsedTuple.get(2); - Assertions.assertNotNull(metadata); - String[] scripts = metadata.getValues("streetAddress"); - Assertions.assertNotNull(scripts); - } } diff --git a/core/src/test/resources/test.parsefilters.json b/core/src/test/resources/test.parsefilters.json index d32c5b526..ceac8bde6 100644 --- a/core/src/test/resources/test.parsefilters.json +++ b/core/src/test/resources/test.parsefilters.json @@ -11,13 +11,6 @@ "title": "//TITLE" } }, - { - "class": "org.apache.stormcrawler.parse.filter.LDJsonParseFilter", - "name": "LDJsonParseFilter", - "params": { - "streetAddress": "/location/streetAddress" - } - }, { "class": "org.apache.stormcrawler.parse.filter.CommaSeparatedToMultivaluedMetadata", "name": "CommaSeparatedToMultivaluedMetadata", diff --git a/docs/src/main/asciidoc/architecture.adoc b/docs/src/main/asciidoc/architecture.adoc index 5b2a22e58..aed5c7d9d 100644 --- a/docs/src/main/asciidoc/architecture.adoc +++ b/docs/src/main/asciidoc/architecture.adoc @@ -76,6 +76,8 @@ processed. Parse Filters are applied within the parsing bolts, following parsing by `SiteMapParser`, `JSoupParser`, or `ParserBolt` (Tika). They can modify extracted text, metadata, and links before the content is indexed or new URLs are emitted. +For HTML documents, `JSoupParser` also applies JSoup Filters, which work directly +on the JSoup document and do not require a DOM. Common Parse Filters include: diff --git a/docs/src/main/asciidoc/configuration.adoc b/docs/src/main/asciidoc/configuration.adoc index f628335b7..17d0be9d0 100644 --- a/docs/src/main/asciidoc/configuration.adoc +++ b/docs/src/main/asciidoc/configuration.adoc @@ -394,6 +394,7 @@ Configures parsing of fetched text and the handling of discovered URIs | detect.mimetype | true | Enable MIME type detection during parsing. | feed.filter.hours.since.published | -1 | Discard feeds older than value hours. | feed.sniffContent | false | Try to detect feeds automatically. +| jsoup.filters.config.file | - | Path to JSON config defining the JSoupFilters used by JSoupParserBolt. See link:https://github.com/apache/stormcrawler/blob/main/archetype/src/main/resources/archetype-resources/src/main/resources/jsoupfilters.json[default jsoupfilters.json]. | jsoup.treat.non.html.as.error | true | If true, non-HTML content is treated as an error by JSoupParserBolt. Note that `text/plain` is always handled directly (the content is used as the text) and never treated as an error. Its stored text honors `textextractor.no.text` and is capped at `textextractor.skip.after`; the raw fetched size is bounded by `http.content.limit`. | parsefilters.config.file | parsefilters.json | Path to JSON config defining ParseFilters. See link:https://github.com/apache/stormcrawler/blob/main/archetype/src/main/resources/archetype-resources/src/main/resources/parsefilters.json[default parsefilters.json]. | parser.emitOutlinks | true | Emit discovered links as DISCOVERED tuples. diff --git a/docs/src/main/asciidoc/internals.adoc b/docs/src/main/asciidoc/internals.adoc index e98fd99d0..6d14ad36b 100644 --- a/docs/src/main/asciidoc/internals.adoc +++ b/docs/src/main/asciidoc/internals.adoc @@ -133,7 +133,7 @@ You can easily build your own custom indexer to integrate with other storage sys The link:https://github.com/apache/stormcrawler/blob/main/core/src/main/java/org/apache/stormcrawler/bolt/JSoupParserBolt.java[JSoupParserBolt] can be used to parse HTML documents and extract the outlinks, text, and metadata it contains. `text/plain` content is also handled: since there is no markup to parse, the content is used directly as the text and no outlinks are extracted. If you want to parse other non-HTML documents, use the link:https://github.com/apache/stormcrawler/tree/main/external/tika/src/main/java/org/apache/stormcrawler/tika[Tika-based ParserBolt] from the external modules. -This parser calls the xref:urlfilters[URLFilters] and xref:parsefilters[ParseFilters] defined in the configuration. Please note that it calls xref:metadatatransfer[MetadataTransfer] prior to calling the xref:parsefilters[ParseFilters]. If you create new Outlinks in your [[ParseFilters]], you'll need to make sure that you use MetadataTransfer there to inherit the Metadata from the parent document. +This parser calls the xref:urlfilters[URLFilters], xref:jsoupfilters[JSoupFilters] and xref:parsefilters[ParseFilters] defined in the configuration. Please note that it calls xref:metadatatransfer[MetadataTransfer] prior to calling the xref:parsefilters[ParseFilters]. If you create new Outlinks in your [[ParseFilters]], you'll need to make sure that you use MetadataTransfer there to inherit the Metadata from the parent document. The **JSoupParserBolt** automatically identifies the charset of the documents. It uses the status stream to report parsing errors but also for the outlinks it extracts from a page. These would typically be used by an extension of link:https://github.com/apache/stormcrawler/blob/main/core/src/main/java/org/apache/stormcrawler/persistence/AbstractStatusUpdaterBolt.java[AbstractStatusUpdaterBolt] and persisted in some form of storage. @@ -214,15 +214,72 @@ The archetype includes a default link:https://github.com/apache/stormcrawler/blo * **CommaSeparatedToMultivaluedMetadata** – link:https://github.com/apache/stormcrawler/blob/main/core/src/main/java/org/apache/stormcrawler/parse/filter/CommaSeparatedToMultivaluedMetadata.java[CommaSeparatedToMultivaluedMetadata] rewrites single metadata values containing comma-separated entries into multiple values for the same key, useful for keyword tags. The number of values produced from a single entry is bounded by the optional `maxTokens` parameter (128 by default), so that a page cannot generate an unbounded amount of metadata. * **DebugParseFilter** – link:https://github.com/apache/stormcrawler/blob/main/core/src/main/java/org/apache/stormcrawler/parse/filter/DebugParseFilter.java[DebugParseFilter] dumps an XML representation of the DOM structure to a temporary file. * **DomainParseFilter** – link:https://github.com/apache/stormcrawler/blob/main/core/src/main/java/org/apache/stormcrawler/parse/filter/DomainParseFilter.java[DomainParseFilter] stores the domain or host name in the metadata for later indexing. -* **LDJsonParseFilter** – link:https://github.com/apache/stormcrawler/blob/main/core/src/main/java/org/apache/stormcrawler/parse/filter/LDJsonParseFilter.java[LDJsonParseFilter] extracts data from JSON-LD representations. -* **LinkParseFilter** – link:https://github.com/apache/stormcrawler/blob/main/core/src/main/java/org/apache/stormcrawler/parse/filter/LinkParseFilter.java[LinkParseFilter] extracts outlinks from documents using XPath expressions defined in the configuration. * **MD5SignatureParseFilter** – link:https://github.com/apache/stormcrawler/blob/main/core/src/main/java/org/apache/stormcrawler/parse/filter/MD5SignatureParseFilter.java[MD5SignatureParseFilter] generates an MD5 signature of a document based on the binary content, text, or URL (as a last resort). It can be combined with content filtering to exclude boilerplate text. * **MimeTypeNormalization** – link:https://github.com/apache/stormcrawler/blob/main/core/src/main/java/org/apache/stormcrawler/parse/filter/MimeTypeNormalization.java[MimeTypeNormalization] converts server-reported or inferred mime-type values into human-readable values such as _pdf_, _html_, or _image_ and stores them in the metadata, useful for indexing and filtering search results. * **XPathFilter** – link:https://github.com/apache/stormcrawler/blob/main/core/src/main/java/org/apache/stormcrawler/parse/filter/XPathFilter.java[XPathFilter] allows extraction of data using XPath expressions and storing them in the Metadata object. +For HTML documents parsed by the JSoupParserBolt, extracting data from JSON-LD or outlinks with XPath expressions is done with xref:jsoupfilters[JSoupFilters]. + You can also implement custom ParseFilters to extend the capabilities of the parsing pipeline. For example, you might create a filter to enrich a document's metadata with additional information, such as language detection, sentiment analysis, named entity recognition, or custom tags extracted from the content. Custom filters can also modify or remove outlinks, normalize text, or integrate external data sources, allowing you to tailor the crawler to your specific processing or indexing requirements. By implementing the ParseFilter interface and configuring the filter in the JSON file, your custom logic will be seamlessly executed within the parsing bolt. +[[jsoupfilters]] +==== JSoup Filters + +JSoupFilters are called by the link:https://github.com/apache/stormcrawler/blob/main/core/src/main/java/org/apache/stormcrawler/bolt/JSoupParserBolt.java[JSoupParserBolt] only. Unlike ParseFilters, they work directly on the document produced by link:https://jsoup.org/[JSoup] and do not require a DOM, which makes them the preferred option for extracting data from HTML documents. They are called before the xref:parsefilters[ParseFilters], which can therefore use the metadata and outlinks generated by the JSoupFilters. + +JSoupFilters need to implement the interface link:https://github.com/apache/stormcrawler/blob/main/core/src/main/java/org/apache/stormcrawler/parse/JSoupFilter.java[JSoupFilter]: + +[source,java] +---- +public void filter(String url, byte[] content, org.jsoup.nodes.Document doc, ParseResult parse); + +public void configure(Map stormConf, JsonNode filterParams); +---- + +The `filter` method has the same role as for the ParseFilters (see above) but gets the JSoup document instead of a DOM. The `configure` method takes the JSON object loaded by the wrapper class link:https://github.com/apache/stormcrawler/blob/main/core/src/main/java/org/apache/stormcrawler/parse/JSoupFilters.java[JSoupFilters]. + +The JSoupFilters are defined in a JSON file whose location is set with `jsoup.filters.config.file`; no JSoupFilters are used if this is not set. The archetype includes a default link:https://github.com/apache/stormcrawler/blob/main/archetype/src/main/resources/archetype-resources/src/main/resources/jsoupfilters.json[jsoupfilters.json]: + +[source,json] +---- +{ + "org.apache.stormcrawler.parse.JSoupFilters": [ + { + "class": "org.apache.stormcrawler.jsoup.XPathFilter", + "name": "XPathFilter", + "params": { + "canonical": "//*[@rel=\"canonical\"]/@href", + "parse.description": [ + "//*[@name=\"description\"]/@content", + "//*[@name=\"Description\"]/@content" + ], + "parse.title": [ + "//TITLE/allText()", + "//META[@name=\"title\"]/@content" + ], + "parse.keywords": "//META[@name=\"keywords\"]/@content" + } + }, + { + "class": "org.apache.stormcrawler.jsoup.LinkParseFilter", + "name": "LinkParseFilter", + "params": { + "pattern": "//FRAME/@src" + } + } + ] +} +---- + +JSoupFilters are executed in the order they appear in the JSON file. + +===== Provided JSoupFilters + +* **XPathFilter** – link:https://github.com/apache/stormcrawler/blob/main/core/src/main/java/org/apache/stormcrawler/jsoup/XPathFilter.java[XPathFilter] extracts data using XPath expressions and stores the values in the Metadata object, under the key used in the configuration. A key can have several expressions; they are tried in order and the values of the first one returning a result are kept. By default, the text of the matching elements is used; an expression can end with `/@attribute` to get the value of an attribute, `/html()` to get the inner HTML of the elements, or `/tidyText()` and `/allText()` for their text. Element names are lowercased, so `//TITLE` and `//title` are equivalent. +* **LinkParseFilter** – link:https://github.com/apache/stormcrawler/blob/main/core/src/main/java/org/apache/stormcrawler/jsoup/LinkParseFilter.java[LinkParseFilter] extracts additional outlinks with XPath expressions, using the same syntax as the XPathFilter, e.g. `//IMG/@src`. The URLs are resolved against the URL of the page, filtered with the xref:urlfilters[URLFilters] and get their metadata from xref:metadatatransfer[MetadataTransfer], like the outlinks extracted by the parser. +* **LDJsonParseFilter** – link:https://github.com/apache/stormcrawler/blob/main/core/src/main/java/org/apache/stormcrawler/jsoup/LDJsonParseFilter.java[LDJsonParseFilter] extracts data from the first link:https://json-ld.org/[JSON-LD] script of a page. The parameters map metadata keys to link:https://datatracker.ietf.org/doc/html/rfc6901[JSON Pointer] expressions, e.g. `"streetAddress": "/location/streetAddress"`. + [[urlfilters]] ==== URL Filters diff --git a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/parse/filter/JSONResourceWrapper.java b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/parse/filter/JSONResourceWrapper.java index 6f6d7be66..566ff7533 100644 --- a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/parse/filter/JSONResourceWrapper.java +++ b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/parse/filter/JSONResourceWrapper.java @@ -70,6 +70,11 @@ public void filter(String url, byte[] content, DocumentFragment doc, ParseResult refresher.getDelegate().filter(url, content, doc, parse); } + @Override + public boolean needsDOM() { + return refresher.getDelegate().needsDOM(); + } + @Override public void cleanup() { if (refresher != null) { diff --git a/external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/parse/filter/JSONResourceWrapperTest.java b/external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/parse/filter/JSONResourceWrapperTest.java new file mode 100644 index 000000000..0b988576d --- /dev/null +++ b/external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/parse/filter/JSONResourceWrapperTest.java @@ -0,0 +1,87 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.stormcrawler.opensearch.parse.filter; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import java.io.InputStream; +import java.util.HashMap; +import java.util.Map; +import org.apache.stormcrawler.JSONResource; +import org.apache.stormcrawler.parse.ParseFilter; +import org.apache.stormcrawler.parse.ParseResult; +import org.jetbrains.annotations.NotNull; +import org.junit.jupiter.api.Test; +import org.w3c.dom.DocumentFragment; + +class JSONResourceWrapperTest { + + private static final ObjectMapper MAPPER = new ObjectMapper(); + + /** Minimal ParseFilter + JSONResource implementation. */ + public static class StubParseFilter extends ParseFilter implements JSONResource { + + @Override + public void configure(@NotNull Map stormConf, @NotNull JsonNode params) {} + + @Override + public void filter(String url, byte[] content, DocumentFragment doc, ParseResult parse) {} + + @Override + public String getResourceFile() { + return "stub.json"; + } + + @Override + public void loadJSONResources(InputStream inputStream) {} + } + + /** ParseFilter which requires a DOM. */ + public static class DOMStubParseFilter extends StubParseFilter { + @Override + public boolean needsDOM() { + return true; + } + } + + private static boolean wrapperNeedsDOM(Class delegateClass) { + ObjectNode delegate = MAPPER.createObjectNode(); + delegate.put("class", delegateClass.getName()); + delegate.set("params", MAPPER.createObjectNode()); + ObjectNode params = MAPPER.createObjectNode(); + params.set("delegate", delegate); + + JSONResourceWrapper wrapper = new JSONResourceWrapper(); + wrapper.configure(new HashMap<>(), params); + try { + return wrapper.needsDOM(); + } finally { + wrapper.cleanup(); + } + } + + @Test + void needsDOMIsDelegated() { + assertTrue(wrapperNeedsDOM(DOMStubParseFilter.class)); + assertFalse(wrapperNeedsDOM(StubParseFilter.class)); + } +}