#158 remove the DOM-based LDJsonParseFilter and LinkParseFilter

core/src/main/java/org/apache/stormcrawler/jsoup/LinkParseFilter.java

@@ -39,7 +39,7 @@
  *
  * <pre>{@code
  * {
- *   "class": "org.apache.stormcrawler.parse.filter.LinkParseFilter",
+ *   "class": "org.apache.stormcrawler.jsoup.LinkParseFilter",
  *   "name": "LinkParseFilter",
  *   "params": {
  *     "pattern": "//IMG/@src",

core/src/main/java/org/apache/stormcrawler/util/RobotsTags.java

@@ -17,16 +17,7 @@
 
 package org.apache.stormcrawler.util;
 
-import javax.xml.xpath.XPath;
-import javax.xml.xpath.XPathConstants;
-import javax.xml.xpath.XPathExpression;
-import javax.xml.xpath.XPathExpressionException;
-import javax.xml.xpath.XPathFactory;
 import org.apache.stormcrawler.Metadata;
-import org.w3c.dom.DocumentFragment;
-import org.w3c.dom.NamedNodeMap;
-import org.w3c.dom.Node;
-import org.w3c.dom.NodeList;
 
 /**
  * Normalises the robots instructions provided by the HTML meta tags or the HTTP X-Robots-Tag
@@ -52,17 +43,6 @@ public class RobotsTags {
 
     private boolean noCache = false;
 
-    private static final XPathExpression expression;
-
-    static {
-        XPath xpath = XPathFactory.newInstance().newXPath();
-        try {
-            expression = xpath.compile("/HTML/*/META");
-        } catch (XPathExpressionException e) {
-            throw new RuntimeException(e);
-        }
-    }
-
     /** Get the values from the fetch metadata. */
     public RobotsTags(Metadata metadata, String protocolMDprefix) {
         // HTTP headers
@@ -80,46 +60,6 @@ public RobotsTags(Metadata metadata, String protocolMDprefix) {
 
     public RobotsTags() {}
 
-    // set the values based on the meta tags
-    // HTML tags
-    // <meta name="robots" content="noarchive, nofollow"/>
-    // called by the parser bolts
-    public void extractMetaTags(DocumentFragment doc) throws XPathExpressionException {
-        NodeList nodes = (NodeList) expression.evaluate(doc, XPathConstants.NODESET);
-        if (nodes == null) {
-            return;
-        }
-        int numNodes = nodes.getLength();
-        for (int i = 0; i < numNodes; i++) {
-            Node n = (Node) nodes.item(i);
-            // iterate on the attributes
-            // and check that it has name=robots and content
-            // whatever the case is
-            boolean isRobots = false;
-            String content = null;
-            NamedNodeMap attrs = n.getAttributes();
-            for (int att = 0; att < attrs.getLength(); att++) {
-                Node keyval = attrs.item(att);
-                if ("name".equalsIgnoreCase(keyval.getNodeName())
-                        && "robots".equalsIgnoreCase(keyval.getNodeValue())) {
-                    isRobots = true;
-                    continue;
-                }
-                if ("content".equalsIgnoreCase(keyval.getNodeName())) {
-                    content = keyval.getNodeValue();
-                    continue;
-                }
-            }
-
-            if (isRobots && content != null) {
-                // got a value - split it
-                String[] vals = content.split(" *, *");
-                parseValues(vals);
-                return;
-            }
-        }
-    }
-
     /** Extracts meta tags based on the value of the content attribute. */
     public void extractMetaTags(String content) {
         if (content == null) {

core/src/test/java/org/apache/stormcrawler/parse/filter/XPathFilterTest.java

@@ -64,16 +64,4 @@ void testScriptExtraction() throws IOException {
         Assertions.assertEquals("", scripts[0].trim());
         Assertions.assertTrue(scripts[1].contains("_paq"));
     }
-
-    @Test
-    void testLDJsonExtraction() throws IOException {
-        prepareParserBolt("test.parsefilters.json");
-        parse("https://stormcrawler.apache.org", "stormcrawler.apache.org.html");
-        Assertions.assertEquals(1, output.getEmitted().size());
-        List<Object> parsedTuple = output.getEmitted().get(0);
-        Metadata metadata = (Metadata) parsedTuple.get(2);
-        Assertions.assertNotNull(metadata);
-        String[] scripts = metadata.getValues("streetAddress");
-        Assertions.assertNotNull(scripts);
-    }
 }

core/src/test/resources/test.parsefilters.json

@@ -11,13 +11,6 @@
         "title": "//TITLE"
       }
     },
-    {
-      "class": "org.apache.stormcrawler.parse.filter.LDJsonParseFilter",
-      "name": "LDJsonParseFilter",
-      "params": {
-        "streetAddress": "/location/streetAddress"
-       }
-    },
     {
       "class": "org.apache.stormcrawler.parse.filter.CommaSeparatedToMultivaluedMetadata",
       "name": "CommaSeparatedToMultivaluedMetadata",

docs/src/main/asciidoc/architecture.adoc

@@ -76,6 +76,8 @@ processed.
 Parse Filters are applied within the parsing bolts, following parsing by
 `SiteMapParser`, `JSoupParser`, or `ParserBolt` (Tika). They can modify extracted text,
 metadata, and links before the content is indexed or new URLs are emitted.
+For HTML documents, `JSoupParser` also applies JSoup Filters, which work directly
+on the JSoup document and do not require a DOM.
 
 Common Parse Filters include:
 

docs/src/main/asciidoc/configuration.adoc

@@ -394,6 +394,7 @@ Configures parsing of fetched text and the handling of discovered URIs
 | detect.mimetype | true | Enable MIME type detection during parsing.
 | feed.filter.hours.since.published | -1 | Discard feeds older than value hours.
 | feed.sniffContent | false | Try to detect feeds automatically.
+| jsoup.filters.config.file | - | Path to JSON config defining the JSoupFilters used by JSoupParserBolt. See link:https://github.com/apache/stormcrawler/blob/main/archetype/src/main/resources/archetype-resources/src/main/resources/jsoupfilters.json[default jsoupfilters.json].
 | jsoup.treat.non.html.as.error | true | If true, non-HTML content is treated as an error by JSoupParserBolt. Note that `text/plain` is always handled directly (the content is used as the text) and never treated as an error. Its stored text honors `textextractor.no.text` and is capped at `textextractor.skip.after`; the raw fetched size is bounded by `http.content.limit`.
 | parsefilters.config.file | parsefilters.json | Path to JSON config defining ParseFilters. See link:https://github.com/apache/stormcrawler/blob/main/archetype/src/main/resources/archetype-resources/src/main/resources/parsefilters.json[default parsefilters.json].
 | parser.emitOutlinks | true | Emit discovered links as DISCOVERED tuples.