TIKA-4966: apply font formatting styles in docx

CHANGES.txt

@@ -1,5 +1,13 @@
 Release 4.2.0 - unreleased
 
+   * The docx parser now applies bold and italics set by a run's character style
+     (e.g. Word's built-in Strong and Emphasis), following basedOn chains; previously
+     only direct formatting was emitted. The new OfficeParserConfig option
+     includeCharacterStyleClasses wraps such runs in <span class="..."> named after
+     the style. Style-derived classes (paragraph and character) now replace every
+     character other than letters, digits, '_' and '-' with '_' and are capped at 128
+     characters, so a style name can no longer yield extra class tokens (TIKA-4966).
+
    * OpenAI-compatible VLM parser can send the output token limit as
      max_completion_tokens for models that reject max_tokens. The new
      useMaxCompletionTokens setting is opt-in; max_tokens remains the default.

tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/OfficeParserConfig.java

@@ -46,6 +46,7 @@ public class OfficeParserConfig implements Serializable {
     private int rtfEmbeddedMaxBytesInKb = 2 * 1024 * 1024; // 2 GB
 
     private boolean includeGlossary = true;
+    private boolean includeCharacterStyleClasses = false;
     private String dateOverrideFormat = null;
     private int maxOverride = 0;//ignore
 
@@ -203,6 +204,23 @@ public void setIncludeGlossary(boolean includeGlossary) {
         this.includeGlossary = includeGlossary;
     }
 
+    public boolean isIncludeCharacterStyleClasses() {
+        return includeCharacterStyleClasses;
+    }
+
+    /**
+     * Whether to wrap docx runs that carry a character style in
+     * <code>&lt;span class="..."&gt;</code>, named after the style the same way
+     * paragraph styles are.
+     * <p/>
+     * Default: <code>false</code>
+     *
+     * @param includeCharacterStyleClasses whether to emit character style classes
+     */
+    public void setIncludeCharacterStyleClasses(boolean includeCharacterStyleClasses) {
+        this.includeCharacterStyleClasses = includeCharacterStyleClasses;
+    }
+
     public boolean isIncludeMissingRows() {
         return includeMissingRows;
     }

tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/WordExtractor.java

@@ -29,6 +29,7 @@
 import java.util.Locale;
 import java.util.Map;
 import java.util.Set;
+import java.util.regex.Pattern;
 
 import org.apache.poi.hwpf.HWPFDocument;
 import org.apache.poi.hwpf.HWPFOldDocument;
@@ -78,6 +79,8 @@ public class WordExtractor extends AbstractPOIFSExtractor {
     private static final String LIST_DELIMITER = " ";
     private static final Map<String, TagAndStyle> fixedParagraphStyles = new HashMap<>();
     private static final TagAndStyle defaultParagraphStyle = new TagAndStyle("p", null);
+    private static final int MAX_STYLE_CLASS_LENGTH = 128;
+    private static final Pattern NON_STYLE_CLASS_CHARS = Pattern.compile("[^\\p{L}\\p{N}_-]");
     private static final Logger LOG = LoggerFactory.getLogger(WordExtractor.class);
 
     static {
@@ -170,14 +173,30 @@ public static TagAndStyle buildParagraphTagAndStyle(String styleName, boolean is
             // Turn it into a H1 - H6 (H7+ isn't valid!)
             tag = "h" + Math.min(num, 6);
         } else {
-            styleClass = styleName.replace(' ', '_');
-            styleClass =
-                    styleClass.substring(0, 1).toLowerCase(Locale.ROOT) + styleClass.substring(1);
+            styleClass = toStyleClass(styleName);
         }
 
         return new TagAndStyle(tag, styleClass);
     }
 
+    /**
+     * @return the XHTML class for a style name, e.g. "List Bullet" -&gt; "list_Bullet",
+     * or null for an empty name. Anything but letters, digits, '_' and '-' becomes '_',
+     * so a style name can never yield more than one class token.
+     */
+    public static String toStyleClass(String styleName) {
+        if (styleName == null || styleName.isEmpty()) {
+            return null;
+        }
+        // sanitize last: lowercasing can expand a letter into a disallowed combining mark
+        String styleClass =
+                styleName.substring(0, 1).toLowerCase(Locale.ROOT) + styleName.substring(1);
+        if (styleClass.length() > MAX_STYLE_CLASS_LENGTH) {
+            styleClass = styleClass.substring(0, MAX_STYLE_CLASS_LENGTH);
+        }
+        return NON_STYLE_CLASS_CHARS.matcher(styleClass).replaceAll("_");
+    }
+
     protected void parse(POIFSFileSystem filesystem, XHTMLContentHandler xhtml)
             throws IOException, SAXException, TikaException {
         parse(filesystem.getRoot(), xhtml);

tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/FormattingTagManager.java

@@ -44,8 +44,9 @@ class FormattingTagManager {
 
     private final XHTMLContentHandler xhtml;
 
-    // Outermost to innermost: hyperlink > bold > italic > strike > underline
+    // Outermost to innermost: hyperlink > span > bold > italic > strike > underline
     private String currentHyperlink = null;
+    private String currentSpanClass = null;
     private boolean wrapperHyperlinkActive = false;
     private boolean isBold = false;
     private boolean isItalics = false;
@@ -65,7 +66,7 @@ void openHyperlink(String url) throws SAXException {
         if (url == null) {
             return;
         }
-        closeFormattingTags();
+        closeInlineTags();
         if (currentHyperlink != null) {
             xhtml.endElement("a");
         }
@@ -79,7 +80,7 @@ void openHyperlink(String url) throws SAXException {
      */
     void closeHyperlink() throws SAXException {
         if (currentHyperlink != null && wrapperHyperlinkActive) {
-            closeFormattingTags();
+            closeInlineTags();
             xhtml.endElement("a");
             currentHyperlink = null;
             wrapperHyperlinkActive = false;
@@ -98,11 +99,19 @@ boolean isHyperlinkActive() {
      * opening and closing XHTML tags as needed to maintain proper nesting.
      */
     void applyFormatting(RunProperties runProperties) throws SAXException {
+        applyFormatting(runProperties, null);
+    }
+
+    /**
+     * As {@link #applyFormatting(RunProperties)}, additionally wrapping the run in
+     * {@code <span class="spanClass">} when spanClass is non-null.
+     */
+    void applyFormatting(RunProperties runProperties, String spanClass) throws SAXException {
         // Run-property hyperlinks only when no wrapper is active
         if (!wrapperHyperlinkActive) {
             String newHyperlink = runProperties.getHlinkClickUrl();
             if (!Objects.equals(newHyperlink, currentHyperlink)) {
-                closeFormattingTags();
+                closeInlineTags();
                 if (currentHyperlink != null) {
                     xhtml.endElement("a");
                 }
@@ -113,6 +122,14 @@ void applyFormatting(RunProperties runProperties) throws SAXException {
             }
         }
 
+        if (!Objects.equals(spanClass, currentSpanClass)) {
+            closeInlineTags();
+            if (spanClass != null) {
+                xhtml.startElement("span", "class", spanClass);
+            }
+            currentSpanClass = spanClass;
+        }
+
         if (runProperties.isBold() != isBold) {
             // Close inner tags before flipping <b>. Nesting is <b><i><s><u>
             // (outermost to innermost), so close innermost first: u, s, i.
@@ -182,14 +199,22 @@ void applyFormatting(RunProperties runProperties) throws SAXException {
      * Closes all currently open tags in proper nesting order.
      */
     void closeAll() throws SAXException {
-        closeFormattingTags();
+        closeInlineTags();
         if (currentHyperlink != null) {
             xhtml.endElement("a");
             currentHyperlink = null;
             wrapperHyperlinkActive = false;
         }
     }
 
+    private void closeInlineTags() throws SAXException {
+        closeFormattingTags();
+        if (currentSpanClass != null) {
+            xhtml.endElement("span");
+            currentSpanClass = null;
+        }
+    }
+
     private void closeFormattingTags() throws SAXException {
         if (isUnderline) {
             xhtml.endElement("u");

tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/OOXMLTikaBodyPartHandler.java

@@ -49,6 +49,8 @@ public class OOXMLTikaBodyPartHandler
     private final XWPFListManager listManager;
     private final boolean includeDeletedText;
     private final boolean includeMoveFromText;
+    private final boolean includeCharacterStyleClasses;
+    private final RunProperties effectiveRunProperties = new RunProperties();
     private final XWPFStylesShim styles;
     private final Metadata metadata;
 
@@ -95,6 +97,7 @@ public OOXMLTikaBodyPartHandler(XHTMLContentHandler xhtml, Metadata metadata) {
         this.listManager = XWPFListManager.EMPTY_LIST;
         this.includeDeletedText = false;
         this.includeMoveFromText = false;
+        this.includeCharacterStyleClasses = false;
     }
 
     public OOXMLTikaBodyPartHandler(XHTMLContentHandler xhtml, XWPFStylesShim styles,
@@ -113,6 +116,7 @@ public OOXMLTikaBodyPartHandler(XHTMLContentHandler xhtml, XWPFStylesShim styles
         this.listManager = listManager;
         this.includeDeletedText = parserConfig.isIncludeDeletedContent();
         this.includeMoveFromText = parserConfig.isIncludeMoveFromContent();
+        this.includeCharacterStyleClasses = parserConfig.isIncludeCharacterStyleClasses();
     }
 
     /**
@@ -128,10 +132,32 @@ public void setInlineBodyPartMap(OOXMLInlineBodyPartMap inlinePartMap,
 
     @Override
     public void run(RunProperties runProperties, String contents) throws SAXException {
-        formattingTags.applyFormatting(runProperties);
+        if (contents.isEmpty()) {
+            return;
+        }
+        String styleId = runProperties.getStyleID();
+        if (styleId == null || styles == null) {
+            formattingTags.applyFormatting(runProperties);
+        } else {
+            effectiveRunProperties.copyFrom(runProperties);
+            if (!runProperties.isBoldSet()) {
+                effectiveRunProperties.bold = styles.isBold(styleId);
+            }
+            if (!runProperties.isItalicsSet()) {
+                effectiveRunProperties.italics = styles.isItalics(styleId);
+            }
+            formattingTags.applyFormatting(effectiveRunProperties, getSpanClass(styleId));
+        }
         xhtml.characters(contents);
     }
 
+    private String getSpanClass(String styleId) {
+        if (!includeCharacterStyleClasses) {
+            return null;
+        }
+        return WordExtractor.toStyleClass(styles.getStyleName(styleId));
+    }
+
     @Override
     public void hyperlinkStart(String link) throws SAXException {
         formattingTags.openHyperlink(link);

tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/OOXMLWordAndPowerPointTextHandler.java

@@ -20,7 +20,6 @@
 import java.util.Date;
 import java.util.Map;
 
-import org.apache.poi.xwpf.usermodel.UnderlinePatterns;
 import org.xml.sax.Attributes;
 import org.xml.sax.SAXException;
 import org.xml.sax.helpers.DefaultHandler;
@@ -56,6 +55,7 @@ public class OOXMLWordAndPowerPointTextHandler extends DefaultHandler {
     private final static String RPR = "rPr";
     private final static String P = "p";
     private final static String P_STYLE = "pStyle";
+    private final static String R_STYLE = "rStyle";
     private final static String PPR = "pPr";
     private final static String T = "t";
     private final static String TAB = "tab";
@@ -290,6 +290,10 @@ public void startElement(String uri, String localName, String qName, Attributes
         } else if (P_STYLE.equals(localName)) {
             String styleId = atts.getValue(W_NS, "val");
             currPProperties.setStyleID(styleId);
+        } else if (R_STYLE.equals(localName)) {
+            if (inR && inRPr) {
+                currRunProperties.setStyleID(atts.getValue(W_NS, "val"));
+            }
         } else if (I.equals(localName)) { //TODO: add iCs
             //rprs don't have to be inR; ignore those that aren't
             if (inR && inRPr) {
@@ -630,11 +634,7 @@ private void handleEndOfRun() throws SAXException {
         bodyContentsHandler.run(currRunProperties, runBuffer.toString());
         inR = false;
         runBuffer.setLength(0);
-        currRunProperties.setBold(false);
-        currRunProperties.setItalics(false);
-        currRunProperties.setStrike(false);
-        currRunProperties.setUnderline(UnderlinePatterns.NONE.name());
-        currRunProperties.setHlinkClickUrl(null);
+        currRunProperties.reset();
     }
 
     @Override

tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/RunProperties.java

@@ -33,12 +33,22 @@ public class RunProperties {
     // PPTX hlinkClick hyperlink URL — set from <a:hlinkClick> inside <a:rPr>
     String hlinkClickUrl = null;
 
+    // docx character style (w:rStyle); direct b/i, once set, override it
+    String styleID = null;
+    boolean boldSet = false;
+    boolean italicsSet = false;
+
     public boolean isItalics() {
         return italics;
     }
 
     public void setItalics(boolean italics) {
         this.italics = italics;
+        this.italicsSet = true;
+    }
+
+    public boolean isItalicsSet() {
+        return italicsSet;
     }
 
     public boolean isBold() {
@@ -47,6 +57,19 @@ public boolean isBold() {
 
     public void setBold(boolean bold) {
         this.bold = bold;
+        this.boldSet = true;
+    }
+
+    public boolean isBoldSet() {
+        return boldSet;
+    }
+
+    public String getStyleID() {
+        return styleID;
+    }
+
+    public void setStyleID(String styleID) {
+        this.styleID = styleID;
     }
 
     public boolean isStrikeThrough() {
@@ -79,4 +102,26 @@ public String getHlinkClickUrl() {
     public void setHlinkClickUrl(String url) {
         this.hlinkClickUrl = url;
     }
+
+    void copyFrom(RunProperties other) {
+        italics = other.italics;
+        bold = other.bold;
+        strikeThrough = other.strikeThrough;
+        underline = other.underline;
+        hlinkClickUrl = other.hlinkClickUrl;
+        styleID = other.styleID;
+        boldSet = other.boldSet;
+        italicsSet = other.italicsSet;
+    }
+
+    public void reset() {
+        italics = false;
+        bold = false;
+        strikeThrough = false;
+        underline = UnderlinePatterns.NONE;
+        hlinkClickUrl = null;
+        styleID = null;
+        boldSet = false;
+        italicsSet = false;
+    }
 }