Terminal: grapheme clusters, with the cell layout kept by the text data

terminal/bundles/org.eclipse.terminal.control/src/org/eclipse/terminal/internal/model/Cells.java

@@ -0,0 +1,171 @@
+/*******************************************************************************
+ * Copyright (c) 2026 Contributors to the Eclipse Foundation.
+ * All rights reserved. This program and the accompanying materials
+ * are made available under the terms of the Eclipse Public License 2.0
+ * which accompanies this distribution, and is available at
+ * https://www.eclipse.org/legal/epl-2.0/
+ *
+ * SPDX-License-Identifier: EPL-2.0
+ *******************************************************************************/
+package org.eclipse.terminal.internal.model;
+
+import java.text.Normalizer;
+
+import org.eclipse.terminal.model.ITerminalTextDataReadOnly;
+
+/**
+ * The characters of a line as its cells hold them, for everything that reads
+ * them: the selection, copying, the word under the pointer, drawing and the
+ * emulator. How many cells a character takes is kept by the text data
+ * ({@link ITerminalTextDataReadOnly#getCellWidth(int, int)}); this puts the
+ * character back together from them: a surrogate pair from its two cells, a
+ * grapheme cluster from beside the line
+ * ({@link ITerminalTextDataReadOnly#getCluster(int, int)}).
+ */
+public final class Cells {
+
+	private Cells() {
+	}
+
+	/** @return the first cell of the character the cell belongs to */
+	public static int start(ITerminalTextDataReadOnly data, int line, int col) {
+		return col > 0 && data.getCellWidth(line, col) == 0 ? col - 1 : col;
+	}
+
+	/**
+	 * @return the character starting in the cell as the program wrote it: a whole
+	 *         grapheme cluster, both halves of a surrogate pair, or the one char (a
+	 *         replacement character for half a pair); "" for the second cell of a
+	 *         character and for a cell never written
+	 */
+	public static String text(ITerminalTextDataReadOnly data, int line, int col) {
+		return text(data, line, col, false);
+	}
+
+	/**
+	 * @param pastMargin whether the text goes on past the margin, where a narrowed
+	 *            screen keeps what it no longer shows: a pair the margin cuts is whole
+	 *            there
+	 */
+	private static String text(ITerminalTextDataReadOnly data, int line, int col, boolean pastMargin) {
+		if (data.getCellWidth(line, col) == 0) {
+			return ""; //$NON-NLS-1$
+		}
+		String cluster = data.getCluster(line, col);
+		if (cluster != null) {
+			return cluster;
+		}
+		char c = charAt(data, line, col);
+		if (c == '\000') {
+			return ""; //$NON-NLS-1$
+		}
+		if (Character.isHighSurrogate(c) && (data.getCellWidth(line, col) == 2
+				|| pastMargin && Character.isLowSurrogate(charAt(data, line, col + 1)))) {
+			return new String(new char[] { c, charAt(data, line, col + 1) });
+		}
+		// half a pair, its other half cut off by a narrowed screen, is no character
+		return Character.isSurrogate(c) ? "\uFFFD" : String.valueOf(c); //$NON-NLS-1$
+	}
+
+	/**
+	 * @return the char in the cell, also one a narrowed screen keeps past its margin
+	 *         (0 where there is none)
+	 */
+	private static char charAt(ITerminalTextDataReadOnly data, int line, int col) {
+		if (col < data.getWidth()) {
+			return data.getChar(line, col);
+		}
+		char[] chars = data.getChars(line);
+		return chars != null && col < chars.length ? chars[col] : 0;
+	}
+
+	/**
+	 * The text of a whole row, also what a narrowed screen keeps past its margin and
+	 * shows again when it widens.
+	 */
+	public static String row(ITerminalTextDataReadOnly data, int line, int from) {
+		char[] chars = data.getChars(line);
+		int to = Math.max(data.getWidth(), chars == null ? 0 : chars.length);
+		return text(data, line, from, to, to > data.getWidth());
+	}
+
+	/**
+	 * @return the character starting in the cell as it is drawn: a cluster with its
+	 *         marks composed where they have a composed form (NFC)
+	 */
+	public static String drawn(ITerminalTextDataReadOnly data, int line, int col) {
+		String text = text(data, line, col);
+		return text.length() > 1 ? Normalizer.normalize(text, Normalizer.Form.NFC) : text;
+	}
+
+	/**
+	 * The characters starting in the cells from <code>from</code> to
+	 * <code>to</code> as they are drawn, a cell never written as a space: text that
+	 * lines up cell for cell as long as each character is drawn as wide as the cells
+	 * it takes.
+	 *
+	 * @param cells the chars of those cells, as
+	 *            {@link ITerminalTextDataReadOnly#getLineSegments(int, int, int)}
+	 *            has them
+	 * @return the text, or null when a character the range starts or ends in is cut
+	 *         by it
+	 */
+	public static String drawn(ITerminalTextDataReadOnly data, int line, int from, String cells) {
+		if (data.isSimpleLine(line)) {
+			return cells.replace('\000', ' '); // the chars are the text
+		}
+		int to = from + cells.length();
+		if (data.getCellWidth(line, from) == 0) {
+			return null;
+		}
+		StringBuilder drawn = new StringBuilder(cells.length());
+		for (int col = from; col < to;) {
+			int width = Math.max(data.getCellWidth(line, col), 1); // always on, whatever the cells hold
+			if (col + width > to) {
+				return null;
+			}
+			String cluster = data.getCluster(line, col);
+			char c = cells.charAt(col - from);
+			if (cluster != null) {
+				drawn.append(Normalizer.normalize(cluster, Normalizer.Form.NFC));
+			} else if (c == '\000') {
+				drawn.append(' ');
+			} else {
+				drawn.append(c);
+				if (width == 2 && Character.isHighSurrogate(c)) {
+					drawn.append(cells.charAt(col - from + 1));
+				}
+			}
+			col += width;
+		}
+		return drawn.toString();
+	}
+
+	/**
+	 * The text of the characters starting in the cells from <code>from</code> to
+	 * <code>to</code>: a cell never written stands for a space, and those at the
+	 * end of the range are left out.
+	 */
+	public static String text(ITerminalTextDataReadOnly data, int line, int from, int to) {
+		return text(data, line, from, to, false);
+	}
+
+	private static String text(ITerminalTextDataReadOnly data, int line, int from, int to, boolean pastMargin) {
+		StringBuilder text = new StringBuilder(to - from);
+		int written = 0;
+		for (int col = from; col < to; col++) {
+			if (data.getCellWidth(line, col) == 0) {
+				continue;
+			}
+			String c = text(data, line, col, pastMargin);
+			if (c.isEmpty()) {
+				text.append(' ');
+			} else {
+				text.append(c);
+				written = text.length();
+			}
+		}
+		text.setLength(written);
+		return text.toString();
+	}
+}

terminal/bundles/org.eclipse.terminal.control/src/org/eclipse/terminal/internal/model/CharWidth.java

@@ -62,45 +62,53 @@ private CharWidth() {
 			0x1FAEF, 0x1FAF8, 0x20000, 0x2FFFD, 0x30000, 0x3FFFD
 	};
 
+	/** the width of every character of the BMP, looked up rather than worked out for each one written */
+	private static final byte[] BMP = new byte[0x10000];
+	static {
+		for (int c = 0; c < BMP.length; c++) {
+			BMP[c] = (byte) compute(c);
+		}
+	}
+
 	/** @return 0 for combining and non-printing, 2 for East Asian W/F, else 1 */
 	public static int of(int codePoint) {
+		return codePoint < BMP.length ? BMP[codePoint] : compute(codePoint);
+	}
+
+	private static int compute(int codePoint) {
 		if (codePoint < 0x0080) {
 			return codePoint < 0x20 || codePoint == 0x7F ? 0 : 1;
 		}
 		return isZeroWidth(codePoint) ? 0 : isWide(codePoint) ? 2 : 1;
 	}
 
-	/** @return the total display width of {@code s} */
-	public static int ofString(String s) {
-		return s.codePoints().map(CharWidth::of).sum();
+	/**
+	 * Cells a code point takes by itself: its width, beyond the BMP as well. A
+	 * narrow one there, such as a mathematical letter, takes one cell, which keeps
+	 * it as a cluster beside the line since its two chars cannot share the cell.
+	 */
+	public static int cells(int codePoint) {
+		return of(codePoint);
 	}
 
 	/**
-	 * A {@code '\000'} means one of two things in a line of cells: the filler that
-	 * a wide character puts in the cell it also covers, which carries no text of
-	 * its own, or a cell that was never written or has been erased, which reads as
-	 * a space. Only a wide character of the BMP leaves a filler: one beyond the
-	 * BMP already covers its two cells with its two surrogates, so a null after it
-	 * is an ordinary empty cell.
-	 *
-	 * @return whether the cell at {@code index} is the filler of the character
-	 *         before it
+	 * Cells a grapheme cluster takes, summed as Windows Terminal sums them: what its
+	 * code points take, VS16 (emoji presentation) counting as two, but at least one
+	 * and no more than two.
 	 */
-	public static boolean isFiller(CharSequence text, int index) {
-		if (index == 0 || text.charAt(index) != '\000') {
-			return false;
+	public static int clusterCells(String cluster) {
+		int cells = 0;
+		for (int i = 0; i < cluster.length();) {
+			int codePoint = cluster.codePointAt(i);
+			cells += codePoint == 0xFE0F ? 2 : cells(codePoint);
+			i += Character.charCount(codePoint);
 		}
-		char before = text.charAt(index - 1);
-		return !Character.isSurrogate(before) && of(before) == 2;
+		return Math.max(1, Math.min(2, cells));
 	}
 
-	/**
-	 * @return whether the cell at {@code index} is the second of the two cells of
-	 *         a wide character: its filler, or the low half of a surrogate pair
-	 */
-	public static boolean isSecondCell(CharSequence text, int index) {
-		return isFiller(text, index) || index > 0 && Character.isLowSurrogate(text.charAt(index))
-				&& Character.isHighSurrogate(text.charAt(index - 1));
+	/** @return the total display width of {@code s} */
+	public static int ofString(String s) {
+		return s.codePoints().map(CharWidth::of).sum();
 	}
 
 	private static boolean isZeroWidth(int codePoint) {

terminal/bundles/org.eclipse.terminal.control/src/org/eclipse/terminal/internal/model/SynchronizedTerminalTextData.java

@@ -162,4 +162,24 @@ synchronized public void setWrappedLine(int line) {
 	synchronized public void clearWrappedLine(int line) {
 		fData.clearWrappedLine(line);
 	}
+
+	@Override
+	synchronized public String getCluster(int line, int column) {
+		return fData.getCluster(line, column);
+	}
+
+	@Override
+	synchronized public int getCellWidth(int line, int column) {
+		return fData.getCellWidth(line, column);
+	}
+
+	@Override
+	synchronized public boolean isSimpleLine(int line) {
+		return fData.isSimpleLine(line);
+	}
+
+	@Override
+	synchronized public void setCluster(int line, int column, String cluster) {
+		fData.setCluster(line, column, cluster);
+	}
 }

terminal/bundles/org.eclipse.terminal.control/src/org/eclipse/terminal/internal/model/TerminalTextData.java

@@ -348,4 +348,25 @@ public void clearWrappedLine(int line) {
 		fData.clearWrappedLine(line);
 		sendLineChangedToSnapshots(line);
 	}
+
+	@Override
+	public String getCluster(int line, int column) {
+		return fData.getCluster(line, column);
+	}
+
+	@Override
+	public int getCellWidth(int line, int column) {
+		return fData.getCellWidth(line, column);
+	}
+
+	@Override
+	public boolean isSimpleLine(int line) {
+		return fData.isSimpleLine(line);
+	}
+
+	@Override
+	public void setCluster(int line, int column, String cluster) {
+		fData.setCluster(line, column, cluster);
+		sendLineChangedToSnapshots(line);
+	}
 }

terminal/bundles/org.eclipse.terminal.control/src/org/eclipse/terminal/internal/model/TerminalTextDataFastScroll.java

@@ -302,6 +302,30 @@ private void validateLineParameter(int line) {
 		}
 	}
 
+	@Override
+	public String getCluster(int line, int column) {
+		validateLineParameter(line);
+		return fData.getCluster(getPositionOfLine(line), column);
+	}
+
+	@Override
+	public int getCellWidth(int line, int column) {
+		validateLineParameter(line);
+		return fData.getCellWidth(getPositionOfLine(line), column);
+	}
+
+	@Override
+	public boolean isSimpleLine(int line) {
+		validateLineParameter(line);
+		return fData.isSimpleLine(getPositionOfLine(line));
+	}
+
+	@Override
+	public void setCluster(int line, int column, String cluster) {
+		validateLineParameter(line);
+		fData.setCluster(getPositionOfLine(line), column, cluster);
+	}
+
 	@Override
 	public void setWrappedLine(int line) {
 		validateLineParameter(line);

terminal/bundles/org.eclipse.terminal.control/src/org/eclipse/terminal/internal/model/TerminalTextDataSnapshot.java

@@ -318,4 +318,19 @@ public ITerminalTextData getTerminalTextData() {
 	public boolean isWrappedLine(int line) {
 		return fSnapshot.isWrappedLine(line);
 	}
+
+	@Override
+	public String getCluster(int line, int column) {
+		return fSnapshot.getCluster(line, column);
+	}
+
+	@Override
+	public int getCellWidth(int line, int column) {
+		return fSnapshot.getCellWidth(line, column);
+	}
+
+	@Override
+	public boolean isSimpleLine(int line) {
+		return fSnapshot.isSimpleLine(line);
+	}
 }