Skip to content

Commit 8dcfd0c

Browse files
committed
Highlight matching delimiters in the REPL
1 parent 5503a5e commit 8dcfd0c

11 files changed

Lines changed: 848 additions & 37 deletions

File tree

‎Doc/tutorial/appendix.rst‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,11 @@ control capabilities.
1717
Since Python 3.13, a new interactive shell is used by default.
1818
This one supports color, multiline editing, history browsing, and
1919
paste mode. To disable color, see :ref:`using-on-controlling-color` for
20-
details. Function keys provide some additional functionality.
20+
details. When the cursor touches a parenthesis, bracket, brace, or string
21+
quote, its matching delimiter is highlighted. This includes triple quotes
22+
and expressions in formatted and template strings. Delimiters within
23+
string contents and comments are ignored. Function keys provide some
24+
additional functionality.
2125
:kbd:`F1` enters the interactive help browser :mod:`pydoc`.
2226
:kbd:`F2` allows for browsing command-line history with neither output nor the
2327
:term:`>>>` and :term:`...` prompts. :kbd:`F3` enters "paste mode", which

‎Lib/_colorize.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -412,6 +412,7 @@ class Syntax(ThemeSection):
412412
number: str = ANSIColors.YELLOW
413413
op: str = ANSIColors.RESET
414414
definition: str = ANSIColors.BOLD
415+
matching_delimiter: str = ANSIColors.INTENSE_BACKGROUND_BLACK
415416
soft_keyword: str = ANSIColors.BOLD_BLUE
416417
reset: str = ANSIColors.RESET
417418

‎Lib/_pyrepl/reader.py‎

Lines changed: 75 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,10 @@
3737
from .layout import LayoutMap, LayoutResult, LayoutRow, WrappedRow, layout_content_lines
3838
from .layout import wrapped_row_end
3939
from .render import RenderCell, RenderLine, RenderedScreen, ScreenOverlay
40-
from .utils import ANSI_ESCAPE_SEQUENCE, ColorSpan, THEME, StyleRef, gen_colors
40+
from .utils import (
41+
ANSI_ESCAPE_SEQUENCE, ColorSpan, THEME, StyleRef, TokenizedBuffer,
42+
find_matching_delimiter, gen_colors,
43+
)
4144
from .trace import trace
4245

4346

@@ -304,8 +307,10 @@ class Reader:
304307
cxy: CursorXY = field(init=False)
305308
lxy: CursorXY = field(init=False)
306309
scheduled_commands: list[CommandName] = field(default_factory=list)
310+
_matching_delimiter: tuple[int, int] | None = field(init=False)
311+
_tokenized_buffer: TokenizedBuffer | None = field(init=False)
307312
can_colorize: bool = False
308-
gen_colors: Callable[[str], Iterator[ColorSpan]] = gen_colors
313+
gen_colors: Callable[[TokenizedBuffer], Iterator[ColorSpan]] = gen_colors
309314
threading_hook: Callback | None = None
310315
invalidation: RefreshInvalidation = field(init=False)
311316

@@ -386,6 +391,8 @@ def __post_init__(self) -> None:
386391
self.input_trans = input.KeymapTranslator(
387392
self.keymap, invalid_cls="invalid-key", character_cls="self-insert"
388393
)
394+
self._matching_delimiter = None
395+
self._tokenized_buffer = None
389396
self.layout = LayoutMap.empty()
390397
self.cxy = self.pos2xy()
391398
self.lxy = (self.pos, 0)
@@ -429,9 +436,8 @@ def calc_screen(self) -> RenderedScreen:
429436
self,
430437
reuse_full=True,
431438
)
432-
assert not self.last_refresh_cache.line_end_offsets or (
433-
self.last_refresh_cache.line_end_offsets[-1] >= len(self.buffer)
434-
), "Buffer modified without invalidate_buffer() call"
439+
# A short screen can cache only a prefix of the buffer.
440+
# get_cached_location() then falls back to a full rebuild.
435441
else:
436442
offset, num_common_lines = self.last_refresh_cache.get_cached_location(
437443
self,
@@ -536,7 +542,8 @@ def _build_content_lines(
536542
prompt_from_cache: bool,
537543
) -> tuple[ContentLine, ...]:
538544
if self.can_colorize:
539-
colors = list(self.gen_colors(self.get_unicode()))
545+
text = self.get_unicode()
546+
colors = list(self.gen_colors(self._get_tokenized_buffer(text)))
540547
else:
541548
colors = None
542549
trace("colors = {colors}", colors=colors)
@@ -602,8 +609,59 @@ def _render_message_lines(self) -> tuple[RenderLine, ...]:
602609
def get_screen_overlays(self) -> tuple[ScreenOverlay, ...]:
603610
return ()
604611

612+
def _get_tokenized_buffer(self, text: str) -> TokenizedBuffer:
613+
if self._tokenized_buffer is None or self._tokenized_buffer.source != text:
614+
self._tokenized_buffer = TokenizedBuffer.from_buffer(text)
615+
return self._tokenized_buffer
616+
617+
def _get_matching_delimiter(self) -> tuple[int, int] | None:
618+
if not self.can_colorize or self.finished:
619+
return None
620+
if not any(
621+
0 <= i < len(self.buffer) and self.buffer[i] in "()[]{}\"'"
622+
for i in (self.pos, self.pos - 1)
623+
):
624+
return None
625+
text = self.get_unicode()
626+
return find_matching_delimiter(
627+
self._get_tokenized_buffer(text), self.pos
628+
)
629+
630+
def _matching_delimiter_overlays(
631+
self, base_screen: RenderedScreen
632+
) -> tuple[ScreenOverlay, ...]:
633+
if (
634+
self._matching_delimiter is None
635+
or not self.last_refresh_cache.line_end_offsets
636+
):
637+
return ()
638+
639+
changed_rows: dict[int, list[RenderCell]] = {}
640+
start, end = self._matching_delimiter
641+
for offset in range(start, end):
642+
# pos_to_xy() clamps positions beyond the cached layout.
643+
if offset >= self.last_refresh_cache.line_end_offsets[-1]:
644+
break
645+
# The delimiter's end disambiguates a wrap boundary.
646+
x, y = self.layout.pos_to_xy(offset + 1)
647+
cells = changed_rows.setdefault(y, list(base_screen.lines[y].cells))
648+
column = 0
649+
for i, cell in enumerate(cells):
650+
if column == x - 1 and cell.width:
651+
cells[i] = replace(
652+
cell, style=StyleRef.from_tag("matching_delimiter")
653+
)
654+
break
655+
column += cell.width
656+
return tuple(
657+
ScreenOverlay(y, (RenderLine.from_cells(cells),))
658+
for y, cells in changed_rows.items()
659+
)
660+
605661
def compose_rendered_screen(self, base_screen: RenderedScreen) -> RenderedScreen:
606-
overlays = list(self.get_screen_overlays())
662+
self._matching_delimiter = self._get_matching_delimiter()
663+
overlays = list(self._matching_delimiter_overlays(base_screen))
664+
overlays.extend(self.get_screen_overlays())
607665
message_lines = self._render_message_lines()
608666
if message_lines:
609667
overlays.append(ScreenOverlay(len(base_screen.lines), message_lines))
@@ -824,8 +882,10 @@ def prepare(self) -> None:
824882
self.console.prepare()
825883
self.arg = None
826884
self.finished = False
885+
self._tokenized_buffer = None
827886
del self.buffer[:]
828887
self.pos = 0
888+
self._matching_delimiter = None
829889
self.layout = LayoutMap.empty()
830890
self.cxy = self.pos2xy()
831891
self.lxy = (self.pos, 0)
@@ -850,6 +910,7 @@ def last_command_is(self, cls: CommandClass) -> bool:
850910

851911
def restore(self) -> None:
852912
"""Clean up after a run."""
913+
self._tokenized_buffer = None
853914
self.console.restore()
854915

855916
@contextmanager
@@ -892,6 +953,11 @@ def error(self, msg: str = "none") -> None:
892953
self.invalidate_message()
893954

894955
def update_screen(self) -> None:
956+
if (
957+
not self.invalidation.needs_screen_refresh
958+
and self._get_matching_delimiter() != self._matching_delimiter
959+
):
960+
self.invalidate_overlay()
895961
if self.invalidation.is_cursor_only:
896962
self.update_cursor()
897963
self.clear_invalidation()
@@ -939,15 +1005,16 @@ def do_cmd(self, cmd: CommandInput) -> None:
9391005
and not self.invalidation.is_cursor_only
9401006
):
9411007
self.invalidate_cursor()
1008+
self.finished = bool(command.finish)
9421009
self.update_screen()
9431010

9441011
if command_type is not commands.digit_arg:
9451012
self.last_command = command_type
9461013

947-
self.finished = bool(command.finish)
9481014
if self.finished:
9491015
self.console.finish()
9501016
self.finish()
1017+
self._tokenized_buffer = None
9511018

9521019
def run_hooks(self) -> None:
9531020
threading_hook = self.threading_hook

‎Lib/_pyrepl/simple_interact.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -159,6 +159,7 @@ def maybe_run_command(statement: str) -> bool:
159159
r.cmpltn_reset()
160160
if r.input_trans is r.isearch_trans:
161161
r.do_cmd(("isearch-end", [""]))
162+
r.finished = True
162163
r.pos = len(r.get_unicode())
163164
r.invalidate_full()
164165
r.refresh()

‎Lib/_pyrepl/utils.py‎

Lines changed: 116 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@
88
import _colorize
99

1010
from collections import deque
11-
from collections.abc import Iterable, Iterator
11+
from collections.abc import Iterable, Iterator, Sequence
1212
from dataclasses import dataclass
1313
from io import StringIO
1414
from re import Match
@@ -46,7 +46,7 @@ def from_re(cls, m: Match[str], group: int | str) -> Self:
4646
return cls(re_span[0], re_span[1] - 1)
4747

4848
@classmethod
49-
def from_token(cls, token: TI, line_len: list[int]) -> Self:
49+
def from_token(cls, token: TI, line_len: Sequence[int]) -> Self:
5050
end_offset = -1
5151
if (token.type in {T.FSTRING_MIDDLE, T.TSTRING_MIDDLE}
5252
and token.string.endswith(("{", "}"))):
@@ -116,20 +116,120 @@ def unbracket(s: str, including_content: bool = False) -> str:
116116
return s.translate(ZERO_WIDTH_TRANS)
117117

118118

119-
def gen_colors(buffer: str) -> Iterator[ColorSpan]:
119+
@dataclass(frozen=True)
120+
class TokenizedBuffer:
121+
"""Replay tokens before reproducing any terminal tokenizer error."""
122+
123+
source: str
124+
tokens: tuple[TI, ...]
125+
line_lengths: tuple[int, ...]
126+
terminal_error: tuple[type[SyntaxError] | type[tokenize.TokenError], tuple] | None
127+
128+
@classmethod
129+
def from_buffer(cls, source: str) -> Self:
130+
sio = StringIO(source)
131+
line_lengths = [0] + [len(line) for line in sio.readlines()]
132+
for i in range(1, len(line_lengths)):
133+
line_lengths[i] += line_lengths[i - 1]
134+
135+
sio.seek(0)
136+
token_generator = tokenize.generate_tokens(sio.readline)
137+
tokens = []
138+
terminal_error = None
139+
try:
140+
tokens.extend(token_generator)
141+
except (SyntaxError, tokenize.TokenError) as exc:
142+
terminal_error = (type(exc), exc.args)
143+
return cls(source, tuple(tokens), tuple(line_lengths), terminal_error)
144+
145+
def replay(self) -> Iterator[TI]:
146+
yield from self.tokens
147+
if self.terminal_error is not None:
148+
error_type, args = self.terminal_error
149+
raise error_type(*args)
150+
151+
152+
def find_matching_delimiter(
153+
buffer: TokenizedBuffer, pos: int
154+
) -> tuple[int, int] | None:
155+
"""Return the half-open span of the partner touching the cursor."""
156+
source = buffer.source
157+
# A delimiter under the cursor takes precedence, even inside literal data.
158+
candidate = next((
159+
i for i in (pos, pos - 1)
160+
if 0 <= i < len(source) and source[i] in "()[]{}\"'"
161+
), None)
162+
if candidate is None:
163+
return None
164+
165+
gen, line_lengths = buffer.replay(), buffer.line_lengths
166+
try:
167+
return _find_matching_delimiter_from_token_stream(
168+
gen, line_lengths, candidate
169+
)
170+
except SyntaxError, tokenize.TokenError:
171+
# Incomplete input can still contain complete pairs.
172+
return None
173+
174+
175+
def _find_matching_delimiter_from_token_stream(
176+
token_generator: Iterator[TI],
177+
line_lengths: Sequence[int],
178+
candidate: int,
179+
) -> tuple[int, int] | None:
180+
brackets: list[tuple[str, int]] = []
181+
strings: list[tuple[int, int]] = []
182+
closing = {")": "(", "]": "[", "}": "{"}
183+
for token in token_generator:
184+
if token.type == T.ERRORTOKEN and token.string in {"\"", "'"}:
185+
# An unfinished short string can expose apparent operators.
186+
break
187+
offset = line_lengths[token.start[0] - 1] + token.start[1]
188+
pair = None
189+
match token.type:
190+
case T.STRING | T.FSTRING_START | T.TSTRING_START:
191+
# Token boundaries already account for prefixes and escapes.
192+
quote_index = next(
193+
i for i, c in enumerate(token.string) if c in "\"'"
194+
)
195+
delimiter = token.string[quote_index]
196+
width = 3 if token.string.startswith(delimiter * 3, quote_index) else 1
197+
opening = (offset + quote_index, offset + quote_index + width)
198+
if token.type == T.STRING:
199+
end = line_lengths[token.end[0] - 1] + token.end[1]
200+
pair = (opening, (end - width, end))
201+
else:
202+
strings.append(opening)
203+
case T.FSTRING_END | T.TSTRING_END:
204+
opening = strings.pop()
205+
pair = (opening, (offset, offset + len(token.string)))
206+
case T.OP if token.string in "()[]{}":
207+
if token.string in "([{":
208+
brackets.append((token.string, offset))
209+
elif brackets and brackets[-1][0] == closing[token.string]:
210+
_, opening_offset = brackets.pop()
211+
pair = (
212+
(opening_offset, opening_offset + 1), (offset, offset + 1)
213+
)
214+
else:
215+
# Do not pair across an incorrectly nested delimiter.
216+
brackets.clear()
217+
if pair is not None:
218+
opening, ending = pair
219+
if opening[0] <= candidate < opening[1]:
220+
return ending
221+
if ending[0] <= candidate < ending[1]:
222+
return opening
223+
return None
224+
225+
226+
def gen_colors(buffer: TokenizedBuffer) -> Iterator[ColorSpan]:
120227
"""Returns a list of index spans to color using the given color tag.
121228
122229
The input `buffer` should be a valid start of a Python code block, i.e.
123230
it cannot be a block starting in the middle of a multiline string.
124231
"""
125-
sio = StringIO(buffer)
126-
line_lengths = [0] + [len(line) for line in sio.readlines()]
127-
# make line_lengths cumulative
128-
for i in range(1, len(line_lengths)):
129-
line_lengths[i] += line_lengths[i-1]
130-
131-
sio.seek(0)
132-
gen = tokenize.generate_tokens(sio.readline)
232+
gen, line_lengths = buffer.replay(), buffer.line_lengths
133233
last_emitted: ColorSpan | None = None
134234
try:
135235
for color in gen_colors_from_token_stream(gen, line_lengths):
@@ -139,13 +239,13 @@ def gen_colors(buffer: str) -> Iterator[ColorSpan]:
139239
return
140240
except tokenize.TokenError as te:
141241
yield from recover_unterminated_string(
142-
te, line_lengths, last_emitted, buffer
242+
te, line_lengths, last_emitted, buffer.source
143243
)
144244

145245

146246
def recover_unterminated_string(
147247
exc: tokenize.TokenError,
148-
line_lengths: list[int],
248+
line_lengths: Sequence[int],
149249
last_emitted: ColorSpan | None,
150250
buffer: str,
151251
) -> Iterator[ColorSpan]:
@@ -186,7 +286,7 @@ def recover_unterminated_string(
186286

187287
def gen_colors_from_token_stream(
188288
token_generator: Iterator[TI],
189-
line_lengths: list[int],
289+
line_lengths: Sequence[int],
190290
) -> Iterator[ColorSpan]:
191291
token_window = prev_next_window(token_generator)
192292

@@ -398,7 +498,8 @@ def disp_str(
398498
(['a', ' ', '=', ' ', '9'], [1, 1, 1, 1, 1])
399499
400500
>>> line = "while 1:"
401-
>>> colors = list(utils.gen_colors(line))
501+
>>> tokenized = utils.TokenizedBuffer.from_buffer(line)
502+
>>> colors = list(utils.gen_colors(tokenized))
402503
>>> utils.disp_str(line, colors=colors)
403504
(['\x1b[1;34mw', 'h', 'i', 'l', 'e\x1b[0m', ' ', '1', ':'], [1, 1, 1, 1, 1, 1, 1, 1])
404505

0 commit comments

Comments
 (0)