Skip to content

Commit 95ec6ed

Browse files
committed
Simplify REPL delimiter matching tokenization
1 parent 9f4d9d6 commit 95ec6ed

6 files changed

Lines changed: 97 additions & 235 deletions

File tree

‎Lib/_pyrepl/reader.py‎

Lines changed: 4 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@
3838
from .layout import wrapped_row_end
3939
from .render import RenderCell, RenderLine, RenderedScreen, ScreenOverlay
4040
from .utils import (
41-
ANSI_ESCAPE_SEQUENCE, ColorSpan, THEME, StyleRef, TokenizedBuffer,
41+
ANSI_ESCAPE_SEQUENCE, ColorSpan, THEME, StyleRef,
4242
find_matching_delimiter, gen_colors,
4343
)
4444
from .trace import trace
@@ -308,9 +308,8 @@ class Reader:
308308
lxy: CursorXY = field(init=False)
309309
scheduled_commands: list[CommandName] = field(default_factory=list)
310310
_matching_delimiter: tuple[int, int] | None = field(init=False)
311-
_tokenized_buffer: TokenizedBuffer | None = field(init=False)
312311
can_colorize: bool = False
313-
gen_colors: Callable[[TokenizedBuffer], Iterator[ColorSpan]] = gen_colors
312+
gen_colors: Callable[[str], Iterator[ColorSpan]] = gen_colors
314313
threading_hook: Callback | None = None
315314
invalidation: RefreshInvalidation = field(init=False)
316315

@@ -392,7 +391,6 @@ def __post_init__(self) -> None:
392391
self.keymap, invalid_cls="invalid-key", character_cls="self-insert"
393392
)
394393
self._matching_delimiter = None
395-
self._tokenized_buffer = None
396394
self.layout = LayoutMap.empty()
397395
self.cxy = self.pos2xy()
398396
self.lxy = (self.pos, 0)
@@ -542,8 +540,7 @@ def _build_content_lines(
542540
prompt_from_cache: bool,
543541
) -> tuple[ContentLine, ...]:
544542
if self.can_colorize:
545-
text = self.get_unicode()
546-
colors = list(self.gen_colors(self._get_tokenized_buffer(text)))
543+
colors = list(self.gen_colors(self.get_unicode()))
547544
else:
548545
colors = None
549546
trace("colors = {colors}", colors=colors)
@@ -609,11 +606,6 @@ def _render_message_lines(self) -> tuple[RenderLine, ...]:
609606
def get_screen_overlays(self) -> tuple[ScreenOverlay, ...]:
610607
return ()
611608

612-
def _get_tokenized_buffer(self, text: str) -> TokenizedBuffer:
613-
if self._tokenized_buffer is None or self._tokenized_buffer.source != text:
614-
self._tokenized_buffer = TokenizedBuffer.from_buffer(text)
615-
return self._tokenized_buffer
616-
617609
def _get_matching_delimiter(self) -> tuple[int, int] | None:
618610
if not self.can_colorize or self.finished:
619611
return None
@@ -622,10 +614,7 @@ def _get_matching_delimiter(self) -> tuple[int, int] | None:
622614
for i in (self.pos, self.pos - 1)
623615
):
624616
return None
625-
text = self.get_unicode()
626-
return find_matching_delimiter(
627-
self._get_tokenized_buffer(text), self.pos
628-
)
617+
return find_matching_delimiter(self.get_unicode(), self.pos)
629618

630619
def _matching_delimiter_overlays(
631620
self, base_screen: RenderedScreen
@@ -882,7 +871,6 @@ def prepare(self) -> None:
882871
self.console.prepare()
883872
self.arg = None
884873
self.finished = False
885-
self._tokenized_buffer = None
886874
del self.buffer[:]
887875
self.pos = 0
888876
self._matching_delimiter = None
@@ -910,7 +898,6 @@ def last_command_is(self, cls: CommandClass) -> bool:
910898

911899
def restore(self) -> None:
912900
"""Clean up after a run."""
913-
self._tokenized_buffer = None
914901
self.console.restore()
915902

916903
@contextmanager
@@ -1014,7 +1001,6 @@ def do_cmd(self, cmd: CommandInput) -> None:
10141001
if self.finished:
10151002
self.console.finish()
10161003
self.finish()
1017-
self._tokenized_buffer = None
10181004

10191005
def run_hooks(self) -> None:
10201006
threading_hook = self.threading_hook

‎Lib/_pyrepl/utils.py‎

Lines changed: 22 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@
88
import _colorize
99

1010
from collections import deque
11-
from collections.abc import Iterable, Iterator, Sequence
11+
from collections.abc import Iterable, Iterator
1212
from dataclasses import dataclass
1313
from io import StringIO
1414
from re import Match
@@ -46,7 +46,7 @@ def from_re(cls, m: Match[str], group: int | str) -> Self:
4646
return cls(re_span[0], re_span[1] - 1)
4747

4848
@classmethod
49-
def from_token(cls, token: TI, line_len: Sequence[int]) -> Self:
49+
def from_token(cls, token: TI, line_len: list[int]) -> Self:
5050
end_offset = -1
5151
if (token.type in {T.FSTRING_MIDDLE, T.TSTRING_MIDDLE}
5252
and token.string.endswith(("{", "}"))):
@@ -116,53 +116,29 @@ def unbracket(s: str, including_content: bool = False) -> str:
116116
return s.translate(ZERO_WIDTH_TRANS)
117117

118118

119-
@dataclass(frozen=True)
120-
class TokenizedBuffer:
121-
"""Replay tokens before reproducing any terminal tokenizer error."""
119+
def _get_token_stream(buffer: str) -> tuple[Iterator[TI], list[int]]:
120+
sio = StringIO(buffer)
121+
line_lengths = [0] + [len(line) for line in sio.readlines()]
122+
# make line_lengths cumulative
123+
for i in range(1, len(line_lengths)):
124+
line_lengths[i] += line_lengths[i-1]
122125

123-
source: str
124-
tokens: tuple[TI, ...]
125-
line_lengths: tuple[int, ...]
126-
terminal_error: tuple[type[SyntaxError] | type[tokenize.TokenError], tuple] | None
126+
sio.seek(0)
127+
gen = tokenize.generate_tokens(sio.readline)
128+
return gen, line_lengths
127129

128-
@classmethod
129-
def from_buffer(cls, source: str) -> Self:
130-
sio = StringIO(source)
131-
line_lengths = [0] + [len(line) for line in sio.readlines()]
132-
for i in range(1, len(line_lengths)):
133-
line_lengths[i] += line_lengths[i - 1]
134-
135-
sio.seek(0)
136-
token_generator = tokenize.generate_tokens(sio.readline)
137-
tokens = []
138-
terminal_error = None
139-
try:
140-
tokens.extend(token_generator)
141-
except (SyntaxError, tokenize.TokenError) as exc:
142-
terminal_error = (type(exc), exc.args)
143-
return cls(source, tuple(tokens), tuple(line_lengths), terminal_error)
144-
145-
def replay(self) -> Iterator[TI]:
146-
yield from self.tokens
147-
if self.terminal_error is not None:
148-
error_type, args = self.terminal_error
149-
raise error_type(*args)
150-
151-
152-
def find_matching_delimiter(
153-
buffer: TokenizedBuffer, pos: int
154-
) -> tuple[int, int] | None:
130+
131+
def find_matching_delimiter(buffer: str, pos: int) -> tuple[int, int] | None:
155132
"""Return the half-open span of the partner touching the cursor."""
156-
source = buffer.source
157133
# A delimiter under the cursor takes precedence, even inside literal data.
158134
candidate = next((
159135
i for i in (pos, pos - 1)
160-
if 0 <= i < len(source) and source[i] in "()[]{}\"'"
136+
if 0 <= i < len(buffer) and buffer[i] in "()[]{}\"'"
161137
), None)
162138
if candidate is None:
163139
return None
164140

165-
gen, line_lengths = buffer.replay(), buffer.line_lengths
141+
gen, line_lengths = _get_token_stream(buffer)
166142
try:
167143
return _find_matching_delimiter_from_token_stream(
168144
gen, line_lengths, candidate
@@ -174,7 +150,7 @@ def find_matching_delimiter(
174150

175151
def _find_matching_delimiter_from_token_stream(
176152
token_generator: Iterator[TI],
177-
line_lengths: Sequence[int],
153+
line_lengths: list[int],
178154
candidate: int,
179155
) -> tuple[int, int] | None:
180156
brackets: list[tuple[str, int]] = []
@@ -223,13 +199,13 @@ def _find_matching_delimiter_from_token_stream(
223199
return None
224200

225201

226-
def gen_colors(buffer: TokenizedBuffer) -> Iterator[ColorSpan]:
202+
def gen_colors(buffer: str) -> Iterator[ColorSpan]:
227203
"""Returns a list of index spans to color using the given color tag.
228204
229205
The input `buffer` should be a valid start of a Python code block, i.e.
230206
it cannot be a block starting in the middle of a multiline string.
231207
"""
232-
gen, line_lengths = buffer.replay(), buffer.line_lengths
208+
gen, line_lengths = _get_token_stream(buffer)
233209
last_emitted: ColorSpan | None = None
234210
try:
235211
for color in gen_colors_from_token_stream(gen, line_lengths):
@@ -239,13 +215,13 @@ def gen_colors(buffer: TokenizedBuffer) -> Iterator[ColorSpan]:
239215
return
240216
except tokenize.TokenError as te:
241217
yield from recover_unterminated_string(
242-
te, line_lengths, last_emitted, buffer.source
218+
te, line_lengths, last_emitted, buffer
243219
)
244220

245221

246222
def recover_unterminated_string(
247223
exc: tokenize.TokenError,
248-
line_lengths: Sequence[int],
224+
line_lengths: list[int],
249225
last_emitted: ColorSpan | None,
250226
buffer: str,
251227
) -> Iterator[ColorSpan]:
@@ -286,7 +262,7 @@ def recover_unterminated_string(
286262

287263
def gen_colors_from_token_stream(
288264
token_generator: Iterator[TI],
289-
line_lengths: Sequence[int],
265+
line_lengths: list[int],
290266
) -> Iterator[ColorSpan]:
291267
token_window = prev_next_window(token_generator)
292268

@@ -498,8 +474,7 @@ def disp_str(
498474
(['a', ' ', '=', ' ', '9'], [1, 1, 1, 1, 1])
499475
500476
>>> line = "while 1:"
501-
>>> tokenized = utils.TokenizedBuffer.from_buffer(line)
502-
>>> colors = list(utils.gen_colors(tokenized))
477+
>>> colors = list(utils.gen_colors(line))
503478
>>> utils.disp_str(line, colors=colors)
504479
(['\x1b[1;34mw', 'h', 'i', 'l', 'e\x1b[0m', ' ', '1', ':'], [1, 1, 1, 1, 1, 1, 1, 1])
505480

‎Lib/pdb.py‎

Lines changed: 6 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -466,14 +466,13 @@ def complete(self, text, state):
466466
return None
467467

468468
def gen_colors(self, buffer):
469-
from _pyrepl.utils import ColorSpan, Span, TokenizedBuffer
469+
from _pyrepl.utils import ColorSpan, Span
470470

471-
source = buffer.source
472-
if not source.strip():
471+
if not buffer.strip():
473472
return
474473

475-
leading_spaces = len(source) - len(source.lstrip())
476-
leading_text = source.split()[0]
474+
leading_spaces = len(buffer) - len(buffer.lstrip())
475+
leading_text = buffer.split()[0]
477476
if hasattr(self.pdb_instance, 'do_' + leading_text):
478477
yield ColorSpan(
479478
Span(leading_spaces, leading_spaces + len(leading_text) - 1),
@@ -482,8 +481,7 @@ def gen_colors(self, buffer):
482481
# Redact the command text with spaces so there will be no duplicated
483482
# color span generated for it later.
484483
redact_length = leading_spaces + len(leading_text)
485-
source = ' ' * redact_length + source[redact_length:]
486-
buffer = TokenizedBuffer.from_buffer(source)
484+
buffer = ' ' * redact_length + buffer[redact_length:]
487485

488486
yield from _pyrepl.utils.gen_colors(buffer)
489487

@@ -1274,8 +1272,7 @@ def handle_command_def(self, line):
12741272

12751273
def _colorize_code(self, code):
12761274
if self.colorize and _pyrepl:
1277-
buffer = _pyrepl.utils.TokenizedBuffer.from_buffer(code)
1278-
colors = list(_pyrepl.utils.gen_colors(buffer))
1275+
colors = list(_pyrepl.utils.gen_colors(code))
12791276
chars, _ = _pyrepl.utils.disp_str(code, colors=colors, force_color=True)
12801277
code = "".join(chars)
12811278
return code

‎Lib/test/test_pdb.py‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5087,8 +5087,7 @@ def test_gen_colors(self):
50875087
])
50885088
]
50895089

5090-
for source, expected in test_cases:
5091-
buffer = pdb._pyrepl.utils.TokenizedBuffer.from_buffer(source)
5090+
for buffer, expected in test_cases:
50925091
for color_span, ((start, end), tag) in zip(gen_colors(buffer), expected, strict=True):
50935092
self.assertEqual(color_span.span.start, start)
50945093
self.assertEqual(color_span.span.end, end)

‎Lib/test/test_pyrepl/test_reader.py‎

Lines changed: 23 additions & 54 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,9 @@
11
import itertools
22
import functools
33
import rlcompleter
4-
import tokenize
54
from textwrap import dedent
65
from unittest import TestCase
7-
from unittest.mock import MagicMock, patch
6+
from unittest.mock import MagicMock
87
from test.support import force_colorized, force_colorized_test_class
98
from test.support import force_not_colorized_test_class
109

@@ -36,67 +35,37 @@ def prepare_reader_multiline_prompt(*args, **kwargs):
3635

3736
@force_not_colorized_test_class
3837
class TestReader(ScreenEqualMixin, TestCase):
39-
def test_tokenization_shared_across_refreshes_and_cursor_moves(self):
38+
def test_buffer_replacement_and_cursor_moves(self):
4039
reader = prepare_reader(prepare_console((), width=80))
4140
reader.can_colorize = True
4241
reader.buffer = list("()[]")
43-
with patch("_pyrepl.utils.tokenize.generate_tokens",
44-
wraps=tokenize.generate_tokens) as generate_tokens:
45-
reader.refresh()
46-
self.assertEqual(generate_tokens.call_count, 1)
47-
self.assertEqual(reader._matching_delimiter, (1, 2))
48-
reader.invalidate_full()
49-
reader.refresh()
50-
reader.do_cmd(("right", []))
51-
reader.do_cmd(("right", []))
52-
self.assertEqual(generate_tokens.call_count, 1)
53-
self.assertEqual(reader._matching_delimiter, (3, 4))
54-
55-
# Replacing the list with different text of the same length must
56-
# invalidate the source snapshot without relying on edit commands.
57-
reader.buffer = list("[{}]")
58-
reader.invalidate_full()
59-
reader.refresh()
60-
self.assertEqual(generate_tokens.call_count, 2)
61-
self.assertEqual(reader._matching_delimiter, (1, 2))
62-
reader.do_cmd(("self-insert", ["x"]))
63-
self.assertEqual(generate_tokens.call_count, 3)
42+
reader.refresh()
43+
self.assertEqual(reader._matching_delimiter, (1, 2))
44+
reader.invalidate_full()
45+
reader.refresh()
46+
reader.do_cmd(("right", []))
47+
reader.do_cmd(("right", []))
48+
self.assertEqual(reader._matching_delimiter, (3, 4))
49+
50+
# Replacing the list with different text of the same length must
51+
# update delimiter matching without relying on edit commands.
52+
reader.buffer = list("[{}]")
53+
reader.invalidate_full()
54+
reader.refresh()
55+
self.assertEqual(reader._matching_delimiter, (1, 2))
6456

65-
def test_tokenization_injected_into_custom_colorizer(self):
57+
def test_custom_colorizer_receives_original_source(self):
6658
reader = prepare_reader(prepare_console((), width=80))
6759
reader.buffer = list("(value)")
60+
reader.gen_colors = MagicMock(return_value=iter(()))
6861
reader.can_colorize = False
69-
with patch("_pyrepl.utils.tokenize.generate_tokens",
70-
wraps=tokenize.generate_tokens) as generate_tokens:
71-
reader.refresh()
72-
self.assertEqual(generate_tokens.call_count, 0)
73-
reader.can_colorize = True
74-
reader.gen_colors = MagicMock(return_value=iter(()))
75-
reader.invalidate_full()
76-
reader.refresh()
77-
reader.gen_colors.assert_called_once()
78-
self.assertIs(reader.gen_colors.call_args.args[0],
79-
reader._tokenized_buffer)
80-
self.assertEqual(reader._tokenized_buffer.source, "(value)")
81-
self.assertEqual(reader._matching_delimiter, (6, 7))
82-
self.assertEqual(generate_tokens.call_count, 1)
83-
84-
def test_tokenization_released_after_accept_and_restore(self):
85-
reader = prepare_reader(prepare_console((), width=80))
86-
reader.can_colorize = True
87-
reader.buffer = list("(value)")
8862
reader.refresh()
89-
self.assertIsNotNone(reader._tokenized_buffer)
90-
with patch("_pyrepl.utils.tokenize.generate_tokens",
91-
wraps=tokenize.generate_tokens) as generate_tokens:
92-
reader.do_cmd(("accept", []))
93-
self.assertEqual(generate_tokens.call_count, 0)
94-
self.assertIsNone(reader._tokenized_buffer)
95-
reader.finished = False
63+
reader.gen_colors.assert_not_called()
64+
reader.can_colorize = True
65+
reader.invalidate_full()
9666
reader.refresh()
97-
self.assertIsNotNone(reader._tokenized_buffer)
98-
reader.restore()
99-
self.assertIsNone(reader._tokenized_buffer)
67+
reader.gen_colors.assert_called_once_with("(value)")
68+
self.assertEqual(reader._matching_delimiter, (6, 7))
10069

10170
def test_matching_delimiter_overlay_and_finished_state(self):
10271
console = prepare_console((), width=80)

0 commit comments

Comments
 (0)