88import _colorize
99
1010from collections import deque
11- from collections .abc import Iterable , Iterator
11+ from collections .abc import Iterable , Iterator , Sequence
1212from dataclasses import dataclass
1313from io import StringIO
1414from re import Match
@@ -46,7 +46,7 @@ def from_re(cls, m: Match[str], group: int | str) -> Self:
4646 return cls (re_span [0 ], re_span [1 ] - 1 )
4747
4848 @classmethod
49- def from_token (cls , token : TI , line_len : list [int ]) -> Self :
49+ def from_token (cls , token : TI , line_len : Sequence [int ]) -> Self :
5050 end_offset = - 1
5151 if (token .type in {T .FSTRING_MIDDLE , T .TSTRING_MIDDLE }
5252 and token .string .endswith (("{" , "}" ))):
@@ -116,20 +116,120 @@ def unbracket(s: str, including_content: bool = False) -> str:
116116 return s .translate (ZERO_WIDTH_TRANS )
117117
118118
119- def gen_colors (buffer : str ) -> Iterator [ColorSpan ]:
119+ @dataclass (frozen = True )
120+ class TokenizedBuffer :
121+ """Replay tokens before reproducing any terminal tokenizer error."""
122+
123+ source : str
124+ tokens : tuple [TI , ...]
125+ line_lengths : tuple [int , ...]
126+ terminal_error : tuple [type [SyntaxError ] | type [tokenize .TokenError ], tuple ] | None
127+
128+ @classmethod
129+ def from_buffer (cls , source : str ) -> Self :
130+ sio = StringIO (source )
131+ line_lengths = [0 ] + [len (line ) for line in sio .readlines ()]
132+ for i in range (1 , len (line_lengths )):
133+ line_lengths [i ] += line_lengths [i - 1 ]
134+
135+ sio .seek (0 )
136+ token_generator = tokenize .generate_tokens (sio .readline )
137+ tokens = []
138+ terminal_error = None
139+ try :
140+ tokens .extend (token_generator )
141+ except (SyntaxError , tokenize .TokenError ) as exc :
142+ terminal_error = (type (exc ), exc .args )
143+ return cls (source , tuple (tokens ), tuple (line_lengths ), terminal_error )
144+
145+ def replay (self ) -> Iterator [TI ]:
146+ yield from self .tokens
147+ if self .terminal_error is not None :
148+ error_type , args = self .terminal_error
149+ raise error_type (* args )
150+
151+
152+ def find_matching_delimiter (
153+ buffer : TokenizedBuffer , pos : int
154+ ) -> tuple [int , int ] | None :
155+ """Return the half-open span of the partner touching the cursor."""
156+ source = buffer .source
157+ # A delimiter under the cursor takes precedence, even inside literal data.
158+ candidate = next ((
159+ i for i in (pos , pos - 1 )
160+ if 0 <= i < len (source ) and source [i ] in "()[]{}\" '"
161+ ), None )
162+ if candidate is None :
163+ return None
164+
165+ gen , line_lengths = buffer .replay (), buffer .line_lengths
166+ try :
167+ return _find_matching_delimiter_from_token_stream (
168+ gen , line_lengths , candidate
169+ )
170+ except SyntaxError , tokenize .TokenError :
171+ # Incomplete input can still contain complete pairs.
172+ return None
173+
174+
175+ def _find_matching_delimiter_from_token_stream (
176+ token_generator : Iterator [TI ],
177+ line_lengths : Sequence [int ],
178+ candidate : int ,
179+ ) -> tuple [int , int ] | None :
180+ brackets : list [tuple [str , int ]] = []
181+ strings : list [tuple [int , int ]] = []
182+ closing = {")" : "(" , "]" : "[" , "}" : "{" }
183+ for token in token_generator :
184+ if token .type == T .ERRORTOKEN and token .string in {"\" " , "'" }:
185+ # An unfinished short string can expose apparent operators.
186+ break
187+ offset = line_lengths [token .start [0 ] - 1 ] + token .start [1 ]
188+ pair = None
189+ match token .type :
190+ case T .STRING | T .FSTRING_START | T .TSTRING_START :
191+ # Token boundaries already account for prefixes and escapes.
192+ quote_index = next (
193+ i for i , c in enumerate (token .string ) if c in "\" '"
194+ )
195+ delimiter = token .string [quote_index ]
196+ width = 3 if token .string .startswith (delimiter * 3 , quote_index ) else 1
197+ opening = (offset + quote_index , offset + quote_index + width )
198+ if token .type == T .STRING :
199+ end = line_lengths [token .end [0 ] - 1 ] + token .end [1 ]
200+ pair = (opening , (end - width , end ))
201+ else :
202+ strings .append (opening )
203+ case T .FSTRING_END | T .TSTRING_END :
204+ opening = strings .pop ()
205+ pair = (opening , (offset , offset + len (token .string )))
206+ case T .OP if token .string in "()[]{}" :
207+ if token .string in "([{" :
208+ brackets .append ((token .string , offset ))
209+ elif brackets and brackets [- 1 ][0 ] == closing [token .string ]:
210+ _ , opening_offset = brackets .pop ()
211+ pair = (
212+ (opening_offset , opening_offset + 1 ), (offset , offset + 1 )
213+ )
214+ else :
215+ # Do not pair across an incorrectly nested delimiter.
216+ brackets .clear ()
217+ if pair is not None :
218+ opening , ending = pair
219+ if opening [0 ] <= candidate < opening [1 ]:
220+ return ending
221+ if ending [0 ] <= candidate < ending [1 ]:
222+ return opening
223+ return None
224+
225+
226+ def gen_colors (buffer : TokenizedBuffer ) -> Iterator [ColorSpan ]:
120227 """Returns a list of index spans to color using the given color tag.
121228
122229 The input `buffer` should be a valid start of a Python code block, i.e.
123230 it cannot be a block starting in the middle of a multiline string.
124231 """
125- sio = StringIO (buffer )
126- line_lengths = [0 ] + [len (line ) for line in sio .readlines ()]
127- # make line_lengths cumulative
128- for i in range (1 , len (line_lengths )):
129- line_lengths [i ] += line_lengths [i - 1 ]
130-
131- sio .seek (0 )
132- gen = tokenize .generate_tokens (sio .readline )
232+ gen , line_lengths = buffer .replay (), buffer .line_lengths
133233 last_emitted : ColorSpan | None = None
134234 try :
135235 for color in gen_colors_from_token_stream (gen , line_lengths ):
@@ -139,13 +239,13 @@ def gen_colors(buffer: str) -> Iterator[ColorSpan]:
139239 return
140240 except tokenize .TokenError as te :
141241 yield from recover_unterminated_string (
142- te , line_lengths , last_emitted , buffer
242+ te , line_lengths , last_emitted , buffer . source
143243 )
144244
145245
146246def recover_unterminated_string (
147247 exc : tokenize .TokenError ,
148- line_lengths : list [int ],
248+ line_lengths : Sequence [int ],
149249 last_emitted : ColorSpan | None ,
150250 buffer : str ,
151251) -> Iterator [ColorSpan ]:
@@ -186,7 +286,7 @@ def recover_unterminated_string(
186286
187287def gen_colors_from_token_stream (
188288 token_generator : Iterator [TI ],
189- line_lengths : list [int ],
289+ line_lengths : Sequence [int ],
190290) -> Iterator [ColorSpan ]:
191291 token_window = prev_next_window (token_generator )
192292
@@ -398,7 +498,8 @@ def disp_str(
398498 (['a', ' ', '=', ' ', '9'], [1, 1, 1, 1, 1])
399499
400500 >>> line = "while 1:"
401- >>> colors = list(utils.gen_colors(line))
501+ >>> tokenized = utils.TokenizedBuffer.from_buffer(line)
502+ >>> colors = list(utils.gen_colors(tokenized))
402503 >>> utils.disp_str(line, colors=colors)
403504 (['\x1b[1;34mw', 'h', 'i', 'l', 'e\x1b[0m', ' ', '1', ':'], [1, 1, 1, 1, 1, 1, 1, 1])
404505
0 commit comments