88import _colorize
99
1010from collections import deque
11- from collections .abc import Iterable , Iterator , Sequence
11+ from collections .abc import Iterable , Iterator
1212from dataclasses import dataclass
1313from io import StringIO
1414from re import Match
@@ -46,7 +46,7 @@ def from_re(cls, m: Match[str], group: int | str) -> Self:
4646 return cls (re_span [0 ], re_span [1 ] - 1 )
4747
4848 @classmethod
49- def from_token (cls , token : TI , line_len : Sequence [int ]) -> Self :
49+ def from_token (cls , token : TI , line_len : list [int ]) -> Self :
5050 end_offset = - 1
5151 if (token .type in {T .FSTRING_MIDDLE , T .TSTRING_MIDDLE }
5252 and token .string .endswith (("{" , "}" ))):
@@ -116,53 +116,29 @@ def unbracket(s: str, including_content: bool = False) -> str:
116116 return s .translate (ZERO_WIDTH_TRANS )
117117
118118
119- @dataclass (frozen = True )
120- class TokenizedBuffer :
121- """Replay tokens before reproducing any terminal tokenizer error."""
119+ def _get_token_stream (buffer : str ) -> tuple [Iterator [TI ], list [int ]]:
120+ sio = StringIO (buffer )
121+ line_lengths = [0 ] + [len (line ) for line in sio .readlines ()]
122+ # make line_lengths cumulative
123+ for i in range (1 , len (line_lengths )):
124+ line_lengths [i ] += line_lengths [i - 1 ]
122125
123- source : str
124- tokens : tuple [TI , ...]
125- line_lengths : tuple [int , ...]
126- terminal_error : tuple [type [SyntaxError ] | type [tokenize .TokenError ], tuple ] | None
126+ sio .seek (0 )
127+ gen = tokenize .generate_tokens (sio .readline )
128+ return gen , line_lengths
127129
128- @classmethod
129- def from_buffer (cls , source : str ) -> Self :
130- sio = StringIO (source )
131- line_lengths = [0 ] + [len (line ) for line in sio .readlines ()]
132- for i in range (1 , len (line_lengths )):
133- line_lengths [i ] += line_lengths [i - 1 ]
134-
135- sio .seek (0 )
136- token_generator = tokenize .generate_tokens (sio .readline )
137- tokens = []
138- terminal_error = None
139- try :
140- tokens .extend (token_generator )
141- except (SyntaxError , tokenize .TokenError ) as exc :
142- terminal_error = (type (exc ), exc .args )
143- return cls (source , tuple (tokens ), tuple (line_lengths ), terminal_error )
144-
145- def replay (self ) -> Iterator [TI ]:
146- yield from self .tokens
147- if self .terminal_error is not None :
148- error_type , args = self .terminal_error
149- raise error_type (* args )
150-
151-
152- def find_matching_delimiter (
153- buffer : TokenizedBuffer , pos : int
154- ) -> tuple [int , int ] | None :
130+
131+ def find_matching_delimiter (buffer : str , pos : int ) -> tuple [int , int ] | None :
155132 """Return the half-open span of the partner touching the cursor."""
156- source = buffer .source
157133 # A delimiter under the cursor takes precedence, even inside literal data.
158134 candidate = next ((
159135 i for i in (pos , pos - 1 )
160- if 0 <= i < len (source ) and source [i ] in "()[]{}\" '"
136+ if 0 <= i < len (buffer ) and buffer [i ] in "()[]{}\" '"
161137 ), None )
162138 if candidate is None :
163139 return None
164140
165- gen , line_lengths = buffer . replay (), buffer . line_lengths
141+ gen , line_lengths = _get_token_stream ( buffer )
166142 try :
167143 return _find_matching_delimiter_from_token_stream (
168144 gen , line_lengths , candidate
@@ -174,7 +150,7 @@ def find_matching_delimiter(
174150
175151def _find_matching_delimiter_from_token_stream (
176152 token_generator : Iterator [TI ],
177- line_lengths : Sequence [int ],
153+ line_lengths : list [int ],
178154 candidate : int ,
179155) -> tuple [int , int ] | None :
180156 brackets : list [tuple [str , int ]] = []
@@ -223,13 +199,13 @@ def _find_matching_delimiter_from_token_stream(
223199 return None
224200
225201
226- def gen_colors (buffer : TokenizedBuffer ) -> Iterator [ColorSpan ]:
202+ def gen_colors (buffer : str ) -> Iterator [ColorSpan ]:
227203 """Returns a list of index spans to color using the given color tag.
228204
229205 The input `buffer` should be a valid start of a Python code block, i.e.
230206 it cannot be a block starting in the middle of a multiline string.
231207 """
232- gen , line_lengths = buffer . replay (), buffer . line_lengths
208+ gen , line_lengths = _get_token_stream ( buffer )
233209 last_emitted : ColorSpan | None = None
234210 try :
235211 for color in gen_colors_from_token_stream (gen , line_lengths ):
@@ -239,13 +215,13 @@ def gen_colors(buffer: TokenizedBuffer) -> Iterator[ColorSpan]:
239215 return
240216 except tokenize .TokenError as te :
241217 yield from recover_unterminated_string (
242- te , line_lengths , last_emitted , buffer . source
218+ te , line_lengths , last_emitted , buffer
243219 )
244220
245221
246222def recover_unterminated_string (
247223 exc : tokenize .TokenError ,
248- line_lengths : Sequence [int ],
224+ line_lengths : list [int ],
249225 last_emitted : ColorSpan | None ,
250226 buffer : str ,
251227) -> Iterator [ColorSpan ]:
@@ -286,7 +262,7 @@ def recover_unterminated_string(
286262
287263def gen_colors_from_token_stream (
288264 token_generator : Iterator [TI ],
289- line_lengths : Sequence [int ],
265+ line_lengths : list [int ],
290266) -> Iterator [ColorSpan ]:
291267 token_window = prev_next_window (token_generator )
292268
@@ -498,8 +474,7 @@ def disp_str(
498474 (['a', ' ', '=', ' ', '9'], [1, 1, 1, 1, 1])
499475
500476 >>> line = "while 1:"
501- >>> tokenized = utils.TokenizedBuffer.from_buffer(line)
502- >>> colors = list(utils.gen_colors(tokenized))
477+ >>> colors = list(utils.gen_colors(line))
503478 >>> utils.disp_str(line, colors=colors)
504479 (['\x1b[1;34mw', 'h', 'i', 'l', 'e\x1b[0m', ' ', '1', ':'], [1, 1, 1, 1, 1, 1, 1, 1])
505480
0 commit comments