diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py index 6d9f7f21f42954..4398a58257584f 100644 --- a/Lib/test/test_tokenize.py +++ b/Lib/test/test_tokenize.py @@ -2687,6 +2687,24 @@ def test_tolerant_fstring_closer_at_expression_entry_depth(self): self._get_tokens(source, extra_tokens=extra_tokens) self.assertEqual(caught.exception.args, error) + def test_fstring_unmatched_paren_before_indent(self): + # gh-162142: on 3.15.1 this input reached _get_col_offsets with a + # byte offset of -1 for the INDENT token, and the same-line fast + # path passed it to _PyPegen_byte_offset_to_character_offset_line, + # which read one byte before the line buffer. The mismatched + # bracket is now a TokenError before any INDENT is produced, so + # the over-read no longer triggers on main, but keep the input + # covered as a regression test for the guard in _get_col_offsets. + source = 'f"{)\n \u00e9\n' + for extra_tokens in (False, True): + with self.subTest(extra_tokens=extra_tokens): + with self.assertRaises(tokenize.TokenError) as caught: + self._get_tokens(source, extra_tokens=extra_tokens) + self.assertEqual( + caught.exception.args, + ("f-string: unmatched ')'", (1, 4)), + ) + def test_int(self): self.check_tokenize('0xff <= 255', """\ diff --git a/Misc/NEWS.d/next/Library/2026-11-19-13-26-04.gh-issue-162142.Kx7QmR.rst b/Misc/NEWS.d/next/Library/2026-11-19-13-26-04.gh-issue-162142.Kx7QmR.rst new file mode 100644 index 00000000000000..7ebc6ad7c9c3e7 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-11-19-13-26-04.gh-issue-162142.Kx7QmR.rst @@ -0,0 +1,5 @@ +Fix a one-byte heap over-read in the tokenizer: ``_get_col_offsets`` could +pass a byte offset of ``-1`` to +``_PyPegen_byte_offset_to_character_offset_line`` when an ``INDENT`` token +started before the current line, which then read one byte before the line +buffer. The same-line fast path now skips the call for a negative offset. diff --git a/Python/Python-tokenize.c b/Python/Python-tokenize.c index 951d4976c147c0..bf697dee1443c2 100644 --- a/Python/Python-tokenize.c +++ b/Python/Python-tokenize.c @@ -206,7 +206,11 @@ _get_col_offsets(tokenizeriterobject *it, const struct token *token, if (token_end >= view->end_line_start) { Py_ssize_t end_byte_offset = token_end - view->end_line_start; - if (lineno == end_lineno) { + // byte_offset is -1 when the token starts before the current line's + // span, as an INDENT token does when the tokenizer still holds + // earlier lines in its buffer. The callee reads data[col_offset], + // so a negative offset would read before the line's buffer. + if (lineno == end_lineno && byte_offset >= 0) { // Avoid rescanning the prefix of a very long line. Py_ssize_t token_col_offset = _PyPegen_byte_offset_to_character_offset_line(line, byte_offset, end_byte_offset); if (token_col_offset < 0) {