From 9895cf82f2934fb634673c42bd7450b7ff873a79 Mon Sep 17 00:00:00 2001 From: 5 <93879550+tydavid5@users.noreply.github.com> Date: Thu, 19 Nov 2026 14:58:05 +0100 Subject: [PATCH] gh-162142: Guard the same-line fast path against a negative byte offset _get_col_offsets can pass a byte offset of -1 to _PyPegen_byte_offset_to_character_offset_line when a token starts before the current line's span, as an INDENT token does when the tokenizer still holds earlier lines in its buffer. That function then reads data[-1], one byte before the line's UTF-8 buffer. The same-line fast path now only runs when the byte offset is non-negative, so the call never sees a negative offset. The other two calls in the function already pass non-negative offsets, and the callee is shared with the parser, so the guard belongs at this call site rather than in the callee. On main this is hardening: the mismatched bracket that used to reach the over-read is now a TokenError before any INDENT is produced, and no input was found that reaches it, so the new test asserts that TokenError rather than a crash. Verified with an AddressSanitizer build: the repro, the edge cases and 3,000 fuzz inputs run clean both before and after the fix, and the edge cases produce identical tokens. --- Lib/test/test_tokenize.py | 18 ++++++++++++++++++ ...6-11-19-13-26-04.gh-issue-162142.Kx7QmR.rst | 5 +++++ Python/Python-tokenize.c | 6 +++++- 3 files changed, 28 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2026-11-19-13-26-04.gh-issue-162142.Kx7QmR.rst diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py index 6d9f7f21f42954..4398a58257584f 100644 --- a/Lib/test/test_tokenize.py +++ b/Lib/test/test_tokenize.py @@ -2687,6 +2687,24 @@ def test_tolerant_fstring_closer_at_expression_entry_depth(self): self._get_tokens(source, extra_tokens=extra_tokens) self.assertEqual(caught.exception.args, error) + def test_fstring_unmatched_paren_before_indent(self): + # gh-162142: on 3.15.1 this input reached _get_col_offsets with a + # byte offset of -1 for the INDENT token, and the same-line fast + # path passed it to _PyPegen_byte_offset_to_character_offset_line, + # which read one byte before the line buffer. The mismatched + # bracket is now a TokenError before any INDENT is produced, so + # the over-read no longer triggers on main, but keep the input + # covered as a regression test for the guard in _get_col_offsets. + source = 'f"{)\n \u00e9\n' + for extra_tokens in (False, True): + with self.subTest(extra_tokens=extra_tokens): + with self.assertRaises(tokenize.TokenError) as caught: + self._get_tokens(source, extra_tokens=extra_tokens) + self.assertEqual( + caught.exception.args, + ("f-string: unmatched ')'", (1, 4)), + ) + def test_int(self): self.check_tokenize('0xff <= 255', """\ diff --git a/Misc/NEWS.d/next/Library/2026-11-19-13-26-04.gh-issue-162142.Kx7QmR.rst b/Misc/NEWS.d/next/Library/2026-11-19-13-26-04.gh-issue-162142.Kx7QmR.rst new file mode 100644 index 00000000000000..7ebc6ad7c9c3e7 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-11-19-13-26-04.gh-issue-162142.Kx7QmR.rst @@ -0,0 +1,5 @@ +Fix a one-byte heap over-read in the tokenizer: ``_get_col_offsets`` could +pass a byte offset of ``-1`` to +``_PyPegen_byte_offset_to_character_offset_line`` when an ``INDENT`` token +started before the current line, which then read one byte before the line +buffer. The same-line fast path now skips the call for a negative offset. diff --git a/Python/Python-tokenize.c b/Python/Python-tokenize.c index 951d4976c147c0..bf697dee1443c2 100644 --- a/Python/Python-tokenize.c +++ b/Python/Python-tokenize.c @@ -206,7 +206,11 @@ _get_col_offsets(tokenizeriterobject *it, const struct token *token, if (token_end >= view->end_line_start) { Py_ssize_t end_byte_offset = token_end - view->end_line_start; - if (lineno == end_lineno) { + // byte_offset is -1 when the token starts before the current line's + // span, as an INDENT token does when the tokenizer still holds + // earlier lines in its buffer. The callee reads data[col_offset], + // so a negative offset would read before the line's buffer. + if (lineno == end_lineno && byte_offset >= 0) { // Avoid rescanning the prefix of a very long line. Py_ssize_t token_col_offset = _PyPegen_byte_offset_to_character_offset_line(line, byte_offset, end_byte_offset); if (token_col_offset < 0) {