From ba20f3810ab37e3c2a028a0f3376cd3c78e145a2 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Sun, 6 Sep 2026 22:22:51 +0100 Subject: [PATCH] gh-153569: report tokenizer diagnostics without rewinding the scanner --- Lib/test/test_codeop.py | 11 +++++ Lib/test/test_source_encoding.py | 26 +++++++++-- Parser/lexer/lexer.c | 13 ++++-- Parser/lexer/state.h | 9 ++++ Parser/lexer/string.c | 75 ++++++++++++++++++++------------ Parser/pegen.c | 4 +- Parser/pegen_errors.c | 34 ++++++++++----- Parser/tokenizer/decoder.c | 14 ++---- Parser/tokenizer/helpers.c | 53 ++++++++++++++-------- Parser/tokenizer/helpers.h | 7 +++ 10 files changed, 167 insertions(+), 79 deletions(-) diff --git a/Lib/test/test_codeop.py b/Lib/test/test_codeop.py index d57452602ce5574..c75c0f8627bf084 100644 --- a/Lib/test/test_codeop.py +++ b/Lib/test/test_codeop.py @@ -113,6 +113,17 @@ def test_valid(self, compiler): av("def f():\n pass\n#foo\n") av("@a.b.c\ndef f():\n pass\n") + @subTests('symbol', ('single', 'exec')) + @subTests('prefix', ('', 'f', 't')) + def test_incomplete_string_diagnostics(self, symbol, prefix): + opening = f' á = {prefix}"""first\n' + source = 'if True:\n' + opening + 'second' + with self.assertRaises(_IncompleteInputError) as cm: + Compile()(source, '', symbol) + text = opening + 'second' + ('\n' if symbol == 'exec' else '') + self.assertEqual(cm.exception.args, ( + 'incomplete input', ('', 2, 9, text, 2, -1))) + @subTests('compiler', COMPILERS) def test_incomplete(self, compiler): ai = functools.partial(self.assertIncomplete, compiler=compiler) diff --git a/Lib/test/test_source_encoding.py b/Lib/test/test_source_encoding.py index 862a20a058be75a..ec98e609c4e98f9 100644 --- a/Lib/test/test_source_encoding.py +++ b/Lib/test/test_source_encoding.py @@ -3,8 +3,8 @@ import unittest from test import support from test.support import script_helper -from test.support.os_helper import TESTFN, unlink, rmtree -from test.support.import_helper import unload +from test.support.os_helper import TESTFN, TESTFN_ASCII, unlink, rmtree +from test.support.import_helper import import_module, unload import importlib import os import sys @@ -83,12 +83,30 @@ def test_truncated_utf8_at_eof(self): self.assertRaises(SyntaxError, compile, seq, '', 'exec') def test_invalid_utf8_offset_after_non_ascii(self): + for name in ('é', 'éé', '𝒜'): + with self.subTest(name=name): + source = ('x = ' + name).encode() + b'\xff\n' + with self.assertRaises(SyntaxError) as caught: + compile(source, '', 'exec') + error = caught.exception + self.assertEqual( + (error.lineno, error.offset, error.end_lineno, error.end_offset), + (1, 5 + len(name), 1, 5 + len(name)), + ) + + @support.cpython_only + def test_invalid_utf8_file_offset_after_non_ascii(self): + _testcapi = import_module('_testcapi') + self.addCleanup(unlink, TESTFN_ASCII) + with open(TESTFN_ASCII, 'wb') as f: + f.write(b'\nx = \xc3\xa9\xc3\xa9\xff\n') with self.assertRaises(SyntaxError) as caught: - compile(b"x = \xc3\xa9\xff\n", "", "exec") + _testcapi.run_file( + os.fsencode(TESTFN_ASCII), _testcapi.Py_file_input, {}) error = caught.exception self.assertEqual( (error.lineno, error.offset, error.end_lineno, error.end_offset), - (1, 6, 1, 6), + (2, 7, 2, 7), ) def test_long_bom_conflict_message_is_not_truncated(self): diff --git a/Parser/lexer/lexer.c b/Parser/lexer/lexer.c index 3bfed0e5d95bd34..77c465eff2a4b7b 100644 --- a/Parser/lexer/lexer.c +++ b/Parser/lexer/lexer.c @@ -99,6 +99,7 @@ verify_identifier(struct tok_state *tok) assert(PyUnicode_GET_LENGTH(s) > 0); if (invalid < PyUnicode_GET_LENGTH(s)) { Py_UCS4 ch = PyUnicode_READ_CHAR(s, invalid); + const char *error_cursor = tok->cur; if (invalid + 1 < PyUnicode_GET_LENGTH(s)) { /* Determine the offset in UTF-8 encoded input */ Py_SETREF(s, PyUnicode_Substring(s, 0, invalid + 1)); @@ -109,14 +110,20 @@ verify_identifier(struct tok_state *tok) tok->done = E_ERROR; return 0; } - tok->cur = (char *)tok->start + PyBytes_GET_SIZE(s); + error_cursor = tok->start + PyBytes_GET_SIZE(s); } Py_DECREF(s); if (Py_UNICODE_ISPRINTABLE(ch)) { - _PyTokenizer_syntaxerror(tok, "invalid character '%c' (U+%04X)", ch, ch); + _PyTokenizer_syntaxerror_at( + tok, tok->line_start, + error_cursor - tok->line_start, tok->lineno, -1, -1, + "invalid character '%c' (U+%04X)", ch, ch); } else { - _PyTokenizer_syntaxerror(tok, "invalid non-printable character U+%04X", ch); + _PyTokenizer_syntaxerror_at( + tok, tok->line_start, + error_cursor - tok->line_start, tok->lineno, -1, -1, + "invalid non-printable character U+%04X", ch); } return 0; } diff --git a/Parser/lexer/state.h b/Parser/lexer/state.h index ff23169e4e3a86b..40ce65345125263 100644 --- a/Parser/lexer/state.h +++ b/Parser/lexer/state.h @@ -78,6 +78,14 @@ typedef struct { indentation_level stack[MAXINDENT]; } lexer_layout_state; +/* Supplemental source context for a terminal error. location is the reporting + cursor, independent of the scanner cursor; lineno == 0 means absent. + The text span may cover multiple physical lines. */ +typedef struct { + _PyTok_Loc location; + _PyTok_Span text_span; +} _PyTokenizer_Diagnostic; + /* Tokenizer state */ struct tok_state { /* Input state; buf <= cur <= inp */ @@ -93,6 +101,7 @@ struct tok_state { lexer_layout_state layout; int lineno; /* Current line number */ _PyTok_Loc start_loc; + _PyTokenizer_Diagnostic diagnostic; int level; /* () [] {} Parentheses nesting level */ /* Used to allow free continuations inside them */ char parenstack[MAXLEVEL]; diff --git a/Parser/lexer/string.c b/Parser/lexer/string.c index 86b5821928643f4..a525a4447fdb886 100644 --- a/Parser/lexer/string.c +++ b/Parser/lexer/string.c @@ -7,13 +7,18 @@ #define MAKE_TOKEN(token_type) _PyLexer_token_setup(tok, token, token_type, p_start, p_end) -static void -rewind_to_string_start(struct tok_state *tok, const char *start, - _PyTok_Loc location) +static int +string_error_token(struct tok_state *tok, struct token *token, + const char *start, _PyTok_Loc location) { - tok->cur = (char *)start + 1; - tok->line_start = start - location.byte_col; - tok->lineno = location.lineno; + tok->diagnostic = (_PyTokenizer_Diagnostic){ + .location = {location.lineno, location.byte_col + 1}, + .text_span = _PyLexer_BufferSpan(tok, start - location.byte_col, tok->inp), + }; + int type = _PyLexer_token_setup(tok, token, ERRORTOKEN, NULL, NULL); + token->start_loc = location; + token->end_loc = (_PyTok_Loc){location.lineno, -1}; + return type; } int @@ -351,44 +356,51 @@ _PyLexer_scan_string(struct tok_state *tok, struct token *token, int c) } if (c == EOF || (quote_size == 1 && c == '\n')) { int end_lineno = tok->lineno; - rewind_to_string_start(tok, tok->start, tok->start_loc); + _PyTok_Loc location = tok->start_loc; + const char *line = tok->start - location.byte_col; + Py_ssize_t cursor_offset = (Py_ssize_t)location.byte_col + 1; const ftstring_state *state = _PyLexer_CurrentFTString(tok); if (state != NULL) { /* A matching quote belongs to the surrounding formatted * string, so the expression is missing its closing brace. */ if (state->quote == quote && state->quote_size == quote_size) { - return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, + _PyTokenizer_syntaxerror_at( + tok, line, cursor_offset, location.lineno, -1, -1, "%c-string: expecting '}'", - _PyLexer_StringPrefix(state->kind))); + _PyLexer_StringPrefix(state->kind)); + return string_error_token(tok, token, tok->start, location); } } if (quote_size == 3) { - _PyTokenizer_syntaxerror(tok, "unterminated triple-quoted string literal" - " (detected at line %d)", end_lineno); + _PyTokenizer_syntaxerror_at( + tok, line, cursor_offset, location.lineno, -1, -1, + "unterminated triple-quoted string literal" + " (detected at line %d)", end_lineno); if (c != '\n') { tok->done = E_EOFS; } - return MAKE_TOKEN(ERRORTOKEN); + return string_error_token(tok, token, tok->start, location); } else { if (has_escaped_quote) { - _PyTokenizer_syntaxerror( - tok, + _PyTokenizer_syntaxerror_at( + tok, line, cursor_offset, location.lineno, -1, -1, "unterminated string literal (detected at line %d); " "perhaps you escaped the end quote?", end_lineno ); } else { - _PyTokenizer_syntaxerror( - tok, "unterminated string literal (detected at line %d)", end_lineno + _PyTokenizer_syntaxerror_at( + tok, line, cursor_offset, location.lineno, -1, -1, + "unterminated string literal (detected at line %d)", end_lineno ); } if (c != '\n') { tok->done = E_EOLS; } - return MAKE_TOKEN(ERRORTOKEN); + return string_error_token(tok, token, tok->start, location); } } if (c == quote) { @@ -452,25 +464,30 @@ _PyLexer_get_ftstring(struct tok_state *tok, ftstring_state *current, struct tok } int end_lineno = tok->lineno; - rewind_to_string_start(tok, - _PyLexer_BufferPointer(tok, current->start), - current->start_loc); + _PyTok_Loc location = current->start_loc; + const char *line = _PyLexer_BufferPointer( + tok, current->start - location.byte_col); + Py_ssize_t cursor_offset = (Py_ssize_t)location.byte_col + 1; if (quote_size == 3) { - _PyTokenizer_syntaxerror(tok, - "unterminated triple-quoted %c-string literal" - " (detected at line %d)", - _PyLexer_StringPrefix(current->kind), end_lineno); + _PyTokenizer_syntaxerror_at( + tok, line, cursor_offset, location.lineno, -1, -1, + "unterminated triple-quoted %c-string literal" + " (detected at line %d)", + _PyLexer_StringPrefix(current->kind), end_lineno); if (c != '\n') { tok->done = E_EOFS; } - return MAKE_TOKEN(ERRORTOKEN); + return string_error_token(tok, token, + _PyLexer_BufferPointer(tok, current->start), location); } else { - return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, - "unterminated %c-string literal (detected at" - " line %d)", - _PyLexer_StringPrefix(current->kind), end_lineno)); + _PyTokenizer_syntaxerror_at( + tok, line, cursor_offset, location.lineno, -1, -1, + "unterminated %c-string literal (detected at line %d)", + _PyLexer_StringPrefix(current->kind), end_lineno); + return string_error_token(tok, token, + _PyLexer_BufferPointer(tok, current->start), location); } } diff --git a/Parser/pegen.c b/Parser/pegen.c index f9bcd1c83944669..96c23be653fc47d 100644 --- a/Parser/pegen.c +++ b/Parser/pegen.c @@ -219,11 +219,11 @@ initialize_token(Parser *p, Token *parser_token, struct token *new_token, int to parser_token->level = new_token->level; parser_token->is_raw = new_token->is_raw; parser_token->lineno = new_token->start_loc.lineno; - parser_token->col_offset = p->tok->lineno == p->starting_lineno + parser_token->col_offset = new_token->end_loc.lineno == p->starting_lineno ? p->starting_col_offset + new_token->start_loc.byte_col : new_token->start_loc.byte_col; parser_token->end_lineno = new_token->end_loc.lineno; - parser_token->end_col_offset = p->tok->lineno == p->starting_lineno + parser_token->end_col_offset = new_token->end_loc.lineno == p->starting_lineno ? p->starting_col_offset + new_token->end_loc.byte_col : new_token->end_loc.byte_col; diff --git a/Parser/pegen_errors.c b/Parser/pegen_errors.c index 275c3d1470e5215..a8ff04d573c0a6b 100644 --- a/Parser/pegen_errors.c +++ b/Parser/pegen_errors.c @@ -202,7 +202,10 @@ _PyPegen_raise_error(Parser *p, PyObject *errtype, int use_mark, const char *err Py_ssize_t col_offset; Py_ssize_t end_col_offset = -1; if (t->col_offset == -1) { - if (p->tok->cur == p->tok->buf) { + _PyTokenizer_Diagnostic diagnostic = p->tok->diagnostic; + if (diagnostic.location.lineno != 0) { + col_offset = diagnostic.location.byte_col; + } else if (p->tok->cur == p->tok->buf) { col_offset = 0; } else { const char* start = p->tok->buf ? p->tok->line_start : p->tok->buf; @@ -251,12 +254,19 @@ _PyPegen_raise_error_known_location(Parser *p, PyObject *errtype, PyObject *error_line = NULL; PyObject *tmp = NULL; p->error_indicator = 1; + _PyTokenizer_Diagnostic diagnostic = p->tok->diagnostic; + _PyTok_Loc location = diagnostic.location.lineno != 0 + ? diagnostic.location : (_PyTok_Loc){p->tok->lineno, + p->tok->line_start == NULL ? -1 : _PyLexer_ByteColumn(p->tok)}; + _PyTok_Span text_span = diagnostic.location.lineno != 0 + ? diagnostic.text_span : _PyLexer_BufferSpan( + p->tok, p->tok->line_start, p->tok->inp); if (end_lineno == CURRENT_POS) { - end_lineno = p->tok->lineno; + end_lineno = location.lineno; } if (end_col_offset == CURRENT_POS) { - end_col_offset = p->tok->cur - p->tok->line_start; + end_col_offset = location.byte_col; } errstr = PyUnicode_FromFormatV(errmsg, va); @@ -264,8 +274,7 @@ _PyPegen_raise_error_known_location(Parser *p, PyObject *errtype, goto error; } - if (_PyTok_ReaderIsInteractive(p->tok) && - _PyTokenizer_RetainedSource(p->tok) != NULL) { + if (_PyTok_ReaderIsInteractive(p->tok) && _PyTokenizer_RetainedSource(p->tok) != NULL) { error_line = get_error_line_from_source(p, lineno); } else if (p->start_rule == Py_file_input) { @@ -281,13 +290,16 @@ _PyPegen_raise_error_known_location(Parser *p, PyObject *errtype, we're actually parsing from a file, which has an E_EOF SyntaxError and in that case `PyErr_ProgramTextObject` fails because lineno points to last_file_line + 1, which does not physically exist */ - assert(p->tok->fp == NULL || p->tok->fp == stdin || p->tok->done == E_EOF); - - if (p->tok->lineno <= lineno && p->tok->inp > p->tok->buf) { - Py_ssize_t size = p->tok->inp - p->tok->line_start; - error_line = PyUnicode_DecodeUTF8(p->tok->line_start, size, "replace"); + assert((p->tok->fp == NULL || p->tok->fp == stdin) || p->tok->done == E_EOF); + + if (location.lineno <= lineno && + p->tok->inp > p->tok->buf) { + Py_ssize_t size; + const char *line = _PyLexer_BufferSpanView( + p->tok, text_span, &size); + error_line = PyUnicode_DecodeUTF8(line, size, "replace"); } - else if (p->tok->fp == NULL || p->tok->fp == stdin) { + else if ((p->tok->fp == NULL || p->tok->fp == stdin)) { error_line = get_error_line_from_source(p, lineno); } else { diff --git a/Parser/tokenizer/decoder.c b/Parser/tokenizer/decoder.c index 023455d29d6d073..c36860237fce0b9 100644 --- a/Parser/tokenizer/decoder.c +++ b/Parser/tokenizer/decoder.c @@ -240,22 +240,14 @@ _PyTok_DetectEncoding(struct tok_state *tok, const _PyTok_Chunk *first, const _PyTok_Chunk *line = cookie_line == 2 ? second : first; const char *line_data = line->data + (cookie_line == 1 ? 3 : 0); Py_ssize_t line_len = line->len - (cookie_line == 1 ? 3 : 0); - const char *saved_line_start = tok->line_start; - char *saved_cur = tok->cur; - int saved_lineno = tok->lineno; - tok->line_start = line_data; - tok->cur = (char *)line_data; - tok->lineno = cookie_line; int end_col = (int)Py_MIN(line_len, INT_MAX); if (end_col > 0 && (line_data[end_col - 1] == '\n' || line_data[end_col - 1] == '\r')) { end_col--; } - _PyTokenizer_syntaxerror_known_range( - tok, 0, end_col, "encoding problem: %s with BOM", cookie); - tok->line_start = saved_line_start; - tok->cur = saved_cur; - tok->lineno = saved_lineno; + _PyTokenizer_syntaxerror_at( + tok, line_data, 0, cookie_line, 0, end_col, + "encoding problem: %s with BOM", cookie); PyMem_Free(cookie); return _PYTOK_ENCODING_ERROR; } diff --git a/Parser/tokenizer/helpers.c b/Parser/tokenizer/helpers.c index 646f8e5932e9546..66a8b160f687871 100644 --- a/Parser/tokenizer/helpers.c +++ b/Parser/tokenizer/helpers.c @@ -24,7 +24,8 @@ byte_col_to_char_col(const char *line, int byte_col) } static int -_syntaxerror_range(struct tok_state *tok, const char *format, +_syntaxerror_range(struct tok_state *tok, const char *line_start, + Py_ssize_t cursor_offset, int lineno, const char *format, int col_offset, int end_col_offset, va_list vargs) { @@ -40,7 +41,7 @@ _syntaxerror_range(struct tok_state *tok, const char *format, goto error; } - errtext = PyUnicode_DecodeUTF8(tok->line_start, tok->cur - tok->line_start, + errtext = PyUnicode_DecodeUTF8(line_start, cursor_offset, "replace"); if (!errtext) { goto error; @@ -50,19 +51,19 @@ _syntaxerror_range(struct tok_state *tok, const char *format, col_offset = (int)PyUnicode_GET_LENGTH(errtext); } else if (col_offset > 0) { - col_offset = byte_col_to_char_col(tok->line_start, col_offset); + col_offset = byte_col_to_char_col(line_start, col_offset); } if (end_col_offset == -1) { end_col_offset = col_offset; } else if (end_col_offset > 0) { - end_col_offset = byte_col_to_char_col(tok->line_start, end_col_offset); + end_col_offset = byte_col_to_char_col(line_start, end_col_offset); } - Py_ssize_t line_len = strcspn(tok->line_start, "\n"); - if (line_len != tok->cur - tok->line_start) { + Py_ssize_t line_len = strcspn(line_start, "\n"); + if (line_len != cursor_offset) { Py_DECREF(errtext); - errtext = PyUnicode_DecodeUTF8(tok->line_start, line_len, + errtext = PyUnicode_DecodeUTF8(line_start, line_len, "replace"); } if (!errtext) { @@ -71,8 +72,8 @@ _syntaxerror_range(struct tok_state *tok, const char *format, args = Py_BuildValue("(O(OiiNii))", errmsg, tok->filename ? tok->filename : Py_None, - tok->lineno, col_offset, errtext, - tok->lineno, end_col_offset); + lineno, col_offset, errtext, + lineno, end_col_offset); if (args) { PyErr_SetObject(PyExc_SyntaxError, args); Py_DECREF(args); @@ -90,7 +91,9 @@ _PyTokenizer_syntaxerror(struct tok_state *tok, const char *format, ...) // These errors are cleaned on startup. Todo: Fix it. va_list vargs; va_start(vargs, format); - int ret = _syntaxerror_range(tok, format, -1, -1, vargs); + int ret = _syntaxerror_range( + tok, tok->line_start, + tok->cur - tok->line_start, tok->lineno, format, -1, -1, vargs); va_end(vargs); return ret; } @@ -102,7 +105,24 @@ _PyTokenizer_syntaxerror_known_range(struct tok_state *tok, { va_list vargs; va_start(vargs, format); - int ret = _syntaxerror_range(tok, format, col_offset, end_col_offset, vargs); + int ret = _syntaxerror_range( + tok, tok->line_start, + tok->cur - tok->line_start, tok->lineno, format, + col_offset, end_col_offset, vargs); + va_end(vargs); + return ret; +} + +int +_PyTokenizer_syntaxerror_at( + struct tok_state *tok, const char *line_start, Py_ssize_t cursor_offset, + int lineno, int col_offset, int end_col_offset, const char *format, ...) +{ + va_list vargs; + va_start(vargs, format); + int ret = _syntaxerror_range( + tok, line_start, cursor_offset, lineno, format, + col_offset, end_col_offset, vargs); va_end(vargs); return ret; } @@ -302,26 +322,21 @@ _PyTokenizer_ensure_utf8(const char *line, struct tok_state *tok, int lineno) const char *badchar = NULL; const char *c; int length; - int col_offset = 0; const char *line_start = line; for (c = line; *c; c += length) { if (!(length = valid_utf8((const unsigned char *)c))) { badchar = c; break; } - col_offset++; if (*c == '\n') { lineno++; - col_offset = 0; line_start = c + 1; } } if (badchar) { - tok->lineno = lineno; - tok->line_start = line_start; - tok->cur = (char *)badchar; - _PyTokenizer_syntaxerror_known_range(tok, - col_offset + 1, col_offset + 1, + _PyTokenizer_syntaxerror_at( + tok, line_start, badchar - line_start + 1, lineno, + -1, -1, "Non-UTF-8 code starting with '\\x%.2x'" "%s%V on line %i, " "but no encoding declared; " diff --git a/Parser/tokenizer/helpers.h b/Parser/tokenizer/helpers.h index 51de0cbb156f833..27bec7f3bf0ffee 100644 --- a/Parser/tokenizer/helpers.h +++ b/Parser/tokenizer/helpers.h @@ -6,10 +6,17 @@ #include "../lexer/state.h" int _PyTokenizer_syntaxerror(struct tok_state *tok, const char *format, ...); +/* Positive range columns are 1-based byte columns. A start column of -1 + derives the character column from the reporting cursor; an end column of + -1 uses the start column. */ int _PyTokenizer_syntaxerror_known_range(struct tok_state *tok, int col_offset, int end_col_offset, const char *format, ...); +int _PyTokenizer_syntaxerror_at( + struct tok_state *tok, const char *line_start, Py_ssize_t cursor_offset, + int lineno, int col_offset, int end_col_offset, const char *format, ...); int _PyTokenizer_indenterror(struct tok_state *tok); int _PyTokenizer_warn_invalid_escape_sequence(struct tok_state *tok, int first_invalid_escape_char); int _PyTokenizer_parser_warn(struct tok_state *tok, PyObject *category, const char *format, ...); + void _PyTokenizer_raise_init_error(PyObject *filename); int _PyTokenizer_ensure_utf8(const char *line, struct tok_state *tok, int lineno);