Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 11 additions & 0 deletions Lib/test/test_codeop.py
Original file line number Diff line number Diff line change
Expand Up @@ -113,6 +113,17 @@ def test_valid(self, compiler):
av("def f():\n pass\n#foo\n")
av("@a.b.c\ndef f():\n pass\n")

@subTests('symbol', ('single', 'exec'))
@subTests('prefix', ('', 'f', 't'))
def test_incomplete_string_diagnostics(self, symbol, prefix):
opening = f' á = {prefix}"""first\n'
source = 'if True:\n' + opening + 'second'
with self.assertRaises(_IncompleteInputError) as cm:
Compile()(source, '<input>', symbol)
text = opening + 'second' + ('\n' if symbol == 'exec' else '')
self.assertEqual(cm.exception.args, (
'incomplete input', ('<input>', 2, 9, text, 2, -1)))

@subTests('compiler', COMPILERS)
def test_incomplete(self, compiler):
ai = functools.partial(self.assertIncomplete, compiler=compiler)
Expand Down
26 changes: 22 additions & 4 deletions Lib/test/test_source_encoding.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,8 @@
import unittest
from test import support
from test.support import script_helper
from test.support.os_helper import TESTFN, unlink, rmtree
from test.support.import_helper import unload
from test.support.os_helper import TESTFN, TESTFN_ASCII, unlink, rmtree
from test.support.import_helper import import_module, unload
import importlib
import os
import sys
Expand Down Expand Up @@ -83,12 +83,30 @@ def test_truncated_utf8_at_eof(self):
self.assertRaises(SyntaxError, compile, seq, '<test>', 'exec')

def test_invalid_utf8_offset_after_non_ascii(self):
for name in ('é', 'éé', '𝒜'):
with self.subTest(name=name):
source = ('x = ' + name).encode() + b'\xff\n'
with self.assertRaises(SyntaxError) as caught:
compile(source, '<test>', 'exec')
error = caught.exception
self.assertEqual(
(error.lineno, error.offset, error.end_lineno, error.end_offset),
(1, 5 + len(name), 1, 5 + len(name)),
)

@support.cpython_only
def test_invalid_utf8_file_offset_after_non_ascii(self):
_testcapi = import_module('_testcapi')
self.addCleanup(unlink, TESTFN_ASCII)
with open(TESTFN_ASCII, 'wb') as f:
f.write(b'\nx = \xc3\xa9\xc3\xa9\xff\n')
with self.assertRaises(SyntaxError) as caught:
compile(b"x = \xc3\xa9\xff\n", "<test>", "exec")
_testcapi.run_file(
os.fsencode(TESTFN_ASCII), _testcapi.Py_file_input, {})
error = caught.exception
self.assertEqual(
(error.lineno, error.offset, error.end_lineno, error.end_offset),
(1, 6, 1, 6),
(2, 7, 2, 7),
)

def test_long_bom_conflict_message_is_not_truncated(self):
Expand Down
13 changes: 10 additions & 3 deletions Parser/lexer/lexer.c
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,7 @@ verify_identifier(struct tok_state *tok)
assert(PyUnicode_GET_LENGTH(s) > 0);
if (invalid < PyUnicode_GET_LENGTH(s)) {
Py_UCS4 ch = PyUnicode_READ_CHAR(s, invalid);
const char *error_cursor = tok->cur;
if (invalid + 1 < PyUnicode_GET_LENGTH(s)) {
/* Determine the offset in UTF-8 encoded input */
Py_SETREF(s, PyUnicode_Substring(s, 0, invalid + 1));
Expand All @@ -109,14 +110,20 @@ verify_identifier(struct tok_state *tok)
tok->done = E_ERROR;
return 0;
}
tok->cur = (char *)tok->start + PyBytes_GET_SIZE(s);
error_cursor = tok->start + PyBytes_GET_SIZE(s);
}
Py_DECREF(s);
if (Py_UNICODE_ISPRINTABLE(ch)) {
_PyTokenizer_syntaxerror(tok, "invalid character '%c' (U+%04X)", ch, ch);
_PyTokenizer_syntaxerror_at(
tok, tok->line_start,
error_cursor - tok->line_start, tok->lineno, -1, -1,
"invalid character '%c' (U+%04X)", ch, ch);
}
else {
_PyTokenizer_syntaxerror(tok, "invalid non-printable character U+%04X", ch);
_PyTokenizer_syntaxerror_at(
tok, tok->line_start,
error_cursor - tok->line_start, tok->lineno, -1, -1,
"invalid non-printable character U+%04X", ch);
}
return 0;
}
Expand Down
9 changes: 9 additions & 0 deletions Parser/lexer/state.h
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,14 @@ typedef struct {
indentation_level stack[MAXINDENT];
} lexer_layout_state;

/* Supplemental source context for a terminal error. location is the reporting
cursor, independent of the scanner cursor; lineno == 0 means absent.
The text span may cover multiple physical lines. */
typedef struct {
_PyTok_Loc location;
_PyTok_Span text_span;
} _PyTokenizer_Diagnostic;

/* Tokenizer state */
struct tok_state {
/* Input state; buf <= cur <= inp */
Expand All @@ -93,6 +101,7 @@ struct tok_state {
lexer_layout_state layout;
int lineno; /* Current line number */
_PyTok_Loc start_loc;
_PyTokenizer_Diagnostic diagnostic;
int level; /* () [] {} Parentheses nesting level */
/* Used to allow free continuations inside them */
char parenstack[MAXLEVEL];
Expand Down
75 changes: 46 additions & 29 deletions Parser/lexer/string.c
Original file line number Diff line number Diff line change
Expand Up @@ -7,13 +7,18 @@

#define MAKE_TOKEN(token_type) _PyLexer_token_setup(tok, token, token_type, p_start, p_end)

static void
rewind_to_string_start(struct tok_state *tok, const char *start,
_PyTok_Loc location)
static int
string_error_token(struct tok_state *tok, struct token *token,
const char *start, _PyTok_Loc location)
{
tok->cur = (char *)start + 1;
tok->line_start = start - location.byte_col;
tok->lineno = location.lineno;
tok->diagnostic = (_PyTokenizer_Diagnostic){
.location = {location.lineno, location.byte_col + 1},
.text_span = _PyLexer_BufferSpan(tok, start - location.byte_col, tok->inp),
};
int type = _PyLexer_token_setup(tok, token, ERRORTOKEN, NULL, NULL);
token->start_loc = location;
token->end_loc = (_PyTok_Loc){location.lineno, -1};
return type;
}

int
Expand Down Expand Up @@ -351,44 +356,51 @@ _PyLexer_scan_string(struct tok_state *tok, struct token *token, int c)
}
if (c == EOF || (quote_size == 1 && c == '\n')) {
int end_lineno = tok->lineno;
rewind_to_string_start(tok, tok->start, tok->start_loc);
_PyTok_Loc location = tok->start_loc;
const char *line = tok->start - location.byte_col;
Py_ssize_t cursor_offset = (Py_ssize_t)location.byte_col + 1;

const ftstring_state *state = _PyLexer_CurrentFTString(tok);
if (state != NULL) {
/* A matching quote belongs to the surrounding formatted
* string, so the expression is missing its closing brace. */
if (state->quote == quote && state->quote_size == quote_size) {
return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok,
_PyTokenizer_syntaxerror_at(
tok, line, cursor_offset, location.lineno, -1, -1,
"%c-string: expecting '}'",
_PyLexer_StringPrefix(state->kind)));
_PyLexer_StringPrefix(state->kind));
return string_error_token(tok, token, tok->start, location);
}
}

if (quote_size == 3) {
_PyTokenizer_syntaxerror(tok, "unterminated triple-quoted string literal"
" (detected at line %d)", end_lineno);
_PyTokenizer_syntaxerror_at(
tok, line, cursor_offset, location.lineno, -1, -1,
"unterminated triple-quoted string literal"
" (detected at line %d)", end_lineno);
if (c != '\n') {
tok->done = E_EOFS;
}
return MAKE_TOKEN(ERRORTOKEN);
return string_error_token(tok, token, tok->start, location);
}
else {
if (has_escaped_quote) {
_PyTokenizer_syntaxerror(
tok,
_PyTokenizer_syntaxerror_at(
tok, line, cursor_offset, location.lineno, -1, -1,
"unterminated string literal (detected at line %d); "
"perhaps you escaped the end quote?",
end_lineno
);
} else {
_PyTokenizer_syntaxerror(
tok, "unterminated string literal (detected at line %d)", end_lineno
_PyTokenizer_syntaxerror_at(
tok, line, cursor_offset, location.lineno, -1, -1,
"unterminated string literal (detected at line %d)", end_lineno
);
}
if (c != '\n') {
tok->done = E_EOLS;
}
return MAKE_TOKEN(ERRORTOKEN);
return string_error_token(tok, token, tok->start, location);
}
}
if (c == quote) {
Expand Down Expand Up @@ -452,25 +464,30 @@ _PyLexer_get_ftstring(struct tok_state *tok, ftstring_state *current, struct tok
}

int end_lineno = tok->lineno;
rewind_to_string_start(tok,
_PyLexer_BufferPointer(tok, current->start),
current->start_loc);
_PyTok_Loc location = current->start_loc;
const char *line = _PyLexer_BufferPointer(
tok, current->start - location.byte_col);
Py_ssize_t cursor_offset = (Py_ssize_t)location.byte_col + 1;

if (quote_size == 3) {
_PyTokenizer_syntaxerror(tok,
"unterminated triple-quoted %c-string literal"
" (detected at line %d)",
_PyLexer_StringPrefix(current->kind), end_lineno);
_PyTokenizer_syntaxerror_at(
tok, line, cursor_offset, location.lineno, -1, -1,
"unterminated triple-quoted %c-string literal"
" (detected at line %d)",
_PyLexer_StringPrefix(current->kind), end_lineno);
if (c != '\n') {
tok->done = E_EOFS;
}
return MAKE_TOKEN(ERRORTOKEN);
return string_error_token(tok, token,
_PyLexer_BufferPointer(tok, current->start), location);
}
else {
return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok,
"unterminated %c-string literal (detected at"
" line %d)",
_PyLexer_StringPrefix(current->kind), end_lineno));
_PyTokenizer_syntaxerror_at(
tok, line, cursor_offset, location.lineno, -1, -1,
"unterminated %c-string literal (detected at line %d)",
_PyLexer_StringPrefix(current->kind), end_lineno);
return string_error_token(tok, token,
_PyLexer_BufferPointer(tok, current->start), location);
}
}

Expand Down
4 changes: 2 additions & 2 deletions Parser/pegen.c
Original file line number Diff line number Diff line change
Expand Up @@ -219,11 +219,11 @@ initialize_token(Parser *p, Token *parser_token, struct token *new_token, int to
parser_token->level = new_token->level;
parser_token->is_raw = new_token->is_raw;
parser_token->lineno = new_token->start_loc.lineno;
parser_token->col_offset = p->tok->lineno == p->starting_lineno
parser_token->col_offset = new_token->end_loc.lineno == p->starting_lineno
? p->starting_col_offset + new_token->start_loc.byte_col
: new_token->start_loc.byte_col;
parser_token->end_lineno = new_token->end_loc.lineno;
parser_token->end_col_offset = p->tok->lineno == p->starting_lineno
parser_token->end_col_offset = new_token->end_loc.lineno == p->starting_lineno
? p->starting_col_offset + new_token->end_loc.byte_col
: new_token->end_loc.byte_col;

Expand Down
34 changes: 23 additions & 11 deletions Parser/pegen_errors.c
Original file line number Diff line number Diff line change
Expand Up @@ -202,7 +202,10 @@ _PyPegen_raise_error(Parser *p, PyObject *errtype, int use_mark, const char *err
Py_ssize_t col_offset;
Py_ssize_t end_col_offset = -1;
if (t->col_offset == -1) {
if (p->tok->cur == p->tok->buf) {
_PyTokenizer_Diagnostic diagnostic = p->tok->diagnostic;
if (diagnostic.location.lineno != 0) {
col_offset = diagnostic.location.byte_col;
} else if (p->tok->cur == p->tok->buf) {
col_offset = 0;
} else {
const char* start = p->tok->buf ? p->tok->line_start : p->tok->buf;
Expand Down Expand Up @@ -251,21 +254,27 @@ _PyPegen_raise_error_known_location(Parser *p, PyObject *errtype,
PyObject *error_line = NULL;
PyObject *tmp = NULL;
p->error_indicator = 1;
_PyTokenizer_Diagnostic diagnostic = p->tok->diagnostic;
_PyTok_Loc location = diagnostic.location.lineno != 0
? diagnostic.location : (_PyTok_Loc){p->tok->lineno,
p->tok->line_start == NULL ? -1 : _PyLexer_ByteColumn(p->tok)};
_PyTok_Span text_span = diagnostic.location.lineno != 0
? diagnostic.text_span : _PyLexer_BufferSpan(
p->tok, p->tok->line_start, p->tok->inp);

if (end_lineno == CURRENT_POS) {
end_lineno = p->tok->lineno;
end_lineno = location.lineno;
}
if (end_col_offset == CURRENT_POS) {
end_col_offset = p->tok->cur - p->tok->line_start;
end_col_offset = location.byte_col;
}

errstr = PyUnicode_FromFormatV(errmsg, va);
if (!errstr) {
goto error;
}

if (_PyTok_ReaderIsInteractive(p->tok) &&
_PyTokenizer_RetainedSource(p->tok) != NULL) {
if (_PyTok_ReaderIsInteractive(p->tok) && _PyTokenizer_RetainedSource(p->tok) != NULL) {
error_line = get_error_line_from_source(p, lineno);
}
else if (p->start_rule == Py_file_input) {
Expand All @@ -281,13 +290,16 @@ _PyPegen_raise_error_known_location(Parser *p, PyObject *errtype,
we're actually parsing from a file, which has an E_EOF SyntaxError and in that case
`PyErr_ProgramTextObject` fails because lineno points to last_file_line + 1, which
does not physically exist */
assert(p->tok->fp == NULL || p->tok->fp == stdin || p->tok->done == E_EOF);

if (p->tok->lineno <= lineno && p->tok->inp > p->tok->buf) {
Py_ssize_t size = p->tok->inp - p->tok->line_start;
error_line = PyUnicode_DecodeUTF8(p->tok->line_start, size, "replace");
assert((p->tok->fp == NULL || p->tok->fp == stdin) || p->tok->done == E_EOF);

if (location.lineno <= lineno &&
p->tok->inp > p->tok->buf) {
Py_ssize_t size;
const char *line = _PyLexer_BufferSpanView(
p->tok, text_span, &size);
error_line = PyUnicode_DecodeUTF8(line, size, "replace");
}
else if (p->tok->fp == NULL || p->tok->fp == stdin) {
else if ((p->tok->fp == NULL || p->tok->fp == stdin)) {
error_line = get_error_line_from_source(p, lineno);
}
else {
Expand Down
14 changes: 3 additions & 11 deletions Parser/tokenizer/decoder.c
Original file line number Diff line number Diff line change
Expand Up @@ -240,22 +240,14 @@ _PyTok_DetectEncoding(struct tok_state *tok, const _PyTok_Chunk *first,
const _PyTok_Chunk *line = cookie_line == 2 ? second : first;
const char *line_data = line->data + (cookie_line == 1 ? 3 : 0);
Py_ssize_t line_len = line->len - (cookie_line == 1 ? 3 : 0);
const char *saved_line_start = tok->line_start;
char *saved_cur = tok->cur;
int saved_lineno = tok->lineno;
tok->line_start = line_data;
tok->cur = (char *)line_data;
tok->lineno = cookie_line;
int end_col = (int)Py_MIN(line_len, INT_MAX);
if (end_col > 0 && (line_data[end_col - 1] == '\n' ||
line_data[end_col - 1] == '\r')) {
end_col--;
}
_PyTokenizer_syntaxerror_known_range(
tok, 0, end_col, "encoding problem: %s with BOM", cookie);
tok->line_start = saved_line_start;
tok->cur = saved_cur;
tok->lineno = saved_lineno;
_PyTokenizer_syntaxerror_at(
tok, line_data, 0, cookie_line, 0, end_col,
"encoding problem: %s with BOM", cookie);
PyMem_Free(cookie);
return _PYTOK_ENCODING_ERROR;
}
Expand Down
Loading
Loading