diff --git a/src/grammar/lexer.l b/src/grammar/lexer.l index 9b42eba..0346380 100644 --- a/src/grammar/lexer.l +++ b/src/grammar/lexer.l @@ -72,11 +72,19 @@ const std::unordered_map& keywordTable() { %x INCFILE IDENT \$?[_A-Za-z][A-Za-z0-9_]* +UTF8CH [\xc2-\xdf][\x80-\xbf]|[\xe0-\xef][\x80-\xbf]{2}|[\xf0-\xf4][\x80-\xbf]{3} NUMLIT 0[xX][0-9A-Fa-f]+|[0-9]+([.][0-9]*)?([eE][+-]?[0-9]+)?|[.][0-9]+([eE][+-]?[0-9]+)? %% [ \t\r\n]+ { /* skip whitespace */ } +\xc2\xa0|\xef\xbb\xbf|\xa0 { + // Also whitespace, as in OpenSCAD's lexer.l: U+00A0 NO-BREAK SPACE + // (C2 A0), which Qt's editor silently turns into a space -- so a file + // holding one parsed in the editor and failed from the command line -- + // U+FEFF (EF BB BF, a byte-order mark, anywhere in the file) and a bare + // A0, Latin-1's no-break space. +} "//"[^\n]* { /* skip line comment */ } "/*"([^*]|\*+[^*/])*\*+"/" { /* skip block comment */ } @@ -162,9 +170,21 @@ NUMLIT 0[xX][0-9A-Fa-f]+|[0-9]+([.][0-9]*)?([eE][+-]?[0-9]+)?|[.][0-9]+([eE][+- ">" { return yy::parser::make_GTOP(driver.currentTokenLoc()); } "#" { return yy::parser::make_HASHOP(driver.currentTokenLoc()); } -. { +{UTF8CH} { + // A whole character, so the message stays valid UTF-8: quoting just its + // first byte made a message no consumer could decode. driver.reportError(driver.currentTokenLoc(), std::string("unexpected character '") + yytext + "'"); } +. { + // A byte that starts no valid UTF-8 character: named, not quoted. + std::string shown(yytext); + const auto byte = static_cast(yytext[0]); + if (byte >= 0x80 || byte < 0x20) { + static const char hex[] = "0123456789ABCDEF"; + shown = std::string("\\x") + hex[byte >> 4] + hex[byte & 0xF]; + } + driver.reportError(driver.currentTokenLoc(), "unexpected character '" + shown + "'"); +} <> { return yy::parser::make_YYEOF(driver.currentTokenLoc()); } diff --git a/tests/test_lexical.cpp b/tests/test_lexical.cpp index e14cd3c..fdf7696 100644 --- a/tests/test_lexical.cpp +++ b/tests/test_lexical.cpp @@ -473,3 +473,30 @@ TEST(LexicalStrings, BackslashNewlineStillAdvancesTheLineNumber) { EXPECT_EQ(ast[0]->position().line, 1); EXPECT_EQ(ast[1]->position().line, 3) << "the continuation consumed a line"; } + +// As in OpenSCAD's lexer: U+00A0, U+FEFF and a bare Latin-1 A0 are whitespace. +TEST(LexicalWhitespace, NoBreakSpaceByteOrderMarkAndLatin1A0) { + for (const char* ws : {"\xc2\xa0", "\xef\xbb\xbf", "\xa0"}) { + auto ast = parseSrc(std::string(ws) + "a" + ws + "=" + ws + "1;" + ws + "\nb = 2;"); + ASSERT_EQ(ast.size(), 2u); + EXPECT_EQ(ast[1]->position().line, 2); + } +} + +// The message quotes a whole character, and names a byte that starts none, +// so it is always valid UTF-8. +TEST(LexicalWhitespace, UnexpectedCharacterMessageIsValidUtf8) { + const std::pair cases[] = { + {"a = 1 \xc3\xa9;", "unexpected character '\xc3\xa9'"}, + {"a = 1 \xc2;", "unexpected character '\\xC2'"}, + {"a = 1 \xff;", "unexpected character '\\xFF'"}, + }; + for (const auto& [src, want] : cases) { + try { + parseSrc(src); + FAIL() << "expected a parse error for " << src; + } catch (const ParseError& e) { + EXPECT_NE(std::string(e.what()).find(want), std::string::npos) << e.what(); + } + } +}