Lexer: no-break space and BOM are whitespace; error text stays UTF-8

src/grammar/lexer.l

@@ -72,11 +72,19 @@ const std::unordered_map<std::string, KeywordFactory>& keywordTable() {
 %x INCFILE
 
 IDENT   \$?[_A-Za-z][A-Za-z0-9_]*
+UTF8CH  [\xc2-\xdf][\x80-\xbf]|[\xe0-\xef][\x80-\xbf]{2}|[\xf0-\xf4][\x80-\xbf]{3}
 NUMLIT  0[xX][0-9A-Fa-f]+|[0-9]+([.][0-9]*)?([eE][+-]?[0-9]+)?|[.][0-9]+([eE][+-]?[0-9]+)?
 
 %%
 
 [ \t\r\n]+              { /* skip whitespace */ }
+\xc2\xa0|\xef\xbb\xbf|\xa0 {
+    // Also whitespace, as in OpenSCAD's lexer.l: U+00A0 NO-BREAK SPACE
+    // (C2 A0), which Qt's editor silently turns into a space -- so a file
+    // holding one parsed in the editor and failed from the command line --
+    // U+FEFF (EF BB BF, a byte-order mark, anywhere in the file) and a bare
+    // A0, Latin-1's no-break space.
+}
 "//"[^\n]*               { /* skip line comment */ }
 "/*"([^*]|\*+[^*/])*\*+"/" { /* skip block comment */ }
 
@@ -162,9 +170,21 @@ NUMLIT  0[xX][0-9A-Fa-f]+|[0-9]+([.][0-9]*)?([eE][+-]?[0-9]+)?|[.][0-9]+([eE][+-
 ">" { return yy::parser::make_GTOP(driver.currentTokenLoc()); }
 "#" { return yy::parser::make_HASHOP(driver.currentTokenLoc()); }
 
-. {
+{UTF8CH} {
+    // A whole character, so the message stays valid UTF-8: quoting just its
+    // first byte made a message no consumer could decode.
     driver.reportError(driver.currentTokenLoc(), std::string("unexpected character '") + yytext + "'");
 }
+. {
+    // A byte that starts no valid UTF-8 character: named, not quoted.
+    std::string shown(yytext);
+    const auto byte = static_cast<unsigned char>(yytext[0]);
+    if (byte >= 0x80 || byte < 0x20) {
+        static const char hex[] = "0123456789ABCDEF";
+        shown = std::string("\\x") + hex[byte >> 4] + hex[byte & 0xF];
+    }
+    driver.reportError(driver.currentTokenLoc(), "unexpected character '" + shown + "'");
+}
 
 <<EOF>> { return yy::parser::make_YYEOF(driver.currentTokenLoc()); }
 

tests/test_lexical.cpp

@@ -473,3 +473,30 @@ TEST(LexicalStrings, BackslashNewlineStillAdvancesTheLineNumber) {
     EXPECT_EQ(ast[0]->position().line, 1);
     EXPECT_EQ(ast[1]->position().line, 3) << "the continuation consumed a line";
 }
+
+// As in OpenSCAD's lexer: U+00A0, U+FEFF and a bare Latin-1 A0 are whitespace.
+TEST(LexicalWhitespace, NoBreakSpaceByteOrderMarkAndLatin1A0) {
+    for (const char* ws : {"\xc2\xa0", "\xef\xbb\xbf", "\xa0"}) {
+        auto ast = parseSrc(std::string(ws) + "a" + ws + "=" + ws + "1;" + ws + "\nb = 2;");
+        ASSERT_EQ(ast.size(), 2u);
+        EXPECT_EQ(ast[1]->position().line, 2);
+    }
+}
+
+// The message quotes a whole character, and names a byte that starts none,
+// so it is always valid UTF-8.
+TEST(LexicalWhitespace, UnexpectedCharacterMessageIsValidUtf8) {
+    const std::pair<const char*, const char*> cases[] = {
+        {"a = 1 \xc3\xa9;", "unexpected character '\xc3\xa9'"},
+        {"a = 1 \xc2;", "unexpected character '\\xC2'"},
+        {"a = 1 \xff;", "unexpected character '\\xFF'"},
+    };
+    for (const auto& [src, want] : cases) {
+        try {
+            parseSrc(src);
+            FAIL() << "expected a parse error for " << src;
+        } catch (const ParseError& e) {
+            EXPECT_NE(std::string(e.what()).find(want), std::string::npos) << e.what();
+        }
+    }
+}