From 0e7e0a3821e0d2e15862489bc8a9d0a6016040a8 Mon Sep 17 00:00:00 2001 From: Revar Desmera Date: Sun, 9 Aug 2026 22:20:41 -0700 Subject: [PATCH] Stop a backslash-newline in a string leaking to stdout Flex's `.` never matches a newline, so `\\.` could not match a backslash escaping the end of a line. With no rule for it the backslash fell through to flex's DEFAULT ECHO rule and was printed straight to stdout -- a stray `\` on every parse of such a file, in the middle of whatever the program was writing. The newline was then picked up by the content-run rule and became part of the string. Both characters are now matched and kept verbatim in the literal. Kept rather than dropped on purpose. StringLiteral::toString() wraps the stored text in quotes to reproduce the source, so cooking escapes here would make the pretty-printer emit real newlines and quotes and corrupt the file it was formatting. Resolving escapes belongs where the literal is evaluated; this side's job is to describe the source faithfully. --- src/grammar/lexer.l | 9 +++++++++ tests/test_lexical.cpp | 37 +++++++++++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+) diff --git a/src/grammar/lexer.l b/src/grammar/lexer.l index 5ffab6e..1a8a704 100644 --- a/src/grammar/lexer.l +++ b/src/grammar/lexer.l @@ -99,6 +99,15 @@ NUMLIT 0[xX][0-9A-Fa-f]+|[0-9]+([.][0-9]*)?([eE][+-]?[0-9]+)?|[.][0-9]+([eE][+- driver.stringStart = driver.tokenStart; BEGIN(STR); } + /* A backslash escaping the end of the line. Flex's `.` never matches a + newline, so without this rule the backslash matched nothing, fell to + flex's default ECHO and was printed to stdout -- a stray `\` on every + parse of such a file -- while the newline was picked up as ordinary + content by the run rule below and ended up inside the string. + Kept verbatim rather than dropped here: the value is cooked when the + literal is evaluated, so the AST keeps text that reproduces the + source exactly, which is what the pretty-printer round-trips. */ +\\\r?\n { driver.stringBuffer.append(yytext, yyleng); } \\. { driver.stringBuffer.append(yytext, yyleng); } [^"\\]+ { driver.stringBuffer.append(yytext, yyleng); } \" { diff --git a/tests/test_lexical.cpp b/tests/test_lexical.cpp index e3919d9..e14cd3c 100644 --- a/tests/test_lexical.cpp +++ b/tests/test_lexical.cpp @@ -436,3 +436,40 @@ TEST(SourceSpans, MultiLineStringReportsItsStartingLine) { EXPECT_EQ(assign->expr->position().line, 2); EXPECT_EQ(spanned(src, *assign->expr), "\"one\ntwo\""); } + +// -- Backslash at end of line inside a string --------------------------- + +// The lexer had no rule for a backslash followed by a newline: flex's `.` +// never matches a newline, so `\\.` could not, and the backslash fell +// through to flex's DEFAULT ECHO rule -- printed straight to stdout, a +// stray `\` on every parse of such a file. The newline was then swallowed +// by the content-run rule and became part of the string. +// +// The literal keeps both characters. Cooking happens where the literal is +// evaluated, not here, so a StringLiteral's text still reproduces the +// source it was parsed from -- which is what pretty-printing relies on. +TEST(LexicalStrings, BackslashNewlineIsKeptVerbatimInTheLiteral) { + struct Case { std::string src; std::string want; const char* what; }; + const Case cases[] = { + {"a = \"x \\\ny\";", "x \\\ny", "LF"}, + {"a = \"x \\\r\ny\";", "x \\\r\ny", "CRLF"}, + }; + for (const Case& c : cases) { + auto ast = parseSrc(c.src); + ASSERT_EQ(ast.size(), 1u) << c.what; + auto* assign = dynamic_cast(ast[0].get()); + ASSERT_NE(assign, nullptr) << c.what; + auto* str = dynamic_cast(assign->expr.get()); + ASSERT_NE(str, nullptr) << c.what; + EXPECT_EQ(str->val, c.want) << c.what; + } +} + +// The continued line still counts, or every position after a multi-line +// string would be reported one line short. +TEST(LexicalStrings, BackslashNewlineStillAdvancesTheLineNumber) { + auto ast = parseSrc("a = \"x \\\ny\";\nb = 2;\n"); + ASSERT_EQ(ast.size(), 2u); + EXPECT_EQ(ast[0]->position().line, 1); + EXPECT_EQ(ast[1]->position().line, 3) << "the continuation consumed a line"; +}