Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions src/grammar/lexer.l
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,15 @@ NUMLIT 0[xX][0-9A-Fa-f]+|[0-9]+([.][0-9]*)?([eE][+-]?[0-9]+)?|[.][0-9]+([eE][+-
driver.stringStart = driver.tokenStart;
BEGIN(STR);
}
/* A backslash escaping the end of the line. Flex's `.` never matches a
newline, so without this rule the backslash matched nothing, fell to
flex's default ECHO and was printed to stdout -- a stray `\` on every
parse of such a file -- while the newline was picked up as ordinary
content by the run rule below and ended up inside the string.
Kept verbatim rather than dropped here: the value is cooked when the
literal is evaluated, so the AST keeps text that reproduces the
source exactly, which is what the pretty-printer round-trips. */
<STR>\\\r?\n { driver.stringBuffer.append(yytext, yyleng); }
<STR>\\. { driver.stringBuffer.append(yytext, yyleng); }
<STR>[^"\\]+ { driver.stringBuffer.append(yytext, yyleng); }
<STR>\" {
Expand Down
37 changes: 37 additions & 0 deletions tests/test_lexical.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -436,3 +436,40 @@ TEST(SourceSpans, MultiLineStringReportsItsStartingLine) {
EXPECT_EQ(assign->expr->position().line, 2);
EXPECT_EQ(spanned(src, *assign->expr), "\"one\ntwo\"");
}

// -- Backslash at end of line inside a string ---------------------------

// The lexer had no rule for a backslash followed by a newline: flex's `.`
// never matches a newline, so `\\.` could not, and the backslash fell
// through to flex's DEFAULT ECHO rule -- printed straight to stdout, a
// stray `\` on every parse of such a file. The newline was then swallowed
// by the content-run rule and became part of the string.
//
// The literal keeps both characters. Cooking happens where the literal is
// evaluated, not here, so a StringLiteral's text still reproduces the
// source it was parsed from -- which is what pretty-printing relies on.
TEST(LexicalStrings, BackslashNewlineIsKeptVerbatimInTheLiteral) {
struct Case { std::string src; std::string want; const char* what; };
const Case cases[] = {
{"a = \"x \\\ny\";", "x \\\ny", "LF"},
{"a = \"x \\\r\ny\";", "x \\\r\ny", "CRLF"},
};
for (const Case& c : cases) {
auto ast = parseSrc(c.src);
ASSERT_EQ(ast.size(), 1u) << c.what;
auto* assign = dynamic_cast<Assignment*>(ast[0].get());
ASSERT_NE(assign, nullptr) << c.what;
auto* str = dynamic_cast<StringLiteral*>(assign->expr.get());
ASSERT_NE(str, nullptr) << c.what;
EXPECT_EQ(str->val, c.want) << c.what;
}
}

// The continued line still counts, or every position after a multi-line
// string would be reported one line short.
TEST(LexicalStrings, BackslashNewlineStillAdvancesTheLineNumber) {
auto ast = parseSrc("a = \"x \\\ny\";\nb = 2;\n");
ASSERT_EQ(ast.size(), 2u);
EXPECT_EQ(ast[0]->position().line, 1);
EXPECT_EQ(ast[1]->position().line, 3) << "the continuation consumed a line";
}
Loading