Update logos to 0.16

bluejay-parser/Cargo.toml

@@ -11,7 +11,7 @@ exclude = [".gitignore", "tests/**/*"]
 description = "A GraphQL parser"
 
 [dependencies]
-logos = { version = "0.15" }
+logos = { version = "0.16" }
 enum-as-inner = "0.7"
 ariadne = { version = "0.5.0" }
 serde = { version = "1.0.203", optional = true }

bluejay-parser/src/lexer/logos_lexer.rs

@@ -24,7 +24,7 @@ pub(crate) struct Extras {
 #[logos(subpattern fixedunicode = r"\\u[0-9A-Fa-f]{4}")]
 #[logos(error = LexError)]
 #[logos(skip r"[\uFEFF\t \n\r,]+")]
-#[logos(skip r"#[^\n\r]*")] // comments
+#[logos(skip(r"#[^\n\r]*", allow_greedy = true))] // comments
 #[logos(extras = Extras)]
 pub(crate) enum Token<'a> {
     // Punctuators
@@ -470,11 +470,9 @@ mod tests {
             vec![(Err(LexError::UnrecognizedToken), 0..1)],
             Token::lexer(".").spanned().collect::<Vec<_>>(),
         );
+        // Two dots give one error that spans the failed match attempt
         assert_eq!(
-            vec![
-                (Err(LexError::UnrecognizedToken), 0..1),
-                (Err(LexError::UnrecognizedToken), 1..2),
-            ],
+            vec![(Err(LexError::UnrecognizedToken), 0..2)],
             Token::lexer("..").spanned().collect::<Vec<_>>(),
         );
         assert_eq!(
@@ -589,21 +587,23 @@ mod tests {
             )],
             Token::lexer(r#""\q""#).spanned().collect::<Vec<_>>(),
         );
-        // A unicode escape sequence with too few digits
+        // A unicode escape sequence with too few digits.
+        // The inner error covers the full failed escape sequence.
         assert_eq!(
             vec![(
                 Err(LexError::StringValueInvalid(vec![
-                    StringValueLexError::InvalidCharacters(Span::from(1..2)),
+                    StringValueLexError::InvalidCharacters(Span::from(1..5)),
                 ])),
                 0..6,
             )],
             Token::lexer(r#""\u12""#).spanned().collect::<Vec<_>>(),
         );
-        // A unicode escape sequence with no digits
+        // A unicode escape sequence with no digits.
+        // The inner error covers the full failed escape sequence.
         assert_eq!(
             vec![(
                 Err(LexError::StringValueInvalid(vec![
-                    StringValueLexError::InvalidCharacters(Span::from(1..2)),
+                    StringValueLexError::InvalidCharacters(Span::from(1..4)),
                 ])),
                 0..6,
             )],
@@ -798,7 +798,17 @@ mod tests {
     fn kitchen_sink_token_stream_test() {
         // Lex a document with all token types and all ignored token types,
         // and compare the full token stream, with spans, to the expected stream.
-        let separators = [" ", ",", "\n", "\t", "\r\n", " # comment\n", "\u{FEFF}"];
+        let separators = [
+            " ",
+            ",",
+            "\n",
+            "\t",
+            "\r\n",
+            " # comment\n",
+            " # comment\r\n",
+            " #comment\r",
+            "\u{FEFF}",
+        ];
         let parts = vec![
             ("query", Token::Name("query")),
             ("MyQuery", Token::Name("MyQuery")),

bluejay-parser/src/lexer/logos_lexer/safety_tests.rs

@@ -242,28 +242,36 @@ fn large_pathological_inputs_terminate() {
         format!("\"\"\"{}", "\\\"\"\"".repeat(25_000)),
         format!("\"\"\"{}\"\"\"", " \n".repeat(50_000)),
         "$".repeat(50_000),
-        // Many small numeric tokens. See large_single_token_runs_terminate
-        // for why one large numeric token is not included here.
         "9 ".repeat(50_000),
         "0 ".repeat(50_000),
         ".".repeat(50_000),
         format!("#{}", "c".repeat(100_000)),
-        // Keep runs of ignored characters below the crash threshold for
-        // unoptimized builds. See large_single_token_runs_terminate.
+        // Large ASCII tokens and large ASCII ignored runs crashed
+        // unoptimized builds with logos 0.15. Logos 0.16 lexes them
+        // with bounded stack usage.
+        "9".repeat(100_000),
+        format!("-1.{0}e-{0}", "9".repeat(50_000)),
+        " ".repeat(100_000),
+        "\t \r\n,".repeat(20_000),
+        // Keep runs of byte order marks below the crash threshold for
+        // unoptimized builds. See large_multibyte_runs_terminate.
         "\u{FEFF}\t \r\n,".repeat(1_000),
     ];
     for input in large_inputs {
         assert_lexes_safely(&input);
     }
 }
 
-/// With logos 0.15, the generated matchers for numeric tokens and for
-/// runs of ignored characters use stack space proportional to the run
-/// length in unoptimized builds. Optimized builds compile the recursion
-/// into loops, so release builds accept runs of all lengths. As a result
-/// of this, one numeric token with approximately 5,000 or more digits,
-/// or one run of approximately 50,000 or more ignored characters,
-/// crashes unoptimized builds.
+/// With logos 0.16, the generated matchers for runs of multi-byte
+/// characters use stack space proportional to the run length in
+/// unoptimized builds. One thousand multi-byte characters use
+/// approximately 1 MiB of stack. Optimized builds compile the
+/// recursion into loops, so release builds accept runs of all lengths.
+/// The affected inputs are strings with many multi-byte characters and
+/// runs of many byte order marks.
+/// Logos 0.15 had the same problem for numeric tokens and for runs of
+/// ASCII ignored characters. Logos 0.16 corrected those cases and
+/// introduced the multi-byte problem for strings.
 /// This test pins the current stack usage with some headroom.
 /// If it starts to abort with a stack overflow after a logos upgrade,
 /// then the stack usage per character became worse, which makes the
@@ -277,27 +285,28 @@ fn long_token_stack_usage() {
             assert_lexes_safely(&format!("-1.{}e-9", "9".repeat(1_000)));
             assert_lexes_safely(&" ".repeat(1_000));
             assert_lexes_safely(&"\u{FEFF}\t \r\n,".repeat(200));
+            assert_lexes_safely(&format!("\"{}\"", "é".repeat(500)));
         })
         .unwrap()
         .join()
         .unwrap();
 }
 
-/// One large numeric token or one large run of ignored characters must
-/// lex safely. With logos 0.15, these inputs overflow the stack in
-/// unoptimized builds, and the process aborts. Optimized builds are not
-/// affected. See long_token_stack_usage for the details.
+/// One large run of multi-byte characters must lex safely. With
+/// logos 0.16, these inputs overflow the stack in unoptimized builds,
+/// and the process aborts. Optimized builds are not affected.
+/// See long_token_stack_usage for the details.
 /// This test is ignored because a failure aborts the full test process.
 /// Try to enable this test again after each logos upgrade:
 /// run `cargo test -p bluejay-parser --lib -- --ignored` in a debug
 /// build. If all tests pass, remove the ignore attribute.
 #[test]
-#[ignore = "logos 0.15 overflows the stack on large single tokens in unoptimized builds; try to re-enable after the next logos upgrade"]
-fn large_single_token_runs_terminate() {
+#[ignore = "logos 0.16 overflows the stack on long runs of multi-byte characters in unoptimized builds; try to re-enable after the next logos upgrade"]
+fn large_multibyte_runs_terminate() {
     let large_inputs = [
-        "9".repeat(100_000),
-        format!("-1.{0}e-{0}", "9".repeat(50_000)),
-        " ".repeat(100_000),
+        format!("\"{}\"", "é".repeat(50_000)),
+        format!("\"{}\"", "🔥".repeat(25_000)),
+        "\u{FEFF}".repeat(30_000),
         "\u{FEFF}\t \r\n,".repeat(20_000),
     ];
     for input in large_inputs {

bluejay-parser/tests/lexer_adversarial_test.rs

@@ -93,17 +93,11 @@ fn truncated_documents_parse_safely() {
     }
 }
 
-/// Documents with one large numeric token or one large run of ignored
-/// characters must parse safely. With logos 0.15, these inputs overflow
-/// the stack in unoptimized builds, and the process aborts. Optimized
-/// builds are not affected. The max tokens limit does not protect
-/// against this, because each input is a single token.
-/// This test is ignored because a failure aborts the full test process.
-/// Try to enable this test again after each logos upgrade:
-/// run `cargo test -p bluejay-parser --test lexer_adversarial_test -- --ignored`
-/// in a debug build. If all tests pass, remove the ignore attribute.
+/// Documents with one large numeric token or one large run of ASCII
+/// ignored characters must parse safely. Logos 0.15 caused a stack
+/// overflow for these inputs in unoptimized builds. Logos 0.16 parses
+/// them with bounded stack usage.
 #[test]
-#[ignore = "logos 0.15 overflows the stack on large single tokens in unoptimized builds; try to re-enable after the next logos upgrade"]
 fn large_single_token_documents_parse_safely() {
     let sources = [
         format!("{{ a(b: {}) }}", "9".repeat(100_000)),
@@ -115,6 +109,27 @@ fn large_single_token_documents_parse_safely() {
     }
 }
 
+/// Documents with one large run of multi-byte characters must parse
+/// safely. With logos 0.16, these inputs overflow the stack in
+/// unoptimized builds, and the process aborts. Optimized builds are
+/// not affected. The max tokens limit does not protect against this,
+/// because each input is a single token.
+/// This test is ignored because a failure aborts the full test process.
+/// Try to enable this test again after each logos upgrade:
+/// run `cargo test -p bluejay-parser --test lexer_adversarial_test -- --ignored`
+/// in a debug build. If all tests pass, remove the ignore attribute.
+#[test]
+#[ignore = "logos 0.16 overflows the stack on long runs of multi-byte characters in unoptimized builds; try to re-enable after the next logos upgrade"]
+fn large_multibyte_documents_parse_safely() {
+    let sources = [
+        format!("{{ a(b: \"{}\") }}", "é".repeat(50_000)),
+        format!("{}{{ a }}", "\u{FEFF}".repeat(30_000)),
+    ];
+    for source in sources {
+        assert_parses_safely(&source);
+    }
+}
+
 /// The max tokens limit is a denial-of-service protection.
 /// It must bound the work for large hostile documents.
 #[test]