From 17d9a58aa5ea1e4c814709087fbd8f52830e5bc6 Mon Sep 17 00:00:00 2001 From: Adam Petro Date: Wed, 12 Aug 2026 15:29:05 -0400 Subject: [PATCH 1/2] Update logos to 0.16 Update the logos dependency from 0.15 to 0.16. Add allow_greedy to the comment skip pattern. Logos 0.16 requires this opt-in for greedy patterns that are equivalent to a dot repetition. The greedy match to the end of the line is the intended behavior for GraphQL comments. Update three test expectations for a change in error recovery. Logos 0.16 reports one error for a failed match attempt. The error span covers all consumed bytes. Logos 0.15 reported an error for only the first byte: - ".." gives one error with span 0..2, not two errors. - The inner error for "\u12" in a string covers the full escape sequence. - The inner error for "\u{}" in a string covers the full escape sequence. Restructure the stack usage tests for the changes in logos 0.16: - Logos 0.16 lexes large ASCII tokens and large ASCII ignored runs with bounded stack usage. Logos 0.15 caused a stack overflow for these inputs in unoptimized builds. Enable the tests for these inputs to prevent a regression. - Logos 0.16 uses stack space proportional to the match length for runs of multi-byte characters, for example emoji in a string. Logos 0.15 lexed these inputs with bounded stack usage. Optimized builds are not affected. Narrow the ignored tests to these inputs and update the documentation. Benchmark results show no performance change outside the measurement noise. Assisted-By: devx/a4890344-dddf-4733-a94e-71230a3953d1 --- Cargo.lock | 37 +++----------- bluejay-parser/Cargo.toml | 2 +- bluejay-parser/src/lexer/logos_lexer.rs | 18 +++---- .../src/lexer/logos_lexer/safety_tests.rs | 49 +++++++++++-------- .../tests/lexer_adversarial_test.rs | 35 +++++++++---- 5 files changed, 71 insertions(+), 70 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c1541a7..b63fbd8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -57,12 +57,6 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" -[[package]] -name = "beef" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a8241f3ebb85c056b509d4327ad0358fbbba6ffb340bf388f26350aeda225b1" - [[package]] name = "bitflags" version = "2.11.0" @@ -659,12 +653,6 @@ dependencies = [ "wasm-bindgen", ] -[[package]] -name = "lazy_static" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" - [[package]] name = "leb128fmt" version = "0.1.0" @@ -691,34 +679,32 @@ checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" [[package]] name = "logos" -version = "0.15.1" +version = "0.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff472f899b4ec2d99161c51f60ff7075eeb3097069a36050d8037a6325eb8154" +checksum = "eb2c55a318a87600ea870ff8c2012148b44bf18b74fad48d0f835c38c7d07c5f" dependencies = [ "logos-derive", ] [[package]] name = "logos-codegen" -version = "0.15.1" +version = "0.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "192a3a2b90b0c05b27a0b2c43eecdb7c415e29243acc3f89cc8247a5b693045c" +checksum = "58b3ffaa284e1350d017a57d04ada118c4583cf260c8fb01e0fe28a2e9cf8970" dependencies = [ - "beef", "fnv", - "lazy_static", "proc-macro2", "quote", + "regex-automata", "regex-syntax", - "rustc_version", "syn 2.0.119", ] [[package]] name = "logos-derive" -version = "0.15.1" +version = "0.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "605d9697bcd5ef3a42d38efc51541aa3d6a4a25f7ab6d1ed0da5ac632a26b470" +checksum = "52d3a9855747c17eaf4383823f135220716ab49bea5fbea7dd42cc9a92f8aa31" dependencies = [ "logos-codegen", ] @@ -867,15 +853,6 @@ version = "0.8.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a" -[[package]] -name = "rustc_version" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" -dependencies = [ - "semver", -] - [[package]] name = "rustix" version = "1.1.4" diff --git a/bluejay-parser/Cargo.toml b/bluejay-parser/Cargo.toml index 43030f1..50fa8f0 100644 --- a/bluejay-parser/Cargo.toml +++ b/bluejay-parser/Cargo.toml @@ -11,7 +11,7 @@ exclude = [".gitignore", "tests/**/*"] description = "A GraphQL parser" [dependencies] -logos = { version = "0.15" } +logos = { version = "0.16" } enum-as-inner = "0.7" ariadne = { version = "0.5.0" } serde = { version = "1.0.203", optional = true } diff --git a/bluejay-parser/src/lexer/logos_lexer.rs b/bluejay-parser/src/lexer/logos_lexer.rs index 371e0ff..b879a60 100644 --- a/bluejay-parser/src/lexer/logos_lexer.rs +++ b/bluejay-parser/src/lexer/logos_lexer.rs @@ -24,7 +24,7 @@ pub(crate) struct Extras { #[logos(subpattern fixedunicode = r"\\u[0-9A-Fa-f]{4}")] #[logos(error = LexError)] #[logos(skip r"[\uFEFF\t \n\r,]+")] -#[logos(skip r"#[^\n\r]*")] // comments +#[logos(skip(r"#[^\n\r]*", allow_greedy = true))] // comments #[logos(extras = Extras)] pub(crate) enum Token<'a> { // Punctuators @@ -470,11 +470,9 @@ mod tests { vec![(Err(LexError::UnrecognizedToken), 0..1)], Token::lexer(".").spanned().collect::>(), ); + // Two dots give one error that spans the failed match attempt assert_eq!( - vec![ - (Err(LexError::UnrecognizedToken), 0..1), - (Err(LexError::UnrecognizedToken), 1..2), - ], + vec![(Err(LexError::UnrecognizedToken), 0..2)], Token::lexer("..").spanned().collect::>(), ); assert_eq!( @@ -589,21 +587,23 @@ mod tests { )], Token::lexer(r#""\q""#).spanned().collect::>(), ); - // A unicode escape sequence with too few digits + // A unicode escape sequence with too few digits. + // The inner error covers the full failed escape sequence. assert_eq!( vec![( Err(LexError::StringValueInvalid(vec![ - StringValueLexError::InvalidCharacters(Span::from(1..2)), + StringValueLexError::InvalidCharacters(Span::from(1..5)), ])), 0..6, )], Token::lexer(r#""\u12""#).spanned().collect::>(), ); - // A unicode escape sequence with no digits + // A unicode escape sequence with no digits. + // The inner error covers the full failed escape sequence. assert_eq!( vec![( Err(LexError::StringValueInvalid(vec![ - StringValueLexError::InvalidCharacters(Span::from(1..2)), + StringValueLexError::InvalidCharacters(Span::from(1..4)), ])), 0..6, )], diff --git a/bluejay-parser/src/lexer/logos_lexer/safety_tests.rs b/bluejay-parser/src/lexer/logos_lexer/safety_tests.rs index 310cd42..b7eda21 100644 --- a/bluejay-parser/src/lexer/logos_lexer/safety_tests.rs +++ b/bluejay-parser/src/lexer/logos_lexer/safety_tests.rs @@ -242,14 +242,19 @@ fn large_pathological_inputs_terminate() { format!("\"\"\"{}", "\\\"\"\"".repeat(25_000)), format!("\"\"\"{}\"\"\"", " \n".repeat(50_000)), "$".repeat(50_000), - // Many small numeric tokens. See large_single_token_runs_terminate - // for why one large numeric token is not included here. "9 ".repeat(50_000), "0 ".repeat(50_000), ".".repeat(50_000), format!("#{}", "c".repeat(100_000)), - // Keep runs of ignored characters below the crash threshold for - // unoptimized builds. See large_single_token_runs_terminate. + // Large ASCII tokens and large ASCII ignored runs crashed + // unoptimized builds with logos 0.15. Logos 0.16 lexes them + // with bounded stack usage. + "9".repeat(100_000), + format!("-1.{0}e-{0}", "9".repeat(50_000)), + " ".repeat(100_000), + "\t \r\n,".repeat(20_000), + // Keep runs of byte order marks below the crash threshold for + // unoptimized builds. See large_multibyte_runs_terminate. "\u{FEFF}\t \r\n,".repeat(1_000), ]; for input in large_inputs { @@ -257,13 +262,16 @@ fn large_pathological_inputs_terminate() { } } -/// With logos 0.15, the generated matchers for numeric tokens and for -/// runs of ignored characters use stack space proportional to the run -/// length in unoptimized builds. Optimized builds compile the recursion -/// into loops, so release builds accept runs of all lengths. As a result -/// of this, one numeric token with approximately 5,000 or more digits, -/// or one run of approximately 50,000 or more ignored characters, -/// crashes unoptimized builds. +/// With logos 0.16, the generated matchers for runs of multi-byte +/// characters use stack space proportional to the run length in +/// unoptimized builds. One thousand multi-byte characters use +/// approximately 1 MiB of stack. Optimized builds compile the +/// recursion into loops, so release builds accept runs of all lengths. +/// The affected inputs are strings with many multi-byte characters and +/// runs of many byte order marks. +/// Logos 0.15 had the same problem for numeric tokens and for runs of +/// ASCII ignored characters. Logos 0.16 corrected those cases and +/// introduced the multi-byte problem for strings. /// This test pins the current stack usage with some headroom. /// If it starts to abort with a stack overflow after a logos upgrade, /// then the stack usage per character became worse, which makes the @@ -277,27 +285,28 @@ fn long_token_stack_usage() { assert_lexes_safely(&format!("-1.{}e-9", "9".repeat(1_000))); assert_lexes_safely(&" ".repeat(1_000)); assert_lexes_safely(&"\u{FEFF}\t \r\n,".repeat(200)); + assert_lexes_safely(&format!("\"{}\"", "é".repeat(500))); }) .unwrap() .join() .unwrap(); } -/// One large numeric token or one large run of ignored characters must -/// lex safely. With logos 0.15, these inputs overflow the stack in -/// unoptimized builds, and the process aborts. Optimized builds are not -/// affected. See long_token_stack_usage for the details. +/// One large run of multi-byte characters must lex safely. With +/// logos 0.16, these inputs overflow the stack in unoptimized builds, +/// and the process aborts. Optimized builds are not affected. +/// See long_token_stack_usage for the details. /// This test is ignored because a failure aborts the full test process. /// Try to enable this test again after each logos upgrade: /// run `cargo test -p bluejay-parser --lib -- --ignored` in a debug /// build. If all tests pass, remove the ignore attribute. #[test] -#[ignore = "logos 0.15 overflows the stack on large single tokens in unoptimized builds; try to re-enable after the next logos upgrade"] -fn large_single_token_runs_terminate() { +#[ignore = "logos 0.16 overflows the stack on long runs of multi-byte characters in unoptimized builds; try to re-enable after the next logos upgrade"] +fn large_multibyte_runs_terminate() { let large_inputs = [ - "9".repeat(100_000), - format!("-1.{0}e-{0}", "9".repeat(50_000)), - " ".repeat(100_000), + format!("\"{}\"", "é".repeat(50_000)), + format!("\"{}\"", "🔥".repeat(25_000)), + "\u{FEFF}".repeat(30_000), "\u{FEFF}\t \r\n,".repeat(20_000), ]; for input in large_inputs { diff --git a/bluejay-parser/tests/lexer_adversarial_test.rs b/bluejay-parser/tests/lexer_adversarial_test.rs index ca5e968..816ee12 100644 --- a/bluejay-parser/tests/lexer_adversarial_test.rs +++ b/bluejay-parser/tests/lexer_adversarial_test.rs @@ -93,17 +93,11 @@ fn truncated_documents_parse_safely() { } } -/// Documents with one large numeric token or one large run of ignored -/// characters must parse safely. With logos 0.15, these inputs overflow -/// the stack in unoptimized builds, and the process aborts. Optimized -/// builds are not affected. The max tokens limit does not protect -/// against this, because each input is a single token. -/// This test is ignored because a failure aborts the full test process. -/// Try to enable this test again after each logos upgrade: -/// run `cargo test -p bluejay-parser --test lexer_adversarial_test -- --ignored` -/// in a debug build. If all tests pass, remove the ignore attribute. +/// Documents with one large numeric token or one large run of ASCII +/// ignored characters must parse safely. Logos 0.15 caused a stack +/// overflow for these inputs in unoptimized builds. Logos 0.16 parses +/// them with bounded stack usage. #[test] -#[ignore = "logos 0.15 overflows the stack on large single tokens in unoptimized builds; try to re-enable after the next logos upgrade"] fn large_single_token_documents_parse_safely() { let sources = [ format!("{{ a(b: {}) }}", "9".repeat(100_000)), @@ -115,6 +109,27 @@ fn large_single_token_documents_parse_safely() { } } +/// Documents with one large run of multi-byte characters must parse +/// safely. With logos 0.16, these inputs overflow the stack in +/// unoptimized builds, and the process aborts. Optimized builds are +/// not affected. The max tokens limit does not protect against this, +/// because each input is a single token. +/// This test is ignored because a failure aborts the full test process. +/// Try to enable this test again after each logos upgrade: +/// run `cargo test -p bluejay-parser --test lexer_adversarial_test -- --ignored` +/// in a debug build. If all tests pass, remove the ignore attribute. +#[test] +#[ignore = "logos 0.16 overflows the stack on long runs of multi-byte characters in unoptimized builds; try to re-enable after the next logos upgrade"] +fn large_multibyte_documents_parse_safely() { + let sources = [ + format!("{{ a(b: \"{}\") }}", "é".repeat(50_000)), + format!("{}{{ a }}", "\u{FEFF}".repeat(30_000)), + ]; + for source in sources { + assert_parses_safely(&source); + } +} + /// The max tokens limit is a denial-of-service protection. /// It must bound the work for large hostile documents. #[test] From 1131153d48489dadbf07856c62caaf49805357a8 Mon Sep 17 00:00:00 2001 From: Adam Petro Date: Wed, 12 Aug 2026 16:04:52 -0400 Subject: [PATCH 2/2] Test comments with all line terminators in the token stream test Add comment separators with carriage return and line feed, and with carriage return alone, to the kitchen sink token stream test. The test asserts exact spans for the tokens that follow each comment. This makes sure that a comment ends at each line terminator and does not consume the tokens on the next line. Assisted-By: devx/a4890344-dddf-4733-a94e-71230a3953d1 --- bluejay-parser/src/lexer/logos_lexer.rs | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/bluejay-parser/src/lexer/logos_lexer.rs b/bluejay-parser/src/lexer/logos_lexer.rs index b879a60..4d65dba 100644 --- a/bluejay-parser/src/lexer/logos_lexer.rs +++ b/bluejay-parser/src/lexer/logos_lexer.rs @@ -798,7 +798,17 @@ mod tests { fn kitchen_sink_token_stream_test() { // Lex a document with all token types and all ignored token types, // and compare the full token stream, with spans, to the expected stream. - let separators = [" ", ",", "\n", "\t", "\r\n", " # comment\n", "\u{FEFF}"]; + let separators = [ + " ", + ",", + "\n", + "\t", + "\r\n", + " # comment\n", + " # comment\r\n", + " #comment\r", + "\u{FEFF}", + ]; let parts = vec![ ("query", Token::Name("query")), ("MyQuery", Token::Name("MyQuery")),