diff --git a/Cargo.lock b/Cargo.lock index c1541a7..b63fbd8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -57,12 +57,6 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" -[[package]] -name = "beef" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a8241f3ebb85c056b509d4327ad0358fbbba6ffb340bf388f26350aeda225b1" - [[package]] name = "bitflags" version = "2.11.0" @@ -659,12 +653,6 @@ dependencies = [ "wasm-bindgen", ] -[[package]] -name = "lazy_static" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" - [[package]] name = "leb128fmt" version = "0.1.0" @@ -691,34 +679,32 @@ checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" [[package]] name = "logos" -version = "0.15.1" +version = "0.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff472f899b4ec2d99161c51f60ff7075eeb3097069a36050d8037a6325eb8154" +checksum = "eb2c55a318a87600ea870ff8c2012148b44bf18b74fad48d0f835c38c7d07c5f" dependencies = [ "logos-derive", ] [[package]] name = "logos-codegen" -version = "0.15.1" +version = "0.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "192a3a2b90b0c05b27a0b2c43eecdb7c415e29243acc3f89cc8247a5b693045c" +checksum = "58b3ffaa284e1350d017a57d04ada118c4583cf260c8fb01e0fe28a2e9cf8970" dependencies = [ - "beef", "fnv", - "lazy_static", "proc-macro2", "quote", + "regex-automata", "regex-syntax", - "rustc_version", "syn 2.0.119", ] [[package]] name = "logos-derive" -version = "0.15.1" +version = "0.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "605d9697bcd5ef3a42d38efc51541aa3d6a4a25f7ab6d1ed0da5ac632a26b470" +checksum = "52d3a9855747c17eaf4383823f135220716ab49bea5fbea7dd42cc9a92f8aa31" dependencies = [ "logos-codegen", ] @@ -867,15 +853,6 @@ version = "0.8.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a" -[[package]] -name = "rustc_version" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" -dependencies = [ - "semver", -] - [[package]] name = "rustix" version = "1.1.4" diff --git a/bluejay-parser/Cargo.toml b/bluejay-parser/Cargo.toml index 43030f1..50fa8f0 100644 --- a/bluejay-parser/Cargo.toml +++ b/bluejay-parser/Cargo.toml @@ -11,7 +11,7 @@ exclude = [".gitignore", "tests/**/*"] description = "A GraphQL parser" [dependencies] -logos = { version = "0.15" } +logos = { version = "0.16" } enum-as-inner = "0.7" ariadne = { version = "0.5.0" } serde = { version = "1.0.203", optional = true } diff --git a/bluejay-parser/src/lexer/logos_lexer.rs b/bluejay-parser/src/lexer/logos_lexer.rs index 371e0ff..4d65dba 100644 --- a/bluejay-parser/src/lexer/logos_lexer.rs +++ b/bluejay-parser/src/lexer/logos_lexer.rs @@ -24,7 +24,7 @@ pub(crate) struct Extras { #[logos(subpattern fixedunicode = r"\\u[0-9A-Fa-f]{4}")] #[logos(error = LexError)] #[logos(skip r"[\uFEFF\t \n\r,]+")] -#[logos(skip r"#[^\n\r]*")] // comments +#[logos(skip(r"#[^\n\r]*", allow_greedy = true))] // comments #[logos(extras = Extras)] pub(crate) enum Token<'a> { // Punctuators @@ -470,11 +470,9 @@ mod tests { vec![(Err(LexError::UnrecognizedToken), 0..1)], Token::lexer(".").spanned().collect::>(), ); + // Two dots give one error that spans the failed match attempt assert_eq!( - vec![ - (Err(LexError::UnrecognizedToken), 0..1), - (Err(LexError::UnrecognizedToken), 1..2), - ], + vec![(Err(LexError::UnrecognizedToken), 0..2)], Token::lexer("..").spanned().collect::>(), ); assert_eq!( @@ -589,21 +587,23 @@ mod tests { )], Token::lexer(r#""\q""#).spanned().collect::>(), ); - // A unicode escape sequence with too few digits + // A unicode escape sequence with too few digits. + // The inner error covers the full failed escape sequence. assert_eq!( vec![( Err(LexError::StringValueInvalid(vec![ - StringValueLexError::InvalidCharacters(Span::from(1..2)), + StringValueLexError::InvalidCharacters(Span::from(1..5)), ])), 0..6, )], Token::lexer(r#""\u12""#).spanned().collect::>(), ); - // A unicode escape sequence with no digits + // A unicode escape sequence with no digits. + // The inner error covers the full failed escape sequence. assert_eq!( vec![( Err(LexError::StringValueInvalid(vec![ - StringValueLexError::InvalidCharacters(Span::from(1..2)), + StringValueLexError::InvalidCharacters(Span::from(1..4)), ])), 0..6, )], @@ -798,7 +798,17 @@ mod tests { fn kitchen_sink_token_stream_test() { // Lex a document with all token types and all ignored token types, // and compare the full token stream, with spans, to the expected stream. - let separators = [" ", ",", "\n", "\t", "\r\n", " # comment\n", "\u{FEFF}"]; + let separators = [ + " ", + ",", + "\n", + "\t", + "\r\n", + " # comment\n", + " # comment\r\n", + " #comment\r", + "\u{FEFF}", + ]; let parts = vec![ ("query", Token::Name("query")), ("MyQuery", Token::Name("MyQuery")), diff --git a/bluejay-parser/src/lexer/logos_lexer/safety_tests.rs b/bluejay-parser/src/lexer/logos_lexer/safety_tests.rs index 310cd42..b7eda21 100644 --- a/bluejay-parser/src/lexer/logos_lexer/safety_tests.rs +++ b/bluejay-parser/src/lexer/logos_lexer/safety_tests.rs @@ -242,14 +242,19 @@ fn large_pathological_inputs_terminate() { format!("\"\"\"{}", "\\\"\"\"".repeat(25_000)), format!("\"\"\"{}\"\"\"", " \n".repeat(50_000)), "$".repeat(50_000), - // Many small numeric tokens. See large_single_token_runs_terminate - // for why one large numeric token is not included here. "9 ".repeat(50_000), "0 ".repeat(50_000), ".".repeat(50_000), format!("#{}", "c".repeat(100_000)), - // Keep runs of ignored characters below the crash threshold for - // unoptimized builds. See large_single_token_runs_terminate. + // Large ASCII tokens and large ASCII ignored runs crashed + // unoptimized builds with logos 0.15. Logos 0.16 lexes them + // with bounded stack usage. + "9".repeat(100_000), + format!("-1.{0}e-{0}", "9".repeat(50_000)), + " ".repeat(100_000), + "\t \r\n,".repeat(20_000), + // Keep runs of byte order marks below the crash threshold for + // unoptimized builds. See large_multibyte_runs_terminate. "\u{FEFF}\t \r\n,".repeat(1_000), ]; for input in large_inputs { @@ -257,13 +262,16 @@ fn large_pathological_inputs_terminate() { } } -/// With logos 0.15, the generated matchers for numeric tokens and for -/// runs of ignored characters use stack space proportional to the run -/// length in unoptimized builds. Optimized builds compile the recursion -/// into loops, so release builds accept runs of all lengths. As a result -/// of this, one numeric token with approximately 5,000 or more digits, -/// or one run of approximately 50,000 or more ignored characters, -/// crashes unoptimized builds. +/// With logos 0.16, the generated matchers for runs of multi-byte +/// characters use stack space proportional to the run length in +/// unoptimized builds. One thousand multi-byte characters use +/// approximately 1 MiB of stack. Optimized builds compile the +/// recursion into loops, so release builds accept runs of all lengths. +/// The affected inputs are strings with many multi-byte characters and +/// runs of many byte order marks. +/// Logos 0.15 had the same problem for numeric tokens and for runs of +/// ASCII ignored characters. Logos 0.16 corrected those cases and +/// introduced the multi-byte problem for strings. /// This test pins the current stack usage with some headroom. /// If it starts to abort with a stack overflow after a logos upgrade, /// then the stack usage per character became worse, which makes the @@ -277,27 +285,28 @@ fn long_token_stack_usage() { assert_lexes_safely(&format!("-1.{}e-9", "9".repeat(1_000))); assert_lexes_safely(&" ".repeat(1_000)); assert_lexes_safely(&"\u{FEFF}\t \r\n,".repeat(200)); + assert_lexes_safely(&format!("\"{}\"", "é".repeat(500))); }) .unwrap() .join() .unwrap(); } -/// One large numeric token or one large run of ignored characters must -/// lex safely. With logos 0.15, these inputs overflow the stack in -/// unoptimized builds, and the process aborts. Optimized builds are not -/// affected. See long_token_stack_usage for the details. +/// One large run of multi-byte characters must lex safely. With +/// logos 0.16, these inputs overflow the stack in unoptimized builds, +/// and the process aborts. Optimized builds are not affected. +/// See long_token_stack_usage for the details. /// This test is ignored because a failure aborts the full test process. /// Try to enable this test again after each logos upgrade: /// run `cargo test -p bluejay-parser --lib -- --ignored` in a debug /// build. If all tests pass, remove the ignore attribute. #[test] -#[ignore = "logos 0.15 overflows the stack on large single tokens in unoptimized builds; try to re-enable after the next logos upgrade"] -fn large_single_token_runs_terminate() { +#[ignore = "logos 0.16 overflows the stack on long runs of multi-byte characters in unoptimized builds; try to re-enable after the next logos upgrade"] +fn large_multibyte_runs_terminate() { let large_inputs = [ - "9".repeat(100_000), - format!("-1.{0}e-{0}", "9".repeat(50_000)), - " ".repeat(100_000), + format!("\"{}\"", "é".repeat(50_000)), + format!("\"{}\"", "🔥".repeat(25_000)), + "\u{FEFF}".repeat(30_000), "\u{FEFF}\t \r\n,".repeat(20_000), ]; for input in large_inputs { diff --git a/bluejay-parser/tests/lexer_adversarial_test.rs b/bluejay-parser/tests/lexer_adversarial_test.rs index ca5e968..816ee12 100644 --- a/bluejay-parser/tests/lexer_adversarial_test.rs +++ b/bluejay-parser/tests/lexer_adversarial_test.rs @@ -93,17 +93,11 @@ fn truncated_documents_parse_safely() { } } -/// Documents with one large numeric token or one large run of ignored -/// characters must parse safely. With logos 0.15, these inputs overflow -/// the stack in unoptimized builds, and the process aborts. Optimized -/// builds are not affected. The max tokens limit does not protect -/// against this, because each input is a single token. -/// This test is ignored because a failure aborts the full test process. -/// Try to enable this test again after each logos upgrade: -/// run `cargo test -p bluejay-parser --test lexer_adversarial_test -- --ignored` -/// in a debug build. If all tests pass, remove the ignore attribute. +/// Documents with one large numeric token or one large run of ASCII +/// ignored characters must parse safely. Logos 0.15 caused a stack +/// overflow for these inputs in unoptimized builds. Logos 0.16 parses +/// them with bounded stack usage. #[test] -#[ignore = "logos 0.15 overflows the stack on large single tokens in unoptimized builds; try to re-enable after the next logos upgrade"] fn large_single_token_documents_parse_safely() { let sources = [ format!("{{ a(b: {}) }}", "9".repeat(100_000)), @@ -115,6 +109,27 @@ fn large_single_token_documents_parse_safely() { } } +/// Documents with one large run of multi-byte characters must parse +/// safely. With logos 0.16, these inputs overflow the stack in +/// unoptimized builds, and the process aborts. Optimized builds are +/// not affected. The max tokens limit does not protect against this, +/// because each input is a single token. +/// This test is ignored because a failure aborts the full test process. +/// Try to enable this test again after each logos upgrade: +/// run `cargo test -p bluejay-parser --test lexer_adversarial_test -- --ignored` +/// in a debug build. If all tests pass, remove the ignore attribute. +#[test] +#[ignore = "logos 0.16 overflows the stack on long runs of multi-byte characters in unoptimized builds; try to re-enable after the next logos upgrade"] +fn large_multibyte_documents_parse_safely() { + let sources = [ + format!("{{ a(b: \"{}\") }}", "é".repeat(50_000)), + format!("{}{{ a }}", "\u{FEFF}".repeat(30_000)), + ]; + for source in sources { + assert_parses_safely(&source); + } +} + /// The max tokens limit is a denial-of-service protection. /// It must bound the work for large hostile documents. #[test]