From 89e11e5cfcea4d935a78fd8885603c94698dd2b7 Mon Sep 17 00:00:00 2001 From: Taus Date: Mon, 6 Jul 2026 21:19:45 +0000 Subject: [PATCH 01/13] unified: Add `swift-syntax-rs` Adds an initial prototype of an interface from Rust to Swift, which enables us to use the `swift-syntax` package for parsing. At present, the parsed AST is passed between Swift and Rust as a JSON string. --- Cargo.lock | 4 + Cargo.toml | 1 + MODULE.bazel | 35 +++- unified/swift-syntax-rs/.gitignore | 2 + unified/swift-syntax-rs/.swift-version | 1 + unified/swift-syntax-rs/BUILD.bazel | 55 ++++++ unified/swift-syntax-rs/Cargo.toml | 14 ++ unified/swift-syntax-rs/README.md | 180 ++++++++++++++++++ unified/swift-syntax-rs/build.rs | 102 ++++++++++ unified/swift-syntax-rs/src/lib.rs | 125 ++++++++++++ unified/swift-syntax-rs/src/main.rs | 42 ++++ .../swift-syntax-rs/swift/Package.resolved | 15 ++ unified/swift-syntax-rs/swift/Package.swift | 30 +++ .../SwiftSyntaxFFI/SwiftSyntaxFFI.swift | 145 ++++++++++++++ 14 files changed, 750 insertions(+), 1 deletion(-) create mode 100644 unified/swift-syntax-rs/.gitignore create mode 100644 unified/swift-syntax-rs/.swift-version create mode 100644 unified/swift-syntax-rs/BUILD.bazel create mode 100644 unified/swift-syntax-rs/Cargo.toml create mode 100644 unified/swift-syntax-rs/README.md create mode 100644 unified/swift-syntax-rs/build.rs create mode 100644 unified/swift-syntax-rs/src/lib.rs create mode 100644 unified/swift-syntax-rs/src/main.rs create mode 100644 unified/swift-syntax-rs/swift/Package.resolved create mode 100644 unified/swift-syntax-rs/swift/Package.swift create mode 100644 unified/swift-syntax-rs/swift/Sources/SwiftSyntaxFFI/SwiftSyntaxFFI.swift diff --git a/Cargo.lock b/Cargo.lock index 76043ec0a439..7a9f19667911 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2850,6 +2850,10 @@ version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" +[[package]] +name = "swift-syntax-rs" +version = "0.1.0" + [[package]] name = "syn" version = "2.0.106" diff --git a/Cargo.toml b/Cargo.toml index 9c15b486062b..9f3780bb1d77 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,6 +10,7 @@ members = [ "ruby/extractor", "unified/extractor", "unified/extractor/tree-sitter-swift", + "unified/swift-syntax-rs", "rust/extractor", "rust/extractor/macros", "rust/ast-generator", diff --git a/MODULE.bazel b/MODULE.bazel index e6f1ddb893dd..aa1f6555e0ca 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -24,13 +24,15 @@ bazel_dep(name = "rules_python", version = "1.9.0") bazel_dep(name = "rules_shell", version = "0.7.1") bazel_dep(name = "bazel_skylib", version = "1.9.0") bazel_dep(name = "abseil-cpp", version = "20260107.1", repo_name = "absl") -bazel_dep(name = "nlohmann_json", version = "3.11.3", repo_name = "json") +bazel_dep(name = "nlohmann_json", version = "3.12.0.bcr.1", repo_name = "json") bazel_dep(name = "fmt", version = "12.1.0-codeql.1") bazel_dep(name = "rules_kotlin", version = "2.2.2-codeql.1") bazel_dep(name = "gazelle", version = "0.50.0") bazel_dep(name = "rules_dotnet", version = "0.21.5-codeql.1") bazel_dep(name = "googletest", version = "1.17.0.bcr.2") bazel_dep(name = "rules_rust", version = "0.69.0") +bazel_dep(name = "rules_swift", version = "4.0.0-rc4") +bazel_dep(name = "swift-syntax", version = "602.0.0.bcr.2") bazel_dep(name = "zstd", version = "1.5.7.bcr.1") bazel_dep(name = "buildifier_prebuilt", version = "6.4.0", dev_dependency = True) @@ -219,6 +221,37 @@ use_repo( "swift-resource-dir-macos", ) +# Hermetic Swift toolchain (from swift.org) for building the `swift-syntax` +# based Rust wrapper in `unified/swift-syntax-rs`. This is the official +# `rules_swift` standalone-toolchain extension and is independent of the +# patched prebuilt toolchain wired up via `swift_deps` above. +# +# Bazel cannot auto-select between Linux distributions, so the toolchain is +# registered explicitly for the distribution our CI runs on (ubuntu24.04, +# x86_64). Add further platforms here if CI grows to cover them. +# +# We register the `exec` toolchain (normal host compilation, targeting +# linux/x86_64) rather than the `embedded` one, which targets `os:none` +# (Embedded Swift) and would not match a normal host build. +# +# The Swift version is read from `unified/swift-syntax-rs/.swift-version`, which +# is the single source of truth shared with the local `cargo` build (via +# swiftly / any Swift install) and `swift/Package.swift`. +swift = use_extension("@rules_swift//swift:extensions.bzl", "swift") +swift.toolchain( + name = "swift_toolchain", + swift_version_file = "//unified/swift-syntax-rs:.swift-version", +) +use_repo( + swift, + "swift_toolchain", + "swift_toolchain_ubuntu24.04", +) + +register_toolchains( + "@swift_toolchain//:swift_toolchain_exec_ubuntu24.04", +) + node = use_extension("@rules_nodejs//nodejs:extensions.bzl", "node") node.toolchain( name = "nodejs", diff --git a/unified/swift-syntax-rs/.gitignore b/unified/swift-syntax-rs/.gitignore new file mode 100644 index 000000000000..0b6496865408 --- /dev/null +++ b/unified/swift-syntax-rs/.gitignore @@ -0,0 +1,2 @@ +/target +/swift/.build diff --git a/unified/swift-syntax-rs/.swift-version b/unified/swift-syntax-rs/.swift-version new file mode 100644 index 000000000000..42cc526d6ca7 --- /dev/null +++ b/unified/swift-syntax-rs/.swift-version @@ -0,0 +1 @@ +6.2.4 diff --git a/unified/swift-syntax-rs/BUILD.bazel b/unified/swift-syntax-rs/BUILD.bazel new file mode 100644 index 000000000000..10d11a1bec11 --- /dev/null +++ b/unified/swift-syntax-rs/BUILD.bazel @@ -0,0 +1,55 @@ +load("@rules_rust//rust:defs.bzl", "rust_binary", "rust_library", "rust_test") +load("@rules_swift//swift:swift.bzl", "swift_library") + +package(default_visibility = ["//visibility:public"]) + +# The pinned Swift version, shared with the local `cargo` build and +# `swift/Package.swift`. Referenced by the `swift.toolchain` extension in +# //:MODULE.bazel via `swift_version_file`. +exports_files([".swift-version"]) + +# The Swift FFI shim: wraps swift-syntax and exposes a small C ABI +# (`ssr_parse_json` / `ssr_string_free`). Built with the hermetic Swift +# toolchain registered in //:MODULE.bazel; provides a `CcInfo` that the Rust +# targets below link against. +swift_library( + name = "swift_syntax_ffi", + srcs = ["swift/Sources/SwiftSyntaxFFI/SwiftSyntaxFFI.swift"], + module_name = "SwiftSyntaxFFI", + deps = [ + "@swift-syntax//:SwiftParser", + "@swift-syntax//:SwiftSyntax", + ], +) + +# Safe Rust bindings on top of the C ABI. `build.rs` is intentionally *not* +# wired up here (no `cargo_build_script`): under Bazel the Swift side is +# provided by `:swift_syntax_ffi`, whereas `build.rs` only exists to build the +# Swift shim in the local `cargo` workflow. +rust_library( + name = "swift_syntax_rs", + srcs = ["src/lib.rs"], + edition = "2024", + deps = [":swift_syntax_ffi"], +) + +rust_binary( + name = "swift-syntax-parse", + srcs = ["src/main.rs"], + # The Swift toolchain propagates a runfiles-relative RPATH to its runtime + # `.so`s via `swift_syntax_ffi`'s `CcInfo`, but (unlike `swift_binary`) + # `rust_binary` does not copy those libraries into runfiles. Add the + # toolchain files as `data` so the runtime is found at execution time. + # (rules_swift does not yet support statically linking the runtime.) + data = ["@swift_toolchain_ubuntu24.04//:files"], + edition = "2024", + deps = [":swift_syntax_rs"], +) + +rust_test( + name = "swift_syntax_rs_test", + size = "small", + crate = ":swift_syntax_rs", + data = ["@swift_toolchain_ubuntu24.04//:files"], + edition = "2024", +) diff --git a/unified/swift-syntax-rs/Cargo.toml b/unified/swift-syntax-rs/Cargo.toml new file mode 100644 index 000000000000..6957fb0e50e9 --- /dev/null +++ b/unified/swift-syntax-rs/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "swift-syntax-rs" +description = "Rust wrapper around the swift-syntax package for parsing Swift source" +version = "0.1.0" +authors = ["GitHub"] +edition = "2024" + +[lib] +name = "swift_syntax_rs" +path = "src/lib.rs" + +[[bin]] +name = "swift-syntax-parse" +path = "src/main.rs" diff --git a/unified/swift-syntax-rs/README.md b/unified/swift-syntax-rs/README.md new file mode 100644 index 000000000000..08796b3a6a7d --- /dev/null +++ b/unified/swift-syntax-rs/README.md @@ -0,0 +1,180 @@ +# swift-syntax-rs + +A Rust wrapper around the [swift-syntax](https://github.com/swiftlang/swift-syntax) +package, allowing Swift source code to be parsed from Rust. + +Parsing is delegated to a small Swift shim (in [`swift/`](swift/)) that links +against `SwiftSyntax`/`SwiftParser` and exposes a tiny C ABI. The Rust crate +builds that shim (via `build.rs`) and provides safe bindings on top of it. + +## Output format + +The emitted JSON tree preserves the AST's named structure. Every node has a +`kind` and a `range` with `start`/`end` positions (UTF-8 `offset` plus 1-based +`line`/`column`). Beyond that: + +- **Tokens** carry `text`, `tokenKind`, and `leadingTrivia`/`trailingTrivia` + arrays of `{ kind, text }` pieces. +- **Layout nodes** (e.g. `functionDecl`) embed their children directly as + members keyed by the child's name in the parent (`name`, `signature`, + `body`, …), alongside `kind`/`range`. Absent optional children are omitted. +- **Collection nodes** (e.g. `codeBlockItemList`) are elided: a list-valued + field is simply a JSON array of its elements (e.g. `parameters`, `statements`). + This drops the collection node's own `kind`/`range`. + +Only meaningful trivia is kept — the four comment kinds (`lineComment`, +`blockComment`, `docLineComment`, `docBlockComment`) and `unexpectedText` +(source the parser skipped). Whitespace trivia is dropped, since node ranges +already encode positions. + +### Example + +Parsing `let x = 1 // c` produces the following (each `range` object is +abbreviated here as `…`): + +```jsonc +{ + "kind": "sourceFile", + "range": …, + "statements": [ // collection node elided to an array + { + "kind": "codeBlockItem", + "range": …, + "item": { + "kind": "variableDecl", + "range": …, + "attributes": [], // empty collection → empty array + "modifiers": [], + "bindingSpecifier": { // a token + "kind": "token", + "text": "let", + "tokenKind": "keyword(SwiftSyntax.Keyword.let)", + "range": …, + "leadingTrivia": [], + "trailingTrivia": [] + }, + "bindings": [ + { + "kind": "patternBinding", + "range": …, + "pattern": { + "kind": "identifierPattern", + "range": …, + "identifier": { "kind": "token", "text": "x", "tokenKind": "identifier(\"x\")", "range": …, "leadingTrivia": [], "trailingTrivia": [] } + }, + "initializer": { + "kind": "initializerClause", + "range": …, + "equal": { "kind": "token", "text": "=", "tokenKind": "equal", "range": …, "leadingTrivia": [], "trailingTrivia": [] }, + "value": { + "kind": "integerLiteralExpr", + "range": …, + "literal": { + "kind": "token", + "text": "1", + "tokenKind": "integerLiteral(\"1\")", + "range": …, + "leadingTrivia": [], + "trailingTrivia": [ { "kind": "lineComment", "text": "// c" } ] + } + } + } + } + ] + } + } + ], + "endOfFileToken": { "kind": "token", "text": "", "tokenKind": "endOfFile", "range": …, "leadingTrivia": [], "trailingTrivia": [] } +} +``` + +Note how `statements`, `bindings`, `attributes`, and `modifiers` are plain +arrays (their collection nodes are elided), layout children such as +`bindingSpecifier` and `initializer` are embedded by name, and the `// c` +comment rides along as `trailingTrivia` on the token it follows. + +## Prerequisites + +The build does not depend on any particular version manager. You need: + +- **Rust** — pinned to `1.88` by the repo-root [`rust-toolchain.toml`](../../rust-toolchain.toml), + which `rustup` picks up automatically. +- **Swift** — pinned to the version in [`.swift-version`](.swift-version) + (currently `6.2.4`), used to build `swift-syntax` `602.0.0`. Install it any way + you like — [swift.org](https://www.swift.org/install/) or + [swiftly](https://www.swift.org/swiftly/) (which reads `.swift-version`), or a + system package. Just make sure `swift` is on your `PATH` (or point `build.rs` + at it with the `SWIFT` environment variable). + +On Debian/Ubuntu the Swift runtime also needs `libncurses6` (and related libs) +available on the system. + +## Building & testing + +With `cargo` and `swift` on `PATH`: + +```sh +cargo build +cargo test +``` + +If your `swift`/`swiftc` are not on `PATH`, point the build at them explicitly: + +```sh +SWIFT=/path/to/swift SWIFTC=/path/to/swiftc cargo build +``` + +The first build compiles `swift-syntax` and can take several minutes. + +## Building with Bazel (CI) + +CI builds this crate hermetically with Bazel. A Swift toolchain is downloaded +from swift.org by the official `rules_swift` standalone toolchain extension +(wired up in the repo-root `MODULE.bazel`), `swift-syntax` is pulled from the +Bazel Central Registry, and the FFI shim is compiled as a `swift_library` that +the Rust targets link against. `build.rs` is not used under Bazel; it only +builds the Swift shim for the local `cargo` workflow. + +```sh +bazel build //unified/swift-syntax-rs:swift-syntax-parse +bazel test //unified/swift-syntax-rs:swift_syntax_rs_test +bazel run //unified/swift-syntax-rs:swift-syntax-parse < some.swift +``` + +Requirements: + +- **`clang`** must be installed on the runner. `rules_swift` requires the Bazel + CC toolchain to use clang; the repo's `.bazelrc` already sets + `--repo_env=CC=clang`, so no extra flags are needed. +- The registered Swift toolchain currently targets **ubuntu24.04 / x86_64** + only (Bazel cannot auto-select between Linux distributions). Add more + platforms in `MODULE.bazel` (`swift.toolchain` + `register_toolchains`) if CI + grows to cover them. + +The Swift compiler version is read from [`.swift-version`](.swift-version) by +both the Bazel toolchain (`swift.toolchain(swift_version_file = …)`) and the +local build, and is kept in sync with the `swift-syntax` release pinned in +`swift/Package.swift`, so local and CI builds behave identically. + +## Usage + +Library: + +```rust +let json = swift_syntax_rs::parse_to_json("let x = 1")?; +println!("{json}"); +``` + +CLI (reads a file argument or stdin, prints the syntax tree as JSON): + +```sh +echo 'let x = 1' | cargo run --bin swift-syntax-parse +``` + +## Layout + +- `swift/` — Swift package exposing the `ssr_parse_json` / `ssr_string_free` C ABI. +- `build.rs` — builds the Swift package and emits link/rpath flags (local `cargo` only). +- `BUILD.bazel` — Bazel targets for the hermetic CI build (swift_library + rust targets). +- `src/lib.rs` — safe Rust bindings (`parse_to_json`). +- `src/main.rs` — demo CLI. diff --git a/unified/swift-syntax-rs/build.rs b/unified/swift-syntax-rs/build.rs new file mode 100644 index 000000000000..0adb140140dc --- /dev/null +++ b/unified/swift-syntax-rs/build.rs @@ -0,0 +1,102 @@ +use std::env; +use std::path::PathBuf; +use std::process::Command; + +fn main() { + let manifest_dir = PathBuf::from(env::var("CARGO_MANIFEST_DIR").unwrap()); + let swift_dir = manifest_dir.join("swift"); + + println!( + "cargo:rerun-if-changed={}", + swift_dir.join("Package.swift").display() + ); + println!( + "cargo:rerun-if-changed={}", + swift_dir + .join("Sources/SwiftSyntaxFFI/SwiftSyntaxFFI.swift") + .display() + ); + println!("cargo:rerun-if-env-changed=SWIFT"); + println!("cargo:rerun-if-env-changed=SWIFTC"); + + // Build the Swift FFI package as a release dynamic library. + let mut command = Command::new(swift_bin()); + command + .args(["build", "-c", "release"]) + .current_dir(&swift_dir); + apply_bare_repository_workaround(&mut command); + let status = command.status().unwrap_or_else(|e| { + panic!( + "failed to run `{swift} build`: {e}\n\ + Install a Swift toolchain (see https://www.swift.org/install/, e.g. via \ + swiftly) and ensure `swift` is on PATH, or set the `SWIFT` environment \ + variable to the `swift` executable. The pinned version is in `.swift-version`.", + swift = swift_bin(), + ) + }); + assert!(status.success(), "`swift build` failed"); + + // Link against the freshly built dynamic library. + let build_dir = swift_dir.join(".build/release"); + println!("cargo:rustc-link-search=native={}", build_dir.display()); + println!("cargo:rustc-link-lib=dylib=SwiftSyntaxFFI"); + println!("cargo:rustc-link-arg=-Wl,-rpath,{}", build_dir.display()); + + // The executable also needs to find the Swift runtime libraries at run time. + if let Some(runtime) = swift_runtime_dir() { + println!("cargo:rustc-link-search=native={}", runtime.display()); + println!("cargo:rustc-link-arg=-Wl,-rpath,{}", runtime.display()); + } +} + +/// Query the active Swift toolchain for the directory containing its runtime +/// shared libraries (e.g. `libswiftCore.so`). +fn swift_runtime_dir() -> Option { + let output = Command::new(swiftc_bin()) + .arg("-print-target-info") + .output() + .ok()?; + if !output.status.success() { + return None; + } + let info = String::from_utf8_lossy(&output.stdout); + + // Extract the value of `"runtimeResourcePath": "..."` without pulling in a + // JSON dependency. + let key = "\"runtimeResourcePath\""; + let start = info.find(key)?; + let rest = &info[start + key.len()..]; + let colon = rest.find(':')?; + let after = &rest[colon + 1..]; + let open = after.find('"')?; + let value_start = &after[open + 1..]; + let close = value_start.find('"')?; + let resource_path = &value_start[..close]; + + Some(PathBuf::from(resource_path).join("linux")) +} + +/// The `swift` driver to invoke: `$SWIFT` if set, otherwise `swift` from `PATH`. +/// This keeps the build tool-agnostic — any Swift install works; no particular +/// version manager is required. +fn swift_bin() -> String { + env::var("SWIFT").unwrap_or_else(|_| "swift".to_string()) +} + +/// The `swiftc` compiler to invoke: `$SWIFTC` if set, otherwise `swiftc` from +/// `PATH`. +fn swiftc_bin() -> String { + env::var("SWIFTC").unwrap_or_else(|_| "swiftc".to_string()) +} + +/// Some environments (notably GitHub Codespaces) inject +/// `GIT_CONFIG_KEY_0=safe.bareRepository` / `GIT_CONFIG_VALUE_0=explicit`, which +/// breaks the cached bare git repositories `swift build` uses. When exactly that +/// key is present, relax it to `all` for the `swift build` subprocess only +/// (rather than unconditionally, which could clobber an unrelated +/// `GIT_CONFIG_VALUE_0`). +fn apply_bare_repository_workaround(command: &mut Command) { + if env::var("GIT_CONFIG_KEY_0").as_deref() == Ok("safe.bareRepository") { + command.env("GIT_CONFIG_VALUE_0", "all"); + } +} diff --git a/unified/swift-syntax-rs/src/lib.rs b/unified/swift-syntax-rs/src/lib.rs new file mode 100644 index 000000000000..83dc5e2c6fbd --- /dev/null +++ b/unified/swift-syntax-rs/src/lib.rs @@ -0,0 +1,125 @@ +//! Rust wrapper around the [swift-syntax](https://github.com/swiftlang/swift-syntax) +//! package, allowing Swift source code to be parsed from Rust. +//! +//! The heavy lifting is done by a small Swift shim (see `swift/`) that links +//! against `SwiftSyntax`/`SwiftParser` and exposes a tiny C ABI. This module +//! provides safe Rust bindings on top of that ABI. + +use std::ffi::{CStr, CString}; +use std::os::raw::c_char; + +// C ABI exported by the `SwiftSyntaxFFI` dynamic library. +unsafe extern "C" { + /// Parse a NUL-terminated Swift source string, returning a heap-allocated + /// NUL-terminated JSON string (or null on failure). The caller owns the + /// returned pointer and must release it with `ssr_string_free`. + fn ssr_parse_json(source: *const c_char) -> *mut c_char; + + /// Free a string previously returned by `ssr_parse_json`. + fn ssr_string_free(ptr: *mut c_char); +} + +/// Errors that can occur while parsing Swift source. +#[derive(Debug)] +pub enum ParseError { + /// The provided source contained an interior NUL byte. + NulByte, + /// The Swift side failed to produce a result. + SwiftFailure, +} + +impl std::fmt::Display for ParseError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + ParseError::NulByte => write!(f, "source contained an interior NUL byte"), + ParseError::SwiftFailure => write!(f, "swift-syntax failed to parse the source"), + } + } +} + +impl std::error::Error for ParseError {} + +/// Parse the given Swift `source` and return a JSON representation of its +/// syntax tree. +/// +/// # Example +/// ```no_run +/// let json = swift_syntax_rs::parse_to_json("let x = 1").unwrap(); +/// println!("{json}"); +/// ``` +pub fn parse_to_json(source: &str) -> Result { + let c_source = CString::new(source).map_err(|_| ParseError::NulByte)?; + + // SAFETY: `c_source` is a valid NUL-terminated string for the duration of + // the call. The returned pointer, if non-null, is owned by us and freed via + // `ssr_string_free` before returning. + unsafe { + let ptr = ssr_parse_json(c_source.as_ptr()); + if ptr.is_null() { + return Err(ParseError::SwiftFailure); + } + let json = CStr::from_ptr(ptr).to_string_lossy().into_owned(); + ssr_string_free(ptr); + Ok(json) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parses_simple_declaration() { + let json = parse_to_json("let x = 1").expect("parsing should succeed"); + assert!( + json.contains("\"kind\":\"sourceFile\""), + "unexpected tree: {json}" + ); + assert!(json.contains("\"text\":\"x\""), "unexpected tree: {json}"); + // Source ranges are emitted for every node. + assert!(json.contains("\"range\""), "missing ranges: {json}"); + assert!( + json.contains("\"line\"") && json.contains("\"column\"") && json.contains("\"offset\""), + "missing location fields: {json}" + ); + } + + #[test] + fn preserves_named_structure() { + let json = parse_to_json("let x = 1").expect("parsing should succeed"); + // Layout children are embedded directly under their field name. + assert!( + json.contains("\"initializer\""), + "missing initializer field: {json}" + ); + // Collection-valued fields are elided to plain JSON arrays. + assert!( + json.contains("\"statements\":["), + "statements should be an array: {json}" + ); + assert!( + json.contains("\"bindings\":["), + "bindings should be an array: {json}" + ); + } + + #[test] + fn parses_empty_source() { + let json = parse_to_json("").expect("parsing empty source should succeed"); + assert!( + json.contains("\"kind\":\"sourceFile\""), + "unexpected tree: {json}" + ); + } + + #[test] + fn captures_trivia() { + // A leading comment is kept as trivia on the token it precedes. + let json = parse_to_json("// hi\nlet x = 1").expect("parsing should succeed"); + assert!(json.contains("\"leadingTrivia\""), "missing trivia: {json}"); + assert!( + json.contains("lineComment"), + "comment trivia not captured: {json}" + ); + } +} diff --git a/unified/swift-syntax-rs/src/main.rs b/unified/swift-syntax-rs/src/main.rs new file mode 100644 index 000000000000..d79cd2ffef1f --- /dev/null +++ b/unified/swift-syntax-rs/src/main.rs @@ -0,0 +1,42 @@ +//! Small demo CLI: parse Swift source and print the syntax tree as JSON. +//! +//! Usage: +//! swift-syntax-parse [FILE] +//! +//! Reads from FILE if given, otherwise from standard input. + +use std::io::Read; +use std::process::ExitCode; + +fn main() -> ExitCode { + let source = match read_source() { + Ok(s) => s, + Err(e) => { + eprintln!("error reading source: {e}"); + return ExitCode::FAILURE; + } + }; + + match swift_syntax_rs::parse_to_json(&source) { + Ok(json) => { + println!("{json}"); + ExitCode::SUCCESS + } + Err(e) => { + eprintln!("error: {e}"); + ExitCode::FAILURE + } + } +} + +fn read_source() -> std::io::Result { + let mut args = std::env::args().skip(1); + match args.next() { + Some(path) => std::fs::read_to_string(path), + None => { + let mut buf = String::new(); + std::io::stdin().read_to_string(&mut buf)?; + Ok(buf) + } + } +} diff --git a/unified/swift-syntax-rs/swift/Package.resolved b/unified/swift-syntax-rs/swift/Package.resolved new file mode 100644 index 000000000000..9cebf34ec543 --- /dev/null +++ b/unified/swift-syntax-rs/swift/Package.resolved @@ -0,0 +1,15 @@ +{ + "originHash" : "ad84ad340ee01aa6e38b877dd29e38fa9cb40603dfdefa4e6a69e27f2db208d1", + "pins" : [ + { + "identity" : "swift-syntax", + "kind" : "remoteSourceControl", + "location" : "https://github.com/swiftlang/swift-syntax.git", + "state" : { + "revision" : "4799286537280063c85a32f09884cfbca301b1a1", + "version" : "602.0.0" + } + } + ], + "version" : 3 +} diff --git a/unified/swift-syntax-rs/swift/Package.swift b/unified/swift-syntax-rs/swift/Package.swift new file mode 100644 index 000000000000..1b0111364ce7 --- /dev/null +++ b/unified/swift-syntax-rs/swift/Package.swift @@ -0,0 +1,30 @@ +// swift-tools-version:6.0 +import PackageDescription + +let package = Package( + name: "SwiftSyntaxFFI", + products: [ + // Dynamic library so the produced .so records its dependency on the + // Swift runtime libraries, keeping the Rust link step simple. + .library( + name: "SwiftSyntaxFFI", + type: .dynamic, + targets: ["SwiftSyntaxFFI"] + ) + ], + dependencies: [ + .package( + url: "https://github.com/swiftlang/swift-syntax.git", + exact: "602.0.0" + ) + ], + targets: [ + .target( + name: "SwiftSyntaxFFI", + dependencies: [ + .product(name: "SwiftSyntax", package: "swift-syntax"), + .product(name: "SwiftParser", package: "swift-syntax"), + ] + ) + ] +) diff --git a/unified/swift-syntax-rs/swift/Sources/SwiftSyntaxFFI/SwiftSyntaxFFI.swift b/unified/swift-syntax-rs/swift/Sources/SwiftSyntaxFFI/SwiftSyntaxFFI.swift new file mode 100644 index 000000000000..177e8ff319d9 --- /dev/null +++ b/unified/swift-syntax-rs/swift/Sources/SwiftSyntaxFFI/SwiftSyntaxFFI.swift @@ -0,0 +1,145 @@ +import Foundation +import SwiftParser + +// `@_spi(RawSyntax)` exposes the `childName(_:)` helper that maps a child's +// key path to its field name in the parent layout; used to emit named fields. +@_spi(RawSyntax) import SwiftSyntax + +#if canImport(Glibc) + import Glibc +#elseif canImport(Darwin) + import Darwin +#endif + +/// Convert an absolute position into an `{ offset, line, column }` dictionary. +/// +/// `offset` is a UTF-8 byte offset; `line`/`column` are 1-based. +private func location( + _ position: AbsolutePosition, + _ converter: SourceLocationConverter +) -> [String: Any] { + let loc = converter.location(for: position) + return [ + "offset": position.utf8Offset, + "line": loc.line, + "column": loc.column, + ] +} + +/// Trivia kinds worth preserving in the serialized tree. Comments carry +/// developer intent (including doc comments), and `unexpectedText` flags source +/// the parser had to skip. Whitespace and multi-line-string escape markers are +/// dropped: node ranges already encode positions, so they would only bloat the +/// output. +private let keptTriviaKinds: Set = [ + "lineComment", + "blockComment", + "docLineComment", + "docBlockComment", + "unexpectedText", +] + +/// Serialize a trivia collection into an array of `{ kind, text }` pieces, +/// keeping only the kinds in `keptTriviaKinds`. +private func serializeTrivia(_ trivia: Trivia) -> [Any] { + trivia.pieces.compactMap { piece -> [String: Any]? in + // The label of an enum case mirror is the case name (e.g. "spaces", + // "lineComment"), which gives us a stable kind without an exhaustive + // switch over every `TriviaPiece` case. + let kind = Mirror(reflecting: piece).children.first?.label ?? "\(piece)" + guard keptTriviaKinds.contains(kind) else { return nil } + return [ + "kind": kind, + "text": Trivia(pieces: [piece]).description, + ] + } +} + +/// Recursively convert a SwiftSyntax node into a JSON-serializable value. +/// +/// * Tokens and layout nodes (e.g. `functionDecl`) become objects carrying +/// `kind` and source `range`. Layout nodes additionally embed their children +/// directly as members keyed by the child's name in the parent (e.g. `name`, +/// `signature`, `body`); absent optional children are omitted. Field names +/// never collide with `kind`/`range`. +/// * Collection nodes (e.g. `codeBlockItemList`) are *elided*: they become a +/// plain array of their serialized elements, taking the place of the +/// collection node itself. A list-valued layout field (e.g. `parameters`) is +/// therefore simply a JSON array. This drops the collection node's own +/// `kind`/`range`, which are unnamed and largely recoverable from the +/// elements. +private func serialize( + _ node: Syntax, + _ converter: SourceLocationConverter +) -> Any { + if node.kind.isSyntaxCollection { + return node.children(viewMode: .sourceAccurate).map { + serialize($0, converter) + } + } + + // Source range covering the node's content, excluding surrounding trivia. + let range: [String: Any] = [ + "start": location(node.positionAfterSkippingLeadingTrivia, converter), + "end": location(node.endPositionBeforeTrailingTrivia, converter), + ] + + if let token = node.as(TokenSyntax.self) { + return [ + "kind": "token", + "tokenKind": "\(token.tokenKind)", + "text": token.text, + "range": range, + "leadingTrivia": serializeTrivia(token.leadingTrivia), + "trailingTrivia": serializeTrivia(token.trailingTrivia), + ] + } + + var result: [String: Any] = [ + "kind": "\(node.kind)", + "range": range, + ] + var unnamed = 0 + for child in node.children(viewMode: .sourceAccurate) { + // Layout children are named; embed each under its field name in the + // parent (the same mechanism SwiftSyntax uses for its debug dump). A + // child that is a collection serializes to an array (see above). + if let keyPath = child.keyPathInParent, let name = childName(keyPath) { + result[name] = serialize(child, converter) + } else { + // Defensive fallback for any unnamed layout child. + result["child\(unnamed)"] = serialize(child, converter) + unnamed += 1 + } + } + return result +} + +/// Parse the given NUL-terminated Swift source string and return a +/// heap-allocated, NUL-terminated JSON representation of the syntax tree. +/// +/// The returned pointer is owned by the caller and MUST be released with +/// `ssr_string_free`. Returns `nil` on failure. +@_cdecl("ssr_parse_json") +public func ssr_parse_json(_ source: UnsafePointer?) -> UnsafeMutablePointer? { + guard let source = source else { return nil } + let code = String(cString: source) + let tree = Parser.parse(source: code) + let converter = SourceLocationConverter(fileName: "", tree: tree) + let json = serialize(Syntax(tree), converter) + + guard + let data = try? JSONSerialization.data( + withJSONObject: json, options: [.sortedKeys]), + let string = String(data: data, encoding: .utf8) + else { + return nil + } + return strdup(string) +} + +/// Free a string previously returned by this library. +@_cdecl("ssr_string_free") +public func ssr_string_free(_ ptr: UnsafeMutablePointer?) { + free(ptr) +} From 87c81731252678b5fe3cb7d14a0a19c65e4243f9 Mon Sep 17 00:00:00 2001 From: Taus Date: Tue, 7 Jul 2026 13:35:07 +0000 Subject: [PATCH 02/13] unified: Don't emit empty trivia These were taking up roughly 20% of the JSON payload. --- .../SwiftSyntaxFFI/SwiftSyntaxFFI.swift | 27 +++++++++++++------ 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/unified/swift-syntax-rs/swift/Sources/SwiftSyntaxFFI/SwiftSyntaxFFI.swift b/unified/swift-syntax-rs/swift/Sources/SwiftSyntaxFFI/SwiftSyntaxFFI.swift index 177e8ff319d9..3dcfcb2c3895 100644 --- a/unified/swift-syntax-rs/swift/Sources/SwiftSyntaxFFI/SwiftSyntaxFFI.swift +++ b/unified/swift-syntax-rs/swift/Sources/SwiftSyntaxFFI/SwiftSyntaxFFI.swift @@ -57,11 +57,14 @@ private func serializeTrivia(_ trivia: Trivia) -> [Any] { /// Recursively convert a SwiftSyntax node into a JSON-serializable value. /// -/// * Tokens and layout nodes (e.g. `functionDecl`) become objects carrying -/// `kind` and source `range`. Layout nodes additionally embed their children -/// directly as members keyed by the child's name in the parent (e.g. `name`, -/// `signature`, `body`); absent optional children are omitted. Field names -/// never collide with `kind`/`range`. +/// * Tokens carry `kind`, `tokenKind`, `text`, and `range`, plus +/// `leadingTrivia`/`trailingTrivia` — but only when non-empty (after +/// filtering, most tokens have no trivia, so the keys are simply absent). +/// * Layout nodes (e.g. `functionDecl`) carry `kind` and source `range`, and +/// additionally embed their children directly as members keyed by the +/// child's name in the parent (e.g. `name`, `signature`, `body`); absent +/// optional children are omitted. Field names never collide with +/// `kind`/`range`. /// * Collection nodes (e.g. `codeBlockItemList`) are *elided*: they become a /// plain array of their serialized elements, taking the place of the /// collection node itself. A list-valued layout field (e.g. `parameters`) is @@ -85,14 +88,22 @@ private func serialize( ] if let token = node.as(TokenSyntax.self) { - return [ + var result: [String: Any] = [ "kind": "token", "tokenKind": "\(token.tokenKind)", "text": token.text, "range": range, - "leadingTrivia": serializeTrivia(token.leadingTrivia), - "trailingTrivia": serializeTrivia(token.trailingTrivia), ] + // Only emit trivia when present; after filtering, most tokens have none. + let leading = serializeTrivia(token.leadingTrivia) + if !leading.isEmpty { + result["leadingTrivia"] = leading + } + let trailing = serializeTrivia(token.trailingTrivia) + if !trailing.isEmpty { + result["trailingTrivia"] = trailing + } + return result } var result: [String: Any] = [ From 446ff7c463bf3d8496b13458fc1a6749b4c2eb43 Mon Sep 17 00:00:00 2001 From: Taus Date: Tue, 7 Jul 2026 13:36:10 +0000 Subject: [PATCH 03/13] unified: Add mapping from swift-syntax JSON to yeast AST Adds a preliminary mapping that decodes the JSON into the yeast AST. The JSON AST itself is still very verbose, but it would be useful to see if it actually contains all of the things we want it to contain. --- Cargo.lock | 4 + shared/yeast/src/lib.rs | 42 ++- unified/swift-syntax-rs/BUILD.bazel | 11 +- unified/swift-syntax-rs/Cargo.toml | 4 + unified/swift-syntax-rs/README.md | 37 ++- unified/swift-syntax-rs/src/lib.rs | 2 + unified/swift-syntax-rs/src/yeast_adapter.rs | 282 +++++++++++++++++++ 7 files changed, 369 insertions(+), 13 deletions(-) create mode 100644 unified/swift-syntax-rs/src/yeast_adapter.rs diff --git a/Cargo.lock b/Cargo.lock index 7a9f19667911..24885fea8563 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2853,6 +2853,10 @@ checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" [[package]] name = "swift-syntax-rs" version = "0.1.0" +dependencies = [ + "serde_json", + "yeast", +] [[package]] name = "syn" diff --git a/shared/yeast/src/lib.rs b/shared/yeast/src/lib.rs index e6759f1ddb92..fa9074805b88 100644 --- a/shared/yeast/src/lib.rs +++ b/shared/yeast/src/lib.rs @@ -333,6 +333,28 @@ impl Ast { ast } + /// Construct an empty AST backed by `schema`, for building a tree + /// programmatically from a source other than a tree-sitter parse (e.g. an + /// external parser's output). Populate it with [`Ast::create_node`] / + /// [`Ast::create_node_with_range`] and then designate the root with + /// [`Ast::set_root`]. The `schema` must already have every node kind and + /// field name registered (see [`schema::Schema`]). + pub fn with_schema(schema: schema::Schema) -> Self { + Self { + root: Id(0), + nodes: Vec::new(), + schema, + source: Vec::new(), + } + } + + /// Set the original source bytes, used to resolve `NodeContent::Range` + /// nodes to text. Nodes built with inline `NodeContent::DynamicString` + /// content do not require this. + pub fn set_source(&mut self, source: Vec) { + self.source = source; + } + /// Returns the source text for `id`, resolving `NodeContent::Range` /// against the stored source bytes when available. pub fn source_text(&self, id: Id) -> String { @@ -463,6 +485,24 @@ impl Ast { Id(id) } + /// Register a named node kind, returning its id (idempotent). Lets callers + /// build an AST in a single pass, registering kinds as nodes are created + /// rather than pre-populating the schema. + pub fn register_kind(&mut self, name: &str) -> KindId { + self.schema.register_kind(name) + } + + /// Register an anonymous (unnamed) token kind, returning its id + /// (idempotent). Anonymous tokens are keyed by their text (e.g. `"func"`). + pub fn register_unnamed_kind(&mut self, name: &str) -> KindId { + self.schema.register_unnamed_kind(name) + } + + /// Register a field name, returning its id (idempotent). + pub fn register_field(&mut self, name: &str) -> FieldId { + self.schema.register_field(name) + } + fn union_source_range_of_children( &self, fields: &BTreeMap>, @@ -604,7 +644,7 @@ impl Ast { } } - fn id_for_unnamed_node_kind(&self, kind: &str) -> Option { + pub fn id_for_unnamed_node_kind(&self, kind: &str) -> Option { let id = self.schema.id_for_unnamed_node_kind(kind).unwrap_or(0); if id == 0 { None diff --git a/unified/swift-syntax-rs/BUILD.bazel b/unified/swift-syntax-rs/BUILD.bazel index 10d11a1bec11..3a2e5ed0796b 100644 --- a/unified/swift-syntax-rs/BUILD.bazel +++ b/unified/swift-syntax-rs/BUILD.bazel @@ -28,9 +28,16 @@ swift_library( # Swift shim in the local `cargo` workflow. rust_library( name = "swift_syntax_rs", - srcs = ["src/lib.rs"], + srcs = glob( + ["src/**/*.rs"], + exclude = ["src/main.rs"], + ), edition = "2024", - deps = [":swift_syntax_ffi"], + deps = [ + ":swift_syntax_ffi", + "//shared/yeast", + "@vendor_ts__serde_json-1.0.145//:serde_json", + ], ) rust_binary( diff --git a/unified/swift-syntax-rs/Cargo.toml b/unified/swift-syntax-rs/Cargo.toml index 6957fb0e50e9..576246468480 100644 --- a/unified/swift-syntax-rs/Cargo.toml +++ b/unified/swift-syntax-rs/Cargo.toml @@ -12,3 +12,7 @@ path = "src/lib.rs" [[bin]] name = "swift-syntax-parse" path = "src/main.rs" + +[dependencies] +serde_json = "1.0" +yeast = { path = "../../shared/yeast" } diff --git a/unified/swift-syntax-rs/README.md b/unified/swift-syntax-rs/README.md index 08796b3a6a7d..bb4f7f667a5d 100644 --- a/unified/swift-syntax-rs/README.md +++ b/unified/swift-syntax-rs/README.md @@ -13,8 +13,8 @@ The emitted JSON tree preserves the AST's named structure. Every node has a `kind` and a `range` with `start`/`end` positions (UTF-8 `offset` plus 1-based `line`/`column`). Beyond that: -- **Tokens** carry `text`, `tokenKind`, and `leadingTrivia`/`trailingTrivia` - arrays of `{ kind, text }` pieces. +- **Tokens** carry `text`, `tokenKind`, and — only when non-empty — + `leadingTrivia`/`trailingTrivia` arrays of `{ kind, text }` pieces. - **Layout nodes** (e.g. `functionDecl`) embed their children directly as members keyed by the child's name in the parent (`name`, `signature`, `body`, …), alongside `kind`/`range`. Absent optional children are omitted. @@ -49,9 +49,7 @@ abbreviated here as `…`): "kind": "token", "text": "let", "tokenKind": "keyword(SwiftSyntax.Keyword.let)", - "range": …, - "leadingTrivia": [], - "trailingTrivia": [] + "range": … }, "bindings": [ { @@ -60,12 +58,12 @@ abbreviated here as `…`): "pattern": { "kind": "identifierPattern", "range": …, - "identifier": { "kind": "token", "text": "x", "tokenKind": "identifier(\"x\")", "range": …, "leadingTrivia": [], "trailingTrivia": [] } + "identifier": { "kind": "token", "text": "x", "tokenKind": "identifier(\"x\")", "range": … } }, "initializer": { "kind": "initializerClause", "range": …, - "equal": { "kind": "token", "text": "=", "tokenKind": "equal", "range": …, "leadingTrivia": [], "trailingTrivia": [] }, + "equal": { "kind": "token", "text": "=", "tokenKind": "equal", "range": … }, "value": { "kind": "integerLiteralExpr", "range": …, @@ -74,7 +72,6 @@ abbreviated here as `…`): "text": "1", "tokenKind": "integerLiteral(\"1\")", "range": …, - "leadingTrivia": [], "trailingTrivia": [ { "kind": "lineComment", "text": "// c" } ] } } @@ -84,14 +81,15 @@ abbreviated here as `…`): } } ], - "endOfFileToken": { "kind": "token", "text": "", "tokenKind": "endOfFile", "range": …, "leadingTrivia": [], "trailingTrivia": [] } + "endOfFileToken": { "kind": "token", "text": "", "tokenKind": "endOfFile", "range": … } } ``` Note how `statements`, `bindings`, `attributes`, and `modifiers` are plain arrays (their collection nodes are elided), layout children such as `bindingSpecifier` and `initializer` are embedded by name, and the `// c` -comment rides along as `trailingTrivia` on the token it follows. +comment rides along as `trailingTrivia` on the token it follows. Tokens without +trivia (most of them) simply omit the `leadingTrivia`/`trailingTrivia` keys. ## Prerequisites @@ -171,10 +169,29 @@ CLI (reads a file argument or stdin, prints the syntax tree as JSON): echo 'let x = 1' | cargo run --bin swift-syntax-parse ``` +## Converting to a yeast AST + +For use in the CodeQL extractor, the JSON tree can be converted into a +[`yeast::Ast`](../../shared/yeast) — the in-memory format the extractor's +rewrite rules operate on — via [`yeast_adapter::json_to_ast`](src/yeast_adapter.rs): + +```rust +let json = swift_syntax_rs::parse_to_json("let x = 1")?; +let ast = swift_syntax_rs::yeast_adapter::json_to_ast(&json)?; +``` + +The adapter mirrors tree-sitter's node model, which is what yeast expects: +layout nodes and varying tokens (identifiers, literals, operators) become +**named** nodes; fixed tokens (keywords, punctuation) become **anonymous** +nodes keyed by their text. It preserves swift-syntax's own kind/field names — +aligning them with the tree-sitter-swift schema so the existing rewrite rules +fire is a separate, later step. + ## Layout - `swift/` — Swift package exposing the `ssr_parse_json` / `ssr_string_free` C ABI. - `build.rs` — builds the Swift package and emits link/rpath flags (local `cargo` only). - `BUILD.bazel` — Bazel targets for the hermetic CI build (swift_library + rust targets). - `src/lib.rs` — safe Rust bindings (`parse_to_json`). +- `src/yeast_adapter.rs` — converts the JSON tree into a `yeast::Ast`. - `src/main.rs` — demo CLI. diff --git a/unified/swift-syntax-rs/src/lib.rs b/unified/swift-syntax-rs/src/lib.rs index 83dc5e2c6fbd..aa711e322a49 100644 --- a/unified/swift-syntax-rs/src/lib.rs +++ b/unified/swift-syntax-rs/src/lib.rs @@ -5,6 +5,8 @@ //! against `SwiftSyntax`/`SwiftParser` and exposes a tiny C ABI. This module //! provides safe Rust bindings on top of that ABI. +pub mod yeast_adapter; + use std::ffi::{CStr, CString}; use std::os::raw::c_char; diff --git a/unified/swift-syntax-rs/src/yeast_adapter.rs b/unified/swift-syntax-rs/src/yeast_adapter.rs new file mode 100644 index 000000000000..5f84e1304bba --- /dev/null +++ b/unified/swift-syntax-rs/src/yeast_adapter.rs @@ -0,0 +1,282 @@ +//! Adapter that converts the swift-syntax JSON tree (see [`crate::parse_to_json`]) +//! into a [`yeast::Ast`], the in-memory format the CodeQL desugaring rules +//! operate on. +//! +//! The mapping mirrors tree-sitter's node model, which is what yeast (and the +//! extractor's rewrite rules) expect: +//! +//! * **Layout nodes** (e.g. `functionDecl`) and **varying tokens** (identifiers, +//! literals, operators — the ones whose text is not determined by their kind) +//! become **named** nodes, keyed by their kind name. +//! * **Fixed tokens** (keywords and punctuation, whose text is fully determined +//! by their kind) become **anonymous** nodes, keyed by their text — exactly +//! how tree-sitter models anonymous tokens (e.g. `"func"`, `"->"`). +//! * Collection nodes are already elided to JSON arrays upstream, so a +//! list-valued field maps directly to that field holding several children. +//! +//! Note: this preserves swift-syntax's own kind/field names. Aligning those +//! names with the tree-sitter-swift schema (so the existing rewrite rules fire) +//! is a separate, later step; this module is only concerned with getting the +//! tree into yeast's format. + +use std::collections::BTreeMap; + +use serde_json::Value; +use yeast::schema::Schema; +use yeast::{Ast, Id, NodeContent}; + +/// swift-syntax `TokenKind` cases whose text is *not* determined by the kind +/// (i.e. `TokenKind.defaultText == nil`). These carry varying information and +/// are modelled as named leaf nodes; every other token is a fixed +/// keyword/punctuation token modelled as an anonymous token keyed by its text. +const VARYING_TOKEN_KINDS: &[&str] = &[ + "identifier", + "integerLiteral", + "floatLiteral", + "stringSegment", + "binaryOperator", + "prefixOperator", + "postfixOperator", + "dollarIdentifier", + "regexLiteralPattern", + "rawStringPoundDelimiter", + "regexPoundDelimiter", + "shebang", + "unknown", +]; + +/// Keys of a node object that carry metadata rather than a structural child. +fn is_metadata_key(key: &str) -> bool { + matches!( + key, + "kind" | "range" | "tokenKind" | "text" | "leadingTrivia" | "trailingTrivia" + ) +} + +/// The classification of a JSON node into a yeast kind name and named-ness. +struct KindInfo { + /// The name under which the kind is registered in the schema. + name: String, + /// `true` for named nodes (layout nodes + varying tokens), `false` for + /// anonymous tokens (fixed keywords/punctuation). + is_named: bool, + /// The leaf text for tokens (empty for layout nodes). + text: String, +} + +/// Determine the kind name / named-ness / text for a JSON node object. +fn classify(node: &Value) -> Result { + let kind = node + .get("kind") + .and_then(Value::as_str) + .ok_or("node object is missing a string `kind`")?; + + if kind != "token" { + // Layout node: named, keyed by its kind, no leaf text. + return Ok(KindInfo { + name: kind.to_string(), + is_named: true, + text: String::new(), + }); + } + + let token_kind = node + .get("tokenKind") + .and_then(Value::as_str) + .ok_or("token is missing a string `tokenKind`")?; + let text = node + .get("text") + .and_then(Value::as_str) + .unwrap_or("") + .to_string(); + + // The case name is the part before any `(payload)` in the debug rendering. + let case_name = token_kind.split('(').next().unwrap_or(token_kind); + + if VARYING_TOKEN_KINDS.contains(&case_name) { + // Varying token: named leaf, keyed by its case name. + Ok(KindInfo { + name: case_name.to_string(), + is_named: true, + text, + }) + } else { + // Fixed token: anonymous, keyed by its text (as tree-sitter does). + // `endOfFile` has empty text, so fall back to the case name there. + let name = if text.is_empty() { + case_name.to_string() + } else { + text.clone() + }; + Ok(KindInfo { + name, + is_named: false, + text, + }) + } +} + +/// Iterate over a node object's structural (field, value) pairs in a stable +/// order, skipping metadata keys. +fn field_entries(node: &Value) -> Vec<(&str, &Value)> { + node.as_object() + .map(|map| { + map.iter() + .filter(|(k, _)| !is_metadata_key(k)) + .map(|(k, v)| (k.as_str(), v)) + .collect() + }) + .unwrap_or_default() +} + +/// The child node objects held by a field value, which is either a single node +/// object or an array of them (an elided collection). +fn children_of(value: &Value) -> Vec<&Value> { + match value { + Value::Array(items) => items.iter().collect(), + other => vec![other], + } +} + +/// Recursively build `node` (and its descendants) into `ast`, returning its id. +/// +/// This is a single traversal: each node's kind and field names are registered +/// in the schema on the fly, immediately before the node is created. Children +/// are built first so a parent's field lists reference existing ids. +fn build(node: &Value, ast: &mut Ast) -> Result { + let info = classify(node)?; + + let mut fields: BTreeMap> = BTreeMap::new(); + for (field, value) in field_entries(node) { + let field_id = ast.register_field(field); + let mut ids = Vec::new(); + for child in children_of(value) { + ids.push(build(child, ast)?); + } + fields.insert(field_id, ids); + } + + let kind_id = if info.is_named { + ast.register_kind(&info.name) + } else { + ast.register_unnamed_kind(&info.name) + }; + + Ok(ast.create_node_with_range( + kind_id, + NodeContent::DynamicString(info.text), + fields, + info.is_named, + None, + )) +} + +/// Convert a swift-syntax JSON tree (as produced by [`crate::parse_to_json`]) +/// into a [`yeast::Ast`]. +pub fn json_to_ast(json: &str) -> Result { + let root: Value = serde_json::from_str(json).map_err(|e| format!("invalid JSON: {e}"))?; + + let mut ast = Ast::with_schema(Schema::new()); + let root_id = build(&root, &mut ast)?; + ast.set_root(root_id); + Ok(ast) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// A hand-written JSON tree exercising layout nodes, a named (varying) + /// token, a fixed keyword token, and an elided collection field — so the + /// adapter is tested without needing the Swift toolchain. + fn sample_json() -> &'static str { + r#"{ + "kind": "sourceFile", + "range": {"start":{"offset":0,"line":1,"column":1},"end":{"offset":9,"line":1,"column":10}}, + "statements": [ + { + "kind": "variableDecl", + "range": {"start":{"offset":0,"line":1,"column":1},"end":{"offset":9,"line":1,"column":10}}, + "bindingSpecifier": { + "kind": "token", + "tokenKind": "keyword(SwiftSyntax.Keyword.let)", + "text": "let", + "range": {"start":{"offset":0,"line":1,"column":1},"end":{"offset":3,"line":1,"column":4}} + }, + "name": { + "kind": "token", + "tokenKind": "identifier(\"x\")", + "text": "x", + "range": {"start":{"offset":4,"line":1,"column":5},"end":{"offset":5,"line":1,"column":6}} + } + } + ] + }"# + } + + #[test] + fn builds_ast_from_json() { + let ast = json_to_ast(sample_json()).expect("adapter should succeed"); + let root = ast.get_root(); + let root_node = ast.get_node(root).expect("root exists"); + assert_eq!(root_node.kind_name(), "sourceFile"); + assert!(root_node.is_named()); + } + + #[test] + fn classifies_named_and_anonymous_tokens() { + let ast = json_to_ast(sample_json()).expect("adapter should succeed"); + // Walk all nodes and collect (kind_name, is_named) for the two tokens. + let mut let_named = None; + let mut ident_named = None; + for node in ast.nodes() { + match node.kind_name() { + "let" => let_named = Some(node.is_named()), + "identifier" => ident_named = Some(node.is_named()), + _ => {} + } + } + // Fixed keyword `let` is anonymous (keyed by its text "let"). + assert_eq!(let_named, Some(false)); + // Varying `identifier` is a named leaf. + assert_eq!(ident_named, Some(true)); + } + + #[test] + fn preserves_leaf_text() { + let ast = json_to_ast(sample_json()).expect("adapter should succeed"); + let ident = ast + .nodes() + .iter() + .enumerate() + .find(|(_, n)| n.kind_name() == "identifier") + .map(|(i, _)| Id(i)) + .expect("identifier node exists"); + assert_eq!(ast.source_text(ident), "x"); + } + + /// End-to-end: real Swift source parsed by the shim, then adapted into a + /// `yeast::Ast`. Requires the Swift toolchain (like the crate's FFI tests). + #[test] + fn end_to_end_from_swift_source() { + let json = crate::parse_to_json("func f(n: Int) -> Int { return n }") + .expect("parsing should succeed"); + let ast = json_to_ast(&json).expect("adapter should succeed"); + + let root = ast.get_node(ast.get_root()).expect("root exists"); + assert_eq!(root.kind_name(), "sourceFile"); + + // The tree contains a `functionDecl` layout node and an anonymous + // `func` keyword token keyed by its text. + let mut kinds: Vec<&str> = ast.nodes().iter().map(|n| n.kind_name()).collect(); + kinds.sort_unstable(); + assert!( + kinds.contains(&"functionDecl"), + "expected a functionDecl node, got kinds: {kinds:?}" + ); + assert!( + kinds.contains(&"func"), + "expected an anonymous `func` token, got kinds: {kinds:?}" + ); + } +} From 8909fae81e7f8b93df2822b0a4c1e3ddce7a4dae Mon Sep 17 00:00:00 2001 From: Taus Date: Tue, 7 Jul 2026 14:28:19 +0000 Subject: [PATCH 04/13] unified: Extract locations from `swift-syntax` AST Introduces new yeast types for `Point`s and `Range`s (corresponding to the tree-sitter equivalents), since it seemed silly to have `swift-syntax-rs` pull in `tree-sitter` just to have those types available. This _does_ mean there is a slight overhead in the shared extractor when converting between these types, but I think this is negligible. --- .../src/extractor/mod.rs | 12 +++- shared/yeast-macros/src/parse.rs | 2 +- shared/yeast/src/build.rs | 10 +-- shared/yeast/src/lib.rs | 61 +++++++++++++------ shared/yeast/src/range.rs | 29 ++++++--- unified/swift-syntax-rs/src/yeast_adapter.rs | 49 ++++++++++++++- 6 files changed, 124 insertions(+), 39 deletions(-) diff --git a/shared/tree-sitter-extractor/src/extractor/mod.rs b/shared/tree-sitter-extractor/src/extractor/mod.rs index 54b01ba5146e..474dc096a2a6 100644 --- a/shared/tree-sitter-extractor/src/extractor/mod.rs +++ b/shared/tree-sitter-extractor/src/extractor/mod.rs @@ -81,10 +81,18 @@ impl AstNode for yeast::Node { yeast::Node::is_extra(self) } fn start_position(&self) -> tree_sitter::Point { - yeast::Node::start_position(self) + let p = yeast::Node::start_position(self); + tree_sitter::Point { + row: p.row, + column: p.column, + } } fn end_position(&self) -> tree_sitter::Point { - yeast::Node::end_position(self) + let p = yeast::Node::end_position(self); + tree_sitter::Point { + row: p.row, + column: p.column, + } } fn byte_range(&self) -> std::ops::Range { yeast::Node::byte_range(self) diff --git a/shared/yeast-macros/src/parse.rs b/shared/yeast-macros/src/parse.rs index 01c0b574b1cf..55ada3588491 100644 --- a/shared/yeast-macros/src/parse.rs +++ b/shared/yeast-macros/src/parse.rs @@ -903,7 +903,7 @@ pub fn parse_rule_top(input: TokenStream) -> Result { Ok(quote! { { let __query = #query_code; - yeast::Rule::new(__query, Box::new(|__ast: &mut yeast::Ast, mut __captures: yeast::captures::Captures, __fresh: &yeast::tree_builder::FreshScope, __source_range: Option, __user_ctx: &mut _, __translator: yeast::TranslatorHandle<'_, _>| { + yeast::Rule::new(__query, Box::new(|__ast: &mut yeast::Ast, mut __captures: yeast::captures::Captures, __fresh: &yeast::tree_builder::FreshScope, __source_range: Option, __user_ctx: &mut _, __translator: yeast::TranslatorHandle<'_, _>| { // Auto-translation prefix: recursively translate every // captured node before invoking the user's transform body, // except for `@@name` captures listed in `__skip` which the diff --git a/shared/yeast/src/build.rs b/shared/yeast/src/build.rs index 68a4c883242a..508a1e731346 100644 --- a/shared/yeast/src/build.rs +++ b/shared/yeast/src/build.rs @@ -2,7 +2,7 @@ use std::collections::BTreeMap; use crate::captures::Captures; use crate::tree_builder::FreshScope; -use crate::{Ast, FieldId, Id, NodeContent, TranslatorHandle}; +use crate::{Ast, FieldId, Id, NodeContent, Range, TranslatorHandle}; /// Context for building new AST nodes during a transformation. /// @@ -34,7 +34,7 @@ pub struct BuildCtx<'a, C: 'a = ()> { pub captures: &'a Captures, pub fresh: &'a FreshScope, /// Source range of the matched node, inherited by synthetic nodes. - pub source_range: Option, + pub source_range: Option, /// User-supplied context, accessible directly via `ctx.field` (via Deref). pub user_ctx: &'a mut C, /// Optional translator handle, populated when the context is built by @@ -63,7 +63,7 @@ impl<'a, C> BuildCtx<'a, C> { ast: &'a mut Ast, captures: &'a Captures, fresh: &'a FreshScope, - source_range: Option, + source_range: Option, user_ctx: &'a mut C, ) -> Self { Self { @@ -82,7 +82,7 @@ impl<'a, C> BuildCtx<'a, C> { ast: &'a mut Ast, captures: &'a Captures, fresh: &'a FreshScope, - source_range: Option, + source_range: Option, user_ctx: &'a mut C, translator: TranslatorHandle<'a, C>, ) -> Self { @@ -143,7 +143,7 @@ impl<'a, C> BuildCtx<'a, C> { &mut self, kind: &'static str, value: &str, - source_range: Option, + source_range: Option, ) -> Id { self.ast.create_named_token_with_range( kind, diff --git a/shared/yeast/src/lib.rs b/shared/yeast/src/lib.rs index fa9074805b88..b2709ab406ee 100644 --- a/shared/yeast/src/lib.rs +++ b/shared/yeast/src/lib.rs @@ -15,11 +15,32 @@ pub mod schema; pub mod tree_builder; mod visitor; +pub use range::{Point, Range}; pub use yeast_macros::{query, rule, rules, tree, trees}; use captures::Captures; use query::QueryNode; +impl From for Point { + fn from(p: tree_sitter::Point) -> Self { + Point { + row: p.row, + column: p.column, + } + } +} + +impl From for Range { + fn from(r: tree_sitter::Range) -> Self { + Range { + start_byte: r.start_byte, + end_byte: r.end_byte, + start_point: r.start_point.into(), + end_point: r.end_point.into(), + } + } +} + /// Node id: an index into the [`Ast`] arena. A newtype around `usize` /// rather than a bare alias so that it can carry its own /// [`YeastDisplay`] / [`YeastSourceRange`] / [`IntoFieldIds`] impls @@ -106,7 +127,7 @@ pub trait YeastDisplay { /// rule's source range. `Id` returns the referenced node's range, letting /// `(kind #{capture})` carry the captured node's location. pub trait YeastSourceRange { - fn yeast_source_range(&self, ast: &Ast) -> Option; + fn yeast_source_range(&self, ast: &Ast) -> Option; } impl YeastDisplay for Id { @@ -116,9 +137,9 @@ impl YeastDisplay for Id { } impl YeastSourceRange for Id { - fn yeast_source_range(&self, ast: &Ast) -> Option { + fn yeast_source_range(&self, ast: &Ast) -> Option { ast.get_node(*self).and_then(|n| match &n.content { - NodeContent::Range(r) => Some(r.clone()), + NodeContent::Range(r) => Some(*r), _ => n.source_range, }) } @@ -134,7 +155,7 @@ macro_rules! impl_yeast_display_via_display { } impl YeastSourceRange for $t { - fn yeast_source_range(&self, _ast: &Ast) -> Option { + fn yeast_source_range(&self, _ast: &Ast) -> Option { None } } @@ -157,7 +178,7 @@ impl YeastDisplay for &T { } impl YeastSourceRange for &T { - fn yeast_source_range(&self, ast: &Ast) -> Option { + fn yeast_source_range(&self, ast: &Ast) -> Option { (**self).yeast_source_range(ast) } } @@ -361,7 +382,7 @@ impl Ast { let Some(node) = self.get_node(id) else { return String::new(); }; - let read_range = |range: &tree_sitter::Range| { + let read_range = |range: &Range| { let start = range.start_byte; let end = range.end_byte; if end <= self.source.len() && start <= end { @@ -459,7 +480,7 @@ impl Ast { content: NodeContent, fields: BTreeMap>, is_named: bool, - source_range: Option, + source_range: Option, ) -> Id { let source_range = match &content { // Parsed nodes already carry an exact source range in their content. @@ -506,11 +527,11 @@ impl Ast { fn union_source_range_of_children( &self, fields: &BTreeMap>, - ) -> Option { + ) -> Option { let mut start_byte: Option = None; let mut end_byte: Option = None; - let mut start_point = tree_sitter::Point { row: 0, column: 0 }; - let mut end_point = tree_sitter::Point { row: 0, column: 0 }; + let mut start_point = Point { row: 0, column: 0 }; + let mut end_point = Point { row: 0, column: 0 }; for child_ids in fields.values() { for &child_id in child_ids { @@ -553,7 +574,7 @@ impl Ast { } match (start_byte, end_byte) { - (Some(start_byte), Some(end_byte)) => Some(tree_sitter::Range { + (Some(start_byte), Some(end_byte)) => Some(Range { start_byte, end_byte, start_point, @@ -571,7 +592,7 @@ impl Ast { &mut self, kind: &'static str, content: String, - source_range: Option, + source_range: Option, ) -> Id { let kind_id = self.schema.id_for_node_kind(kind).unwrap_or_else(|| { panic!("create_named_token: node kind '{kind}' not found in schema") @@ -664,7 +685,7 @@ pub struct Node { /// For synthetic nodes, the source range of the original node they /// were desugared from. Used for location information in TRAP output. #[serde(skip)] - source_range: Option, + source_range: Option, is_named: bool, is_missing: bool, is_extra: bool, @@ -692,11 +713,11 @@ impl Node { self.is_error } - fn fake_point(&self) -> tree_sitter::Point { - tree_sitter::Point { row: 0, column: 0 } + fn fake_point(&self) -> Point { + Point { row: 0, column: 0 } } - pub fn start_position(&self) -> tree_sitter::Point { + pub fn start_position(&self) -> Point { match self.content { NodeContent::Range(range) => range.start_point, _ => self @@ -705,7 +726,7 @@ impl Node { } } - pub fn end_position(&self) -> tree_sitter::Point { + pub fn end_position(&self) -> Point { match self.content { NodeContent::Range(range) => range.end_point, _ => self @@ -754,7 +775,7 @@ impl Node { /// or a new string if the node is synthesized. #[derive(PartialEq, Eq, Debug, Clone, Serialize)] pub enum NodeContent { - Range(#[serde(with = "range::Range")] tree_sitter::Range), + Range(Range), String(&'static str), DynamicString(String), } @@ -767,7 +788,7 @@ impl From<&'static str> for NodeContent { impl From for NodeContent { fn from(value: tree_sitter::Range) -> Self { - NodeContent::Range(value) + NodeContent::Range(value.into()) } } @@ -894,7 +915,7 @@ pub type Transform = Box< &mut Ast, Captures, &tree_builder::FreshScope, - Option, + Option, &mut C, TranslatorHandle<'_, C>, ) -> Result, String> diff --git a/shared/yeast/src/range.rs b/shared/yeast/src/range.rs index ec670b438d56..e56bbcc6fb85 100644 --- a/shared/yeast/src/range.rs +++ b/shared/yeast/src/range.rs @@ -1,21 +1,32 @@ -//! (de)-serialize helpers for tree_sitter::Range +//! Location types for yeast ASTs. +//! +//! These are plain owned structs (mirroring tree-sitter's `Point`/`Range`) so +//! that code building an AST programmatically can populate node locations +//! without depending on tree-sitter. Conversions from the tree-sitter types +//! live in the crate root, next to the `from_tree` construction path. use serde::{Deserialize, Serialize}; -#[derive(Serialize, Deserialize)] -#[serde(remote = "tree_sitter::Point")] +/// A position in a source file: a 0-based `row` (line) and 0-based `column` +/// (UTF-8 byte offset within the line), matching tree-sitter's convention. +#[derive(Serialize, Deserialize, Clone, Copy, Debug, Default, PartialEq, Eq, Hash)] pub struct Point { pub row: usize, pub column: usize, } -#[derive(Serialize, Deserialize)] -#[serde(remote = "tree_sitter::Range")] +impl Point { + pub fn new(row: usize, column: usize) -> Self { + Self { row, column } + } +} + +/// A half-open source range: byte offsets plus start/end [`Point`]s. The end is +/// exclusive, matching tree-sitter's convention. +#[derive(Serialize, Deserialize, Clone, Copy, Debug, Default, PartialEq, Eq, Hash)] pub struct Range { pub start_byte: usize, pub end_byte: usize, - #[serde(with = "Point")] - pub start_point: tree_sitter::Point, - #[serde(with = "Point")] - pub end_point: tree_sitter::Point, + pub start_point: Point, + pub end_point: Point, } diff --git a/unified/swift-syntax-rs/src/yeast_adapter.rs b/unified/swift-syntax-rs/src/yeast_adapter.rs index 5f84e1304bba..8a0baf7a4cd6 100644 --- a/unified/swift-syntax-rs/src/yeast_adapter.rs +++ b/unified/swift-syntax-rs/src/yeast_adapter.rs @@ -23,7 +23,7 @@ use std::collections::BTreeMap; use serde_json::Value; use yeast::schema::Schema; -use yeast::{Ast, Id, NodeContent}; +use yeast::{Ast, Id, NodeContent, Point, Range}; /// swift-syntax `TokenKind` cases whose text is *not* determined by the kind /// (i.e. `TokenKind.defaultText == nil`). These carry varying information and @@ -167,10 +167,39 @@ fn build(node: &Value, ast: &mut Ast) -> Result { NodeContent::DynamicString(info.text), fields, info.is_named, - None, + parse_range(node), )) } +/// Parse a node's `range` into a [`yeast::Range`]. +/// +/// The JSON carries, for `start` and `end`, a 0-based UTF-8 file byte `offset`, +/// a 1-based `line`, and a 1-based UTF-8 byte `column`. yeast (like tree-sitter) +/// uses byte offsets with 0-based rows/columns and an exclusive end, so the +/// line/column are shifted down by one. swift-syntax's end position is already +/// exclusive, so the byte offsets map across directly. +fn parse_range(node: &Value) -> Option { + let range = node.get("range")?; + let point = |key: &str| -> Option<(usize, Point)> { + let p = range.get(key)?; + let offset = p.get("offset")?.as_u64()? as usize; + let line = p.get("line")?.as_u64()? as usize; + let column = p.get("column")?.as_u64()? as usize; + Some(( + offset, + Point::new(line.saturating_sub(1), column.saturating_sub(1)), + )) + }; + let (start_byte, start_point) = point("start")?; + let (end_byte, end_point) = point("end")?; + Some(Range { + start_byte, + end_byte, + start_point, + end_point, + }) +} + /// Convert a swift-syntax JSON tree (as produced by [`crate::parse_to_json`]) /// into a [`yeast::Ast`]. pub fn json_to_ast(json: &str) -> Result { @@ -255,6 +284,22 @@ mod tests { assert_eq!(ast.source_text(ident), "x"); } + #[test] + fn maps_source_locations() { + let ast = json_to_ast(sample_json()).expect("adapter should succeed"); + let ident = ast + .nodes() + .iter() + .find(|n| n.kind_name() == "identifier") + .expect("identifier node exists"); + // `x` is at file offset 4..5, line 1, column 5 (1-based) in the JSON, + // which maps to 0-based row 0, column 4 and byte range 4..5. + assert_eq!(ident.start_byte(), 4); + assert_eq!(ident.end_byte(), 5); + assert_eq!(ident.start_position(), Point::new(0, 4)); + assert_eq!(ident.end_position(), Point::new(0, 5)); + } + /// End-to-end: real Swift source parsed by the shim, then adapted into a /// `yeast::Ast`. Requires the Swift toolchain (like the crate's FFI tests). #[test] From ddab6ccb35691f368d09ef0dfa63a5e1bb47c104 Mon Sep 17 00:00:00 2001 From: Taus Date: Tue, 7 Jul 2026 14:43:59 +0000 Subject: [PATCH 05/13] unified: Emit locations for comments Also gathers these into a separate side-channel during the initial AST construction. That way, we don't encounter these as weird extra nodes while running yeast. --- unified/swift-syntax-rs/README.md | 13 +- unified/swift-syntax-rs/src/yeast_adapter.rs | 147 ++++++++++++++++-- .../SwiftSyntaxFFI/SwiftSyntaxFFI.swift | 47 ++++-- 3 files changed, 179 insertions(+), 28 deletions(-) diff --git a/unified/swift-syntax-rs/README.md b/unified/swift-syntax-rs/README.md index bb4f7f667a5d..08cabeb736b0 100644 --- a/unified/swift-syntax-rs/README.md +++ b/unified/swift-syntax-rs/README.md @@ -177,15 +177,20 @@ rewrite rules operate on — via [`yeast_adapter::json_to_ast`](src/yeast_adapte ```rust let json = swift_syntax_rs::parse_to_json("let x = 1")?; -let ast = swift_syntax_rs::yeast_adapter::json_to_ast(&json)?; +let adapted = swift_syntax_rs::yeast_adapter::json_to_ast(&json)?; +let ast = adapted.ast; // the yeast::Ast +let comments = adapted.trivia; // side-channel comment/unexpectedText tokens ``` The adapter mirrors tree-sitter's node model, which is what yeast expects: layout nodes and varying tokens (identifiers, literals, operators) become **named** nodes; fixed tokens (keywords, punctuation) become **anonymous** -nodes keyed by their text. It preserves swift-syntax's own kind/field names — -aligning them with the tree-sitter-swift schema so the existing rewrite rules -fire is a separate, later step. +nodes keyed by their text. Comments (and `unexpectedText`) are harvested into a +side channel (`adapted.trivia`) during the same traversal rather than embedded +in the tree, matching how the extractor treats tree-sitter `extra` nodes. It +preserves swift-syntax's own kind/field names — aligning them with the +tree-sitter-swift schema so the existing rewrite rules fire is a separate, +later step. ## Layout diff --git a/unified/swift-syntax-rs/src/yeast_adapter.rs b/unified/swift-syntax-rs/src/yeast_adapter.rs index 8a0baf7a4cd6..e2adf1df5a6a 100644 --- a/unified/swift-syntax-rs/src/yeast_adapter.rs +++ b/unified/swift-syntax-rs/src/yeast_adapter.rs @@ -25,6 +25,29 @@ use serde_json::Value; use yeast::schema::Schema; use yeast::{Ast, Id, NodeContent, Point, Range}; +/// A comment (or `unexpectedText`) recovered from the syntax tree's trivia. +/// +/// These are collected into a side channel rather than embedded in the +/// [`yeast::Ast`], mirroring how the extractor treats tree-sitter `extra` +/// nodes: they carry a location and text but are not attached to a parent. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct TriviaToken { + /// The trivia kind (e.g. `lineComment`, `blockComment`, `docLineComment`, + /// `docBlockComment`, `unexpectedText`). + pub kind: String, + /// The verbatim source text of the piece (e.g. `// comment`). + pub text: String, + /// The source range the piece occupies. + pub range: Range, +} + +/// The result of adapting a swift-syntax JSON tree: the [`yeast::Ast`] plus the +/// comment/`unexpectedText` trivia harvested from it (in source order). +pub struct AdaptedTree { + pub ast: Ast, + pub trivia: Vec, +} + /// swift-syntax `TokenKind` cases whose text is *not* determined by the kind /// (i.e. `TokenKind.defaultText == nil`). These carry varying information and /// are modelled as named leaf nodes; every other token is a fixed @@ -142,16 +165,19 @@ fn children_of(value: &Value) -> Vec<&Value> { /// /// This is a single traversal: each node's kind and field names are registered /// in the schema on the fly, immediately before the node is created. Children -/// are built first so a parent's field lists reference existing ids. -fn build(node: &Value, ast: &mut Ast) -> Result { +/// are built first so a parent's field lists reference existing ids. Any +/// comment/`unexpectedText` trivia carried by a token is harvested into +/// `trivia` during the same pass rather than embedded in the tree. +fn build(node: &Value, ast: &mut Ast, trivia: &mut Vec) -> Result { let info = classify(node)?; + collect_trivia(node, trivia); let mut fields: BTreeMap> = BTreeMap::new(); for (field, value) in field_entries(node) { let field_id = ast.register_field(field); let mut ids = Vec::new(); for child in children_of(value) { - ids.push(build(child, ast)?); + ids.push(build(child, ast, trivia)?); } fields.insert(field_id, ids); } @@ -171,6 +197,35 @@ fn build(node: &Value, ast: &mut Ast) -> Result { )) } +/// Harvest a token's `leadingTrivia`/`trailingTrivia` pieces (each already +/// filtered to comments/`unexpectedText` upstream) into `out`. Non-token nodes +/// have no trivia keys, so this is a no-op for them. +fn collect_trivia(node: &Value, out: &mut Vec) { + for key in ["leadingTrivia", "trailingTrivia"] { + let Some(Value::Array(pieces)) = node.get(key) else { + continue; + }; + for piece in pieces { + let (Some(kind), Some(range)) = ( + piece.get("kind").and_then(Value::as_str), + parse_range(piece), + ) else { + continue; + }; + let text = piece + .get("text") + .and_then(Value::as_str) + .unwrap_or("") + .to_string(); + out.push(TriviaToken { + kind: kind.to_string(), + text, + range, + }); + } + } +} + /// Parse a node's `range` into a [`yeast::Range`]. /// /// The JSON carries, for `start` and `end`, a 0-based UTF-8 file byte `offset`, @@ -201,14 +256,20 @@ fn parse_range(node: &Value) -> Option { } /// Convert a swift-syntax JSON tree (as produced by [`crate::parse_to_json`]) -/// into a [`yeast::Ast`]. -pub fn json_to_ast(json: &str) -> Result { +/// into a [`yeast::Ast`] plus the comment/`unexpectedText` trivia harvested +/// from it. Both are produced in a single traversal. +pub fn json_to_ast(json: &str) -> Result { let root: Value = serde_json::from_str(json).map_err(|e| format!("invalid JSON: {e}"))?; let mut ast = Ast::with_schema(Schema::new()); - let root_id = build(&root, &mut ast)?; + let mut trivia = Vec::new(); + let root_id = build(&root, &mut ast, &mut trivia)?; ast.set_root(root_id); - Ok(ast) + + // Emit trivia in source order (the traversal visits nodes bottom-up). + trivia.sort_by_key(|t| t.range.start_byte); + + Ok(AdaptedTree { ast, trivia }) } #[cfg(test)] @@ -245,7 +306,9 @@ mod tests { #[test] fn builds_ast_from_json() { - let ast = json_to_ast(sample_json()).expect("adapter should succeed"); + let ast = json_to_ast(sample_json()) + .expect("adapter should succeed") + .ast; let root = ast.get_root(); let root_node = ast.get_node(root).expect("root exists"); assert_eq!(root_node.kind_name(), "sourceFile"); @@ -254,7 +317,9 @@ mod tests { #[test] fn classifies_named_and_anonymous_tokens() { - let ast = json_to_ast(sample_json()).expect("adapter should succeed"); + let ast = json_to_ast(sample_json()) + .expect("adapter should succeed") + .ast; // Walk all nodes and collect (kind_name, is_named) for the two tokens. let mut let_named = None; let mut ident_named = None; @@ -273,7 +338,9 @@ mod tests { #[test] fn preserves_leaf_text() { - let ast = json_to_ast(sample_json()).expect("adapter should succeed"); + let ast = json_to_ast(sample_json()) + .expect("adapter should succeed") + .ast; let ident = ast .nodes() .iter() @@ -286,7 +353,9 @@ mod tests { #[test] fn maps_source_locations() { - let ast = json_to_ast(sample_json()).expect("adapter should succeed"); + let ast = json_to_ast(sample_json()) + .expect("adapter should succeed") + .ast; let ident = ast .nodes() .iter() @@ -300,13 +369,55 @@ mod tests { assert_eq!(ident.end_position(), Point::new(0, 5)); } + #[test] + fn collects_trivia_into_side_channel() { + // A token carrying a trailing line comment in its trivia. + let json = r#"{ + "kind": "sourceFile", + "range": {"start":{"offset":0,"line":1,"column":1},"end":{"offset":14,"line":1,"column":15}}, + "value": { + "kind": "token", + "tokenKind": "integerLiteral(\"1\")", + "text": "1", + "range": {"start":{"offset":0,"line":1,"column":1},"end":{"offset":1,"line":1,"column":2}}, + "trailingTrivia": [ + { + "kind": "lineComment", + "text": "// c", + "range": {"start":{"offset":2,"line":1,"column":3},"end":{"offset":6,"line":1,"column":7}} + } + ] + } + }"#; + let adapted = json_to_ast(json).expect("adapter should succeed"); + + // The comment is in the side channel, with its text and location. + assert_eq!(adapted.trivia.len(), 1); + let comment = &adapted.trivia[0]; + assert_eq!(comment.kind, "lineComment"); + assert_eq!(comment.text, "// c"); + assert_eq!(comment.range.start_byte, 2); + assert_eq!(comment.range.end_byte, 6); + + // It is not embedded in the AST as a node. + assert!( + adapted + .ast + .nodes() + .iter() + .all(|n| n.kind_name() != "lineComment"), + "comment should not appear as an AST node" + ); + } + /// End-to-end: real Swift source parsed by the shim, then adapted into a /// `yeast::Ast`. Requires the Swift toolchain (like the crate's FFI tests). #[test] fn end_to_end_from_swift_source() { - let json = crate::parse_to_json("func f(n: Int) -> Int { return n }") + let json = crate::parse_to_json("func f(n: Int) -> Int { return n } // trailing") .expect("parsing should succeed"); - let ast = json_to_ast(&json).expect("adapter should succeed"); + let adapted = json_to_ast(&json).expect("adapter should succeed"); + let ast = &adapted.ast; let root = ast.get_node(ast.get_root()).expect("root exists"); assert_eq!(root.kind_name(), "sourceFile"); @@ -323,5 +434,15 @@ mod tests { kinds.contains(&"func"), "expected an anonymous `func` token, got kinds: {kinds:?}" ); + + // The trailing comment is recovered into the side channel. + assert!( + adapted + .trivia + .iter() + .any(|t| t.kind == "lineComment" && t.text == "// trailing"), + "expected the trailing comment in the trivia side channel, got: {:?}", + adapted.trivia + ); } } diff --git a/unified/swift-syntax-rs/swift/Sources/SwiftSyntaxFFI/SwiftSyntaxFFI.swift b/unified/swift-syntax-rs/swift/Sources/SwiftSyntaxFFI/SwiftSyntaxFFI.swift index 3dcfcb2c3895..6305b1610d06 100644 --- a/unified/swift-syntax-rs/swift/Sources/SwiftSyntaxFFI/SwiftSyntaxFFI.swift +++ b/unified/swift-syntax-rs/swift/Sources/SwiftSyntaxFFI/SwiftSyntaxFFI.swift @@ -39,20 +39,39 @@ private let keptTriviaKinds: Set = [ "unexpectedText", ] -/// Serialize a trivia collection into an array of `{ kind, text }` pieces, -/// keeping only the kinds in `keptTriviaKinds`. -private func serializeTrivia(_ trivia: Trivia) -> [Any] { - trivia.pieces.compactMap { piece -> [String: Any]? in +/// Serialize a trivia collection into an array of `{ kind, text, range }` +/// pieces, keeping only the kinds in `keptTriviaKinds`. +/// +/// `start` is the absolute position of the first piece (a token's leading +/// trivia starts at `token.position`; its trailing trivia at +/// `token.endPositionBeforeTrailingTrivia`). Each piece's range is derived by +/// accumulating piece lengths, so kept pieces carry an exact source location. +private func serializeTrivia( + _ trivia: Trivia, + startingAt start: AbsolutePosition, + _ converter: SourceLocationConverter +) -> [Any] { + var result: [Any] = [] + var offset = start.utf8Offset + for piece in trivia.pieces { + let length = piece.sourceLength.utf8Length // The label of an enum case mirror is the case name (e.g. "spaces", // "lineComment"), which gives us a stable kind without an exhaustive // switch over every `TriviaPiece` case. let kind = Mirror(reflecting: piece).children.first?.label ?? "\(piece)" - guard keptTriviaKinds.contains(kind) else { return nil } - return [ - "kind": kind, - "text": Trivia(pieces: [piece]).description, - ] + if keptTriviaKinds.contains(kind) { + result.append([ + "kind": kind, + "text": Trivia(pieces: [piece]).description, + "range": [ + "start": location(AbsolutePosition(utf8Offset: offset), converter), + "end": location(AbsolutePosition(utf8Offset: offset + length), converter), + ], + ]) + } + offset += length } + return result } /// Recursively convert a SwiftSyntax node into a JSON-serializable value. @@ -95,11 +114,17 @@ private func serialize( "range": range, ] // Only emit trivia when present; after filtering, most tokens have none. - let leading = serializeTrivia(token.leadingTrivia) + // Leading trivia starts at the token's own position; trailing trivia + // starts just after the token's content. + let leading = serializeTrivia( + token.leadingTrivia, startingAt: token.position, converter) if !leading.isEmpty { result["leadingTrivia"] = leading } - let trailing = serializeTrivia(token.trailingTrivia) + let trailing = serializeTrivia( + token.trailingTrivia, + startingAt: token.endPositionBeforeTrailingTrivia, + converter) if !trailing.isEmpty { result["trailingTrivia"] = trailing } From 58ddeacde08e66e31f5dd8a05b39bcceecfedd4f Mon Sep 17 00:00:00 2001 From: Taus Date: Tue, 7 Jul 2026 15:56:56 +0000 Subject: [PATCH 06/13] unified: Hook up swift-syntax AST At this point it's only a proof-of-concept translation -- it translates `sourceFile` nodes, but everything else gets mapped to 'unsupported_node`. --- Cargo.lock | 4 - shared/yeast-schema/src/schema.rs | 22 ++ shared/yeast/src/lib.rs | 39 +++- shared/yeast/tests/test.rs | 53 +++++ unified/extractor/src/languages/mod.rs | 9 + .../src/languages/swift/adapter.rs} | 51 +---- .../extractor/src/languages/swift/swift.rs | 19 ++ .../tests/fixtures/let_x.swiftsyntax.json | 196 ++++++++++++++++++ .../extractor/tests/swift_syntax_pipeline.rs | 49 +++++ unified/swift-syntax-rs/BUILD.bazel | 2 - unified/swift-syntax-rs/Cargo.toml | 4 - unified/swift-syntax-rs/README.md | 27 +-- unified/swift-syntax-rs/src/lib.rs | 9 +- 13 files changed, 404 insertions(+), 80 deletions(-) rename unified/{swift-syntax-rs/src/yeast_adapter.rs => extractor/src/languages/swift/adapter.rs} (88%) create mode 100644 unified/extractor/tests/fixtures/let_x.swiftsyntax.json create mode 100644 unified/extractor/tests/swift_syntax_pipeline.rs diff --git a/Cargo.lock b/Cargo.lock index 24885fea8563..7a9f19667911 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2853,10 +2853,6 @@ checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" [[package]] name = "swift-syntax-rs" version = "0.1.0" -dependencies = [ - "serde_json", - "yeast", -] [[package]] name = "syn" diff --git a/shared/yeast-schema/src/schema.rs b/shared/yeast-schema/src/schema.rs index 4acd14377a4d..9c438d7c22e8 100644 --- a/shared/yeast-schema/src/schema.rs +++ b/shared/yeast-schema/src/schema.rs @@ -167,6 +167,28 @@ impl Schema { id } + /// Register every kind (named and unnamed) and field *name* from `other` + /// into this schema (idempotent). Ids are assigned in this schema's own id + /// space; existing ids are unchanged. + /// + /// This is used when running desugaring rules over an AST that was built + /// against a different schema (e.g. from an external parser): the rules + /// build output nodes whose kind/field names come from `other`, and those + /// names must resolve in the AST's own schema. Only names are needed — the + /// rule engine resolves kinds/fields by name and does not consult + /// `other`'s field-type or supertype information. + pub fn register_names_from(&mut self, other: &Schema) { + for name in other.kind_ids.keys() { + self.register_kind(name); + } + for name in other.unnamed_kind_ids.keys() { + self.register_unnamed_kind(name); + } + for name in other.field_ids.keys() { + self.register_field(name); + } + } + /// Track a name for a kind ID without registering it as named or /// unnamed. Useful when importing tree-sitter ID tables that may /// contain duplicate IDs across the named/unnamed split. diff --git a/shared/yeast/src/lib.rs b/shared/yeast/src/lib.rs index b2709ab406ee..89facfaea99f 100644 --- a/shared/yeast/src/lib.rs +++ b/shared/yeast/src/lib.rs @@ -524,10 +524,15 @@ impl Ast { self.schema.register_field(name) } - fn union_source_range_of_children( - &self, - fields: &BTreeMap>, - ) -> Option { + /// Register every kind and field name from `schema` into this AST's schema + /// (idempotent). Used before desugaring an externally-built AST so that + /// rules can build output nodes whose kind/field names come from the + /// desugarer's output schema. + pub fn register_names_from_schema(&mut self, schema: &schema::Schema) { + self.schema.register_names_from(schema); + } + + fn union_source_range_of_children(&self, fields: &BTreeMap>) -> Option { let mut start_byte: Option = None; let mut end_byte: Option = None; let mut start_point = Point { row: 0, column: 0 }; @@ -1448,6 +1453,18 @@ impl<'a, C: Clone + Default> Runner<'a, C> { let mut user_ctx = C::default(); self.run_with_ctx(input, &mut user_ctx) } + + /// Run all phases over an already-built `ast`, using the default context + /// (`C::default()`). Unlike [`run_from_tree`](Self::run_from_tree), the AST + /// is supplied by the caller (e.g. built from an external parser's output) + /// rather than constructed from a tree-sitter tree. The caller is + /// responsible for ensuring the AST's schema knows any output kind/field + /// names the rules will build (see [`Ast::register_names_from_schema`]). + pub fn run_from_ast(&self, mut ast: Ast) -> Result { + let mut user_ctx = C::default(); + self.run_phases(&mut ast, &mut user_ctx)?; + Ok(ast) + } } // --------------------------------------------------------------------------- @@ -1470,6 +1487,12 @@ pub trait Desugarer: Send + Sync { /// Parse `tree` against `source` and run the desugaring pipeline. /// Each call constructs a fresh default user context internally. fn run_from_tree(&self, tree: &tree_sitter::Tree, source: &[u8]) -> Result; + + /// Run the desugaring pipeline over an already-built `ast` (e.g. produced + /// by an external parser rather than tree-sitter). The desugarer ensures + /// the AST's schema knows its output kind/field names before running the + /// rules. Each call constructs a fresh default user context internally. + fn run_from_ast(&self, ast: Ast) -> Result; } /// A concrete [`Desugarer`] backed by a [`DesugaringConfig`] for a @@ -1507,4 +1530,12 @@ impl Desugarer for ConcreteDesugarer let runner = Runner::with_schema(self.language.clone(), &self.schema, &self.config.phases); runner.run_from_tree(tree, source) } + + fn run_from_ast(&self, mut ast: Ast) -> Result { + // The AST was built against its own (external) schema; make sure the + // output kind/field names the rules build are resolvable in it. + ast.register_names_from_schema(&self.schema); + let runner = Runner::with_schema(self.language.clone(), &self.schema, &self.config.phases); + runner.run_from_ast(ast) + } } diff --git a/shared/yeast/tests/test.rs b/shared/yeast/tests/test.rs index 3a24709dd9ff..756e219bd890 100644 --- a/shared/yeast/tests/test.rs +++ b/shared/yeast/tests/test.rs @@ -266,6 +266,59 @@ fn test_query_match() { assert!(captures.get_var("right").is_ok()); } +#[test] +fn test_run_from_ast_desugars_hand_built_tree() { + use std::collections::BTreeMap; + + // Output schema for the desugared tree. Its kind/field names must become + // resolvable in the hand-built AST's schema for the rule to build them. + let schema_yaml = r#" +named: + assignment: + left: leaf + leaf: +"#; + + // A rule over an *input* kind (`wrapper`) that is not in the output schema, + // rewriting to an output `assignment` node. + let rules: Vec = vec![yeast::rule!( + (wrapper) + => + (assignment left: (leaf "lit")) + )]; + + let lang: tree_sitter::Language = tree_sitter_ruby::LANGUAGE.into(); + let config = DesugaringConfig::<()>::new() + .add_phase("test", PhaseKind::OneShot, rules) + .with_output_node_types_yaml(schema_yaml); + let desugarer = ConcreteDesugarer::new(lang, config).unwrap(); + + // Build the input AST by hand, as an external parser adapter would. The + // schema starts empty and gains the `wrapper` input kind on the fly. + let mut ast = Ast::with_schema(yeast::schema::Schema::new()); + let wrapper_kind = ast.register_kind("wrapper"); + let root = ast.create_node_with_range( + wrapper_kind, + NodeContent::DynamicString(String::new()), + BTreeMap::new(), + true, + None, + ); + ast.set_root(root); + + // Desugaring the hand-built AST applies the rule, producing `assignment` + // even though the AST was built against a schema with no output kinds. + let out = desugarer + .run_from_ast(ast) + .expect("run_from_ast should succeed"); + let out_root = out.get_node(out.get_root()).expect("root exists"); + assert_eq!(out_root.kind_name(), "assignment"); + let dump = dump_ast(&out, out.get_root(), ""); + assert!(dump.contains("assignment"), "unexpected dump: {dump}"); + assert!(dump.contains("left"), "unexpected dump: {dump}"); + assert!(dump.contains("leaf"), "unexpected dump: {dump}"); +} + #[test] fn test_query_no_match() { let runner: Runner = Runner::new(tree_sitter_ruby::LANGUAGE.into(), &[]); diff --git a/unified/extractor/src/languages/mod.rs b/unified/extractor/src/languages/mod.rs index 20ad599edfb6..52a1bd40ffc7 100644 --- a/unified/extractor/src/languages/mod.rs +++ b/unified/extractor/src/languages/mod.rs @@ -3,6 +3,15 @@ use codeql_extractor::extractor::simple; #[path = "swift/swift.rs"] mod swift; +/// swift-syntax JSON -> `yeast::Ast` adapter for the Swift front-end. +/// +/// Currently exercised by tests and the forthcoming runtime extraction path; +/// `allow(dead_code)` because this is a binary crate, so its public API isn't +/// counted as used until the binary itself calls it. +#[path = "swift/adapter.rs"] +#[allow(dead_code)] +pub mod swift_adapter; + /// Shared YEAST output AST schema for all languages. pub(crate) const OUTPUT_AST_SCHEMA: &str = include_str!("../../ast_types.yml"); diff --git a/unified/swift-syntax-rs/src/yeast_adapter.rs b/unified/extractor/src/languages/swift/adapter.rs similarity index 88% rename from unified/swift-syntax-rs/src/yeast_adapter.rs rename to unified/extractor/src/languages/swift/adapter.rs index e2adf1df5a6a..37d5aac00d2d 100644 --- a/unified/swift-syntax-rs/src/yeast_adapter.rs +++ b/unified/extractor/src/languages/swift/adapter.rs @@ -1,6 +1,10 @@ -//! Adapter that converts the swift-syntax JSON tree (see [`crate::parse_to_json`]) -//! into a [`yeast::Ast`], the in-memory format the CodeQL desugaring rules -//! operate on. +//! Converts the swift-syntax JSON syntax tree into a [`yeast::Ast`], the +//! in-memory format the CodeQL desugaring rules operate on. +//! +//! The JSON tree is produced by the `swift-syntax-rs` crate's Swift FFI shim +//! (`parse_to_json`). This module is pure Rust (only `yeast` + `serde_json`), +//! so the extractor consumes swift-syntax output without pulling in the Swift +//! toolchain (the JSON is produced out-of-process). //! //! The mapping mirrors tree-sitter's node model, which is what yeast (and the //! extractor's rewrite rules) expect: @@ -15,9 +19,8 @@ //! list-valued field maps directly to that field holding several children. //! //! Note: this preserves swift-syntax's own kind/field names. Aligning those -//! names with the tree-sitter-swift schema (so the existing rewrite rules fire) -//! is a separate, later step; this module is only concerned with getting the -//! tree into yeast's format. +//! names with the tree-sitter-swift schema (so the rewrite rules in +//! [`super::swift`] fire) is done incrementally in the rules. use std::collections::BTreeMap; @@ -409,40 +412,4 @@ mod tests { "comment should not appear as an AST node" ); } - - /// End-to-end: real Swift source parsed by the shim, then adapted into a - /// `yeast::Ast`. Requires the Swift toolchain (like the crate's FFI tests). - #[test] - fn end_to_end_from_swift_source() { - let json = crate::parse_to_json("func f(n: Int) -> Int { return n } // trailing") - .expect("parsing should succeed"); - let adapted = json_to_ast(&json).expect("adapter should succeed"); - let ast = &adapted.ast; - - let root = ast.get_node(ast.get_root()).expect("root exists"); - assert_eq!(root.kind_name(), "sourceFile"); - - // The tree contains a `functionDecl` layout node and an anonymous - // `func` keyword token keyed by its text. - let mut kinds: Vec<&str> = ast.nodes().iter().map(|n| n.kind_name()).collect(); - kinds.sort_unstable(); - assert!( - kinds.contains(&"functionDecl"), - "expected a functionDecl node, got kinds: {kinds:?}" - ); - assert!( - kinds.contains(&"func"), - "expected an anonymous `func` token, got kinds: {kinds:?}" - ); - - // The trailing comment is recovered into the side channel. - assert!( - adapted - .trivia - .iter() - .any(|t| t.kind == "lineComment" && t.text == "// trailing"), - "expected the trailing comment in the trivia side channel, got: {:?}", - adapted.trivia - ); - } } diff --git a/unified/extractor/src/languages/swift/swift.rs b/unified/extractor/src/languages/swift/swift.rs index 74809ea18e41..13f1a6beadf0 100644 --- a/unified/extractor/src/languages/swift/swift.rs +++ b/unified/extractor/src/languages/swift/swift.rs @@ -135,6 +135,25 @@ fn translation_rules() -> Vec> { // Declarations may be wrapped in local/global wrapper nodes. rule!((global_declaration _ @inner) => stmt { inner }), rule!((local_declaration _ @inner) => stmt { inner }), + // ---- swift-syntax front-end (minimal hook-up) ---- + // These rules target the swift-syntax AST (camelCase kind names), + // produced by the sibling `adapter` module. They coexist with the + // tree-sitter rules (snake_case names): rules are dispatched by exact + // kind name, and the two name spaces never collide, so these are inert + // on the tree-sitter path. Only the minimal top-level mapping lives here + // to demonstrate the pipeline end-to-end; the full translation is added + // separately. Unmatched swift-syntax nodes fall through to the + // `unsupported_node` fallback at the end. + // + // `sourceFile` holds its top-level statements in an (elided) + // `statements` collection; each element is a `codeBlockItem` wrapping + // the real node. + rule!( + (sourceFile statements: _* @items) + => + (top_level body: (block stmt: {items})) + ), + rule!((codeBlockItem item: @item) => stmt { item }), // ---- Literals ---- rule!((integer_literal) => (int_literal)), rule!((hex_literal) => (int_literal)), diff --git a/unified/extractor/tests/fixtures/let_x.swiftsyntax.json b/unified/extractor/tests/fixtures/let_x.swiftsyntax.json new file mode 100644 index 000000000000..6c3d18e2fef0 --- /dev/null +++ b/unified/extractor/tests/fixtures/let_x.swiftsyntax.json @@ -0,0 +1,196 @@ +{ + "endOfFileToken": { + "kind": "token", + "range": { + "end": { + "column": 1, + "line": 2, + "offset": 10 + }, + "start": { + "column": 1, + "line": 2, + "offset": 10 + } + }, + "text": "", + "tokenKind": "endOfFile" + }, + "kind": "sourceFile", + "range": { + "end": { + "column": 1, + "line": 2, + "offset": 10 + }, + "start": { + "column": 1, + "line": 1, + "offset": 0 + } + }, + "statements": [ + { + "item": { + "attributes": [], + "bindings": [ + { + "initializer": { + "equal": { + "kind": "token", + "range": { + "end": { + "column": 8, + "line": 1, + "offset": 7 + }, + "start": { + "column": 7, + "line": 1, + "offset": 6 + } + }, + "text": "=", + "tokenKind": "equal" + }, + "kind": "initializerClause", + "range": { + "end": { + "column": 10, + "line": 1, + "offset": 9 + }, + "start": { + "column": 7, + "line": 1, + "offset": 6 + } + }, + "value": { + "kind": "integerLiteralExpr", + "literal": { + "kind": "token", + "range": { + "end": { + "column": 10, + "line": 1, + "offset": 9 + }, + "start": { + "column": 9, + "line": 1, + "offset": 8 + } + }, + "text": "1", + "tokenKind": "integerLiteral(\"1\")" + }, + "range": { + "end": { + "column": 10, + "line": 1, + "offset": 9 + }, + "start": { + "column": 9, + "line": 1, + "offset": 8 + } + } + } + }, + "kind": "patternBinding", + "pattern": { + "identifier": { + "kind": "token", + "range": { + "end": { + "column": 6, + "line": 1, + "offset": 5 + }, + "start": { + "column": 5, + "line": 1, + "offset": 4 + } + }, + "text": "x", + "tokenKind": "identifier(\"x\")" + }, + "kind": "identifierPattern", + "range": { + "end": { + "column": 6, + "line": 1, + "offset": 5 + }, + "start": { + "column": 5, + "line": 1, + "offset": 4 + } + } + }, + "range": { + "end": { + "column": 10, + "line": 1, + "offset": 9 + }, + "start": { + "column": 5, + "line": 1, + "offset": 4 + } + } + } + ], + "bindingSpecifier": { + "kind": "token", + "range": { + "end": { + "column": 4, + "line": 1, + "offset": 3 + }, + "start": { + "column": 1, + "line": 1, + "offset": 0 + } + }, + "text": "let", + "tokenKind": "keyword(SwiftSyntax.Keyword.let)" + }, + "kind": "variableDecl", + "modifiers": [], + "range": { + "end": { + "column": 10, + "line": 1, + "offset": 9 + }, + "start": { + "column": 1, + "line": 1, + "offset": 0 + } + } + }, + "kind": "codeBlockItem", + "range": { + "end": { + "column": 10, + "line": 1, + "offset": 9 + }, + "start": { + "column": 1, + "line": 1, + "offset": 0 + } + } + } + ] +} diff --git a/unified/extractor/tests/swift_syntax_pipeline.rs b/unified/extractor/tests/swift_syntax_pipeline.rs new file mode 100644 index 000000000000..cdaae1f47c6e --- /dev/null +++ b/unified/extractor/tests/swift_syntax_pipeline.rs @@ -0,0 +1,49 @@ +//! Integration test for the swift-syntax front-end pipeline: +//! +//! swift-syntax JSON -> `swift_adapter::json_to_ast` -> yeast `Ast` +//! -> `Desugarer::run_from_ast` (the real Swift translation rules) -> dump. +//! +//! This exercises the whole chain *without* the Swift toolchain: the JSON +//! fixture is a real `parse_to_json` dump (see the file header) fed through the +//! pure-Rust adapter module. It verifies the desugarer runs end-to-end over an +//! externally-built AST. + +use yeast::dump::dump_ast; + +#[path = "../src/languages/mod.rs"] +mod languages; + +/// A real `swift-syntax-rs` JSON dump of the Swift source `let x = 1`. +const LET_X_JSON: &str = include_str!("fixtures/let_x.swiftsyntax.json"); + +#[test] +fn swift_syntax_json_runs_through_the_desugarer() { + let lang = languages::all_language_specs() + .into_iter() + .find(|l| l.file_globs.iter().any(|g| g.contains("swift"))) + .expect("swift language spec"); + let desugarer = lang.desugar.as_deref().expect("swift desugarer"); + + // Adapt the swift-syntax JSON into a yeast AST (pure Rust, no Swift FFI). + let adapted = + languages::swift_adapter::json_to_ast(LET_X_JSON).expect("adapter should succeed"); + assert_eq!( + adapted + .ast + .get_node(adapted.ast.get_root()) + .unwrap() + .kind_name(), + "sourceFile" + ); + + // Run the real Swift desugaring rules over the externally-built AST. The + // top-level swift-syntax rules map `sourceFile` to `top_level`/`block`; + // kinds without swift-syntax rules yet fall back to `unsupported_node`. + let desugared = desugarer + .run_from_ast(adapted.ast) + .expect("desugaring an externally-built AST should not error"); + + let dump = dump_ast(&desugared, desugared.get_root(), ""); + assert!(dump.contains("top_level"), "unexpected dump: {dump}"); + assert!(dump.contains("block"), "unexpected dump: {dump}"); +} diff --git a/unified/swift-syntax-rs/BUILD.bazel b/unified/swift-syntax-rs/BUILD.bazel index 3a2e5ed0796b..9db40d4db30c 100644 --- a/unified/swift-syntax-rs/BUILD.bazel +++ b/unified/swift-syntax-rs/BUILD.bazel @@ -35,8 +35,6 @@ rust_library( edition = "2024", deps = [ ":swift_syntax_ffi", - "//shared/yeast", - "@vendor_ts__serde_json-1.0.145//:serde_json", ], ) diff --git a/unified/swift-syntax-rs/Cargo.toml b/unified/swift-syntax-rs/Cargo.toml index 576246468480..6957fb0e50e9 100644 --- a/unified/swift-syntax-rs/Cargo.toml +++ b/unified/swift-syntax-rs/Cargo.toml @@ -12,7 +12,3 @@ path = "src/lib.rs" [[bin]] name = "swift-syntax-parse" path = "src/main.rs" - -[dependencies] -serde_json = "1.0" -yeast = { path = "../../shared/yeast" } diff --git a/unified/swift-syntax-rs/README.md b/unified/swift-syntax-rs/README.md index 08cabeb736b0..a2d66495a541 100644 --- a/unified/swift-syntax-rs/README.md +++ b/unified/swift-syntax-rs/README.md @@ -171,26 +171,12 @@ echo 'let x = 1' | cargo run --bin swift-syntax-parse ## Converting to a yeast AST -For use in the CodeQL extractor, the JSON tree can be converted into a -[`yeast::Ast`](../../shared/yeast) — the in-memory format the extractor's -rewrite rules operate on — via [`yeast_adapter::json_to_ast`](src/yeast_adapter.rs): - -```rust -let json = swift_syntax_rs::parse_to_json("let x = 1")?; -let adapted = swift_syntax_rs::yeast_adapter::json_to_ast(&json)?; -let ast = adapted.ast; // the yeast::Ast -let comments = adapted.trivia; // side-channel comment/unexpectedText tokens -``` - -The adapter mirrors tree-sitter's node model, which is what yeast expects: -layout nodes and varying tokens (identifiers, literals, operators) become -**named** nodes; fixed tokens (keywords, punctuation) become **anonymous** -nodes keyed by their text. Comments (and `unexpectedText`) are harvested into a -side channel (`adapted.trivia`) during the same traversal rather than embedded -in the tree, matching how the extractor treats tree-sitter `extra` nodes. It -preserves swift-syntax's own kind/field names — aligning them with the -tree-sitter-swift schema so the existing rewrite rules fire is a separate, -later step. +The JSON tree is consumed by the CodeQL extractor, which converts it into a +[`yeast::Ast`](../../shared/yeast) — the in-memory format its rewrite rules +operate on. That adapter is a pure-Rust module living in the extractor +(`unified/extractor/src/languages/swift/adapter.rs`), so the extractor never +needs the Swift toolchain: it consumes the JSON produced out-of-process by this +crate's `parse_to_json` / the `swift-syntax-parse` binary. ## Layout @@ -198,5 +184,4 @@ later step. - `build.rs` — builds the Swift package and emits link/rpath flags (local `cargo` only). - `BUILD.bazel` — Bazel targets for the hermetic CI build (swift_library + rust targets). - `src/lib.rs` — safe Rust bindings (`parse_to_json`). -- `src/yeast_adapter.rs` — converts the JSON tree into a `yeast::Ast`. - `src/main.rs` — demo CLI. diff --git a/unified/swift-syntax-rs/src/lib.rs b/unified/swift-syntax-rs/src/lib.rs index aa711e322a49..2544c0bc408d 100644 --- a/unified/swift-syntax-rs/src/lib.rs +++ b/unified/swift-syntax-rs/src/lib.rs @@ -3,9 +3,12 @@ //! //! The heavy lifting is done by a small Swift shim (see `swift/`) that links //! against `SwiftSyntax`/`SwiftParser` and exposes a tiny C ABI. This module -//! provides safe Rust bindings on top of that ABI. - -pub mod yeast_adapter; +//! provides safe Rust bindings on top of that ABI, exposing [`parse_to_json`] +//! which turns Swift source into a JSON syntax tree. +//! +//! Converting that JSON into a `yeast::Ast` (for the CodeQL extractor) is done +//! by the extractor's own pure-Rust adapter module, keeping the Swift toolchain +//! out of the extractor's build. use std::ffi::{CStr, CString}; use std::os::raw::c_char; From 3e2daf2258da21c31f6a657295de20c4da0a9708 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 13 Jul 2026 14:12:30 +0000 Subject: [PATCH 07/13] Support building swift-syntax-rs on macOS --- MODULE.bazel | 34 +++---- unified/swift-syntax-rs/BUILD.bazel | 53 ++++++----- unified/swift-syntax-rs/README.md | 14 ++- unified/swift-syntax-rs/build.rs | 2 +- unified/swift-syntax-rs/swift/Package.swift | 5 ++ unified/swift-syntax-rs/xcode_transition.bzl | 94 ++++++++++++++++++++ 6 files changed, 161 insertions(+), 41 deletions(-) create mode 100644 unified/swift-syntax-rs/xcode_transition.bzl diff --git a/MODULE.bazel b/MODULE.bazel index aa1f6555e0ca..4c9e3316c018 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -33,6 +33,11 @@ bazel_dep(name = "googletest", version = "1.17.0.bcr.2") bazel_dep(name = "rules_rust", version = "0.69.0") bazel_dep(name = "rules_swift", version = "4.0.0-rc4") bazel_dep(name = "swift-syntax", version = "602.0.0.bcr.2") + +# Needed so we can `use_repo` `local_config_xcode` and +# `local_config_apple_cc_toolchains` below (referenced by the per-target +# Xcode-config transition in `unified/swift-syntax-rs/xcode_transition.bzl`). +bazel_dep(name = "apple_support", version = "2.6.1") bazel_dep(name = "zstd", version = "1.5.7.bcr.1") bazel_dep(name = "buildifier_prebuilt", version = "6.4.0", dev_dependency = True) @@ -221,22 +226,11 @@ use_repo( "swift-resource-dir-macos", ) -# Hermetic Swift toolchain (from swift.org) for building the `swift-syntax` -# based Rust wrapper in `unified/swift-syntax-rs`. This is the official -# `rules_swift` standalone-toolchain extension and is independent of the -# patched prebuilt toolchain wired up via `swift_deps` above. -# -# Bazel cannot auto-select between Linux distributions, so the toolchain is -# registered explicitly for the distribution our CI runs on (ubuntu24.04, -# x86_64). Add further platforms here if CI grows to cover them. -# -# We register the `exec` toolchain (normal host compilation, targeting -# linux/x86_64) rather than the `embedded` one, which targets `os:none` -# (Embedded Swift) and would not match a normal host build. -# -# The Swift version is read from `unified/swift-syntax-rs/.swift-version`, which -# is the single source of truth shared with the local `cargo` build (via -# swiftly / any Swift install) and `swift/Package.swift`. +# Swift toolchain for building `unified/swift-syntax-rs`. On Linux we register +# a hermetic swift.org toolchain as the exec toolchain; on macOS `rules_swift` +# auto-registers `xcode_swift_toolchain` (host Xcode + OS-provided Swift +# runtime), which is not hermetic. Version comes from +# `unified/swift-syntax-rs/.swift-version`. swift = use_extension("@rules_swift//swift:extensions.bzl", "swift") swift.toolchain( name = "swift_toolchain", @@ -252,6 +246,14 @@ register_toolchains( "@swift_toolchain//:swift_toolchain_exec_ubuntu24.04", ) +# `apple_support`'s xcode_config and CC toolchains, needed by the Xcode +# transition in `unified/swift-syntax-rs/xcode_transition.bzl`. +xcode_configure = use_extension("@apple_support//xcode:xcode_configure.bzl", "xcode_configure_extension") +use_repo(xcode_configure, "local_config_xcode") + +apple_cc_configure = use_extension("@apple_support//crosstool:setup.bzl", "apple_cc_configure_extension") +use_repo(apple_cc_configure, "local_config_apple_cc_toolchains") + node = use_extension("@rules_nodejs//nodejs:extensions.bzl", "node") node.toolchain( name = "nodejs", diff --git a/unified/swift-syntax-rs/BUILD.bazel b/unified/swift-syntax-rs/BUILD.bazel index 9db40d4db30c..be7d41eaf871 100644 --- a/unified/swift-syntax-rs/BUILD.bazel +++ b/unified/swift-syntax-rs/BUILD.bazel @@ -1,31 +1,36 @@ load("@rules_rust//rust:defs.bzl", "rust_binary", "rust_library", "rust_test") -load("@rules_swift//swift:swift.bzl", "swift_library") +load(":xcode_transition.bzl", "xcode_transition_swift_library") package(default_visibility = ["//visibility:public"]) -# The pinned Swift version, shared with the local `cargo` build and -# `swift/Package.swift`. Referenced by the `swift.toolchain` extension in -# //:MODULE.bazel via `swift_version_file`. +# Pinned Swift version, shared with `cargo` and `swift/Package.swift`. +# Referenced by `swift.toolchain(swift_version_file = ...)` in //:MODULE.bazel. exports_files([".swift-version"]) -# The Swift FFI shim: wraps swift-syntax and exposes a small C ABI -# (`ssr_parse_json` / `ssr_string_free`). Built with the hermetic Swift -# toolchain registered in //:MODULE.bazel; provides a `CcInfo` that the Rust -# targets below link against. -swift_library( +# Targets in this package require a Swift toolchain (Linux or macOS). +# `select()` gives us OR-of-OSes; other platforms get marked incompatible +# so `bazel build/test //...` skips them cleanly. +_SWIFT_SUPPORTED_PLATFORMS = select({ + "@platforms//os:linux": [], + "@platforms//os:macos": [], + "//conditions:default": ["@platforms//:incompatible"], +}) + +# Swift FFI shim: wraps swift-syntax and exposes a small C ABI. The Rust +# targets below link against its `CcInfo`. +xcode_transition_swift_library( name = "swift_syntax_ffi", srcs = ["swift/Sources/SwiftSyntaxFFI/SwiftSyntaxFFI.swift"], module_name = "SwiftSyntaxFFI", + target_compatible_with = _SWIFT_SUPPORTED_PLATFORMS, deps = [ "@swift-syntax//:SwiftParser", "@swift-syntax//:SwiftSyntax", ], ) -# Safe Rust bindings on top of the C ABI. `build.rs` is intentionally *not* -# wired up here (no `cargo_build_script`): under Bazel the Swift side is -# provided by `:swift_syntax_ffi`, whereas `build.rs` only exists to build the -# Swift shim in the local `cargo` workflow. +# Safe Rust bindings on top of the C ABI. Under Bazel the Swift side comes +# from `:swift_syntax_ffi`; `build.rs` is only used by the `cargo` workflow. rust_library( name = "swift_syntax_rs", srcs = glob( @@ -33,6 +38,7 @@ rust_library( exclude = ["src/main.rs"], ), edition = "2024", + target_compatible_with = _SWIFT_SUPPORTED_PLATFORMS, deps = [ ":swift_syntax_ffi", ], @@ -41,13 +47,16 @@ rust_library( rust_binary( name = "swift-syntax-parse", srcs = ["src/main.rs"], - # The Swift toolchain propagates a runfiles-relative RPATH to its runtime - # `.so`s via `swift_syntax_ffi`'s `CcInfo`, but (unlike `swift_binary`) - # `rust_binary` does not copy those libraries into runfiles. Add the - # toolchain files as `data` so the runtime is found at execution time. - # (rules_swift does not yet support statically linking the runtime.) - data = ["@swift_toolchain_ubuntu24.04//:files"], + # `rust_binary` doesn't copy the Swift runtime into runfiles the way + # `swift_binary` does. On Linux, ship the standalone toolchain's runtime; + # on macOS the OS provides it at `/usr/lib/swift` (rpath'd by + # `xcode_swift_toolchain`). + data = select({ + "@platforms//os:macos": [], + "@platforms//os:linux": ["@swift_toolchain_ubuntu24.04//:files"], + }), edition = "2024", + target_compatible_with = _SWIFT_SUPPORTED_PLATFORMS, deps = [":swift_syntax_rs"], ) @@ -55,6 +64,10 @@ rust_test( name = "swift_syntax_rs_test", size = "small", crate = ":swift_syntax_rs", - data = ["@swift_toolchain_ubuntu24.04//:files"], + data = select({ + "@platforms//os:macos": [], + "@platforms//os:linux": ["@swift_toolchain_ubuntu24.04//:files"], + }), edition = "2024", + target_compatible_with = _SWIFT_SUPPORTED_PLATFORMS, ) diff --git a/unified/swift-syntax-rs/README.md b/unified/swift-syntax-rs/README.md index a2d66495a541..6a714137d8c7 100644 --- a/unified/swift-syntax-rs/README.md +++ b/unified/swift-syntax-rs/README.md @@ -144,10 +144,16 @@ Requirements: - **`clang`** must be installed on the runner. `rules_swift` requires the Bazel CC toolchain to use clang; the repo's `.bazelrc` already sets `--repo_env=CC=clang`, so no extra flags are needed. -- The registered Swift toolchain currently targets **ubuntu24.04 / x86_64** - only (Bazel cannot auto-select between Linux distributions). Add more - platforms in `MODULE.bazel` (`swift.toolchain` + `register_toolchains`) if CI - grows to cover them. +- The registered Swift toolchains cover **ubuntu24.04 / x86_64** and + **macOS / `xcode`** (Apple Silicon and Intel). Bazel selects the toolchain + matching the host. Targets are marked `target_compatible_with` these two + OSes, so on Windows Bazel skips them cleanly. +- **macOS only:** the Swift toolchain comes from the host Xcode installation + (`rules_swift` auto-registers `xcode_swift_toolchain`), which also needs + Xcode's CC toolchain and xcode_config; these are applied to the Swift + target via an incoming-edge Starlark transition (see + [`xcode_transition.bzl`](xcode_transition.bzl)), so other targets on macOS + keep using Bazel's default CC toolchain. The Swift compiler version is read from [`.swift-version`](.swift-version) by both the Bazel toolchain (`swift.toolchain(swift_version_file = …)`) and the diff --git a/unified/swift-syntax-rs/build.rs b/unified/swift-syntax-rs/build.rs index 0adb140140dc..3cd2e2858742 100644 --- a/unified/swift-syntax-rs/build.rs +++ b/unified/swift-syntax-rs/build.rs @@ -73,7 +73,7 @@ fn swift_runtime_dir() -> Option { let close = value_start.find('"')?; let resource_path = &value_start[..close]; - Some(PathBuf::from(resource_path).join("linux")) + Some(PathBuf::from(resource_path).join(if cfg!(target_os = "macos") { "macosx" } else { "linux" })) } /// The `swift` driver to invoke: `$SWIFT` if set, otherwise `swift` from `PATH`. diff --git a/unified/swift-syntax-rs/swift/Package.swift b/unified/swift-syntax-rs/swift/Package.swift index 1b0111364ce7..fcd9fcc07bca 100644 --- a/unified/swift-syntax-rs/swift/Package.swift +++ b/unified/swift-syntax-rs/swift/Package.swift @@ -3,6 +3,11 @@ import PackageDescription let package = Package( name: "SwiftSyntaxFFI", + platforms: [ + // swift-syntax 602 requires macOS 10.15; declare it explicitly + // rather than relying on the swift-tools-version default (10.13). + .macOS(.v10_15), + ], products: [ // Dynamic library so the produced .so records its dependency on the // Swift runtime libraries, keeping the Rust link step simple. diff --git a/unified/swift-syntax-rs/xcode_transition.bzl b/unified/swift-syntax-rs/xcode_transition.bzl new file mode 100644 index 000000000000..c6f293a0452c --- /dev/null +++ b/unified/swift-syntax-rs/xcode_transition.bzl @@ -0,0 +1,94 @@ +"""Per-target Xcode configuration for `//unified/swift-syntax-rs` on macOS. + +`rules_swift`'s auto-registered `xcode_swift_toolchain` reads +`cc_toolchain.target_gnu_system_name`, which is literally "local" on +Bazel's built-in `local_config_cc`. To make it usable we need: + +- `--xcode_version_config=@local_config_xcode//:host_xcodes` — selects + `apple_support`'s xcode_config (matches `rules_swift`'s `system_sdk` keys). +- `--extra_toolchains=@local_config_apple_cc_toolchains//:all` — forces + `apple_support`'s CC toolchain ahead of `local_config_cc`. + +Applied via an incoming-edge Starlark transition on the `swift_library` +target only (via the `xcode_transition_swift_library` macro), so downstream +Rust targets and the rest of the repo stay on the default CC toolchain and +the `@local_config_*` repos are not materialized otherwise. No-op off macOS. +""" + +load("@rules_cc//cc/common:cc_info.bzl", "CcInfo") +load("@rules_swift//swift:swift.bzl", "swift_library") +load("//misc/bazel:os.bzl", "os_select") + +_XCODE_VERSION_CONFIG = "//command_line_option:xcode_version_config" +_EXTRA_TOOLCHAINS = "//command_line_option:extra_toolchains" + +def _transition_impl(settings, attr): + if attr.os != "macos": + # Preserve inputs so the configuration is identical to the incoming one. + return { + _XCODE_VERSION_CONFIG: settings[_XCODE_VERSION_CONFIG], + _EXTRA_TOOLCHAINS: settings[_EXTRA_TOOLCHAINS], + } + return { + _XCODE_VERSION_CONFIG: "@local_config_xcode//:host_xcodes", + _EXTRA_TOOLCHAINS: ( + list(settings[_EXTRA_TOOLCHAINS]) + + ["@local_config_apple_cc_toolchains//:all"] + ), + } + +_xcode_transition = transition( + implementation = _transition_impl, + inputs = [_XCODE_VERSION_CONFIG, _EXTRA_TOOLCHAINS], + outputs = [_XCODE_VERSION_CONFIG, _EXTRA_TOOLCHAINS], +) + +def _wrapper_impl(ctx): + src = ctx.attr.actual[0] + # Forward the providers a downstream `rust_*` target reads from `deps`: + # `DefaultInfo`, `CcInfo` (linking info), and `OutputGroupInfo`. + providers = [src[DefaultInfo]] + for p in (CcInfo, OutputGroupInfo): + if p in src: + providers.append(src[p]) + return providers + +_xcode_transition_swift_library_rule = rule( + implementation = _wrapper_impl, + attrs = { + "actual": attr.label( + mandatory = True, + cfg = _xcode_transition, + providers = [CcInfo], + ), + "os": attr.string(mandatory = True), + "_allowlist_function_transition": attr.label( + default = "@bazel_tools//tools/allowlists/function_transition_allowlist", + ), + }, +) + +def xcode_transition_swift_library(name, visibility = None, tags = None, target_compatible_with = None, **kwargs): + """`swift_library` wrapped in the macOS Xcode-config transition. + + Emits a private inner `_impl_` `swift_library` (tagged `manual`) + and a public `name` that applies the transition on macOS and forwards + the inner target's providers. Downstream `rust_*` targets depend on + `name` as usual; only the `swift_library` sub-graph flips toolchain. + """ + inner_name = "_impl_%s" % name + swift_library( + name = inner_name, + visibility = ["//visibility:private"], + tags = (tags or []) + ["manual"], + target_compatible_with = target_compatible_with, + **kwargs + ) + _xcode_transition_swift_library_rule( + name = name, + visibility = visibility, + tags = tags, + target_compatible_with = target_compatible_with, + actual = ":" + inner_name, + os = os_select(linux = "linux", macos = "macos", default = "other"), + ) From 1577d82ebd9d4f528ebf5ca579b1af76956dac4f Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 14 Jul 2026 14:45:46 +0000 Subject: [PATCH 08/13] Upgrade swift-syntax-rs to swift-syntax 603.0.2 / Swift 6.3.2 --- MODULE.bazel | 2 +- unified/swift-syntax-rs/.swift-version | 2 +- unified/swift-syntax-rs/README.md | 2 +- unified/swift-syntax-rs/swift/Package.resolved | 6 +++--- unified/swift-syntax-rs/swift/Package.swift | 4 ++-- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/MODULE.bazel b/MODULE.bazel index 4c9e3316c018..37a71583198f 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -32,7 +32,7 @@ bazel_dep(name = "rules_dotnet", version = "0.21.5-codeql.1") bazel_dep(name = "googletest", version = "1.17.0.bcr.2") bazel_dep(name = "rules_rust", version = "0.69.0") bazel_dep(name = "rules_swift", version = "4.0.0-rc4") -bazel_dep(name = "swift-syntax", version = "602.0.0.bcr.2") +bazel_dep(name = "swift-syntax", version = "603.0.2") # Needed so we can `use_repo` `local_config_xcode` and # `local_config_apple_cc_toolchains` below (referenced by the per-target diff --git a/unified/swift-syntax-rs/.swift-version b/unified/swift-syntax-rs/.swift-version index 42cc526d6ca7..91e4a9f26224 100644 --- a/unified/swift-syntax-rs/.swift-version +++ b/unified/swift-syntax-rs/.swift-version @@ -1 +1 @@ -6.2.4 +6.3.2 diff --git a/unified/swift-syntax-rs/README.md b/unified/swift-syntax-rs/README.md index 6a714137d8c7..9b14d4026f37 100644 --- a/unified/swift-syntax-rs/README.md +++ b/unified/swift-syntax-rs/README.md @@ -98,7 +98,7 @@ The build does not depend on any particular version manager. You need: - **Rust** — pinned to `1.88` by the repo-root [`rust-toolchain.toml`](../../rust-toolchain.toml), which `rustup` picks up automatically. - **Swift** — pinned to the version in [`.swift-version`](.swift-version) - (currently `6.2.4`), used to build `swift-syntax` `602.0.0`. Install it any way + (currently `6.3.2`), used to build `swift-syntax` `603.0.2`. Install it any way you like — [swift.org](https://www.swift.org/install/) or [swiftly](https://www.swift.org/swiftly/) (which reads `.swift-version`), or a system package. Just make sure `swift` is on your `PATH` (or point `build.rs` diff --git a/unified/swift-syntax-rs/swift/Package.resolved b/unified/swift-syntax-rs/swift/Package.resolved index 9cebf34ec543..dafd46e961b5 100644 --- a/unified/swift-syntax-rs/swift/Package.resolved +++ b/unified/swift-syntax-rs/swift/Package.resolved @@ -1,13 +1,13 @@ { - "originHash" : "ad84ad340ee01aa6e38b877dd29e38fa9cb40603dfdefa4e6a69e27f2db208d1", + "originHash" : "169957f8fbd882866f8f9f1c68cde813dca3560577939a8ffec366e3d77548e4", "pins" : [ { "identity" : "swift-syntax", "kind" : "remoteSourceControl", "location" : "https://github.com/swiftlang/swift-syntax.git", "state" : { - "revision" : "4799286537280063c85a32f09884cfbca301b1a1", - "version" : "602.0.0" + "revision" : "79e4b74a295b6eb74a8b585e3a39d29e70c1dbd1", + "version" : "603.0.2" } } ], diff --git a/unified/swift-syntax-rs/swift/Package.swift b/unified/swift-syntax-rs/swift/Package.swift index fcd9fcc07bca..a58b7c1a479b 100644 --- a/unified/swift-syntax-rs/swift/Package.swift +++ b/unified/swift-syntax-rs/swift/Package.swift @@ -4,7 +4,7 @@ import PackageDescription let package = Package( name: "SwiftSyntaxFFI", platforms: [ - // swift-syntax 602 requires macOS 10.15; declare it explicitly + // swift-syntax 603 requires macOS 10.15; declare it explicitly // rather than relying on the swift-tools-version default (10.13). .macOS(.v10_15), ], @@ -20,7 +20,7 @@ let package = Package( dependencies: [ .package( url: "https://github.com/swiftlang/swift-syntax.git", - exact: "602.0.0" + exact: "603.0.2" ) ], targets: [ From 3eb353ecf76a7f61b6193129e64a9c803ecb7605 Mon Sep 17 00:00:00 2001 From: Taus Date: Wed, 15 Jul 2026 12:10:50 +0000 Subject: [PATCH 09/13] swift-syntax-rs: Address PR review feedback MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - BUILD.bazel: require x86_64 on the Linux branch of the compatibility select. The only registered standalone Linux Swift toolchain is x86_64-only, so without the CPU constraint Linux/aarch64 targets stayed "compatible" and then failed toolchain resolution instead of being skipped cleanly. - build.rs: emitting `rerun-if-changed` disables Cargo's default whole-package scan, so list every input to `swift build` — the package manifests, `Package.resolved`, `.swift-version`, and the whole Sources tree — so stale shim/toolchain output can't linger. (`.build/` is left unwatched, as it is this build's own output.) - src/lib.rs: a null result from the shim is not a parse failure — SwiftParser recovers from invalid syntax and always yields a tree — so it means the shim failed to serialize/allocate the JSON. Fix the error message and the variant doc accordingly. - README.md: `.swift-version` is not honored by the macOS Bazel build (which uses the host Xcode toolchain); document that it pins the Linux and local builds only. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- unified/swift-syntax-rs/BUILD.bazel | 15 ++++++++++++++- unified/swift-syntax-rs/README.md | 13 +++++++++---- unified/swift-syntax-rs/build.rs | 24 ++++++++++++++---------- unified/swift-syntax-rs/src/lib.rs | 9 +++++++-- 4 files changed, 44 insertions(+), 17 deletions(-) diff --git a/unified/swift-syntax-rs/BUILD.bazel b/unified/swift-syntax-rs/BUILD.bazel index be7d41eaf871..fed7df4e611c 100644 --- a/unified/swift-syntax-rs/BUILD.bazel +++ b/unified/swift-syntax-rs/BUILD.bazel @@ -10,8 +10,21 @@ exports_files([".swift-version"]) # Targets in this package require a Swift toolchain (Linux or macOS). # `select()` gives us OR-of-OSes; other platforms get marked incompatible # so `bazel build/test //...` skips them cleanly. +# +# The Linux branch additionally requires x86_64: the only registered standalone +# Linux Swift toolchain (`swift_toolchain_exec_ubuntu24.04`, see //:MODULE.bazel) +# is x86_64-only. Without the CPU constraint, Linux/aarch64 targets would stay +# "compatible" and then fail toolchain resolution instead of being skipped. +config_setting( + name = "linux_x86_64", + constraint_values = [ + "@platforms//os:linux", + "@platforms//cpu:x86_64", + ], +) + _SWIFT_SUPPORTED_PLATFORMS = select({ - "@platforms//os:linux": [], + ":linux_x86_64": [], "@platforms//os:macos": [], "//conditions:default": ["@platforms//:incompatible"], }) diff --git a/unified/swift-syntax-rs/README.md b/unified/swift-syntax-rs/README.md index 9b14d4026f37..990af77d26fd 100644 --- a/unified/swift-syntax-rs/README.md +++ b/unified/swift-syntax-rs/README.md @@ -155,10 +155,15 @@ Requirements: [`xcode_transition.bzl`](xcode_transition.bzl)), so other targets on macOS keep using Bazel's default CC toolchain. -The Swift compiler version is read from [`.swift-version`](.swift-version) by -both the Bazel toolchain (`swift.toolchain(swift_version_file = …)`) and the -local build, and is kept in sync with the `swift-syntax` release pinned in -`swift/Package.swift`, so local and CI builds behave identically. +The Swift compiler version in [`.swift-version`](.swift-version) is kept in sync +with the `swift-syntax` release pinned in `swift/Package.swift`. It is honored on +**Linux**, where the hermetic swift.org Bazel toolchain +(`swift.toolchain(swift_version_file = …)`) and the local `cargo`/`swift build` +both read it. On **macOS** it is *not* honored by the Bazel build: `rules_swift` +auto-registers the host `xcode_swift_toolchain`, which uses whichever Swift ships +with the installed Xcode and ignores `.swift-version`. So the pinned version +governs Linux (and local) builds, while the macOS compiler version depends on the +host Xcode. ## Usage diff --git a/unified/swift-syntax-rs/build.rs b/unified/swift-syntax-rs/build.rs index 3cd2e2858742..c599cdf4e70b 100644 --- a/unified/swift-syntax-rs/build.rs +++ b/unified/swift-syntax-rs/build.rs @@ -6,16 +6,20 @@ fn main() { let manifest_dir = PathBuf::from(env::var("CARGO_MANIFEST_DIR").unwrap()); let swift_dir = manifest_dir.join("swift"); - println!( - "cargo:rerun-if-changed={}", - swift_dir.join("Package.swift").display() - ); - println!( - "cargo:rerun-if-changed={}", - swift_dir - .join("Sources/SwiftSyntaxFFI/SwiftSyntaxFFI.swift") - .display() - ); + // Emitting any `rerun-if-changed` disables Cargo's default whole-package + // scan, so we must list every input to the `swift build` below. Watch the + // package manifests, the pinned dependency lockfile, the pinned compiler + // version, and the entire Swift sources tree (a directory is scanned + // recursively). `.build/` is intentionally *not* watched (it is this + // build's output; watching it would cause perpetual rebuilds). + for input in [ + swift_dir.join("Package.swift"), + swift_dir.join("Package.resolved"), + swift_dir.join("Sources"), + manifest_dir.join(".swift-version"), + ] { + println!("cargo:rerun-if-changed={}", input.display()); + } println!("cargo:rerun-if-env-changed=SWIFT"); println!("cargo:rerun-if-env-changed=SWIFTC"); diff --git a/unified/swift-syntax-rs/src/lib.rs b/unified/swift-syntax-rs/src/lib.rs index 2544c0bc408d..2a8fe0411af8 100644 --- a/unified/swift-syntax-rs/src/lib.rs +++ b/unified/swift-syntax-rs/src/lib.rs @@ -29,7 +29,10 @@ unsafe extern "C" { pub enum ParseError { /// The provided source contained an interior NUL byte. NulByte, - /// The Swift side failed to produce a result. + /// The Swift shim returned no result. `SwiftParser` recovers from invalid + /// syntax (it always produces a tree, possibly with error nodes), so this + /// does *not* indicate a syntax error in the source — it means the shim + /// failed to serialize the tree to JSON or to allocate the result string. SwiftFailure, } @@ -37,7 +40,9 @@ impl std::fmt::Display for ParseError { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { ParseError::NulByte => write!(f, "source contained an interior NUL byte"), - ParseError::SwiftFailure => write!(f, "swift-syntax failed to parse the source"), + ParseError::SwiftFailure => { + write!(f, "the swift-syntax shim failed to produce a JSON result") + } } } } From 46d6a118f4598833a89be79cdf6f06816146500b Mon Sep 17 00:00:00 2001 From: Taus Date: Wed, 15 Jul 2026 12:10:50 +0000 Subject: [PATCH 10/13] yeast: Clarify Ast::with_schema registration doc The doc claimed the schema passed to `Ast::with_schema` must already have every node kind and field name registered, contradicting the registration methods just below and the swift-syntax adapter, which starts from `Schema::new()` and registers names on demand during construction. Document that up-front registration is optional. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- shared/yeast/src/lib.rs | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/shared/yeast/src/lib.rs b/shared/yeast/src/lib.rs index 89facfaea99f..6719a5498e2b 100644 --- a/shared/yeast/src/lib.rs +++ b/shared/yeast/src/lib.rs @@ -358,8 +358,14 @@ impl Ast { /// programmatically from a source other than a tree-sitter parse (e.g. an /// external parser's output). Populate it with [`Ast::create_node`] / /// [`Ast::create_node_with_range`] and then designate the root with - /// [`Ast::set_root`]. The `schema` must already have every node kind and - /// field name registered (see [`schema::Schema`]). + /// [`Ast::set_root`]. + /// + /// Node kind and field names may be registered in `schema` up front, or on + /// demand while building via [`Ast::register_kind`], + /// [`Ast::register_unnamed_kind`], and [`Ast::register_field`] (which return + /// the ids to pass to [`Ast::create_node_with_range`]). Passing a fresh + /// [`schema::Schema::new`] and registering names during construction is + /// therefore fine — see the swift-syntax adapter for an example. pub fn with_schema(schema: schema::Schema) -> Self { Self { root: Id(0), From 205c9a934674b327dccaca3ef4d09ba9c5ff3c11 Mon Sep 17 00:00:00 2001 From: Taus Date: Wed, 15 Jul 2026 12:19:25 +0000 Subject: [PATCH 11/13] swift-syntax-rs: Fix bazel formatting aCo-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- unified/swift-syntax-rs/xcode_transition.bzl | 1 + 1 file changed, 1 insertion(+) diff --git a/unified/swift-syntax-rs/xcode_transition.bzl b/unified/swift-syntax-rs/xcode_transition.bzl index c6f293a0452c..b1612e762b15 100644 --- a/unified/swift-syntax-rs/xcode_transition.bzl +++ b/unified/swift-syntax-rs/xcode_transition.bzl @@ -45,6 +45,7 @@ _xcode_transition = transition( def _wrapper_impl(ctx): src = ctx.attr.actual[0] + # Forward the providers a downstream `rust_*` target reads from `deps`: # `DefaultInfo`, `CcInfo` (linking info), and `OutputGroupInfo`. providers = [src[DefaultInfo]] From 7474a33132cec2331e75c278567416f11f300606 Mon Sep 17 00:00:00 2001 From: Taus Date: Wed, 15 Jul 2026 12:47:26 +0000 Subject: [PATCH 12/13] unified: Hardcode swift_version It seems that using swift_version_file has some issues when `codeql` is consumed as a dependency module. I'm hoping hardcoding the version instead will fix this, but ideally we should find a more robust solution. --- MODULE.bazel | 13 ++++++++++--- unified/swift-syntax-rs/BUILD.bazel | 4 ---- unified/swift-syntax-rs/README.md | 23 ++++++++++++++--------- 3 files changed, 24 insertions(+), 16 deletions(-) diff --git a/MODULE.bazel b/MODULE.bazel index 37a71583198f..3638a07289b8 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -229,12 +229,19 @@ use_repo( # Swift toolchain for building `unified/swift-syntax-rs`. On Linux we register # a hermetic swift.org toolchain as the exec toolchain; on macOS `rules_swift` # auto-registers `xcode_swift_toolchain` (host Xcode + OS-provided Swift -# runtime), which is not hermetic. Version comes from -# `unified/swift-syntax-rs/.swift-version`. +# runtime), which is not hermetic. +# +# The version is pinned as a literal rather than read from +# `unified/swift-syntax-rs/.swift-version` via `swift_version_file`: the latter +# makes the extension `module_ctx.read` a `//unified/...` label, which fails to +# resolve when this repo is consumed as a dependency module (`@@ql+`) whose +# `unified/swift-syntax-rs` package is not loadable in that context. Keep this +# in sync with `unified/swift-syntax-rs/.swift-version` (used by the `cargo` +# build) and the `swift-syntax` release in `swift/Package.swift`. swift = use_extension("@rules_swift//swift:extensions.bzl", "swift") swift.toolchain( name = "swift_toolchain", - swift_version_file = "//unified/swift-syntax-rs:.swift-version", + swift_version = "6.3.2", ) use_repo( swift, diff --git a/unified/swift-syntax-rs/BUILD.bazel b/unified/swift-syntax-rs/BUILD.bazel index fed7df4e611c..02fda50cf502 100644 --- a/unified/swift-syntax-rs/BUILD.bazel +++ b/unified/swift-syntax-rs/BUILD.bazel @@ -3,10 +3,6 @@ load(":xcode_transition.bzl", "xcode_transition_swift_library") package(default_visibility = ["//visibility:public"]) -# Pinned Swift version, shared with `cargo` and `swift/Package.swift`. -# Referenced by `swift.toolchain(swift_version_file = ...)` in //:MODULE.bazel. -exports_files([".swift-version"]) - # Targets in this package require a Swift toolchain (Linux or macOS). # `select()` gives us OR-of-OSes; other platforms get marked incompatible # so `bazel build/test //...` skips them cleanly. diff --git a/unified/swift-syntax-rs/README.md b/unified/swift-syntax-rs/README.md index 990af77d26fd..d43969199f91 100644 --- a/unified/swift-syntax-rs/README.md +++ b/unified/swift-syntax-rs/README.md @@ -155,15 +155,20 @@ Requirements: [`xcode_transition.bzl`](xcode_transition.bzl)), so other targets on macOS keep using Bazel's default CC toolchain. -The Swift compiler version in [`.swift-version`](.swift-version) is kept in sync -with the `swift-syntax` release pinned in `swift/Package.swift`. It is honored on -**Linux**, where the hermetic swift.org Bazel toolchain -(`swift.toolchain(swift_version_file = …)`) and the local `cargo`/`swift build` -both read it. On **macOS** it is *not* honored by the Bazel build: `rules_swift` -auto-registers the host `xcode_swift_toolchain`, which uses whichever Swift ships -with the installed Xcode and ignores `.swift-version`. So the pinned version -governs Linux (and local) builds, while the macOS compiler version depends on the -host Xcode. +The Swift compiler version is kept in sync across three places: the +[`.swift-version`](.swift-version) file (read by the local `cargo`/`swift build` +and by [swiftly](https://www.swift.org/swiftly/)), the literal `swift_version` +pinned on `swift.toolchain(...)` in the root `MODULE.bazel` (the hermetic +swift.org **Linux** Bazel toolchain), and the `swift-syntax` release in +`swift/Package.swift`. On **macOS** the version is *not* pinned by the Bazel +build: `rules_swift` auto-registers the host `xcode_swift_toolchain`, which uses +whichever Swift ships with the installed Xcode. So the pin governs Linux (and +local) builds, while the macOS compiler version depends on the host Xcode. + +(The Bazel toolchain pins a literal rather than reading `.swift-version` via +`swift_version_file`, because the latter makes the module extension read a +`//unified/...` label, which fails when this repo is consumed as a dependency +module.) ## Usage From da1bbb7fac53fb8097e317ae4f2c22ffaec6fc7d Mon Sep 17 00:00:00 2001 From: Taus Date: Wed, 15 Jul 2026 13:07:06 +0000 Subject: [PATCH 13/13] Bazel: regenerate vendored cargo dependencies Registers the `unified/swift-syntax-rs` workspace member (added in "unified: Add swift-syntax-rs") in the tree-sitter extractors crate universe. It has no external Rust dependencies, so its dependency entries are empty. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../tree_sitter_extractors_deps/BUILD.bazel | 12 ++++++++ .../tree_sitter_extractors_deps/defs.bzl | 28 +++++++++++++++++++ 2 files changed, 40 insertions(+) diff --git a/misc/bazel/3rdparty/tree_sitter_extractors_deps/BUILD.bazel b/misc/bazel/3rdparty/tree_sitter_extractors_deps/BUILD.bazel index bb32aa97a5e2..ce1d79a772b9 100644 --- a/misc/bazel/3rdparty/tree_sitter_extractors_deps/BUILD.bazel +++ b/misc/bazel/3rdparty/tree_sitter_extractors_deps/BUILD.bazel @@ -553,6 +553,18 @@ alias( tags = ["manual"], ) +alias( + name = "swift-syntax-rs-0.1.0", + actual = "@vendor_ts__swift-syntax-rs-0.1.0//:swift_syntax_rs", + tags = ["manual"], +) + +alias( + name = "swift-syntax-rs", + actual = "@vendor_ts__swift-syntax-rs-0.1.0//:swift_syntax_rs", + tags = ["manual"], +) + alias( name = "syn-2.0.106", actual = "@vendor_ts__syn-2.0.106//:syn", diff --git a/misc/bazel/3rdparty/tree_sitter_extractors_deps/defs.bzl b/misc/bazel/3rdparty/tree_sitter_extractors_deps/defs.bzl index 7fbdfc4bbd4b..f3b5edb59cd6 100644 --- a/misc/bazel/3rdparty/tree_sitter_extractors_deps/defs.bzl +++ b/misc/bazel/3rdparty/tree_sitter_extractors_deps/defs.bzl @@ -429,6 +429,10 @@ _NORMAL_DEPENDENCIES = { "tree-sitter-language": Label("@vendor_ts__tree-sitter-language-0.1.5//:tree_sitter_language"), }, }, + "unified/swift-syntax-rs": { + _COMMON_CONDITION: { + }, + }, } _NORMAL_ALIASES = { @@ -475,6 +479,10 @@ _NORMAL_ALIASES = { _COMMON_CONDITION: { }, }, + "unified/swift-syntax-rs": { + _COMMON_CONDITION: { + }, + }, } _NORMAL_DEV_DEPENDENCIES = { @@ -505,6 +513,8 @@ _NORMAL_DEV_DEPENDENCIES = { }, "unified/extractor/tree-sitter-swift": { }, + "unified/swift-syntax-rs": { + }, } _NORMAL_DEV_ALIASES = { @@ -532,6 +542,8 @@ _NORMAL_DEV_ALIASES = { }, "unified/extractor/tree-sitter-swift": { }, + "unified/swift-syntax-rs": { + }, } _PROC_MACRO_DEPENDENCIES = { @@ -557,6 +569,8 @@ _PROC_MACRO_DEPENDENCIES = { }, "unified/extractor/tree-sitter-swift": { }, + "unified/swift-syntax-rs": { + }, } _PROC_MACRO_ALIASES = { @@ -582,6 +596,8 @@ _PROC_MACRO_ALIASES = { }, "unified/extractor/tree-sitter-swift": { }, + "unified/swift-syntax-rs": { + }, } _PROC_MACRO_DEV_DEPENDENCIES = { @@ -607,6 +623,8 @@ _PROC_MACRO_DEV_DEPENDENCIES = { }, "unified/extractor/tree-sitter-swift": { }, + "unified/swift-syntax-rs": { + }, } _PROC_MACRO_DEV_ALIASES = { @@ -634,6 +652,8 @@ _PROC_MACRO_DEV_ALIASES = { }, "unified/extractor/tree-sitter-swift": { }, + "unified/swift-syntax-rs": { + }, } _BUILD_DEPENDENCIES = { @@ -663,6 +683,8 @@ _BUILD_DEPENDENCIES = { "tree-sitter-generate": Label("@vendor_ts__tree-sitter-generate-0.26.8//:tree_sitter_generate"), }, }, + "unified/swift-syntax-rs": { + }, } _BUILD_ALIASES = { @@ -690,6 +712,8 @@ _BUILD_ALIASES = { _COMMON_CONDITION: { }, }, + "unified/swift-syntax-rs": { + }, } _BUILD_PROC_MACRO_DEPENDENCIES = { @@ -715,6 +739,8 @@ _BUILD_PROC_MACRO_DEPENDENCIES = { }, "unified/extractor/tree-sitter-swift": { }, + "unified/swift-syntax-rs": { + }, } _BUILD_PROC_MACRO_ALIASES = { @@ -740,6 +766,8 @@ _BUILD_PROC_MACRO_ALIASES = { }, "unified/extractor/tree-sitter-swift": { }, + "unified/swift-syntax-rs": { + }, } _CONDITIONS = {