diff --git a/Cargo.lock b/Cargo.lock index 76043ec0a439..7a9f19667911 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2850,6 +2850,10 @@ version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" +[[package]] +name = "swift-syntax-rs" +version = "0.1.0" + [[package]] name = "syn" version = "2.0.106" diff --git a/Cargo.toml b/Cargo.toml index 9c15b486062b..9f3780bb1d77 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,6 +10,7 @@ members = [ "ruby/extractor", "unified/extractor", "unified/extractor/tree-sitter-swift", + "unified/swift-syntax-rs", "rust/extractor", "rust/extractor/macros", "rust/ast-generator", diff --git a/MODULE.bazel b/MODULE.bazel index e6f1ddb893dd..3638a07289b8 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -24,13 +24,20 @@ bazel_dep(name = "rules_python", version = "1.9.0") bazel_dep(name = "rules_shell", version = "0.7.1") bazel_dep(name = "bazel_skylib", version = "1.9.0") bazel_dep(name = "abseil-cpp", version = "20260107.1", repo_name = "absl") -bazel_dep(name = "nlohmann_json", version = "3.11.3", repo_name = "json") +bazel_dep(name = "nlohmann_json", version = "3.12.0.bcr.1", repo_name = "json") bazel_dep(name = "fmt", version = "12.1.0-codeql.1") bazel_dep(name = "rules_kotlin", version = "2.2.2-codeql.1") bazel_dep(name = "gazelle", version = "0.50.0") bazel_dep(name = "rules_dotnet", version = "0.21.5-codeql.1") bazel_dep(name = "googletest", version = "1.17.0.bcr.2") bazel_dep(name = "rules_rust", version = "0.69.0") +bazel_dep(name = "rules_swift", version = "4.0.0-rc4") +bazel_dep(name = "swift-syntax", version = "603.0.2") + +# Needed so we can `use_repo` `local_config_xcode` and +# `local_config_apple_cc_toolchains` below (referenced by the per-target +# Xcode-config transition in `unified/swift-syntax-rs/xcode_transition.bzl`). +bazel_dep(name = "apple_support", version = "2.6.1") bazel_dep(name = "zstd", version = "1.5.7.bcr.1") bazel_dep(name = "buildifier_prebuilt", version = "6.4.0", dev_dependency = True) @@ -219,6 +226,41 @@ use_repo( "swift-resource-dir-macos", ) +# Swift toolchain for building `unified/swift-syntax-rs`. On Linux we register +# a hermetic swift.org toolchain as the exec toolchain; on macOS `rules_swift` +# auto-registers `xcode_swift_toolchain` (host Xcode + OS-provided Swift +# runtime), which is not hermetic. +# +# The version is pinned as a literal rather than read from +# `unified/swift-syntax-rs/.swift-version` via `swift_version_file`: the latter +# makes the extension `module_ctx.read` a `//unified/...` label, which fails to +# resolve when this repo is consumed as a dependency module (`@@ql+`) whose +# `unified/swift-syntax-rs` package is not loadable in that context. Keep this +# in sync with `unified/swift-syntax-rs/.swift-version` (used by the `cargo` +# build) and the `swift-syntax` release in `swift/Package.swift`. +swift = use_extension("@rules_swift//swift:extensions.bzl", "swift") +swift.toolchain( + name = "swift_toolchain", + swift_version = "6.3.2", +) +use_repo( + swift, + "swift_toolchain", + "swift_toolchain_ubuntu24.04", +) + +register_toolchains( + "@swift_toolchain//:swift_toolchain_exec_ubuntu24.04", +) + +# `apple_support`'s xcode_config and CC toolchains, needed by the Xcode +# transition in `unified/swift-syntax-rs/xcode_transition.bzl`. +xcode_configure = use_extension("@apple_support//xcode:xcode_configure.bzl", "xcode_configure_extension") +use_repo(xcode_configure, "local_config_xcode") + +apple_cc_configure = use_extension("@apple_support//crosstool:setup.bzl", "apple_cc_configure_extension") +use_repo(apple_cc_configure, "local_config_apple_cc_toolchains") + node = use_extension("@rules_nodejs//nodejs:extensions.bzl", "node") node.toolchain( name = "nodejs", diff --git a/misc/bazel/3rdparty/tree_sitter_extractors_deps/BUILD.bazel b/misc/bazel/3rdparty/tree_sitter_extractors_deps/BUILD.bazel index bb32aa97a5e2..ce1d79a772b9 100644 --- a/misc/bazel/3rdparty/tree_sitter_extractors_deps/BUILD.bazel +++ b/misc/bazel/3rdparty/tree_sitter_extractors_deps/BUILD.bazel @@ -553,6 +553,18 @@ alias( tags = ["manual"], ) +alias( + name = "swift-syntax-rs-0.1.0", + actual = "@vendor_ts__swift-syntax-rs-0.1.0//:swift_syntax_rs", + tags = ["manual"], +) + +alias( + name = "swift-syntax-rs", + actual = "@vendor_ts__swift-syntax-rs-0.1.0//:swift_syntax_rs", + tags = ["manual"], +) + alias( name = "syn-2.0.106", actual = "@vendor_ts__syn-2.0.106//:syn", diff --git a/misc/bazel/3rdparty/tree_sitter_extractors_deps/defs.bzl b/misc/bazel/3rdparty/tree_sitter_extractors_deps/defs.bzl index 7fbdfc4bbd4b..f3b5edb59cd6 100644 --- a/misc/bazel/3rdparty/tree_sitter_extractors_deps/defs.bzl +++ b/misc/bazel/3rdparty/tree_sitter_extractors_deps/defs.bzl @@ -429,6 +429,10 @@ _NORMAL_DEPENDENCIES = { "tree-sitter-language": Label("@vendor_ts__tree-sitter-language-0.1.5//:tree_sitter_language"), }, }, + "unified/swift-syntax-rs": { + _COMMON_CONDITION: { + }, + }, } _NORMAL_ALIASES = { @@ -475,6 +479,10 @@ _NORMAL_ALIASES = { _COMMON_CONDITION: { }, }, + "unified/swift-syntax-rs": { + _COMMON_CONDITION: { + }, + }, } _NORMAL_DEV_DEPENDENCIES = { @@ -505,6 +513,8 @@ _NORMAL_DEV_DEPENDENCIES = { }, "unified/extractor/tree-sitter-swift": { }, + "unified/swift-syntax-rs": { + }, } _NORMAL_DEV_ALIASES = { @@ -532,6 +542,8 @@ _NORMAL_DEV_ALIASES = { }, "unified/extractor/tree-sitter-swift": { }, + "unified/swift-syntax-rs": { + }, } _PROC_MACRO_DEPENDENCIES = { @@ -557,6 +569,8 @@ _PROC_MACRO_DEPENDENCIES = { }, "unified/extractor/tree-sitter-swift": { }, + "unified/swift-syntax-rs": { + }, } _PROC_MACRO_ALIASES = { @@ -582,6 +596,8 @@ _PROC_MACRO_ALIASES = { }, "unified/extractor/tree-sitter-swift": { }, + "unified/swift-syntax-rs": { + }, } _PROC_MACRO_DEV_DEPENDENCIES = { @@ -607,6 +623,8 @@ _PROC_MACRO_DEV_DEPENDENCIES = { }, "unified/extractor/tree-sitter-swift": { }, + "unified/swift-syntax-rs": { + }, } _PROC_MACRO_DEV_ALIASES = { @@ -634,6 +652,8 @@ _PROC_MACRO_DEV_ALIASES = { }, "unified/extractor/tree-sitter-swift": { }, + "unified/swift-syntax-rs": { + }, } _BUILD_DEPENDENCIES = { @@ -663,6 +683,8 @@ _BUILD_DEPENDENCIES = { "tree-sitter-generate": Label("@vendor_ts__tree-sitter-generate-0.26.8//:tree_sitter_generate"), }, }, + "unified/swift-syntax-rs": { + }, } _BUILD_ALIASES = { @@ -690,6 +712,8 @@ _BUILD_ALIASES = { _COMMON_CONDITION: { }, }, + "unified/swift-syntax-rs": { + }, } _BUILD_PROC_MACRO_DEPENDENCIES = { @@ -715,6 +739,8 @@ _BUILD_PROC_MACRO_DEPENDENCIES = { }, "unified/extractor/tree-sitter-swift": { }, + "unified/swift-syntax-rs": { + }, } _BUILD_PROC_MACRO_ALIASES = { @@ -740,6 +766,8 @@ _BUILD_PROC_MACRO_ALIASES = { }, "unified/extractor/tree-sitter-swift": { }, + "unified/swift-syntax-rs": { + }, } _CONDITIONS = { diff --git a/shared/tree-sitter-extractor/src/extractor/mod.rs b/shared/tree-sitter-extractor/src/extractor/mod.rs index 54b01ba5146e..474dc096a2a6 100644 --- a/shared/tree-sitter-extractor/src/extractor/mod.rs +++ b/shared/tree-sitter-extractor/src/extractor/mod.rs @@ -81,10 +81,18 @@ impl AstNode for yeast::Node { yeast::Node::is_extra(self) } fn start_position(&self) -> tree_sitter::Point { - yeast::Node::start_position(self) + let p = yeast::Node::start_position(self); + tree_sitter::Point { + row: p.row, + column: p.column, + } } fn end_position(&self) -> tree_sitter::Point { - yeast::Node::end_position(self) + let p = yeast::Node::end_position(self); + tree_sitter::Point { + row: p.row, + column: p.column, + } } fn byte_range(&self) -> std::ops::Range { yeast::Node::byte_range(self) diff --git a/shared/yeast-macros/src/parse.rs b/shared/yeast-macros/src/parse.rs index 01c0b574b1cf..55ada3588491 100644 --- a/shared/yeast-macros/src/parse.rs +++ b/shared/yeast-macros/src/parse.rs @@ -903,7 +903,7 @@ pub fn parse_rule_top(input: TokenStream) -> Result { Ok(quote! { { let __query = #query_code; - yeast::Rule::new(__query, Box::new(|__ast: &mut yeast::Ast, mut __captures: yeast::captures::Captures, __fresh: &yeast::tree_builder::FreshScope, __source_range: Option, __user_ctx: &mut _, __translator: yeast::TranslatorHandle<'_, _>| { + yeast::Rule::new(__query, Box::new(|__ast: &mut yeast::Ast, mut __captures: yeast::captures::Captures, __fresh: &yeast::tree_builder::FreshScope, __source_range: Option, __user_ctx: &mut _, __translator: yeast::TranslatorHandle<'_, _>| { // Auto-translation prefix: recursively translate every // captured node before invoking the user's transform body, // except for `@@name` captures listed in `__skip` which the diff --git a/shared/yeast-schema/src/schema.rs b/shared/yeast-schema/src/schema.rs index 4acd14377a4d..9c438d7c22e8 100644 --- a/shared/yeast-schema/src/schema.rs +++ b/shared/yeast-schema/src/schema.rs @@ -167,6 +167,28 @@ impl Schema { id } + /// Register every kind (named and unnamed) and field *name* from `other` + /// into this schema (idempotent). Ids are assigned in this schema's own id + /// space; existing ids are unchanged. + /// + /// This is used when running desugaring rules over an AST that was built + /// against a different schema (e.g. from an external parser): the rules + /// build output nodes whose kind/field names come from `other`, and those + /// names must resolve in the AST's own schema. Only names are needed — the + /// rule engine resolves kinds/fields by name and does not consult + /// `other`'s field-type or supertype information. + pub fn register_names_from(&mut self, other: &Schema) { + for name in other.kind_ids.keys() { + self.register_kind(name); + } + for name in other.unnamed_kind_ids.keys() { + self.register_unnamed_kind(name); + } + for name in other.field_ids.keys() { + self.register_field(name); + } + } + /// Track a name for a kind ID without registering it as named or /// unnamed. Useful when importing tree-sitter ID tables that may /// contain duplicate IDs across the named/unnamed split. diff --git a/shared/yeast/src/build.rs b/shared/yeast/src/build.rs index 68a4c883242a..508a1e731346 100644 --- a/shared/yeast/src/build.rs +++ b/shared/yeast/src/build.rs @@ -2,7 +2,7 @@ use std::collections::BTreeMap; use crate::captures::Captures; use crate::tree_builder::FreshScope; -use crate::{Ast, FieldId, Id, NodeContent, TranslatorHandle}; +use crate::{Ast, FieldId, Id, NodeContent, Range, TranslatorHandle}; /// Context for building new AST nodes during a transformation. /// @@ -34,7 +34,7 @@ pub struct BuildCtx<'a, C: 'a = ()> { pub captures: &'a Captures, pub fresh: &'a FreshScope, /// Source range of the matched node, inherited by synthetic nodes. - pub source_range: Option, + pub source_range: Option, /// User-supplied context, accessible directly via `ctx.field` (via Deref). pub user_ctx: &'a mut C, /// Optional translator handle, populated when the context is built by @@ -63,7 +63,7 @@ impl<'a, C> BuildCtx<'a, C> { ast: &'a mut Ast, captures: &'a Captures, fresh: &'a FreshScope, - source_range: Option, + source_range: Option, user_ctx: &'a mut C, ) -> Self { Self { @@ -82,7 +82,7 @@ impl<'a, C> BuildCtx<'a, C> { ast: &'a mut Ast, captures: &'a Captures, fresh: &'a FreshScope, - source_range: Option, + source_range: Option, user_ctx: &'a mut C, translator: TranslatorHandle<'a, C>, ) -> Self { @@ -143,7 +143,7 @@ impl<'a, C> BuildCtx<'a, C> { &mut self, kind: &'static str, value: &str, - source_range: Option, + source_range: Option, ) -> Id { self.ast.create_named_token_with_range( kind, diff --git a/shared/yeast/src/lib.rs b/shared/yeast/src/lib.rs index e6759f1ddb92..6719a5498e2b 100644 --- a/shared/yeast/src/lib.rs +++ b/shared/yeast/src/lib.rs @@ -15,11 +15,32 @@ pub mod schema; pub mod tree_builder; mod visitor; +pub use range::{Point, Range}; pub use yeast_macros::{query, rule, rules, tree, trees}; use captures::Captures; use query::QueryNode; +impl From for Point { + fn from(p: tree_sitter::Point) -> Self { + Point { + row: p.row, + column: p.column, + } + } +} + +impl From for Range { + fn from(r: tree_sitter::Range) -> Self { + Range { + start_byte: r.start_byte, + end_byte: r.end_byte, + start_point: r.start_point.into(), + end_point: r.end_point.into(), + } + } +} + /// Node id: an index into the [`Ast`] arena. A newtype around `usize` /// rather than a bare alias so that it can carry its own /// [`YeastDisplay`] / [`YeastSourceRange`] / [`IntoFieldIds`] impls @@ -106,7 +127,7 @@ pub trait YeastDisplay { /// rule's source range. `Id` returns the referenced node's range, letting /// `(kind #{capture})` carry the captured node's location. pub trait YeastSourceRange { - fn yeast_source_range(&self, ast: &Ast) -> Option; + fn yeast_source_range(&self, ast: &Ast) -> Option; } impl YeastDisplay for Id { @@ -116,9 +137,9 @@ impl YeastDisplay for Id { } impl YeastSourceRange for Id { - fn yeast_source_range(&self, ast: &Ast) -> Option { + fn yeast_source_range(&self, ast: &Ast) -> Option { ast.get_node(*self).and_then(|n| match &n.content { - NodeContent::Range(r) => Some(r.clone()), + NodeContent::Range(r) => Some(*r), _ => n.source_range, }) } @@ -134,7 +155,7 @@ macro_rules! impl_yeast_display_via_display { } impl YeastSourceRange for $t { - fn yeast_source_range(&self, _ast: &Ast) -> Option { + fn yeast_source_range(&self, _ast: &Ast) -> Option { None } } @@ -157,7 +178,7 @@ impl YeastDisplay for &T { } impl YeastSourceRange for &T { - fn yeast_source_range(&self, ast: &Ast) -> Option { + fn yeast_source_range(&self, ast: &Ast) -> Option { (**self).yeast_source_range(ast) } } @@ -333,13 +354,41 @@ impl Ast { ast } + /// Construct an empty AST backed by `schema`, for building a tree + /// programmatically from a source other than a tree-sitter parse (e.g. an + /// external parser's output). Populate it with [`Ast::create_node`] / + /// [`Ast::create_node_with_range`] and then designate the root with + /// [`Ast::set_root`]. + /// + /// Node kind and field names may be registered in `schema` up front, or on + /// demand while building via [`Ast::register_kind`], + /// [`Ast::register_unnamed_kind`], and [`Ast::register_field`] (which return + /// the ids to pass to [`Ast::create_node_with_range`]). Passing a fresh + /// [`schema::Schema::new`] and registering names during construction is + /// therefore fine — see the swift-syntax adapter for an example. + pub fn with_schema(schema: schema::Schema) -> Self { + Self { + root: Id(0), + nodes: Vec::new(), + schema, + source: Vec::new(), + } + } + + /// Set the original source bytes, used to resolve `NodeContent::Range` + /// nodes to text. Nodes built with inline `NodeContent::DynamicString` + /// content do not require this. + pub fn set_source(&mut self, source: Vec) { + self.source = source; + } + /// Returns the source text for `id`, resolving `NodeContent::Range` /// against the stored source bytes when available. pub fn source_text(&self, id: Id) -> String { let Some(node) = self.get_node(id) else { return String::new(); }; - let read_range = |range: &tree_sitter::Range| { + let read_range = |range: &Range| { let start = range.start_byte; let end = range.end_byte; if end <= self.source.len() && start <= end { @@ -437,7 +486,7 @@ impl Ast { content: NodeContent, fields: BTreeMap>, is_named: bool, - source_range: Option, + source_range: Option, ) -> Id { let source_range = match &content { // Parsed nodes already carry an exact source range in their content. @@ -463,14 +512,37 @@ impl Ast { Id(id) } - fn union_source_range_of_children( - &self, - fields: &BTreeMap>, - ) -> Option { + /// Register a named node kind, returning its id (idempotent). Lets callers + /// build an AST in a single pass, registering kinds as nodes are created + /// rather than pre-populating the schema. + pub fn register_kind(&mut self, name: &str) -> KindId { + self.schema.register_kind(name) + } + + /// Register an anonymous (unnamed) token kind, returning its id + /// (idempotent). Anonymous tokens are keyed by their text (e.g. `"func"`). + pub fn register_unnamed_kind(&mut self, name: &str) -> KindId { + self.schema.register_unnamed_kind(name) + } + + /// Register a field name, returning its id (idempotent). + pub fn register_field(&mut self, name: &str) -> FieldId { + self.schema.register_field(name) + } + + /// Register every kind and field name from `schema` into this AST's schema + /// (idempotent). Used before desugaring an externally-built AST so that + /// rules can build output nodes whose kind/field names come from the + /// desugarer's output schema. + pub fn register_names_from_schema(&mut self, schema: &schema::Schema) { + self.schema.register_names_from(schema); + } + + fn union_source_range_of_children(&self, fields: &BTreeMap>) -> Option { let mut start_byte: Option = None; let mut end_byte: Option = None; - let mut start_point = tree_sitter::Point { row: 0, column: 0 }; - let mut end_point = tree_sitter::Point { row: 0, column: 0 }; + let mut start_point = Point { row: 0, column: 0 }; + let mut end_point = Point { row: 0, column: 0 }; for child_ids in fields.values() { for &child_id in child_ids { @@ -513,7 +585,7 @@ impl Ast { } match (start_byte, end_byte) { - (Some(start_byte), Some(end_byte)) => Some(tree_sitter::Range { + (Some(start_byte), Some(end_byte)) => Some(Range { start_byte, end_byte, start_point, @@ -531,7 +603,7 @@ impl Ast { &mut self, kind: &'static str, content: String, - source_range: Option, + source_range: Option, ) -> Id { let kind_id = self.schema.id_for_node_kind(kind).unwrap_or_else(|| { panic!("create_named_token: node kind '{kind}' not found in schema") @@ -604,7 +676,7 @@ impl Ast { } } - fn id_for_unnamed_node_kind(&self, kind: &str) -> Option { + pub fn id_for_unnamed_node_kind(&self, kind: &str) -> Option { let id = self.schema.id_for_unnamed_node_kind(kind).unwrap_or(0); if id == 0 { None @@ -624,7 +696,7 @@ pub struct Node { /// For synthetic nodes, the source range of the original node they /// were desugared from. Used for location information in TRAP output. #[serde(skip)] - source_range: Option, + source_range: Option, is_named: bool, is_missing: bool, is_extra: bool, @@ -652,11 +724,11 @@ impl Node { self.is_error } - fn fake_point(&self) -> tree_sitter::Point { - tree_sitter::Point { row: 0, column: 0 } + fn fake_point(&self) -> Point { + Point { row: 0, column: 0 } } - pub fn start_position(&self) -> tree_sitter::Point { + pub fn start_position(&self) -> Point { match self.content { NodeContent::Range(range) => range.start_point, _ => self @@ -665,7 +737,7 @@ impl Node { } } - pub fn end_position(&self) -> tree_sitter::Point { + pub fn end_position(&self) -> Point { match self.content { NodeContent::Range(range) => range.end_point, _ => self @@ -714,7 +786,7 @@ impl Node { /// or a new string if the node is synthesized. #[derive(PartialEq, Eq, Debug, Clone, Serialize)] pub enum NodeContent { - Range(#[serde(with = "range::Range")] tree_sitter::Range), + Range(Range), String(&'static str), DynamicString(String), } @@ -727,7 +799,7 @@ impl From<&'static str> for NodeContent { impl From for NodeContent { fn from(value: tree_sitter::Range) -> Self { - NodeContent::Range(value) + NodeContent::Range(value.into()) } } @@ -854,7 +926,7 @@ pub type Transform = Box< &mut Ast, Captures, &tree_builder::FreshScope, - Option, + Option, &mut C, TranslatorHandle<'_, C>, ) -> Result, String> @@ -1387,6 +1459,18 @@ impl<'a, C: Clone + Default> Runner<'a, C> { let mut user_ctx = C::default(); self.run_with_ctx(input, &mut user_ctx) } + + /// Run all phases over an already-built `ast`, using the default context + /// (`C::default()`). Unlike [`run_from_tree`](Self::run_from_tree), the AST + /// is supplied by the caller (e.g. built from an external parser's output) + /// rather than constructed from a tree-sitter tree. The caller is + /// responsible for ensuring the AST's schema knows any output kind/field + /// names the rules will build (see [`Ast::register_names_from_schema`]). + pub fn run_from_ast(&self, mut ast: Ast) -> Result { + let mut user_ctx = C::default(); + self.run_phases(&mut ast, &mut user_ctx)?; + Ok(ast) + } } // --------------------------------------------------------------------------- @@ -1409,6 +1493,12 @@ pub trait Desugarer: Send + Sync { /// Parse `tree` against `source` and run the desugaring pipeline. /// Each call constructs a fresh default user context internally. fn run_from_tree(&self, tree: &tree_sitter::Tree, source: &[u8]) -> Result; + + /// Run the desugaring pipeline over an already-built `ast` (e.g. produced + /// by an external parser rather than tree-sitter). The desugarer ensures + /// the AST's schema knows its output kind/field names before running the + /// rules. Each call constructs a fresh default user context internally. + fn run_from_ast(&self, ast: Ast) -> Result; } /// A concrete [`Desugarer`] backed by a [`DesugaringConfig`] for a @@ -1446,4 +1536,12 @@ impl Desugarer for ConcreteDesugarer let runner = Runner::with_schema(self.language.clone(), &self.schema, &self.config.phases); runner.run_from_tree(tree, source) } + + fn run_from_ast(&self, mut ast: Ast) -> Result { + // The AST was built against its own (external) schema; make sure the + // output kind/field names the rules build are resolvable in it. + ast.register_names_from_schema(&self.schema); + let runner = Runner::with_schema(self.language.clone(), &self.schema, &self.config.phases); + runner.run_from_ast(ast) + } } diff --git a/shared/yeast/src/range.rs b/shared/yeast/src/range.rs index ec670b438d56..e56bbcc6fb85 100644 --- a/shared/yeast/src/range.rs +++ b/shared/yeast/src/range.rs @@ -1,21 +1,32 @@ -//! (de)-serialize helpers for tree_sitter::Range +//! Location types for yeast ASTs. +//! +//! These are plain owned structs (mirroring tree-sitter's `Point`/`Range`) so +//! that code building an AST programmatically can populate node locations +//! without depending on tree-sitter. Conversions from the tree-sitter types +//! live in the crate root, next to the `from_tree` construction path. use serde::{Deserialize, Serialize}; -#[derive(Serialize, Deserialize)] -#[serde(remote = "tree_sitter::Point")] +/// A position in a source file: a 0-based `row` (line) and 0-based `column` +/// (UTF-8 byte offset within the line), matching tree-sitter's convention. +#[derive(Serialize, Deserialize, Clone, Copy, Debug, Default, PartialEq, Eq, Hash)] pub struct Point { pub row: usize, pub column: usize, } -#[derive(Serialize, Deserialize)] -#[serde(remote = "tree_sitter::Range")] +impl Point { + pub fn new(row: usize, column: usize) -> Self { + Self { row, column } + } +} + +/// A half-open source range: byte offsets plus start/end [`Point`]s. The end is +/// exclusive, matching tree-sitter's convention. +#[derive(Serialize, Deserialize, Clone, Copy, Debug, Default, PartialEq, Eq, Hash)] pub struct Range { pub start_byte: usize, pub end_byte: usize, - #[serde(with = "Point")] - pub start_point: tree_sitter::Point, - #[serde(with = "Point")] - pub end_point: tree_sitter::Point, + pub start_point: Point, + pub end_point: Point, } diff --git a/shared/yeast/tests/test.rs b/shared/yeast/tests/test.rs index 3a24709dd9ff..756e219bd890 100644 --- a/shared/yeast/tests/test.rs +++ b/shared/yeast/tests/test.rs @@ -266,6 +266,59 @@ fn test_query_match() { assert!(captures.get_var("right").is_ok()); } +#[test] +fn test_run_from_ast_desugars_hand_built_tree() { + use std::collections::BTreeMap; + + // Output schema for the desugared tree. Its kind/field names must become + // resolvable in the hand-built AST's schema for the rule to build them. + let schema_yaml = r#" +named: + assignment: + left: leaf + leaf: +"#; + + // A rule over an *input* kind (`wrapper`) that is not in the output schema, + // rewriting to an output `assignment` node. + let rules: Vec = vec![yeast::rule!( + (wrapper) + => + (assignment left: (leaf "lit")) + )]; + + let lang: tree_sitter::Language = tree_sitter_ruby::LANGUAGE.into(); + let config = DesugaringConfig::<()>::new() + .add_phase("test", PhaseKind::OneShot, rules) + .with_output_node_types_yaml(schema_yaml); + let desugarer = ConcreteDesugarer::new(lang, config).unwrap(); + + // Build the input AST by hand, as an external parser adapter would. The + // schema starts empty and gains the `wrapper` input kind on the fly. + let mut ast = Ast::with_schema(yeast::schema::Schema::new()); + let wrapper_kind = ast.register_kind("wrapper"); + let root = ast.create_node_with_range( + wrapper_kind, + NodeContent::DynamicString(String::new()), + BTreeMap::new(), + true, + None, + ); + ast.set_root(root); + + // Desugaring the hand-built AST applies the rule, producing `assignment` + // even though the AST was built against a schema with no output kinds. + let out = desugarer + .run_from_ast(ast) + .expect("run_from_ast should succeed"); + let out_root = out.get_node(out.get_root()).expect("root exists"); + assert_eq!(out_root.kind_name(), "assignment"); + let dump = dump_ast(&out, out.get_root(), ""); + assert!(dump.contains("assignment"), "unexpected dump: {dump}"); + assert!(dump.contains("left"), "unexpected dump: {dump}"); + assert!(dump.contains("leaf"), "unexpected dump: {dump}"); +} + #[test] fn test_query_no_match() { let runner: Runner = Runner::new(tree_sitter_ruby::LANGUAGE.into(), &[]); diff --git a/unified/extractor/src/languages/mod.rs b/unified/extractor/src/languages/mod.rs index 20ad599edfb6..52a1bd40ffc7 100644 --- a/unified/extractor/src/languages/mod.rs +++ b/unified/extractor/src/languages/mod.rs @@ -3,6 +3,15 @@ use codeql_extractor::extractor::simple; #[path = "swift/swift.rs"] mod swift; +/// swift-syntax JSON -> `yeast::Ast` adapter for the Swift front-end. +/// +/// Currently exercised by tests and the forthcoming runtime extraction path; +/// `allow(dead_code)` because this is a binary crate, so its public API isn't +/// counted as used until the binary itself calls it. +#[path = "swift/adapter.rs"] +#[allow(dead_code)] +pub mod swift_adapter; + /// Shared YEAST output AST schema for all languages. pub(crate) const OUTPUT_AST_SCHEMA: &str = include_str!("../../ast_types.yml"); diff --git a/unified/extractor/src/languages/swift/adapter.rs b/unified/extractor/src/languages/swift/adapter.rs new file mode 100644 index 000000000000..37d5aac00d2d --- /dev/null +++ b/unified/extractor/src/languages/swift/adapter.rs @@ -0,0 +1,415 @@ +//! Converts the swift-syntax JSON syntax tree into a [`yeast::Ast`], the +//! in-memory format the CodeQL desugaring rules operate on. +//! +//! The JSON tree is produced by the `swift-syntax-rs` crate's Swift FFI shim +//! (`parse_to_json`). This module is pure Rust (only `yeast` + `serde_json`), +//! so the extractor consumes swift-syntax output without pulling in the Swift +//! toolchain (the JSON is produced out-of-process). +//! +//! The mapping mirrors tree-sitter's node model, which is what yeast (and the +//! extractor's rewrite rules) expect: +//! +//! * **Layout nodes** (e.g. `functionDecl`) and **varying tokens** (identifiers, +//! literals, operators — the ones whose text is not determined by their kind) +//! become **named** nodes, keyed by their kind name. +//! * **Fixed tokens** (keywords and punctuation, whose text is fully determined +//! by their kind) become **anonymous** nodes, keyed by their text — exactly +//! how tree-sitter models anonymous tokens (e.g. `"func"`, `"->"`). +//! * Collection nodes are already elided to JSON arrays upstream, so a +//! list-valued field maps directly to that field holding several children. +//! +//! Note: this preserves swift-syntax's own kind/field names. Aligning those +//! names with the tree-sitter-swift schema (so the rewrite rules in +//! [`super::swift`] fire) is done incrementally in the rules. + +use std::collections::BTreeMap; + +use serde_json::Value; +use yeast::schema::Schema; +use yeast::{Ast, Id, NodeContent, Point, Range}; + +/// A comment (or `unexpectedText`) recovered from the syntax tree's trivia. +/// +/// These are collected into a side channel rather than embedded in the +/// [`yeast::Ast`], mirroring how the extractor treats tree-sitter `extra` +/// nodes: they carry a location and text but are not attached to a parent. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct TriviaToken { + /// The trivia kind (e.g. `lineComment`, `blockComment`, `docLineComment`, + /// `docBlockComment`, `unexpectedText`). + pub kind: String, + /// The verbatim source text of the piece (e.g. `// comment`). + pub text: String, + /// The source range the piece occupies. + pub range: Range, +} + +/// The result of adapting a swift-syntax JSON tree: the [`yeast::Ast`] plus the +/// comment/`unexpectedText` trivia harvested from it (in source order). +pub struct AdaptedTree { + pub ast: Ast, + pub trivia: Vec, +} + +/// swift-syntax `TokenKind` cases whose text is *not* determined by the kind +/// (i.e. `TokenKind.defaultText == nil`). These carry varying information and +/// are modelled as named leaf nodes; every other token is a fixed +/// keyword/punctuation token modelled as an anonymous token keyed by its text. +const VARYING_TOKEN_KINDS: &[&str] = &[ + "identifier", + "integerLiteral", + "floatLiteral", + "stringSegment", + "binaryOperator", + "prefixOperator", + "postfixOperator", + "dollarIdentifier", + "regexLiteralPattern", + "rawStringPoundDelimiter", + "regexPoundDelimiter", + "shebang", + "unknown", +]; + +/// Keys of a node object that carry metadata rather than a structural child. +fn is_metadata_key(key: &str) -> bool { + matches!( + key, + "kind" | "range" | "tokenKind" | "text" | "leadingTrivia" | "trailingTrivia" + ) +} + +/// The classification of a JSON node into a yeast kind name and named-ness. +struct KindInfo { + /// The name under which the kind is registered in the schema. + name: String, + /// `true` for named nodes (layout nodes + varying tokens), `false` for + /// anonymous tokens (fixed keywords/punctuation). + is_named: bool, + /// The leaf text for tokens (empty for layout nodes). + text: String, +} + +/// Determine the kind name / named-ness / text for a JSON node object. +fn classify(node: &Value) -> Result { + let kind = node + .get("kind") + .and_then(Value::as_str) + .ok_or("node object is missing a string `kind`")?; + + if kind != "token" { + // Layout node: named, keyed by its kind, no leaf text. + return Ok(KindInfo { + name: kind.to_string(), + is_named: true, + text: String::new(), + }); + } + + let token_kind = node + .get("tokenKind") + .and_then(Value::as_str) + .ok_or("token is missing a string `tokenKind`")?; + let text = node + .get("text") + .and_then(Value::as_str) + .unwrap_or("") + .to_string(); + + // The case name is the part before any `(payload)` in the debug rendering. + let case_name = token_kind.split('(').next().unwrap_or(token_kind); + + if VARYING_TOKEN_KINDS.contains(&case_name) { + // Varying token: named leaf, keyed by its case name. + Ok(KindInfo { + name: case_name.to_string(), + is_named: true, + text, + }) + } else { + // Fixed token: anonymous, keyed by its text (as tree-sitter does). + // `endOfFile` has empty text, so fall back to the case name there. + let name = if text.is_empty() { + case_name.to_string() + } else { + text.clone() + }; + Ok(KindInfo { + name, + is_named: false, + text, + }) + } +} + +/// Iterate over a node object's structural (field, value) pairs in a stable +/// order, skipping metadata keys. +fn field_entries(node: &Value) -> Vec<(&str, &Value)> { + node.as_object() + .map(|map| { + map.iter() + .filter(|(k, _)| !is_metadata_key(k)) + .map(|(k, v)| (k.as_str(), v)) + .collect() + }) + .unwrap_or_default() +} + +/// The child node objects held by a field value, which is either a single node +/// object or an array of them (an elided collection). +fn children_of(value: &Value) -> Vec<&Value> { + match value { + Value::Array(items) => items.iter().collect(), + other => vec![other], + } +} + +/// Recursively build `node` (and its descendants) into `ast`, returning its id. +/// +/// This is a single traversal: each node's kind and field names are registered +/// in the schema on the fly, immediately before the node is created. Children +/// are built first so a parent's field lists reference existing ids. Any +/// comment/`unexpectedText` trivia carried by a token is harvested into +/// `trivia` during the same pass rather than embedded in the tree. +fn build(node: &Value, ast: &mut Ast, trivia: &mut Vec) -> Result { + let info = classify(node)?; + collect_trivia(node, trivia); + + let mut fields: BTreeMap> = BTreeMap::new(); + for (field, value) in field_entries(node) { + let field_id = ast.register_field(field); + let mut ids = Vec::new(); + for child in children_of(value) { + ids.push(build(child, ast, trivia)?); + } + fields.insert(field_id, ids); + } + + let kind_id = if info.is_named { + ast.register_kind(&info.name) + } else { + ast.register_unnamed_kind(&info.name) + }; + + Ok(ast.create_node_with_range( + kind_id, + NodeContent::DynamicString(info.text), + fields, + info.is_named, + parse_range(node), + )) +} + +/// Harvest a token's `leadingTrivia`/`trailingTrivia` pieces (each already +/// filtered to comments/`unexpectedText` upstream) into `out`. Non-token nodes +/// have no trivia keys, so this is a no-op for them. +fn collect_trivia(node: &Value, out: &mut Vec) { + for key in ["leadingTrivia", "trailingTrivia"] { + let Some(Value::Array(pieces)) = node.get(key) else { + continue; + }; + for piece in pieces { + let (Some(kind), Some(range)) = ( + piece.get("kind").and_then(Value::as_str), + parse_range(piece), + ) else { + continue; + }; + let text = piece + .get("text") + .and_then(Value::as_str) + .unwrap_or("") + .to_string(); + out.push(TriviaToken { + kind: kind.to_string(), + text, + range, + }); + } + } +} + +/// Parse a node's `range` into a [`yeast::Range`]. +/// +/// The JSON carries, for `start` and `end`, a 0-based UTF-8 file byte `offset`, +/// a 1-based `line`, and a 1-based UTF-8 byte `column`. yeast (like tree-sitter) +/// uses byte offsets with 0-based rows/columns and an exclusive end, so the +/// line/column are shifted down by one. swift-syntax's end position is already +/// exclusive, so the byte offsets map across directly. +fn parse_range(node: &Value) -> Option { + let range = node.get("range")?; + let point = |key: &str| -> Option<(usize, Point)> { + let p = range.get(key)?; + let offset = p.get("offset")?.as_u64()? as usize; + let line = p.get("line")?.as_u64()? as usize; + let column = p.get("column")?.as_u64()? as usize; + Some(( + offset, + Point::new(line.saturating_sub(1), column.saturating_sub(1)), + )) + }; + let (start_byte, start_point) = point("start")?; + let (end_byte, end_point) = point("end")?; + Some(Range { + start_byte, + end_byte, + start_point, + end_point, + }) +} + +/// Convert a swift-syntax JSON tree (as produced by [`crate::parse_to_json`]) +/// into a [`yeast::Ast`] plus the comment/`unexpectedText` trivia harvested +/// from it. Both are produced in a single traversal. +pub fn json_to_ast(json: &str) -> Result { + let root: Value = serde_json::from_str(json).map_err(|e| format!("invalid JSON: {e}"))?; + + let mut ast = Ast::with_schema(Schema::new()); + let mut trivia = Vec::new(); + let root_id = build(&root, &mut ast, &mut trivia)?; + ast.set_root(root_id); + + // Emit trivia in source order (the traversal visits nodes bottom-up). + trivia.sort_by_key(|t| t.range.start_byte); + + Ok(AdaptedTree { ast, trivia }) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// A hand-written JSON tree exercising layout nodes, a named (varying) + /// token, a fixed keyword token, and an elided collection field — so the + /// adapter is tested without needing the Swift toolchain. + fn sample_json() -> &'static str { + r#"{ + "kind": "sourceFile", + "range": {"start":{"offset":0,"line":1,"column":1},"end":{"offset":9,"line":1,"column":10}}, + "statements": [ + { + "kind": "variableDecl", + "range": {"start":{"offset":0,"line":1,"column":1},"end":{"offset":9,"line":1,"column":10}}, + "bindingSpecifier": { + "kind": "token", + "tokenKind": "keyword(SwiftSyntax.Keyword.let)", + "text": "let", + "range": {"start":{"offset":0,"line":1,"column":1},"end":{"offset":3,"line":1,"column":4}} + }, + "name": { + "kind": "token", + "tokenKind": "identifier(\"x\")", + "text": "x", + "range": {"start":{"offset":4,"line":1,"column":5},"end":{"offset":5,"line":1,"column":6}} + } + } + ] + }"# + } + + #[test] + fn builds_ast_from_json() { + let ast = json_to_ast(sample_json()) + .expect("adapter should succeed") + .ast; + let root = ast.get_root(); + let root_node = ast.get_node(root).expect("root exists"); + assert_eq!(root_node.kind_name(), "sourceFile"); + assert!(root_node.is_named()); + } + + #[test] + fn classifies_named_and_anonymous_tokens() { + let ast = json_to_ast(sample_json()) + .expect("adapter should succeed") + .ast; + // Walk all nodes and collect (kind_name, is_named) for the two tokens. + let mut let_named = None; + let mut ident_named = None; + for node in ast.nodes() { + match node.kind_name() { + "let" => let_named = Some(node.is_named()), + "identifier" => ident_named = Some(node.is_named()), + _ => {} + } + } + // Fixed keyword `let` is anonymous (keyed by its text "let"). + assert_eq!(let_named, Some(false)); + // Varying `identifier` is a named leaf. + assert_eq!(ident_named, Some(true)); + } + + #[test] + fn preserves_leaf_text() { + let ast = json_to_ast(sample_json()) + .expect("adapter should succeed") + .ast; + let ident = ast + .nodes() + .iter() + .enumerate() + .find(|(_, n)| n.kind_name() == "identifier") + .map(|(i, _)| Id(i)) + .expect("identifier node exists"); + assert_eq!(ast.source_text(ident), "x"); + } + + #[test] + fn maps_source_locations() { + let ast = json_to_ast(sample_json()) + .expect("adapter should succeed") + .ast; + let ident = ast + .nodes() + .iter() + .find(|n| n.kind_name() == "identifier") + .expect("identifier node exists"); + // `x` is at file offset 4..5, line 1, column 5 (1-based) in the JSON, + // which maps to 0-based row 0, column 4 and byte range 4..5. + assert_eq!(ident.start_byte(), 4); + assert_eq!(ident.end_byte(), 5); + assert_eq!(ident.start_position(), Point::new(0, 4)); + assert_eq!(ident.end_position(), Point::new(0, 5)); + } + + #[test] + fn collects_trivia_into_side_channel() { + // A token carrying a trailing line comment in its trivia. + let json = r#"{ + "kind": "sourceFile", + "range": {"start":{"offset":0,"line":1,"column":1},"end":{"offset":14,"line":1,"column":15}}, + "value": { + "kind": "token", + "tokenKind": "integerLiteral(\"1\")", + "text": "1", + "range": {"start":{"offset":0,"line":1,"column":1},"end":{"offset":1,"line":1,"column":2}}, + "trailingTrivia": [ + { + "kind": "lineComment", + "text": "// c", + "range": {"start":{"offset":2,"line":1,"column":3},"end":{"offset":6,"line":1,"column":7}} + } + ] + } + }"#; + let adapted = json_to_ast(json).expect("adapter should succeed"); + + // The comment is in the side channel, with its text and location. + assert_eq!(adapted.trivia.len(), 1); + let comment = &adapted.trivia[0]; + assert_eq!(comment.kind, "lineComment"); + assert_eq!(comment.text, "// c"); + assert_eq!(comment.range.start_byte, 2); + assert_eq!(comment.range.end_byte, 6); + + // It is not embedded in the AST as a node. + assert!( + adapted + .ast + .nodes() + .iter() + .all(|n| n.kind_name() != "lineComment"), + "comment should not appear as an AST node" + ); + } +} diff --git a/unified/extractor/src/languages/swift/swift.rs b/unified/extractor/src/languages/swift/swift.rs index 74809ea18e41..13f1a6beadf0 100644 --- a/unified/extractor/src/languages/swift/swift.rs +++ b/unified/extractor/src/languages/swift/swift.rs @@ -135,6 +135,25 @@ fn translation_rules() -> Vec> { // Declarations may be wrapped in local/global wrapper nodes. rule!((global_declaration _ @inner) => stmt { inner }), rule!((local_declaration _ @inner) => stmt { inner }), + // ---- swift-syntax front-end (minimal hook-up) ---- + // These rules target the swift-syntax AST (camelCase kind names), + // produced by the sibling `adapter` module. They coexist with the + // tree-sitter rules (snake_case names): rules are dispatched by exact + // kind name, and the two name spaces never collide, so these are inert + // on the tree-sitter path. Only the minimal top-level mapping lives here + // to demonstrate the pipeline end-to-end; the full translation is added + // separately. Unmatched swift-syntax nodes fall through to the + // `unsupported_node` fallback at the end. + // + // `sourceFile` holds its top-level statements in an (elided) + // `statements` collection; each element is a `codeBlockItem` wrapping + // the real node. + rule!( + (sourceFile statements: _* @items) + => + (top_level body: (block stmt: {items})) + ), + rule!((codeBlockItem item: @item) => stmt { item }), // ---- Literals ---- rule!((integer_literal) => (int_literal)), rule!((hex_literal) => (int_literal)), diff --git a/unified/extractor/tests/fixtures/let_x.swiftsyntax.json b/unified/extractor/tests/fixtures/let_x.swiftsyntax.json new file mode 100644 index 000000000000..6c3d18e2fef0 --- /dev/null +++ b/unified/extractor/tests/fixtures/let_x.swiftsyntax.json @@ -0,0 +1,196 @@ +{ + "endOfFileToken": { + "kind": "token", + "range": { + "end": { + "column": 1, + "line": 2, + "offset": 10 + }, + "start": { + "column": 1, + "line": 2, + "offset": 10 + } + }, + "text": "", + "tokenKind": "endOfFile" + }, + "kind": "sourceFile", + "range": { + "end": { + "column": 1, + "line": 2, + "offset": 10 + }, + "start": { + "column": 1, + "line": 1, + "offset": 0 + } + }, + "statements": [ + { + "item": { + "attributes": [], + "bindings": [ + { + "initializer": { + "equal": { + "kind": "token", + "range": { + "end": { + "column": 8, + "line": 1, + "offset": 7 + }, + "start": { + "column": 7, + "line": 1, + "offset": 6 + } + }, + "text": "=", + "tokenKind": "equal" + }, + "kind": "initializerClause", + "range": { + "end": { + "column": 10, + "line": 1, + "offset": 9 + }, + "start": { + "column": 7, + "line": 1, + "offset": 6 + } + }, + "value": { + "kind": "integerLiteralExpr", + "literal": { + "kind": "token", + "range": { + "end": { + "column": 10, + "line": 1, + "offset": 9 + }, + "start": { + "column": 9, + "line": 1, + "offset": 8 + } + }, + "text": "1", + "tokenKind": "integerLiteral(\"1\")" + }, + "range": { + "end": { + "column": 10, + "line": 1, + "offset": 9 + }, + "start": { + "column": 9, + "line": 1, + "offset": 8 + } + } + } + }, + "kind": "patternBinding", + "pattern": { + "identifier": { + "kind": "token", + "range": { + "end": { + "column": 6, + "line": 1, + "offset": 5 + }, + "start": { + "column": 5, + "line": 1, + "offset": 4 + } + }, + "text": "x", + "tokenKind": "identifier(\"x\")" + }, + "kind": "identifierPattern", + "range": { + "end": { + "column": 6, + "line": 1, + "offset": 5 + }, + "start": { + "column": 5, + "line": 1, + "offset": 4 + } + } + }, + "range": { + "end": { + "column": 10, + "line": 1, + "offset": 9 + }, + "start": { + "column": 5, + "line": 1, + "offset": 4 + } + } + } + ], + "bindingSpecifier": { + "kind": "token", + "range": { + "end": { + "column": 4, + "line": 1, + "offset": 3 + }, + "start": { + "column": 1, + "line": 1, + "offset": 0 + } + }, + "text": "let", + "tokenKind": "keyword(SwiftSyntax.Keyword.let)" + }, + "kind": "variableDecl", + "modifiers": [], + "range": { + "end": { + "column": 10, + "line": 1, + "offset": 9 + }, + "start": { + "column": 1, + "line": 1, + "offset": 0 + } + } + }, + "kind": "codeBlockItem", + "range": { + "end": { + "column": 10, + "line": 1, + "offset": 9 + }, + "start": { + "column": 1, + "line": 1, + "offset": 0 + } + } + } + ] +} diff --git a/unified/extractor/tests/swift_syntax_pipeline.rs b/unified/extractor/tests/swift_syntax_pipeline.rs new file mode 100644 index 000000000000..cdaae1f47c6e --- /dev/null +++ b/unified/extractor/tests/swift_syntax_pipeline.rs @@ -0,0 +1,49 @@ +//! Integration test for the swift-syntax front-end pipeline: +//! +//! swift-syntax JSON -> `swift_adapter::json_to_ast` -> yeast `Ast` +//! -> `Desugarer::run_from_ast` (the real Swift translation rules) -> dump. +//! +//! This exercises the whole chain *without* the Swift toolchain: the JSON +//! fixture is a real `parse_to_json` dump (see the file header) fed through the +//! pure-Rust adapter module. It verifies the desugarer runs end-to-end over an +//! externally-built AST. + +use yeast::dump::dump_ast; + +#[path = "../src/languages/mod.rs"] +mod languages; + +/// A real `swift-syntax-rs` JSON dump of the Swift source `let x = 1`. +const LET_X_JSON: &str = include_str!("fixtures/let_x.swiftsyntax.json"); + +#[test] +fn swift_syntax_json_runs_through_the_desugarer() { + let lang = languages::all_language_specs() + .into_iter() + .find(|l| l.file_globs.iter().any(|g| g.contains("swift"))) + .expect("swift language spec"); + let desugarer = lang.desugar.as_deref().expect("swift desugarer"); + + // Adapt the swift-syntax JSON into a yeast AST (pure Rust, no Swift FFI). + let adapted = + languages::swift_adapter::json_to_ast(LET_X_JSON).expect("adapter should succeed"); + assert_eq!( + adapted + .ast + .get_node(adapted.ast.get_root()) + .unwrap() + .kind_name(), + "sourceFile" + ); + + // Run the real Swift desugaring rules over the externally-built AST. The + // top-level swift-syntax rules map `sourceFile` to `top_level`/`block`; + // kinds without swift-syntax rules yet fall back to `unsupported_node`. + let desugared = desugarer + .run_from_ast(adapted.ast) + .expect("desugaring an externally-built AST should not error"); + + let dump = dump_ast(&desugared, desugared.get_root(), ""); + assert!(dump.contains("top_level"), "unexpected dump: {dump}"); + assert!(dump.contains("block"), "unexpected dump: {dump}"); +} diff --git a/unified/swift-syntax-rs/.gitignore b/unified/swift-syntax-rs/.gitignore new file mode 100644 index 000000000000..0b6496865408 --- /dev/null +++ b/unified/swift-syntax-rs/.gitignore @@ -0,0 +1,2 @@ +/target +/swift/.build diff --git a/unified/swift-syntax-rs/.swift-version b/unified/swift-syntax-rs/.swift-version new file mode 100644 index 000000000000..91e4a9f26224 --- /dev/null +++ b/unified/swift-syntax-rs/.swift-version @@ -0,0 +1 @@ +6.3.2 diff --git a/unified/swift-syntax-rs/BUILD.bazel b/unified/swift-syntax-rs/BUILD.bazel new file mode 100644 index 000000000000..02fda50cf502 --- /dev/null +++ b/unified/swift-syntax-rs/BUILD.bazel @@ -0,0 +1,82 @@ +load("@rules_rust//rust:defs.bzl", "rust_binary", "rust_library", "rust_test") +load(":xcode_transition.bzl", "xcode_transition_swift_library") + +package(default_visibility = ["//visibility:public"]) + +# Targets in this package require a Swift toolchain (Linux or macOS). +# `select()` gives us OR-of-OSes; other platforms get marked incompatible +# so `bazel build/test //...` skips them cleanly. +# +# The Linux branch additionally requires x86_64: the only registered standalone +# Linux Swift toolchain (`swift_toolchain_exec_ubuntu24.04`, see //:MODULE.bazel) +# is x86_64-only. Without the CPU constraint, Linux/aarch64 targets would stay +# "compatible" and then fail toolchain resolution instead of being skipped. +config_setting( + name = "linux_x86_64", + constraint_values = [ + "@platforms//os:linux", + "@platforms//cpu:x86_64", + ], +) + +_SWIFT_SUPPORTED_PLATFORMS = select({ + ":linux_x86_64": [], + "@platforms//os:macos": [], + "//conditions:default": ["@platforms//:incompatible"], +}) + +# Swift FFI shim: wraps swift-syntax and exposes a small C ABI. The Rust +# targets below link against its `CcInfo`. +xcode_transition_swift_library( + name = "swift_syntax_ffi", + srcs = ["swift/Sources/SwiftSyntaxFFI/SwiftSyntaxFFI.swift"], + module_name = "SwiftSyntaxFFI", + target_compatible_with = _SWIFT_SUPPORTED_PLATFORMS, + deps = [ + "@swift-syntax//:SwiftParser", + "@swift-syntax//:SwiftSyntax", + ], +) + +# Safe Rust bindings on top of the C ABI. Under Bazel the Swift side comes +# from `:swift_syntax_ffi`; `build.rs` is only used by the `cargo` workflow. +rust_library( + name = "swift_syntax_rs", + srcs = glob( + ["src/**/*.rs"], + exclude = ["src/main.rs"], + ), + edition = "2024", + target_compatible_with = _SWIFT_SUPPORTED_PLATFORMS, + deps = [ + ":swift_syntax_ffi", + ], +) + +rust_binary( + name = "swift-syntax-parse", + srcs = ["src/main.rs"], + # `rust_binary` doesn't copy the Swift runtime into runfiles the way + # `swift_binary` does. On Linux, ship the standalone toolchain's runtime; + # on macOS the OS provides it at `/usr/lib/swift` (rpath'd by + # `xcode_swift_toolchain`). + data = select({ + "@platforms//os:macos": [], + "@platforms//os:linux": ["@swift_toolchain_ubuntu24.04//:files"], + }), + edition = "2024", + target_compatible_with = _SWIFT_SUPPORTED_PLATFORMS, + deps = [":swift_syntax_rs"], +) + +rust_test( + name = "swift_syntax_rs_test", + size = "small", + crate = ":swift_syntax_rs", + data = select({ + "@platforms//os:macos": [], + "@platforms//os:linux": ["@swift_toolchain_ubuntu24.04//:files"], + }), + edition = "2024", + target_compatible_with = _SWIFT_SUPPORTED_PLATFORMS, +) diff --git a/unified/swift-syntax-rs/Cargo.toml b/unified/swift-syntax-rs/Cargo.toml new file mode 100644 index 000000000000..6957fb0e50e9 --- /dev/null +++ b/unified/swift-syntax-rs/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "swift-syntax-rs" +description = "Rust wrapper around the swift-syntax package for parsing Swift source" +version = "0.1.0" +authors = ["GitHub"] +edition = "2024" + +[lib] +name = "swift_syntax_rs" +path = "src/lib.rs" + +[[bin]] +name = "swift-syntax-parse" +path = "src/main.rs" diff --git a/unified/swift-syntax-rs/README.md b/unified/swift-syntax-rs/README.md new file mode 100644 index 000000000000..d43969199f91 --- /dev/null +++ b/unified/swift-syntax-rs/README.md @@ -0,0 +1,203 @@ +# swift-syntax-rs + +A Rust wrapper around the [swift-syntax](https://github.com/swiftlang/swift-syntax) +package, allowing Swift source code to be parsed from Rust. + +Parsing is delegated to a small Swift shim (in [`swift/`](swift/)) that links +against `SwiftSyntax`/`SwiftParser` and exposes a tiny C ABI. The Rust crate +builds that shim (via `build.rs`) and provides safe bindings on top of it. + +## Output format + +The emitted JSON tree preserves the AST's named structure. Every node has a +`kind` and a `range` with `start`/`end` positions (UTF-8 `offset` plus 1-based +`line`/`column`). Beyond that: + +- **Tokens** carry `text`, `tokenKind`, and — only when non-empty — + `leadingTrivia`/`trailingTrivia` arrays of `{ kind, text }` pieces. +- **Layout nodes** (e.g. `functionDecl`) embed their children directly as + members keyed by the child's name in the parent (`name`, `signature`, + `body`, …), alongside `kind`/`range`. Absent optional children are omitted. +- **Collection nodes** (e.g. `codeBlockItemList`) are elided: a list-valued + field is simply a JSON array of its elements (e.g. `parameters`, `statements`). + This drops the collection node's own `kind`/`range`. + +Only meaningful trivia is kept — the four comment kinds (`lineComment`, +`blockComment`, `docLineComment`, `docBlockComment`) and `unexpectedText` +(source the parser skipped). Whitespace trivia is dropped, since node ranges +already encode positions. + +### Example + +Parsing `let x = 1 // c` produces the following (each `range` object is +abbreviated here as `…`): + +```jsonc +{ + "kind": "sourceFile", + "range": …, + "statements": [ // collection node elided to an array + { + "kind": "codeBlockItem", + "range": …, + "item": { + "kind": "variableDecl", + "range": …, + "attributes": [], // empty collection → empty array + "modifiers": [], + "bindingSpecifier": { // a token + "kind": "token", + "text": "let", + "tokenKind": "keyword(SwiftSyntax.Keyword.let)", + "range": … + }, + "bindings": [ + { + "kind": "patternBinding", + "range": …, + "pattern": { + "kind": "identifierPattern", + "range": …, + "identifier": { "kind": "token", "text": "x", "tokenKind": "identifier(\"x\")", "range": … } + }, + "initializer": { + "kind": "initializerClause", + "range": …, + "equal": { "kind": "token", "text": "=", "tokenKind": "equal", "range": … }, + "value": { + "kind": "integerLiteralExpr", + "range": …, + "literal": { + "kind": "token", + "text": "1", + "tokenKind": "integerLiteral(\"1\")", + "range": …, + "trailingTrivia": [ { "kind": "lineComment", "text": "// c" } ] + } + } + } + } + ] + } + } + ], + "endOfFileToken": { "kind": "token", "text": "", "tokenKind": "endOfFile", "range": … } +} +``` + +Note how `statements`, `bindings`, `attributes`, and `modifiers` are plain +arrays (their collection nodes are elided), layout children such as +`bindingSpecifier` and `initializer` are embedded by name, and the `// c` +comment rides along as `trailingTrivia` on the token it follows. Tokens without +trivia (most of them) simply omit the `leadingTrivia`/`trailingTrivia` keys. + +## Prerequisites + +The build does not depend on any particular version manager. You need: + +- **Rust** — pinned to `1.88` by the repo-root [`rust-toolchain.toml`](../../rust-toolchain.toml), + which `rustup` picks up automatically. +- **Swift** — pinned to the version in [`.swift-version`](.swift-version) + (currently `6.3.2`), used to build `swift-syntax` `603.0.2`. Install it any way + you like — [swift.org](https://www.swift.org/install/) or + [swiftly](https://www.swift.org/swiftly/) (which reads `.swift-version`), or a + system package. Just make sure `swift` is on your `PATH` (or point `build.rs` + at it with the `SWIFT` environment variable). + +On Debian/Ubuntu the Swift runtime also needs `libncurses6` (and related libs) +available on the system. + +## Building & testing + +With `cargo` and `swift` on `PATH`: + +```sh +cargo build +cargo test +``` + +If your `swift`/`swiftc` are not on `PATH`, point the build at them explicitly: + +```sh +SWIFT=/path/to/swift SWIFTC=/path/to/swiftc cargo build +``` + +The first build compiles `swift-syntax` and can take several minutes. + +## Building with Bazel (CI) + +CI builds this crate hermetically with Bazel. A Swift toolchain is downloaded +from swift.org by the official `rules_swift` standalone toolchain extension +(wired up in the repo-root `MODULE.bazel`), `swift-syntax` is pulled from the +Bazel Central Registry, and the FFI shim is compiled as a `swift_library` that +the Rust targets link against. `build.rs` is not used under Bazel; it only +builds the Swift shim for the local `cargo` workflow. + +```sh +bazel build //unified/swift-syntax-rs:swift-syntax-parse +bazel test //unified/swift-syntax-rs:swift_syntax_rs_test +bazel run //unified/swift-syntax-rs:swift-syntax-parse < some.swift +``` + +Requirements: + +- **`clang`** must be installed on the runner. `rules_swift` requires the Bazel + CC toolchain to use clang; the repo's `.bazelrc` already sets + `--repo_env=CC=clang`, so no extra flags are needed. +- The registered Swift toolchains cover **ubuntu24.04 / x86_64** and + **macOS / `xcode`** (Apple Silicon and Intel). Bazel selects the toolchain + matching the host. Targets are marked `target_compatible_with` these two + OSes, so on Windows Bazel skips them cleanly. +- **macOS only:** the Swift toolchain comes from the host Xcode installation + (`rules_swift` auto-registers `xcode_swift_toolchain`), which also needs + Xcode's CC toolchain and xcode_config; these are applied to the Swift + target via an incoming-edge Starlark transition (see + [`xcode_transition.bzl`](xcode_transition.bzl)), so other targets on macOS + keep using Bazel's default CC toolchain. + +The Swift compiler version is kept in sync across three places: the +[`.swift-version`](.swift-version) file (read by the local `cargo`/`swift build` +and by [swiftly](https://www.swift.org/swiftly/)), the literal `swift_version` +pinned on `swift.toolchain(...)` in the root `MODULE.bazel` (the hermetic +swift.org **Linux** Bazel toolchain), and the `swift-syntax` release in +`swift/Package.swift`. On **macOS** the version is *not* pinned by the Bazel +build: `rules_swift` auto-registers the host `xcode_swift_toolchain`, which uses +whichever Swift ships with the installed Xcode. So the pin governs Linux (and +local) builds, while the macOS compiler version depends on the host Xcode. + +(The Bazel toolchain pins a literal rather than reading `.swift-version` via +`swift_version_file`, because the latter makes the module extension read a +`//unified/...` label, which fails when this repo is consumed as a dependency +module.) + +## Usage + +Library: + +```rust +let json = swift_syntax_rs::parse_to_json("let x = 1")?; +println!("{json}"); +``` + +CLI (reads a file argument or stdin, prints the syntax tree as JSON): + +```sh +echo 'let x = 1' | cargo run --bin swift-syntax-parse +``` + +## Converting to a yeast AST + +The JSON tree is consumed by the CodeQL extractor, which converts it into a +[`yeast::Ast`](../../shared/yeast) — the in-memory format its rewrite rules +operate on. That adapter is a pure-Rust module living in the extractor +(`unified/extractor/src/languages/swift/adapter.rs`), so the extractor never +needs the Swift toolchain: it consumes the JSON produced out-of-process by this +crate's `parse_to_json` / the `swift-syntax-parse` binary. + +## Layout + +- `swift/` — Swift package exposing the `ssr_parse_json` / `ssr_string_free` C ABI. +- `build.rs` — builds the Swift package and emits link/rpath flags (local `cargo` only). +- `BUILD.bazel` — Bazel targets for the hermetic CI build (swift_library + rust targets). +- `src/lib.rs` — safe Rust bindings (`parse_to_json`). +- `src/main.rs` — demo CLI. diff --git a/unified/swift-syntax-rs/build.rs b/unified/swift-syntax-rs/build.rs new file mode 100644 index 000000000000..c599cdf4e70b --- /dev/null +++ b/unified/swift-syntax-rs/build.rs @@ -0,0 +1,106 @@ +use std::env; +use std::path::PathBuf; +use std::process::Command; + +fn main() { + let manifest_dir = PathBuf::from(env::var("CARGO_MANIFEST_DIR").unwrap()); + let swift_dir = manifest_dir.join("swift"); + + // Emitting any `rerun-if-changed` disables Cargo's default whole-package + // scan, so we must list every input to the `swift build` below. Watch the + // package manifests, the pinned dependency lockfile, the pinned compiler + // version, and the entire Swift sources tree (a directory is scanned + // recursively). `.build/` is intentionally *not* watched (it is this + // build's output; watching it would cause perpetual rebuilds). + for input in [ + swift_dir.join("Package.swift"), + swift_dir.join("Package.resolved"), + swift_dir.join("Sources"), + manifest_dir.join(".swift-version"), + ] { + println!("cargo:rerun-if-changed={}", input.display()); + } + println!("cargo:rerun-if-env-changed=SWIFT"); + println!("cargo:rerun-if-env-changed=SWIFTC"); + + // Build the Swift FFI package as a release dynamic library. + let mut command = Command::new(swift_bin()); + command + .args(["build", "-c", "release"]) + .current_dir(&swift_dir); + apply_bare_repository_workaround(&mut command); + let status = command.status().unwrap_or_else(|e| { + panic!( + "failed to run `{swift} build`: {e}\n\ + Install a Swift toolchain (see https://www.swift.org/install/, e.g. via \ + swiftly) and ensure `swift` is on PATH, or set the `SWIFT` environment \ + variable to the `swift` executable. The pinned version is in `.swift-version`.", + swift = swift_bin(), + ) + }); + assert!(status.success(), "`swift build` failed"); + + // Link against the freshly built dynamic library. + let build_dir = swift_dir.join(".build/release"); + println!("cargo:rustc-link-search=native={}", build_dir.display()); + println!("cargo:rustc-link-lib=dylib=SwiftSyntaxFFI"); + println!("cargo:rustc-link-arg=-Wl,-rpath,{}", build_dir.display()); + + // The executable also needs to find the Swift runtime libraries at run time. + if let Some(runtime) = swift_runtime_dir() { + println!("cargo:rustc-link-search=native={}", runtime.display()); + println!("cargo:rustc-link-arg=-Wl,-rpath,{}", runtime.display()); + } +} + +/// Query the active Swift toolchain for the directory containing its runtime +/// shared libraries (e.g. `libswiftCore.so`). +fn swift_runtime_dir() -> Option { + let output = Command::new(swiftc_bin()) + .arg("-print-target-info") + .output() + .ok()?; + if !output.status.success() { + return None; + } + let info = String::from_utf8_lossy(&output.stdout); + + // Extract the value of `"runtimeResourcePath": "..."` without pulling in a + // JSON dependency. + let key = "\"runtimeResourcePath\""; + let start = info.find(key)?; + let rest = &info[start + key.len()..]; + let colon = rest.find(':')?; + let after = &rest[colon + 1..]; + let open = after.find('"')?; + let value_start = &after[open + 1..]; + let close = value_start.find('"')?; + let resource_path = &value_start[..close]; + + Some(PathBuf::from(resource_path).join(if cfg!(target_os = "macos") { "macosx" } else { "linux" })) +} + +/// The `swift` driver to invoke: `$SWIFT` if set, otherwise `swift` from `PATH`. +/// This keeps the build tool-agnostic — any Swift install works; no particular +/// version manager is required. +fn swift_bin() -> String { + env::var("SWIFT").unwrap_or_else(|_| "swift".to_string()) +} + +/// The `swiftc` compiler to invoke: `$SWIFTC` if set, otherwise `swiftc` from +/// `PATH`. +fn swiftc_bin() -> String { + env::var("SWIFTC").unwrap_or_else(|_| "swiftc".to_string()) +} + +/// Some environments (notably GitHub Codespaces) inject +/// `GIT_CONFIG_KEY_0=safe.bareRepository` / `GIT_CONFIG_VALUE_0=explicit`, which +/// breaks the cached bare git repositories `swift build` uses. When exactly that +/// key is present, relax it to `all` for the `swift build` subprocess only +/// (rather than unconditionally, which could clobber an unrelated +/// `GIT_CONFIG_VALUE_0`). +fn apply_bare_repository_workaround(command: &mut Command) { + if env::var("GIT_CONFIG_KEY_0").as_deref() == Ok("safe.bareRepository") { + command.env("GIT_CONFIG_VALUE_0", "all"); + } +} diff --git a/unified/swift-syntax-rs/src/lib.rs b/unified/swift-syntax-rs/src/lib.rs new file mode 100644 index 000000000000..2a8fe0411af8 --- /dev/null +++ b/unified/swift-syntax-rs/src/lib.rs @@ -0,0 +1,135 @@ +//! Rust wrapper around the [swift-syntax](https://github.com/swiftlang/swift-syntax) +//! package, allowing Swift source code to be parsed from Rust. +//! +//! The heavy lifting is done by a small Swift shim (see `swift/`) that links +//! against `SwiftSyntax`/`SwiftParser` and exposes a tiny C ABI. This module +//! provides safe Rust bindings on top of that ABI, exposing [`parse_to_json`] +//! which turns Swift source into a JSON syntax tree. +//! +//! Converting that JSON into a `yeast::Ast` (for the CodeQL extractor) is done +//! by the extractor's own pure-Rust adapter module, keeping the Swift toolchain +//! out of the extractor's build. + +use std::ffi::{CStr, CString}; +use std::os::raw::c_char; + +// C ABI exported by the `SwiftSyntaxFFI` dynamic library. +unsafe extern "C" { + /// Parse a NUL-terminated Swift source string, returning a heap-allocated + /// NUL-terminated JSON string (or null on failure). The caller owns the + /// returned pointer and must release it with `ssr_string_free`. + fn ssr_parse_json(source: *const c_char) -> *mut c_char; + + /// Free a string previously returned by `ssr_parse_json`. + fn ssr_string_free(ptr: *mut c_char); +} + +/// Errors that can occur while parsing Swift source. +#[derive(Debug)] +pub enum ParseError { + /// The provided source contained an interior NUL byte. + NulByte, + /// The Swift shim returned no result. `SwiftParser` recovers from invalid + /// syntax (it always produces a tree, possibly with error nodes), so this + /// does *not* indicate a syntax error in the source — it means the shim + /// failed to serialize the tree to JSON or to allocate the result string. + SwiftFailure, +} + +impl std::fmt::Display for ParseError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + ParseError::NulByte => write!(f, "source contained an interior NUL byte"), + ParseError::SwiftFailure => { + write!(f, "the swift-syntax shim failed to produce a JSON result") + } + } + } +} + +impl std::error::Error for ParseError {} + +/// Parse the given Swift `source` and return a JSON representation of its +/// syntax tree. +/// +/// # Example +/// ```no_run +/// let json = swift_syntax_rs::parse_to_json("let x = 1").unwrap(); +/// println!("{json}"); +/// ``` +pub fn parse_to_json(source: &str) -> Result { + let c_source = CString::new(source).map_err(|_| ParseError::NulByte)?; + + // SAFETY: `c_source` is a valid NUL-terminated string for the duration of + // the call. The returned pointer, if non-null, is owned by us and freed via + // `ssr_string_free` before returning. + unsafe { + let ptr = ssr_parse_json(c_source.as_ptr()); + if ptr.is_null() { + return Err(ParseError::SwiftFailure); + } + let json = CStr::from_ptr(ptr).to_string_lossy().into_owned(); + ssr_string_free(ptr); + Ok(json) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parses_simple_declaration() { + let json = parse_to_json("let x = 1").expect("parsing should succeed"); + assert!( + json.contains("\"kind\":\"sourceFile\""), + "unexpected tree: {json}" + ); + assert!(json.contains("\"text\":\"x\""), "unexpected tree: {json}"); + // Source ranges are emitted for every node. + assert!(json.contains("\"range\""), "missing ranges: {json}"); + assert!( + json.contains("\"line\"") && json.contains("\"column\"") && json.contains("\"offset\""), + "missing location fields: {json}" + ); + } + + #[test] + fn preserves_named_structure() { + let json = parse_to_json("let x = 1").expect("parsing should succeed"); + // Layout children are embedded directly under their field name. + assert!( + json.contains("\"initializer\""), + "missing initializer field: {json}" + ); + // Collection-valued fields are elided to plain JSON arrays. + assert!( + json.contains("\"statements\":["), + "statements should be an array: {json}" + ); + assert!( + json.contains("\"bindings\":["), + "bindings should be an array: {json}" + ); + } + + #[test] + fn parses_empty_source() { + let json = parse_to_json("").expect("parsing empty source should succeed"); + assert!( + json.contains("\"kind\":\"sourceFile\""), + "unexpected tree: {json}" + ); + } + + #[test] + fn captures_trivia() { + // A leading comment is kept as trivia on the token it precedes. + let json = parse_to_json("// hi\nlet x = 1").expect("parsing should succeed"); + assert!(json.contains("\"leadingTrivia\""), "missing trivia: {json}"); + assert!( + json.contains("lineComment"), + "comment trivia not captured: {json}" + ); + } +} diff --git a/unified/swift-syntax-rs/src/main.rs b/unified/swift-syntax-rs/src/main.rs new file mode 100644 index 000000000000..d79cd2ffef1f --- /dev/null +++ b/unified/swift-syntax-rs/src/main.rs @@ -0,0 +1,42 @@ +//! Small demo CLI: parse Swift source and print the syntax tree as JSON. +//! +//! Usage: +//! swift-syntax-parse [FILE] +//! +//! Reads from FILE if given, otherwise from standard input. + +use std::io::Read; +use std::process::ExitCode; + +fn main() -> ExitCode { + let source = match read_source() { + Ok(s) => s, + Err(e) => { + eprintln!("error reading source: {e}"); + return ExitCode::FAILURE; + } + }; + + match swift_syntax_rs::parse_to_json(&source) { + Ok(json) => { + println!("{json}"); + ExitCode::SUCCESS + } + Err(e) => { + eprintln!("error: {e}"); + ExitCode::FAILURE + } + } +} + +fn read_source() -> std::io::Result { + let mut args = std::env::args().skip(1); + match args.next() { + Some(path) => std::fs::read_to_string(path), + None => { + let mut buf = String::new(); + std::io::stdin().read_to_string(&mut buf)?; + Ok(buf) + } + } +} diff --git a/unified/swift-syntax-rs/swift/Package.resolved b/unified/swift-syntax-rs/swift/Package.resolved new file mode 100644 index 000000000000..dafd46e961b5 --- /dev/null +++ b/unified/swift-syntax-rs/swift/Package.resolved @@ -0,0 +1,15 @@ +{ + "originHash" : "169957f8fbd882866f8f9f1c68cde813dca3560577939a8ffec366e3d77548e4", + "pins" : [ + { + "identity" : "swift-syntax", + "kind" : "remoteSourceControl", + "location" : "https://github.com/swiftlang/swift-syntax.git", + "state" : { + "revision" : "79e4b74a295b6eb74a8b585e3a39d29e70c1dbd1", + "version" : "603.0.2" + } + } + ], + "version" : 3 +} diff --git a/unified/swift-syntax-rs/swift/Package.swift b/unified/swift-syntax-rs/swift/Package.swift new file mode 100644 index 000000000000..a58b7c1a479b --- /dev/null +++ b/unified/swift-syntax-rs/swift/Package.swift @@ -0,0 +1,35 @@ +// swift-tools-version:6.0 +import PackageDescription + +let package = Package( + name: "SwiftSyntaxFFI", + platforms: [ + // swift-syntax 603 requires macOS 10.15; declare it explicitly + // rather than relying on the swift-tools-version default (10.13). + .macOS(.v10_15), + ], + products: [ + // Dynamic library so the produced .so records its dependency on the + // Swift runtime libraries, keeping the Rust link step simple. + .library( + name: "SwiftSyntaxFFI", + type: .dynamic, + targets: ["SwiftSyntaxFFI"] + ) + ], + dependencies: [ + .package( + url: "https://github.com/swiftlang/swift-syntax.git", + exact: "603.0.2" + ) + ], + targets: [ + .target( + name: "SwiftSyntaxFFI", + dependencies: [ + .product(name: "SwiftSyntax", package: "swift-syntax"), + .product(name: "SwiftParser", package: "swift-syntax"), + ] + ) + ] +) diff --git a/unified/swift-syntax-rs/swift/Sources/SwiftSyntaxFFI/SwiftSyntaxFFI.swift b/unified/swift-syntax-rs/swift/Sources/SwiftSyntaxFFI/SwiftSyntaxFFI.swift new file mode 100644 index 000000000000..6305b1610d06 --- /dev/null +++ b/unified/swift-syntax-rs/swift/Sources/SwiftSyntaxFFI/SwiftSyntaxFFI.swift @@ -0,0 +1,181 @@ +import Foundation +import SwiftParser + +// `@_spi(RawSyntax)` exposes the `childName(_:)` helper that maps a child's +// key path to its field name in the parent layout; used to emit named fields. +@_spi(RawSyntax) import SwiftSyntax + +#if canImport(Glibc) + import Glibc +#elseif canImport(Darwin) + import Darwin +#endif + +/// Convert an absolute position into an `{ offset, line, column }` dictionary. +/// +/// `offset` is a UTF-8 byte offset; `line`/`column` are 1-based. +private func location( + _ position: AbsolutePosition, + _ converter: SourceLocationConverter +) -> [String: Any] { + let loc = converter.location(for: position) + return [ + "offset": position.utf8Offset, + "line": loc.line, + "column": loc.column, + ] +} + +/// Trivia kinds worth preserving in the serialized tree. Comments carry +/// developer intent (including doc comments), and `unexpectedText` flags source +/// the parser had to skip. Whitespace and multi-line-string escape markers are +/// dropped: node ranges already encode positions, so they would only bloat the +/// output. +private let keptTriviaKinds: Set = [ + "lineComment", + "blockComment", + "docLineComment", + "docBlockComment", + "unexpectedText", +] + +/// Serialize a trivia collection into an array of `{ kind, text, range }` +/// pieces, keeping only the kinds in `keptTriviaKinds`. +/// +/// `start` is the absolute position of the first piece (a token's leading +/// trivia starts at `token.position`; its trailing trivia at +/// `token.endPositionBeforeTrailingTrivia`). Each piece's range is derived by +/// accumulating piece lengths, so kept pieces carry an exact source location. +private func serializeTrivia( + _ trivia: Trivia, + startingAt start: AbsolutePosition, + _ converter: SourceLocationConverter +) -> [Any] { + var result: [Any] = [] + var offset = start.utf8Offset + for piece in trivia.pieces { + let length = piece.sourceLength.utf8Length + // The label of an enum case mirror is the case name (e.g. "spaces", + // "lineComment"), which gives us a stable kind without an exhaustive + // switch over every `TriviaPiece` case. + let kind = Mirror(reflecting: piece).children.first?.label ?? "\(piece)" + if keptTriviaKinds.contains(kind) { + result.append([ + "kind": kind, + "text": Trivia(pieces: [piece]).description, + "range": [ + "start": location(AbsolutePosition(utf8Offset: offset), converter), + "end": location(AbsolutePosition(utf8Offset: offset + length), converter), + ], + ]) + } + offset += length + } + return result +} + +/// Recursively convert a SwiftSyntax node into a JSON-serializable value. +/// +/// * Tokens carry `kind`, `tokenKind`, `text`, and `range`, plus +/// `leadingTrivia`/`trailingTrivia` — but only when non-empty (after +/// filtering, most tokens have no trivia, so the keys are simply absent). +/// * Layout nodes (e.g. `functionDecl`) carry `kind` and source `range`, and +/// additionally embed their children directly as members keyed by the +/// child's name in the parent (e.g. `name`, `signature`, `body`); absent +/// optional children are omitted. Field names never collide with +/// `kind`/`range`. +/// * Collection nodes (e.g. `codeBlockItemList`) are *elided*: they become a +/// plain array of their serialized elements, taking the place of the +/// collection node itself. A list-valued layout field (e.g. `parameters`) is +/// therefore simply a JSON array. This drops the collection node's own +/// `kind`/`range`, which are unnamed and largely recoverable from the +/// elements. +private func serialize( + _ node: Syntax, + _ converter: SourceLocationConverter +) -> Any { + if node.kind.isSyntaxCollection { + return node.children(viewMode: .sourceAccurate).map { + serialize($0, converter) + } + } + + // Source range covering the node's content, excluding surrounding trivia. + let range: [String: Any] = [ + "start": location(node.positionAfterSkippingLeadingTrivia, converter), + "end": location(node.endPositionBeforeTrailingTrivia, converter), + ] + + if let token = node.as(TokenSyntax.self) { + var result: [String: Any] = [ + "kind": "token", + "tokenKind": "\(token.tokenKind)", + "text": token.text, + "range": range, + ] + // Only emit trivia when present; after filtering, most tokens have none. + // Leading trivia starts at the token's own position; trailing trivia + // starts just after the token's content. + let leading = serializeTrivia( + token.leadingTrivia, startingAt: token.position, converter) + if !leading.isEmpty { + result["leadingTrivia"] = leading + } + let trailing = serializeTrivia( + token.trailingTrivia, + startingAt: token.endPositionBeforeTrailingTrivia, + converter) + if !trailing.isEmpty { + result["trailingTrivia"] = trailing + } + return result + } + + var result: [String: Any] = [ + "kind": "\(node.kind)", + "range": range, + ] + var unnamed = 0 + for child in node.children(viewMode: .sourceAccurate) { + // Layout children are named; embed each under its field name in the + // parent (the same mechanism SwiftSyntax uses for its debug dump). A + // child that is a collection serializes to an array (see above). + if let keyPath = child.keyPathInParent, let name = childName(keyPath) { + result[name] = serialize(child, converter) + } else { + // Defensive fallback for any unnamed layout child. + result["child\(unnamed)"] = serialize(child, converter) + unnamed += 1 + } + } + return result +} + +/// Parse the given NUL-terminated Swift source string and return a +/// heap-allocated, NUL-terminated JSON representation of the syntax tree. +/// +/// The returned pointer is owned by the caller and MUST be released with +/// `ssr_string_free`. Returns `nil` on failure. +@_cdecl("ssr_parse_json") +public func ssr_parse_json(_ source: UnsafePointer?) -> UnsafeMutablePointer? { + guard let source = source else { return nil } + let code = String(cString: source) + let tree = Parser.parse(source: code) + let converter = SourceLocationConverter(fileName: "", tree: tree) + let json = serialize(Syntax(tree), converter) + + guard + let data = try? JSONSerialization.data( + withJSONObject: json, options: [.sortedKeys]), + let string = String(data: data, encoding: .utf8) + else { + return nil + } + return strdup(string) +} + +/// Free a string previously returned by this library. +@_cdecl("ssr_string_free") +public func ssr_string_free(_ ptr: UnsafeMutablePointer?) { + free(ptr) +} diff --git a/unified/swift-syntax-rs/xcode_transition.bzl b/unified/swift-syntax-rs/xcode_transition.bzl new file mode 100644 index 000000000000..b1612e762b15 --- /dev/null +++ b/unified/swift-syntax-rs/xcode_transition.bzl @@ -0,0 +1,95 @@ +"""Per-target Xcode configuration for `//unified/swift-syntax-rs` on macOS. + +`rules_swift`'s auto-registered `xcode_swift_toolchain` reads +`cc_toolchain.target_gnu_system_name`, which is literally "local" on +Bazel's built-in `local_config_cc`. To make it usable we need: + +- `--xcode_version_config=@local_config_xcode//:host_xcodes` — selects + `apple_support`'s xcode_config (matches `rules_swift`'s `system_sdk` keys). +- `--extra_toolchains=@local_config_apple_cc_toolchains//:all` — forces + `apple_support`'s CC toolchain ahead of `local_config_cc`. + +Applied via an incoming-edge Starlark transition on the `swift_library` +target only (via the `xcode_transition_swift_library` macro), so downstream +Rust targets and the rest of the repo stay on the default CC toolchain and +the `@local_config_*` repos are not materialized otherwise. No-op off macOS. +""" + +load("@rules_cc//cc/common:cc_info.bzl", "CcInfo") +load("@rules_swift//swift:swift.bzl", "swift_library") +load("//misc/bazel:os.bzl", "os_select") + +_XCODE_VERSION_CONFIG = "//command_line_option:xcode_version_config" +_EXTRA_TOOLCHAINS = "//command_line_option:extra_toolchains" + +def _transition_impl(settings, attr): + if attr.os != "macos": + # Preserve inputs so the configuration is identical to the incoming one. + return { + _XCODE_VERSION_CONFIG: settings[_XCODE_VERSION_CONFIG], + _EXTRA_TOOLCHAINS: settings[_EXTRA_TOOLCHAINS], + } + return { + _XCODE_VERSION_CONFIG: "@local_config_xcode//:host_xcodes", + _EXTRA_TOOLCHAINS: ( + list(settings[_EXTRA_TOOLCHAINS]) + + ["@local_config_apple_cc_toolchains//:all"] + ), + } + +_xcode_transition = transition( + implementation = _transition_impl, + inputs = [_XCODE_VERSION_CONFIG, _EXTRA_TOOLCHAINS], + outputs = [_XCODE_VERSION_CONFIG, _EXTRA_TOOLCHAINS], +) + +def _wrapper_impl(ctx): + src = ctx.attr.actual[0] + + # Forward the providers a downstream `rust_*` target reads from `deps`: + # `DefaultInfo`, `CcInfo` (linking info), and `OutputGroupInfo`. + providers = [src[DefaultInfo]] + for p in (CcInfo, OutputGroupInfo): + if p in src: + providers.append(src[p]) + return providers + +_xcode_transition_swift_library_rule = rule( + implementation = _wrapper_impl, + attrs = { + "actual": attr.label( + mandatory = True, + cfg = _xcode_transition, + providers = [CcInfo], + ), + "os": attr.string(mandatory = True), + "_allowlist_function_transition": attr.label( + default = "@bazel_tools//tools/allowlists/function_transition_allowlist", + ), + }, +) + +def xcode_transition_swift_library(name, visibility = None, tags = None, target_compatible_with = None, **kwargs): + """`swift_library` wrapped in the macOS Xcode-config transition. + + Emits a private inner `_impl_` `swift_library` (tagged `manual`) + and a public `name` that applies the transition on macOS and forwards + the inner target's providers. Downstream `rust_*` targets depend on + `name` as usual; only the `swift_library` sub-graph flips toolchain. + """ + inner_name = "_impl_%s" % name + swift_library( + name = inner_name, + visibility = ["//visibility:private"], + tags = (tags or []) + ["manual"], + target_compatible_with = target_compatible_with, + **kwargs + ) + _xcode_transition_swift_library_rule( + name = name, + visibility = visibility, + tags = tags, + target_compatible_with = target_compatible_with, + actual = ":" + inner_name, + os = os_select(linux = "linux", macos = "macos", default = "other"), + )