unified: Add mapping from swift-syntax JSON to yeast AST

Adds a preliminary mapping that decodes the JSON into the yeast AST. The
JSON AST itself is still very verbose, but it would be useful to see if
it actually contains all of the things we want it to contain.

Author: tausbn

Cargo.lock

@@ -333,6 +333,28 @@ impl Ast {
         ast
     }
 
+    /// Construct an empty AST backed by `schema`, for building a tree
+    /// programmatically from a source other than a tree-sitter parse (e.g. an
+    /// external parser's output). Populate it with [`Ast::create_node`] /
+    /// [`Ast::create_node_with_range`] and then designate the root with
+    /// [`Ast::set_root`]. The `schema` must already have every node kind and
+    /// field name registered (see [`schema::Schema`]).
+    pub fn with_schema(schema: schema::Schema) -> Self {
+        Self {
+            root: Id(0),
+            nodes: Vec::new(),
+            schema,
+            source: Vec::new(),
+        }
+    }
+
+    /// Set the original source bytes, used to resolve `NodeContent::Range`
+    /// nodes to text. Nodes built with inline `NodeContent::DynamicString`
+    /// content do not require this.
+    pub fn set_source(&mut self, source: Vec<u8>) {
+        self.source = source;
+    }
+
     /// Returns the source text for `id`, resolving `NodeContent::Range`
     /// against the stored source bytes when available.
     pub fn source_text(&self, id: Id) -> String {
@@ -463,6 +485,24 @@ impl Ast {
         Id(id)
     }
 
+    /// Register a named node kind, returning its id (idempotent). Lets callers
+    /// build an AST in a single pass, registering kinds as nodes are created
+    /// rather than pre-populating the schema.
+    pub fn register_kind(&mut self, name: &str) -> KindId {
+        self.schema.register_kind(name)
+    }
+
+    /// Register an anonymous (unnamed) token kind, returning its id
+    /// (idempotent). Anonymous tokens are keyed by their text (e.g. `"func"`).
+    pub fn register_unnamed_kind(&mut self, name: &str) -> KindId {
+        self.schema.register_unnamed_kind(name)
+    }
+
+    /// Register a field name, returning its id (idempotent).
+    pub fn register_field(&mut self, name: &str) -> FieldId {
+        self.schema.register_field(name)
+    }
+
     fn union_source_range_of_children(
         &self,
         fields: &BTreeMap<FieldId, Vec<Id>>,
@@ -604,7 +644,7 @@ impl Ast {
         }
     }
 
-    fn id_for_unnamed_node_kind(&self, kind: &str) -> Option<KindId> {
+    pub fn id_for_unnamed_node_kind(&self, kind: &str) -> Option<KindId> {
         let id = self.schema.id_for_unnamed_node_kind(kind).unwrap_or(0);
         if id == 0 {
             None

shared/yeast/src/lib.rs

@@ -28,9 +28,16 @@ swift_library(
 # Swift shim in the local `cargo` workflow.
 rust_library(
     name = "swift_syntax_rs",
-    srcs = ["src/lib.rs"],
+    srcs = glob(
+        ["src/**/*.rs"],
+        exclude = ["src/main.rs"],
+    ),
     edition = "2024",
-    deps = [":swift_syntax_ffi"],
+    deps = [
+        ":swift_syntax_ffi",
+        "//shared/yeast",
+        "@vendor_ts__serde_json-1.0.145//:serde_json",
+    ],
 )
 
 rust_binary(

unified/swift-syntax-rs/BUILD.bazel

@@ -12,3 +12,7 @@ path = "src/lib.rs"
 [[bin]]
 name = "swift-syntax-parse"
 path = "src/main.rs"
+
+[dependencies]
+serde_json = "1.0"
+yeast = { path = "../../shared/yeast" }

unified/swift-syntax-rs/Cargo.toml

@@ -13,8 +13,8 @@ The emitted JSON tree preserves the AST's named structure. Every node has a
 `kind` and a `range` with `start`/`end` positions (UTF-8 `offset` plus 1-based
 `line`/`column`). Beyond that:
 
-- **Tokens** carry `text`, `tokenKind`, and `leadingTrivia`/`trailingTrivia`
-  arrays of `{ kind, text }` pieces.
+- **Tokens** carry `text`, `tokenKind`, and — only when non-empty —
+  `leadingTrivia`/`trailingTrivia` arrays of `{ kind, text }` pieces.
 - **Layout nodes** (e.g. `functionDecl`) embed their children directly as
   members keyed by the child's name in the parent (`name`, `signature`,
   `body`, …), alongside `kind`/`range`. Absent optional children are omitted.
@@ -49,9 +49,7 @@ abbreviated here as `…`):
           "kind": "token",
           "text": "let",
           "tokenKind": "keyword(SwiftSyntax.Keyword.let)",
-          "range": …,
-          "leadingTrivia": [],
-          "trailingTrivia": []
+          "range": …
         },
         "bindings": [
           {
@@ -60,12 +58,12 @@ abbreviated here as `…`):
             "pattern": {
               "kind": "identifierPattern",
               "range": …,
-              "identifier": { "kind": "token", "text": "x", "tokenKind": "identifier(\"x\")", "range": …, "leadingTrivia": [], "trailingTrivia": [] }
+              "identifier": { "kind": "token", "text": "x", "tokenKind": "identifier(\"x\")", "range": … }
             },
             "initializer": {
               "kind": "initializerClause",
               "range": …,
-              "equal": { "kind": "token", "text": "=", "tokenKind": "equal", "range": …, "leadingTrivia": [], "trailingTrivia": [] },
+              "equal": { "kind": "token", "text": "=", "tokenKind": "equal", "range": … },
               "value": {
                 "kind": "integerLiteralExpr",
                 "range": …,
@@ -74,7 +72,6 @@ abbreviated here as `…`):
                   "text": "1",
                   "tokenKind": "integerLiteral(\"1\")",
                   "range": …,
-                  "leadingTrivia": [],
                   "trailingTrivia": [ { "kind": "lineComment", "text": "// c" } ]
                 }
               }
@@ -84,14 +81,15 @@ abbreviated here as `…`):
       }
     }
   ],
-  "endOfFileToken": { "kind": "token", "text": "", "tokenKind": "endOfFile", "range": …, "leadingTrivia": [], "trailingTrivia": [] }
+  "endOfFileToken": { "kind": "token", "text": "", "tokenKind": "endOfFile", "range": … }
 }
 ```
 
 Note how `statements`, `bindings`, `attributes`, and `modifiers` are plain
 arrays (their collection nodes are elided), layout children such as
 `bindingSpecifier` and `initializer` are embedded by name, and the `// c`
-comment rides along as `trailingTrivia` on the token it follows.
+comment rides along as `trailingTrivia` on the token it follows. Tokens without
+trivia (most of them) simply omit the `leadingTrivia`/`trailingTrivia` keys.
 
 ## Prerequisites
 
@@ -171,10 +169,29 @@ CLI (reads a file argument or stdin, prints the syntax tree as JSON):
 echo 'let x = 1' | cargo run --bin swift-syntax-parse
 ```
 
+## Converting to a yeast AST
+
+For use in the CodeQL extractor, the JSON tree can be converted into a
+[`yeast::Ast`](../../shared/yeast) — the in-memory format the extractor's
+rewrite rules operate on — via [`yeast_adapter::json_to_ast`](src/yeast_adapter.rs):
+
+```rust
+let json = swift_syntax_rs::parse_to_json("let x = 1")?;
+let ast = swift_syntax_rs::yeast_adapter::json_to_ast(&json)?;
+```
+
+The adapter mirrors tree-sitter's node model, which is what yeast expects:
+layout nodes and varying tokens (identifiers, literals, operators) become
+**named** nodes; fixed tokens (keywords, punctuation) become **anonymous**
+nodes keyed by their text. It preserves swift-syntax's own kind/field names —
+aligning them with the tree-sitter-swift schema so the existing rewrite rules
+fire is a separate, later step.
+
 ## Layout
 
 - `swift/` — Swift package exposing the `ssr_parse_json` / `ssr_string_free` C ABI.
 - `build.rs` — builds the Swift package and emits link/rpath flags (local `cargo` only).
 - `BUILD.bazel` — Bazel targets for the hermetic CI build (swift_library + rust targets).
 - `src/lib.rs` — safe Rust bindings (`parse_to_json`).
+- `src/yeast_adapter.rs` — converts the JSON tree into a `yeast::Ast`.
 - `src/main.rs` — demo CLI.

unified/swift-syntax-rs/README.md

@@ -5,6 +5,8 @@
 //! against `SwiftSyntax`/`SwiftParser` and exposes a tiny C ABI. This module
 //! provides safe Rust bindings on top of that ABI.
 
+pub mod yeast_adapter;
+
 use std::ffi::{CStr, CString};
 use std::os::raw::c_char;