Split SwiftSyntaxParser into a separate module

Changelog.md

@@ -5,6 +5,9 @@ Note: This is in reverse chronological order, so newer entries are added to the
 ## `main`
 
 * To clarify that the edits passed to `IncrementalParseTransition` are applied concurrently, introduce a new `ConcurrentEdit` type that provides the guarantee and allows translation of sequentially applied edits to the expected concurrent form.
+* The `SwiftSyntaxParser` type and a few related types now live in their own module (also named `SwiftSyntaxParser`). This allows using `SwiftSyntax` for code generation purposes without having a compatible `_InternalSwiftSyntaxParser.dylib` around. 
+
+  `import SwiftSyntaxParser` where necessary.
 
 ## Swift 5.3
 

Package.swift

@@ -7,11 +7,13 @@ let package = Package(
   name: "SwiftSyntax",
   targets: [
     .target(name: "_CSwiftSyntax"),
-    .testTarget(name: "SwiftSyntaxTest", dependencies: ["SwiftSyntax"], exclude: ["Inputs"]),
+    .testTarget(name: "SwiftSyntaxTest", dependencies: ["SwiftSyntax"]),
     .target(name: "SwiftSyntaxBuilder", dependencies: ["SwiftSyntax"]),
     .testTarget(name: "SwiftSyntaxBuilderTest", dependencies: ["SwiftSyntaxBuilder"]),
-    .target(name: "lit-test-helper", dependencies: ["SwiftSyntax"]),
-    .testTarget(name: "PerformanceTest", dependencies: ["SwiftSyntax"])
+    .target(name: "SwiftSyntaxParser", dependencies: ["SwiftSyntax"]),
+    .testTarget(name: "SwiftSyntaxParserTest", dependencies: ["SwiftSyntaxParser"], exclude: ["Inputs"]),
+    .target(name: "lit-test-helper", dependencies: ["SwiftSyntax", "SwiftSyntaxParser"]),
+    .testTarget(name: "PerformanceTest", dependencies: ["SwiftSyntax", "SwiftSyntaxParser"])
     // Also see targets added below
   ]
 )
@@ -52,4 +54,5 @@ if ProcessInfo.processInfo.environment["SWIFT_BUILD_SCRIPT_ENVIRONMENT"] != nil
 }
 
 package.products.append(.library(name: "SwiftSyntax", type: libraryType, targets: ["SwiftSyntax"]))
+package.products.append(.library(name: "SwiftSyntaxParser", type: libraryType, targets: ["SwiftSyntaxParser"]))
 package.products.append(.library(name: "SwiftSyntaxBuilder", type: libraryType, targets: ["SwiftSyntaxBuilder"]))

Sources/SwiftSyntax/CNodes.swift

@@ -0,0 +1,72 @@
+//===---------------------------- CNodes.swift ----------------------------===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2021 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+@_implementationOnly import _CSwiftSyntax
+
+// These two types need to be exposed publicly, so they can't rely on the C
+// types defined in _CSwiftSyntax.
+public typealias CSyntaxKind = UInt16
+public typealias CClientNode = UnsafeMutableRawPointer
+
+typealias CSyntaxNode = swiftparse_syntax_node_t
+typealias CSyntaxNodePtr = UnsafePointer<CSyntaxNode>
+typealias CTokenKind = swiftparse_token_kind_t
+typealias CTriviaKind = swiftparse_trivia_kind_t
+typealias CTokenData = swiftparse_token_data_t
+typealias CLayoutData = swiftparse_layout_data_t
+typealias CTriviaPiecePtr = UnsafePointer<CTriviaPiece>
+typealias CTriviaPiece = swiftparse_trivia_piece_t
+
+/// Computes a hash value that describes the layout of all C nodes which are
+/// passed as opaque values between `SwiftSyntaxParser` and `SwiftSyntax`.
+/// This should match the value returned by the `cNodeLayoutHash` function in
+/// the `SwiftSyntaxParser` module.
+public func cNodeLayoutHash() -> Int {
+  var hasher = Hasher()
+
+  // These two types are not defined in terms of the C types in SwiftSyntax.
+  // Let's include them specifically in the hash as well.
+  hasher.combine(MemoryLayout<CSyntaxKind>.size)
+  hasher.combine(MemoryLayout<CClientNode>.size)
+
+  hasher.combine(MemoryLayout<swiftparse_range_t>.size)
+  hasher.combine(MemoryLayout<swiftparse_range_t>.offset(of: \.offset))
+  hasher.combine(MemoryLayout<swiftparse_range_t>.offset(of: \.length))
+
+  hasher.combine(MemoryLayout<swiftparse_trivia_kind_t>.size)
+  hasher.combine(MemoryLayout<swiftparse_token_kind_t>.size)
+  hasher.combine(MemoryLayout<swiftparse_syntax_kind_t>.size)
+
+  hasher.combine(MemoryLayout<swiftparse_client_node_t>.size)
+
+  hasher.combine(MemoryLayout<swiftparse_trivia_piece_t>.size)
+  hasher.combine(MemoryLayout<swiftparse_trivia_piece_t>.offset(of: \.length))
+  hasher.combine(MemoryLayout<swiftparse_trivia_piece_t>.offset(of: \.kind))
+
+  hasher.combine(MemoryLayout<swiftparse_token_data_t>.size)
+  hasher.combine(MemoryLayout<swiftparse_token_data_t>.offset(of: \.leading_trivia))
+  hasher.combine(MemoryLayout<swiftparse_token_data_t>.offset(of: \.trailing_trivia))
+  hasher.combine(MemoryLayout<swiftparse_token_data_t>.offset(of: \.leading_trivia_count))
+  hasher.combine(MemoryLayout<swiftparse_token_data_t>.offset(of: \.trailing_trivia_count))
+  hasher.combine(MemoryLayout<swiftparse_token_data_t>.offset(of: \.kind))
+  hasher.combine(MemoryLayout<swiftparse_token_data_t>.offset(of: \.range))
+
+  hasher.combine(MemoryLayout<swiftparse_layout_data_t>.size)
+  hasher.combine(MemoryLayout<swiftparse_layout_data_t>.offset(of: \.nodes))
+  hasher.combine(MemoryLayout<swiftparse_layout_data_t>.offset(of: \.nodes_count))
+
+  hasher.combine(MemoryLayout<swiftparse_syntax_node_t>.size)
+  hasher.combine(MemoryLayout<swiftparse_syntax_node_t>.offset(of: \.kind))
+  hasher.combine(MemoryLayout<swiftparse_syntax_node_t>.offset(of: \.present))
+
+  return hasher.finalize()
+}

Sources/SwiftSyntax/IncrementalParseTransition.swift

@@ -199,11 +199,11 @@ public struct ConcurrentEdits {
 
 /// Provides a mechanism for the parser to skip regions of an incrementally
 /// updated source that was already parsed during a previous parse invocation.
-internal struct IncrementalParseLookup {
+public struct IncrementalParseLookup {
   fileprivate let transition: IncrementalParseTransition
   fileprivate var cursor: SyntaxCursor
 
-  init(transition: IncrementalParseTransition) {
+  public init(transition: IncrementalParseTransition) {
     self.transition = transition
     self.cursor = .init(root: transition.previousTree.data.absoluteRaw)
   }
@@ -224,16 +224,16 @@ internal struct IncrementalParseLookup {
   ///
   /// - Parameters:
   ///   - offset: The byte offset of the source string that is currently parsed.
-  ///   - kind: The `SyntaxKind` that the parser expects at this position.
+  ///   - kind: The `CSyntaxKind` that the parser expects at this position.
   /// - Returns: A `SyntaxNode` node from the previous parse invocation,
   ///            representing the contents of this region, if it is still valid
   ///            to re-use. `nil` otherwise.
-  mutating func lookUp(_ newOffset: Int, kind: SyntaxKind) -> SyntaxNode? {
+  public mutating func lookUp(_ newOffset: Int, kind: CSyntaxKind) -> SyntaxNode? {
     guard let prevOffset = translateToPreEditOffset(newOffset) else {
       return nil
     }
     let prevPosition = AbsolutePosition(utf8Offset: prevOffset)
-    let node = cursorLookup(prevPosition: prevPosition, kind: kind)
+    let node = cursorLookup(prevPosition: prevPosition, kind: .fromRawValue(kind))
     if let delegate = reusedDelegate, let node = node {
       delegate.parserReusedNode(
         range: ByteSourceRange(offset: newOffset, length: node.byteSize),

Sources/SwiftSyntax/Misc.swift.gyb

@@ -19,8 +19,6 @@
 //
 //===----------------------------------------------------------------------===//
 
-@_implementationOnly import _InternalSwiftSyntaxParser
-
 extension SyntaxNode {
   public var isUnknown: Bool { return raw.kind.isUnknown }
   public var asUnknown: UnknownSyntax? {
@@ -63,10 +61,3 @@ extension Syntax {
     }
   }
 }
-
-extension SyntaxParser {
-  static func verifyNodeDeclarationHash() -> Bool {
-    return String(cString: swiftparse_syntax_structure_versioning_identifier()!) ==
-      "${calculate_node_hash()}"
-  }
-}

Sources/SwiftSyntax/RawSyntax.swift

@@ -1216,12 +1216,12 @@ final class RawSyntax: ManagedBuffer<RawSyntaxBase, RawSyntaxDataElement> {
 }
 
 extension RawSyntax {
+  static func moveFromOpaque(_ cn: CClientNode) -> RawSyntax {
+    return Unmanaged<RawSyntax>.fromOpaque(cn).takeRetainedValue()
+  }
+
   static func moveFromOpaque(_ cn: CClientNode?) -> RawSyntax? {
-    if let subnode = cn {
-      return Unmanaged<RawSyntax>.fromOpaque(subnode).takeRetainedValue()
-    } else {
-      return nil
-    }
+    return cn.map(moveFromOpaque)
   }
 
   static func getFromOpaque(_ cn: CClientNode?) -> RawSyntax? {

Sources/SwiftSyntax/_SyntaxParserInterop.swift

@@ -0,0 +1,54 @@
+//===--------------------- _SyntaxParserInterop.swift ---------------------===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2021 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+/// Namespace for methods needed by the `SwiftSyntaxParser` module to
+/// efficiently create `SwiftSyntax` nodes from the C nodes created by the
+/// parser.
+public enum _SyntaxParserInterop {
+  private static func getRetainedOpaque(rawSyntax: RawSyntax) -> CClientNode {
+    return Unmanaged.passRetained(rawSyntax).toOpaque()
+  }
+
+  /// Create a `RawSyntax` node for the given `cnode` and return an opaque
+  /// pointer to the `RawSyntax` node (a `CClientNode`).
+  /// After this method finishes, the `RawSyntax` node has a retain count of 1
+  /// and is owned by whoever manages the returned `CClientNode`. Passing the
+  /// `CClientNode` to `nodeFromRetainedOpaqueRawSyntax` transfers ownership
+  /// back to `SwiftSyntax`.
+  public static func getRetainedOpaqueRawSyntax(
+    cnode: UnsafeRawPointer, source: String
+  ) -> CClientNode {
+    let cnode = cnode.assumingMemoryBound(to: CSyntaxNode.self)
+    // Transfer ownership of the object to the C parser. We get ownership back
+    // via `moveFromCRawNode()`.
+    let node = RawSyntax.create(from: cnode, source: source)
+    return getRetainedOpaque(rawSyntax: node)
+  }
+
+  /// Return an opaque pointer to the given `node`.
+  /// After this method finishes, the `RawSyntax` node has a retain count of 1
+  /// and is owned by whoever manages the returned `CClientNode`. Passing the
+  /// `CClientNode` to `nodeFromRetainedOpaqueRawSyntax` transfers ownership
+  /// back to `SwiftSyntax`.
+  public static func getRetainedOpaqueRawSyntax(node: SyntaxNode)
+      -> CClientNode {
+    return getRetainedOpaque(rawSyntax: node.raw)
+  }
+
+  /// After an opaque pointer to a `RawSyntax` node has been created using one
+  /// of the methods above, transfer its ownership back to a `Syntax` node,
+  /// which is managed by `SwiftSyntax`.
+  public static func nodeFromRetainedOpaqueRawSyntax(_ cRoot: CClientNode)
+      -> Syntax {
+    return Syntax(SyntaxData.forRoot(RawSyntax.moveFromOpaque(cRoot)))
+  }
+}

Sources/SwiftSyntax/gyb_generated/Misc.swift

@@ -12,8 +12,6 @@
 //
 //===----------------------------------------------------------------------===//
 
-@_implementationOnly import _InternalSwiftSyntaxParser
-
 extension SyntaxNode {
   public var isUnknown: Bool { return raw.kind.isUnknown }
   public var asUnknown: UnknownSyntax? {
@@ -1964,10 +1962,3 @@ extension Syntax {
     }
   }
 }
-
-extension SyntaxParser {
-  static func verifyNodeDeclarationHash() -> Bool {
-    return String(cString: swiftparse_syntax_structure_versioning_identifier()!) ==
-      "e9565bceebb81b9c3a69c442a8576b029d7eaf9c"
-  }
-}

Sources/SwiftSyntaxParser/CNodes.swift

@@ -0,0 +1,73 @@
+//===---------------------------- CNodes.swift ----------------------------===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2021 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+@_implementationOnly import _InternalSwiftSyntaxParser
+
+typealias CSyntaxNode = swiftparse_syntax_node_t
+typealias CTriviaPiece = swiftparse_trivia_piece_t
+typealias CSyntaxNodePtr = UnsafePointer<CSyntaxNode>
+typealias CTriviaPiecePtr = UnsafePointer<CTriviaPiece>
+typealias CSyntaxKind = swiftparse_syntax_kind_t
+typealias CTokenKind = swiftparse_token_kind_t
+typealias CTriviaKind = swiftparse_trivia_kind_t
+typealias CTokenData = swiftparse_token_data_t
+typealias CLayoutData = swiftparse_layout_data_t
+typealias CParseLookupResult = swiftparse_lookup_result_t
+typealias CClientNode = swiftparse_client_node_t
+typealias CDiagnostic = swiftparser_diagnostic_t
+typealias CFixit = swiftparse_diagnostic_fixit_t
+typealias CRange = swiftparse_range_t
+
+/// Computes a hash value that describes the layout of all C nodes which are
+/// passed as opaque values between `SwiftSyntaxParser` and `SwiftSyntax`.
+/// This should match the value returned by the `cNodeLayoutHash` function in
+/// the `SwiftSyntax` module.
+public func cNodeLayoutHash() -> Int {
+  var hasher = Hasher()
+
+  // These two types are not defined in terms of the C types in SwiftSyntax.
+  // Let's include them specifically in the hash as well.
+  hasher.combine(MemoryLayout<CSyntaxKind>.size)
+  hasher.combine(MemoryLayout<CClientNode>.size)
+
+  hasher.combine(MemoryLayout<swiftparse_range_t>.size)
+  hasher.combine(MemoryLayout<swiftparse_range_t>.offset(of: \.offset))
+  hasher.combine(MemoryLayout<swiftparse_range_t>.offset(of: \.length))
+
+  hasher.combine(MemoryLayout<swiftparse_trivia_kind_t>.size)
+  hasher.combine(MemoryLayout<swiftparse_token_kind_t>.size)
+  hasher.combine(MemoryLayout<swiftparse_syntax_kind_t>.size)
+
+  hasher.combine(MemoryLayout<swiftparse_client_node_t>.size)
+
+  hasher.combine(MemoryLayout<swiftparse_trivia_piece_t>.size)
+  hasher.combine(MemoryLayout<swiftparse_trivia_piece_t>.offset(of: \.length))
+  hasher.combine(MemoryLayout<swiftparse_trivia_piece_t>.offset(of: \.kind))
+
+  hasher.combine(MemoryLayout<swiftparse_token_data_t>.size)
+  hasher.combine(MemoryLayout<swiftparse_token_data_t>.offset(of: \.leading_trivia))
+  hasher.combine(MemoryLayout<swiftparse_token_data_t>.offset(of: \.trailing_trivia))
+  hasher.combine(MemoryLayout<swiftparse_token_data_t>.offset(of: \.leading_trivia_count))
+  hasher.combine(MemoryLayout<swiftparse_token_data_t>.offset(of: \.trailing_trivia_count))
+  hasher.combine(MemoryLayout<swiftparse_token_data_t>.offset(of: \.kind))
+  hasher.combine(MemoryLayout<swiftparse_token_data_t>.offset(of: \.range))
+
+  hasher.combine(MemoryLayout<swiftparse_layout_data_t>.size)
+  hasher.combine(MemoryLayout<swiftparse_layout_data_t>.offset(of: \.nodes))
+  hasher.combine(MemoryLayout<swiftparse_layout_data_t>.offset(of: \.nodes_count))
+
+  hasher.combine(MemoryLayout<swiftparse_syntax_node_t>.size)
+  hasher.combine(MemoryLayout<swiftparse_syntax_node_t>.offset(of: \.kind))
+  hasher.combine(MemoryLayout<swiftparse_syntax_node_t>.offset(of: \.present))
+
+  return hasher.finalize()
+}

Sources/SwiftSyntax/Diagnostic.swift → Sources/SwiftSyntaxParser/Diagnostic.swift

@@ -12,6 +12,8 @@
 // This file provides the Diagnostic, Note, and FixIt types.
 //===----------------------------------------------------------------------===//
 
+import SwiftSyntax
+
 /// A FixIt represents a change to source code in order to "correct" a
 /// diagnostic.
 public enum FixIt: Codable, CustomDebugStringConvertible {

Sources/SwiftSyntax/DiagnosticEngine.swift → Sources/SwiftSyntaxParser/DiagnosticEngine.swift

@@ -14,6 +14,7 @@
 //===----------------------------------------------------------------------===//
 
 import Foundation
+import SwiftSyntax
 
 /// The DiagnosticEngine allows Swift tools to emit diagnostics.
 public class DiagnosticEngine {

Sources/SwiftSyntax/JSONDiagnosticConsumer.swift → Sources/SwiftSyntaxParser/JSONDiagnosticConsumer.swift

@@ -14,6 +14,7 @@
 //===----------------------------------------------------------------------===//
 
 import Foundation
+import SwiftSyntax
 
 public final class JSONDiagnosticConsumer: DiagnosticConsumer {
   /// Enumerates the possible places this consumer might output diagnostics.

Sources/SwiftSyntaxParser/NodeDeclarationHash.swift.gyb

@@ -0,0 +1,27 @@
+%{
+  # -*- mode: Swift -*-
+  from gyb_syntax_support import calculate_node_hash
+  # Ignore the following admonition it applies to the resulting .swift file only
+}%
+//// Automatically Generated From NodeDeclarationHash.swift.gyb.
+//// Do Not Edit Directly!
+//===--------------------- NodeDeclarationHash.swift ----------------------===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2021 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+@_implementationOnly import _InternalSwiftSyntaxParser
+
+extension SyntaxParser {
+  static func verifyNodeDeclarationHash() -> Bool {
+    return String(cString: swiftparse_syntax_structure_versioning_identifier()!) ==
+      "${calculate_node_hash()}"
+  }
+}