Audit the public API of the SwiftSyntax module

CodeGeneration/Sources/SyntaxSupport/Classification.swift

@@ -28,16 +28,15 @@ public class SyntaxClassification {
 public class ChildClassification {
   public let parent: Node
   public let childIndex: Int
-  public let isToken: Bool
-  public let classification: SyntaxClassification?
-  public let force: Bool
+  public let child: Child
+  public var isToken: Bool { child.isToken }
+  public var classification: SyntaxClassification? { child.classification }
+  public var force: Bool { child.forceClassification }
 
   public init(node: Node, childIndex: Int, child: Child) {
     self.parent = node
     self.childIndex = childIndex
-    self.isToken = child.syntaxKind.hasSuffix("Token")
-    self.classification = child.classification
-    self.force = child.forceClassification
+    self.child = child
   }
 }
 

CodeGeneration/Sources/generate-swiftsyntax/GenerateSwiftSyntax.swift

@@ -22,6 +22,7 @@ private let generatedDirName = "generated"
 private let swiftBasicFormatGeneratedDir = ["SwiftBasicFormat", generatedDirName]
 private let ideUtilsGeneratedDir = ["IDEUtils", generatedDirName]
 private let swiftParserGeneratedDir = ["SwiftParser", generatedDirName]
+private let swiftParserDiagnosticsGeneratedDir = ["SwiftParserDiagnostics", generatedDirName]
 private let swiftSyntaxGeneratedDir = ["SwiftSyntax", generatedDirName]
 private let swiftSyntaxBuilderGeneratedDir = ["SwiftSyntaxBuilder", generatedDirName]
 private let BASE_KIND_FILES = [
@@ -86,13 +87,18 @@ struct GenerateSwiftSyntax: ParsableCommand {
 
         // SwiftParser
         GeneratedFileSpec(swiftParserGeneratedDir + ["DeclarationModifier.swift"], declarationModifierFile),
+        GeneratedFileSpec(swiftParserGeneratedDir + ["IsLexerClassified.swift"], isLexerClassifiedFile),
         GeneratedFileSpec(swiftParserGeneratedDir + ["Parser+Entry.swift"], parserEntryFile),
         GeneratedFileSpec(swiftParserGeneratedDir + ["TokenSpecStaticMembers.swift"], tokenSpecStaticMembersFile),
         GeneratedFileSpec(swiftParserGeneratedDir + ["TypeAttribute.swift"], typeAttributeFile),
 
+        // SwiftParserDiagnostics
+        GeneratedFileSpec(swiftParserDiagnosticsGeneratedDir + ["ChildNameForDiagnostics.swift"], childNameForDiagnosticFile),
+        GeneratedFileSpec(swiftParserDiagnosticsGeneratedDir + ["SyntaxKindNameForDiagnostics.swift"], syntaxKindNameForDiagnosticFile),
+        GeneratedFileSpec(swiftParserDiagnosticsGeneratedDir + ["TokenNameForDiagnostics.swift"], tokenNameForDiagnosticFile),
+
         // SwiftSyntax
         GeneratedFileSpec(swiftSyntaxGeneratedDir + ["Keyword.swift"], keywordFile),
-        GeneratedFileSpec(swiftSyntaxGeneratedDir + ["Misc.swift"], miscFile),
         GeneratedFileSpec(swiftSyntaxGeneratedDir + ["raw", "RawSyntaxNodes.swift"], rawSyntaxNodesFile),
         GeneratedFileSpec(swiftSyntaxGeneratedDir + ["raw", "RawSyntaxValidation.swift"], rawSyntaxValidationFile),
         GeneratedFileSpec(swiftSyntaxGeneratedDir + ["SyntaxAnyVisitor.swift"], syntaxAnyVisitorFile),

CodeGeneration/Sources/generate-swiftsyntax/templates/basicformat/BasicFormatFile.swift

@@ -62,19 +62,19 @@ let basicFormatFile = SourceFileSyntax(leadingTrivia: copyrightHeader) {
     DeclSyntax(
       """
       open override func visitPre(_ node: Syntax) {
-        if let keyPath = getKeyPath(node), shouldIndent(keyPath) {
+        if let keyPath = node.keyPathInParent, shouldIndent(keyPath) {
           indentationLevel += 1
         }
         if let parent = node.parent, childrenSeparatedByNewline(parent) {
-          putNextTokenOnNewLine = true && node.previousToken != nil
+          putNextTokenOnNewLine = true && node.previousToken(viewMode: .sourceAccurate) != nil
         }
       }
       """
     )
     DeclSyntax(
       """
       open override func visitPost(_ node: Syntax) {
-        if let keyPath = getKeyPath(node), shouldIndent(keyPath) {
+        if let keyPath = node.keyPathInParent, shouldIndent(keyPath) {
           indentationLevel -= 1
         }
       }
@@ -92,7 +92,7 @@ let basicFormatFile = SourceFileSyntax(leadingTrivia: copyrightHeader) {
         if requiresTrailingSpace(node) && trailingTrivia.isEmpty {
           trailingTrivia += .space
         }
-        if let keyPath = getKeyPath(Syntax(node)), requiresLeadingNewline(keyPath), !(leadingTrivia.first?.isNewline ?? false), !shouldOmitNewline(node) {
+        if let keyPath = node.keyPathInParent, requiresLeadingNewline(keyPath), !(leadingTrivia.first?.isNewline ?? false), !shouldOmitNewline(node) {
           leadingTrivia = .newline + leadingTrivia
         }
         var isOnNewline: Bool = (lastRewrittenToken?.trailingTrivia.pieces.last?.isNewline == true)
@@ -206,7 +206,7 @@ let basicFormatFile = SourceFileSyntax(leadingTrivia: copyrightHeader) {
     try FunctionDeclSyntax("open func requiresLeadingSpace(_ token: TokenSyntax) -> Bool") {
       StmtSyntax(
         """
-        if let keyPath = getKeyPath(token), let requiresLeadingSpace = requiresLeadingSpace(keyPath) {
+        if let keyPath = token.keyPathInParent, let requiresLeadingSpace = requiresLeadingSpace(keyPath) {
           return requiresLeadingSpace
         }
         """
@@ -268,7 +268,7 @@ let basicFormatFile = SourceFileSyntax(leadingTrivia: copyrightHeader) {
     try FunctionDeclSyntax("open func requiresTrailingSpace(_ token: TokenSyntax) -> Bool") {
       StmtSyntax(
         """
-        if let keyPath = getKeyPath(token), let requiresTrailingSpace = requiresTrailingSpace(keyPath) {
+        if let keyPath = token.keyPathInParent, let requiresTrailingSpace = requiresTrailingSpace(keyPath) {
           return requiresTrailingSpace
         }
         """
@@ -311,19 +311,5 @@ let basicFormatFile = SourceFileSyntax(leadingTrivia: copyrightHeader) {
         }
       }
     }
-
-    DeclSyntax(
-      """
-      private func getKeyPath<T: SyntaxProtocol>(_ node: T) -> AnyKeyPath? {
-        guard let parent = node.parent else {
-          return nil
-        }
-        guard case .layout(let childrenKeyPaths) = parent.kind.syntaxNodeType.structure else {
-          return nil
-        }
-        return childrenKeyPaths[node.indexInParent]
-      }
-      """
-    )
   }
 }

CodeGeneration/Sources/generate-swiftsyntax/templates/ideutils/SyntaxClassificationFile.swift

@@ -53,35 +53,17 @@ let syntaxClassificationFile = SourceFileSyntax(leadingTrivia: copyrightHeader)
       ///   - childKind: The node syntax kind.
       /// - Returns: A pair of classification and whether it is "forced", or nil if
       ///   no classification is attached.
-      internal static func classify(
-          parentKind: SyntaxKind, indexInParent: Int, childKind: SyntaxKind
-        ) -> (SyntaxClassification, Bool)?
+      internal static func classify(_ keyPath: AnyKeyPath) -> (SyntaxClassification, Bool)?
       """
     ) {
-      try IfExprSyntax(
-        """
-        // Separate checks for token nodes (most common checks) versus checks for layout nodes.
-        if childKind == .token
-        """
-      ) {
-        try SwitchExprSyntax("switch (parentKind, indexInParent)") {
-          for childClassification in node_child_classifications where childClassification.isToken {
-            SwitchCaseSyntax("case (.\(raw: childClassification.parent.swiftSyntaxKind), \(raw: childClassification.childIndex)):") {
-              StmtSyntax("return (.\(raw: childClassification.classification!.swiftName), \(raw: childClassification.force))")
-            }
+      try SwitchExprSyntax("switch keyPath") {
+        for childClassification in node_child_classifications {
+          SwitchCaseSyntax("case \\\(raw: childClassification.parent.type.syntaxBaseName).\(raw: childClassification.child.swiftName):") {
+            StmtSyntax("return (.\(raw: childClassification.classification!.swiftName), \(raw: childClassification.force))")
           }
-
-          SwitchCaseSyntax("default: return nil")
         }
-      } else: {
-        try SwitchExprSyntax("switch (parentKind, indexInParent)") {
-          for childClassification in node_child_classifications where !childClassification.isToken {
-            SwitchCaseSyntax("case (.\(raw: childClassification.parent.swiftSyntaxKind), \(raw: childClassification.childIndex)):") {
-              StmtSyntax("return (.\(raw: childClassification.classification!.swiftName), \(raw: childClassification.force))")
-            }
-          }
-
-          SwitchCaseSyntax("default: return nil")
+        SwitchCaseSyntax("default:") {
+          StmtSyntax("return nil")
         }
       }
     }

CodeGeneration/Sources/generate-swiftsyntax/templates/swiftparser/IsLexerClassifiedFile.swift

@@ -0,0 +1,77 @@
+//===----------------------------------------------------------------------===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2023 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+import SwiftSyntax
+import SwiftSyntaxBuilder
+import SyntaxSupport
+import Utils
+
+let isLexerClassifiedFile = SourceFileSyntax(leadingTrivia: copyrightHeader) {
+  DeclSyntax("import SwiftSyntax")
+
+  try! ExtensionDeclSyntax(
+    """
+    extension Keyword
+    """
+  ) {
+    try! VariableDeclSyntax(
+      """
+      /// Whether the token kind is switched from being an identifier to being a keyword in the lexer.
+      /// This is true for keywords that used to be considered non-contextual.
+      var isLexerClassified: Bool
+      """
+    ) {
+      try! SwitchExprSyntax("switch self") {
+        for keyword in KEYWORDS {
+          if keyword.isLexerClassified {
+            SwitchCaseSyntax("case .\(raw: keyword.escapedName): return true")
+          }
+        }
+        SwitchCaseSyntax("default: return false")
+      }
+    }
+  }
+
+  try! ExtensionDeclSyntax(
+    """
+    extension TokenKind
+    """
+  ) {
+    try! VariableDeclSyntax(
+      """
+      /// Returns `true` if the token is a Swift keyword.
+      ///
+      /// Keywords are reserved unconditionally for use by Swift and may not
+      /// appear as identifiers in any position without being escaped. For example,
+      /// `class`, `func`, or `import`.
+      public var isLexerClassifiedKeyword: Bool
+      """
+    ) {
+      try! SwitchExprSyntax("switch self") {
+        SwitchCaseSyntax("case .eof:") {
+          StmtSyntax("return false")
+        }
+
+        for token in SYNTAX_TOKENS where token.isKeyword {
+          SwitchCaseSyntax("case .\(raw: token.swiftKind):") {
+            StmtSyntax("return true")
+          }
+        }
+
+        SwitchCaseSyntax("case .keyword(let keyword):") {
+          StmtSyntax("return keyword.isLexerClassified")
+        }
+        SwitchCaseSyntax("default: return false")
+      }
+    }
+  }
+}

CodeGeneration/Sources/generate-swiftsyntax/templates/swiftparser/TokenSpecStaticMembersFile.swift

@@ -16,7 +16,7 @@ import SyntaxSupport
 import Utils
 
 let tokenSpecStaticMembersFile = SourceFileSyntax(leadingTrivia: copyrightHeader) {
-  DeclSyntax("import SwiftSyntax")
+  DeclSyntax("@_spi(RawSyntax) import SwiftSyntax")
 
   try! ExtensionDeclSyntax("extension TokenSpec") {
     DeclSyntax("static var eof: TokenSpec { return TokenSpec(.eof) }")

CodeGeneration/Sources/generate-swiftsyntax/templates/swiftparserdiagnostics/ChildNameForDiagnosticsFile.swift

@@ -0,0 +1,56 @@
+//===----------------------------------------------------------------------===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2023 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+import SwiftSyntax
+import SwiftSyntaxBuilder
+import SyntaxSupport
+import Utils
+
+let childNameForDiagnosticFile = SourceFileSyntax(leadingTrivia: copyrightHeader) {
+  DeclSyntax("import SwiftSyntax")
+
+  try! FunctionDeclSyntax(
+    "private func childNameForDiagnostics(_ keyPath: AnyKeyPath) -> String?"
+  ) {
+    try! SwitchExprSyntax("switch keyPath") {
+      for node in NON_BASE_SYNTAX_NODES where !node.isSyntaxCollection {
+        for child in node.children {
+          if let nameForDiagnostics = child.nameForDiagnostics {
+            SwitchCaseSyntax("case \\\(raw: node.type.syntaxBaseName).\(raw: child.swiftName):") {
+              StmtSyntax(#"return "\#(raw: nameForDiagnostics)""#)
+            }
+          }
+        }
+      }
+      SwitchCaseSyntax(
+        """
+        default:
+          return nil
+        """
+      )
+    }
+  }
+
+  DeclSyntax(
+    """
+    extension SyntaxProtocol {
+      var childNameInParent: String? {
+        guard let keyPath = self.keyPathInParent else {
+          return nil
+        }
+        return childNameForDiagnostics(keyPath)
+      }
+    }
+    """
+  )
+
+}

CodeGeneration/Sources/generate-swiftsyntax/templates/swiftparserdiagnostics/SyntaxKindNameForDiagnosticsFile.swift

@@ -0,0 +1,44 @@
+//===----------------------------------------------------------------------===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2023 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+import SwiftSyntax
+import SwiftSyntaxBuilder
+import SyntaxSupport
+import Utils
+
+let syntaxKindNameForDiagnosticFile = SourceFileSyntax(leadingTrivia: copyrightHeader) {
+  DeclSyntax("import SwiftSyntax")
+
+  try! ExtensionDeclSyntax("extension SyntaxKind") {
+    try VariableDeclSyntax("var nameForDiagnostics: String?") {
+      try SwitchExprSyntax("switch self") {
+        SwitchCaseSyntax("case .token:") {
+          StmtSyntax(#"return "token""#)
+        }
+
+        for node in NON_BASE_SYNTAX_NODES {
+          if let nameForDiagnostics = node.nameForDiagnostics {
+            SwitchCaseSyntax("case .\(raw: node.swiftSyntaxKind):") {
+              StmtSyntax("return \"\(raw: nameForDiagnostics)\"")
+            }
+          }
+        }
+        SwitchCaseSyntax(
+          """
+          default:
+            return nil
+          """
+        )
+      }
+    }
+  }
+}

CodeGeneration/Sources/generate-swiftsyntax/templates/swiftparserdiagnostics/TokenNameForDiagnosticsFile.swift

@@ -0,0 +1,39 @@
+//===----------------------------------------------------------------------===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2023 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+import SwiftSyntax
+import SwiftSyntaxBuilder
+import SyntaxSupport
+import Utils
+
+let tokenNameForDiagnosticFile = SourceFileSyntax(leadingTrivia: copyrightHeader) {
+  DeclSyntax("@_spi(RawSyntax) import SwiftSyntax")
+
+  try! ExtensionDeclSyntax("extension TokenKind") {
+    try! VariableDeclSyntax("var nameForDiagnostics: String") {
+      try! SwitchExprSyntax("switch self") {
+        SwitchCaseSyntax("case .eof:") {
+          StmtSyntax(#"return "end of file""#)
+        }
+
+        for token in SYNTAX_TOKENS where token.swiftKind != "keyword" {
+          SwitchCaseSyntax("case .\(raw: token.swiftKind):") {
+            StmtSyntax("return #\"\(raw: token.nameForDiagnostics)\"#")
+          }
+        }
+        SwitchCaseSyntax("case .keyword(let keyword):") {
+          StmtSyntax("return String(syntaxText: keyword.defaultText)")
+        }
+      }
+    }
+  }
+}