Merge main into release/6.0

.github/ISSUE_TEMPLATE/BUG_REPORT.yml

@@ -18,8 +18,8 @@ body:
     id: editor
     attributes:
       label: Editor
-      description: Which text editor are you using?
-      placeholder: Eg. Visual Studio Code with Swift plugin 1.9.0, neovim
+      description: Which text editor are you using (and LSP extension/plugin if applicable)?
+      placeholder: Eg. Visual Studio Code with Swift extension 1.9.0, Neovim
   - type: dropdown
     id: reproduces-with-swift-6
     attributes:
@@ -50,4 +50,4 @@ body:
       label: Logging
       description: |
         If you are using SourceKit-LSP from Swift 6, running `sourcekit-lsp diagnose` in terminal and attaching the generated bundle helps us diagnose the issue.
-        The generated bundle might contain portions of your source code, so please only attach it if you feel comfortable sharing it.
+        The generated bundle may contain paths to files on disk as well as portions of your source code. This greatly helps in reproducing issues, but you should only attach it if you feel comfortable doing so.

.gitignore

@@ -2,6 +2,7 @@
 default.profraw
 Package.resolved
 /.build
+/.index-build
 /Packages
 /*.xcodeproj
 /*.sublime-project

Package.swift

@@ -173,6 +173,8 @@ let package = Package(
     .target(
       name: "SemanticIndex",
       dependencies: [
+        "CAtomics",
+        "LanguageServerProtocol",
         "LSPLogging",
         "SKCore",
         .product(name: "IndexStoreDB", package: "indexstore-db"),

Sources/Diagnose/CMakeLists.txt

@@ -1,4 +1,5 @@
 add_library(Diagnose STATIC
+  CommandConfiguration+Sendable.swift
   CommandLineArgumentsReducer.swift
   DiagnoseCommand.swift
   MergeSwiftFiles.swift
@@ -11,9 +12,10 @@ add_library(Diagnose STATIC
   ReproducerBundle.swift
   RequestInfo.swift
   SourceKitD+RunWithYaml.swift
+  SourcekitdRequestCommand.swift
   SourceKitDRequestExecutor.swift
   SourceReducer.swift
-  SourcekitdRequestCommand.swift
+  StderrStreamConcurrencySafe.swift
   SwiftFrontendCrashScraper.swift
   Toolchain+SwiftFrontend.swift)
 

Sources/Diagnose/CommandConfiguration+Sendable.swift

@@ -0,0 +1,21 @@
+//===----------------------------------------------------------------------===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2024 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+import ArgumentParser
+
+// If `CommandConfiguration` is not sendable, commands can't have static `configuration` properties.
+// Needed until we update Swift CI to swift-argument-parser 1.3.1, which has this conformance (rdar://128042447).
+#if compiler(<5.11)
+extension CommandConfiguration: @unchecked Sendable {}
+#else
+extension CommandConfiguration: @unchecked @retroactive Sendable {}
+#endif

Sources/Diagnose/CommandLineArgumentsReducer.swift

@@ -16,6 +16,7 @@ import LSPLogging
 // MARK: - Entry point
 
 extension RequestInfo {
+  @MainActor
   func reduceCommandLineArguments(
     using executor: SourceKitRequestExecutor,
     progressUpdate: (_ progress: Double, _ message: String) -> Void
@@ -49,6 +50,7 @@ fileprivate class CommandLineArgumentReducer {
     self.progressUpdate = progressUpdate
   }
 
+  @MainActor
   func run(initialRequestInfo: RequestInfo) async throws -> RequestInfo {
     var requestInfo = initialRequestInfo
     requestInfo = try await reduce(initialRequestInfo: requestInfo, simultaneousRemove: 10)
@@ -113,6 +115,7 @@ fileprivate class CommandLineArgumentReducer {
     return requestInfo
   }
 
+  @MainActor
   private func tryRemoving(
     _ argumentsToRemove: ClosedRange<Int>,
     from requestInfo: RequestInfo

Sources/Diagnose/DiagnoseCommand.swift

@@ -16,13 +16,13 @@ import SKCore
 
 import struct TSCBasic.AbsolutePath
 import class TSCBasic.Process
-import var TSCBasic.stderrStream
 import class TSCUtility.PercentProgressAnimation
 
 /// When diagnosis is started, a progress bar displayed on the terminal that shows how far the diagnose command has
 /// progressed.
 /// Can't be a member of `DiagnoseCommand` because then `DiagnoseCommand` is no longer codable, which it needs to be
 /// to be a `AsyncParsableCommand`.
+@MainActor
 private var progressBar: PercentProgressAnimation? = nil
 
 /// A component of the diagnostic bundle that's collected in independent stages.
@@ -35,7 +35,7 @@ fileprivate enum BundleComponent: String, CaseIterable, ExpressibleByArgument {
 }
 
 public struct DiagnoseCommand: AsyncParsableCommand {
-  public static var configuration: CommandConfiguration = CommandConfiguration(
+  public static let configuration: CommandConfiguration = CommandConfiguration(
     commandName: "diagnose",
     abstract: "Creates a bundle containing information that help diagnose issues with sourcekit-lsp"
   )
@@ -72,6 +72,7 @@ public struct DiagnoseCommand: AsyncParsableCommand {
     }
   }
 
+  @MainActor
   var toolchain: Toolchain? {
     get async throws {
       if let toolchainOverride {
@@ -96,6 +97,7 @@ public struct DiagnoseCommand: AsyncParsableCommand {
 
   public init() {}
 
+  @MainActor
   private func addSourcekitdCrashReproducer(toBundle bundlePath: URL) async throws {
     reportProgress(.reproducingSourcekitdCrash(progress: 0), message: "Trying to reduce recent sourcekitd crashes")
     for (name, requestInfo) in try requestInfos() {
@@ -121,6 +123,7 @@ public struct DiagnoseCommand: AsyncParsableCommand {
     }
   }
 
+  @MainActor
   private func addSwiftFrontendCrashReproducer(toBundle bundlePath: URL) async throws {
     reportProgress(
       .reproducingSwiftFrontendCrash(progress: 0),
@@ -182,14 +185,16 @@ public struct DiagnoseCommand: AsyncParsableCommand {
   }
 
   /// Execute body and if it throws, log the error.
-  private func orPrintError(_ body: () async throws -> Void) async {
+  @MainActor
+  private func orPrintError(_ body: @MainActor () async throws -> Void) async {
     do {
       try await body()
     } catch {
       print(error)
     }
   }
 
+  @MainActor
   private func addOsLog(toBundle bundlePath: URL) async throws {
     #if os(macOS)
     reportProgress(.collectingLogMessages(progress: 0), message: "Collecting log messages")
@@ -227,6 +232,7 @@ public struct DiagnoseCommand: AsyncParsableCommand {
     #endif
   }
 
+  @MainActor
   private func addCrashLogs(toBundle bundlePath: URL) throws {
     #if os(macOS)
     reportProgress(.collectingCrashReports, message: "Collecting crash reports")
@@ -252,6 +258,7 @@ public struct DiagnoseCommand: AsyncParsableCommand {
     #endif
   }
 
+  @MainActor
   private func addSwiftVersion(toBundle bundlePath: URL) async throws {
     let outputFileUrl = bundlePath.appendingPathComponent("swift-versions.txt")
     FileManager.default.createFile(atPath: outputFileUrl.path, contents: nil)
@@ -283,10 +290,12 @@ public struct DiagnoseCommand: AsyncParsableCommand {
     }
   }
 
+  @MainActor
   private func reportProgress(_ state: DiagnoseProgressState, message: String) {
     progressBar?.update(step: Int(state.progress * 100), total: 100, text: message)
   }
 
+  @MainActor
   public func run() async throws {
     print(
       """
@@ -303,7 +312,10 @@ public struct DiagnoseCommand: AsyncParsableCommand {
       """
     )
 
-    progressBar = PercentProgressAnimation(stream: stderrStream, header: "Diagnosing sourcekit-lsp issues")
+    progressBar = PercentProgressAnimation(
+      stream: stderrStreamConcurrencySafe,
+      header: "Diagnosing sourcekit-lsp issues"
+    )
 
     let dateFormatter = ISO8601DateFormatter()
     dateFormatter.timeZone = NSTimeZone.local
@@ -342,6 +354,7 @@ public struct DiagnoseCommand: AsyncParsableCommand {
 
   }
 
+  @MainActor
   private func reduce(
     requestInfo: RequestInfo,
     toolchain: Toolchain?,

Sources/Diagnose/MergeSwiftFiles.swift

@@ -17,6 +17,7 @@ extension RequestInfo {
   /// Check if the issue reproduces when merging all `.swift` input files into a single file.
   ///
   /// Returns `nil` if the issue didn't reproduce with all `.swift` files merged.
+  @MainActor
   func mergeSwiftFiles(
     using executor: SourceKitRequestExecutor,
     progressUpdate: (_ progress: Double, _ message: String) -> Void

Sources/Diagnose/ReduceCommand.swift

@@ -20,7 +20,7 @@ import var TSCBasic.stderrStream
 import class TSCUtility.PercentProgressAnimation
 
 public struct ReduceCommand: AsyncParsableCommand {
-  public static var configuration: CommandConfiguration = CommandConfiguration(
+  public static let configuration: CommandConfiguration = CommandConfiguration(
     commandName: "reduce",
     abstract: "Reduce a single sourcekitd crash",
     shouldDisplay: false
@@ -56,6 +56,7 @@ public struct ReduceCommand: AsyncParsableCommand {
   private var nsPredicate: NSPredicate? { nil }
   #endif
 
+  @MainActor
   var toolchain: Toolchain? {
     get async throws {
       if let toolchainOverride {
@@ -68,6 +69,7 @@ public struct ReduceCommand: AsyncParsableCommand {
 
   public init() {}
 
+  @MainActor
   public func run() async throws {
     guard let sourcekitd = try await toolchain?.sourcekitd else {
       throw ReductionError("Unable to find sourcekitd.framework")

Sources/Diagnose/ReduceFrontendCommand.swift

@@ -20,7 +20,7 @@ import var TSCBasic.stderrStream
 import class TSCUtility.PercentProgressAnimation
 
 public struct ReduceFrontendCommand: AsyncParsableCommand {
-  public static var configuration: CommandConfiguration = CommandConfiguration(
+  public static let configuration: CommandConfiguration = CommandConfiguration(
     commandName: "reduce-frontend",
     abstract: "Reduce a single swift-frontend crash",
     shouldDisplay: false
@@ -64,6 +64,7 @@ public struct ReduceFrontendCommand: AsyncParsableCommand {
   )
   var frontendArgs: [String]
 
+  @MainActor
   var toolchain: Toolchain? {
     get async throws {
       if let toolchainOverride {
@@ -76,6 +77,7 @@ public struct ReduceFrontendCommand: AsyncParsableCommand {
 
   public init() {}
 
+  @MainActor
   public func run() async throws {
     guard let sourcekitd = try await toolchain?.sourcekitd else {
       throw ReductionError("Unable to find sourcekitd.framework")
@@ -84,7 +86,10 @@ public struct ReduceFrontendCommand: AsyncParsableCommand {
       throw ReductionError("Unable to find swift-frontend")
     }
 
-    let progressBar = PercentProgressAnimation(stream: stderrStream, header: "Reducing swift-frontend crash")
+    let progressBar = PercentProgressAnimation(
+      stream: stderrStream,
+      header: "Reducing swift-frontend crash"
+    )
 
     let executor = OutOfProcessSourceKitRequestExecutor(
       sourcekitd: sourcekitd.asURL,

Sources/Diagnose/ReduceSourceKitDRequest.swift

@@ -12,6 +12,7 @@
 
 extension RequestInfo {
   /// Reduce the input file of this request and the command line arguments.
+  @MainActor
   func reduce(
     using executor: SourceKitRequestExecutor,
     progressUpdate: (_ progress: Double, _ message: String) -> Void

Sources/Diagnose/ReduceSwiftFrontend.swift

@@ -10,6 +10,7 @@
 //
 //===----------------------------------------------------------------------===//
 
+@MainActor
 @_spi(Testing)
 public func reduceFrontendIssue(
   frontendArgs: [String],

Sources/Diagnose/RequestInfo.swift

@@ -15,7 +15,7 @@ import RegexBuilder
 
 /// All the information necessary to replay a sourcektid request.
 @_spi(Testing)
-public struct RequestInfo {
+public struct RequestInfo: Sendable {
   /// The JSON request object. Contains the following dynamic placeholders:
   ///  - `$OFFSET`: To be replaced by `offset` before running the request
   ///  - `$FILE`: Will be replaced with a path to the file that contains the reduced source code.
@@ -51,7 +51,7 @@ public struct RequestInfo {
   }
 
   /// A fake value that is used to indicate that we are reducing a `swift-frontend` issue instead of a sourcekitd issue.
-  static var fakeRequestTemplateForFrontendIssues = """
+  static let fakeRequestTemplateForFrontendIssues = """
     {
       key.request: sourcekit-lsp-fake-request-for-frontend-crash
       key.compilerargs: [

Sources/Diagnose/SourceKitDRequestExecutor.swift

@@ -19,7 +19,7 @@ import struct TSCBasic.ProcessResult
 
 /// The different states in which a sourcekitd request can finish.
 @_spi(Testing)
-public enum SourceKitDRequestResult {
+public enum SourceKitDRequestResult: Sendable {
   /// The request succeeded.
   case success(response: String)
 
@@ -46,11 +46,12 @@ fileprivate extension String {
 /// An executor that can run a sourcekitd request and indicate whether the request reprodes a specified issue.
 @_spi(Testing)
 public protocol SourceKitRequestExecutor {
-  func runSourceKitD(request: RequestInfo) async throws -> SourceKitDRequestResult
-  func runSwiftFrontend(request: RequestInfo) async throws -> SourceKitDRequestResult
+  @MainActor func runSourceKitD(request: RequestInfo) async throws -> SourceKitDRequestResult
+  @MainActor func runSwiftFrontend(request: RequestInfo) async throws -> SourceKitDRequestResult
 }
 
 extension SourceKitRequestExecutor {
+  @MainActor
   func run(request: RequestInfo) async throws -> SourceKitDRequestResult {
     if request.requestTemplate == RequestInfo.fakeRequestTemplateForFrontendIssues {
       return try await runSwiftFrontend(request: request)

Sources/Diagnose/SourceReducer.swift

@@ -21,6 +21,7 @@ import SwiftSyntax
 
 extension RequestInfo {
   @_spi(Testing)
+  @MainActor
   public func reduceInputFile(
     using executor: SourceKitRequestExecutor,
     progressUpdate: (_ progress: Double, _ message: String) -> Void
@@ -61,6 +62,7 @@ fileprivate enum ReductionStepResult {
 }
 
 /// Reduces an input source file while continuing to reproduce the crash
+@MainActor
 fileprivate class SourceReducer {
   /// The executor that is used to run a sourcekitd request and check whether it
   /// still crashes.
@@ -84,6 +86,7 @@ fileprivate class SourceReducer {
   }
 
   /// Reduce the file contents in `initialRequest` to a smaller file that still reproduces a crash.
+  @MainActor
   func run(initialRequestInfo: RequestInfo) async throws -> RequestInfo {
     var requestInfo = initialRequestInfo
     self.initialImportCount = Parser.parse(source: requestInfo.fileContents).numberOfImports
@@ -242,6 +245,7 @@ fileprivate class SourceReducer {
   ///
   /// If the request still crashes after applying the edits computed by `reduce`, return the reduced request info.
   /// Otherwise, return `nil`
+  @MainActor
   private func runReductionStep(
     requestInfo: RequestInfo,
     reportProgress: Bool = true,
@@ -608,6 +612,7 @@ fileprivate class FirstImportFinder: SyntaxAnyVisitor {
 /// the file that imports the module. If `areFallbackArgs` is set, we have synthesized fallback arguments that only
 /// contain a target and SDK. This is useful when reducing a swift-frontend crash because sourcekitd requires driver
 /// arguments but the swift-frontend crash has frontend args.
+@MainActor
 fileprivate func getSwiftInterface(
   _ moduleName: String,
   executor: SourceKitRequestExecutor,
@@ -680,6 +685,7 @@ fileprivate func getSwiftInterface(
   return try JSONDecoder().decode(String.self, from: sanitizedData)
 }
 
+@MainActor
 fileprivate func inlineFirstImport(
   in tree: SourceFileSyntax,
   executor: SourceKitRequestExecutor,

Sources/Diagnose/SourcekitdRequestCommand.swift

@@ -18,7 +18,7 @@ import SourceKitD
 import struct TSCBasic.AbsolutePath
 
 public struct SourceKitdRequestCommand: AsyncParsableCommand {
-  public static var configuration = CommandConfiguration(
+  public static let configuration = CommandConfiguration(
     commandName: "run-sourcekitd-request",
     abstract: "Run a sourcekitd request and print its result",
     shouldDisplay: false