Add infrastructure for test projects using IndexStoreDB's "Tibs"

Package.swift

@@ -29,7 +29,7 @@ let package = Package(
 
       .target(
         name: "SKTestSupport",
-        dependencies: ["SourceKit"]),
+        dependencies: ["SourceKit", "ISDBTestSupport", "tibs"]),
       .testTarget(
         name: "SourceKitTests",
         dependencies: ["SourceKit", "SKTestSupport"]),

Sources/LanguageServerProtocol/DefinitionRequest.swift

@@ -31,4 +31,9 @@ public struct DefinitionRequest: TextDocumentRequest, Hashable {
 
   /// The document location at which to lookup symbol information.
   public var position: Position
+
+  public init(textDocument: TextDocumentIdentifier, position: Position) {
+    self.textDocument = textDocument
+    self.position = position
+  }
 }

Sources/LanguageServerProtocol/ReferencesRequest.swift

@@ -35,4 +35,10 @@ public struct ReferencesRequest: RequestType, Hashable {
 
   /// Whether to include the declaration in the list of symbols, or just the references.
   public var includeDeclaration: Bool?
+
+  public init(textDocument: TextDocumentIdentifier, position: Position, includeDeclaration: Bool? = nil) {
+    self.textDocument = textDocument
+    self.position = position
+    self.includeDeclaration = includeDeclaration
+  }
 }

Sources/SKTestSupport/SKTibsTestWorkspace.swift

@@ -0,0 +1,154 @@
+//===----------------------------------------------------------------------===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2019 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+import SourceKit
+import LanguageServerProtocol
+import SKCore
+import IndexStoreDB
+import ISDBTibs
+import ISDBTestSupport
+import Basic
+import SPMUtility
+import XCTest
+import Foundation
+
+public typealias URL = Foundation.URL
+
+public final class SKTibsTestWorkspace {
+
+  public let tibsWorkspace: TibsTestWorkspace
+  public let testServer = TestSourceKitServer(connectionKind: .local)
+
+  public var index: IndexStoreDB { tibsWorkspace.index }
+  public var builder: TibsBuilder { tibsWorkspace.builder }
+  public var sources: TestSources { tibsWorkspace.sources }
+  public var sk: TestClient { testServer.client }
+
+  public init(
+    immutableProjectDir: URL,
+    persistentBuildDir: URL,
+    tmpDir: URL,
+    toolchain: Toolchain) throws
+  {
+    self.tibsWorkspace = try TibsTestWorkspace(
+      immutableProjectDir: immutableProjectDir,
+      persistentBuildDir: persistentBuildDir,
+      tmpDir: tmpDir,
+      toolchain: TibsToolchain(toolchain))
+
+    sk.allowUnexpectedNotification = true
+    initWorkspace()
+  }
+
+  public init(projectDir: URL, tmpDir: URL, toolchain: Toolchain) throws {
+    self.tibsWorkspace = try TibsTestWorkspace(
+      projectDir: projectDir,
+      tmpDir: tmpDir,
+      toolchain: TibsToolchain(toolchain))
+
+    sk.allowUnexpectedNotification = true
+    initWorkspace()
+  }
+
+  func initWorkspace() {
+    let buildPath = AbsolutePath(builder.buildRoot.path)
+    testServer.server!.workspace = Workspace(
+      rootPath: AbsolutePath(sources.rootDirectory.path),
+      clientCapabilities: ClientCapabilities(),
+      buildSettings: CompilationDatabaseBuildSystem(projectRoot: buildPath),
+      index: index,
+      buildSetup: BuildSetup(configuration: .debug, path: buildPath, flags: BuildFlags()))
+  }
+}
+
+extension SKTibsTestWorkspace {
+
+  public func testLoc(_ name: String) -> TestLocation { sources.locations[name]! }
+
+  public func buildAndIndex() throws {
+    try tibsWorkspace.buildAndIndex()
+  }
+}
+
+extension SKTibsTestWorkspace {
+  public func openDocument(_ url: URL, language: Language) throws {
+    sk.send(DidOpenTextDocument(textDocument: TextDocumentItem(
+      url: url,
+      language: language,
+      version: 1,
+      text: try sources.sourceCache.get(url))))
+  }
+}
+
+extension XCTestCase {
+
+  public func staticSourceKitTibsWorkspace(name: String, testFile: String = #file) throws -> SKTibsTestWorkspace? {
+    let testDirName = testDirectoryName
+    let workspace = try SKTibsTestWorkspace(
+      immutableProjectDir: inputsDirectory(testFile: testFile)
+        .appendingPathComponent(name, isDirectory: true),
+      persistentBuildDir: XCTestCase.productsDirectory
+        .appendingPathComponent("sk-tests/\(testDirName)", isDirectory: true),
+      tmpDir: URL(fileURLWithPath: NSTemporaryDirectory(), isDirectory: true)
+        .appendingPathComponent("sk-test-data/\(testDirName)", isDirectory: true),
+      toolchain: ToolchainRegistry.shared.default!)
+
+    if workspace.builder.targets.contains(where: { target in !target.clangTUs.isEmpty })
+      && !workspace.builder.toolchain.clangHasIndexSupport {
+      fputs("warning: skipping test because '\(workspace.builder.toolchain.clang.path)' does not " +
+            "have indexstore support; use swift-clang\n", stderr)
+      return nil
+    }
+
+    return workspace
+  }
+}
+
+extension TestLocation {
+  public var position: Position {
+    Position(self)
+  }
+}
+
+extension Position {
+  public init(_ loc: TestLocation) {
+    // FIXME: utf16 vfs utf8 column
+    self.init(line: loc.line - 1, utf16index: loc.column - 1)
+  }
+}
+
+extension Location {
+  public init(_ loc: TestLocation) {
+    self.init(url: loc.url, range: Range(Position(loc)))
+  }
+}
+
+extension TibsToolchain {
+  public convenience init(_ sktc: Toolchain) {
+    let ninja: URL?
+    if let ninjaPath = ProcessInfo.processInfo.environment["NINJA_BIN"] {
+      ninja = URL(fileURLWithPath: ninjaPath, isDirectory: false)
+    } else {
+      ninja = findTool(name: "ninja")
+    }
+    self.init(
+      swiftc: sktc.swiftc!.asURL,
+      clang: sktc.clang!.asURL,
+      libIndexStore: sktc.libIndexStore!.asURL,
+      tibs: XCTestCase.productsDirectory.appendingPathComponent("tibs", isDirectory: false),
+      ninja: ninja)
+  }
+}
+
+extension TestLocation {
+  public var docIdentifier: TextDocumentIdentifier { TextDocumentIdentifier(url) }
+}

Sources/SourceKit/SourceKitServer.swift

@@ -40,7 +40,7 @@ public final class SourceKitServer: LanguageServer {
 
   var languageService: [LanguageServiceKey: Connection] = [:]
 
-  var workspace: Workspace?
+  public var workspace: Workspace?
 
   let fs: FileSystem
 

Tests/INPUTS/CodeCompleteSingleModule/CodeCompleteSingleModuleA.swift

@@ -0,0 +1,3 @@
+struct A {
+  func method(a b: Int) {}
+}

Tests/INPUTS/CodeCompleteSingleModule/CodeCompleteSingleModuleB.swift

@@ -0,0 +1,3 @@
+func test(a: A) {
+  a . /*cc:A*/
+}

Tests/INPUTS/CodeCompleteSingleModule/project.json

@@ -0,0 +1 @@
+{ "sources": ["CodeCompleteSingleModuleA.swift", "CodeCompleteSingleModuleB.swift"] }

Tests/INPUTS/SwiftModules/a.swift

@@ -0,0 +1 @@
+public func /*aaa:def*/aaa() {}

Tests/INPUTS/SwiftModules/b.swift

@@ -0,0 +1,4 @@
+import A
+public func /*bbb:def*/bbb() {
+  /*aaa:call*/aaa()
+}

Tests/INPUTS/SwiftModules/c.swift

@@ -0,0 +1,6 @@
+import A
+import B
+public func ccc() {
+  /*aaa:call:c*/aaa()
+  /*bbb:call*/bbb()
+}

Tests/INPUTS/SwiftModules/project.json

@@ -0,0 +1,7 @@
+{
+  "targets": [
+    { "name": "A", "sources": ["a.swift"] },
+    { "name": "B", "sources": ["b.swift"], "dependencies": ["A"] },
+    { "name": "C", "sources": ["c.swift"], "dependencies": ["B"] },
+  ]
+}

Tests/INPUTS/proj1/a.swift

@@ -0,0 +1,4 @@
+func /*a:def*/a() {
+  /*b:call*/b()
+  /*c:call*/c()
+}

Tests/INPUTS/proj1/b.swift

@@ -0,0 +1,3 @@
+func /*b:def*/b() {
+  /*a:call*/a()
+}

Tests/INPUTS/proj1/project.json

@@ -0,0 +1 @@
+{"sources": ["a.swift", "b.swift", "rec/c.swift"]}

Tests/INPUTS/proj1/rec/c.swift

@@ -0,0 +1,2 @@
+func /*c*/c() {
+}

Tests/SourceKitTests/SourceKitTests.swift

@@ -12,9 +12,17 @@
 
 @testable import SourceKit
 import LanguageServerProtocol
+import Basic
+import SPMUtility
+import SKCore
 import SKTestSupport
+import IndexStoreDB
+import ISDBTibs
+import ISDBTestSupport
 import XCTest
 
+public typealias URL = Foundation.URL
+
 final class SKTests: XCTestCase {
 
     func testInitLocal() {
@@ -52,4 +60,69 @@ final class SKTests: XCTestCase {
       XCTAssertEqual(initResult.capabilities.textDocumentSync?.openClose, true)
       XCTAssertNotNil(initResult.capabilities.completionProvider)
     }
+
+  func testIndexSwiftModules() throws {
+    guard let ws = try staticSourceKitTibsWorkspace(name: "SwiftModules") else { return }
+    try ws.buildAndIndex()
+
+    let locDef = ws.testLoc("aaa:def")
+    let locRef = ws.testLoc("aaa:call:c")
+
+    try ws.openDocument(locDef.url, language: .swift)
+    try ws.openDocument(locRef.url, language: .swift)
+
+    // MARK: Jump to definition
+
+    let jump = try ws.sk.sendSync(DefinitionRequest(
+      textDocument: locRef.docIdentifier,
+      position: locRef.position))
+
+    XCTAssertEqual(jump.count, 1)
+    XCTAssertEqual(jump.first?.url, locDef.url)
+    XCTAssertEqual(jump.first?.range.lowerBound, locDef.position)
+
+    // MARK: Find references
+
+    let refs = try ws.sk.sendSync(ReferencesRequest(
+      textDocument: locDef.docIdentifier,
+      position: locDef.position))
+
+    XCTAssertEqual(Set(refs), [
+      Location(locDef),
+      Location(locRef),
+      Location(ws.testLoc("aaa:call")),
+    ])
+  }
+
+  func testCodeCompleteSwiftTibs() throws {
+    guard let ws = try staticSourceKitTibsWorkspace(name: "CodeCompleteSingleModule") else { return }
+    let loc = ws.testLoc("cc:A")
+    try ws.openDocument(loc.url, language: .swift)
+
+    let results = try ws.sk.sendSync(
+      CompletionRequest(textDocument: loc.docIdentifier, position: loc.position))
+
+    XCTAssertEqual(results, CompletionList(isIncomplete: false, items: [
+      CompletionItem(
+        label: "method(a: Int)",
+        detail: "Void",
+        sortText: nil,
+        filterText: "method(a:)",
+        textEdit: nil,
+        insertText: "method(a: )",
+        insertTextFormat: .plain,
+        kind: .method,
+        deprecated: nil),
+      CompletionItem(
+        label: "self",
+        detail: "A",
+        sortText: nil,
+        filterText: "self",
+        textEdit: nil,
+        insertText: "self",
+        insertTextFormat: .plain,
+        kind: .keyword,
+        deprecated: nil),
+    ]))
+  }
 }

Tests/SourceKitTests/XCTestManifests.swift

@@ -80,6 +80,8 @@ extension SKTests {
     //   `swift test --generate-linuxmain`
     // to regenerate.
     static let __allTests__SKTests = [
+        ("testCodeCompleteSwiftTibs", testCodeCompleteSwiftTibs),
+        ("testIndexSwiftModules", testIndexSwiftModules),
         ("testInitJSON", testInitJSON),
         ("testInitLocal", testInitLocal),
     ]

Utilities/build-script-helper.py

@@ -5,6 +5,7 @@
 import argparse
 import os
 import platform
+import shutil
 import subprocess
 import sys
 
@@ -13,6 +14,11 @@ def swiftpm(action, swift_exec, swiftpm_args, env=None):
   print(' '.join(cmd))
   subprocess.check_call(cmd, env=env)
 
+def swiftpm_bin_path(swift_exec, swiftpm_args, env=None):
+  cmd = [swift_exec, 'build', '--show-bin-path'] + swiftpm_args
+  print(' '.join(cmd))
+  return subprocess.check_output(cmd, env=env).strip()
+
 def get_swiftpm_options(args):
   swiftpm_args = [
     '--package-path', args.package_path,
@@ -79,6 +85,10 @@ def add_common_args(parser):
   if args.action == 'build':
     swiftpm('build', swift_exec, swiftpm_args, env)
   elif args.action == 'test':
+    bin_path = swiftpm_bin_path(swift_exec, swiftpm_args, env)
+    tests = os.path.join(bin_path, 'sk-tests')
+    print('Cleaning ' + tests)
+    shutil.rmtree(tests, ignore_errors=True)
     swiftpm('test', swift_exec, swiftpm_args, env)
   else:
     assert False, 'unknown action \'{}\''.format(args.action)