Add infrastructure for testing IndexStoreDB

Package.swift

@@ -11,6 +11,12 @@ let package = Package(
     .library(
       name: "IndexStoreDB_CXX",
       targets: ["IndexStoreDB_Index"]),
+    .library(
+      name: "ISDBTestSupport",
+      targets: ["ISDBTestSupport"]),
+    .executable(
+      name: "tibs",
+      targets: ["tibs"])
   ],
   dependencies: [],
   targets: [
@@ -23,7 +29,28 @@ let package = Package(
 
     .testTarget(
       name: "IndexStoreDBTests",
-      dependencies: ["IndexStoreDB"]),
+      dependencies: ["IndexStoreDB", "ISDBTestSupport"]),
+
+    // MARK: Swift Test Infrastructure
+
+    // The Test Index Build System (tibs) library.
+    .target(
+      name: "ISDBTibs",
+      dependencies: []),
+
+    .testTarget(
+      name: "ISDBTibsTests",
+      dependencies: ["ISDBTibs"]),
+
+    // Commandline tool for working with tibs projects.
+    .target(
+      name: "tibs",
+      dependencies: ["ISDBTibs"]),
+
+    // Test support library, built on top of tibs.
+    .target(
+      name: "ISDBTestSupport",
+      dependencies: ["IndexStoreDB", "ISDBTibs"]),
 
     // MARK: C++ interface
 

Sources/ISDBTestSupport/SourceFileCache.swift

@@ -0,0 +1,48 @@
+//===----------------------------------------------------------------------===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2019 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+import Foundation
+
+/// Reads and caches file contents by URL.
+///
+/// Use `cache.get(url)` to read a file, or get its cached contents. The contents can be overridden
+/// or removed from the cache by calling `cache.set(url, to: "new contents")`
+public final class SourceFileCache {
+  var cache: [URL: String] = [:]
+
+  public init(_ cache: [URL: String] = [:]) {
+    self.cache = cache
+  }
+
+  /// Read the contents of `file`, or retrieve them from the cache if available.
+  ///
+  /// * parameter file: The file to read.
+  /// * returns: The file contents as a String.
+  /// * throws: If there are any errors reading the file.
+  public func get(_ file: URL) throws -> String {
+    if let content = cache[file] {
+      return content
+    }
+    let content = try String(contentsOfFile: file.path, encoding: .utf8)
+    cache[file] = content
+    return content
+  }
+
+  /// Set the cached contents of `file` to `content`.
+  ///
+  /// * parameters
+  ///   * file: The file to read.
+  ///   * content: The new file content.
+  public func set(_ file: URL, to content: String?) {
+    cache[file] = content
+  }
+}

Sources/ISDBTestSupport/TestLocation.swift

@@ -0,0 +1,65 @@
+//===----------------------------------------------------------------------===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2019 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+import Foundation
+import IndexStoreDB
+
+/// A source location (file:line:column) in a test project, for use with the TestLocationScanner.
+public struct TestLocation: Hashable {
+
+  /// The path/url of the source file.
+  public var url: URL
+
+  /// The one-based line number.
+  public var line: Int
+
+  /// The one-based column number.
+  ///
+  /// FIXME: define utf8 vs. utf16 column index.
+  public var column: Int
+
+  public init(url: URL, line: Int, column: Int) {
+    self.url = url
+    self.line = line
+    self.column = column
+  }
+}
+
+extension TestLocation: Comparable {
+  public static func <(a: TestLocation, b: TestLocation) -> Bool {
+    return (a.url.path, a.line, a.column) < (b.url.path, b.line, b.column)
+  }
+}
+
+extension SymbolLocation {
+
+  /// Constructs a SymbolLocation from a TestLocation, using a non-system path by default.
+  public init(_ loc: TestLocation, isSystem: Bool = false) {
+    self.init(
+      path: loc.url.path,
+      isSystem: isSystem,
+      line: loc.line,
+      utf8Column: loc.column)
+  }
+}
+
+extension Symbol {
+
+  /// Returns a SymbolOccurrence with the given location and roles.
+  public func at(_ location: TestLocation, roles: SymbolRole) -> SymbolOccurrence {
+    return self.at(SymbolLocation(location), roles: roles)
+  }
+}
+
+extension TestLocation: CustomStringConvertible {
+  public var description: String { "\(url.path):\(line):\(column)" }
+}

Sources/ISDBTestSupport/TestLocationScanner.swift

@@ -0,0 +1,144 @@
+//===----------------------------------------------------------------------===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2019 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+import Foundation
+
+/// A builder object for scanning source files for TestLocations specified in /*inline comments*/.
+///
+/// The scanner searches source files for inline comments /*foo*/ and builds up a mapping from name
+/// (the contents of the inline comment) to the location in the source file that it was found.
+///
+/// For example:
+///
+/// ```
+/// var scanner = TestLocationScanner()
+/// scanner.scan("""
+///   func /*foo:def*/foo() {}
+///   """, url: myURL)
+/// scanner.result == ["foo:def": TestLocation(url: myURL, line: 1, column: 17)]
+/// ```
+public struct TestLocationScanner {
+
+  /// The result of the scan (so far), mapping name to test location.
+  public var result: [String: TestLocation] = [:]
+
+  public init() {}
+
+  public enum Error: Swift.Error {
+    case noSuchFileOrDirectory(URL)
+
+    /// The sources contained a `/*/*nested*/*/` inline comment, which is not supported.
+    case nestedComment(TestLocation)
+
+    /// The same test location name was used in multiple places.
+    case duplicateKey(String, TestLocation, TestLocation)
+  }
+
+  public mutating func scan(_ str: String, url: URL) throws {
+    if str.count < 4 {
+      return
+    }
+
+    enum State {
+      /// Outside any comment.
+      case normal(prev: Character)
+
+      /// Inside a comment. The payload contains the previous character and the index of the first
+      /// character after the '*' (i.e. the start of the comment body).
+      ///
+      ///       bodyStart
+      ///       |
+      ///     /*XXX*/
+      ///       ^^^
+      case comment(bodyStart: String.Index, prev: Character)
+    }
+
+    var state = State.normal(prev: "_")
+    var i = str.startIndex
+    var line = 1
+    var column = 1
+
+    while i != str.endIndex {
+      let c = str[i]
+
+      switch (state, c) {
+      case (.normal("/"), "*"):
+        state = .comment(bodyStart: str.index(after: i), prev: "_")
+      case (.normal(_), _):
+        state = .normal(prev: c)
+
+      case (.comment(let start, "*"), "/"):
+        let name = String(str[start..<str.index(before: i)])
+        let loc =  TestLocation(url: url, line: line, column: column + 1)
+        if let prevLoc = result.updateValue(loc, forKey: name) {
+          throw Error.duplicateKey(name, prevLoc, loc)
+        }
+        state = .normal(prev: "_")
+
+      case (.comment(_, "/"), "*"):
+        throw Error.nestedComment(TestLocation(url: url, line: line, column: column))
+
+      case (.comment(let start, _), _):
+        state = .comment(bodyStart: start, prev: c)
+      }
+
+      if c == "\n" {
+        line += 1
+        column = 1
+      } else {
+        column += 1
+      }
+
+      i = str.index(after: i)
+    }
+  }
+
+  public mutating func scan(file: URL, sourceCache: SourceFileCache) throws {
+    let content = try sourceCache.get(file)
+    try scan(content, url: file)
+  }
+
+  public mutating func scan(rootDirectory: URL, sourceCache: SourceFileCache) throws {
+    let fm = FileManager.default
+
+    guard let generator = fm.enumerator(at: rootDirectory, includingPropertiesForKeys: []) else {
+      throw Error.noSuchFileOrDirectory(rootDirectory)
+    }
+
+    while let url = generator.nextObject() as? URL {
+      if isSourceFileExtension(url.pathExtension) {
+        try scan(file: url, sourceCache: sourceCache)
+      }
+    }
+  }
+}
+
+/// Scans `rootDirectory` for test locations, returning a mapping from name to location.
+///
+/// See TestLocationScanner.
+public func scanLocations(
+  rootDirectory: URL,
+  sourceCache: SourceFileCache
+) throws -> [String: TestLocation] {
+  var scanner = TestLocationScanner()
+  try scanner.scan(rootDirectory: rootDirectory, sourceCache: sourceCache)
+  return scanner.result
+}
+
+func isSourceFileExtension(_ ext: String) -> Bool {
+  switch ext {
+    case "swift", "c", "cpp", "m", "mm", "h", "hpp":
+      return true
+    default:
+      return false
+  }
+}