[test] Split TestLocation column into explicit utf8/utf16 indices

Sources/ISDBTestSupport/TestLocation.swift

@@ -22,21 +22,23 @@ public struct TestLocation: Hashable {
   /// The one-based line number.
   public var line: Int
 
-  /// The one-based column number.
-  ///
-  /// FIXME: define utf8 vs. utf16 column index.
-  public var column: Int
+  /// The one-based UTF-8 column index.
+  public var utf8Column: Int
 
-  public init(url: URL, line: Int, column: Int) {
+  /// The one-based UTF-16 column index.
+  public var utf16Column: Int
+
+  public init(url: URL, line: Int, utf8Column: Int, utf16Column: Int) {
     self.url = url
     self.line = line
-    self.column = column
+    self.utf8Column = utf8Column
+    self.utf16Column = utf16Column
   }
 }
 
 extension TestLocation: Comparable {
   public static func <(a: TestLocation, b: TestLocation) -> Bool {
-    return (a.url.path, a.line, a.column) < (b.url.path, b.line, b.column)
+    return (a.url.path, a.line, a.utf8Column) < (b.url.path, b.line, b.utf8Column)
   }
 }
 
@@ -48,7 +50,7 @@ extension SymbolLocation {
       path: loc.url.path,
       isSystem: isSystem,
       line: loc.line,
-      utf8Column: loc.column)
+      utf8Column: loc.utf8Column)
   }
 }
 
@@ -61,5 +63,5 @@ extension Symbol {
 }
 
 extension TestLocation: CustomStringConvertible {
-  public var description: String { "\(url.path):\(line):\(column)" }
+  public var description: String { "\(url.path):\(line):\(utf8Column)" }
 }

Sources/ISDBTestSupport/TestLocationScanner.swift

@@ -75,7 +75,7 @@ public struct TestLocationScanner {
     var state = State.normal(prev: "_")
     var i = str.startIndex
     var line = 1
-    var column = 1
+    var lineStart = i
 
     while i != str.endIndex {
       let c = str[i]
@@ -87,39 +87,47 @@ public struct TestLocationScanner {
         state = .normal(prev: c)
 
       case (.comment(let start, "*"), "/"):
-        let name: String
-        let col: Int
+        let nameStart: String.Index
+        let locIndex: String.Index
         if str[start] == "<" {
-          // Location of the leading '/'.
-          name = String(str[str.index(after: start)..<str.index(before: i)])
-          col = column - str.distance(from: start, to: i) - 2
+          nameStart = str.index(after: start)
+          locIndex = str.index(start, offsetBy: -2) // subtract '/' and '*'
         } else {
-          // Location after the trailing '/'.
-          name = String(str[start..<str.index(before: i)])
-          col = column + 1
+          nameStart = start
+          locIndex = str.index(after: i) // after trailing '/'
         }
 
-        let loc =  TestLocation(url: url, line: line, column: col)
+        let name = String(str[nameStart..<str.index(before: i)])
+
+        let loc = TestLocation(
+          url: url,
+          line: line,
+          utf8Column: 1 + str.utf8.distance(from: lineStart, to: locIndex),
+          utf16Column: 1 + str.utf16.distance(from: lineStart, to: locIndex))
+
         if let prevLoc = result.updateValue(loc, forKey: name) {
           throw Error.duplicateKey(name, prevLoc, loc)
         }
+
         state = .normal(prev: "_")
 
       case (.comment(_, "/"), "*"):
-        throw Error.nestedComment(TestLocation(url: url, line: line, column: column))
+        throw Error.nestedComment(TestLocation(
+          url: url,
+          line: line,
+          utf8Column: 1 + str.utf8.distance(from: lineStart, to: i),
+          utf16Column: 1 + str.utf16.distance(from: lineStart, to: i)))
 
       case (.comment(let start, _), _):
         state = .comment(bodyStart: start, prev: c)
       }
 
+      i = str.index(after: i)
+
       if c == "\n" {
         line += 1
-        column = 1
-      } else {
-        column += 1
+        lineStart = i
       }
-
-      i = str.index(after: i)
     }
   }
 

Tests/IndexStoreDBTests/LocationScannerTests.swift

@@ -22,22 +22,37 @@ final class LocationScannerTests: XCTestCase {
     var url: URL
     var name: String
     var line: Int
-    var column: Int
-    init(url: URL = LocationScannerTests.magicURL, _ name: String, _ line: Int, _ column: Int) {
+    var utf8Column: Int
+    var utf16Column: Int
+
+    init(
+      url: URL = LocationScannerTests.magicURL,
+      _ name: String,
+      _ line: Int,
+      utf8Column: Int,
+      utf16Column: Int)
+    {
       self.url = url
       self.name = name
       self.line = line
-      self.column = column
+      self.utf8Column = utf8Column
+      self.utf16Column = utf16Column
+    }
+
+    init(url: URL = LocationScannerTests.magicURL, _ name: String, _ line: Int, _ column: Int) {
+      self.init(url: url, name, line, utf8Column: column, utf16Column: column)
     }
+
     init(_ name: String, _ loc: TestLocation) {
-      self.url = loc.url
-      self.name = name
-      self.line = loc.line
-      self.column = loc.column
+      self.init(
+        url: loc.url,
+        name, loc.line,
+        utf8Column: loc.utf8Column,
+        utf16Column: loc.utf16Column)
     }
     static func <(a: Loc, b: Loc) -> Bool {
-      return (a.url.absoluteString, a.line, a.column, a.name) <
-             (b.url.absoluteString, b.line, b.column, b.name)
+      return (a.url.absoluteString, a.line, a.utf8Column, a.name) <
+             (b.url.absoluteString, b.line, b.utf8Column, b.name)
     }
   }
 
@@ -153,4 +168,13 @@ final class LocationScannerTests: XCTestCase {
       Loc(url: proj1.appendingPathComponent("rec/c.swift", isDirectory: false), "c", 1, 11),
     ])
   }
+
+  func testUnicode() throws {
+    XCTAssertEqual(try scanString("😃/*a*/"), [Loc("a", 1, utf8Column: 10, utf16Column: 8)])
+    XCTAssertEqual(try scanString("😃/*<a*/"), [Loc("a", 1, utf8Column: 5, utf16Column: 3)])
+    XCTAssertEqual(try scanString("Ć/*a*/"), [Loc("a", 1, utf8Column: 8, utf16Column: 7)])
+    XCTAssertEqual(try scanString("Ć/*<a*/"), [Loc("a", 1, utf8Column: 3, utf16Column: 2)])
+    XCTAssertEqual(try scanString("👩‍👩‍👧‍👧/*a*/"), [Loc("a", 1, utf8Column: 31, utf16Column: 17)])
+    XCTAssertEqual(try scanString("👩‍👩‍👧‍👧/*<a*/"), [Loc("a", 1, utf8Column: 26, utf16Column: 12)])
+  }
 }

Tests/IndexStoreDBTests/XCTestManifests.swift

@@ -35,6 +35,7 @@ extension LocationScannerTests {
         ("testName", testName),
         ("testNested", testNested),
         ("testSmall", testSmall),
+        ("testUnicode", testUnicode),
     ]
 }