[WIP] Automatic Linux test manifest generation

.travis.yml

@@ -0,0 +1,18 @@
+os:
+  - linux
+  - osx
+language: generic
+sudo: required
+dist: trusty
+osx_image: xcode7.3
+env:
+  - SWIFT_VERSION=DEVELOPMENT-SNAPSHOT-2016-02-25-a
+install:
+  - eval "$(curl -sL https://gist.githubusercontent.com/kylef/5c0475ff02b7c7671d2a/raw/02090c7ede5a637b76e6df1710e83cd0bbe7dcdf/swiftenv-install.sh)"
+script:
+  - Utilities/bootstrap
+  - .build/bin/swift-test
+notifications:
+  email:
+    on_success: never
+    on_failure: change

Documentation/SourceLayouts.md

@@ -35,6 +35,6 @@ Where `foo` is an executable and `bar.a` a static library.
 
 ## Other Rules
 
-* Directories named `Tests` are ignored
-* Sub directories of a directory named `Sources`, `Source`, `srcs` or `src` become modules
+* Sub directories of directory named `Tests` become test-modules and are executed by `swift test`. `Tests` or any subdirectory can be excluded via Manifest file.
+* Sub directories of a directory named `Sources`, `Source`, `srcs` or `src` become modules.
 * It is acceptable to have no `Sources` directory, in which case the root directory is treated as a single module (place your sources there) or sub directories of the root are considered modules. Use this layout convention for simple projects.

Sources/Build/TestMetadata.swift

@@ -0,0 +1,206 @@
+/*
+ This source file is part of the Swift.org open source project
+
+ Copyright 2015 - 2016 Apple Inc. and the Swift project authors
+ Licensed under Apache License v2.0 with Runtime Library Exception
+
+ See http://swift.org/LICENSE.txt for license information
+ See http://swift.org/CONTRIBUTORS.txt for Swift project authors
+ */
+
+import PackageType
+import Utility
+import POSIX
+import func libc.fclose
+
+struct TestMetadata {
+
+    /// Map from product name -> XCTestMain absolute path
+    var productMainPaths: [String: String]
+
+    /// Map from module name -> XCTestManifest absolute path
+    var moduleManifestPaths: [String: String]
+}
+
+struct ProductTestMetadata {
+    let xctestMainPath: String
+    let product: Product
+    let metadata: [ModuleTestMetadata]
+}
+
+
+/// Generates a XCTestManifest.swift file for each module and XCTestMain.swift file for
+/// each product. Note that no files are generated for modules with no XCTestCase
+/// subclasses and no files are generated for packages with no XCTest-able modules.
+/// Returns TestMetadata, which informs of which files need to be added for 
+/// compilation into which module/product.
+func generateTestManifestFilesForProducts(products: [Product], prefix: String) throws -> TestMetadata {
+
+    //only work with test products
+    let testProducts = products.filter { $0.isTest }
+
+    //create our .build subfolder
+    let testManifestFolder = Path.join(prefix, "xctestmanifests")
+    try mkdir(testManifestFolder)
+
+    //TODO: what do we do when user deletes an XCTestCase file?
+    //we should somehow know to delete its manifest.
+    //here we could keep track of touched manifest files and delete the 
+    //untouched ones at the end.
+
+    //produce manifest file for each test module
+    let productsMetadata = try testProducts.flatMap { testProduct in
+        return try generateTestManifestFiles(testProduct, testManifestFolder: testManifestFolder)
+    }
+
+    //collect the paths of generated files
+    var productPaths = [String: String]()
+    var modulePaths = [String: String]()
+
+    for productMetadata in productsMetadata {
+        productPaths[productMetadata.product.name] = productMetadata.xctestMainPath
+        for moduleMetadata in productMetadata.metadata {
+            modulePaths[moduleMetadata.module.name] = moduleMetadata.testManifestPath
+        }
+    }
+
+    return TestMetadata(productMainPaths: productPaths, moduleManifestPaths: modulePaths)
+}
+
+/// Generates one XCTestManifest.swift per test module and one XCTestMain.swift
+/// for the whole product. All returned paths need to be added for compilation.
+func generateTestManifestFiles(product: Product, testManifestFolder: String) throws -> ProductTestMetadata? {
+
+    // Now parse and generate files for each test module.
+    // First parse each module's tests to get the list of classes
+    // which we'll use to generate the XCTestMain.swift file later.
+    let metadata = try product
+        .modules
+        .flatMap{ $0 as? TestModule }
+        .flatMap { try generateXCTestManifests($0, testManifestFolder: testManifestFolder) }
+
+    // Don't continue if no modules with testable classes were found
+    guard metadata.count > 0 else { return nil }
+
+    //TODO: Decide what to do when users already have
+    //the manual XCTestCaseProvider extension.
+    //Ignore that file? Error out and ask them to remove it once and for all?
+    //I'd prefer the second, because it'll get everyone on the same page quickly.
+
+    // With the test information, generate the XCTestMain file
+    let mainPath = try generateXCTestMain(product, metadata: metadata, testManifestFolder: testManifestFolder)
+
+    return ProductTestMetadata(xctestMainPath: mainPath, product: product, metadata: metadata)
+}
+
+/// Updates the contents of and returns the path of XCTestMain.swift file.
+func generateXCTestMain(product: Product, metadata: [ModuleTestMetadata], testManifestFolder: String) throws -> String {
+
+    let main = Path.join(testManifestFolder, "\(product.name)-XCTestMain.swift")
+
+    // Generate the XCTestMain.swift's contents.
+    try writeXCTestMain(metadata, path: main)
+
+    return main
+}
+
+/// Returns a list of class names that are subclasses of XCTestCase
+func generateXCTestManifests(module: TestModule, testManifestFolder: String) throws -> ModuleTestMetadata? {
+
+    let root = module.sources.root
+    let testManifestPath = Path.join(testManifestFolder, "\(module.name)-XCTestManifest.swift")
+
+    // Replace the String based parser with AST parser once it's ready
+    let parser: TestMetadataParser = StringTestMetadataParser()
+
+    let classes = try module
+        .sources
+        .relativePaths
+        .map { Path.join(root, $0) }
+        .flatMap { try parser.parseTestClasses($0) }
+        .flatMap { $0 }
+
+    guard classes.count > 0 else { return nil }
+
+    let metadata = ModuleTestMetadata(module: module, testManifestPath: testManifestPath, classes: classes)
+
+    //now generate the XCTestManifest.swift file
+    try writeXCTestManifest(metadata, path: testManifestPath)
+
+    return metadata
+}
+
+func writeXCTestManifest(metadata: ModuleTestMetadata, path: String) throws {
+
+    let file = try fopen(path, mode: .Write)
+    defer {
+        fclose(file)
+    }
+
+    //imports
+    try fputs("import XCTest\n", file)
+    try fputs("\n", file)
+
+    try fputs("#if os(Linux)\n", file)
+
+    //for each class
+    try metadata.classes.sort { $0.name < $1.name }.forEach { classMetadata in
+        try fputs("extension \(classMetadata.name): XCTestCaseProvider {\n", file)
+        try fputs("    var allTests : [(String, () throws -> Void)] {\n", file)
+        try fputs("        return [\n", file)
+
+        try classMetadata.testNames.sort().forEach {
+            try fputs("            (\"\($0)\", \($0)),\n", file)
+        }
+
+        try fputs("        ]\n", file)
+        try fputs("    }\n", file)
+        try fputs("}\n", file)
+    }
+
+    try fputs("#endif\n\n", file)
+}
+
+func writeXCTestMain(metadata: [ModuleTestMetadata], path: String) throws {
+
+    //don't write anything if no classes are available
+    guard metadata.count > 0 else { return }
+
+    let file = try fopen(path, mode: .Write)
+    defer {
+        fclose(file)
+    }
+
+    //imports
+    try fputs("import XCTest\n", file)
+    try metadata.flatMap { $0.module.importableName() }.sort().forEach {
+        try fputs("@testable import \($0)\n", file)
+    }
+    try fputs("\n", file)
+
+    try fputs("XCTMain([\n", file)
+
+    //for each class
+    for moduleMetadata in metadata {
+        try moduleMetadata
+            .classes
+            .sort { $0.name < $1.name }
+            .forEach { classMetadata in
+                try fputs("    \(moduleMetadata.module.importableName()).\(classMetadata.name)(),\n", file)
+        }
+    }
+
+    try fputs("])\n\n", file)
+}
+
+extension Module {
+
+    func importableName() -> String {
+        //module name e.g. "Build.test" needs to be imported as "Buildtest"
+        //so remove all occurences of '.'
+        let name = self.name.splitWithCharactersInString(".").joinWithSeparator("")
+        return name
+    }
+}
+
+

Sources/Build/TestMetadataParser.swift

@@ -0,0 +1,114 @@
+/*
+ This source file is part of the Swift.org open source project
+
+ Copyright 2015 - 2016 Apple Inc. and the Swift project authors
+ Licensed under Apache License v2.0 with Runtime Library Exception
+
+ See http://swift.org/LICENSE.txt for license information
+ See http://swift.org/CONTRIBUTORS.txt for Swift project authors
+ */
+
+import Utility
+import PackageType
+
+struct TestClassMetadata {
+    let name: String
+    let testNames: [String]
+}
+
+struct ModuleTestMetadata {
+    let module: TestModule
+    let testManifestPath: String
+    let classes: [TestClassMetadata]
+}
+
+protocol TestMetadataParser {
+    func parseTestClasses(testFilePath: String) throws -> [TestClassMetadata]?
+}
+
+//up for grabs
+//struct ASTTestMetadataParser: TestMetadataParser {
+//    
+//
+//}
+
+struct StringTestMetadataParser: TestMetadataParser {
+
+    /// Based on the absolute path passed in parses test class metadata
+    /// Returns array because there might be more test classes in one file
+    func parseTestClasses(testFilePath: String) throws -> [TestClassMetadata]? {
+
+        guard let lines = try? File(path: testFilePath).enumerate() else { return nil }
+
+        var classes = [TestClassMetadata]()
+        func finishClass(className: String?, _ testNames: [String]) {
+            if let className = className where testNames.count > 0 {
+                let testClass = TestClassMetadata(name: className, testNames: testNames)
+                classes.append(testClass)
+            }
+        }
+
+        //parse lines one by one, no need to load the whole into memory
+        var className: String? = nil
+        var testNames = [String]()
+        while true {
+
+            guard let line = lines.next() else {
+                //EOF, no more classes
+                finishClass(className, testNames)
+                testNames = []
+                return classes.nilIfEmpty()
+            }
+
+            //look for a class name
+            if let newClassName = parseClassName(line) {
+                //found class name, finish old class and start new
+                finishClass(className, testNames)
+                className = newClassName
+                testNames = []
+            }
+
+            //look for a test case
+            if let testName = parseTestName(line) {
+                //found another test name
+                testNames.append(testName)
+            }
+        }
+    }
+
+    func parseClassName(line: String) -> String? {
+        var comps = line.splitWithCharactersInString(":\r\t\n ").filter { !$0.isEmpty }
+        guard comps.count >= 2 else { return nil }
+        //see if this line is a candidate
+        guard Set(comps).contains("class") else { return nil }
+        while true {
+            let allowedFirst = Set(["private", "public", "internal", "final"])
+            if allowedFirst.contains(comps[0]) {
+                //drop first
+                comps = Array(comps.dropFirst())
+            } else {
+                break
+            }
+        }
+        guard comps[0] == "class" else { return nil }
+        guard comps[2] == "XCTestCase" else { return nil }
+        return comps[1]
+    }
+
+    func parseTestName(line: String) -> String? {
+        let comps = line.splitWithCharactersInString(" \t\n\r{").filter { !$0.isEmpty }
+        guard comps.count >= 2 else { return nil }
+        guard comps[0] == "func" else { return nil }
+        guard comps[1].hasPrefix("test") else { return nil }
+        guard comps[1].hasSuffix("()") else { return nil }
+        //remove trailing parentheses
+        return String(comps[1].characters.dropLast(2))
+    }
+}
+
+extension Array {
+    public func nilIfEmpty() -> Array? {
+        return self.isEmpty ? nil : self
+    }
+}
+