More swiftbuild testing (#8333)

Add more verifications of the swiftbuild build system to ensure
correctness moving forward. Tag various discovered swift-build
integration issues with the tag `SWBINTTODO` with a general description
of the problem encountered to enable categorization for further
exploration activities.

### Motivation:

The swiftbuild integration has only the most basic smoke testing of its
functionality. Add more automated tests, and explicit skips where there
are functionality gaps that need to be addressed.

---------

Co-authored-by: Sam Khouri <bkhouri@apple.com>

Author: cmcgee1024

IntegrationTests/Tests/IntegrationTests/SwiftPMTests.swift

@@ -57,11 +57,27 @@ final class SwiftPMTests: XCTestCase {
         #endif
 
         // Test SwiftBuildSystem
-        try withTemporaryDirectory { tmpDir in
-            let packagePath = tmpDir.appending(component: "foo")
-            try localFileSystem.createDirectory(packagePath)
-            try sh(swiftPackage, "--package-path", packagePath, "init", "--type", "executable")
-            try sh(swiftBuild, "--package-path", packagePath, "--build-system", "swiftbuild")
+        do {
+            try withTemporaryDirectory { tmpDir in
+                let packagePath = tmpDir.appending(component: "foo")
+                try localFileSystem.createDirectory(packagePath)
+                try sh(swiftPackage, "--package-path", packagePath, "init", "--type", "executable")
+                try sh(swiftBuild, "--package-path", packagePath, "--build-system", "swiftbuild")
+                // SWBINTTODO: Path issues related to swift run of the output from swiftbuild buildsystem
+                //let (stdout, stderr)  = try sh(swiftRun, "--package-path", packagePath, "--build-system", "swiftbuild")
+                //XCTAssertMatch(stdout, .contains("Hello, world!"))
+            }
+        }
+
+        do {
+            try withTemporaryDirectory { tmpDir in
+                let packagePath = tmpDir.appending(component: "foo")
+                try localFileSystem.createDirectory(packagePath)
+                try sh(swiftPackage, "--package-path", packagePath, "init", "--type", "library")
+                try sh(swiftBuild, "--package-path", packagePath, "--build-system", "swiftbuild")
+                // SWBINTTODO: Path issues related to swift test of the output from a swiftbuild buildsystem
+                //try sh(swiftTest, "--package-path", packagePath, "--build-system", "swiftbuild")
+            }
         }
     }
 

Sources/CoreCommands/SwiftCommandState.swift

@@ -1029,7 +1029,7 @@ extension BuildSystemProvider.Kind {
             return false
         }
     }
-    
+
     fileprivate var additionalFileRules: [FileRuleDescription] {
         switch self {
         case .xcode, .swiftbuild:

Sources/SwiftBuildSupport/SwiftBuildSystem.swift

@@ -256,16 +256,18 @@ public final class SwiftBuildSystem: SPMBuildCore.BuildSystem {
                 header: ""
             )
 
-        do {
-            try await withSession(service: service, name: self.buildParameters.pifManifest.pathString) { session, _ in
-                // Load the workspace, and set the system information to the default
-                do {
-                    try await session.loadWorkspace(containerPath: self.buildParameters.pifManifest.pathString)
-                    try await session.setSystemInfo(.default())
-                } catch {
-                    self.observabilityScope.emit(error: error.localizedDescription)
-                    throw error
-                }
+            do {
+                try await withSession(service: service, name: self.buildParameters.pifManifest.pathString) { session, _ in
+                    self.outputStream.send("Building for \(self.buildParameters.configuration == .debug ? "debugging" : "production")...\n")
+
+                    // Load the workspace, and set the system information to the default
+                    do {
+                        try await session.loadWorkspace(containerPath: self.buildParameters.pifManifest.pathString)
+                        try await session.setSystemInfo(.default())
+                    } catch {
+                        self.observabilityScope.emit(error: error.localizedDescription)
+                        throw error
+                    }
 
                     // Find the targets to build.
                     let configuredTargets: [SWBConfiguredTarget]

Sources/_InternalTestSupport/misc.swift

@@ -117,6 +117,10 @@ public func testWithTemporaryDirectory<Result>(
     }
 }
 
+public enum TestError: Error {
+    case platformNotSupported
+}
+
 @discardableResult public func fixture<T>(
     name: String,
     createGitRepo: Bool = true,
@@ -252,7 +256,7 @@ public func getBuildSystemArgs(for buildSystem: BuildSystemProvider.Kind?) -> [S
 
 @discardableResult
 public func executeSwiftBuild(
-    _ packagePath: AbsolutePath,
+    _ packagePath: AbsolutePath?,
     configuration: Configuration = .Debug,
     extraArgs: [String] = [],
     Xcc: [String] = [],
@@ -274,8 +278,8 @@ public func executeSwiftBuild(
 
 @discardableResult
 public func executeSwiftRun(
-    _ packagePath: AbsolutePath,
-    _ executable: String,
+    _ packagePath: AbsolutePath?,
+    _ executable: String?,
     configuration: Configuration = .Debug,
     extraArgs: [String] = [],
     Xcc: [String] = [],
@@ -292,13 +296,15 @@ public func executeSwiftRun(
         Xswiftc: Xswiftc,
         buildSystem: buildSystem
     )
-    args.append(executable)
+    if let executable {
+        args.append(executable)
+    }
     return try await SwiftPM.Run.execute(args, packagePath: packagePath, env: env)
 }
 
 @discardableResult
 public func executeSwiftPackage(
-    _ packagePath: AbsolutePath,
+    _ packagePath: AbsolutePath?,
     configuration: Configuration = .Debug,
     extraArgs: [String] = [],
     Xcc: [String] = [],
@@ -320,7 +326,7 @@ public func executeSwiftPackage(
 
 @discardableResult
 public func executeSwiftPackageRegistry(
-    _ packagePath: AbsolutePath,
+    _ packagePath: AbsolutePath?,
     configuration: Configuration = .Debug,
     extraArgs: [String] = [],
     Xcc: [String] = [],
@@ -342,13 +348,14 @@ public func executeSwiftPackageRegistry(
 
 @discardableResult
 public func executeSwiftTest(
-    _ packagePath: AbsolutePath,
+    _ packagePath: AbsolutePath?,
     configuration: Configuration = .Debug,
     extraArgs: [String] = [],
     Xcc: [String] = [],
     Xld: [String] = [],
     Xswiftc: [String] = [],
     env: Environment? = nil,
+    throwIfCommandFails: Bool = false,
     buildSystem: BuildSystemProvider.Kind = .native
 ) async throws -> (stdout: String, stderr: String) {
     let args = swiftArgs(
@@ -359,7 +366,7 @@ public func executeSwiftTest(
         Xswiftc: Xswiftc,
         buildSystem: buildSystem
     )
-    return try await SwiftPM.Test.execute(args, packagePath: packagePath, env: env)
+    return try await SwiftPM.Test.execute(args, packagePath: packagePath, env: env, throwIfCommandFails: throwIfCommandFails)
 }
 
 private func swiftArgs(

Tests/BuildTests/BuildPlanTests.swift

@@ -47,7 +47,7 @@ extension Build.BuildPlan {
 
 class BuildPlanTestCase: BuildSystemProviderTestCase {
     override func setUpWithError() throws {
-        try XCTSkipIf(type(of: self) == BuildPlanTestCase.self, "Pay no attention to the class behind the curtain.")
+        try XCTSkipIf(type(of: self) == BuildPlanTestCase.self, "Skipping this test since it will be run in subclasses that will provide different build systems to test.")
     }
 
     let inputsDir = AbsolutePath(#file).parentDirectory.appending(components: "Inputs")

Tests/CommandsTests/APIDiffTests.swift

@@ -13,6 +13,7 @@
 import Basics
 import Build
 import Commands
+import SPMBuildCore
 
 @_spi(SwiftPMInternal)
 import DriverSupport
@@ -24,7 +25,11 @@ import _InternalTestSupport
 import Workspace
 import XCTest
 
-final class APIDiffTests: CommandsTestCase {
+class APIDiffTestCase: CommandsBuildProviderTestCase {
+    override func setUpWithError() throws {
+        try XCTSkipIf(type(of: self) == APIDiffTestCase.self, "Skipping this test since it will be run in subclasses that will provide different build systems to test.")
+    }
+
     @discardableResult
     private func execute(
         _ args: [String],
@@ -34,7 +39,12 @@ final class APIDiffTests: CommandsTestCase {
         var environment = env ?? [:]
         // don't ignore local packages when caching
         environment["SWIFTPM_TESTS_PACKAGECACHE"] = "1"
-        return try await SwiftPM.Package.execute(args, packagePath: packagePath, env: environment)
+        return try await executeSwiftPackage(
+            packagePath,
+            extraArgs: args,
+            env: environment,
+            buildSystem: buildSystemProvider
+        )
     }
 
     func skipIfApiDigesterUnsupportedOrUnset() throws {
@@ -444,3 +454,26 @@ final class APIDiffTests: CommandsTestCase {
         }
     }
 }
+
+class APIDiffNativeTests: APIDiffTestCase {
+
+    override open var buildSystemProvider: BuildSystemProvider.Kind {
+        return .native
+    }
+
+    override func skipIfApiDigesterUnsupportedOrUnset() throws {
+        try super.skipIfApiDigesterUnsupportedOrUnset()
+    }
+
+}
+
+class APIDiffSwiftBuildTests: APIDiffTestCase {
+
+    override open var buildSystemProvider: BuildSystemProvider.Kind {
+        return .swiftbuild
+    }
+
+    override func skipIfApiDigesterUnsupportedOrUnset() throws {
+        try super.skipIfApiDigesterUnsupportedOrUnset()
+    }
+}