Run build tool plugins for C-family targets (#6516)

neonichu · MaxDesiatov · web-flow · commit 678e6831c7b1 · 2023-05-10T16:46:32.000-07:00
While we intentionally don't support generating C sources from plugins today since we haven't figured out how to deal with headers etc, not running plugins at all without any diagnostics whatsoever seems like an implementation oversight based on the fact that we have two completely different implementations for Swift and C-family targets (which is something we also need to rectify at some point).

With this change, we're running build-tool plugins in the exact same way as we are doing it for Swift targets. We are only doing this for packages with tools-version 5.9 or higher in order to have any unintentional impact on existing packages.

rdar://101671614

Co-authored-by: Max Desiatov &lt;m_desiatov@apple.com&gt;
diff --git a/Sources/Build/BuildDescription/ClangTargetBuildDescription.swift b/Sources/Build/BuildDescription/ClangTargetBuildDescription.swift
@@ -15,6 +15,8 @@ import PackageLoading
 import PackageModel
 import class PackageGraph.ResolvedTarget
 import struct SPMBuildCore.BuildParameters
+import struct SPMBuildCore.BuildToolPluginInvocationResult
+import struct SPMBuildCore.PrebuildCommandResult
 
 import enum TSCBasic.ProcessEnv
 
@@ -41,9 +43,22 @@ public final class ClangTargetBuildDescription {
         buildParameters.buildEnvironment
     }
 
+    /// The list of all resource files in the target, including the derived ones.
+    public var resources: [Resource] {
+        self.target.underlyingTarget.resources + self.pluginDerivedResources
+    }
+
     /// Path to the bundle generated for this module (if any).
     var bundlePath: AbsolutePath? {
-        target.underlyingTarget.bundleName.map(buildParameters.bundlePath(named:))
+        guard !resources.isEmpty else {
+            return .none
+        }
+
+        if let bundleName = target.underlyingTarget.potentialBundleName {
+            return self.buildParameters.bundlePath(named: bundleName)
+        } else {
+            return .none
+        }
     }
 
     /// The modulemap file for this target, if any.
@@ -57,6 +72,12 @@ public final class ClangTargetBuildDescription {
     /// These are the source files generated during the build.
     private var derivedSources: Sources
 
+    /// These are the source files derived from plugins.
+    private var pluginDerivedSources: Sources
+
+    /// These are the resource files derived from plugins.
+    private var pluginDerivedResources: [Resource]
+
     /// Path to the resource accessor header file, if generated.
     public private(set) var resourceAccessorHeaderFile: AbsolutePath?
 
@@ -84,12 +105,19 @@ public final class ClangTargetBuildDescription {
         target.type == .test
     }
 
+    /// The results of applying any build tool plugins to this target.
+    public let buildToolPluginInvocationResults: [BuildToolPluginInvocationResult]
+
     /// Create a new target description with target and build parameters.
     init(
         target: ResolvedTarget,
         toolsVersion: ToolsVersion,
+        additionalFileRules: [FileRuleDescription] = [],
         buildParameters: BuildParameters,
-        fileSystem: FileSystem
+        buildToolPluginInvocationResults: [BuildToolPluginInvocationResult] = [],
+        prebuildCommandResults: [PrebuildCommandResult] = [],
+        fileSystem: FileSystem,
+        observabilityScope: ObservabilityScope
     ) throws {
         guard target.underlyingTarget is ClangTarget else {
             throw InternalError("underlying target type mismatch \(target)")
@@ -102,6 +130,25 @@ public final class ClangTargetBuildDescription {
         self.tempsPath = buildParameters.buildPath.appending(component: target.c99name + ".build")
         self.derivedSources = Sources(paths: [], root: tempsPath.appending("DerivedSources"))
 
+        // We did not use to apply package plugins to C-family targets in prior tools-versions, this preserves the behavior.
+        if toolsVersion >= .v5_9 {
+            self.buildToolPluginInvocationResults = buildToolPluginInvocationResults
+
+            (self.pluginDerivedSources, self.pluginDerivedResources) = SharedTargetBuildDescription.computePluginGeneratedFiles(
+                target: target,
+                toolsVersion: toolsVersion,
+                additionalFileRules: additionalFileRules,
+                buildParameters: buildParameters,
+                buildToolPluginInvocationResults: buildToolPluginInvocationResults,
+                prebuildCommandResults: prebuildCommandResults,
+                observabilityScope: observabilityScope
+            )
+        } else {
+            self.buildToolPluginInvocationResults = []
+            self.pluginDerivedSources = Sources(paths: [], root: buildParameters.dataPath)
+            self.pluginDerivedResources = []
+        }
+
         // Try computing modulemap path for a C library.  This also creates the file in the file system, if needed.
         if target.type == .library {
             // If there's a custom module map, use it as given.
@@ -141,6 +188,7 @@ public final class ClangTargetBuildDescription {
         let sources = [
             target.sources.root: target.sources.relativePaths,
             derivedSources.root: derivedSources.relativePaths,
+            pluginDerivedSources.root: pluginDerivedSources.relativePaths
         ]
 
         return try sources.flatMap { root, relativePaths in
diff --git a/Sources/Build/BuildDescription/SharedTargetBuildDescription.swift b/Sources/Build/BuildDescription/SharedTargetBuildDescription.swift
@@ -0,0 +1,67 @@
+//===----------------------------------------------------------------------===//
+//
+// This source file is part of the Swift open source project
+//
+// Copyright (c) 2023 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See http://swift.org/LICENSE.txt for license information
+// See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+import Basics
+import PackageGraph
+import PackageLoading
+import PackageModel
+import SPMBuildCore
+
+import struct Basics.AbsolutePath
+
+/// Shared functionality between `ClangTargetBuildDescription` and `SwiftTargetBuildDescription` with the eventual hope of having a single type.
+struct SharedTargetBuildDescription {
+    static func computePluginGeneratedFiles(
+        target: ResolvedTarget,
+        toolsVersion: ToolsVersion,
+        additionalFileRules: [FileRuleDescription],
+        buildParameters: BuildParameters,
+        buildToolPluginInvocationResults: [BuildToolPluginInvocationResult],
+        prebuildCommandResults: [PrebuildCommandResult],
+        observabilityScope: ObservabilityScope
+    ) -> (pluginDerivedSources: Sources, pluginDerivedResources: [Resource]) {
+        var pluginDerivedSources = Sources(paths: [], root: buildParameters.dataPath)
+
+        // Add any derived files that were declared for any commands from plugin invocations.
+        var pluginDerivedFiles = [AbsolutePath]()
+        for command in buildToolPluginInvocationResults.reduce([], { $0 + $1.buildCommands }) {
+            for absPath in command.outputFiles {
+                pluginDerivedFiles.append(absPath)
+            }
+        }
+
+        // Add any derived files that were discovered from output directories of prebuild commands.
+        for result in prebuildCommandResults {
+            for path in result.derivedFiles {
+                pluginDerivedFiles.append(path)
+            }
+        }
+
+        // Let `TargetSourcesBuilder` compute the treatment of plugin generated files.
+        let (derivedSources, derivedResources) = TargetSourcesBuilder.computeContents(
+            for: pluginDerivedFiles,
+            toolsVersion: toolsVersion,
+            additionalFileRules: additionalFileRules,
+            defaultLocalization: target.defaultLocalization,
+            targetName: target.name,
+            targetPath: target.underlyingTarget.path,
+            observabilityScope: observabilityScope
+        )
+        let pluginDerivedResources = derivedResources
+        derivedSources.forEach { absPath in
+            let relPath = absPath.relative(to: pluginDerivedSources.root)
+            pluginDerivedSources.relativePaths.append(relPath)
+        }
+
+        return (pluginDerivedSources, pluginDerivedResources)
+    }
+}
diff --git a/Sources/Build/BuildDescription/SwiftTargetBuildDescription.swift b/Sources/Build/BuildDescription/SwiftTargetBuildDescription.swift
@@ -251,42 +251,20 @@ public final class SwiftTargetBuildDescription {
         self.fileSystem = fileSystem
         self.tempsPath = buildParameters.buildPath.appending(component: target.c99name + ".build")
         self.derivedSources = Sources(paths: [], root: self.tempsPath.appending("DerivedSources"))
-        self.pluginDerivedSources = Sources(paths: [], root: buildParameters.dataPath)
         self.buildToolPluginInvocationResults = buildToolPluginInvocationResults
         self.prebuildCommandResults = prebuildCommandResults
         self.requiredMacroProducts = requiredMacroProducts
         self.observabilityScope = observabilityScope
 
-        // Add any derived files that were declared for any commands from plugin invocations.
-        var pluginDerivedFiles = [AbsolutePath]()
-        for command in buildToolPluginInvocationResults.reduce([], { $0 + $1.buildCommands }) {
-            for absPath in command.outputFiles {
-                pluginDerivedFiles.append(absPath)
-            }
-        }
-
-        // Add any derived files that were discovered from output directories of prebuild commands.
-        for result in self.prebuildCommandResults {
-            for path in result.derivedFiles {
-                pluginDerivedFiles.append(path)
-            }
-        }
-
-        // Let `TargetSourcesBuilder` compute the treatment of plugin generated files.
-        let (derivedSources, derivedResources) = TargetSourcesBuilder.computeContents(
-            for: pluginDerivedFiles,
+        (self.pluginDerivedSources, self.pluginDerivedResources) = SharedTargetBuildDescription.computePluginGeneratedFiles(
+            target: target,
             toolsVersion: toolsVersion,
             additionalFileRules: additionalFileRules,
-            defaultLocalization: target.defaultLocalization,
-            targetName: target.name,
-            targetPath: target.underlyingTarget.path,
+            buildParameters: buildParameters,
+            buildToolPluginInvocationResults: buildToolPluginInvocationResults,
+            prebuildCommandResults: prebuildCommandResults,
             observabilityScope: observabilityScope
         )
-        self.pluginDerivedResources = derivedResources
-        derivedSources.forEach { absPath in
-            let relPath = absPath.relative(to: self.pluginDerivedSources.root)
-            self.pluginDerivedSources.relativePaths.append(relPath)
-        }
 
         if self.shouldEmitObjCCompatibilityHeader {
             self.moduleMap = try self.generateModuleMap()
diff --git a/Sources/Build/BuildDescription/TargetBuildDescription.swift b/Sources/Build/BuildDescription/TargetBuildDescription.swift
@@ -13,6 +13,7 @@
 import Basics
 import class PackageGraph.ResolvedTarget
 import struct PackageModel.Resource
+import struct SPMBuildCore.BuildToolPluginInvocationResult
 
 /// A target description which can either be for a Swift or Clang target.
 public enum TargetBuildDescription {
@@ -40,8 +41,7 @@ public enum TargetBuildDescription {
         case .swift(let target):
             return target.resources
         case .clang(let target):
-            // TODO: Clang targets should support generated resources in the future.
-            return target.target.underlyingTarget.resources
+            return target.resources
         }
     }
 
@@ -82,4 +82,13 @@ public enum TargetBuildDescription {
             return target.resourceBundleInfoPlistPath
         }
     }
+
+    var buildToolPluginInvocationResults: [BuildToolPluginInvocationResult] {
+        switch self {
+        case .swift(let target):
+            return target.buildToolPluginInvocationResults
+        case .clang(let target):
+            return target.buildToolPluginInvocationResults
+        }
+    }
 }
diff --git a/Sources/Build/BuildPlan.swift b/Sources/Build/BuildPlan.swift
@@ -496,8 +496,13 @@ public class BuildPlan: SPMBuildCore.BuildPlan {
                 targetMap[target] = try .clang(ClangTargetBuildDescription(
                     target: target,
                     toolsVersion: toolsVersion,
+                    additionalFileRules: additionalFileRules,
                     buildParameters: buildParameters,
-                    fileSystem: fileSystem))
+                    buildToolPluginInvocationResults: buildToolPluginInvocationResults[target] ?? [],
+                    prebuildCommandResults: prebuildCommandResults[target] ?? [],
+                    fileSystem: fileSystem,
+                    observabilityScope: observabilityScope)
+                )
             case is PluginTarget:
                 guard let package = graph.package(for: target) else {
                     throw InternalError("package not found for \(target)")
diff --git a/Sources/Build/CMakeLists.txt b/Sources/Build/CMakeLists.txt
@@ -10,6 +10,7 @@ add_library(Build
   BuildDescription/ClangTargetBuildDescription.swift
   BuildDescription/PluginDescription.swift
   BuildDescription/ProductBuildDescription.swift
+  BuildDescription/SharedTargetBuildDescription.swift
   BuildDescription/SwiftTargetBuildDescription.swift
   BuildDescription/TargetBuildDescription.swift
   BuildOperationBuildSystemDelegateHandler.swift
diff --git a/Sources/Build/LLBuildManifestBuilder.swift b/Sources/Build/LLBuildManifestBuilder.swift
@@ -207,6 +207,41 @@ extension LLBuildManifestBuilder {
     }
 }
 
+// MARK: - Compilation
+
+extension LLBuildManifestBuilder {
+    private func addBuildToolPlugins(_ target: TargetBuildDescription) throws {
+        // Add any regular build commands created by plugins for the target.
+        for result in target.buildToolPluginInvocationResults {
+            // Only go through the regular build commands — prebuild commands are handled separately.
+            for command in result.buildCommands {
+                // Create a shell command to invoke the executable. We include the path of the executable as a
+                // dependency, and make sure the name is unique.
+                let execPath = command.configuration.executable
+                let uniquedName = ([execPath.pathString] + command.configuration.arguments).joined(separator: "|")
+                let displayName = command.configuration.displayName ?? execPath.basename
+                var commandLine = [execPath.pathString] + command.configuration.arguments
+                if !self.disableSandboxForPluginCommands {
+                    commandLine = try Sandbox.apply(
+                        command: commandLine,
+                        strictness: .writableTemporaryDirectory,
+                        writableDirectories: [result.pluginOutputDirectory]
+                    )
+                }
+                self.manifest.addShellCmd(
+                    name: displayName + "-" + ByteString(encodingAsUTF8: uniquedName).sha256Checksum,
+                    description: displayName,
+                    inputs: command.inputFiles.map { .file($0) },
+                    outputs: command.outputFiles.map { .file($0) },
+                    arguments: commandLine,
+                    environment: command.configuration.environment,
+                    workingDirectory: command.configuration.workingDirectory?.pathString
+                )
+            }
+        }
+    }
+}
+
 // MARK: - Compile Swift
 
 extension LLBuildManifestBuilder {
@@ -681,34 +716,7 @@ extension LLBuildManifestBuilder {
             }
         }
 
-        // Add any regular build commands created by plugins for the target.
-        for result in target.buildToolPluginInvocationResults {
-            // Only go through the regular build commands — prebuild commands are handled separately.
-            for command in result.buildCommands {
-                // Create a shell command to invoke the executable. We include the path of the executable as a
-                // dependency, and make sure the name is unique.
-                let execPath = command.configuration.executable
-                let uniquedName = ([execPath.pathString] + command.configuration.arguments).joined(separator: "|")
-                let displayName = command.configuration.displayName ?? execPath.basename
-                var commandLine = [execPath.pathString] + command.configuration.arguments
-                if !self.disableSandboxForPluginCommands {
-                    commandLine = try Sandbox.apply(
-                        command: commandLine,
-                        strictness: .writableTemporaryDirectory,
-                        writableDirectories: [result.pluginOutputDirectory]
-                    )
-                }
-                self.manifest.addShellCmd(
-                    name: displayName + "-" + ByteString(encodingAsUTF8: uniquedName).sha256Checksum,
-                    description: displayName,
-                    inputs: command.inputFiles.map { .file($0) },
-                    outputs: command.outputFiles.map { .file($0) },
-                    arguments: commandLine,
-                    environment: command.configuration.environment,
-                    workingDirectory: command.configuration.workingDirectory?.pathString
-                )
-            }
-        }
+        try addBuildToolPlugins(.swift(target))
 
         // Depend on any required macro product's output.
         try target.requiredMacroProducts.forEach { macro in
@@ -882,6 +890,8 @@ extension LLBuildManifestBuilder {
             )
         }
 
+        try addBuildToolPlugins(.clang(target))
+
         // Create a phony node to represent the entire target.
         let targetName = target.target.getLLBuildTargetName(config: self.buildConfig)
         let output: Node = .virtual(targetName)

Original file line number	Diff line number	Diff line change
`@@ -13,6 +13,7 @@`
`13`	`13`	`import Basics`
`14`	`14`	`import class PackageGraph.ResolvedTarget`
`15`	`15`	`import struct PackageModel.Resource`
	`16`	`+import struct SPMBuildCore.BuildToolPluginInvocationResult`
`16`	`17`
`17`	`18`	`/// A target description which can either be for a Swift or Clang target.`
`18`	`19`	`public enum TargetBuildDescription {`
`@@ -40,8 +41,7 @@ public enum TargetBuildDescription {`
`40`	`41`	`case .swift(let target):`
`41`	`42`	`return target.resources`
`42`	`43`	`case .clang(let target):`
`43`		`- // TODO: Clang targets should support generated resources in the future.`
`44`		`- return target.target.underlyingTarget.resources`
	`44`	`+ return target.resources`
`45`	`45`	`}`
`46`	`46`	`}`
`47`	`47`
`@@ -82,4 +82,13 @@ public enum TargetBuildDescription {`
`82`	`82`	`return target.resourceBundleInfoPlistPath`
`83`	`83`	`}`
`84`	`84`	`}`
	`85`	`+`
	`86`	`+ var buildToolPluginInvocationResults: [BuildToolPluginInvocationResult] {`
	`87`	`+ switch self {`
	`88`	`+ case .swift(let target):`
	`89`	`+ return target.buildToolPluginInvocationResults`
	`90`	`+ case .clang(let target):`
	`91`	`+ return target.buildToolPluginInvocationResults`
	`92`	`+ }`
	`93`	`+ }`
`85`	`94`	`}`