Add option to explain a given module dependency

Adds support for '-explain-dependency <MODULE_NAME>', which causes the driver to run a dependency scan, and find all dependency paths that cause the main module to depend, directly or transitively, on a given module dependency.

Author: artemcm

Sources/SwiftDriver/Driver/Driver.swift

@@ -1332,6 +1332,37 @@ extension Driver {
       return
     }
 
+    // If we're only supposed to explain a dependency on a given module, do so now.
+    if let explainModuleName = parsedOptions.getLastArgument(.explainModuleDependency) {
+      guard let dependencyPlanner = explicitDependencyBuildPlanner else {
+        fatalError("Cannot explain dependency without Explicit Build Planner")
+      }
+      guard let dependencyPaths = try dependencyPlanner.explainDependency(explainModuleName.asSingle) else {
+        diagnosticEngine.emit(.remark("No such module dependency found: '\(explainModuleName.asSingle)'"))
+        return
+      }
+      diagnosticEngine.emit(.remark("Module '\(moduleOutputInfo.name)' depends on '\(explainModuleName.asSingle)'"))
+      for path in dependencyPaths {
+        var pathString:String = ""
+        for (index, moduleId) in path.enumerated() {
+          switch moduleId {
+          case .swift(let moduleName):
+            pathString = pathString + "[" + moduleName + "]"
+          case .swiftPrebuiltExternal(let moduleName):
+            pathString = pathString + "[" + moduleName + "]"
+          case .clang(let moduleName):
+            pathString = pathString + "[" + moduleName + "](ObjC)"
+          case .swiftPlaceholder(_):
+            fatalError("Unexpected unresolved Placeholder module")
+          }
+          if index < path.count - 1 {
+            pathString = pathString + " -> "
+          }
+        }
+        diagnosticEngine.emit(.note(pathString))
+      }
+    }
+
     if parsedOptions.contains(.driverPrintOutputFileMap) {
       if let outputFileMap = self.outputFileMap {
         stderrStream <<< outputFileMap.description

Sources/SwiftDriver/ExplicitModuleBuilds/ExplicitDependencyBuildPlanner.swift

@@ -487,6 +487,12 @@ public typealias ExternalTargetModuleDetailsMap = [ModuleDependencyId: ExternalT
   }
 }
 
+internal extension ExplicitDependencyBuildPlanner {
+  func explainDependency(_ dependencyModuleName: String) throws -> [[ModuleDependencyId]]? {
+    return try dependencyGraph.explainDependency(dependencyModuleName: dependencyModuleName)
+  }
+}
+
 // InterModuleDependencyGraph printing, useful for debugging
 internal extension InterModuleDependencyGraph {
   func prettyPrintString() throws -> String {

Sources/SwiftDriver/ExplicitModuleBuilds/InterModuleDependencies/CommonDependencyOperations.swift

@@ -235,3 +235,48 @@ extension InterModuleDependencyGraph {
                       details: .clang(combinedModuleDetails))
   }
 }
+
+internal extension InterModuleDependencyGraph {
+  func explainDependency(dependencyModuleName: String) throws -> [[ModuleDependencyId]]? {
+    guard modules.contains(where: { $0.key.moduleName == dependencyModuleName }) else { return nil }
+    var results = [[ModuleDependencyId]]()
+    try findAllPaths(source: .swift(mainModuleName),
+                     to: dependencyModuleName,
+                     pathSoFar: [.swift(mainModuleName)],
+                     results: &results)
+    return Array(results)
+  }
+
+
+  private func findAllPaths(source: ModuleDependencyId,
+                            to moduleName: String,
+                            pathSoFar: [ModuleDependencyId],
+                            results: inout [[ModuleDependencyId]]) throws {
+    let sourceInfo = try moduleInfo(of: source)
+    // If the source is our target, we are done
+    guard source.moduleName != moduleName else {
+      // If the source is a target Swift module, also check if it
+      // depends on a corresponding Clang module with the same name.
+      // If it does, add it to the path as well.
+      var completePath = pathSoFar
+      if let dependencies = sourceInfo.directDependencies,
+         dependencies.contains(.clang(moduleName)) {
+        completePath.append(.clang(moduleName))
+      }
+      results.append(completePath)
+      return
+    }
+
+    // If no more dependencies, this is a leaf node, we are done
+    guard let dependencies = sourceInfo.directDependencies else {
+      return
+    }
+
+    for dependency in dependencies {
+      try findAllPaths(source: dependency,
+                       to: moduleName,
+                       pathSoFar: pathSoFar + [dependency],
+                       results: &results)
+    }
+  }
+}

Sources/SwiftDriver/Jobs/Planning.swift

@@ -140,7 +140,8 @@ extension Driver {
                                                           IncrementalCompilationState.InitialStateForPlanning?)
   throws -> InterModuleDependencyGraph? {
     let interModuleDependencyGraph: InterModuleDependencyGraph?
-    if parsedOptions.contains(.driverExplicitModuleBuild) {
+    if parsedOptions.contains(.driverExplicitModuleBuild) ||
+       parsedOptions.contains(.explainModuleDependency) {
       interModuleDependencyGraph =
         try initialIncrementalState?.maybeUpToDatePriorInterModuleDependencyGraph ??
             gatherModuleDependencies()
@@ -195,13 +196,15 @@ extension Driver {
     dependencyGraph: InterModuleDependencyGraph?,
     addJob: (Job) -> Void)
   throws {
-    // If asked, add jobs to precompile module dependencies
-    guard parsedOptions.contains(.driverExplicitModuleBuild) else { return }
     guard let resolvedDependencyGraph = dependencyGraph else {
-      fatalError("Attempting to plan explicit dependency build without a dependency graph")
+      return
     }
     let modulePrebuildJobs =
         try generateExplicitModuleDependenciesJobs(dependencyGraph: resolvedDependencyGraph)
+    // If asked, add jobs to precompile module dependencies. Otherwise exit.
+    // We may have a dependency graph but not be required to add pre-compile jobs to the build plan,
+    // for example when `-explain-dependency` is being used.
+    guard parsedOptions.contains(.driverExplicitModuleBuild) else { return }
     modulePrebuildJobs.forEach(addJob)
   }
 

Sources/SwiftOptions/Options.swift

@@ -20,6 +20,7 @@ extension Option {
   public static let accessNotesPathEQ: Option = Option("-access-notes-path=", .joined, alias: Option.accessNotesPath, attributes: [.frontend, .argumentIsPath])
   public static let accessNotesPath: Option = Option("-access-notes-path", .separate, attributes: [.frontend, .argumentIsPath], helpText: "Specify YAML file to override attributes on Swift declarations in this module")
   public static let aliasModuleNamesInModuleInterface: Option = Option("-alias-module-names-in-module-interface", .flag, attributes: [.helpHidden, .frontend, .noDriver], helpText: "When emitting a module interface, disambiguate modules using distinct alias names")
+  public static let allowUnstableCacheKeyForTesting: Option = Option("-allow-unstable-cache-key-for-testing", .flag, attributes: [.helpHidden, .frontend, .noDriver], helpText: "Allow compilation caching with unstable inputs for testing purpose")
   public static let allowableClient: Option = Option("-allowable-client", .separate, attributes: [.frontend], metaVar: "<vers>", helpText: "Module names that are allowed to import this module")
   public static let analyzeRequirementMachine: Option = Option("-analyze-requirement-machine", .flag, attributes: [.helpHidden, .frontend, .noDriver], helpText: "Print out requirement machine statistics at the end of the compilation job")
   public static let apiDiffDataDir: Option = Option("-api-diff-data-dir", .separate, attributes: [.frontend, .noInteractive, .doesNotAffectIncrementalBuild, .argumentIsPath], metaVar: "<path>", helpText: "Load platform and version specific API migration data files from <path>. Ignored if -api-diff-data-file is specified.")
@@ -329,6 +330,7 @@ extension Option {
   public static let enableBatchMode: Option = Option("-enable-batch-mode", .flag, attributes: [.helpHidden, .frontend, .noInteractive], helpText: "Enable combining frontend jobs into batches")
   public static let enableBridgingPch: Option = Option("-enable-bridging-pch", .flag, attributes: [.helpHidden], helpText: "Enable automatic generation of bridging PCH files")
   public static let enableBuiltinModule: Option = Option("-enable-builtin-module", .flag, attributes: [.frontend, .moduleInterface], helpText: "Enables the explicit import of the Builtin module")
+  public static let enableCas: Option = Option("-enable-cas", .flag, attributes: [.frontend, .noDriver], helpText: "Enable CAS for swift-frontend")
   public static let enableCollocateMetadataFunctions: Option = Option("-enable-collocate-metadata-functions", .flag, attributes: [.helpHidden, .frontend, .noDriver], helpText: "Enable collocate metadata functions")
   public static let enableColocateTypeDescriptors: Option = Option("-enable-colocate-type-descriptors", .flag, attributes: [.helpHidden, .frontend, .noDriver], helpText: "Enable colocate type descriptors")
   public static let enableConformanceAvailabilityErrors: Option = Option("-enable-conformance-availability-errors", .flag, attributes: [.helpHidden, .frontend, .noDriver], helpText: "Diagnose conformance availability violations as errors")
@@ -418,13 +420,14 @@ extension Option {
   public static let driverExperimentalExplicitModuleBuild: Option = Option("-experimental-explicit-module-build", .flag, alias: Option.driverExplicitModuleBuild, attributes: [.helpHidden], helpText: "Prebuild module dependencies to make them explicit")
   public static let experimentalHermeticSealAtLink: Option = Option("-experimental-hermetic-seal-at-link", .flag, attributes: [.helpHidden, .frontend], helpText: "Library code can assume that all clients are visible at linktime, and aggressively strip unused code")
   public static let experimentalOneWayClosureParams: Option = Option("-experimental-one-way-closure-params", .flag, attributes: [.helpHidden, .frontend, .noDriver], helpText: "Enable experimental support for one-way closure parameters")
-  public static let ExperimentalPerformanceAnnotations: Option = Option("-experimental-performance-annotations", .flag, attributes: [.helpHidden, .frontend], helpText: "Enable experimental performance annotations")
+  public static let ExperimentalPerformanceAnnotations: Option = Option("-experimental-performance-annotations", .flag, attributes: [.helpHidden, .frontend], helpText: "Deprecated, has no effect")
   public static let experimentalPrintFullConvention: Option = Option("-experimental-print-full-convention", .flag, attributes: [.helpHidden, .frontend, .noDriver], helpText: "When emitting a module interface or SIL, emit additional @convention arguments, regardless of whether they were written in the source. Also requires -use-clang-function-types to be enabled.")
   public static let experimentalSkipAllFunctionBodies: Option = Option("-experimental-skip-all-function-bodies", .flag, attributes: [.helpHidden, .frontend, .noDriver], helpText: "Skip type-checking function bodies and all SIL generation")
   public static let experimentalSkipNonInlinableFunctionBodiesWithoutTypes: Option = Option("-experimental-skip-non-inlinable-function-bodies-without-types", .flag, attributes: [.helpHidden, .frontend], helpText: "Skip work on non-inlinable function bodies that do not declare nested types")
   public static let experimentalSkipNonInlinableFunctionBodies: Option = Option("-experimental-skip-non-inlinable-function-bodies", .flag, attributes: [.helpHidden, .frontend], helpText: "Skip type-checking and SIL generation for non-inlinable function bodies")
   public static let experimentalSpiImports: Option = Option("-experimental-spi-imports", .flag, attributes: [.helpHidden, .frontend, .noDriver], helpText: "Enable experimental support for SPI imports")
   public static let experimentalSpiOnlyImports: Option = Option("-experimental-spi-only-imports", .flag, attributes: [.helpHidden, .frontend, .noDriver], helpText: "Enable use of @_spiOnly imports")
+  public static let explainModuleDependency: Option = Option("-explain-module-dependency", .separate, attributes: [], helpText: "Emit remark/notes describing why compilaiton may depend on a module with a given name.")
   public static let explicitDependencyGraphFormat: Option = Option("-explicit-dependency-graph-format=", .joined, attributes: [.helpHidden, .doesNotAffectIncrementalBuild], helpText: "Specify the explicit dependency graph output format to either 'json' or 'dot'")
   public static let explicitInterfaceModuleBuild: Option = Option("-explicit-interface-module-build", .flag, attributes: [.helpHidden, .frontend, .noDriver], helpText: "Use the specified command-line to build the module from interface, instead of flags specified in the interface")
   public static let driverExplicitModuleBuild: Option = Option("-explicit-module-build", .flag, attributes: [.helpHidden], helpText: "Prebuild module dependencies to make them explicit")
@@ -792,6 +795,7 @@ extension Option {
       Option.accessNotesPathEQ,
       Option.accessNotesPath,
       Option.aliasModuleNamesInModuleInterface,
+      Option.allowUnstableCacheKeyForTesting,
       Option.allowableClient,
       Option.analyzeRequirementMachine,
       Option.apiDiffDataDir,
@@ -823,6 +827,8 @@ extension Option {
       Option.buildModuleFromParseableInterface,
       Option.bypassBatchModeChecks,
       Option.candidateModuleFile,
+      Option.casFs,
+      Option.casPath,
       Option.checkApiAvailabilityOnly,
       Option.checkOnoneCompleteness,
       Option.clangBuildSessionFile,
@@ -1101,6 +1107,7 @@ extension Option {
       Option.enableBatchMode,
       Option.enableBridgingPch,
       Option.enableBuiltinModule,
+      Option.enableCas,
       Option.enableCollocateMetadataFunctions,
       Option.enableColocateTypeDescriptors,
       Option.enableConformanceAvailabilityErrors,
@@ -1197,6 +1204,7 @@ extension Option {
       Option.experimentalSkipNonInlinableFunctionBodies,
       Option.experimentalSpiImports,
       Option.experimentalSpiOnlyImports,
+      Option.explainModuleDependency,
       Option.explicitDependencyGraphFormat,
       Option.explicitInterfaceModuleBuild,
       Option.driverExplicitModuleBuild,

Sources/makeOptions/makeOptions.cpp

@@ -64,6 +64,7 @@ enum SwiftFlags {
   SwiftAPIDigesterOption = (1 << 17),
   NewDriverOnlyOption = (1 << 18),
   ModuleInterfaceOptionIgnorable = (1 << 19),
+  ModuleInterfaceOptionIgnorablePrivate = (1 << 20)
 };
 
 static std::set<std::string> swiftKeywords = { "internal", "static" };

Tests/SwiftDriverTests/ExplicitModuleBuildTests.swift

@@ -1613,6 +1613,53 @@ final class ExplicitModuleBuildTests: XCTestCase {
     XCTAssertEqual(moduleMap[1].sourceInfoPath!.path.description, "B.swiftsourceinfo")
     XCTAssertEqual(moduleMap[1].isFramework, false)
   }
+
+
+  func testTraceDependency() throws {
+    try withTemporaryDirectory { path in
+      try localFileSystem.changeCurrentWorkingDirectory(to: path)
+      let moduleCachePath = path.appending(component: "ModuleCache")
+      try localFileSystem.createDirectory(moduleCachePath)
+      let main = path.appending(component: "testTraceDependency.swift")
+      try localFileSystem.writeFileContents(main) {
+        $0 <<< "import C;"
+        $0 <<< "import E;"
+        $0 <<< "import G;"
+      }
+
+      let cHeadersPath: AbsolutePath =
+          testInputsPath.appending(component: "ExplicitModuleBuilds")
+                        .appending(component: "CHeaders")
+      let swiftModuleInterfacesPath: AbsolutePath =
+          testInputsPath.appending(component: "ExplicitModuleBuilds")
+                        .appending(component: "Swift")
+      let sdkArgumentsForTesting = (try? Driver.sdkArgumentsForTesting()) ?? []
+      var driver = try Driver(args: ["swiftc",
+                                     "-I", cHeadersPath.nativePathString(escaped: true),
+                                     "-I", swiftModuleInterfacesPath.nativePathString(escaped: true),
+                                     "-explicit-module-build", "-v",
+                                     "-module-cache-path", moduleCachePath.nativePathString(escaped: true),
+                                     "-working-directory", path.nativePathString(escaped: true),
+                                     "-explain-module-dependency", "A",
+                                     main.nativePathString(escaped: true)] + sdkArgumentsForTesting,
+                              env: ProcessEnv.vars)
+      let jobs = try driver.planBuild()
+      try driver.run(jobs: jobs)
+      XCTAssertTrue(!driver.diagnosticEngine.diagnostics.isEmpty)
+      XCTAssertTrue(driver.diagnosticEngine.diagnostics.contains { $0.behavior == .remark &&
+                                                                   $0.message.text == "Module 'testTraceDependency' depends on 'A'"})
+
+      for diag in driver.diagnosticEngine.diagnostics {
+        print(diag.behavior)
+        print(diag.message)
+      }
+      XCTAssertTrue(driver.diagnosticEngine.diagnostics.contains { $0.behavior == .note &&
+                                                                   $0.message.text == "[testTraceDependency] -> [A] -> [A](ObjC)"})
+      XCTAssertTrue(driver.diagnosticEngine.diagnostics.contains { $0.behavior == .note &&
+                                                                   $0.message.text == "[testTraceDependency] -> [C](ObjC) -> [B](ObjC) -> [A](ObjC)"})
+    }
+  }
+
 // We only care about prebuilt modules in macOS.
 #if os(macOS)
   func testPrebuiltModuleGenerationJobs() throws {