[ParseableInterfaces] Add -prebuilt-module-cache-path to the frontend

include/swift/Basic/STLExtras.h

@@ -792,6 +792,30 @@ inline T accumulate(const Container &C, T init, BinaryOperation op) {
   return std::accumulate(C.begin(), C.end(), init, op);
 }
 
+/// Returns true if the range defined by \p mainBegin ..< \p mainEnd starts with
+/// the same elements as the range defined by \p prefixBegin ..< \p prefixEnd.
+///
+/// This includes cases where the prefix range is empty, as well as when the two
+/// ranges are the same length and contain the same elements.
+template <typename MainInputIterator, typename PrefixInputIterator>
+inline bool hasPrefix(MainInputIterator mainBegin,
+                      const MainInputIterator mainEnd,
+                      PrefixInputIterator prefixBegin,
+                      const PrefixInputIterator prefixEnd) {
+  while (prefixBegin != prefixEnd) {
+    // If "main" is shorter than "prefix", it does not start with "prefix".
+    if (mainBegin == mainEnd)
+      return false;
+    // If there's a mismatch, "main" does not start with "prefix".
+    if (*mainBegin != *prefixBegin)
+      return false;
+    ++prefixBegin;
+    ++mainBegin;
+  }
+  // If we checked every element of "prefix", "main" does start with "prefix".
+  return true;
+}
+
 /// Provides default implementations of !=, <=, >, and >= based on == and <.
 template <typename T>
 class RelationalOperationsBase {

include/swift/Frontend/FrontendOptions.h

@@ -72,6 +72,10 @@ class FrontendOptions {
   /// The path to which we should store indexing data, if any.
   std::string IndexStorePath;
 
+  /// The path to look in when loading a parseable interface file, to see if a
+  /// binary module has already been built for use by the compiler.
+  std::string PrebuiltModuleCachePath;
+
   /// Emit index data for imported serialized swift system modules.
   bool IndexSystemModules = false;
 

include/swift/Frontend/ParseableInterfaceSupport.h

@@ -61,13 +61,15 @@ getModuleCachePathFromClang(const clang::CompilerInstance &Instance);
 /// directory, and loading the serialized .swiftmodules from there.
 class ParseableInterfaceModuleLoader : public SerializedModuleLoaderBase {
   explicit ParseableInterfaceModuleLoader(ASTContext &ctx, StringRef cacheDir,
+                                          StringRef prebuiltCacheDir,
                                           DependencyTracker *tracker,
                                           ModuleLoadingMode loadMode)
     : SerializedModuleLoaderBase(ctx, tracker, loadMode),
-      CacheDir(cacheDir)
+      CacheDir(cacheDir), PrebuiltCacheDir(prebuiltCacheDir)
   {}
 
   std::string CacheDir;
+  std::string PrebuiltCacheDir;
 
   /// Wire up the SubInvocation's InputsAndOutputs to contain both input and
   /// output filenames.
@@ -86,11 +88,11 @@ class ParseableInterfaceModuleLoader : public SerializedModuleLoaderBase {
 
 public:
   static std::unique_ptr<ParseableInterfaceModuleLoader>
-  create(ASTContext &ctx, StringRef cacheDir,
-         DependencyTracker *tracker,
-         ModuleLoadingMode loadMode) {
+  create(ASTContext &ctx, StringRef cacheDir, StringRef prebuiltCacheDir,
+         DependencyTracker *tracker, ModuleLoadingMode loadMode) {
     return std::unique_ptr<ParseableInterfaceModuleLoader>(
-        new ParseableInterfaceModuleLoader(ctx, cacheDir, tracker, loadMode));
+        new ParseableInterfaceModuleLoader(ctx, cacheDir, prebuiltCacheDir,
+                                           tracker, loadMode));
   }
 
   /// Unconditionally build \p InPath (a swiftinterface file) to \p OutPath (as

include/swift/Option/FrontendOptions.td

@@ -518,6 +518,13 @@ def build_module_from_parseable_interface :
   HelpText<"Treat the (single) input as a swiftinterface and produce a module">,
   ModeOpt;
 
+def prebuilt_module_cache_path :
+  Separate<["-"], "prebuilt-module-cache-path">,
+  HelpText<"Directory of prebuilt modules for loading parseable interfaces">;
+def prebuilt_module_cache_path_EQ :
+  Joined<["-"], "prebuilt-module-cache-path=">,
+  Alias<prebuilt_module_cache_path>;
+
 def enable_resilience_bypass : Flag<["-"], "enable-resilience-bypass">,
    HelpText<"Completely bypass resilience when accessing types in resilient frameworks">;
 

lib/Frontend/ArgsToFrontendOptionsConverter.cpp

@@ -60,6 +60,9 @@ bool ArgsToFrontendOptionsConverter::convert(
   if (const Arg *A = Args.getLastArg(OPT_index_store_path)) {
     Opts.IndexStorePath = A->getValue();
   }
+  if (const Arg *A = Args.getLastArg(OPT_prebuilt_module_cache_path)) {
+    Opts.PrebuiltModuleCachePath = A->getValue();
+  }
 
   Opts.IndexSystemModules |= Args.hasArg(OPT_index_system_modules);
 

lib/Frontend/Frontend.cpp

@@ -331,8 +331,11 @@ bool CompilerInstance::setUpModuleLoaders() {
   if (MLM != ModuleLoadingMode::OnlySerialized) {
     auto const &Clang = clangImporter->getClangInstance();
     std::string ModuleCachePath = getModuleCachePathFromClang(Clang);
+    StringRef PrebuiltModuleCachePath =
+        Invocation.getFrontendOptions().PrebuiltModuleCachePath;
     auto PIML = ParseableInterfaceModuleLoader::create(*Context,
                                                        ModuleCachePath,
+                                                       PrebuiltModuleCachePath,
                                                        getDependencyTracker(),
                                                        MLM);
     Context->addModuleLoader(std::move(PIML));

lib/Frontend/ParseableInterfaceSupport.cpp

@@ -19,6 +19,7 @@
 #include "swift/AST/FileSystem.h"
 #include "swift/AST/Module.h"
 #include "swift/AST/ProtocolConformance.h"
+#include "swift/Basic/STLExtras.h"
 #include "swift/Frontend/Frontend.h"
 #include "swift/Frontend/ParseableInterfaceSupport.h"
 #include "swift/Frontend/PrintingDiagnosticConsumer.h"
@@ -443,6 +444,8 @@ std::error_code ParseableInterfaceModuleLoader::openModuleFiles(
     std::unique_ptr<llvm::MemoryBuffer> *ModuleDocBuffer,
     llvm::SmallVectorImpl<char> &Scratch) {
 
+  namespace path = llvm::sys::path;
+
   // If running in OnlySerialized mode, ParseableInterfaceModuleLoader
   // should not have been constructed at all.
   assert(LoadMode != ModuleLoadingMode::OnlySerialized);
@@ -453,11 +456,11 @@ std::error_code ParseableInterfaceModuleLoader::openModuleFiles(
 
   // First check to see if the .swiftinterface exists at all. Bail if not.
   ModPath = DirName;
-  llvm::sys::path::append(ModPath, ModuleFilename);
+  path::append(ModPath, ModuleFilename);
 
   auto Ext = file_types::getExtension(file_types::TY_SwiftParseableInterfaceFile);
   InPath = ModPath;
-  llvm::sys::path::replace_extension(InPath, Ext);
+  path::replace_extension(InPath, Ext);
   if (!FS.exists(InPath))
     return std::make_error_code(std::errc::no_such_file_or_directory);
 
@@ -472,31 +475,67 @@ std::error_code ParseableInterfaceModuleLoader::openModuleFiles(
     return std::make_error_code(std::errc::not_supported);
   }
 
-  // At this point we're either in PreferParseable mode or there's no credible
-  // adjacent .swiftmodule so we'll go ahead and start trying to convert the
-  // .swiftinterface.
+  // If we have a prebuilt cache path, check that too if the interface comes
+  // from the SDK.
+  if (!PrebuiltCacheDir.empty()) {
+    StringRef SDKPath = Ctx.SearchPathOpts.SDKPath;
+    if (!SDKPath.empty() && hasPrefix(path::begin(InPath),
+                                      path::end(InPath),
+                                      path::begin(SDKPath),
+                                      path::end(SDKPath))) {
+      // Assemble the expected path: $PREBUILT_CACHE/Foo.swiftmodule or
+      // $PREBUILT_CACHE/Foo.swiftmodule/arch.swiftmodule. Note that there's no
+      // cache key here.
+      OutPath = PrebuiltCacheDir;
+
+      // FIXME: Would it be possible to only have architecture-specific names
+      // here? Then we could skip this check.
+      StringRef InParentDirName = path::filename(path::parent_path(InPath));
+      if (path::extension(InParentDirName) == ".swiftmodule") {
+        assert(path::stem(InParentDirName) == ModuleID.first.str());
+        path::append(OutPath, InParentDirName);
+      }
+      path::append(OutPath, ModuleFilename);
 
-  // Set up a _potential_ sub-invocation to consume the .swiftinterface and emit
-  // the .swiftmodule.
-  CompilerInvocation SubInvocation =
-      createInvocationForBuildingFromInterface(Ctx, ModuleID.first.str(), CacheDir);
-  computeCachedOutputPath(Ctx, SubInvocation, InPath, OutPath);
-  configureSubInvocationInputsAndOutputs(SubInvocation, InPath, OutPath);
+      if (!swiftModuleIsUpToDate(FS, ModuleID, OutPath, Diags,
+                                 dependencyTracker)) {
+        OutPath.clear();
+      }
+    }
+  }
 
-  // Evaluate if we need to run this sub-invocation, and if so run it.
-  if (!swiftModuleIsUpToDate(FS, ModuleID, OutPath, Diags, dependencyTracker)) {
-    if (::buildSwiftModuleFromSwiftInterface(FS, Diags, ModuleID.second,
-                                             SubInvocation, CacheDir,
-                                             dependencyTracker))
-      return std::make_error_code(std::errc::invalid_argument);
+  if (OutPath.empty()) {
+    // At this point we're either in PreferParseable mode or there's no credible
+    // adjacent .swiftmodule so we'll go ahead and start trying to convert the
+    // .swiftinterface.
+
+    // Set up a _potential_ sub-invocation to consume the .swiftinterface and
+    // emit the .swiftmodule.
+    CompilerInvocation SubInvocation =
+        createInvocationForBuildingFromInterface(Ctx, ModuleID.first.str(),
+                                                 CacheDir);
+    computeCachedOutputPath(Ctx, SubInvocation, InPath, OutPath);
+    configureSubInvocationInputsAndOutputs(SubInvocation, InPath, OutPath);
+
+    // Evaluate if we need to run this sub-invocation, and if so run it.
+    if (!swiftModuleIsUpToDate(FS, ModuleID, OutPath, Diags,
+                               dependencyTracker)) {
+      if (::buildSwiftModuleFromSwiftInterface(FS, Diags, ModuleID.second,
+                                               SubInvocation, CacheDir,
+                                               dependencyTracker))
+        return std::make_error_code(std::errc::invalid_argument);
+    }
   }
 
   // Finish off by delegating back up to the SerializedModuleLoaderBase
   // routine that can load the recently-manufactured serialized module.
   LLVM_DEBUG(llvm::dbgs() << "Loading " << OutPath
              << " via normal module loader\n");
+  // FIXME: This will never find the swiftdoc file, because that's in the
+  // original directory. We should probably just not try to match the signature
+  // of the overridable entry point.
   auto ErrorCode = SerializedModuleLoaderBase::openModuleFiles(
-      ModuleID, CacheDir, llvm::sys::path::filename(OutPath),
+      ModuleID, path::parent_path(OutPath), path::filename(OutPath),
       ModuleDocFilename, ModuleBuffer, ModuleDocBuffer, Scratch);
   LLVM_DEBUG(llvm::dbgs() << "Loaded " << OutPath
              << " via normal module loader");

test/ParseableInterface/ModuleCache/Inputs/prebuilt-module-cache/Lib.swiftinterface

@@ -0,0 +1,9 @@
+// swift-interface-format-version: 1.0
+// swift-module-flags: -parse-stdlib -module-name Lib
+
+public struct FromInterface {
+  @inlinable public init() {}
+}
+public var testValue: FromInterface {
+  @inlinable get { return FromInterface() }
+}

test/ParseableInterface/ModuleCache/prebuilt-module-cache-archs.swift

@@ -0,0 +1,34 @@
+// RUN: %empty-directory(%t)
+// RUN: %empty-directory(%t/include/Lib.swiftmodule)
+// RUN: cp %S/Inputs/prebuilt-module-cache/Lib.swiftinterface %t/include/Lib.swiftmodule/$(basename %target-swiftmodule-name .swiftmodule).swiftinterface
+
+// Baseline check: if the prebuilt cache path does not exist, everything should
+// still work.
+// RUN: not %target-swift-frontend -typecheck -enable-parseable-module-interface -parse-stdlib -module-cache-path %t/MCP -sdk %t/include -I %t/include/ -prebuilt-module-cache-path %t/prebuilt-cache %s 2>&1 | %FileCheck -check-prefix=FROM-INTERFACE %s
+
+// Baseline check: if the module is not in the prebuilt cache, build it
+// normally.
+// RUN: %empty-directory(%t/prebuilt-cache)
+// RUN: not %target-swift-frontend -typecheck -enable-parseable-module-interface -parse-stdlib -module-cache-path %t/MCP -sdk %t/include -I %t/include/ -prebuilt-module-cache-path %t/prebuilt-cache %s 2>&1 | %FileCheck -check-prefix=FROM-INTERFACE %s
+
+// Do a manual prebuild, and see if it gets picked up.
+// RUN: %empty-directory(%t/MCP)
+// RUN: %empty-directory(%t/prebuilt-cache/Lib.swiftmodule)
+// RUN: sed -e 's/FromInterface/FromPrebuilt/g' %S/Inputs/prebuilt-module-cache/Lib.swiftinterface | %target-swift-frontend -parse-stdlib -module-cache-path %t/MCP -emit-module-path %t/prebuilt-cache/Lib.swiftmodule/%target-swiftmodule-name - -module-name Lib
+// RUN: not %target-swift-frontend -typecheck -enable-parseable-module-interface -parse-stdlib -module-cache-path %t/MCP -sdk %t/include -I %t/include/ -prebuilt-module-cache-path %t/prebuilt-cache %s 2>&1 | %FileCheck -check-prefix=FROM-PREBUILT %s
+// RUN: ls %t/MCP | grep -v swiftmodule
+
+// What if the module is invalid?
+// RUN: rm %t/prebuilt-cache/Lib.swiftmodule/%target-swiftmodule-name && touch %t/prebuilt-cache/Lib.swiftmodule/%target-swiftmodule-name
+// RUN: not %target-swift-frontend -typecheck -enable-parseable-module-interface -parse-stdlib -module-cache-path %t/MCP -sdk %t/include -I %t/include/ -prebuilt-module-cache-path %t/prebuilt-cache %s 2>&1 | %FileCheck -check-prefix=FROM-INTERFACE %s
+
+// What if the arch is missing?
+// RUN: rm %t/prebuilt-cache/Lib.swiftmodule/%target-swiftmodule-name
+// RUN: not %target-swift-frontend -typecheck -enable-parseable-module-interface -parse-stdlib -module-cache-path %t/MCP -sdk %t/include -I %t/include/ -prebuilt-module-cache-path %t/prebuilt-cache %s 2>&1 | %FileCheck -check-prefix=FROM-INTERFACE %s
+
+import Lib
+
+struct X {}
+let _: X = Lib.testValue
+// FROM-INTERFACE: [[@LINE-1]]:16: error: cannot convert value of type 'FromInterface' to specified type 'X'
+// FROM-PREBUILT: [[@LINE-2]]:16: error: cannot convert value of type 'FromPrebuilt' to specified type 'X'

test/ParseableInterface/ModuleCache/prebuilt-module-cache.swift

@@ -0,0 +1,37 @@
+// RUN: %empty-directory(%t)
+
+// Baseline check: if the prebuilt cache path does not exist, everything should
+// still work.
+// RUN: not %target-swift-frontend -typecheck -enable-parseable-module-interface -parse-stdlib -module-cache-path %t/MCP -sdk %S/Inputs -I %S/Inputs/prebuilt-module-cache/ -prebuilt-module-cache-path %t/prebuilt-cache %s 2>&1 | %FileCheck -check-prefix=FROM-INTERFACE %s
+
+// Baseline check: if the module is not in the prebuilt cache, build it
+// normally.
+// RUN: %empty-directory(%t/prebuilt-cache)
+// RUN: not %target-swift-frontend -typecheck -enable-parseable-module-interface -parse-stdlib -module-cache-path %t/MCP -sdk %S/Inputs -I %S/Inputs/prebuilt-module-cache/ -prebuilt-module-cache-path %t/prebuilt-cache %s 2>&1 | %FileCheck -check-prefix=FROM-INTERFACE %s
+
+// Do a manual prebuild, and see if it gets picked up.
+// RUN: %empty-directory(%t/MCP)
+// RUN: sed -e 's/FromInterface/FromPrebuilt/g' %S/Inputs/prebuilt-module-cache/Lib.swiftinterface | %target-swift-frontend -parse-stdlib -module-cache-path %t/MCP -emit-module-path %t/prebuilt-cache/Lib.swiftmodule - -module-name Lib
+// RUN: not %target-swift-frontend -typecheck -enable-parseable-module-interface -parse-stdlib -module-cache-path %t/MCP -sdk %S/Inputs -I %S/Inputs/prebuilt-module-cache/ -prebuilt-module-cache-path %t/prebuilt-cache %s 2>&1 | %FileCheck -check-prefix=FROM-PREBUILT %s
+// RUN: ls %t/MCP | grep -v swiftmodule
+
+// Try some variations on the detection that the search path is in the SDK:
+// RUN: not %target-swift-frontend -typecheck -enable-parseable-module-interface -parse-stdlib -module-cache-path %t/MCP -sdk %S/Inputs -I %S/Inputs/prebuilt-module-cache/ -prebuilt-module-cache-path %t/prebuilt-cache %s 2>&1 | %FileCheck -check-prefix=FROM-PREBUILT %s
+// RUN: not %target-swift-frontend -typecheck -enable-parseable-module-interface -parse-stdlib -module-cache-path %t/MCP -sdk %S//Inputs -I %S/Inputs/prebuilt-module-cache/ -prebuilt-module-cache-path %t/prebuilt-cache %s 2>&1 | %FileCheck -check-prefix=FROM-PREBUILT %s
+// RUN: not %target-swift-frontend -typecheck -enable-parseable-module-interface -parse-stdlib -module-cache-path %t/MCP -sdk %S/Inputs/prebuilt-module-cache -I %S/Inputs/prebuilt-module-cache/ -prebuilt-module-cache-path %t/prebuilt-cache %s 2>&1 | %FileCheck -check-prefix=FROM-PREBUILT %s
+// RUN: not %target-swift-frontend -typecheck -enable-parseable-module-interface -parse-stdlib -module-cache-path %t/MCP -sdk / -I %S/Inputs/prebuilt-module-cache/ -prebuilt-module-cache-path %t/prebuilt-cache %s 2>&1 | %FileCheck -check-prefix=FROM-PREBUILT %s
+
+// RUN: not %target-swift-frontend -typecheck -enable-parseable-module-interface -parse-stdlib -module-cache-path %t/MCP -sdk %S/Inputs/p -I %S/Inputs/prebuilt-module-cache/ -prebuilt-module-cache-path %t/prebuilt-cache %s 2>&1 | %FileCheck -check-prefix=FROM-INTERFACE %s
+// RUN: not %target-swift-frontend -typecheck -enable-parseable-module-interface -parse-stdlib -module-cache-path %t/MCP -sdk %S/garbage-path -I %S/Inputs/prebuilt-module-cache/ -prebuilt-module-cache-path %t/prebuilt-cache %s 2>&1 | %FileCheck -check-prefix=FROM-INTERFACE %s
+// RUN: not %target-swift-frontend -typecheck -enable-parseable-module-interface -parse-stdlib -module-cache-path %t/MCP -sdk "" -I %S/Inputs/prebuilt-module-cache/ -prebuilt-module-cache-path %t/prebuilt-cache %s 2>&1 | %FileCheck -check-prefix=FROM-INTERFACE %s
+
+// What if the module is invalid?
+// RUN: rm %t/prebuilt-cache/Lib.swiftmodule && touch %t/prebuilt-cache/Lib.swiftmodule
+// RUN: not %target-swift-frontend -typecheck -enable-parseable-module-interface -parse-stdlib -module-cache-path %t/MCP -sdk %S/Inputs/ -I %S/Inputs/prebuilt-module-cache/ -prebuilt-module-cache-path %t/prebuilt-cache %s 2>&1 | %FileCheck -check-prefix=FROM-INTERFACE %s
+
+import Lib
+
+struct X {}
+let _: X = Lib.testValue
+// FROM-INTERFACE: [[@LINE-1]]:16: error: cannot convert value of type 'FromInterface' to specified type 'X'
+// FROM-PREBUILT: [[@LINE-2]]:16: error: cannot convert value of type 'FromPrebuilt' to specified type 'X'