Skip to content

Commit c33cc0f

Browse files
cachemeifyoucancachemeifyoucan
committed
[Caching] Create clang importer from cc1 args directly
When caching build is enabled, teach dependency scanner to report command-lines with `-direct-clang-cc1-module-build` so the later compilation can instantiate clang importer with cc1 args directly. This avoids running clang driver code, which might involve file system lookups, which are the file deps that are not captured and might result in different compilation mode. rdar://119275464
1 parent 5feb998 commit c33cc0f

File tree

11 files changed

+180
-68
lines changed

11 files changed

+180
-68
lines changed

‎include/swift/ClangImporter/ClangImporter.h

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -503,6 +503,9 @@ class ClangImporter final : public ClangModuleLoader {
503503

504504
std::string getClangModuleHash() const;
505505

506+
/// Get clang import creation cc1 args for swift explicit module build.
507+
std::vector<std::string> getSwiftExplicitModuleDirectCC1Args() const;
508+
506509
/// If we already imported a given decl successfully, return the corresponding
507510
/// Swift decl as an Optional<Decl *>, but if we previously tried and failed
508511
/// to import said decl then return nullptr.

‎lib/ClangImporter/ClangImporter.cpp

Lines changed: 109 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -58,7 +58,9 @@
5858
#include "clang/Basic/Module.h"
5959
#include "clang/Basic/TargetInfo.h"
6060
#include "clang/Basic/Version.h"
61+
#include "clang/CAS/CASOptions.h"
6162
#include "clang/CodeGen/ObjectFilePCHContainerOperations.h"
63+
#include "clang/Frontend/CompilerInvocation.h"
6264
#include "clang/Frontend/FrontendActions.h"
6365
#include "clang/Frontend/TextDiagnosticPrinter.h"
6466
#include "clang/Frontend/Utils.h"
@@ -75,6 +77,7 @@
7577
#include "clang/Serialization/ASTWriter.h"
7678
#include "llvm/ADT/IntrusiveRefCntPtr.h"
7779
#include "llvm/ADT/STLExtras.h"
80+
#include "llvm/ADT/SmallVector.h"
7881
#include "llvm/ADT/StringExtras.h"
7982
#include "llvm/Support/CrashRecoveryContext.h"
8083
#include "llvm/Support/FileCollector.h"
@@ -1064,26 +1067,7 @@ std::optional<std::vector<std::string>> ClangImporter::getClangCC1Arguments(
10641067
ClangImporter *importer, ASTContext &ctx,
10651068
llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> VFS,
10661069
bool ignoreClangTarget) {
1067-
// If using direct cc1 module build, return extra args only.
1068-
if (ctx.ClangImporterOpts.DirectClangCC1ModuleBuild)
1069-
return ctx.ClangImporterOpts.ExtraArgs;
1070-
1071-
// Otherwise, create cc1 arguments from driver args.
1072-
auto driverArgs = getClangDriverArguments(ctx, ignoreClangTarget);
1073-
1074-
llvm::SmallVector<const char *> invocationArgs;
1075-
invocationArgs.reserve(driverArgs.size());
1076-
llvm::for_each(driverArgs, [&](const std::string &Arg) {
1077-
invocationArgs.push_back(Arg.c_str());
1078-
});
1079-
1080-
if (ctx.ClangImporterOpts.DumpClangDiagnostics) {
1081-
llvm::errs() << "clang importer driver args: '";
1082-
llvm::interleave(
1083-
invocationArgs, [](StringRef arg) { llvm::errs() << arg; },
1084-
[] { llvm::errs() << "' '"; });
1085-
llvm::errs() << "'\n";
1086-
}
1070+
std::unique_ptr<clang::CompilerInvocation> CI;
10871071

10881072
// Set up a temporary diagnostic client to report errors from parsing the
10891073
// command line, which may be important for Swift clients if, for example,
@@ -1095,24 +1079,62 @@ std::optional<std::vector<std::string>> ClangImporter::getClangCC1Arguments(
10951079
// clang::CompilerInstance is created.
10961080
llvm::IntrusiveRefCntPtr<clang::DiagnosticOptions> tempDiagOpts{
10971081
new clang::DiagnosticOptions};
1098-
10991082
auto *tempDiagClient =
11001083
new ClangDiagnosticConsumer(importer->Impl, *tempDiagOpts,
11011084
ctx.ClangImporterOpts.DumpClangDiagnostics);
1102-
11031085
auto clangDiags = clang::CompilerInstance::createDiagnostics(
11041086
tempDiagOpts.get(), tempDiagClient,
11051087
/*owned*/ true);
11061088

1107-
clang::CreateInvocationOptions CIOpts;
1108-
CIOpts.VFS = VFS;
1109-
CIOpts.Diags = clangDiags;
1110-
CIOpts.RecoverOnError = false;
1111-
CIOpts.ProbePrecompiled = true;
1112-
auto CI = clang::createInvocation(invocationArgs, std::move(CIOpts));
1089+
// If using direct cc1 module build, use extra args to setup ClangImporter.
1090+
if (ctx.ClangImporterOpts.DirectClangCC1ModuleBuild) {
1091+
llvm::SmallVector<const char *> clangArgs;
1092+
clangArgs.reserve(ctx.ClangImporterOpts.ExtraArgs.size());
1093+
llvm::for_each(
1094+
ctx.ClangImporterOpts.ExtraArgs,
1095+
[&](const std::string &Arg) { clangArgs.push_back(Arg.c_str()); });
1096+
1097+
// Try parse extra args, if failed, return nullopt.
1098+
CI = std::make_unique<clang::CompilerInvocation>();
1099+
if (!clang::CompilerInvocation::CreateFromArgs(*CI, clangArgs,
1100+
*clangDiags))
1101+
return std::nullopt;
1102+
1103+
// Forwards some options from swift to clang even using direct mode. This is
1104+
// to reduce the number of argument passing on the command-line and swift
1105+
// compiler can be more efficient to compute swift cache key without having
1106+
// the knowledge about clang command-line options.
1107+
if (ctx.ClangImporterOpts.CASOpts) {
1108+
CI->getFrontendOpts().IncludeTimestamps = false;
1109+
CI->getCASOpts() = *ctx.ClangImporterOpts.CASOpts;
1110+
}
1111+
} else {
1112+
// Otherwise, create cc1 arguments from driver args.
1113+
auto driverArgs = getClangDriverArguments(ctx, ignoreClangTarget);
1114+
1115+
llvm::SmallVector<const char *> invocationArgs;
1116+
invocationArgs.reserve(driverArgs.size());
1117+
llvm::for_each(driverArgs, [&](const std::string &Arg) {
1118+
invocationArgs.push_back(Arg.c_str());
1119+
});
1120+
1121+
if (ctx.ClangImporterOpts.DumpClangDiagnostics) {
1122+
llvm::errs() << "clang importer driver args: '";
1123+
llvm::interleave(
1124+
invocationArgs, [](StringRef arg) { llvm::errs() << arg; },
1125+
[] { llvm::errs() << "' '"; });
1126+
llvm::errs() << "'\n";
1127+
}
11131128

1114-
if (!CI)
1115-
return std::nullopt;
1129+
clang::CreateInvocationOptions CIOpts;
1130+
CIOpts.VFS = VFS;
1131+
CIOpts.Diags = clangDiags;
1132+
CIOpts.RecoverOnError = false;
1133+
CIOpts.ProbePrecompiled = true;
1134+
CI = clang::createInvocation(invocationArgs, std::move(CIOpts));
1135+
if (!CI)
1136+
return std::nullopt;
1137+
}
11161138

11171139
// FIXME: clang fails to generate a module if there is a `-fmodule-map-file`
11181140
// argument pointing to a missing file.
@@ -3894,6 +3916,63 @@ std::string ClangImporter::getClangModuleHash() const {
38943916
return Impl.Invocation->getModuleHash(Impl.Instance->getDiagnostics());
38953917
}
38963918

3919+
std::vector<std::string>
3920+
ClangImporter::getSwiftExplicitModuleDirectCC1Args() const {
3921+
llvm::SmallVector<const char*> clangArgs;
3922+
clangArgs.reserve(Impl.ClangArgs.size());
3923+
llvm::for_each(Impl.ClangArgs, [&](const std::string &Arg) {
3924+
clangArgs.push_back(Arg.c_str());
3925+
});
3926+
3927+
clang::CompilerInvocation instance;
3928+
clang::DiagnosticsEngine clangDiags(new clang::DiagnosticIDs(),
3929+
new clang::DiagnosticOptions(),
3930+
new clang::IgnoringDiagConsumer());
3931+
bool success = clang::CompilerInvocation::CreateFromArgs(instance, clangArgs,
3932+
clangDiags);
3933+
(void)success;
3934+
assert(success && "clang options from clangImporter failed to parse");
3935+
3936+
if (!Impl.SwiftContext.ClangImporterOpts.CASOpts)
3937+
return instance.getCC1CommandLine();
3938+
3939+
// Clear some options that are not needed.
3940+
// CASOpts are forwarded from swift arguments.
3941+
instance.getCASOpts() = clang::CASOptions();
3942+
3943+
// HeaderSearchOptions.
3944+
auto &HSOpts = instance.getHeaderSearchOpts();
3945+
HSOpts.BuildSessionTimestamp = 0;
3946+
HSOpts.ImplicitModuleMaps = 0;
3947+
HSOpts.VFSOverlayFiles.clear();
3948+
3949+
// LanguageOptions.
3950+
auto &LangOpts = instance.getLangOpts();
3951+
LangOpts.ImplicitModules = false;
3952+
3953+
// FrontendOptions.
3954+
auto &FEOpts = instance.getFrontendOpts();
3955+
FEOpts.IncludeTimestamps = false;
3956+
FEOpts.ModuleMapFiles.clear();
3957+
3958+
if (Impl.SwiftContext.ClangImporterOpts.UseClangIncludeTree) {
3959+
HSOpts.UserEntries.clear();
3960+
HSOpts.SystemHeaderPrefixes.clear();
3961+
3962+
// FileSystemOptions.
3963+
auto FSOpts = instance.getFileSystemOpts();
3964+
FSOpts.WorkingDir.clear();
3965+
3966+
// PreprocessorOptions.
3967+
auto &PPOpts = instance.getPreprocessorOpts();
3968+
PPOpts.Macros.clear();
3969+
PPOpts.MacroIncludes.clear();
3970+
PPOpts.Includes.clear();
3971+
}
3972+
3973+
return instance.getCC1CommandLine();
3974+
}
3975+
38973976
llvm::Optional<Decl *>
38983977
ClangImporter::importDeclCached(const clang::NamedDecl *ClangDecl) {
38993978
return Impl.importDeclCached(ClangDecl, Impl.CurrentVersion);

‎lib/DependencyScan/ModuleDependencyScanner.cpp

Lines changed: 15 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -345,6 +345,7 @@ ModuleDependencyScanner::getMainModuleDependencyInfo(
345345
{"-Xcc", "-target", "-Xcc", ScanASTContext.LangOpts.Target.str()});
346346

347347
std::string rootID;
348+
std::vector<std::string> buildArgs;
348349
if (tracker) {
349350
tracker->startTracking();
350351
for (auto fileUnit : mainModule->getFiles()) {
@@ -370,10 +371,22 @@ ModuleDependencyScanner::getMainModuleDependencyInfo(
370371
return std::make_error_code(std::errc::io_error);
371372
}
372373
rootID = root->getID().toString();
374+
375+
buildArgs.push_back("-direct-clang-cc1-module-build");
376+
for (auto &arg : clangImporter->getSwiftExplicitModuleDirectCC1Args()) {
377+
buildArgs.push_back("-Xcc");
378+
buildArgs.push_back(arg);
379+
}
373380
}
374381

375-
auto mainDependencies =
376-
ModuleDependencyInfo::forSwiftSourceModule(rootID, {}, {}, ExtraPCMArgs);
382+
llvm::SmallVector<StringRef> buildCommands;
383+
buildCommands.reserve(buildArgs.size());
384+
llvm::for_each(buildArgs, [&](const std::string &arg) {
385+
buildCommands.emplace_back(arg);
386+
});
387+
388+
auto mainDependencies = ModuleDependencyInfo::forSwiftSourceModule(
389+
rootID, buildCommands, {}, ExtraPCMArgs);
377390

378391
llvm::StringSet<> alreadyAddedModules;
379392
// Compute Implicit dependencies of the main module

‎lib/DependencyScan/ScanDependencies.cpp

Lines changed: 3 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -293,19 +293,11 @@ static llvm::Error resolveExplicitModuleInputs(
293293
}
294294
}
295295
if (!clangDepDetails->moduleCacheKey.empty()) {
296-
auto appendXclang = [&]() {
297-
if (!resolvingDepInfo.isClangModule()) {
298-
// clang module build using cc1 arg so this is not needed.
299-
commandLine.push_back("-Xcc");
300-
commandLine.push_back("-Xclang");
301-
}
302-
commandLine.push_back("-Xcc");
303-
};
304-
appendXclang();
296+
commandLine.push_back("-Xcc");
305297
commandLine.push_back("-fmodule-file-cache-key");
306-
appendXclang();
298+
commandLine.push_back("-Xcc");
307299
commandLine.push_back(remapPath(clangDepDetails->pcmOutputPath));
308-
appendXclang();
300+
commandLine.push_back("-Xcc");
309301
commandLine.push_back(clangDepDetails->moduleCacheKey);
310302
}
311303

‎lib/Frontend/ModuleInterfaceLoader.cpp

Lines changed: 17 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -2002,10 +2002,16 @@ InterfaceSubContextDelegateImpl::runInSubCompilerInstance(StringRef moduleName,
20022002
== originalTargetTriple.getEnvironment()) {
20032003
parsedTargetTriple.setArchName(originalTargetTriple.getArchName());
20042004
subInvocation.setTargetTriple(parsedTargetTriple.str());
2005+
}
20052006

2006-
// Overload the target in the BuildArgs as well
2007-
BuildArgs.push_back("-target");
2008-
BuildArgs.push_back(parsedTargetTriple.str());
2007+
// Find and overload all "-target" to be parsedTargetTriple. This make sure
2008+
// the build command for the interface is the same no matter what the parent
2009+
// triple is so there is no need to spawn identical jobs.
2010+
assert(llvm::find(BuildArgs, "-target") != BuildArgs.end() &&
2011+
"missing target option");
2012+
for (unsigned idx = 0, end = BuildArgs.size(); idx < end; ++idx) {
2013+
if (BuildArgs[idx] == "-target" && ++idx < end)
2014+
BuildArgs[idx] = parsedTargetTriple.str();
20092015
}
20102016

20112017
// restore `StrictImplicitModuleContext`
@@ -2354,6 +2360,11 @@ struct ExplicitCASModuleLoader::Implementation {
23542360

23552361
std::set<std::string> moduleMapsSeen;
23562362
std::vector<std::string> &extraClangArgs = Ctx.ClangImporterOpts.ExtraArgs;
2363+
// Append -Xclang if we are not in direct cc1 mode.
2364+
auto appendXclang = [&]() {
2365+
if (!Ctx.ClangImporterOpts.DirectClangCC1ModuleBuild)
2366+
extraClangArgs.push_back("-Xclang");
2367+
};
23572368
for (auto &entry : ExplicitClangModuleMap) {
23582369
const auto &moduleMapPath = entry.getValue().moduleMapPath;
23592370
if (!moduleMapPath.empty() &&
@@ -2372,11 +2383,11 @@ struct ExplicitCASModuleLoader::Implementation {
23722383
}
23732384
auto cachePath = entry.getValue().moduleCacheKey;
23742385
if (cachePath) {
2375-
extraClangArgs.push_back("-Xclang");
2386+
appendXclang();
23762387
extraClangArgs.push_back("-fmodule-file-cache-key");
2377-
extraClangArgs.push_back("-Xclang");
2388+
appendXclang();
23782389
extraClangArgs.push_back(modulePath);
2379-
extraClangArgs.push_back("-Xclang");
2390+
appendXclang();
23802391
extraClangArgs.push_back(*cachePath);
23812392
}
23822393
}

‎lib/Serialization/ScanningLoaders.cpp

Lines changed: 17 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -170,10 +170,23 @@ SwiftModuleScanner::scanInterfaceFile(Twine moduleInterfacePath,
170170
// Add explicit Swift dependency compilation flags
171171
Args.push_back("-explicit-interface-module-build");
172172
Args.push_back("-disable-implicit-swift-modules");
173-
Args.push_back("-Xcc");
174-
Args.push_back("-fno-implicit-modules");
175-
Args.push_back("-Xcc");
176-
Args.push_back("-fno-implicit-module-maps");
173+
174+
// Handle clang arguments. For caching build, all arguments are passed
175+
// with `-direct-clang-cc1-module-build`.
176+
if (Ctx.ClangImporterOpts.CASOpts) {
177+
Args.push_back("-direct-clang-cc1-module-build");
178+
auto *importer =
179+
static_cast<ClangImporter *>(Ctx.getClangModuleLoader());
180+
for (auto &Arg : importer->getSwiftExplicitModuleDirectCC1Args()) {
181+
Args.push_back("-Xcc");
182+
Args.push_back(Arg);
183+
}
184+
} else {
185+
Args.push_back("-Xcc");
186+
Args.push_back("-fno-implicit-modules");
187+
Args.push_back("-Xcc");
188+
Args.push_back("-fno-implicit-module-maps");
189+
}
177190
for (const auto &candidate : compiledCandidates) {
178191
Args.push_back("-candidate-module-file");
179192
Args.push_back(candidate);

‎test/CAS/can-import.swift

Lines changed: 2 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,3 @@
1-
// rdar://119964830 Temporarily disabling in Linux
2-
// UNSUPPORTED: OS=linux-gnu
3-
41
// RUN: %empty-directory(%t)
52
// RUN: split-file %s %t
63

@@ -24,7 +21,7 @@
2421
// RUN: %{python} %S/Inputs/SwiftDepsExtractor.py %t/deps.json clang:SwiftShims moduleCacheKey | tr -d '\n' > %t/Shims.key
2522
// RUN: %{python} %S/Inputs/SwiftDepsExtractor.py %t/deps.json clang:A moduleCacheKey | tr -d '\n' > %t/A.key
2623
// RUN: %{python} %S/Inputs/SwiftDepsExtractor.py %t/deps.json clang:B moduleCacheKey | tr -d '\n' > %t/B.key
27-
// RUN: %{python} %S/Inputs/BuildCommandExtractor.py %t/deps.json MyApp > %t/MyApp.cmd
24+
// RUN: %{python} %S/Inputs/BuildCommandExtractor.py %t/deps.json Test > %t/MyApp.cmd
2825

2926
// RUN: echo "[{" > %/t/map.json
3027
// RUN: echo "\"moduleName\": \"Swift\"," >> %/t/map.json
@@ -64,7 +61,7 @@
6461
// RUN: -typecheck -cache-compile-job -cas-path %t/cas \
6562
// RUN: -swift-version 5 -disable-implicit-swift-modules \
6663
// RUN: -disable-implicit-string-processing-module-import -disable-implicit-concurrency-module-import \
67-
// RUN: -module-name MyApp -explicit-swift-module-map-file @%t/map.casid \
64+
// RUN: -module-name Test -explicit-swift-module-map-file @%t/map.casid \
6865
// RUN: %t/main.swift @%t/MyApp.cmd
6966

7067
//--- main.swift

‎test/CAS/include-tree.swift

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,3 @@
1-
// rdar://119964830 Temporarily disabling in Linux
2-
// UNSUPPORTED: OS=linux-gnu
3-
41
// RUN: %empty-directory(%t)
52
// RUN: split-file %s %t
63

‎test/CAS/module_deps.swift

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -31,9 +31,8 @@
3131
// RUN: %{python} %S/Inputs/SwiftDepsExtractor.py %t/deps.json clang:F casFSRootID > %t/F_fs.casid
3232
// RUN: llvm-cas --cas %t/cas --ls-tree-recursive @%t/F_fs.casid | %FileCheck %s -check-prefix FS_ROOT_F
3333

34-
/// make sure the number of CASFS is correct. 10 entries with 2 line each + 1 extra bracket
35-
// RUN: NUM_CMDS=$(%{python} %S/Inputs/SwiftDepsExtractor.py %t/deps.json deps commandLine | wc -l)
36-
// RUN: if [ ! $NUM_CMDS -eq 21 ]; then echo "wrong number of CASFS from scanning"; exit 1; fi
34+
// RUN: %{python} %S/Inputs/SwiftDepsExtractor.py %t/deps.json deps commandLine > %t/deps.cmd
35+
// RUN: %FileCheck %s -check-prefix MAIN_CMD -input-file=%t/deps.cmd
3736

3837
// FS_ROOT_E-DAG: E.swiftinterface
3938
// FS_ROOT_E-DAG: SDKSettings.json
@@ -49,6 +48,10 @@
4948
// FS_ROOT_F: CHeaders/X.h
5049
// FS_ROOT_F: CHeaders/module.modulemap
5150

51+
// MAIN_CMD: -direct-clang-cc1-module-build
52+
// MAIN_CMD: -cas-fs
53+
// MAIN_CMD-NOT: -clang-include-tree-root
54+
5255
import C
5356
import E
5457
import G

0 commit comments

Comments
 (0)