Skip to content

Commit c5a2b7a

Browse files
cachemeifyoucancachemeifyoucan
committed
[Caching] Create clang importer from cc1 args directly
When caching build is enabled, teach dependency scanner to report command-lines with `-direct-clang-cc1-module-build` so the later compilation can instantiate clang importer with cc1 args directly. This avoids running clang driver code, which might involve file system lookups, which are the file deps that are not captured and might result in different compilation mode. rdar://119275464
1 parent c9fcab1 commit c5a2b7a

14 files changed

+180
-75
lines changed

‎include/swift/ClangImporter/ClangImporter.h

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -503,6 +503,9 @@ class ClangImporter final : public ClangModuleLoader {
503503

504504
std::string getClangModuleHash() const;
505505

506+
/// Get clang import creation cc1 args for swift explicit module build.
507+
std::vector<std::string> getSwiftExplicitModuleDirectCC1Args() const;
508+
506509
/// If we already imported a given decl successfully, return the corresponding
507510
/// Swift decl as an Optional<Decl *>, but if we previously tried and failed
508511
/// to import said decl then return nullptr.

‎lib/ClangImporter/ClangImporter.cpp

Lines changed: 109 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -58,7 +58,9 @@
5858
#include "clang/Basic/Module.h"
5959
#include "clang/Basic/TargetInfo.h"
6060
#include "clang/Basic/Version.h"
61+
#include "clang/CAS/CASOptions.h"
6162
#include "clang/CodeGen/ObjectFilePCHContainerOperations.h"
63+
#include "clang/Frontend/CompilerInvocation.h"
6264
#include "clang/Frontend/FrontendActions.h"
6365
#include "clang/Frontend/TextDiagnosticPrinter.h"
6466
#include "clang/Frontend/Utils.h"
@@ -75,6 +77,7 @@
7577
#include "clang/Serialization/ASTWriter.h"
7678
#include "llvm/ADT/IntrusiveRefCntPtr.h"
7779
#include "llvm/ADT/STLExtras.h"
80+
#include "llvm/ADT/SmallVector.h"
7881
#include "llvm/ADT/StringExtras.h"
7982
#include "llvm/Support/CrashRecoveryContext.h"
8083
#include "llvm/Support/FileCollector.h"
@@ -1072,26 +1075,7 @@ std::optional<std::vector<std::string>> ClangImporter::getClangCC1Arguments(
10721075
ClangImporter *importer, ASTContext &ctx,
10731076
llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> VFS,
10741077
bool ignoreClangTarget) {
1075-
// If using direct cc1 module build, return extra args only.
1076-
if (ctx.ClangImporterOpts.DirectClangCC1ModuleBuild)
1077-
return ctx.ClangImporterOpts.ExtraArgs;
1078-
1079-
// Otherwise, create cc1 arguments from driver args.
1080-
auto driverArgs = getClangDriverArguments(ctx, ignoreClangTarget);
1081-
1082-
llvm::SmallVector<const char *> invocationArgs;
1083-
invocationArgs.reserve(driverArgs.size());
1084-
llvm::for_each(driverArgs, [&](const std::string &Arg) {
1085-
invocationArgs.push_back(Arg.c_str());
1086-
});
1087-
1088-
if (ctx.ClangImporterOpts.DumpClangDiagnostics) {
1089-
llvm::errs() << "clang importer driver args: '";
1090-
llvm::interleave(
1091-
invocationArgs, [](StringRef arg) { llvm::errs() << arg; },
1092-
[] { llvm::errs() << "' '"; });
1093-
llvm::errs() << "'\n";
1094-
}
1078+
std::unique_ptr<clang::CompilerInvocation> CI;
10951079

10961080
// Set up a temporary diagnostic client to report errors from parsing the
10971081
// command line, which may be important for Swift clients if, for example,
@@ -1103,24 +1087,62 @@ std::optional<std::vector<std::string>> ClangImporter::getClangCC1Arguments(
11031087
// clang::CompilerInstance is created.
11041088
llvm::IntrusiveRefCntPtr<clang::DiagnosticOptions> tempDiagOpts{
11051089
new clang::DiagnosticOptions};
1106-
11071090
auto *tempDiagClient =
11081091
new ClangDiagnosticConsumer(importer->Impl, *tempDiagOpts,
11091092
ctx.ClangImporterOpts.DumpClangDiagnostics);
1110-
11111093
auto clangDiags = clang::CompilerInstance::createDiagnostics(
11121094
tempDiagOpts.get(), tempDiagClient,
11131095
/*owned*/ true);
11141096

1115-
clang::CreateInvocationOptions CIOpts;
1116-
CIOpts.VFS = VFS;
1117-
CIOpts.Diags = clangDiags;
1118-
CIOpts.RecoverOnError = false;
1119-
CIOpts.ProbePrecompiled = true;
1120-
auto CI = clang::createInvocation(invocationArgs, std::move(CIOpts));
1097+
// If using direct cc1 module build, use extra args to setup ClangImporter.
1098+
if (ctx.ClangImporterOpts.DirectClangCC1ModuleBuild) {
1099+
llvm::SmallVector<const char *> clangArgs;
1100+
clangArgs.reserve(ctx.ClangImporterOpts.ExtraArgs.size());
1101+
llvm::for_each(
1102+
ctx.ClangImporterOpts.ExtraArgs,
1103+
[&](const std::string &Arg) { clangArgs.push_back(Arg.c_str()); });
1104+
1105+
// Try parse extra args, if failed, return nullopt.
1106+
CI = std::make_unique<clang::CompilerInvocation>();
1107+
if (!clang::CompilerInvocation::CreateFromArgs(*CI, clangArgs,
1108+
*clangDiags))
1109+
return std::nullopt;
1110+
1111+
// Forwards some options from swift to clang even using direct mode. This is
1112+
// to reduce the number of argument passing on the command-line and swift
1113+
// compiler can be more efficient to compute swift cache key without having
1114+
// the knowledge about clang command-line options.
1115+
if (ctx.ClangImporterOpts.CASOpts) {
1116+
CI->getFrontendOpts().IncludeTimestamps = false;
1117+
CI->getCASOpts() = *ctx.ClangImporterOpts.CASOpts;
1118+
}
1119+
} else {
1120+
// Otherwise, create cc1 arguments from driver args.
1121+
auto driverArgs = getClangDriverArguments(ctx, ignoreClangTarget);
1122+
1123+
llvm::SmallVector<const char *> invocationArgs;
1124+
invocationArgs.reserve(driverArgs.size());
1125+
llvm::for_each(driverArgs, [&](const std::string &Arg) {
1126+
invocationArgs.push_back(Arg.c_str());
1127+
});
1128+
1129+
if (ctx.ClangImporterOpts.DumpClangDiagnostics) {
1130+
llvm::errs() << "clang importer driver args: '";
1131+
llvm::interleave(
1132+
invocationArgs, [](StringRef arg) { llvm::errs() << arg; },
1133+
[] { llvm::errs() << "' '"; });
1134+
llvm::errs() << "'\n";
1135+
}
11211136

1122-
if (!CI)
1123-
return std::nullopt;
1137+
clang::CreateInvocationOptions CIOpts;
1138+
CIOpts.VFS = VFS;
1139+
CIOpts.Diags = clangDiags;
1140+
CIOpts.RecoverOnError = false;
1141+
CIOpts.ProbePrecompiled = true;
1142+
CI = clang::createInvocation(invocationArgs, std::move(CIOpts));
1143+
if (!CI)
1144+
return std::nullopt;
1145+
}
11241146

11251147
// FIXME: clang fails to generate a module if there is a `-fmodule-map-file`
11261148
// argument pointing to a missing file.
@@ -3924,6 +3946,63 @@ std::string ClangImporter::getClangModuleHash() const {
39243946
return Impl.Invocation->getModuleHash(Impl.Instance->getDiagnostics());
39253947
}
39263948

3949+
std::vector<std::string>
3950+
ClangImporter::getSwiftExplicitModuleDirectCC1Args() const {
3951+
llvm::SmallVector<const char*> clangArgs;
3952+
clangArgs.reserve(Impl.ClangArgs.size());
3953+
llvm::for_each(Impl.ClangArgs, [&](const std::string &Arg) {
3954+
clangArgs.push_back(Arg.c_str());
3955+
});
3956+
3957+
clang::CompilerInvocation instance;
3958+
clang::DiagnosticsEngine clangDiags(new clang::DiagnosticIDs(),
3959+
new clang::DiagnosticOptions(),
3960+
new clang::IgnoringDiagConsumer());
3961+
bool success = clang::CompilerInvocation::CreateFromArgs(instance, clangArgs,
3962+
clangDiags);
3963+
(void)success;
3964+
assert(success && "clang options from clangImporter failed to parse");
3965+
3966+
if (!Impl.SwiftContext.ClangImporterOpts.CASOpts)
3967+
return instance.getCC1CommandLine();
3968+
3969+
// Clear some options that are not needed.
3970+
// CASOpts are forwarded from swift arguments.
3971+
instance.getCASOpts() = clang::CASOptions();
3972+
3973+
// HeaderSearchOptions.
3974+
auto &HSOpts = instance.getHeaderSearchOpts();
3975+
HSOpts.BuildSessionTimestamp = 0;
3976+
HSOpts.ImplicitModuleMaps = 0;
3977+
HSOpts.VFSOverlayFiles.clear();
3978+
3979+
// LanguageOptions.
3980+
auto &LangOpts = instance.getLangOpts();
3981+
LangOpts.ImplicitModules = false;
3982+
3983+
// FrontendOptions.
3984+
auto &FEOpts = instance.getFrontendOpts();
3985+
FEOpts.IncludeTimestamps = false;
3986+
FEOpts.ModuleMapFiles.clear();
3987+
3988+
if (Impl.SwiftContext.ClangImporterOpts.UseClangIncludeTree) {
3989+
HSOpts.UserEntries.clear();
3990+
HSOpts.SystemHeaderPrefixes.clear();
3991+
3992+
// FileSystemOptions.
3993+
auto &FSOpts = instance.getFileSystemOpts();
3994+
FSOpts.WorkingDir.clear();
3995+
3996+
// PreprocessorOptions.
3997+
auto &PPOpts = instance.getPreprocessorOpts();
3998+
PPOpts.Macros.clear();
3999+
PPOpts.MacroIncludes.clear();
4000+
PPOpts.Includes.clear();
4001+
}
4002+
4003+
return instance.getCC1CommandLine();
4004+
}
4005+
39274006
llvm::Optional<Decl *>
39284007
ClangImporter::importDeclCached(const clang::NamedDecl *ClangDecl) {
39294008
return Impl.importDeclCached(ClangDecl, Impl.CurrentVersion);

‎lib/DependencyScan/ModuleDependencyScanner.cpp

Lines changed: 15 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -346,6 +346,7 @@ ModuleDependencyScanner::getMainModuleDependencyInfo(
346346
{"-Xcc", "-target", "-Xcc", ScanASTContext.LangOpts.Target.str()});
347347

348348
std::string rootID;
349+
std::vector<std::string> buildArgs;
349350
if (tracker) {
350351
tracker->startTracking();
351352
for (auto fileUnit : mainModule->getFiles()) {
@@ -371,10 +372,22 @@ ModuleDependencyScanner::getMainModuleDependencyInfo(
371372
return std::make_error_code(std::errc::io_error);
372373
}
373374
rootID = root->getID().toString();
375+
376+
buildArgs.push_back("-direct-clang-cc1-module-build");
377+
for (auto &arg : clangImporter->getSwiftExplicitModuleDirectCC1Args()) {
378+
buildArgs.push_back("-Xcc");
379+
buildArgs.push_back(arg);
380+
}
374381
}
375382

376-
auto mainDependencies =
377-
ModuleDependencyInfo::forSwiftSourceModule(rootID, {}, {}, ExtraPCMArgs);
383+
llvm::SmallVector<StringRef> buildCommands;
384+
buildCommands.reserve(buildArgs.size());
385+
llvm::for_each(buildArgs, [&](const std::string &arg) {
386+
buildCommands.emplace_back(arg);
387+
});
388+
389+
auto mainDependencies = ModuleDependencyInfo::forSwiftSourceModule(
390+
rootID, buildCommands, {}, ExtraPCMArgs);
378391

379392
llvm::StringSet<> alreadyAddedModules;
380393
// Compute Implicit dependencies of the main module

‎lib/DependencyScan/ScanDependencies.cpp

Lines changed: 3 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -293,19 +293,11 @@ static llvm::Error resolveExplicitModuleInputs(
293293
}
294294
}
295295
if (!clangDepDetails->moduleCacheKey.empty()) {
296-
auto appendXclang = [&]() {
297-
if (!resolvingDepInfo.isClangModule()) {
298-
// clang module build using cc1 arg so this is not needed.
299-
commandLine.push_back("-Xcc");
300-
commandLine.push_back("-Xclang");
301-
}
302-
commandLine.push_back("-Xcc");
303-
};
304-
appendXclang();
296+
commandLine.push_back("-Xcc");
305297
commandLine.push_back("-fmodule-file-cache-key");
306-
appendXclang();
298+
commandLine.push_back("-Xcc");
307299
commandLine.push_back(remapPath(clangDepDetails->pcmOutputPath));
308-
appendXclang();
300+
commandLine.push_back("-Xcc");
309301
commandLine.push_back(clangDepDetails->moduleCacheKey);
310302
}
311303

‎lib/Frontend/ModuleInterfaceLoader.cpp

Lines changed: 19 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1691,6 +1691,8 @@ void InterfaceSubContextDelegateImpl::inheritOptionsForBuildingInterface(
16911691
GenericArgs.push_back(ArgSaver.save(pair));
16921692
}
16931693
}
1694+
// ClangIncludeTree is default on when caching is enabled.
1695+
genericSubInvocation.getClangImporterOptions().UseClangIncludeTree = true;
16941696
}
16951697

16961698
if (!clangImporterOpts.UseClangIncludeTree) {
@@ -2040,10 +2042,16 @@ InterfaceSubContextDelegateImpl::runInSubCompilerInstance(StringRef moduleName,
20402042
== originalTargetTriple.getEnvironment()) {
20412043
parsedTargetTriple.setArchName(originalTargetTriple.getArchName());
20422044
subInvocation.setTargetTriple(parsedTargetTriple.str());
2045+
}
20432046

2044-
// Overload the target in the BuildArgs as well
2045-
BuildArgs.push_back("-target");
2046-
BuildArgs.push_back(parsedTargetTriple.str());
2047+
// Find and overload all "-target" to be parsedTargetTriple. This make sure
2048+
// the build command for the interface is the same no matter what the parent
2049+
// triple is so there is no need to spawn identical jobs.
2050+
assert(llvm::find(BuildArgs, "-target") != BuildArgs.end() &&
2051+
"missing target option");
2052+
for (unsigned idx = 0, end = BuildArgs.size(); idx < end; ++idx) {
2053+
if (BuildArgs[idx] == "-target" && ++idx < end)
2054+
BuildArgs[idx] = parsedTargetTriple.str();
20472055
}
20482056

20492057
// restore `StrictImplicitModuleContext`
@@ -2394,6 +2402,11 @@ struct ExplicitCASModuleLoader::Implementation {
23942402

23952403
std::set<std::string> moduleMapsSeen;
23962404
std::vector<std::string> &extraClangArgs = Ctx.ClangImporterOpts.ExtraArgs;
2405+
// Append -Xclang if we are not in direct cc1 mode.
2406+
auto appendXclang = [&]() {
2407+
if (!Ctx.ClangImporterOpts.DirectClangCC1ModuleBuild)
2408+
extraClangArgs.push_back("-Xclang");
2409+
};
23972410
for (auto &entry : ExplicitClangModuleMap) {
23982411
const auto &moduleMapPath = entry.getValue().moduleMapPath;
23992412
if (!moduleMapPath.empty() &&
@@ -2412,11 +2425,11 @@ struct ExplicitCASModuleLoader::Implementation {
24122425
}
24132426
auto cachePath = entry.getValue().moduleCacheKey;
24142427
if (cachePath) {
2415-
extraClangArgs.push_back("-Xclang");
2428+
appendXclang();
24162429
extraClangArgs.push_back("-fmodule-file-cache-key");
2417-
extraClangArgs.push_back("-Xclang");
2430+
appendXclang();
24182431
extraClangArgs.push_back(modulePath);
2419-
extraClangArgs.push_back("-Xclang");
2432+
appendXclang();
24202433
extraClangArgs.push_back(*cachePath);
24212434
}
24222435
}

‎lib/Serialization/ScanningLoaders.cpp

Lines changed: 17 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -170,10 +170,23 @@ SwiftModuleScanner::scanInterfaceFile(Twine moduleInterfacePath,
170170
// Add explicit Swift dependency compilation flags
171171
Args.push_back("-explicit-interface-module-build");
172172
Args.push_back("-disable-implicit-swift-modules");
173-
Args.push_back("-Xcc");
174-
Args.push_back("-fno-implicit-modules");
175-
Args.push_back("-Xcc");
176-
Args.push_back("-fno-implicit-module-maps");
173+
174+
// Handle clang arguments. For caching build, all arguments are passed
175+
// with `-direct-clang-cc1-module-build`.
176+
if (Ctx.ClangImporterOpts.CASOpts) {
177+
Args.push_back("-direct-clang-cc1-module-build");
178+
auto *importer =
179+
static_cast<ClangImporter *>(Ctx.getClangModuleLoader());
180+
for (auto &Arg : importer->getSwiftExplicitModuleDirectCC1Args()) {
181+
Args.push_back("-Xcc");
182+
Args.push_back(Arg);
183+
}
184+
} else {
185+
Args.push_back("-Xcc");
186+
Args.push_back("-fno-implicit-modules");
187+
Args.push_back("-Xcc");
188+
Args.push_back("-fno-implicit-module-maps");
189+
}
177190
for (const auto &candidate : compiledCandidates) {
178191
Args.push_back("-candidate-module-file");
179192
Args.push_back(candidate);

‎test/CAS/cache_replay.swift

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,3 @@
1-
// rdar://119964830 Temporarily disabling in Linux
2-
// UNSUPPORTED: OS=linux-gnu
3-
41
// RUN: %empty-directory(%t)
52
// RUN: split-file %s %t
63

‎test/CAS/cached_diagnostics.swift

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,3 @@
1-
// rdar://119964830 Temporarily disabling in Linux
2-
// UNSUPPORTED: OS=linux-gnu
3-
41
// RUN: %empty-directory(%t)
52

63
// RUN: %target-swift-frontend -scan-dependencies -module-name Test -O -import-objc-header %S/Inputs/objc.h \

‎test/CAS/cached_diagnostics_remap.swift

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,3 @@
1-
// rdar://119964830 Temporarily disabling in Linux
2-
// UNSUPPORTED: OS=linux-gnu
3-
41
// RUN: %empty-directory(%t)
52
// RUN: split-file %s %t
63

‎test/CAS/can-import.swift

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,3 @@
1-
// rdar://119964830 Temporarily disabling in Linux
2-
// UNSUPPORTED: OS=linux-gnu
3-
41
// RUN: %empty-directory(%t)
52
// RUN: split-file %s %t
63

‎test/CAS/include-tree.swift

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,3 @@
1-
// rdar://119964830 Temporarily disabling in Linux
2-
// UNSUPPORTED: OS=linux-gnu
3-
41
// RUN: %empty-directory(%t)
52
// RUN: split-file %s %t
63

‎test/CAS/module_deps.swift

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -31,9 +31,8 @@
3131
// RUN: %{python} %S/Inputs/SwiftDepsExtractor.py %t/deps.json clang:F casFSRootID > %t/F_fs.casid
3232
// RUN: llvm-cas --cas %t/cas --ls-tree-recursive @%t/F_fs.casid | %FileCheck %s -check-prefix FS_ROOT_F
3333

34-
/// make sure the number of CASFS is correct. 10 entries with 2 line each + 1 extra bracket
35-
// RUN: NUM_CMDS=$(%{python} %S/Inputs/SwiftDepsExtractor.py %t/deps.json deps commandLine | wc -l)
36-
// RUN: if [ ! $NUM_CMDS -eq 21 ]; then echo "wrong number of CASFS from scanning"; exit 1; fi
34+
// RUN: %{python} %S/Inputs/SwiftDepsExtractor.py %t/deps.json deps commandLine > %t/deps.cmd
35+
// RUN: %FileCheck %s -check-prefix MAIN_CMD -input-file=%t/deps.cmd
3736

3837
// FS_ROOT_E-DAG: E.swiftinterface
3938
// FS_ROOT_E-DAG: SDKSettings.json
@@ -49,6 +48,10 @@
4948
// FS_ROOT_F: CHeaders/X.h
5049
// FS_ROOT_F: CHeaders/module.modulemap
5150

51+
// MAIN_CMD: -direct-clang-cc1-module-build
52+
// MAIN_CMD: -cas-fs
53+
// MAIN_CMD-NOT: -clang-include-tree-root
54+
5255
import C
5356
import E
5457
import G

0 commit comments

Comments
 (0)