Skip to content

Commit 7acfb5a

Browse files
kateinoigakukunkateinoigakukun
committed
[LTO] Setup LTO pipeline and swift-lto tool
Start to implement the LTO library and set up a basis of incremental testing tool named `swift-lto`. The swiftLTO library accepts SIB files for multi-modules, optimizes them at SIL level and emit LLVM bitcode. It will be dynamically loaded into linker to support language specific LTO.
1 parent 932a91e commit 7acfb5a

File tree

13 files changed

+336
-0
lines changed

13 files changed

+336
-0
lines changed

‎include/swift/AST/DiagnosticsFrontend.def

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -358,6 +358,10 @@ REMARK(interface_file_lock_failure,none,
358358

359359
REMARK(interface_file_lock_timed_out,none,
360360
"timed out waiting to acquire lock file for module interface '%0'", (StringRef))
361+
ERROR(invalid_serialized_module,none,
362+
"unable to register invalid serialized module", ())
363+
ERROR(unable_to_load_serialized_module,none,
364+
"unable to load serialized module '%0'", (StringRef))
361365

362366
// Dependency Verifier Diagnostics
363367
ERROR(dependency_cascading_mismatch,none,

‎include/swift/LTO/LTO.h

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
//===--- LTO.cpp - Swift LTO ----------------------------------------------===//
2+
//
3+
// This source file is part of the Swift.org open source project
4+
//
5+
// Copyright (c) 2014 - 2017 Apple Inc. and the Swift project authors
6+
// Licensed under Apache License v2.0 with Runtime Library Exception
7+
//
8+
// See https://swift.org/LICENSE.txt for license information
9+
// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
10+
//
11+
//===----------------------------------------------------------------------===//
12+
13+
#ifndef SWIFT_LTO_H
14+
#define SWIFT_LTO_H
15+
16+
#include "llvm/ADT/SmallVector.h"
17+
#include "llvm/ADT/StringRef.h"
18+
#include "llvm/Support/MemoryBuffer.h"
19+
#include <functional>
20+
#include <memory>
21+
22+
#include "swift/AST/ASTContext.h"
23+
#include "swift/AST/IRGenOptions.h"
24+
#include "swift/ClangImporter/ClangImporterOptions.h"
25+
#include "swift/Frontend/ModuleInterfaceLoader.h"
26+
#include "swift/Frontend/PrintingDiagnosticConsumer.h"
27+
#include "swift/Serialization/Validation.h"
28+
29+
namespace swift {
30+
31+
class ASTContext;
32+
33+
namespace lto {
34+
35+
using GetStreamFn = std::function<std::unique_ptr<llvm::raw_ostream>(
36+
llvm::StringRef ModuleName)>;
37+
38+
class LTOPipeline {
39+
llvm::SmallVector<llvm::StringRef, 4> RuntimeLibraryPaths;
40+
llvm::SmallVector<llvm::StringRef, 4> RuntimeLibraryImportPaths;
41+
llvm::StringRef RuntimeResourcePath;
42+
llvm::SmallVector<Identifier, 2> ModuleNames;
43+
LangOptions LangOpts;
44+
ClangImporterOptions ClangOpts;
45+
TypeCheckerOptions TCOpts;
46+
SearchPathOptions SearchPathOpts;
47+
SourceManager SM;
48+
DiagnosticEngine Diags;
49+
PrintingDiagnosticConsumer PrintDiags;
50+
std::unique_ptr<ASTContext> Ctx;
51+
MemoryBufferSerializedModuleLoader *MBL;
52+
53+
public:
54+
LTOPipeline(llvm::SmallVector<llvm::StringRef, 4> RuntimeLibraryPaths,
55+
llvm::SmallVector<llvm::StringRef, 4> RuntimeLibraryImportPaths,
56+
llvm::StringRef RuntimeResourcePath)
57+
: RuntimeLibraryPaths(RuntimeLibraryPaths),
58+
RuntimeLibraryImportPaths(RuntimeLibraryImportPaths),
59+
RuntimeResourcePath(RuntimeResourcePath), Diags(SM) {}
60+
bool addModule(std::unique_ptr<llvm::MemoryBuffer> Buffer);
61+
bool emitLLVMModules(GetStreamFn GetStream);
62+
63+
private:
64+
ASTContext *createASTContext(serialization::ValidationInfo info,
65+
serialization::ExtendedValidationInfo extInfo);
66+
};
67+
68+
} // namespace lto
69+
} // namespace swift
70+
71+
#endif // SWIFT_LTO_H

‎lib/CMakeLists.txt

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ add_subdirectory(IDE)
2727
add_subdirectory(Immediate)
2828
add_subdirectory(IRGen)
2929
add_subdirectory(LLVMPasses)
30+
add_subdirectory(LTO)
3031
add_subdirectory(Markup)
3132
add_subdirectory(Migrator)
3233
add_subdirectory(Option)

‎lib/FrontendTool/FrontendTool.cpp

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1384,6 +1384,7 @@ static bool serializeSIB(SILModule *SM, const PrimarySpecificPaths &PSPs,
13841384
serializationOpts.OutputPath = moduleOutputPath.c_str();
13851385
serializationOpts.SerializeAllSIL = true;
13861386
serializationOpts.IsSIB = true;
1387+
serializationOpts.SerializeOptionsForDebugging = true;
13871388

13881389
serialize(MSF, serializationOpts, SM);
13891390
return Context.hadError();

‎lib/LTO/CMakeLists.txt

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
add_swift_host_library(swiftLTO STATIC
2+
LTO.cpp)
3+
target_link_libraries(swiftLTO PRIVATE
4+
swiftFrontend
5+
swiftIRGen
6+
swiftSILGen
7+
swiftSILOptimizer)

‎lib/LTO/LTO.cpp

Lines changed: 122 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,122 @@
1+
//===--- LTO.cpp - Swift LTO ----------------------------------------------===//
2+
//
3+
// This source file is part of the Swift.org open source project
4+
//
5+
// Copyright (c) 2014 - 2020 Apple Inc. and the Swift project authors
6+
// Licensed under Apache License v2.0 with Runtime Library Exception
7+
//
8+
// See https://swift.org/LICENSE.txt for license information
9+
// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
10+
//
11+
//===----------------------------------------------------------------------===//
12+
13+
#include "swift/LTO/LTO.h"
14+
#include "swift/AST/DiagnosticsFrontend.h"
15+
#include "swift/AST/IRGenRequests.h"
16+
#include "swift/ClangImporter/ClangImporter.h"
17+
#include "swift/SIL/SILModule.h"
18+
#include "swift/Serialization/SerializedModuleLoader.h"
19+
#include "llvm/Bitcode/BitcodeWriter.h"
20+
#include "llvm/IR/Module.h"
21+
22+
namespace swift {
23+
24+
namespace lto {
25+
26+
using namespace llvm;
27+
28+
bool LTOPipeline::addModule(std::unique_ptr<MemoryBuffer> Buffer) {
29+
serialization::ExtendedValidationInfo extendedInfo;
30+
serialization::ValidationInfo info =
31+
serialization::validateSerializedAST(Buffer->getBuffer(), &extendedInfo);
32+
if (info.status != serialization::Status::Valid) {
33+
Diags.diagnose(SourceLoc(), diag::invalid_serialized_module);
34+
return true;
35+
}
36+
37+
if (!Ctx) {
38+
Ctx.reset(createASTContext(info, extendedInfo));
39+
}
40+
41+
MBL->registerMemoryBuffer(info.name, std::move(Buffer));
42+
43+
ModuleNames.emplace_back(Ctx->getIdentifier(info.name));
44+
return false;
45+
}
46+
47+
bool LTOPipeline::emitLLVMModules(GetStreamFn GetStream) {
48+
IRGenOptions Opts = {};
49+
Opts.OutputKind = IRGenOutputKind::Module;
50+
51+
for (auto &ModuleName : ModuleNames) {
52+
std::vector<swift::Located<swift::Identifier>> AccessPath;
53+
AccessPath.emplace_back(ModuleName, SourceLoc());
54+
auto SwiftModule = Ctx->getModule(AccessPath);
55+
if (!SwiftModule) {
56+
Diags.diagnose(SourceLoc(), diag::unable_to_load_serialized_module,
57+
ModuleName.get());
58+
return true;
59+
}
60+
Lowering::TypeConverter Types(*SwiftModule);
61+
SILOptions SILOpts = {};
62+
auto SM = performASTLowering(SwiftModule, Types, SILOpts);
63+
// TODO: Propagate input file name through SIB to enable debug info
64+
const PrimarySpecificPaths PSPs;
65+
auto GeneratedMod =
66+
performIRGeneration(Opts, SM->getSwiftModule(), std::move(SM),
67+
ModuleName.get(), PSPs, ArrayRef<std::string>());
68+
auto LLVMMod = GeneratedMod.getModule();
69+
if (auto OS = GetStream(LLVMMod->getName())) {
70+
WriteBitcodeToFile(*LLVMMod, *OS);
71+
}
72+
}
73+
return false;
74+
}
75+
76+
ASTContext *
77+
LTOPipeline::createASTContext(serialization::ValidationInfo info,
78+
serialization::ExtendedValidationInfo extInfo) {
79+
auto Ctx = ASTContext::get(LangOpts, TCOpts, SearchPathOpts, SM, Diags);
80+
Diags.addConsumer(PrintDiags);
81+
LangOpts.setTarget(Triple(info.targetTriple));
82+
SearchPathOpts.SDKPath = extInfo.getSDKPath();
83+
84+
SearchPathOpts.RuntimeLibraryPaths.insert(
85+
SearchPathOpts.RuntimeLibraryPaths.end(), RuntimeLibraryPaths.begin(),
86+
RuntimeLibraryPaths.end());
87+
SearchPathOpts.RuntimeLibraryImportPaths.insert(
88+
SearchPathOpts.RuntimeLibraryImportPaths.end(),
89+
RuntimeLibraryImportPaths.begin(), RuntimeLibraryImportPaths.end());
90+
SearchPathOpts.RuntimeResourcePath = RuntimeResourcePath;
91+
92+
// MARK: Setup module loaders
93+
std::unique_ptr<ClangImporter> clangImporter =
94+
ClangImporter::create(*Ctx, ClangOpts, "", nullptr);
95+
auto const &Clang = clangImporter->getClangInstance();
96+
std::string ModuleCachePath = getModuleCachePathFromClang(Clang);
97+
98+
auto MIL = ModuleInterfaceLoader::create(*Ctx, ModuleCachePath, "", nullptr,
99+
ModuleLoadingMode::PreferSerialized);
100+
Ctx->addModuleLoader(std::move(MIL));
101+
auto MBL = MemoryBufferSerializedModuleLoader::create(
102+
*Ctx, nullptr, ModuleLoadingMode::OnlySerialized, true);
103+
this->MBL = MBL.get();
104+
105+
auto SML = SerializedModuleLoader::create(
106+
*Ctx, nullptr, ModuleLoadingMode::OnlySerialized, true);
107+
108+
Ctx->addModuleLoader(std::move(MBL));
109+
Ctx->addModuleLoader(std::move(SML));
110+
Ctx->addModuleLoader(std::move(clangImporter), /*isClang*/ true);
111+
112+
registerIRGenRequestFunctions(Ctx->evaluator);
113+
registerSILOptimizerRequestFunctions(Ctx->evaluator);
114+
registerParseRequestFunctions(Ctx->evaluator);
115+
registerTypeCheckerRequestFunctions(Ctx->evaluator);
116+
registerSILGenRequestFunctions(Ctx->evaluator);
117+
registerIRGenSILTransforms(*Ctx);
118+
return Ctx;
119+
}
120+
121+
} // namespace lto
122+
} // namespace swift

‎test/LTO/Inputs/lib.swift

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
public struct LibX {}
2+
3+
public func getLibX() -> LibX { return LibX() }

‎test/LTO/Inputs/main.swift

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
import lib
2+
3+
_ = getLibX()

‎test/LTO/pipeline.swift

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
// RUN: %empty-directory(%t)
2+
// RUN: cd %t && %target-swiftc_driver -emit-module %S/Inputs/lib.swift
3+
// RUN: cd %t && %target-swift-frontend -emit-sib %S/Inputs/lib.swift -parse-as-library
4+
// RUN: cd %t && %target-swift-frontend -emit-sib -I%t %S/Inputs/main.swift
5+
6+
// Examine loading order
7+
// RUN: cd %t && %swift-lto main.sib lib.sib
8+
// RUN: cd %t && %llvm-dis lib.bc -o - | %FileCheck %s -check-prefix=CHECK-LIB
9+
// RUN: cd %t && %llvm-dis main.bc -o - | %FileCheck %s -check-prefix=CHECK-MAIN
10+
11+
// RUN: cd %t && %swift-lto lib.sib main.sib
12+
// RUN: cd %t && %llvm-dis lib.bc -o - | %FileCheck %s -check-prefix=CHECK-LIB
13+
// RUN: cd %t && %llvm-dis main.bc -o - | %FileCheck %s -check-prefix=CHECK-MAIN
14+
15+
// CHECK-LIB: ModuleID = 'lib.bc'
16+
// CHECK-LIB: define hidden swiftcc void @"$s3lib4LibXVACycfC"()
17+
// CHECK-LIB: define swiftcc void @"$s3lib7getLibXAA0C1XVyF"()
18+
// CHECK-LIB: call swiftcc void @"$s3lib4LibXVACycfC"
19+
20+
// CHECK-MAIN: ModuleID = 'main.bc'
21+
// CHECK-MAIN: define i32 @main
22+
// CHECK-MAIN: call swiftcc void @"$s3lib7getLibXAA0C1XVyF"
23+
// CHECK-MAIN: declare swiftcc void @"$s3lib7getLibXAA0C1XVyF"()
24+

‎test/lit.cfg

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -398,6 +398,7 @@ if '-disable-astscope-lookup' in config.swift_test_options:
398398
if '-disable-parser-lookup' in config.swift_test_options:
399399
config.available_features.add("disable-parser-lookup")
400400
config.substitutions.append( ('%swift-indent', config.swift_indent) )
401+
config.substitutions.append( ('%swift-lto', '%s lto' % config.sil_llvm_gen) )
401402
config.substitutions.append( ('%llvm-link', config.llvm_link) )
402403
config.substitutions.append( ('%swift-llvm-opt', config.swift_llvm_opt) )
403404
config.substitutions.append( ('%llvm-dwarfdump', config.llvm_dwarfdump) )

‎tools/sil-llvm-gen/CMakeLists.txt

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
add_swift_host_tool(sil-llvm-gen
22
SILLLVMGen.cpp
3+
swift_lto_main.cpp
34
SWIFT_COMPONENT tools
45
)
56
target_link_libraries(sil-llvm-gen
@@ -8,6 +9,7 @@ target_link_libraries(sil-llvm-gen
89
swiftIRGen
910
swiftSILGen
1011
swiftSILOptimizer
12+
swiftLTO
1113
# Clang libraries included to appease the linker on linux.
1214
clangBasic
1315
clangCodeGen)

‎tools/sil-llvm-gen/SILLLVMGen.cpp

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -111,12 +111,19 @@ static llvm::cl::opt<bool>
111111
// without being given the address of a function in the main executable).
112112
void anchorForGetMainExecutable() {}
113113

114+
extern llvm::cl::SubCommand LTOSubcommand;
115+
int swift_lto_main(int argc, char **argv);
116+
114117
int main(int argc, char **argv) {
115118
PROGRAM_START(argc, argv);
116119
INITIALIZE_LLVM();
117120

118121
llvm::cl::ParseCommandLineOptions(argc, argv, "Swift LLVM IR Generator\n");
119122

123+
if (LTOSubcommand) {
124+
return swift_lto_main(argc, argv);
125+
}
126+
120127
if (PrintStats)
121128
llvm::EnableStatistics();
122129

‎tools/sil-llvm-gen/swift_lto_main.cpp

Lines changed: 90 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,90 @@
1+
//===--- swift_lto_main.cpp -----------------------------------------------===//
2+
//
3+
// This source file is part of the Swift.org open source project
4+
//
5+
// Copyright (c) 2014 - 2020 Apple Inc. and the Swift project authors
6+
// Licensed under Apache License v2.0 with Runtime Library Exception
7+
//
8+
// See https://swift.org/LICENSE.txt for license information
9+
// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
10+
//
11+
//===----------------------------------------------------------------------===//
12+
///
13+
/// \file
14+
///
15+
/// This is a tool for reading sib files and running LTO passes upon them.
16+
///
17+
//===----------------------------------------------------------------------===//
18+
19+
#include "swift/Basic/LLVMInitialize.h"
20+
#include "swift/Frontend/Frontend.h"
21+
#include "swift/LTO/LTO.h"
22+
#include "swift/Strings.h"
23+
#include "llvm/Support/CommandLine.h"
24+
#include "llvm/Support/MemoryBuffer.h"
25+
#include <cstdio>
26+
#include <memory>
27+
using namespace swift;
28+
29+
llvm::cl::SubCommand LTOSubcommand("lto", "Swift LTO Tool");
30+
31+
static llvm::cl::list<std::string>
32+
InputFilenames(llvm::cl::Positional, llvm::cl::desc("[input files...]"),
33+
llvm::cl::OneOrMore, llvm::cl::sub(LTOSubcommand));
34+
35+
// This function isn't referenced outside its translation unit, but it
36+
// can't use the "static" keyword because its address is used for
37+
// getMainExecutable (since some platforms don't support taking the
38+
// address of main, and some platforms can't implement getMainExecutable
39+
// without being given the address of a function in the main executable).
40+
void anchorForGetMainExecutableInSwiftLTO() {}
41+
42+
// This tool is combined with sil-llvm-gen to reduce link time.
43+
// This entrypoint is invoked from SILLLVMGen.cpp when user invoke
44+
// lto subcommand.
45+
int swift_lto_main(int argc, char **argv) {
46+
CompilerInvocation Invocation;
47+
48+
Invocation.setMainExecutablePath(llvm::sys::fs::getMainExecutable(
49+
argv[0],
50+
reinterpret_cast<void *>(&anchorForGetMainExecutableInSwiftLTO)));
51+
52+
auto SearchPathOpts = Invocation.getSearchPathOptions();
53+
llvm::SmallVector<llvm::StringRef, 4> RuntimeLibraryPaths;
54+
llvm::SmallVector<llvm::StringRef, 4> RuntimeLibraryImportPaths;
55+
RuntimeLibraryPaths.insert(RuntimeLibraryPaths.begin(),
56+
SearchPathOpts.RuntimeLibraryPaths.begin(),
57+
SearchPathOpts.RuntimeLibraryPaths.end());
58+
RuntimeLibraryImportPaths.insert(
59+
RuntimeLibraryImportPaths.begin(),
60+
SearchPathOpts.RuntimeLibraryImportPaths.begin(),
61+
SearchPathOpts.RuntimeLibraryImportPaths.end());
62+
lto::LTOPipeline Pipeline(RuntimeLibraryPaths, RuntimeLibraryImportPaths,
63+
SearchPathOpts.RuntimeResourcePath);
64+
65+
for (auto InputFilename : InputFilenames) {
66+
// Load the input file.
67+
auto FileBufOrErr = llvm::MemoryBuffer::getFileOrSTDIN(InputFilename);
68+
if (!FileBufOrErr) {
69+
fprintf(stderr, "Error! Failed to open file: %s\n",
70+
InputFilename.c_str());
71+
exit(-1);
72+
}
73+
74+
if (Pipeline.addModule(std::move(FileBufOrErr.get()))) {
75+
fprintf(stderr, "Error! Failed to load serialized module: %s\n",
76+
InputFilename.c_str());
77+
exit(-1);
78+
}
79+
}
80+
81+
Pipeline.emitLLVMModules([&](StringRef ModuleName) {
82+
std::error_code EC;
83+
std::unique_ptr<llvm::raw_ostream> RawOS =
84+
std::make_unique<llvm::raw_fd_ostream>(ModuleName.str() + ".bc", EC);
85+
if (EC)
86+
return std::unique_ptr<llvm::raw_ostream>(nullptr);
87+
return RawOS;
88+
});
89+
return 0;
90+
}

0 commit comments

Comments
 (0)