[lld][WebAssemlby] Implement --thinlto-object-suffix-replace/--thinlt…

…o-prefix-replace (llvm#114625)

Fixes: llvm#79604

lld/test/wasm/lto/thinlto-emit-index.ll

@@ -0,0 +1,108 @@
+;; Copied from ELF/lto/thinlto-index-only.ll
+;; First ensure that the ThinLTO handling in lld handles
+;; bitcode without summary sections gracefully and generates index file.
+; RUN: rm -rf %t && mkdir %t && cd %t
+; RUN: mkdir d
+; RUN: llvm-as %s -o 1.o
+; RUN: llvm-as %p/Inputs/thinlto.ll -o d/2.o
+; RUN: wasm-ld --thinlto-emit-index-files -shared 1.o d/2.o -o 3
+; RUN: ls d/2.o.thinlto.bc
+; RUN: ls 3
+; RUN: wasm-ld -shared 1.o d/2.o -o 3
+; RUN: llvm-nm 3 | FileCheck %s --check-prefix=NM
+
+;; Basic ThinLTO tests.
+; RUN: opt -module-summary %s -o 1.o
+; RUN: opt -module-summary %p/Inputs/thinlto.ll -o d/2.o
+; RUN: opt -module-summary %p/Inputs/thinlto_empty.ll -o 3.o
+; RUN: cp 3.o 4.o
+
+;; Ensure lld generates an index and also a binary if requested.
+; RUN: wasm-ld --thinlto-emit-index-files -shared 1.o --start-lib d/2.o 3.o --end-lib 4.o -o 4
+; RUN: ls 4
+; RUN: llvm-bcanalyzer -dump 1.o.thinlto.bc | FileCheck %s --check-prefix=BACKEND1
+; RUN: llvm-bcanalyzer -dump d/2.o.thinlto.bc | FileCheck %s --check-prefix=BACKEND2
+; RUN: llvm-dis < 3.o.thinlto.bc | FileCheck %s --check-prefix=BACKEND3
+; RUN: llvm-dis < 4.o.thinlto.bc | FileCheck %s --check-prefix=BACKEND4
+
+; IMPORTS1: d/2.o
+
+;; Ensure lld generates an index and not a binary if both emit-index and index-only are present.
+; RUN: wasm-ld --thinlto-emit-index-files --thinlto-index-only -shared 1.o d/2.o -o 5
+; RUN: not ls 5
+
+;; Test that LLD generates an empty index even for lazy object file that is not added to link.
+;; Test that LLD also generates empty imports file with the --thinlto-emit-imports-files option.
+; RUN: rm -f 1.o.thinlto.bc 1.o.imports
+; RUN: wasm-ld --thinlto-emit-index-files -shared d/2.o --start-lib 1.o --end-lib \
+; RUN: --thinlto-emit-imports-files -o 7
+; RUN: ls 7
+; RUN: ls 1.o.thinlto.bc
+; RUN: ls 1.o.imports
+
+;; Ensure LLD generates an empty index for each bitcode file even if all bitcode files are lazy.
+; RUN: rm -f 1.o.thinlto.bc
+; RUN: llvm-mc -filetype=obj -triple=wasm32-unknown-linux /dev/null -o dummy.o
+; RUN: wasm-ld --thinlto-emit-index-files -shared dummy.o --start-lib 1.o --end-lib -o 8
+; RUN: ls 8
+; RUN: ls 1.o.thinlto.bc
+
+;; Test that LLD errors out when run with suffix replacement, or prefix replacement
+; RUN: not wasm-ld --thinlto-emit-index-files -shared d/2.o --start-lib 1.o --end-lib \
+; RUN: --thinlto-prefix-replace="abc;xyz" 2>&1 | FileCheck %s --check-prefix=ERR1
+; ERR1: --thinlto-prefix-replace is not supported with --thinlto-emit-index-files
+
+; RUN: not wasm-ld --thinlto-emit-index-files -shared d/2.o --start-lib 1.o --end-lib \
+; RUN: --thinlto-object-suffix-replace="abc;xyz" 2>&1 | FileCheck %s --check-prefix=ERR2
+; ERR2: --thinlto-object-suffix-replace is not supported with --thinlto-emit-index-files
+
+;; But not when passed with index only as well
+; RUN: wasm-ld --thinlto-emit-index-files -shared d/2.o --start-lib 1.o --end-lib \
+; RUN: --thinlto-prefix-replace="abc;xyz" --thinlto-index-only
+
+; RUN: wasm-ld --thinlto-emit-index-files -shared d/2.o --start-lib 1.o --end-lib \
+; RUN: --thinlto-object-suffix-replace="abc;xyz" --thinlto-index-only
+
+; NM: T f
+
+;; The backend index for this module contains summaries from itself and
+;; Inputs/thinlto.ll, as it imports from the latter.
+; BACKEND1: <MODULE_STRTAB_BLOCK
+; BACKEND1-NEXT: <ENTRY {{.*}} record string = '1.o'
+; BACKEND1-NEXT: <ENTRY {{.*}} record string = 'd/2.o'
+; BACKEND1-NEXT: </MODULE_STRTAB_BLOCK
+; BACKEND1: <GLOBALVAL_SUMMARY_BLOCK
+; BACKEND1: <VERSION
+; BACKEND1: <FLAGS
+; BACKEND1: <VALUE_GUID {{.*}} op0={{1|2}} {{op1=3060885059 op2=1207956914|op1=3432075125 op2=3712786831}}
+; BACKEND1: <VALUE_GUID {{.*}} op0={{1|2}} {{op1=3060885059 op2=1207956914|op1=3432075125 op2=3712786831}}
+; BACKEND1: <COMBINED
+; BACKEND1: <COMBINED
+; BACKEND1: </GLOBALVAL_SUMMARY_BLOCK
+
+;; The backend index for Input/thinlto.ll contains summaries from itself only,
+;; as it does not import anything.
+; BACKEND2: <MODULE_STRTAB_BLOCK
+; BACKEND2-NEXT: <ENTRY {{.*}} record string = 'd/2.o'
+; BACKEND2-NEXT: </MODULE_STRTAB_BLOCK
+; BACKEND2-NEXT: <GLOBALVAL_SUMMARY_BLOCK
+; BACKEND2-NEXT: <VERSION
+; BACKEND2-NEXT: <FLAGS
+; BACKEND2-NEXT: <VALUE_GUID {{.*}} op0=1 op1=3060885059 op2=1207956914
+; BACKEND2-NEXT: <COMBINED
+; BACKEND2-NEXT: </GLOBALVAL_SUMMARY_BLOCK
+
+; BACKEND3: ^0 = flags:
+
+; BACKEND4: ^0 = module: (path: "4.o", hash: (0, 0, 0, 0, 0))
+
+target datalayout = "e-m:e-p:32:32-p10:8:8-p20:8:8-i64:64-n32:64-S128-ni:1:10:20"
+target triple = "wasm32-unknown-unknown"
+
+declare void @g(...)
+
+define void @f() {
+entry:
+  call void (...) @g()
+  ret void
+}

lld/test/wasm/lto/thinlto-object-suffix-replace.ll

@@ -0,0 +1,47 @@
+;; Copied from ELF/lto/thinlto-object-suffix-replace.ll
+;; Test to make sure the thinlto-object-suffix-replace option is handled
+;; correctly.
+; RUN: rm -rf %t && mkdir %t && cd %t
+
+;; Generate bitcode file with summary, as well as a minimized bitcode without
+; the debug metadata for the thin link.
+; RUN: opt --thinlto-bc %s -thin-link-bitcode-file=1.thinlink.bc -o 1.o
+
+;; First perform the thin link on the normal bitcode file, and save the
+;; resulting index.
+; RUN: wasm-ld --thinlto-index-only -shared 1.o -o 3
+; RUN: cp 1.o.thinlto.bc 1.o.thinlto.bc.orig
+
+;; Next perform the thin link on the minimized bitcode file, and compare dump
+;; of the resulting index to the above dump to ensure they are identical.
+; RUN: rm -f 1.o.thinlto.bc
+;; Make sure it isn't inadvertently using the regular bitcode file.
+; RUN: rm -f 1.o
+; RUN: wasm-ld --thinlto-index-only --thinlto-object-suffix-replace=".thinlink.bc;.o" \
+; RUN:   -shared 1.thinlink.bc -o 3
+; RUN: cmp 1.o.thinlto.bc.orig 1.o.thinlto.bc
+
+;; Ensure lld generates error if object suffix replace option does not have 'old;new' format
+; RUN: rm -f 1.o.thinlto.bc
+; RUN: not wasm-ld --thinlto-index-only --thinlto-object-suffix-replace="abc:def" -shared 1.thinlink.bc \
+; RUN:   -o 3 2>&1 | FileCheck %s --check-prefix=ERR1
+; ERR1: --thinlto-object-suffix-replace= expects 'old;new' format, but got abc:def
+
+;; If filename does not end with old suffix, no suffix change should occur,
+;; so ".thinlto.bc" will simply be appended to the input file name.
+; RUN: rm -f 1.thinlink.bc.thinlto.bc
+; RUN: wasm-ld --thinlto-index-only --thinlto-object-suffix-replace=".abc;.o" -shared 1.thinlink.bc -o /dev/null
+; RUN: ls 1.thinlink.bc.thinlto.bc
+
+target datalayout = "e-m:e-p:32:32-p10:8:8-p20:8:8-i64:64-n32:64-S128-ni:1:10:20"
+target triple = "wasm32-unknown-unknown"
+
+define void @f() {
+entry:
+  ret void
+}
+
+!llvm.dbg.cu = !{}
+
+!1 = !{i32 2, !"Debug Info Version", i32 3}
+!llvm.module.flags = !{!1}

lld/test/wasm/lto/thinlto-prefix-replace.ll

@@ -0,0 +1,23 @@
+; Copied from ELF/lto/thinlto-prefix-replace.ll
+; Check that changing the output path via thinlto-prefix-replace works
+; RUN: mkdir -p %t/oldpath
+; RUN: opt -module-summary %s -o %t/oldpath/thinlto_prefix_replace.o
+
+; Ensure that there is no existing file at the new path, so we properly
+; test the creation of the new file there.
+; RUN: rm -f %t/newpath/thinlto_prefix_replace.o.thinlto.bc
+; RUN: wasm-ld --thinlto-index-only --thinlto-prefix-replace="%t/oldpath/;%t/newpath/" -shared %t/oldpath/thinlto_prefix_replace.o -o %t/thinlto_prefix_replace
+; RUN: ls %t/newpath/thinlto_prefix_replace.o.thinlto.bc
+
+; Ensure that lld generates error if prefix replace option does not have 'old;new' format.
+; RUN: rm -f %t/newpath/thinlto_prefix_replace.o.thinlto.bc
+; RUN: not wasm-ld --thinlto-index-only --thinlto-prefix-replace=abc:def -shared %t/oldpath/thinlto_prefix_replace.o -o /dev/null 2>&1 | FileCheck %s --check-prefix=ERR
+; ERR: --thinlto-prefix-replace= expects 'old;new' format, but got abc:def
+
+target datalayout = "e-m:e-p:32:32-p10:8:8-p20:8:8-i64:64-n32:64-S128-ni:1:10:20"
+target triple = "wasm32-unknown-unknown"
+
+define void @f() {
+entry:
+  ret void
+}

lld/wasm/Config.h

@@ -110,6 +110,10 @@ struct Configuration {
   llvm::StringRef thinLTOCacheDir;
   llvm::StringRef thinLTOJobs;
   llvm::StringRef thinLTOIndexOnlyArg;
+  std::pair<llvm::StringRef, llvm::StringRef> thinLTOObjectSuffixReplace;
+  llvm::StringRef thinLTOPrefixReplaceOld;
+  llvm::StringRef thinLTOPrefixReplaceNew;
+  llvm::StringRef thinLTOPrefixReplaceNativeObject;
   llvm::StringRef whyExtract;
 
   llvm::StringSet<> allowUndefinedSymbols;

lld/wasm/Driver.cpp

@@ -425,6 +425,33 @@ void LinkerDriver::createFiles(opt::InputArgList &args) {
     error("no input files");
 }
 
+static StringRef getAliasSpelling(opt::Arg *arg) {
+  if (const opt::Arg *alias = arg->getAlias())
+    return alias->getSpelling();
+  return arg->getSpelling();
+}
+
+static std::pair<StringRef, StringRef> getOldNewOptions(opt::InputArgList &args,
+                                                        unsigned id) {
+  auto *arg = args.getLastArg(id);
+  if (!arg)
+    return {"", ""};
+
+  StringRef s = arg->getValue();
+  std::pair<StringRef, StringRef> ret = s.split(';');
+  if (ret.second.empty())
+    error(getAliasSpelling(arg) + " expects 'old;new' format, but got " + s);
+  return ret;
+}
+
+// Parse options of the form "old;new[;extra]".
+static std::tuple<StringRef, StringRef, StringRef>
+getOldNewOptionsExtra(opt::InputArgList &args, unsigned id) {
+  auto [oldDir, second] = getOldNewOptions(args, id);
+  auto [newDir, extraDir] = second.split(';');
+  return {oldDir, newDir, extraDir};
+}
+
 static StringRef getEntry(opt::InputArgList &args) {
   auto *arg = args.getLastArg(OPT_entry, OPT_no_entry);
   if (!arg) {
@@ -577,6 +604,24 @@ static void readConfigs(opt::InputArgList &args) {
   config->thinLTOIndexOnly = args.hasArg(OPT_thinlto_index_only) ||
                              args.hasArg(OPT_thinlto_index_only_eq);
   config->thinLTOIndexOnlyArg = args.getLastArgValue(OPT_thinlto_index_only_eq);
+  config->thinLTOObjectSuffixReplace =
+      getOldNewOptions(args, OPT_thinlto_object_suffix_replace_eq);
+  std::tie(config->thinLTOPrefixReplaceOld, config->thinLTOPrefixReplaceNew,
+           config->thinLTOPrefixReplaceNativeObject) =
+      getOldNewOptionsExtra(args, OPT_thinlto_prefix_replace_eq);
+  if (config->thinLTOEmitIndexFiles && !config->thinLTOIndexOnly) {
+    if (args.hasArg(OPT_thinlto_object_suffix_replace_eq))
+      error("--thinlto-object-suffix-replace is not supported with "
+            "--thinlto-emit-index-files");
+    else if (args.hasArg(OPT_thinlto_prefix_replace_eq))
+      error("--thinlto-prefix-replace is not supported with "
+            "--thinlto-emit-index-files");
+  }
+  if (!config->thinLTOPrefixReplaceNativeObject.empty() &&
+      config->thinLTOIndexOnlyArg.empty()) {
+    error("--thinlto-prefix-replace=old_dir;new_dir;obj_dir must be used with "
+          "--thinlto-index-only=");
+  }
   config->unresolvedSymbols = getUnresolvedSymbolPolicy(args);
   config->whyExtract = args.getLastArgValue(OPT_why_extract);
   errorHandler().verbose = args.hasArg(OPT_verbose);
@@ -721,7 +766,7 @@ static void checkOptions(opt::InputArgList &args) {
   if (config->pie && config->shared)
     error("-shared and -pie may not be used together");
 
-  if (config->outputFile.empty())
+  if (config->outputFile.empty() && !config->thinLTOIndexOnly)
     error("no output file specified");
 
   if (config->importTable && config->exportTable)

lld/wasm/InputFiles.cpp

@@ -46,6 +46,13 @@ std::string toString(const wasm::InputFile *file) {
 
 namespace wasm {
 
+std::string replaceThinLTOSuffix(StringRef path) {
+  auto [suffix, repl] = config->thinLTOObjectSuffixReplace;
+  if (path.consume_back(suffix))
+    return (path + repl).str();
+  return std::string(path);
+}
+
 void InputFile::checkArch(Triple::ArchType arch) const {
   bool is64 = arch == Triple::wasm64;
   if (is64 && !config->is64) {
@@ -837,6 +844,8 @@ BitcodeFile::BitcodeFile(MemoryBufferRef m, StringRef archiveName,
   this->archiveName = std::string(archiveName);
 
   std::string path = mb.getBufferIdentifier().str();
+  if (config->thinLTOIndexOnly)
+    path = replaceThinLTOSuffix(mb.getBufferIdentifier());
 
   // ThinLTO assumes that all MemoryBufferRefs given to it have a unique
   // name. If two archives define two members with the same name, this

lld/wasm/InputFiles.h

@@ -195,6 +195,8 @@ InputFile *createObjectFile(MemoryBufferRef mb, StringRef archiveName = "",
 // Opens a given file.
 std::optional<MemoryBufferRef> readFile(StringRef path);
 
+std::string replaceThinLTOSuffix(StringRef path);
+
 } // namespace wasm
 
 std::string toString(const wasm::InputFile *file);

lld/wasm/LTO.cpp

@@ -43,6 +43,11 @@ using namespace llvm;
 using namespace lld::wasm;
 using namespace lld;
 
+static std::string getThinLTOOutputFile(StringRef modulePath) {
+  return lto::getThinLTOOutputFile(modulePath, config->thinLTOPrefixReplaceOld,
+                                   config->thinLTOPrefixReplaceNew);
+}
+
 static lto::Config createConfig() {
   lto::Config c;
   c.Options = initTargetOptionsFromCodeGenFlags();
@@ -84,7 +89,10 @@ BitcodeCompiler::BitcodeCompiler() {
   auto onIndexWrite = [&](StringRef s) { thinIndices.erase(s); };
   if (config->thinLTOIndexOnly) {
     backend = lto::createWriteIndexesThinBackend(
-        llvm::hardware_concurrency(config->thinLTOJobs), "", "", "",
+        llvm::hardware_concurrency(config->thinLTOJobs),
+        std::string(config->thinLTOPrefixReplaceOld),
+        std::string(config->thinLTOPrefixReplaceNew),
+        std::string(config->thinLTOPrefixReplaceNativeObject),
         config->thinLTOEmitImportsFiles, indexFile.get(), onIndexWrite);
   } else {
     backend = lto::createInProcessThinBackend(
@@ -158,7 +166,8 @@ static void thinLTOCreateEmptyIndexFiles() {
       continue;
     if (linkedBitCodeFiles.contains(f->getName()))
       continue;
-    std::string path(f->obj->getName());
+    std::string path =
+        replaceThinLTOSuffix(getThinLTOOutputFile(f->obj->getName()));
     std::unique_ptr<raw_fd_ostream> os = openFile(path + ".thinlto.bc");
     if (!os)
       continue;

lld/wasm/Options.td

@@ -309,6 +309,8 @@ def thinlto_index_only: FF<"thinlto-index-only">;
 def thinlto_index_only_eq: JJ<"thinlto-index-only=">;
 def thinlto_jobs: JJ<"thinlto-jobs=">,
   HelpText<"Number of ThinLTO jobs. Default to --threads=">;
+def thinlto_object_suffix_replace_eq: JJ<"thinlto-object-suffix-replace=">;
+def thinlto_prefix_replace_eq: JJ<"thinlto-prefix-replace=">;
 def lto_debug_pass_manager: FF<"lto-debug-pass-manager">,
   HelpText<"Debug new pass manager">;