Swift: turn extractor into a swift-frontend plugin

This change turns the extractor into a plugin to a slightly modified
`swift-extractor` binary, providing only a `swift::FrontendObserver`
instance.

This makes it so that we inherit the behaviours of the released
`swift-frontend`, among the known solved issues are:
* work around the issue preventing the Linux extractor from compiling
  stdlib `.swiftinterface` files, which was preventing the extractor
  from working on top of earlier versions of the swift SDK
* `@`-prefixed parameter files are now supported

Under the hood the Swift build system has been tweaked so that calls to
`performFrontend` in the driver executable use a `FrontendObserver`
built from a dynamic library, and the extractor now provides an
alternative frontend observer dynamic library.

Some special treatment needed to retain the same features:
* a new `finished` method was added to `swift::FrontendObserver` to
  implement the `file_is_successfully_extracted` predicate
* part of the swift AST library (`Identifier`) needed to be extracted
  into a separate dynamic library as well, as behaviour was depending on
  the value of static pointers that therefore needed to be shared
  between `swift-frontend` and the extractor
* the `CODEQL_EXTRACTOR_SWIFT_RUN_UNDER` functionality has been moved
  to the `extractor` bash wrapper, which also makes it work better to
  catch crashes happening before `main` is even entered (which can
  happen with tracer problems)

Author: redsun82

swift/extractor/BUILD.bazel

@@ -3,11 +3,12 @@ load("//misc/bazel/cmake:cmake.bzl", "generate_cmake")
 load("//misc/bazel:pkg_runfiles.bzl", "pkg_runfiles")
 
 swift_cc_binary(
-    name = "extractor.real",
+    name = "swiftFrontendObserver",
     srcs = glob([
         "*.h",
         "*.cpp",
     ]),
+    linkshared = True,
     deps = [
         "//swift/extractor/config",
         "//swift/extractor/infra",
@@ -21,19 +22,27 @@ swift_cc_binary(
 
 generate_cmake(
     name = "cmake",
-    targets = [":extractor.real"],
+    targets = [":swiftFrontendObserver"],
     visibility = ["//visibility:public"],
 )
 
 sh_binary(
     name = "extractor",
     srcs = ["extractor.sh"],
-    data = [":extractor.real"],
+    data = [
+        ":swiftFrontendObserver",
+        "//swift/third_party/swift-llvm-support:swift-frontend",
+    ],
 )
 
 pkg_runfiles(
     name = "pkg",
     srcs = [":extractor"],
     excludes = ["extractor.sh"],  # script gets copied as "extractor", no need for the original .sh file
+    renames = select({
+        # workaround for https://github.com/bazelbuild/bazel/issues/11082 (wrongly marked as closed)
+        "@platforms//os:macos": {":swiftFrontendObserver": "libswiftFrontendObserver.dylib"},
+        "//conditions:default": {},
+    }),
     visibility = ["//swift:__pkg__"],
 )

swift/extractor/extractor.sh

@@ -1,9 +1,19 @@
 #!/bin/bash
 
+EXE_DIR="$(dirname "$0")"
+
 if [[ "$(uname)" == Darwin ]]; then
-  export DYLD_LIBRARY_PATH=$(dirname "$0")
+  export DYLD_LIBRARY_PATH="$EXE_DIR"
 else
-  export LD_LIBRARY_PATH=$(dirname "$0")
+  export LD_LIBRARY_PATH="$EXE_DIR"
+fi
+
+TOOL="$CODEQL_EXTRACTOR_SWIFT_RUN_UNDER"
+
+if [[ -n "$CODEQL_EXTRACTOR_SWIFT_RUN_UNDER_FILTER" ]]; then
+  if [[ ! "$*" =~ $CODEQL_EXTRACTOR_SWIFT_RUN_UNDER_FILTER ]]; then
+    TOOL=
+  fi
 fi
 
-exec -a "$0" "$0.real" "$@"
+exec -a swift-frontend $TOOL "$EXE_DIR/swift-frontend" "$@"

swift/extractor/main.cpp

@@ -6,7 +6,8 @@
 #include <chrono>
 
 #include <swift/Basic/LLVMInitialize.h>
-#include <swift/FrontendTool/FrontendTool.h>
+#include <swift/DriverTool/DriverTool.h>
+#include <swift/DriverTool/FrontendObserver.h>
 #include <swift/Basic/InitializeSwiftModules.h>
 
 #include "swift/extractor/SwiftExtractor.h"
@@ -92,7 +93,8 @@ class Observer : public swift::FrontendObserver {
     codeql::extractExtractLazyDeclarations(state, compiler);
   }
 
-  void markSuccessfullyExtractedFiles() {
+  void finished(int status) override {
+    if (status != 0) return;
     codeql::SwiftLocationExtractor locExtractor{invocationTrap};
     for (const auto& src : state.sourceFiles) {
       auto fileLabel = locExtractor.emitFile(src);
@@ -102,6 +104,8 @@ class Observer : public swift::FrontendObserver {
 
  private:
   codeql::SwiftExtractorState state;
+  std::shared_ptr<codeql::FileInterceptor> openInterception{
+      codeql::setupFileInterception(state.configuration)};
   codeql::TrapDomain invocationTrap{invocationTrapDomain(state)};
   codeql::SwiftDiagnosticsConsumer diagConsumer{invocationTrap};
 };
@@ -113,49 +117,6 @@ static std::string getenv_or(const char* envvar, const std::string& def) {
   return def;
 }
 
-static bool checkRunUnderFilter(int argc, char* const* argv) {
-  auto runUnderFilter = getenv("CODEQL_EXTRACTOR_SWIFT_RUN_UNDER_FILTER");
-  if (runUnderFilter == nullptr) {
-    return true;
-  }
-  std::string call = argv[0];
-  for (auto i = 1; i < argc; ++i) {
-    call += ' ';
-    call += argv[i];
-  }
-  std::regex filter{runUnderFilter, std::regex_constants::basic | std::regex_constants::nosubs};
-  return std::regex_search(call, filter);
-}
-
-// if `CODEQL_EXTRACTOR_SWIFT_RUN_UNDER` env variable is set, and either
-// * `CODEQL_EXTRACTOR_SWIFT_RUN_UNDER_FILTER` is not set, or
-// * it is set to a regexp matching any substring of the extractor call
-// then the running process is substituted with the command (and possibly
-// options) stated in `CODEQL_EXTRACTOR_SWIFT_RUN_UNDER`, followed by `argv`.
-// Before calling `exec`, `CODEQL_EXTRACTOR_SWIFT_RUN_UNDER` is unset to avoid
-// unpleasant loops.
-// An example usage is to run the extractor under `gdbserver :1234` when the
-// arguments match a given source file.
-static void checkWhetherToRunUnderTool(int argc, char* const* argv) {
-  assert(argc > 0);
-
-  auto runUnder = getenv("CODEQL_EXTRACTOR_SWIFT_RUN_UNDER");
-  if (runUnder == nullptr || !checkRunUnderFilter(argc, argv)) {
-    return;
-  }
-  std::vector<char*> args;
-  // split RUN_UNDER value by spaces to get args vector
-  for (auto word = std::strtok(runUnder, " "); word != nullptr; word = std::strtok(nullptr, " ")) {
-    args.push_back(word);
-  }
-  // append process args, including extractor executable path
-  args.insert(args.end(), argv, argv + argc);
-  args.push_back(nullptr);
-  // avoid looping on this function
-  unsetenv("CODEQL_EXTRACTOR_SWIFT_RUN_UNDER");
-  execvp(args[0], args.data());
-}
-
 // Creates a target file that should store per-invocation info, e.g. compilation args,
 // compilations, diagnostics, etc.
 codeql::TrapDomain invocationTrapDomain(codeql::SwiftExtractorState& state) {
@@ -170,39 +131,19 @@ codeql::TrapDomain invocationTrapDomain(codeql::SwiftExtractorState& state) {
   return std::move(maybeDomain.value());
 }
 
-codeql::SwiftExtractorConfiguration configure(int argc, char** argv) {
+codeql::SwiftExtractorConfiguration configure(llvm::ArrayRef<const char*> argv) {
   codeql::SwiftExtractorConfiguration configuration{};
   configuration.trapDir = getenv_or("CODEQL_EXTRACTOR_SWIFT_TRAP_DIR", ".");
   configuration.sourceArchiveDir = getenv_or("CODEQL_EXTRACTOR_SWIFT_SOURCE_ARCHIVE_DIR", ".");
   configuration.scratchDir = getenv_or("CODEQL_EXTRACTOR_SWIFT_SCRATCH_DIR", ".");
-  configuration.frontendOptions.assign(argv + 1, argv + argc);
+  configuration.frontendOptions.assign(argv.begin() + 1, argv.end());
   return configuration;
 }
 
-int main(int argc, char** argv) {
-  checkWhetherToRunUnderTool(argc, argv);
-
-  if (argc == 1) {
-    // TODO: print usage
-    return 1;
-  }
-
-  // Required by Swift/LLVM
-  PROGRAM_START(argc, argv);
-  INITIALIZE_LLVM();
-  initializeSwiftModules();
-
-  const auto configuration = configure(argc, argv);
-
-  auto openInterception = codeql::setupFileInterception(configuration);
-
-  Observer observer(configuration);
-  int frontend_rc = swift::performFrontend(configuration.frontendOptions, "swift-extractor",
-                                           (void*)main, &observer);
-
-  if (frontend_rc == 0) {
-    observer.markSuccessfullyExtractedFiles();
-  }
+namespace swift {
 
-  return frontend_rc;
+FrontendObserver* getFrontendObserver(llvm::ArrayRef<const char*> argv) {
+  static Observer observer{configure(argv)};
+  return &observer;
 }
+}  // namespace swift

swift/ql/test/extractor-tests/run_under/Strings.expected

@@ -1,2 +1,2 @@
-| run_under: $CODEQL_EXTRACTOR_SWIFT_ROOT/tools/$CODEQL_PLATFORM/extractor -sdk $CODEQL_EXTRACTOR_SWIFT_ROOT/qltest/$CODEQL_PLATFORM/sdk -c -primary-file filtered_in.swift |
-| run_under: $CODEQL_EXTRACTOR_SWIFT_ROOT/tools/$CODEQL_PLATFORM/extractor -sdk $CODEQL_EXTRACTOR_SWIFT_ROOT/qltest/$CODEQL_PLATFORM/sdk -c -primary-file unfiltered.swift |
+| run_under: $CODEQL_EXTRACTOR_SWIFT_ROOT/tools/$CODEQL_PLATFORM/swift-frontend -sdk $CODEQL_EXTRACTOR_SWIFT_ROOT/qltest/$CODEQL_PLATFORM/sdk -c -primary-file filtered_in.swift |
+| run_under: $CODEQL_EXTRACTOR_SWIFT_ROOT/tools/$CODEQL_PLATFORM/swift-frontend -sdk $CODEQL_EXTRACTOR_SWIFT_ROOT/qltest/$CODEQL_PLATFORM/sdk -c -primary-file unfiltered.swift |

swift/third_party/load.bzl

@@ -1,10 +1,10 @@
 load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
 
-_swift_prebuilt_version = "swift-5.7.3-RELEASE.142"
+_swift_prebuilt_version = "swift-5.7.3-RELEASE.150"
 _swift_sha_map = {
-    "Linux-X64": "398d8de54c8775c939dff95ed5bb0e04a9308a1982b4c1900cd4a5d01223f63b",
-    "macOS-ARM64": "397dd67ea99b9c9455794c6eb0f1664b6179fe542c7c1d3010314a3e8a905ae4",
-    "macOS-X64": "4b9d8e4e89f16a7c1e7edc7893aa189b37d5b4412be724a86ef59c49d11a6f75",
+    "Linux-X64": "8465b5ad6b34c723786ae56478ece1a3a778c812c4d0a0521236a12c6544f57d",
+    "macOS-ARM64": "38b84b24366841fc2b7c5fe081d87310d8e62051884147e155bc74a929a770b7",
+    "macOS-X64": "1be86d88e8a9be530a8c48ba73cd000daaae9d0435ae1db9ff3a368444639b99",
 }
 
 _swift_arch_map = {

swift/third_party/swift-llvm-support/BUILD.bazel

@@ -17,5 +17,5 @@ _arch_override = {
             for arch in ("linux", "darwin_x86_64", "darwin_arm64")
         }),
     )
-    for name in ("swift-llvm-support", "swift-test-sdk")
+    for name in ("swift-llvm-support", "swift-test-sdk", "swift-frontend")
 ]

swift/third_party/swift-llvm-support/BUILD.swift-llvm-support.bazel

@@ -2,23 +2,16 @@ load("@rules_pkg//:mappings.bzl", "pkg_files", "strip_prefix")
 
 cc_library(
     name = "swift-llvm-support",
-    srcs = [
-        "libCodeQLSwiftFrontendTool.a",
-    ] + select({
-        "@platforms//os:linux": [
-            "libCodeQLSwiftFrontendTool.so",
-            "libswiftCore.so",
-        ],
-        "@platforms//os:macos": [
-            "libCodeQLSwiftFrontendTool.dylib",
-            "libswiftCore.dylib",
-            "libswiftCompatibility50.a",
-            "libswiftCompatibility51.a",
-            "libswiftCompatibilityConcurrency.a",
-            "libswiftCompatibilityDynamicReplacements.a",
-        ],
-    }),
-    hdrs = glob(["include/**/*", "stdlib/**/*" ]),
+    srcs = glob([
+        "lib*.a",
+        "lib*.so",
+        "lib*.dylib",
+    ]),
+    hdrs = glob([
+        "include/**/*",
+        "stdlib/**/*",
+    ]),
+    includes = ["include"],
     linkopts = [
         "-lm",
         "-lz",
@@ -34,7 +27,6 @@ cc_library(
         ],
         "//conditions:default": [],
     }),
-    includes = [ "include" ],
     visibility = ["//visibility:public"],
 )
 
@@ -46,3 +38,5 @@ pkg_files(
     strip_prefix = strip_prefix.from_pkg(),
     visibility = ["//visibility:public"],
 )
+
+exports_files(["swift-frontend"])