Merge commit 'e5ed7b6e2fd3' (Feb 27th)

commit e5ed7b6
Author: rohit-rao <rohitrao@google.com>
Date:   Tue Feb 27 15:23:00 2024 -0500

    [clang] Extend define-target-os-macros to support XROS. (llvm#82833)

Author: mgehre-amd

.ci/monolithic-windows.sh

@@ -38,6 +38,12 @@ targets="${2}"
 
 echo "--- cmake"
 pip install -q -r ${MONOREPO_ROOT}/mlir/python/requirements.txt
+
+# The CMAKE_*_LINKER_FLAGS to disable the manifest come from research
+# on fixing a build reliability issue on the build server, please
+# see https://github.com/llvm/llvm-project/pull/82393 and
+# https://discourse.llvm.org/t/rfc-future-of-windows-pre-commit-ci/76840/40
+# for further information.
 cmake -S ${MONOREPO_ROOT}/llvm -B ${BUILD_DIR} \
       -D LLVM_ENABLE_PROJECTS="${projects}" \
       -G Ninja \
@@ -49,7 +55,10 @@ cmake -S ${MONOREPO_ROOT}/llvm -B ${BUILD_DIR} \
       -D COMPILER_RT_BUILD_ORC=OFF \
       -D CMAKE_C_COMPILER_LAUNCHER=sccache \
       -D CMAKE_CXX_COMPILER_LAUNCHER=sccache \
-      -D MLIR_ENABLE_BINDINGS_PYTHON=ON
+      -D MLIR_ENABLE_BINDINGS_PYTHON=ON \
+      -D CMAKE_EXE_LINKER_FLAGS="/MANIFEST:NO" \
+      -D CMAKE_MODULE_LINKER_FLAGS="/MANIFEST:NO" \
+      -D CMAKE_SHARED_LINKER_FLAGS="/MANIFEST:NO"
 
 echo "--- ninja"
 # Targets are not escaped as they are passed as separate arguments.

.git-blame-ignore-revs

@@ -73,3 +73,11 @@ f6d557ee34b6bbdb1dc32f29e34b4a4a8ad35e81
 
 # [libc++] Format the code base (#74334)
 9783f28cbb155e4a8d49c12e1c60ce14dcfaf0c7
+
+# [RFC] compiler-rt builtins cleanup and refactoring
+082b89b25faae3e45a023caf51b65ca0f02f377f
+0ba22f51d128bee9d69756c56c4678097270e10b
+84da0e1bb75f8666cf222d2f600f37bebb9ea389
+
+# [NFC] clang-format utils/TableGen (#80973)
+b9079baaddfed5e604fbfaa1d81a7a1c38e78c26

.github/CODEOWNERS

@@ -101,3 +101,6 @@
 
 # MLIR Sparsifier.
 /mlir/**/*SparseTensor*/ @aartbik @PeimingLiu @yinying-lisa-li @matthias-springer
+
+# BOLT
+/bolt/ @aaupov @maksfb @rafaelauler @ayermolo @dcci

.github/new-prs-labeler.yml

@@ -75,12 +75,49 @@ mc:
 clang:driver:
   - clang/*/Driver/**
 
+compiler-rt:asan:
+  - compiler-rt/lib/asan/**
+  - compiler-rt/include/sanitizer/asan_interface.h
+  - compiler-rt/test/asan/**
+  - compiler-rt/lib/asan_abi/**
+  - compiler-rt/test/asan_abi/**
+
+compiler-rt:builtins:
+  - compiler-rt/lib/builtins/**
+  - compiler-rt/test/builtins/**
+
+compiler-rt:cfi:
+  - compiler-rt/lib/cfi/**
+  - compiler-rt/test/cfi/**
+
+compiler-rt:fuzzer:
+  - compiler-rt/lib/fuzzer/**
+  - compiler-rt/include/fuzzer/**
+  - compiler-rt/test/fuzzer/**
+
+compiler-rt:hwasan:
+  - compiler-rt/lib/hwasan/**
+  - compiler-rt/include/sanitizer/hwasan_interface.h
+  - compiler-rt/test/hwasan/**
+
+compiler-rt:lsan:
+  - compiler-rt/lib/lsan/**
+  - compiler-rt/include/sanitizer/lsan_interface.h
+  - compiler-rt/test/lsan/**
+
+compiler-rt:msan:
+  - compiler-rt/lib/msan/**
+  - compiler-rt/include/sanitizer/msan_interface.h
+  - compiler-rt/test/msan/**
+
 compiler-rt:sanitizer:
   - llvm/lib/Transforms/Instrumentation/*Sanitizer*
   - compiler-rt/lib/interception/**
   - compiler-rt/lib/*san*/**
+  - compiler-rt/include/sanitizer/**
   - compiler-rt/test/*san*/**
   - compiler-rt/lib/fuzzer/**
+  - compiler-rt/include/fuzzer/**
   - compiler-rt/test/fuzzer/**
   - compiler-rt/lib/scudo/**
   - compiler-rt/test/scudo/**
@@ -89,6 +126,19 @@ compiler-rt:scudo:
   - compiler-rt/lib/scudo/**
   - compiler-rt/test/scudo/**
 
+compiler-rt:tsan:
+  - compiler-rt/lib/tsan/**
+  - compiler-rt/include/sanitizer/tsan_interface.h
+  - compiler-rt/include/sanitizer/tsan_interface_atomic.h
+  - compiler-rt/test/tsan/**
+
+compiler-rt:ubsan:
+  - compiler-rt/lib/ubsan/**
+  - compiler-rt/include/sanitizer/ubsan_interface.h
+  - compiler-rt/test/ubsan/**
+  - compiler-rt/lib/ubsan_minimal/**
+  - compiler-rt/test/ubsan_minimal/**
+
 xray:
   - llvm/tools/llvm-xray/**
   - compiler-rt/*/xray/**
@@ -772,6 +822,50 @@ backend:X86:
   - llvm/lib/TargetParser/X86*
   - llvm/utils/TableGen/X86*
 
+backend:PowerPC:
+  - llvm/include/llvm/BinaryFormat/ELFRelocs/PowerPC*
+  - llvm/include/llvm/BinaryFormat/XCOFF.h
+  - llvm/include/llvm/IR/IntrinsicsPowerPC.td
+  - llvm/lib/CodeGen/AsmPrinter/AIXException.cpp
+  - llvm/lib/Target/PowerPC/**
+  - llvm/test/Analysis/**/PowerPC/**
+  - llvm/test/CodeGen/PowerPC/**
+  - llvm/test/CodeGen/MIR/PowerPC/**
+  - llvm/test/DebugInfo/XCOFF/**
+  - llvm/test/DebugInfo/PowerPC/**
+  - llvm/test/LTO/PowerPC/**
+  - llvm/test/MC/Disassembler/PowerPC/**
+  - llvm/test/MC/PowerPC/**
+  - llvm/test/MC/XCOFF/**
+  - llvm/test/Transforms/**/PowerPC/**
+  - clang/include/clang/Basic/BuiltinsPPC.*
+  - clang/lib/Basic/Targets/PPC.*
+  - clang/lib/CodeGen/Targets/PPC.cpp
+  - clang/lib/Driver/ToolChains/PPC*
+  - clang/lib/Driver/ToolChains/AIX*
+  - clang/lib/Driver/ToolChains/Arch/PPC.*
+  - clang/test/CodeGen/PowerPC/**
+
+backend:SystemZ:
+  - llvm/include/llvm/BinaryFormat/ELFRelocs/SystemZ*
+  - llvm/include/llvm/BinaryFormat/GOFF.h
+  - llvm/include/llvm/IR/IntrinsicsSystemZ.td
+  - llvm/lib/Target/SystemZ/**
+  - llvm/test/Analysis/**/SystemZ/**
+  - llvm/test/CodeGen/SystemZ/**
+  - llvm/test/DebugInfo/SystemZ/**
+  - llvm/test/ExecutionEngine/**/SystemZ/**
+  - llvm/test/MC/Disassembler/SystemZ/**
+  - llvm/test/MC/GOFF/**
+  - llvm/test/MC/SystemZ/**
+  - llvm/test/Transforms/**/SystemZ/**
+  - clang/include/clang/Basic/BuiltinsSystemZ.*
+  - clang/lib/Basic/Targets/SystemZ.*
+  - clang/lib/CodeGen/Targets/SystemZ.cpp
+  - clang/lib/Driver/ToolChains/ZOS*
+  - clang/lib/Driver/ToolChains/Arch/SystemZ.*
+  - clang/test/CodeGen/SystemZ/**
+
 third-party:unittests:
   - third-party/unittests/**
 

.github/workflows/build-ci-container.yml

@@ -1,4 +1,3 @@
-
 name: Build CI Container
 
 permissions:
@@ -19,9 +18,41 @@ on:
       - '.github/workflows/containers/github-action-ci/**'
 
 jobs:
-  build-ci-container:
+  # TODO(boomanaiden154): Switch this back to a single stage build when we can
+  # run this on the self-hosted runners and don't have to do it this way to
+  # avoid timeouts.
+  build-ci-container-stage1:
     if: github.repository_owner == 'llvm'
     runs-on: ubuntu-latest
+    steps:
+      - name: Checkout LLVM
+        uses: actions/checkout@v4
+        with:
+          sparse-checkout: .github/workflows/containers/github-action-ci/
+      - name: Change podman Root Direcotry
+        run: |
+          mkdir -p ~/.config/containers
+          sudo mkdir -p /mnt/podman
+          sudo chown `whoami`:`whoami` /mnt/podman
+          cp ./.github/workflows/containers/github-action-ci/storage.conf ~/.config/containers/storage.conf
+          podman info
+      - name: Build container stage1
+        working-directory: ./.github/workflows/containers/github-action-ci/
+        run: |
+          podman build -t stage1-toolchain --target stage1-toolchain -f stage1.Dockerfile .
+      - name: Save container image
+        run: |
+          podman save stage1-toolchain > stage1-toolchain.tar
+      - name: Upload container image
+        uses: actions/upload-artifact@v4
+        with:
+          name: stage1-toolchain
+          path: stage1-toolchain.tar
+          retention-days: 1
+  build-ci-container-stage2:
+    if: github.repository_owner == 'llvm'
+    runs-on: ubuntu-latest
+    needs: build-ci-container-stage1
     permissions:
       packages: write
     steps:
@@ -38,10 +69,27 @@ jobs:
         with:
           sparse-checkout: .github/workflows/containers/github-action-ci/
 
+      - name: Change podman Root Direcotry
+        run: |
+          mkdir -p ~/.config/containers
+          sudo mkdir -p /mnt/podman
+          sudo chown `whoami`:`whoami` /mnt/podman
+          cp ./.github/workflows/containers/github-action-ci/storage.conf ~/.config/containers/storage.conf
+          podman info
+
+      - name: Download stage1-toolchain
+        uses: actions/download-artifact@v4
+        with:
+          name: stage1-toolchain
+
+      - name: Load stage1-toolchain
+        run: |
+          podman load -i stage1-toolchain.tar
+
       - name: Build Container
         working-directory: ./.github/workflows/containers/github-action-ci/
         run: |
-          podman build -t ${{ steps.vars.outputs.container-name-tag }} .
+          podman build -t ${{ steps.vars.outputs.container-name-tag }} -f stage2.Dockerfile .
           podman tag ${{ steps.vars.outputs.container-name-tag }} ${{ steps.vars.outputs.container-name }}:latest
 
       - name: Test Container

.github/workflows/containers/github-action-ci/Dockerfile

@@ -0,0 +1,13 @@
+diff --git a/clang/cmake/caches/BOLT-PGO.cmake b/clang/cmake/caches/BOLT-PGO.cmake
+index 1a04ca9a74e5..d092820e4115 100644
+--- a/clang/cmake/caches/BOLT-PGO.cmake
++++ b/clang/cmake/caches/BOLT-PGO.cmake
+@@ -4,6 +4,8 @@ set(CLANG_BOOTSTRAP_TARGETS
+   stage2-clang-bolt
+   stage2-distribution
+   stage2-install-distribution
++  clang
++  lld
+   CACHE STRING "")
+ set(BOOTSTRAP_CLANG_BOOTSTRAP_TARGETS
+   clang-bolt

.github/workflows/containers/github-action-ci/bootstrap.patch

@@ -0,0 +1,44 @@
+FROM docker.io/library/ubuntu:22.04 as base
+ENV LLVM_SYSROOT=/opt/llvm
+
+FROM base as stage1-toolchain
+ENV LLVM_VERSION=17.0.6
+
+RUN apt-get update && \
+    apt-get install -y \
+    wget \
+    gcc \
+    g++ \
+    cmake \
+    ninja-build \
+    python3 \
+    git \
+    curl
+
+RUN curl -O -L https://github.com/llvm/llvm-project/archive/refs/tags/llvmorg-$LLVM_VERSION.tar.gz && tar -xf llvmorg-$LLVM_VERSION.tar.gz
+
+WORKDIR /llvm-project-llvmorg-$LLVM_VERSION
+
+COPY bootstrap.patch /
+
+# TODO(boomanaiden154): Remove the patch pulled from a LLVM PR once we bump
+# the toolchain to version 18 and the patch is in-tree.
+# TODO(boomanaiden154): Remove the bootstrap patch once we unsplit the build
+# and no longer need to explicitly build the stage2 dependencies.
+RUN curl https://github.com/llvm/llvm-project/commit/dd0356d741aefa25ece973d6cc4b55dcb73b84b4.patch | patch -p1 && cat /bootstrap.patch | patch -p1
+
+RUN mkdir build
+
+RUN cmake -B ./build -G Ninja ./llvm \
+  -C ./clang/cmake/caches/BOLT-PGO.cmake \
+  -DBOOTSTRAP_LLVM_ENABLE_LLD=ON \
+  -DBOOTSTRAP_BOOTSTRAP_LLVM_ENABLE_LLD=ON \
+  -DPGO_INSTRUMENT_LTO=Thin \
+  -DLLVM_ENABLE_RUNTIMES="compiler-rt" \
+  -DCMAKE_INSTALL_PREFIX="$LLVM_SYSROOT" \
+  -DLLVM_ENABLE_PROJECTS="bolt;clang;lld;clang-tools-extra" \
+  -DLLVM_DISTRIBUTION_COMPONENTS="lld;compiler-rt;clang-format" \
+  -DCLANG_DEFAULT_LINKER="lld" \
+  -DBOOTSTRAP_CLANG_PGO_TRAINING_DATA_SOURCE_DIR=/llvm-project-llvmorg-$LLVM_VERSION/llvm
+
+RUN ninja -C ./build stage2-instrumented-clang stage2-instrumented-lld

.github/workflows/containers/github-action-ci/stage1.Dockerfile

@@ -0,0 +1,27 @@
+FROM docker.io/library/ubuntu:22.04 as base
+ENV LLVM_SYSROOT=/opt/llvm
+
+FROM stage1-toolchain AS stage2-toolchain
+
+RUN ninja -C ./build stage2-clang-bolt stage2-install-distribution && ninja -C ./build install-distribution && rm -rf ./build
+
+FROM base
+
+COPY --from=stage2-toolchain $LLVM_SYSROOT $LLVM_SYSROOT
+
+# Need to install curl for hendrikmuhs/ccache-action
+# Need nodejs for some of the GitHub actions.
+# Need perl-modules for clang analyzer tests.
+RUN apt-get update && \
+    apt-get install -y \
+    binutils \
+    cmake \
+    curl \
+    libstdc++-11-dev \
+    ninja-build \
+    nodejs \
+    perl-modules \
+    python3-psutil
+
+ENV LLVM_SYSROOT=$LLVM_SYSROOT
+ENV PATH=${LLVM_SYSROOT}/bin:${PATH}

.github/workflows/containers/github-action-ci/stage2.Dockerfile

@@ -0,0 +1,4 @@
+[storage]
+  driver = "overlay"
+  runroot = "/mnt/podman/container"
+  graphroot = "/mnt/podman/image"