Use Clang 13 in non-sanitizer Linux x86_64 builds

Summary: Use Clang 13 instead of Clang 12 in non-sanitizer Linux x86_64 builds. Also switch LLVM URL detection to use the llvm-installer module (https://pypi.org/project/llvm-installer/). Test Plan: Jenkins Reviewers: steve.varnau Reviewed By: steve.varnau Subscribers: ybase Differential Revision: https://phabricator.dev.yugabyte.com/D18136
norachoukri · Jul 7, 2022 · 429ba0d · 429ba0d
1 parent f166e71
commit 429ba0d
Show file tree

Hide file tree

Showing 9 changed files with 44 additions and 109 deletions.
diff --git a/build-support/common-build-env-test.sh b/build-support/common-build-env-test.sh
@@ -189,9 +189,17 @@ test_set_cmake_build_type_and_compiler_type   release    linux-gnu gcc        re
 test_set_cmake_build_type_and_compiler_type   release    linux-gnu gcc8       release    gcc8    0
 test_set_cmake_build_type_and_compiler_type   release    linux-gnu gcc9       release    gcc9    0
 test_set_cmake_build_type_and_compiler_type   asan       linux-gnu auto       fastdebug  clang12 0
-test_set_cmake_build_type_and_compiler_type   debug      linux-gnu auto       debug      clang12 0
-test_set_cmake_build_type_and_compiler_type   FaStDeBuG  linux-gnu auto       fastdebug  clang12 0
-test_set_cmake_build_type_and_compiler_type   release    linux-gnu auto       release    clang12 0
+
+# Test cases where there is difference between architectures.
+if [[ $arch == "x86_64" ]]; then
+  clangN=clang13
+else
+  clangN=clang12
+fi
+
+test_set_cmake_build_type_and_compiler_type   debug      linux-gnu auto       debug      $clangN 0
+test_set_cmake_build_type_and_compiler_type   FaStDeBuG  linux-gnu auto       fastdebug  $clangN 0
+test_set_cmake_build_type_and_compiler_type   release    linux-gnu auto       release    $clangN 0
 
 # -------------------------------------------------------------------------------------------------
 

diff --git a/build-support/common-build-env.sh b/build-support/common-build-env.sh
@@ -343,7 +343,7 @@ decide_whether_to_use_linuxbrew() {
         YB_USE_LINUXBREW=1
       fi
     elif [[ -n ${YB_LINUXBREW_DIR:-} ||
-            ( ${YB_COMPILER_TYPE} == "clang12" &&
+            ( ${YB_COMPILER_TYPE} =~ ^clang[0-9]+$ &&
               $build_type == "release" &&
               "$( uname -m )" == "x86_64" ) ]]; then
       YB_USE_LINUXBREW=1
@@ -495,7 +495,13 @@ set_default_compiler_type() {
     if is_mac; then
       YB_COMPILER_TYPE=clang
     elif [[ $OSTYPE =~ ^linux ]]; then
-      YB_COMPILER_TYPE=clang12
+      detect_architecture
+      if [[ ${build_type} =~ ^(debug|fastdebug|release)$ &&
+            ${YB_TARGET_ARCH} == "x86_64" ]]; then
+        YB_COMPILER_TYPE=clang13
+      else
+        YB_COMPILER_TYPE=clang12
+      fi
     else
       fatal "Cannot set default compiler type on OS $OSTYPE"
     fi
@@ -1207,6 +1213,7 @@ download_thirdparty() {
 }
 
 download_toolchain() {
+  expect_vars_to_be_set YB_COMPILER_TYPE
   local toolchain_urls=()
   local linuxbrew_url=""
   if [[ -n ${YB_THIRDPARTY_DIR:-} && -f "$YB_THIRDPARTY_DIR/linuxbrew_url.txt" ]]; then
@@ -1231,6 +1238,19 @@ download_toolchain() {
   if [[ -n ${linuxbrew_url:-} ]]; then
     toolchain_urls+=( "$linuxbrew_url" )
   fi
+  if [[ -z ${YB_LLVM_TOOLCHAIN_URL:-} &&
+        ${YB_COMPILER_TYPE:-} =~ ^clang[0-9]+$ ]] && is_linux; then
+    YB_LLVM_TOOLCHAIN_URL=$(
+      activate_virtualenv;
+      python3 -m llvm_installer --print-url --llvm-major-version=${YB_COMPILER_TYPE#clang}
+    )
+    if [[ ${YB_LLVM_TOOLCHAIN_URL} != https://* ]]; then
+      fatal "Failed to determine LLVM toolchain URL using the llvm-installer utility." \
+            "YB_LLVM_TOOLCHAIN_URL=${YB_LLVM_TOOLCHAIN_URL}. See" \
+            "https://github.com/yugabyte/llvm-installer for details."
+    fi
+    export YB_LLVM_TOOLCHAIN_URL
+  fi
   if [[ -n ${YB_LLVM_TOOLCHAIN_URL:-} ]]; then
     toolchain_urls+=( "${YB_LLVM_TOOLCHAIN_URL}" )
   fi
@@ -1255,7 +1275,7 @@ download_toolchain() {
     fi
 
     download_and_extract_archive "$toolchain_url" "$toolchain_dir_parent"
-    if "$is_linuxbrew"; then
+    if [[ ${is_linuxbrew} == "true" ]]; then
       if [[ -n ${YB_LINUXBREW_DIR:-} &&
             $YB_LINUXBREW_DIR != "$extracted_dir" ]]; then
         log_thirdparty_and_toolchain_details
@@ -1267,7 +1287,7 @@ download_toolchain() {
       save_brew_path_to_build_dir
     fi
 
-    if "$is_llvm"; then
+    if [[ ${is_llvm} == "true" ]]; then
       if [[ -n ${YB_LLVM_TOOLCHAIN_DIR:-} &&
             $YB_LLVM_TOOLCHAIN_DIR != "$extracted_dir" ]]; then
         log_thirdparty_and_toolchain_details
@@ -2442,7 +2462,6 @@ set_prebuilt_thirdparty_url() {
       local thirdparty_tool_cmd_line=(
         "$YB_BUILD_SUPPORT_DIR/thirdparty_tool"
         --save-thirdparty-url-to-file "$thirdparty_url_file_path"
-        --save-llvm-url-to-file "$llvm_url_file_path"
         --compiler-type "$YB_COMPILER_TYPE"
       )
       if [[ -n ${YB_USE_LINUXBREW:-} ]]; then

diff --git a/build-support/jenkins/build-and-test.sh b/build-support/jenkins/build-and-test.sh
@@ -204,7 +204,7 @@ log "YB_DOWNLOAD_THIRDPARTY=$YB_DOWNLOAD_THIRDPARTY"
 decide_whether_to_use_linuxbrew
 
 if [[ -z ${YB_LINKING_TYPE:-} ]]; then
-  if using_linuxbrew && [[ "${YB_COMPILER_TYPE}" =~ clang1[234] && "${BUILD_TYPE}" == "release" ]]
+  if using_linuxbrew && [[ "${YB_COMPILER_TYPE}" =~ ^clang[0-9]+$ && "${BUILD_TYPE}" == "release" ]]
   then
     export YB_LINKING_TYPE=full-lto
   else

diff --git a/ent/build-support/upload_package.sh b/ent/build-support/upload_package.sh
@@ -99,12 +99,6 @@ upload_package() {
       log "Skipping package upload for a non-release build (build type: $BUILD_TYPE)"
       return
     fi
-
-    if is_linux && [[ $YB_COMPILER_TYPE != "gcc" &&
-                      $YB_COMPILER_TYPE != "clang12" ]]; then
-      log "Skipping package upload on Linux (compiler type: $YB_COMPILER_TYPE)"
-      return
-    fi
   fi
 
   package_upload_skipped=false

diff --git a/jenkins_jobs.yml b/jenkins_jobs.yml
@@ -12,12 +12,12 @@ jobs:
     release_artifact: false
 
   - os: centos7
-    compiler: clang12
+    compiler: clang13
     build_type: debug
     release_artifact: false
 
   - os: alma8
-    compiler: clang12
+    compiler: clang13
     build_type: release
     release_artifact: true
 
@@ -49,6 +49,6 @@ jobs:
     release_artifact: true
 
   - os: ubuntu20.04
-    compiler: clang12
+    compiler: clang13
     build_type: debug
     release_artifact: false
diff --git a/python/yb/llvm_urls.py b/python/yb/llvm_urls.py
diff --git a/python/yb/thirdparty_tool.py b/python/yb/thirdparty_tool.py
@@ -52,8 +52,6 @@
 
 from yb.os_versions import adjust_os_type, is_compatible_os
 
-from yb.llvm_urls import get_llvm_url
-
 
 ruamel_yaml_object = ruamel.yaml.YAML()
 
@@ -331,11 +329,6 @@ def parse_args() -> argparse.Namespace:
         '--save-thirdparty-url-to-file',
         help='Determine the third-party archive download URL for the combination of criteria, '
              'including the compiler type, and write it to the file specified by this argument.')
-    parser.add_argument(
-        '--save-llvm-url-to-file',
-        help='Determine the LLVM toolchain archive download URL and write it to the file '
-             'specified by this argument. Similar to --save-download-url-to-file but also '
-             'takes the OS into account.')
     parser.add_argument(
         '--compiler-type',
         help='Compiler type, to help us decide which third-party archive to choose. '
@@ -519,7 +512,7 @@ def update_archive_metadata_file(self) -> None:
                 groups_to_use.append(releases_by_commit[extra_commit])
 
         new_metadata: Dict[str, Any] = {
-            SHA_FOR_LOCAL_CHECKOUT_KEY: sha,
+            SHA_FOR_LOCAL_CHECKOUT_KEY: latest_release_sha,
             'archives': []
         }
         releases_to_use: List[GitHubThirdPartyRelease] = [
@@ -724,7 +717,7 @@ def main() -> None:
             print(compiler)
         return
 
-    if args.save_thirdparty_url_to_file or args.save_llvm_url_to_file:
+    if args.save_thirdparty_url_to_file:
         if not args.compiler_type:
             raise ValueError("Compiler type not specified")
         thirdparty_release: Optional[MetadataItem] = get_third_party_release(
@@ -741,19 +734,6 @@ def main() -> None:
         if args.save_thirdparty_url_to_file:
             make_parent_dir(args.save_thirdparty_url_to_file)
             write_file(thirdparty_url, args.save_thirdparty_url_to_file)
-        if (args.save_llvm_url_to_file and
-                thirdparty_release.compiler_type.startswith('clang') and
-                thirdparty_release.is_linuxbrew):
-            llvm_url = get_llvm_url(thirdparty_release.compiler_type)
-            if llvm_url is not None:
-                logging.info(f"Download URL for the LLVM toolchain: {llvm_url}")
-                make_parent_dir(args.save_llvm_url_to_file)
-                write_file(llvm_url, args.save_llvm_url_to_file)
-            else:
-                logging.info("Could not determine LLVM URL for compiler type %s" %
-                             thirdparty_release.compiler_type)
-        else:
-            logging.info("Not a Linuxbrew URL, not saving LLVM URL to file")
 
 
 if __name__ == '__main__':

diff --git a/requirements.txt b/requirements.txt
@@ -18,3 +18,4 @@ wheel
 yugabyte_pycommon
 compiler-identification
 pytest
+llvm-installer
diff --git a/requirements_frozen.txt b/requirements_frozen.txt
@@ -35,7 +35,7 @@ ruamel.yaml.clib==0.2.6
 semantic-version==2.8.5
 shutilwhich==1.1.0
 six==1.16.0
-sys-detection==1.2.0
+sys-detection==1.3.0
 toml==0.10.2
 typed-ast==1.5.1
 typing-extensions==3.10.0.2
@@ -44,3 +44,4 @@ urllib3==1.26.7
 wrapt==1.12.1
 yugabyte-pycommon==1.9.15
 zipp==3.6.0
+llvm_installer==1.1.3