diff --git a/build-support/common-build-env-test.sh b/build-support/common-build-env-test.sh index 159eade2ae6c..dd1d300c3011 100755 --- a/build-support/common-build-env-test.sh +++ b/build-support/common-build-env-test.sh @@ -189,9 +189,17 @@ test_set_cmake_build_type_and_compiler_type release linux-gnu gcc re test_set_cmake_build_type_and_compiler_type release linux-gnu gcc8 release gcc8 0 test_set_cmake_build_type_and_compiler_type release linux-gnu gcc9 release gcc9 0 test_set_cmake_build_type_and_compiler_type asan linux-gnu auto fastdebug clang12 0 -test_set_cmake_build_type_and_compiler_type debug linux-gnu auto debug clang12 0 -test_set_cmake_build_type_and_compiler_type FaStDeBuG linux-gnu auto fastdebug clang12 0 -test_set_cmake_build_type_and_compiler_type release linux-gnu auto release clang12 0 + +# Test cases where there is difference between architectures. +if [[ $arch == "x86_64" ]]; then + clangN=clang13 +else + clangN=clang12 +fi + +test_set_cmake_build_type_and_compiler_type debug linux-gnu auto debug $clangN 0 +test_set_cmake_build_type_and_compiler_type FaStDeBuG linux-gnu auto fastdebug $clangN 0 +test_set_cmake_build_type_and_compiler_type release linux-gnu auto release $clangN 0 # ------------------------------------------------------------------------------------------------- diff --git a/build-support/common-build-env.sh b/build-support/common-build-env.sh index 3c588a0670fa..1f2950419dd4 100644 --- a/build-support/common-build-env.sh +++ b/build-support/common-build-env.sh @@ -343,7 +343,7 @@ decide_whether_to_use_linuxbrew() { YB_USE_LINUXBREW=1 fi elif [[ -n ${YB_LINUXBREW_DIR:-} || - ( ${YB_COMPILER_TYPE} == "clang12" && + ( ${YB_COMPILER_TYPE} =~ ^clang[0-9]+$ && $build_type == "release" && "$( uname -m )" == "x86_64" ) ]]; then YB_USE_LINUXBREW=1 @@ -495,7 +495,13 @@ set_default_compiler_type() { if is_mac; then YB_COMPILER_TYPE=clang elif [[ $OSTYPE =~ ^linux ]]; then - YB_COMPILER_TYPE=clang12 + detect_architecture + if [[ ${build_type} =~ ^(debug|fastdebug|release)$ && + ${YB_TARGET_ARCH} == "x86_64" ]]; then + YB_COMPILER_TYPE=clang13 + else + YB_COMPILER_TYPE=clang12 + fi else fatal "Cannot set default compiler type on OS $OSTYPE" fi @@ -1207,6 +1213,7 @@ download_thirdparty() { } download_toolchain() { + expect_vars_to_be_set YB_COMPILER_TYPE local toolchain_urls=() local linuxbrew_url="" if [[ -n ${YB_THIRDPARTY_DIR:-} && -f "$YB_THIRDPARTY_DIR/linuxbrew_url.txt" ]]; then @@ -1231,6 +1238,19 @@ download_toolchain() { if [[ -n ${linuxbrew_url:-} ]]; then toolchain_urls+=( "$linuxbrew_url" ) fi + if [[ -z ${YB_LLVM_TOOLCHAIN_URL:-} && + ${YB_COMPILER_TYPE:-} =~ ^clang[0-9]+$ ]] && is_linux; then + YB_LLVM_TOOLCHAIN_URL=$( + activate_virtualenv; + python3 -m llvm_installer --print-url --llvm-major-version=${YB_COMPILER_TYPE#clang} + ) + if [[ ${YB_LLVM_TOOLCHAIN_URL} != https://* ]]; then + fatal "Failed to determine LLVM toolchain URL using the llvm-installer utility." \ + "YB_LLVM_TOOLCHAIN_URL=${YB_LLVM_TOOLCHAIN_URL}. See" \ + "https://github.com/yugabyte/llvm-installer for details." + fi + export YB_LLVM_TOOLCHAIN_URL + fi if [[ -n ${YB_LLVM_TOOLCHAIN_URL:-} ]]; then toolchain_urls+=( "${YB_LLVM_TOOLCHAIN_URL}" ) fi @@ -1255,7 +1275,7 @@ download_toolchain() { fi download_and_extract_archive "$toolchain_url" "$toolchain_dir_parent" - if "$is_linuxbrew"; then + if [[ ${is_linuxbrew} == "true" ]]; then if [[ -n ${YB_LINUXBREW_DIR:-} && $YB_LINUXBREW_DIR != "$extracted_dir" ]]; then log_thirdparty_and_toolchain_details @@ -1267,7 +1287,7 @@ download_toolchain() { save_brew_path_to_build_dir fi - if "$is_llvm"; then + if [[ ${is_llvm} == "true" ]]; then if [[ -n ${YB_LLVM_TOOLCHAIN_DIR:-} && $YB_LLVM_TOOLCHAIN_DIR != "$extracted_dir" ]]; then log_thirdparty_and_toolchain_details @@ -2442,7 +2462,6 @@ set_prebuilt_thirdparty_url() { local thirdparty_tool_cmd_line=( "$YB_BUILD_SUPPORT_DIR/thirdparty_tool" --save-thirdparty-url-to-file "$thirdparty_url_file_path" - --save-llvm-url-to-file "$llvm_url_file_path" --compiler-type "$YB_COMPILER_TYPE" ) if [[ -n ${YB_USE_LINUXBREW:-} ]]; then diff --git a/build-support/jenkins/build-and-test.sh b/build-support/jenkins/build-and-test.sh index 9abd1927f830..fc2701e01644 100755 --- a/build-support/jenkins/build-and-test.sh +++ b/build-support/jenkins/build-and-test.sh @@ -204,7 +204,7 @@ log "YB_DOWNLOAD_THIRDPARTY=$YB_DOWNLOAD_THIRDPARTY" decide_whether_to_use_linuxbrew if [[ -z ${YB_LINKING_TYPE:-} ]]; then - if using_linuxbrew && [[ "${YB_COMPILER_TYPE}" =~ clang1[234] && "${BUILD_TYPE}" == "release" ]] + if using_linuxbrew && [[ "${YB_COMPILER_TYPE}" =~ ^clang[0-9]+$ && "${BUILD_TYPE}" == "release" ]] then export YB_LINKING_TYPE=full-lto else diff --git a/ent/build-support/upload_package.sh b/ent/build-support/upload_package.sh index b8e84b0efbe2..956348abf76b 100644 --- a/ent/build-support/upload_package.sh +++ b/ent/build-support/upload_package.sh @@ -99,12 +99,6 @@ upload_package() { log "Skipping package upload for a non-release build (build type: $BUILD_TYPE)" return fi - - if is_linux && [[ $YB_COMPILER_TYPE != "gcc" && - $YB_COMPILER_TYPE != "clang12" ]]; then - log "Skipping package upload on Linux (compiler type: $YB_COMPILER_TYPE)" - return - fi fi package_upload_skipped=false diff --git a/jenkins_jobs.yml b/jenkins_jobs.yml index d3aa1bc56ba4..0b9a390eb962 100644 --- a/jenkins_jobs.yml +++ b/jenkins_jobs.yml @@ -12,12 +12,12 @@ jobs: release_artifact: false - os: centos7 - compiler: clang12 + compiler: clang13 build_type: debug release_artifact: false - os: alma8 - compiler: clang12 + compiler: clang13 build_type: release release_artifact: true @@ -49,6 +49,6 @@ jobs: release_artifact: true - os: ubuntu20.04 - compiler: clang12 + compiler: clang13 build_type: debug release_artifact: false diff --git a/python/yb/llvm_urls.py b/python/yb/llvm_urls.py deleted file mode 100644 index c219c8e9a2db..000000000000 --- a/python/yb/llvm_urls.py +++ /dev/null @@ -1,68 +0,0 @@ -# Copyright (c) Yugabyte, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except -# in compliance with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software distributed under the License -# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -# or implied. See the License for the specific language governing permissions and limitations -# under the License. - -from sys_detection import local_sys_conf - -from typing import Optional - -from yb.os_versions import is_compatible_os - - -def _llvm_url_for_tag(tag: str) -> str: - return 'https://github.com/yugabyte/build-clang/releases/download/%s/yb-llvm-%s.tar.gz' % ( - tag, tag) - - -COMPILER_TYPE_TO_ARCH_TO_OS_TYPE_TO_LLVM_URL = { - 'clang11': { - 'x86_64': { - 'centos7': _llvm_url_for_tag('v11.1.0-yb-1-1633099975-130bd22e-centos7-x86_64'), - 'almalinux8': _llvm_url_for_tag('v11.1.0-yb-1-1633143292-130bd22e-almalinux8-x86_64'), - }, - 'aarch64': { - 'centos8': _llvm_url_for_tag('v11.1.0-yb-1-1633544021-130bd22e-centos8-aarch64'), - }, - }, - 'clang12': { - 'x86_64': { - 'centos7': _llvm_url_for_tag('v12.0.1-yb-1-1633099823-bdb147e6-centos7-x86_64'), - 'almalinux8': _llvm_url_for_tag('v12.0.1-yb-1-1633143152-bdb147e6-almalinux8-x86_64'), - }, - 'aarch64': { - 'almalinux8': _llvm_url_for_tag('v12.0.1-yb-1-1648458260-bdb147e6-almalinux8-aarch64'), - }, - } -} - - -def get_llvm_url(compiler_type: str) -> Optional[str]: - os_type_to_llvm_url = ( - COMPILER_TYPE_TO_ARCH_TO_OS_TYPE_TO_LLVM_URL.get(compiler_type) or {} - ).get(local_sys_conf().architecture) - if os_type_to_llvm_url is None: - return None - - os_type = local_sys_conf().short_os_name_and_version() - if os_type in os_type_to_llvm_url: - return os_type_to_llvm_url[os_type] - - candidate_urls = [ - os_type_to_llvm_url[os_type_key] - for os_type_key in os_type_to_llvm_url - if is_compatible_os(os_type_key, os_type) - ] - if len(candidate_urls) > 1: - raise ValueError("Ambiguous LLVM URLs: %s" % candidate_urls) - if not candidate_urls: - return None - candidate_url = candidate_urls[0] - return candidate_url diff --git a/python/yb/thirdparty_tool.py b/python/yb/thirdparty_tool.py index 7e349943f300..9de8c738570a 100644 --- a/python/yb/thirdparty_tool.py +++ b/python/yb/thirdparty_tool.py @@ -52,8 +52,6 @@ from yb.os_versions import adjust_os_type, is_compatible_os -from yb.llvm_urls import get_llvm_url - ruamel_yaml_object = ruamel.yaml.YAML() @@ -331,11 +329,6 @@ def parse_args() -> argparse.Namespace: '--save-thirdparty-url-to-file', help='Determine the third-party archive download URL for the combination of criteria, ' 'including the compiler type, and write it to the file specified by this argument.') - parser.add_argument( - '--save-llvm-url-to-file', - help='Determine the LLVM toolchain archive download URL and write it to the file ' - 'specified by this argument. Similar to --save-download-url-to-file but also ' - 'takes the OS into account.') parser.add_argument( '--compiler-type', help='Compiler type, to help us decide which third-party archive to choose. ' @@ -519,7 +512,7 @@ def update_archive_metadata_file(self) -> None: groups_to_use.append(releases_by_commit[extra_commit]) new_metadata: Dict[str, Any] = { - SHA_FOR_LOCAL_CHECKOUT_KEY: sha, + SHA_FOR_LOCAL_CHECKOUT_KEY: latest_release_sha, 'archives': [] } releases_to_use: List[GitHubThirdPartyRelease] = [ @@ -724,7 +717,7 @@ def main() -> None: print(compiler) return - if args.save_thirdparty_url_to_file or args.save_llvm_url_to_file: + if args.save_thirdparty_url_to_file: if not args.compiler_type: raise ValueError("Compiler type not specified") thirdparty_release: Optional[MetadataItem] = get_third_party_release( @@ -741,19 +734,6 @@ def main() -> None: if args.save_thirdparty_url_to_file: make_parent_dir(args.save_thirdparty_url_to_file) write_file(thirdparty_url, args.save_thirdparty_url_to_file) - if (args.save_llvm_url_to_file and - thirdparty_release.compiler_type.startswith('clang') and - thirdparty_release.is_linuxbrew): - llvm_url = get_llvm_url(thirdparty_release.compiler_type) - if llvm_url is not None: - logging.info(f"Download URL for the LLVM toolchain: {llvm_url}") - make_parent_dir(args.save_llvm_url_to_file) - write_file(llvm_url, args.save_llvm_url_to_file) - else: - logging.info("Could not determine LLVM URL for compiler type %s" % - thirdparty_release.compiler_type) - else: - logging.info("Not a Linuxbrew URL, not saving LLVM URL to file") if __name__ == '__main__': diff --git a/requirements.txt b/requirements.txt index 06d937b67c5a..71bcf9fc13fb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -18,3 +18,4 @@ wheel yugabyte_pycommon compiler-identification pytest +llvm-installer diff --git a/requirements_frozen.txt b/requirements_frozen.txt index 88b5549d9339..165632c58dc5 100644 --- a/requirements_frozen.txt +++ b/requirements_frozen.txt @@ -35,7 +35,7 @@ ruamel.yaml.clib==0.2.6 semantic-version==2.8.5 shutilwhich==1.1.0 six==1.16.0 -sys-detection==1.2.0 +sys-detection==1.3.0 toml==0.10.2 typed-ast==1.5.1 typing-extensions==3.10.0.2 @@ -44,3 +44,4 @@ urllib3==1.26.7 wrapt==1.12.1 yugabyte-pycommon==1.9.15 zipp==3.6.0 +llvm_installer==1.1.3