From ce06510a39bb1b8e5a97bbe87b349a4c642710f6 Mon Sep 17 00:00:00 2001 From: Mikhail Bautin Date: Thu, 30 Dec 2021 13:03:50 -0800 Subject: [PATCH] [#10920] Support build with Clang 12 and Linuxbrew Summary: Support a build with Clang 12 and Linuxbrew. This is the same version of Linuxbrew that we have been using with GCC 5. The LLVM toolchain is not built for Linuxbrew but for the native OS, so we specify all libraries and include directories explicitly in order to build with Linuxbrew includes and libraries. The resulting package should have the same portability across target Linux distributions as the package we have been building with GCC 5. This will allow us to drop support for GCC 5 and use a much newer compiler for our production release package. Enhancing thirdparty_tool.py to allow using Linuxbrew-based packages built on a different OS, because they should not depend on the Linux distribution. Relevant PR in yugabyte-db-thirdparty: https://github.com/yugabyte/yugabyte-db-thirdparty/pull/95 Other changes: - Removing automatic installation of Linuxbrew into `~/.linuxbrew-yb-build`. Our standard practice is to install it into a path like `/opt/yb-build/brew/linuxbrew-20181203T161736v9`. - Remove unused constant SHARED_LINUXBREW_BUILDS_DIR. - Do not use Ninja from the Linuxbrew directory. - Activate the Python virtual environment early in the build-and-test.sh script used by Jenkins. - Add a python_with_venv.sh script to simplify running Python commands with the build virtualenv. - Simplify checking for Clang and GCC compiler families in CMake scripts (IS_CLANG and IS_GCC variables). - Add type annotations dependency_graph.py and refactor it. Add it to codecheck.ini. - Add a Python function arg_str_to_bool useful for creating user-friendly boolean arguments using argparse. Test Plan: Jenkins Run the build in various configurations on CentOS 7, AlmaLinux 8, macOS, across x86_64 and ARM architectures. Reviewers: steve.varnau, sergei Reviewed By: sergei Subscribers: ybase Differential Revision: https://phabricator.dev.yugabyte.com/D14487 --- CMakeLists.txt | 68 +-- build-support/common-build-env-test.sh | 94 +-- build-support/common-build-env.sh | 414 ++++++------- build-support/common-test-env.sh | 10 +- .../compiler-wrappers/compiler-wrapper.sh | 4 +- build-support/jenkins/build-and-test.sh | 11 +- build-support/python_with_venv.sh | 8 + build-support/show_build_root_name_regex.sh | 3 + build-support/thirdparty_archives.yml | 114 ++-- build-support/thirdparty_archives_manual.yml | 1 + cmake_modules/CompilerInfo.cmake | 10 + cmake_modules/YugabyteCMakeUnitTest.cmake | 32 +- cmake_modules/YugabyteFunctions.cmake | 171 ++++-- codecheck.ini | 21 +- jenkins_jobs.yml | 16 +- python/yb/common_util.py | 33 +- python/yb/dependency_graph.py | 554 +++++++++++------- python/yb/llvm_urls.py | 64 ++ python/yb/os_versions.py | 36 ++ python/yb/thirdparty_tool.py | 238 +++++--- src/postgres/CMakeLists.txt | 4 +- src/postgres/src/backend/tcop/postgres.c | 2 +- yb_build.sh | 11 +- 23 files changed, 1165 insertions(+), 754 deletions(-) create mode 100755 build-support/python_with_venv.sh create mode 100755 build-support/show_build_root_name_regex.sh create mode 100644 python/yb/llvm_urls.py create mode 100644 python/yb/os_versions.py diff --git a/CMakeLists.txt b/CMakeLists.txt index 68f745db3dda..cefb231c8e84 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -54,6 +54,7 @@ if("$ENV{YB_USE_PCH}" STREQUAL "1") endif() endif() include(YugabyteFunctions) +yb_initialize_constants() include(YugabyteTesting) ADD_CXX_FLAGS("-Werror") @@ -164,8 +165,6 @@ CHECK_YB_COMPILER_PATH(${CMAKE_C_COMPILER}) message("CMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}") CHECK_YB_COMPILER_PATH(${CMAKE_CXX_COMPILER}) -set(BUILD_SUPPORT_DIR ${CMAKE_CURRENT_SOURCE_DIR}/build-support) - # Provide a 'latest' symlink to this build directory if the "blessed" multi-build layout is # detected: # @@ -227,36 +226,17 @@ endif() INIT_COMPILER_TYPE_FROM_BUILD_ROOT() -# TODO: this is complicated by the fact that we are trying to build third-party dependencies here if -# necessary, and that the compiler is sometimes part of those third-party dependencies (e.g. Clang 7 -# with the Linuxbrew-based setup). Therefore, we don't yet know the exact compiler version. If we -# know that we are not using a third-party setup where we build our own Clang, or if we are not -# planning to invoke the third-party dependency build from here, we include the CompilerInfo module -# early. We also always do that on macOS where we only use Clang. - -set(INCLUDE_COMPILER_INFO_EARLY FALSE) -if(NOT "${YB_COMPILER_TYPE}" STREQUAL "clang" OR - NOT REBUILD_THIRDPARTY OR - APPLE) - set(INCLUDE_COMPILER_INFO_EARLY TRUE) -endif() -if(INCLUDE_COMPILER_INFO_EARLY) - message("Including the CompilerInfo module before determining the third-party instrumentation " - "type to use. Criteria: " - "REBUILD_THIRDPARTY=${REBUILD_THIRDPARTY}, " - "YB_COMPILER_TYPE=${YB_COMPILER_TYPE}.") - include(CompilerInfo) -endif() +include(CompilerInfo) # This helps find the right third-party build directory. if ("${YB_BUILD_TYPE}" MATCHES "^(asan|tsan)$") set(THIRDPARTY_INSTRUMENTATION_TYPE "${YB_BUILD_TYPE}") -elseif ("${COMPILER_FAMILY}" STREQUAL "clang") +elseif (IS_CLANG) set(THIRDPARTY_INSTRUMENTATION_TYPE "uninstrumented") -elseif ("${COMPILER_FAMILY}" STREQUAL "gcc") +elseif (IS_GCC) set(THIRDPARTY_INSTRUMENTATION_TYPE "uninstrumented") else() - message(FATAL_ERROR "Unknown or still undetermined compiler family: '${COMPILER_FAMILY}'.") + message(FATAL_ERROR "Unknown compiler family: '${COMPILER_FAMILY}'.") endif() message("THIRDPARTY_INSTRUMENTATION_TYPE=${THIRDPARTY_INSTRUMENTATION_TYPE}") @@ -318,7 +298,6 @@ DETECT_BREW() message("Using COMPILER_FAMILY=${COMPILER_FAMILY}") -# We can only use IS_CLANG after calling VALIDATE_COMPILER_TYPE. if (NOT APPLE AND "${CMAKE_SYSTEM_PROCESSOR}" STREQUAL "x86_64") # To enable 16-byte atomics support we should specify appropriate architecture. ADD_CXX_FLAGS("-march=ivybridge") @@ -520,7 +499,7 @@ if (IS_CLANG) if ("${COMPILER_VERSION}" VERSION_GREATER_EQUAL "12.0.0" AND APPLE) ADD_CXX_FLAGS("-Wno-c++17-compat-mangling") endif() -elseif("${COMPILER_FAMILY}" STREQUAL "gcc") +elseif(IS_GCC) if ("${CMAKE_SYSTEM_PROCESSOR}" STREQUAL "x86_64") ADD_CXX_FLAGS("-mno-abm -mno-movbe") endif() @@ -530,15 +509,15 @@ endif() set(YB_PREFIX_COMMON "${YB_THIRDPARTY_DIR}/installed/common") -# Flag to enable thread sanitizer (clang or gcc 4.8) -if ("${YB_BUILD_TYPE}" MATCHES "^(asan|tsan)$") - YB_SETUP_SANITIZER() -elseif (NOT APPLE AND IS_CLANG) +if(NOT APPLE AND IS_CLANG) YB_SETUP_CLANG() + if ("${YB_BUILD_TYPE}" MATCHES "^(asan|tsan)$") + YB_SETUP_SANITIZER() + endif() endif() if (USING_LINUXBREW) - include_directories(SYSTEM ${LINUXBREW_DIR}/include) + include_directories(SYSTEM "${LINUXBREW_DIR}/include") endif() if (NOT IS_CLANG) @@ -551,8 +530,8 @@ endif() # - disable 'alignment' because unaligned access is really OK on Nehalem and we do it # all over the place. if ("${YB_BUILD_TYPE}" STREQUAL "asan") - if(NOT ("${COMPILER_FAMILY}" STREQUAL "clang" OR - "${COMPILER_FAMILY}" STREQUAL "gcc" AND "${COMPILER_VERSION}" VERSION_GREATER "4.9")) + if(NOT (IS_CLANG OR + IS_GCC AND "${COMPILER_VERSION}" VERSION_GREATER "4.9")) message(SEND_ERROR "Cannot use UBSAN without clang or gcc >= 4.9") endif() set(CXX_NO_SANITIZE_FLAG "alignment") @@ -567,7 +546,7 @@ endif () if ("${YB_BUILD_TYPE}" MATCHES "^(asan|tsan)$") # GCC 4.8 and 4.9 (latest as of this writing) don't allow you to specify a # sanitizer blacklist. - if("${COMPILER_FAMILY}" STREQUAL "clang") + if(IS_CLANG) # Require clang 3.4 or newer; clang 3.3 has issues with TSAN and pthread # symbol interception. if("${COMPILER_VERSION}" VERSION_LESS "3.4") @@ -605,6 +584,11 @@ include_directories(src) enable_testing() +if (USING_LINUXBREW) + ADD_GLOBAL_RPATH_ENTRY_AND_LIB_DIR("${YB_BUILD_ROOT}/postgres/lib") + ADD_GLOBAL_RPATH_ENTRY_AND_LIB_DIR("${LINUXBREW_LIB_DIR}") +endif() + ############################################################ # Dependencies ############################################################ @@ -710,7 +694,7 @@ set(YB_CMAKE_CXX_EXTRA_FLAGS "-Wextra -Wno-unused-parameter") set(CMAKE_C_FLAGS "${CMAKE_CXX_FLAGS}") # C++-only flags. -if (("${COMPILER_FAMILY}" STREQUAL "gcc") AND +if ((IS_GCC) AND ("${CMAKE_CXX_COMPILER_VERSION}" VERSION_GREATER_EQUAL "7.0")) # Used with GCC 7. ADD_CXX_FLAGS("-faligned-new") @@ -726,7 +710,7 @@ message("CMAKE_C_FLAGS ${CMAKE_C_FLAGS}") set(CMAKE_CXX_STANDARD 14) -if ("${COMPILER_FAMILY}" STREQUAL "gcc" AND +if (IS_GCC AND "${COMPILER_VERSION}" VERSION_GREATER_EQUAL "8.0.0") # Remove after switching to C++17 ADD_CXX_FLAGS("-Wno-aligned-new") @@ -734,16 +718,6 @@ endif() set(CMAKE_CXX_STANDARD_REQUIRED ON) -if (USING_LINUXBREW) - # We need to add the postgres library directory prior to /usr/lib64 specifically in order to avoid - # using the system libpq. - ADD_GLOBAL_RPATH_ENTRY_AND_LIB_DIR("${YB_BUILD_ROOT}/postgres/lib") - - # Add the Linuxbrew library directory and the system library directory last. - ADD_GLOBAL_RPATH_ENTRY_AND_LIB_DIR("${LINUXBREW_LIB_DIR}") - ADD_GLOBAL_RPATH_ENTRY_AND_LIB_DIR("/usr/lib64") -endif() - message("Linker flags for executables: ${CMAKE_EXE_LINKER_FLAGS}") # Define full paths to libpq and libyb_pgbackend shared libraries. These libraries are built as part diff --git a/build-support/common-build-env-test.sh b/build-support/common-build-env-test.sh index 2d9d6aad2b58..dd3ae4359238 100755 --- a/build-support/common-build-env-test.sh +++ b/build-support/common-build-env-test.sh @@ -19,6 +19,7 @@ # set -euo pipefail +# shellcheck source=build-support/common-build-env.sh . "${0%/*}/common-build-env.sh" assert_equals() { @@ -90,7 +91,7 @@ test_compiler_detection_by_jenkins_job_name() { unset YB_COMPILER_TYPE JOB_NAME="$jenkins_job_name" set_compiler_type_based_on_jenkins_job_name - assert_equals "$expected_compiler_type" "$YB_COMPILER_TYPE" + assert_equals "$expected_compiler_type" "$YB_COMPILER_TYPE" "compiler type" ) } @@ -138,53 +139,70 @@ test_set_cmake_build_type_and_compiler_type() { OSTYPE=$os_type yb_fatal_quiet=true set_cmake_build_type_and_compiler_type - assert_equals "$expected_cmake_build_type" "$cmake_build_type" "$test_case_details" - assert_equals "$expected_compiler_type" "$YB_COMPILER_TYPE" "$test_case_details" + assert_equals "$expected_cmake_build_type" "$cmake_build_type" "$test_case_details" \ + "Note: comparing CMake build type." + assert_equals "$expected_compiler_type" "$YB_COMPILER_TYPE" "$test_case_details" \ + "Note: comparing compiler type." ) local exit_code=$? set -e assert_equals "$expected_exit_code" "$exit_code" } +arch=$( uname -m ) + # Parameters: build_type OSTYPE Compiler Expected Expected # type build_type YB_COMPILER_ # preference TYPE -test_set_cmake_build_type_and_compiler_type asan darwin auto fastdebug clang 0 -test_set_cmake_build_type_and_compiler_type asan darwin clang fastdebug clang 0 -test_set_cmake_build_type_and_compiler_type asan darwin gcc N/A N/A 1 -test_set_cmake_build_type_and_compiler_type asan linux-gnu auto fastdebug clang7 0 -test_set_cmake_build_type_and_compiler_type asan linux-gnu clang7 fastdebug clang7 0 -test_set_cmake_build_type_and_compiler_type asan linux-gnu gcc N/A N/A 1 -test_set_cmake_build_type_and_compiler_type asan linux-gnu gcc8 fastdebug gcc8 0 -test_set_cmake_build_type_and_compiler_type asan linux-gnu gcc9 fastdebug gcc9 0 -test_set_cmake_build_type_and_compiler_type tsan linux-gnu auto fastdebug clang7 0 -test_set_cmake_build_type_and_compiler_type tsan linux-gnu clang7 fastdebug clang7 0 -test_set_cmake_build_type_and_compiler_type tsan linux-gnu gcc N/A N/A 1 -test_set_cmake_build_type_and_compiler_type tsan linux-gnu gcc8 fastdebug gcc8 0 -test_set_cmake_build_type_and_compiler_type tsan linux-gnu gcc9 fastdebug gcc9 0 -test_set_cmake_build_type_and_compiler_type debug darwin auto debug clang 0 -test_set_cmake_build_type_and_compiler_type debug darwin clang debug clang 0 -test_set_cmake_build_type_and_compiler_type debug darwin gcc N/A N/A 1 -test_set_cmake_build_type_and_compiler_type debug linux-gnu auto debug gcc 0 -test_set_cmake_build_type_and_compiler_type debug linux-gnu clang debug clang 0 -test_set_cmake_build_type_and_compiler_type debug linux-gnu gcc debug gcc 0 -test_set_cmake_build_type_and_compiler_type debug linux-gnu gcc8 debug gcc8 0 -test_set_cmake_build_type_and_compiler_type debug linux-gnu gcc9 debug gcc9 0 -test_set_cmake_build_type_and_compiler_type FaStDeBuG darwin auto fastdebug clang 0 -test_set_cmake_build_type_and_compiler_type FaStDeBuG darwin clang fastdebug clang 0 -test_set_cmake_build_type_and_compiler_type FaStDeBuG darwin gcc N/A N/A 1 -test_set_cmake_build_type_and_compiler_type FaStDeBuG linux-gnu auto fastdebug gcc 0 -test_set_cmake_build_type_and_compiler_type FaStDeBuG linux-gnu clang fastdebug clang 0 -test_set_cmake_build_type_and_compiler_type FaStDeBuG linux-gnu gcc fastdebug gcc 0 -test_set_cmake_build_type_and_compiler_type release darwin auto release clang 0 -test_set_cmake_build_type_and_compiler_type release darwin clang release clang 0 -test_set_cmake_build_type_and_compiler_type release darwin gcc N/A N/A 1 -test_set_cmake_build_type_and_compiler_type release linux-gnu auto release gcc 0 -test_set_cmake_build_type_and_compiler_type release linux-gnu clang release clang 0 -test_set_cmake_build_type_and_compiler_type release linux-gnu gcc release gcc 0 -test_set_cmake_build_type_and_compiler_type release linux-gnu gcc8 release gcc8 0 -test_set_cmake_build_type_and_compiler_type release linux-gnu gcc9 release gcc9 0 +# The last parameter is expected exit code (0 or 1). + +test_set_cmake_build_type_and_compiler_type asan darwin auto fastdebug clang 0 +test_set_cmake_build_type_and_compiler_type asan darwin clang fastdebug clang 0 +test_set_cmake_build_type_and_compiler_type asan darwin gcc N/A N/A 1 +test_set_cmake_build_type_and_compiler_type asan linux-gnu clang7 fastdebug clang7 0 +test_set_cmake_build_type_and_compiler_type asan linux-gnu gcc N/A N/A 1 +test_set_cmake_build_type_and_compiler_type asan linux-gnu gcc8 N/A gcc8 1 +test_set_cmake_build_type_and_compiler_type asan linux-gnu gcc9 N/A gcc9 1 +test_set_cmake_build_type_and_compiler_type tsan linux-gnu auto fastdebug clang7 0 +test_set_cmake_build_type_and_compiler_type tsan linux-gnu clang7 fastdebug clang7 0 +test_set_cmake_build_type_and_compiler_type tsan linux-gnu gcc N/A N/A 1 +test_set_cmake_build_type_and_compiler_type tsan linux-gnu gcc8 N/A gcc8 1 +test_set_cmake_build_type_and_compiler_type tsan linux-gnu gcc9 N/A gcc9 1 +test_set_cmake_build_type_and_compiler_type debug darwin auto debug clang 0 +test_set_cmake_build_type_and_compiler_type debug darwin clang debug clang 0 +test_set_cmake_build_type_and_compiler_type debug darwin gcc N/A N/A 1 +test_set_cmake_build_type_and_compiler_type debug linux-gnu clang debug clang 0 +test_set_cmake_build_type_and_compiler_type debug linux-gnu gcc debug gcc 0 +test_set_cmake_build_type_and_compiler_type debug linux-gnu gcc8 debug gcc8 0 +test_set_cmake_build_type_and_compiler_type debug linux-gnu gcc9 debug gcc9 0 +test_set_cmake_build_type_and_compiler_type FaStDeBuG darwin auto fastdebug clang 0 +test_set_cmake_build_type_and_compiler_type FaStDeBuG darwin clang fastdebug clang 0 +test_set_cmake_build_type_and_compiler_type FaStDeBuG darwin gcc N/A N/A 1 +test_set_cmake_build_type_and_compiler_type FaStDeBuG linux-gnu clang fastdebug clang 0 +test_set_cmake_build_type_and_compiler_type FaStDeBuG linux-gnu gcc fastdebug gcc 0 +test_set_cmake_build_type_and_compiler_type release darwin auto release clang 0 +test_set_cmake_build_type_and_compiler_type release darwin clang release clang 0 +test_set_cmake_build_type_and_compiler_type release darwin gcc N/A N/A 1 +test_set_cmake_build_type_and_compiler_type release linux-gnu clang release clang 0 +test_set_cmake_build_type_and_compiler_type release linux-gnu gcc release gcc 0 +test_set_cmake_build_type_and_compiler_type release linux-gnu gcc8 release gcc8 0 +test_set_cmake_build_type_and_compiler_type release linux-gnu gcc9 release gcc9 0 + +# TODO: use Clang 12 uniformly for all achitectures, and get rid of conditional checks here. +if [[ $arch == "x86_64" && $OSTYPE == linux* ]] && is_redhat_family; then + test_set_cmake_build_type_and_compiler_type asan linux-gnu auto fastdebug clang12 0 + test_set_cmake_build_type_and_compiler_type debug linux-gnu auto debug clang12 0 + test_set_cmake_build_type_and_compiler_type FaStDeBuG linux-gnu auto fastdebug clang12 0 + test_set_cmake_build_type_and_compiler_type release linux-gnu auto release clang12 0 +fi + +if [[ $arch == "aarch64" && $OSTYPE == linux* ]]; then + test_set_cmake_build_type_and_compiler_type asan linux-gnu auto fastdebug clang11 0 + test_set_cmake_build_type_and_compiler_type debug linux-gnu auto debug clang11 0 + test_set_cmake_build_type_and_compiler_type FaStDeBuG linux-gnu auto fastdebug clang11 0 + test_set_cmake_build_type_and_compiler_type release linux-gnu auto release clang11 0 +fi # ------------------------------------------------------------------------------------------------- diff --git a/build-support/common-build-env.sh b/build-support/common-build-env.sh index f97ef4ddbf38..6284afc676c5 100644 --- a/build-support/common-build-env.sh +++ b/build-support/common-build-env.sh @@ -111,13 +111,11 @@ initialize_yugabyte_bash_common declare -i MAX_JAVA_BUILD_ATTEMPTS=5 # Reuse the C errno value for this. +# shellcheck disable=SC2034 declare -r -i YB_EXIT_CODE_NO_SUCH_FILE_OR_DIRECTORY=2 readonly YB_JENKINS_NFS_HOME_DIR=/Volumes/n/jenkins -# In our NFS environment, we keep Linuxbrew builds in this directory. -readonly SHARED_LINUXBREW_BUILDS_DIR="$YB_JENKINS_NFS_HOME_DIR/linuxbrew" - # This is the parent directory for all kinds of thirdparty and toolchain tarballs. readonly OPT_YB_BUILD_DIR="/opt/yb-build" @@ -141,8 +139,6 @@ readonly YB_NUM_CORES_PER_BUILD_WORKER=8 readonly MIN_EFFECTIVE_NUM_BUILD_WORKERS=5 readonly MAX_EFFECTIVE_NUM_BUILD_WORKERS=10 -readonly YB_LINUXBREW_LOCAL_ROOT=$HOME/.linuxbrew-yb-build - if [[ -z ${is_run_test_script:-} ]]; then is_run_test_script=false fi @@ -212,8 +208,9 @@ make_regex_from_list VALID_ARCHITECTURES "${VALID_ARCHITECTURES[@]}" readonly BUILD_ROOT_BASENAME_RE=\ "^($VALID_BUILD_TYPES_RAW_RE)-\ -($VALID_COMPILER_TYPES_RAW_RE)-\ -($VALID_LINKING_TYPES_RAW_RE)\ +($VALID_COMPILER_TYPES_RAW_RE)\ +(-linuxbrew)?\ +(-($VALID_LINKING_TYPES_RAW_RE))\ (-($VALID_ARCHITECTURES_RAW_RE))?\ (-ninja)?\ (-clion)?$" @@ -352,7 +349,35 @@ set_build_root() { fi validate_compiler_type "$YB_COMPILER_TYPE" - BUILD_ROOT=$YB_BUILD_PARENT_DIR/$build_type-$YB_COMPILER_TYPE-dynamic + BUILD_ROOT=$YB_BUILD_PARENT_DIR/$build_type-$YB_COMPILER_TYPE + + if [[ -z ${YB_USE_LINUXBREW:-} ]]; then + if [[ -n ${predefined_build_root:-} ]]; then + if [[ ${predefined_build_root##*/} == *-linuxbrew-* ]]; then + YB_USE_LINUXBREW=1 + else + YB_USE_LINUXBREW=0 + fi + elif [[ -n ${YB_LINUXBREW_DIR:-} || ${YB_COMPILER_TYPE} =~ ^gcc5?$ ]]; then + YB_USE_LINUXBREW=1 + elif [[ ${YB_COMPILER_TYPE} == "clang12" ]]; then + # For Clang 12 in particular, we have prebuilt third-party archives with and without + # Linuxbrew. Use Linuxbrew by default for the release build. + if [[ $build_type == "release" && "$( uname -m )" == "x86_64" ]]; then + YB_USE_LINUXBREW=1 + else + YB_USE_LINUXBREW=0 + fi + fi + export YB_USE_LINUXBREW=${YB_USE_LINUXBREW:-0} + fi + # Now we've finalized our decision about whether we are using Linuxbrew. + + if using_linuxbrew; then + BUILD_ROOT+="-linuxbrew" + fi + + BUILD_ROOT+="-dynamic" if is_apple_silicon; then # Append the target architecture (x86_64 or arm64). BUILD_ROOT+=-${YB_TARGET_ARCH} @@ -462,14 +487,30 @@ set_build_type_based_on_jenkins_job_name() { } set_default_compiler_type() { - if [[ -z "${YB_COMPILER_TYPE:-}" ]]; then - if [[ "$OSTYPE" =~ ^darwin ]]; then + if [[ -z ${YB_COMPILER_TYPE:-} ]]; then + if is_mac; then YB_COMPILER_TYPE=clang + elif [[ $OSTYPE =~ ^linux ]]; then + if [[ ${build_type:-} == "tsan" ]]; then + # TODO: upgrade Clang version used for TSAN as well. + YB_COMPILER_TYPE=clang7 + elif [[ $( uname -m ) == "aarch64" ]] || ! is_redhat_family; then + # TODO: produce a third-party build for aarch64 with Clang 12, and on Ubuntu. + YB_COMPILER_TYPE=clang11 + else + YB_COMPILER_TYPE=clang12 + fi else - YB_COMPILER_TYPE=gcc + fatal "Cannot set default compiler type on OS $OSTYPE" fi export YB_COMPILER_TYPE readonly YB_COMPILER_TYPE + else + if is_mac; then + if [[ $YB_COMPILER_TYPE != "clang" ]]; then + fatal "YB_COMPILER_TYPE is $YB_COMPILER_TYPE on macOS, but only 'clang' is supported" + fi + fi fi } @@ -606,30 +647,17 @@ set_cmake_build_type_and_compiler_type() { validate_cmake_build_type "$cmake_build_type" readonly cmake_build_type - if is_mac; then - if [[ -z ${YB_COMPILER_TYPE:-} ]]; then - YB_COMPILER_TYPE=clang - elif [[ $YB_COMPILER_TYPE != "clang" ]]; then - fatal "YB_COMPILER_TYPE can only be 'clang' on Mac OS X," \ - "found YB_COMPILER_TYPE=$YB_COMPILER_TYPE." - fi - elif [[ -z ${YB_COMPILER_TYPE:-} ]]; then - if [[ $build_type =~ ^(asan|tsan)$ ]]; then - # Use Clang by default for ASAN/TSAN builds. - YB_COMPILER_TYPE=clang7 - else - # The default on Linux. - YB_COMPILER_TYPE=gcc - fi - elif [[ $build_type =~ ^(asan|tsan)$ && $YB_COMPILER_TYPE == "gcc" ]]; then - fatal "Cannot use ASAN/TSAN with the 'gcc' compiler type. Has to be a specific version of " \ - "GCC such as gcc8 or gcc9." - fi + set_default_compiler_type validate_compiler_type readonly YB_COMPILER_TYPE export YB_COMPILER_TYPE + if [[ $build_type =~ ^asan|tsan|tsan_slow$ && $YB_COMPILER_TYPE == gcc* ]]; then + fatal "Build type $build_type not supported with compiler type $YB_COMPILER_TYPE." \ + "Sanitizers are only supported with Clang." + fi + # We need to set CMAKE_C_COMPILER and CMAKE_CXX_COMPILER outside of CMake. We used to do that from # CMakeLists.txt, and got into an infinite loop where CMake kept saying: # @@ -930,7 +958,7 @@ find_compiler_by_type() { unset cc_executable unset cxx_executable case "$YB_COMPILER_TYPE" in - gcc) + gcc|gcc5) if [[ -n ${YB_GCC_PREFIX:-} ]]; then if [[ ! -d $YB_GCC_PREFIX/bin ]]; then fatal "Directory YB_GCC_PREFIX/bin ($YB_GCC_PREFIX/bin) does not exist" @@ -1184,74 +1212,72 @@ download_thirdparty() { export YB_THIRDPARTY_DIR=$extracted_dir yb_thirdparty_dir_origin=" (downloaded from $YB_THIRDPARTY_URL)" save_thirdparty_info_to_build_dir - - if ! is_redhat_family; then - return - fi download_toolchain } download_toolchain() { - local toolchain_url_path="" - local toolchain_dir_parent="" - local is_linuxbrew=false - - if [[ -f $YB_THIRDPARTY_DIR/linuxbrew_url.txt || - -n ${YB_THIRDPARTY_URL:-} && ${YB_THIRDPARTY_URL##*/} == *linuxbrew* ]]; then - # Only attempt to download Linuxbrew if the third-party tarball name explicitly mentions it. - # Read a linuxbrew_url.txt file in the third-party directory that we downloaded, and follow that - # link to download and install the appropriate Linuxbrew package. - toolchain_url_path=$YB_THIRDPARTY_DIR/linuxbrew_url.txt - is_linuxbrew=true + local toolchain_urls=() + if [[ -n ${YB_THIRDPARTY_URL:-} && ${YB_THIRDPARTY_URL##*/} == *linuxbrew* ]]; then + # TODO: get rid of this and always include linuxbrew_url.txt in the thirdparty archives that are + # built for Linuxbrew. + toolchain_urls+=( "https://github.com/yugabyte/brew-build/releases/download/\ +20181203T161736v9/linuxbrew-20181203T161736v9.tar.gz" ) + if [[ -f $BUILD_ROOT/llvm_url.txt ]]; then + toolchain_urls+=( "$(<"$BUILD_ROOT/llvm_url.txt")" ) + fi else - toolchain_url_path=$YB_THIRDPARTY_DIR/toolchain_url.txt + for file_name_part in linuxbrew toolchain; do + local url_file_path="$YB_THIRDPARTY_DIR/${file_name_part}_url.txt" + if [[ -f $url_file_path ]]; then + toolchain_urls+=( "$(<"$url_file_path")" ) + break + fi + done + fi + if [[ ${#toolchain_urls[@]} -eq 0 ]]; then + return fi - if [[ -n $toolchain_url_path ]]; then - if [[ -f $toolchain_url_path ]]; then - local toolchain_url - toolchain_url=$(<"$toolchain_url_path") - local toolchain_url_basename=${toolchain_url##*/} + for toolchain_url in "${toolchain_urls[@]}"; do + local toolchain_url_basename=${toolchain_url##*/} + local is_llvm=false + local is_linuxbrew=false + if [[ $toolchain_url_basename == yb-llvm-* ]]; then + toolchain_dir_parent=$TOOLCHAIN_PARENT_DIR_LLVM + is_llvm=true + elif [[ $toolchain_url_basename =~ ^(yb-)?linuxbrew-.*$ ]]; then + toolchain_dir_parent=$TOOLCHAIN_PARENT_DIR_LINUXBREW + is_linuxbrew=true + else + fatal "Unable to determine the installation parent directory for the toolchain archive" \ + "named '$toolchain_url_basename'. Toolchain URL: '$toolchain_url'." + fi - is_llvm=false - if [[ $toolchain_url_basename == yb-llvm-* ]]; then - toolchain_dir_parent=$TOOLCHAIN_PARENT_DIR_LLVM - is_llvm=true - elif "$is_linuxbrew"; then - toolchain_dir_parent=$TOOLCHAIN_PARENT_DIR_LINUXBREW - else - fatal "Unable to determine the installation parent directory for the toolchain archive" \ - "named '$toolchain_url_basename'. Toolchain URL: '$toolchain_url'." - fi - - download_and_extract_archive "$toolchain_url" "$toolchain_dir_parent" - if "$is_linuxbrew"; then - if [[ -n ${YB_LINUXBREW_DIR:-} && - $YB_LINUXBREW_DIR != "$extracted_dir" ]]; then - log_thirdparty_and_toolchain_details - fatal "YB_LINUXBREW_DIR is already set to '$YB_LINUXBREW_DIR', cannot set it to" \ - "'$extracted_dir'" - fi - export YB_LINUXBREW_DIR=$extracted_dir - yb_linuxbrew_dir_origin=" (downloaded from $toolchain_url)" - save_brew_path_to_build_dir + download_and_extract_archive "$toolchain_url" "$toolchain_dir_parent" + if "$is_linuxbrew"; then + if [[ -n ${YB_LINUXBREW_DIR:-} && + $YB_LINUXBREW_DIR != "$extracted_dir" ]]; then + log_thirdparty_and_toolchain_details + fatal "YB_LINUXBREW_DIR is already set to '$YB_LINUXBREW_DIR', cannot set it to" \ + "'$extracted_dir'" fi + export YB_LINUXBREW_DIR=$extracted_dir + yb_linuxbrew_dir_origin=" (downloaded from $toolchain_url)" + save_brew_path_to_build_dir + fi - if "$is_llvm"; then - if [[ -n ${YB_LLVM_TOOLCHAIN_DIR:-} && - $YB_LLVM_TOOLCHAIN_DIR != "$extracted_dir" ]]; then - log_thirdparty_and_toolchain_details - fatal "YB_LLVM_TOOLCHAIN_DIR is already set to '$YB_LLVM_TOOLCHAIN_DIR', cannot set it " \ - "to '$extracted_dir'" - fi - export YB_LLVM_TOOLCHAIN_DIR=$extracted_dir - yb_llvm_toolchain_dir_origin=" (downloaded from $toolchain_url)" - save_llvm_toolchain_path_to_build_dir + if "$is_llvm"; then + if [[ -n ${YB_LLVM_TOOLCHAIN_DIR:-} && + $YB_LLVM_TOOLCHAIN_DIR != "$extracted_dir" ]]; then + log_thirdparty_and_toolchain_details + fatal "YB_LLVM_TOOLCHAIN_DIR is already set to '$YB_LLVM_TOOLCHAIN_DIR', cannot set it " \ + "to '$extracted_dir'" fi - elif "$is_linuxbrew"; then - fatal "Cannot download Linuxbrew: file '$toolchain_url_path' does not exist" + export YB_LLVM_TOOLCHAIN_DIR=$extracted_dir + yb_llvm_toolchain_dir_origin=" (downloaded from $toolchain_url)" + save_llvm_toolchain_path_to_build_dir fi - fi + done } # ------------------------------------------------------------------------------------------------- @@ -1259,7 +1285,7 @@ download_toolchain() { # ------------------------------------------------------------------------------------------------- disable_linuxbrew() { - export YB_DISABLE_LINUXBREW=1 + export YB_USE_LINUXBREW=0 unset YB_LINUXBREW_DIR } @@ -1269,19 +1295,7 @@ detect_toolchain() { } detect_brew() { - if [[ ${YB_DISABLE_LINUXBREW:-0} == "1" ]]; then - disable_linuxbrew - return - fi - if [[ -n ${YB_COMPILER_TYPE:-} && - # YB_COMPILER_TYPE could be set to a specific compiler version, like clang10 or gcc8, and - # in those cases we know we don't use Linuxbrew. - $YB_COMPILER_TYPE != "gcc" && - $YB_COMPILER_TYPE != "clang" ]]; then - disable_linuxbrew - return - fi - if is_ubuntu; then + if [[ ${YB_USE_LINUXBREW:-} == "0" ]]; then disable_linuxbrew return fi @@ -1296,45 +1310,6 @@ detect_brew() { fi } -install_linuxbrew() { - if ! is_linux; then - fatal "Expected this function to only be called on Linux" - fi - if is_ubuntu; then - return - fi - local version=$1 - local linuxbrew_dirname=linuxbrew-$version - local linuxbrew_dir=$YB_LINUXBREW_LOCAL_ROOT/$linuxbrew_dirname - local linuxbrew_archive="${linuxbrew_dir}.tar.gz" - local linuxbrew_archive_checksum="${linuxbrew_archive}.sha256" - local url="https://github.com/YugaByte/brew-build/releases/download/$version/\ -linuxbrew-$version.tar.gz" - mkdir -p "$YB_LINUXBREW_LOCAL_ROOT" - if [[ ! -f $linuxbrew_archive ]]; then - echo "Downloading Linuxbrew from $url..." - rm -f "$linuxbrew_archive_checksum" - curl -L "$url" -o "$linuxbrew_archive" - fi - if [[ ! -f $linuxbrew_archive_checksum ]]; then - echo "Downloading Linuxbrew archive checksum file for $url..." - curl -L "$url.sha256" -o "$linuxbrew_archive_checksum" - fi - echo "Verifying Linuxbrew archive checksum ..." - pushd "$YB_LINUXBREW_LOCAL_ROOT" - sha256sum -c --strict "$linuxbrew_archive_checksum" - popd - echo "Installing Linuxbrew into $linuxbrew_dir..." - local tmp=$YB_LINUXBREW_LOCAL_ROOT/tmp/$$_$RANDOM$RANDOM - mkdir -p "$tmp" - tar zxf "$linuxbrew_archive" -C "$tmp" - if mv "$tmp/$linuxbrew_dirname" "$YB_LINUXBREW_LOCAL_ROOT/"; then - pushd "$linuxbrew_dir" - ./post_install.sh - popd - fi -} - try_set_linuxbrew_dir() { local linuxbrew_dir=$1 if [[ -d "$linuxbrew_dir" && @@ -1388,27 +1363,16 @@ save_paths_to_build_dir() { detect_linuxbrew() { expect_vars_to_be_set YB_COMPILER_TYPE - if [[ -z ${BUILD_ROOT:-} ]]; then - fatal "BUILD_ROOT is not set, not trying to use the default version of Linuxbrew." - fi if ! is_linux; then - fatal "Expected this function to only be called on Linux" - fi - if is_ubuntu; then - # Not using Linuxbrew on Ubuntu. - unset YB_LINUXBREW_DIR - return - fi - if [[ $YB_COMPILER_TYPE =~ ^.*[0-9]+$ ]]; then - # Not allowing to use Linuxbrew if the compiler type mentions a specific compiler version, e.g. - # gcc9 or clang11. - unset YB_LINUXBREW_DIR return fi if [[ -n ${YB_LINUXBREW_DIR:-} ]]; then export YB_LINUXBREW_DIR return fi + if [[ -n ${YB_USE_LINUXBREW:-} && ${YB_USE_LINUXBREW:-} != "1" ]]; then + return + fi if ! "$is_clean_build" && [[ -n ${BUILD_ROOT:-} && -f $BUILD_ROOT/linuxbrew_path.txt ]]; then YB_LINUXBREW_DIR=$(<"$BUILD_ROOT/linuxbrew_path.txt") @@ -1416,56 +1380,6 @@ detect_linuxbrew() { yb_linuxbrew_dir_origin=" (from file '$BUILD_ROOT/linuxbrew_path.txt')" return fi - - local version_file=$YB_SRC_ROOT/build-support/linuxbrew_version.txt - if [[ ! -f $version_file ]]; then - fatal "'$version_file' does not exist" - fi - local linuxbrew_version - linuxbrew_version=$( read_file_and_trim "$version_file" ) - local linuxbrew_dirname - linuxbrew_dirname="linuxbrew-$linuxbrew_version" - - local candidates=() - local jenkins_linuxbrew_dir="$SHARED_LINUXBREW_BUILDS_DIR/$linuxbrew_dirname" - if [[ -d $jenkins_linuxbrew_dir ]]; then - candidates=( "$jenkins_linuxbrew_dir" ) - elif is_jenkins; then - if is_src_root_on_nfs; then - wait_for_directory_existence "$jenkins_linuxbrew_dir" - candidates=( "$jenkins_linuxbrew_dir" ) - else - yb_fatal_exit_code=$YB_EXIT_CODE_NO_SUCH_FILE_OR_DIRECTORY - fatal "Warning: Linuxbrew directory referenced by '$version_file' does not" \ - "exist: '$jenkins_linuxbrew_dir', refusing to proceed to prevent " \ - "non-deterministic builds." - fi - fi - - if [[ ${#candidates[@]} -gt 0 ]]; then - local linuxbrew_dir - for linuxbrew_dir in "${candidates[@]}"; do - if try_set_linuxbrew_dir "$linuxbrew_dir"; then - yb_linuxbrew_dir_origin=" (from '$version_file')" - return - fi - done - fi - - local linuxbrew_local_dir="$YB_LINUXBREW_LOCAL_ROOT/$linuxbrew_dirname" - if ! is_jenkins && [[ ! -d $linuxbrew_local_dir ]]; then - install_linuxbrew "$linuxbrew_version" - fi - if try_set_linuxbrew_dir "$linuxbrew_local_dir"; then - yb_linuxbrew_dir_origin=" (local installation)" - else - if [[ ${#candidates[@]} -gt 0 ]]; then - log "Could not find Linuxbrew in any of these directories: ${candidates[*]}." - else - log "Could not find Linuxbrew candidate directories." - fi - log "Failed to install Linuxbrew $linuxbrew_version into $linuxbrew_local_dir." - fi } detect_llvm_toolchain() { @@ -1482,12 +1396,18 @@ detect_llvm_toolchain() { } using_linuxbrew() { - if is_linux && [[ -n ${YB_LINUXBREW_DIR:-} ]]; then + if is_linux && [[ -n ${YB_LINUXBREW_DIR:-} || ${YB_USE_LINUXBREW:-} == "1" ]]; then return 0 # true in bash fi return 1 } +ensure_linuxbrew_dir_is_set() { + if [[ -z ${YB_LINUXBREW_DIR:-} ]]; then + fatal "YB_LINUXBREW_DIR is not set. YB_USE_LINUXBREW=${YB_USE_LINUXBREW:-undefined}" + fi +} + decide_whether_to_use_ninja() { if [[ -z ${YB_USE_NINJA:-} ]]; then # Autodetect whether we need to use Ninja at all, based on whether it is available. @@ -1499,6 +1419,7 @@ decide_whether_to_use_ninja() { elif command -v ninja >/dev/null || [[ -x /usr/local/bin/ninja ]]; then export YB_USE_NINJA=1 elif using_linuxbrew; then + ensure_linuxbrew_dir_is_set local yb_ninja_path_candidate=$YB_LINUXBREW_DIR/bin/ninja if [[ -x $yb_ninja_path_candidate ]]; then export YB_USE_NINJA=1 @@ -1531,14 +1452,7 @@ find_ninja_executable() { return fi - if using_linuxbrew; then - # On Linux, try to use Linux from Linuxbrew as a last resort. - local yb_ninja_path_candidate=$YB_LINUXBREW_DIR/bin/ninja - if [[ -x $yb_ninja_path_candidate ]]; then - export YB_NINJA_PATH=$yb_ninja_path_candidate - return - fi - fi + # We used to get Ninja from Linuxbrew here as a last resort but we don't do that anymore. # ----------------------------------------------------------------------------------------------- # Ninja not found @@ -1562,6 +1476,7 @@ using_ninja() { add_brew_bin_to_path() { if using_linuxbrew; then + ensure_linuxbrew_dir_is_set # We need to add Linuxbrew's bin directory to PATH so that we can find the right compiler and # linker. put_path_entry_first "$YB_LINUXBREW_DIR/bin" @@ -1937,7 +1852,7 @@ find_or_download_thirdparty() { export NO_REBUILD_THIRDPARTY=1 log "Using downloaded third-party directory: $YB_THIRDPARTY_DIR" if using_linuxbrew; then - log "Using Linuxbrew directory: $YB_LINUXBREW_DIR" + log "Using Linuxbrew directory: ${YB_LINUXBREW_DIR:-undefined}" fi fi @@ -2019,18 +1934,27 @@ handle_predefined_build_root() { local basename=${predefined_build_root##*/} if [[ $basename =~ $BUILD_ROOT_BASENAME_RE ]]; then - local _build_type=${BASH_REMATCH[1]} - local _compiler_type=${BASH_REMATCH[2]} + local group_idx=1 + local _build_type=${BASH_REMATCH[$group_idx]} + + (( group_idx+=1 )) + local _compiler_type=${BASH_REMATCH[$group_idx]} + (( group_idx+=1 )) + local _dash_linuxbrew=${BASH_REMATCH[$group_idx]} + + (( group_idx+=2 )) # _linking_type is unused. We always use dynamic linking, but we plan to support static linking. # shellcheck disable=SC2034 - local _linking_type=${BASH_REMATCH[3]} + local _linking_type=${BASH_REMATCH[$group_idx]} - # The architecture field is of the form (-(...))?, so it utilizes two capture groups (4 and 5). - # Group 4 has a leading dash and group 5 does not have it. - local _architecture=${BASH_REMATCH[5]} + # The architecture field is of the form (-(...))?, so it utilizes two capture groups. The first + # one has a leading dash and the second one does not. We jump directly to the second one. + (( group_idx+=2 )) + local _architecture=${BASH_REMATCH[$group_idx]} - local _dash_ninja=${BASH_REMATCH[6]} + (( group_idx+=1 )) + local _dash_ninja=${BASH_REMATCH[$group_idx]} else fatal "Could not parse build root directory name '$basename'" \ "(full path: '$predefined_build_root'). Expected to match '$BUILD_ROOT_BASENAME_RE'." @@ -2058,6 +1982,19 @@ handle_predefined_build_root() { "does not match YB_COMPILER_TYPE ('$YB_COMPILER_TYPE')." fi + local use_linuxbrew + if [[ -z ${_dash_linuxbrew:-} ]]; then + use_linuxbrew=false + if [[ -n ${YB_LINUXBREW_DIR:-} ]]; then + fatal "YB_LINUXBREW_DIR is set but the build root directory name '$basename' does not" \ + "contain a '-linuxbrew-' component." + fi + else + # We will ensure that YB_LINUXBREW_DIR is set later. We might need to set it based on the + # build root itself. + use_linuxbrew=true + fi + if [[ -n $_architecture ]]; then if [[ -z ${YB_TARGET_ARCH:-} ]]; then export YB_TARGET_ARCH=$_architecture @@ -2087,6 +2024,17 @@ handle_predefined_build_root() { export YB_USE_NINJA=$should_use_ninja decide_whether_to_use_ninja + detect_brew + + if [[ ${use_linuxbrew} == "true" && -z ${YB_LINUXBREW_DIR:-} ]]; then + if [[ -f "$predefined_build_root/linuxbrew_path.txt" ]]; then + YB_LINUXBREW_DIR=$(<"$predefined_build_root/linuxbrew_path.txt") + export YB_LINUXBREW_DIR + else + fatal "YB_LINUXBREW_DIR is not set but the build root directory name '$basename' contains a" \ + "'-linuxbrew-' component." + fi + fi } # Remove the build/latest symlink to prevent Jenkins from showing every test twice in test results. @@ -2192,13 +2140,20 @@ run_shellcheck() { ) pushd "$YB_SRC_ROOT" local script_path + local shellcheck_had_errors=false for script_path in "${scripts_to_check[@]}"; do # We skip errors 2030 and 2031 that say that a variable has been modified in a subshell and that # the modification is local to the subshell. Seeing a lot of false positivies for these with # the version 0.7.2 of Shellcheck. - ( set -x; shellcheck --external-sources --exclude=2030,2031 --shell=bash "$script_path" ) + if ! ( set -x; shellcheck --external-sources --exclude=2030,2031 --shell=bash "$script_path" ) + then + shellcheck_had_errors=true + fi done popd + if [[ $shellcheck_had_errors == "true" ]]; then + exit 1 + fi } activate_virtualenv() { @@ -2458,12 +2413,23 @@ set_prebuilt_thirdparty_url() { if [[ ${YB_DOWNLOAD_THIRDPARTY:-} == "1" ]]; then if [[ -z ${YB_THIRDPARTY_URL:-} ]]; then local thirdparty_url_file_path="$BUILD_ROOT/thirdparty_url.txt" + local llvm_url_file_path="$BUILD_ROOT/llvm_url.txt" if [[ -f $thirdparty_url_file_path ]]; then rm -f "$thirdparty_url_file_path" fi + local is_linuxbrew_arg="" + if [[ -n ${YB_USE_LINUXBREW:-} ]]; then + if [[ $YB_USE_LINUXBREW == "1" ]]; then + is_linuxbrew_arg="--is-linuxbrew=true" + else + is_linuxbrew_arg="--is-linuxbrew=false" + fi + fi "$YB_BUILD_SUPPORT_DIR/thirdparty_tool" \ - --save-download-url-to-file "$thirdparty_url_file_path" \ - --compiler-type "$YB_COMPILER_TYPE" + --save-thirdparty-url-to-file "$thirdparty_url_file_path" \ + --save-llvm-url-to-file "$llvm_url_file_path" \ + --compiler-type "$YB_COMPILER_TYPE" \ + $is_linuxbrew_arg YB_THIRDPARTY_URL=$(<"$BUILD_ROOT/thirdparty_url.txt") export YB_THIRDPARTY_URL yb_thirdparty_url_origin=" (determined automatically based on the OS and compiler type)" diff --git a/build-support/common-test-env.sh b/build-support/common-test-env.sh index 059aafbda06e..5635c3f569b0 100644 --- a/build-support/common-test-env.sh +++ b/build-support/common-test-env.sh @@ -177,7 +177,7 @@ validate_test_descriptor() { "$test_descriptor" fi # Remove $TEST_DESCRIPTOR_SEPARATOR and all that follows and get a relative binary path. - validate_relative_test_binary_path "${test_descriptor%$TEST_DESCRIPTOR_SEPARATOR*}" + validate_relative_test_binary_path "${test_descriptor%"$TEST_DESCRIPTOR_SEPARATOR"*}" else validate_relative_test_binary_path "$test_descriptor" fi @@ -454,8 +454,8 @@ prepare_for_running_cxx_test() { fi if [[ "$test_descriptor" =~ $TEST_DESCRIPTOR_SEPARATOR ]]; then - rel_test_binary=${test_descriptor%$TEST_DESCRIPTOR_SEPARATOR*} - test_name=${test_descriptor#*$TEST_DESCRIPTOR_SEPARATOR} + rel_test_binary=${test_descriptor%"${TEST_DESCRIPTOR_SEPARATOR}"*} + test_name=${test_descriptor#*"${TEST_DESCRIPTOR_SEPARATOR}"} run_at_once=false junit_test_case_id=$test_name else @@ -1906,7 +1906,7 @@ resolve_and_run_java_test() { "'$module_name' and '$current_module_name' are valid candidates." fi module_name=$current_module_name - rel_module_dir=${module_dir##$YB_SRC_ROOT/} + rel_module_dir=${module_dir##"${YB_SRC_ROOT}"/} fi done fi @@ -1942,7 +1942,7 @@ resolve_and_run_java_test() { "'$module_name' and '$current_module_name' are valid candidates." fi module_name=$current_module_name - rel_module_dir=${module_dir##$YB_SRC_ROOT/} + rel_module_dir=${module_dir##"${YB_SRC_ROOT}"/} if [[ ${#candidate_files[@]} -gt 1 ]]; then fatal "Ambiguous source files for Java/Scala test '$java_test_name': " \ diff --git a/build-support/compiler-wrappers/compiler-wrapper.sh b/build-support/compiler-wrappers/compiler-wrapper.sh index 96fd96c8c1ca..0effe97a3a98 100755 --- a/build-support/compiler-wrappers/compiler-wrapper.sh +++ b/build-support/compiler-wrappers/compiler-wrapper.sh @@ -713,7 +713,7 @@ if [[ $PWD == $BUILD_ROOT/postgres_build || if [[ -d $include_dir ]]; then include_dir=$( cd "$include_dir" && pwd ) if [[ $include_dir == $BUILD_ROOT/postgres_build/* ]]; then - rel_include_dir=${include_dir#$BUILD_ROOT/postgres_build/} + rel_include_dir=${include_dir#"${BUILD_ROOT}"/postgres_build/} updated_include_dir=$YB_SRC_ROOT/src/postgres/$rel_include_dir if [[ -d $updated_include_dir ]]; then new_cmd+=( -I"$updated_include_dir" ) @@ -723,7 +723,7 @@ if [[ $PWD == $BUILD_ROOT/postgres_build || new_cmd+=( "$arg" ) elif [[ -f $arg && $arg != "conftest.c" ]]; then file_path=$PWD/${arg#./} - rel_file_path=${file_path#$BUILD_ROOT/postgres_build/} + rel_file_path=${file_path#"${BUILD_ROOT}"/postgres_build/} updated_file_path=$YB_SRC_ROOT/src/postgres/$rel_file_path if [[ -f $updated_file_path ]] && cmp --quiet "$file_path" "$updated_file_path"; then new_cmd+=( "$updated_file_path" ) diff --git a/build-support/jenkins/build-and-test.sh b/build-support/jenkins/build-and-test.sh index 26d8ab66ce93..9cc36f5a48e1 100755 --- a/build-support/jenkins/build-and-test.sh +++ b/build-support/jenkins/build-and-test.sh @@ -161,6 +161,9 @@ log "Removing old JSON-based test report files" rm -f test_results.json test_failures.json ) +activate_virtualenv +set_pythonpath + # We change YB_RUN_JAVA_TEST_METHODS_SEPARATELY in a subshell in a few places and that is OK. # shellcheck disable=SC2031 export YB_RUN_JAVA_TEST_METHODS_SEPARATELY=1 @@ -172,8 +175,6 @@ if is_mac; then export PATH=/usr/local/bin:$PATH fi -MAX_NUM_PARALLEL_TESTS=3 - # gather core dumps ulimit -c unlimited @@ -400,13 +401,7 @@ if [[ $BUILD_TYPE != "asan" ]]; then export YB_TEST_ULIMIT_CORE=unlimited fi -# Cap the number of parallel tests to run at $MAX_NUM_PARALLEL_TESTS detect_num_cpus -if [[ $YB_NUM_CPUS -gt $MAX_NUM_PARALLEL_TESTS ]]; then - NUM_PARALLEL_TESTS=$MAX_NUM_PARALLEL_TESTS -else - NUM_PARALLEL_TESTS=$YB_NUM_CPUS -fi declare -i EXIT_STATUS=0 diff --git a/build-support/python_with_venv.sh b/build-support/python_with_venv.sh new file mode 100755 index 000000000000..aa048654ba7a --- /dev/null +++ b/build-support/python_with_venv.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +. "${BASH_SOURCE[0]%/*}/../build-support/common-build-env.sh" + +activate_virtualenv +set_pythonpath +set -x +python3 "$@" diff --git a/build-support/show_build_root_name_regex.sh b/build-support/show_build_root_name_regex.sh new file mode 100755 index 000000000000..c88cf18c61ae --- /dev/null +++ b/build-support/show_build_root_name_regex.sh @@ -0,0 +1,3 @@ +#!/usr/bin/env bash +. "${BASH_SOURCE[0]%/*}/../build-support/common-build-env.sh" +echo "${BUILD_ROOT_BASENAME_RE}" diff --git a/build-support/thirdparty_archives.yml b/build-support/thirdparty_archives.yml index ce924d921e85..4e5c0af04e16 100644 --- a/build-support/thirdparty_archives.yml +++ b/build-support/thirdparty_archives.yml @@ -1,102 +1,134 @@ -sha_for_local_checkout: bf5bec16ebb714db0e0dc2d53de64a3a8ebe9508 +sha_for_local_checkout: dd4872fe56e973d28e4171648629fc45a3c44f00 archives: - os_type: almalinux8 architecture: x86_64 compiler_type: clang11 - sha: bf5bec16ebb714db0e0dc2d53de64a3a8ebe9508 - tag: v20211217015051-bf5bec16eb-almalinux8-x86_64-clang11 + is_linuxbrew: false + sha: dd4872fe56e973d28e4171648629fc45a3c44f00 + tag: v20211222064143-dd4872fe56-almalinux8-x86_64-clang11 - os_type: almalinux8 architecture: x86_64 compiler_type: clang12 - sha: bf5bec16ebb714db0e0dc2d53de64a3a8ebe9508 - tag: v20211217015101-bf5bec16eb-almalinux8-x86_64-clang12 + is_linuxbrew: false + sha: dd4872fe56e973d28e4171648629fc45a3c44f00 + tag: v20211222064150-dd4872fe56-almalinux8-x86_64-clang12 + - os_type: almalinux8 + architecture: x86_64 + compiler_type: clang12 + is_linuxbrew: true + sha: dd4872fe56e973d28e4171648629fc45a3c44f00 + tag: v20211222064126-dd4872fe56-almalinux8-x86_64-clang12-linuxbrew - os_type: almalinux8 architecture: x86_64 compiler_type: gcc8 - sha: bf5bec16ebb714db0e0dc2d53de64a3a8ebe9508 - tag: v20211217015122-bf5bec16eb-almalinux8-x86_64-gcc8 + is_linuxbrew: false + sha: dd4872fe56e973d28e4171648629fc45a3c44f00 + tag: v20211222064129-dd4872fe56-almalinux8-x86_64-gcc8 - os_type: almalinux8 architecture: x86_64 compiler_type: gcc9 - sha: bf5bec16ebb714db0e0dc2d53de64a3a8ebe9508 - tag: v20211217015123-bf5bec16eb-almalinux8-x86_64-gcc9 + is_linuxbrew: false + sha: dd4872fe56e973d28e4171648629fc45a3c44f00 + tag: v20211222064204-dd4872fe56-almalinux8-x86_64-gcc9 - os_type: centos7 architecture: x86_64 compiler_type: clang11 - sha: bf5bec16ebb714db0e0dc2d53de64a3a8ebe9508 - tag: v20211217015110-bf5bec16eb-centos7-x86_64-clang11 + is_linuxbrew: false + sha: dd4872fe56e973d28e4171648629fc45a3c44f00 + tag: v20211222064132-dd4872fe56-centos7-x86_64-clang11 - os_type: centos7 architecture: x86_64 compiler_type: clang12 - sha: bf5bec16ebb714db0e0dc2d53de64a3a8ebe9508 - tag: v20211217015140-bf5bec16eb-centos7-x86_64-clang12 + is_linuxbrew: false + sha: dd4872fe56e973d28e4171648629fc45a3c44f00 + tag: v20211222064215-dd4872fe56-centos7-x86_64-clang12 - os_type: centos7 architecture: x86_64 compiler_type: clang7 - sha: bf5bec16ebb714db0e0dc2d53de64a3a8ebe9508 - tag: v20211217015053-bf5bec16eb-centos7-x86_64-clang7 + is_linuxbrew: false + sha: dd4872fe56e973d28e4171648629fc45a3c44f00 + tag: v20211222064218-dd4872fe56-centos7-x86_64-clang7 - os_type: centos7 architecture: x86_64 compiler_type: gcc5 - sha: bf5bec16ebb714db0e0dc2d53de64a3a8ebe9508 - tag: v20211217015050-bf5bec16eb-centos7-x86_64-linuxbrew-gcc5 + is_linuxbrew: true + sha: dd4872fe56e973d28e4171648629fc45a3c44f00 + tag: v20211222064200-dd4872fe56-centos7-x86_64-linuxbrew-gcc5 - os_type: centos7 architecture: x86_64 compiler_type: gcc8 - sha: bf5bec16ebb714db0e0dc2d53de64a3a8ebe9508 - tag: v20211217015111-bf5bec16eb-centos7-x86_64-gcc8 + is_linuxbrew: false + sha: dd4872fe56e973d28e4171648629fc45a3c44f00 + tag: v20211222064222-dd4872fe56-centos7-x86_64-gcc8 - os_type: centos7 architecture: x86_64 compiler_type: gcc9 - sha: bf5bec16ebb714db0e0dc2d53de64a3a8ebe9508 - tag: v20211217015144-bf5bec16eb-centos7-x86_64-gcc9 + is_linuxbrew: false + sha: dd4872fe56e973d28e4171648629fc45a3c44f00 + tag: v20211222064145-dd4872fe56-centos7-x86_64-gcc9 - os_type: centos8 architecture: aarch64 compiler_type: clang11 - sha: bf5bec16ebb714db0e0dc2d53de64a3a8ebe9508 - tag: v20211217070927-bf5bec16eb-centos8-aarch64-clang11 + is_linuxbrew: false + sha: dd4872fe56e973d28e4171648629fc45a3c44f00 + tag: v20211224053420-dd4872fe56-centos8-aarch64-clang11 - os_type: centos8 architecture: x86_64 compiler_type: gcc8 - sha: bf5bec16ebb714db0e0dc2d53de64a3a8ebe9508 - tag: v20211217015053-bf5bec16eb-centos8-x86_64-gcc8 + is_linuxbrew: false + sha: dd4872fe56e973d28e4171648629fc45a3c44f00 + tag: v20211222064134-dd4872fe56-centos8-x86_64-gcc8 - os_type: centos8 architecture: x86_64 compiler_type: gcc9 - sha: bf5bec16ebb714db0e0dc2d53de64a3a8ebe9508 - tag: v20211217015104-bf5bec16eb-centos8-x86_64-gcc9 + is_linuxbrew: false + sha: dd4872fe56e973d28e4171648629fc45a3c44f00 + tag: v20211222064151-dd4872fe56-centos8-x86_64-gcc9 + - os_type: macos + architecture: arm64 + compiler_type: clang + is_linuxbrew: false + sha: dd4872fe56e973d28e4171648629fc45a3c44f00 + tag: v20211223213110-dd4872fe56-macos-arm64 - os_type: macos architecture: x86_64 compiler_type: clang - sha: bf5bec16ebb714db0e0dc2d53de64a3a8ebe9508 - tag: v20211217015038-bf5bec16eb-macos-x86_64 + is_linuxbrew: false + sha: dd4872fe56e973d28e4171648629fc45a3c44f00 + tag: v20211222064205-dd4872fe56-macos-x86_64 - os_type: ubuntu18.04 architecture: x86_64 compiler_type: clang10 - sha: bf5bec16ebb714db0e0dc2d53de64a3a8ebe9508 - tag: v20211217015135-bf5bec16eb-ubuntu1804-x86_64-clang10 + is_linuxbrew: false + sha: dd4872fe56e973d28e4171648629fc45a3c44f00 + tag: v20211222064157-dd4872fe56-ubuntu1804-x86_64-clang10 - os_type: ubuntu18.04 architecture: x86_64 compiler_type: clang11 - sha: bf5bec16ebb714db0e0dc2d53de64a3a8ebe9508 - tag: v20211217015135-bf5bec16eb-ubuntu1804-x86_64-clang11 + is_linuxbrew: false + sha: dd4872fe56e973d28e4171648629fc45a3c44f00 + tag: v20211222064210-dd4872fe56-ubuntu1804-x86_64-clang11 - os_type: ubuntu18.04 architecture: x86_64 compiler_type: gcc7 - sha: bf5bec16ebb714db0e0dc2d53de64a3a8ebe9508 - tag: v20211217015136-bf5bec16eb-ubuntu1804-x86_64-gcc7 + is_linuxbrew: false + sha: dd4872fe56e973d28e4171648629fc45a3c44f00 + tag: v20211222064304-dd4872fe56-ubuntu1804-x86_64-gcc7 - os_type: ubuntu18.04 architecture: x86_64 compiler_type: gcc8 - sha: bf5bec16ebb714db0e0dc2d53de64a3a8ebe9508 - tag: v20211217015135-bf5bec16eb-ubuntu1804-x86_64-gcc8 + is_linuxbrew: false + sha: dd4872fe56e973d28e4171648629fc45a3c44f00 + tag: v20211222064157-dd4872fe56-ubuntu1804-x86_64-gcc8 - os_type: ubuntu20.04 architecture: x86_64 compiler_type: clang11 - sha: bf5bec16ebb714db0e0dc2d53de64a3a8ebe9508 - tag: v20211217015144-bf5bec16eb-ubuntu2004-x86_64-clang11 + is_linuxbrew: false + sha: dd4872fe56e973d28e4171648629fc45a3c44f00 + tag: v20211222064157-dd4872fe56-ubuntu2004-x86_64-clang11 - os_type: ubuntu20.04 architecture: x86_64 compiler_type: gcc9 - sha: bf5bec16ebb714db0e0dc2d53de64a3a8ebe9508 - tag: v20211217015119-bf5bec16eb-ubuntu2004-x86_64-gcc9 + is_linuxbrew: false + sha: dd4872fe56e973d28e4171648629fc45a3c44f00 + tag: v20211222064144-dd4872fe56-ubuntu2004-x86_64-gcc9 diff --git a/build-support/thirdparty_archives_manual.yml b/build-support/thirdparty_archives_manual.yml index 828e5e9133d0..aeffd4be9a51 100644 --- a/build-support/thirdparty_archives_manual.yml +++ b/build-support/thirdparty_archives_manual.yml @@ -7,3 +7,4 @@ archives: compiler_type: gcc9 tag: v20210817191252-9a29e26965-centos8-aarch64-gcc9 sha: 9a29e269659a6b46eb05747b4431afdc34e59695 + is_linuxbrew: false diff --git a/cmake_modules/CompilerInfo.cmake b/cmake_modules/CompilerInfo.cmake index cbe6090bb74b..df18b253874b 100644 --- a/cmake_modules/CompilerInfo.cmake +++ b/cmake_modules/CompilerInfo.cmake @@ -97,3 +97,13 @@ else() message(FATAL_ERROR "Unknown compiler. Version info:\n${COMPILER_VERSION_FULL}") endif() message("Selected compiler family '${COMPILER_FAMILY}', version '${COMPILER_VERSION}'") + +set(IS_CLANG FALSE) +if("${COMPILER_FAMILY}" STREQUAL "clang") + set(IS_CLANG TRUE) +endif() + +set(IS_GCC FALSE) +if("${COMPILER_FAMILY}" STREQUAL "gcc") + set(IS_GCC TRUE) +endif() diff --git a/cmake_modules/YugabyteCMakeUnitTest.cmake b/cmake_modules/YugabyteCMakeUnitTest.cmake index 583372f72e90..6b3b0532e0ea 100644 --- a/cmake_modules/YugabyteCMakeUnitTest.cmake +++ b/cmake_modules/YugabyteCMakeUnitTest.cmake @@ -37,7 +37,8 @@ function(check_parse_build_root_basename BUILD_ROOT_BASENAME EXPECTED_BUILD_TYPE EXPECTED_COMPILER_TYPE - EXPECTED_LINKING_TYPE) + EXPECTED_LINKING_TYPE + EXPECTED_USING_LINUXBREW) set(CMAKE_CURRENT_BINARY_DIR "/somedir/${BUILD_ROOT_BASENAME}") unset(ENV{YB_COMPILER_TYPE}) parse_build_root_basename() @@ -45,16 +46,29 @@ function(check_parse_build_root_basename assert_equals("${EXPECTED_COMPILER_TYPE}" "${YB_COMPILER_TYPE}" "(CMake var)") assert_equals("${EXPECTED_COMPILER_TYPE}" "$ENV{YB_COMPILER_TYPE}" "(env var)") assert_equals("${EXPECTED_LINKING_TYPE}" "${YB_LINKING_TYPE}") + assert_equals("${EXPECTED_USING_LINUXBREW}" "${YB_USING_LINUXBREW_FROM_BUILD_ROOT}") endfunction() - function(test_parse_build_root_basename) - check_parse_build_root_basename("debug-clang-dynamic-ninja" "debug" "clang" "dynamic") - check_parse_build_root_basename("release-gcc8-static-ninja" "release" "gcc8" "static") - check_parse_build_root_basename("debug-clang-dynamic" "debug" "clang" "dynamic") - check_parse_build_root_basename("asan-gcc9-dynamic-ninja" "asan" "gcc9" "dynamic") - check_parse_build_root_basename("debug-clang-dynamic" "debug" "clang" "dynamic") - check_parse_build_root_basename("tsan-clang11-dynamic" "tsan" "clang11" "dynamic") + # BUILD_ROOT_BASENAME BUILD_TYPE COMPILER_TYPE LINK_TYPE USING_LINUXBREW + check_parse_build_root_basename( + "debug-clang-dynamic-ninja" "debug" "clang" "dynamic" OFF) + check_parse_build_root_basename( + "release-gcc8-static-ninja" "release" "gcc8" "static" OFF) + check_parse_build_root_basename( + "debug-clang-dynamic" "debug" "clang" "dynamic" OFF) + check_parse_build_root_basename( + "asan-gcc9-dynamic-ninja" "asan" "gcc9" "dynamic" OFF) + check_parse_build_root_basename( + "debug-clang-dynamic" "debug" "clang" "dynamic" OFF) + check_parse_build_root_basename( + "tsan-clang11-dynamic" "tsan" "clang11" "dynamic" OFF) + check_parse_build_root_basename( + "tsan-clang11-dynamic" "tsan" "clang11" "dynamic" OFF) + check_parse_build_root_basename( + "release-clang12-linuxbrew-dynamic" "release" "clang12" "dynamic" ON) + check_parse_build_root_basename( + "release-gcc5-linuxbrew-dynamic" "release" "gcc5" "dynamic" ON) endfunction() # ------------------------------------------------------------------------------------------------- @@ -65,6 +79,8 @@ set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/cmake_modules") include(YugabyteFunctions) +yb_initialize_constants() + test_parse_build_root_basename() message("All Yugabyte CMake tests passed successfully") diff --git a/cmake_modules/YugabyteFunctions.cmake b/cmake_modules/YugabyteFunctions.cmake index c638d5a3eab9..695fd9578829 100644 --- a/cmake_modules/YugabyteFunctions.cmake +++ b/cmake_modules/YugabyteFunctions.cmake @@ -12,6 +12,10 @@ # Various CMake functions. This should eventually be organized and refactored. +macro(yb_initialize_constants) + set(BUILD_SUPPORT_DIR "${CMAKE_CURRENT_SOURCE_DIR}/build-support") +endmacro() + function(CHECK_YB_COMPILER_PATH COMPILER_PATH) if(NOT "${COMPILER_PATH}" MATCHES "/compiler-wrappers/(cc|c[+][+])$" AND NOT "${CMAKE_COMMAND}" MATCHES "/[.]?(CLion|clion)") @@ -93,20 +97,13 @@ function(DETECT_BREW) # TODO: consolidate Linuxbrew detection logic between here and detect_brew in common-build-env.sh. # As of 10/2020 we only check the compiler version here but not in detect_brew. set(USING_LINUXBREW FALSE) - if(NOT APPLE AND - # In practice, we only use Linuxbrew with Clang 7.x. - (NOT IS_CLANG OR "${COMPILER_VERSION}" VERSION_LESS "8.0.0") AND - # In practice, we only use Linuxbrew with GCC 5.x. - (NOT IS_GCC OR "${COMPILER_VERSION}" VERSION_LESS "6.0.0") AND - # Only a few compiler types could be used with Linuxbrew. The "clang7" compiler type is - # explicitly NOT included. - ("${YB_COMPILE_TYPE}" MATCHES "^(gcc|gcc5|clang)$")) + if(NOT APPLE) + set(LINUXBREW_DIR "$ENV{YB_LINUXBREW_DIR}") message("Trying to detect whether we should use Linuxbrew. " "IS_CLANG=${IS_CLANG}, " "IS_GCC=${IS_GCC}, " "COMPILER_VERSION=${COMPILER_VERSION}, " "LINUXBREW_DIR=${LINUXBREW_DIR}") - set(LINUXBREW_DIR "$ENV{YB_LINUXBREW_DIR}") if("${LINUXBREW_DIR}" STREQUAL "") if(EXISTS "${CMAKE_CURRENT_BINARY_DIR}/linuxbrew_path.txt") file(STRINGS "${CMAKE_CURRENT_BINARY_DIR}/linuxbrew_path.txt" LINUXBREW_DIR) @@ -157,7 +154,6 @@ function(INIT_COMPILER_TYPE_FROM_BUILD_ROOT) endif() endif() - message("YB_COMPILER_TYPE env var: $ENV{YB_COMPILER_TYPE}") # Make sure we can use $ENV{YB_COMPILER_TYPE} and ${YB_COMPILER_TYPE} interchangeably. SET(YB_COMPILER_TYPE "$ENV{YB_COMPILER_TYPE}" PARENT_SCOPE) endfunction() @@ -176,14 +172,14 @@ function(VALIDATE_COMPILER_TYPE) endif() if ("$ENV{YB_COMPILER_TYPE}" STREQUAL "zapcc") - if (NOT "${COMPILER_FAMILY}" STREQUAL "clang") + if (NOT IS_CLANG) message(FATAL_ERROR "Compiler type is zapcc but the compiler family is '${COMPILER_FAMILY}' " "(expected to be clang)") endif() endif() - if("${YB_COMPILER_TYPE}" MATCHES "^gcc.*$" AND NOT "${COMPILER_FAMILY}" STREQUAL "gcc") + if("${YB_COMPILER_TYPE}" MATCHES "^gcc.*$" AND NOT IS_GCC) message(FATAL_ERROR "Compiler type '${YB_COMPILER_TYPE}' does not match the compiler family " "'${COMPILER_FAMILY}'.") @@ -196,20 +192,10 @@ function(VALIDATE_COMPILER_TYPE) "'${YB_COMPILER_TYPE}'.") endif() - if (NOT "${COMPILER_FAMILY}" STREQUAL "gcc" AND - NOT "${COMPILER_FAMILY}" STREQUAL "clang") + if (NOT IS_GCC AND + NOT IS_CLANG) message(FATAL_ERROR "Unknown compiler family: ${COMPILER_FAMILY} (expected 'gcc' or 'clang').") endif() - - set(IS_CLANG FALSE PARENT_SCOPE) - if ("${COMPILER_FAMILY}" STREQUAL "clang") - set(IS_CLANG TRUE PARENT_SCOPE) - endif() - - set(IS_GCC FALSE PARENT_SCOPE) - if ("${COMPILER_FAMILY}" STREQUAL "gcc") - set(IS_GCC TRUE PARENT_SCOPE) - endif() endfunction() # Linker flags applied to both executables and shared libraries. We append this both to @@ -397,7 +383,33 @@ macro(YB_SETUP_CLANG) # the Linuxbrew include directory too. include_directories(SYSTEM "${LIBCXX_INCLUDE_DIR}") + if(NOT APPLE) + execute_process(COMMAND "${CMAKE_CXX_COMPILER}" -print-search-dirs + OUTPUT_VARIABLE CLANG_PRINT_SEARCH_DIRS_OUTPUT) + + if ("${CLANG_PRINT_SEARCH_DIRS_OUTPUT}" MATCHES ".*libraries: =([^:]+)(:.*|$)" ) + # We get a directory like this: + # .../yb-llvm-v12.0.1-yb-1-1639783720-bdb147e6-almalinux8-x86_64/lib/clang/12.0.1 + set(CLANG_LIB_DIR "${CMAKE_MATCH_1}") + set(CLANG_RUNTIME_LIB_DIR "${CMAKE_MATCH_1}/lib/linux") + else() + message(FATAL_ERROR + "Could not parse the output of 'clang -print-search-dirs': " + "${CLANG_PRINT_SEARCH_DIRS_OUTPUT}") + endif() + if(USING_LINUXBREW) + set(CLANG_INCLUDE_DIR "${CLANG_LIB_DIR}/include") + if(NOT EXISTS "${CLANG_INCLUDE_DIR}") + message(FATAL_ERROR "Clang include directory '${CLANG_INCLUDE_DIR}' does not exist") + endif() + ADD_CXX_FLAGS("-isystem ${CLANG_INCLUDE_DIR}") + endif() + endif() + ADD_CXX_FLAGS("-nostdinc++") + if(USING_LINUXBREW) + ADD_CXX_FLAGS("-nostdinc") + endif() ADD_LINKER_FLAGS("-L${LIBCXX_DIR}/lib") if(NOT EXISTS "${LIBCXX_DIR}/lib") message(FATAL_ERROR "libc++ library directory does not exist: '${LIBCXX_DIR}/lib'") @@ -418,7 +430,7 @@ macro(YB_SETUP_SANITIZER) "Build type: ${YB_BUILD_TYPE}.") endif() - if("${COMPILER_FAMILY}" STREQUAL "clang") + if(IS_CLANG) message("Using instrumented libc++ (build type: ${YB_BUILD_TYPE})") YB_SETUP_CLANG("${YB_BUILD_TYPE}") else() @@ -427,7 +439,7 @@ macro(YB_SETUP_SANITIZER) endif() if("${YB_BUILD_TYPE}" STREQUAL "asan") - if("${COMPILER_FAMILY}" STREQUAL "clang" AND + if(IS_CLANG AND "${COMPILER_VERSION}" VERSION_GREATER_EQUAL "10.0.0" AND NOT APPLE) # TODO: see if we can use static libasan instead (requires third-party changes). @@ -439,20 +451,13 @@ macro(YB_SETUP_SANITIZER) # dependency on libc++ so it gets resolved using our rpath. ADD_LINKER_FLAGS("-lc++") - execute_process( - COMMAND "${CMAKE_CXX_COMPILER}" -print-search-dirs - OUTPUT_VARIABLE CLANG_PRINT_SEARCH_DIRS_OUTPUT) - if ("${CLANG_PRINT_SEARCH_DIRS_OUTPUT}" MATCHES ".*libraries: =([^:]+)(:.*|$)" ) - set(CLANG_RUNTIME_LIB_DIR "${CMAKE_MATCH_1}/lib/linux") - if(NOT EXISTS "${CLANG_RUNTIME_LIB_DIR}") - message(FATAL_ERROR "Clang runtime directory does not exist: ${CLANG_RUNTIME_LIB_DIR}") - endif() - ADD_GLOBAL_RPATH_ENTRY("${CLANG_RUNTIME_LIB_DIR}") - else() - message(FATAL_ERROR - "Could not parse the output of 'clang -print-search-dirs': " - "${CLANG_PRINT_SEARCH_DIRS_OUTPUT}") + if("${CLANG_RUNTIME_LIB_DIR}" STREQUAL "") + message(FATAL_ERROR "CLANG_RUNTIME_LIB_DIR is not set") + endif() + if(NOT EXISTS "${CLANG_RUNTIME_LIB_DIR}") + message(FATAL_ERROR "Clang runtime directory does not exist: ${CLANG_RUNTIME_LIB_DIR}") endif() + ADD_GLOBAL_RPATH_ENTRY("${CLANG_RUNTIME_LIB_DIR}") endif() ADD_CXX_FLAGS("-fsanitize=address") @@ -460,7 +465,7 @@ macro(YB_SETUP_SANITIZER) # Compile and link against the thirdparty ASAN instrumented libstdcxx. ADD_EXE_LINKER_FLAGS("-fsanitize=address") - if("${COMPILER_FAMILY}" STREQUAL "gcc") + if(IS_GCC) ADD_EXE_LINKER_FLAGS("-lubsan -ldl") ADD_CXX_FLAGS("-Wno-error=maybe-uninitialized") endif() @@ -475,7 +480,7 @@ macro(YB_SETUP_SANITIZER) # Compile and link against the thirdparty TSAN instrumented libstdcxx. ADD_EXE_LINKER_FLAGS("-fsanitize=thread") - if("${COMPILER_FAMILY}" STREQUAL "clang" AND + if(IS_CLANG AND "${COMPILER_VERSION}" VERSION_GREATER_EQUAL "10.0.0") # To avoid issues with missing libunwind symbols: # https://gist.githubusercontent.com/mbautin/5bc53ed2d342eab300aec7120eb42996/raw @@ -598,19 +603,31 @@ function(ADD_POSTGRES_SHARED_LIBRARY LIB_NAME SHARED_LIB_PATH) endfunction() function(parse_build_root_basename) + if ("${BUILD_SUPPORT_DIR}" STREQUAL "") + message(FATAL_ERROR "BUILD_SUPPORT_DIR is not set in parse_build_root_basename") + endif() get_filename_component(YB_BUILD_ROOT_BASENAME "${CMAKE_CURRENT_BINARY_DIR}" NAME) - string(REPLACE "-" ";" YB_BUILD_ROOT_BASENAME_COMPONENTS ${YB_BUILD_ROOT_BASENAME}) - list(LENGTH YB_BUILD_ROOT_BASENAME_COMPONENTS YB_BUILD_ROOT_BASENAME_COMPONENTS_LENGTH) - if(YB_BUILD_ROOT_BASENAME_COMPONENTS_LENGTH LESS 3 OR - YB_BUILD_ROOT_BASENAME_COMPONENTS_LENGTH GREATER 5) - message( - FATAL_ERROR - "Wrong number of components of the build root basename: " - "${YB_BUILD_ROOT_BASENAME_COMPONENTS_LENGTH}. Expected 3, 4, or 5 components. " - "Basename: ${YB_BUILD_ROOT_BASENAME}") + + EXEC_PROGRAM("${BUILD_SUPPORT_DIR}/show_build_root_name_regex.sh" + OUTPUT_VARIABLE BUILD_ROOT_BASENAME_RE) + string(REGEX MATCH "${BUILD_ROOT_BASENAME_RE}" RE_MATCH_RESULT "${YB_BUILD_ROOT_BASENAME}") + if("$ENV{YB_DEBUG_BUILD_ROOT_BASENAME_PARSING}" STREQUAL "1") + message("Parsing build root basename: ${YB_BUILD_ROOT_BASENAME}") + message("Regular expression: ${BUILD_ROOT_BASENAME_RE}") + message("Capture groups (note that some components are repeated with and without a leading " + "dash):") + foreach(MATCH_INDEX RANGE 1 9) + message(" CMAKE_MATCH_${MATCH_INDEX}=${CMAKE_MATCH_${MATCH_INDEX}}") + endforeach() endif() - list(GET YB_BUILD_ROOT_BASENAME_COMPONENTS 0 YB_BUILD_TYPE) - set(YB_BUILD_TYPE "${YB_BUILD_TYPE}" PARENT_SCOPE) + + set(YB_BUILD_TYPE "${CMAKE_MATCH_1}" PARENT_SCOPE) + + # ----------------------------------------------------------------------------------------------- + # YB_COMPILER_TYPE + # ----------------------------------------------------------------------------------------------- + + set(YB_COMPILER_TYPE_FROM_BUILD_ROOT_BASENAME "${CMAKE_MATCH_2}") if(NOT "${YB_COMPILER_TYPE}" STREQUAL "" AND NOT "${YB_COMPILER_TYPE}" STREQUAL "${YB_COMPILER_TYPE_FROM_BUILD_ROOT_BASENAME}") message( @@ -620,7 +637,6 @@ function(parse_build_root_basename) "different: '${YB_COMPILER_TYPE_FROM_BUILD_ROOT_BASENAME}'.") endif() - list(GET YB_BUILD_ROOT_BASENAME_COMPONENTS 1 YB_COMPILER_TYPE_FROM_BUILD_ROOT_BASENAME) if(NOT "$ENV{YB_COMPILER_TYPE}" STREQUAL "" AND NOT "$ENV{YB_COMPILER_TYPE}" STREQUAL "${YB_COMPILER_TYPE_FROM_BUILD_ROOT_BASENAME}") message( @@ -629,10 +645,32 @@ function(parse_build_root_basename) "the value auto-detected from the build root basename '${YB_BUILD_ROOT_BASENAME}' is " "different: '${YB_COMPILER_TYPE_FROM_BUILD_ROOT_BASENAME}'.") endif() + set(YB_COMPILER_TYPE "${YB_COMPILER_TYPE_FROM_BUILD_ROOT_BASENAME}" PARENT_SCOPE) set(ENV{YB_COMPILER_TYPE} "${YB_COMPILER_TYPE_FROM_BUILD_ROOT_BASENAME}") - list(GET YB_BUILD_ROOT_BASENAME_COMPONENTS 2 YB_LINKING_TYPE) + # ----------------------------------------------------------------------------------------------- + # YB_USING_LINUXBREW_FROM_BUILD_ROOT + # ----------------------------------------------------------------------------------------------- + + if (NOT "${CMAKE_MATCH_3}" STREQUAL "-linuxbrew" AND + NOT "${CMAKE_MATCH_3}" STREQUAL "") + message(FATAL_ERROR + "Invalid value of the 3rd capture group for build root basename" + "'${YB_BUILD_ROOT_BASENAME}': either '-linuxbrew' or an empty string.") + endif() + + if ("${CMAKE_MATCH_3}" STREQUAL "-linuxbrew") + set(YB_USING_LINUXBREW_FROM_BUILD_ROOT ON PARENT_SCOPE) + else() + set(YB_USING_LINUXBREW_FROM_BUILD_ROOT OFF PARENT_SCOPE) + endif() + + # ----------------------------------------------------------------------------------------------- + # YB_LINKING_TYPE + # ----------------------------------------------------------------------------------------------- + + set(YB_LINKING_TYPE "${CMAKE_MATCH_5}") if(NOT "${YB_LINKING_TYPE}" MATCHES "^(static|dynamic)$") message( FATAL_ERROR @@ -641,15 +679,20 @@ function(parse_build_root_basename) endif() set(YB_LINKING_TYPE "${YB_LINKING_TYPE}" PARENT_SCOPE) - if (YB_BUILD_ROOT_BASENAME_COMPONENTS_LENGTH GREATER 3) - list(GET YB_BUILD_ROOT_BASENAME_COMPONENTS 3 YB_TARGET_ARCH_FROM_BUILD_ROOT) - if(NOT "${YB_TARGET_ARCH_FROM_BUILD_ROOT}" STREQUAL "ninja") - if(NOT "${CMAKE_SYSTEM_PROCESSOR}" STREQUAL "${YB_TARGET_ARCH_FROM_BUILD_ROOT}") - message( - FATAL_ERROR - "Target architecture inferred from build root is '${YB_TARGET_ARCH_FROM_BUILD_ROOT}', " - "but CMAKE_SYSTEM_PROCESSOR is ${CMAKE_SYSTEM_PROCESSOR}") - endif() - endif() + set(OPTIONAL_DASH_NINJA "${CMAKE_MATCH_8}") + if(NOT "${OPTIONAL_DASH_NINJA}" STREQUAL "" AND + NOT "${OPTIONAL_DASH_NINJA}" STREQUAL "-ninja") + message(FATAL_ERROR + "Invalid value of the 8th capture group for build root basename" + "'${YB_BUILD_ROOT_BASENAME}': either '-ninja' or an empty string.") + endif() + + set(YB_TARGET_ARCH_FROM_BUILD_ROOT "${CMAKE_MATCH_7}") + if (NOT "${YB_TARGET_ARCH_FROM_BUILD_ROOT}" STREQUAL "" AND + NOT "${CMAKE_SYSTEM_PROCESSOR}" STREQUAL "${YB_TARGET_ARCH_FROM_BUILD_ROOT}") + message( + FATAL_ERROR + "Target architecture inferred from build root is '${YB_TARGET_ARCH_FROM_BUILD_ROOT}', " + "but CMAKE_SYSTEM_PROCESSOR is ${CMAKE_SYSTEM_PROCESSOR}") endif() endfunction() diff --git a/codecheck.ini b/codecheck.ini index 123e638b32fb..2e6a7e83dc49 100644 --- a/codecheck.ini +++ b/codecheck.ini @@ -28,20 +28,21 @@ shellcheck = off # TODO: add codecheck support for a list of plain file paths (not regexes) and use it here. included_regex_list = + ^build-support/is_same_path[.]py$ + ^build-support/kill_long_running_minicluster_daemons[.]py$ + ^build-support/run_tests_on_spark[.]py$ + ^build-support/split_long_command_line[.]py$ + ^build-support/validate_build_root[.]py$ ^python/yb/__init__[.]py$ + ^python/yb/build_postgres[.]py$ ^python/yb/command_util[.]py$ - ^python/yb/library_packager[.]py$ - ^python/yb/linuxbrew[.]py$ ^python/yb/common_util[.]py$ - ^python/yb/tool_base[.]py$ ^python/yb/compile_commands[.]py$ - ^python/yb/run_pvs_studio_analyzer[.]py$ - ^python/yb/build_postgres[.]py$ + ^python/yb/dependency_graph[.]py$ + ^python/yb/library_packager[.]py$ + ^python/yb/linuxbrew[.]py$ ^python/yb/os_detection[.]py$ + ^python/yb/run_pvs_studio_analyzer[.]py$ ^python/yb/thirdparty_tool[.]py$ + ^python/yb/tool_base[.]py$ ^python/yb/yb_dist_tests[.]py$ - ^build-support/is_same_path[.]py$ - ^build-support/kill_long_running_minicluster_daemons[.]py$ - ^build-support/split_long_command_line[.]py$ - ^build-support/validate_build_root[.]py$ - ^build-support/run_tests_on_spark[.]py$ diff --git a/jenkins_jobs.yml b/jenkins_jobs.yml index 502c3e9d258d..dd59ee6b4586 100644 --- a/jenkins_jobs.yml +++ b/jenkins_jobs.yml @@ -4,35 +4,49 @@ jobs: - os: alma8 compiler: clang12 build_type: asan + - os: centos7 compiler: clang7 build_type: tsan + - os: centos7 compiler: clang11 build_type: debug + + - os: alma8 + compiler: clang12 + build_type: release + - os: centos8 compiler: clang11 build_type: release architecture: aarch64 + - os: centos8 compiler: clang11 build_type: debug architecture: aarch64 + - os: centos7 compiler: gcc5 build_type: debug + - os: centos7 compiler: gcc5 build_type: release + - os: alma8 compiler: gcc9 - build_type: debug + build_type: debug + - os: macos compiler: clang build_type: debug + - os: macos compiler: clang build_type: release + - os: ubuntu20.04 compiler: gcc9 build_type: debug diff --git a/python/yb/common_util.py b/python/yb/common_util.py index 1f6597a764db..a78386457b71 100644 --- a/python/yb/common_util.py +++ b/python/yb/common_util.py @@ -24,6 +24,7 @@ import shlex import io import platform +import argparse import typing from typing import ( @@ -70,7 +71,7 @@ def set_thirdparty_dir(thirdparty_dir: str) -> None: def sorted_grouped_by( - arr: List[_GroupElementType], + arr: Iterable[_GroupElementType], key_fn: Callable[[_GroupElementType], _GroupKeyType] ) -> List[Tuple[_GroupKeyType, List[_GroupElementType]]]: """ @@ -82,7 +83,7 @@ def sorted_grouped_by( def group_by( - arr: List[_GroupElementType], + arr: Iterable[_GroupElementType], key_fn: Callable[[_GroupElementType], _GroupKeyType] ) -> Dict[_GroupKeyType, List[_GroupElementType]]: """ @@ -230,6 +231,13 @@ def get_yb_src_root_from_build_root( return yb_src_root +def ensure_yb_src_root_from_build_root(build_dir: str, verbose: bool = False) -> str: + yb_src_root = get_yb_src_root_from_build_root( + build_dir=build_dir, verbose=verbose, must_succeed=True) + assert yb_src_root is not None + return yb_src_root + + def is_macos() -> bool: return platform.system() == 'Darwin' @@ -379,6 +387,15 @@ def write_yaml_file(content: Any, output_file_path: str) -> None: yaml.dump(content, output_file) +def write_file(content: str, output_file_path: str) -> None: + with open(output_file_path, 'w') as output_file: + output_file.write(content) + + +def make_parent_dir(path: str) -> None: + os.makedirs(os.path.dirname(os.path.abspath(path)), exist_ok=True) + + def to_yaml_str(content: Any) -> str: out = io.StringIO() yaml = get_ruamel_yaml_instance() @@ -403,3 +420,15 @@ def check_arch() -> None: def is_macos_arm64() -> bool: return is_macos() and get_target_arch() == 'arm64' + + +# From https://stackoverflow.com/questions/15008758/parsing-boolean-values-with-argparse +# To be used for boolean arguments. +def arg_str_to_bool(v: Any) -> bool: + if isinstance(v, bool): + return v + if v.lower() in ('yes', 'true', 't', 'y', '1'): + return True + if v.lower() in ('no', 'false', 'f', 'n', '0'): + return False + raise argparse.ArgumentTypeError('Boolean value expected, got: %s' % v) diff --git a/python/yb/dependency_graph.py b/python/yb/dependency_graph.py index 67ee1d933547..d86c1287a741 100755 --- a/python/yb/dependency_graph.py +++ b/python/yb/dependency_graph.py @@ -16,32 +16,43 @@ import unittest import pipes import platform +import time from datetime import datetime +from enum import Enum +from typing import Optional, List, Dict, TypeVar, Set, Any, Iterable, cast +from pathlib import Path + +from yb.common_util import ( + group_by, + make_set, + get_build_type_from_build_root, + ensure_yb_src_root_from_build_root, + convert_to_non_ninja_build_root, + get_bool_env_var, + is_ninja_build_root, + shlex_join, + read_file, +) +from yb.command_util import mkdir_p +from yugabyte_pycommon import WorkDirContext # type: ignore + + +def make_extensions(exts_without_dot: List[str]) -> List[str]: + for ext in exts_without_dot: + assert not ext.startswith('.') -from typing import Optional, List, Dict - -sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) - -from yb.common_util import \ - group_by, make_set, get_build_type_from_build_root, get_yb_src_root_from_build_root, \ - convert_to_non_ninja_build_root, get_bool_env_var, is_ninja_build_root # nopep8 -from yb.command_util import mkdir_p # nopep8 -from yugabyte_pycommon import WorkDirContext # nopep8 - - -def make_extensions(exts_without_dot): return ['.' + ext for ext in exts_without_dot] -def ends_with_one_of(path, exts): +def ends_with_any_of(path: str, exts: List[str]) -> bool: for ext in exts: if path.endswith(ext): return True return False -def get_relative_path_or_none(abs_path, relative_to): +def get_relative_path_or_none(abs_path: str, relative_to: str) -> Optional[str]: """ If the given path starts with another directory path, return the relative path, or else None. """ @@ -49,6 +60,7 @@ def get_relative_path_or_none(abs_path, relative_to): relative_to += '/' if abs_path.startswith(relative_to): return abs_path[len(relative_to):] + return None SOURCE_FILE_EXTENSIONS = make_extensions(['c', 'cc', 'cpp', 'cxx', 'h', 'hpp', 'hxx', 'proto', @@ -73,19 +85,18 @@ def get_relative_path_or_none(abs_path, relative_to): SELF_TEST_CMD = 'self-test' DEBUG_DUMP_CMD = 'debug-dump' -COMMANDS = [LIST_DEPS_CMD, - LIST_REVERSE_DEPS_CMD, - LIST_AFFECTED_CMD, - SELF_TEST_CMD, - DEBUG_DUMP_CMD] +COMMANDS = [ + LIST_DEPS_CMD, + LIST_REVERSE_DEPS_CMD, + LIST_AFFECTED_CMD, + SELF_TEST_CMD, + DEBUG_DUMP_CMD, +] COMMANDS_NOT_NEEDING_TARGET_SET = [SELF_TEST_CMD, DEBUG_DUMP_CMD] HOME_DIR = os.path.realpath(os.path.expanduser('~')) -# This will match any node type (node types being sources/libraries/tests/etc.) -NODE_TYPE_ANY = 'any' - # As of August 2019, there is nothing in the "bin", "managed" and "www" directories that # is being used by tests. # If that changes, this needs to be updated. Note that the "bin" directory here is the @@ -113,68 +124,88 @@ def get_relative_path_or_none(abs_path, relative_to): DYLIB_SUFFIX = '.dylib' if platform.system() == 'Darwin' else '.so' -def is_object_file(path): +class NodeType(Enum): + # A special value used to match any node type. + ANY = 'any' + + SOURCE = 'source' + LIBRARY = 'library' + TEST = 'test' + OBJECT = 'object' + EXECUTABLE = 'executable' + OTHER = 'other' + + def __str__(self) -> str: + return self.value + + +def is_object_file(path: str) -> bool: return path.endswith('.o') def is_shared_library(path: str) -> bool: return ( - ends_with_one_of(path, LIBRARY_FILE_EXTENSIONS) and + ends_with_any_of(path, LIBRARY_FILE_EXTENSIONS) and not os.path.basename(path) in LIBRARY_FILE_EXTENSIONS and not path.startswith('-')) -def append_to_list_in_dict(dest, key, new_item): +K = TypeVar('K') +V = TypeVar('V') + + +def append_to_list_in_dict(dest: Dict[K, List[V]], key: K, new_item: V) -> None: if key in dest: dest[key].append(new_item) else: dest[key] = [new_item] -def get_node_type_by_path(path): +def get_node_type_by_path(path: str) -> NodeType: """ - >>> get_node_type_by_path('my_source_file.cc') + >>> get_node_type_by_path('my_source_file.cc').value 'source' - >>> get_node_type_by_path('my_library.so') + >>> get_node_type_by_path('my_library.so').value 'library' - >>> get_node_type_by_path('/bin/bash') + >>> get_node_type_by_path('/bin/bash').value 'executable' - >>> get_node_type_by_path('my_object_file.o') + >>> get_node_type_by_path('my_object_file.o').value 'object' - >>> get_node_type_by_path('tests-integration/some_file') + >>> get_node_type_by_path('tests-integration/some_file').value 'test' - >>> get_node_type_by_path('tests-integration/some_file.txt') + >>> get_node_type_by_path('tests-integration/some_file.txt').value 'other' - >>> get_node_type_by_path('something/my-test') + >>> get_node_type_by_path('something/my-test').value 'test' - >>> get_node_type_by_path('something/my_test') + >>> get_node_type_by_path('something/my_test').value 'test' - >>> get_node_type_by_path('something/my-itest') + >>> get_node_type_by_path('something/my-itest').value 'test' - >>> get_node_type_by_path('something/my_itest') + >>> get_node_type_by_path('something/my_itest').value 'test' - >>> get_node_type_by_path('some-dir/some_file') + >>> get_node_type_by_path('some-dir/some_file').value 'other' """ - if ends_with_one_of(path, SOURCE_FILE_EXTENSIONS): - return 'source' + if ends_with_any_of(path, SOURCE_FILE_EXTENSIONS): + return NodeType.SOURCE - if ends_with_one_of(path, LIBRARY_FILE_EXTENSIONS): - return 'library' + if (ends_with_any_of(path, LIBRARY_FILE_EXTENSIONS) or + any([ext + '.' in path for ext in LIBRARY_FILE_EXTENSIONS])): + return NodeType.LIBRARY - if (ends_with_one_of(path, TEST_FILE_SUFFIXES) or + if (ends_with_any_of(path, TEST_FILE_SUFFIXES) or (os.path.basename(os.path.dirname(path)).startswith('tests-') and '.' not in os.path.basename(path))): - return 'test' + return NodeType.TEST if is_object_file(path): - return 'object' + return NodeType.OBJECT if os.path.exists(path) and os.access(path, os.X_OK): # This will only work if the code has been fully built. - return 'executable' + return NodeType.EXECUTABLE - return 'other' + return NodeType.OTHER class Node: @@ -182,7 +213,21 @@ class Node: A node in the dependency graph. Could be a source file, a header file, an object file, a dynamic library, or an executable. """ - def __init__(self, path, dep_graph, source_str): + + path: str + deps: Set['Node'] + reverse_deps: Set['Node'] + node_type: NodeType + dep_graph: 'DependencyGraph' + conf: 'Configuration' + source_str: Optional[str] + is_proto_lib: bool + link_cmd: Optional[List[str]] + _cached_proto_lib_deps: Optional[List['Node']] + _cached_containing_binaries: Optional[List['Node']] + _cached_cmake_target: Optional[str] + + def __init__(self, path: str, dep_graph: 'DependencyGraph', source_str: Optional[str]) -> None: path = os.path.realpath(path) self.path = path @@ -197,8 +242,8 @@ def __init__(self, path, dep_graph, source_str): self.conf = dep_graph.conf self.source_str = source_str - self.is_proto_lib = ( - self.node_type == 'library' and + self.is_proto_lib = bool( + self.node_type == NodeType.LIBRARY and PROTO_LIBRARY_FILE_NAME_RE.match(os.path.basename(self.path))) self._cached_proto_lib_deps = None @@ -206,30 +251,32 @@ def __init__(self, path, dep_graph, source_str): self._cached_cmake_target = None self._has_no_cmake_target = False - def add_dependency(self, dep): + self.link_cmd = None + + def add_dependency(self, dep: 'Node') -> None: assert self is not dep self.deps.add(dep) dep.reverse_deps.add(self) - def __eq__(self, other): + def __eq__(self, other: Any) -> bool: if not isinstance(other, Node): return False return self.path == other.path - def __hash__(self): + def __hash__(self) -> int: return hash(self.path) - def is_source(self): - return self.node_type == 'source' + def is_source(self) -> bool: + return self.node_type == NodeType.SOURCE - def validate_existence(self): + def validate_existence(self) -> None: if not os.path.exists(self.path) and not self.dep_graph.conf.incomplete_build: raise RuntimeError( "Path does not exist: '{}'. This node was found in: {}".format( self.path, self.source_str)) - def get_pretty_path(self): + def get_pretty_path(self) -> str: for prefix, alias in [(self.conf.build_root, '$BUILD_ROOT'), (self.conf.yb_src_root, '$YB_SRC_ROOT'), (HOME_DIR, '~')]: @@ -238,20 +285,20 @@ def get_pretty_path(self): return self.path - def path_rel_to_build_root(self): + def path_rel_to_build_root(self) -> Optional[str]: return get_relative_path_or_none(self.path, self.conf.build_root) - def path_rel_to_src_root(self): + def path_rel_to_src_root(self) -> Optional[str]: return get_relative_path_or_none(self.path, self.conf.yb_src_root) - def __str__(self): + def __str__(self) -> str: return "Node(\"{}\", type={}, {} deps, {} rev deps)".format( self.get_pretty_path(), self.node_type, len(self.deps), len(self.reverse_deps)) - def __repr__(self): + def __repr__(self) -> str: return self.__str__() - def get_cmake_target(self): + def get_cmake_target(self) -> Optional[str]: """ @return the CMake target based on the current node's path. E.g. this would be "master" for the "libmaster.so" library, "yb-master" for the "yb-master" executable. @@ -300,7 +347,7 @@ def get_cmake_target(self): self._has_no_cmake_target = True return None - def get_containing_binaries(self): + def get_containing_binaries(self) -> List['Node']: """ Returns nodes (usually one node) corresponding to executables or dynamic libraries that the given object file is compiled into. @@ -314,14 +361,17 @@ def get_containing_binaries(self): "%s. All set of reverse dependencies: %s" % (self, self.reverse_deps)) return cc_o_rev_deps[0].get_containing_binaries() - if self.node_type != 'object': - return None + if self.node_type != NodeType.OBJECT: + raise ValueError( + "get_containing_binaries can only be called on nodes of type 'object'. Node: " + + str(self)) + if self._cached_containing_binaries: return self._cached_containing_binaries binaries = [] for rev_dep in self.reverse_deps: - if rev_dep.node_type in ['library', 'test', 'executable']: + if rev_dep.node_type in [NodeType.LIBRARY, NodeType.TEST, NodeType.EXECUTABLE]: binaries.append(rev_dep) if len(binaries) > 1: logging.warning( @@ -331,46 +381,59 @@ def get_containing_binaries(self): self._cached_containing_binaries = binaries return binaries - def get_recursive_deps(self): + def get_recursive_deps( + self, + skip_node_types: Set[NodeType] = set()) -> Set['Node']: """ Returns a set of all dependencies that this node depends on. + skip_node_types specifies the set of categories of nodes, except the initial node, to stop + the recursive search at. """ - recursive_deps = set() - visited = set() + recursive_deps: Set[Node] = set() + visited: Set[Node] = set() - def walk(node, add_self=True): - if add_self: + def walk(node: Node, is_initial: bool) -> None: + if not is_initial: + # Only skip the given subset of node types for nodes other than the initial node. + if node.node_type in skip_node_types: + return + # Also, never add the initial node to the result. recursive_deps.add(node) for dep in node.deps: if dep not in recursive_deps: - walk(dep) - walk(self, add_self=False) + walk(dep, is_initial=False) + + walk(self, is_initial=True) return recursive_deps - def get_proto_lib_deps(self): + def get_proto_lib_deps(self) -> List['Node']: if self._cached_proto_lib_deps is None: self._cached_proto_lib_deps = [ dep for dep in self.get_recursive_deps() if dep.is_proto_lib ] return self._cached_proto_lib_deps - def get_containing_proto_lib(self): + def get_containing_proto_lib(self) -> Optional['Node']: """ For a .pb.cc file node, return the node corresponding to the containing protobuf library. """ if not self.path.endswith('.pb.cc.o'): - return + return None - proto_libs = [binary for binary in self.get_containing_binaries()] + containing_binaries: List[Node] = self.get_containing_binaries() - if len(proto_libs) != 1: + if len(containing_binaries) != 1: logging.info("Reverse deps:\n %s" % ("\n ".join( [str(dep) for dep in self.reverse_deps]))) - raise RuntimeError("Invalid number of proto libs for %s: %s" % (node, proto_libs)) - return proto_libs[0] + raise RuntimeError( + "Invalid number of proto libs for %s. Expected 1 but found %d: %s" % ( + self, + len(containing_binaries), + containing_binaries)) + return containing_binaries[0] - def get_proto_gen_cmake_target(self): + def get_proto_gen_cmake_target(self) -> Optional[str]: """ For .pb.{h,cc} nodes this will return a CMake target of the form gen_..., e.g. gen_src_yb_common_wire_protocol. @@ -389,25 +452,53 @@ def get_proto_gen_cmake_target(self): [match.group(1), 'proto'] ) + def set_link_command(self, link_cmd: List[str]) -> None: + assert self.link_cmd is None + self.link_cmd = link_cmd + + def as_json(self) -> Dict[str, Any]: + d: Dict[str, Any] = dict( + path=self.path + ) + if self.link_cmd: + d['link_cmd'] = self.link_cmd + return d + + def update_from_json(self, json_data: Dict[str, Any]) -> None: + if 'link_cmd' in json_data: + self.link_cmd = json_data['link_cmd'] -def set_to_str(items): - return ",\n".join(sorted(items)) +def set_to_str(items: Set[Any]) -> str: + return ",\n".join(sorted(list(items))) -def is_abs_path(path): + +def is_abs_path(path: str) -> bool: return path.startswith('/') class Configuration: - - def __init__(self, args): + args: argparse.Namespace + verbose: bool + build_root: str + is_ninja: bool + build_type: str + yb_src_root: str + src_dir_path: str + ent_src_dir_path: str + rel_path_base_dirs: Set[str] + incomplete_build: bool + file_regex: str + src_dir_paths: List[str] + + def __init__(self, args: argparse.Namespace) -> None: self.args = args self.verbose = args.verbose self.build_root = os.path.realpath(args.build_root) self.is_ninja = is_ninja_build_root(self.build_root) self.build_type = get_build_type_from_build_root(self.build_root) - self.yb_src_root = get_yb_src_root_from_build_root(self.build_root, must_succeed=True) + self.yb_src_root = ensure_yb_src_root_from_build_root(self.build_root) self.src_dir_path = os.path.join(self.yb_src_root, 'src') self.ent_src_dir_path = os.path.join(self.yb_src_root, 'ent', 'src') self.rel_path_base_dirs = set([self.build_root, os.path.join(self.src_dir_path, 'yb')]) @@ -430,14 +521,18 @@ class CMakeDepGraph: yb_cmake_deps.txt file that we generate in our top-level CMakeLists.txt. This dependency graph does not have any nodes for source files and object files. """ - def __init__(self, build_root): + + build_root: str + cmake_targets: Set[str] + cmake_deps_path: str + cmake_deps: Dict[str, Set[str]] + + def __init__(self, build_root: str) -> None: self.build_root = build_root - self.cmake_targets = None self.cmake_deps_path = os.path.join(self.build_root, 'yb_cmake_deps.txt') - self.cmake_deps = None self._load() - def _load(self): + def _load(self) -> None: logging.info("Loading dependencies between CMake targets from '{}'".format( self.cmake_deps_path)) self.cmake_deps = {} @@ -468,7 +563,7 @@ def _load(self): logging.info("Found {} CMake targets in '{}'".format( len(self.cmake_targets), self.cmake_deps_path)) - def _get_cmake_dep_set_of(self, target): + def _get_cmake_dep_set_of(self, target: str) -> Set[str]: """ Get CMake dependencies of the given target. What is returned is a mutable set. Modifying this set modifies this CMake dependency graph. @@ -480,17 +575,17 @@ def _get_cmake_dep_set_of(self, target): self.cmake_targets.add(target) return deps - def add_dependency(self, from_target, to_target): + def add_dependency(self, from_target: str, to_target: str) -> None: if from_target in IGNORED_CMAKE_TARGETS or to_target in IGNORED_CMAKE_TARGETS: return self._get_cmake_dep_set_of(from_target).add(to_target) self.cmake_targets.add(to_target) - def get_recursive_cmake_deps(self, cmake_target): - result = set() - visited = set() + def get_recursive_cmake_deps(self, cmake_target: str) -> Set[str]: + result: Set[str] = set() + visited: Set[str] = set() - def walk(cur_target, add_this_target=True): + def walk(cur_target: str, add_this_target: bool = True) -> None: if cur_target in visited: return visited.add(cur_target) @@ -508,20 +603,27 @@ class DependencyGraphBuilder: Builds a dependency graph from the contents of the build directory. Each node of the graph is a file (an executable, a dynamic library, or a source file). """ - def __init__(self, conf): + conf: Configuration + compile_dirs: Set[str] + compile_commands: Dict[str, Any] + useful_base_dirs: Set[str] + unresolvable_rel_paths: Set[str] + resolved_rel_paths: Dict[str, str] + dep_graph: 'DependencyGraph' + cmake_dep_graph: CMakeDepGraph + + def __init__(self, conf: Configuration) -> None: self.conf = conf self.compile_dirs = set() - self.compile_commands = None self.useful_base_dirs = set() self.unresolvable_rel_paths = set() self.resolved_rel_paths = {} self.dep_graph = DependencyGraph(conf) - self.cmake_dep_graph = None - def load_cmake_deps(self): + def load_cmake_deps(self) -> None: self.cmake_dep_graph = CMakeDepGraph(self.conf.build_root) - def parse_link_and_depend_files_for_make(self): + def parse_link_and_depend_files_for_make(self) -> None: logging.info( "Parsing link.txt and depend.make files from the build tree at '{}'".format( self.conf.build_root)) @@ -546,7 +648,7 @@ def parse_link_and_depend_files_for_make(self): logging.info("Parsed link.txt and depend.make files in %.2f seconds" % (datetime.now() - start_time).total_seconds()) - def find_proto_files(self): + def find_proto_files(self) -> None: for src_subtree_root in self.conf.src_dir_paths: logging.info("Finding .proto files in the source tree at '{}'".format(src_subtree_root)) source_str = 'proto files in {}'.format(src_subtree_root) @@ -557,7 +659,7 @@ def find_proto_files(self): os.path.join(root, file_name), source_str=source_str) - def find_flex_bison_files(self): + def find_flex_bison_files(self) -> None: """ CMake commands generally include the C file compilation, but misses the case where flex or bison generates those files, somewhat similiar to .proto files. @@ -578,9 +680,9 @@ def find_flex_bison_files(self): os.path.join(root, file_name), source_str) - def match_cmake_targets_with_files(self): + def match_cmake_targets_with_files(self) -> None: logging.info("Matching CMake targets with the files found") - self.cmake_target_to_nodes = {} + self.cmake_target_to_nodes: Dict[str, Set[Node]] = {} for node in self.dep_graph.get_nodes(): node_cmake_target = node.get_cmake_target() if node_cmake_target: @@ -590,7 +692,7 @@ def match_cmake_targets_with_files(self): self.cmake_target_to_nodes[node_cmake_target] = node_set node_set.add(node) - self.cmake_target_to_node = {} + self.cmake_target_to_node: Dict[str, Node] = {} unmatched_cmake_targets = set() cmake_dep_graph = self.dep_graph.get_cmake_dep_graph() for cmake_target in cmake_dep_graph.cmake_targets: @@ -620,7 +722,7 @@ def match_cmake_targets_with_files(self): continue node.add_dependency(self.cmake_target_to_node[cmake_target_dep]) - def resolve_rel_path(self, rel_path): + def resolve_rel_path(self, rel_path: str) -> Optional[str]: if is_abs_path(rel_path): return rel_path @@ -649,7 +751,7 @@ def resolve_rel_path(self, rel_path): self.resolved_rel_paths[rel_path] = resolved return resolved - def resolve_dependent_rel_path(self, rel_path): + def resolve_dependent_rel_path(self, rel_path: str) -> str: if is_abs_path(rel_path): return rel_path if is_object_file(rel_path): @@ -658,7 +760,7 @@ def resolve_dependent_rel_path(self, rel_path): "Don't know how to resolve relative path of a 'dependent': {}".format( rel_path)) - def parse_ninja_metadata(self): + def parse_ninja_metadata(self) -> None: with WorkDirContext(self.conf.build_root): ninja_path = os.environ.get('YB_NINJA_PATH', 'ninja') logging.info("Ninja executable path: %s", ninja_path) @@ -666,26 +768,33 @@ def parse_ninja_metadata(self): subprocess.check_call('{} -t commands >ninja_commands.txt'.format( pipes.quote(ninja_path)), shell=True) logging.info("Parsing the output of 'ninja -t commands' for linker commands") + start_time_sec = time.time() self.parse_link_txt_file('ninja_commands.txt') + logging.info("Parsing linker commands took %.1f seconds", time.time() - start_time_sec) logging.info("Running 'ninja -t deps'") subprocess.check_call('{} -t deps >ninja_deps.txt'.format( pipes.quote(ninja_path)), shell=True) + start_time = time.time() logging.info("Parsing the output of 'ninja -t deps' to infer dependencies") + logging.info("Parsing dependencies took %.1f seconds", time.time() - start_time_sec) self.parse_depend_file('ninja_deps.txt') - def register_dependency(self, dependent, dependency, dependency_came_from): + def register_dependency(self, + dependent: str, + dependency: str, + dependency_came_from: str) -> None: dependent = self.resolve_dependent_rel_path(dependent.strip()) dependency = dependency.strip() - dependency = self.resolve_rel_path(dependency) - if dependency: + dependency_resolved: Optional[str] = self.resolve_rel_path(dependency) + if dependency_resolved: dependent_node = self.dep_graph.find_or_create_node( dependent, source_str=dependency_came_from) dependency_node = self.dep_graph.find_or_create_node( - dependency, source_str=dependency_came_from) + dependency_resolved, source_str=dependency_came_from) dependent_node.add_dependency(dependency_node) - def parse_depend_file(self, depend_make_path): + def parse_depend_file(self, depend_make_path: str) -> None: """ Parse either a depend.make file from a CMake-generated Unix Makefile project (fully built) or the output of "ninja -t deps" in a Ninja project. @@ -717,27 +826,14 @@ def parse_depend_file(self, depend_make_path): raise ValueError("Could not parse this line from %s:\n%s" % (depend_make_path, line_orig)) - def find_node_by_rel_path(self, rel_path): - if is_abs_path(rel_path): - return self.find_node(rel_path, must_exist=True) - candidates = [] - for path, node in self.node_by_path.items(): - if path.endswith('/' + rel_path): - candidates.append(node) - if not candidates: - raise RuntimeError("Could not find node by relative path '{}'".format(rel_path)) - if len(candidates) > 1: - raise RuntimeError("Ambiguous nodes for relative path '{}'".format(rel_path)) - return candidates[0] - - def parse_link_txt_file(self, link_txt_path): + def parse_link_txt_file(self, link_txt_path: str) -> None: with open(link_txt_path) as link_txt_file: for line in link_txt_file: line = line.strip() if line: self.parse_link_command(line, link_txt_path) - def parse_link_command(self, link_command, link_txt_path): + def parse_link_command(self, link_command: str, link_txt_path: str) -> None: link_args = link_command.split() output_path = None inputs = [] @@ -795,8 +891,9 @@ def parse_link_command(self, link_command, link_txt_path): ("Cannot add a dependency from a node to itself: %s. " "Parsed from command: %s") % (output_node, link_command)) output_node.add_dependency(dependency_node) + output_node.set_link_command(link_args) - def build(self): + def build(self) -> 'DependencyGraph': compile_commands_path = os.path.join(self.conf.build_root, 'compile_commands.json') cmake_deps_path = os.path.join(self.conf.build_root, 'yb_cmake_deps.txt') if not os.path.exists(compile_commands_path) or not os.path.exists(cmake_deps_path): @@ -818,7 +915,7 @@ def build(self): self.compile_commands = json.load(commands_file) for entry in self.compile_commands: - self.compile_dirs.add(entry['directory']) + self.compile_dirs.add(cast(Dict[str, Any], entry)['directory']) if self.conf.is_ninja: self.parse_ninja_metadata() @@ -837,9 +934,16 @@ def build(self): class DependencyGraph: - canonicalization_cache = {} + conf: Configuration + node_by_path: Dict[str, Node] + nodes_by_basename: Optional[Dict[str, Set[Node]]] + cmake_dep_graph: Optional[CMakeDepGraph] - def __init__(self, conf, json_data=None): + canonicalization_cache: Dict[str, str] = {} + + def __init__(self, + conf: Configuration, + json_data: Optional[List[Dict[str, Any]]] = None) -> None: """ @param json_data optional results of JSON parsing """ @@ -850,7 +954,7 @@ def __init__(self, conf, json_data=None): self.nodes_by_basename = None self.cmake_dep_graph = None - def get_cmake_dep_graph(self): + def get_cmake_dep_graph(self) -> CMakeDepGraph: if self.cmake_dep_graph: return self.cmake_dep_graph @@ -863,7 +967,10 @@ def get_cmake_dep_graph(self): self.cmake_dep_graph = cmake_dep_graph return cmake_dep_graph - def find_node(self, path, must_exist=True, source_str=None): + def find_node(self, + path: str, + must_exist: bool = True, + source_str: Optional[str] = None) -> Node: assert source_str is None or not must_exist path = os.path.abspath(path) node = self.node_by_path.get(path) @@ -877,7 +984,7 @@ def find_node(self, path, must_exist=True, source_str=None): self.node_by_path[path] = node return node - def find_or_create_node(self, path, source_str=None): + def find_or_create_node(self, path: str, source_str: Optional[str] = None) -> Node: """ Finds a node with the given path or creates it if it does not exist. @param source_str a string description of how we came up with this node's path @@ -893,33 +1000,45 @@ def find_or_create_node(self, path, source_str=None): return self.find_node(canonical_path, must_exist=False, source_str=source_str) - def init_from_json(self, json_nodes): + def create_node_from_json(self, node_json_data: Dict[str, Any]) -> Node: + node = self.find_or_create_node(node_json_data['path']) + node.update_from_json(node_json_data) + return node + + def init_from_json(self, json_nodes: List[Dict[str, Any]]) -> None: id_to_node = {} id_to_dep_ids = {} for node_json in json_nodes: node_id = node_json['id'] - id_to_node[node_id] = self.find_or_create_node(node_json['path']) - id_to_dep_ids[node_id] = node_json['deps'] + id_to_node[node_id] = self.create_node_from_json(node_json) + id_to_dep_ids[node_id] = node_json.get('deps') or [] for node_id, dep_ids in id_to_dep_ids.items(): node = id_to_node[node_id] for dep_id in dep_ids: node.add_dependency(id_to_node[dep_id]) - def find_nodes_by_regex(self, regex_str): + def find_nodes_by_regex(self, regex_str: str) -> List[Node]: filter_re = re.compile(regex_str) return [node for node in self.get_nodes() if filter_re.match(node.path)] - def find_nodes_by_basename(self, basename): + def find_nodes_by_basename(self, basename: str) -> Optional[Set[Node]]: if not self.nodes_by_basename: # We are lazily initializing the basename -> node map, and any changes to the graph # after this point will not get reflected in it. This is OK as we're only using this # function after the graph has been built. - self.nodes_by_basename = group_by( + self.nodes_by_basename = { + k: set(v) + for k, v in group_by( self.get_nodes(), - lambda node: os.path.basename(node.path)) + lambda node: os.path.basename(node.path)).items() + } + assert self.nodes_by_basename is not None return self.nodes_by_basename.get(basename) - def find_affected_nodes(self, start_nodes, requested_node_type=NODE_TYPE_ANY): + def find_affected_nodes( + self, + start_nodes: Set[Node], + requested_node_type: NodeType = NodeType.ANY) -> Set[Node]: if self.conf.verbose: logging.info("Starting with the following initial nodes:") for node in start_nodes: @@ -928,9 +1047,9 @@ def find_affected_nodes(self, start_nodes, requested_node_type=NODE_TYPE_ANY): results = set() visited = set() - def dfs(node): - if ((requested_node_type == NODE_TYPE_ANY or node.node_type == requested_node_type) and - node not in start_nodes): + def dfs(node: Node) -> None: + if ((requested_node_type == NodeType.ANY or + node.node_type == requested_node_type) and node not in start_nodes): results.add(node) if node in visited: return @@ -943,7 +1062,8 @@ def dfs(node): return results - def affected_basenames_by_basename_for_test(self, basename, node_type=NODE_TYPE_ANY): + def affected_basenames_by_basename_for_test( + self, basename: str, node_type: NodeType = NodeType.ANY) -> Set[str]: nodes_for_basename = self.find_nodes_by_basename(basename) if not nodes_for_basename: self.dump_debug_info() @@ -953,17 +1073,17 @@ def affected_basenames_by_basename_for_test(self, basename, node_type=NODE_TYPE_ return set([os.path.basename(node.path) for node in self.find_affected_nodes(nodes_for_basename, node_type)]) - def save_as_json(self, output_path): + def save_as_json(self, output_path: str) -> None: """ Converts the dependency graph into a JSON representation, where every node is given an id, so that dependencies are represented concisely. """ with open(output_path, 'w') as output_file: next_node_id = [1] # Use a mutable object so we can modify it from closure. - path_to_id = {} + path_to_id: Dict[str, int] = {} output_file.write("[") - def get_node_id(node): + def get_node_id(node: Node) -> int: node_id = path_to_id.get(node.path) if not node_id: node_id = next_node_id[0] @@ -973,11 +1093,11 @@ def get_node_id(node): is_first = True for node_path, node in self.node_by_path.items(): - node_json = dict( - id=get_node_id(node), - path=node_path, - deps=[get_node_id(dep) for dep in node.deps] - ) + node_json = node.as_json() + dep_ids = [get_node_id(dep) for dep in node.deps] + node_json.update(id=get_node_id(node)) + if dep_ids: + node_json.update(deps=dep_ids) if not is_first: output_file.write(",\n") is_first = False @@ -986,21 +1106,21 @@ def get_node_id(node): logging.info("Saved dependency graph to '{}'".format(output_path)) - def validate_node_existence(self): + def validate_node_existence(self) -> None: logging.info("Validating existence of build artifacts") for node in self.get_nodes(): node.validate_existence() - def get_nodes(self): + def get_nodes(self) -> Iterable[Node]: return self.node_by_path.values() - def dump_debug_info(self): + def dump_debug_info(self) -> None: logging.info("Dumping all graph nodes for debugging ({} nodes):".format( len(self.node_by_path))) for node in sorted(self.get_nodes(), key=lambda node: str(node)): logging.info(node) - def _add_proto_generation_deps(self): + def _add_proto_generation_deps(self) -> None: """ Add dependencies of .pb.{h,cc} files on the corresponding .proto file. We do that by finding .proto and .pb.{h,cc} nodes in the graph independently and matching them @@ -1011,10 +1131,11 @@ def _add_proto_generation_deps(self): generates these files (e.g. gen_src_yb_rocksdb_db_version_edit_proto). We add these inferred dependencies to the separate CMake dependency graph. """ - proto_node_by_rel_path = {} - pb_h_cc_nodes_by_rel_path = {} + proto_node_by_rel_path: Dict[str, Node] = {} + pb_h_cc_nodes_by_rel_path: Dict[str, List[Node]] = {} cmake_dep_graph = self.get_cmake_dep_graph() + assert cmake_dep_graph is not None for node in self.get_nodes(): basename = os.path.basename(node.path) @@ -1050,8 +1171,10 @@ def _add_proto_generation_deps(self): proto_gen_cmake_target = node.get_proto_gen_cmake_target() for containing_binary in node.get_containing_binaries(): containing_cmake_target = containing_binary.get_cmake_target() + assert containing_cmake_target is not None proto_gen_cmake_target = node.get_proto_gen_cmake_target() - self.cmake_dep_graph.add_dependency( + assert proto_gen_cmake_target is not None + cmake_dep_graph.add_dependency( containing_cmake_target, proto_gen_cmake_target ) @@ -1077,12 +1200,12 @@ def _add_proto_generation_deps(self): for pb_h_cc_node in pb_h_cc_nodes_by_rel_path[rel_path]: pb_h_cc_node.add_dependency(proto_node) - def _check_for_circular_dependencies(self): + def _check_for_circular_dependencies(self) -> None: logging.info("Checking for circular dependencies") visited = set() stack = [] - def walk(node): + def walk(node: Node) -> None: if node in visited: return try: @@ -1093,17 +1216,17 @@ def walk(node): if dep in stack: raise RuntimeError("Circular dependency loop found: %s", stack) return - walk(dep) + walk(dep) # type: ignore finally: stack.pop() for node in self.get_nodes(): - walk(node) + walk(node) # type: ignore logging.info("No circular dependencies found -- this is good (visited %d nodes)", len(visited)) - def validate_proto_deps(self): + def validate_proto_deps(self) -> None: """ Make sure that code that depends on protobuf-generated files also depends on the corresponding protobuf library target. @@ -1115,9 +1238,6 @@ def validate_proto_deps(self): self._add_proto_generation_deps() self._check_for_circular_dependencies() - header_node_map = {} - lib_node_map = {} - # For any .pb.h file, we want to find all .cc.o files that depend on it, meaning that they # directly or indirectly include that .pb.h file. # @@ -1145,8 +1265,11 @@ def validate_proto_deps(self): for rev_dep in pb_h_node.reverse_deps: if rev_dep.path.endswith('.cc.o'): - for binary in rev_dep.get_containing_binaries(): - binary_cmake_target = binary.get_cmake_target() + containing_binaries: Optional[List[Node]] = rev_dep.get_containing_binaries() + assert containing_binaries is not None + for binary in containing_binaries: + binary_cmake_target: Optional[str] = binary.get_cmake_target() + assert binary_cmake_target is not None recursive_cmake_deps = self.get_cmake_dep_graph().get_recursive_cmake_deps( binary_cmake_target) @@ -1166,24 +1289,32 @@ def validate_proto_deps(self): class DependencyGraphTest(unittest.TestCase): - dep_graph = None + dep_graph: Optional[DependencyGraph] = None # Basename -> basenames affected by it. - affected_basenames_cache = {} - - def get_affected_basenames(self, initial_basename): - affected_basenames = self.affected_basenames_cache.get(initial_basename) - if not affected_basenames: - affected_basenames = self.dep_graph.affected_basenames_by_basename_for_test( - initial_basename) - self.affected_basenames_cache[initial_basename] = affected_basenames - if self.dep_graph.conf.verbose: - # This is useful to get inspiration for new tests. - logging.info("Files affected by {}:\n {}".format( - initial_basename, "\n ".join(sorted(affected_basenames)))) - return affected_basenames - - def assert_affected_by(self, expected_affected_basenames, initial_basename): + affected_basenames_cache: Dict[str, Set[str]] = {} + + def get_affected_basenames(self, initial_basename: str) -> Set[str]: + affected_basenames_from_cache: Optional[Set[str]] = self.affected_basenames_cache.get( + initial_basename) + if affected_basenames_from_cache is not None: + return affected_basenames_from_cache + + assert self.dep_graph + affected_basenames_for_test: Set[str] = \ + self.dep_graph.affected_basenames_by_basename_for_test(initial_basename) + self.affected_basenames_cache[initial_basename] = affected_basenames_for_test + assert self.dep_graph is not None + if self.dep_graph.conf.verbose: + # This is useful to get inspiration for new tests. + logging.info("Files affected by {}:\n {}".format( + initial_basename, "\n ".join(sorted(affected_basenames_for_test)))) + return affected_basenames_for_test + + def assert_affected_by( + self, + expected_affected_basenames: List[str], + initial_basename: str) -> None: """ Asserts that all given files are affected by the given file. Other files might also be affected and that's OK. @@ -1196,12 +1327,15 @@ def assert_affected_by(self, expected_affected_basenames, initial_basename): sorted(remaining_basenames), initial_basename)) - def assert_unaffected_by(self, unaffected_basenames, initial_basename): + def assert_unaffected_by( + self, + unaffected_basenames: List[str], + initial_basename: str) -> None: """ Asserts that the given files are unaffected by the given file. """ - affected_basenames = self.get_affected_basenames(initial_basename) - incorrectly_affected = make_set(unaffected_basenames) & affected_basenames + affected_basenames: Set[str] = self.get_affected_basenames(initial_basename) + incorrectly_affected: Set[str] = make_set(unaffected_basenames) & affected_basenames if incorrectly_affected: self.assertFalse( ("Expected files {} to be unaffected by {}, but they are. Other affected " @@ -1210,20 +1344,23 @@ def assert_unaffected_by(self, unaffected_basenames, initial_basename): initial_basename, sorted(affected_basenames - incorrectly_affected))) - def assert_affected_exactly_by(self, expected_affected_basenames, initial_basename): + def assert_affected_exactly_by( + self, + expected_affected_basenames: List[str], + initial_basename: str) -> None: """ Checks the exact set of files affected by the given file. """ affected_basenames = self.get_affected_basenames(initial_basename) - self.assertEquals(make_set(expected_affected_basenames), affected_basenames) + self.assertEqual(make_set(expected_affected_basenames), affected_basenames) - def test_master_main(self): + def test_master_main(self) -> None: self.assert_affected_by([ 'libintegration-tests' + DYLIB_SUFFIX, 'yb-master' ], 'master_main.cc') - def test_tablet_server_main(self): + def test_tablet_server_main(self) -> None: self.assert_affected_by([ 'libintegration-tests' + DYLIB_SUFFIX, 'linked_list-test' @@ -1231,14 +1368,14 @@ def test_tablet_server_main(self): self.assert_unaffected_by(['yb-master'], 'tablet_server_main.cc') - def test_bulk_load_tool(self): + def test_bulk_load_tool(self) -> None: self.assert_affected_exactly_by([ 'yb-bulk_load', 'yb-bulk_load-test', 'yb-bulk_load.cc.o' ], 'yb-bulk_load.cc') - def test_flex_bison(self): + def test_flex_bison(self) -> None: self.assert_affected_by([ 'scanner_lex.l.cc' ], 'scanner_lex.l') @@ -1246,11 +1383,12 @@ def test_flex_bison(self): 'parser_gram.y.cc' ], 'parser_gram.y') - def test_proto_deps_validity(self): + def test_proto_deps_validity(self) -> None: + assert self.dep_graph is not None self.dep_graph.validate_proto_deps() -def run_self_test(dep_graph): +def run_self_test(dep_graph: DependencyGraph) -> None: logging.info("Running a self-test of the {} tool".format(os.path.basename(__file__))) DependencyGraphTest.dep_graph = dep_graph suite = unittest.TestLoader().loadTestsFromTestCase(DependencyGraphTest) @@ -1261,7 +1399,7 @@ def run_self_test(dep_graph): sys.exit(1) -def get_file_category(rel_path): +def get_file_category(rel_path: str) -> str: """ Categorize file changes so that we can decide what tests to run. @@ -1302,7 +1440,7 @@ def get_file_category(rel_path): return 'other' -def main(): +def main() -> None: parser = argparse.ArgumentParser( description='A tool for working with the dependency graph') parser.add_argument('--verbose', action='store_true', @@ -1312,8 +1450,9 @@ def main(): help='Rebuild the dependecy graph and save it to a file') parser.add_argument('--node-type', help='Node type to look for', - default='any', - choices=['test', 'object', 'library', 'source', 'any']) + type=NodeType, + choices=list(NodeType), + default=NodeType.ANY) parser.add_argument('--file-regex', help='Regular expression for file names to select as initial nodes for ' 'querying the dependency graph.') @@ -1406,6 +1545,7 @@ def main(): updated_categories = set() file_changes = [] + initial_nodes: Iterable[Node] if args.git_diff: old_working_dir = os.getcwd() with WorkDirContext(conf.yb_src_root): @@ -1450,9 +1590,9 @@ def main(): logging.info(" %s", change) updated_categories = set(file_changes_by_category.keys()) - results = set() + results: Set[Node] = set() if cmd == LIST_AFFECTED_CMD: - results = dep_graph.find_affected_nodes(initial_nodes, args.node_type) + results = dep_graph.find_affected_nodes(set(initial_nodes), args.node_type) elif cmd == LIST_DEPS_CMD: for node in initial_nodes: results.update(node.deps) @@ -1460,7 +1600,7 @@ def main(): for node in initial_nodes: results.update(node.reverse_deps) else: - raise RuntimeError("Unimplemented command '{}'".format(command)) + raise ValueError("Unimplemented command '{}'".format(cmd)) if args.output_test_config: test_basename_list = sorted( diff --git a/python/yb/llvm_urls.py b/python/yb/llvm_urls.py new file mode 100644 index 000000000000..f4c06f12f21f --- /dev/null +++ b/python/yb/llvm_urls.py @@ -0,0 +1,64 @@ +# Copyright (c) Yugabyte, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except +# in compliance with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed under the License +# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +# or implied. See the License for the specific language governing permissions and limitations +# under the License. + +from sys_detection import local_sys_conf + +from typing import Optional + +from yb.os_versions import is_compatible_os + + +def _llvm_url_for_tag(tag: str) -> str: + return 'https://github.com/yugabyte/build-clang/releases/download/%s/yb-llvm-%s.tar.gz' % ( + tag, tag) + + +COMPILER_TYPE_TO_ARCH_TO_OS_TYPE_TO_LLVM_URL = { + 'clang11': { + 'x86_64': { + 'centos7': _llvm_url_for_tag('v11.1.0-yb-1-1633099975-130bd22e-centos7-x86_64'), + 'almalinux8': _llvm_url_for_tag('v11.1.0-yb-1-1633143292-130bd22e-almalinux8-x86_64'), + }, + 'aarch64': { + 'centos8': _llvm_url_for_tag('v11.1.0-yb-1-1633544021-130bd22e-centos8-aarch64'), + }, + }, + 'clang12': { + 'x86_64': { + 'centos7': _llvm_url_for_tag('v12.0.1-yb-1-1633099823-bdb147e6-centos7-x86_64'), + 'almalinux8': _llvm_url_for_tag('v12.0.1-yb-1-1633143152-bdb147e6-almalinux8-x86_64'), + }, + } +} + + +def get_llvm_url(compiler_type: str) -> Optional[str]: + os_type_to_llvm_url = ( + COMPILER_TYPE_TO_ARCH_TO_OS_TYPE_TO_LLVM_URL.get(compiler_type) or {} + ).get(local_sys_conf().architecture) + if os_type_to_llvm_url is None: + return None + + os_type = local_sys_conf().short_os_name_and_version() + if os_type in os_type_to_llvm_url: + return os_type_to_llvm_url[os_type] + + candidate_urls = [ + os_type_to_llvm_url[os_type_key] + for os_type_key in os_type_to_llvm_url + if is_compatible_os(os_type_key, os_type) + ] + if len(candidate_urls) > 1: + raise ValueError("Ambiguous LLVM URLs: %s" % candidate_urls) + if not candidate_urls: + return None + return candidate_urls[0] diff --git a/python/yb/os_versions.py b/python/yb/os_versions.py new file mode 100644 index 000000000000..2c6746242e71 --- /dev/null +++ b/python/yb/os_versions.py @@ -0,0 +1,36 @@ +# Copyright (c) Yugabyte, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except +# in compliance with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed under the License +# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +# or implied. See the License for the specific language governing permissions and limitations +# under the License. + +import re + + +UBUNTU_OS_TYPE_RE = re.compile(r'^(ubuntu)([0-9]{2})([0-9]{2})$') +RHEL_FAMILY_RE = re.compile(r'^(almalinux|centos|rhel)([0-9]+)$') + + +def adjust_os_type(os_type: str) -> str: + match = UBUNTU_OS_TYPE_RE.match(os_type) + if match: + # Convert e.g. ubuntu2004 -> ubuntu20.04 for clarity. + return f'{match.group(1)}{match.group(2)}.{match.group(3)}' + return os_type + + +def is_compatible_os(archive_os: str, target_os: str) -> bool: + """ + Check if two combinations of OS name and version are compatible. + """ + rhel_like1 = RHEL_FAMILY_RE.match(archive_os) + rhel_like2 = RHEL_FAMILY_RE.match(target_os) + if rhel_like1 and rhel_like2 and rhel_like1.group(2) == rhel_like2.group(2): + return True + return archive_os == target_os diff --git a/python/yb/thirdparty_tool.py b/python/yb/thirdparty_tool.py index f40c6402f7b5..a3a3d8d8fd08 100644 --- a/python/yb/thirdparty_tool.py +++ b/python/yb/thirdparty_tool.py @@ -23,6 +23,8 @@ import os import logging import argparse +import ruamel.yaml + from autorepr import autorepr # type: ignore from github import Github @@ -32,11 +34,27 @@ from datetime import datetime from yb.common_util import ( - init_env, YB_SRC_ROOT, read_file, load_yaml_file, write_yaml_file, to_yaml_str) + init_env, + YB_SRC_ROOT, + read_file, + load_yaml_file, + write_yaml_file, + to_yaml_str, + arg_str_to_bool, + write_file, + make_parent_dir, + ) from sys_detection import local_sys_conf, SHORT_OS_NAME_REGEX_STR from collections import defaultdict +from yb.os_versions import adjust_os_type, is_compatible_os + +from yb.llvm_urls import get_llvm_url + + +ruamel_yaml_object = ruamel.yaml.YAML() + THIRDPARTY_ARCHIVES_REL_PATH = os.path.join('build-support', 'thirdparty_archives.yml') MANUAL_THIRDPARTY_ARCHIVES_REL_PATH = os.path.join( 'build-support', 'thirdparty_archives_manual.yml') @@ -63,18 +81,16 @@ def get_arch_regex(index: int) -> str: get_arch_regex(1), r'(?:-(?P(?:%s)[a-z0-9.]*))' % SHORT_OS_NAME_REGEX_STR, get_arch_regex(2), - r'(?:-(?Plinuxbrew))?', + r'(?:-(?Plinuxbrew))?', # "devtoolset" really means just "gcc" here. We should replace it with "gcc" in release names. r'(?:-(?P(?:gcc|clang|devtoolset-?)[a-z0-9.]+))?', + r'(?:-(?Plinuxbrew))?', r'$', ])) # We will store the SHA1 to be used for the local third-party checkout under this key. SHA_FOR_LOCAL_CHECKOUT_KEY = 'sha_for_local_checkout' -UBUNTU_OS_TYPE_RE = re.compile(r'^(ubuntu)([0-9]{2})([0-9]{2})$') -RHEL_FAMILY_RE = re.compile(r'^(almalinux|centos|rhel)([0-9]+)$') - # Skip these problematic tags. BROKEN_TAGS = set(['v20210907234210-47a70bc7dc-centos7-x86_64-linuxbrew-gcc5']) @@ -83,37 +99,33 @@ def get_archive_name_from_tag(tag: str) -> str: return f'yugabyte-db-thirdparty-{tag}.tar.gz' -def adjust_os_type(os_type: str) -> str: - match = UBUNTU_OS_TYPE_RE.match(os_type) - if match: - # Convert e.g. ubuntu2004 -> ubuntu20.04 for clarity. - return f'{match.group(1)}{match.group(2)}.{match.group(3)}' - return os_type - - -def compatible_os(archive_os: str, target_os: str) -> bool: - rhel_like1 = RHEL_FAMILY_RE.match(archive_os) - rhel_like2 = RHEL_FAMILY_RE.match(target_os) - if rhel_like1 and rhel_like2 and rhel_like1.group(2) == rhel_like2.group(2): - return True - return archive_os == target_os - - -class YBDependenciesRelease: - +class ThirdPartyReleaseBase: # The list of fields without the release tag. The tag is special because it includes the # timestamp, so by repeating a build on the same commit in yugabyte-db-thirdparty, we could get # multiple releases that have the same OS/architecture/compiler type/SHA but different tags. # Therefore we distinguish between "key with tag" and "key with no tag" - KEY_FIELDS_NO_TAG = ['os_type', 'architecture', 'compiler_type', 'sha'] + KEY_FIELDS_NO_TAG = ['os_type', 'architecture', 'compiler_type', 'is_linuxbrew', 'sha'] KEY_FIELDS_WITH_TAG = KEY_FIELDS_NO_TAG + ['tag'] os_type: str architecture: str compiler_type: str + is_linuxbrew: bool sha: str tag: str + __str__ = __repr__ = autorepr(KEY_FIELDS_WITH_TAG) + + def as_dict(self) -> Dict[str, str]: + return {k: getattr(self, k) for k in self.KEY_FIELDS_WITH_TAG} + + def get_sort_key(self, include_tag: bool = True) -> Tuple[str, ...]: + return tuple( + getattr(self, k) for k in + (self.KEY_FIELDS_WITH_TAG if include_tag else self.KEY_FIELDS_NO_TAG)) + + +class GitHubThirdPartyRelease(ThirdPartyReleaseBase): github_release: GitRelease timestamp: str @@ -145,7 +157,8 @@ def __init__(self, github_release: GitRelease) -> None: if arch1 is not None and arch2 is not None and arch1 != arch2: raise ValueError("Contradicting values of arhitecture in tag '%s'" % tag) self.architecture = arch1 or arch2 - self.is_linuxbrew = bool(group_dict.get('is_linuxbrew')) + self.is_linuxbrew = (bool(group_dict.get('is_linuxbrew1')) or + bool(group_dict.get('is_linuxbrew2'))) compiler_type = group_dict.get('compiler_type') if compiler_type is None and self.os_type == 'macos': @@ -198,24 +211,29 @@ def validate_url(self) -> bool: return True - def as_dict(self) -> Dict[str, str]: - return {k: getattr(self, k) for k in self.KEY_FIELDS_WITH_TAG} - - def get_sort_key(self, include_tag: bool = True) -> Tuple[str, ...]: - return tuple( - getattr(self, k) for k in - (self.KEY_FIELDS_WITH_TAG if include_tag else self.KEY_FIELDS_NO_TAG)) - def is_consistent_with_yb_version(self, yb_version: str) -> bool: return (self.branch_name is None or yb_version.startswith((self.branch_name + '.', self.branch_name + '-'))) - __str__ = __repr__ = autorepr(KEY_FIELDS_WITH_TAG) + +@ruamel_yaml_object.register_class +class MetadataItem(ThirdPartyReleaseBase): + """ + A metadata item for a third-party download archive loaded from the thirdparty_archives.yml + file. + """ + + def __init__(self, json_data: Dict[str, Any]) -> None: + for field_name in GitHubThirdPartyRelease.KEY_FIELDS_WITH_TAG: + setattr(self, field_name, json_data[field_name]) + + def url(self) -> str: + return f'{DOWNLOAD_URL_PREFIX}{self.tag}/{get_archive_name_from_tag(self.tag)}' class ReleaseGroup: sha: str - releases: List[YBDependenciesRelease] + releases: List[GitHubThirdPartyRelease] creation_timestamps: List[datetime] def __init__(self, sha: str) -> None: @@ -223,7 +241,7 @@ def __init__(self, sha: str) -> None: self.releases = [] self.creation_timestamps = [] - def add_release(self, release: YBDependenciesRelease) -> None: + def add_release(self, release: GitHubThirdPartyRelease) -> None: if release.sha != self.sha: raise ValueError( f"Adding a release with wrong SHA. Expected: {self.sha}, got: " @@ -258,9 +276,14 @@ def parse_args() -> argparse.Namespace: help='Show the Git SHA1 of the commit to use in the yugabyte-db-thirdparty repo ' 'in case we are building the third-party dependencies from scratch.') parser.add_argument( - '--save-download-url-to-file', + '--save-thirdparty-url-to-file', help='Determine the third-party archive download URL for the combination of criteria, ' 'including the compiler type, and write it to the file specified by this argument.') + parser.add_argument( + '--save-llvm-url-to-file', + help='Determine the LLVM toolchain archive download URL and write it to the file ' + 'specified by this argument. Similar to --save-download-url-to-file but also ' + 'takes the OS into account.') parser.add_argument( '--compiler-type', help='Compiler type, to help us decide which third-party archive to choose. ' @@ -274,6 +297,11 @@ def parse_args() -> argparse.Namespace: '--architecture', help='Machine architecture, to help us decide which third-party archive to choose. ' 'The default value is determined automatically based on the current platform.') + parser.add_argument( + '--is-linuxbrew', + help='Whether the archive shget_download_urlould be based on Linuxbrew.', + type=arg_str_to_bool, + default=None) parser.add_argument( '--verbose', help='Verbose debug information') @@ -350,7 +378,7 @@ def update_archive_metadata_file(self) -> None: logging.info(f'Skipping tag {tag_name}, does not match the filter') continue - yb_dep_release = YBDependenciesRelease(release) + yb_dep_release = GitHubThirdPartyRelease(release) if not yb_dep_release.is_consistent_with_yb_version(yb_version): logging.debug( f"Skipping release tag: {tag_name} (does not match version {yb_version}") @@ -395,7 +423,7 @@ def update_archive_metadata_file(self) -> None: if rel.tag not in BROKEN_TAGS ] - releases_by_key_without_tag: DefaultDict[Tuple[str, ...], List[YBDependenciesRelease]] = \ + releases_by_key_without_tag: DefaultDict[Tuple[str, ...], List[GitHubThirdPartyRelease]] = \ defaultdict(list) num_valid_releases = 0 @@ -426,7 +454,7 @@ def update_archive_metadata_file(self) -> None: else: filtered_releases_for_one_commit.append(releases_for_key[0]) - filtered_releases_for_one_commit.sort(key=YBDependenciesRelease.get_sort_key) + filtered_releases_for_one_commit.sort(key=GitHubThirdPartyRelease.get_sort_key) for yb_thirdparty_release in filtered_releases_for_one_commit: new_metadata['archives'].append(yb_thirdparty_release.as_dict()) @@ -445,20 +473,20 @@ def load_manual_metadata() -> Dict[str, Any]: return load_yaml_file(get_manual_archive_metadata_file_path()) -def filter_for_os(archive_candidates: List[Dict[str, str]], os_type: str) -> List[Dict[str, str]]: +def filter_for_os(archive_candidates: List[MetadataItem], os_type: str) -> List[MetadataItem]: filtered_exactly = [ - candidate for candidate in archive_candidates if candidate['os_type'] == os_type + candidate for candidate in archive_candidates if candidate.os_type == os_type ] if filtered_exactly: return filtered_exactly return [ candidate for candidate in archive_candidates - if compatible_os(candidate['os_type'], os_type) + if is_compatible_os(candidate.os_type, os_type) ] def get_compilers( - metadata: Dict[str, Any], + metadata_items: List[MetadataItem], os_type: Optional[str], architecture: Optional[str]) -> list: if not os_type: @@ -466,64 +494,79 @@ def get_compilers( if not architecture: architecture = local_sys_conf().architecture - candidates = [i for i in metadata['archives'] if i['architecture'] == architecture] + candidates: List[MetadataItem] = [ + metadata_item + for metadata_item in metadata_items + if metadata_item.architecture == architecture + ] candidates = filter_for_os(candidates, os_type) - - compilers = [x['compiler_type'] for x in candidates] + compilers = sorted(set([metadata_item.compiler_type for metadata_item in candidates])) return compilers -def get_download_url( - metadata: Dict[str, Any], +def get_third_party_release( + available_archives: List[MetadataItem], compiler_type: str, os_type: Optional[str], - architecture: Optional[str]) -> str: + architecture: Optional[str], + is_linuxbrew: Optional[bool]) -> MetadataItem: if not os_type: os_type = local_sys_conf().short_os_name_and_version() if not architecture: architecture = local_sys_conf().architecture candidates: List[Any] = [] - available_archives = metadata['archives'] - for archive in available_archives: - if archive['compiler_type'] == compiler_type and archive['architecture'] == architecture: - candidates.append(archive) - candidates = filter_for_os(candidates, os_type) + candidates = [ + archive for archive in available_archives + if archive.compiler_type == compiler_type and archive.architecture == architecture + ] + + if is_linuxbrew is not None: + candidates = [ + candidate for candidate in candidates + if candidate.is_linuxbrew == is_linuxbrew + ] + + if is_linuxbrew is None or not is_linuxbrew or len(candidates) > 1: + # If a Linuxbrew archive is requested, we don't have to filter by OS, because archives + # should be OS-independent. But still do that if we have more than one candidate. + candidates = filter_for_os(candidates, os_type) if len(candidates) == 1: - tag = candidates[0]['tag'] - return f'{DOWNLOAD_URL_PREFIX}{tag}/{get_archive_name_from_tag(tag)}' + return candidates[0] if not candidates: - if os_type == 'centos7' and compiler_type == 'gcc': - logging.info( - "Assuming that the compiler type of 'gcc' means 'gcc5'. " - "This will change when we stop using Linuxbrew and update the compiler.") - return get_download_url(metadata, 'gcc5', os_type, architecture) - if os_type == 'ubuntu18.04' and compiler_type == 'gcc': - logging.info( - "Assuming that the compiler type of 'gcc' means 'gcc7'. " - "This will change when we stop using Linuxbrew and update the compiler.") - return get_download_url(metadata, 'gcc7', os_type, architecture) + if compiler_type == 'gcc': + if os_type == 'ubuntu18.04': + logging.info( + "Assuming that the compiler type of 'gcc' means 'gcc7' on Ubuntu 18.04.") + return get_third_party_release( + available_archives, 'gcc7', os_type, architecture, is_linuxbrew) + logging.info( + "Assuming that the compiler type of 'gcc' means 'gcc5' with Linuxbrew." + "This will not be needed when we phase out our use of GCC 5.") + if is_linuxbrew is None: + is_linuxbrew = True + return get_third_party_release( + available_archives, 'gcc5', os_type, architecture, is_linuxbrew) + + if candidates: + i = 1 + for candidate in candidates: + logging.warning("Third-party release archive candidate #%d: %s", i, candidate) + i += 1 + wrong_count_str = 'more than one' + else: logging.info(f"Available release archives:\n{to_yaml_str(available_archives)}") - raise ValueError( - "Could not find a third-party release archive to download for " - f"OS type '{os_type}', " - f"compiler type '{compiler_type}', and " - f"architecture '{architecture}'. " - "Please see the list of available thirdparty archives above.") - - i = 1 - for candidate in candidates: - logging.warning("Third-party release archive candidate #%d: %s", i, candidate) - i += 1 + wrong_count_str = 'no' raise ValueError( - f"Found too many third-party release archives to download for OS type " - f"{os_type} and compiler type matching {compiler_type}: {candidates}.") + f"Found {wrong_count_str} third-party release archives to download for OS type " + f"{os_type}, compiler type matching {compiler_type}, architecture {architecture}, " + f"is_linuxbrew={is_linuxbrew}. See more details above.") def main() -> None: @@ -542,32 +585,41 @@ def main() -> None: print(metadata[SHA_FOR_LOCAL_CHECKOUT_KEY]) return - metadata['archives'].extend(manual_metadata['archives']) + metadata_items = [ + MetadataItem(item_json_data) + for item_json_data in metadata['archives'] + manual_metadata['archives'] + ] if args.list_compilers: compiler_list = get_compilers( - metadata=metadata, + metadata_items=metadata_items, os_type=args.os_type, architecture=args.architecture) for compiler in compiler_list: print(compiler) return - if args.save_download_url_to_file: + if args.save_thirdparty_url_to_file or args.save_llvm_url_to_file: if not args.compiler_type: raise ValueError("Compiler type not specified") - url = get_download_url( - metadata=metadata, + thirdparty_release: Optional[MetadataItem] = get_third_party_release( + available_archives=metadata_items, compiler_type=args.compiler_type, os_type=args.os_type, - architecture=args.architecture) - if url is None: - raise RuntimeError("Could not determine download URL") - logging.info(f"Download URL for the third-party dependencies: {url}") - output_file_dir = os.path.dirname(os.path.abspath(args.save_download_url_to_file)) - os.makedirs(output_file_dir, exist_ok=True) - with open(args.save_download_url_to_file, 'w') as output_file: - output_file.write(url) + architecture=args.architecture, + is_linuxbrew=args.is_linuxbrew) + if thirdparty_release is None: + raise RuntimeError("Could not determine third-party archive download URL") + thirdparty_url = thirdparty_release.url() + logging.info(f"Download URL for the third-party dependencies: {thirdparty_url}") + if args.save_thirdparty_url_to_file: + make_parent_dir(args.save_thirdparty_url_to_file) + write_file(thirdparty_url, args.save_thirdparty_url_to_file) + if args.save_llvm_url_to_file and thirdparty_release.is_linuxbrew: + llvm_url = get_llvm_url(thirdparty_release.compiler_type) + if llvm_url is not None: + make_parent_dir(args.save_llvm_url_to_file) + write_file(llvm_url, args.save_llvm_url_to_file) if __name__ == '__main__': diff --git a/src/postgres/CMakeLists.txt b/src/postgres/CMakeLists.txt index 012c7679d288..4983343020b3 100644 --- a/src/postgres/CMakeLists.txt +++ b/src/postgres/CMakeLists.txt @@ -17,7 +17,7 @@ get_property(yb_cmake_include_dirs DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} foreach(include_dir ${yb_cmake_include_dirs}) set(POSTGRES_EXTRA_C_CXX_FLAGS "${POSTGRES_EXTRA_C_CXX_FLAGS} -I${include_dir}") endforeach(include_dir) -if("${COMPILER_FAMILY}" STREQUAL "gcc" AND "${COMPILER_VERSION}" VERSION_GREATER "8.0") +if(IS_GCC AND "${COMPILER_VERSION}" VERSION_GREATER "8.0") string(CONCAT POSTGRES_EXTRA_C_CXX_FLAGS "${POSTGRES_EXTRA_C_CXX_FLAGS}" " -Wno-format-truncation" " -Wno-maybe-uninitialized" @@ -32,7 +32,7 @@ endif() set(POSTGRES_EXTRA_LD_FLAGS "") if(NOT APPLE AND "${YB_BUILD_TYPE}" STREQUAL "asan" AND - "${COMPILER_FAMILY}" STREQUAL "clang" AND + IS_CLANG AND "${COMPILER_VERSION}" VERSION_GREATER_EQUAL "10.0.0") set(POSTGRES_EXTRA_LD_FLAGS "${POSTGRES_EXTRA_LD_FLAGS} -ldl") endif() diff --git a/src/postgres/src/backend/tcop/postgres.c b/src/postgres/src/backend/tcop/postgres.c index 16abc3b95f95..7e30ef770593 100644 --- a/src/postgres/src/backend/tcop/postgres.c +++ b/src/postgres/src/backend/tcop/postgres.c @@ -1,4 +1,4 @@ -/*------------------------------------------------------------------------- + /*------------------------------------------------------------------------- * * postgres.c * POSTGRES C Backend Interface diff --git a/yb_build.sh b/yb_build.sh index e58688728f59..6aee689df56d 100755 --- a/yb_build.sh +++ b/yb_build.sh @@ -207,6 +207,10 @@ Options: --arch Build for the given architecture. Currently only relevant for Apple Silicon where we can build for x86_64 and arm64 (no cross-compilation support yet). + --linuxbrew, --no-linuxbrew + Specify in order to do a Linuxbrew based build, or specifically prohibit doing so. This + influences the choice of prebuilt third-party archive. This can also be specified using the + YB_USE_LINUXBREW environment variable. -- Pass all arguments after -- to repeat_unit_test. @@ -1087,8 +1091,13 @@ while [[ $# -gt 0 ]]; do export YB_TARGET_ARCH=$2 shift ;; + --linuxbrew) + export YB_USE_LINUXBREW=1 + ;; + --no-linuxbrew) + export YB_USE_LINUXBREW=0 + ;; *) - if [[ $1 =~ ^(YB_[A-Z0-9_]+|postgres_FLAGS_[a-zA-Z0-9_]+)=(.*)$ ]]; then env_var_name=${BASH_REMATCH[1]} # Use "the ultimate fix" from http://bit.ly/setenvvar to set a variable with the name stored