Merge branch 'master' into add-snapshot-time

yugabyte · Jan 1, 2022 · 58cafd4 · 58cafd4
2 parents c814231 + fbe661a
commit 58cafd4
Show file tree

Hide file tree

Showing 987 changed files with 24,755 additions and 10,202 deletions.
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -54,6 +54,7 @@ if("$ENV{YB_USE_PCH}" STREQUAL "1")
   endif()
 endif()
 include(YugabyteFunctions)
+yb_initialize_constants()
 include(YugabyteTesting)
 
 ADD_CXX_FLAGS("-Werror")
@@ -164,8 +165,6 @@ CHECK_YB_COMPILER_PATH(${CMAKE_C_COMPILER})
 message("CMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}")
 CHECK_YB_COMPILER_PATH(${CMAKE_CXX_COMPILER})
 
-set(BUILD_SUPPORT_DIR ${CMAKE_CURRENT_SOURCE_DIR}/build-support)
-
 # Provide a 'latest' symlink to this build directory if the "blessed" multi-build layout is
 # detected:
 #
@@ -227,36 +226,17 @@ endif()
 
 INIT_COMPILER_TYPE_FROM_BUILD_ROOT()
 
-# TODO: this is complicated by the fact that we are trying to build third-party dependencies here if
-# necessary, and that the compiler is sometimes part of those third-party dependencies (e.g. Clang 7
-# with the Linuxbrew-based setup). Therefore, we don't yet know the exact compiler version. If we
-# know that we are not using a third-party setup where we build our own Clang, or if we are not
-# planning to invoke the third-party dependency build from here, we include the CompilerInfo module
-# early. We also always do that on macOS where we only use Clang.
-
-set(INCLUDE_COMPILER_INFO_EARLY FALSE)
-if(NOT "${YB_COMPILER_TYPE}" STREQUAL "clang" OR
-   NOT REBUILD_THIRDPARTY OR
-   APPLE)
-  set(INCLUDE_COMPILER_INFO_EARLY TRUE)
-endif()
-if(INCLUDE_COMPILER_INFO_EARLY)
-  message("Including the CompilerInfo module before determining the third-party instrumentation "
-          "type to use. Criteria: "
-          "REBUILD_THIRDPARTY=${REBUILD_THIRDPARTY}, "
-          "YB_COMPILER_TYPE=${YB_COMPILER_TYPE}.")
-  include(CompilerInfo)
-endif()
+include(CompilerInfo)
 
 # This helps find the right third-party build directory.
 if ("${YB_BUILD_TYPE}" MATCHES "^(asan|tsan)$")
   set(THIRDPARTY_INSTRUMENTATION_TYPE "${YB_BUILD_TYPE}")
-elseif ("${COMPILER_FAMILY}" STREQUAL "clang")
+elseif (IS_CLANG)
   set(THIRDPARTY_INSTRUMENTATION_TYPE "uninstrumented")
-elseif ("${COMPILER_FAMILY}" STREQUAL "gcc")
+elseif (IS_GCC)
   set(THIRDPARTY_INSTRUMENTATION_TYPE "uninstrumented")
 else()
-  message(FATAL_ERROR "Unknown or still undetermined compiler family: '${COMPILER_FAMILY}'.")
+  message(FATAL_ERROR "Unknown compiler family: '${COMPILER_FAMILY}'.")
 endif()
 message("THIRDPARTY_INSTRUMENTATION_TYPE=${THIRDPARTY_INSTRUMENTATION_TYPE}")
 
@@ -318,7 +298,6 @@ DETECT_BREW()
 
 message("Using COMPILER_FAMILY=${COMPILER_FAMILY}")
 
-# We can only use IS_CLANG after calling VALIDATE_COMPILER_TYPE.
 if (NOT APPLE AND "${CMAKE_SYSTEM_PROCESSOR}" STREQUAL "x86_64")
   # To enable 16-byte atomics support we should specify appropriate architecture.
   ADD_CXX_FLAGS("-march=ivybridge")
@@ -520,7 +499,7 @@ if (IS_CLANG)
   if ("${COMPILER_VERSION}" VERSION_GREATER_EQUAL "12.0.0" AND APPLE)
     ADD_CXX_FLAGS("-Wno-c++17-compat-mangling")
   endif()
-elseif("${COMPILER_FAMILY}" STREQUAL "gcc")
+elseif(IS_GCC)
   if ("${CMAKE_SYSTEM_PROCESSOR}" STREQUAL "x86_64")
     ADD_CXX_FLAGS("-mno-abm -mno-movbe")
   endif()
@@ -530,15 +509,15 @@ endif()
 
 set(YB_PREFIX_COMMON "${YB_THIRDPARTY_DIR}/installed/common")
 
-# Flag to enable thread sanitizer (clang or gcc 4.8)
-if ("${YB_BUILD_TYPE}" MATCHES "^(asan|tsan)$")
-  YB_SETUP_SANITIZER()
-elseif (NOT APPLE AND IS_CLANG)
+if(NOT APPLE AND IS_CLANG)
   YB_SETUP_CLANG()
+  if ("${YB_BUILD_TYPE}" MATCHES "^(asan|tsan)$")
+    YB_SETUP_SANITIZER()
+  endif()
 endif()
 
 if (USING_LINUXBREW)
-  include_directories(SYSTEM ${LINUXBREW_DIR}/include)
+  include_directories(SYSTEM "${LINUXBREW_DIR}/include")
 endif()
 
 if (NOT IS_CLANG)
@@ -551,8 +530,8 @@ endif()
 # - disable 'alignment' because unaligned access is really OK on Nehalem and we do it
 #   all over the place.
 if ("${YB_BUILD_TYPE}" STREQUAL "asan")
-  if(NOT ("${COMPILER_FAMILY}" STREQUAL "clang" OR
-          "${COMPILER_FAMILY}" STREQUAL "gcc" AND "${COMPILER_VERSION}" VERSION_GREATER "4.9"))
+  if(NOT (IS_CLANG OR
+          IS_GCC AND "${COMPILER_VERSION}" VERSION_GREATER "4.9"))
     message(SEND_ERROR "Cannot use UBSAN without clang or gcc >= 4.9")
   endif()
   set(CXX_NO_SANITIZE_FLAG "alignment")
@@ -567,7 +546,7 @@ endif ()
 if ("${YB_BUILD_TYPE}" MATCHES "^(asan|tsan)$")
   # GCC 4.8 and 4.9 (latest as of this writing) don't allow you to specify a
   # sanitizer blacklist.
-  if("${COMPILER_FAMILY}" STREQUAL "clang")
+  if(IS_CLANG)
     # Require clang 3.4 or newer; clang 3.3 has issues with TSAN and pthread
     # symbol interception.
     if("${COMPILER_VERSION}" VERSION_LESS "3.4")
@@ -605,6 +584,11 @@ include_directories(src)
 
 enable_testing()
 
+if (USING_LINUXBREW)
+  ADD_GLOBAL_RPATH_ENTRY_AND_LIB_DIR("${YB_BUILD_ROOT}/postgres/lib")
+  ADD_GLOBAL_RPATH_ENTRY_AND_LIB_DIR("${LINUXBREW_LIB_DIR}")
+endif()
+
 ############################################################
 # Dependencies
 ############################################################
@@ -710,7 +694,7 @@ set(YB_CMAKE_CXX_EXTRA_FLAGS "-Wextra -Wno-unused-parameter")
 set(CMAKE_C_FLAGS "${CMAKE_CXX_FLAGS}")
 
 # C++-only flags.
-if (("${COMPILER_FAMILY}" STREQUAL "gcc") AND
+if ((IS_GCC) AND
     ("${CMAKE_CXX_COMPILER_VERSION}" VERSION_GREATER_EQUAL "7.0"))
   # Used with GCC 7.
   ADD_CXX_FLAGS("-faligned-new")
@@ -726,24 +710,14 @@ message("CMAKE_C_FLAGS ${CMAKE_C_FLAGS}")
 
 set(CMAKE_CXX_STANDARD 14)
 
-if ("${COMPILER_FAMILY}" STREQUAL "gcc" AND
+if (IS_GCC AND
     "${COMPILER_VERSION}" VERSION_GREATER_EQUAL "8.0.0")
   # Remove after switching to C++17
   ADD_CXX_FLAGS("-Wno-aligned-new")
 endif()
 
 set(CMAKE_CXX_STANDARD_REQUIRED ON)
 
-if (USING_LINUXBREW)
-  # We need to add the postgres library directory prior to /usr/lib64 specifically in order to avoid
-  # using the system libpq.
-  ADD_GLOBAL_RPATH_ENTRY_AND_LIB_DIR("${YB_BUILD_ROOT}/postgres/lib")
-
-  # Add the Linuxbrew library directory and the system library directory last.
-  ADD_GLOBAL_RPATH_ENTRY_AND_LIB_DIR("${LINUXBREW_LIB_DIR}")
-  ADD_GLOBAL_RPATH_ENTRY_AND_LIB_DIR("/usr/lib64")
-endif()
-
 message("Linker flags for executables: ${CMAKE_EXE_LINKER_FLAGS}")
 
 # Define full paths to libpq and libyb_pgbackend shared libraries. These libraries are built as part

diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md
@@ -144,3 +144,4 @@ This is a list of people who have contributed code to the [YugabyteDB](https://g
 * [rthallamko3](https://github.com/rthallamko3)
 * [SteveSong](https://github.com/SteveXiSong)
 * [viko600](https://github.com/viko600)
+* [lnguyen](https://github.com/lnguyen-yugabyte)
diff --git a/build-support/common-build-env-test.sh b/build-support/common-build-env-test.sh
@@ -19,6 +19,7 @@
 #
 set -euo pipefail
 
+# shellcheck source=build-support/common-build-env.sh
 . "${0%/*}/common-build-env.sh"
 
 assert_equals() {
@@ -90,7 +91,7 @@ test_compiler_detection_by_jenkins_job_name() {
     unset YB_COMPILER_TYPE
     JOB_NAME="$jenkins_job_name"
     set_compiler_type_based_on_jenkins_job_name
-    assert_equals "$expected_compiler_type" "$YB_COMPILER_TYPE"
+    assert_equals "$expected_compiler_type" "$YB_COMPILER_TYPE" "compiler type"
   )
 }
 
@@ -138,53 +139,70 @@ test_set_cmake_build_type_and_compiler_type() {
     OSTYPE=$os_type
     yb_fatal_quiet=true
     set_cmake_build_type_and_compiler_type
-    assert_equals "$expected_cmake_build_type" "$cmake_build_type" "$test_case_details"
-    assert_equals "$expected_compiler_type" "$YB_COMPILER_TYPE" "$test_case_details"
+    assert_equals "$expected_cmake_build_type" "$cmake_build_type" "$test_case_details" \
+                  "Note: comparing CMake build type."
+    assert_equals "$expected_compiler_type" "$YB_COMPILER_TYPE" "$test_case_details" \
+                  "Note: comparing compiler type."
   )
   local exit_code=$?
   set -e
   assert_equals "$expected_exit_code" "$exit_code"
 }
 
+arch=$( uname -m )
+
 # Parameters:                               build_type OSTYPE    Compiler   Expected   Expected
 #                                                                type       build_type YB_COMPILER_
 #                                                                preference            TYPE
 
-test_set_cmake_build_type_and_compiler_type asan       darwin    auto       fastdebug  clang  0
-test_set_cmake_build_type_and_compiler_type asan       darwin    clang      fastdebug  clang  0
-test_set_cmake_build_type_and_compiler_type asan       darwin    gcc        N/A        N/A    1
-test_set_cmake_build_type_and_compiler_type asan       linux-gnu auto       fastdebug  clang7 0
-test_set_cmake_build_type_and_compiler_type asan       linux-gnu clang7     fastdebug  clang7 0
-test_set_cmake_build_type_and_compiler_type asan       linux-gnu gcc        N/A        N/A    1
-test_set_cmake_build_type_and_compiler_type asan       linux-gnu gcc8       fastdebug  gcc8   0
-test_set_cmake_build_type_and_compiler_type asan       linux-gnu gcc9       fastdebug  gcc9   0
-test_set_cmake_build_type_and_compiler_type tsan       linux-gnu auto       fastdebug  clang7 0
-test_set_cmake_build_type_and_compiler_type tsan       linux-gnu clang7     fastdebug  clang7 0
-test_set_cmake_build_type_and_compiler_type tsan       linux-gnu gcc        N/A        N/A    1
-test_set_cmake_build_type_and_compiler_type tsan       linux-gnu gcc8       fastdebug  gcc8   0
-test_set_cmake_build_type_and_compiler_type tsan       linux-gnu gcc9       fastdebug  gcc9   0
-test_set_cmake_build_type_and_compiler_type debug      darwin    auto       debug      clang  0
-test_set_cmake_build_type_and_compiler_type debug      darwin    clang      debug      clang  0
-test_set_cmake_build_type_and_compiler_type debug      darwin    gcc        N/A        N/A    1
-test_set_cmake_build_type_and_compiler_type debug      linux-gnu auto       debug      gcc    0
-test_set_cmake_build_type_and_compiler_type debug      linux-gnu clang      debug      clang  0
-test_set_cmake_build_type_and_compiler_type debug      linux-gnu gcc        debug      gcc    0
-test_set_cmake_build_type_and_compiler_type debug      linux-gnu gcc8       debug      gcc8   0
-test_set_cmake_build_type_and_compiler_type debug      linux-gnu gcc9       debug      gcc9   0
-test_set_cmake_build_type_and_compiler_type FaStDeBuG  darwin    auto       fastdebug  clang  0
-test_set_cmake_build_type_and_compiler_type FaStDeBuG  darwin    clang      fastdebug  clang  0
-test_set_cmake_build_type_and_compiler_type FaStDeBuG  darwin    gcc        N/A        N/A    1
-test_set_cmake_build_type_and_compiler_type FaStDeBuG  linux-gnu auto       fastdebug  gcc    0
-test_set_cmake_build_type_and_compiler_type FaStDeBuG  linux-gnu clang      fastdebug  clang  0
-test_set_cmake_build_type_and_compiler_type FaStDeBuG  linux-gnu gcc        fastdebug  gcc    0
-test_set_cmake_build_type_and_compiler_type release    darwin    auto       release    clang  0
-test_set_cmake_build_type_and_compiler_type release    darwin    clang      release    clang  0
-test_set_cmake_build_type_and_compiler_type release    darwin    gcc        N/A        N/A    1
-test_set_cmake_build_type_and_compiler_type release    linux-gnu auto       release    gcc    0
-test_set_cmake_build_type_and_compiler_type release    linux-gnu clang      release    clang  0
-test_set_cmake_build_type_and_compiler_type release    linux-gnu gcc        release    gcc    0
-test_set_cmake_build_type_and_compiler_type release    linux-gnu gcc8       release    gcc8   0
-test_set_cmake_build_type_and_compiler_type release    linux-gnu gcc9       release    gcc9   0
+# The last parameter is expected exit code (0 or 1).
+
+test_set_cmake_build_type_and_compiler_type   asan       darwin    auto       fastdebug  clang   0
+test_set_cmake_build_type_and_compiler_type   asan       darwin    clang      fastdebug  clang   0
+test_set_cmake_build_type_and_compiler_type   asan       darwin    gcc        N/A        N/A     1
+test_set_cmake_build_type_and_compiler_type   asan       linux-gnu clang7     fastdebug  clang7  0
+test_set_cmake_build_type_and_compiler_type   asan       linux-gnu gcc        N/A        N/A     1
+test_set_cmake_build_type_and_compiler_type   asan       linux-gnu gcc8       N/A        gcc8    1
+test_set_cmake_build_type_and_compiler_type   asan       linux-gnu gcc9       N/A        gcc9    1
+test_set_cmake_build_type_and_compiler_type   tsan       linux-gnu auto       fastdebug  clang7  0
+test_set_cmake_build_type_and_compiler_type   tsan       linux-gnu clang7     fastdebug  clang7  0
+test_set_cmake_build_type_and_compiler_type   tsan       linux-gnu gcc        N/A        N/A     1
+test_set_cmake_build_type_and_compiler_type   tsan       linux-gnu gcc8       N/A        gcc8    1
+test_set_cmake_build_type_and_compiler_type   tsan       linux-gnu gcc9       N/A        gcc9    1
+test_set_cmake_build_type_and_compiler_type   debug      darwin    auto       debug      clang   0
+test_set_cmake_build_type_and_compiler_type   debug      darwin    clang      debug      clang   0
+test_set_cmake_build_type_and_compiler_type   debug      darwin    gcc        N/A        N/A     1
+test_set_cmake_build_type_and_compiler_type   debug      linux-gnu clang      debug      clang   0
+test_set_cmake_build_type_and_compiler_type   debug      linux-gnu gcc        debug      gcc     0
+test_set_cmake_build_type_and_compiler_type   debug      linux-gnu gcc8       debug      gcc8    0
+test_set_cmake_build_type_and_compiler_type   debug      linux-gnu gcc9       debug      gcc9    0
+test_set_cmake_build_type_and_compiler_type   FaStDeBuG  darwin    auto       fastdebug  clang   0
+test_set_cmake_build_type_and_compiler_type   FaStDeBuG  darwin    clang      fastdebug  clang   0
+test_set_cmake_build_type_and_compiler_type   FaStDeBuG  darwin    gcc        N/A        N/A     1
+test_set_cmake_build_type_and_compiler_type   FaStDeBuG  linux-gnu clang      fastdebug  clang   0
+test_set_cmake_build_type_and_compiler_type   FaStDeBuG  linux-gnu gcc        fastdebug  gcc     0
+test_set_cmake_build_type_and_compiler_type   release    darwin    auto       release    clang   0
+test_set_cmake_build_type_and_compiler_type   release    darwin    clang      release    clang   0
+test_set_cmake_build_type_and_compiler_type   release    darwin    gcc        N/A        N/A     1
+test_set_cmake_build_type_and_compiler_type   release    linux-gnu clang      release    clang   0
+test_set_cmake_build_type_and_compiler_type   release    linux-gnu gcc        release    gcc     0
+test_set_cmake_build_type_and_compiler_type   release    linux-gnu gcc8       release    gcc8    0
+test_set_cmake_build_type_and_compiler_type   release    linux-gnu gcc9       release    gcc9    0
+
+# TODO: use Clang 12 uniformly for all achitectures, and get rid of conditional checks here.
+if [[ $arch == "x86_64" && $OSTYPE == linux* ]] && is_redhat_family; then
+  test_set_cmake_build_type_and_compiler_type asan       linux-gnu auto       fastdebug  clang12 0
+  test_set_cmake_build_type_and_compiler_type debug      linux-gnu auto       debug      clang12 0
+  test_set_cmake_build_type_and_compiler_type FaStDeBuG  linux-gnu auto       fastdebug  clang12 0
+  test_set_cmake_build_type_and_compiler_type release    linux-gnu auto       release    clang12 0
+fi
+
+if [[ $arch == "aarch64" && $OSTYPE == linux* ]]; then
+  test_set_cmake_build_type_and_compiler_type asan       linux-gnu auto       fastdebug  clang11 0
+  test_set_cmake_build_type_and_compiler_type debug      linux-gnu auto       debug      clang11 0
+  test_set_cmake_build_type_and_compiler_type FaStDeBuG  linux-gnu auto       fastdebug  clang11 0
+  test_set_cmake_build_type_and_compiler_type release    linux-gnu auto       release    clang11 0
+fi
 
 # -------------------------------------------------------------------------------------------------