[yugabyte#17268] Consolidate type-checked Python-based build scripts …

…into the python/yugabyte directory

Summary:
Consolidate Python-based build scripts from python/yb and build-support into the python/yugabyte directory. This simplifies the codecheck.ini file: we don't have to list all the files where we need to check types, we just put it in that directory. As part of this, adding type annotations and fixing a few related errors in some of the scripts that were not previously type-checked.

Also, when these Python scripts are invoked from Bash scripts, we are now using intermediate variables named YB_SCRIPT_PATH_..., which are set in common-build-env.sh and the existence of the corresponding paths is checked in common-build-env-test.sh.

Test Plan: Jenkins

Reviewers: steve.varnau, hsunder

Reviewed By: steve.varnau, hsunder

Subscribers: jenkins-bot, ybase

Differential Revision: https://phabricator.dev.yugabyte.com/D25188

.arclint

@@ -15,13 +15,14 @@
      "(^managed/src/main/resources/swagger-strict[.]json$)",
      "(^managed/build[.]sbt$)",
      "(^managed/src/main/resources/aws_pricing/.*$)",
-     "(^python/yb/test_data/.*[.](log|out)$)",
+     "(^python/yugabyte/test_data/.*[.](log|out)$)",
      "(^managed/src/main/resources/metric/Dashboard[.]json$)",
      "(^managed/src/main/resources/metric/recording_rules[.]yml$)",
      "(^managed/devops/replicated[.]yml$)",
      "(^python/yb/py[.]typed$)",
      "(^managed/RUNTIME-FLAGS[.]md$)",
-     "(^[.]clang-tidy)"
+     "(^[.]clang-tidy)",
+     "(.*/py.typed)"
   ],
   "linters": {
     "go-files": {
@@ -67,7 +68,8 @@
         "(java/yb-cdc/README.md)",
         "(^docs/.*[.]md$)",
         "(^[.]fossa[.]yml$)",
-        "(^managed/.*[.]conf)"
+        "(^managed/.*[.]conf)",
+        "(^[.]gitignore$)"
       ]
     },
     "pycodestyle": {

.gitignore

@@ -112,7 +112,7 @@ submodules/
 .netlify/
 
 *.log
-!python/yb/test_data/org_yb_pgsql_TestDropTableWithConcurrentTxn_testDmlTxnDrop_1pct_sample.log
+!python/yugabyte/test_data/org_yb_pgsql_TestDropTableWithConcurrentTxn_testDmlTxnDrop_1pct_sample.log
 
 managed/yba-installer/bin/yba-ctl
 managed/yba-installer/yba-ctl

bin/ccmd

@@ -6,4 +6,4 @@ set -euo pipefail
 activate_virtualenv
 set_pythonpath
 
-python3 "${YB_SRC_ROOT}/python/yb/ccmd_tool.py" "$@"
+python3 "$YB_SCRIPT_PATH_CCMD_TOOL" "$@"

bin/remote_build.py

@@ -26,8 +26,8 @@
     os.path.dirname(os.path.dirname(os.path.abspath(__file__))), 'python'))  # noqa
 
 
-from yb import remote
-from yb.common_util import init_logging
+from yugabyte import remote
+from yugabyte.common_util import init_logging
 
 
 def add_extra_yb_build_args(yb_build_args: List[str], extra_args: List[str]) -> List[str]:

bin/remote_release.py

@@ -23,7 +23,7 @@
 sys.path.insert(0, os.path.join(
     os.path.dirname(os.path.dirname(os.path.abspath(__file__))), 'python'))  # noqa
 
-from yb import remote
+from yugabyte import remote
 
 
 def main() -> None:

build-support/build_postgres

@@ -28,4 +28,4 @@ activate_virtualenv
 set_sanitizer_runtime_options
 
 set_pythonpath
-"$YB_SRC_ROOT"/python/yb/build_postgres.py "$@"
+python3 "$YB_SCRIPT_PATH_BUILD_POSTGRES" "$@"

build-support/common-build-env-test.sh

@@ -190,6 +190,36 @@ test_set_cmake_build_type_and_compiler_type   release    linux-gnu auto       re
 test_set_cmake_build_type_and_compiler_type   tsan       linux-gnu auto       fastdebug  clang15 0
 test_set_cmake_build_type_and_compiler_type   asan       linux-gnu auto       fastdebug  clang15 0
 
+# -------------------------------------------------------------------------------------------------
+# Test existence of scripts pointed to by specical "script path" variables.
+# -------------------------------------------------------------------------------------------------
+
+list_yb_script_path_var_names() {
+  env | grep -E '^YB_SCRIPT_PATH_' | sed 's/=.*//g'
+}
+
+# Unset all script path variables in case some of them are set from outside.
+for script_path_var_name in $( list_yb_script_path_var_names ); do
+  unset "${script_path_var_name}"
+done
+
+# Then set them again from scratch.
+yb_script_paths_are_set=false
+set_script_paths
+
+# Verify that the script pointed to by each of these variables exists.
+for script_path_var_name in $( list_yb_script_path_var_names ); do
+  script_path_var_value=${!script_path_var_name}
+  if [[ ! -f ${script_path_var_value} ]]; then
+    fatal "Script path variable '$script_path_var_name' points to a non-existent file: " \
+          "'$script_path_var_value'"
+  fi
+  if [[ ! -x ${script_path_var_value} ]]; then
+    fatal "Script path variable '$script_path_var_name' points to a non-executable file: " \
+          "'$script_path_var_value'"
+  fi
+done
+
 # -------------------------------------------------------------------------------------------------
 
 echo "${0##/*} succeeded"

build-support/common-build-env.sh

@@ -29,10 +29,59 @@ fi
 
 readonly YB_COMMON_BUILD_ENV_SOURCED=1
 
+yb_script_paths_are_set=false
+
 # -------------------------------------------------------------------------------------------------
 # Functions used during initialization
 # -------------------------------------------------------------------------------------------------
 
+set_script_paths() {
+  if [[ $yb_script_paths_are_set == "true" ]]; then
+    return
+  fi
+  yb_script_paths_are_set=true
+
+  # We check that all of these scripts actually exist in common-build-env-test.sh, that's why it is
+  # useful to invoke them using these constants.
+  export YB_SCRIPT_PATH_AGGREGATE_TEST_REPORTS=\
+$YB_SRC_ROOT/python/yugabyte/aggregate_test_reports.py
+  export YB_SCRIPT_PATH_BUILD_POSTGRES=$YB_SRC_ROOT/python/yugabyte/build_postgres.py
+  export YB_SCRIPT_PATH_CCMD_TOOL=$YB_SRC_ROOT/python/yugabyte/ccmd_tool.py
+  export YB_SCRIPT_PATH_CHECK_PYTHON_SYNTAX=$YB_SRC_ROOT/python/yugabyte/check_python_syntax.py
+  export YB_SCRIPT_PATH_DEDUP_THREAD_STACKS=$YB_SRC_ROOT/python/yugabyte/dedup_thread_stacks.py
+  export YB_SCRIPT_PATH_DEPENDENCY_GRAPH=$YB_SRC_ROOT/python/yugabyte/dependency_graph.py
+  export YB_SCRIPT_PATH_DOWNLOAD_AND_EXTRACT_ARCHIVE=\
+$YB_SRC_ROOT/python/yugabyte/download_and_extract_archive.py
+  export YB_SCRIPT_PATH_FIX_PATHS_IN_COMPILE_ERRORS=\
+$YB_SRC_ROOT/python/yugabyte/fix_paths_in_compile_errors.py
+  export YB_SCRIPT_PATH_FOSSA_ANALYSIS=$YB_SRC_ROOT/python/yugabyte/fossa_analysis.py
+  export YB_SCRIPT_PATH_GEN_AUTO_FLAGS_JSON=$YB_SRC_ROOT/python/yugabyte/gen_auto_flags_json.py
+  export YB_SCRIPT_PATH_GEN_FLAGS_METADATA=$YB_SRC_ROOT/python/yugabyte/gen_flags_metadata.py
+  export YB_SCRIPT_PATH_GEN_INITIAL_SYS_CATALOG_SNAPSHOT=\
+$YB_SRC_ROOT/python/yugabyte/gen_initial_sys_catalog_snapshot.py
+  export YB_SCRIPT_PATH_GEN_VERSION_INFO=$YB_SRC_ROOT/python/yugabyte/gen_version_info.py
+  export YB_SCRIPT_PATH_IS_SAME_PATH=$YB_SRC_ROOT/python/yugabyte/is_same_path.py
+  export YB_SCRIPT_PATH_KILL_LONG_RUNNING_MINICLUSTER_DAEMONS=\
+$YB_SRC_ROOT/python/yugabyte/kill_long_running_minicluster_daemons.py
+  export YB_SCRIPT_PATH_MAKE_RPATH_RELATIVE=$YB_SRC_ROOT/python/yugabyte/make_rpath_relative.py
+  export YB_SCRIPT_PATH_PARSE_TEST_FAILURE=$YB_SRC_ROOT/python/yugabyte/parse_test_failure.py
+  export YB_SCRIPT_PATH_POSTPROCESS_TEST_RESULT=\
+$YB_SRC_ROOT/python/yugabyte/postprocess_test_result.py
+  export YB_SCRIPT_PATH_PROCESS_TREE_SUPERVISOR=\
+$YB_SRC_ROOT/python/yugabyte/process_tree_supervisor.py
+  export YB_SCRIPT_PATH_REWRITE_TEST_LOG=$YB_SRC_ROOT/python/yugabyte/rewrite_test_log.py
+  export YB_SCRIPT_PATH_RUN_PVS_STUDIO_ANALYZER=\
+$YB_SRC_ROOT/python/yugabyte/run_pvs_studio_analyzer.py
+  export YB_SCRIPT_PATH_RUN_TESTS_ON_SPARK=$YB_SRC_ROOT/python/yugabyte/run_tests_on_spark.py
+  export YB_SCRIPT_PATH_SPLIT_LONG_COMMAND_LINE=\
+$YB_SRC_ROOT/python/yugabyte/split_long_command_line.py
+  export YB_SCRIPT_PATH_THIRDPARTY_TOOL=$YB_SRC_ROOT/python/yugabyte/thirdparty_tool.py
+  export YB_SCRIPT_PATH_UPDATE_TEST_RESULT_XML=\
+$YB_SRC_ROOT/python/yugabyte/update_test_result_xml.py
+  export YB_SCRIPT_PATH_YB_RELEASE_CORE_DB=$YB_SRC_ROOT/python/yugabyte/yb_release_core_db.py
+  export YB_SCRIPT_PATH_LIST_PACKAGED_TARGETS=$YB_SRC_ROOT/python/yugabyte/list_packaged_targets.py
+}
+
 set_yb_src_root() {
   export YB_SRC_ROOT=$1
   YB_BUILD_SUPPORT_DIR=$YB_SRC_ROOT/build-support
@@ -42,6 +91,8 @@ set_yb_src_root() {
   YB_COMPILER_WRAPPER_CC=$YB_BUILD_SUPPORT_DIR/compiler-wrappers/cc
   YB_COMPILER_WRAPPER_CXX=$YB_BUILD_SUPPORT_DIR/compiler-wrappers/c++
   yb_java_project_dirs=( "$YB_SRC_ROOT/java" )
+
+  set_script_paths
 }
 
 # Puts the current Git SHA1 in the current directory into the current_sha1 variable.
@@ -394,7 +445,7 @@ set_build_root() {
 
   if [[ -n ${predefined_build_root:-} &&
         $predefined_build_root != "$BUILD_ROOT" ]] &&
-     ! "$YB_BUILD_SUPPORT_DIR/is_same_path.py" "$predefined_build_root" "$BUILD_ROOT"; then
+     ! "$YB_SCRIPT_PATH_IS_SAME_PATH" "$predefined_build_root" "$BUILD_ROOT"; then
     fatal "An inconsistency between predefined BUILD_ROOT ('$predefined_build_root') and" \
           "computed BUILD_ROOT ('$BUILD_ROOT')."
   fi
@@ -1147,7 +1198,7 @@ download_and_extract_archive() {
           log "[Host $(hostname)] $FLOCK_MSG: $lock_path, proceeding with archive installation."
           (
             set -x
-            "$YB_SRC_ROOT/python/yb/download_and_extract_archive.py" \
+            "$YB_SCRIPT_PATH_DOWNLOAD_AND_EXTRACT_ARCHIVE" \
               --url "$url" \
               --dest-dir-parent "$dest_dir_parent" \
               --local-cache-dir "$LOCAL_DOWNLOAD_DIR"
@@ -2167,7 +2218,7 @@ check_python_script_syntax() {
   git ls-files -t '*.py' \
     | grep -v '^S' \
     | sed 's/^[[:alpha:]] //' \
-    | xargs -P 8 -n 1 "$YB_BUILD_SUPPORT_DIR/check_python_syntax.py"
+    | xargs -P 8 -n 1 "$YB_SCRIPT_PATH_CHECK_PYTHON_SYNTAX"
   popd
 }
 

build-support/common-test-env.sh

@@ -598,7 +598,7 @@ analyze_existing_core_file() {
     echo "$debugger_input" |
       "${debugger_cmd[@]}" 2>&1 |
       grep -Ev "^\[New LWP [0-9]+\]$" |
-      "$YB_SRC_ROOT"/build-support/dedup_thread_stacks.py |
+      "$YB_SCRIPT_PATH_DEDUP_THREAD_STACKS" |
       tee -a "$append_output_to"
   ) >&2
   set -e
@@ -790,8 +790,8 @@ handle_cxx_test_xml_output() {
       # parse_test_failure will also generate XML file in this case.
     fi
     echo "Generating an XML output file using parse_test_failure.py: $xml_output_file" >&2
-    "$YB_SRC_ROOT"/build-support/parse_test_failure.py -x \
-        "$junit_test_case_id" "$test_log_path" >"$xml_output_file"
+    "$YB_SCRIPT_PATH_PARSE_TEST_FAILURE" -x "$junit_test_case_id" "$test_log_path" \
+      >"$xml_output_file"
   fi
 
   process_tree_supervisor_append_log_to_on_error=$test_log_path
@@ -808,7 +808,7 @@ handle_cxx_test_xml_output() {
     log "Test succeeded, updating $xml_output_file"
   fi
   update_test_result_xml_cmd=(
-    "$YB_SRC_ROOT"/build-support/update_test_result_xml.py
+    "$YB_SCRIPT_PATH_UPDATE_TEST_RESULT_XML"
     --result-xml "$xml_output_file"
     --mark-as-failed "$test_failed"
   )
@@ -1023,7 +1023,7 @@ run_postproces_test_result_script() {
   fi
   (
     set_pythonpath
-    "$VIRTUAL_ENV/bin/python" "${YB_SRC_ROOT}/python/yb/postprocess_test_result.py" \
+    "$VIRTUAL_ENV/bin/python" "$YB_SCRIPT_PATH_POSTPROCESS_TEST_RESULT" \
       "${args[@]}" "$@"
   )
 }
@@ -1038,7 +1038,7 @@ rewrite_test_log() {
   (
     # TODO: we should just set PYTHONPATH globally, e.g. at the time we activate virtualenv.
     set_pythonpath
-    "${VIRTUAL_ENV}/bin/python" "${YB_SRC_ROOT}/python/yb/rewrite_test_log.py" \
+    "${VIRTUAL_ENV}/bin/python" "$YB_SCRIPT_PATH_REWRITE_TEST_LOG" \
         --input-log-path "${test_log_path}" \
         --replace-original \
         --yb-src-root "${YB_SRC_ROOT}" \
@@ -1371,7 +1371,7 @@ run_tests_on_spark() {
     # Finished task 2791.0 in stage 0.0 (TID 2791) in 10436 ms on <ip> (executor 3) (2900/2908)
     time "$spark_submit_cmd_path" \
       --driver-cores "$INITIAL_SPARK_DRIVER_CORES" \
-      "$YB_SRC_ROOT/build-support/run_tests_on_spark.py" \
+      "$YB_SCRIPT_PATH_RUN_TESTS_ON_SPARK" \
       "${run_tests_args[@]}" "$@" 2>&1 | \
       grep -Ev "TaskSetManager: (Starting task|Finished task .* \([0-9]+[1-9]/[0-9]+\))" \
            --line-buffered
@@ -1715,7 +1715,7 @@ run_java_test() {
     else
       log "Process tree supervisor script reported an error, marking the test as failed in" \
           "$junit_xml_path"
-      "$YB_SRC_ROOT"/build-support/update_test_result_xml.py \
+      "$YB_SCRIPT_PATH_UPDATE_TEST_RESULT_XML" \
         --result-xml "$junit_xml_path" \
         --mark-as-failed true \
         --extra-message "Process supervisor script reported errors (e.g. unterminated processes)."

build-support/compiler-wrappers/compiler-wrapper.sh

@@ -133,7 +133,7 @@ show_compiler_command_line() {
 
   local command_line_filter=cat
   if [[ -n ${YB_SPLIT_LONG_COMPILER_CMD_LINES:-} ]]; then
-    command_line_filter=$YB_SRC_ROOT/build-support/split_long_command_line.py
+    command_line_filter=$YB_SCRIPT_PATH_SPLIT_LONG_COMMAND_LINE
   fi
 
   # Split the failed compilation command over multiple lines for easier reading.
@@ -553,7 +553,7 @@ local_build_exit_handler() {
                 echo "Output file (from -o): $output_file"
               fi
 
-            ) | "$YB_SRC_ROOT/build-support/fix_paths_in_compile_errors.py"
+            ) | "$YB_SCRIPT_PATH_FIX_PATHS_IN_COMPILE_ERRORS"
 
             unset IFS
             echo "\-------------------------------------------------------------------------------"
@@ -739,7 +739,7 @@ if [[ ${YB_DISABLE_RELATIVE_RPATH:-0} == "0" ]] &&
       case $arg in
         -Wl,-rpath,*)
           new_rpath_arg=$(
-            "$YB_BUILD_SUPPORT_DIR/make_rpath_relative.py" "$output_file" "$arg"
+          "$YB_SCRIPT_PATH_MAKE_RPATH_RELATIVE" "$output_file" "$arg"
           )
           new_cmd+=( "$new_rpath_arg" )
         ;;

build-support/dependency_graph

@@ -17,4 +17,4 @@ set -euo pipefail
 
 activate_virtualenv
 set_pythonpath
-"$YB_SRC_ROOT"/python/yb/dependency_graph.py "$@"
+"$YB_SCRIPT_PATH_DEPENDENCY_GRAPH" "$@"

build-support/fossa_analysis

@@ -25,4 +25,4 @@ if [[ -n ${BUILD_ROOT:-} ]]; then
 fi
 find_or_download_thirdparty
 
-python3 "$YB_SRC_ROOT"/python/yb/fossa_analysis.py "$@"
+python3 "$YB_SCRIPT_PATH_FOSSA_ANALYSIS" "$@"

build-support/gen_auto_flags

@@ -18,4 +18,4 @@ set -euo pipefail
 
 activate_virtualenv
 set_pythonpath
-"$YB_SRC_ROOT"/python/yb/gen_auto_flags_json.py "$@"
+"$YB_SCRIPT_PATH_GEN_AUTO_FLAGS_JSON" "$@"

build-support/gen_flags_metadata_wrapper

@@ -17,4 +17,4 @@ set -euo pipefail
 
 activate_virtualenv
 set_pythonpath
-"$YB_SRC_ROOT"/python/yb/gen_flags_metadata.py "$@"
+"$YB_SCRIPT_PATH_GEN_FLAGS_METADATA" "$@"

build-support/gen_initial_sys_catalog_snapshot_wrapper

@@ -17,4 +17,4 @@ set -euo pipefail
 
 activate_virtualenv
 set_pythonpath
-"$YB_SRC_ROOT"/python/yb/gen_initial_sys_catalog_snapshot.py "$@"
+"$YB_SCRIPT_PATH_GEN_INITIAL_SYS_CATALOG_SNAPSHOT" "$@"

build-support/jenkins/test.sh

@@ -257,7 +257,7 @@ export YB_SKIP_INITIAL_SYS_CATALOG_SNAPSHOT=0
 
 if [[ $YB_RUN_AFFECTED_TESTS_ONLY == "1" ]]; then
   if ! ( set -x
-         "${YB_SRC_ROOT}/python/yb/dependency_graph.py" \
+         "$YB_SCRIPT_PATH_DEPENDENCY_GRAPH" \
            --build-root "${BUILD_ROOT}" \
            self-test \
            --rebuild-graph ); then
@@ -309,7 +309,7 @@ if [[ ${YB_COMPILE_ONLY} != "1" ]]; then
         # of tests to run. Useful when testing this script.
         (
           set -x
-          "${YB_SRC_ROOT}/python/yb/dependency_graph.py" \
+          "$YB_SCRIPT_PATH_DEPENDENCY_GRAPH" \
               --build-root "${BUILD_ROOT}" \
               --git-commit "${YB_GIT_COMMIT_FOR_DETECTING_TESTS:-$current_git_commit}" \
               --output-test-config "${test_conf_path}" \
@@ -390,7 +390,7 @@ fi
 remove_latest_symlink
 
 log "Aggregating test reports"
-"$YB_SRC_ROOT/python/yb/aggregate_test_reports.py" \
+"$YB_SCRIPT_PATH_AGGREGATE_TEST_REPORTS" \
       --yb-src-root "${YB_SRC_ROOT}" \
       --output-dir "${YB_SRC_ROOT}" \
       --build-type "${build_type}" \

build-support/jenkins/yb-jenkins-build.sh

@@ -103,7 +103,7 @@ echo
 show_disk_usage
 
 if is_mac; then
-  "$YB_BUILD_SUPPORT_DIR"/kill_long_running_minicluster_daemons.py
+  "$YB_SCRIPT_PATH_KILL_LONG_RUNNING_MINICLUSTER_DAEMONS"
 fi
 
 set +e