diff --git a/.arclint b/.arclint index 1fc2ca667e1d..b5ce7866345e 100644 --- a/.arclint +++ b/.arclint @@ -15,13 +15,14 @@ "(^managed/src/main/resources/swagger-strict[.]json$)", "(^managed/build[.]sbt$)", "(^managed/src/main/resources/aws_pricing/.*$)", - "(^python/yb/test_data/.*[.](log|out)$)", + "(^python/yugabyte/test_data/.*[.](log|out)$)", "(^managed/src/main/resources/metric/Dashboard[.]json$)", "(^managed/src/main/resources/metric/recording_rules[.]yml$)", "(^managed/devops/replicated[.]yml$)", "(^python/yb/py[.]typed$)", "(^managed/RUNTIME-FLAGS[.]md$)", - "(^[.]clang-tidy)" + "(^[.]clang-tidy)", + "(.*/py.typed)" ], "linters": { "go-files": { @@ -67,7 +68,8 @@ "(java/yb-cdc/README.md)", "(^docs/.*[.]md$)", "(^[.]fossa[.]yml$)", - "(^managed/.*[.]conf)" + "(^managed/.*[.]conf)", + "(^[.]gitignore$)" ] }, "pycodestyle": { diff --git a/.gitignore b/.gitignore index 94b14d6cd922..3ff10224622c 100644 --- a/.gitignore +++ b/.gitignore @@ -112,7 +112,7 @@ submodules/ .netlify/ *.log -!python/yb/test_data/org_yb_pgsql_TestDropTableWithConcurrentTxn_testDmlTxnDrop_1pct_sample.log +!python/yugabyte/test_data/org_yb_pgsql_TestDropTableWithConcurrentTxn_testDmlTxnDrop_1pct_sample.log managed/yba-installer/bin/yba-ctl managed/yba-installer/yba-ctl diff --git a/bin/ccmd b/bin/ccmd index 41404b86a17c..72d9f797f351 100755 --- a/bin/ccmd +++ b/bin/ccmd @@ -6,4 +6,4 @@ set -euo pipefail activate_virtualenv set_pythonpath -python3 "${YB_SRC_ROOT}/python/yb/ccmd_tool.py" "$@" +python3 "$YB_SCRIPT_PATH_CCMD_TOOL" "$@" diff --git a/bin/remote_build.py b/bin/remote_build.py index 1a2a8b40eac2..06fa3ff2122d 100755 --- a/bin/remote_build.py +++ b/bin/remote_build.py @@ -26,8 +26,8 @@ os.path.dirname(os.path.dirname(os.path.abspath(__file__))), 'python')) # noqa -from yb import remote -from yb.common_util import init_logging +from yugabyte import remote +from yugabyte.common_util import init_logging def add_extra_yb_build_args(yb_build_args: List[str], extra_args: List[str]) -> List[str]: diff --git a/bin/remote_release.py b/bin/remote_release.py index d79d356370f0..29ebaa8465f2 100755 --- a/bin/remote_release.py +++ b/bin/remote_release.py @@ -23,7 +23,7 @@ sys.path.insert(0, os.path.join( os.path.dirname(os.path.dirname(os.path.abspath(__file__))), 'python')) # noqa -from yb import remote +from yugabyte import remote def main() -> None: diff --git a/build-support/build_postgres b/build-support/build_postgres index b98d01dcecb9..d5164315b3d4 100755 --- a/build-support/build_postgres +++ b/build-support/build_postgres @@ -28,4 +28,4 @@ activate_virtualenv set_sanitizer_runtime_options set_pythonpath -"$YB_SRC_ROOT"/python/yb/build_postgres.py "$@" +python3 "$YB_SCRIPT_PATH_BUILD_POSTGRES" "$@" diff --git a/build-support/common-build-env-test.sh b/build-support/common-build-env-test.sh index 18c5d2e9ec4c..6a8b76f0360d 100755 --- a/build-support/common-build-env-test.sh +++ b/build-support/common-build-env-test.sh @@ -190,6 +190,36 @@ test_set_cmake_build_type_and_compiler_type release linux-gnu auto re test_set_cmake_build_type_and_compiler_type tsan linux-gnu auto fastdebug clang15 0 test_set_cmake_build_type_and_compiler_type asan linux-gnu auto fastdebug clang15 0 +# ------------------------------------------------------------------------------------------------- +# Test existence of scripts pointed to by specical "script path" variables. +# ------------------------------------------------------------------------------------------------- + +list_yb_script_path_var_names() { + env | grep -E '^YB_SCRIPT_PATH_' | sed 's/=.*//g' +} + +# Unset all script path variables in case some of them are set from outside. +for script_path_var_name in $( list_yb_script_path_var_names ); do + unset "${script_path_var_name}" +done + +# Then set them again from scratch. +yb_script_paths_are_set=false +set_script_paths + +# Verify that the script pointed to by each of these variables exists. +for script_path_var_name in $( list_yb_script_path_var_names ); do + script_path_var_value=${!script_path_var_name} + if [[ ! -f ${script_path_var_value} ]]; then + fatal "Script path variable '$script_path_var_name' points to a non-existent file: " \ + "'$script_path_var_value'" + fi + if [[ ! -x ${script_path_var_value} ]]; then + fatal "Script path variable '$script_path_var_name' points to a non-executable file: " \ + "'$script_path_var_value'" + fi +done + # ------------------------------------------------------------------------------------------------- echo "${0##/*} succeeded" diff --git a/build-support/common-build-env.sh b/build-support/common-build-env.sh index 4607dfc2a411..232e77f4bf7e 100644 --- a/build-support/common-build-env.sh +++ b/build-support/common-build-env.sh @@ -29,10 +29,59 @@ fi readonly YB_COMMON_BUILD_ENV_SOURCED=1 +yb_script_paths_are_set=false + # ------------------------------------------------------------------------------------------------- # Functions used during initialization # ------------------------------------------------------------------------------------------------- +set_script_paths() { + if [[ $yb_script_paths_are_set == "true" ]]; then + return + fi + yb_script_paths_are_set=true + + # We check that all of these scripts actually exist in common-build-env-test.sh, that's why it is + # useful to invoke them using these constants. + export YB_SCRIPT_PATH_AGGREGATE_TEST_REPORTS=\ +$YB_SRC_ROOT/python/yugabyte/aggregate_test_reports.py + export YB_SCRIPT_PATH_BUILD_POSTGRES=$YB_SRC_ROOT/python/yugabyte/build_postgres.py + export YB_SCRIPT_PATH_CCMD_TOOL=$YB_SRC_ROOT/python/yugabyte/ccmd_tool.py + export YB_SCRIPT_PATH_CHECK_PYTHON_SYNTAX=$YB_SRC_ROOT/python/yugabyte/check_python_syntax.py + export YB_SCRIPT_PATH_DEDUP_THREAD_STACKS=$YB_SRC_ROOT/python/yugabyte/dedup_thread_stacks.py + export YB_SCRIPT_PATH_DEPENDENCY_GRAPH=$YB_SRC_ROOT/python/yugabyte/dependency_graph.py + export YB_SCRIPT_PATH_DOWNLOAD_AND_EXTRACT_ARCHIVE=\ +$YB_SRC_ROOT/python/yugabyte/download_and_extract_archive.py + export YB_SCRIPT_PATH_FIX_PATHS_IN_COMPILE_ERRORS=\ +$YB_SRC_ROOT/python/yugabyte/fix_paths_in_compile_errors.py + export YB_SCRIPT_PATH_FOSSA_ANALYSIS=$YB_SRC_ROOT/python/yugabyte/fossa_analysis.py + export YB_SCRIPT_PATH_GEN_AUTO_FLAGS_JSON=$YB_SRC_ROOT/python/yugabyte/gen_auto_flags_json.py + export YB_SCRIPT_PATH_GEN_FLAGS_METADATA=$YB_SRC_ROOT/python/yugabyte/gen_flags_metadata.py + export YB_SCRIPT_PATH_GEN_INITIAL_SYS_CATALOG_SNAPSHOT=\ +$YB_SRC_ROOT/python/yugabyte/gen_initial_sys_catalog_snapshot.py + export YB_SCRIPT_PATH_GEN_VERSION_INFO=$YB_SRC_ROOT/python/yugabyte/gen_version_info.py + export YB_SCRIPT_PATH_IS_SAME_PATH=$YB_SRC_ROOT/python/yugabyte/is_same_path.py + export YB_SCRIPT_PATH_KILL_LONG_RUNNING_MINICLUSTER_DAEMONS=\ +$YB_SRC_ROOT/python/yugabyte/kill_long_running_minicluster_daemons.py + export YB_SCRIPT_PATH_MAKE_RPATH_RELATIVE=$YB_SRC_ROOT/python/yugabyte/make_rpath_relative.py + export YB_SCRIPT_PATH_PARSE_TEST_FAILURE=$YB_SRC_ROOT/python/yugabyte/parse_test_failure.py + export YB_SCRIPT_PATH_POSTPROCESS_TEST_RESULT=\ +$YB_SRC_ROOT/python/yugabyte/postprocess_test_result.py + export YB_SCRIPT_PATH_PROCESS_TREE_SUPERVISOR=\ +$YB_SRC_ROOT/python/yugabyte/process_tree_supervisor.py + export YB_SCRIPT_PATH_REWRITE_TEST_LOG=$YB_SRC_ROOT/python/yugabyte/rewrite_test_log.py + export YB_SCRIPT_PATH_RUN_PVS_STUDIO_ANALYZER=\ +$YB_SRC_ROOT/python/yugabyte/run_pvs_studio_analyzer.py + export YB_SCRIPT_PATH_RUN_TESTS_ON_SPARK=$YB_SRC_ROOT/python/yugabyte/run_tests_on_spark.py + export YB_SCRIPT_PATH_SPLIT_LONG_COMMAND_LINE=\ +$YB_SRC_ROOT/python/yugabyte/split_long_command_line.py + export YB_SCRIPT_PATH_THIRDPARTY_TOOL=$YB_SRC_ROOT/python/yugabyte/thirdparty_tool.py + export YB_SCRIPT_PATH_UPDATE_TEST_RESULT_XML=\ +$YB_SRC_ROOT/python/yugabyte/update_test_result_xml.py + export YB_SCRIPT_PATH_YB_RELEASE_CORE_DB=$YB_SRC_ROOT/python/yugabyte/yb_release_core_db.py + export YB_SCRIPT_PATH_LIST_PACKAGED_TARGETS=$YB_SRC_ROOT/python/yugabyte/list_packaged_targets.py +} + set_yb_src_root() { export YB_SRC_ROOT=$1 YB_BUILD_SUPPORT_DIR=$YB_SRC_ROOT/build-support @@ -42,6 +91,8 @@ set_yb_src_root() { YB_COMPILER_WRAPPER_CC=$YB_BUILD_SUPPORT_DIR/compiler-wrappers/cc YB_COMPILER_WRAPPER_CXX=$YB_BUILD_SUPPORT_DIR/compiler-wrappers/c++ yb_java_project_dirs=( "$YB_SRC_ROOT/java" ) + + set_script_paths } # Puts the current Git SHA1 in the current directory into the current_sha1 variable. @@ -394,7 +445,7 @@ set_build_root() { if [[ -n ${predefined_build_root:-} && $predefined_build_root != "$BUILD_ROOT" ]] && - ! "$YB_BUILD_SUPPORT_DIR/is_same_path.py" "$predefined_build_root" "$BUILD_ROOT"; then + ! "$YB_SCRIPT_PATH_IS_SAME_PATH" "$predefined_build_root" "$BUILD_ROOT"; then fatal "An inconsistency between predefined BUILD_ROOT ('$predefined_build_root') and" \ "computed BUILD_ROOT ('$BUILD_ROOT')." fi @@ -1147,7 +1198,7 @@ download_and_extract_archive() { log "[Host $(hostname)] $FLOCK_MSG: $lock_path, proceeding with archive installation." ( set -x - "$YB_SRC_ROOT/python/yb/download_and_extract_archive.py" \ + "$YB_SCRIPT_PATH_DOWNLOAD_AND_EXTRACT_ARCHIVE" \ --url "$url" \ --dest-dir-parent "$dest_dir_parent" \ --local-cache-dir "$LOCAL_DOWNLOAD_DIR" @@ -2167,7 +2218,7 @@ check_python_script_syntax() { git ls-files -t '*.py' \ | grep -v '^S' \ | sed 's/^[[:alpha:]] //' \ - | xargs -P 8 -n 1 "$YB_BUILD_SUPPORT_DIR/check_python_syntax.py" + | xargs -P 8 -n 1 "$YB_SCRIPT_PATH_CHECK_PYTHON_SYNTAX" popd } diff --git a/build-support/common-test-env.sh b/build-support/common-test-env.sh index 82c9f37ce4de..c67a25fba127 100644 --- a/build-support/common-test-env.sh +++ b/build-support/common-test-env.sh @@ -598,7 +598,7 @@ analyze_existing_core_file() { echo "$debugger_input" | "${debugger_cmd[@]}" 2>&1 | grep -Ev "^\[New LWP [0-9]+\]$" | - "$YB_SRC_ROOT"/build-support/dedup_thread_stacks.py | + "$YB_SCRIPT_PATH_DEDUP_THREAD_STACKS" | tee -a "$append_output_to" ) >&2 set -e @@ -790,8 +790,8 @@ handle_cxx_test_xml_output() { # parse_test_failure will also generate XML file in this case. fi echo "Generating an XML output file using parse_test_failure.py: $xml_output_file" >&2 - "$YB_SRC_ROOT"/build-support/parse_test_failure.py -x \ - "$junit_test_case_id" "$test_log_path" >"$xml_output_file" + "$YB_SCRIPT_PATH_PARSE_TEST_FAILURE" -x "$junit_test_case_id" "$test_log_path" \ + >"$xml_output_file" fi process_tree_supervisor_append_log_to_on_error=$test_log_path @@ -808,7 +808,7 @@ handle_cxx_test_xml_output() { log "Test succeeded, updating $xml_output_file" fi update_test_result_xml_cmd=( - "$YB_SRC_ROOT"/build-support/update_test_result_xml.py + "$YB_SCRIPT_PATH_UPDATE_TEST_RESULT_XML" --result-xml "$xml_output_file" --mark-as-failed "$test_failed" ) @@ -1023,7 +1023,7 @@ run_postproces_test_result_script() { fi ( set_pythonpath - "$VIRTUAL_ENV/bin/python" "${YB_SRC_ROOT}/python/yb/postprocess_test_result.py" \ + "$VIRTUAL_ENV/bin/python" "$YB_SCRIPT_PATH_POSTPROCESS_TEST_RESULT" \ "${args[@]}" "$@" ) } @@ -1038,7 +1038,7 @@ rewrite_test_log() { ( # TODO: we should just set PYTHONPATH globally, e.g. at the time we activate virtualenv. set_pythonpath - "${VIRTUAL_ENV}/bin/python" "${YB_SRC_ROOT}/python/yb/rewrite_test_log.py" \ + "${VIRTUAL_ENV}/bin/python" "$YB_SCRIPT_PATH_REWRITE_TEST_LOG" \ --input-log-path "${test_log_path}" \ --replace-original \ --yb-src-root "${YB_SRC_ROOT}" \ @@ -1371,7 +1371,7 @@ run_tests_on_spark() { # Finished task 2791.0 in stage 0.0 (TID 2791) in 10436 ms on (executor 3) (2900/2908) time "$spark_submit_cmd_path" \ --driver-cores "$INITIAL_SPARK_DRIVER_CORES" \ - "$YB_SRC_ROOT/build-support/run_tests_on_spark.py" \ + "$YB_SCRIPT_PATH_RUN_TESTS_ON_SPARK" \ "${run_tests_args[@]}" "$@" 2>&1 | \ grep -Ev "TaskSetManager: (Starting task|Finished task .* \([0-9]+[1-9]/[0-9]+\))" \ --line-buffered @@ -1715,7 +1715,7 @@ run_java_test() { else log "Process tree supervisor script reported an error, marking the test as failed in" \ "$junit_xml_path" - "$YB_SRC_ROOT"/build-support/update_test_result_xml.py \ + "$YB_SCRIPT_PATH_UPDATE_TEST_RESULT_XML" \ --result-xml "$junit_xml_path" \ --mark-as-failed true \ --extra-message "Process supervisor script reported errors (e.g. unterminated processes)." diff --git a/build-support/compiler-wrappers/compiler-wrapper.sh b/build-support/compiler-wrappers/compiler-wrapper.sh index b292b7a6881c..a57188e63e6e 100755 --- a/build-support/compiler-wrappers/compiler-wrapper.sh +++ b/build-support/compiler-wrappers/compiler-wrapper.sh @@ -133,7 +133,7 @@ show_compiler_command_line() { local command_line_filter=cat if [[ -n ${YB_SPLIT_LONG_COMPILER_CMD_LINES:-} ]]; then - command_line_filter=$YB_SRC_ROOT/build-support/split_long_command_line.py + command_line_filter=$YB_SCRIPT_PATH_SPLIT_LONG_COMMAND_LINE fi # Split the failed compilation command over multiple lines for easier reading. @@ -553,7 +553,7 @@ local_build_exit_handler() { echo "Output file (from -o): $output_file" fi - ) | "$YB_SRC_ROOT/build-support/fix_paths_in_compile_errors.py" + ) | "$YB_SCRIPT_PATH_FIX_PATHS_IN_COMPILE_ERRORS" unset IFS echo "\-------------------------------------------------------------------------------" @@ -739,7 +739,7 @@ if [[ ${YB_DISABLE_RELATIVE_RPATH:-0} == "0" ]] && case $arg in -Wl,-rpath,*) new_rpath_arg=$( - "$YB_BUILD_SUPPORT_DIR/make_rpath_relative.py" "$output_file" "$arg" + "$YB_SCRIPT_PATH_MAKE_RPATH_RELATIVE" "$output_file" "$arg" ) new_cmd+=( "$new_rpath_arg" ) ;; diff --git a/build-support/dependency_graph b/build-support/dependency_graph index af0eeb1298f9..957e2ca81002 100755 --- a/build-support/dependency_graph +++ b/build-support/dependency_graph @@ -17,4 +17,4 @@ set -euo pipefail activate_virtualenv set_pythonpath -"$YB_SRC_ROOT"/python/yb/dependency_graph.py "$@" +"$YB_SCRIPT_PATH_DEPENDENCY_GRAPH" "$@" diff --git a/build-support/fossa_analysis b/build-support/fossa_analysis index beb4df0d62c6..21b5597d8a45 100755 --- a/build-support/fossa_analysis +++ b/build-support/fossa_analysis @@ -25,4 +25,4 @@ if [[ -n ${BUILD_ROOT:-} ]]; then fi find_or_download_thirdparty -python3 "$YB_SRC_ROOT"/python/yb/fossa_analysis.py "$@" +python3 "$YB_SCRIPT_PATH_FOSSA_ANALYSIS" "$@" diff --git a/build-support/gen_auto_flags b/build-support/gen_auto_flags index 91f92853de2f..f76b0a9f67be 100755 --- a/build-support/gen_auto_flags +++ b/build-support/gen_auto_flags @@ -18,4 +18,4 @@ set -euo pipefail activate_virtualenv set_pythonpath -"$YB_SRC_ROOT"/python/yb/gen_auto_flags_json.py "$@" +"$YB_SCRIPT_PATH_GEN_AUTO_FLAGS_JSON" "$@" diff --git a/build-support/gen_flags_metadata_wrapper b/build-support/gen_flags_metadata_wrapper index b4e564aaa0c7..aef2b933ec9f 100755 --- a/build-support/gen_flags_metadata_wrapper +++ b/build-support/gen_flags_metadata_wrapper @@ -17,4 +17,4 @@ set -euo pipefail activate_virtualenv set_pythonpath -"$YB_SRC_ROOT"/python/yb/gen_flags_metadata.py "$@" +"$YB_SCRIPT_PATH_GEN_FLAGS_METADATA" "$@" diff --git a/build-support/gen_initial_sys_catalog_snapshot_wrapper b/build-support/gen_initial_sys_catalog_snapshot_wrapper index 2b9678e69d96..7f774787b4ba 100755 --- a/build-support/gen_initial_sys_catalog_snapshot_wrapper +++ b/build-support/gen_initial_sys_catalog_snapshot_wrapper @@ -17,4 +17,4 @@ set -euo pipefail activate_virtualenv set_pythonpath -"$YB_SRC_ROOT"/python/yb/gen_initial_sys_catalog_snapshot.py "$@" +"$YB_SCRIPT_PATH_GEN_INITIAL_SYS_CATALOG_SNAPSHOT" "$@" diff --git a/build-support/get_source_rel_path.py b/build-support/get_source_rel_path.py deleted file mode 100755 index ba587c1d5805..000000000000 --- a/build-support/get_source_rel_path.py +++ /dev/null @@ -1,22 +0,0 @@ -#!/usr/bin/env python - -# Get the relative path for a source file. Relative to: -# - source root (YB_SRC_ROOT) -# - build root (YB_BUILD_ROOT) - - -import os -import sys - - -if __name__ == '__main__': - file_path = os.path.realpath(sys.argv[1]) - base_path = None - for base_path_env_var in ['YB_BUILD_ROOT', 'YB_SRC_ROOT']: - base_path_candidate = os.path.realpath(os.getenv(base_path_env_var)) - if file_path.startswith(base_path_candidate + '/'): - base_path = base_path_candidate - break - if base_path is not None: - file_path = os.path.relpath(file_path, base_path) - print(file_path) diff --git a/build-support/is_same_path.py b/build-support/is_same_path.py deleted file mode 100755 index 38a3c7b8ad32..000000000000 --- a/build-support/is_same_path.py +++ /dev/null @@ -1,11 +0,0 @@ -#!/usr/bin/env python - -import os -import sys - - -if __name__ == '__main__': - if os.path.realpath(sys.argv[1]) == os.path.realpath(sys.argv[2]): - sys.exit(0) - - sys.exit(1) diff --git a/build-support/jenkins/test.sh b/build-support/jenkins/test.sh index b9ab5540f3f2..0740b859d580 100755 --- a/build-support/jenkins/test.sh +++ b/build-support/jenkins/test.sh @@ -257,7 +257,7 @@ export YB_SKIP_INITIAL_SYS_CATALOG_SNAPSHOT=0 if [[ $YB_RUN_AFFECTED_TESTS_ONLY == "1" ]]; then if ! ( set -x - "${YB_SRC_ROOT}/python/yb/dependency_graph.py" \ + "$YB_SCRIPT_PATH_DEPENDENCY_GRAPH" \ --build-root "${BUILD_ROOT}" \ self-test \ --rebuild-graph ); then @@ -309,7 +309,7 @@ if [[ ${YB_COMPILE_ONLY} != "1" ]]; then # of tests to run. Useful when testing this script. ( set -x - "${YB_SRC_ROOT}/python/yb/dependency_graph.py" \ + "$YB_SCRIPT_PATH_DEPENDENCY_GRAPH" \ --build-root "${BUILD_ROOT}" \ --git-commit "${YB_GIT_COMMIT_FOR_DETECTING_TESTS:-$current_git_commit}" \ --output-test-config "${test_conf_path}" \ @@ -390,7 +390,7 @@ fi remove_latest_symlink log "Aggregating test reports" -"$YB_SRC_ROOT/python/yb/aggregate_test_reports.py" \ +"$YB_SCRIPT_PATH_AGGREGATE_TEST_REPORTS" \ --yb-src-root "${YB_SRC_ROOT}" \ --output-dir "${YB_SRC_ROOT}" \ --build-type "${build_type}" \ diff --git a/build-support/jenkins/yb-jenkins-build.sh b/build-support/jenkins/yb-jenkins-build.sh index 03656e2d13d0..a7597b546946 100755 --- a/build-support/jenkins/yb-jenkins-build.sh +++ b/build-support/jenkins/yb-jenkins-build.sh @@ -103,7 +103,7 @@ echo show_disk_usage if is_mac; then - "$YB_BUILD_SUPPORT_DIR"/kill_long_running_minicluster_daemons.py + "$YB_SCRIPT_PATH_KILL_LONG_RUNNING_MINICLUSTER_DAEMONS" fi set +e diff --git a/build-support/run-test.sh b/build-support/run-test.sh index 35aeebda5e84..feeff4c9205d 100755 --- a/build-support/run-test.sh +++ b/build-support/run-test.sh @@ -118,7 +118,7 @@ trap cleanup EXIT readonly process_supervisor_log_path=\ ${TEST_TMPDIR:-/tmp}/yb_process_supervisor_for_pid_$$__$RANDOM.log -"$YB_SRC_ROOT/python/yb/process_tree_supervisor.py" \ +"$YB_SCRIPT_PATH_PROCESS_TREE_SUPERVISOR" \ --pid $$ \ --terminate-subtree \ --timeout-sec "$PROCESS_TREE_SUPERVISOR_TEST_TIMEOUT_SEC" \ diff --git a/build-support/run_pvs_studio_analyzer b/build-support/run_pvs_studio_analyzer index 954ddae2b485..e8e5f34162af 100755 --- a/build-support/run_pvs_studio_analyzer +++ b/build-support/run_pvs_studio_analyzer @@ -17,4 +17,4 @@ set -euo pipefail activate_virtualenv set_pythonpath -python3 "$YB_SRC_ROOT"/python/yb/run_pvs_studio_analyzer.py "$@" +python3 "$YB_SCRIPT_PATH_RUN_PVS_STUDIO_ANALYZER" "$@" diff --git a/build-support/thirdparty_tool b/build-support/thirdparty_tool index e956bdad761f..9fe2fefc6b6e 100755 --- a/build-support/thirdparty_tool +++ b/build-support/thirdparty_tool @@ -18,4 +18,4 @@ set -euo pipefail activate_virtualenv set_pythonpath -python3 "$YB_SRC_ROOT"/python/yb/thirdparty_tool.py "$@" +python3 "$YB_SCRIPT_PATH_THIRDPARTY_TOOL" "$@" diff --git a/build-support/tserver_lto.sh b/build-support/tserver_lto.sh index 42977b88bba2..48206ad86dcd 100755 --- a/build-support/tserver_lto.sh +++ b/build-support/tserver_lto.sh @@ -28,7 +28,7 @@ if [[ $( uname -m ) == "x86_64" ]]; then fi dep_graph_cmd=( - "${YB_SRC_ROOT}/python/yb/dependency_graph.py" + "$YB_SCRIPT_PATH_DEPENDENCY_GRAPH" "--build-root=${YB_SRC_ROOT}/build/${build_root_basename}" "--file-regex=^.*/yb-tserver-dynamic$" ) diff --git a/codecheck.ini b/codecheck.ini index 41facb0b47bd..b42c65fbe8b3 100644 --- a/codecheck.ini +++ b/codecheck.ini @@ -38,70 +38,12 @@ included_regex_list = ^build-support/common-test-env.sh$ ^build-support/compiler-wrappers/compiler-wrapper.sh$ ^build-support/find_linuxbrew.sh$ - ^build-support/is_same_path[.]py$ ^build-support/jenkins/build.sh$ ^build-support/jenkins/common-lto.sh$ ^build-support/jenkins/test.sh$ ^build-support/jenkins/yb-jenkins-build.sh$ ^build-support/jenkins/yb-jenkins-test.sh$ - ^build-support/kill_long_running_minicluster_daemons[.]py$ - ^build-support/list_packaged_targets[.]$ - ^build-support/run_tests_on_spark[.]py$ ^build-support/run-test.sh$ - ^build-support/split_long_command_line[.]py$ - ^build-support/stabilize_auto_flags_list[.]py$ - ^build-support/validate_build_root[.]py$ - ^build-support/yb_release[.]py$ ^python/yb/__init__[.]py$ - ^python/yb/aggregate_test_reports[.]py$ - ^python/yb/build_paths[.]py$ - ^python/yb/build_postgres[.]py$ - ^python/yb/ccmd_tool[.]py$ - ^python/yb/clang_tidy_runner[.]py$ - ^python/yb/cmake_cache[.]py$ - ^python/yb/command_util[.]py$ - ^python/yb/common_util[.]py$ - ^python/yb/compile_commands_processor[.]py - ^python/yb/compile_commands[.]py$ - ^python/yb/compiler_args[.]py - ^python/yb/compiler_invocation[.]py$ - ^python/yb/compiler_parallel_runner[.]py$ - ^python/yb/dep_graph_common[.]py$ - ^python/yb/dependency_graph[.]py$ - ^python/yb/file_util[.]py$ - ^python/yb/fossa_analysis[.]py$ - ^python/yb/gen_auto_flags_json[.]py$ - ^python/yb/gen_flags_metadata[.]py$ - ^python/yb/gen_initial_sys_catalog_snapshot[.]py$ - ^python/yb/include_path_rewriter[.]py$ - ^python/yb/install_wrapper[.]py$ - ^python/yb/json_util[.]py$ - ^python/yb/library_packager[.]py$ - ^python/yb/link_cmd[.]py$ - ^python/yb/linuxbrew[.]py$ - ^python/yb/logging_util[.]py$ - ^python/yb/lto[.]py$ - ^python/yb/mac_library_packager[.]py$ - ^python/yb/optional_components[.]py$ - ^python/yb/os_detection[.]py$ - ^python/yb/os_versions[.]py$ - ^python/yb/parallel_task_runner[.]py$ - ^python/yb/postgres_build_util[.]py$ - ^python/yb/postprocess_test_result[.]py$ - ^python/yb/preprocessor[.]py$ - ^python/yb/release_util[.]py$ - ^python/yb/remote[.]py$ - ^python/yb/rewrite_test_log[.]py$ - ^python/yb/rpath[.]py$ - ^python/yb/run_pvs_studio_analyzer[.]py$ - ^python/yb/source_files[.]py$ - ^python/yb/string_util[.]py$ - ^python/yb/sys_libs[.]py$ - ^python/yb/test_postprocess_test_result[.]py$ - ^python/yb/test_rewrite_test_log[.]py$ - ^python/yb/thirdparty_tool[.]py$ - ^python/yb/timestamp_saver[.]py$ - ^python/yb/tool_base[.]py$ - ^python/yb/type_util[.]py$ - ^python/yb/yb_dist_tests[.]py$ + ^python/yugabyte/.*[.]py$ ^yb_build.sh$ diff --git a/java/yb-pgsql/src/test/java/org/yb/pgsql/PgRegressBuilder.java b/java/yb-pgsql/src/test/java/org/yb/pgsql/PgRegressBuilder.java index c7a7dd9e1671..99fb371ad084 100644 --- a/java/yb-pgsql/src/test/java/org/yb/pgsql/PgRegressBuilder.java +++ b/java/yb-pgsql/src/test/java/org/yb/pgsql/PgRegressBuilder.java @@ -187,7 +187,7 @@ public PgRegressBuilder setEnvVars(Map envVars) throws IOExcepti private void addPostProcessEnvVar() throws IOException { File postprocessScript = new File( - TestUtils.findYbRootDir() + "/build-support/pg_regress_postprocess_output.py"); + TestUtils.findYbRootDir() + "/python/yugabyte/pg_regress_postprocess_output.py"); if (!postprocessScript.exists()) { throw new IOException("File does not exist: " + postprocessScript); diff --git a/managed/build.sbt b/managed/build.sbt index e61b232af4ce..0c82e2093d3f 100644 --- a/managed/build.sbt +++ b/managed/build.sbt @@ -328,7 +328,7 @@ lazy val moveYbcPackage = getBoolEnvVar(moveYbcPackageEnvName) versionGenerate := { val buildType = sys.env.getOrElse("BUILD_TYPE", "release") - val status = Process("../build-support/gen_version_info.py --build-type=" + buildType + " " + + val status = Process("../python/yugabyte/gen_version_info.py --build-type=" + buildType + " " + (Compile / resourceDirectory).value / "version_metadata.json").! ybLog("version_metadata.json Generated") Process("rm -f " + (Compile / resourceDirectory).value / "gen_version_info.log").! diff --git a/python/yb/log/summarize_tcmalloc_trace.py b/python/yb/log/summarize_tcmalloc_trace.py index 80b8632dac66..63fecc30ca09 100644 --- a/python/yb/log/summarize_tcmalloc_trace.py +++ b/python/yb/log/summarize_tcmalloc_trace.py @@ -21,7 +21,7 @@ import sys from typing import Dict, Iterator, List -from yb.log.log_iter import LogIterator +from yugabyte.log.log_iter import LogIterator MALLOC_CALL_RE = re.compile(r'^Malloc Call: size = (?P\d+)$') diff --git a/python/yugabyte/__init__.py b/python/yugabyte/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/python/yb/aggregate_test_reports.py b/python/yugabyte/aggregate_test_reports.py similarity index 99% rename from python/yb/aggregate_test_reports.py rename to python/yugabyte/aggregate_test_reports.py index 89262c3a8b2c..d9e5c0efd451 100755 --- a/python/yb/aggregate_test_reports.py +++ b/python/yugabyte/aggregate_test_reports.py @@ -85,7 +85,7 @@ from typing import List, Dict, Optional, Any, Tuple, Set, cast, Callable -from yb.common_util import init_logging +from yugabyte.common_util import init_logging def is_test_failure(report: Dict[str, Any]) -> bool: diff --git a/python/yb/build_paths.py b/python/yugabyte/build_paths.py similarity index 98% rename from python/yb/build_paths.py rename to python/yugabyte/build_paths.py index 6aff87380964..31aae5432cf3 100644 --- a/python/yb/build_paths.py +++ b/python/yugabyte/build_paths.py @@ -12,7 +12,7 @@ import os -from yb.file_util import read_file +from yugabyte.file_util import read_file from typing import Optional, Set diff --git a/python/yb/build_postgres.py b/python/yugabyte/build_postgres.py similarity index 98% rename from python/yb/build_postgres.py rename to python/yugabyte/build_postgres.py index d72b0577077c..2c5120b2eedc 100755 --- a/python/yb/build_postgres.py +++ b/python/yugabyte/build_postgres.py @@ -37,8 +37,8 @@ is_verbose_mode ) -from yb.tool_base import YbBuildToolBase -from yb.common_util import ( +from yugabyte.tool_base import YbBuildToolBase +from yugabyte.common_util import ( YB_SRC_ROOT, get_build_type_from_build_root, get_bool_env_var, @@ -49,14 +49,17 @@ is_macos_arm64, init_logging, ) -from yb.json_util import write_json_file, read_json_file -from yb import compile_commands -from yb.compile_commands import create_compile_commands_symlink, get_compile_commands_file_path -from yb.compile_commands_processor import CompileCommandProcessor -from yb.cmake_cache import CMakeCache, load_cmake_cache -from yb.file_util import mkdir_p -from yb.string_util import compute_sha256 -from yb.timestamp_saver import TimestampSaver +from yugabyte.json_util import write_json_file, read_json_file +from yugabyte import compile_commands +from yugabyte.compile_commands import ( + create_compile_commands_symlink, + get_compile_commands_file_path, +) +from yugabyte.compile_commands_processor import CompileCommandProcessor +from yugabyte.cmake_cache import CMakeCache, load_cmake_cache +from yugabyte.file_util import mkdir_p +from yugabyte.string_util import compute_sha256 +from yugabyte.timestamp_saver import TimestampSaver ALLOW_REMOTE_COMPILATION = True @@ -317,7 +320,7 @@ def adjust_cflags_in_makefile(self) -> None: ]) line = ''.join([ install_cmd_line_prefix, - os.path.join('$(YB_SRC_ROOT)', 'python', 'yb', 'install_wrapper.py'), + os.path.join('$(YB_SRC_ROOT)', 'python', 'yugabyte', 'install_wrapper.py'), ' ', line[len(install_cmd_line_prefix):] ]) @@ -619,7 +622,7 @@ def get_build_stamp(self, include_env_vars: bool) -> str: pathspec = [ 'src/postgres', 'src/yb/yql/pggate', - 'python/yb/build_postgres.py', + 'python/yugabyte/build_postgres.py', 'build-support/build_postgres', 'CMakeLists.txt', ] diff --git a/python/yb/ccmd_tool.py b/python/yugabyte/ccmd_tool.py old mode 100644 new mode 100755 similarity index 97% rename from python/yb/ccmd_tool.py rename to python/yugabyte/ccmd_tool.py index b9d592e147e5..ef1256f15a59 --- a/python/yb/ccmd_tool.py +++ b/python/yugabyte/ccmd_tool.py @@ -33,19 +33,19 @@ from overrides import overrides # type: ignore -from yb.tool_base import YbBuildToolBase, UserFriendlyToolError -from yb.compile_commands import ( +from yugabyte.tool_base import YbBuildToolBase, UserFriendlyToolError +from yugabyte.compile_commands import ( get_compile_commands_file_path, CompileCommand, COMBINED_POSTPROCESSED_DIR_NAME, ) -from yb.common_util import YB_SRC_ROOT, ensure_enclosed_by -from yb.json_util import read_json_file +from yugabyte.common_util import YB_SRC_ROOT, ensure_enclosed_by +from yugabyte.json_util import read_json_file -from yb.command_util import shlex_join -from yb.clang_tidy_runner import ClangTidyRunner -from yb.compiler_parallel_runner import CompilerParallelRunner -from yb.preprocessor import run_preprocessor +from yugabyte.command_util import shlex_join +from yugabyte.clang_tidy_runner import ClangTidyRunner +from yugabyte.compiler_parallel_runner import CompilerParallelRunner +from yugabyte.preprocessor import run_preprocessor # We create methods with this suffix in the tool class that implement various subcommands. diff --git a/build-support/check_python_syntax.py b/python/yugabyte/check_python_syntax.py similarity index 95% rename from build-support/check_python_syntax.py rename to python/yugabyte/check_python_syntax.py index cb6533191b94..f25b297b9b1b 100755 --- a/build-support/check_python_syntax.py +++ b/python/yugabyte/check_python_syntax.py @@ -20,11 +20,11 @@ import traceback -def horizontal_line(): +def horizontal_line() -> None: sys.stderr.write("-" * 80 + "\n") -if __name__ == '__main__': +def main() -> None: filename = sys.argv[1] if not os.path.exists(filename): sys.stderr.write(f"Python file does not exist, cannot check syntax: {filename}\n") @@ -40,3 +40,7 @@ def horizontal_line(): traceback.print_exc() horizontal_line() sys.exit(1) + + +if __name__ == '__main__': + main() diff --git a/python/yb/clang_tidy_runner.py b/python/yugabyte/clang_tidy_runner.py similarity index 96% rename from python/yb/clang_tidy_runner.py rename to python/yugabyte/clang_tidy_runner.py index a13c287819a9..a4d09b4fa940 100644 --- a/python/yb/clang_tidy_runner.py +++ b/python/yugabyte/clang_tidy_runner.py @@ -19,14 +19,14 @@ from overrides import overrides # type: ignore -from yb.parallel_task_runner import ParallelTaskRunner, ReportHelper -from yb.compile_commands import CompileCommand -from yb.common_util import YB_SRC_ROOT, shlex_join -from yb.file_util import read_file, write_file -from yb.command_util import decode_cmd_output - -from yb.type_util import checked_cast -from yb.compiler_invocation import run_compiler +from yugabyte.parallel_task_runner import ParallelTaskRunner, ReportHelper +from yugabyte.compile_commands import CompileCommand +from yugabyte.common_util import YB_SRC_ROOT, shlex_join +from yugabyte.file_util import read_file, write_file +from yugabyte.command_util import decode_cmd_output + +from yugabyte.type_util import checked_cast +from yugabyte.compiler_invocation import run_compiler def text_document_to_lines(s: str) -> List[str]: diff --git a/python/yb/cmake_cache.py b/python/yugabyte/cmake_cache.py similarity index 100% rename from python/yb/cmake_cache.py rename to python/yugabyte/cmake_cache.py diff --git a/python/yb/command_util.py b/python/yugabyte/command_util.py similarity index 98% rename from python/yb/command_util.py rename to python/yugabyte/command_util.py index 8814d22be5b5..41525fbfc65c 100644 --- a/python/yb/command_util.py +++ b/python/yugabyte/command_util.py @@ -25,8 +25,8 @@ from typing import Union, List, Optional, IO -from yb.common_util import shlex_join -from yb.file_util import mkdir_p +from yugabyte.common_util import shlex_join +from yugabyte.file_util import mkdir_p class ProgramResult: diff --git a/python/yb/common_util.py b/python/yugabyte/common_util.py similarity index 100% rename from python/yb/common_util.py rename to python/yugabyte/common_util.py diff --git a/python/yb/compile_commands.py b/python/yugabyte/compile_commands.py similarity index 97% rename from python/yb/compile_commands.py rename to python/yugabyte/compile_commands.py index fd45e17c395e..87caa26a7e2c 100644 --- a/python/yb/compile_commands.py +++ b/python/yugabyte/compile_commands.py @@ -18,9 +18,9 @@ from typing import List, Dict, Any, Optional, cast, Hashable -from yb.common_util import YB_SRC_ROOT -from yb.json_util import read_json_file, write_json_file -from yb.compiler_args import CompilerArguments +from yugabyte.common_util import YB_SRC_ROOT +from yugabyte.json_util import read_json_file, write_json_file +from yugabyte.compiler_args import CompilerArguments # We build PostgreSQL code in a separate directory (postgres_build) rsynced from the source tree to # support out-of-source builds. Then, after generating the compilation commands, we rewrite them diff --git a/python/yb/compile_commands_processor.py b/python/yugabyte/compile_commands_processor.py similarity index 96% rename from python/yb/compile_commands_processor.py rename to python/yugabyte/compile_commands_processor.py index 59e39cf807b3..91471bfdf6ca 100644 --- a/python/yb/compile_commands_processor.py +++ b/python/yugabyte/compile_commands_processor.py @@ -16,12 +16,12 @@ from typing import Set, Dict, List, Any, Optional, Hashable -from yb.postgres_build_util import POSTGRES_BUILD_SUBDIR -from yb.common_util import YB_SRC_ROOT, get_absolute_path_aliases, join_paths_if_needed -from yb.compiler_args import get_preprocessor_definition_values -from yb.include_path_rewriter import IncludePathRewriter -from yb.compile_commands import get_arguments_from_compile_command_item, CompileCommand -from yb.file_util import assert_absolute_path +from yugabyte.postgres_build_util import POSTGRES_BUILD_SUBDIR +from yugabyte.common_util import YB_SRC_ROOT, get_absolute_path_aliases, join_paths_if_needed +from yugabyte.compiler_args import get_preprocessor_definition_values +from yugabyte.include_path_rewriter import IncludePathRewriter +from yugabyte.compile_commands import get_arguments_from_compile_command_item, CompileCommand +from yugabyte.file_util import assert_absolute_path class CompileCommandProcessor: diff --git a/python/yb/compiler_args.py b/python/yugabyte/compiler_args.py similarity index 99% rename from python/yb/compiler_args.py rename to python/yugabyte/compiler_args.py index 930f511b9c08..4d4fb41cbafd 100644 --- a/python/yb/compiler_args.py +++ b/python/yugabyte/compiler_args.py @@ -15,7 +15,7 @@ import functools -from yb.command_util import shlex_join +from yugabyte.command_util import shlex_join from collections import defaultdict diff --git a/python/yb/compiler_invocation.py b/python/yugabyte/compiler_invocation.py similarity index 90% rename from python/yb/compiler_invocation.py rename to python/yugabyte/compiler_invocation.py index 158f8d4f37a1..42b0835ca623 100644 --- a/python/yb/compiler_invocation.py +++ b/python/yugabyte/compiler_invocation.py @@ -10,9 +10,9 @@ # or implied. See the License for the specific language governing permissions and limitations # under the License. -from yb.compile_commands import CompileCommand -from yb.command_util import ProgramResult, run_program -from yb.file_util import delete_file_if_exists +from yugabyte.compile_commands import CompileCommand +from yugabyte.command_util import ProgramResult, run_program +from yugabyte.file_util import delete_file_if_exists import copy diff --git a/python/yb/compiler_parallel_runner.py b/python/yugabyte/compiler_parallel_runner.py similarity index 93% rename from python/yb/compiler_parallel_runner.py rename to python/yugabyte/compiler_parallel_runner.py index 50e908983ccc..8839cf36f13f 100644 --- a/python/yb/compiler_parallel_runner.py +++ b/python/yugabyte/compiler_parallel_runner.py @@ -19,13 +19,13 @@ from overrides import overrides # type: ignore -from yb.parallel_task_runner import ParallelTaskRunner, ReportHelper -from yb.compile_commands import CompileCommand -from yb.common_util import YB_SRC_ROOT, shlex_join -from yb.file_util import read_file, write_file +from yugabyte.parallel_task_runner import ParallelTaskRunner, ReportHelper +from yugabyte.compile_commands import CompileCommand +from yugabyte.common_util import YB_SRC_ROOT, shlex_join +from yugabyte.file_util import read_file, write_file -from yb.type_util import checked_cast -from yb.compiler_invocation import run_compiler +from yugabyte.type_util import checked_cast +from yugabyte.compiler_invocation import run_compiler @dataclass diff --git a/build-support/dedup_thread_stacks.py b/python/yugabyte/dedup_thread_stacks.py similarity index 81% rename from build-support/dedup_thread_stacks.py rename to python/yugabyte/dedup_thread_stacks.py index eac53d229b4c..1630d4bed584 100755 --- a/build-support/dedup_thread_stacks.py +++ b/python/yugabyte/dedup_thread_stacks.py @@ -7,26 +7,32 @@ import re import sys +from typing import List, Optional, Dict -THREAD_HEADER_RE = re.compile('^Thread (\d+) \(LWP (\d+)\):$') + +THREAD_HEADER_RE = re.compile(r'^Thread (\d+) \(LWP (\d+)\):$') HEX_NUMBER_RE_STR = r'\b0x[0-9a-fA-f]+\b' STACK_FRAME_RE = re.compile(r'^#(\d+)\s+(.*)$') PARAM_VALUE_RE = re.compile(r'=' + HEX_NUMBER_RE_STR) class StackTrace: + thread_id: int + lwp_id: int + frames: List[str] + raw_frames: List[str] - def __init__(self, thread_id, lwp_id): + def __init__(self, thread_id: int, lwp_id: int) -> None: self.thread_id = thread_id self.lwp_id = lwp_id self.frames = [] self.raw_frames = [] - def append_line(self, line): + def append_line(self, line: str) -> None: self.raw_frames.append(line) self.frames.append(line) - def append_frame(self, index, line): + def append_frame(self, index: int, line: str) -> bool: if index == len(self.frames): self.raw_frames.append(line) line = PARAM_VALUE_RE.sub('=0x...', line) @@ -34,22 +40,24 @@ def append_frame(self, index, line): return True return False - def frames_key(self): + def frames_key(self) -> str: return "\n".join(self.frames) class Collector: + stacks: List[StackTrace] + current_stack: Optional[StackTrace] - def __init__(self): + def __init__(self) -> None: self.stacks = [] self.current_stack = None - def stack_finished(self): - if self.current_stack: + def stack_finished(self) -> None: + if self.current_stack is not None: self.stacks.append(self.current_stack) self.current_stack = None - def process_line(self, line): + def process_line(self, line: str) -> bool: """ :param line: a line from gdb output :return: True if the line was appended to the current stack trace, False if its format was @@ -75,14 +83,13 @@ def process_line(self, line): index = int(frame_match.group(1)) if self.current_stack: return self.current_stack.append_frame(index, line) - else: - return False + return False self.stack_finished() return False - def print_grouped_stacks(self): - groups = {} + def print_grouped_stacks(self) -> None: + groups: Dict[str, List[StackTrace]] = {} for stack in self.stacks: key = stack.frames_key() if key not in groups: diff --git a/python/yb/dep_graph_common.py b/python/yugabyte/dep_graph_common.py similarity index 99% rename from python/yb/dep_graph_common.py rename to python/yugabyte/dep_graph_common.py index d2f77024bc2f..6df6574a953f 100644 --- a/python/yb/dep_graph_common.py +++ b/python/yugabyte/dep_graph_common.py @@ -23,7 +23,7 @@ from enum import Enum from typing import Any, Set, FrozenSet, List, Optional, Dict, Union, Iterable, FrozenSet -from yb.common_util import ( +from yugabyte.common_util import ( append_to_list_in_dict, assert_sets_equal, assert_set_contains_all, @@ -35,7 +35,7 @@ is_ninja_build_root, ) -from yb.cmake_cache import CMakeCache, load_cmake_cache +from yugabyte.cmake_cache import CMakeCache, load_cmake_cache def make_extensions(exts_without_dot: List[str]) -> List[str]: diff --git a/python/yb/dependency_graph.py b/python/yugabyte/dependency_graph.py similarity index 99% rename from python/yb/dependency_graph.py rename to python/yugabyte/dependency_graph.py index e2b4426a30df..0202831f831d 100755 --- a/python/yb/dependency_graph.py +++ b/python/yugabyte/dependency_graph.py @@ -33,7 +33,7 @@ from typing import Optional, List, Dict, TypeVar, Set, Any, Iterable, cast from pathlib import Path -from yb.common_util import ( +from yugabyte.common_util import ( arg_str_to_bool, get_bool_env_var, group_by, @@ -41,13 +41,13 @@ make_set, shlex_join, ) -from yb.command_util import mkdir_p -from yb.source_files import ( +from yugabyte.command_util import mkdir_p +from yugabyte.source_files import ( CATEGORIES_NOT_CAUSING_RERUN_OF_ALL_TESTS, get_file_category, SourceFileCategory, ) -from yb.dep_graph_common import ( +from yugabyte.dep_graph_common import ( CMakeDepGraph, DependencyGraph, DepGraphConf, @@ -58,7 +58,7 @@ NodeType, ) from yugabyte_pycommon import WorkDirContext # type: ignore -from yb.lto import link_whole_program +from yugabyte.lto import link_whole_program class Command(Enum): diff --git a/python/yb/download_and_extract_archive.py b/python/yugabyte/download_and_extract_archive.py similarity index 94% rename from python/yb/download_and_extract_archive.py rename to python/yugabyte/download_and_extract_archive.py index e6273b2112c0..51454a2e4a77 100755 --- a/python/yb/download_and_extract_archive.py +++ b/python/yugabyte/download_and_extract_archive.py @@ -25,24 +25,23 @@ import re import logging import socket -import random import atexit import subprocess import argparse import tempfile import time import getpass -import platform -import fcntl import errno +from typing import List + g_verbose = False EXPECTED_ARCHIVE_EXTENSION = '.tar.gz' CHECKSUM_EXTENSION = '.sha256' -def remove_ignore_errors(file_path): +def remove_ignore_errors(file_path: str) -> None: file_path = os.path.abspath(file_path) if os.path.isfile(file_path): try: @@ -51,7 +50,7 @@ def remove_ignore_errors(file_path): logging.warning("Error removing %s: %s, ignoring", file_path, e) -def run_cmd(args): +def run_cmd(args: List[str]) -> None: if g_verbose: logging.info("Running command: %s", args) try: @@ -61,17 +60,17 @@ def run_cmd(args): raise -def validate_sha256sum(checksum_str): +def validate_sha256sum(checksum_str: str) -> None: if not re.match(r'^[0-9a-f]{64}$', checksum_str): raise ValueError("Invalid SHA256 checksum: '%s', expected 64 hex characters", checksum_str) -def read_file_and_strip(file_path): +def read_file_and_strip(file_path: str) -> str: with open(file_path) as f: return f.read().strip() -def compute_sha256sum(file_path): +def compute_sha256sum(file_path: str) -> str: cmd_line = None if sys.platform.startswith('linux'): cmd_line = ['sha256sum', file_path] @@ -85,7 +84,7 @@ def compute_sha256sum(file_path): return checksum_str -def verify_sha256sum(checksum_file_path, data_file_path): +def verify_sha256sum(checksum_file_path: str, data_file_path: str) -> bool: if not os.path.exists(checksum_file_path): raise IOError("Checksum file does not exist: %s" % checksum_file_path) @@ -114,7 +113,7 @@ def verify_sha256sum(checksum_file_path, data_file_path): return False -def download_url(url, dest_path): +def download_url(url: str, dest_path: str) -> None: start_time_sec = time.time() logging.info("Downloading %s to %s", url, dest_path) dest_dir = os.path.dirname(dest_path) @@ -127,7 +126,7 @@ def download_url(url, dest_path): logging.info("Downloaded %s to %s in %.1fs" % (url, dest_path, elapsed_sec)) -def move_file(src_path, dest_path): +def move_file(src_path: str, dest_path: str) -> None: if g_verbose: logging.info("Trying to move file %s to %s", src_path, dest_path) if not os.path.exists(src_path): @@ -145,7 +144,7 @@ def move_file(src_path, dest_path): os.rename(src_path, dest_path) -def check_dir_exists_and_is_writable(dir_path, description): +def check_dir_exists_and_is_writable(dir_path: str, description: str) -> None: if not os.path.isdir(dir_path): raise IOError("%s directory %s does not exist" % (description, dir_path)) if not os.access(dir_path, os.W_OK): @@ -154,7 +153,7 @@ def check_dir_exists_and_is_writable(dir_path, description): # From https://github.com/ianlini/mkdir-p/blob/master/mkdir_p/mkdir_p.py -def mkdir_p(path, mode=0o777): +def mkdir_p(path: str, mode: int = 0o777) -> None: try: os.makedirs(path, mode=mode) except OSError as exc: @@ -164,7 +163,7 @@ def mkdir_p(path, mode=0o777): raise -def exists_or_is_link(dest): +def exists_or_is_link(dest: str) -> bool: """ A file could be a link to a non-existent directory, or to a directory owned by a different user in a directory with sticky bit set. In such cases os.path.exists might return false, but @@ -173,7 +172,8 @@ def exists_or_is_link(dest): return os.path.exists(dest) or os.path.islink(dest) -def download_and_extract(url, dest_dir_parent, local_cache_dir, nfs_cache_dir): +def download_and_extract( + url: str, dest_dir_parent: str, local_cache_dir: str, nfs_cache_dir: str) -> None: tar_gz_name = os.path.basename(url) checksum_file_name = tar_gz_name + CHECKSUM_EXTENSION install_dir_name = tar_gz_name[:-len(EXPECTED_ARCHIVE_EXTENSION)] @@ -207,7 +207,7 @@ def download_and_extract(url, dest_dir_parent, local_cache_dir, nfs_cache_dir): mkdir_p(dest_dir_parent) tmp_dir = tempfile.mkdtemp(prefix=tmp_dir_prefix) - def cleanup(): + def cleanup() -> None: if os.path.isdir(tmp_dir): run_cmd(['rm', '-rf', tmp_dir]) @@ -277,7 +277,7 @@ def cleanup(): "installed it concurrently. This is OK." % ( orig_brew_home, dest_dir)) - def create_brew_symlink_if_needed(): + def create_brew_symlink_if_needed() -> None: brew_link_src = os.path.basename(orig_brew_home) # dest_dir will now be a symlink pointing to brew_link_src. We are NOT creating a # symlink inside dest_dir. @@ -342,7 +342,7 @@ def create_brew_symlink_if_needed(): logging.info("Installation of %s took %.1f sec", dest_dir, time.time() - start_time_sec) -def main(): +def main() -> None: # Created files/directories should be writable by the group. os.umask(2) diff --git a/python/yb/file_util.py b/python/yugabyte/file_util.py similarity index 100% rename from python/yb/file_util.py rename to python/yugabyte/file_util.py diff --git a/build-support/fix_paths_in_compile_errors.py b/python/yugabyte/fix_paths_in_compile_errors.py similarity index 79% rename from build-support/fix_paths_in_compile_errors.py rename to python/yugabyte/fix_paths_in_compile_errors.py index 40a22c886b94..8b5b037a741b 100755 --- a/build-support/fix_paths_in_compile_errors.py +++ b/python/yugabyte/fix_paths_in_compile_errors.py @@ -26,15 +26,20 @@ YB_SRC_ROOT = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) -if __name__ == '__main__': - def rewrite_path(match): - original_path = match.group() - if os.path.exists(original_path): - real_path = os.path.realpath(original_path) - if real_path.startswith(YB_SRC_ROOT + '/'): - return os.path.relpath(real_path, YB_SRC_ROOT) - return original_path +def rewrite_path(match: re.Match) -> str: + original_path = match.group() + if os.path.exists(original_path): + real_path = os.path.realpath(original_path) + if real_path.startswith(YB_SRC_ROOT + '/'): + return os.path.relpath(real_path, YB_SRC_ROOT) + return original_path + +def main() -> None: for line in sys.stdin: # We're using .rstrip() and not .strip() because we want to preserve original indentation. print(PATH_RE.sub(rewrite_path, line.rstrip())) + + +if __name__ == '__main__': + main() diff --git a/python/yb/fossa_analysis.py b/python/yugabyte/fossa_analysis.py old mode 100644 new mode 100755 similarity index 99% rename from python/yb/fossa_analysis.py rename to python/yugabyte/fossa_analysis.py index 21d8989d0454..c508ab48c3fe --- a/python/yb/fossa_analysis.py +++ b/python/yugabyte/fossa_analysis.py @@ -24,7 +24,7 @@ from packaging import version -from yb.common_util import ( +from yugabyte.common_util import ( YB_SRC_ROOT, get_thirdparty_dir, load_yaml_file, shlex_join, write_yaml_file, init_logging ) diff --git a/python/yb/gen_auto_flags_json.py b/python/yugabyte/gen_auto_flags_json.py similarity index 99% rename from python/yb/gen_auto_flags_json.py rename to python/yugabyte/gen_auto_flags_json.py index c5f8da1de506..1253ef3826e4 100755 --- a/python/yb/gen_auto_flags_json.py +++ b/python/yugabyte/gen_auto_flags_json.py @@ -27,7 +27,7 @@ from typing import List, Dict, cast from yugabyte_pycommon import run_program, WorkDirContext # type: ignore -from yb.common_util import init_logging +from yugabyte.common_util import init_logging def get_auto_flags( diff --git a/python/yb/gen_flags_metadata.py b/python/yugabyte/gen_flags_metadata.py similarity index 97% rename from python/yb/gen_flags_metadata.py rename to python/yugabyte/gen_flags_metadata.py index 0176d5006e1a..f151685e7d2a 100755 --- a/python/yb/gen_flags_metadata.py +++ b/python/yugabyte/gen_flags_metadata.py @@ -23,7 +23,7 @@ from yugabyte_pycommon import run_program, WorkDirContext # type: ignore -from yb.common_util import init_logging +from yugabyte.common_util import init_logging def main() -> None: diff --git a/python/yb/gen_initial_sys_catalog_snapshot.py b/python/yugabyte/gen_initial_sys_catalog_snapshot.py similarity index 98% rename from python/yb/gen_initial_sys_catalog_snapshot.py rename to python/yugabyte/gen_initial_sys_catalog_snapshot.py index 8e3d3ea9153b..8f75990cf8c9 100755 --- a/python/yb/gen_initial_sys_catalog_snapshot.py +++ b/python/yugabyte/gen_initial_sys_catalog_snapshot.py @@ -25,7 +25,7 @@ from yugabyte_pycommon import run_program, WorkDirContext, mkdir_p # type: ignore -from yb.common_util import YB_SRC_ROOT, init_logging +from yugabyte.common_util import YB_SRC_ROOT, init_logging def main() -> None: diff --git a/build-support/gen_version_info.py b/python/yugabyte/gen_version_info.py similarity index 88% rename from build-support/gen_version_info.py rename to python/yugabyte/gen_version_info.py index 9a79ac0cbdeb..1648b219ff3d 100755 --- a/build-support/gen_version_info.py +++ b/python/yugabyte/gen_version_info.py @@ -45,26 +45,29 @@ import sys import time import socket + +from typing import Optional + from time import strftime, localtime sys.path.append(os.path.join(os.path.dirname(os.path.dirname(__file__)), 'python')) -from yb.common_util import get_yb_src_root_from_build_root # noqa +from yugabyte.common_util import get_yb_src_root_from_build_root # noqa -def is_git_repo_clean(git_repo_dir): +def is_git_repo_clean(git_repo_dir: str) -> bool: return subprocess.call( "cd {} && git diff --quiet && git diff --cached --quiet".format( shlex.quote(git_repo_dir)), shell=True) == 0 -def boolean_to_json_str(bool_flag): +def boolean_to_json_str(bool_flag: bool) -> str: return str(bool_flag).lower() -def get_git_sha1(git_repo_dir): +def get_git_sha1(git_repo_dir: str) -> Optional[str]: try: sha1 = subprocess.check_output( 'cd {} && git rev-parse HEAD'.format(shlex.quote(git_repo_dir)), shell=True @@ -73,12 +76,14 @@ def get_git_sha1(git_repo_dir): if re.match(r'^[0-9a-f]{40}$', sha1): return sha1 logging.warning("Invalid git SHA1 in directory '%s': %s", git_repo_dir, sha1) + return None except Exception as e: logging.warning("Failed to get git SHA1 in directory: %s", git_repo_dir) + return None -def main(): +def main() -> int: logging.basicConfig( level=logging.INFO, format="[" + os.path.basename(__file__) + "] %(asctime)s %(levelname)s: %(message)s") @@ -94,6 +99,7 @@ def main(): hostname = socket.gethostname() build_time = "%s %s" % (strftime("%d %b %Y %H:%M:%S", localtime()), time.tzname[0]) + username: Optional[str] try: username = os.getlogin() except OSError as ex: @@ -105,7 +111,7 @@ def main(): except KeyError: username = os.getenv('USER') if not username: - id_output = subprocess.check_output('id').strip() + id_output = subprocess.check_output('id').decode('utf-8').strip() ID_OUTPUT_RE = re.compile(r'^uid=\d+[(]([^)]+)[)]\s.*') match = ID_OUTPUT_RE.match(id_output) if match: @@ -117,23 +123,29 @@ def main(): raise git_repo_dir = get_yb_src_root_from_build_root(os.getcwd(), must_succeed=False, verbose=True) - clean_repo = bool(git_repo_dir) and is_git_repo_clean(git_repo_dir) + if git_repo_dir: + clean_repo = is_git_repo_clean(git_repo_dir) + else: + clean_repo = False + git_hash: Optional[str] if args.git_hash: # Git hash provided on the command line. git_hash = args.git_hash elif 'YB_VERSION_INFO_GIT_SHA1' in os.environ: git_hash = os.environ['YB_VERSION_INFO_GIT_SHA1'] logging.info("Git SHA1 provided using the YB_VERSION_INFO_GIT_SHA1 env var: %s", git_hash) - else: + elif git_repo_dir: # No command line git hash, find it in the local git repository. git_hash = get_git_sha1(git_repo_dir) + else: + git_hash = None path_to_version_file = os.path.join( - os.path.dirname(os.path.realpath(__file__)), "..", "version.txt") + os.path.dirname(os.path.realpath(__file__)), "..", "..", "version.txt") with open(path_to_version_file) as version_file: version_string = version_file.read().strip() - match = re.match("(\d+\.\d+\.\d+\.\d+)", version_string) + match = re.match(r"(\d+\.\d+\.\d+\.\d+)", version_string) if not match: parser.error("Invalid version specified: {}".format(version_string)) sys.exit(1) @@ -178,7 +190,7 @@ def main(): except IOError as ex: if attempts_left == 0: raise ex - if 'Resource temporarily unavailable' in ex.message: + if 'Resource temporarily unavailable' in str(ex): time.sleep(0.1) attempts_left -= 1 diff --git a/python/yb/include_path_rewriter.py b/python/yugabyte/include_path_rewriter.py similarity index 97% rename from python/yb/include_path_rewriter.py rename to python/yugabyte/include_path_rewriter.py index 4b238d58e008..6871dab9531e 100644 --- a/python/yb/include_path_rewriter.py +++ b/python/yugabyte/include_path_rewriter.py @@ -14,11 +14,11 @@ from typing import List, Set, Optional -from yb.postgres_build_util import POSTGRES_BUILD_SUBDIR -from yb.common_util import YB_SRC_ROOT +from yugabyte.postgres_build_util import POSTGRES_BUILD_SUBDIR +from yugabyte.common_util import YB_SRC_ROOT -from yb.compiler_args import get_include_path_arg -from yb.file_util import assert_absolute_path +from yugabyte.compiler_args import get_include_path_arg +from yugabyte.file_util import assert_absolute_path class IncludePathRewriter: diff --git a/python/yb/install_wrapper.py b/python/yugabyte/install_wrapper.py similarity index 98% rename from python/yb/install_wrapper.py rename to python/yugabyte/install_wrapper.py index aefc9fbf6350..ffc12b6eceaa 100755 --- a/python/yb/install_wrapper.py +++ b/python/yugabyte/install_wrapper.py @@ -24,7 +24,7 @@ sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) # nopep8 -from yb.common_util import shlex_join, are_files_equal +from yugabyte.common_util import shlex_join, are_files_equal def main() -> None: diff --git a/python/yugabyte/is_same_path.py b/python/yugabyte/is_same_path.py new file mode 100755 index 000000000000..548a85ff5639 --- /dev/null +++ b/python/yugabyte/is_same_path.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python + +# Copyright (c) Yugabyte, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except +# in compliance with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed under the License +# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +# or implied. See the License for the specific language governing permissions and limitations +# under the License. + +import os +import sys + + +if __name__ == '__main__': + if os.path.realpath(sys.argv[1]) == os.path.realpath(sys.argv[2]): + sys.exit(0) + + sys.exit(1) diff --git a/python/yb/json_util.py b/python/yugabyte/json_util.py similarity index 96% rename from python/yb/json_util.py rename to python/yugabyte/json_util.py index 67bb1b072b70..9f983e9a384c 100644 --- a/python/yb/json_util.py +++ b/python/yugabyte/json_util.py @@ -15,7 +15,7 @@ from typing import Any, Optional -from yb.logging_util import log_info +from yugabyte.logging_util import log_info JSON_INDENTATION = 2 diff --git a/build-support/kill_long_running_minicluster_daemons.py b/python/yugabyte/kill_long_running_minicluster_daemons.py similarity index 98% rename from build-support/kill_long_running_minicluster_daemons.py rename to python/yugabyte/kill_long_running_minicluster_daemons.py index f294ef756c08..c50384de4ae0 100755 --- a/build-support/kill_long_running_minicluster_daemons.py +++ b/python/yugabyte/kill_long_running_minicluster_daemons.py @@ -29,7 +29,7 @@ PS_OUTPUT_LINE_RE = re.compile(r'^\s*(\d+)\s+\S+\s+(\S+\s+\d+\s\d+:\d+:\d+ \d+)\s+(.*)$') -if __name__ == '__main__': +def main() -> None: logging.basicConfig( level=logging.INFO, format="[%(filename)s:%(lineno)d] %(asctime)s %(levelname)s: %(message)s") @@ -88,3 +88,7 @@ "Examined %d processes, found %d masters/tservers, killed %d long-running masters/tservers" ", failed to kill %d processes.", num_processes_found, num_daemons_found, num_killed, num_failed_to_kill) + + +if __name__ == '__main__': + main() diff --git a/python/yb/library_packager.py b/python/yugabyte/library_packager.py similarity index 98% rename from python/yb/library_packager.py rename to python/yugabyte/library_packager.py index 108b871d0865..c37d63510bbf 100644 --- a/python/yb/library_packager.py +++ b/python/yugabyte/library_packager.py @@ -22,7 +22,7 @@ Run doctest tests as follows: - python -m doctest python/yb/library_packager.py + python -m doctest python/yugabyte/library_packager.py """ @@ -38,14 +38,14 @@ from functools import total_ordering from queue import Queue -from yb.command_util import run_program, mkdir_p, copy_deep -from yb.common_util import ( +from yugabyte.command_util import run_program, mkdir_p, copy_deep +from yugabyte.common_util import ( get_thirdparty_dir, sorted_grouped_by, YB_SRC_ROOT, ) -from yb.rpath import set_rpath, remove_rpath -from yb.linuxbrew import get_linuxbrew_home, using_linuxbrew, LinuxbrewHome +from yugabyte.rpath import set_rpath, remove_rpath +from yugabyte.linuxbrew import get_linuxbrew_home, using_linuxbrew, LinuxbrewHome from typing import List, Optional, Any, Set, Tuple diff --git a/python/yb/linuxbrew.py b/python/yugabyte/linuxbrew.py similarity index 98% rename from python/yb/linuxbrew.py rename to python/yugabyte/linuxbrew.py index 6259202e7537..5b5549bcf1da 100644 --- a/python/yb/linuxbrew.py +++ b/python/yugabyte/linuxbrew.py @@ -19,7 +19,7 @@ import sys import subprocess -from yb.common_util import safe_path_join, YB_SRC_ROOT, shlex_join +from yugabyte.common_util import safe_path_join, YB_SRC_ROOT, shlex_join from typing import Optional diff --git a/build-support/list_packaged_targets.py b/python/yugabyte/list_packaged_targets.py similarity index 88% rename from build-support/list_packaged_targets.py rename to python/yugabyte/list_packaged_targets.py index abdcd0c645b3..04fc707d52d5 100755 --- a/build-support/list_packaged_targets.py +++ b/python/yugabyte/list_packaged_targets.py @@ -7,8 +7,8 @@ import re import argparse -from yb.release_util import read_release_manifest, filter_bin_items -from yb.optional_components import ( +from yugabyte.release_util import read_release_manifest, filter_bin_items +from yugabyte.optional_components import ( add_optional_component_arguments, optional_components_from_args, ) diff --git a/python/yb/log/log_iter.py b/python/yugabyte/log/log_iter.py similarity index 100% rename from python/yb/log/log_iter.py rename to python/yugabyte/log/log_iter.py diff --git a/python/yugabyte/log/summarize_tcmalloc_trace.py b/python/yugabyte/log/summarize_tcmalloc_trace.py new file mode 100644 index 000000000000..63fecc30ca09 --- /dev/null +++ b/python/yugabyte/log/summarize_tcmalloc_trace.py @@ -0,0 +1,96 @@ +# +# Copyright (c) YugaByte, Inc. +# # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except +# in compliance with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed under the License +# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +# or implied. See the License for the specific language governing permissions and limitations +# under the License. +# +""" +Combs a log generated with tcmalloc_trace_enabled and provides statistics about each stack +trace found. +""" + +import argparse +import itertools +import re +import sys +from typing import Dict, Iterator, List + +from yugabyte.log.log_iter import LogIterator + + +MALLOC_CALL_RE = re.compile(r'^Malloc Call: size = (?P\d+)$') + +EXCLUDED_CALLS = [ + r' MallocHook::.*', + r' tcmalloc::*', +] + +EXCLUDED_CALL_RE = [re.compile(regex) for regex in EXCLUDED_CALLS] + + +def check_excluded(s: str) -> bool: + return any(r.search(s) for r in EXCLUDED_CALL_RE) + + +def get_malloc_trace_dict(lines: Iterator[str]) -> Dict[str, List[int]]: + stacks: Dict[str, List[int]] = {} + + for log in LogIterator(lines): + if len(log.message_lines) < 2: + continue + first_line = log.message_lines[0] + + match = MALLOC_CALL_RE.match(first_line) + if match: + stack_trace = '\n'.join(line for line in log.message_lines if not check_excluded(line)) + if stack_trace not in stacks: + stacks[stack_trace] = [] + stacks[stack_trace].append(int(match.group('size'))) + + return stacks + + +def print_malloc_trace_summary(trace_dict: Dict[str, List[int]], limit: int = -1) -> None: + trace_stats = sorted([ + (trace, sum(sizes), len(sizes), min(sizes), max(sizes)) + for trace, sizes in trace_dict.items() + ], key=lambda x: -x[1]) + + overall_total_alloc = sum(x[1] for x in trace_stats) + print(f'TOTAL: {overall_total_alloc}\n') + + limit = limit if limit != -1 else len(trace_stats) + for stack, total_alloc, num_alloc, min_alloc, max_alloc in trace_stats[:limit]: + avg_alloc = total_alloc / num_alloc + print( + f'SIZE: {total_alloc} over {num_alloc} allocations ' + f'(min {min_alloc}, avg {avg_alloc}, max {max_alloc})\n' + f'{stack}\n') + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + '--limit', + help='Max number of stack traces to output.', + type=int, + default=-1) + parser.add_argument( + 'files', + help='Log files to scan through. Reads from stdin if not specified.', + nargs='*') + args = parser.parse_args() + + files = (open(file, 'rt') for file in args.files) if args.files else [sys.stdin] + lines = itertools.chain.from_iterable(files) + print_malloc_trace_summary(get_malloc_trace_dict(lines), args.limit) + + +if __name__ == '__main__': + main() diff --git a/python/yb/logging_util.py b/python/yugabyte/logging_util.py similarity index 98% rename from python/yb/logging_util.py rename to python/yugabyte/logging_util.py index babdafce7782..c0daf019a419 100644 --- a/python/yb/logging_util.py +++ b/python/yugabyte/logging_util.py @@ -13,7 +13,7 @@ import logging import os -from yb.common_util import get_home_dir_aliases, YB_SRC_ROOT +from yugabyte.common_util import get_home_dir_aliases, YB_SRC_ROOT from typing import Dict, List, Any, Optional, cast diff --git a/python/yb/lto.py b/python/yugabyte/lto.py similarity index 99% rename from python/yb/lto.py rename to python/yugabyte/lto.py index 2418cd084395..c71daa433c0a 100644 --- a/python/yb/lto.py +++ b/python/yugabyte/lto.py @@ -18,16 +18,16 @@ import psutil # type: ignore from typing import Iterable, Set, List, Optional, Tuple, Dict -from yb.dep_graph_common import ( +from yugabyte.dep_graph_common import ( DependencyGraph, Node, NodeType, DYLIB_SUFFIX, ) -from yb.common_util import shlex_join, create_symlink -from yb.file_util import read_file, write_file -from yb.build_paths import BuildPaths +from yugabyte.common_util import shlex_join, create_symlink +from yugabyte.file_util import read_file, write_file +from yugabyte.build_paths import BuildPaths from yugabyte_pycommon import WorkDirContext # type: ignore diff --git a/python/yb/mac_library_packager.py b/python/yugabyte/mac_library_packager.py similarity index 99% rename from python/yb/mac_library_packager.py rename to python/yugabyte/mac_library_packager.py index c9865ad6ef04..3d21178d75e3 100644 --- a/python/yb/mac_library_packager.py +++ b/python/yugabyte/mac_library_packager.py @@ -9,8 +9,8 @@ from argparse import ArgumentParser -from yb.command_util import run_program -from yb.common_util import get_thirdparty_dir +from yugabyte.command_util import run_program +from yugabyte.common_util import get_thirdparty_dir from typing import List, Dict, Optional, Tuple diff --git a/build-support/make_rpath_relative.py b/python/yugabyte/make_rpath_relative.py similarity index 84% rename from build-support/make_rpath_relative.py rename to python/yugabyte/make_rpath_relative.py index 2db9f7168434..6af0d68ab76a 100755 --- a/build-support/make_rpath_relative.py +++ b/python/yugabyte/make_rpath_relative.py @@ -15,16 +15,18 @@ import os import sys +from typing import List + RPATH_ARG_PREFIX = '-Wl,-rpath,' -def path_to_components(p): +def path_to_components(p: str) -> List[str]: items = p.split('/') return [item for item in items if len(item) > 0] -def num_common_path_entries(path1, path2): +def num_common_path_entries(path1: str, path2: str) -> int: """ :return The number of common initial path entries for the two given paths. """ @@ -39,30 +41,30 @@ def num_common_path_entries(path1, path2): class RpathRelativizer: - def __init__(self, output_path): + def __init__(self, output_path: str) -> None: self.output_dir_abspath = os.path.dirname(os.path.abspath(output_path)) - def relativize_arg(self, rpath_arg): + def relativize_arg(self, rpath_arg: str) -> str: """ :param rpath_arg: a compiler argument of the '-Wl,-rpath,...' form. """ - if not rpath_option.startswith(RPATH_ARG_PREFIX): + if not rpath_arg.startswith(RPATH_ARG_PREFIX): raise ValueError('RPATH option does not start with %s: %s' % ( - RPATH_ARG_PREFIX, rpath_option + RPATH_ARG_PREFIX, rpath_arg )) relative_rpath_entries = [ self.relativize_rpath_entry(entry) - for entry in rpath_option[len(RPATH_ARG_PREFIX):].split(',') + for entry in rpath_arg[len(RPATH_ARG_PREFIX):].split(',') if entry ] return RPATH_ARG_PREFIX + ','.join(relative_rpath_entries) - def relativize_rpath_entry(self, entry): + def relativize_rpath_entry(self, entry: str) -> str: if not entry: - entry + return entry if os.path.isdir(entry): # Try both absolute path and "real" path to support cases when the entire source @@ -86,10 +88,14 @@ def relativize_rpath_entry(self, entry): return entry -if __name__ == '__main__': +def main() -> None: if len(sys.argv) != 3: raise RuntimeError("Exactly 2 arguments expected: output path and rpath linker option") output_path, rpath_option = sys.argv[1:] relativizer = RpathRelativizer(output_path) print(relativizer.relativize_arg(rpath_option)) + + +if __name__ == '__main__': + main() diff --git a/python/yb/optional_components.py b/python/yugabyte/optional_components.py similarity index 100% rename from python/yb/optional_components.py rename to python/yugabyte/optional_components.py diff --git a/python/yb/os_versions.py b/python/yugabyte/os_versions.py similarity index 100% rename from python/yb/os_versions.py rename to python/yugabyte/os_versions.py diff --git a/python/yb/parallel_task_runner.py b/python/yugabyte/parallel_task_runner.py similarity index 98% rename from python/yb/parallel_task_runner.py rename to python/yugabyte/parallel_task_runner.py index b0a1ab9f1fcc..65c8554cfb67 100644 --- a/python/yb/parallel_task_runner.py +++ b/python/yugabyte/parallel_task_runner.py @@ -22,9 +22,9 @@ from typing import List, Any, Type from dataclasses import dataclass -from yb.type_util import assert_type +from yugabyte.type_util import assert_type -from yb.file_util import write_file +from yugabyte.file_util import write_file @dataclass diff --git a/build-support/parse_test_failure.py b/python/yugabyte/parse_test_failure.py similarity index 81% rename from build-support/parse_test_failure.py rename to python/yugabyte/parse_test_failure.py index 3662f5305d93..785659260288 100755 --- a/build-support/parse_test_failure.py +++ b/python/yugabyte/parse_test_failure.py @@ -35,11 +35,15 @@ # This script parses a test log (provided on stdin) and returns # a summary of the error which caused the test to fail. -from xml.sax.saxutils import quoteattr import argparse +import os import re import sys -import os.path + +from xml.sax.saxutils import quoteattr + +from typing import Iterable, List, Pattern, Match, Dict, Tuple, Optional, TextIO + # Read at most 100MB of a test log. # Rarely would this be exceeded, but we don't want to end up @@ -62,39 +66,39 @@ UNRECOGNIZED_ERROR = "Unrecognized error type. Please see the error log for more information." -def consume_rest(line_iter): +def consume_rest(line_iter: Iterable[Match]) -> List[str]: """ Consume and return the rest of the lines in the iterator. """ - return [l.group(0) for l in line_iter] + return [line_match.group(0) for line_match in line_iter] -def consume_until(line_iter, end_re): +def consume_until(line_iter: Iterable[Match], end_re: Pattern) -> List[str]: """ Consume and return lines from the iterator until one matches 'end_re'. The line matching 'end_re' will not be returned, but will be consumed. """ - ret = [] - for l in line_iter: - line = l.group(0) + ret: List[str] = [] + for line_match in line_iter: + line = line_match.group(0) if end_re.search(line): break ret.append(line) return ret -def remove_glog_lines(lines): +def remove_glog_lines(lines: List[str]) -> List[str]: """ Remove any lines from the list of strings which appear to be GLog messages. """ - return [l for l in lines if not GLOG_LINE_RE.search(l)] + return [line for line in lines if not GLOG_LINE_RE.search(line)] -def record_error(errors, name, error): +def record_error(errors: Dict[Optional[str], List[str]], name: Optional[str], error: str) -> None: errors.setdefault(name, []).append(error) -def extract_failures(log_text): +def extract_failures(log_text: str) -> Tuple[List[str], Dict[Optional[str], List[str]]]: cur_test_case = None tests_seen = set() - tests_seen_in_order = list() - errors_by_test = dict() + tests_seen_in_order: List[str] = list() + errors_by_test: Dict[Optional[str], List[str]] = dict() # Iterate over the lines, using finditer instead of .split() # so that we don't end up doubling memory usage. @@ -132,23 +136,28 @@ def extract_failures(log_text): # Look for test failures # - slight micro-optimization to check for substring before running the regex - m = 'Failure' in line and TEST_FAILURE_RE.search(line) - if m: - error_signature = m.group(0) + "\n" - error_signature += "\n".join(remove_glog_lines( - consume_until(line_iter, END_TESTCASE_RE))) - record_error(errors_by_test, cur_test_case, error_signature) + if 'Failure' in line: + m = TEST_FAILURE_RE.search(line) + if m: + error_signature = m.group(0) + "\n" + error_signature += "\n".join(remove_glog_lines( + consume_until(line_iter, END_TESTCASE_RE))) + record_error(errors_by_test, cur_test_case, error_signature) # Look for fatal log messages (including CHECK failures) # - slight micro-optimization to check for 'F' before running the regex - m = line and line[0] == 'F' and FATAL_LOG_RE.search(line) - if m: - error_signature = m.group(1) + "\n" - remaining_lines = consume_rest(line_iter) - remaining_lines = [l for l in remaining_lines if STACKTRACE_ELEM_RE.search(l) - and not IGNORED_STACKTRACE_ELEM_RE.search(l)] - error_signature += "\n".join(remaining_lines) - record_error(errors_by_test, cur_test_case, error_signature) + if line and line[0] == 'F': + m = FATAL_LOG_RE.search(line) + if m: + error_signature = m.group(1) + "\n" + remaining_lines = consume_rest(line_iter) + remaining_lines = [ + line for line in remaining_lines + if STACKTRACE_ELEM_RE.search(line) and + not IGNORED_STACKTRACE_ELEM_RE.search(line) + ] + error_signature += "\n".join(remaining_lines) + record_error(errors_by_test, cur_test_case, error_signature) # Look for leak check summary (comes at the end of a log, not part of a single test) m = LEAK_CHECK_SUMMARY_RE.search(line) @@ -171,7 +180,7 @@ def extract_failures(log_text): # Return failure summary formatted as text. -def text_failure_summary(tests, errors_by_test): +def text_failure_summary(tests: List[str], errors_by_test: Dict[Optional[str], List[str]]) -> str: msg = '' for test_name in tests: if test_name not in errors_by_test: @@ -183,19 +192,14 @@ def text_failure_summary(tests, errors_by_test): return msg -# Parse log lines and return failure summary formatted as text. -# -# This helper function is part of a public API called from test_result_server.py -def extract_failure_summary(log_text): - (tests, errors_by_test) = extract_failures(log_text) - return text_failure_summary(tests, errors_by_test) - - # Print failure summary based on desired output format. # 'tests' is a list of all tests run (in order), not just the failed ones. # This allows us to print the test results in the order they were run. # 'errors_by_test' is a dict of lists, keyed by test name. -def print_failure_summary(tests, errors_by_test, is_xml): +def print_failure_summary( + tests: List[str], + errors_by_test: Dict[Optional[str], List[str]], + is_xml: bool) -> None: # Plain text dump. if not is_xml: sys.stdout.write(text_failure_summary(tests, errors_by_test)) @@ -246,7 +250,7 @@ def print_failure_summary(tests, errors_by_test, is_xml): print('') -def main(): +def main() -> None: parser = argparse.ArgumentParser() parser.add_argument( "-x", "--xml", @@ -265,11 +269,15 @@ def main(): test_case_id = args.junit_test_case_id + in_file: Optional[TextIO] if args.path: in_file = open(args.path) if os.path.isfile(args.path) else None else: in_file = sys.stdin + tests: List[str] + errors_by_test: Dict[Optional[str], List[str]] + if in_file: log_text = in_file.read(MAX_MEMORY) (tests, errors_by_test) = extract_failures(log_text) diff --git a/build-support/pg_regress_postprocess_output.py b/python/yugabyte/pg_regress_postprocess_output.py similarity index 98% rename from build-support/pg_regress_postprocess_output.py rename to python/yugabyte/pg_regress_postprocess_output.py index 3bb4f5775823..a28eb45dcac8 100755 --- a/build-support/pg_regress_postprocess_output.py +++ b/python/yugabyte/pg_regress_postprocess_output.py @@ -18,7 +18,7 @@ SANITIZER_SEPARATOR_LINE = '-' * 53 -def main(): +def main() -> None: file_path = sys.argv[1] if not os.path.exists(file_path): @@ -38,7 +38,7 @@ def main(): # 2) Also mask any uuids since they are randomly generated and will vary across runs. lines = [line.rstrip() for line in lines] - def mask_uuid4s(unmasked_line): + def mask_uuid4s(unmasked_line: str) -> str: uuid_start_indices = [] line_copy_with_uuid4s_masked = "" for m in re.finditer( diff --git a/python/yb/postgres_build_util.py b/python/yugabyte/postgres_build_util.py similarity index 100% rename from python/yb/postgres_build_util.py rename to python/yugabyte/postgres_build_util.py diff --git a/python/yb/postprocess_test_result.py b/python/yugabyte/postprocess_test_result.py similarity index 99% rename from python/yb/postprocess_test_result.py rename to python/yugabyte/postprocess_test_result.py index b690a3917b3e..00037fbe6c90 100755 --- a/python/yb/postprocess_test_result.py +++ b/python/yugabyte/postprocess_test_result.py @@ -30,7 +30,7 @@ from typing import Any, Dict, AnyStr sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -from yb.common_util import init_logging # noqa +from yugabyte.common_util import init_logging # noqa # Example test failure (from C++) diff --git a/python/yb/preprocessor.py b/python/yugabyte/preprocessor.py similarity index 93% rename from python/yb/preprocessor.py rename to python/yugabyte/preprocessor.py index c2a3959a29e1..cd95f1b195a7 100644 --- a/python/yb/preprocessor.py +++ b/python/yugabyte/preprocessor.py @@ -12,8 +12,8 @@ import copy -from yb.compile_commands import CompileCommand -from yb.command_util import run_program, ProgramResult +from yugabyte.compile_commands import CompileCommand +from yugabyte.command_util import run_program, ProgramResult from typing import List, Tuple diff --git a/python/yb/process_tree_supervisor.py b/python/yugabyte/process_tree_supervisor.py similarity index 91% rename from python/yb/process_tree_supervisor.py rename to python/yugabyte/process_tree_supervisor.py index bea9583d9390..2c255d79db19 100755 --- a/python/yb/process_tree_supervisor.py +++ b/python/yugabyte/process_tree_supervisor.py @@ -22,6 +22,8 @@ import os import logging +from typing import List, Any, Optional, Set + # https://stackoverflow.com/questions/2549939/get-signal-names-from-numbers-in-python/35996948 SIGNAL_TO_NAME = dict( @@ -35,29 +37,32 @@ g_signal_caught = None -def get_cmdline(process): +def get_cmdline(process: Any) -> Optional[List[str]]: try: return process.cmdline() except psutil.NoSuchProcess as e: logging.warning("Newly added child process disappeared right away") + return None except psutil.AccessDenied as e: logging.warning( "Access denied trying to get the command line for pid %d (%s), " "ignoring this process", process.pid, str(e)) + return None except OSError as e: logging.warning( "OSError trying to get the command line for pid %d (%s), ignoring this process", process.pid, str(e)) + return None -def get_cmdline_for_log(process): +def get_cmdline_for_log(process: Any) -> List[str]: cmdline = get_cmdline(process) if cmdline: return cmdline - return '[failed to get cmdline for pid %d]' % process.pid + return ['failed to get cmdline for pid %d' % process.pid] -def get_process_by_pid(pid): +def get_process_by_pid(pid: int) -> Optional[psutil.Process]: try: process = psutil.Process(pid) if process.status() == psutil.STATUS_ZOMBIE: @@ -68,13 +73,18 @@ def get_process_by_pid(pid): class ProcessTreeSupervisor(): - def __init__(self, args): + args: argparse.Namespace + ancestor_pid: int + pid_set: Set[int] + my_pid: int + + def __init__(self, args: argparse.Namespace) -> None: self.args = args self.ancestor_pid = args.pid self.pid_set = set([self.ancestor_pid]) self.my_pid = os.getpid() - def run(self): + def run(self) -> None: start_time_sec = time.time() while g_signal_caught is None: elapsed_time_sec = time.time() - start_time_sec @@ -129,7 +139,7 @@ def run(self): time.sleep(1) self.handle_termination() - def new_process_found(self, process): + def new_process_found(self, process: psutil.Process) -> bool: pid = process.pid cmdline = get_cmdline(process) if not cmdline: @@ -138,17 +148,17 @@ def new_process_found(self, process): logging.info("Tracking a child pid: %s: %s", pid, cmdline) return True - def process_terminated(self, existing_pid): + def process_terminated(self, existing_pid: int) -> None: logging.info("Process finished by itself: %d", existing_pid) - def report_stray_process(self, process, will_kill=False): + def report_stray_process(self, process: psutil.Process, will_kill: bool = False) -> None: # YB_STRAY_PROCESS is something we'll grep for in an enclosing script. logging.info( "YB_STRAY_PROCESS: Found a stray process%s: pid: %d, command line: %s", (', killing' if will_kill else ', not killing'), process.pid, get_cmdline_for_log(process)) - def handle_termination(self): + def handle_termination(self) -> None: killed_pids = [] for pid in self.pid_set: if pid != self.ancestor_pid: @@ -176,12 +186,12 @@ def handle_termination(self): logging.info("All %d stray processes successfully killed", len(killed_pids)) -def signal_handler(signum, frame): +def signal_handler(signum: int, unused_frame: Any) -> None: global g_signal_caught g_signal_caught = signum -def main(): +def main() -> None: signal.signal(signal.SIGUSR1, signal_handler) signal.signal(signal.SIGTERM, signal_handler) signal.signal(signal.SIGINT, signal_handler) diff --git a/python/yugabyte/py.typed b/python/yugabyte/py.typed new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/python/yb/release_util.py b/python/yugabyte/release_util.py similarity index 98% rename from python/yb/release_util.py rename to python/yugabyte/release_util.py index ce23992cde24..6e10b9908ac5 100644 --- a/python/yb/release_util.py +++ b/python/yugabyte/release_util.py @@ -16,13 +16,13 @@ from subprocess import call, check_output from xml.dom import minidom -from yb.command_util import run_program, mkdir_p, copy_deep -from yb.common_util import ( +from yugabyte.command_util import run_program, mkdir_p, copy_deep +from yugabyte.common_util import ( get_thirdparty_dir, get_compiler_type_from_build_root, YB_SRC_ROOT, ) -from yb.optional_components import OptionalComponents +from yugabyte.optional_components import OptionalComponents from typing import Dict, Any, Optional, cast, List diff --git a/python/yb/remote.py b/python/yugabyte/remote.py similarity index 99% rename from python/yb/remote.py rename to python/yugabyte/remote.py index b7c70b003667..033bd31c4ffe 100644 --- a/python/yb/remote.py +++ b/python/yugabyte/remote.py @@ -15,7 +15,7 @@ from typing import Sequence, Union, Tuple, List, Set, Optional, Dict, Any -from yb.common_util import shlex_join +from yugabyte.common_util import shlex_join REMOTE_BUILD_HOST_ENV_VAR = 'YB_REMOTE_BUILD_HOST' DEFAULT_UPSTREAM = 'origin' diff --git a/python/yb/rewrite_test_log.py b/python/yugabyte/rewrite_test_log.py similarity index 99% rename from python/yb/rewrite_test_log.py rename to python/yugabyte/rewrite_test_log.py index e2904f8b562c..7890c427ac2a 100755 --- a/python/yb/rewrite_test_log.py +++ b/python/yugabyte/rewrite_test_log.py @@ -30,7 +30,7 @@ from typing import List, Optional, Set, Any, Tuple, Dict sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -from yb.common_util import init_logging, shlex_join # noqa +from yugabyte.common_util import init_logging, shlex_join # noqa UUID_RE_STR = '[0-9a-f]{32}' TABLET_OR_PEER_ID_RE_STR = r'\b[T|P] [0-9a-f]{32}\b' diff --git a/python/yb/rpath.py b/python/yugabyte/rpath.py similarity index 96% rename from python/yb/rpath.py rename to python/yugabyte/rpath.py index 69af7f7ba3d4..7749dc67e48c 100644 --- a/python/yb/rpath.py +++ b/python/yugabyte/rpath.py @@ -18,8 +18,8 @@ from sys_detection import local_sys_conf, OsReleaseVars -from yb.common_util import find_executable -from yb.linuxbrew import get_linuxbrew_home, LinuxbrewHome +from yugabyte.common_util import find_executable +from yugabyte.linuxbrew import get_linuxbrew_home, LinuxbrewHome g_chrpath_path_initialized: bool = False diff --git a/python/yb/run_pvs_studio_analyzer.py b/python/yugabyte/run_pvs_studio_analyzer.py similarity index 97% rename from python/yb/run_pvs_studio_analyzer.py rename to python/yugabyte/run_pvs_studio_analyzer.py index 1d9c250d4364..126ded775f60 100755 --- a/python/yb/run_pvs_studio_analyzer.py +++ b/python/yugabyte/run_pvs_studio_analyzer.py @@ -25,9 +25,9 @@ from overrides import overrides from yugabyte_pycommon import mkdir_p # type: ignore -from yb.common_util import YB_SRC_ROOT, find_executable, rm_rf, check_call_and_log -from yb.tool_base import YbBuildToolBase -from yb.compile_commands import ( +from yugabyte.common_util import YB_SRC_ROOT, find_executable, rm_rf, check_call_and_log +from yugabyte.tool_base import YbBuildToolBase +from yugabyte.compile_commands import ( get_compile_commands_file_path, COMBINED_RAW_DIR_NAME, filter_compile_commands) diff --git a/build-support/run_tests_on_spark.py b/python/yugabyte/run_tests_on_spark.py similarity index 98% rename from build-support/run_tests_on_spark.py rename to python/yugabyte/run_tests_on_spark.py index c76499af1cf8..6316d1e7f8f7 100755 --- a/build-support/run_tests_on_spark.py +++ b/python/yugabyte/run_tests_on_spark.py @@ -23,7 +23,7 @@ Run all C++ tests: "$SPARK_INSTALLATION_DIR/bin/spark-submit" \ - build-support/run_tests_on_spark.py \ + python/yugabyte/run_tests_on_spark.py \ --spark-master-url=spark://$SPARK_HOST:$SPARK_PORT \ --build-root "$PWD/build/release-gcc-dynamic-ninja" \ --verbose \ @@ -36,7 +36,7 @@ Run Java tests satisfying a particular regex: "$SPARK_INSTALLATION_DIR/bin/spark-submit" \ - build-support/run_tests_on_spark.py \ + python/yugabyte/run_tests_on_spark.py \ --spark-master-url=spark://$SPARK_HOST:$SPARK_PORT \ --build-root "$PWD/build/release-gcc-dynamic-ninja" \ --verbose \ @@ -120,9 +120,9 @@ def wait_for_path_to_exist(target_path: str) -> None: sys.path.append(os.path.join(os.path.dirname(__file__), '..', 'python')) -from yb import yb_dist_tests # noqa -from yb import command_util # noqa -from yb.common_util import set_to_comma_sep_str, is_macos # noqa +from yugabyte import yb_dist_tests # noqa +from yugabyte import command_util # noqa +from yugabyte.common_util import set_to_comma_sep_str, is_macos # noqa # Special Jenkins environment variables. They are propagated to tasks running in a distributed way # on Spark. @@ -251,14 +251,17 @@ def init_spark_context(details: List[str] = []) -> None: details.append('URL: {}'.format(os.environ['BUILD_URL'])) spark_context = SparkContext(spark_master_url, "YB tests: {}".format(' '.join(details))) - yb_python_zip_path = yb_dist_tests.get_tmp_filename( - prefix='yb_python_module_for_spark_workers_', suffix='.zip', auto_remove=True) - logging.info("Creating a zip archive with the yb python module at %s", yb_python_zip_path) - subprocess.check_call( - ['zip', '--recurse-paths', '--quiet', yb_python_zip_path, 'yb', - '-x', '*.sw?', '-x', '*.pyc'], - cwd=os.path.join(global_conf.yb_src_root, 'python')) - spark_context.addPyFile(yb_python_zip_path) + for module_name in ['yb', 'yugabyte']: + yb_python_zip_path = yb_dist_tests.get_tmp_filename( + prefix=f'{module_name}_python_module_for_spark_workers_', + suffix='.zip', auto_remove=True) + logging.info("Creating a zip archive with the '%s' python module at %s", + module_name, yb_python_zip_path) + zip_cmd_args = [ + 'zip', '--recurse-paths', '--quiet', yb_python_zip_path, module_name, + '-x', '*.sw?', '-x', '*.pyc'] + subprocess.check_call(zip_cmd_args, cwd=os.path.join(global_conf.yb_src_root, 'python')) + spark_context.addPyFile(yb_python_zip_path) if global_conf.archive_for_workers is not None: logging.info("Will send the archive %s to all Spark workers", global_conf.archive_for_workers) @@ -317,7 +320,7 @@ def parallel_run_test(test_descriptor_str: str, fail_count: Any) -> yb_dist_test copy_spark_stderr(test_descriptor_str, build_host) raise e - from yb import yb_dist_tests + from yugabyte import yb_dist_tests wait_for_path_to_exist(global_conf.build_root) @@ -628,7 +631,7 @@ def parallel_list_test_descriptors(rel_test_path: str) -> List[str]: minutes in debug. """ - from yb import yb_dist_tests, command_util + from yugabyte import yb_dist_tests, command_util global_conf = initialize_remote_task() os.environ['BUILD_ROOT'] = global_conf.build_root diff --git a/python/yb/source_files.py b/python/yugabyte/source_files.py similarity index 99% rename from python/yb/source_files.py rename to python/yugabyte/source_files.py index fc5be46507bc..fa447f9d2fb3 100644 --- a/python/yb/source_files.py +++ b/python/yugabyte/source_files.py @@ -105,7 +105,7 @@ def get_file_category(rel_path: str) -> SourceFileCategory: if rel_path.startswith('src/'): return SourceFileCategory.CPP - if rel_path.startswith('build-support/') or rel_path.startswith('python/yb/build_postgres.py'): + if rel_path.startswith('build-support/') or rel_path.startswith('python/yugabyte/'): return SourceFileCategory.BUILD_SCRIPTS if rel_path.startswith('python/'): diff --git a/build-support/split_long_command_line.py b/python/yugabyte/split_long_command_line.py similarity index 100% rename from build-support/split_long_command_line.py rename to python/yugabyte/split_long_command_line.py diff --git a/build-support/stabilize_auto_flags_list.py b/python/yugabyte/stabilize_auto_flags_list.py similarity index 94% rename from build-support/stabilize_auto_flags_list.py rename to python/yugabyte/stabilize_auto_flags_list.py index e3e41801c145..66ac63da4aa8 100755 --- a/build-support/stabilize_auto_flags_list.py +++ b/python/yugabyte/stabilize_auto_flags_list.py @@ -20,7 +20,12 @@ import ruamel.yaml # type: ignore sys.path.append(os.path.join(os.path.dirname(os.path.dirname(__file__)), 'python')) -from yb.common_util import YB_SRC_ROOT, create_temp_dir, init_logging, shlex_join # noqa: E402 +from yugabyte.common_util import ( + YB_SRC_ROOT, + create_temp_dir, + init_logging, + shlex_join, +) # noqa: E402 def main() -> None: diff --git a/python/yb/string_util.py b/python/yugabyte/string_util.py similarity index 100% rename from python/yb/string_util.py rename to python/yugabyte/string_util.py diff --git a/python/yb/test_data/org_yb_pgsql_TestDropTableWithConcurrentTxn_testDmlTxnDrop_1pct_sample.log b/python/yugabyte/test_data/org_yb_pgsql_TestDropTableWithConcurrentTxn_testDmlTxnDrop_1pct_sample.log similarity index 100% rename from python/yb/test_data/org_yb_pgsql_TestDropTableWithConcurrentTxn_testDmlTxnDrop_1pct_sample.log rename to python/yugabyte/test_data/org_yb_pgsql_TestDropTableWithConcurrentTxn_testDmlTxnDrop_1pct_sample.log diff --git a/python/yb/test_data/org_yb_pgsql_TestDropTableWithConcurrentTxn_testDmlTxnDrop_1pct_sample.out b/python/yugabyte/test_data/org_yb_pgsql_TestDropTableWithConcurrentTxn_testDmlTxnDrop_1pct_sample.out similarity index 100% rename from python/yb/test_data/org_yb_pgsql_TestDropTableWithConcurrentTxn_testDmlTxnDrop_1pct_sample.out rename to python/yugabyte/test_data/org_yb_pgsql_TestDropTableWithConcurrentTxn_testDmlTxnDrop_1pct_sample.out diff --git a/python/yb/test_postprocess_test_result.py b/python/yugabyte/test_postprocess_test_result.py similarity index 98% rename from python/yb/test_postprocess_test_result.py rename to python/yugabyte/test_postprocess_test_result.py index 6a189708e10a..3be9508d3581 100644 --- a/python/yb/test_postprocess_test_result.py +++ b/python/yugabyte/test_postprocess_test_result.py @@ -14,7 +14,7 @@ import pytest import random from typing import Dict, Any -from yb.postprocess_test_result import Postprocessor, FAIL_TAG_AND_PATTERN, SIGNALS +from yugabyte.postprocess_test_result import Postprocessor, FAIL_TAG_AND_PATTERN, SIGNALS LOG_CONTENTS = \ diff --git a/python/yb/test_rewrite_test_log.py b/python/yugabyte/test_rewrite_test_log.py similarity index 96% rename from python/yb/test_rewrite_test_log.py rename to python/yugabyte/test_rewrite_test_log.py index 2cf3a7a9a614..ae7bdc91a74a 100644 --- a/python/yb/test_rewrite_test_log.py +++ b/python/yugabyte/test_rewrite_test_log.py @@ -14,7 +14,7 @@ import os import subprocess -from yb.rewrite_test_log import LogRewriterConf, LogRewriter +from yugabyte.rewrite_test_log import LogRewriterConf, LogRewriter def test_java_test_log_rewrite(tmp_path: pathlib.Path) -> None: diff --git a/python/yb/thirdparty_tool.py b/python/yugabyte/thirdparty_tool.py old mode 100644 new mode 100755 similarity index 99% rename from python/yb/thirdparty_tool.py rename to python/yugabyte/thirdparty_tool.py index 81013b7011a2..74641fc3adb7 --- a/python/yb/thirdparty_tool.py +++ b/python/yugabyte/thirdparty_tool.py @@ -35,7 +35,7 @@ from typing import DefaultDict, Dict, List, Any, Optional, Pattern, Tuple from datetime import datetime -from yb.common_util import ( +from yugabyte.common_util import ( init_logging, YB_SRC_ROOT, load_yaml_file, @@ -44,13 +44,13 @@ arg_str_to_bool, make_parent_dir, ) -from yb.file_util import read_file, write_file +from yugabyte.file_util import read_file, write_file from sys_detection import local_sys_conf, SHORT_OS_NAME_REGEX_STR, is_macos from collections import defaultdict -from yb.os_versions import adjust_os_type, is_compatible_os +from yugabyte.os_versions import adjust_os_type, is_compatible_os ruamel_yaml_object = ruamel.yaml.YAML() diff --git a/python/yb/timestamp_saver.py b/python/yugabyte/timestamp_saver.py similarity index 98% rename from python/yb/timestamp_saver.py rename to python/yugabyte/timestamp_saver.py index e46a02cb6977..e0eb23fb5b8e 100644 --- a/python/yb/timestamp_saver.py +++ b/python/yugabyte/timestamp_saver.py @@ -16,7 +16,7 @@ from typing import Optional, List, Dict, Tuple, Any -from yb.file_util import compute_file_sha256 +from yugabyte.file_util import compute_file_sha256 class TimestampSaver: diff --git a/python/yb/tool_base.py b/python/yugabyte/tool_base.py similarity index 98% rename from python/yb/tool_base.py rename to python/yugabyte/tool_base.py index 919c75401a04..2a40f26a33dc 100644 --- a/python/yb/tool_base.py +++ b/python/yugabyte/tool_base.py @@ -7,7 +7,7 @@ from typing import Dict, List, Any, Union, Optional -from yb.common_util import set_env_vars_from_build_root, init_logging +from yugabyte.common_util import set_env_vars_from_build_root, init_logging from overrides import overrides, EnforceOverrides diff --git a/python/yb/type_util.py b/python/yugabyte/type_util.py similarity index 100% rename from python/yb/type_util.py rename to python/yugabyte/type_util.py diff --git a/build-support/update_test_result_xml.py b/python/yugabyte/update_test_result_xml.py similarity index 93% rename from build-support/update_test_result_xml.py rename to python/yugabyte/update_test_result_xml.py index d05f0aca9039..d41cd067a9e6 100755 --- a/build-support/update_test_result_xml.py +++ b/python/yugabyte/update_test_result_xml.py @@ -20,6 +20,8 @@ import signal import time +from typing import Any, Optional + from xml.dom import minidom @@ -32,31 +34,35 @@ # https://stackoverflow.com/questions/2281850/timeout-function-if-it-takes-too-long-to-finish class Timeout: - def __init__(self, seconds=1, error_message='Timeout'): + seconds: int + start_time: Optional[float] + + def __init__(self, seconds: int = 1, error_message: str = 'Timeout'): self.seconds = seconds self.start_time = None - def handle_timeout(self, signum, frame): + def handle_timeout(self, unused_signum: Any, unused_frame: Any) -> None: if self.start_time is None: raise RuntimeError("Timed out") raise RuntimeError("Timed out after %.2f seconds" % (time.time() - self.start_time)) - def __enter__(self): + def __enter__(self) -> 'Timeout': self.start_time = time.time() signal.signal(signal.SIGALRM, self.handle_timeout) signal.alarm(self.seconds) + return self - def __exit__(self, type, value, traceback): + def __exit__(self, unused_type: Any, unused_value: Any, unused_traceback: Any) -> None: signal.alarm(0) -def initialize(): +def initialize() -> None: logging.basicConfig( level=logging.INFO, format="[" + os.path.basename(__file__) + "] %(asctime)s %(levelname)s: %(message)s") -def parse_args(): +def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser( usage="usage: %(prog)s ", description="Updates a JUnit-style test result XML file, e.g. to add a log URL or " @@ -94,7 +100,7 @@ def parse_args(): return parser.parse_args() -def update_test_result_xml(args): +def update_test_result_xml(args: argparse.Namespace) -> bool: result_xml_size = os.stat(args.result_xml).st_size if result_xml_size > MAX_RESULT_XML_SIZE_BYTES: logging.error( @@ -178,7 +184,7 @@ def update_test_result_xml(args): return True -def main(): +def main() -> bool: args = None try: with Timeout(seconds=TIMEOUT_SEC): diff --git a/build-support/validate_build_root.py b/python/yugabyte/validate_build_root.py similarity index 100% rename from build-support/validate_build_root.py rename to python/yugabyte/validate_build_root.py diff --git a/python/yb/yb_dist_tests.py b/python/yugabyte/yb_dist_tests.py similarity index 98% rename from python/yb/yb_dist_tests.py rename to python/yugabyte/yb_dist_tests.py index dff7f68eacad..71613e79c505 100644 --- a/python/yb/yb_dist_tests.py +++ b/python/yugabyte/yb_dist_tests.py @@ -28,11 +28,11 @@ from functools import total_ordering -from yb import command_util -from yb.common_util import get_build_type_from_build_root, \ +from yugabyte import command_util +from yugabyte.common_util import get_build_type_from_build_root, \ get_compiler_type_from_build_root, \ is_macos # nopep8 -from yb.postgres_build_util import POSTGRES_BUILD_SUBDIR +from yugabyte.postgres_build_util import POSTGRES_BUILD_SUBDIR from typing import Optional, List, Set, Dict, cast # This is used to separate relative binary path from gtest_filter for C++ tests in what we call @@ -243,7 +243,7 @@ def set_env_on_spark_worker( def set_global_conf_from_args(args: argparse.Namespace) -> GlobalTestConfig: build_root = os.path.realpath(args.build_root) - # This module is expected to be under python/yb. + # This module is expected to be under python/yugabyte. yb_src_root = os.path.dirname(os.path.dirname(os.path.dirname(os.path.realpath(__file__)))) # Ensure that build_root is consistent with yb_src_root above. @@ -275,7 +275,7 @@ def set_global_conf_from_args(args: argparse.Namespace) -> GlobalTestConfig: raise ValueError( "Build root '%s' implies compiler type '%s' but YB_COMPILER_TYPE is '%s'" % ( build_root, compiler_type, compiler_type_from_env)) - from yb import common_util + from yugabyte import common_util os.environ['YB_COMPILER_TYPE'] = compiler_type global global_conf diff --git a/python/yb/yb_release_core_db.py b/python/yugabyte/yb_release_core_db.py similarity index 98% rename from python/yb/yb_release_core_db.py rename to python/yugabyte/yb_release_core_db.py index 81202d0517c4..badd3a38e59f 100755 --- a/python/yb/yb_release_core_db.py +++ b/python/yugabyte/yb_release_core_db.py @@ -23,10 +23,10 @@ import traceback import ruamel.yaml -from yb.library_packager import LibraryPackager, add_common_arguments -from yb.mac_library_packager import MacLibraryPackager, add_common_arguments -from yb.release_util import ReleaseUtil, check_for_local_changes -from yb.common_util import ( +from yugabyte.library_packager import LibraryPackager, add_common_arguments +from yugabyte.mac_library_packager import MacLibraryPackager, add_common_arguments +from yugabyte.release_util import ReleaseUtil, check_for_local_changes +from yugabyte.common_util import ( init_logging, get_build_type_from_build_root, set_thirdparty_dir, @@ -34,8 +34,8 @@ create_temp_dir, shlex_join, ) -from yb.linuxbrew import set_build_root -from yb.optional_components import ( +from yugabyte.linuxbrew import set_build_root +from yugabyte.optional_components import ( OptionalComponents, add_optional_component_arguments, optional_components_from_args, diff --git a/requirements.txt b/requirements.txt index 6b799e37cf35..5c6a5397aa84 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,20 +3,21 @@ boto botocore codecheck compiledb +compiler-identification distro downloadutil +llvm-installer mypy overrides packaging psutil pycodestyle pygithub +pytest ruamel.yaml semantic-version six sys-detection +types-psutil wheel yugabyte_pycommon -compiler-identification -pytest -llvm-installer diff --git a/requirements_frozen.txt b/requirements_frozen.txt index f754a200275a..e2fbdc1e7129 100644 --- a/requirements_frozen.txt +++ b/requirements_frozen.txt @@ -1,3 +1,7 @@ +Deprecated==1.2.13 +PyGithub==1.55 +PyJWT==2.1.0 +PyNaCl==1.4.0 attrs==21.4.0 autorepr==0.3.0 bashlex==0.16 @@ -10,39 +14,36 @@ click==8.0.1 codecheck==1.3.1 compiledb==0.10.1 compiler-identification==1.0.3 -Deprecated==1.2.13 distro==1.6.0 downloadutil==1.0.2 idna==3.2 importlib-metadata==4.8.3 iniconfig==1.1.1 -mypy==0.991 +llvm_installer==1.2.11 mypy-extensions==0.4.3 +mypy==0.991 overrides==6.1.0 packaging==21.0 pluggy==1.0.0 -psutil==5.8.0 +psutil==5.9.5 py==1.11.0 pycodestyle==2.7.0 pycparser==2.20 -PyGithub==1.55 -PyJWT==2.1.0 -PyNaCl==1.4.0 pyparsing==2.4.7 pytest==6.2.5 requests==2.26.0 -ruamel.yaml==0.17.16 ruamel.yaml.clib==0.2.6 +ruamel.yaml==0.17.16 semantic-version==2.8.5 shutilwhich==1.1.0 six==1.16.0 sys-detection==1.3.0 toml==0.10.2 typed-ast==1.5.1 +types-psutil==5.9.5.12 typing-extensions==3.10.0.2 typing-utils==0.1.0 urllib3==1.26.7 wrapt==1.12.1 yugabyte-pycommon==1.9.15 zipp==3.6.0 -llvm_installer==1.2.11 diff --git a/src/yb/util/CMakeLists.txt b/src/yb/util/CMakeLists.txt index 5485ba370427..d256e12acb6a 100644 --- a/src/yb/util/CMakeLists.txt +++ b/src/yb/util/CMakeLists.txt @@ -106,7 +106,7 @@ ADD_YB_LIBRARY(opid_proto # the build directory. set(VERSION_STAMP_FILE ${CMAKE_BINARY_DIR}/version_metadata.json) -list(APPEND GEN_VERSION_INFO_COMMAND "${BUILD_SUPPORT_DIR}/gen_version_info.py") +list(APPEND GEN_VERSION_INFO_COMMAND "${YB_SRC_ROOT}/python/yugabyte/gen_version_info.py") list(APPEND GEN_VERSION_INFO_COMMAND "--build-type=${CMAKE_BUILD_TYPE}") if(YB_GIT_HASH) message(STATUS "Provided git hash: ${YB_GIT_HASH}") diff --git a/yb_build.sh b/yb_build.sh index e4e4c3aab3ec..7ea69de3daf1 100755 --- a/yb_build.sh +++ b/yb_build.sh @@ -822,7 +822,7 @@ use_packaged_targets() { done < <( activate_virtualenv &>/dev/null set_pythonpath - "$YB_SRC_ROOT"/build-support/list_packaged_targets.py "${optional_components_args[@]}" + "$YB_SCRIPT_PATH_LIST_PACKAGED_TARGETS" "${optional_components_args[@]}" ) if [[ ${#packaged_targets[@]} -eq 0 ]]; then fatal "Failed to identify the set of targets to build for the release package" diff --git a/yb_release b/yb_release index aa0a826a2741..3ffcb9114335 100755 --- a/yb_release +++ b/yb_release @@ -17,4 +17,4 @@ set -euo pipefail activate_virtualenv set_pythonpath -python3 "$YB_SRC_ROOT"/python/yb/yb_release_core_db.py "$@" +python3 "$YB_SCRIPT_PATH_YB_RELEASE_CORE_DB" "$@"