[2.5.3][Platform]: Revert python packaging changes made for python3.

Summary:
Revert the python3 prioritization packaging changes. 2.5.3 will use python2 and later versions
will move towards python3.

Test Plan:
bin/install_python_requirements.sh --create_package
bin/install_python_requirements.sh --use_package

---

Reviewers: daniel, sanketh

Reviewed By: sanketh

Subscribers: jenkins-bot, yugaware

Differential Revision: https://phabricator.dev.yugabyte.com/D10908

Author: WesleyW

managed/devops/bin/common.sh

@@ -47,120 +47,20 @@ regex_from_list() {
   echo "^($regex)$"
 }
 
-# This just logs to stderr.
-log() {
-  BEGIN_COLOR='\033[0;32m'
-  END_COLOR='\033[0m'
-  GREEN='\033[0;32m'
-  RED='\033[0;31m'
-
-  case ${_log_level:-info} in
-    error)
-      BEGIN_COLOR='\033[0;31m'
-      shift
-      ;;
-    warn)
-      BEGIN_COLOR='\033[0;33m'
-      shift
-      ;;
-  esac
-  echo -e "${BEGIN_COLOR}[$( get_timestamp ) \
-${BASH_SOURCE[1]##*/}:${BASH_LINENO[0]} ${FUNCNAME[1]}]${END_COLOR}" $* >&2
-}
-
-fatal() {
-  log "$@"
-  exit 1
-}
-
-get_timestamp() {
-  date +%Y-%m-%d_%H_%M_%S
-}
-
-check_python_executables() {
-  executables=("$@")
-  for py_executable in "${executables[@]}"; do
-    if which "$py_executable" > /dev/null 2>&1; then
-      PYTHON_EXECUTABLE="$py_executable"
-      return 0
-    fi
-  done
-  return 1
-}
-
-set_python_vars() {
-  # Prioritize python3 over python2.
-  if [ -z "$YB_MANAGED_DEVOPS_USE_PYTHON3" ]; then
-    if check_python_executables "${PYTHON3_EXECUTABLES[@]}"; then
-      YB_MANAGED_DEVOPS_USE_PYTHON3="1"
-    elif check_python_executables "${PYTHON2_EXECUTABLES[@]}"; then
-      YB_MANAGED_DEVOPS_USE_PYTHON3="0"
-    fi
-
-    if [ -z "$PYTHON_EXECUTABLE" ]; then
-      if which python > /dev/null 2>&1; then
-        PYTHON_EXECUTABLE="python"
-        if python -c 'import sys; sys.exit(1) if sys.version_info[0] != 2 else sys.exit(0)';  then
-          YB_MANAGED_DEVOPS_USE_PYTHON3="0"
-        else
-          YB_MANAGED_DEVOPS_USE_PYTHON3="1"
-        fi
-      fi
-    fi
-  else
-    if which python > /dev/null 2>&1; then
-      if python -c 'import sys; sys.exit(1) if sys.version_info[0] != 2 else sys.exit(0)';  then
-        if [[ "$YB_MANAGED_DEVOPS_USE_PYTHON3" == "0" ]]; then
-          PYTHON_EXECUTABLE="python"
-        fi
-      elif [[ "$YB_MANAGED_DEVOPS_USE_PYTHON3" == "1" ]]; then
-        PYTHON_EXECUTABLE="python"
-      fi
-    fi
-
-    if [ -z "$PYTHON_EXECUTABLE" ]; then
-      POSSIBLE_EXECUTABLES="$PYTHON3_EXECUTABLES"
-      if [[ "$YB_MANAGED_DEVOPS_USE_PYTHON3" == "0" ]]; then
-        POSSIBLE_EXECUTABLES="$PYTHON2_EXECUTABLES"
-      fi
-      check_python_executables "${POSSIBLE_EXECUTABLES[@]}"
-    fi
-  fi
-
-  if [[ $YB_MANAGED_DEVOPS_USE_PYTHON3 != "0" &&
-        $YB_MANAGED_DEVOPS_USE_PYTHON3 != "1" ]]; then
-    fatal "Invalid value of YB_MANAGED_DEVOPS_USE_PYTHON3: $YB_MANAGED_DEVOPS_USE_PYTHON3," \
-          "expected 0 or 1"
-  fi
-  if [ -z "$PYTHON_EXECUTABLE" ]; then
-    fatal "Failed to find python executable."
-  fi
-
-  if [[ $YB_MANAGED_DEVOPS_USE_PYTHON3 == "1" ]]; then
-    YB_VIRTUALENV_BASENAME=venv
-    REQUIREMENTS_FILE_NAME="$yb_devops_home/python3_requirements.txt"
-    FROZEN_REQUIREMENTS_FILE="$yb_devops_home/python3_requirements_frozen.txt"
-    YB_PYTHON_MODULES_DIR="$yb_devops_home/python3_modules"
-    YB_PYTHON_MODULES_PACKAGE="$yb_devops_home/python3_modules.tar.gz"
-    YB_INSTALLED_MODULES_DIR="$yb_devops_home/python3_installed_modules"
-  else
-    YB_VIRTUALENV_BASENAME=python_virtual_env
-    REQUIREMENTS_FILE_NAME="$yb_devops_home/python_requirements.txt"
-    FROZEN_REQUIREMENTS_FILE="$yb_devops_home/python_requirements_frozen.txt"
-    YB_PYTHON_MODULES_DIR="$yb_devops_home/python2_modules"
-    YB_PYTHON_MODULES_PACKAGE="$yb_devops_home/python2_modules.tar.gz"
-    YB_INSTALLED_MODULES_DIR="$yb_devops_home/python2_installed_modules"
-  fi
-}
-
 # -------------------------------------------------------------------------------------------------
 # Constants
 # -------------------------------------------------------------------------------------------------
-readonly PYTHON2_EXECUTABLES=('python2' 'python2.7')
-readonly PYTHON3_EXECUTABLES=('python3' 'python3.6' 'python3.7' 'python3.8')
-PYTHON_EXECUTABLE=""
-USER_INPUT_MANAGED_DEVOPS_USE_PYTHON3=${YB_MANAGED_DEVOPS_USE_PYTHON3:-""}
-YB_MANAGED_DEVOPS_USE_PYTHON3=""
+DEFAULT_USE_PY3_VALUE="1"
+if python -c 'import sys; sys.exit(1) if sys.version_info[0] != 2 else sys.exit(0)'; then
+  DEFAULT_USE_PY3_VALUE="0"
+fi
+
+readonly YB_MANAGED_DEVOPS_USE_PYTHON3=${YB_MANAGED_DEVOPS_USE_PYTHON3:-$DEFAULT_USE_PY3_VALUE}
+if [[ $YB_MANAGED_DEVOPS_USE_PYTHON3 != "0" &&
+      $YB_MANAGED_DEVOPS_USE_PYTHON3 != "1" ]]; then
+  fatal "Invalid value of YB_MANAGED_DEVOPS_USE_PYTHON3: $YB_MANAGED_DEVOPS_USE_PYTHON3," \
+        "expected 0 or 1"
+fi
 
 readonly yb_script_name=${0##*/}
 readonly yb_script_name_no_extension=${yb_script_name%.sh}
@@ -189,12 +89,22 @@ set +u
 readonly MANAGED_PYTHONPATH_ORIGINAL="${PYTHONPATH:-}"
 set -u
 
-YB_VIRTUALENV_BASENAME=venv
-REQUIREMENTS_FILE_NAME="$yb_devops_home/python3_requirements.txt"
-FROZEN_REQUIREMENTS_FILE="$yb_devops_home/python3_requirements_frozen.txt"
-YB_PYTHON_MODULES_DIR="$yb_devops_home/python3_modules"
-YB_PYTHON_MODULES_PACKAGE="$yb_devops_home/python3_modules.tar.gz"
-YB_INSTALLED_MODULES_DIR="$yb_devops_home/python3_installed_modules"
+# Basename (i.e. name excluding the directory path) of our virtualenv.
+if [[ $YB_MANAGED_DEVOPS_USE_PYTHON3 == "1" ]]; then
+  readonly YB_VIRTUALENV_BASENAME=venv
+  readonly REQUIREMENTS_FILE_NAME="$yb_devops_home/python3_requirements.txt"
+  readonly FROZEN_REQUIREMENTS_FILE="$yb_devops_home/python3_requirements_frozen.txt"
+  readonly YB_PYTHON_MODULES_DIR="$yb_devops_home/python3_modules"
+  readonly YB_PYTHON_MODULES_PACKAGE="$yb_devops_home/python3_modules.tar.gz"
+  readonly YB_INSTALLED_MODULES_DIR="$yb_devops_home/python3_installed_modules"
+else
+  readonly YB_VIRTUALENV_BASENAME=python_virtual_env
+  readonly REQUIREMENTS_FILE_NAME="$yb_devops_home/python_requirements.txt"
+  readonly FROZEN_REQUIREMENTS_FILE="$yb_devops_home/python_requirements_frozen.txt"
+  readonly YB_PYTHON_MODULES_DIR="$yb_devops_home/python2_modules"
+  readonly YB_PYTHON_MODULES_PACKAGE="$yb_devops_home/python2_modules.tar.gz"
+  readonly YB_INSTALLED_MODULES_DIR="$yb_devops_home/python2_installed_modules"
+fi
 
 readonly YBOPS_TOP_LEVEL_DIR_BASENAME=opscli
 readonly YBOPS_PACKAGE_NAME=ybops
@@ -217,6 +127,35 @@ log_error() {
  log "$@"
 }
 
+# This just logs to stderr.
+log() {
+  BEGIN_COLOR='\033[0;32m'
+  END_COLOR='\033[0m'
+  GREEN='\033[0;32m'
+  RED='\033[0;31m'
+
+  case ${_log_level:-info} in
+    error)
+      BEGIN_COLOR='\033[0;31m'
+      shift
+      ;;
+    warn)
+      BEGIN_COLOR='\033[0;33m'
+      shift
+      ;;
+  esac
+  echo -e "${BEGIN_COLOR}[$( get_timestamp ) ${BASH_SOURCE[1]##*/}:${BASH_LINENO[0]} ${FUNCNAME[1]}]${END_COLOR}" $* >&2
+}
+
+fatal() {
+  log "$@"
+  exit 1
+}
+
+get_timestamp() {
+  date +%Y-%m-%d_%H_%M_%S
+}
+
 ensure_log_dir_defined() {
   if [[ -z ${log_dir:-} ]]; then
     fatal "log_dir is not set"
@@ -268,7 +207,7 @@ deactivate_virtualenv() {
     return
   fi
 
-  if [[ -d "$YB_INSTALLED_MODULES_DIR" ]]; then
+  if [[ -f "$YB_INSTALLED_MODULES_DIR" ]]; then
     export PYTHONPATH=$MANAGED_PYTHONPATH_ORIGINAL
     return
   fi
@@ -306,7 +245,7 @@ activate_virtualenv() {
     return
   fi
 
-  if [[ -d "$YB_INSTALLED_MODULES_DIR" ]]; then
+  if [[ -f "$YB_INSTALLED_MODULES_DIR" ]]; then
     export PYTHONPATH="${YB_INSTALLED_MODULES_DIR}:${MANAGED_PYTHONPATH_ORIGINAL}"
     return
   fi
@@ -318,10 +257,6 @@ activate_virtualenv() {
   if [[ ! -d $virtualenv_dir ]]; then
     # We need to be using system python to install the virtualenv module or create a new virtualenv.
     deactivate_virtualenv
-    # Reset python variables after deactivating virtualenv
-    PYTHON_EXECUTABLE=""
-    YB_MANAGED_DEVOPS_USE_PYTHON3=$USER_INPUT_MANAGED_DEVOPS_USE_PYTHON3
-    set_python_vars
 
     if [[ $YB_MANAGED_DEVOPS_USE_PYTHON3 == "0" ]]; then
       pip_install "virtualenv<20"
@@ -331,10 +266,10 @@ activate_virtualenv() {
       set -x
       cd "${virtualenv_dir%/*}"
       if [[ $YB_MANAGED_DEVOPS_USE_PYTHON3 == "1" ]]; then
-        "$PYTHON_EXECUTABLE" -m venv "$YB_VIRTUALENV_BASENAME"
+        python3 -m venv "$YB_VIRTUALENV_BASENAME"
       else
         # Assuming that the default python binary is pointing to Python 2.7.
-        "$PYTHON_EXECUTABLE" -m virtualenv --no-setuptools "$YB_VIRTUALENV_BASENAME"
+        python -m virtualenv --no-setuptools "$YB_VIRTUALENV_BASENAME"
       fi
     )
   elif "$is_linux"; then
@@ -431,9 +366,9 @@ verbose_mkdir_p() {
 
 run_pip() {
   if [[ $YB_MANAGED_DEVOPS_USE_PYTHON3 == "1" ]]; then
-    "$PYTHON_EXECUTABLE" -m pip "$@"
+    pip3 "$@"
   else
-    "$PYTHON_EXECUTABLE" "$(which pip)" "$@"
+    python "$(which pip)" "$@"
   fi
 }
 
@@ -531,7 +466,8 @@ install_ybops_package() {
   fi
   (
     cd "$yb_devops_home/$YBOPS_TOP_LEVEL_DIR_BASENAME"
-    "$PYTHON_EXECUTABLE" setup.py install $user_flag
+    log "Using python: $( which python )"
+    python setup.py install $user_flag
     rm -rf build dist "$YBOPS_PACKAGE_NAME.egg-info"
   )
   virtualenv_aware_log "Installed the ybops package"
@@ -576,7 +512,6 @@ detect_os() {
 # Initialization
 # -------------------------------------------------------------------------------------------------
 
-set_python_vars
 detect_os
 
 #

managed/devops/bin/ybcloud.sh

@@ -12,9 +12,6 @@ set -e
 . "${BASH_SOURCE%/*}"/common.sh
 
 activate_virtualenv
+cd "$yb_devops_home"
 
-if [[ -d "$YB_INSTALLED_MODULES_DIR" ]]; then
-  "$PYTHON_EXECUTABLE" "$yb_devops_home"/opscli/ybops/scripts/ybcloud.py "$@"
-else
-  "$PYTHON_EXECUTABLE" "$(which ybcloud.py)" "$@"
-fi
+python "$(which ybcloud.py)" "$@"

managed/devops/opscli/ybops/cloud/aws/utils.py

@@ -210,7 +210,7 @@ def __init__(self, metadata, host_vpc_id, host_vpc_region):
         self._validate_cidr_overlap()
 
     def _validate_cidr_overlap(self):
-        region_networks = [ip_network(cidr) for cidr in self.region_cidrs.values()]
+        region_networks = [ip_network(cidr.decode('utf-8')) for cidr in self.region_cidrs.values()]
         all_networks = region_networks
         for i in range(len(all_networks)):
             for j in range(i + 1, len(all_networks)):

managed/devops/opscli/ybops/cloud/common/ansible.py

@@ -126,7 +126,7 @@ def run(self, filename, extra_vars=dict(), host_info={}, print_output=True):
         p = subprocess.Popen(process_args, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
         stdout, stderr = p.communicate()
         if print_output:
-            print(stdout.decode('utf-8'))
+            print(stdout)
         EXCEPTION_MSG_FORMAT = ("Playbook run of {} against {} with args {} " +
                                 "failed with return code {} and error '{}'")
         if p.returncode != 0:

managed/devops/opscli/ybops/cloud/gcp/utils.py

@@ -108,7 +108,7 @@ def _query_endpoint(endpoint):
         try:
             url = "{}/{}".format(GcpMetadata.METADATA_URL_BASE, endpoint)
             req = requests.get(url, headers=GcpMetadata.CUSTOM_HEADERS, timeout=2)
-            return req.content.decode('utf-8') if req.status_code == requests.codes.ok else None
+            return req.content if req.status_code == requests.codes.ok else None
         except requests.exceptions.ConnectionError as e:
             return None
 

managed/devops/opscli/ybops/utils/__init__.py

@@ -448,9 +448,9 @@ def format_rsa_key(key, public_key):
         key (str): Encoded key in OpenSSH or PEM format based on the flag (public key or not).
     """
     if public_key:
-        return key.publickey().exportKey("OpenSSH").decode('utf-8')
+        return str(key.publickey().exportKey("OpenSSH"))
     else:
-        return key.exportKey("PEM").decode('utf-8')
+        return str(key.exportKey("PEM"))
 
 
 def validated_key_file(key_file):
@@ -602,7 +602,7 @@ def validate_cron_status(host_name, port, username, ssh_key_file):
 
         _, stdout, stderr = ssh_client.exec_command("crontab -l")
         cronjobs = ["clean_cores.sh", "zip_purge_yb_logs.sh", "yb-server-ctl.sh tserver"]
-        stdout = stdout.read().decode('utf-8')
+        stdout = stdout.read()
         return len(stderr.readlines()) == 0 and all(c in stdout for c in cronjobs)
     except (paramiko.ssh_exception.NoValidConnectionsError,
             paramiko.ssh_exception.AuthenticationException,

managed/devops/python3_requirements.txt

@@ -19,24 +19,24 @@
 # TODO: see if we can move to using Pundler (https://github.com/Deepwalker/pundler) instead.
 
 ansible==2.5.4
-ansible-vault==2.1.0
+ansible-vault==1.0.4
 apache-libcloud>=1.2.1
 auth0-python==3.0.0
+awscli==1.11.1
 azure-common==1.1.25
 azure-mgmt-compute==12.1.0
-azure-mgmt-privatedns==0.1.0
 azure-mgmt-network==10.2.0
 azure-mgmt-resource==10.0.0
-boto==2.49.0
-boto3==1.17.22
-botocore==1.20.22
-cryptography==3.3.2
+azure-mgmt-nspkg==3.0.2
+boto>=2.42.0
+boto3>=1.4.0
+botocore>=1.4.59
 distro==1.5.0
+docker-compose==1.9.0
 docker-py==1.7.0
+enum34==1.1.6
 fabric==2.2.1
-google-api-python-client==1.10.0
-google-api-core==1.22.0
-google-auth==1.19.2
+google-api-python-client
 graphviz
 jmespath==0.9.0
 ipaddress==1.0.22
@@ -47,7 +47,6 @@ pycryptodome==3.6.1
 python-dateutil==2.5.3
 python-dotenv==0.6.4
 PyYAML==3.13
-s3transfer>=0.3.0,<0.4.0
 six==1.10.0
 slackclient
 stormssh