PYTHON-5480 Update Python 3.9-specific tests to use Python 3.10 (mongodb#2560)

Author: blink1073

.evergreen/generated_configs/tasks.yml

@@ -142,17 +142,16 @@ buildvariants:
       - rhel87-small
 
   # Disable test commands tests
-  - name: disable-test-commands-rhel8-python3.9
+  - name: disable-test-commands-rhel8
     tasks:
       - name: .test-standard .server-latest
-    display_name: Disable test commands RHEL8 Python3.9
+    display_name: Disable test commands RHEL8
     run_on:
       - rhel87-small
     expansions:
       AUTH: auth
       SSL: ssl
       DISABLE_TEST_COMMANDS: "1"
-      PYTHON_BINARY: /opt/python/3.9/bin/python3
 
   # Doctests tests
   - name: doctests-rhel8
@@ -491,14 +490,14 @@ buildvariants:
       SUB_TEST_NAME: pyopenssl
 
   # Search index tests
-  - name: search-index-helpers-rhel8-python3.9
+  - name: search-index-helpers-rhel8-python3.10
     tasks:
       - name: .search_index
-    display_name: Search Index Helpers RHEL8 Python3.9
+    display_name: Search Index Helpers RHEL8 Python3.10
     run_on:
       - rhel87-small
     expansions:
-      PYTHON_BINARY: /opt/python/3.9/bin/python3
+      PYTHON_BINARY: /opt/python/3.10/bin/python3
 
   # Server version tests
   - name: mongodb-v4.2

.evergreen/generated_configs/variants.yml

@@ -20,8 +20,13 @@ fi
 set -o xtrace
 
 # Install python with pip.
-PYTHON_VER="python3.9"
+PYTHON_VER="python3.10"
 apt-get -qq update  < /dev/null > /dev/null
+apt-get -q install -y software-properties-common
+# Use openpgp to avoid gpg key timeout.
+mkdir -p $HOME/.gnupg
+echo "keyserver keys.openpgp.org" >> $HOME/.gnupg/gpg.conf
+add-apt-repository -y 'ppa:deadsnakes/ppa'
 apt-get -qq install $PYTHON_VER $PYTHON_VER-venv build-essential $PYTHON_VER-dev -y  < /dev/null > /dev/null
 
 export PYTHON_BINARY=$PYTHON_VER

.evergreen/run-mongodb-aws-ecs-test.sh

@@ -330,10 +330,9 @@ def create_mod_wsgi_variants():
 def create_disable_test_commands_variants():
     host = DEFAULT_HOST
     expansions = dict(AUTH="auth", SSL="ssl", DISABLE_TEST_COMMANDS="1")
-    python = CPYTHONS[0]
-    display_name = get_variant_name("Disable test commands", host, python=python)
+    display_name = get_variant_name("Disable test commands", host)
     tasks = [".test-standard .server-latest"]
-    return [create_variant(tasks, display_name, host=host, python=python, expansions=expansions)]
+    return [create_variant(tasks, display_name, host=host, expansions=expansions)]
 
 
 def create_oidc_auth_variants():

.evergreen/scripts/generate_config.py

@@ -22,7 +22,7 @@
 ##############
 
 ALL_VERSIONS = ["4.2", "4.4", "5.0", "6.0", "7.0", "8.0", "rapid", "latest"]
-CPYTHONS = ["3.9", "3.10", "3.11", "3.12", "3.13", "3.14"]
+CPYTHONS = ["3.10", "3.9", "3.11", "3.12", "3.13", "3.14"]
 PYPYS = ["pypy3.10"]
 ALL_PYTHONS = CPYTHONS + PYPYS
 MIN_MAX_PYTHON = [CPYTHONS[0], CPYTHONS[-1]]

.evergreen/scripts/generate_config_utils.py

@@ -33,7 +33,7 @@ def _setup_azure_vm(base_env: dict[str, str]) -> None:
     env["AZUREKMS_CMD"] = "sudo apt-get install -y python3-dev build-essential"
     run_command(f"{azure_dir}/run-command.sh", env=env)
 
-    env["AZUREKMS_CMD"] = "bash .evergreen/just.sh setup-tests kms azure-remote"
+    env["AZUREKMS_CMD"] = "NO_EXT=1 bash .evergreen/just.sh setup-tests kms azure-remote"
     run_command(f"{azure_dir}/run-command.sh", env=env)
     LOGGER.info("Setting up Azure VM... done.")
 
@@ -53,7 +53,7 @@ def _setup_gcp_vm(base_env: dict[str, str]) -> None:
     env["GCPKMS_CMD"] = "sudo apt-get install -y python3-dev build-essential"
     run_command(f"{gcp_dir}/run-command.sh", env=env)
 
-    env["GCPKMS_CMD"] = "bash ./.evergreen/just.sh setup-tests kms gcp-remote"
+    env["GCPKMS_CMD"] = "NO_EXT=1 bash ./.evergreen/just.sh setup-tests kms gcp-remote"
     run_command(f"{gcp_dir}/run-command.sh", env=env)
     LOGGER.info("Setting up GCP VM...")
 
@@ -98,6 +98,13 @@ def setup_kms(sub_test_name: str) -> None:
         if sub_test_target == "azure":
             os.environ["AZUREKMS_VMNAME_PREFIX"] = "PYTHON_DRIVER"
 
+            # Found using "az vm image list --output table"
+            os.environ[
+                "AZUREKMS_IMAGE"
+            ] = "Canonical:0001-com-ubuntu-server-jammy:22_04-lts-gen2:latest"
+        else:
+            os.environ["GCPKMS_IMAGEFAMILY"] = "debian-12"
+
         run_command("./setup.sh", cwd=kms_dir)
         base_env = _load_kms_config(sub_test_target)
 

.evergreen/scripts/kms_tester.py

@@ -42,6 +42,11 @@ def setup_oidc(sub_test_name: str) -> dict[str, str] | None:
     if sub_test_name == "azure":
         env["AZUREOIDC_VMNAME_PREFIX"] = "PYTHON_DRIVER"
     if "-remote" not in sub_test_name:
+        if sub_test_name == "azure":
+            # Found using "az vm image list --output table"
+            env["AZUREOIDC_IMAGE"] = "Canonical:0001-com-ubuntu-server-jammy:22_04-lts-gen2:latest"
+        else:
+            env["GCPKMS_IMAGEFAMILY"] = "debian-12"
         run_command(f"bash {target_dir}/setup.sh", env=env)
     if sub_test_name in K8S_NAMES:
         run_command(f"bash {target_dir}/setup-pod.sh {sub_test_name}")
@@ -84,7 +89,7 @@ def test_oidc_send_to_remote(sub_test_name: str) -> None:
         env[f"{upper_name}OIDC_DRIVERS_TAR_FILE"] = TMP_DRIVER_FILE
         env[
             f"{upper_name}OIDC_TEST_CMD"
-        ] = f"OIDC_ENV={sub_test_name} ./.evergreen/run-mongodb-oidc-test.sh"
+        ] = f"NO_EXT=1 OIDC_ENV={sub_test_name} ./.evergreen/run-mongodb-oidc-test.sh"
     elif sub_test_name in K8S_NAMES:
         env["K8S_DRIVERS_TAR_FILE"] = TMP_DRIVER_FILE
         env["K8S_TEST_CMD"] = "OIDC_ENV=k8s ./.evergreen/run-mongodb-oidc-test.sh"

.evergreen/scripts/oidc_tester.py

@@ -53,7 +53,7 @@
 GROUP_MAP = dict(mockupdb="mockupdb", perf="perf")
 
 # The python version used for perf tests.
-PERF_PYTHON_VERSION = "3.9.13"
+PERF_PYTHON_VERSION = "3.10.11"
 
 
 def is_set(var: str) -> bool:
@@ -90,6 +90,13 @@ def setup_libmongocrypt():
         distro = get_distro()
         if distro.name.startswith("Debian"):
             target = f"debian{distro.version_id}"
+        elif distro.name.startswith("Ubuntu"):
+            if distro.version_id == "20.04":
+                target = "debian11"
+            elif distro.version_id == "22.04":
+                target = "debian12"
+            elif distro.version_id == "24.04":
+                target = "debian13"
         elif distro.name.startswith("Red Hat"):
             if distro.version_id.startswith("7"):
                 target = "rhel-70-64-bit"

.evergreen/scripts/setup_tests.py

@@ -2059,7 +2059,7 @@ async def _test_handshake(self, env_vars, expected_env):
     async def test_handshake_01_aws(self):
         await self._test_handshake(
             {
-                "AWS_EXECUTION_ENV": "AWS_Lambda_python3.9",
+                "AWS_EXECUTION_ENV": "AWS_Lambda_python3.10",
                 "AWS_REGION": "us-east-2",
                 "AWS_LAMBDA_FUNCTION_MEMORY_SIZE": "1024",
             },
@@ -2097,7 +2097,7 @@ async def test_handshake_04_vercel(self):
 
     async def test_handshake_05_multiple(self):
         await self._test_handshake(
-            {"AWS_EXECUTION_ENV": "AWS_Lambda_python3.9", "FUNCTIONS_WORKER_RUNTIME": "python"},
+            {"AWS_EXECUTION_ENV": "AWS_Lambda_python3.10", "FUNCTIONS_WORKER_RUNTIME": "python"},
             None,
         )
         # Extra cases for other combos.
@@ -2109,13 +2109,16 @@ async def test_handshake_05_multiple(self):
 
     async def test_handshake_06_region_too_long(self):
         await self._test_handshake(
-            {"AWS_EXECUTION_ENV": "AWS_Lambda_python3.9", "AWS_REGION": "a" * 512},
+            {"AWS_EXECUTION_ENV": "AWS_Lambda_python3.10", "AWS_REGION": "a" * 512},
             {"name": "aws.lambda"},
         )
 
     async def test_handshake_07_memory_invalid_int(self):
         await self._test_handshake(
-            {"AWS_EXECUTION_ENV": "AWS_Lambda_python3.9", "AWS_LAMBDA_FUNCTION_MEMORY_SIZE": "big"},
+            {
+                "AWS_EXECUTION_ENV": "AWS_Lambda_python3.10",
+                "AWS_LAMBDA_FUNCTION_MEMORY_SIZE": "big",
+            },
             {"name": "aws.lambda"},
         )
 

test/asynchronous/test_client.py

@@ -485,7 +485,7 @@ async def test_rtt_connection_is_disabled_poll(self):
 
     async def test_rtt_connection_is_disabled_auto(self):
         envs = [
-            {"AWS_EXECUTION_ENV": "AWS_Lambda_python3.9"},
+            {"AWS_EXECUTION_ENV": "AWS_Lambda_python3.10"},
             {"FUNCTIONS_WORKER_RUNTIME": "python"},
             {"K_SERVICE": "gcpservicename"},
             {"FUNCTION_NAME": "gcpfunctionname"},