HTTP client creation helper

tools/azure-sdk-tools/devtools_testutils/helpers.py

@@ -5,6 +5,7 @@
 # --------------------------------------------------------------------------
 import os
 import sys
+from urllib3 import PoolManager, Retry
 
 from azure_devtools.scenario_tests.config import TestConfig
 
@@ -15,6 +16,24 @@
 this.recording_ids = {}
 
 
+def get_http_client(**kwargs):
+    """Returns a `urllib3` client that provides the test proxy's self-signed certificate if it's available.
+
+    This helper method was implemented since the REQUESTS_CA_BUNDLE environment variable is only automatically set after
+    `proxy_startup.py` starts up the tool. Module-level HTTP clients could be created before this variable is set and
+    therefore fail subsequent SSL requests.
+    """
+    if os.getenv("REQUESTS_CA_BUNDLE"):
+        http_client = PoolManager(
+            retries=Retry(total=3, raise_on_status=kwargs.get("raise_on_status", True)),
+            cert_reqs="CERT_REQUIRED",
+            ca_certs=os.getenv("REQUESTS_CA_BUNDLE"),
+        )
+    else:
+        http_client = PoolManager(retries=Retry(total=3, raise_on_status=kwargs.get("raise_on_status", True)))
+    return http_client
+
+
 def get_recording_id():
     """Fetches the recording ID for the currently executing test"""
     test_id = get_test_id()
@@ -26,8 +45,7 @@ def set_recording_id(test_id, recording_id):
     this.recording_ids[test_id] = recording_id
 
 
-def get_test_id():
-    # type: () -> str
+def get_test_id() -> str:
     # pytest sets the current running test in an environment variable
     # the path to the test can depend on the environment, so we can't assume this is the path from the repo root
     setting_value = os.getenv("PYTEST_CURRENT_TEST")

tools/azure-sdk-tools/devtools_testutils/proxy_startup.py

@@ -19,13 +19,12 @@
 from dotenv import load_dotenv, find_dotenv
 import pytest
 import subprocess
-from urllib3 import PoolManager, Retry
 from urllib3.exceptions import SSLError
 
 from ci_tools.variables import in_ci
 
 from .config import PROXY_URL
-from .helpers import is_live_and_not_recording
+from .helpers import get_http_client, is_live_and_not_recording
 from .sanitizers import add_remove_header_sanitizer, set_custom_default_matcher
 
 
@@ -82,15 +81,6 @@
 
 discovered_roots = []
 
-if os.getenv("REQUESTS_CA_BUNDLE"):
-    http_client = PoolManager(
-        retries=Retry(total=3, raise_on_status=False),
-        cert_reqs="CERT_REQUIRED",
-        ca_certs=os.getenv("REQUESTS_CA_BUNDLE"),
-    )
-else:
-    http_client = PoolManager(retries=Retry(total=1, raise_on_status=False))
-
 
 def get_target_version(repo_root: str) -> str:
     """Gets the target test-proxy version from the target_version.txt file in /eng/common/testproxy"""
@@ -143,6 +133,7 @@ def ascend_to_root(start_dir_or_file: str) -> str:
 def check_availability() -> None:
     """Attempts request to /Info/Available. If a test-proxy instance is responding, we should get a response."""
     try:
+        http_client = get_http_client(raise_on_status=False)
         response = http_client.request(method="GET", url=PROXY_CHECK_URL, timeout=10)
         return response.status
     # We get an SSLError if the container is started but the endpoint isn't available yet
@@ -237,7 +228,7 @@ def prepare_local_tool(repo_root: str) -> str:
                 download_url = PROXY_DOWNLOAD_URL.format(target_proxy_version, target_info["file_name"])
                 download_file = os.path.join(download_folder, target_info["file_name"])
 
-                http_client = PoolManager()
+                http_client = get_http_client()
                 with open(download_file, "wb") as out:
                     r = http_client.request("GET", download_url, preload_content=False)
                     shutil.copyfileobj(r, out)
@@ -329,7 +320,7 @@ def start_test_proxy(request) -> None:
 
     # Wait for the proxy server to become available
     check_proxy_availability()
-    # remove headers from recordings if we don't need them, and ignore them if present
+    # Remove headers from recordings if we don't need them, and ignore them if present
     # Authorization, for example, can contain sensitive info and can cause matching failures during challenge auth
     headers_to_ignore = "Authorization, x-ms-client-request-id, x-ms-request-id"
     add_remove_header_sanitizer(headers=headers_to_ignore)

tools/azure-sdk-tools/devtools_testutils/proxy_testcase.py

@@ -19,9 +19,8 @@
 from azure_devtools.scenario_tests.utilities import trim_kwargs_from_test_function
 
 from .config import PROXY_URL
-from .helpers import get_test_id, is_live, is_live_and_not_recording, set_recording_id
+from .helpers import get_http_client, get_test_id, is_live, is_live_and_not_recording, set_recording_id
 from .proxy_startup import discovered_roots
-from urllib3 import PoolManager, Retry
 from urllib3.exceptions import HTTPError
 import json
 
@@ -32,15 +31,6 @@
 # To learn about how to migrate SDK tests to the test proxy, please refer to the migration guide at
 # https://github.com/Azure/azure-sdk-for-python/blob/main/doc/dev/test_proxy_migration_guide.md
 
-if os.getenv("REQUESTS_CA_BUNDLE"):
-    http_client = PoolManager(
-        retries=Retry(total=3, raise_on_status=False),
-        cert_reqs="CERT_REQUIRED",
-        ca_certs=os.getenv("REQUESTS_CA_BUNDLE"),
-    )
-else:
-    http_client = PoolManager(retries=Retry(total=3, raise_on_status=False))
-
 # defaults
 RECORDING_START_URL = "{}/record/start".format(PROXY_URL)
 RECORDING_STOP_URL = "{}/record/stop".format(PROXY_URL)
@@ -49,9 +39,7 @@
 
 
 def get_recording_assets(test_id: str) -> str:
-    """
-    Used to retrieve the assets.json given a PYTEST_CURRENT_TEST test id.
-    """
+    """Used to retrieve the assets.json given a PYTEST_CURRENT_TEST test id."""
     for root in discovered_roots:
         current_dir = os.path.dirname(test_id)
         while current_dir is not None and not (os.path.dirname(current_dir) == current_dir):
@@ -85,6 +73,7 @@ def start_record_or_playback(test_id: str) -> "Tuple[str, Dict[str, str]]":
         json_payload["x-recording-assets-file"] = assets_json
 
     encoded_payload = json.dumps(json_payload).encode("utf-8")
+    http_client = get_http_client()
 
     if is_live():
         result = http_client.request(
@@ -127,6 +116,7 @@ def start_record_or_playback(test_id: str) -> "Tuple[str, Dict[str, str]]":
 
 def stop_record_or_playback(test_id: str, recording_id: str, test_variables: "Dict[str, str]") -> None:
     try:
+        http_client = get_http_client()
         if is_live():
             http_client.request(
                 method="POST",

tools/azure-sdk-tools/devtools_testutils/sanitizers.py

@@ -3,26 +3,15 @@
 # Licensed under the MIT License. See License.txt in the project root for
 # license information.
 # --------------------------------------------------------------------------
+import json
 from typing import TYPE_CHECKING
 
-
-from urllib3 import PoolManager, Retry
-import json, os
-
 from .config import PROXY_URL
-from .helpers import get_recording_id, is_live, is_live_and_not_recording
+from .helpers import get_http_client, get_recording_id, is_live, is_live_and_not_recording
 
 if TYPE_CHECKING:
     from typing import Optional
 
-if os.getenv("REQUESTS_CA_BUNDLE"):
-    http_client = PoolManager(
-        retries=Retry(total=3, raise_on_status=True),
-        cert_reqs="CERT_REQUIRED",
-        ca_certs=os.getenv("REQUESTS_CA_BUNDLE"),
-    )
-else:
-    http_client = PoolManager(retries=Retry(total=3, raise_on_status=True))
 
 # This file contains methods for adjusting many aspects of test proxy behavior:
 #
@@ -635,6 +624,7 @@ def _send_matcher_request(matcher: str, headers: dict, parameters: "Optional[dic
         if headers[key] is not None:
             headers_to_send[key] = headers[key]
 
+    http_client = get_http_client()
     http_client.request(
         method="POST",
         url=f"{PROXY_URL}/Admin/SetMatcher",
@@ -662,6 +652,7 @@ def _send_recording_options_request(parameters: dict, headers: "Optional[dict]"
         if headers[key] is not None:
             headers_to_send[key] = headers[key]
 
+    http_client = get_http_client()
     http_client.request(
         method="POST",
         url=f"{PROXY_URL}/Admin/SetRecordingOptions",
@@ -687,6 +678,7 @@ def _send_reset_request(headers: dict) -> None:
         if headers[key] is not None:
             headers_to_send[key] = headers[key]
 
+    http_client = get_http_client()
     http_client.request(method="POST", url=f"{PROXY_URL}/Admin/Reset", headers=headers_to_send)
 
 
@@ -708,6 +700,7 @@ def _send_sanitizer_request(sanitizer: str, parameters: dict, headers: "Optional
         if headers[key] is not None:
             headers_to_send[key] = headers[key]
 
+    http_client = get_http_client()
     http_client.request(
         method="POST",
         url="{}/Admin/AddSanitizer".format(PROXY_URL),
@@ -733,6 +726,7 @@ def _send_transform_request(transform: str, parameters: dict, headers: "Optional
         if headers[key] is not None:
             headers_to_send[key] = headers[key]
 
+    http_client = get_http_client()
     http_client.request(
         method="POST",
         url=f"{PROXY_URL}/Admin/AddTransform",