Merge pull request MaterializeInc#21349 from nrainer-materialize/clou…

…dtests/repatriate-envconfig

misc/python/materialize/cloudtest/config/environment_config.py

@@ -1,121 +0,0 @@
-# Copyright Materialize, Inc. and contributors. All rights reserved.
-#
-# Use of this software is governed by the Business Source License
-# included in the LICENSE file at the root of this repository.
-#
-# As of the Change Date specified in that file, in accordance with
-# the Business Source License, use of this software will be governed
-# by the Apache License, Version 2.0.
-
-from dataclasses import dataclass, field
-from typing import Callable, Optional
-
-import pytest
-
-from materialize.cloudtest.util.authentication import AuthConfig, get_auth
-from materialize.cloudtest.util.controller import ControllerDefinition
-
-
-@dataclass
-class ControllerDefinitions:
-    region = ControllerDefinition(
-        "region-controller",
-        "8002",
-        has_configurable_address=False,
-    )
-    environment = ControllerDefinition(
-        "environment-controller",
-        "8001",
-        has_configurable_address=False,
-    )
-    sync_server = ControllerDefinition("sync-server", "8003")
-    internal_api_server = ControllerDefinition("internal-api-server", "8032")
-    region_api_server = ControllerDefinition("region-api-server", "8033")
-
-    def all(self) -> list[ControllerDefinition]:
-        return [
-            self.region,
-            self.environment,
-            self.sync_server,
-            self.internal_api_server,
-            self.region_api_server,
-        ]
-
-
-@dataclass
-class EnvironmentConfig:
-    region: str
-    stack: str
-
-    system_context: str
-    environment_context: str
-
-    controllers: ControllerDefinitions
-
-    e2e_test_user_email: Optional[str]
-
-    refresh_auth_fn: Callable[[], AuthConfig]
-    auth: AuthConfig = field(init=False)
-
-    def __post_init__(self) -> None:
-        self.refresh_auth()
-
-    def refresh_auth(self) -> None:
-        self.auth = self.refresh_auth_fn()
-
-
-@dataclass
-class PgTestDBConfig:
-    testdb_dbname: str
-    testdb_user: str
-    testdb_password: str
-    testdb_host: str
-    testdb_port: str
-
-
-def load_environment_config(pytestconfig: pytest.Config) -> EnvironmentConfig:
-    args = pytestconfig.option
-    args_dict = vars(args)
-
-    controllers = ControllerDefinitions()
-
-    for controller in controllers.all():
-        if controller.has_configurable_address:
-            controller.endpoint = args_dict[
-                f"{controller.name.replace('-', '_')}_address"
-            ]
-
-    config = EnvironmentConfig(
-        system_context=args.system_context,
-        environment_context=args.environment_context,
-        controllers=controllers,
-        e2e_test_user_email=args.e2e_test_user_email,
-        refresh_auth_fn=lambda: get_auth(args),
-        region=args.region,
-        stack=args.stack,
-    )
-
-    return config
-
-
-def load_pg_test_db_config(pytestconfig: pytest.Config) -> PgTestDBConfig:
-    args = pytestconfig.option
-
-    testdb_dbname = args.testdb_dbname
-    assert testdb_dbname is not None
-    testdb_user = args.testdb_user
-    assert testdb_user is not None
-    testdb_password = args.testdb_password
-    assert testdb_password is not None
-    testdb_host = args.testdb_host
-    assert testdb_host is not None
-    testdb_port = args.testdb_port
-    assert testdb_port is not None
-
-    return PgTestDBConfig(
-        testdb_dbname=testdb_dbname,
-        testdb_user=testdb_user,
-        testdb_password=testdb_password,
-        testdb_host=testdb_host,
-        testdb_port=testdb_port,
-    )

misc/python/materialize/cloudtest/util/authentication.py

@@ -6,10 +6,11 @@
 # As of the Change Date specified in that file, in accordance with
 # the Business Source License, use of this software will be governed
 # by the Apache License, Version 2.0.
+from __future__ import annotations
 
 import argparse
-from dataclasses import dataclass
-from typing import Optional
+from dataclasses import dataclass, fields
+from typing import Callable, Optional
 
 import requests
 from requests.exceptions import ConnectionError
@@ -22,20 +23,33 @@
 class AuthConfig:
     organization_id: str
     token: str
+    app_user: Optional[str]
     app_password: Optional[str]
 
+    refresh_fn: Callable[[AuthConfig], None]
+
+    def refresh(self) -> None:
+        self.refresh_fn(self)
+
 
 DEFAULT_ORG_ID = "80b1a04a-2277-11ed-a1ce-5405dbb9e0f7"
 
+
 # TODO: this retry loop should not be necessary, but we are seeing
 # connections getting frequently (but sporadically) interrupted here - we
 # should track this down and remove these retries
-def get_auth(args: argparse.Namespace) -> AuthConfig:
-    config: AuthConfig = retry(lambda: _get_auth(args), 5, [ConnectionError])
+def create_auth(
+    args: argparse.Namespace, refresh_fn: Callable[[AuthConfig], None]
+) -> AuthConfig:
+    config: AuthConfig = retry(
+        lambda: _create_auth(args, refresh_fn), 5, [ConnectionError]
+    )
     return config
 
 
-def _get_auth(args: argparse.Namespace) -> AuthConfig:
+def _create_auth(
+    args: argparse.Namespace, refresh_fn: Callable[[AuthConfig], None]
+) -> AuthConfig:
     if args.e2e_test_user_email is not None:
         assert args.e2e_test_user_password is not None
         assert args.frontegg_host is not None
@@ -55,19 +69,30 @@ def _get_auth(args: argparse.Namespace) -> AuthConfig:
         response.raise_for_status()
 
         organization_id = response.json()["tenantId"]
+        app_user = args.e2e_test_user_email
         app_password = make_app_password(args.frontegg_host, token)
     else:
         organization_id = DEFAULT_ORG_ID
         token = make_jwt(tenant_id=organization_id)
+        app_user = None
         app_password = None
 
     return AuthConfig(
         organization_id=organization_id,
         token=token,
+        app_user=app_user,
         app_password=app_password,
+        refresh_fn=refresh_fn,
     )
 
 
+def update_auth(args: argparse.Namespace, auth: AuthConfig) -> None:
+    new_auth = create_auth(args, auth.refresh_fn)
+
+    for field in fields(new_auth):
+        setattr(auth, field.name, getattr(new_auth, field.name))
+
+
 def make_app_password(frontegg_host: str, token: str) -> str:
     response = requests.post(
         f"https://{frontegg_host}/frontegg/identity/resources/users/api-tokens/v1",

misc/python/materialize/cloudtest/util/controller.py

@@ -14,8 +14,9 @@
 from dataclasses import dataclass
 from typing import Any, Optional, Union
 
+from materialize.cloudtest.util.authentication import AuthConfig
 from materialize.cloudtest.util.common import eprint, retry
-from materialize.cloudtest.util.docker_env import docker_env
+from materialize.cloudtest.util.web_request import WebRequests
 
 
 @dataclass
@@ -56,6 +57,11 @@ def configured_base_url(self) -> str:
 
         return self.endpoint.base_url
 
+    def requests(
+        self, auth: Optional[AuthConfig], client_cert: Optional[tuple[str, str]] = None
+    ) -> WebRequests:
+        return WebRequests(auth, self.configured_base_url(), client_cert)
+
 
 def wait_for_connectable(
     address: Union[tuple[Any, int], str],
@@ -102,7 +108,7 @@ def parse_url(s: str) -> urllib.parse.ParseResult:
     return parsed
 
 
-def launch_controllers(controller_names: list[str]) -> None:
+def launch_controllers(controller_names: list[str], docker_env: dict[str, str]) -> None:
     try:
         subprocess.run(
             [
@@ -113,7 +119,7 @@ def launch_controllers(controller_names: list[str]) -> None:
             ],
             capture_output=True,
             check=True,
-            env=docker_env(),
+            env=docker_env,
         )
     except subprocess.CalledProcessError as e:
         eprint(e.returncode, e.stdout, e.stderr)
@@ -126,13 +132,13 @@ def wait_for_controllers(*endpoints: Endpoint) -> None:
         wait_for_connectable(endpoint.host_port)
 
 
-def cleanup_controllers() -> None:
+def cleanup_controllers(docker_env: dict[str, str]) -> None:
     try:
         subprocess.run(
             ["bin/compose", "down", "-v"],
             capture_output=True,
             check=True,
-            env=docker_env(),
+            env=docker_env,
         )
     except subprocess.CalledProcessError as e:
         eprint(e.returncode, e.stdout, e.stderr)

misc/python/materialize/cloudtest/util/docker_env.py

@@ -1,32 +0,0 @@
-# Copyright Materialize, Inc. and contributors. All rights reserved.
-#
-# Use of this software is governed by the Business Source License
-# included in the LICENSE file at the root of this repository.
-#
-# As of the Change Date specified in that file, in accordance with
-# the Business Source License, use of this software will be governed
-# by the Apache License, Version 2.0.
-
-import os
-import uuid
-from typing import Any
-
-from materialize.cloudtest.util.jwt_key import JWK_PUBLIC_KEY
-
-
-def docker_env() -> dict[Any, str]:
-    docker_env = os.environ.copy()
-    docker_env.update(
-        FRONTEGG_JWK=JWK_PUBLIC_KEY.decode("utf8"),
-        FRONTEGG_URL="https://cloud.materialize.com",
-        # Since these tests don't run against a valid Frontegg workspace, bypass account blocking checks
-        MZCLOUD_SYNC_SERVER_FRONTEGG_DISABLED="true",
-        FRONTEGG_CLIENT_ID=str(uuid.uuid4()),
-        FRONTEGG_SECRET_KEY=str(uuid.uuid4()),
-        # End account blocking bypass block
-        ENVIRONMENTD_IAM_ROLE_ARN="arn:aws:ec2:us-east-1:123445667:iam-role/environmentd-11223344551122334",
-        IAM_ROLE_ARN="arn:aws:ec2:us-east-1:123445667:iam-role/controller-11223344551122334",
-        STACK_TYPE="kind",
-        CLOUD_PROVIDER="aws",
-    )
-    return docker_env