From ffe4eed76d7cd709ea422bfd5f36a391612af86a Mon Sep 17 00:00:00 2001 From: Yu Date: Mon, 12 Aug 2024 12:24:24 +0200 Subject: [PATCH 1/4] test host_manager is running in the container --- .../tests/test_start_host_manager.py | 195 ++++++++++++++++++ 1 file changed, 195 insertions(+) create mode 100644 emulation-system/tests/test_start_host_manager.py diff --git a/emulation-system/tests/test_start_host_manager.py b/emulation-system/tests/test_start_host_manager.py new file mode 100644 index 000000000..7065a6305 --- /dev/null +++ b/emulation-system/tests/test_start_host_manager.py @@ -0,0 +1,195 @@ +import pytest +import docker +import logging +from unittest.mock import MagicMock, patch +from docker.types import IPAMConfig, IPAMPool +import time +from csle_common.dao.emulation_config.emulation_env_config import EmulationEnvConfig +from csle_common.util.emulation_util import EmulationUtil +import csle_common.constants.constants as constants +from csle_common.controllers.host_controller import HostController +import csle_collector.host_manager.host_manager_pb2_grpc +import csle_collector.host_manager.host_manager_pb2 + + +@pytest.fixture(scope="module") +def docker_client() -> None: + """ + Initialize and Provide a Docker client instance for the test + + :return: None + """ + return docker.from_env() + + +@pytest.fixture(scope="module") +def network(docker_client) -> None: + """ + Create a custom network with a specific subnet + + :param docker_client: docker_client + :yield: network + + :return: None + """ + ipam_pool = IPAMPool(subnet="15.15.15.0/24") + ipam_config = IPAMConfig(pool_configs=[ipam_pool]) + network = docker_client.networks.create("test_network", driver="bridge", ipam=ipam_config) + yield network + network.remove() + + +def get_derived_containers(docker_client, excluded_tag="blank") -> None: + """ + Get all the containers except the blank ones + + :param docker_client: docker_client + + :return: None + """ + # Get all images except those with the excluded tag + match_tag = "0.6.0" + all_images = docker_client.images.list() + derived_images = [ + image + for image in all_images + if any(match_tag in tag for tag in image.tags) + and all("base" not in tag for tag in image.tags) + and all(excluded_tag not in tag for tag in image.tags) + ] + return derived_images[:2] + + +@pytest.fixture(scope="module", params=get_derived_containers(docker.from_env())) +def container_setup(request, docker_client, network) -> None: + """ + Starts a Docker container before running tests and ensures its stopped and removed after tests complete. + + :param request: request + :param docker_client: docker_client + :yield: container + + :return: None + """ + # Create and start each derived container + image = request.param + container = docker_client.containers.create( + image.tags[0], # Use the first tag for the image + command="sh -c 'apt-get update && apt-get install -y iputils-ping && while true; do sleep 3600; done'", + detach=True, + ) + network.connect(container) + container.start() + yield container + container.stop() + container.remove() + + +@patch("csle_common.util.emulation_util.EmulationUtil.connect_admin") +@patch("csle_common.util.emulation_util.EmulationUtil.execute_ssh_cmd") +@patch("time.sleep", return_value=None) +@patch("grpc.insecure_channel") +def test_start_host_manager( + mock_grpc_channel, mock_sleep, mock_execute_ssh_cmd, mock_connect_admin, container_setup +) -> None: + """ + Test start_host_manager and verify if it runs correctly within a container. + + :param mock_grpc_channel: mock_grpc_channel + :param mock_sleep: mock_sleep + :param mock_execute_ssh_cmd: mock_execute_ssh_cmd + :param mock_connect_admin: mock_connect_admin + :param container_setup: container_setup + + :return: None + """ + + # Set up mock return values + mock_connect_admin.return_value = None + mock_execute_ssh_cmd.side_effect = [ + ("", "", 0), # Initial check, no host_manager process found + ("", "", 0), # Stopping any old process (even if none exist) + ("", "", 0), # Starting host_manager + (constants.COMMANDS.SEARCH_HOST_MANAGER, "", 0), # Final check, host_manager is now running + ] + # Create a mock EmulationEnvConfig object with necessary attributes + emulation_env_config = MagicMock(spec=EmulationEnvConfig) + emulation_env_config.get_connection.return_value = MagicMock() + # Set up mock host_manager_config attributes + host_manager_config = MagicMock() + emulation_env_config.host_manager_config = host_manager_config + emulation_env_config.host_manager_config.host_manager_port = 8080 + emulation_env_config.host_manager_config.host_manager_log_dir = "/var/log/host_manager" + emulation_env_config.host_manager_config.host_manager_log_file = "host_manager.log" + emulation_env_config.host_manager_config.host_manager_max_workers = 4 + # Create a mock logger + logger = logging.getLogger("test_logger") + ip = "15.15.15.15" + try: + container_setup.reload() + assert container_setup.status == "running", f"Container {container_setup.name} is not running" + print(f"Container {container_setup} is running.") + # Connect using the mocked connect_admin method + EmulationUtil.connect_admin(emulation_env_config=emulation_env_config, ip=ip) + # Check if the host_manager is already running + cmd = ( + constants.COMMANDS.PS_AUX + + " | " + + constants.COMMANDS.GREP + + constants.COMMANDS.SPACE_DELIM + + constants.TRAFFIC_COMMANDS.HOST_MANAGER_FILE_NAME + ) + o, e, _ = EmulationUtil.execute_ssh_cmd(cmd=cmd, conn=emulation_env_config.get_connection(ip=ip)) + + if constants.COMMANDS.SEARCH_HOST_MANAGER not in str(o): + logger.info( + f"Host manager is not running on: {ip}, starting it. Output of {cmd} was: {str(o)}, " + f"err output was: {str(e)}" + ) + # Stop old background job if running + cmd = ( + constants.COMMANDS.SUDO + + constants.COMMANDS.SPACE_DELIM + + constants.COMMANDS.PKILL + + constants.COMMANDS.SPACE_DELIM + + constants.TRAFFIC_COMMANDS.HOST_MANAGER_FILE_NAME + ) + o, e, _ = EmulationUtil.execute_ssh_cmd(cmd=cmd, conn=emulation_env_config.get_connection(ip=ip)) + # Start the host_manager + cmd = constants.COMMANDS.START_HOST_MANAGER.format( + emulation_env_config.host_manager_config.host_manager_port, + emulation_env_config.host_manager_config.host_manager_log_dir, + emulation_env_config.host_manager_config.host_manager_log_file, + emulation_env_config.host_manager_config.host_manager_max_workers, + ) + o, e, _ = EmulationUtil.execute_ssh_cmd(cmd=cmd, conn=emulation_env_config.get_connection(ip=ip)) + time.sleep(5) + cmd = ( + constants.COMMANDS.PS_AUX + + " | " + + constants.COMMANDS.GREP + + constants.COMMANDS.SPACE_DELIM + + constants.TRAFFIC_COMMANDS.HOST_MANAGER_FILE_NAME + ) + o, e, _ = EmulationUtil.execute_ssh_cmd(cmd=cmd, conn=emulation_env_config.get_connection(ip=ip)) + assert constants.COMMANDS.SEARCH_HOST_MANAGER in str( + o + ), "Host manager is not running in the container after start attempt" + else: + logger.info( + f"Host manager is already running on: {ip}. Output of {cmd} was: {str(o)}, " f"err output was: {str(e)}" + ) + mock_channel = MagicMock() + mock_grpc_channel.return_value = mock_channel + # Mock the stub and its method + mock_stub = MagicMock() + mock_channel.__enter__.return_value = mock_stub + mock_stub.GetHostStatus.return_value = csle_collector.host_manager.host_manager_pb2.HostStatusDTO() + # Make the gRPC call + status = HostController.get_host_monitor_thread_status_by_port_and_ip( + ip, emulation_env_config.host_manager_config.host_manager_port + ) + # Validate the status response + assert status is not None, "Failed to get host manager status via gRPC" + except Exception as e: + pytest.fail(f"Exception occurred while starting host manager: {str(e)}") From cd5e3556227f0a7ea2dd076db841ce6589eb3312 Mon Sep 17 00:00:00 2001 From: Yu Date: Mon, 12 Aug 2024 16:01:40 +0200 Subject: [PATCH 2/4] remove print --- emulation-system/tests/test_start_host_manager.py | 1 - 1 file changed, 1 deletion(-) diff --git a/emulation-system/tests/test_start_host_manager.py b/emulation-system/tests/test_start_host_manager.py index 7065a6305..775ee90e5 100644 --- a/emulation-system/tests/test_start_host_manager.py +++ b/emulation-system/tests/test_start_host_manager.py @@ -128,7 +128,6 @@ def test_start_host_manager( try: container_setup.reload() assert container_setup.status == "running", f"Container {container_setup.name} is not running" - print(f"Container {container_setup} is running.") # Connect using the mocked connect_admin method EmulationUtil.connect_admin(emulation_env_config=emulation_env_config, ip=ip) # Check if the host_manager is already running From cc39789906ff7d9cc94b44915424f50d0d39827a Mon Sep 17 00:00:00 2001 From: Yu Date: Tue, 13 Aug 2024 15:16:20 +0200 Subject: [PATCH 3/4] test_start_host_manager via cmd --- .../tests/test_start_host_manager.py | 153 +++++++----------- 1 file changed, 62 insertions(+), 91 deletions(-) diff --git a/emulation-system/tests/test_start_host_manager.py b/emulation-system/tests/test_start_host_manager.py index 775ee90e5..6708cc2f4 100644 --- a/emulation-system/tests/test_start_host_manager.py +++ b/emulation-system/tests/test_start_host_manager.py @@ -1,6 +1,7 @@ import pytest import docker import logging +import grpc from unittest.mock import MagicMock, patch from docker.types import IPAMConfig, IPAMPool import time @@ -10,6 +11,7 @@ from csle_common.controllers.host_controller import HostController import csle_collector.host_manager.host_manager_pb2_grpc import csle_collector.host_manager.host_manager_pb2 +from IPython.lib.editorhooks import emacs @pytest.fixture(scope="module") @@ -57,7 +59,7 @@ def get_derived_containers(docker_client, excluded_tag="blank") -> None: and all("base" not in tag for tag in image.tags) and all(excluded_tag not in tag for tag in image.tags) ] - return derived_images[:2] + return derived_images @pytest.fixture(scope="module", params=get_derived_containers(docker.from_env())) @@ -85,110 +87,79 @@ def container_setup(request, docker_client, network) -> None: container.remove() -@patch("csle_common.util.emulation_util.EmulationUtil.connect_admin") -@patch("csle_common.util.emulation_util.EmulationUtil.execute_ssh_cmd") -@patch("time.sleep", return_value=None) -@patch("grpc.insecure_channel") -def test_start_host_manager( - mock_grpc_channel, mock_sleep, mock_execute_ssh_cmd, mock_connect_admin, container_setup -) -> None: +def test_start_host_manager(container_setup) -> None: """ - Test start_host_manager and verify if it runs correctly within a container. + Start host_manager in a container - :param mock_grpc_channel: mock_grpc_channel - :param mock_sleep: mock_sleep - :param mock_execute_ssh_cmd: mock_execute_ssh_cmd - :param mock_connect_admin: mock_connect_admin :param container_setup: container_setup :return: None """ - - # Set up mock return values - mock_connect_admin.return_value = None - mock_execute_ssh_cmd.side_effect = [ - ("", "", 0), # Initial check, no host_manager process found - ("", "", 0), # Stopping any old process (even if none exist) - ("", "", 0), # Starting host_manager - (constants.COMMANDS.SEARCH_HOST_MANAGER, "", 0), # Final check, host_manager is now running - ] - # Create a mock EmulationEnvConfig object with necessary attributes + failed_containers = [] + containers_info = [] + container_setup.reload() + if container_setup.status != "running": + failed_containers.append(container_setup.name) + containers_info.append( + {"container_id": container_setup.id, "name": container_setup.name, "status": container_setup.status} + ) + print(f"Container {container_setup.name} is not running. Skipping host manager start.") + return + # Mock emulation_env_config emulation_env_config = MagicMock(spec=EmulationEnvConfig) emulation_env_config.get_connection.return_value = MagicMock() - # Set up mock host_manager_config attributes - host_manager_config = MagicMock() - emulation_env_config.host_manager_config = host_manager_config + emulation_env_config.host_manager_config = MagicMock() emulation_env_config.host_manager_config.host_manager_port = 8080 emulation_env_config.host_manager_config.host_manager_log_dir = "/var/log/host_manager" emulation_env_config.host_manager_config.host_manager_log_file = "host_manager.log" emulation_env_config.host_manager_config.host_manager_max_workers = 4 - # Create a mock logger - logger = logging.getLogger("test_logger") - ip = "15.15.15.15" + + ip = container_setup.attrs["NetworkSettings"]["IPAddress"] + port = emulation_env_config.host_manager_config.host_manager_port try: - container_setup.reload() - assert container_setup.status == "running", f"Container {container_setup.name} is not running" - # Connect using the mocked connect_admin method - EmulationUtil.connect_admin(emulation_env_config=emulation_env_config, ip=ip) - # Check if the host_manager is already running + # Start host_manager command cmd = ( - constants.COMMANDS.PS_AUX - + " | " - + constants.COMMANDS.GREP - + constants.COMMANDS.SPACE_DELIM - + constants.TRAFFIC_COMMANDS.HOST_MANAGER_FILE_NAME + f"/root/miniconda3/bin/python3 /host_manager.py " + f"--port {emulation_env_config.host_manager_config.host_manager_port} " + f"--logdir {emulation_env_config.host_manager_config.host_manager_log_dir} " + f"--logfile {emulation_env_config.host_manager_config.host_manager_log_file} " + f"--maxworkers {emulation_env_config.host_manager_config.host_manager_max_workers}" ) - o, e, _ = EmulationUtil.execute_ssh_cmd(cmd=cmd, conn=emulation_env_config.get_connection(ip=ip)) - - if constants.COMMANDS.SEARCH_HOST_MANAGER not in str(o): - logger.info( - f"Host manager is not running on: {ip}, starting it. Output of {cmd} was: {str(o)}, " - f"err output was: {str(e)}" - ) - # Stop old background job if running - cmd = ( - constants.COMMANDS.SUDO - + constants.COMMANDS.SPACE_DELIM - + constants.COMMANDS.PKILL - + constants.COMMANDS.SPACE_DELIM - + constants.TRAFFIC_COMMANDS.HOST_MANAGER_FILE_NAME - ) - o, e, _ = EmulationUtil.execute_ssh_cmd(cmd=cmd, conn=emulation_env_config.get_connection(ip=ip)) - # Start the host_manager - cmd = constants.COMMANDS.START_HOST_MANAGER.format( - emulation_env_config.host_manager_config.host_manager_port, - emulation_env_config.host_manager_config.host_manager_log_dir, - emulation_env_config.host_manager_config.host_manager_log_file, - emulation_env_config.host_manager_config.host_manager_max_workers, - ) - o, e, _ = EmulationUtil.execute_ssh_cmd(cmd=cmd, conn=emulation_env_config.get_connection(ip=ip)) - time.sleep(5) - cmd = ( - constants.COMMANDS.PS_AUX - + " | " - + constants.COMMANDS.GREP - + constants.COMMANDS.SPACE_DELIM - + constants.TRAFFIC_COMMANDS.HOST_MANAGER_FILE_NAME - ) - o, e, _ = EmulationUtil.execute_ssh_cmd(cmd=cmd, conn=emulation_env_config.get_connection(ip=ip)) - assert constants.COMMANDS.SEARCH_HOST_MANAGER in str( - o - ), "Host manager is not running in the container after start attempt" - else: - logger.info( - f"Host manager is already running on: {ip}. Output of {cmd} was: {str(o)}, " f"err output was: {str(e)}" - ) - mock_channel = MagicMock() - mock_grpc_channel.return_value = mock_channel - # Mock the stub and its method - mock_stub = MagicMock() - mock_channel.__enter__.return_value = mock_stub - mock_stub.GetHostStatus.return_value = csle_collector.host_manager.host_manager_pb2.HostStatusDTO() - # Make the gRPC call - status = HostController.get_host_monitor_thread_status_by_port_and_ip( - ip, emulation_env_config.host_manager_config.host_manager_port + # Run cmd in the container + print(f"Running command: {cmd}") + result = container_setup.exec_run(cmd, detach=True) + print("Command is running in the background.") + # Check if host_manager starts + cmd = ( + f"sh -c '{constants.COMMANDS.PS_AUX} | {constants.COMMANDS.GREP} " + f"{constants.COMMANDS.SPACE_DELIM}{constants.TRAFFIC_COMMANDS.HOST_MANAGER_FILE_NAME}'" ) - # Validate the status response - assert status is not None, "Failed to get host manager status via gRPC" + result = container_setup.exec_run(cmd) + output = result.output.decode("utf-8") + print(f"Process check output: {output}") + assert constants.COMMANDS.SEARCH_HOST_MANAGER in output, "Host manager is not running in the container" + # Print logfile + log_file_path = "/var/log/host_managerhost_manager.log" + time.sleep(5) + cmd = f"cat {log_file_path}" + result = container_setup.exec_run(cmd) + log_content = result.output.decode("utf-8") + print(f"Log file content from {container_setup.name}:\n{log_content}") + # Call grpc + print(f"Attempting to connect to host manager at {ip}:{port}") + with grpc.insecure_channel(f"{ip}:{port}", options=constants.GRPC_SERVERS.GRPC_OPTIONS) as channel: + stub = csle_collector.host_manager.host_manager_pb2_grpc.HostManagerStub(channel) + status = csle_collector.host_manager.query_host_manager.get_host_status(stub=stub) + if status: + print(f"Host Manager Status: {status}") + else: + print(f"Host Manager status is empty for container {container_setup.name}.") except Exception as e: - pytest.fail(f"Exception occurred while starting host manager: {str(e)}") + print(f"Error occurred in container {container_setup.name}: {e}") + failed_containers.append(container_setup.name) + containers_info.append({"container_id": container_setup.id, "name": container_setup.name, "error": str(e)}) + if failed_containers: + print("Containers that failed to start the host manager:") + for info in containers_info: + print(info) + assert not failed_containers, f"The following containers failed to start the host manager: {failed_containers}" From babfac476a069de60c35e45bad50cddb9905efd1 Mon Sep 17 00:00:00 2001 From: Yu Date: Tue, 13 Aug 2024 19:17:10 +0200 Subject: [PATCH 4/4] fixed bugs --- .../tests/test_start_host_manager.py | 41 +++++++------------ 1 file changed, 14 insertions(+), 27 deletions(-) diff --git a/emulation-system/tests/test_start_host_manager.py b/emulation-system/tests/test_start_host_manager.py index 6708cc2f4..f056518ab 100644 --- a/emulation-system/tests/test_start_host_manager.py +++ b/emulation-system/tests/test_start_host_manager.py @@ -2,7 +2,7 @@ import docker import logging import grpc -from unittest.mock import MagicMock, patch +from unittest.mock import MagicMock from docker.types import IPAMConfig, IPAMPool import time from csle_common.dao.emulation_config.emulation_env_config import EmulationEnvConfig @@ -77,7 +77,7 @@ def container_setup(request, docker_client, network) -> None: image = request.param container = docker_client.containers.create( image.tags[0], # Use the first tag for the image - command="sh -c 'apt-get update && apt-get install -y iputils-ping && while true; do sleep 3600; done'", + command="sh -c 'while true; do sleep 3600; done'", detach=True, ) network.connect(container) @@ -98,13 +98,7 @@ def test_start_host_manager(container_setup) -> None: failed_containers = [] containers_info = [] container_setup.reload() - if container_setup.status != "running": - failed_containers.append(container_setup.name) - containers_info.append( - {"container_id": container_setup.id, "name": container_setup.name, "status": container_setup.status} - ) - print(f"Container {container_setup.name} is not running. Skipping host manager start.") - return + assert container_setup.status == "running" # Mock emulation_env_config emulation_env_config = MagicMock(spec=EmulationEnvConfig) emulation_env_config.get_connection.return_value = MagicMock() @@ -126,9 +120,7 @@ def test_start_host_manager(container_setup) -> None: f"--maxworkers {emulation_env_config.host_manager_config.host_manager_max_workers}" ) # Run cmd in the container - print(f"Running command: {cmd}") result = container_setup.exec_run(cmd, detach=True) - print("Command is running in the background.") # Check if host_manager starts cmd = ( f"sh -c '{constants.COMMANDS.PS_AUX} | {constants.COMMANDS.GREP} " @@ -136,30 +128,25 @@ def test_start_host_manager(container_setup) -> None: ) result = container_setup.exec_run(cmd) output = result.output.decode("utf-8") - print(f"Process check output: {output}") assert constants.COMMANDS.SEARCH_HOST_MANAGER in output, "Host manager is not running in the container" - # Print logfile - log_file_path = "/var/log/host_managerhost_manager.log" time.sleep(5) - cmd = f"cat {log_file_path}" - result = container_setup.exec_run(cmd) - log_content = result.output.decode("utf-8") - print(f"Log file content from {container_setup.name}:\n{log_content}") # Call grpc - print(f"Attempting to connect to host manager at {ip}:{port}") with grpc.insecure_channel(f"{ip}:{port}", options=constants.GRPC_SERVERS.GRPC_OPTIONS) as channel: stub = csle_collector.host_manager.host_manager_pb2_grpc.HostManagerStub(channel) status = csle_collector.host_manager.query_host_manager.get_host_status(stub=stub) - if status: - print(f"Host Manager Status: {status}") - else: - print(f"Host Manager status is empty for container {container_setup.name}.") + assert status except Exception as e: print(f"Error occurred in container {container_setup.name}: {e}") failed_containers.append(container_setup.name) - containers_info.append({"container_id": container_setup.id, "name": container_setup.name, "error": str(e)}) + containers_info.append( + { + "container_status": container_setup.status, + "container_image": container_setup.image.tags, + "name": container_setup.name, + "error": str(e), + } + ) if failed_containers: print("Containers that failed to start the host manager:") - for info in containers_info: - print(info) - assert not failed_containers, f"The following containers failed to start the host manager: {failed_containers}" + print(containers_info) + assert not failed_containers, f"T{failed_containers} failed"