Merge pull request #1254 from microsoft/enhancement-ci-windows-test-system

[ci] Support System Tests in Windows

Author: ppenna

tools/ci/config/test/catnapw.yaml

@@ -0,0 +1,21 @@
+catnapw:
+  udp_ping_pong: {}
+  udp_push_pop: {}
+  tcp_close:
+    nclients: [32]
+    run_mode: [sequential, concurrent]
+    who_closes: [client, server]
+  tcp_wait:
+    nclients: [32]
+    scenario: [push_close_wait, push_async_close_wait,
+               push_async_close_pending_wait, pop_close_wait,
+               pop_async_close_wait,
+               pop_async_close_pending_wait]
+  tcp_ping_pong: {}
+  tcp_push_pop: {}
+  tcp_echo:
+    bufsize: [64, 1024]
+    nclients: [1, 32]
+    nrequests: [128, 1024]
+    run_mode: [sequential, concurrent]
+    nthreads: [1, 2, 4]

tools/ci/job/factory.py

@@ -68,7 +68,22 @@ def integration_test(self, run_mode="") -> BaseJob:
 
     def system_test(self, test_name: str, niterations: int = 0, run_mode: str = "", nclients: int = 0, bufsize: int = 0, nrequests: int = 0, nthreads: int = 1, who_closes: str = "", scenario: str = "") -> BaseJob:
         if self.config["platform"] == "windows":
-            raise Exception("System tests are not supported on Windows")
+            if test_name == "tcp_echo":
+                return windows.TcpEchoTest(self.config, run_mode, nclients, bufsize, nrequests, nthreads)
+            elif test_name == "tcp_close":
+                return windows.TcpCloseTest(self.config, run_mode=run_mode, who_closes=who_closes, nclients=nclients)
+            elif test_name == "tcp_wait":
+                return windows.TcpWaitTest(self.config, scenario=scenario, nclients=nclients)
+            elif test_name == "tcp_ping_pong":
+                return windows.TcpPingPongTest(self.config)
+            elif test_name == "tcp_push_pop":
+                return windows.TcpPushPopTest(self.config)
+            elif test_name == "udp_ping_pong":
+                return windows.UdpPingPongTest(self.config)
+            elif test_name == "udp_push_pop":
+                return windows.UdpPushPopTest(self.config)
+            else:
+                raise Exception("Invalid test name")
         else:
             if self.config["libos"] == "catmem":
                 if test_name == "open-close":

tools/ci/job/windows.py

@@ -98,6 +98,110 @@ def execute(self) -> bool:
         return super().execute()
 
 
+class EndToEndTestJobOnWindows(BaseWindowsJob):
+
+    def __init__(self, config, job_name: str):
+        self.job_name = job_name
+        self.all_pass = config["all_pass"]
+        super().__init__(config, job_name)
+
+    def execute(self, server_cmd: str, client_cmd: str) -> bool:
+        serverTask: RunOnWindows = RunOnWindows(
+            super().server(), super().repository(), server_cmd, super().is_debug(), super().is_sudo(), super().config_path())
+        jobs: dict[str, subprocess.Popen[str]] = {}
+        jobs[self.job_name + "-server-" +
+             super().server()] = serverTask.execute()
+        time.sleep(super().delay())
+        clientTask: RunOnWindows = RunOnWindows(
+            super().client(), super().repository(), client_cmd, super().is_debug(), super().is_sudo(), super().config_path())
+        jobs[self.job_name + "-client-" +
+             super().client()] = clientTask.execute()
+        return wait_and_report(self.job_name, super().log_directory(), jobs, self.all_pass)
+
+
+class SystemTestJobOnWindows(EndToEndTestJobOnWindows):
+    def __init__(self, config: dict):
+        self.test_name = config["test_name"]
+        self.server_args = config["server_args"]
+        self.client_args = config["client_args"]
+        super().__init__(config, f"system-test-{config['test_alias']}")
+
+    def execute(self) -> bool:
+        server_cmd: str = f"test-system-rust LIBOS={super().libos()} TEST={self.test_name} ARGS=\\'{self.server_args}\\'"
+        client_cmd: str = f"test-system-rust LIBOS={super().libos()} TEST={self.test_name} ARGS=\\'{self.client_args}\\'"
+        return super().execute(server_cmd, client_cmd)
+
+
+class TcpCloseTest(SystemTestJobOnWindows):
+    def __init__(self, config: dict, run_mode: str, who_closes: str, nclients: int):
+        config["test_name"] = "tcp-close"
+        config["test_alias"] = f"tcp-close-{run_mode}-{who_closes}-closes-sockets"
+        config["all_pass"] = True
+        config["server_args"] = f"--peer server --address {config['server_addr']}:12345 --nclients {nclients} --run-mode {run_mode} --whocloses {who_closes}"
+        config["client_args"] = f"--peer client --address {config['server_addr']}:12345 --nclients {nclients} --run-mode {run_mode} --whocloses {who_closes}"
+        super().__init__(config)
+
+
+class TcpEchoTest(SystemTestJobOnWindows):
+    def __init__(self, config: dict, run_mode: str, nclients: int, bufsize: int, nrequests: int, nthreads: int):
+        config["test_name"] = "tcp-echo"
+        config["test_alias"] = f"tcp-echo-{run_mode}-{nclients}-{bufsize}-{nrequests}-{nthreads}"
+        config["all_pass"] = True
+        config["server_args"] = f"--peer server --address {config['server_addr']}:12345 --nthreads {nthreads}"
+        config["client_args"] = f"--peer client --address {config['server_addr']}:12345 --nclients {nclients} --nrequests {nrequests} --bufsize {bufsize} --run-mode {run_mode}"
+        super().__init__(config)
+
+
+class TcpPingPongTest(SystemTestJobOnWindows):
+    def __init__(self, config: dict):
+        config["test_name"] = "tcp-ping-pong"
+        config["test_alias"] = "tcp-ping-pong"
+        config["all_pass"] = True
+        config["server_args"] = f"--server {config['server_addr']}:12345"
+        config["client_args"] = f"--client {config['server_addr']}:12345"
+        super().__init__(config)
+
+
+class TcpPushPopTest(SystemTestJobOnWindows):
+    def __init__(self, config: dict):
+        config["test_name"] = "tcp-push-pop"
+        config["test_alias"] = "tcp-push-pop"
+        config["all_pass"] = True
+        config["server_args"] = f"--server {config['server_addr']}:12345"
+        config["client_args"] = f"--client {config['server_addr']}:12345"
+        super().__init__(config)
+
+
+class TcpWaitTest(SystemTestJobOnWindows):
+    def __init__(self, config: dict, scenario: str, nclients: int):
+        config["test_name"] = "tcp-wait"
+        config["test_alias"] = f"tcp-wait-scenario-{scenario}"
+        config["all_pass"] = True
+        config["server_args"] = f"--peer server --address {config['server_addr']}:12345 --nclients {nclients} --scenario {scenario}"
+        config["client_args"] = f"--peer client --address {config['server_addr']}:12345 --nclients {nclients} --scenario {scenario}"
+        super().__init__(config)
+
+
+class UdpPingPongTest(SystemTestJobOnWindows):
+    def __init__(self, config: dict):
+        config["test_name"] = "udp-ping-pong"
+        config["test_alias"] = "udp-ping-pong"
+        config["all_pass"] = False
+        config["server_args"] = f"--server {config['server_addr']}:12345 {config['client_addr']}:23456"
+        config["client_args"] = f"--client {config['client_addr']}:23456 {config['server_addr']}:12345"
+        super().__init__(config)
+
+
+class UdpPushPopTest(SystemTestJobOnWindows):
+    def __init__(self, config: dict):
+        config["test_name"] = "udp-push-pop"
+        config["test_alias"] = "udp-push-pop"
+        config["all_pass"] = True
+        config["server_args"] = f"--server {config['server_addr']}:12345 {config['client_addr']}:23456"
+        config["client_args"] = f"--client {config['client_addr']}:23456 {config['server_addr']}:12345"
+        super().__init__(config)
+
+
 class IntegrationTestJobOnWindows(BaseWindowsJob):
     def __init__(self, config: dict, name: str):
         super().__init__(config, name)

tools/demikernel_ci.py

@@ -81,7 +81,7 @@ def run_pipeline(
                 status["integration_tests"] = factory.integration_test("pop-wait-async").execute()
 
     # STEP 5: Run system tests.
-    if test_system and config["platform"] == "linux":
+    if test_system and config["platform"]:
         if status["checkout"] and status["compile"]:
             ci_map = read_yaml(libos)
             # Run pipe-open test.