factor out into execution context

Author: justinchiu-cohere

commit0/harness/execution_context.py

@@ -1,10 +1,9 @@
-""" Remote code execution contexts
+"""Remote code execution contexts
 
 Implements the interface for local docker containers, remote modal sandboxes,
 and HTTP servers.
 """
 
-
 from abc import ABC
 import docker
 import logging
@@ -15,7 +14,6 @@
 from commit0.harness.spec import Spec
 from commit0.harness.docker_build import (
     close_logger,
-    setup_logger,
 )
 from commit0.harness.docker_utils import (
     cleanup_container,
@@ -26,15 +24,14 @@
     delete_file_from_container,
     exec_run_with_timeout,
 )
-from commit0.harness.utils import (
-    EvaluationError,
-    extract_test_output,
-    get_hash_string,
-    get_ip,
-    get_user,
-)
 
 
+def read_stream(stream: modal.io_streams.StreamReader) -> str:
+    strings = []
+    for line in stream:
+        strings.append(line)
+    return "\n".join(strings)
+
 class ExecutionContext(ABC):
     def __init__(
         self,
@@ -44,29 +41,29 @@ def __init__(
         timeout: int,
         log_dir: Path,
     ):
-        """ Create the remote execution context
+        """Create the remote execution context
 
         The execution context will persist for the lifetime of this object.
         The execution context can be a Docker container or Modal sandbox.
         """
         raise NotImplementedError
 
-    def copy_ssh_pubkey_from_remote(self):
+    def copy_ssh_pubkey_from_remote(self) -> None:
         raise NotImplementedError
 
-    def copy_to_remote(self, local_path, remote_path):
+    def copy_to_remote(self, local_path: Path, remote_path: Path) -> None:
         raise NotImplementedError
 
-    def exec_run_with_timeout(self, command, timeout):
+    def exec_run_with_timeout(self, command: str, timeout: int) -> None:
         raise NotImplementedError
 
-    def exec_run(self, command):
+    def exec_run(self, command: str) -> None:
         raise NotImplementedError
 
-    def copy_from_remote(self, remote_path, local_path):
+    def copy_from_remote(self, remote_path, local_path) -> None:
         raise NotImplementedError
 
-    def delete_file_from_remote(self, remote_path):
+    def delete_file_from_remote(self, remote_path) -> None:
         raise NotImplementedError
 
     def __enter__(self):
@@ -85,10 +82,10 @@ def __init__(
         timeout: int,
         log_dir: Path,
     ):
-        client = docker.from_env()
+        self.client = docker.from_env()
         self.logger = logger
         self.container = create_container(
-            client=client,
+            client=self.client,
             image_name=spec.repo_image_key,
             container_name=spec.get_container_name(),
             logger=logger,
@@ -103,9 +100,7 @@ def copy_to_remote(self, local_file: Path, remote_path: Path) -> None:
         copy_to_container(self.container, local_file, remote_path)
 
     def exec_run_with_timeout(self, command: str, timeout: int) -> ():
-        return exec_run_with_timeout(
-            self.container, command, timeout
-        )
+        return exec_run_with_timeout(self.container, command, timeout)
 
     def exec_run(self, command: str) -> None:
         return self.container.exec_run(command, demux=True)
@@ -147,6 +142,8 @@ def __init__(
             network_file_systems={
                 "/vol": self.nfs,
             },
+            cpu=8.0,
+            timeout=30,
         )
 
         self.copy_ssh_pubkey_from_remote()
@@ -174,32 +171,30 @@ def copy_ssh_pubkey_from_remote(self):
                 authorized_keys_file.write(public_key + "\n")
 
     def copy_to_remote(self, local_path: Path, remote_path: Path) -> None:
+        tempname = "tmpfile"
         with local_path.open("rb") as f:
-            self.nfs.write_file(str(local_path), f)
-        self.sandbox.exec("bash", "-c", f"cp /vol/{str(local_path)} {str(remote_path)}")
+            self.nfs.write_file(tempname, f)
+        self.sandbox.exec("bash", "-c", f"cp /vol/{tempname} {str(remote_path)}")
 
     def exec_run_with_timeout(self, command: str, timeout: int) -> None:
         """Execute command on modal sandbox"""
+        print("Executing:", command)
         process = self.sandbox.exec("bash", "-c", command)
-        stdout = []
-        for line in process.stdout:
-            stdout.append(line)
-        stderr = []
-        for line in process.stderr:
-            stderr.append(line)
-        return "\n".join(stdout), False, 1
-        return "\n".join(stdout), "\n".join(stderr)
+        print("stdout")
+        stdout = read_stream(process.stdout)
+        print("stderr")
+        stderr = read_stream(process.stderr)
+        print(stderr)
+        return stdout, False, 1
+        return stdout, stderr
 
     def exec_run(self, command: str) -> None:
         """Execute command on modal sandbox"""
         process = self.sandbox.exec("bash", "-c", command)
-        stdout = []
-        for line in process.stdout:
-            stdout.append(line)
-        stderr = []
-        for line in process.stderr:
-            stderr.append(line)
-        return 1, "\n".join(stdout)
+        stdout = read_stream(process.stdout)
+        stderr = read_stream(process.stderr)
+        print(stderr)
+        return 1, stdout
 
     def copy_from_remote(self, remote_path: Path, local_path: Path) -> None:
         """Copy file from modal sandbox"""
@@ -215,4 +210,5 @@ def __enter__(self):
         return self
 
     def __exit__(self, exc_type, exc_value, exc_traceback):
-        self.nfs.__exit__()
+        # self.nfs.__exit__()
+        pass

commit0/harness/modal_utils.py

@@ -17,6 +17,7 @@ def create_sandbox(image: modal.Image, nfs: modal.NetworkFileSystem) -> modal.Sa
         network_file_systems={
             "/vol": nfs,
         },
+        timeout=60,
     )
 
 
@@ -38,9 +39,10 @@ def copy_file_to_sandbox(
     sandbox: modal.Sandbox, nfs: modal.NetworkFileSystem, src: Path, dst: Path
 ) -> None:
     """Copy file to modal sandbox"""
+    path = "tmpfile"
     with src.open("rb") as f:
-        nfs.write_file(str(src), f)
-    sandbox.exec("bash", "-c", f"cp /vol/{str(src)} {str(dst)}")
+        nfs.write_file(path, f)
+    sandbox.exec("bash", "-c", f"cp /vol/{path} {str(dst)}")
 
 
 def copy_from_sandbox(sandbox: modal.Sandbox, src: Path, dst: Path) -> None:

commit0/harness/run_pytest_ids.py

@@ -35,7 +35,6 @@
 )
 
 
-
 class ExecutionBackend(StrEnum):
     LOCAL = auto()
     MODAL = auto()
@@ -149,40 +148,15 @@ def run_modal(
         copy_file_to_sandbox(sandbox, nfs, eval_file, Path("/eval.sh"))
 
         # DBG: check if eval file properly copied
+        print("checking for eval.sh")
         print(execute_command(sandbox, "ls /")[0])
 
         # execute tests
         output, error = execute_command(sandbox, "/bin/bash /eval.sh")
         # TODO: add timeout
-        #print(output)
-        #print(error)
-
-        timed_out = False
-        test_output = extract_test_output(
-            output, "--json-report --json-report-file=report.json"
-        )
-
-        # stdout might be more straightforward
-        print(test_output)
-        test_output_path = log_dir / "test_output.txt"
-        with open(test_output_path, "w") as f:
-            f.write(test_output)
-            if timed_out:
-                f.write(f"\n\nTimeout error: {timeout} seconds exceeded.")
-                raise EvaluationError(
-                    spec.repo,
-                    f"Test timed out after {timeout} seconds.",
-                    logger,
-                )
-
-        # copy back report.json if there is any
-        report_file = Path(spec.repo_directory) / "report.json"
-        # Run the test command inside the container to check if the file exists
-        exit_code, output = container.exec_run(f"test -e {report_file}", demux=True)
-        # Check the exit code of the command
-        if exit_code == 0:
-            copy_from_sandbox(container, report_file, Path(log_dir / "report.json"))
-            delete_file_from_sandbox(container, str(report_file))
+        print(output)
+        print(error)
+        return
 
 
 def main(
@@ -235,6 +209,11 @@ def main(
     eval_file = Path(log_dir / "eval.sh")
     eval_file.write_text(eval_script)
 
+    #run_docker(spec, logger, eval_file, timeout, log_dir)
+    run_modal(spec, logger, eval_file, timeout, log_dir)
+    return
+
+    #backend = "local"
     backend = "modal"
     execution_context = None
     if ExecutionBackend(backend) == ExecutionBackend.MODAL:
@@ -244,7 +223,12 @@ def main(
     print(backend, execution_context)
 
     with execution_context(spec, logger, eval_file, timeout, log_dir) as context:
-        context.copy_to_remote(eval_file, "/eval.sh")
+        context.copy_to_remote(eval_file, Path("/eval.sh"))
+        print(context.exec_run("ls /vol")[1])
+        print(context.exec_run("ls /")[1])
+
+        print(context.exec_run("cat /eval.sh")[1])
+
         output, timed_out, total_runtime = context.exec_run_with_timeout(
             "/bin/bash /eval.sh", timeout
         )
@@ -269,19 +253,17 @@ def main(
         # copy back report.json if there is any
         report_file = Path(spec.repo_directory) / "report.json"
         # Run the test command inside the container to check if the file exists
-        #exit_code, output = container.exec_run(f"test -e {report_file}", demux=True)
+        # exit_code, output = container.exec_run(f"test -e {report_file}", demux=True)
         exit_code, output = context.exec_run(f"test -e {report_file}")
         # Check the exit code of the command
         if exit_code == 0:
             context.copy_from_remote(report_file, Path(log_dir / "report.json"))
             context.delete_file_from_remote(report_file)
 
-
     """
     if ExecutionBackend(backend) == ExecutionBackend.LOCAL:
         run_docker(spec, logger, eval_file, timeout, log_dir)
     elif ExecutionBackend(backend) == ExecutionBackend.MODAL:
-        run_modal(spec, logger, eval_file, timeout, log_dir)
     """