start refactor of sudo modal

Author: justinchiu-cohere

commit0/harness/modal_utils.py

@@ -0,0 +1,72 @@
+""" Modal utility functions
+
+A mirror of the docker utility functions.
+"""
+
+import modal
+import os
+
+from commit0.harness.docker_utils import HEREDOC_DELIMITER
+
+
+def create_sandbox(image: modal.Image, nfs: model.NetworkFileSystem) -> modal.Sandbox:
+    return modal.Sandbox.create(
+        "sleep",
+        "infinity",
+        image=image,
+        network_file_systems={
+            "/vol": nfs,
+        },
+    )
+
+
+def execute_command(sandbox: modal.Sandbox, command: str, timeout=) -> tuple[str,str]:
+    process = sandbox.exec("bash", "-c", command)
+    stdout = []
+    for line in process.stdout:
+        stdout.append(line)
+    stderr = []
+    for line in process.stderr:
+        stderr.append(line)
+    return "\n".join(stdout), "\n".join(stderr)
+
+
+def copy_file_to_sandbox(sandbox: modal.Sandbox, nfs: modal.NetworkFileSystem, src: Path, dst: Path) -> None:
+    with src.open("rb") as f:
+        nfs.write_file(str(src), f)
+    sandbox.exec("bash", "-c", f"cp /vol/{str(src)} {str(dst)}")
+
+
+def copy_from_sandbox(sandbox: modal.Sandbox, src: Path, dst: Path) -> None:
+    pass
+
+
+def delete_file_from_sandbox(sandbox: modal.Sandbox, file_path: str) -> None:
+    pass
+
+
+def copy_ssh_pubkey_from_sandbox(sandbox: modal.Sandbox) -> None:
+    process = sandbox.exec("bash", "-c", "cat /root/.ssh/id_rsa.pub")
+    public_key = "".join([line for line in process.stdout]).strip()
+
+    # add to authorized keys locally. copy-pasted from utils
+    local_authorized_keys_path = os.path.expanduser("~/.ssh/authorized_keys")
+    os.makedirs(os.path.dirname(local_authorized_keys_path), exist_ok=True)
+    if not os.path.exists(local_authorized_keys_path):
+        # Since the file does not exist, create it
+        open(local_authorized_keys_path, "a").close()
+        write = True
+    else:
+        with open(local_authorized_keys_path, "r") as authorized_keys_file:
+            content = authorized_keys_file.read()
+            if public_key not in content:
+                write = True
+            else:
+                write = False
+    if write:
+        with open(local_authorized_keys_path, "a") as authorized_keys_file:
+            authorized_keys_file.write(public_key + "\n")
+
+
+def write_to_sandbox(sandbox: modal.Sandbox, data: str, dst: Path) -> None:
+    pass

commit0/harness/run_pytest_ids.py

@@ -40,6 +40,14 @@ class ExecutionBackend(StrEnum):
 def run_docker(
     spec: Spec, logger: logging.Logger, eval_file: Path, timeout: int, log_dir: Path
 ) -> None:
+    """ Runs the tests in a local docker container.
+
+    1. Creates docker container.
+    2. Copies ssh public key from container to local machine.
+    3. Copies eval.sh from local to container.
+    4. Runs evaluation and saves to {log_dir}/test_output.txt.
+    5. Copies over report.json if it exists.
+    """
     client = docker.from_env()
     container = None
     try:
@@ -107,78 +115,48 @@ def run_docker(
 def run_modal(
     spec: Spec, logger: logging.Logger, eval_file: Path, timeout: int, log_dir: Path
 ) -> None:
-    # get image name to pull from dockerhub
-    # spec.repo_image_key
+    """ Runs the tests in a remote Modal container.
+
+    1. Creates modal container.
+    2. Copies ssh public key from container to local machine.
+    3. Copies eval.sh from local to container.
+    4. Runs evaluation and saves to {log_dir}/test_output.txt.
+    5. Copies over report.json if it exists.
+    """
     import modal
+    from commit0.harness.modal_utils import (
+        create_sandbox,
+        copy_to_sandbox,
+        copy_from_sandbox,
+        delete_file_from_sandbox,
+        copy_ssh_pubkey_from_sandbox,
+        write_to_sandbox,
+    )
 
+    # the image must exist on dockerhub
     reponame = spec.repo.split("/")[-1]
     image_name = f"wentingzhao/{reponame}"
     image = modal.Image.from_registry(image_name)
 
     with modal.NetworkFileSystem.ephemeral() as nfs:
-        # create sleepy sandbox
-        sandbox = modal.Sandbox.create(
-            "sleep",
-            "infinity",
-            image=image,
-            network_file_systems={
-                "/vol": nfs,
-            },
-        )
+        sandbox = create_sandbox(image, nfs)
 
-        # get ssh pubkey
-        process = sandbox.exec("bash", "-c", "cat /root/.ssh/id_rsa.pub")
-        public_key = "".join([line for line in process.stdout]).strip()
-
-        # add to authorized keys locally. copy-pasted from utils
-        local_authorized_keys_path = os.path.expanduser("~/.ssh/authorized_keys")
-        os.makedirs(os.path.dirname(local_authorized_keys_path), exist_ok=True)
-        if not os.path.exists(local_authorized_keys_path):
-            # Since the file does not exist, create it
-            open(local_authorized_keys_path, "a").close()
-            write = True
-        else:
-            with open(local_authorized_keys_path, "r") as authorized_keys_file:
-                content = authorized_keys_file.read()
-                if public_key not in content:
-                    write = True
-                else:
-                    write = False
-        if write:
-            with open(local_authorized_keys_path, "a") as authorized_keys_file:
-                authorized_keys_file.write(public_key + "\n")
+        copy_ssh_pubkey_from_sandbox(sandbox)
 
         # copy eval file
-        with open(eval_file, "rb") as f:
-            nfs.write_file("eval.sh", f)
-        sandbox.exec("bash", "-c", "cp /vol/eval.sh /eval.sh")
+        copy_file_to_sandbox(sandbox, nfs, eval_file, Path("/eval.sh"))
 
         # DBG: check if eval file properly copied
-        process = sandbox.exec("bash", "-c", "ls /")
-        for line in process.stdout:
-            print(line)
-        # /DBG
+        print(execute_command("ls /")[0])
 
         # execute tests
-        process = sandbox.exec("bash", "-c", "/bin/bash /eval.sh")
-        output = []
-        line = ""
-        for line in process.stdout:
-            output.append(line)
-        output = "".join(line)
-        logger.info(output)
+        output, error = execute_command("/bin/bash /eval.sh")
         print(output)
-
-        output = []
-        for line in process.stderr:
-            output.append(line)
-        output_s = "".join(line)
-        logger.info(output_s)
-        print(output_s)
+        print(error)
 
         timed_out = False
         test_output = extract_test_output(
-            output_s, "--json-report --json-report-file=report.json"
+            output, "--json-report --json-report-file=report.json"
         )
 
         # stdout might be more straightforward
@@ -205,6 +183,11 @@ def main(
     backend: str,
     timeout: int,
 ) -> None:
+    """ Runs the pytests for repos in a dataset.
+
+    Tests are run either locally through docker
+    or remotely through Modal.
+    """
     dataset: Iterator[RepoInstance] = load_dataset(dataset_name, split=dataset_split)  # type: ignore
     spec = None
     example = None