Merge remote-tracking branch 'origin/main' into package-without-editable-flag

Author: VukW

cli/medperf/_version.py

@@ -1 +1 @@
-__version__ = "0.1.0"
+__version__ = "0.1.2"

docs/mlcubes/mlcube_models.md

@@ -55,7 +55,7 @@ docker_image_name [docker/image:latest]: # (6)!
 2. Determines how the MLCube root folder will be named.
 3. Gives a Human-readable description to the MLCube Project.
 4. Documents the MLCube implementation by specifying the author.
-5. Indicates how many GPUs should be visible by the MLCube.
+5. Set it to 0. This is now ignored and will be removed in the next release. Please check the last section to learn how to use MLCube with GPUs.
 6. MLCubes use Docker containers under the hood. Here, you can provide an image tag to the image that will be created by this MLCube. **You should use a valid name that allows you to upload it to a Docker registry.**
 
 After filling the configuration options, the following directory structure will be generated:
@@ -232,9 +232,6 @@ accelerator_count [0]: 0
 docker_image_name [docker/image:latest]: repository/model-tutorial:0.0.0
 ```
 
-!!! note
-    This example is built to be used with a CPU. See the [last section](#using-the-example-with-gpus) to know how to configure this example with a GPU.
-
 Note that `docker_image_name` is arbitrarily chosen. Use a valid docker image.
 
 ### Move your Codebase
@@ -355,6 +352,11 @@ The provided example codebase runs only on CPU. You can modify it to have `pytor
 
 The general instructions for building an MLCube to work with a GPU are the same as the provided instructions, but with the following slight modifications:
 
-- Use a number different than `0` for the `accelerator_count` that you will be prompted with when creating the MLCube template.
-- Inside the `docker` section of the `mlcube.yaml`, add a key value pair: `gpu_args: --gpus=all`. These `gpu_args` will be passed to `docker run` under the hood by MLCube. You may add more than just `--gpus=all`.
 - Make sure you install the required GPU dependencies in the docker image. For instance, this may be done by simply modifying the `pip` dependencies in the `requirements.txt` file to download `pytorch` with cuda, or by changing the base image of the dockerfile.
+
+For testing your MLCube with GPUs using the MLCube tool as in the previous section, make sure you run the `mlcube run` command with a `--gpus` argument. Example: `mlcube run --gpus=all ...`
+
+For testing your MLCube with GPUs using MedPerf, make sure you pass as well the `--gpus` argument to the MedPerf command. Example: `medperf --gpus=all <subcommand> ...`.
+
+!!!tip
+    Run `medperf --help` to see the possible options you can use for the `--gpus` argument.

scripts/get_dataset_hashes.py

@@ -0,0 +1,50 @@
+import hashlib
+import os
+import yaml
+
+from medperf import config
+from medperf.init import initialize
+from typer import Option
+
+
+def sha256sum(filename):
+    h = hashlib.sha256()
+    b = bytearray(128 * 1024)
+    mv = memoryview(b)
+    with open(filename, "rb", buffering=0) as f:
+        while n := f.readinto(mv):
+            h.update(mv[:n])
+    return h.hexdigest()
+
+
+def generate_hash_dict(path):
+    hash_dict = {}
+    contents = os.listdir(path)
+
+    for item in contents:
+        item_path = os.path.join(path, item)
+        if os.path.isdir(item_path):
+            hash_dict[item] = generate_hash_dict(item_path)
+        else:
+            hash_dict[item] = sha256sum(item_path)
+
+    return hash_dict
+
+
+def main(
+    dataset_uid: str = Option(None, "-d", "--dataset"),
+    output_file: str = Option("dataset_hashes.yaml", "-f", "--file"),
+):
+    initialize()
+    dset_path = os.path.join(config.datasets_folder, dataset_uid)
+
+    # Get hashes of tree
+    hash_dict = generate_hash_dict(dset_path)
+
+    # Write results to a file
+    with open(output_file, "w") as f:
+        yaml.dump(hash_dict, f)
+
+
+if __name__ == "__main__":
+    run(main)

scripts/get_reviewed_cases_hashes.py

@@ -0,0 +1,56 @@
+import tarfile
+import hashlib
+import shutil
+import os
+import yaml
+
+
+def sha256sum(filename):
+    h = hashlib.sha256()
+    b = bytearray(128 * 1024)
+    mv = memoryview(b)
+    with open(filename, "rb", buffering=0) as f:
+        while n := f.readinto(mv):
+            h.update(mv[:n])
+    return h.hexdigest()
+
+
+def generate_hash_dict(path):
+    hash_dict = {}
+    contents = os.listdir(path)
+
+    for item in contents:
+        item_path = os.path.join(path, item)
+        if os.path.isdir(item_path):
+            hash_dict[item] = generate_hash_dict(item_path)
+        else:
+            hash_dict[item] = sha256sum(item_path)
+
+    return hash_dict
+
+
+def main():
+    dst = ".reviewed_cases_contents"
+    hashes_file = "reviewed_cases_hashes.yaml"
+
+    # Create destination folder
+    shutil.rmtree(dst, ignore_errors=True)
+    os.makedirs(dst, exist_ok=True)
+
+    # Extract contents
+    with tarfile.open("reviewed_cases.tar.gz") as file:
+        file.extractall(dst)
+
+    # Get hashes of tree
+    hash_dict = generate_hash_dict(dst)
+
+    # Write results to a file
+    with open(hashes_file, "w") as f:
+        yaml.dump(hash_dict, f)
+
+    # Delete generated files and folders
+    shutil.rmtree(dst, ignore_errors=True)
+
+
+if __name__ == "__main__":
+    main()

scripts/monitor/rano_monitor/__main__.py

@@ -7,13 +7,12 @@
     DEFAULT_STAGES_PATH,
     STAGES_HELP,
     DSET_LOC_HELP,
-    OUT_HELP
+    OUT_HELP,
 )
 from rano_monitor.dataset_browser import DatasetBrowser
 from rano_monitor.handlers import InvalidHandler
 from rano_monitor.handlers import PromptHandler
 from rano_monitor.handlers import ReportHandler, ReportState
-from rano_monitor.handlers import ReviewedHandler
 from rano_monitor.handlers import TarballReviewedHandler
 from rano_monitor.tarball_browser import TarballBrowser
 from typer import Option
@@ -40,13 +39,11 @@ def run_dset_app(dset_path, stages_path, output_path):
     report_state = ReportState(report_path, t_app)
     report_watchdog = ReportHandler(report_state)
     prompt_watchdog = PromptHandler(dset_data_path, t_app)
-    reviewed_watchdog = ReviewedHandler(dset_data_path, t_app)
     invalid_watchdog = InvalidHandler(invalid_path, t_app)
 
     t_app.set_vars(
         dset_data_path,
         stages_path,
-        reviewed_watchdog,
         output_path,
         invalid_path,
         invalid_watchdog,
@@ -56,7 +53,6 @@ def run_dset_app(dset_path, stages_path, output_path):
     observer = Observer()
     observer.schedule(report_watchdog, dset_path)
     observer.schedule(prompt_watchdog, os.path.join(dset_path, "data"))
-    observer.schedule(reviewed_watchdog, ".")
     observer.schedule(invalid_watchdog, os.path.dirname(invalid_path))
     observer.start()
     t_app.run()
@@ -89,13 +85,8 @@ def run_tarball_app(tarball_path):
 
 @app.command()
 def main(
-    dataset_uid: str = Option(None, "-d", "--dataset", help=DSET_HELP),
-    stages_path: str = Option(
-        DEFAULT_STAGES_PATH,
-        "-s",
-        "--stages",
-        help=STAGES_HELP
-    ),
+    dataset_uid: str = Option(..., "-d", "--dataset", help=DSET_HELP),
+    stages_path: str = Option(DEFAULT_STAGES_PATH, "-s", "--stages", help=STAGES_HELP),
     dset_path: str = Option(
         None,
         "-p",

scripts/monitor/rano_monitor/assets/subject-browser.tcss

@@ -71,11 +71,6 @@ SubjectDetails Button {
     column-span: 2;
 }
 
-SubjectDetails {
-    overflow-y: scroll;
-    height: 100%;
-}
-
 SubjectDetails CopyableItem {
     layout: grid;
     grid-size: 12 1;
@@ -121,7 +116,6 @@ Summary {
     padding: 3;
     align: center middle;
     content-align: center middle;
-    overflow-y: scroll;
 }
 
 Summary Static {
@@ -148,4 +142,9 @@ ListItem .subtitle {
 MarkdownViewer {
     height: auto;
     min-height: 2;
+}
+
+.review-btn {
+    width: 100%;
+    margin: 1;
 }

scripts/monitor/rano_monitor/constants.py

@@ -29,4 +29,5 @@
 LISTITEM_MAX_LEN = 30
 
 REVIEWED_PATTERN = r".*\/(.*)\/(.*)\/finalized\/(.*\.nii\.gz)"
+UNDER_REVIEW_PATTERN = r".*\/(.*)\/(.*)\/under_review\/(.*\.nii\.gz)"
 BRAINMASK_PATTERN = r".*\/(.*)\/(.*)\/brainMask_fused.nii.gz"

scripts/monitor/rano_monitor/dataset_browser.py

@@ -11,7 +11,7 @@
 from rano_monitor.widgets.summary import Summary
 from textual.app import App, ComposeResult
 from textual.binding import Binding
-from textual.containers import Container, Horizontal
+from textual.containers import Container, Horizontal, VerticalScroll
 from textual.reactive import reactive, var
 from textual.widgets import (
     Button,
@@ -42,15 +42,13 @@ def set_vars(
         self,
         dset_data_path,
         stages_path,
-        reviewed_watchdog,
         output_path,
         invalid_path,
         invalid_watchdog,
         prompt_watchdog,
     ):
         self.dset_data_path = dset_data_path
         self.stages_path = stages_path
-        self.reviewed_watchdog = reviewed_watchdog
         self.output_path = output_path
         self.invalid_path = invalid_path
         self.invalid_watchdog = invalid_watchdog
@@ -65,8 +63,9 @@ def compose(self) -> ComposeResult:
         with Container():
             with Container(id="list-container"):
                 yield SubjectListView(id="subjects-list")
-            yield Summary(id="summary")
-            yield SubjectDetails(id="details")
+            with VerticalScroll():
+                yield Summary(id="summary")
+                yield SubjectDetails(id="details")
         with Container(id="confirm-prompt"):
             yield Static(self.prompt, id="confirm-details")
             yield Horizontal(

scripts/monitor/rano_monitor/handlers/__init__.py

@@ -1,14 +1,12 @@
 from .invalid_handler import InvalidHandler
 from .prompt_handler import PromptHandler
 from .report_handler import ReportHandler, ReportState
-from .reviewed_handler import ReviewedHandler
 from .tarball_reviewed_watchdog import TarballReviewedHandler
 
 __all__ = [
     InvalidHandler,
     PromptHandler,
     ReportHandler,
     ReportState,
-    ReviewedHandler,
     TarballReviewedHandler,
 ]