Web UI

.gitignore

@@ -60,7 +60,7 @@ cover/
 local_settings.py
 db.sqlite3
 db.sqlite3-journal
-static/
+server/static/
 *.crt
 *.key
 *.pem
@@ -147,6 +147,9 @@ cython_debug/
 # Dev Environment Specific
 .vscode
 .venv
+
+# Medperf tutorial files
+medperf_tutorial
 server/keys
 
 # exclude fl example

cli/medperf/cli.py

@@ -21,7 +21,7 @@
 import medperf.commands.ca.ca as ca
 import medperf.commands.certificate.certificate as certificate
 import medperf.commands.storage as storage
-
+import medperf.web_ui.app as web_ui
 from medperf.utils import check_for_updates
 from medperf.logging.utils import log_machine_details
 
@@ -40,6 +40,7 @@
 app.add_typer(aggregator.app, name="aggregator", help="Manage aggregators")
 app.add_typer(ca.app, name="ca", help="Manage CAs")
 app.add_typer(certificate.app, name="certificate", help="Manage certificates")
+app.add_typer(web_ui.app, name="web-ui", help="local web UI to manage medperf entities")
 
 
 @app.command("run")

cli/medperf/commands/benchmark/submit.py

@@ -41,6 +41,8 @@ def run(
         ui.print("Uploaded")
         submission.to_permanent_path(updated_benchmark_body)
         submission.write(updated_benchmark_body)
+        print(submission.bmk.id)
+        return submission.bmk.id
 
     def __init__(
         self,
@@ -103,5 +105,5 @@ def to_permanent_path(self, bmk_dict: dict):
         os.rename(old_bmk_loc, new_bmk_loc)
 
     def write(self, updated_body):
-        bmk = Benchmark(**updated_body)
-        bmk.write()
+        self.bmk = Benchmark(**updated_body)
+        self.bmk.write()

cli/medperf/commands/compatibility_test/run.py

@@ -7,6 +7,7 @@
 from medperf.exceptions import InvalidArgumentError
 from .validate_params import CompatibilityTestParamsValidator
 from .utils import download_demo_data, prepare_cube, get_cube, create_test_dataset
+import medperf.config as config
 
 
 class CompatibilityTestExecution:
@@ -87,8 +88,9 @@ def run(
         test_exec.validate()
         test_exec.set_data_source()
         test_exec.process_benchmark()
-        test_exec.prepare_cubes()
-        test_exec.prepare_dataset()
+        with config.ui.interactive():
+            test_exec.prepare_cubes()
+            test_exec.prepare_dataset()
         test_exec.initialize_report()
         results = test_exec.cached_results()
         if results is None:

cli/medperf/commands/dataset/dataset.py

@@ -238,7 +238,7 @@ def import_dataset(
     raw_path: str = typer.Option(
         None,
         "--raw_dataset_path",
-        help="New path of the DEVELOPMENT dataset raw data to be saved.",
+        help="New path of the DEVELOPMENT dataset raw data to be saved. Directory should be empty or doesn't exist.",
     ),
 ):
     """Imports dataset files from specified tar.gz file."""

cli/medperf/commands/dataset/import_dataset.py

@@ -24,17 +24,19 @@
 
     def validate_input(self):
         # The input archive file should exist and be a file
         if not os.path.exists(self.input_path):
             raise InvalidArgumentError(f"File {self.input_path} doesn't exist.")
         if not os.path.isfile(self.input_path):
             raise InvalidArgumentError(f"{self.input_path} is not a file.")
 
         # raw_data_path should be provided if the imported dataset is in dev
         if self.dataset.state == "DEVELOPMENT" and (
-            self.raw_data_path is None or os.path.exists(self.raw_data_path)
+            self.raw_data_path is None
+            or os.path.isfile(self.raw_data_path)
+            or (os.path.exists(self.raw_data_path) and os.listdir(self.raw_data_path))
         ):
             raise InvalidArgumentError(
-                "Output raw data path must be specified and shouldn't exist."
+                "Output raw data path must be specified and, the directory should be empty or does not exist."
             )
 
     def untar_files(self):
@@ -83,11 +85,11 @@
         archive_config = os.path.join(
             root_archive_folder, config.archive_config_filename
         )
         if not os.path.exists(archive_config):
             raise ExecutionError(
                 "Dataset archive is invalid, config file doesn't exist"
             )
         with open(archive_config) as f:
             archive_config = yaml.safe_load(f)
 
         # validate config
@@ -97,7 +99,7 @@
         archive_prepared_dataset_path = os.path.join(
             root_archive_folder, str(self.dataset_id)
         )
         if not os.path.exists(archive_prepared_dataset_path):
             raise ExecutionError("No prepared dataset in archive")
 
         if os.path.exists(self.dataset.data_path) or os.path.exists(
@@ -114,8 +116,8 @@
             archive_raw_labels_path = os.path.join(
                 root_archive_folder, archive_config["raw_labels"]
             )
             if not os.path.exists(archive_raw_data_path) or not os.path.exists(
                 archive_raw_labels_path
             ):
                 raise ExecutionError("No raw data in archive")
 
@@ -137,8 +139,8 @@
             return
 
         # For development datasets, move raw data as well
         os.makedirs(self.raw_data_path, exist_ok=True)
         self.raw_data_path = str(Path(self.raw_data_path).resolve())
         new_raw_data_path = os.path.join(
             self.raw_data_path, os.path.basename(self.archive_raw_data_path)
         )
@@ -147,7 +149,7 @@
         )
 
         same_raw_data_and_labels = os.path.samefile(
             self.archive_raw_data_path, self.archive_raw_labels_path
         )
         move_folder(self.archive_raw_data_path, new_raw_data_path)
         if not same_raw_data_and_labels:

cli/medperf/commands/dataset/prepare.py

@@ -89,7 +89,8 @@ def run(cls, dataset_id: int, approve_sending_reports: bool = False):
             preparation.prompt_for_report_sending_approval()
 
         if preparation.should_run_prepare():
-            preparation.run_prepare()
+            with preparation.ui.interactive():
+                preparation.run_prepare()
 
         with preparation.ui.interactive():
             preparation.run_sanity_check()
@@ -277,6 +278,8 @@ def __generate_report_dict(self):
             with open(self.report_path, "r") as f:
                 report_dict = yaml.safe_load(f)
 
+            # TODO: this specific logic with status is very tuned to the RANO. Hope we'd
+            #  make it more general once
             report = pd.DataFrame(report_dict)
             if "status" in report.keys():
                 report_status = report.status.value_counts() / len(report)
@@ -288,6 +291,7 @@ def __generate_report_dict(self):
 
         return report_status_dict
 
+    @staticmethod
     def prompt_for_report_sending_approval(self):
         example = {
             "execution_status": "running",

cli/medperf/commands/dataset/set_operational.py

@@ -50,6 +50,8 @@ def set_operational(self):
         self.dataset.state = "OPERATION"
 
     def update(self):
+        msg = "This is the information that is going to be transmitted to the medperf server"
+        config.ui.print_warning(msg)
         body = self.todict()
         dict_pretty_print(body)
         msg = "Do you approve sending the presented data to MedPerf? [Y/n] "

cli/medperf/commands/dataset/submit.py

@@ -43,11 +43,17 @@
             submit_as_prepared,
             for_test,
         )
-        preparation.validate()
-        preparation.validate_prep_cube()
-        preparation.create_dataset_object()
-        if submit_as_prepared:
-            preparation.make_dataset_prepared()
+        submission_dict = preparation.prepare_dict(submit_as_prepared)
+        dict_pretty_print(submission_dict)
+
+        msg = "Do you approve the registration of the presented data to MedPerf? [Y/n] "
+        warning = (
+            "Upon submission, your email address will be visible to the Data Preparation"
+            + " Owner for traceability and debugging purposes."
+        )
+        config.ui.print_warning(warning)
+        preparation.approved = preparation.approved or approval_prompt(msg)
+
         updated_dataset_dict = preparation.upload()
         preparation.to_permanent_path(updated_dataset_dict)
         preparation.write(updated_dataset_dict)
@@ -69,8 +75,8 @@
         for_test: bool,
     ):
         self.ui = config.ui
         self.data_path = str(Path(data_path).resolve())
         self.labels_path = str(Path(labels_path).resolve())
         self.metadata_path = metadata_path
         self.name = name
         self.description = description
@@ -82,9 +88,9 @@
         self.for_test = for_test
 
     def validate(self):
         if not os.path.exists(self.data_path):
             raise InvalidArgumentError("The provided data path doesn't exist")
         if not os.path.exists(self.labels_path):
             raise InvalidArgumentError("The provided labels path doesn't exist")
 
         if not self.submit_as_prepared and self.metadata_path:
@@ -137,8 +143,8 @@
         self.dataset = dataset
 
     def make_dataset_prepared(self):
         shutil.copytree(self.data_path, self.dataset.data_path)
         shutil.copytree(self.labels_path, self.dataset.labels_path)
         if self.metadata_path:
             shutil.copytree(self.metadata_path, self.dataset.metadata_path)
         else:
@@ -147,17 +153,16 @@
             # have prepared datasets with no the metadata information
             os.makedirs(self.dataset.metadata_path, exist_ok=True)
 
-    def upload(self):
-        submission_dict = self.dataset.todict()
-        dict_pretty_print(submission_dict)
-        msg = "Do you approve the registration of the presented data to MedPerf? [Y/n] "
-        warning = (
-            "Upon submission, your email address will be visible to the Data Preparation"
-            + " Owner for traceability and debugging purposes."
-        )
-        self.ui.print_warning(warning)
-        self.approved = self.approved or approval_prompt(msg)
+    def prepare_dict(self, submit_as_prepared: bool):
+        self.validate()
+        self.validate_prep_cube()
+        self.create_dataset_object()
+        if submit_as_prepared:
+            self.make_dataset_prepared()
 
+        return self.dataset.todict()
+
+    def upload(self):
         if self.approved:
             updated_body = self.dataset.upload()
             return updated_body

cli/medperf/commands/list.py

@@ -19,6 +19,8 @@ def run(
         """Lists all local datasets
 
         Args:
+            entity_class: entity class to instantiate (Dataset, Model, etc.)
+            fields (list[str]): list of fields to display
             unregistered (bool, optional): Display only local unregistered results. Defaults to False.
             mine_only (bool, optional): Display all registered current-user results. Defaults to False.
             kwargs (dict): Additional parameters for filtering entity lists.

cli/medperf/commands/mlcube/associate.py

@@ -1,6 +1,7 @@
 from medperf import config
 from medperf.entities.cube import Cube
 from medperf.entities.benchmark import Benchmark
+from medperf.exceptions import CleanExit
 from medperf.utils import dict_pretty_print, approval_prompt
 from medperf.commands.compatibility_test.run import CompatibilityTestExecution
 
@@ -42,4 +43,4 @@ def run(
             metadata = {"test_result": results}
             comms.associate_benchmark_model(cube_uid, benchmark_uid, metadata)
         else:
-            ui.print("Model association operation cancelled")
+            raise CleanExit("Model association operation cancelled")

cli/medperf/commands/mlcube/submit.py

@@ -23,6 +23,7 @@ def run(cls, submit_info: dict):
             updated_cube_dict = submission.upload()
             submission.to_permanent_path(updated_cube_dict)
             submission.write(updated_cube_dict)
+        return submission.cube.id
 
     def __init__(self, submit_info: dict):
         self.comms = config.comms
@@ -49,5 +50,5 @@ def to_permanent_path(self, cube_dict):
         os.rename(old_cube_loc, new_cube_loc)
 
     def write(self, updated_cube_dict):
-        cube = Cube(**updated_cube_dict)
-        cube.write()
+        self.cube = Cube(**updated_cube_dict)
+        self.cube.write()

cli/medperf/commands/profile.py

@@ -20,7 +20,7 @@ def activate(profile: str):
     config_p = read_config()
 
     if profile not in config_p:
-        raise InvalidArgumentError("The provided profile does not exists")
+        raise InvalidArgumentError("The provided profile does not exist")
 
     config_p.activate(profile)
     write_config(config_p)
@@ -81,6 +81,8 @@ def view(profile: str = typer.Argument(None)):
     config_p = read_config()
     profile_config = config_p.active_profile
     if profile:
+        if profile not in config_p:
+            raise InvalidArgumentError("The provided profile does not exist")
         profile_config = config_p[profile]
 
     profile_config.pop(config.credentials_keyword, None)
@@ -99,7 +101,7 @@ def delete(profile: str):
     """
     config_p = read_config()
     if profile not in config_p.profiles:
-        raise InvalidArgumentError("The provided profile does not exists")
+        raise InvalidArgumentError("The provided profile does not exist")
 
     if profile in [
         config.default_profile_name,

cli/medperf/commands/result/create.py

@@ -29,7 +29,7 @@ def run(
         ignore_failed_experiments=False,
         no_cache=False,
         show_summary=False,
-    ):
+    ) -> list[Result]:
         """Benchmark execution flow.
 
         Args:
@@ -48,7 +48,8 @@ def run(
             ignore_model_errors,
             ignore_failed_experiments,
         )
-        execution.prepare()
+        with execution.ui.interactive():
+            execution.prepare()
         execution.validate()
         execution.prepare_models()
         if not no_cache:
@@ -166,7 +167,7 @@ def __get_cube(self, uid: int, name: str) -> Cube:
         self.ui.print(f"> Container '{name}' download complete")
         return cube
 
-    def run_experiments(self):
+    def run_experiments(self) -> list[Result]:
         for model_uid in self.models_uids:
             if model_uid in self.cached_results:
                 self.experiments.append(

cli/medperf/comms/auth/auth0.py

@@ -39,11 +39,14 @@ def login(self, email):
         interval = device_code_response["interval"]
 
         config.ui.print(
-            "\nPlease go to the following link to complete your login request:\n"
-            f"\t{verification_uri_complete}\n\n"
-            "Make sure that you will be presented with the following code:\n"
-            f"\t{user_code}\n\n"
+            "\nPlease go to the following link to complete your login request:\n\t"
         )
+        config.ui.print_url(verification_uri_complete)
+        config.ui.print(
+            "\n\nMake sure that you will be presented with the following code:\n\t"
+        )
+        config.ui.print_code(user_code)
+        config.ui.print("\n\n")
         config.ui.print_warning(
             "Keep this terminal open until you complete your login request. "
             "The command will exit on its own once you complete the request. "

cli/medperf/comms/factory.py

@@ -6,9 +6,11 @@
 class CommsFactory:
     @staticmethod
     def create_comms(name: str, host: str) -> Comms:
-        name = name.lower()
-        if name == "rest":
-            return REST(host)
-        else:
-            msg = "the indicated communication interface doesn't exist"
-            raise InvalidArgumentError(msg)
+        if isinstance(name, str):
+            name = name.lower()
+            if name == "rest":
+                return REST(host)
+            else:
+                msg = "the indicated communication interface doesn't exist"
+                raise InvalidArgumentError(msg)
+        return REST(host)

cli/medperf/config.py

@@ -220,6 +220,7 @@
 logs_backup_count = 100
 cleanup = True
 ui = "CLI"
+webui = "WEBUI"
 
 default_profile_name = "default"
 testauth_profile_name = "testauth"

cli/medperf/entities/association.py

@@ -0,0 +1,16 @@
+from datetime import datetime
+from typing import Optional
+
+from medperf.entities.schemas import ApprovableSchema, MedperfSchema
+
+
+class Association(MedperfSchema, ApprovableSchema):
+    id: int
+    metadata: dict
+    dataset: Optional[int]
+    model_mlcube: Optional[int]
+    benchmark: int
+    initiated_by: int
+    created_at: Optional[datetime]
+    modified_at: Optional[datetime]
+    name: str = "Association"  # The server data doesn't have name, while MedperfSchema requires it