SPA metadata source should include Images-Disc (#736)

Merging into main, needs running in test setup before live deployment

The Images-Disc part of the source needs to be included as part of the tag, but this meant that the EpuSession.dm was being searched for in the wrong location.

This PR makes sure the Images-Disc is always sent for ensure_dcg_exists in SPA, and then that searches for the dm file in the parent folder. Also ensures dcg exists for all images folders when metadata files are seen.

Author: stephen-riggs

src/murfey/client/context.py

@@ -45,9 +45,12 @@ def ensure_dcg_exists(
     if collection_type == "tomo":
         experiment_type_id = 36
         session_file = metadata_source / "Session.dm"
+        source_visit_dir = metadata_source.parent
     elif collection_type == "spa":
         experiment_type_id = 37
-        session_file = metadata_source / "EpuSession.dm"
+        # For SPA the metadata source sent should include the Images-Disc
+        session_file = metadata_source.parent / "EpuSession.dm"
+        source_visit_dir = metadata_source.parent.parent
         for h in entry_points(group="murfey.hooks"):
             try:
                 if h.name == "get_epu_session_metadata":
@@ -90,7 +93,6 @@ def ensure_dcg_exists(
         partial_path = "/".join(windows_path.split("\\")[visit_index + 1 :])
         logger.info("Partial Linux path successfully constructed from Windows path")
 
-        source_visit_dir = metadata_source.parent
         logger.info(
             f"Looking for atlas XML file in metadata directory {str((source_visit_dir / partial_path).parent)}"
         )
@@ -120,28 +122,14 @@ def ensure_dcg_exists(
             atlas=Path(partial_path), sample=sample
         )
 
-        dcg_search_dir = (
+        dcg_tag = (
             str(metadata_source).replace(f"/{environment.visit}", "").replace("//", "/")
         )
-        if collection_type == "tomo":
-            dcg_tag = dcg_search_dir
-        else:
-            dcg_images_dirs = sorted(
-                Path(dcg_search_dir).glob("Images-Disc*"),
-                key=lambda x: x.stat().st_ctime,
-            )
-            if not dcg_images_dirs:
-                logger.warning(
-                    f"Cannot find Images-Disc* in {dcg_search_dir}, falling back to Images-Disc1"
-                )
-                dcg_images_dirs = [Path(dcg_search_dir) / "Images-Disc1"]
-            dcg_tag = str(dcg_images_dirs[-1])
-
         dcg_data = {
             "experiment_type_id": experiment_type_id,
             "tag": dcg_tag,
             "atlas": str(
-                _atlas_destination(environment, metadata_source, token)
+                _atlas_destination(environment, session_file.parent, token)
                 / environment.samples[metadata_source].atlas.parent
                 / atlas_xml_path.with_suffix(".jpg").name
             ).replace("//", "/"),

src/murfey/client/contexts/spa_metadata.py

@@ -106,35 +106,39 @@ def post_transfer(
                 return
 
             if source:
-                dcg_tag = ensure_dcg_exists(
-                    collection_type="spa",
-                    metadata_source=source,
-                    environment=environment,
-                    token=self._token,
-                )
                 gs_pix_positions = get_grid_square_atlas_positions(
                     source.parent / partial_path
                 )
-                for gs, pos_data in gs_pix_positions.items():
-                    if pos_data:
-                        capture_post(
-                            base_url=str(environment.url.geturl()),
-                            router_name="session_control.spa_router",
-                            function_name="register_grid_square",
-                            token=self._token,
-                            session_id=environment.murfey_session,
-                            gsid=int(gs),
-                            data={
-                                "tag": dcg_tag,
-                                "x_location": pos_data[0],
-                                "y_location": pos_data[1],
-                                "x_stage_position": pos_data[2],
-                                "y_stage_position": pos_data[3],
-                                "width": pos_data[4],
-                                "height": pos_data[5],
-                                "angle": pos_data[6],
-                            },
-                        )
+                for images_disc in list(source.glob("Images-Disc*")) or [
+                    source / "Images-Disc1"
+                ]:
+                    # Do the dcg registration for every Images-Disc with this session file
+                    dcg_tag = ensure_dcg_exists(
+                        collection_type="spa",
+                        metadata_source=images_disc,
+                        environment=environment,
+                        token=self._token,
+                    )
+                    for gs, pos_data in gs_pix_positions.items():
+                        if pos_data:
+                            capture_post(
+                                base_url=str(environment.url.geturl()),
+                                router_name="session_control.spa_router",
+                                function_name="register_grid_square",
+                                token=self._token,
+                                session_id=environment.murfey_session,
+                                gsid=int(gs),
+                                data={
+                                    "tag": dcg_tag,
+                                    "x_location": pos_data[0],
+                                    "y_location": pos_data[1],
+                                    "x_stage_position": pos_data[2],
+                                    "y_stage_position": pos_data[3],
+                                    "width": pos_data[4],
+                                    "height": pos_data[5],
+                                    "angle": pos_data[6],
+                                },
+                            )
 
         elif (
             transferred_file.suffix == ".dm"
@@ -145,12 +149,15 @@ def post_transfer(
             source = _get_source(transferred_file, environment=environment)
             if source is None:
                 return None
-            ensure_dcg_exists(
-                collection_type="spa",
-                metadata_source=source,
-                environment=environment,
-                token=self._token,
-            )
+            for images_disc in list(source.glob("Images-Disc*")) or [
+                source / "Images-Disc1"
+            ]:
+                ensure_dcg_exists(
+                    collection_type="spa",
+                    metadata_source=images_disc,
+                    environment=environment,
+                    token=self._token,
+                )
 
             gs_name = int(transferred_file.stem.split("_")[1])
             logger.info(
@@ -168,7 +175,9 @@ def post_transfer(
                 logger.warning(
                     f"Cannot find Images-Disc* in {visitless_source_search_dir}"
                 )
-                return
+                visitless_source_images_dirs = [
+                    Path(visitless_source_search_dir) / "Images-Disc1"
+                ]
             visitless_source = str(visitless_source_images_dirs[-1])
 
             if fh_positions:

tests/client/test_context.py

@@ -99,7 +99,7 @@ def test_ensure_dcg_exists_spa(mock_capture_post, tmp_path):
 
     ensure_dcg_exists(
         collection_type="spa",
-        metadata_source=metadata_source,
+        metadata_source=metadata_source / "Images-Disc1",
         environment=env,
         token="token",
     )