Initial implementation of brain-atlas endpoints

See #168 for the specification.
Briefly; a `BrainAtlas` is an named entity that has an asset `annotation.nrrd` attached to it.
It is associated with a particular `BrainRegionHierarchy`
In addition, `BrainAtlasRegion` gives metadata for all the regions within a `BrainAtlas`;
* their volume in the `annotation.nrrd` if they are a leaf, -1 otherwise.
* an attached asset with the .obj mesh

Author: mgeplf

Makefile

@@ -53,7 +53,7 @@ import:  ## Run the import on a database, assumes mba_hierarchy.json and out are
 	@$(call load_env,run-local)
 	@test -n "$(PROJECT_ID_IMPORT)" || (echo "Please set the variable PROJECT_ID_IMPORT"; exit 1)
 	@test -n "$(VIRTUAL_LAB_ID_IMPORT)" || (echo "Please set the variable VIRTUAL_LAB_ID_IMPORT"; exit 1)
-	docker compose up --wait db
+	#docker compose up --wait db
 	uv run -m alembic upgrade head
 	uv run -m app.cli.import_data --seed 0 hierarchy $(HIERARCHY_NAME) mba_hierarchy.json
 	uv run -m app.cli.import_data --seed 1 run ./out --virtual-lab-id $(VIRTUAL_LAB_ID_IMPORT) --project-id $(PROJECT_ID_IMPORT) --hierarchy-name $(HIERARCHY_NAME)

app/cli/brain_region_data.py

@@ -1,5 +1,6 @@
 import datetime
 import glob
+import hashlib
 import json
 import os
 import random
@@ -17,6 +18,7 @@
 from tqdm import tqdm
 
 from app.cli import curate, utils
+from app.cli.brain_region_data import BRAIN_ATLAS_REGION_VOLUMES
 from app.cli.curation import electrical_cell_recording
 from app.cli.types import ContentType
 from app.cli.utils import (
@@ -31,6 +33,7 @@
     Annotation,
     Asset,
     BrainAtlas,
+    BrainAtlasRegion,
     BrainRegion,
     BrainRegionHierarchy,
     CellComposition,
@@ -47,14 +50,14 @@
     Measurement,
     MeasurementAnnotation,
     MEModel,
-    Mesh,
     METypeDensity,
     MTypeClass,
     MTypeClassification,
     Organization,
     Person,
     ReconstructionMorphology,
     SingleNeuronSimulation,
+    Species,
 )
 from app.db.session import configure_database_session_manager
 from app.db.types import (
@@ -71,6 +74,9 @@
 from app.cli.curation import electrical_cell_recording, cell_composition
 from app.cli.types import ContentType
 
+
+BRAIN_ATLAS_NAME = "BlueBrain Atlas"
+
 REQUIRED_PATH = click.Path(exists=True, readable=True, dir_okay=False, resolve_path=True)
 REQUIRED_PATH_DIR = click.Path(
     exists=True, readable=True, file_okay=False, dir_okay=True, resolve_path=True
@@ -555,47 +561,97 @@ def ingest(
         db.commit()
 
 
-class ImportBrainRegionMeshes(Import):
-    name = "BrainRegionMeshes"
+class ImportBrainAtlas(Import):
+    name = "BrainAtlas"
 
     @staticmethod
     def is_correct_type(data):
-        types = ensurelist(data["@type"])
-        return "BrainParcellationMesh" in types
+        # for reasons unknown, the annotations are tagged as v1.2.0
+        # the contents of the annotations is the same as in
+        # s3://openbluebrain/Model_Data/Brain_atlas/Mouse/resolution_25_um/version_1.1.0/Annotation_volume/annotation_ccfv3_l23split_barrelsplit_validated.nrrd
+        # but their sha256 hashes differ
+        return (
+            utils.is_type(data, "BrainParcellationMesh")
+            and "atlasRelease" in data
+            and data["atlasRelease"].get("tag") == "v1.1.0"
+        ) or (
+            utils.is_type(data, "BrainParcellationDataLayer")
+            and "atlasRelease" in data
+            and data["atlasRelease"].get("tag") == "v1.2.0"
+        )
 
     @staticmethod
     def ingest(db, project_context, data_list, all_data_by_id, hierarchy_name: str):
-        for data in tqdm(data_list):
-            if "atlasRelease" not in data or data["atlasRelease"].get("tag", "") != "v1.1.0":
-                continue
+        meshes, annotations = [], []
+        for d in data_list:
+            if utils.is_type(d, "BrainParcellationMesh"):
+                meshes.append(d)
+            else:
+                annotations.append(d)
+
+        assert len(annotations) == 1
+        brain_atlas = db.execute(
+            sa.select(BrainAtlas).filter(BrainAtlas.name == BRAIN_ATLAS_NAME)
+        ).scalar_one_or_none()
+        if brain_atlas is None:
+            hierarchy = db.execute(
+                sa.select(BrainRegionHierarchy).filter(BrainRegionHierarchy.name == hierarchy_name)
+            ).scalar_one()
+
+            mouse = db.execute(
+                sa.select(Species).filter(Species.name == "Mus musculus")
+            ).scalar_one()
+
+            brain_atlas = BrainAtlas(
+                name=BRAIN_ATLAS_NAME,
+                description="version v1.1.0 from NEXUS",
+                species_id=mouse.id,
+                hierarchy_id=hierarchy.id,
+                authorized_project_id=project_context.project_id,
+                authorized_public=True,
+            )
+            db.add(brain_atlas)
+            db.commit()
 
-            legacy_id = data["@id"]
-            legacy_self = data["_self"]
-            rm = utils._find_by_legacy_id(legacy_id, Mesh, db)
-            if rm:
-                continue
+            utils.import_distribution(
+                annotations[0], brain_atlas.id, EntityType.brain_atlas, db, project_context
+            )
 
-            try:
-                brain_region_id = utils.get_brain_region(data, hierarchy_name, db)
-            except RuntimeError:
-                L.exception("Cannot import mesh")
-                continue
+        for mesh in tqdm(meshes):
+            brain_region_data = mesh["brainLocation"]["brainRegion"]
+            brain_region_id = utils.get_brain_region_by_hier_id(
+                brain_region_data, hierarchy_name, db
+            )
 
-            createdAt, updatedAt = utils.get_created_and_updated(data)
+            atlas_region = db.execute(
+                sa.select(BrainAtlasRegion).filter(
+                    BrainAtlasRegion.brain_region_id == str(brain_region_id)
+                )
+            ).scalar_one_or_none()
+
+            if atlas_region is None:
+                annotation_id = curate.curate_brain_region(brain_region_data)["@id"]
+                volume = -1
+
+                leaf_region = False
+                if annotation_id in BRAIN_ATLAS_REGION_VOLUMES:
+                    volume = BRAIN_ATLAS_REGION_VOLUMES[annotation_id]
+                    leaf_region = True
+
+                atlas_region = BrainAtlasRegion(
+                    brain_atlas_id=brain_atlas.id,
+                    brain_region_id=brain_region_id,
+                    volume=volume,
+                    leaf_region=leaf_region,
+                    authorized_project_id=project_context.project_id,
+                    authorized_public=True,
+                )
+                db.add(atlas_region)
+                db.commit()
 
-            db_item = Mesh(
-                name=data["name"],
-                description=data["description"],
-                legacy_id=[legacy_id],
-                legacy_self=[legacy_self],
-                brain_region_id=brain_region_id,
-                creation_date=createdAt,
-                update_date=updatedAt,
-                authorized_project_id=project_context.project_id,
-                authorized_public=AUTHORIZED_PUBLIC,
+            utils.import_distribution(
+                mesh, atlas_region.id, EntityType.brain_atlas, db, project_context
             )
-            db.add(db_item)
-        db.commit()
 
 
 class ImportMorphologies(Import):
@@ -743,8 +799,6 @@ def ingest(db, project_context, data_list, all_data_by_id, hierarchy_name):
             brain_region_id = utils.get_brain_region(data, hierarchy_name, db)
 
             license_id = utils.get_license_mixin(data, db)
-            # species_id, strain_id = utils.get_species_mixin(data, db)
-
             subject_id = utils.get_or_create_subject(data, project_context, db)
             createdAt, updatedAt = utils.get_created_and_updated(data)
 
@@ -1036,50 +1090,6 @@ def ingest(
                 create_annotation(annotation, db_item.id, db)
 
 
-class ImportBrainAtlas(Import):
-    name = "Brain Atlas"
-
-    @staticmethod
-    def is_correct_type(data):
-        return "BrainAtlasRelease" in ensurelist(data["@type"]) and data["@id"] == BRAIN_ATLAS_ID
-
-    @staticmethod
-    def ingest(
-        db: Session,
-        project_context: ProjectContext,
-        data_list: list[dict],
-        all_data_by_id: dict,
-        hierarchy_name: str,
-    ):
-        for data in data_list:
-            legacy_id, legacy_self = data["@id"], data["_self"]
-            rm = utils._find_by_legacy_id(legacy_id, BrainAtlas, db)
-            if rm:
-                continue
-
-            created_by_id, updated_by_id = utils.get_agent_mixin(data, db)
-            createdAt, updatedAt = utils.get_created_and_updated(data)
-            species_id, strain_id = utils.get_species_mixin(data, db)
-
-            brain_region_id = utils.get_brain_region(data, hierarchy_name, db)
-
-            db_item = BrainAtlas(
-                legacy_id=[legacy_id],
-                legacy_self=[legacy_self],
-                name=data["name"],
-                description=data.get("description", ""),
-                brain_region_id=brain_region_id,
-                species_id=species_id,
-                strain_id=strain_id,
-                created_by_id=created_by_id,
-                updated_by_id=updated_by_id,
-                creation_date=createdAt,
-                update_date=updatedAt,
-                authorized_project_id=project_context.project_id,
-                authorized_public=AUTHORIZED_PUBLIC,
-            )
-
-
 class ImportDistribution(Import):
     name = "Distribution"
 
@@ -1098,8 +1108,7 @@ def ingest(
         ignored: dict[tuple[dict], int] = Counter()
         for data in tqdm(data_list):
             legacy_id = data["@id"]
-            root = utils._find_by_legacy_id(legacy_id, Entity, db)
-            if root:
+            if root := utils._find_by_legacy_id(legacy_id, Entity, db):
                 utils.import_distribution(data, root.id, root.type, db, project_context)
             else:
                 dt = data["@type"]
@@ -1339,9 +1348,7 @@ def _do_import(db, input_dir, project_context, hierarchy_name):
         ImportAgent,
         ImportMETypeDensity,
         ImportCellComposition,
-        ImportBrainAtlas,
         ImportAnalysisSoftwareSourceCode,
-        ImportBrainRegionMeshes,
         ImportMorphologies,
         ImportEModels,
         ImportExperimentalNeuronDensities,
@@ -1350,6 +1357,7 @@ def _do_import(db, input_dir, project_context, hierarchy_name):
         ImportMEModel,
         ImportElectricalCellRecording,
         ImportSingleNeuronSimulation,
+        ImportBrainAtlas,
         ImportDistribution,
         ImportNeuronMorphologyFeatureAnnotation,
     ]
@@ -1579,7 +1587,7 @@ def curate_files(input_digest_path, output_digest_path, out_dir, dry_run):
         closing(configure_database_session_manager()) as database_session_manager,
         database_session_manager.session() as db,
     ):
-        # Group assets by ntity_type/entity_id/content_type
+        # Group assets by entity_type/entity_id/content_type
         assets_per_entity_type = defaultdict(lambda: defaultdict(lambda: defaultdict(list)))
         for asset in db.query(Asset).all():
             entity_type = asset.full_path.split("/")[4]

app/cli/curate.py

@@ -60,13 +60,9 @@ def _find_by_legacy_id(legacy_id, db_type, db, _cache={}):
     return res
 
 
-def get_brain_region_by_hier_id(brain_region, hierarchy_name, db, _cache={}):
-    brain_region = curate.curate_brain_region(brain_region)
-
-    brain_region_id = int(brain_region["@id"])
+def get_brain_region_by_annotation_id(brain_region_id: int, hierarchy_name: str, db, _cache={}):
     if (hierarchy_name, brain_region_id) in _cache:
         return _cache[hierarchy_name, brain_region_id]
-
     br = db.execute(
         sa.select(BrainRegion)
         .join(BrainRegionHierarchy, BrainRegion.hierarchy_id == BrainRegionHierarchy.id)
@@ -77,13 +73,22 @@ def get_brain_region_by_hier_id(brain_region, hierarchy_name, db, _cache={}):
     ).scalar_one_or_none()
 
     if br is None:
-        msg = f"({hierarchy_name}, {brain_region}) not found in database"
+        msg = f"({hierarchy_name}, {brain_region_id}) not found in database"
         raise RuntimeError(msg)
 
     _cache[hierarchy_name, brain_region_id] = br.id
+
     return br.id
 
 
+def get_brain_region_by_hier_id(brain_region, hierarchy_name, db):
+    brain_region = curate.curate_brain_region(brain_region)
+
+    brain_region_id = int(brain_region["@id"])
+
+    return get_brain_region_by_annotation_id(brain_region_id, hierarchy_name, db)
+
+
 def get_or_create_species(species, db, _cache={}):
     id_ = species["@id"]
     if id_ in _cache:

app/cli/import_data.py

@@ -464,12 +464,6 @@ class EModel(
     __mapper_args__ = {"polymorphic_identity": __tablename__}  # noqa: RUF012
 
 
-class Mesh(LocationMixin, NameDescriptionVectorMixin, Entity):
-    __tablename__ = EntityType.mesh.value
-    id: Mapped[uuid.UUID] = mapped_column(ForeignKey("entity.id"), primary_key=True)
-    __mapper_args__ = {"polymorphic_identity": __tablename__}  # noqa: RUF012
-
-
 class MEModel(
     MTypesMixin, ETypesMixin, SpeciesMixin, LocationMixin, NameDescriptionVectorMixin, Entity
 ):
@@ -875,9 +869,30 @@ class METypeDensity(
     __mapper_args__ = {"polymorphic_identity": __tablename__}  # noqa: RUF012
 
 
-class BrainAtlas(NameDescriptionVectorMixin, LocationMixin, SpeciesMixin, Entity):
+class BrainAtlas(NameDescriptionVectorMixin, SpeciesMixin, Entity):
     __tablename__ = EntityType.brain_atlas
+
+    id: Mapped[uuid.UUID] = mapped_column(ForeignKey("entity.id"), primary_key=True)
+
+    hierarchy_id: Mapped[uuid.UUID] = mapped_column(
+        ForeignKey("brain_region_hierarchy.id"), index=True
+    )
+
+    __mapper_args__ = {"polymorphic_identity": __tablename__}  # noqa: RUF012
+
+
+class BrainAtlasRegion(Entity, LocationMixin):
+    __tablename__ = EntityType.brain_atlas_region
+
     id: Mapped[uuid.UUID] = mapped_column(ForeignKey("entity.id"), primary_key=True)
+
+    # only the volume for leaf nodes is saved; the consumer must calculate
+    # volumes depending on which view of the hierarchy they are using
+    volume: Mapped[float] = mapped_column(nullable=True)
+    leaf_region: Mapped[bool] = mapped_column(default=False)
+
+    brain_atlas_id: Mapped[uuid.UUID] = mapped_column(ForeignKey("brain_atlas.id"), index=True)
+
     __mapper_args__ = {"polymorphic_identity": __tablename__}  # noqa: RUF012
 
 

app/cli/utils.py

@@ -50,6 +50,7 @@ class EntityType(StrEnum):
 
     analysis_software_source_code = auto()
     brain_atlas = auto()
+    brain_atlas_region = auto()
     emodel = auto()
     cell_composition = auto()
     experimental_bouton_density = auto()