Merge pull request #1082 from fractal-analytics-platform/1063-bis

Add `project` relationship to `Dataset` and `Workflow`

CHANGELOG.md

@@ -4,15 +4,19 @@
 
 * API:
     * Add `GET /admin/job/{job_id}/stop/` and `GET /admin/job/{job_id}/download/` endpoints (\#1059).
-    * Use `DatasetRead` and `WorkflowRead` models for "dump" attributes of `ApplyWorkflowRead` (\#1049).
+    * Use `DatasetDump` and `WorkflowDump` models for "dump" attributes of `ApplyWorkflowRead` (\#1049, \#1082).
     * Add `slurm_accounts` to `User` schemas and add `slurm_account` to `ApplyWorkflow` schemas (\#1067).
     * Prevent providing a `package_version` for task collection from a `.whl` local package (\#1069).
+    * Add `DatasetRead.project` and `WorkflowRead.project` attributes (\#1082).
 * Database:
     * Make `ApplyWorkflow.workflow_dump` column non-nullable (\#1049).
     * Add `UserOAuth.slurm_accounts` and `ApplyWorkflow.slurm_account` columns (\#1067).
     * Add script for adding `ApplyWorkflow.user_email` (\#1058).
+    * Add `Dataset.project` and `Workflow.project` relationships (\#1082).
+    * Avoid using `Project` relationships `dataset_list` or `workflow_list` within some `GET` endpoints (\#1082).
 * Testing:
-    *  Only use ubuntu-22.04 in GitHub actions (\#1061).
+    * Only use ubuntu-22.04 in GitHub actions (\#1061).
+    * Improve unit testing of database models (\#1082).
 * Dependencies:
     * Pin `bcrypt` to 4.0.1 to avoid warning in passlib (\#1060).
 * Runner:

fractal_server/app/models/dataset.py

@@ -38,6 +38,10 @@ class Dataset(_DatasetBase, SQLModel, table=True):
 
     id: Optional[int] = Field(default=None, primary_key=True)
     project_id: int = Field(foreign_key="project.id")
+    project: "Project" = Relationship(  # noqa: F821
+        back_populates="dataset_list",
+        sa_relationship_kwargs=dict(lazy="selectin"),
+    )
 
     list_jobs_input: list[ApplyWorkflow] = Relationship(  # noqa: F821
         sa_relationship_kwargs=dict(

fractal_server/app/models/project.py

@@ -27,7 +27,7 @@ class Project(_ProjectBase, SQLModel, table=True):
         sa_relationship_kwargs={
             "lazy": "selectin",
             "cascade": "all, delete-orphan",
-        }
+        },
     )
 
     workflow_list: list[Workflow] = Relationship(

fractal_server/app/models/workflow.py

@@ -106,6 +106,10 @@ class Workflow(_WorkflowBase, SQLModel, table=True):
 
     id: Optional[int] = Field(default=None, primary_key=True)
     project_id: int = Field(foreign_key="project.id")
+    project: "Project" = Relationship(  # noqa: F821
+        back_populates="workflow_list",
+        sa_relationship_kwargs=dict(lazy="selectin"),
+    )
 
     task_list: list[WorkflowTask] = Relationship(
         sa_relationship_kwargs=dict(

fractal_server/app/routes/api/v1/_aux_functions.py

@@ -100,6 +100,10 @@ async def _get_workflow_check_owner(
             detail=(f"Invalid {project_id=} for {workflow_id=}."),
         )
 
+    # Refresh so that workflow.project relationship is loaded (see discussion
+    # in issue #1063)
+    await db.refresh(workflow)
+
     return workflow
 
 
@@ -257,6 +261,11 @@ async def _get_dataset_check_owner(
             status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
             detail=f"Invalid {project_id=} for {dataset_id=}",
         )
+
+    # Refresh so that dataset.project relationship is loaded (see discussion
+    # in issue #1063)
+    await db.refresh(dataset)
+
     return dict(dataset=dataset, project=project)
 
 

fractal_server/app/routes/api/v1/dataset.py

@@ -14,6 +14,7 @@
 from ....db import get_db
 from ....models import ApplyWorkflow
 from ....models import Dataset
+from ....models import Project
 from ....models import Resource
 from ....runner._common import HISTORY_FILENAME
 from ....schemas import DatasetCreate
@@ -74,10 +75,17 @@ async def read_dataset_list(
     """
     Get dataset list for given project
     """
+    # Access control
     project = await _get_project_check_owner(
         project_id=project_id, user_id=user.id, db=db
     )
-    return project.dataset_list
+    # Find datasets of the current project. Note: this select/where approach
+    # has much better scaling than refreshing all elements of
+    # `project.dataset_list` - ref
+    # https://github.com/fractal-analytics-platform/fractal-server/pull/1082#issuecomment-1856676097.
+    stm = select(Dataset).where(Dataset.project_id == project.id)
+    dataset_list = (await db.execute(stm)).scalars().all()
+    return dataset_list
 
 
 @router.get(
@@ -484,14 +492,13 @@ async def get_workflowtask_status(
 @router.get("/dataset/", response_model=list[DatasetRead])
 async def get_user_datasets(
     user: User = Depends(current_active_user),
+    db: AsyncSession = Depends(get_db),
 ) -> list[DatasetRead]:
     """
     Returns all the datasets of the current user
     """
-    dataset_list = [
-        dataset
-        for project in user.project_list
-        for dataset in project.dataset_list
-    ]
-
+    stm = select(Dataset)
+    stm = stm.join(Project).where(Project.user_list.any(User.id == user.id))
+    res = await db.execute(stm)
+    dataset_list = res.scalars().all()
     return dataset_list

fractal_server/app/routes/api/v1/job.py

@@ -6,9 +6,12 @@
 from fastapi import Response
 from fastapi import status
 from fastapi.responses import StreamingResponse
+from sqlmodel import select
 
 from ....db import AsyncSession
 from ....db import get_db
+from ....models import ApplyWorkflow
+from ....models import Project
 from ....schemas import ApplyWorkflowRead
 from ....security import current_active_user
 from ....security import User
@@ -26,14 +29,16 @@
 @router.get("/job/", response_model=list[ApplyWorkflowRead])
 async def get_user_jobs(
     user: User = Depends(current_active_user),
+    db: AsyncSession = Depends(get_db),
 ) -> list[ApplyWorkflowRead]:
     """
     Returns all the jobs of the current user
     """
 
-    job_list = [
-        job for project in user.project_list for job in project.job_list
-    ]
+    stm = select(ApplyWorkflow)
+    stm = stm.join(Project).where(Project.user_list.any(User.id == user.id))
+    res = await db.execute(stm)
+    job_list = res.scalars().all()
 
     return job_list
 

fractal_server/app/routes/api/v1/workflow.py

@@ -24,6 +24,7 @@
 from ....db import AsyncSession
 from ....db import get_db
 from ....models import ApplyWorkflow
+from ....models import Project
 from ....models import Task
 from ....models import Workflow
 from ....schemas import WorkflowCreate
@@ -55,10 +56,17 @@ async def get_workflow_list(
     """
     Get workflow list for given project
     """
+    # Access control
     project = await _get_project_check_owner(
         project_id=project_id, user_id=user.id, db=db
     )
-    return project.workflow_list
+    # Find workflows of the current project. Note: this select/where approach
+    # has much better scaling than refreshing all elements of
+    # `project.workflow_list` - ref
+    # https://github.com/fractal-analytics-platform/fractal-server/pull/1082#issuecomment-1856676097.
+    stm = select(Workflow).where(Workflow.project_id == project.id)
+    workflow_list = (await db.execute(stm)).scalars().all()
+    return workflow_list
 
 
 @router.post(
@@ -317,14 +325,13 @@ async def import_workflow(
 @router.get("/workflow/", response_model=list[WorkflowRead])
 async def get_user_workflows(
     user: User = Depends(current_active_user),
+    db: AsyncSession = Depends(get_db),
 ) -> list[WorkflowRead]:
     """
     Returns all the workflows of the current user
     """
-    workflow_list = [
-        workflow
-        for project in user.project_list
-        for workflow in project.workflow_list
-    ]
-
+    stm = select(Workflow)
+    stm = stm.join(Project).where(Project.user_list.any(User.id == user.id))
+    res = await db.execute(stm)
+    workflow_list = res.scalars().all()
     return workflow_list

fractal_server/app/schemas/applyworkflow.py

@@ -7,8 +7,6 @@
 from pydantic.types import StrictStr
 
 from ._validators import valstr
-from .dataset import DatasetRead
-from .workflow import WorkflowRead
 
 __all__ = (
     "_ApplyWorkflowBase",
@@ -17,6 +15,47 @@
 )
 
 
+class TaskDump(BaseModel):
+    id: int
+    source: str
+    name: str
+    command: str
+    input_type: str
+    output_type: str
+    owner: Optional[str]
+    version: Optional[str]
+
+
+class WorkflowTaskDump(BaseModel):
+    id: int
+    order: Optional[int]
+    workflow_id: int
+    task_id: int
+    task: TaskDump
+
+
+class WorkflowDump(BaseModel):
+    id: int
+    name: str
+    project_id: int
+    task_list: list[WorkflowTaskDump]
+
+
+class ResourceDump(BaseModel):
+    id: int
+    path: str
+    dataset_id: int
+
+
+class DatasetDump(BaseModel):
+    id: int
+    name: str
+    type: Optional[str]
+    read_only: bool
+    resource_list: list[ResourceDump]
+    project_id: int
+
+
 class JobStatusType(str, Enum):
     """
     Define the available job statuses
@@ -135,11 +174,11 @@ class ApplyWorkflowRead(_ApplyWorkflowBase):
     user_email: str
     slurm_account: Optional[str]
     workflow_id: Optional[int]
-    workflow_dump: Optional[WorkflowRead]
+    workflow_dump: Optional[WorkflowDump]
     input_dataset_id: Optional[int]
-    input_dataset_dump: Optional[DatasetRead]
+    input_dataset_dump: Optional[DatasetDump]
     output_dataset_id: Optional[int]
-    output_dataset_dump: Optional[DatasetRead]
+    output_dataset_dump: Optional[DatasetDump]
     start_timestamp: datetime
     end_timestamp: Optional[datetime]
     status: str

fractal_server/app/schemas/dataset.py

@@ -7,10 +7,10 @@
 
 from ._validators import val_absolute_path
 from ._validators import valstr
+from .project import ProjectRead
 from .workflow import WorkflowTaskRead
 from .workflow import WorkflowTaskStatusType
 
-
 __all__ = (
     "DatasetUpdate",
     "DatasetCreate",
@@ -136,15 +136,17 @@ class DatasetRead(_DatasetBase):
 
     Attributes:
         id:
+        read_only:
         resource_list:
         project_id:
-        read_only:
+        project:
     """
 
     id: int
     resource_list: list[ResourceRead]
     project_id: int
     read_only: bool
+    project: ProjectRead
 
 
 class DatasetStatusRead(BaseModel):

fractal_server/app/schemas/project.py

@@ -41,6 +41,8 @@ class ProjectRead(_ProjectBase):
 
     Attributes:
         id:
+        name:
+        read_only:
     """
 
     id: int

fractal_server/app/schemas/workflow.py

@@ -7,11 +7,11 @@
 
 from ._validators import valint
 from ._validators import valstr
+from .project import ProjectRead
 from .task import TaskExport
 from .task import TaskImport
 from .task import TaskRead
 
-
 __all__ = (
     "WorkflowCreate",
     "WorkflowRead",
@@ -124,11 +124,13 @@ class WorkflowRead(_WorkflowBase):
         id:
         project_id:
         task_list:
+        project:
     """
 
     id: int
     project_id: int
     task_list: list[WorkflowTaskRead]
+    project: ProjectRead
 
 
 class WorkflowCreate(_WorkflowBase):