Version 1.0.4 (#87)

* Bumped version to 1.0.4

* File access control (#86)

* added subproject requirement to uploaded files

* Adjusted file saving mechanism

* Added comment for clarity

* Implemented FileAccessControl based on subproject access

* Improved robustness of MySQLStatementBuilder

* Fixed static acces_level issue in FileAccessChecker

* Mysql statement builder refactor (#88)

* Removed mysqlutils and instead imported mysql-statement-builder package

* Added mysql-statement-builder to requirements.txt

* Reconnected mysqlsb logger with fastapi logger

* Enable sorting user listings, and searching for users (#89)

* Search for user functionality

* Safe generic order-by for user lists

* user sorting and search tests

Added tests for new functionality

* fixed failing cvs simulation test

* Projects administration end-points for SED portal (#91)

* Updated subprojects and projects data structures

* Added min_length constraint to sub-project and project names

* Also list project creation date and participant count

* Project lists entries are now properly populated

* Functionality to avoid accidental project overwrites

* Fixed test fail and bumped up desim version (#90)

* Fixed test that fails

* desim-tool version 0.3.3

* replaced generic exception

---------

Co-authored-by: EppChops <erik.00.berg@gmail.com>
Co-authored-by: Erik Berg <57296415+EppChops@users.noreply.github.com>

* 83 Endpoint for deleting files (#95)

* Functionality for deleting files

* reset utils to previous state

* 83 Update project details (#96)

* Partial implementation of necessary methods

* Post multiple participants at the same time

* Fixed incorrect project access checker instantiation

* Tests

* Fixed broken test request for project update

* Projects can now be updated

* Improved log message

---------

Co-authored-by: Oscar Bennet <oscar.bennet@hotmail.se>
Co-authored-by: EppChops <erik.00.berg@gmail.com>
Co-authored-by: Erik Berg <57296415+EppChops@users.noreply.github.com>

requirements.txt

@@ -1,5 +1,5 @@
 bcrypt==4.0.1
-desim-tool==0.3.1
+desim-tool==0.3.3
 fastapi==0.95.1
 mvmlib==0.5.9
 mysql-connector-python==8.0.33
@@ -11,6 +11,7 @@ python-multipart==0.0.6
 starlette==0.26.1
 uvicorn==0.21.1
 openpyxl==3.1.2
+mysql-statement-builder==0.*
 
 pytest==7.3.1
 httpx==0.24.0

sedbackend/apps/core/authentication/storage.py

@@ -6,7 +6,7 @@
 from sedbackend.apps.core.authentication.models import UserAuth, SSOResolutionData
 from sedbackend.apps.core.users.exceptions import UserNotFoundException
 from sedbackend.apps.core.authentication.exceptions import InvalidNonceException, FaultyNonceOperation
-from sedbackend.libs.mysqlutils.builder import MySQLStatementBuilder, FetchType
+from mysqlsb.builder import MySQLStatementBuilder, FetchType
 
 from mysql.connector.pooling import PooledMySQLConnection
 

sedbackend/apps/core/files/dependencies.py

@@ -0,0 +1,26 @@
+from typing import List
+
+from fastapi import Request
+from fastapi.logger import logger
+
+from sedbackend.apps.core.projects.dependencies import SubProjectAccessChecker
+from sedbackend.apps.core.projects.models import AccessLevel
+from sedbackend.apps.core.projects.implementation import impl_get_subproject_by_id
+from sedbackend.apps.core.files.implementation import impl_get_file_mapped_subproject_id
+
+
+class FileAccessChecker:
+    def __init__(self, allowed_levels: List[AccessLevel]):
+        self.access_levels = allowed_levels
+
+    def __call__(self, file_id: int, request: Request):
+        logger.debug(f'Is user with id {request.state.user_id} '
+                     f'allowed to access file with id {file_id}?')
+        user_id = request.state.user_id
+
+        # Get subproject ID
+        subproject_id = impl_get_file_mapped_subproject_id(file_id)
+
+        # Run subproject access check
+        subproject = impl_get_subproject_by_id(subproject_id)
+        return SubProjectAccessChecker.check_user_subproject_access(subproject, self.access_levels, user_id)

sedbackend/apps/core/files/exceptions.py

@@ -8,3 +8,15 @@ class FileNotFoundException(Exception):
 
 class FileParsingException(Exception):
     pass
+
+
+class SubprojectMappingNotFound(Exception):
+    pass
+
+
+class FileNotDeletedException(Exception):
+    pass
+
+
+class PathMismatchException(Exception):
+    pass

sedbackend/apps/core/files/implementation.py

@@ -39,6 +39,16 @@ def impl_delete_file(file_id: int, current_user_id: int) -> bool:
             status_code=status.HTTP_403_FORBIDDEN,
             detail=f"User does not have access to a file with id = {file_id}"
         )
+    except exc.FileNotDeletedException:
+        raise HTTPException(
+            status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+            detail=f"File could not be deleted"
+        )
+    except exc.PathMismatchException:
+        raise HTTPException(
+            status_code=status.HTTP_400_BAD_REQUEST,
+            detail=f'Path to file does not match internal path'
+        )
 
 
 def impl_get_file_path(file_id: int, current_user_id: int) -> models.StoredFilePath:
@@ -102,3 +112,15 @@ def impl_get_file(file_id: int, current_user_id: int):
             status_code=status.HTTP_403_FORBIDDEN,
             detail="User does not have access to requested file."
         )
+
+
+def impl_get_file_mapped_subproject_id(file_id):
+    try:
+        with get_connection() as con:
+            subproject_id = storage.db_get_file_mapped_subproject_id(con, file_id)
+            return subproject_id
+    except exc.SubprojectMappingNotFound:
+        raise HTTPException(
+            status_code=status.HTTP_400_BAD_REQUEST,
+            detail=f"No subproject mapping found for file with id = {file_id}"
+        )

sedbackend/apps/core/files/models.py

@@ -1,7 +1,6 @@
 from typing import Any
 from datetime import datetime
 import os
-from tempfile import SpooledTemporaryFile
 
 from pydantic import BaseModel
 from fastapi.datastructures import UploadFile
@@ -12,15 +11,17 @@ class StoredFilePost(BaseModel):
     owner_id: int
     extension: str
     file_object: Any
+    subproject_id: int
 
     @staticmethod
-    def import_fastapi_file(file: UploadFile, current_user_id: int):
+    def import_fastapi_file(file: UploadFile, current_user_id: int, subproject_id: int):
         filename = file.filename
         extension = os.path.splitext(file.filename)[1]
         return StoredFilePost(filename=filename,
                               extension=extension,
                               owner_id=current_user_id,
-                              file_object=file.file)
+                              file_object=file.file,
+                              subproject_id=subproject_id)
 
 
 class StoredFileEntry(BaseModel):
@@ -30,6 +31,7 @@ class StoredFileEntry(BaseModel):
     insert_timestamp: datetime
     owner_id: int
     extension: str
+    subproject_id: int
 
 
 class StoredFile(BaseModel):

sedbackend/apps/core/files/router.py

@@ -2,7 +2,9 @@
 from fastapi.responses import FileResponse
 
 import sedbackend.apps.core.files.implementation as impl
+from sedbackend.apps.core.files.dependencies import FileAccessChecker
 from sedbackend.apps.core.authentication.utils import get_current_active_user
+from sedbackend.apps.core.projects.models import AccessLevel
 from sedbackend.apps.core.users.models import User
 
 
@@ -11,7 +13,9 @@
 
 @router.get("/{file_id}/download",
              summary="Download file",
-             response_class=FileResponse)
+             response_class=FileResponse,
+             dependencies=[Depends(FileAccessChecker(AccessLevel.list_can_read()))]
+             )
 async def get_file(file_id: int, current_user: User = Depends(get_current_active_user)):
     """
     Download an uploaded file
@@ -22,3 +26,16 @@ async def get_file(file_id: int, current_user: User = Depends(get_current_active
         filename=stored_file_path.filename
     )
     return resp
+
+
+@router.delete("/{file_id}/delete",
+               summary="Delete file",
+               response_model=bool,
+               dependencies=[Depends(FileAccessChecker(AccessLevel.list_are_admins()))])
+async def delete_file(file_id: int, current_user: User = Depends(get_current_active_user)):
+    """
+    Delete a file. 
+    Only accessible to admins and the owner of the file. 
+    """
+    return impl.impl_delete_file(file_id, current_user.id)
+

sedbackend/apps/core/files/storage.py

@@ -3,14 +3,19 @@
 import os
 
 from mysql.connector.pooling import PooledMySQLConnection
+from fastapi.logger import logger
+import os
 
 import sedbackend.apps.core.files.models as models
 import sedbackend.apps.core.files.exceptions as exc
-from sedbackend.libs.mysqlutils import MySQLStatementBuilder, exclude_cols, FetchType
+import sedbackend.apps.core.files.implementation as impl
+from mysqlsb import MySQLStatementBuilder, exclude_cols, FetchType
 
 FILES_RELATIVE_UPLOAD_DIR = f'{os.path.abspath(os.sep)}sed_lab/uploaded_files/'
 FILES_TABLE = 'files'
+FILES_TO_SUBPROJECTS_MAP_TABLE = 'files_subprojects_map'
 FILES_COLUMNS = ['id', 'temp', 'uuid', 'filename', 'insert_timestamp', 'directory', 'owner_id', 'extension']
+FILES_TO_SUBPROJECTS_MAP_COLUMNS = ['id', 'file_id', 'subproject_id']
 
 
 def db_save_file(con: PooledMySQLConnection, file: models.StoredFilePost) -> models.StoredFileEntry:
@@ -28,38 +33,75 @@ def db_save_file(con: PooledMySQLConnection, file: models.StoredFilePost) -> mod
 
     file_id = insert_stmnt.last_insert_id
 
+    # Store mapping between file id and subproject id in database
+    insert_mapping_stmnt = MySQLStatementBuilder(con)
+    insert_mapping_stmnt.insert(FILES_TO_SUBPROJECTS_MAP_TABLE, ['file_id', 'subproject_id'])\
+        .set_values([file_id, file.subproject_id])\
+        .execute()
+
     return db_get_file_entry(con, file_id, file.owner_id)
 
 
 def db_delete_file(con: PooledMySQLConnection, file_id: int, current_user_id: int) -> bool:
+    stored_file_path = impl.impl_get_file_path(file_id, current_user_id)
+
+    if os.path.commonpath([FILES_RELATIVE_UPLOAD_DIR]) != os.path.commonpath([FILES_RELATIVE_UPLOAD_DIR, os.path.abspath(stored_file_path.path)]):
+        raise exc.PathMismatchException
+
+    try:
+        os.remove(stored_file_path.path)
+        delete_stmnt = MySQLStatementBuilder(con)
+        delete_stmnt.delete(FILES_TABLE) \
+            .where('id=?', [file_id]) \
+            .execute(fetch_type=FetchType.FETCH_NONE)
+
+    except Exception:
+        raise exc.FileNotDeletedException
+
     return True
 
 
 def db_get_file_entry(con: PooledMySQLConnection, file_id: int, current_user_id: int) -> models.StoredFileEntry:
-    select_stmnt = MySQLStatementBuilder(con)
-    res = select_stmnt.select(FILES_TABLE, exclude_cols(FILES_COLUMNS, ['uuid', 'directory']))\
-        .where('id = ?', [file_id])\
-        .execute(dictionary=True, fetch_type=FetchType.FETCH_ONE)
+    res_dict = None
+    with con.cursor(prepared=True) as cursor:
+        # This expression uses two tables (files and files_to_subprojects_map)
+        query = f"SELECT {', '.join(['f.id', 'f.temp', 'f.uuid', 'f.filename', 'f.insert_timestamp', 'f.directory', 'f.owner_id', 'f.extension'])}, fsm.`subproject_id` " \
+                f"FROM `{FILES_TABLE}` f " \
+                f"INNER JOIN {FILES_TO_SUBPROJECTS_MAP_TABLE} fsm ON (f.id = fsm.file_id) " \
+                f"WHERE f.`id` = ?"
+        values = [file_id]
 
-    if res is None:
-        raise exc.FileNotFoundException
+        # Log for sanity-check
+        logger.debug(f"db_get_file_entry query: '{query}' with values: {values}")
+
+        # Execute query
+        cursor.execute(query, values)
+
+        # Handle results
+        results = cursor.fetchone()
 
-    stored_file = models.StoredFileEntry(**res)
+        if results is None:
+            raise exc.FileNotFoundException
+
+        res_dict = dict(zip(cursor.column_names, results))
+
+    stored_file = models.StoredFileEntry(**res_dict)
     return stored_file
 
 
 def db_get_file_path(con: PooledMySQLConnection, file_id: int, current_user_id: int) -> models.StoredFilePath:
     select_stmnt = MySQLStatementBuilder(con)
     res = select_stmnt\
-        .select(FILES_TABLE, ['filename', 'uuid', 'directory', 'extension'])\
+        .select(FILES_TABLE, ['filename', 'uuid', 'directory', 'owner_id', 'extension'])\
         .where('id=?', [file_id])\
         .execute(dictionary=True, fetch_type=FetchType.FETCH_ONE)
 
     if res is None:
         raise exc.FileNotFoundException('File not found in DB')
 
     path = res['directory'] + res['uuid']
-    stored_path = models.StoredFilePath(id=file_id, filename=res['filename'], path=path, extension=res['extension'])
+    stored_path = models.StoredFilePath(
+        id=file_id, filename=res['filename'], path=path, owner_id=res['owner_id'], extension=res['extension'])
     return stored_path
 
 
@@ -71,3 +113,15 @@ def db_put_file_temp(con: PooledMySQLConnection, file_id: int, temp: bool, curre
 def db_put_filename(con: PooledMySQLConnection, file_id: int, filename_new: str, current_user_id: int) \
         -> models.StoredFileEntry:
     pass
+
+
+def db_get_file_mapped_subproject_id(con: PooledMySQLConnection, file_id) -> int:
+    select_stmnt = MySQLStatementBuilder(con)
+    res = select_stmnt.select(FILES_TO_SUBPROJECTS_MAP_TABLE, ['subproject_id'])\
+        .where('file_id=?', [file_id])\
+        .execute(dictionary=True, fetch_type=FetchType.FETCH_ONE)
+
+    if res is None:
+        raise exc.SubprojectMappingNotFound('Mapping could not be found.')
+
+    return res['subproject_id']

sedbackend/apps/core/individuals/storage.py

@@ -5,7 +5,7 @@
 
 import sedbackend.apps.core.individuals.models as models
 import sedbackend.apps.core.individuals.exceptions as ex
-from sedbackend.libs.mysqlutils import MySQLStatementBuilder, FetchType, exclude_cols
+from mysqlsb import MySQLStatementBuilder, FetchType, exclude_cols
 
 INDIVIDUALS_TABLE = 'individuals'
 INDIVIDUALS_COLUMNS = ['id', 'name', 'is_archetype']

sedbackend/apps/core/measurements/implementation.py

@@ -119,9 +119,9 @@ def impl_post_measurement_result(measurement_id: int, mr: models.MeasurementResu
         return res
 
 
-def impl_post_upload_set(file, current_user_id: int, csv_delimiter: Optional[str] = None) -> List:
+def impl_post_upload_set(file, current_user_id: int, subproject_id: int, csv_delimiter: Optional[str] = None) -> List:
     try:
-        stored_file_post = models_files.StoredFilePost.import_fastapi_file(file, current_user_id)
+        stored_file_post = models_files.StoredFilePost.import_fastapi_file(file, current_user_id, subproject_id)
         with get_connection() as con:
             file_entry = storage_files.db_save_file(con, stored_file_post)
             file_path = storage_files.db_get_file_path(con, file_entry.id, current_user_id)

sedbackend/apps/core/measurements/router.py

@@ -30,9 +30,9 @@ async def get_measurement_sets(subproject_id: Optional[int] = None):
              response_model=List[str],
              description="Upload a measurement set using a CSV or Excel file. Leaving csv_delimiter as None will "
                          "result in the value being inferred automatically.")
-async def post_upload_set(file: UploadFile = File(...), current_user: User = Depends(get_current_active_user),
+async def post_upload_set(subproject_id: int, file: UploadFile = File(...), current_user: User = Depends(get_current_active_user),
                           csv_delimiter: Optional[str] = None):
-    return impl.impl_post_upload_set(file, current_user.id, csv_delimiter=csv_delimiter)
+    return impl.impl_post_upload_set(file, current_user.id, subproject_id, csv_delimiter=csv_delimiter)
 
 
 @router.get("/sets/{measurement_set_id}",

sedbackend/apps/core/measurements/storage.py

@@ -3,7 +3,7 @@
 
 from fastapi.logger import logger
 
-from sedbackend.libs.mysqlutils import MySQLStatementBuilder, FetchType, exclude_cols
+from mysqlsb import MySQLStatementBuilder, FetchType, exclude_cols
 import sedbackend.apps.core.measurements.models as models
 import sedbackend.apps.core.measurements.exceptions as exc
 

sedbackend/apps/core/projects/dependencies.py

@@ -3,7 +3,7 @@
 from fastapi import HTTPException, Request, status
 from fastapi.logger import logger
 
-from sedbackend.apps.core.projects.models import AccessLevel
+from sedbackend.apps.core.projects.models import AccessLevel, SubProject
 from sedbackend.apps.core.projects.implementation import impl_get_project, impl_get_subproject_native
 
 
@@ -53,18 +53,23 @@ def __call__(self, native_project_id: int, request: Request):
         # Get subproject
         subproject = impl_get_subproject_native(self.application_sid, native_project_id)
 
+        return SubProjectAccessChecker.check_user_subproject_access(subproject, self.access_levels, user_id)
+
+    @staticmethod
+    def check_user_subproject_access(subproject: SubProject, access_levels: List[AccessLevel], user_id: int):
         if subproject.project_id is not None:
             # Get project
             project = impl_get_project(subproject.project_id)   # <-- This can throw
             # Check user access level in that project
             access = project.participants_access[user_id]
-            if access in self.access_levels:
+            if access in access_levels:
                 logger.debug(f"Yes, user {user_id} has access level {access}")
                 return True
         else:
             # Fallback solution: Check if user is the owner/creator of the subproject.
-            if request.state.user_id == subproject.owner_id:
-                logger.debug("User is owner of subproject.")
+            if user_id == subproject.owner_id:
+                logger.debug(f"User with id {user_id} is the owner of subproject with id {subproject.id} "
+                             f"(owner_id = {subproject.owner_id}).")
                 return True
 
         logger.debug(f"No, user {user_id} does not have the minimum required access level")
@@ -73,3 +78,4 @@ def __call__(self, native_project_id: int, request: Request):
             detail="User does not have the necessary access level",
         )
 
+

sedbackend/apps/core/projects/exceptions.py

@@ -36,3 +36,7 @@ class SubProjectDuplicateException(Exception):
 
 class ParticipantInconsistencyException(Exception):
     pass
+
+
+class ConflictingProjectAssociationException(Exception):
+    pass