Merge pull request #941 from mild-blue/process_excel_to_json

Process Belgian excel to json

.isort.cfg

@@ -1,2 +1,2 @@
 [settings]
-known_third_party = dacite,flask,flask_bcrypt,flask_restx,flask_sqlalchemy,graph_tool,gunicorn,itsdangerous,jinja2,jwt,mip,networkx,numpy,openapi_spec_validator,pandas,pdfkit,pyotp,requests,responses,sentry_sdk,sqlalchemy,swagger_unittest,werkzeug,yaml,yoyo
+known_third_party = dacite,distutils,flask,flask_bcrypt,flask_restx,flask_sqlalchemy,graph_tool,gunicorn,itsdangerous,jinja2,jwt,mip,networkx,numpy,openapi_spec_validator,pandas,pdfkit,pyotp,requests,responses,sentry_sdk,sqlalchemy,swagger_unittest,werkzeug,yaml,yoyo

local_testing_utilities/change-password.http

@@ -37,6 +37,6 @@ PUT {{url}}/v1/user/reset-password
 Content-Type: application/json
 
 {
-  "token": {{change_password_token}},
-  "password": "bag2r"
+  "token": "{{change_password_token}}",
+  "password": "{{new_password}}"
 }

local_testing_utilities/generate_patients.py

@@ -42,6 +42,8 @@
 SMALL_DATA_FOLDER_WITH_NO_SOLUTION = get_absolute_path('tests/resources/high_res_example_small_data_with_no_solution/')
 
 
+# is needed here because kw_args in dataclass is not handled very well by pylint
+# pylint: disable=unexpected-keyword-arg
 def generate_waiting_since() -> str:
     return f'{random.choice(range(2018, 2020))}-{random.choice(range(1, 12))}-{random.choice(range(1, 28))}'
 

local_testing_utilities/process_excel_data_to_json.py

@@ -0,0 +1,34 @@
+import dataclasses
+import json
+import logging
+import os
+
+from txmatching.utils.country_enum import Country
+from txmatching.utils.excel_parsing.parse_excel_data import (ExcelSource,
+                                                             parse_excel_data)
+from txmatching.web import create_app
+
+logger = logging.getLogger(__name__)
+
+# set the path here to excel with data
+PATH_TO_DATA_FOR_UPLOAD = '/home/honza/Downloads/LDEP KUL - UCL July 22..xlsx'
+TXM_EVENT_NAME = 'TEST-BEL-PRIVATE-2022-02'
+
+
+if __name__ == '__main__':
+    app = create_app()
+    with app.app_context():
+        patients = parse_excel_data(os.path.join(PATH_TO_DATA_FOR_UPLOAD),
+                                    txm_event_name=TXM_EVENT_NAME,
+                                    country=Country.BEL_2,
+                                    excel_source=ExcelSource.BEL_2
+                                    )
+        # here we are assuming currently for simplicity that the data is from one country only
+        patients_together = patients[0]
+        patients_together.add_to_existing_patients = False
+        for patient in patients[1:]:
+            patients_together.donors += patient.donors
+            patients_together.recipients += patient.recipients
+
+        with open('tmp.json', 'w', encoding='utf-8') as f:
+            json.dump(dataclasses.asdict(patients_together), f)

txmatching/data_transfer_objects/hla/parsing_issue_dto.py

@@ -19,6 +19,8 @@ class ParsingIssuePublicDTO:
     confirmed_at: Optional[datetime]
 
 
+# pylint: disable=too-many-instance-attributes
+# It is reasonable to have many attributes here
 @dataclass
 class ParsingIssue(PersistentlyHashable):
     hla_code_or_group: Optional[str]

txmatching/data_transfer_objects/patients/out_dtos/donor_dto_out.py

@@ -1,8 +1,9 @@
 from dataclasses import dataclass
-from typing import List, Optional, Dict
+from typing import Dict, List, Optional
 
 from txmatching.auth.exceptions import InvalidArgumentException
-from txmatching.data_transfer_objects.hla.parsing_issue_dto import ParsingIssueConfirmationDTO
+from txmatching.data_transfer_objects.hla.parsing_issue_dto import \
+    ParsingIssueConfirmationDTO
 from txmatching.patients.patient import Donor
 from txmatching.utils.hla_system.detailed_score import DetailedScoreForHLAGroup
 
@@ -14,7 +15,7 @@ class DonorDTOOut(Donor):
     max_score_with_related_recipient: Optional[float] = None
     detailed_score_with_related_recipient: Optional[List[DetailedScoreForHLAGroup]] = None
     compatible_blood_with_related_recipient: Optional[str] = None
-    # this attribute has default value because fields without default values cannot appear 
+    # this attribute has default value because fields without default values cannot appear
     # after fields with default values. That is why we check if this attribute is set during init
     active_and_valid_pair: bool = None
 

txmatching/data_transfer_objects/patients/patient_base_dto.py

@@ -0,0 +1,49 @@
+from dataclasses import dataclass
+from datetime import date
+from typing import Optional
+
+from txmatching.auth.exceptions import InvalidArgumentException
+
+Kilograms = float
+Centimeters = int
+THIS_YEAR = date.today().year
+
+
+@dataclass(kw_only=True)
+class PatientBaseDTO:
+    height: Optional[Centimeters] = None
+    weight: Optional[Kilograms] = None
+    year_of_birth: Optional[int] = None
+
+    def __post_init__(self):
+        if self.weight:
+            _is_weight_valid(self.weight)
+        if self.height:
+            _is_height_valid(self.height)
+        if self.year_of_birth:
+            _is_year_of_birth_valid(self.year_of_birth)
+
+
+def _is_height_valid(height: Centimeters):
+    if height < 0:
+        raise InvalidArgumentException(f'Invalid patient height {height}cm.')
+
+
+def _is_weight_valid(weight: Kilograms):
+    if weight < 0:
+        raise InvalidArgumentException(f'Invalid patient weight {weight}kg.')
+
+
+def _is_year_of_birth_valid(year_of_birth: Centimeters):
+    if year_of_birth < 1900 or year_of_birth > THIS_YEAR:
+        raise InvalidArgumentException(f'Invalid patient year of birth {year_of_birth}')
+
+
+@dataclass(kw_only=True)
+class RecipientBaseDTO:
+    previous_transplants: Optional[int] = None
+
+    def __post_init__(self):
+        if self.previous_transplants and self.previous_transplants < 0:
+            raise InvalidArgumentException(
+                f'Invalid recipient number of previous transplants {self.previous_transplants}.')

txmatching/data_transfer_objects/patients/update_dtos/donor_update_dto.py

@@ -3,19 +3,8 @@
 
 from txmatching.data_transfer_objects.patients.update_dtos.patient_update_dto import \
     PatientUpdateDTO
-from txmatching.patients.patient import is_height_valid, is_weight_valid, is_year_of_birth_valid
 
 
 @dataclass
 class DonorUpdateDTO(PatientUpdateDTO):
     active: Optional[bool] = None
-
-    def __post_init__(self):
-        if self.height:
-            is_height_valid("donor", self.height)
-
-        if self.weight:
-            is_weight_valid("donor", self.weight)
-
-        if self.year_of_birth:
-            is_year_of_birth_valid("donor", self.year_of_birth)

txmatching/data_transfer_objects/patients/update_dtos/patient_update_dto.py

@@ -1,23 +1,21 @@
 from dataclasses import dataclass
 from typing import Optional
 
+from txmatching.data_transfer_objects.patients.patient_base_dto import (
+    PatientBaseDTO, RecipientBaseDTO)
 from txmatching.data_transfer_objects.patients.update_dtos.hla_code_update_dtos import \
     HLATypingUpdateDTO
-from txmatching.patients.patient_parameters import Centimeters, Kilograms
 from txmatching.utils.blood_groups import BloodGroup
 from txmatching.utils.enums import Sex
 
 
-@dataclass
-class PatientUpdateDTO:
+@dataclass(kw_only=True)
+class PatientUpdateDTO(PatientBaseDTO, RecipientBaseDTO):
     # pylint: disable=too-many-instance-attributes
     # It is reasonable to have many attributes here
     db_id: int
     etag: int
     blood_group: Optional[BloodGroup] = None
     hla_typing: Optional[HLATypingUpdateDTO] = None
     sex: Optional[Sex] = None
-    height: Optional[Centimeters] = None
-    weight: Optional[Kilograms] = None
-    year_of_birth: Optional[int] = None
     note: Optional[str] = None

txmatching/data_transfer_objects/patients/update_dtos/recipient_update_dto.py

@@ -3,31 +3,19 @@
 
 from txmatching.data_transfer_objects.patients.hla_antibodies_dto import \
     HLAAntibodiesUpdateDTO
+from txmatching.data_transfer_objects.patients.patient_base_dto import \
+    RecipientBaseDTO
 from txmatching.data_transfer_objects.patients.update_dtos.patient_update_dto import \
     PatientUpdateDTO
-from txmatching.patients.patient import (is_height_valid, is_number_of_previous_transplants_valid,
-    is_weight_valid, is_year_of_birth_valid, RecipientRequirements)
-
+from txmatching.patients.patient import RecipientRequirements
 
 # pylint: disable=too-many-instance-attributes
+
+
 @dataclass
-class RecipientUpdateDTO(PatientUpdateDTO):
+class RecipientUpdateDTO(PatientUpdateDTO, RecipientBaseDTO):
     acceptable_blood_groups: Optional[List[str]] = None
     hla_antibodies: Optional[HLAAntibodiesUpdateDTO] = None
     recipient_requirements: Optional[RecipientRequirements] = None
     cutoff: Optional[int] = None
     waiting_since: Optional[str] = None
-    previous_transplants: Optional[int] = None
-
-    def __post_init__(self):
-        if self.height:
-            is_height_valid("recipient", self.height)
-
-        if self.weight:
-            is_weight_valid("recipient", self.weight)
-
-        if self.year_of_birth:
-            is_year_of_birth_valid("recipient", self.year_of_birth)
-
-        if self.previous_transplants:
-            is_number_of_previous_transplants_valid(self.previous_transplants)

txmatching/data_transfer_objects/patients/upload_dtos/donor_upload_dto.py

@@ -1,14 +1,15 @@
 from dataclasses import dataclass
 from typing import List, Optional
 
-from txmatching.patients.patient import is_height_valid, is_weight_valid, is_year_of_birth_valid
+from txmatching.data_transfer_objects.patients.patient_base_dto import \
+    PatientBaseDTO
 from txmatching.patients.patient_parameters import Centimeters, Kilograms
 from txmatching.utils.blood_groups import BloodGroup
 from txmatching.utils.enums import Sex
 
 
-@dataclass
-class DonorUploadDTO:
+@dataclass(kw_only=True)
+class DonorUploadDTO(PatientBaseDTO):
     # pylint: disable=too-many-instance-attributes
     medical_id: str
     blood_group: BloodGroup
@@ -21,13 +22,3 @@ class DonorUploadDTO:
     year_of_birth: Optional[int] = None
     note: str = ''
     internal_medical_id: Optional[str] = None
-
-    def __post_init__(self):
-        if self.height:
-            is_height_valid("donor", self.height)
-
-        if self.weight:
-            is_weight_valid("donor", self.weight)
-
-        if self.year_of_birth:
-            is_year_of_birth_valid("donor", self.year_of_birth)

txmatching/data_transfer_objects/patients/upload_dtos/recipient_upload_dto.py

@@ -1,41 +1,23 @@
 from dataclasses import dataclass
 from typing import List, Optional
 
+from txmatching.data_transfer_objects.patients.patient_base_dto import (
+    PatientBaseDTO, RecipientBaseDTO)
 from txmatching.data_transfer_objects.patients.upload_dtos.hla_antibodies_upload_dto import \
     HLAAntibodiesUploadDTO
-from txmatching.patients.patient import (is_height_valid, is_number_of_previous_transplants_valid,
-    is_weight_valid, is_year_of_birth_valid)
-from txmatching.patients.patient_parameters import Centimeters, Kilograms
 from txmatching.utils.blood_groups import BloodGroup
 from txmatching.utils.enums import Sex
 
 
-@dataclass
-class RecipientUploadDTO:
+@dataclass(kw_only=True)
+class RecipientUploadDTO(PatientBaseDTO, RecipientBaseDTO):
     # pylint: disable=too-many-instance-attributes
     acceptable_blood_groups: Optional[List[BloodGroup]]
     medical_id: str
     blood_group: BloodGroup
     hla_typing: List[str]
     hla_antibodies: List[HLAAntibodiesUploadDTO]
     sex: Optional[Sex] = None
-    height: Optional[Centimeters] = None
-    weight: Optional[Kilograms] = None
-    year_of_birth: Optional[int] = None
     note: str = ''
     waiting_since: Optional[str] = None
-    previous_transplants: Optional[int] = None
     internal_medical_id: Optional[str] = None
-
-    def __post_init__(self):
-        if self.height:
-            is_height_valid("recipient", self.height)
-
-        if self.weight:
-            is_weight_valid("recipient", self.weight)
-
-        if self.year_of_birth:
-            is_year_of_birth_valid("recipient", self.year_of_birth)
-
-        if self.previous_transplants:
-            is_number_of_previous_transplants_valid(self.previous_transplants)

txmatching/data_transfer_objects/patients/utils.py

@@ -14,7 +14,8 @@
 
 logger = logging.getLogger(__name__)
 
-
+# is needed here because kw_args in dataclass is not handled well by pylint
+# pylint: disable=unexpected-keyword-arg
 def parsing_issue_to_dto(parsing_issue: ParsingIssue, txm_event: TxmEventBase) -> ParsingIssuePublicDTO:
     return ParsingIssuePublicDTO(
         hla_code_or_group=parsing_issue.hla_code_or_group,

txmatching/local_config.py

@@ -9,7 +9,7 @@
 USE_2FA = 'FALSE'
 ENVIRONMENT = 'DEVELOPMENT'
 COLOUR_SCHEME = 'IKEM'
-
-AUTHENTIC_CLIENT_ID='f5c6b6a72ff4f7bbdde383a26bdac192b2200707'
-AUTHENTIC_CLIENT_SECRET='37e841e70b842a0d1237b3f7753b5d7461307562568b5add7edcfa6630d578fdffb7ff4d5c0f845d10f8f82bc1d80cec62cb397fd48795a5b1bee6090e0fa409'
-AUTHENTIC_REDIRECT_URI="http://localhost:8080/v1/user/authentik-login"
+# pylint: disable=line-too-long
+AUTHENTIC_CLIENT_ID = 'f5c6b6a72ff4f7bbdde383a26bdac192b2200707'
+AUTHENTIC_CLIENT_SECRET = '37e841e70b842a0d1237b3f7753b5d7461307562568b5add7edcfa6630d578fdffb7ff4d5c0f845d10f8f82bc1d80cec62cb397fd48795a5b1bee6090e0fa409'
+AUTHENTIC_REDIRECT_URI = 'http://localhost:8080/v1/user/authentik-login'

txmatching/patients/hla_code.py

@@ -1,17 +1,17 @@
 import re
-
 from dataclasses import dataclass
 from typing import Optional
 
-from txmatching.utils.enums import GENE_HLA_GROUPS_WITH_OTHER_DETAILED, HLA_GROUPS_PROPERTIES, HLAGroup
+from txmatching.utils.enums import (GENE_HLA_GROUPS_WITH_OTHER_DETAILED,
+                                    HLA_GROUPS_PROPERTIES, HLAGroup)
 
 
 @dataclass
 class HLACode:
     high_res: Optional[str]
     split: Optional[str]
     broad: Optional[str]
-    group: Optional[HLAGroup]
+    group: Optional[HLAGroup] = None
 
     @property
     def display_code(self) -> str:
@@ -24,12 +24,15 @@ def display_code(self) -> str:
         else:
             raise AssertionError('This should never happen. At least one code should be specified.')
 
-    def __init__(self, high_res: Optional[str], split: Optional[str], broad: Optional[str], group = None):
+    def __init__(self, high_res: Optional[str], split: Optional[str], broad: Optional[str], group: HLAGroup = None):
         assert high_res is not None or broad is not None
         self.high_res = high_res
         self.split = split
         self.broad = broad
-        self.group = self.group_from_hla_code
+        if group:
+            self.group = group
+        else:
+            self.group = self.group_from_hla_code
 
     def __repr__(self):
         return f'HLACode({repr(self.high_res)}, {repr(self.split)}, {repr(self.broad)})'