Project, event validation and protobuf update (#823)

* Project, event validation and protobuf update

* Site and project search when creating candidate

* missing import

* correct pscid search

* Events validation

* flake rules update

* review, and NULL value filtered out for site and project

* flake

* flake

Co-authored-by: regisoc <regis.ongaro-carcy@mcin.ca>

Author: laemtl

.github/workflows/flake8_python_linter.yml

@@ -35,6 +35,8 @@ jobs:
           #     E241: Multiple spaces after ','
           #     E251: Unexpected spaces around keyword / parameter equals
           #     E272: Multiple spaces before keyword
-          flake8_args: "--ignore=W291,W292,W293,E202,E203,E221,E241,E251,E272 --max-line-length 120"
+          flake8_args: "--ignore=W291,W292,W293,W503,E202,E203,E221,E241,E251,E272
+                        --max-line-length 120
+                        --exclude python/react-series-data-viewer/protocol_buffers/"
           github_token: ${{ secrets.GITHUB_TOKEN }}
 

python/lib/candidate.py

@@ -83,53 +83,87 @@ def create_candidate(self, db, participants_info):
             if 'age' in row:
                 self.age = row['age']
 
-            if 'site' in row:
+            # three steps to find site:
+            #   1. try matching full name from 'site' column in participants.tsv in db
+            #   2. try extracting alias from pscid
+            #   3. try finding previous site in candidate table
+
+            if 'site' in row and row['site'].lower() not in ("null", ""):
+                # search site id in psc table by its full name
                 site_info = db.pselect(
                     "SELECT CenterID FROM psc WHERE Name = %s",
                     [row['site'], ]
                 )
-                if(len(site_info) > 0):
+                if len(site_info) > 0:
                     self.center_id = site_info[0]['CenterID']
-            else:
+
+            if self.center_id is None:
+                # search site id in psc table by its alias extracted from pscid
                 db_sites = db.pselect("SELECT CenterID, Alias FROM psc")
                 for site in db_sites:
                     if site['Alias'] in row['participant_id']:
                         self.center_id = site['CenterID']
 
-            if 'project' in row:
+            if self.center_id is None:
+                # try to find participant site in db
+                candidate_site_project = db.pselect(
+                    "SELECT RegistrationCenterID FROM candidate WHERE pscid = %s",
+                    [self.psc_id, ]
+                )
+                if len(candidate_site_project) > 0:
+                    self.center_id = candidate_site_project[0]['RegistrationCenterID']
+
+            # two steps to find project:
+            #   1. find full name in 'project' column in participants.tsv
+            #   2. find previous in candidate table
+
+            if 'project' in row and row['project'].lower() not in ("null", ""):
+                # search project id in Project table by its full name
                 project_info = db.pselect(
                     "SELECT ProjectID FROM Project WHERE Name = %s",
                     [row['project'], ]
                 )
-                if(len(project_info) > 0):
+                if len(project_info) > 0:
                     self.project_id = project_info[0]['ProjectID']
 
+            if self.project_id is None:
+                # try to find participant project
+                candidate_site_project = db.pselect(
+                    "SELECT RegistrationProjectID FROM candidate WHERE pscid = %s",
+                    [self.psc_id, ]
+                )
+                if len(candidate_site_project) > 0:
+                    self.center_id = candidate_site_project[0]['RegistrationProjectID']
+
         if not self.center_id:
             print("ERROR: could not determine site for " + self.psc_id + "."
                   + " Please check that your psc table contains a site with an"
                   + " alias matching the BIDS participant_id or a name matching the site mentioned in"
                   + " participants.tsv's site column")
             sys.exit(lib.exitcode.PROJECT_CUSTOMIZATION_FAILURE)
 
+        if not self.project_id:
+            print("ERROR: could not determine project for " + self.psc_id + "."
+                  + " Please check that your project table contains a project with a"
+                  + " name matching the participants.tsv's project column")
+            sys.exit(lib.exitcode.PROJECT_CUSTOMIZATION_FAILURE)
+
         if self.verbose:
             print("Creating candidate with \n"
                   + "PSCID     = " + self.psc_id + ",\n"
                   + "CandID    = " + str(self.cand_id) + ",\n"
                   + "CenterID  = " + str(self.center_id)  + ",\n"
                   + "ProjectID = " + str(self.project_id))
 
-        insert_col = ('PSCID', 'CandID', 'RegistrationCenterID')
-        insert_val = (self.psc_id, str(self.cand_id), str(self.center_id))
+        insert_col = ('PSCID', 'CandID', 'RegistrationCenterID', 'RegistrationProjectID')
+        insert_val = (self.psc_id, str(self.cand_id), str(self.center_id), str(self.project_id))
 
         if self.sex:
             insert_col = insert_col + ('Sex',)
             insert_val = insert_val + (self.sex,)
         if self.dob:
             insert_col = insert_col + ('DoB',)
             insert_val = insert_val + (self.dob,)
-        if self.project_id:
-            insert_col = insert_col + ('RegistrationProjectID',)
-            insert_val = insert_val + (str(self.project_id),)
 
         db.insert(
             table_name='candidate',

python/lib/physiological.py

@@ -535,13 +535,12 @@ def insert_event_file(self, event_data, event_file, physiological_file_id,
                 if field not in row.keys():
                     row[field] = None
 
-            # TODO: remove the following if once received confirmation from
-            # TODO: James it was an error.
-            if "NaN" in row['duration']:
-                row['duration'] = 0
+            duration = 0
+            if (type(row['duration']) == int or type(row['duration']) == float):
+                duration = row['duration']
 
             sample = None
-            if row['event_sample'] and (type(row['event_sample']) == int or type(row['event_sample']) == float):
+            if type(row['event_sample']) == int or type(row['event_sample']) == float:
                 sample = row['event_sample']
             if row['sample'] and (type(row['sample']) == int or type(row['sample']) == float):
                 sample = row['sample']
@@ -552,12 +551,16 @@ def insert_event_file(self, event_data, event_file, physiological_file_id,
             elif row['value']:
                 event_value = str(row['value'])
 
+            response_time = None
+            if type(row['response_time']) == int or type(row['response_time']) == float:
+                response_time = row['response_time']
+
             values_tuple = (
                 str(physiological_file_id),
                 row['onset'],
-                row['duration'],
+                duration,
                 row['trial_type'],
-                row['response_time'],
+                response_time,
                 row['event_code'],
                 event_value,
                 sample,

python/react-series-data-viewer/chunking.py

@@ -7,10 +7,7 @@
 from scipy import signal
 import sys
 
-try:
-    from .protocol_buffers import chunk_pb2 as chunk_pb
-except Exception:
-    from protocol_buffers import chunk_pb2 as chunk_pb
+from protocol_buffers import chunk_pb2 as chunk_pb
 
 
 def pad_channels(channels, chunk_size):