Facade timezone fix redux

augur/application/db/lib.py

@@ -4,7 +4,7 @@
 import logging
 import sqlalchemy as s
 from sqlalchemy import func 
 from sqlalchemy.exc import DataError
 from sqlalchemy.dialects import postgresql
 from sqlalchemy.exc import OperationalError
 from psycopg2.errors import DeadlockDetected
@@ -167,7 +167,7 @@
 
     try:
         working_commits = fetchall_data_from_sql_text(query)
     except:
         working_commits = []
 
     return working_commits
@@ -183,7 +183,7 @@
 
     try:
         missing_commit_hashes = fetchall_data_from_sql_text(fetch_missing_hashes_sql)
     except:
         missing_commit_hashes = []
 
     return missing_commit_hashes
@@ -203,7 +203,7 @@
         return session.query(CollectionStatus).filter(getattr(CollectionStatus,f"{collection_type}_status" ) == CollectionState.COLLECTING.value).count()
 
 
 def facade_bulk_insert_commits(logger, records):
 
     with get_session() as session:
 
@@ -225,24 +225,46 @@
 
                 facade_bulk_insert_commits(logger, firsthalfRecords)
                 facade_bulk_insert_commits(logger, secondhalfRecords)
-            elif len(records) == 1 and isinstance(e,DataError) and "time zone displacement" in f"{e}":
+            elif len(records) == 1:
                 commit_record = records[0]
                 #replace incomprehensible dates with epoch.
                 #2021-10-11 11:57:46 -0500
 
                 # placeholder_date = "1970-01-01 00:00:15 -0500"
-                placeholder_date = commit_record['author_timestamp']
+                placeholder_date = commit_record['cmt_author_timestamp']
+
+                postgres_valid_timezones = {
+                    -1200, -1100, -1000, -930, -900, -800, -700,
+                    -600, -500, -400, -300, -230, -200, -100, 000,
+                    100, 200, 300, 330, 400, 430, 500, 530, 545, 600,
+                    630, 700, 800, 845, 900, 930, 1000, 1030, 1100, 1200,
+                    1245, 1300, 1400
+                }
 
                 # Reconstruct timezone portion of the date string to UTC
-                placeholder_date = re.split("[-+]", placeholder_date)
-                placeholder_date.pop()
-                placeholder_date = "-".join(placeholder_date) + "+0000"
+                placeholder_date_segments = re.split(" ", placeholder_date)
+                tzdata = placeholder_date_segments.pop()
+
+                if ":" in tzdata:
+                    tzdata = tzdata.replace(":", "")
+
+                if int(tzdata) not in postgres_valid_timezones:
+                    tzdata = "+0000"
+                else:
+                    raise e
+
+                placeholder_date_segments.append(tzdata)
+
+                placeholder_date = " ".join(placeholder_date_segments)
 
                 #Check for improper utc timezone offset
                 #UTC timezone offset should be between -14:00 and +14:00
 
-                commit_record['author_timestamp'] = placeholder_date
-                commit_record['committer_timestamp'] = placeholder_date
+                # analyzecommit.generate_commit_record() defines the keys on the commit_record dictionary
+                commit_record['cmt_author_timestamp'] = placeholder_date
+                commit_record['cmt_committer_timestamp'] = placeholder_date
+
+                logger.warning(f"commit with invalid timezone set to UTC: {commit_record['cmt_commit_hash']}")
 
                 session.execute(
                     s.insert(Commit),
@@ -253,7 +275,7 @@
                 raise e
 
 
 def batch_insert_contributors(logger, data: Union[List[dict], dict]) -> Optional[List[dict]]:
 
     batch_size = 1000
 
@@ -264,7 +286,7 @@
 
 
 
 def bulk_insert_dicts(logger, data: Union[List[dict], dict], table, natural_keys: List[str], return_columns: Optional[List[str]] = None, string_fields: Optional[List[str]] = None, on_conflict_update:bool = True) -> Optional[List[dict]]:
 
     if isinstance(data, list) is False: