👌 IMPROVE: work towards more modularity

.pre-commit-config.yaml

@@ -1,14 +1,14 @@
 repos:
-  - repo: https://github.com/asottile/seed-isort-config
-    rev: v2.2.0
-    hooks:
-    - id: seed-isort-config
-      log_file: logs/hooks.log
-  - repo: https://github.com/pre-commit/mirrors-isort
-    rev: v5.10.1
-    hooks:
-    - id: isort
-      log_file: logs/hooks.log
+  # - repo: https://github.com/asottile/seed-isort-config
+  #   rev: v2.2.0
+  #   hooks:
+  #   - id: seed-isort-config
+  #     log_file: logs/hooks.log
+  # - repo: https://github.com/pre-commit/mirrors-isort
+  #   rev: v5.10.1
+  #   hooks:
+  #   - id: isort
+  #     log_file: logs/hooks.log
   - repo: https://github.com/psf/black
     rev: 23.3.0
     hooks:

src/common.py

@@ -23,16 +23,16 @@ def collect_deltas(df: pd.DataFrame, watermark: dt.datetime) -> pd.DataFrame:
     return filtered_df
 
 
-def generate_db_connection() -> sessionmaker.object_session:
-    engine = create_engine(Settings().db_connection_string)
+def generate_db_connection(connection_string) -> sessionmaker.object_session:
+    engine = create_engine(connection_string)
     Session = sessionmaker(bind=engine)
     session = Session()
     return session
 
 
-def get_watermark() -> dt.datetime:
-    query = text("SELECT MAX(date) FROM {};".format(Settings().target_table))
-    session = generate_db_connection()
+def get_watermark(target_table, connection_string) -> dt.datetime:
+    query = text("SELECT MAX(date) FROM {};".format(target_table))
+    session = generate_db_connection(connection_string)
     result = session.execute(query).scalar()
     watermark = pd.Timestamp("1900-01-01") if result is None else result
     return watermark

src/database.py

@@ -1,8 +1,7 @@
+from config import Settings
 from sqlalchemy import DECIMAL, Column, DateTime, Integer, String, create_engine
 from sqlalchemy.ext.declarative import declarative_base
 
-from config import Settings
-
 Base = declarative_base()
 
 

src/file_watcher.py

@@ -3,6 +3,8 @@
 from datetime import datetime as dt
 
 import pandas as pd
+from config import Settings
+from database import WorkoutData
 from loguru import logger as log
 from watchdog.events import FileSystemEventHandler
 
@@ -15,8 +17,6 @@
     get_watermark,
     load_data_into_dataframe,
 )
-from config import Settings
-from database import WorkoutData
 
 
 class MonitorFolder(FileSystemEventHandler):
@@ -47,7 +47,9 @@ def process_data() -> pd.DataFrame:
             try:
                 processed_data = collect_deltas(
                     load_data_into_dataframe(get_staged_file_full_path()),
-                    get_watermark(),
+                    get_watermark(
+                        Settings().target_table, Settings().db_connection_string
+                    ),
                 )
                 return processed_data
             except Exception as e:

src/main.py

@@ -1,5 +1,4 @@
 from dotenv import load_dotenv
-
 from file_watcher import MonitorFolder
 
 load_dotenv()  # Load environment variables from .env file or docker environmental variables