👌️ IMPROVE: Stable Developement Branch Improvements (#18)

* 📖 DOC: make file comments

* 👌 IMPROVE: change deploy to version

* 🐛 FIX: docker build will only work off release branch

* Revert "👌 IMPROVE: change deploy to version"

This reverts commit 487564e.

* 📦 NEW: add pre-commit and other configs

* ❗️ BREAKING: remove the common directory in favor of submodule

* 📦 NEW: added common submodule

* 👌 IMPROVE: add cmd for submodule updating

* 📖 DOC: working on updating readme

* 🧪️ TEST: test code coverage action

* 🐛 FIX: put python function back to 3.9

* 🧪️ TEST: change way poetry is installed

* 🧪️ TEST: correct poetry install cmd

* 🐛 FIX: add pytest-cov dependency

* 🧪️ TEST: finished testing workflow test

* 📖 DOC: change README

* 🐛 FIX: fix doc issues from linter

* 🐛 FIX: appease the allmighty linter

* 👌 IMPROVE: work towards more modularity

* 🐛 FIX: forgot to change the template text portions

---------

Signed-off-by: Bryan Jenks <bryan@bryanjenks.dev>

.github/ISSUE_TEMPLATE/config.yml

@@ -4,5 +4,5 @@ contact_links:
     url: https://github.com/sponsors/<GH USERNAME>
     about: Help support continued development on this project
   - name: 👨🏻‍💻 More about the maintainer
-    url: <WEBSITE>
-    about: get to know the maintainer of <REPO NAME>
+    url: https://www.bryanjenks.dev
+    about: get to know the maintainer of etl_strong_app

.github/PULL_REQUEST_TEMPLATE.md

@@ -8,7 +8,7 @@ Create a pull request to help us improve. See [CONTRIBUTING](/CONTRIBUTING.md).
 
 ---
 
-- [ ] I have communicated my intent to create this pull request via repo [Discussions](https://github.com/<REPO NAME>/discussions) or [Issues](https://github.com/<REPO NAME>/issues) before undertaking this work
+- [ ] I have communicated my intent to create this pull request via repo [Discussions](https://github.com/etl_strong_app/discussions) or [Issues](https://github.com/etl_strong_app/issues) before undertaking this work
 - [ ] I have read [CONTRIBUTING](/CONTRIBUTING.md)
 - [ ] My commits are granular and descriptive
 - [ ] I have used [Emoji Log](https://github.com/ahmadawais/Emoji-Log) for my commit messages and pull request title

.pre-commit-config.yaml

@@ -1,14 +1,14 @@
 repos:
-  - repo: https://github.com/asottile/seed-isort-config
-    rev: v2.2.0
-    hooks:
-    - id: seed-isort-config
-      log_file: logs/hooks.log
-  - repo: https://github.com/pre-commit/mirrors-isort
-    rev: v5.10.1
-    hooks:
-    - id: isort
-      log_file: logs/hooks.log
+  # - repo: https://github.com/asottile/seed-isort-config
+  #   rev: v2.2.0
+  #   hooks:
+  #   - id: seed-isort-config
+  #     log_file: logs/hooks.log
+  # - repo: https://github.com/pre-commit/mirrors-isort
+  #   rev: v5.10.1
+  #   hooks:
+  #   - id: isort
+  #     log_file: logs/hooks.log
   - repo: https://github.com/psf/black
     rev: 23.3.0
     hooks:

src/common.py

@@ -23,16 +23,16 @@ def collect_deltas(df: pd.DataFrame, watermark: dt.datetime) -> pd.DataFrame:
     return filtered_df
 
 
-def generate_db_connection() -> sessionmaker.object_session:
-    engine = create_engine(Settings().db_connection_string)
+def generate_db_connection(connection_string) -> sessionmaker.object_session:
+    engine = create_engine(connection_string)
     Session = sessionmaker(bind=engine)
     session = Session()
     return session
 
 
-def get_watermark() -> dt.datetime:
-    query = text("SELECT MAX(date) FROM {};".format(Settings().target_table))
-    session = generate_db_connection()
+def get_watermark(target_table, connection_string) -> dt.datetime:
+    query = text("SELECT MAX(date) FROM {};".format(target_table))
+    session = generate_db_connection(connection_string)
     result = session.execute(query).scalar()
     watermark = pd.Timestamp("1900-01-01") if result is None else result
     return watermark

src/database.py

@@ -1,8 +1,7 @@
+from config import Settings
 from sqlalchemy import DECIMAL, Column, DateTime, Integer, String, create_engine
 from sqlalchemy.ext.declarative import declarative_base
 
-from config import Settings
-
 Base = declarative_base()
 
 

src/file_watcher.py

@@ -3,6 +3,8 @@
 from datetime import datetime as dt
 
 import pandas as pd
+from config import Settings
+from database import WorkoutData
 from loguru import logger as log
 from watchdog.events import FileSystemEventHandler
 
@@ -15,8 +17,6 @@
     get_watermark,
     load_data_into_dataframe,
 )
-from config import Settings
-from database import WorkoutData
 
 
 class MonitorFolder(FileSystemEventHandler):
@@ -47,7 +47,9 @@ def process_data() -> pd.DataFrame:
             try:
                 processed_data = collect_deltas(
                     load_data_into_dataframe(get_staged_file_full_path()),
-                    get_watermark(),
+                    get_watermark(
+                        Settings().target_table, Settings().db_connection_string
+                    ),
                 )
                 return processed_data
             except Exception as e:

src/main.py

@@ -1,5 +1,4 @@
 from dotenv import load_dotenv
-
 from file_watcher import MonitorFolder
 
 load_dotenv()  # Load environment variables from .env file or docker environmental variables