👌️IMPROVE: System Overhaul

Bryan-Jenks-Home-Lab · Sep 1, 2023 · fa5eb6f · fa5eb6f
2 parents cb0fbda + 701990f
commit fa5eb6f
Show file tree

Hide file tree

Showing 25 changed files with 474 additions and 320 deletions.
diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md
@@ -8,7 +8,7 @@ Create a pull request to help us improve. See [CONTRIBUTING](/CONTRIBUTING.md).
 
 ---
 
-- [ ] I have communicated my intent to create this pull request via repo [Discussions](https://github.com/watchr/discussions) or [Issues](https://github.com/watchr/issues) before undertaking this work
+- [ ] I have communicated my intent to create this pull request via repo [Discussions](https://github.com/etl_strong_app/discussions) or [Issues](https://github.com/etl_strong_app/issues) before undertaking this work
 - [ ] I have read [CONTRIBUTING](/CONTRIBUTING.md)
 - [ ] My commits are granular and descriptive
 - [ ] I have used [Emoji Log](https://github.com/ahmadawais/Emoji-Log) for my commit messages and pull request title

diff --git a/.gitignore b/.gitignore
@@ -10,3 +10,5 @@ logs/
 # temporary ignores
 test-and-build-azure-pipeline.yml
 TODO.md
+output.csv
+eda.ipynb
diff --git a/.gitmodules b/.gitmodules
@@ -1,3 +1,6 @@
 [submodule "common"]
 	path = common
 	url = https://github.com/Bryan-Jenks-Home-Lab/common
+[submodule "src/common_python"]
+	path = src/common_python
+	url = https://github.com/Bryan-Jenks-Home-Lab/common_python
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -96,10 +96,10 @@ repos:
     rev: 2.7.1
     hooks:
     - id: editorconfig-checker
-  - repo: https://github.com/econchick/interrogate
-    rev: 1.5.0
-    hooks:
-    - id: interrogate
+  # - repo: https://github.com/econchick/interrogate
+  #   rev: 1.5.0
+  #   hooks:
+  #   - id: interrogate
   - repo: https://github.com/asottile/blacken-docs
     rev: 1.13.0
     hooks:

diff --git a/Dockerfile b/Dockerfile
@@ -12,4 +12,4 @@ COPY . .
 # Install python dependencies
 RUN pip install --no-cache-dir -r requirements.txt
 
-CMD ["python", "src/main.py"]
+CMD ["python", "src"]
diff --git a/README.md b/README.md
@@ -1,11 +1,11 @@
-# watchr
+# Watchr
 
 <!-- markdownlint-disable MD033-->
 <!-- Header & Preview Image -->
 
 <h1 align="center">
-  <a href="https://www.strong.app/" target="_blank">
-    <img src="images/strongapp_banner.png" height="50%" width="50%">
+  <a href="https://www.apple.com/ios/health/" target="_blank">
+    <img src="images/applehealth.jpeg" height="50%" width="50%">
   </a>
 </h1>
 
@@ -38,17 +38,19 @@
 
 <!-- Description -->
 
-> <++>
+> `Watchr` Is a file watching service for importing records into a Postgres database.
 
-<++>
+Watchr is organized through the usage of Pydantic models to govern the behavior of how a file get processed, the actions around its artifact cleanup, and other configuration.
+
+This project is still rough and narrowly focused for my own personal use cases at the moment but in the future i may put forth more intention in making this more general and accessible to others.
 
 ---
 
 ## Table of Contents
 
 ---
 
-- [watchr](#watchr)
+- [Watchr](#watchr)
   - [Table of Contents](#table-of-contents)
   - [Features](#features)
   - [Requirements](#requirements)
@@ -148,7 +150,27 @@ See [CONTRIBUTING](CONTRIBUTING.md)
 
 [Return To Top](#table-of-contents)
 
-<++>
+Pull image from ghcr.io
+
+```sh
+docker pull ghcr.io/bryan-jenks-home-lab/etl_strong_app:main
+```
+
+Run image
+
+```sh
+docker run -d \
+    -e DB_CONNECTION_STRING='postgresql+psycopg2://<USER>:<PASSWORD>@<SERVER>:<PORT>/<DATABASE>' \
+    -e EXPECTED_FILE=strong.csv \
+    -e TARGET_TABLE=<DATABASE>.<SCHEMA>.<TABLE_NAME> \
+    -e WATCH_PATH=/data/inbound \
+    -e STAGING_PATH=/data/outbound \
+    -e PROCESSED_PATH=/data/processed \
+    -v /<PATH_ON_YOUR_MACHINE>/strong_app_import/inbound:/data/inbound \
+    -v /<PATH_ON_YOUR_MACHINE>/strong_app_import/outbound:/data/outbound \
+    -v /<PATH_ON_YOUR_MACHINE>/strong_app_import/processed:/data/processed \
+    ghcr.io/bryan-jenks-home-lab/etl_strong_app:main
+```
 
 ---
 
@@ -178,7 +200,9 @@ See [CONTRIBUTING](CONTRIBUTING.md)
 
 [Return To Top](#table-of-contents)
 
-<++>
+1. `python -m pip install poetry python-dotenv pre-commit`
+1. `pre-commit install`
+1. `pre-commit autoupdate`
 
 See [Road Map](ROADMAP.md)
 

diff --git a/READMEold.md b/READMEold.md
diff --git a/common b/common
diff --git a/docker-compose.yaml b/docker-compose.yaml
@@ -1,14 +1,13 @@
 version: "3.7"
 services:
-  data_importer:
+  watchr:
     image: tallguyjenks/watchr:latest
     container_name: watchr
     restart: unless-stopped
     networks:
-      - proxy
-    volumes:
-      - /mnt/sql:/mnt/sql
-      - /mnt/homelab:/mnt/homelab
+      proxy:
+        ipv4_address: 1.1.1.1
+    volumes: ["/mnt/homelab:/mnt/homelab"]
 
 networks:
   proxy:

diff --git a/images/applehealth.jpeg b/images/applehealth.jpeg
diff --git a/images/interrogate_badge.svg b/images/interrogate_badge.svg
diff --git a/images/strongapp_banner.png b/images/strongapp_banner.png
diff --git a/makefile b/makefile
@@ -45,11 +45,9 @@ format:
 	black ./src
 run:
 	@echo "\n🏃🏻‍♂️️ ${GREEN}Running Application${NC}\n"
-	poetry run python3 src/main.py
+	poetry run python3 src
 test:
 	@echo "\n🧪️ ${GREEN}Running Test Suite${NC}\n"
-	pre-commit install
-	pre-commit autoupdate
 	poetry run python3 -m pytest
 	@echo "\n🧪️ ${GREEN}Cleaning Up Test Suite Artifacts${NC}\n"
 	@$(MAKE) clean
@@ -59,6 +57,8 @@ clean:
 	rm -rf **/.pytest_cache
 	rm -rf .pytest_cache
 	rm -rf **/__pycache__
+	rm -rf **/common_python/__pycache__
+	rm -rf **/apple_health/__pycache__
 	@echo "\n🧹️ ${GREEN}Cleaned${NC}\n"
 build:
 	@echo "\n🛠️ ${GREEN}Beginning Build Process${NC}\n"

diff --git a/pyproject.toml b/pyproject.toml
@@ -1,7 +1,7 @@
 [tool.poetry]
 name = "watchr"
-version = "0.1.3"
-description = "Picking up and processing files for personal finances and accounting"
+version = "0.2.3"
+description = "Picking up and processing files"
 authors = ["Bryan Jenks <bryan@bryanjenks.dev>"]
 
 [tool.poetry.dependencies]
@@ -16,6 +16,8 @@ pydantic = "^1.10.7"
 alembic = "^1.10.4"
 uvicorn = "^0.22.0"
 python-dotenv = "^1.0.0"
+pretty-errors = "^1.2.25"
+requests = "^2.31.0"
 
 [tool.poetry.dev-dependencies]
 pytest = "^7.3.1"

diff --git a/src/main.py → src/__main__.py b/src/main.py → src/__main__.py
@@ -1,7 +1,17 @@
+import sys
+
+import pretty_errors  # NOQA: F401
+from config import Settings
 from dotenv import load_dotenv
 from file_watcher import MonitorFolder
+from loguru import logger as log
 
-load_dotenv()  # Load environment variables from .env file or docker environmental variables
+# Load environment variables from .env file or docker environmental variables
+load_dotenv()
+# Remove the default logger configuration
+log.remove()
+# Add a new logger configuration with the desired log level
+log.add(sink=sys.stdout, level=Settings().log_level)
 
 
 def main():

diff --git a/src/apple_health/__init__.py b/src/apple_health/__init__.py