Merge branch 'main' into oidc

Author: MikeWent

.github/workflows/ci.yml

@@ -26,7 +26,7 @@ jobs:
           make ci-test
       - name: Stop and remove containers
         run: |
-          make down
+          make ci-down
 
   docker:
     name: Docker Build

.gitignore

@@ -3,4 +3,5 @@ __pycache__
 *.db
 .pytest_cache
 .venv
-secrets.env
+secrets.prod.env
+secrets.dev.env

.pre-commit-config.yaml

@@ -0,0 +1,11 @@
+repos:
+  - repo: https://github.com/PyCQA/isort
+    rev: 6.0.1
+    hooks:
+      - id: isort
+        args: ["--profile=black"]
+  - repo: https://github.com/psf/black
+    rev: 25.1.0
+    hooks:
+      - id: black
+        language_version: python3.12

Makefile

@@ -30,9 +30,12 @@ ci-daemon:
 ci-test:
 	docker compose -f $(COMPOSE_BASE) -f $(CI_COMPOSE) exec api pytest -v -s --log-level DEBUG
 
+ci-down:
+	docker compose -f $(COMPOSE_BASE) -f $(CI_COMPOSE) down
+
 # Stops all running services
 down:
-	docker compose down
+	docker compose -f $(COMPOSE_BASE) -f $(DEV_COMPOSE) -f $(PROD_COMPOSE) down
 
 test: dev-daemon
 	docker compose -f $(COMPOSE_BASE) -f $(DEV_COMPOSE) exec api pytest

README.md

@@ -73,6 +73,12 @@ uv remove packagename
 uv sync
 ```
 
+install pre-commit hook
+```
+source ./.venv/bin/activate
+pre-commit install
+```
+
 ### tests
 ```
 make test
@@ -108,14 +114,15 @@ open http://localhost:8000/docs and http://localhost:9000
 - [x] add split participants by a tag
 - [x] ~~grafana~~, statistics
 - [x] treasuries
-- [ ] logging
-- [ ] permissions?
-- [ ] misc validation of amounts (>0.00)
-- [ ] improve split ux
-- [ ] pass tags as a list, not as add/delete operations
-- [ ] migrations
+- [x] logging
+- [x] postgres
 
 ## todo techdebt
+- [ ] migrations
+- [x] pass tags as a list, not as add/delete operations
+    - [ ] fix ui tag management
+- [ ] misc validation of amounts (>0.00)
+- [ ] improve split ux
 - [ ] make a uniform deposit api CRUD, provider should be enum
 - [ ] update all boolean attrs to status enums
 - [ ] mobile ui
@@ -124,6 +131,7 @@ open http://localhost:8000/docs and http://localhost:9000
 - [ ] remove base service class
 
 ## todo future features
+- [ ] permissions?
 - [ ] deposit ui
 - [ ] donation categories (entities?)
 - [ ] easy payment urls

api/app/app.py

@@ -1,24 +1,26 @@
 """FastAPI app initialization, exception handling"""
 
+import json
 import logging
 import traceback
 
 import uvicorn
 from app.config import Config, get_config
 from app.errors.base import ApplicationError
+from app.errors.token import TokenInvalid
 from app.routes.balance import balance_router
 from app.routes.currency_exchange import currency_exchange_router
 from app.routes.deposit_provider_callbacks import deposit_provider_callbacks_router
 from app.routes.deposits import deposits_router
 from app.routes.entity import entity_router
+from app.routes.oidc import router as oidc_router
 from app.routes.resident_fee import router as resident_fee_router
 from app.routes.split import split_router
 from app.routes.stats import router as stats_router
 from app.routes.tag import tag_router
 from app.routes.token import token_router
 from app.routes.transaction import transaction_router
 from app.routes.treasury import treasury_router
-from app.routes.oidc import router as oidc_router
 from fastapi import FastAPI, Request
 from fastapi.exceptions import ResponseValidationError
 from fastapi.middleware.cors import CORSMiddleware
@@ -42,7 +44,9 @@
     secret_key=config.secret_key,
 )
 if not config.secret_key:
-    raise ValueError("SECRET_KEY is missing in the configuration. Please set a valid secret key.")
+    raise ValueError(
+        "SECRET_KEY is missing in the configuration. Please set a valid secret key."
+    )
 
 
 @app.exception_handler(ResponseValidationError)
@@ -60,24 +64,31 @@ async def response_validation_exception_handler(
 
 @app.exception_handler(ApplicationError)
 def application_exception_handler(request: Request, exc: ApplicationError):
-    traceback.print_exception(exc)
+    c = {
+        "error_code": exc.error_code,
+        "error": exc.error,
+        "where": exc.where,
+    }
+    logger.error(c)
+    # Only print full traceback when in debug logging
+    if logger.isEnabledFor(logging.DEBUG):
+        traceback.print_exception(exc)
     e = JSONResponse(
         status_code=exc.http_code or 418,
-        content={
-            "error_code": exc.error_code,
-            "error": exc.error,
-            "where": exc.where,
-        },
+        content=c,
     )
     return e
 
 
 @app.exception_handler(SQLAlchemyError)
 def sqlite_exception_handler(request: Request, exc: SQLAlchemyError):
-    traceback.print_exception(exc)
+    logger.error(exc)
+    # Only print full traceback when in debug logging
+    if logger.isEnabledFor(logging.DEBUG):
+        traceback.print_exception(exc)
     e = JSONResponse(
         status_code=418,
-        content={"error_code": 4000, "error": exc._message()},
+        content={"error_code": 1500, "error": exc._message()},
     )
     return e
 
@@ -97,4 +108,4 @@ def sqlite_exception_handler(request: Request, exc: SQLAlchemyError):
 app.include_router(oidc_router)
 
 if __name__ == "__main__":
-    uvicorn.run(app, host="0.0.0.0", port=8000)
+    uvicorn.run(app, host="0.0.0.0", port=8000, log_level="info")

api/app/config.py

@@ -24,6 +24,8 @@ class Config:
     cryptapi_address_trc20_usdt: str | None = field(
         default=getenv("REFINANCE_CRYPTAPI_ADDRESS_TRC20_USDT", "")
     )
+    # Optional database URL for Postgres or other databases
+    database_url_env: str | None = field(default=getenv("REFINANCE_DATABASE_URL", None))
 
     # OIDC configuration
     oidc_client_id: str | None = field(default=getenv("REFINANCE_OIDC_CLIENT_ID", ""))
@@ -43,6 +45,9 @@ def database_path(self) -> Path:
 
     @property
     def database_url(self) -> str:
+        # Use provided DATABASE_URL if available, else fall back to SQLite file
+        if self.database_url_env:
+            return self.database_url_env
         return f"sqlite:///{self.database_path}"
 
 

api/app/db.py

@@ -5,9 +5,9 @@
 from contextlib import contextmanager
 from typing import Any, Generator, List, Type
 
-from app.bootstrap import BOOTSTRAP
 from app.config import Config, get_config
 from app.models.base import BaseModel
+from app.seeding import SEEDING
 from fastapi import Depends
 from sqlalchemy import Engine, create_engine, text
 from sqlalchemy.exc import SQLAlchemyError
@@ -25,9 +25,14 @@ class DatabaseConnection:
     def __init__(self, config: Config = Depends(get_config)) -> None:
         # Ensure the database folder exists.
         os.makedirs(config.database_path.parent, exist_ok=True)
-        self.engine = create_engine(
-            config.database_url, connect_args={"check_same_thread": False}
-        )
+        # Create engine with appropriate connection args for SQLite or other databases
+        db_url = config.database_url
+        if db_url.startswith("sqlite"):  # SQLite needs check_same_thread
+            self.engine = create_engine(
+                db_url, connect_args={"check_same_thread": False}
+            )
+        else:
+            self.engine = create_engine(db_url)
         self.session_local = sessionmaker(
             autocommit=False,
             autoflush=False,
@@ -62,7 +67,7 @@ def seed_bootstrap_data(self) -> None:
         This method is called once during initialization.
         """
         with self.get_session() as session:
-            for model, seeds in BOOTSTRAP.items():
+            for model, seeds in SEEDING.items():
                 try:
                     self._seed_model(
                         session=session,
@@ -104,7 +109,6 @@ def _seed_model(
         logger.info("Seeding data for table '%s'", table_name)
 
         for seed in seeds:
-            logger.debug("Merging seed with id %s for table '%s'", seed.id, table_name)
             session.merge(seed)
         session.flush()
         logger.info("Merged %d seed(s) for table '%s'", len(seeds), table_name)
@@ -130,7 +134,24 @@ def _seed_model(
                 logger.info(
                     "Current sequence for table '%s': %d", table_name, current_seq
                 )
-                if current_seq < (sequence_start - 1):
+                # Determine the current max ID in the table
+                max_id_result = session.execute(
+                    text(f"SELECT MAX(id) FROM {table_name}")
+                ).fetchone()
+                max_id = (
+                    max_id_result[0]
+                    if max_id_result and max_id_result[0] is not None
+                    else 0
+                )
+                logger.info("Max id for table '%s': %d", table_name, max_id)
+                if max_id >= sequence_start:
+                    logger.info(
+                        "Skipping sequence update for table '%s' as max id %d >= desired start %d",
+                        table_name,
+                        max_id,
+                        sequence_start,
+                    )
+                elif current_seq < (sequence_start - 1):
                     logger.info(
                         "Updating sequence for table '%s' to %d",
                         table_name,
@@ -161,7 +182,24 @@ def _seed_model(
                 sequence_name,
                 current_seq,
             )
-            if current_seq < (sequence_start - 1):
+            # Determine the current max ID in the table
+            max_id_result = session.execute(
+                text(f"SELECT MAX(id) FROM {table_name}")
+            ).fetchone()
+            max_id = (
+                max_id_result[0]
+                if max_id_result and max_id_result[0] is not None
+                else 0
+            )
+            logger.info("Max id for table '%s': %d", table_name, max_id)
+            if max_id >= sequence_start:
+                logger.info(
+                    "Skipping sequence update for table '%s' as max id %d >= desired start %d",
+                    table_name,
+                    max_id,
+                    sequence_start,
+                )
+            elif current_seq < (sequence_start - 1):
                 logger.info(
                     "Updating sequence for table '%s' (sequence: '%s') to %d",
                     table_name,

api/app/models/tag.py

@@ -8,11 +8,3 @@ class Tag(BaseModel):
     __tablename__ = "tags"
 
     name: Mapped[str] = mapped_column(unique=True)
-
-
-# this list is used by db.py to create system tags
-TAG_BOOTSTRAP: list[Tag] = [
-    Tag(id=1, name="sys", comment="things defined in refinance code logic"),
-    Tag(id=2, name="resident", comment="hackerspace residents"),
-    Tag(id=3, name="fee", comment="monthly resident's fee"),
-]

api/app/routes/entity.py

@@ -9,7 +9,6 @@
     EntitySchema,
     EntityUpdateSchema,
 )
-from app.schemas.tag import TagSchema
 from app.services.entity import EntityService
 from fastapi import APIRouter, Depends
 
@@ -60,23 +59,3 @@ def update_entity(
     actor_entity: Entity = Depends(get_entity_from_token),
 ):
     return entity_service.update(entity_id, entity_update)
-
-
-@entity_router.post("/{entity_id}/tags", response_model=TagSchema)
-def add_tag_to_entity(
-    entity_id: int,
-    tag_id: int,
-    entity_service: EntityService = Depends(),
-    actor_entity: Entity = Depends(get_entity_from_token),
-):
-    return entity_service.add_tag(entity_id, tag_id)
-
-
-@entity_router.delete("/{entity_id}/tags", response_model=TagSchema)
-def remove_tag_from_entity(
-    entity_id: int,
-    tag_id: int,
-    entity_service: EntityService = Depends(),
-    actor_entity: Entity = Depends(get_entity_from_token),
-):
-    return entity_service.remove_tag(entity_id, tag_id)