Use postgresql for tests (#450)

### Description

Fix #403 

Add Postgresql databse support for tests. Use a TestAppClient per test
file, to prevent tests from a module to impact other modules

Configure a Postgresql database for the test workflow.
Run tests with Postgresql. Run migration tests with SQLite and
Postgresql.

### Checklist

- [x] Created tests which fail without the change (if possible)
- [x] All tests passing
- [x] Extended the documentation, if necessary

---------

Co-authored-by: julien4215 <120588494+julien4215@users.noreply.github.com>

.env.test

@@ -53,4 +53,4 @@ POSTGRES_USER = "hyperion"
 POSTGRES_PASSWORD = "somerealpassword"
 POSTGRES_DB = "hyperion"
 POSTGRES_TZ = "Etc/UTC"
-DATABASE_DEBUG = False # If True, will print all SQL queries in the console
+DATABASE_DEBUG = False # If True, will print all SQL queries in the console

.github/workflows/test.yml

@@ -30,6 +30,23 @@ jobs:
         ports:
           # Maps port 6379 on service container to the host
           - 6379:6379
+      postgres:
+        # Docker Hub image
+        image: postgres
+        # Provide the password for postgres
+        env:
+          POSTGRES_PASSWORD: "somerealpassword"
+          POSTGRES_USER: "hyperion"
+          POSTGRES_DB: "hyperion"
+        # Set health checks to wait until postgres has started
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          # Maps tcp port 5432 on service container to the host
+          - 5432:5432
 
     steps:
       - name: Check out the code
@@ -61,9 +78,14 @@ jobs:
         with:
           path: .pytest_cache
           key: pytest_cache-${{ github.head_ref }}
+
+      - name: Run migration unit tests with SQLite
+        run: python -m pytest tests/test_migrations.py
 
-      - name: Run unit tests
+      - name: Run unit tests with Postgresql
         run: python -m pytest --cov
+        env:
+          SQLITE_DB: ""
 
       - name: Upload coverage reports to Codecov
         uses: codecov/codecov-action@v4

.gitignore

@@ -27,7 +27,3 @@ logs/
 
 # Pytest-cov
 .coverage
-
-# Migration tests
-test_migration.db
-test_migration.db-journal

migrations/versions/0-6-init.py

@@ -2,7 +2,7 @@
 
 Revision ID: 17b92dc4b50d
 Revises:
-Create Date: 2024-04-12 18:19:12.439926
+Create Date: 2024-05-26 16:25:22.390935
 
 """
 
@@ -929,6 +929,74 @@ def downgrade() -> None:
     op.drop_table("amap_delivery")
     op.drop_index(op.f("ix_advert_advertisers_id"), table_name="advert_advertisers")
     op.drop_table("advert_advertisers")
+    sa.Enum(
+        "Autre",
+        "Adoma",
+        "Exte",
+        "T1",
+        "T2",
+        "T3",
+        "T4",
+        "T56",
+        "U1",
+        "U2",
+        "U3",
+        "U4",
+        "U56",
+        "V1",
+        "V2",
+        "V3",
+        "V45",
+        "V6",
+        "X1",
+        "X2",
+        "X3",
+        "X4",
+        "X5",
+        "X6",
+        name="floorstype",
+    ).drop(op.get_bind())
+    sa.Enum(
+        "cinema",
+        "advert",
+        "bookingadmin",
+        "amap",
+        "booking",
+        "event",
+        "loan",
+        "raffle",
+        "vote",
+        name="topic",
+    ).drop(op.get_bind())
+    sa.Enum(
+        "eventAE",
+        "eventUSE",
+        "independentAssociation",
+        "happyHour",
+        "direction",
+        "nightParty",
+        "other",
+        name="calendareventtype",
+    ).drop(op.get_bind())
+    sa.Enum(
+        "comity",
+        "section_ae",
+        "club_ae",
+        "section_use",
+        "club_use",
+        "association_independant",
+        name="kinds",
+    ).drop(op.get_bind())
+    sa.Enum(
+        "waiting",
+        "open",
+        "closed",
+        "counting",
+        "published",
+        name="statustype",
+    ).drop(op.get_bind())
+    sa.Enum("midi", "soir", name="amapslottype").drop(op.get_bind())
+    sa.Enum("creation", "open", "lock", name="rafflestatustype").drop(op.get_bind())
     # ### end Alembic commands ###
 
 

migrations/versions/8-recommendation_primary_key.py

@@ -6,6 +6,7 @@
 from collections.abc import Sequence
 from datetime import UTC, datetime
 from typing import TYPE_CHECKING
+from uuid import UUID
 
 import sqlalchemy as sa
 from alembic import op
@@ -61,4 +62,10 @@ def test_upgrade(
         sa.text("SELECT id from recommendation"),
     ).fetchall()
 
-    assert ("66c363bc-c71f-4eae-8376-c37712a312f6",) in rows
+    assert len(rows) > 0
+
+    if isinstance(rows[0][0], UUID):
+        assert (UUID("66c363bc-c71f-4eae-8376-c37712a312f6"),) in rows
+    else:
+        # SQLite does not support UUID and will use strings
+        assert ("66c363bc-c71f-4eae-8376-c37712a312f6",) in rows

migrations/versions/9-external_users.py

@@ -78,6 +78,11 @@ def downgrade() -> None:
     op.drop_column("core_user_unconfirmed", "external")
     op.drop_column("core_user", "external")
 
+    core_user = sa.sql.table("core_user", sa.Column("floor"))
+    op.execute(
+        core_user.update().where(core_user.c.floor.is_(None)).values(floor="Autre"),
+    )
+
     with op.batch_alter_table("core_user") as batch_op:
         batch_op.alter_column(
             "floor",
@@ -108,6 +113,7 @@ def downgrade() -> None:
                 "X6",
                 name="floorstype",
             ),
+            # We make this column non nullable, we must provide a default value
             nullable=False,
         )
 

pyproject.toml

@@ -114,7 +114,13 @@ ignore_missing_imports = true
 
 [tool.pytest.ini_options]
 asyncio_mode = "auto"
-filterwarnings = "error"
+filterwarnings = [
+    "error",
+    # We don't know how to fix the warning:
+    # `ResourceWarning: connection <psycopg.Connection [IDLE] (host=... user=... database=...) at 0x1166bf310> was deleted while still open. Please use 'with' or '.close()' to close the connection`
+    # It only happen when running tests with Postgresql
+    "ignore:connection <.*> was deleted while still open.:ResourceWarning",
+]
 
 [tool.coverage.run]
 source_pkgs = ["app"]

tests/commons.py

@@ -5,18 +5,17 @@
 
 import redis
 from fastapi import Depends
-from fastapi.testclient import TestClient
+from sqlalchemy import NullPool
 from sqlalchemy.exc import IntegrityError
 from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine
 
-from app.app import get_application
 from app.core import models_core, security
 from app.core.auth import schemas_auth
 from app.core.config import Settings
 from app.core.groups import cruds_groups
 from app.core.groups.groups_type import GroupType
 from app.core.users import cruds_users
-from app.dependencies import get_db, get_redis_client, get_settings
+from app.dependencies import get_settings
 from app.types.floors_type import FloorsType
 from app.types.sqlalchemy import Base
 from app.utils.redis import connect, disconnect
@@ -31,17 +30,23 @@ def override_get_settings() -> Settings:
 
 settings = override_get_settings()
 
-test_app = get_application(settings=settings, drop_db=True)  # Create the test's app
 
+# Connect to the test's database
 if settings.SQLITE_DB:
-    SQLALCHEMY_DATABASE_URL = (
-        f"sqlite+aiosqlite:///./{settings.SQLITE_DB}"  # Connect to the test's database
-    )
+    SQLALCHEMY_DATABASE_URL = f"sqlite+aiosqlite:///./{settings.SQLITE_DB}"
+    SQLALCHEMY_DATABASE_URL_SYNC = f"sqlite:///./{settings.SQLITE_DB}"
 else:
     SQLALCHEMY_DATABASE_URL = f"postgresql+asyncpg://{settings.POSTGRES_USER}:{settings.POSTGRES_PASSWORD}@{settings.POSTGRES_HOST}/{settings.POSTGRES_DB}"
+    SQLALCHEMY_DATABASE_URL_SYNC = f"postgresql+psycopg://{settings.POSTGRES_USER}:{settings.POSTGRES_PASSWORD}@{settings.POSTGRES_HOST}/{settings.POSTGRES_DB}"
 
 
-engine = create_async_engine(SQLALCHEMY_DATABASE_URL, echo=True)
+engine = create_async_engine(
+    SQLALCHEMY_DATABASE_URL,
+    echo=True,
+    # We need to use NullPool to run tests with Postgresql
+    # See https://docs.sqlalchemy.org/en/20/orm/extensions/asyncio.html#using-multiple-asyncio-event-loops
+    poolclass=NullPool,
+)
 
 TestingSessionLocal = async_sessionmaker(
     engine,
@@ -91,17 +96,6 @@ def change_redis_client_status(activated: bool) -> None:
         redis_client = False
 
 
-test_app.dependency_overrides[get_db] = override_get_db
-test_app.dependency_overrides[get_settings] = override_get_settings
-test_app.dependency_overrides[get_redis_client] = override_get_redis_client
-
-
-client = TestClient(test_app)  # Create a client to execute tests
-
-with client:  # That syntax trigger the lifespan defined in main.py
-    pass
-
-
 async def create_user_with_groups(
     groups: list[GroupType],
     user_id: str | None = None,

tests/conftest.py

@@ -0,0 +1,22 @@
+import pytest
+from fastapi.testclient import TestClient
+
+from app.app import get_application
+from app.dependencies import get_db, get_redis_client, get_settings
+from tests.commons import (
+    override_get_db,
+    override_get_redis_client,
+    override_get_settings,
+    settings,
+)
+
+
+@pytest.fixture(scope="module", autouse=True)
+def client() -> TestClient:
+    test_app = get_application(settings=settings, drop_db=True)  # Create the test's app
+
+    test_app.dependency_overrides[get_db] = override_get_db
+    test_app.dependency_overrides[get_settings] = override_get_settings
+    test_app.dependency_overrides[get_redis_client] = override_get_redis_client
+
+    return TestClient(test_app)  # Create a client to execute tests

tests/test_PH.py

@@ -3,13 +3,13 @@
 from pathlib import Path
 
 import pytest_asyncio
+from fastapi.testclient import TestClient
 
 from app.core import models_core
 from app.core.groups.groups_type import GroupType
 from app.modules.ph import models_ph
 from tests.commons import (
     add_object_to_db,
-    client,
     create_api_access_token,
     create_user_with_groups,
 )
@@ -25,7 +25,7 @@
 
 
 @pytest_asyncio.fixture(scope="module", autouse=True)
-async def init_objects():
+async def init_objects() -> None:
     global ph_user_ph
     ph_user_ph = await create_user_with_groups([GroupType.ph])
 
@@ -55,7 +55,7 @@ async def init_objects():
     await add_object_to_db(paper2)
 
 
-def test_create_paper():
+def test_create_paper(client: TestClient) -> None:
     response = client.post(
         "/ph/",
         json={
@@ -68,7 +68,7 @@ def test_create_paper():
     assert response.status_code == 201
 
 
-def test_get_papers():
+def test_get_papers(client: TestClient) -> None:
     response = client.get(
         "/ph/",
         headers={"Authorization": f"Bearer {token_simple}"},
@@ -81,7 +81,7 @@ def test_get_papers():
     ]
 
 
-def test_get_papers_admin():
+def test_get_papers_admin(client: TestClient) -> None:
     response = client.get(
         "/ph/admin",
         headers={"Authorization": f"Bearer {token_ph}"},
@@ -92,7 +92,7 @@ def test_get_papers_admin():
     assert str(paper2.id) in [response_paper["id"] for response_paper in response_json]
 
 
-def test_create_paper_pdf_and_cover():
+def test_create_paper_pdf_and_cover(client: TestClient) -> None:
     with Path("assets/pdf/default_ph.pdf").open("rb") as pdf:
         response = client.post(
             f"/ph/{paper.id}/pdf",
@@ -105,23 +105,23 @@ def test_create_paper_pdf_and_cover():
     assert Path(f"data/ph/cover/{paper.id}.jpg").is_file()
 
 
-def test_get_paper_pdf():
+def test_get_paper_pdf(client: TestClient) -> None:
     response = client.get(
         f"/ph/{paper.id}/pdf",
         headers={"Authorization": f"Bearer {token_simple}"},
     )
     assert response.status_code == 200
 
 
-def test_get_cover():
+def test_get_cover(client: TestClient) -> None:
     response = client.get(
         f"/ph/{paper.id}/cover",
         headers={"Authorization": f"Bearer {token_simple}"},
     )
     assert response.status_code == 200
 
 
-def test_update_paper():
+def test_update_paper(client: TestClient) -> None:
     response = client.patch(
         f"/ph/{paper.id}",
         json={
@@ -133,7 +133,7 @@ def test_update_paper():
     assert response.status_code == 204
 
 
-def test_delete_paper():
+def test_delete_paper(client: TestClient) -> None:
     with Path("assets/pdf/default_PDF.pdf").open("rb") as pdf:
         client.post(
             f"/ph/{paper.id}/pdf",