Add test to cover the end-to-end flow, fixed laike9m#48

Author: laike9m

tests/fixture.py

@@ -0,0 +1,10 @@
+"""
+I can't get fixture to work well with parameterized test, so just create some constants here
+"""
+
+from pathlib import Path
+
+QUESTION = "question.py"
+CHALLENGES_DIR = Path(__file__).parent.parent / "challenges"
+ALL_QUESTIONS = list(CHALLENGES_DIR.glob("**/question.py"))
+ALL_SOLUTIONS = list(CHALLENGES_DIR.glob("**/solution.py"))

tests/test_api.py

@@ -0,0 +1,24 @@
+from pathlib import Path
+
+import pytest
+from werkzeug.test import TestResponse
+from app import app
+from .fixture import ALL_QUESTIONS
+
+
+@pytest.mark.parametrize("question_file", ALL_QUESTIONS, ids=lambda x: x.parent.name)
+def test_get_challenge(question_file: Path):
+    level, challenge_name = question_file.parent.name.split("-", maxsplit=1)
+    response = app.test_client().get(f"/{level}/{challenge_name}")
+
+    # Verify the returned HTML contains a challenge's source code
+    assert "TODO" in response.text
+
+
+@pytest.mark.parametrize("question_file", ALL_QUESTIONS, ids=lambda x: x.parent.name)
+def test_run_challenge(question_file: Path):
+    level, challenge_name = question_file.parent.name.split("-", maxsplit=1)
+    response = app.test_client().post(f"run/{level}/{challenge_name}", data="")
+
+    # Verify backend can run type check and return results
+    assert response.json is not None

tests/test_identical.py

@@ -7,10 +7,7 @@
 import pytest
 
 from views.challenge import Challenge, Level
-
-CHALLENGES_DIR = Path(__file__).parent.parent / "challenges"
-ALL_SOLUTIONS = list(CHALLENGES_DIR.glob("**/solution.py"))
-QUESTION = "question.py"
+from .fixture import ALL_SOLUTIONS, QUESTION
 
 
 @pytest.mark.parametrize("solution_file", ALL_SOLUTIONS, ids=lambda x: x.parent.name)

tests/test_questions.py

@@ -7,18 +7,13 @@
 import pytest
 
 from views.challenge import ChallengeManager
-
-CHALLENGES_DIR = Path(__file__).parent.parent / "challenges"
-ALL_QUESTIONS = list(CHALLENGES_DIR.glob("**/question.py"))
+from .fixture import ALL_QUESTIONS
 
 
 @pytest.mark.parametrize("question_file", ALL_QUESTIONS, ids=lambda x: x.parent.name)
 def test_solution_valid(question_file: Path):
-    test_id = question_file.parent.name
-
-    # TODO: remove "advanced-generic-param" once fixed
     # Skip the challenges whose question can't fail type check no matter how many tests are added.
-    if test_id in ("basic-any"):
+    if question_file.parent.name == "basic-any":
         pytest.skip(f"Skipping test: {question_file}")
 
     with question_file.open() as f:

tests/test_solutions.py

@@ -7,9 +7,7 @@
 import pytest
 
 from views.challenge import ChallengeManager
-
-CHALLENGES_DIR = Path(__file__).parent.parent / "challenges"
-ALL_SOLUTIONS = list(CHALLENGES_DIR.glob("**/solution.py"))
+from .fixture import ALL_SOLUTIONS
 
 
 @pytest.mark.parametrize("solution_file", ALL_SOLUTIONS, ids=lambda x: x.parent.name)

views/challenge.py

@@ -72,8 +72,8 @@ def run_challenge(self, key: ChallengeKey, user_code: str) -> TypeCheckResult:
     def _load_challenges() -> dict[ChallengeKey, Challenge]:
         challenges = {}
         for filename in glob.glob(f"{ROOT_DIR}/challenges/*/question.py"):
-            dir_name = os.path.basename(os.path.dirname(filename))
-            level, challenge_name = dir_name.split("-", maxsplit=1)
+            challenge_folder_name = os.path.basename(os.path.dirname(filename))
+            level, challenge_name = challenge_folder_name.split("-", maxsplit=1)
             with open(filename, "r") as file:
                 code = file.read()
             challenges[ChallengeKey(Level(level), challenge_name)] = Challenge(