feat: Refactor backend to a rest api (#346)

* refactor: Refactor backend to a rest api and make weak changes to the front to make it works

* fix: remove result bool in Success response object since it's useless, remove sensitive data from the Success Response object, make the integration unit test compatible

* fix: clean query processor return types, remove deprecated fields, better handling for few errors

* fix: unit tests, remove deprecated is_index from jinja templates

---------

Co-authored-by: ix <n.guintini@protonmail.com>

Author: ix-56h

src/server/main.py

@@ -6,14 +6,13 @@
 from pathlib import Path
 
 from dotenv import load_dotenv
-from fastapi import FastAPI, Request
+from fastapi import FastAPI
 from fastapi.responses import FileResponse, HTMLResponse
 from fastapi.staticfiles import StaticFiles
 from slowapi.errors import RateLimitExceeded
 from starlette.middleware.trustedhost import TrustedHostMiddleware
 
-from server.routers import download, dynamic, index
-from server.server_config import templates
+from server.routers import dynamic, index, ingest
 from server.server_utils import lifespan, limiter, rate_limit_exception_handler
 
 # Load environment variables from .env file
@@ -58,7 +57,7 @@ async def health_check() -> dict[str, str]:
     return {"status": "healthy"}
 
 
-@app.head("/")
+@app.head("/", include_in_schema=False)
 async def head_root() -> HTMLResponse:
     """Respond to HTTP HEAD requests for the root URL.
 
@@ -73,26 +72,7 @@ async def head_root() -> HTMLResponse:
     return HTMLResponse(content=None, headers={"content-type": "text/html; charset=utf-8"})
 
 
-@app.get("/api/", response_class=HTMLResponse)
-@app.get("/api", response_class=HTMLResponse)
-async def api_docs(request: Request) -> HTMLResponse:
-    """Render the API documentation page.
-
-    Parameters
-    ----------
-    request : Request
-        The incoming HTTP request.
-
-    Returns
-    -------
-    HTMLResponse
-        A rendered HTML page displaying API documentation.
-
-    """
-    return templates.TemplateResponse("api.jinja", {"request": request})
-
-
-@app.get("/robots.txt")
+@app.get("/robots.txt", include_in_schema=False)
 async def robots() -> FileResponse:
     """Serve the ``robots.txt`` file to guide search engine crawlers.
 
@@ -120,5 +100,5 @@ async def llm_txt() -> FileResponse:
 
 # Include routers for modular endpoints
 app.include_router(index)
-app.include_router(download)
+app.include_router(ingest)
 app.include_router(dynamic)

src/server/models.py

@@ -2,12 +2,116 @@
 
 from __future__ import annotations
 
-from pydantic import BaseModel
+from enum import Enum
+from typing import Union
+
+from pydantic import BaseModel, Field, field_validator
 
 # needed for type checking (pydantic)
 from server.form_types import IntForm, OptStrForm, StrForm  # noqa: TC001 (typing-only-first-party-import)
 
 
+class PatternType(str, Enum):
+    """Enumeration for pattern types used in file filtering."""
+
+    INCLUDE = "include"
+    EXCLUDE = "exclude"
+
+
+class IngestRequest(BaseModel):
+    """Request model for the /api/ingest endpoint.
+
+    Attributes
+    ----------
+    input_text : str
+        The Git repository URL or slug to ingest.
+    max_file_size : int
+        Maximum file size slider position (0-500) for filtering files.
+    pattern_type : PatternType
+        Type of pattern to use for file filtering (include or exclude).
+    pattern : str
+        Glob/regex pattern string for file filtering.
+    token : str | None
+        GitHub personal access token (PAT) for accessing private repositories.
+
+    """
+
+    input_text: str = Field(..., description="Git repository URL or slug to ingest")
+    max_file_size: int = Field(..., ge=0, le=500, description="File size slider position (0-500)")
+    pattern_type: PatternType = Field(default=PatternType.EXCLUDE, description="Pattern type for file filtering")
+    pattern: str = Field(default="", description="Glob/regex pattern for file filtering")
+    token: str | None = Field(default=None, description="GitHub PAT for private repositories")
+
+    @field_validator("input_text")
+    @classmethod
+    def validate_input_text(cls, v: str) -> str:
+        """Validate that input_text is not empty."""
+        if not v.strip():
+            err = "input_text cannot be empty"
+            raise ValueError(err)
+        return v.strip()
+
+    @field_validator("pattern")
+    @classmethod
+    def validate_pattern(cls, v: str) -> str:
+        """Validate pattern field."""
+        return v.strip()
+
+
+class IngestSuccessResponse(BaseModel):
+    """Success response model for the /api/ingest endpoint.
+
+    Attributes
+    ----------
+    repo_url : str
+        The original repository URL that was processed.
+    short_repo_url : str
+        Short form of repository URL (user/repo).
+    summary : str
+        Summary of the ingestion process including token estimates.
+    tree : str
+        File tree structure of the repository.
+    content : str
+        Processed content from the repository files.
+    default_max_file_size : int
+        The file size slider position used.
+    pattern_type : str
+        The pattern type used for filtering.
+    pattern : str
+        The pattern used for filtering.
+
+    """
+
+    repo_url: str = Field(..., description="Original repository URL")
+    short_repo_url: str = Field(..., description="Short repository URL (user/repo)")
+    summary: str = Field(..., description="Ingestion summary with token estimates")
+    tree: str = Field(..., description="File tree structure")
+    content: str = Field(..., description="Processed file content")
+    default_max_file_size: int = Field(..., description="File size slider position used")
+    pattern_type: str = Field(..., description="Pattern type used")
+    pattern: str = Field(..., description="Pattern used")
+
+
+class IngestErrorResponse(BaseModel):
+    """Error response model for the /api/ingest endpoint.
+
+    Attributes
+    ----------
+    error : str
+        Error message describing what went wrong.
+    repo_url : str
+        The repository URL that failed to process.
+
+    """
+
+    error: str = Field(..., description="Error message")
+    repo_url: str = Field(..., description="Repository URL that failed")
+
+
+# Union type for API responses
+IngestResponse = Union[IngestSuccessResponse, IngestErrorResponse]
+
+
 class QueryForm(BaseModel):
     """Form data for the query.
 

src/server/query_processor.py

@@ -2,46 +2,35 @@
 
 from __future__ import annotations
 
-from functools import partial
 from pathlib import Path
-from typing import TYPE_CHECKING, cast
+from typing import cast
 
 from gitingest.clone import clone_repo
 from gitingest.ingestion import ingest_query
 from gitingest.query_parser import IngestionQuery, parse_query
 from gitingest.utils.git_utils import validate_github_token
+from server.models import IngestErrorResponse, IngestResponse, IngestSuccessResponse
 from server.server_config import (
-    DEFAULT_FILE_SIZE_KB,
-    EXAMPLE_REPOS,
+    DEFAULT_MAX_FILE_SIZE_KB,
     MAX_DISPLAY_SIZE,
-    templates,
 )
 from server.server_utils import Colors, log_slider_to_size
 
-if TYPE_CHECKING:
-    from fastapi import Request
-    from starlette.templating import _TemplateResponse
-
 
 async def process_query(
-    request: Request,
-    *,
     input_text: str,
     slider_position: int,
     pattern_type: str = "exclude",
     pattern: str = "",
-    is_index: bool = False,
     token: str | None = None,
-) -> _TemplateResponse:
+) -> IngestResponse:
     """Process a query by parsing input, cloning a repository, and generating a summary.
 
     Handle user input, process Git repository data, and prepare
     a response for rendering a template with the processed results or an error message.
 
     Parameters
     ----------
-    request : Request
-        The HTTP request object.
     input_text : str
         Input text provided by the user, typically a Git repository URL or slug.
     slider_position : int
@@ -50,15 +39,13 @@ async def process_query(
         Type of pattern to use (either "include" or "exclude") (default: ``"exclude"``).
     pattern : str
         Pattern to include or exclude in the query, depending on the pattern type.
-    is_index : bool
-        Flag indicating whether the request is for the index page (default: ``False``).
     token : str | None
         GitHub personal access token (PAT) for accessing private repositories.
 
     Returns
     -------
-    _TemplateResponse
-        Rendered template response containing the processed results or an error message.
+    IngestResponse
+        A union type, corresponding to IngestErrorResponse or IngestSuccessResponse
 
     Raises
     ------
@@ -79,21 +66,10 @@ async def process_query(
     if token:
         validate_github_token(token)
 
-    template = "index.jinja" if is_index else "git.jinja"
-    template_response = partial(templates.TemplateResponse, name=template)
     max_file_size = log_slider_to_size(slider_position)
 
-    context = {
-        "request": request,
-        "repo_url": input_text,
-        "examples": EXAMPLE_REPOS if is_index else [],
-        "default_file_size": slider_position,
-        "pattern_type": pattern_type,
-        "pattern": pattern,
-        "token": token,
-    }
-
     query: IngestionQuery | None = None
+    short_repo_url = ""
 
     try:
         query = await parse_query(
@@ -107,7 +83,7 @@ async def process_query(
         query.ensure_url()
 
         # Sets the "<user>/<repo>" for the page title
-        context["short_repo_url"] = f"{query.user_name}/{query.repo_name}"
+        short_repo_url = f"{query.user_name}/{query.repo_name}"
 
         clone_config = query.extract_clone_config()
         await clone_repo(clone_config, token=token)
@@ -126,10 +102,10 @@ async def process_query(
             print(f"{Colors.BROWN}WARN{Colors.END}: {Colors.RED}<-  {Colors.END}", end="")
             print(f"{Colors.RED}{exc}{Colors.END}")
 
-        context["error_message"] = f"Error: {exc}"
-        if "405" in str(exc):
-            context["error_message"] = "Repository not found. Please make sure it is public."
-        return template_response(context=context)
+        return IngestErrorResponse(
+            error="Repository not found. Please make sure it is public." if "405" in str(exc) else "",
+            repo_url=short_repo_url,
+        )
 
     if len(content) > MAX_DISPLAY_SIZE:
         content = (
@@ -148,18 +124,17 @@ async def process_query(
         summary=summary,
     )
 
-    context.update(
-        {
-            "result": True,
-            "summary": summary,
-            "tree": tree,
-            "content": content,
-            "ingest_id": query.id,
-        },
+    return IngestSuccessResponse(
+        repo_url=input_text,
+        short_repo_url=short_repo_url,
+        summary=summary,
+        tree=tree,
+        content=content,
+        default_max_file_size=slider_position,
+        pattern_type=pattern_type,
+        pattern=pattern,
     )
 
-    return template_response(context=context)
-
 
 def _print_query(url: str, max_file_size: int, pattern_type: str, pattern: str) -> None:
     """Print a formatted summary of the query details for debugging.
@@ -177,7 +152,7 @@ def _print_query(url: str, max_file_size: int, pattern_type: str, pattern: str)
 
     """
     print(f"{Colors.WHITE}{url:<20}{Colors.END}", end="")
-    if int(max_file_size / 1024) != DEFAULT_FILE_SIZE_KB:
+    if int(max_file_size / 1024) != DEFAULT_MAX_FILE_SIZE_KB:
         print(
             f" | {Colors.YELLOW}Size: {int(max_file_size / 1024)}kb{Colors.END}",
             end="",

src/server/routers/__init__.py

@@ -1,7 +1,7 @@
 """Module containing the routers for the FastAPI application."""
 
-from server.routers.download import router as download
 from server.routers.dynamic import router as dynamic
 from server.routers.index import router as index
+from server.routers.ingest import router as ingest
 
-__all__ = ["download", "dynamic", "index"]
+__all__ = ["dynamic", "index", "ingest"]