Refactor process_query

rohanv11 · Dec 15, 2024 · 11a662c · 11a662c
1 parent d54e6a5
commit 11a662c
Show file tree

Hide file tree

Showing 4 changed files with 56 additions and 100 deletions.
diff --git a/src/process_query.py b/src/process_query.py
@@ -1,14 +1,55 @@
 from ingest import ingest_from_query
 from utils.clone import clone_repo
+from utils.parse_url import parse_url
+from utils.log_convert import logSliderToSize
+from fastapi.templating import Jinja2Templates
+from fastapi import Request
+from config import MAX_DISPLAY_SIZE, EXAMPLE_REPOS
 
-async def process_query(query: dict) -> str:
-
-    await clone_repo(query)
 
-    result = ingest_from_query(query)
-    txt_dump = result[1] + "\n" + result[2]
-    with open(f"{query['local_path']}.txt", "w") as f:
-        f.write(txt_dump)
 
+templates = Jinja2Templates(directory="templates")
+
+async def process_query(request: Request, input_text: str, max_file_size: int, is_index: bool) -> str:
+
+
+    template = "index.jinja.html" if is_index else "github.jinja.html"
+    slider_position = max_file_size
+    size_in_kb = logSliderToSize(max_file_size)
+
+    try:
+        query = parse_url(input_text)
+        query["max_file_size"] = int(size_in_kb) * 1024
+        await clone_repo(query)
+        summary, tree, content = ingest_from_query(query)
+        with open(f"{query['local_path']}.txt", "w") as f:
+            f.write(tree + "\n" + content)
+    except Exception as e:
+        return templates.TemplateResponse(
+            template, 
+            {
+                "request": request,
+                "github_url": input_text,
+                "error_message": f"Error: {e}",
+                "examples": EXAMPLE_REPOS if is_index else [],
+                "default_file_size": slider_position,
+            }
+        )
+
+    if len(content) > MAX_DISPLAY_SIZE:
+        content = f"(Files content cropped to {int(MAX_DISPLAY_SIZE/1000)}k characters, download full ingest to see more)\n" + content[:MAX_DISPLAY_SIZE]
 
-    return result
+    return templates.TemplateResponse(
+        "index.jinja.html", 
+        {
+            "request": request, 
+            "github_url": input_text,
+            "result": True, 
+            "summary": summary,
+            "tree": tree, 
+            "content": content,
+            "examples": EXAMPLE_REPOS if is_index else [],
+            "ingest_id": query['id'],
+            "default_file_size": slider_position,  
+        }
+    )
diff --git a/src/routers/dynamic.py b/src/routers/dynamic.py
@@ -1,25 +1,19 @@
-from fastapi import APIRouter, Request, Form, Depends
+from fastapi import APIRouter, Request, Form
 from fastapi.responses import HTMLResponse
 from fastapi.templating import Jinja2Templates
 
-from utils.parse_url import parse_url   
 from process_query import process_query
-from config import MAX_DISPLAY_SIZE
 from utils.limiter import limiter
-from utils.log_convert import logSliderToSize
 
 router = APIRouter()
 templates = Jinja2Templates(directory="templates")
 
-
 @router.get("/{full_path:path}")
 async def catch_all(request: Request, full_path: str):
     return templates.TemplateResponse(
         "github.jinja.html",
         {
             "request": request,
-            "result": False,
-            "loading": True,
             "github_url": f"https://github.com/{full_path}",
             "default_file_size": 243
         }
@@ -32,45 +26,5 @@ async def process_catch_all(
     input_text: str = Form(...),
     max_file_size: int = Form(...)
 ):
-
-    slider_position = max_file_size
-    size_in_kb = logSliderToSize(max_file_size)
-    try:
-        parsed_query = parse_url(input_text, size_in_kb)
-
-        summary, tree, content = await process_query(
-            parsed_query,
-
-        )
-    except Exception as e:
-        print(e)
-        return templates.TemplateResponse(
-            "github.jinja.html", 
-            {
-                "request": request, 
-                "result": False, 
-                "loading": False,
-                "github_url": input_text,
-                "error_message": f"Error: \n {e}",
-                "default_file_size": slider_position,
-
-            }
-        )
-
-    if len(content) > MAX_DISPLAY_SIZE:
-        content = f"(Files content cropped to {int(MAX_DISPLAY_SIZE/1000)}k characters, download full digest to see more)\n" + content[:MAX_DISPLAY_SIZE]
-
-    return templates.TemplateResponse(
-        "github.jinja.html", 
-        {
-            "request": request, 
-            "result": True, 
-            "loading": False,
-            "summary": summary,
-            "tree": tree, 
-            "content": content,
-            "ingest_id": parsed_query["id"],
-            "github_url": input_text,
-            "default_file_size": max_file_size  
-        }
-    )
+    return await process_query(request, input_text, max_file_size, is_index=False)
+
diff --git a/src/routers/index.py b/src/routers/index.py
@@ -1,21 +1,16 @@
 from fastapi import APIRouter, Request, Form
-from fastapi.responses import HTMLResponse, JSONResponse
+from fastapi.responses import HTMLResponse
 from fastapi.templating import Jinja2Templates
 
-from utils.parse_url import parse_url
 from utils.limiter import limiter
 from process_query import process_query
-from config import MAX_DISPLAY_SIZE, EXAMPLE_REPOS
-from utils.log_convert import logSliderToSize
+from config import EXAMPLE_REPOS
 
 
 router = APIRouter()
 templates = Jinja2Templates(directory="templates")
 
 
-
-
-
 @router.get("/", response_class=HTMLResponse)
 async def home(request: Request):
     return templates.TemplateResponse(
@@ -35,42 +30,9 @@ async def index_post(
     input_text: str = Form(...),
     max_file_size: int = Form(...)
 ):
-    slider_position = max_file_size
-    size_in_kb = logSliderToSize(max_file_size)
+    return await process_query(request, input_text, max_file_size, is_index=True)
 
-    try:
-        parsed_query = parse_url(input_text, size_in_kb)
-        summary, tree, content = await process_query(parsed_query)
-    except Exception as e:
-        print(e)
-        return templates.TemplateResponse(
-            "index.jinja.html", 
-            {
-                "request": request,
-                "error_message": f"Error: {e}",
-                "examples": EXAMPLE_REPOS,
-                "default_file_size": slider_position,  # Return original slider position
-                "github_url": input_text  # Add the URL to the error response
-            }
-        )
 
-    if len(content) > MAX_DISPLAY_SIZE:
-        content = f"(Files content cropped to {int(MAX_DISPLAY_SIZE/1000)}k characters, download full ingest to see more)\n" + content[:MAX_DISPLAY_SIZE]
-
-    return templates.TemplateResponse(
-        "index.jinja.html", 
-        {
-            "request": request, 
-            "summary": summary,
-            "result": True, 
-            "tree": tree, 
-            "content": content,
-            "examples": EXAMPLE_REPOS,
-            "ingest_id": parsed_query['id'],
-            "default_file_size": slider_position,  
-            "github_url": input_text 
-        }
-    )
 
 
 
diff --git a/src/utils/parse_url.py b/src/utils/parse_url.py
@@ -10,7 +10,7 @@ def validate_github_url(url: str) -> bool:
     github_pattern = r'^https://github\.com/[^/]+/[^/]+'
     return bool(re.match(github_pattern, url))
 
-def parse_url(url: str, max_file_size: int) -> dict:
+def parse_url(url: str) -> dict:
     parsed = {
         "user_name": None,
         "repo_name": None,
@@ -20,7 +20,6 @@ def parse_url(url: str, max_file_size: int) -> dict:
         "subpath": "/",
         "local_path": None,
         "url": None,
-        "max_file_size": int(max_file_size) * 1024
     }
 
     if not validate_github_url(url):