Revert "Modified Crawling and graceful error handling with streamlined UI. "

README.md

@@ -1,8 +1,8 @@
 # DevDocs by CyberAGI 🚀
 
 <div align="center">
-  <img src="assets/image.png" alt="DevDocs Interface" width="800">
-
+  <img src="https://github.com/user-attachments/assets/6d4cc4df-fe5d-4483-9218-3d621f572e49" alt="DevDocs Interface" width="800">
+  <img src="https://github.com/user-attachments/assets/00350dc6-2ff3-40cf-b0b3-8b3e387d983d" alt="DevDocs Interface" width="800">
 
   <p align="center">
     <strong>Turn Weeks of Documentation Research into Hours of Productive Development</strong>
@@ -108,43 +108,18 @@ git clone https://github.com/cyberagiinc/DevDocs.git
 # Navigate to the project directory
 cd DevDocs
 
-# Configure environment variables
-# Copy the template file to .env
-cp .env.template .env
-
-# Ensure NEXT_PUBLIC_BACKEND_URL in .env is set correctly (e.g., http://localhost:24125)
-# This allows the frontend (running in your browser) to communicate with the backend service.
-
-
 # Start all services using Docker
 ./docker-start.sh
 ```
 
-For Windows users: Experimental Only (Not Tested Yet)
+For Windows users:
 ```cmd
 # Clone the repository
 git clone https://github.com/cyberagiinc/DevDocs.git
 
 # Navigate to the project directory
-
 cd DevDocs
 
-# Configure environment variables
-# Copy the template file to .env
-
-copy .env.template .env
-
-# Ensure NEXT_PUBLIC_BACKEND_URL in .env is set correctly (e.g., http://localhost:24125)
-
-# This allows the frontend (running in your browser) to communicate with the backend service.
-
-# Prerequisites: Install WSL 2 and Docker Desktop
-# Docker Desktop for Windows requires WSL 2. Please ensure you have WSL 2 installed and running first.
-# 1. Install WSL 2: Follow the official Microsoft guide: https://learn.microsoft.com/en-us/windows/wsl/install
-# 2. Install Docker Desktop for Windows: Download and install from the official Docker website. Docker Desktop includes Docker Compose.
-
-
-
 # Start all services using Docker
 docker-start.bat
 ```

app/api/all-files/route.ts

@@ -90,7 +90,8 @@ export async function GET(request: Request) {
     // Get in-memory files from the backend
     let memoryFiles = []
     try {
-      const memoryResponse = await fetch('http://backend:24125/api/memory-files')
+      const backendUrl = process.env.NEXT_PUBLIC_BACKEND_URL || process.env.BACKEND_URL || 'http://localhost:24125'
+      const memoryResponse = await fetch(`${backendUrl}/api/memory-files`)
       if (memoryResponse.ok) {
         const memoryData = await memoryResponse.json()
         if (memoryData.success && Array.isArray(memoryData.files)) {

app/api/memory-file/route.ts

@@ -0,0 +1,48 @@
+import { NextResponse } from 'next/server'
+
+export async function GET(request: Request) {
+  try {
+    const { searchParams } = new URL(request.url)
+    const id = searchParams.get('id')
+
+    if (!id) {
+      return NextResponse.json(
+        { success: false, error: 'Missing file ID' },
+        { status: 400 }
+      )
+    }
+
+    // Fetch the in-memory file from the backend
+    const backendUrl = process.env.NEXT_PUBLIC_BACKEND_URL || process.env.BACKEND_URL || 'http://localhost:24125'
+    const response = await fetch(`${backendUrl}/api/memory-files/${id}`)
+
+    if (!response.ok) {
+      const errorData = await response.json()
+      return NextResponse.json(
+        { success: false, error: errorData.error || 'Failed to fetch in-memory file' },
+        { status: response.status }
+      )
+    }
+
+    const data = await response.json()
+
+    if (!data.success) {
+      return NextResponse.json(
+        { success: false, error: data.error || 'Failed to fetch in-memory file' },
+        { status: 500 }
+      )
+    }
+
+    return NextResponse.json({
+      success: true,
+      content: data.content,
+      metadata: data.metadata
+    })
+  } catch (error) {
+    console.error('Error fetching in-memory file:', error)
+    return NextResponse.json(
+      { success: false, error: error instanceof Error ? error.message : 'Failed to fetch in-memory file' },
+      { status: 500 }
+    )
+  }
+}

app/api/storage/route.ts

@@ -41,23 +41,8 @@ export async function GET(request: Request) {
       const mdFiles = files.filter(f => f.endsWith('.md'))
       const jsonFiles = files.filter(f => f.endsWith('.json'))
 
-      // Define interface for disk file details
-      interface DiskFileDetail {
-        name: string;
-        jsonPath: string;
-        markdownPath: string;
-        timestamp: Date;
-        size: number;
-        wordCount: number;
-        charCount: number;
-        isConsolidated: boolean;
-        pagesCount: number;
-        rootUrl: string;
-        isInMemory: boolean;
-      }
-
       // Get disk files
-      const diskFileDetails: DiskFileDetail[] = await Promise.all(
+      const diskFileDetails = await Promise.all(
         mdFiles.map(async (filename) => {
           const mdPath = path.join(STORAGE_DIR, filename)
           const jsonPath = path.join(STORAGE_DIR, filename.replace('.md', '.json'))
@@ -177,13 +162,42 @@ export async function GET(request: Request) {
         metadata?: any;
       }
 
-      // Removed fetching and combining of in-memory files as that feature was removed.
-      // We now only work with files read from disk.
-      const allFiles = diskFileDetails // Keep variable name for minimal diff, even though it's just disk files now
-
+      // Get in-memory files from the backend
+      let memoryFiles = []
+      try {
+        const backendUrl = process.env.NEXT_PUBLIC_BACKEND_URL || process.env.BACKEND_URL || 'http://localhost:24125'
+        const memoryResponse = await fetch(`${backendUrl}/api/memory-files`)
+        if (memoryResponse.ok) {
+          const memoryData = await memoryResponse.json()
+          if (memoryData.success && Array.isArray(memoryData.files)) {
+            // Convert in-memory files to the same format as disk files
+            memoryFiles = memoryData.files
+              .filter((file: MemoryFile) => !file.isJson) // Only include markdown files
+              .map((file: MemoryFile) => ({
+                name: file.name,
+                jsonPath: file.path.replace('.md', '.json'),
+                markdownPath: file.path,
+                timestamp: new Date(file.timestamp),
+                size: file.size,
+                wordCount: file.wordCount,
+                charCount: file.charCount,
+                isConsolidated: false,
+                pagesCount: 1,
+                rootUrl: '',
+                isInMemory: true
+              }))
+          }
+        }
+      } catch (e) {
+        console.error('Error fetching in-memory files:', e)
+      }
+
+      // Combine disk and memory files
+      const allFiles = [...diskFileDetails, ...memoryFiles]
+
       // Filter out individual files (non-consolidated files)
       // Only show consolidated files in the Stored Files section
-      const consolidatedFiles = allFiles.filter((file: DiskFileDetail) => file.isConsolidated)
+      const consolidatedFiles = allFiles.filter(file => file.isConsolidated)
 
       // Additional filter to exclude files with UUID-like names
       // UUID pattern: 8-4-4-4-12 hex digits (e.g., 095104d8-8e90-48f0-8670-9e45c914f115)