Dev

README.md

@@ -171,22 +171,6 @@ VITE_BACKEND_API_URL=${VITE_BACKEND_API_URL-backendurl}
 7. Enjoy Graph Building.
 ---
 
-## Additional Configuration
-
-### **LLM Models**
-Configure LLM models using the `VITE_LLM_MODELS_PROD` variable. Example:
-```bash
-VITE_LLM_MODELS_PROD="openai_gpt_4o,openai_gpt_4o_mini,diffbot,gemini_1.5_flash"
-```
-
-### **Input Sources**
-The default input sources are: `local`, `YouTube`, `Wikipedia`, `AWS S3`, and `web`. 
-
-To enable GCS integration, include `gcs` and your Google client ID:
-```bash
-VITE_REACT_APP_SOURCES="local,youtube,wiki,s3,gcs,web"
-VITE_GOOGLE_CLIENT_ID="your-google-client-id"
-```
 
 ## Usage
 1. Connect to Neo4j Aura Instance which can be both AURA DS or AURA DB by passing URI and password through Backend env, fill using login dialog or drag and drop the Neo4j credentials file.
@@ -237,16 +221,16 @@ VITE_GOOGLE_CLIENT_ID="your-google-client-id"
 | RAGAS_EMBEDDING_MODEL         | Optional      | openai              | embedding model used by ragas evaluation framework                               |
 |                                                                                                                                                                        |
 | **FRONTEND ENV** 
-| VITE_BACKEND_API_URL         | Optional           | http://localhost:8000 | URL for backend API                                                                       |
-| VITE_BLOOM_URL               | Optional           | https://workspace-preview.neo4j.io/workspace/explore?connectURL={CONNECT_URL}&search=Show+me+a+graph&featureGenAISuggestions=true&featureGenAISuggestionsInternal=true | URL for Bloom visualization |
+| VITE_BLOOM_URL               | Mandatory           | https://workspace-preview.neo4j.io/workspace/explore?connectURL={CONNECT_URL}&search=Show+me+a+graph&featureGenAISuggestions=true&featureGenAISuggestionsInternal=true | URL for Bloom visualization |
 | VITE_REACT_APP_SOURCES       | Mandatory          | local,youtube,wiki,s3 | List of input sources that will be available                                               |
 | VITE_CHAT_MODES              | Mandatory          | vector,graph+vector,graph,hybrid | Chat modes available for Q&A
-| VITE_ENV                     | Mandatory          | DEV or PROD           | Environment variable for the app                                                                 |
+| VITE_ENV                     | Mandatory          | DEV or PROD           | Environment variable for the app|
+| VITE_LLM_MODELS              | Mandatory | 'diffbot,openai_gpt_3.5,openai_gpt_4o,openai_gpt_4o_mini,gemini_1.5_pro,gemini_1.5_flash,azure_ai_gpt_35,azure_ai_gpt_4o,ollama_llama3,groq_llama3_70b,anthropic_claude_3_5_sonnet' | Supported Models For the application
+| VITE_BACKEND_API_URL         | Optional           | http://localhost:8000 | URL for backend API    |
 | VITE_TIME_PER_PAGE          | Optional           | 50             | Time per page for processing                                                                    |
 | VITE_CHUNK_SIZE              | Optional           | 5242880       | Size of each chunk of file for upload                                                                |
 | VITE_GOOGLE_CLIENT_ID        | Optional           |               | Client ID for Google authentication                                                              |
-| VITE_LLM_MODELS_PROD         | Optional      | openai_gpt_4o,openai_gpt_4o_mini,diffbot,gemini_1.5_flash | To Distinguish models based on the Enviornment PROD or DEV 
-| VITE_LLM_MODELS              | Optional | 'diffbot,openai_gpt_3.5,openai_gpt_4o,openai_gpt_4o_mini,gemini_1.5_pro,gemini_1.5_flash,azure_ai_gpt_35,azure_ai_gpt_4o,ollama_llama3,groq_llama3_70b,anthropic_claude_3_5_sonnet' | Supported Models For the application
+| VITE_LLM_MODELS_PROD         | Optional      | openai_gpt_4o,openai_gpt_4o_mini,diffbot,gemini_1.5_flash | To Distinguish models based on the Enviornment PROD or DEV |
 | VITE_AUTH0_CLIENT_ID | Mandatory if you are enabling Authentication otherwise it is optional |       |Okta Oauth Client ID for authentication
 | VITE_AUTH0_DOMAIN | Mandatory if you are enabling Authentication otherwise it is optional |           | Okta Oauth Cliend Domain
 | VITE_SKIP_AUTH | Optional | true | Flag to skip the authentication 

backend/src/document_sources/wikipedia.py

@@ -11,6 +11,6 @@ def get_documents_from_Wikipedia(wiki_query:str, language:str):
   except Exception as e:
     message="Failed To Process Wikipedia Query"
     error_message = str(e)
-    logging.exception(f'Failed To Process Wikipedia Query: {file_name}, Exception Stack trace: {error_message}')
+    logging.exception(f'Failed To Process Wikipedia Query, Exception Stack trace: {error_message}')
     raise LLMGraphBuilderException(error_message+' '+message)
 

backend/test_integrationqa.py

@@ -8,8 +8,7 @@
 from dotenv import load_dotenv
 from src.main import *
 from src.QA_integration import QA_RAG
-from src.ragas_eval import get_ragas_metrics
-from datasets import Dataset
+
 # Load environment variables
 load_dotenv()
 URI = os.getenv('NEO4J_URI')
@@ -49,7 +48,7 @@ def test_graph_from_file_local(model_name):
        create_source_node_local(graph, model_name, file_name)
        result = asyncio.run(
            extract_graph_from_file_local_file(
-               URI, USERNAME, PASSWORD, DATABASE, model_name, merged_file_path, file_name, '', '', None, ''
+               URI, USERNAME, PASSWORD, DATABASE, model_name, merged_file_path, file_name, '', '',100,20,1, None,''
            )
        )
        logging.info(f"Local file test result: {result}")
@@ -67,7 +66,7 @@ def test_graph_from_wikipedia(model_name):
        create_source_node_graph_url_wikipedia(graph, model_name, wiki_query, "Wikipedia")
        result = asyncio.run(
            extract_graph_from_file_Wikipedia(
-               URI, USERNAME, PASSWORD, DATABASE, model_name, file_name, 'en', file_name, '', '', None, ''
+               URI, USERNAME, PASSWORD, DATABASE, model_name, file_name, 'en', file_name, '', '', 100,20,1,None,''
            )
        )
        logging.info(f"Wikipedia test result: {result}")
@@ -85,9 +84,9 @@ def test_graph_from_youtube_video(model_name):
        create_source_node_graph_url_youtube(graph, model_name, source_url, "youtube")
        result = asyncio.run(
            extract_graph_from_file_youtube(
-               URI, USERNAME, PASSWORD, DATABASE, model_name, source_url, file_name, '', '', None, ''
+               URI, USERNAME, PASSWORD, DATABASE, model_name, source_url, file_name, '', '',100,20,1, None,''
            )
-       )
+      )
        logging.info(f"YouTube video test result: {result}")
        if isinstance(result, dict) and result.get("status") == "Failed":
            return {"status": "Failed", "error": result.get("error", "Unknown error")}
@@ -104,7 +103,7 @@ def test_graph_website(model_name):
        create_source_node_graph_web_url(graph, model_name, source_url, "web-url")
        result = asyncio.run(
            extract_graph_from_web_page(
-               URI, USERNAME, PASSWORD, DATABASE, model_name, source_url, "Google Cloud Skills Boost", '', '', None, ''
+               URI, USERNAME, PASSWORD, DATABASE, model_name, source_url, "Google Cloud Skills Boost-www", '', '',100,20,1, None,''
            )
        )
        logging.info(f"Web URL test result: {result}")
@@ -122,7 +121,8 @@ def test_chatbot_qna(model_name, mode='vector'):
        result = QA_RAG(graph, model_name, 'Tell me about Amazon', '[]', 1, mode)
     #    assert len(result['message']) > 20
        logging.info(f"Chatbot QnA test passed for mode: {mode}")
-       return result
+       final_result = {'model_name':model_name,'mode':mode,'result':result}
+       return final_result
    except Exception as e:
        logging.error(f"Error in chatbot QnA: {e}")
        return {"status": "Failed", "error": str(e)}
@@ -170,15 +170,34 @@ def get_duplicate_nodes():
            return "Data successfully loaded"
        else:
            return "Unable to load data"
-
+
+def flatten_extract_dataframe(df: pd.DataFrame):
+   rows = []
+   for _, row in df.iterrows():
+       try:
+           col1, col2, execution_date = row[0], row[1], row[2] if len(row) > 2 else None
+           data_dict = ast.literal_eval(col1) if isinstance(col1, str) and col1.startswith("{") else {}
+           meta_dict = ast.literal_eval(col2) if isinstance(col2, str) and col2.startswith("{") else {}
+           combined_dict = {**data_dict, **meta_dict}
+           for key in combined_dict.keys():
+               if isinstance(combined_dict[key], dict) and key.startswith("processed_chunk_detail"):
+                   combined_dict[key] = str(combined_dict[key])
+           combined_dict["execution_date"] = execution_date
+           rows.append(combined_dict)
+       except (SyntaxError, ValueError, TypeError) as e:
+           print(f"Error parsing row: {row} - {e}")
+           continue
+   flat_df = pd.DataFrame(rows)
+   return flat_df
+
 def run_tests():
    """Runs all integration tests and logs results."""
    extract_list = []
    extract_error_list = []
    chatbot_list = []
    chatbot_error_list = []
    other_api_list = []
-   models = ['openai_gpt_4','openai_gpt_4o','openai_gpt_4o_mini','gemini_1.5_pro','gemini_1.5_flash','gemini_2.0_flash','bedrock_nova_micro_v1','bedrock_nova_lite_v1','bedrock_nova_pro_v1','fireworks_qwen72b_instruct']
+   models = ['openai_gpt_4o','openai_gpt_4o_mini','openai_gpt_4.1','openai_gpt_4.1_mini','gemini_2.0_flash','fireworks_llama4_maverick','bedrock_nova_pro_v1']
    chatbot_modes = [
        "vector",
        "graph+vector",
@@ -218,9 +237,7 @@ def run_tests():
 
        try:
             schema_result = test_populate_graph_schema_from_text(model_name)
-            print("KAUSTUBH : ",schema_result)
             other_api_list.append({f"{model_name}":schema_result}) 
-            print("other_api_list : ",other_api_list)
        except Exception as e:
            logging.error(f"Error in test_populate_graph_schema_from_text for {model_name}: {e}")
            other_api_list.append({f"{model_name}":str(e)}) 

frontend/package.json

@@ -32,9 +32,11 @@
     "react": "^18.3.1",
     "react-dom": "^18.3.1",
     "react-icons": "^5.5.0",
-    "react-markdown": "^9.0.1",
+    "react-markdown": "^10.1.0",
     "react-router": "^6.23.1",
     "react-router-dom": "^6.23.1",
+    "rehype-raw": "^7.0.0",
+    "remark-gfm": "^4.0.1",
     "uuid": "^11.1.0"
   },
   "devDependencies": {

frontend/src/App.css

@@ -469,4 +469,5 @@
 .animate-highlight {
   animation: highlightAnimation 1.5s ease-in-out;
   border: 2px solid rgb(var(--theme-palette-primary-bg-strong));
-}
+}
+

frontend/src/components/ChatBot/Chatbot.tsx

@@ -41,6 +41,8 @@ import { downloadClickHandler, getDateTime } from '../../utils/Utils';
 import ChatModesSwitch from './ChatModesSwitch';
 import CommonActions from './CommonChatActions';
 import Loader from '../../utils/Loader';
+import remarkGfm from 'remark-gfm';
+import rehypeRaw from 'rehype-raw';
 const InfoModal = lazy(() => import('./ChatInfoModal'));
 if (typeof window !== 'undefined') {
   if (!sessionStorage.getItem('session_id')) {
@@ -467,9 +469,17 @@ const Chatbot: FC<ChatbotProps> = (props) => {
                         chat.isLoading && index === listMessages.length - 1 && chat.user === 'chatbot' ? 'loader' : ''
                       }`}
                     >
-                      <ReactMarkdown className={!isFullScreen ? 'max-w-[250px]' : ''}>
-                        {chat.modes[chat.currentMode]?.message || ''}
-                      </ReactMarkdown>
+                      <div
+                        className={
+                          !isFullScreen
+                            ? 'max-w-[250px] prose prose-sm sm:prose lg:prose-lg xl:prose-xl'
+                            : 'prose prose-sm sm:prose lg:prose-lg xl:prose-xl max-w-none'
+                        }
+                      >
+                        <ReactMarkdown remarkPlugins={[remarkGfm]} rehypePlugins={[rehypeRaw] as any}>
+                          {chat.modes[chat.currentMode]?.message || ''}
+                        </ReactMarkdown>
+                      </div>
                     </div>
                     <div>
                       <div>

frontend/src/components/ChatBot/ChunkInfo.tsx

@@ -13,7 +13,8 @@ import { chatModeLables } from '../../utils/Constants';
 import GraphViewModal from '../Graph/GraphViewModal';
 import { handleGraphNodeClick } from './chatInfo';
 import { IconButtonWithToolTip } from '../UI/IconButtonToolTip';
-
+import remarkGfm from 'remark-gfm';
+import rehypeRaw from 'rehype-raw';
 const ChunkInfo: FC<ChunkProps> = ({ loading, chunks, mode }) => {
   const themeUtils = useContext(ThemeWrapperContext);
   const [neoNodes, setNeoNodes] = useState<any[]>([]);
@@ -257,8 +258,10 @@ const ChunkInfo: FC<ChunkProps> = ({ loading, chunks, mode }) => {
                     </div>
                   </>
                 )}
-                <div className='mt-2'>
-                  <ReactMarkdown>{chunk?.text}</ReactMarkdown>
+                <div className='mt-2 prose prose-sm sm:prose lg:prose-lg xl:prose-xl max-w-none'>
+                  <ReactMarkdown remarkPlugins={[remarkGfm]} rehypePlugins={[rehypeRaw] as any}>
+                    {chunk?.text}
+                  </ReactMarkdown>
                 </div>
               </li>
             ))}

frontend/src/components/ChatBot/CommunitiesInfo.tsx

@@ -5,7 +5,8 @@ import { CommunitiesProps } from '../../types';
 import { chatModeLables } from '../../utils/Constants';
 import GraphViewModal from '../Graph/GraphViewModal';
 import { handleGraphNodeClick } from './chatInfo';
-
+import remarkGfm from 'remark-gfm';
+import rehypeRaw from 'rehype-raw';
 const CommunitiesInfo: FC<CommunitiesProps> = ({ loading, communities, mode }) => {
   const [neoNodes, setNeoNodes] = useState<any[]>([]);
   const [neoRels, setNeoRels] = useState<any[]>([]);
@@ -51,7 +52,11 @@ const CommunitiesInfo: FC<CommunitiesProps> = ({ loading, communities, mode }) =
                       <Typography variant='subheading-medium'>{community.score}</Typography>
                     </Flex>
                   )}
-                  <ReactMarkdown>{community.summary}</ReactMarkdown>
+                  <div className='prose prose-sm sm:prose lg:prose-lg xl:prose-xl max-w-none'>
+                    <ReactMarkdown remarkPlugins={[remarkGfm]} rehypePlugins={[rehypeRaw] as any}>
+                      {community.summary}
+                    </ReactMarkdown>
+                  </div>
                 </div>
               </li>
             ))}

frontend/src/components/Content.tsx

@@ -52,6 +52,7 @@ import { useHasSelections } from '../hooks/useHasSelections';
 import { ChevronUpIconOutline, ChevronDownIconOutline } from '@neo4j-ndl/react/icons';
 import { ThemeWrapperContext } from '../context/ThemeWrapper';
 import { useAuth0 } from '@auth0/auth0-react';
+import React from 'react';
 
 const ConfirmationDialog = lazy(() => import('./Popups/LargeFilePopUp/ConfirmationDialog'));
 
@@ -168,10 +169,10 @@ const Content: React.FC<ContentProps> = ({
               ? postProcessingTasks.filter((task) => task !== 'graph_schema_consolidation')
               : postProcessingTasks
             : hasSelections
-            ? postProcessingTasks.filter(
-                (task) => task !== 'graph_schema_consolidation' && task !== 'enable_communities'
-              )
-            : postProcessingTasks.filter((task) => task !== 'enable_communities');
+              ? postProcessingTasks.filter(
+                  (task) => task !== 'graph_schema_consolidation' && task !== 'enable_communities'
+                )
+              : postProcessingTasks.filter((task) => task !== 'enable_communities');
           if (payload.length) {
             const response = await postProcessing(payload);
             if (response.data.status === 'Success') {
@@ -236,7 +237,7 @@ const Content: React.FC<ContentProps> = ({
           ...curfile,
           model:
             curfile.status === 'New' || curfile.status === 'Ready to Reprocess'
-              ? selectedOption?.value ?? ''
+              ? (selectedOption?.value ?? '')
               : curfile.model,
         };
       });
@@ -536,7 +537,7 @@ const Content: React.FC<ContentProps> = ({
     }
   };
 
-  const processWaitingFilesOnRefresh = () => {
+  const processWaitingFilesOnRefresh = useCallback(() => {
     let data = [];
     const processingFilesCount = filesData.filter((f) => f.status === 'Processing').length;
 
@@ -556,7 +557,7 @@ const Content: React.FC<ContentProps> = ({
         .filter((f) => f.status === 'New' || f.status == 'Ready to Reprocess');
       addFilesToQueue(selectedNewFiles as CustomFile[]);
     }
-  };
+  }, [filesData, queue]);
 
   const handleOpenGraphClick = () => {
     const bloomUrl = process.env.VITE_BLOOM_URL;
@@ -966,26 +967,29 @@ const Content: React.FC<ContentProps> = ({
         <FileTable
           connectionStatus={connectionStatus}
           setConnectionStatus={setConnectionStatus}
-          onInspect={(name) => {
+          onInspect={useCallback((name) => {
             setInspectedName(name);
             setOpenGraphView(true);
             setViewPoint('tableView');
-          }}
-          onRetry={(id) => {
+          }, [])}
+          onRetry={useCallback((id) => {
             setRetryFile(id);
             toggleRetryPopup();
-          }}
-          onChunkView={async (name) => {
-            setDocumentName(name);
-            if (name != documentName) {
-              toggleChunkPopup();
-              if (totalPageCount) {
-                setTotalPageCount(null);
+          }, [])}
+          onChunkView={useCallback(
+            async (name) => {
+              setDocumentName(name);
+              if (name != documentName) {
+                toggleChunkPopup();
+                if (totalPageCount) {
+                  setTotalPageCount(null);
+                }
+                setCurrentPage(1);
+                await getChunks(name, 1);
               }
-              setCurrentPage(1);
-              await getChunks(name, 1);
-            }
-          }}
+            },
+            [documentName, totalPageCount]
+          )}
           ref={childRef}
           handleGenerateGraph={processWaitingFilesOnRefresh}
         ></FileTable>
@@ -1089,4 +1093,4 @@ const Content: React.FC<ContentProps> = ({
   );
 };
 
-export default Content;
+export default React.memo(Content);

frontend/src/components/FileTable.tsx

@@ -67,6 +67,7 @@ import { ThemeWrapperContext } from '../context/ThemeWrapper';
 import BreakDownPopOver from './BreakDownPopOver';
 import { InformationCircleIconOutline } from '@neo4j-ndl/react/icons';
 import { useAuth0 } from '@auth0/auth0-react';
+import React from 'react';
 
 let onlyfortheFirstRender = true;
 
@@ -89,7 +90,6 @@ const FileTable: ForwardRefRenderFunction<ChildRef, FileTableProps> = (props, re
   const islargeDesktop = useMediaQuery(`(min-width:1440px )`);
   const tableRef = useRef(null);
   const { isAuthenticated } = useAuth0();
-
   const { updateStatusForLargeFiles } = useServerSideEvent(
     (inMinutes, time, fileName) => {
       showNormalToast(`${fileName} will take approx ${time} ${inMinutes ? 'Min' : 'Sec'}`);
@@ -729,8 +729,8 @@ const FileTable: ForwardRefRenderFunction<ChildRef, FileTableProps> = (props, re
                   name: item?.fileName,
                   size: item?.fileSize ?? 0,
                   type: item?.fileType?.includes('.')
-                    ? item?.fileType?.substring(1)?.toUpperCase() ?? 'None'
-                    : item?.fileType?.toUpperCase() ?? 'None',
+                    ? (item?.fileType?.substring(1)?.toUpperCase() ?? 'None')
+                    : (item?.fileType?.toUpperCase() ?? 'None'),
                   nodesCount: item?.nodeCount ?? 0,
                   processingTotalTime: item?.processingTime ?? 'None',
                   relationshipsCount: item?.relationshipCount ?? 0,
@@ -1061,4 +1061,4 @@ const FileTable: ForwardRefRenderFunction<ChildRef, FileTableProps> = (props, re
   );
 };
 
-export default forwardRef(FileTable);
+export default React.memo(forwardRef(FileTable));