User flow changes for recreating supported vector index (#682)

* removed the if check

* Add one more check for create vector index when chunks are exist without embeddings

* removed local files

* condition changes

* chunks exists check

* chunk exists without embeddings check

* vector Index issue fixed

* vector index with different dimension

* Update graphDB_dataAccess.py

---------

Co-authored-by: Pravesh Kumar <121786590+praveshkumar1988@users.noreply.github.com>

Author: kartikpersistent

backend/files/About Amazon.pdf

@@ -611,11 +611,11 @@ async def merge_duplicate_nodes(uri=Form(), userName=Form(), password=Form(), da
         gc.collect()
 
 @app.post("/drop_create_vector_index")
-async def merge_duplicate_nodes(uri=Form(), userName=Form(), password=Form(), database=Form(), is_vector_index_recreate=Form()):
+async def merge_duplicate_nodes(uri=Form(), userName=Form(), password=Form(), database=Form(), isVectorIndexExist=Form()):
     try:
         graph = create_graph_database_connection(uri, userName, password, database)
         graphDb_data_Access = graphDBdataAccess(graph)
-        result = graphDb_data_Access.drop_create_vector_index(is_vector_index_recreate)
+        result = graphDb_data_Access.drop_create_vector_index(isVectorIndexExist)
         return create_api_response('Success',message=result)
     except Exception as e:
         job_status = "Failed"

backend/files/Copy1_patrick_pichette_-_wikipedia.pdf

@@ -158,23 +158,34 @@ def connection_check_and_get_vector_dimensions(self):
                                     WHERE type = 'VECTOR' AND name = 'vector'
                                     RETURN options.indexConfig['vector.dimensions'] AS vector_dimensions
                                 """)
+        
+        result_chunks = self.graph.query("""match (c:Chunk) return size(c.embedding) as embeddingSize, count(*) as chunks, 
+                                                    count(c.embedding) as hasEmbedding
+                                """)
+        
         embedding_model = os.getenv('EMBEDDING_MODEL')
         embeddings, application_dimension = load_embedding_model(embedding_model)
         logging.info(f'embedding model:{embeddings} and dimesion:{application_dimension}')
+        # print(chunks_exists)
 
         if self.graph:
             if len(db_vector_dimension) > 0:
                 return {'db_vector_dimension': db_vector_dimension[0]['vector_dimensions'], 'application_dimension':application_dimension, 'message':"Connection Successful"}
             else:
-                logging.info("Vector index does not exist in database")
-                return {'db_vector_dimension': 0, 'application_dimension':application_dimension, 'message':"Connection Successful"}
+                if len(db_vector_dimension) == 0 and len(result_chunks) == 0:
+                    logging.info("Chunks and vector index does not exists in database")
+                    return {'db_vector_dimension': 0, 'application_dimension':application_dimension, 'message':"Connection Successful","chunks_exists":False}
+                elif len(db_vector_dimension) == 0 and result_chunks[0]['hasEmbedding']==0 and result_chunks[0]['chunks'] > 0:
+                    return {'db_vector_dimension': 0, 'application_dimension':application_dimension, 'message':"Connection Successful","chunks_exists":True}
+                else:
+                    return {'message':"Connection Successful"}
 
     def execute_query(self, query, param=None):
         return self.graph.query(query, param)
 
     def get_current_status_document_node(self, file_name):
         query = """
-                MATCH(d:Document {fileName : $file_name}) RETURN d.status AS Status , d.processingTime AS processingTime, 
+                MATCH(d:Document {fileName : $file_name}) RETURN d.stats AS Status , d.processingTime AS processingTime, 
                 d.nodeCount AS nodeCount, d.model as model, d.relationshipCount as relationshipCount,
                 d.total_pages AS total_pages, d.total_chunks AS total_chunks , d.fileSize as fileSize, 
                 d.is_cancelled as is_cancelled, d.processed_chunk as processed_chunk, d.fileSource as fileSource
@@ -322,15 +333,16 @@ def merge_duplicate_nodes(self,duplicate_nodes_list):
         param = {"rows":nodes_list}
         return self.execute_query(query,param)
 
-    def drop_create_vector_index(self, is_vector_index_recreate):
+    def drop_create_vector_index(self, isVectorIndexExist):
         """
         drop and create the vector index when vector index dimesion are different.
         """
         embedding_model = os.getenv('EMBEDDING_MODEL')
         embeddings, dimension = load_embedding_model(embedding_model)
-        if is_vector_index_recreate == 'true':
-            self.graph.query("""drop index vector""")
 
+        if isVectorIndexExist == 'true':
+            self.graph.query("""drop index vector""")
+        # self.graph.query("""drop index vector""")
         self.graph.query("""CREATE VECTOR INDEX `vector` if not exists for (c:Chunk) on (c.embedding)
                             OPTIONS {indexConfig: {
                             `vector.dimensions`: $dimensions,
@@ -341,4 +353,4 @@ def drop_create_vector_index(self, is_vector_index_recreate):
                             "dimensions" : dimension
                         }
                         )
-        return "Drop and Re-Create vector index succesfully"
+        return "Drop and Re-Create vector index succesfully"

backend/files/Copy_IndianStockMkt,1.pdf

@@ -47,9 +47,10 @@ const Content: React.FC<ContentProps> = ({
   const isTablet = useMediaQuery(`(min-width:${breakpoints.xs}) and (max-width: ${breakpoints.lg})`);
   const [init, setInit] = useState<boolean>(false);
   const [openConnection, setOpenConnection] = useState<connectionState>({
-    isvectorIndexMatch: true,
     openPopUp: false,
-    novectorindexInDB: true,
+    chunksExists: false,
+    vectorIndexMisMatch: false,
+    chunksExistsWithDifferentDimension: false,
   });
   const [openGraphView, setOpenGraphView] = useState<boolean>(false);
   const [inspectedName, setInspectedName] = useState<string>('');
@@ -418,14 +419,24 @@ const Content: React.FC<ContentProps> = ({
               userDbVectorIndex: response.data.data.db_vector_dimension,
             })
           );
-          if (response.data.data.application_dimension === response.data.data.db_vector_dimension) {
+          if (
+            (response.data.data.application_dimension === response.data.data.db_vector_dimension ||
+              response.data.data.db_vector_dimension == 0) &&
+            !response.data.data.chunks_exists
+          ) {
             setConnectionStatus(true);
             setOpenConnection((prev) => ({ ...prev, openPopUp: false }));
           } else {
             setOpenConnection({
-              isvectorIndexMatch: false,
               openPopUp: true,
-              novectorindexInDB: response.data.data.db_vector_dimension === 0,
+              chunksExists: response.data.data.chunks_exists as boolean,
+              vectorIndexMisMatch:
+                response.data.data.db_vector_dimension > 0 &&
+                response.data.data.db_vector_dimension != response.data.data.application_dimension,
+              chunksExistsWithDifferentDimension:
+                response.data.data.db_vector_dimension > 0 &&
+                response.data.data.db_vector_dimension != response.data.data.application_dimension &&
+                (response.data.data.chunks_exists ?? true),
             });
             setConnectionStatus(false);
           }
@@ -539,8 +550,9 @@ const Content: React.FC<ContentProps> = ({
               open={openConnection.openPopUp}
               setOpenConnection={setOpenConnection}
               setConnectionStatus={setConnectionStatus}
-              isVectorIndexMatch={openConnection.isvectorIndexMatch}
-              noVectorIndexFound={openConnection.novectorindexInDB}
+              isVectorIndexMatch={openConnection.vectorIndexMisMatch}
+              chunksExistsWithoutEmbedding={openConnection.chunksExists}
+              chunksExistsWithDifferentEmbedding={openConnection.chunksExistsWithDifferentDimension}
             />
           </Suspense>