Deduplication tab

backend/example.env

@@ -25,6 +25,8 @@ NEO4J_USER_AGENT=""
 ENABLE_USER_AGENT = ""
 LLM_MODEL_CONFIG_model_version=""
 ENTITY_EMBEDDING="" True or False
+DUPLICATE_SCORE_VALUE = ""
+DUPLICATE_TEXT_DISTANCE = ""
 #examples
 LLM_MODEL_CONFIG_azure_ai_gpt_35="azure_deployment_name,azure_endpoint or base_url,azure_api_key,api_version"
 LLM_MODEL_CONFIG_azure_ai_gpt_4o="gpt-4o,https://YOUR-ENDPOINT.openai.azure.com/,azure_api_key,api_version"

backend/score.py

@@ -600,6 +600,42 @@ async def get_unconnected_nodes_list(uri=Form(), userName=Form(), password=Form(
         if graph is not None:
             close_db_connection(graph,"delete_unconnected_nodes")
         gc.collect()
+
+@app.post("/get_duplicate_nodes")
+async def get_duplicate_nodes(uri=Form(), userName=Form(), password=Form(), database=Form()):
+    try:
+        graph = create_graph_database_connection(uri, userName, password, database)
+        graphDb_data_Access = graphDBdataAccess(graph)
+        nodes_list, total_nodes = graphDb_data_Access.get_duplicate_nodes_list()
+        return create_api_response('Success',data=nodes_list, message=total_nodes)
+    except Exception as e:
+        job_status = "Failed"
+        message="Unable to get the list of duplicate nodes"
+        error_message = str(e)
+        logging.exception(f'Exception in getting list of duplicate nodes:{error_message}')
+        return create_api_response(job_status, message=message, error=error_message)
+    finally:
+        if graph is not None:
+            close_db_connection(graph,"get_duplicate_nodes")
+        gc.collect()
+
+@app.post("/merge_duplicate_nodes")
+async def merge_duplicate_nodes(uri=Form(), userName=Form(), password=Form(), database=Form(),duplicate_nodes_list=Form()):
+    try:
+        graph = create_graph_database_connection(uri, userName, password, database)
+        graphDb_data_Access = graphDBdataAccess(graph)
+        result = graphDb_data_Access.merge_duplicate_nodes(duplicate_nodes_list)
+        return create_api_response('Success',data=result,message="Duplicate entities merged successfully")
+    except Exception as e:
+        job_status = "Failed"
+        message="Unable to merge the duplicate nodes"
+        error_message = str(e)
+        logging.exception(f'Exception in merge the duplicate nodes:{error_message}')
+        return create_api_response(job_status, message=message, error=error_message)
+    finally:
+        if graph is not None:
+            close_db_connection(graph,"merge_duplicate_nodes")
+        gc.collect()
 
 if __name__ == "__main__":
     uvicorn.run(app)

backend/src/graphDB_dataAccess.py

@@ -241,4 +241,63 @@ def delete_unconnected_nodes(self,unconnected_entities_list):
         DETACH DELETE e
         """
         param = {"elementIds":entities_list}
+        return self.execute_query(query,param)
+
+    def get_duplicate_nodes_list(self):
+        score_value = float(os.environ.get('DUPLICATE_SCORE_VALUE'))
+        text_distance = int(os.environ.get('DUPLICATE_TEXT_DISTANCE'))
+        query_duplicate_nodes = """
+                MATCH (n:!Chunk&!Document) with n 
+                WHERE n.embedding is not null and n.id is not null // and size(n.id) > 3
+                WITH n ORDER BY count {{ (n)--() }} DESC, size(n.id) DESC // updated
+                WITH collect(n) as nodes
+                UNWIND nodes as n
+                WITH n, [other in nodes 
+                // only one pair, same labels e.g. Person with Person
+                WHERE elementId(n) < elementId(other) and labels(n) = labels(other)
+                // at least embedding similarity of X
+                AND 
+                (vector.similarity.cosine(other.embedding, n.embedding) > $duplicate_score_value
+                OR
+                // either contains each other as substrings or has a text edit distinct of less than 3
+                toLower(n.id) CONTAINS toLower(other.id) OR 
+                toLower(other.id) CONTAINS toLower(n.id)
+                OR (size(n.id)>5 AND apoc.text.distance(toLower(n.id), toLower(other.id)) < $duplicate_text_distance)
+                )] as similar
+                WHERE size(similar) > 0 
+                OPTIONAL MATCH (doc:Document)<-[:PART_OF]-(c:Chunk)-[:HAS_ENTITY]->(n)
+                {return_statement}
+                """
+        return_query_duplicate_nodes = """
+                RETURN n {.*, embedding:null, elementId:elementId(n), labels:labels(n)} as e, 
+                [s in similar | s {.id, .description, labels:labels(s), elementId: elementId(s)}] as similar,
+                collect(distinct doc.fileName) as documents, count(distinct c) as chunkConnections
+                ORDER BY e.id ASC
+                """
+        return_query_duplicate_nodes_total = "RETURN COUNT(DISTINCT(n)) as total"
+
+        param = {"duplicate_score_value": score_value, "duplicate_text_distance" : text_distance}
+
+        nodes_list = self.execute_query(query_duplicate_nodes.format(return_statement=return_query_duplicate_nodes),param=param)
+        total_nodes = self.execute_query(query_duplicate_nodes.format(return_statement=return_query_duplicate_nodes_total),param=param)
+        return nodes_list, total_nodes[0]
+
+    def merge_duplicate_nodes(self,duplicate_nodes_list):
+        nodes_list = json.loads(duplicate_nodes_list)
+        print(f'Nodes list to merge {nodes_list}')
+        query = """
+        UNWIND $rows AS row
+        CALL { with row
+        MATCH (first) WHERE elementId(first) = row.firstElementId
+        MATCH (rest) WHERE elementId(rest) IN row.similarElementIds
+        WITH first, collect (rest) as rest
+        WITH [first] + rest as nodes
+        CALL apoc.refactor.mergeNodes(nodes, 
+        {properties:"discard",mergeRels:true, produceSelfRel:false, preserveExistingSelfRels:false, singleElementAsArray:true}) 
+        YIELD node
+        RETURN size(nodes) as mergedCount
+        }
+        RETURN sum(mergedCount) as totalMerged
+        """
+        param = {"rows":nodes_list}
         return self.execute_query(query,param)

frontend/src/components/ChatBot/ChatInfoModal.tsx

@@ -293,7 +293,7 @@ const ChatInfoModal: React.FC<chatInfoMessage> = ({
                     >
                       <div style={{ backgroundColor: calcWordColor(Object.keys(label)[0]) }} className='legend mr-2'>
                         {
-                          //@ts-ignore
+                          // @ts-ignore
                           label[Object.keys(label)[0]].id ?? Object.keys(label)[0]
                         }
                       </div>