-
Notifications
You must be signed in to change notification settings - Fork 20
Expand file tree
/
Copy pathgraph_manager.py
More file actions
355 lines (287 loc) · 11.7 KB
/
Copy pathgraph_manager.py
File metadata and controls
355 lines (287 loc) · 11.7 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
from entity_graph.config import setup_logger
from entity_graph.models.entity_graph import Object
logger = setup_logger(__name__)
class EntityManager:
def __init__(self, default_collection: str = "default"):
self.entities = {}
self.name_map = {}
self.collection_name_maps = {}
self.default_collection = default_collection
def refresh_name_map(self, collection: str = None):
"""
Refresh the name to entity mapping, optionally filtering by collection
:param collection: The collection to filter by (None for all collections)
:return: Updated name map
"""
if collection is None:
self.name_map = {}
for obj in self.entities.values():
if not hasattr(obj, "name"):
continue
self.name_map[obj.name] = obj
logger.debug(
f"Creating name map, from {len(self.entities)} objects created {len(self.name_map)} unique keys"
)
return self.name_map
filtered_map = {}
for obj in self.entities.values():
if not hasattr(obj, "name"):
continue
obj_collection = getattr(obj, "collection", self.default_collection)
if obj_collection == collection:
filtered_map[obj.name] = obj
self.collection_name_maps[collection] = filtered_map
logger.debug(
f"Creating collection name map, from {len(self.entities)} objects created {len(filtered_map)} unique keys"
)
return filtered_map
def get_name_map(self, collection: str = None):
"""
Get the name map for a specific collection
:param collection: Collection name (None for full name map)
:return: Dictionary mapping names to entities
"""
if collection is None:
if not self.name_map:
self.refresh_name_map()
return self.name_map
if collection not in self.collection_name_maps:
self.refresh_name_map(collection)
return self.collection_name_maps.get(collection, {})
def named_entity(self, name, collection: str = None):
"""
Get an entity by name, optionally filtering by collection
:param name: Name of the entity to retrieve
:param collection: The collection to search in (None for all collections)
:return: Entity with the specified name, or None if not found
"""
name_map = self.get_name_map(collection)
return name_map.get(name)
def get_named_entity_relations(self, name, relation_type, collection: str = None):
"""
Get relations of a named entity, optionally filtering by collection
:param name: Name of the entity
:param relation_type: Type of relation to filter by
:param collection: The collection to search in
:return: List of related entities
"""
entity = self.find_entity(name, collection=collection)
if not entity:
return []
related_ids = entity.get_relations(relation_type)
if collection is None:
return [self.get_entity(eid) for eid in related_ids]
return [
self.get_entity(eid)
for eid in related_ids
if getattr(self.get_entity(eid), "collection", self.default_collection)
== collection
]
def add_entity(self, entity, collection: str = None):
"""
Add an entity to the manager
:param entity: The entity to add
:param collection: Collection to assign to the entity
"""
if collection is not None:
setattr(entity, "collection", collection)
self.entities[entity.entity_id] = entity
# Clear all name maps to force refresh
self.name_map = {}
self.collection_name_maps = {}
def list_entities_by_collection(self, collection: str = None):
"""
List entities filtered by collection
:param collection: Collection to filter by (None for all)
:return: List of entities
"""
if collection is None:
return list(self.entities.values())
return [
entity
for entity in self.entities.values()
if getattr(entity, "collection", self.default_collection) == collection
]
def get_entity(self, entity_id):
"""Retrieve an entity by its ID."""
return self.entities.get(entity_id)
def find_entity(self, entity_name, filtering_func=None, collection: str = None):
"""
Retrieve an entity by its name, optionally filtering by collection
:param entity_name: Name of the entity to find
:param filtering_func: Additional filtering function
:param collection: The collection to search in
:return: Entity with the specified name, or None if not found
"""
if collection is not None:
entity = self.named_entity(entity_name, collection)
if entity and (filtering_func is None or filtering_func(entity)):
return entity
return None
entities = []
for entity in self.entities.values():
if not hasattr(entity, "name") or entity.name != entity_name:
continue
if (
collection is not None
and getattr(entity, "collection", self.default_collection) != collection
):
continue
if filtering_func is not None and not filtering_func(entity):
continue
entities.append(entity)
if len(entities) == 1:
return entities[0]
if len(entities) == 0:
return None
raise ValueError(
f"Multiple entities with name {entity_name} found"
+ (f" in collection {collection}" if collection else "")
)
def add_relation(
self, entity_id1, entity_id2, relation_type="edge", is_bidirectional=True
):
"""Create a relation between two entities."""
entity1 = self.get_entity(entity_id1)
entity2 = self.get_entity(entity_id2)
if entity1 and entity2:
if is_bidirectional:
entity1.add_relation(entity2, relation_type)
entity2.add_relation(
entity1, relation_type
) # Assuming bidirectional relationships
else:
raise ValueError("Unidirectional relationships not implemented")
def get_related_entities(
self, entity_id, relation_type=None, collection: str = None
):
"""
Retrieve all related entities of a given entity
:param entity_id: ID of the entity
:param relation_type: Type of relation to filter by (None for all types)
:param collection: The collection to filter related entities by
:return: List of related entities
"""
entity = self.get_entity(entity_id)
if not entity:
return []
if relation_type:
relations = [
(rtype, eid, rdata)
for rtype, eid, rdata in entity.relations
if rtype == relation_type
]
else:
relations = entity.relations
related_entities = []
for _, eid, _ in relations:
related_entity = self.get_entity(eid)
if related_entity is None:
continue
if (
collection is None
or getattr(related_entity, "collection", self.default_collection)
== collection
):
related_entities.append(related_entity)
return related_entities
def get_table_rows(self, table, collection: str = None):
"""
Get rows from a table, optionally filtering by collection
:param table: The table to get rows from
:param collection: The collection to filter rows by
:return: List of row entities
"""
row_ids = table.get_rows_ids()
rows = []
for row_id in row_ids:
row = self.get_entity(row_id)
if row is None:
continue
if (
collection is None
or getattr(row, "collection", self.default_collection) == collection
):
rows.append(row)
return rows
def get_table_contents_dict(self, table, idx_col, collection: str = None):
"""
Get table contents as a dictionary, optionally filtering by collection
:param table: The table to get contents from
:param idx_col: Column to use as index
:param collection: The collection to filter contents by
:return: Dictionary of table contents
"""
row_ids = table.get_rows_ids()
data = {}
for row_id in row_ids:
row = self.get_entity(row_id)
if row is None or not hasattr(row, "data") or idx_col not in row.data:
continue
if (
collection is not None
and getattr(row, "collection", self.default_collection) != collection
):
continue
data[str(row.data[idx_col])] = {"source_entity": row_id, **row.data}
return data
def export_objects_graph(self, base_relation="parent_of", collection: str = None):
"""
Export objects graph, optionally filtering by collection
:param base_relation: Relation type to include
:param collection: The collection to filter by
:return: Tuple of nodes, edges, and metadata
"""
objects = []
for obj_id, obj in self.entities.items():
if not (
isinstance(obj, Object) or getattr(obj, "entity_type", None) == "object"
):
continue
if (
collection is not None
and getattr(obj, "collection", self.default_collection) != collection
):
continue
objects.append((obj_id, obj))
nodes = [obj[1].name for obj in objects]
metadata = {}
for _, obj in objects:
metadata[obj.name] = {
**obj.attributes,
**obj.extraction_metadata,
"collection": getattr(obj, "collection", self.default_collection),
}
edges = []
for _, obj in objects:
parent_relations = obj.get_relations(base_relation)
for o_id in parent_relations:
related_entity = self.get_entity(o_id)
if related_entity is None or related_entity.name not in nodes:
continue
if (
collection is not None
and getattr(related_entity, "collection", self.default_collection)
!= collection
):
continue
edges.append((obj.name, related_entity.name))
return nodes, edges, metadata
def list_collections(self):
"""
Get a list of all collections in use by the entities
:return: List of collection names
"""
collections = set()
for entity in self.entities.values():
coll = getattr(entity, "collection", self.default_collection)
if coll:
collections.add(coll)
return sorted(list(collections))
def __repr__(self):
"""Return a summary of stored entities."""
collections = self.list_collections()
collection_counts = {
c: len(self.list_entities_by_collection(c)) for c in collections
}
return f"EntityManager({len(self.entities)} entities across {len(collections)} collections: {collection_counts})"