Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add a Vector Database Service to allow stages to read and write to VDBs #1225

Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
25 commits
Select commit Hold shift + click to select a range
5d7b3cb
Added milvus vdb prototype impl
bsuryadevara Sep 26, 2023
4807f3d
Added milvus vdb prototype impl
bsuryadevara Sep 26, 2023
b1f94fb
Added llamaindex and langchain prototypes
bsuryadevara Sep 27, 2023
d912645
doc updates
bsuryadevara Sep 27, 2023
4ecd37f
updates to milvus vd service
bsuryadevara Sep 30, 2023
c18125a
updated search and upsert functions
bsuryadevara Oct 2, 2023
a6ef60e
Added write_to_vector_db stage
bsuryadevara Oct 3, 2023
7389542
Added tests to get started
bsuryadevara Oct 3, 2023
3a31cee
Added tests to get started
bsuryadevara Oct 3, 2023
4cfba55
Added MilvusClient extension class to support missing functions
bsuryadevara Oct 4, 2023
b83f517
Added tests for Milvus vector database serivce
bsuryadevara Oct 4, 2023
b7fee57
Added tests for Milvus vector database service
bsuryadevara Oct 4, 2023
cde18b2
Added tests for Milvus vector database service
bsuryadevara Oct 4, 2023
c9316c0
Added milvus lite to pipeline tests
bsuryadevara Oct 9, 2023
36f1f18
Added tests with milvus lite
bsuryadevara Oct 11, 2023
2f24cc2
Updated Milvus VDB tests
bsuryadevara Oct 11, 2023
9670c97
Merge remote-tracking branch 'upstream/branch-23.11' into 1177-fea-ad…
bsuryadevara Oct 11, 2023
e4b8a02
Updated Milvus VDB tests
bsuryadevara Oct 11, 2023
a5e742e
Added tests with milvus lite
bsuryadevara Oct 11, 2023
3d0e01b
Renamed a file
bsuryadevara Oct 11, 2023
cd52a5f
Feedback changes
bsuryadevara Oct 12, 2023
5ce3402
Feedback changes
bsuryadevara Oct 12, 2023
9e6989a
Removed register stage decorator
bsuryadevara Oct 12, 2023
cf327b5
Ignore pymilvus in the docs
bsuryadevara Oct 13, 2023
a6a6f43
Update variable names
bsuryadevara Oct 13, 2023
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Updated Milvus VDB tests
  • Loading branch information
bsuryadevara committed Oct 11, 2023
commit e4b8a029f73191bf3efc320ceefbc30ddee89f64
31 changes: 25 additions & 6 deletions morpheus/service/milvus_vector_db_service.py
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,7 @@ def wrapper(self, name, *args, **kwargs):

class MilvusVectorDBService(VectorDBService):
"""
Service class for Milvus Vector Database implementation. This class provides methods for interacting
Service class for Milvus Vector Database implementation. This class provides functions for interacting
with a Milvus vector database.

Parameters
Expand Down Expand Up @@ -219,7 +219,6 @@ def create(self, name: str, overwrite: bool = False, **kwargs: dict[str, typing.
schema = pymilvus.CollectionSchema(fields=schema_fields, **schema_conf)

if index_conf:
# Preserve original index configuration.
field_name = index_conf.pop("field_name")
metric_type = index_conf.pop("metric_type")
index_param = self._client.prepare_index_params(field_name=field_name,
Expand All @@ -241,8 +240,7 @@ def create(self, name: str, overwrite: bool = False, **kwargs: dict[str, typing.
self._client.create_partition(collection_name=name, partition_name=part["name"], timeout=timeout)

@with_collection_lock
def insert(self, name: str, data: typing.Union[list[list], list[dict], dict], **kwargs: dict[str,
typing.Any]) -> dict:
def insert(self, name: str, data: typing.Any, **kwargs: dict[str, typing.Any]) -> dict:
"""
Insert a collection specific data in the Milvus vector database.

Expand All @@ -265,6 +263,7 @@ def insert(self, name: str, data: typing.Union[list[list], list[dict], dict], **
RuntimeError
If the collection not exists exists.
"""

return self._collection_insert(name, data, **kwargs)

def _collection_insert(self,
Expand Down Expand Up @@ -404,6 +403,11 @@ def update(self, name: str, data: list[dict], **kwargs: dict[str, typing.Any]) -
Data to be updated in the resource.
**kwargs : dict[str, typing.Any]
Extra keyword arguments specific to upsert operation.

Returns
-------
dict
Returns result of the updated operation stats.
"""

if not isinstance(data, list):
Expand Down Expand Up @@ -440,7 +444,7 @@ def delete_by_keys(self, name: str, keys: typing.Union[int, str, list], **kwargs
Returns
-------
typing.Any
Returns vectors of the given keys that are delete from the resource.
Returns result of the given keys that are delete from the collection.
"""

response = self._client.delete(collection_name=name, pks=keys, **kwargs)
Expand All @@ -464,7 +468,7 @@ def delete(self, name: str, expr: str, **kwargs: dict[str, typing.Any]) -> typin
Returns
-------
typing.Any
Returns vectors of the given keys that are delete from the resource.
Returns result of the given keys that are delete from the collection.
"""

return self._client.delete_by_expr(collection_name=name, expression=expr, **kwargs)
Expand All @@ -484,6 +488,11 @@ def retrieve_by_keys(self, name: str, keys: typing.Union[int, str, list], **kwar
or a list of either.
**kwargs : dict[str, typing.Any]
Additional keyword arguments for the retrieval operation.

Returns
-------
list[dict]
Returns result rows of the given keys from the collection.
"""

result = None
Expand All @@ -509,6 +518,11 @@ def count(self, name: str, **kwargs: dict[str, typing.Any]) -> int:
Name of the collection.
**kwargs : dict[str, typing.Any]
Additional keyword arguments for the count operation.

Returns
-------
int
Returns number of entities in the collection.
"""

return self._client.num_entities(collection_name=name, **kwargs)
Expand Down Expand Up @@ -575,6 +589,11 @@ def describe(self, name: str, **kwargs: dict[str, typing.Any]) -> dict:
Name of the collection.
**kwargs : dict[str, typing.Any]
Additional keyword arguments specific to the Milvus vector database.

Returns
-------
dict
Returns collection information.
"""

return self._client.describe_collection(collection_name=name, **kwargs)
Expand Down
14 changes: 12 additions & 2 deletions morpheus/service/vector_db_service.py
Original file line number Diff line number Diff line change
Expand Up @@ -118,7 +118,7 @@ def drop(self, name: str, **kwargs: dict[str, typing.Any]) -> None:
pass

@abstractmethod
def update(self, name: str, data: typing.Any, **kwargs: dict[str, typing.Any]) -> None:
def update(self, name: str, data: typing.Any, **kwargs: dict[str, typing.Any]) -> dict:
"""
Update data in the vector database.

Expand All @@ -130,6 +130,11 @@ def update(self, name: str, data: typing.Any, **kwargs: dict[str, typing.Any]) -
Data to be updated in the resource.
**kwargs : dict[str, typing.Any]
Extra keyword arguments specific to the vector database implementation.

Returns
-------
dict
Returns result of the updated operation stats.
"""

pass
Expand Down Expand Up @@ -179,6 +184,11 @@ def describe(self, name: str, **kwargs: dict[str, typing.Any]) -> dict:
Name of the resource.
**kwargs : dict[str, typing.Any]
Extra keyword arguments specific to the vector database implementation.

Returns
-------
dict
Returns resource information.
"""

pass
Expand Down Expand Up @@ -263,7 +273,7 @@ def retrieve_by_keys(self, name: str, keys: typing.Any, **kwargs: dict[str, typi
Returns
-------
typing.Any
Returns inserted vectors of the given keys that exists in the resource.
Returns rows of the given keys that exists in the resource.
"""
pass

Expand Down
Loading