Add embed to Index configure calls (#515)

## Problem
`embed` was never exposed as an argument for calling `configure` on
`IndexResource`.

## Solution
- Add a new simple `ConfigureIndexEmbed(TypedDict)` class for
representing the argument dictionary shape. I went with this because it
aligned with the existing `CreateIndexForModelEmbedTypedDict`, but I'm
not sure if this is best practice in the repo at this point. Maybe a
class would be better.
- Update factory, sync, and async resources to pass through `embed` on
`configure` calls.
- Update legacy `Pinecone.configure_index` method to support `embed`.
- Add integration tests to serverless resources to validate converting
an existing serverless index to an integrated index using `configure` or
`configure_index`.

## Type of Change
- [ ] Bug fix (non-breaking change which fixes an issue)
- [X] New feature (non-breaking change which adds functionality)
- [ ] Breaking change (fix or feature that would cause existing
functionality to not work as expected)
- [ ] This change requires a documentation update
- [ ] Infrastructure change (CI configs, etc)
- [ ] Non-code change (docs, etc)
- [ ] None of the above: (explain here)

## Test Plan
New integration tests added. You can pull this branch down and use
poetry to run a repl and quickly evaluate things yourself:

```python
poetry run repl

>>> from pinecone import Pinecone
>>> pc = Pinecone(api_key="YOUR_API_KEY")

>>> pc.create_index(name="test-int-inf-convert", dimension=1024, metric="cosine", spec={"serverless": {"cloud": "aws", "region": "us-east-1"}})
{
    "name": "test-int-inf-convert",
    "metric": "cosine",
    "host": "test-int-inf-convert-bt8x3su.svc.preprod-aws-0.pinecone.io",
    "spec": {
        "serverless": {
            "cloud": "aws",
            "region": "us-east-1"
        }
    },
    "status": {
        "ready": true,
        "state": "Ready"
    },
    "vector_type": "dense",
    "dimension": 1024,
    "deletion_protection": "disabled",
    "tags": null
}

>>> pc.db.index.configure(name="test-int-inf-convert", embed={"model": "multilingual-e5-large", "field_map":{"text": "chunk_text"}})
>>> pc.db.index.describe(name="test-int-inf-convert")
{
    "name": "test-int-inf-convert",
    "metric": "cosine",
    "host": "test-int-inf-convert-bt8x3su.svc.preprod-aws-0.pinecone.io",
    "spec": {
        "serverless": {
            "cloud": "aws",
            "region": "us-east-1"
        }
    },
    "status": {
        "ready": true,
        "state": "Ready"
    },
    "vector_type": "dense",
    "dimension": 1024,
    "deletion_protection": "disabled",
    "tags": null,
    "embed": {
        "model": "multilingual-e5-large",
        "field_map": {
            "text": "chunk_text"
        },
        "dimension": 1024,
        "metric": "cosine",
        "write_parameters": {
            "input_type": "passage",
            "truncate": "END"
        },
        "read_parameters": {
            "input_type": "query",
            "truncate": "END"
        },
        "vector_type": "dense"
    }
}

### repeat with async resources / pc.configure_index()
```

---
- To see the specific tasks where the Asana app for GitHub is being
used, see below:
  - https://app.asana.com/0/0/1210417294961252

Author: austin-denoble

pinecone/__init__.py

@@ -98,6 +98,11 @@
     "RestoreJobList": ("pinecone.db_control.models", "RestoreJobList"),
     "BackupModel": ("pinecone.db_control.models", "BackupModel"),
     "BackupList": ("pinecone.db_control.models", "BackupList"),
+    "ConfigureIndexEmbed": ("pinecone.db_control.types", "ConfigureIndexEmbed"),
+    "CreateIndexForModelEmbedTypedDict": (
+        "pinecone.db_control.types",
+        "CreateIndexForModelEmbedTypedDict",
+    ),
 }
 
 _config_lazy_imports = {

pinecone/__init__.pyi

@@ -78,6 +78,10 @@ from pinecone.db_control.models import (
     PodSpec,
     PodSpecDefinition,
 )
+from pinecone.db_control.types import (
+    ConfigureIndexEmbed,
+    CreateIndexForModelEmbedTypedDict,
+)
 from pinecone.pinecone import Pinecone
 from pinecone.pinecone_asyncio import PineconeAsyncio
 
@@ -160,4 +164,7 @@ __all__ = [
     "ServerlessSpecDefinition",
     "PodSpec",
     "PodSpecDefinition",
+    # Control plane types
+    "ConfigureIndexEmbed",
+    "CreateIndexForModelEmbedTypedDict",
 ]

pinecone/db_control/__init__.py

@@ -1,5 +1,6 @@
 from .enums import *
 from .models import *
+from .types import *
 from .db_control import DBControl
 from .db_control_asyncio import DBControlAsyncio
 from .repr_overrides import install_repr_overrides
@@ -30,6 +31,9 @@
     "BackupList",
     "RestoreJobModel",
     "RestoreJobList",
+    # from .types
+    "ConfigureIndexEmbed",
+    "CreateIndexForModelEmbedTypedDict",
     # direct imports
     "DBControl",
     "DBControlAsyncio",

pinecone/db_control/request_factory.py

@@ -19,6 +19,9 @@
 from pinecone.core.openapi.db_control.model.configure_index_request_spec_pod import (
     ConfigureIndexRequestSpecPod,
 )
+from pinecone.core.openapi.db_control.model.configure_index_request_embed import (
+    ConfigureIndexRequestEmbed,
+)
 from pinecone.core.openapi.db_control.model.deletion_protection import (
     DeletionProtection as DeletionProtectionModel,
 )
@@ -45,7 +48,7 @@
     GcpRegion,
     AzureRegion,
 )
-from .types import CreateIndexForModelEmbedTypedDict
+from .types import CreateIndexForModelEmbedTypedDict, ConfigureIndexEmbed
 
 
 logger = logging.getLogger(__name__)
@@ -241,6 +244,7 @@ def configure_index_request(
         pod_type: Optional[Union[PodType, str]] = None,
         deletion_protection: Optional[Union[DeletionProtection, str]] = None,
         tags: Optional[Dict[str, str]] = None,
+        embed: Optional[Union[ConfigureIndexEmbed, Dict]] = None,
     ):
         if deletion_protection is None:
             dp = DeletionProtectionModel(description.deletion_protection)
@@ -271,13 +275,24 @@ def configure_index_request(
         if replicas:
             pod_config_args.update(replicas=replicas)
 
-        if pod_config_args != {}:
+        embed_config = None
+        if embed is not None:
+            embed_config = ConfigureIndexRequestEmbed(**dict(embed))
+
+        spec = None
+        if pod_config_args:
             spec = ConfigureIndexRequestSpec(pod=ConfigureIndexRequestSpecPod(**pod_config_args))
-            req = ConfigureIndexRequest(deletion_protection=dp, spec=spec, tags=IndexTags(**tags))
-        else:
-            req = ConfigureIndexRequest(deletion_protection=dp, tags=IndexTags(**tags))
 
-        return req
+        args_dict = parse_non_empty_args(
+            [
+                ("deletion_protection", dp),
+                ("tags", IndexTags(**tags)),
+                ("spec", spec),
+                ("embed", embed_config),
+            ]
+        )
+
+        return ConfigureIndexRequest(**args_dict)
 
     @staticmethod
     def create_collection_request(name: str, source: str) -> CreateCollectionRequest:

pinecone/db_control/resources/asyncio/index.py

@@ -27,6 +27,7 @@
 from pinecone.db_control.request_factory import PineconeDBControlRequestFactory
 from pinecone.core.openapi.db_control import API_VERSION
 from pinecone.utils import require_kwargs
+from pinecone.db_control.types.configure_index_embed import ConfigureIndexEmbed
 
 logger = logging.getLogger(__name__)
 """ :meta private: """
@@ -183,6 +184,7 @@ async def configure(
         pod_type: Optional[Union[PodType, str]] = None,
         deletion_protection: Optional[Union[DeletionProtection, str]] = None,
         tags: Optional[Dict[str, str]] = None,
+        embed: Optional[Union[ConfigureIndexEmbed, Dict]] = None,
     ):
         description = await self.describe(name=name)
 
@@ -192,5 +194,6 @@ async def configure(
             pod_type=pod_type,
             deletion_protection=deletion_protection,
             tags=tags,
+            embed=embed,
         )
         await self._index_api.configure_index(name, configure_index_request=req)

pinecone/db_control/resources/sync/index.py

@@ -10,6 +10,7 @@
 from pinecone.db_control.types import CreateIndexForModelEmbedTypedDict
 from pinecone.db_control.request_factory import PineconeDBControlRequestFactory
 from pinecone.core.openapi.db_control import API_VERSION
+from pinecone.db_control.types.configure_index_embed import ConfigureIndexEmbed
 
 logger = logging.getLogger(__name__)
 """ :meta private: """
@@ -224,6 +225,7 @@ def configure(
         pod_type: Optional[Union["PodType", str]] = None,
         deletion_protection: Optional[Union["DeletionProtection", str]] = None,
         tags: Optional[Dict[str, str]] = None,
+        embed: Optional[Union["ConfigureIndexEmbed", Dict]] = None,
     ) -> None:
         api_instance = self._index_api
         description = self.describe(name=name)
@@ -234,6 +236,7 @@ def configure(
             pod_type=pod_type,
             deletion_protection=deletion_protection,
             tags=tags,
+            embed=embed,
         )
         api_instance.configure_index(name, configure_index_request=req)
 

pinecone/db_control/types/__init__.py

@@ -1,3 +1,4 @@
 from .create_index_for_model_embed import CreateIndexForModelEmbedTypedDict
+from .configure_index_embed import ConfigureIndexEmbed
 
-__all__ = ["CreateIndexForModelEmbedTypedDict"]
+__all__ = ["CreateIndexForModelEmbedTypedDict", "ConfigureIndexEmbed"]

pinecone/db_control/types/configure_index_embed.py

@@ -0,0 +1,8 @@
+from typing import TypedDict, Dict, Any, Optional
+
+
+class ConfigureIndexEmbed(TypedDict):
+    model: str
+    field_map: Dict[str, str]
+    read_parameters: Optional[Dict[str, Any]]
+    write_parameters: Optional[Dict[str, Any]]

pinecone/legacy_pinecone_interface.py

@@ -26,7 +26,7 @@
         GcpRegion,
         AzureRegion,
     )
-    from pinecone.db_control.types import CreateIndexForModelEmbedTypedDict
+    from pinecone.db_control.types import CreateIndexForModelEmbedTypedDict, ConfigureIndexEmbed
 
 
 class LegacyPineconeDBControlInterface(ABC):
@@ -438,6 +438,7 @@ def configure_index(
         pod_type: Optional[Union["PodType", str]] = None,
         deletion_protection: Optional[Union["DeletionProtection", str]] = None,
         tags: Optional[Dict[str, str]] = None,
+        embed: Optional[Union["ConfigureIndexEmbed", Dict]] = None,
     ):
         """
         :param name: the name of the Index
@@ -452,6 +453,10 @@ def configure_index(
         :type deletion_protection: str or DeletionProtection, optional
         :param tags: A dictionary of tags to apply to the index. Tags are key-value pairs that can be used to organize and manage indexes. To remove a tag, set the value to "". Tags passed to configure_index will be merged with existing tags and any with the value empty string will be removed.
         :type tags: Dict[str, str], optional
+        :param embed: configures the integrated inference embedding settings for the index. You can convert an existing index to an integrated index by specifying the embedding model and field_map.
+            The index vector type and dimension must match the model vector type and dimension, and the index similarity metric must be supported by the model.
+            You can later change the embedding configuration to update the field_map, read_parameters, or write_parameters. Once set, the model cannot be changed.
+        :type embed: Optional[Union[ConfigureIndexEmbed, Dict]], optional
 
         This method is used to modify an index's configuration. It can be used to:
 

pinecone/pinecone.py

@@ -18,7 +18,7 @@
     from pinecone.db_data import _Index as Index, _IndexAsyncio as IndexAsyncio
     from pinecone.db_control.index_host_store import IndexHostStore
     from pinecone.core.openapi.db_control.api.manage_indexes_api import ManageIndexesApi
-    from pinecone.db_control.types import CreateIndexForModelEmbedTypedDict
+    from pinecone.db_control.types import CreateIndexForModelEmbedTypedDict, ConfigureIndexEmbed
     from pinecone.db_control.enums import (
         Metric,
         VectorType,
@@ -399,13 +399,15 @@ def configure_index(
         pod_type: Optional[Union["PodType", str]] = None,
         deletion_protection: Optional[Union["DeletionProtection", str]] = None,
         tags: Optional[Dict[str, str]] = None,
+        embed: Optional[Union["ConfigureIndexEmbed", Dict]] = None,
     ):
         return self.db.index.configure(
             name=name,
             replicas=replicas,
             pod_type=pod_type,
             deletion_protection=deletion_protection,
             tags=tags,
+            embed=embed,
         )
 
     def create_collection(self, name: str, source: str) -> None: