Add Gemini

langchain_benchmarks/model_registration.py

@@ -252,6 +252,18 @@
         },
     ),
 ]
+_GOOGLE_GENAI_MODELS = [
+    RegisteredModel(
+        provider="google-genai",
+        name="gemini-pro",
+        description="Gemini Pro is a large model from Google trained on a diverse set of tasks.",
+        type="chat",
+        params={
+            "model": "gemini-pro",
+            "convert_system_message_to_human": True,
+        },
+    )
+]
 
 _ANYSCALE_MODELS = [
     RegisteredModel(
@@ -270,4 +282,5 @@
     + _FIREWORKS_MODELS
     + _ANYSCALE_MODELS
     + _ANTHROPIC_MODELS
+    + _GOOGLE_GENAI_MODELS
 )

langchain_benchmarks/schema.py

@@ -253,7 +253,7 @@ def add(self, task: BaseTask) -> None:
 
 Provider = Literal["fireworks", "openai", "anthropic", "anyscale"]
 ModelType = Literal["chat", "llm"]
-AUTHORIZED_NAMESPACES = {"langchain"}
+AUTHORIZED_NAMESPACES = {"langchain", "langchain_google_genai"}
 
 
 def _get_model_class_from_path(
@@ -289,6 +289,10 @@ def _get_default_path(provider: str, type_: ModelType) -> str:
         ("openai", "chat"): "langchain.chat_models.openai.ChatOpenAI",
         ("openai", "llm"): "langchain.llms.openai.OpenAI",
         ("anthropic", "chat"): "langchain.chat_models.anthropic.ChatAnthropic",
+        (
+            "google-genai",
+            "chat",
+        ): "langchain_google_genai.chat_models.ChatGoogleGenerativeAI",
     }
 
     if (provider, type_) not in paths:
@@ -307,6 +311,8 @@ def _get_default_url(provider: str, type_: ModelType) -> Optional[str]:
         return "https://docs.anthropic.com/claude/reference/selecting-a-model"
     elif provider == "anyscale":
         return "https://docs.endpoints.anyscale.com/category/supported-models"
+    elif provider == "google-genai":
+        return "https://ai.google.dev/"
     else:
         return None