Add LangChain Docs Benchmark Evals

langchain-docs-benchmarking/README.md

@@ -0,0 +1,4 @@
+# Benchmarking on LangChain Docs
+
+
+Evaluating various approaches for RAG on LangChain docs.

langchain-docs-benchmarking/app/server.py

@@ -0,0 +1,26 @@
+from fastapi import FastAPI
+from langserve import add_routes
+from chat_langchain.chain import chain
+from openai_functions_agent import agent_executor as openai_functions_agent_chain
+
+
+app = FastAPI()
+
+# Edit this to add the chain you want to add
+add_routes(
+    app,
+    chain,
+    path="/chat",
+    # include_callback_events=True, # TODO: Include when fixed
+)
+
+add_routes(app, openai_functions_agent_chain, path="/openai-functions-agent")
+
+def run_server(port: int = 1983):
+    import uvicorn
+
+    uvicorn.run(app, host="0.0.0.0", port=port)
+
+
+if __name__ == "__main__":
+    run_server()

langchain-docs-benchmarking/main.py

@@ -0,0 +1,18 @@
+from langchain.schema.runnable import RunnableLambda
+from langserve import add_routes
+from fastapi import FastAPI
+import uuid
+
+def foo(uid: uuid.UUID) -> str:
+    return f"The id is {uid}"
+
+chain = RunnableLambda(foo)
+
+app = FastAPI()
+
+add_routes(app, chain)
+
+import uvicorn
+
+uvicorn.run(app, port=8122)
+

langchain-docs-benchmarking/packages/anthropic-iterative-search/README.md

@@ -0,0 +1,69 @@
+
+# anthropic-iterative-search
+
+This template will create a virtual research assistant with the ability to search Wikipedia to find answers to your questions.
+
+It is heavily inspired by [this notebook](https://github.com/anthropics/anthropic-cookbook/blob/main/long_context/wikipedia-search-cookbook.ipynb).
+
+## Environment Setup
+
+Set the `ANTHROPIC_API_KEY` environment variable to access the Anthropic models.
+
+## Usage
+
+To use this package, you should first have the LangChain CLI installed:
+
+```shell
+pip install -U "langchain-cli[serve]"
+```
+
+To create a new LangChain project and install this as the only package, you can do:
+
+```shell
+langchain app new my-app --package anthropic-iterative-search
+```
+
+If you want to add this to an existing project, you can just run:
+
+```shell
+langchain app add anthropic-iterative-search
+```
+
+And add the following code to your `server.py` file:
+```python
+from anthropic_iterative_search import chain as anthropic_iterative_search_chain
+
+add_routes(app, anthropic_iterative_search_chain, path="/anthropic-iterative-search")
+```
+
+(Optional) Let's now configure LangSmith. 
+LangSmith will help us trace, monitor and debug LangChain applications. 
+LangSmith is currently in private beta, you can sign up [here](https://smith.langchain.com/). 
+If you don't have access, you can skip this section
+
+
+```shell
+export LANGCHAIN_TRACING_V2=true
+export LANGCHAIN_API_KEY=<your-api-key>
+export LANGCHAIN_PROJECT=<your-project>  # if not specified, defaults to "default"
+```
+
+If you are inside this directory, then you can spin up a LangServe instance directly by:
+
+```shell
+langchain serve
+```
+
+This will start the FastAPI app with a server is running locally at 
+[http://localhost:8000](http://localhost:8000)
+
+We can see all templates at [http://127.0.0.1:8000/docs](http://127.0.0.1:8000/docs)
+We can access the playground at [http://127.0.0.1:8000/anthropic-iterative-search/playground](http://127.0.0.1:8000/anthropic-iterative-search/playground)  
+
+We can access the template from code with:
+
+```python
+from langserve.client import RemoteRunnable
+
+runnable = RemoteRunnable("http://localhost:8000/anthropic-iterative-search")
+```

langchain-docs-benchmarking/packages/anthropic-iterative-search/anthropic_iterative_search/__init__.py

@@ -0,0 +1,11 @@
+from langchain.schema.runnable import ConfigurableField
+
+from .chain import chain
+from .retriever_agent import executor
+
+final_chain = chain.configurable_alternatives(
+    ConfigurableField(id="chain"),
+    default_key="response",
+    # This adds a new option, with name `openai` that is equal to `ChatOpenAI()`
+    retrieve=executor,
+)

langchain-docs-benchmarking/packages/anthropic-iterative-search/anthropic_iterative_search/agent_scratchpad.py

@@ -0,0 +1,16 @@
+def _format_docs(docs):
+    result = "\n".join(
+        [
+            f'<item index="{i+1}">\n<page_content>\n{r}\n</page_content>\n</item>'
+            for i, r in enumerate(docs)
+        ]
+    )
+    return result
+
+
+def format_agent_scratchpad(intermediate_steps):
+    thoughts = ""
+    for action, observation in intermediate_steps:
+        thoughts += action.log
+        thoughts += "</search_query>" + _format_docs(observation)
+    return thoughts

langchain-docs-benchmarking/packages/anthropic-iterative-search/anthropic_iterative_search/chain.py

@@ -0,0 +1,29 @@
+from langchain.chat_models import ChatAnthropic
+from langchain.prompts import ChatPromptTemplate
+from langchain.pydantic_v1 import BaseModel
+from langchain.schema.output_parser import StrOutputParser
+from langchain.schema.runnable import RunnableLambda
+
+from .prompts import answer_prompt
+from .retriever_agent import executor
+
+prompt = ChatPromptTemplate.from_template(answer_prompt)
+
+model = ChatAnthropic(model="claude-2", temperature=0, max_tokens_to_sample=1000)
+
+chain = (
+    RunnableLambda(lambda x: {"query": x["question"]})
+    | {"query": lambda x: x["query"], "information": executor | (lambda x: x["output"])}
+    | prompt
+    | model
+    | StrOutputParser()
+)
+
+# Add typing for the inputs to be used in the playground
+
+
+class Inputs(BaseModel):
+    question: str
+
+
+chain = chain.with_types(input_type=Inputs)

langchain-docs-benchmarking/packages/anthropic-iterative-search/anthropic_iterative_search/output_parser.py

@@ -0,0 +1,37 @@
+import re
+
+from langchain.schema.agent import AgentAction, AgentFinish
+
+from .agent_scratchpad import _format_docs
+
+
+def extract_between_tags(tag: str, string: str, strip: bool = True) -> str:
+    ext_list = re.findall(f"<{tag}\s?>(.+?)</{tag}\s?>", string, re.DOTALL)
+    if strip:
+        ext_list = [e.strip() for e in ext_list]
+    if ext_list:
+        if len(ext_list) != 1:
+            raise ValueError
+        # Only return the first one
+        return ext_list[0]
+
+
+def parse_output(outputs):
+    partial_completion = outputs["partial_completion"]
+    steps = outputs["intermediate_steps"]
+    search_query = extract_between_tags(
+        "search_query", partial_completion + "</search_query>"
+    )
+    if search_query is None:
+        docs = []
+        str_output = ""
+        for action, observation in steps:
+            docs.extend(observation)
+            str_output += action.log
+            str_output += "</search_query>" + _format_docs(observation)
+        str_output += partial_completion
+        return AgentFinish({"docs": docs, "output": str_output}, log=partial_completion)
+    else:
+        return AgentAction(
+            tool="search", tool_input=search_query, log=partial_completion
+        )

langchain-docs-benchmarking/packages/anthropic-iterative-search/anthropic_iterative_search/prompts.py

@@ -0,0 +1,7 @@
+retrieval_prompt = """{retriever_description} Before beginning to research the user's question, first think for a moment inside <scratchpad> tags about what information is necessary for a well-informed answer. If the user's question is complex, you may need to decompose the query into multiple subqueries and execute them individually. Sometimes the search engine will return empty search results, or the search results may not contain the information you need. In such cases, feel free to try again with a different query. 
+
+After each call to the Search Engine Tool, reflect briefly inside <search_quality></search_quality> tags about whether you now have enough information to answer, or whether more information is needed. If you have all the relevant information, write it in <information></information> tags, WITHOUT actually answering the question. Otherwise, issue a new search.
+
+Here is the user's question: <question>{query}</question> Remind yourself to make short queries in your scratchpad as you plan out your strategy."""  # noqa: E501
+
+answer_prompt = "Here is a user query: <query>{query}</query>. Here is some relevant information: <information>{information}</information>. Please answer the question using the relevant information."  # noqa: E501

langchain-docs-benchmarking/packages/anthropic-iterative-search/anthropic_iterative_search/retriever.py

@@ -0,0 +1,17 @@
+from langchain.tools import tool
+from langchain_docs_retriever.retriever import get_retriever
+
+# This is used to tell the model how to best use the retriever.
+
+retriever_description = """You will be asked a question by a human user. You have access to the following tool to help answer the question. <tool_description> Search Engine Tool * The search engine will exclusively search over the LangChain documentation for pages similar to your query. It returns for each page its title and full page content. Use this tool if you want to get up-to-date and comprehensive information on a topic to help answer queries. Queries should be as atomic as possible -- they only need to address one part of the user's question. For example, if the user's query is "what is the color of a basketball?", your search query should be "basketball". Here's another example: if the user's question is "Who created the first neural network?", your first query should be "neural network". As you can see, these queries are quite short. Think keywords, not phrases. * At any time, you can make a call to the search engine using the following syntax: <search_query>query_word</search_query>. * You'll then get results back in <search_result> tags.</tool_description>"""  # noqa: E501
+
+retriever = get_retriever()
+
+# This should be the same as the function name below
+RETRIEVER_TOOL_NAME = "search"
+
+
+@tool
+def search(query, callbacks = None):
+    """Search the LangChain docs with the retriever."""
+    return retriever.get_relevant_documents(query, callbacks=callbacks)

langchain-docs-benchmarking/packages/anthropic-iterative-search/anthropic_iterative_search/retriever_agent.py

@@ -0,0 +1,41 @@
+from langchain.agents import AgentExecutor
+from langchain.chat_models import ChatAnthropic
+from langchain.prompts import ChatPromptTemplate
+from langchain.schema.output_parser import StrOutputParser
+from langchain.schema.runnable import RunnableMap, RunnablePassthrough
+
+from .agent_scratchpad import format_agent_scratchpad
+from .output_parser import parse_output
+from .prompts import retrieval_prompt
+from .retriever import retriever_description, search
+
+prompt = ChatPromptTemplate.from_messages(
+    [
+        ("user", retrieval_prompt),
+        ("ai", "{agent_scratchpad}"),
+    ]
+)
+prompt = prompt.partial(retriever_description=retriever_description)
+
+model = ChatAnthropic(model="claude-2", temperature=0, max_tokens_to_sample=1000)
+
+chain = (
+    RunnablePassthrough.assign(
+        agent_scratchpad=lambda x: format_agent_scratchpad(x["intermediate_steps"])
+    )
+    | prompt
+    | model.bind(stop_sequences=["</search_query>"])
+    | StrOutputParser()
+)
+
+agent_chain = (
+    RunnableMap(
+        {
+            "partial_completion": chain,
+            "intermediate_steps": lambda x: x["intermediate_steps"],
+        }
+    )
+    | parse_output
+)
+
+executor = AgentExecutor(agent=agent_chain, tools=[search])

langchain-docs-benchmarking/packages/anthropic-iterative-search/main.py

@@ -0,0 +1,12 @@
+from anthropic_iterative_search import final_chain
+
+if __name__ == "__main__":
+    query = (
+        "Which movie came out first: Oppenheimer, or "
+        "Are You There God It's Me Margaret?"
+    )
+    print(
+        final_chain.with_config(configurable={"chain": "retrieve"}).invoke(
+            {"query": query}
+        )
+    )