diff --git a/trulens_eval/examples/expositional/vector-dbs/mongodb_atlas/atlas_quickstart.ipynb b/trulens_eval/examples/expositional/vector-dbs/mongodb_atlas/atlas_quickstart.ipynb index d2948de91..a99d980f1 100644 --- a/trulens_eval/examples/expositional/vector-dbs/mongodb_atlas/atlas_quickstart.ipynb +++ b/trulens_eval/examples/expositional/vector-dbs/mongodb_atlas/atlas_quickstart.ipynb @@ -38,60 +38,9 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/Users/jreini/opt/anaconda3/envs/trulens_dev_empty/lib/python3.11/site-packages/_distutils_hack/__init__.py:33: UserWarning: Setuptools is replacing distutils.\n", - " warnings.warn(\"Setuptools is replacing distutils.\")\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "🦑 Tru initialized with db url sqlite:///default.sqlite .\n", - "🛑 Secret keys may be written to the database. See the `database_redact_keys` option of Tru` to prevent this.\n", - "Starting dashboard ...\n", - "Config file already exists. Skipping writing process.\n", - "Credentials file already exists. Skipping writing process.\n" - ] - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "6ac29f1d94da4896903fea8e3b565857", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "Accordion(children=(VBox(children=(VBox(children=(Label(value='STDOUT'), Output())), VBox(children=(Label(valu…" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Dashboard started at http://192.168.4.23:8501 .\n" - ] - }, - { - "data": { - "text/plain": [ - "" - ] - }, - "execution_count": 1, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "from trulens_eval import Tru\n", "\n", @@ -111,7 +60,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -128,7 +77,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -139,7 +88,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -160,38 +109,9 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "--2024-04-05 16:06:56-- https://query.prod.cms.rt.microsoft.com/cms/api/am/binary/RE4HkJP\n", - "Resolving query.prod.cms.rt.microsoft.com (query.prod.cms.rt.microsoft.com)... 23.47.30.189\n", - "Connecting to query.prod.cms.rt.microsoft.com (query.prod.cms.rt.microsoft.com)|23.47.30.189|:443... connected.\n", - "HTTP request sent, awaiting response... 200 OK\n", - "Length: unspecified [application/pdf]\n", - "Saving to: ‘data/atlas_best_practices.pdf’\n", - "\n", - "data/atlas_best_pra [ <=> ] 500.64K --.-KB/s in 0.04s \n", - "\n", - "2024-04-05 16:06:56 (11.2 MB/s) - ‘data/atlas_best_practices.pdf’ saved [512653]\n", - "\n", - "--2024-04-05 16:06:56-- http://fondamentidibasididati.it/wp-content/uploads/2020/11/DBEssential-2021-C30-11-21.pdf\n", - "Resolving fondamentidibasididati.it (fondamentidibasididati.it)... 157.138.8.2\n", - "Connecting to fondamentidibasididati.it (fondamentidibasididati.it)|157.138.8.2|:80... connected.\n", - "HTTP request sent, awaiting response... 200 OK\n", - "Length: 595602 (582K) [application/pdf]\n", - "Saving to: ‘data/DBEssential-2021.pdf’\n", - "\n", - "data/DBEssential-20 100%[===================>] 581.64K 914KB/s in 0.6s \n", - "\n", - "2024-04-05 16:06:57 (914 KB/s) - ‘data/DBEssential-2021.pdf’ saved [595602/595602]\n", - "\n" - ] - } - ], + "outputs": [], "source": [ "# Load the sample data\n", "!mkdir -p 'data/'\n", @@ -201,7 +121,10 @@ "!wget 'http://fondamentidibasididati.it/wp-content/uploads/2020/11/DBEssential-2021-C30-11-21.pdf' -O 'data/DBEssential-2021.pdf'\n", "db_essentials = SimpleDirectoryReader(input_files=[\"./data/DBEssential-2021.pdf\"]).load_data()\n", "\n", - "documents = atlas_best_practices + db_essentials\n" + "!wget 'https://courses.edx.org/asset-v1:Databricks+LLM101x+2T2023+type@asset+block@Module_2_slides.pdf' -O 'data/DataBrick_vector_search.pdf'\n", + "databrick_vector_search = SimpleDirectoryReader(input_files=[\"./data/DataBrick_vector_search.pdf\"]).load_data()\n", + "\n", + "documents = atlas_best_practices + db_essentials + databrick_vector_search\n" ] }, { @@ -234,38 +157,9 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "969f14f3a741474d998a3fc7a6393a23", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "Parsing nodes: 0%| | 0/133 [00:00\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
GroundednessAnswer RelevanceContext Relevancelatencytotal_cost
app_id
Basic RAG0.6250.7750.851.250.000388
\n", - "" - ], - "text/plain": [ - " Groundedness Answer Relevance Context Relevance latency \\\n", - "app_id \n", - "Basic RAG 0.625 0.775 0.85 1.25 \n", - "\n", - " total_cost \n", - "app_id \n", - "Basic RAG 0.000388 " - ] - }, - "execution_count": 16, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "tru.get_leaderboard()" ] @@ -587,7 +360,7 @@ }, { "cell_type": "code", - "execution_count": 20, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -599,13 +372,17 @@ " filters=[ExactMatchFilter(key=\"metadata.file_name\", value=\"atlas_best_practices.pdf\")]\n", ")\n", "\n", + "metadata_filters_databrick = MetadataFilters(\n", + " filters=[ExactMatchFilter(key=\"metadata.file_name\", value=\"DataBrick_vector_search.pdf\")]\n", + ")\n", "# Instantiate Atlas Vector Search as a retriever for each set of filters\n", "vector_store_retriever_db_essentials = VectorIndexRetriever(index=vector_store_index, filters=metadata_filters_db_essentials, similarity_top_k=5)\n", "vector_store_retriever_atlas = VectorIndexRetriever(index=vector_store_index, filters=metadata_filters_atlas, similarity_top_k=5)\n", - "\n", + "vector_store_retriever_databrick = VectorIndexRetriever(index=vector_store_index, filters=metadata_filters_databrick, similarity_top_k=5)\n", "# Pass the retrievers into the query engines\n", "query_engine_with_filters_db_essentials = RetrieverQueryEngine(retriever=vector_store_retriever_db_essentials)\n", "query_engine_with_filters_atlas = RetrieverQueryEngine(retriever=vector_store_retriever_atlas)\n", + "query_engine_with_filters_databrick = RetrieverQueryEngine(retriever=vector_store_retriever_databrick)\n", "\n", "from llama_index.core.tools import QueryEngineTool\n", "\n", @@ -625,6 +402,13 @@ " ),\n", ")\n", "\n", + "databrick_tool = QueryEngineTool.from_defaults(\n", + " query_engine=query_engine_with_filters_databrick,\n", + " description = (\n", + " \"Useful for retrieving context about Databrick's course on Vector Databases and Search\"\n", + " )\n", + ")\n", + "\n", "# Create the router query engine\n", "\n", "from llama_index.core.query_engine import RouterQueryEngine\n", @@ -640,77 +424,21 @@ " query_engine_tools=[\n", " essentials_tool,\n", " atlas_tool,\n", + " databrick_tool\n", " ],\n", ")\n", "\n", "from trulens_eval import TruLlama\n", "tru_query_engine_recorder_with_router = TruLlama(router_query_engine,\n", - " app_id='Router Query Engine + Filters',\n", + " app_id='Router Query Engine + Filters v2',\n", " feedbacks=[f_groundedness, f_answer_relevance, f_context_relevance])" ] }, { "cell_type": "code", - "execution_count": 21, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "6058752dca6c4459b5012891056c5c25", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "Groundedness per statement in source: 0%| | 0/1 [00:00\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
GroundednessAnswer RelevanceContext Relevancelatencytotal_cost
app_id
Router Query Engine + Filters0.7250.9500.7791671.250.000926
Basic RAG0.6250.7750.8500001.250.000388
\n", - "" - ], - "text/plain": [ - " Groundedness Answer Relevance \\\n", - "app_id \n", - "Router Query Engine + Filters 0.725 0.950 \n", - "Basic RAG 0.625 0.775 \n", - "\n", - " Context Relevance latency total_cost \n", - "app_id \n", - "Router Query Engine + Filters 0.779167 1.25 0.000926 \n", - "Basic RAG 0.850000 1.25 0.000388 " - ] - }, - "execution_count": 22, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "tru.get_leaderboard()" ] diff --git a/trulens_eval/trulens_eval/feedback/provider/base.py b/trulens_eval/trulens_eval/feedback/provider/base.py index e161b3573..734a0c99b 100644 --- a/trulens_eval/trulens_eval/feedback/provider/base.py +++ b/trulens_eval/trulens_eval/feedback/provider/base.py @@ -398,7 +398,7 @@ def context_relevance_with_cot_reasons(self, question: str, "RELEVANCE:", prompts.COT_REASONS_TEMPLATE ) - return self.generate_score_and_reasons(system_prompt, user_prompt, temperature) + return self.generate_score_and_reasons(system_prompt=system_prompt, user_prompt=user_prompt, temperature=temperature) def qs_relevance_with_cot_reasons(self, question: str, context: str) -> Tuple[float, Dict]: