update: codebase

finance_multi_modal_rag/README.md

@@ -4,6 +4,8 @@ A simple RAG (Retrieval-Augmented Generation) system for question-answering on [
 
 The codebase demonstrates how to build multi-modal RAG pipelines for question-answering systems and other downstream tasks in the finance domain using [Llama3.2 Vision](https://huggingface.co/meta-llama/Llama-3.2-90B-Vision). The codebase is also tightly integrated with [Weave](https://weave-docs.wandb.ai/), a lightweight toolkit for tracking and evaluating LLM applications, built by Weights & Biases. You cam explore all the traces and datasets used in this project at [geekyrakshit/finance_multi_modal_rag](https://wandb.ai/geekyrakshit/finance_multi_modal_rag).
 
+In order to learn more, check out the report [Llama3.2-Vision for Multi-modal RAG in Finance](https://wandb.ai/geekyrakshit/finance_multi_modal_rag/reports/Llama3-2-Vision-for-Multi-modal-RAG-in-Finance--Vmlldzo5NTIyODkw).
+
 ## Installation
 
 Install dependencies using the following commands:

finance_multi_modal_rag/finance_multi_modal_rag/chunking.py

@@ -241,8 +241,6 @@ def chunk_single_document(
     )
     filing_date = document["filing_date"]
     text = f"# Report filed at {filing_date}\n\n" + str(document["content"])
-    # text += "\n\n# Report Summary and Keywords\n\n" + str(document["summary"])
-    # text += "\n\n# Image Descriptions\n" + str(document["image_descriptions"])
     chunks = chunker.split_text(text)
     return [
         {

finance_multi_modal_rag/finance_multi_modal_rag/response_generation.py

@@ -35,7 +35,7 @@ def __init__(
         ]
 
     @weave.op()
-    def frame_user_prompt(self, query: str, retrieved_chunks: list[dict]) -> str:
+    def frame_user_prompt(self, query: str, retrieved_chunks: list[dict], use_image_descriptions: bool) -> str:
         user_prompt = ""
         for chunk in retrieved_chunks:
             date = chunk["metadata"]["filing_date"]
@@ -53,7 +53,7 @@ def frame_user_prompt(self, query: str, retrieved_chunks: list[dict]) -> str:
 
 {summary}\n\n
 """
-            if chunk["metadata"]["number_of_images"] > 0:
+            if chunk["metadata"]["number_of_images"] > 0 and use_image_descriptions:
                 image_descriptions = "\n\n".join(
                     self._dataset[chunk["metadata"]["document_idx"]][
                         "image_descriptions"
@@ -90,17 +90,18 @@ def fetch_most_relevant_image(
         return None
 
     @weave.op()
-    def predict(self, query: str):
+    def predict(self, query: str, use_relevant_image: bool = True, use_image_descriptions: bool = True):
         retrieved_chunks = self.text_retriever.predict(query=query, top_k=self.top_k)
-        user_prompt = self.frame_user_prompt(query, retrieved_chunks)
+        user_prompt = self.frame_user_prompt(query, retrieved_chunks, use_image_descriptions)
         user_prompts = [
             """
 # Instructions
 
 You are an expert and highly experienced financial analyst. You are provided with the
 following excerpts from the companies 10-k filings for Tesla, the electric car company.
-You may also be provided with an image that might be relevant to the user's query.
-Your job is to answer the user's question is detail based on the information provided.
+You may also be provided with an image and several image descriptions that might be
+relevant to the user's query. Your job is to answer the user's question is detail based
+on the information provided.
 
 Here are a few rules to follow:
 1. You should pay close attention to the excerpts, especially the dates and other numbers in them.
@@ -116,7 +117,11 @@ def predict(self, query: str):
 """
             + user_prompt
         ]
-        most_relevant_image = self.fetch_most_relevant_image(query, retrieved_chunks)
+        most_relevant_image = (
+            self.fetch_most_relevant_image(query, retrieved_chunks)
+            if use_relevant_image
+            else None
+        )
         if most_relevant_image:
             user_prompts.append(most_relevant_image)
         return {

finance_multi_modal_rag/test.py

@@ -57,5 +57,7 @@
     weave_corpus_dataset_address="TSLA_sec_filings:v8",
 )
 finace_qa_bot.predict(
-    query="What are the major risks or considerations highlighted for investors?"
+    query="What are the most important figures that a Tesla investor should know?",
+    use_relevant_image=False,
+    use_image_descriptions=True,
 )