07_document_loading_and_embedding_using_chromadb_openai.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "07acc2df-ddd4-454b-9948-a64daae25235",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import json\n",
    "from dotenv import load_dotenv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "d348afd7-7ad2-4826-b223-d4ad41d72fd6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "load_dotenv()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "270e9202-1a37-4c02-9322-03cb87b4e880",
   "metadata": {},
   "outputs": [],
   "source": [
    "#os.getenv('OPENAI_API_KEY')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2a736f57-da4e-47f4-adc3-1fb9c0675efc",
   "metadata": {},
   "source": [
    "### Load Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "d8ec0f61-2fd9-4c90-940d-43c64b3ba4ca",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "274"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from langchain.document_loaders import DirectoryLoader, TextLoader\n",
    "\n",
    "loader = DirectoryLoader(\"data/articles\", glob=\"./*.html\", loader_cls=TextLoader)\n",
    "documents = loader.load()\n",
    "len(documents)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0e6cc602-494a-4f07-81d8-a8f71784fe97",
   "metadata": {},
   "source": [
    "### Split Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "0c3215a8-80ab-46be-ac7f-1aff143e99c4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1551"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from langchain.text_splitter import RecursiveCharacterTextSplitter\n",
    "\n",
    "text_splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=200)\n",
    "documents = text_splitter.split_documents(documents)\n",
    "len(documents)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cb38d4d7-dd56-45a3-acbb-78971c19326a",
   "metadata": {},
   "source": [
    "### Create ChromaDB Database (Vector Database)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "f6735788-583f-438a-962e-545a4cd7345c",
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain import embeddings\n",
    "from langchain_openai import OpenAIEmbeddings\n",
    "\n",
    "db_directory = 'db'\n",
    "\n",
    "embedding = OpenAIEmbeddings(model=\"text-embedding-3-small\")\n",
    "# vector_db = Chroma.from_documents(\n",
    "#    documents=documents,\n",
    "#    embedding=embedding,\n",
    "#    persist_directory=db_directory\n",
    "# )"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "508d855c-b663-4089-94f0-48d00eb8d8b5",
   "metadata": {},
   "source": [
    "### Load Vector Data for Usage"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "4883e0ae-6c80-4ac0-a5f7-0e7dc06124b8",
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain.vectorstores import Chroma\n",
    "\n",
    "vector_db = Chroma(persist_directory=db_directory, embedding_function=embedding)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8f70decd-703d-4446-b1f3-4ec432d4a2a3",
   "metadata": {},
   "source": [
    "### Data Retrieval For Similarity Search"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "553b3f02-4296-4ba9-ba9d-342be7b98158",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[Document(page_content='Stifel  analyst Brad Reback   maintains Microsoft (NASDAQ:<a class=\"ticker\" href=\"https://www.benzinga.com/stock/MSFT#NASDAQ\">MSFT</a>) with a Buy and raises the price target from $290 to $310.<p>Copyright &#169; Benzinga. All rights reserved. Write to editorial@benzinga.com with any questions about this content. Benzinga does not provide investment advice.</p>', metadata={'source': 'articles_data/2023-04-14-MSFT-BZ$143319c3.html'}),\n",
       " Document(page_content='BMO Capital  analyst Keith Bachman   maintains Microsoft (NASDAQ:<a class=\"ticker\" href=\"https://www.benzinga.com/stock/MSFT#NASDAQ\">MSFT</a>) with a Market Perform and raises the price target from $305 to $310.<p>Copyright &#169; Benzinga. All rights reserved. Write to editorial@benzinga.com with any questions about this content. Benzinga does not provide investment advice.</p>', metadata={'source': 'articles_data/2023-04-14-MSFT-BZ$14331e25.html'}),\n",
       " Document(page_content='total cost received by the writing party (or parties) was <strong>$74.8K</strong>, with a price of <strong>$505.0</strong> per contract. There were <strong>1106</strong> open contracts at this strike prior to today, and today <strong>1421</strong> contract(s) were bought and sold. </p><br><p>&#x2022;  Regarding <strong>MSFT</strong> (NASDAQ:<a class=\"ticker\" href=\"https://www.benzinga.com/stock/MSFT#NASDAQ\">MSFT</a>), we observe a <strong>call</strong> option <strong>trade</strong> with <strong>bearish</strong> sentiment. It expires in 17 day(s) on <strong>April 21, 2023</strong>. Parties traded <strong>50</strong> contract(s) at a <strong>$285.00</strong> strike. The total cost received by the writing party (or parties) was <strong>$41.5K</strong>, with a price of <strong>$830.0</strong> per contract. There were <strong>13978</strong> open contracts at this strike prior to today, and today <strong>1260</strong> contract(s) were bought and sold. </p><br><p>&#x2022;  Regarding', metadata={'source': 'articles_data/2023-04-04-TSM-BZ$141fd773.html'}),\n",
       " Document(page_content='total cost received by the writing party (or parties) was <strong>$74.8K</strong>, with a price of <strong>$505.0</strong> per contract. There were <strong>1106</strong> open contracts at this strike prior to today, and today <strong>1421</strong> contract(s) were bought and sold. </p><br><p>&#x2022;  Regarding <strong>MSFT</strong> (NASDAQ:<a class=\"ticker\" href=\"https://www.benzinga.com/stock/MSFT#NASDAQ\">MSFT</a>), we observe a <strong>call</strong> option <strong>trade</strong> with <strong>bearish</strong> sentiment. It expires in 17 day(s) on <strong>April 21, 2023</strong>. Parties traded <strong>50</strong> contract(s) at a <strong>$285.00</strong> strike. The total cost received by the writing party (or parties) was <strong>$41.5K</strong>, with a price of <strong>$830.0</strong> per contract. There were <strong>13978</strong> open contracts at this strike prior to today, and today <strong>1260</strong> contract(s) were bought and sold. </p><br><p>&#x2022;  Regarding', metadata={'source': 'articles_data/2023-04-04-NVDA-BZ$141fd773.html'})]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "retriever = vector_db.as_retriever()\n",
    "retriever.invoke(\"How much microsoft raised?\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e993e65e-b1a8-49aa-bf38-fa4e68bff712",
   "metadata": {},
   "source": [
    "### Feed Retrieved Documents to LLM to get Exact Answer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "8f502fd1-52e4-47f6-8afe-1a76a5da246e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'query': 'what is the news about microsoft?',\n",
       " 'result': \" The news is about a leaked video showing Microsoft working on a Windows handheld gaming mode that would allow gamers to play their favorite games on the go. Microsoft has also signed a 10-year partnership with BT Group's EE to expand their cloud gaming offerings. Additionally, the U.K.'s Competition and Markets Authority has provided a provisional update indicating that their inquiry does not anticipate any significant anticompetitive effects resulting from Microsoft's acquisition of Activision Blizzard.\"}"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from langchain_openai import OpenAI\n",
    "from langchain.chains import RetrievalQA\n",
    "\n",
    "chain = RetrievalQA.from_chain_type(\n",
    "    llm=OpenAI(),\n",
    "    chain_type=\"stuff\",\n",
    "    retriever=retriever,\n",
    "    return_source_documents=False\n",
    ")\n",
    "chain.invoke(\"what is the news about microsoft?\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "b21f72d4-c41e-4b66-92cc-873113f4a842",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'query': 'what is the news about apple?',\n",
       " 'result': \" The news is about Apple's upcoming Worldwide Developers Conference, where they are expected to unveil a new mixed-reality headset and update their products, including MacBooks and the Apple Watch. There may also be changes to iOS, potentially allowing for sideloading of apps, but this is not a major focus for this year's updates.\"}"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "chain.invoke(\"what is the news about apple?\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0e75c6fb-fa97-483d-be98-853c4c2d19c7",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2a1b388c-5ed8-4597-a787-e2d474b92d48",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}