08_document_loading_and_embedding_using_pinecone_openai.ipynb.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "07acc2df-ddd4-454b-9948-a64daae25235",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import json\n",
    "from dotenv import load_dotenv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "d348afd7-7ad2-4826-b223-d4ad41d72fd6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "load_dotenv()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "270e9202-1a37-4c02-9322-03cb87b4e880",
   "metadata": {},
   "outputs": [],
   "source": [
    "#os.getenv('OPENAI_API_KEY')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2a736f57-da4e-47f4-adc3-1fb9c0675efc",
   "metadata": {},
   "source": [
    "### Load Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "d8ec0f61-2fd9-4c90-940d-43c64b3ba4ca",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "24"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from langchain.document_loaders import PyPDFDirectoryLoader\n",
    "\n",
    "loader = PyPDFDirectoryLoader(\"data/resumes\")\n",
    "documents = loader.load()\n",
    "len(documents)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0e6cc602-494a-4f07-81d8-a8f71784fe97",
   "metadata": {},
   "source": [
    "### Split Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "0c3215a8-80ab-46be-ac7f-1aff143e99c4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "69"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from langchain.text_splitter import RecursiveCharacterTextSplitter\n",
    "\n",
    "text_splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=200)\n",
    "documents = text_splitter.split_documents(documents)\n",
    "len(documents)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cb38d4d7-dd56-45a3-acbb-78971c19326a",
   "metadata": {},
   "source": [
    "### Create Pinecone Database (Vector Database)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "f6735788-583f-438a-962e-545a4cd7345c",
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain import embeddings\n",
    "from langchain_openai import OpenAIEmbeddings\n",
    "from langchain_pinecone import PineconeVectorStore\n",
    "\n",
    "pinecone_index = \"langchain-practice\"\n",
    "\n",
    "embedding = OpenAIEmbeddings(model=\"text-embedding-3-small\")\n",
    "#docsearch = PineconeVectorStore.from_documents(documents, embedding, index_name=pinecone_index)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "508d855c-b663-4089-94f0-48d00eb8d8b5",
   "metadata": {},
   "source": [
    "### Load Vector Data for Usage"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "4883e0ae-6c80-4ac0-a5f7-0e7dc06124b8",
   "metadata": {},
   "outputs": [],
   "source": [
    "vector_db = PineconeVectorStore.from_existing_index(pinecone_index, embedding)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8f70decd-703d-4446-b1f3-4ec432d4a2a3",
   "metadata": {},
   "source": [
    "### Data Retrieval For Similarity Search"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "553b3f02-4296-4ba9-ba9d-342be7b98158",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[Document(page_content='Contact\\nwww.linkedin.com/in/jericka-henry\\n(LinkedIn)\\nTop Skills\\nCustomer Satisfaction\\nRelationship Building\\nBusiness Development\\nCertifications\\nOccupational Studies of Health and\\nScience\\nTech Sales Career Track\\nHonors-Awards\\nI Can AfricanJericka Henry\\nBDR | SDR | SaaS| CRM | B2B\\nNew York City Metropolitan Area\\nSummary\\nGrowing up in New York City brings out every natural gift that sleeps\\ninside of you. I learned early on having effective communication\\nskills is the key to building, learning and growing your brand\\n(you). Tech Sales is definitely a place I can expand this skill.\\nAt Springboard I conducted effective qualification calls and\\nimplemented objection handling strategies. I also executed on\\ncritical tech sales fundamentals including pipeline development and\\nprospect research and as a result I successfully passed over 20\\ncourse assessments to demonstrate understanding and delivery of\\nimpactful tech sales practices.\\nAs a massage therapist for 5 years and 10+ years in the food', metadata={'page': 0.0, 'source': 'resumes/Jericka_Henry_AEMAAD4rQLQBVoi6Pq2ZOrJFtzy_YpfShh7Ev4c.pdf'}),\n",
       " Document(page_content='strengths can add value to your team!\\nExperience\\nRE:WORK TRAINING\\nBDR Candidate\\nJuly 2023\\xa0-\\xa0Present\\xa0 (10 months)\\nPoughkeepsie, New York\\nRe:Work Training, a 501(c)3 nonprofit organization, is creating workplaces that\\nreflect the diversity of our neighborhoods. \\nAs a BDR Candidate,I help RE:Work scale their impact by genera qualified\\nprospects through my participation in their 12 week Tech Sales Training\\nProgram,including: -Generate outbound leads and handle inbound leads\\nfor the company’s 8-week Tech Sales Training Program, Donor Program &\\nCorporate Partnership Program -Qualify and develop a robust sales pipeline\\nfor RE:Work’s spectrum of programming-Schedule and conduct sales calls\\nand demos -Work closely with sales and marketing teams to develop regular\\nstrategic plans to meet the company’s monthly, quarterly and annual goals -\\nMaintain and update prospect records in our CRM to ensure accurate reports -\\nCommunicate and Identify areas of improvement in the sales process \\n\\xa0 Page 1 of 2', metadata={'page': 0.0, 'source': 'resumes/Mona_Moore_AEMAAEAT7NYB1U2LOo6jlk_k74xbEn174XASCcw.pdf'}),\n",
       " Document(page_content=\"Contact\\nwww.linkedin.com/in/edgar-\\nguzman-390a1a164  (LinkedIn)\\nTop Skills\\nTeam Leadership\\nTeamwork\\nProblem SolvingEdgar Guzman\\nBDR @ Regal | Squad Leader | US Marine Non-Comissioned Officer\\nand Jazz drummer\\nNew York, New York, United States\\nSummary\\nAs a seasoned sales professional, I have honed my skills at\\nJitjatjo and Lenscrafters where I was responsible for developing\\nnew business, building relationships with clients/customers, and\\nexceeding sales targets. During my time in the military, I served as a\\nmanager and leader, gaining invaluable experience in team building,\\nstrategic planning, and decision-making. I proudly served my country\\nin the Marines, exemplifying discipline and dedication to duty.\\nIn addition, I'm a jazz drummer, bringing that unique flavor to\\neverywhere I work- the ability to think on my feet, improvise and\\nadapt to changing circumstances, and excel in high-pressure\\nsituations. These skills will be invaluable to my current role as a\", metadata={'page': 0.0, 'source': 'resumes/Edgar_Guzman_AEMAACdOTpwBelGXTlto3knfHufcg4frfUQhFoE.pdf'}),\n",
       " Document(page_content='Contact\\nwww.linkedin.com/in/abiolabishop\\n(LinkedIn)\\nTop Skills\\nWritten Communication\\nSocial Perceptiveness\\nProblem SolvingAbiola B., M.S.P.M\\nIs your website accessible? Let’s chat!\\nNew York City Metropolitan Area\\nSummary\\nA multi-hyphenated New York native with a passion for fashion,\\ntechnology, and developing strategies for growth. No business is too\\nbig or small for improvement. With no two clients alike, my diverse\\nbackground in fashion and tech has taught me to be highly skilled in\\nclient engagement, problem-solving, customer-business interactions,\\nand high emotional intelligence.\\nExperience\\naccessiBe\\nStrategic Partnerships\\nApril 2024\\xa0-\\xa0Present\\xa0 (1 month)\\nVisulon Inc.\\nLead BDR\\nAugust 2023\\xa0-\\xa0April 2024\\xa0 (9 months)\\nEducation\\nVirginia State University\\nMaster of Science - MS\\xa0\\nVirginia State University\\nBachelor of Science - BS\\xa0\\nActivity\\n04/18/2024, Imeila Fonua added candidate to BDR- Test Sample\\n\\xa0 Page 1 of 1', metadata={'page': 0.0, 'source': 'resumes/Abiola_B.M.S.P.M_AEMAABV6FPwBrrEavgKfp5c3356t6JJvgVOhMgU.pdf'})]"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "retriever = vector_db.as_retriever()\n",
    "retriever.invoke(\"list top skills mentioned in the resumes?\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e993e65e-b1a8-49aa-bf38-fa4e68bff712",
   "metadata": {},
   "source": [
    "### Feed Retrieved Documents to LLM to get Exact Answer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "8f502fd1-52e4-47f6-8afe-1a76a5da246e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'query': 'list top skills mentioned in the resumes?',\n",
       " 'result': ' Customer Satisfaction, Relationship Building, Business Development, Team Leadership, Teamwork, Problem Solving, Written Communication, Social Perceptiveness'}"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from langchain_openai import OpenAI\n",
    "from langchain.chains import RetrievalQA\n",
    "\n",
    "chain = RetrievalQA.from_chain_type(\n",
    "    llm=OpenAI(),\n",
    "    chain_type=\"stuff\",\n",
    "    retriever=retriever,\n",
    "    return_source_documents=False\n",
    ")\n",
    "chain.invoke(\"list top skills mentioned in the resumes?\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "b21f72d4-c41e-4b66-92cc-873113f4a842",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "From the provided documents, the top resumes for python would be Amanze Emeziem and Gus Medina. Both individuals have Python listed as one of their top skills and have experience working with it in their previous roles.\n"
     ]
    }
   ],
   "source": [
    "print(chain.invoke(\"can you give me the top resumes for python based on provided documents?\")['result'].strip())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0e75c6fb-fa97-483d-be98-853c4c2d19c7",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2b944066-95c3-4ac1-83e5-1313a31a67be",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}