langchain_semantic_search.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/venuv/langchain_semantic_search/blob/main/langchain_semantic_search.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "51Wn8g88JCpu",
        "outputId": "a687699f-f2ab-4121-b77d-065a55a819dc"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting langchain\n",
            "  Downloading langchain-0.0.70-py3-none-any.whl (175 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m175.4/175.4 KB\u001b[0m \u001b[31m3.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: requests<3,>=2 in /usr/local/lib/python3.8/dist-packages (from langchain) (2.25.1)\n",
            "Requirement already satisfied: pydantic<2,>=1 in /usr/local/lib/python3.8/dist-packages (from langchain) (1.10.4)\n",
            "Requirement already satisfied: numpy<2,>=1 in /usr/local/lib/python3.8/dist-packages (from langchain) (1.21.6)\n",
            "Requirement already satisfied: SQLAlchemy<2,>=1 in /usr/local/lib/python3.8/dist-packages (from langchain) (1.4.46)\n",
            "Requirement already satisfied: PyYAML<7,>=6 in /usr/local/lib/python3.8/dist-packages (from langchain) (6.0)\n",
            "Requirement already satisfied: typing-extensions>=4.2.0 in /usr/local/lib/python3.8/dist-packages (from pydantic<2,>=1->langchain) (4.4.0)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.8/dist-packages (from requests<3,>=2->langchain) (2.10)\n",
            "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.8/dist-packages (from requests<3,>=2->langchain) (1.24.3)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.8/dist-packages (from requests<3,>=2->langchain) (2022.12.7)\n",
            "Requirement already satisfied: chardet<5,>=3.0.2 in /usr/local/lib/python3.8/dist-packages (from requests<3,>=2->langchain) (4.0.0)\n",
            "Requirement already satisfied: greenlet!=0.4.17 in /usr/local/lib/python3.8/dist-packages (from SQLAlchemy<2,>=1->langchain) (2.0.1)\n",
            "Installing collected packages: langchain\n",
            "Successfully installed langchain-0.0.70\n",
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting openai\n",
            "  Downloading openai-0.26.4.tar.gz (55 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m55.6/55.6 KB\u001b[0m \u001b[31m1.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n",
            "  Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n",
            "  Installing backend dependencies ... \u001b[?25l\u001b[?25hdone\n",
            "  Preparing metadata (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.8/dist-packages (from openai) (4.64.1)\n",
            "Requirement already satisfied: aiohttp in /usr/local/lib/python3.8/dist-packages (from openai) (3.8.3)\n",
            "Requirement already satisfied: requests>=2.20 in /usr/local/lib/python3.8/dist-packages (from openai) (2.25.1)\n",
            "Requirement already satisfied: chardet<5,>=3.0.2 in /usr/local/lib/python3.8/dist-packages (from requests>=2.20->openai) (4.0.0)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.8/dist-packages (from requests>=2.20->openai) (2022.12.7)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.8/dist-packages (from requests>=2.20->openai) (2.10)\n",
            "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.8/dist-packages (from requests>=2.20->openai) (1.24.3)\n",
            "Requirement already satisfied: charset-normalizer<3.0,>=2.0 in /usr/local/lib/python3.8/dist-packages (from aiohttp->openai) (2.1.1)\n",
            "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.8/dist-packages (from aiohttp->openai) (1.3.1)\n",
            "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.8/dist-packages (from aiohttp->openai) (6.0.4)\n",
            "Requirement already satisfied: async-timeout<5.0,>=4.0.0a3 in /usr/local/lib/python3.8/dist-packages (from aiohttp->openai) (4.0.2)\n",
            "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.8/dist-packages (from aiohttp->openai) (1.8.2)\n",
            "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.8/dist-packages (from aiohttp->openai) (1.3.3)\n",
            "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.8/dist-packages (from aiohttp->openai) (22.2.0)\n",
            "Building wheels for collected packages: openai\n",
            "  Building wheel for openai (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for openai: filename=openai-0.26.4-py3-none-any.whl size=67744 sha256=33f1797599669c4c578cb1a6495ca9dfa236e702b1a1c341c5f85c3bf890e74c\n",
            "  Stored in directory: /root/.cache/pip/wheels/2b/d8/4e/268f029bd3277c1dd9e8781a0e0296e0a63822665bfa2429fc\n",
            "Successfully built openai\n",
            "Installing collected packages: openai\n",
            "Successfully installed openai-0.26.4\n",
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.8/dist-packages (2.25.1)\n",
            "Collecting transformers\n",
            "  Downloading transformers-4.26.0-py3-none-any.whl (6.3 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m6.3/6.3 MB\u001b[0m \u001b[31m45.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting faiss-cpu\n",
            "  Downloading faiss_cpu-1.7.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (17.0 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m17.0/17.0 MB\u001b[0m \u001b[31m61.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.8/dist-packages (from requests) (2022.12.7)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.8/dist-packages (from requests) (2.10)\n",
            "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.8/dist-packages (from requests) (1.24.3)\n",
            "Requirement already satisfied: chardet<5,>=3.0.2 in /usr/local/lib/python3.8/dist-packages (from requests) (4.0.0)\n",
            "Collecting tokenizers!=0.11.3,<0.14,>=0.11.1\n",
            "  Downloading tokenizers-0.13.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (7.6 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m7.6/7.6 MB\u001b[0m \u001b[31m78.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting huggingface-hub<1.0,>=0.11.0\n",
            "  Downloading huggingface_hub-0.12.0-py3-none-any.whl (190 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m190.3/190.3 KB\u001b[0m \u001b[31m19.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.8/dist-packages (from transformers) (2022.6.2)\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.8/dist-packages (from transformers) (1.21.6)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.8/dist-packages (from transformers) (6.0)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.8/dist-packages (from transformers) (21.3)\n",
            "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.8/dist-packages (from transformers) (4.64.1)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.8/dist-packages (from transformers) (3.9.0)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.8/dist-packages (from huggingface-hub<1.0,>=0.11.0->transformers) (4.4.0)\n",
            "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /usr/local/lib/python3.8/dist-packages (from packaging>=20.0->transformers) (3.0.9)\n",
            "Installing collected packages: tokenizers, faiss-cpu, huggingface-hub, transformers\n",
            "Successfully installed faiss-cpu-1.7.3 huggingface-hub-0.12.0 tokenizers-0.13.2 transformers-4.26.0\n",
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting PyPDF2\n",
            "  Downloading pypdf2-3.0.1-py3-none-any.whl (232 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m232.6/232.6 KB\u001b[0m \u001b[31m4.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: typing_extensions>=3.10.0.0 in /usr/local/lib/python3.8/dist-packages (from PyPDF2) (4.4.0)\n",
            "Installing collected packages: PyPDF2\n",
            "Successfully installed PyPDF2-3.0.1\n"
          ]
        }
      ],
      "source": [
        "!pip install langchain\n",
        "!pip install openai\n",
        "!pip install requests transformers faiss-cpu\n",
        "!pip install PyPDF2"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "56d08n5RJTE9"
      },
      "outputs": [],
      "source": [
        "from langchain.llms import OpenAI\n",
        "from langchain.chains.qa_with_sources import load_qa_with_sources_chain\n",
        "from langchain.docstore.document import Document\n",
        "import requests\n",
        "from google.colab import drive\n",
        "import os\n",
        "from PyPDF2 import PdfReader\n",
        "from langchain.embeddings.openai import OpenAIEmbeddings\n",
        "from langchain.vectorstores.faiss import FAISS\n",
        "from langchain.text_splitter import CharacterTextSplitter\n",
        "import pickle\n",
        "import os\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "VDoiAEP_u7rE",
        "outputId": "1fb52aea-e72e-478f-cc8e-04bacd6756d9"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "env: OPENAI_API_KEY=<your key here. NO QUOTES>\n"
          ]
        }
      ],
      "source": [
        "%env OPENAI_API_KEY = <your key here. NO QUOTES>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "FtcEDfHQOg2J",
        "outputId": "78f3690d-7e0c-4c15-d2bd-ffe4b1e1fc26"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n"
          ]
        }
      ],
      "source": [
        "drive.mount('/content/drive')\n",
        "gdrive_path = '/content/drive/MyDrive/sleep_pdf/' # or whatever your Google Folder path is"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "WgvWik96Gi8R"
      },
      "outputs": [],
      "source": [
        "# Langchain Document - https://github.com/hwchase17/langchain/blob/master/langchain/docstore/document.py\n",
        "def get_pdf_data(file_path, num_pages = 1):\n",
        "  reader = PdfReader(gdrive_path+file_path)\n",
        "  full_doc_text = \"\"\n",
        "  for page in range(len(reader.pages)):\n",
        "    current_page = reader.pages[page]\n",
        "    text = current_page.extract_text()\n",
        "    full_doc_text += text\n",
        "\n",
        "\n",
        "  return Document(\n",
        "        page_content=full_doc_text,\n",
        "        metadata = {\"source\": file_path} \n",
        "    )"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "x5o7bn2J1Vrn"
      },
      "outputs": [],
      "source": [
        "def source_docs():\n",
        "    return [get_pdf_data(file) for file in os.listdir(gdrive_path)]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "UOG8Dtht8APG"
      },
      "outputs": [],
      "source": [
        "def search_index(source_docs):\n",
        "    source_chunks = []\n",
        "    splitter = CharacterTextSplitter(separator=\" \", chunk_size=1024, chunk_overlap=0)\n",
        "\n",
        "    for source in source_docs:\n",
        "        for chunk in splitter.split_text(source.page_content):\n",
        "            source_chunks.append(Document(page_content=chunk, metadata=source.metadata))\n",
        "    #more likely you'll pickle this inde\n",
        "    with open(\"search_index.pickle\", \"wb\") as f:\n",
        "        pickle.dump(FAISS.from_documents(source_chunks, OpenAIEmbeddings()), f)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "x6d-fA3v8Nnf"
      },
      "outputs": [],
      "source": [
        "#faster but more expensive in $ - chain = load_qa_with_sources_chain(OpenAI(temperature=0),verbose=False,chain_type=\"map_reduce\")\n",
        "chain = load_qa_with_sources_chain(OpenAI(temperature=0),verbose=False, chain_type=\"stuff\")\n",
        "def print_answer(question):\n",
        "    with open(\"search_index.pickle\", \"rb\") as f:\n",
        "        search_index = pickle.load(f)\n",
        "        #print(\"type - \", type(search_index))\n",
        "    print(\n",
        "        chain(\n",
        "            {\n",
        "                \"input_documents\": search_index.similarity_search(question, k=3),\n",
        "                \"question\": question,\n",
        "            },\n",
        "            return_only_outputs=True,\n",
        "        )[\"output_text\"]\n",
        "    \n",
        "    )\n",
        "    "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "g2tTFCZ9Wivd",
        "outputId": "f2c555b3-bd9c-42e8-81e0-8ff99e678223"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "WARNING:root:Created a chunk of size 1092, which is longer than the specified 1024\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            " Sleep devices and actigraphy can both be used to assess sleep quality. Sleep devices measure sleep quality on an accelerometer basis, while actigraphy measures sleep/vigilance patterns.\n",
            "SOURCES: sleep device vs actigraphy.pdf, teeth grinding sleep quality.pdf\n"
          ]
        }
      ],
      "source": [
        "import pdb\n",
        "sources = source_docs()\n",
        "search_index(sources)\n",
        "try:\n",
        "  print_answer(<your question on the document corpus here>) # your question here\n",
        "except Exception as e:\n",
        "  print(f\"Unexpected error: {e}\")"
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "pRnBwxMC0Upj"
      },
      "execution_count": null,
      "outputs": []
    }
  ],
  "metadata": {
    "colab": {
      "provenance": [],
      "authorship_tag": "ABX9TyMac6gKYMjgF3mKpSEoj/3Q",
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}