AWS_EC2Chatbot_KG.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/kushjaggi/AWS-EC2-Chatbot-KG/blob/main/AWS_EC2Chatbot_KG.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Installing all the dependencies\n",
        "\n",
        "*   langchain\n",
        "*   neo4j\n",
        "*   openai\n",
        "*   unstructured[all-docs]\n",
        "*   tiktoken\n",
        "\n",
        "\n",
        "\n"
      ],
      "metadata": {
        "id": "isM7ohE4E3jI"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "collapsed": true,
        "id": "cllpFMqCUaA9",
        "outputId": "16377b41-f6de-40f4-8d3b-0e4d7547b6e6"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Requirement already satisfied: langchain in /usr/local/lib/python3.10/dist-packages (0.0.352)\n",
            "Requirement already satisfied: neo4j in /usr/local/lib/python3.10/dist-packages (5.15.0)\n",
            "Requirement already satisfied: openai in /usr/local/lib/python3.10/dist-packages (1.6.0)\n",
            "Requirement already satisfied: unstructured[all-docs] in /usr/local/lib/python3.10/dist-packages (0.11.6)\n",
            "Requirement already satisfied: tiktoken in /usr/local/lib/python3.10/dist-packages (0.5.2)\n",
            "Requirement already satisfied: PyYAML>=5.3 in /usr/local/lib/python3.10/dist-packages (from langchain) (6.0.1)\n",
            "Requirement already satisfied: SQLAlchemy<3,>=1.4 in /usr/local/lib/python3.10/dist-packages (from langchain) (2.0.23)\n",
            "Requirement already satisfied: aiohttp<4.0.0,>=3.8.3 in /usr/local/lib/python3.10/dist-packages (from langchain) (3.9.1)\n",
            "Requirement already satisfied: async-timeout<5.0.0,>=4.0.0 in /usr/local/lib/python3.10/dist-packages (from langchain) (4.0.3)\n",
            "Requirement already satisfied: dataclasses-json<0.7,>=0.5.7 in /usr/local/lib/python3.10/dist-packages (from langchain) (0.6.3)\n",
            "Requirement already satisfied: jsonpatch<2.0,>=1.33 in /usr/local/lib/python3.10/dist-packages (from langchain) (1.33)\n",
            "Requirement already satisfied: langchain-community<0.1,>=0.0.2 in /usr/local/lib/python3.10/dist-packages (from langchain) (0.0.5)\n",
            "Requirement already satisfied: langchain-core<0.2,>=0.1 in /usr/local/lib/python3.10/dist-packages (from langchain) (0.1.2)\n",
            "Requirement already satisfied: langsmith<0.1.0,>=0.0.70 in /usr/local/lib/python3.10/dist-packages (from langchain) (0.0.72)\n",
            "Requirement already satisfied: numpy<2,>=1 in /usr/local/lib/python3.10/dist-packages (from langchain) (1.23.5)\n",
            "Requirement already satisfied: pydantic<3,>=1 in /usr/local/lib/python3.10/dist-packages (from langchain) (1.10.13)\n",
            "Requirement already satisfied: requests<3,>=2 in /usr/local/lib/python3.10/dist-packages (from langchain) (2.31.0)\n",
            "Requirement already satisfied: tenacity<9.0.0,>=8.1.0 in /usr/local/lib/python3.10/dist-packages (from langchain) (8.2.3)\n",
            "Requirement already satisfied: pytz in /usr/local/lib/python3.10/dist-packages (from neo4j) (2023.3.post1)\n",
            "Requirement already satisfied: anyio<5,>=3.5.0 in /usr/local/lib/python3.10/dist-packages (from openai) (3.7.1)\n",
            "Requirement already satisfied: distro<2,>=1.7.0 in /usr/lib/python3/dist-packages (from openai) (1.7.0)\n",
            "Requirement already satisfied: httpx<1,>=0.23.0 in /usr/local/lib/python3.10/dist-packages (from openai) (0.26.0)\n",
            "Requirement already satisfied: sniffio in /usr/local/lib/python3.10/dist-packages (from openai) (1.3.0)\n",
            "Requirement already satisfied: tqdm>4 in /usr/local/lib/python3.10/dist-packages (from openai) (4.66.1)\n",
            "Requirement already satisfied: typing-extensions<5,>=4.7 in /usr/local/lib/python3.10/dist-packages (from openai) (4.9.0)\n",
            "Requirement already satisfied: chardet in /usr/local/lib/python3.10/dist-packages (from unstructured[all-docs]) (5.2.0)\n",
            "Requirement already satisfied: filetype in /usr/local/lib/python3.10/dist-packages (from unstructured[all-docs]) (1.2.0)\n",
            "Requirement already satisfied: python-magic in /usr/local/lib/python3.10/dist-packages (from unstructured[all-docs]) (0.4.27)\n",
            "Requirement already satisfied: lxml in /usr/local/lib/python3.10/dist-packages (from unstructured[all-docs]) (4.9.3)\n",
            "Requirement already satisfied: nltk in /usr/local/lib/python3.10/dist-packages (from unstructured[all-docs]) (3.8.1)\n",
            "Requirement already satisfied: tabulate in /usr/local/lib/python3.10/dist-packages (from unstructured[all-docs]) (0.9.0)\n",
            "Requirement already satisfied: beautifulsoup4 in /usr/local/lib/python3.10/dist-packages (from unstructured[all-docs]) (4.11.2)\n",
            "Requirement already satisfied: emoji in /usr/local/lib/python3.10/dist-packages (from unstructured[all-docs]) (2.9.0)\n",
            "Requirement already satisfied: python-iso639 in /usr/local/lib/python3.10/dist-packages (from unstructured[all-docs]) (2023.12.11)\n",
            "Requirement already satisfied: langdetect in /usr/local/lib/python3.10/dist-packages (from unstructured[all-docs]) (1.0.9)\n",
            "Requirement already satisfied: rapidfuzz in /usr/local/lib/python3.10/dist-packages (from unstructured[all-docs]) (3.5.2)\n",
            "Requirement already satisfied: backoff in /usr/local/lib/python3.10/dist-packages (from unstructured[all-docs]) (2.2.1)\n",
            "Requirement already satisfied: unstructured-client in /usr/local/lib/python3.10/dist-packages (from unstructured[all-docs]) (0.15.0)\n",
            "Requirement already satisfied: wrapt in /usr/local/lib/python3.10/dist-packages (from unstructured[all-docs]) (1.14.1)\n",
            "Requirement already satisfied: pypandoc in /usr/local/lib/python3.10/dist-packages (from unstructured[all-docs]) (1.12)\n",
            "Requirement already satisfied: pdf2image in /usr/local/lib/python3.10/dist-packages (from unstructured[all-docs]) (1.16.3)\n",
            "Requirement already satisfied: pdfminer.six in /usr/local/lib/python3.10/dist-packages (from unstructured[all-docs]) (20221105)\n",
            "Requirement already satisfied: xlrd in /usr/local/lib/python3.10/dist-packages (from unstructured[all-docs]) (2.0.1)\n",
            "Requirement already satisfied: markdown in /usr/local/lib/python3.10/dist-packages (from unstructured[all-docs]) (3.5.1)\n",
            "Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from unstructured[all-docs]) (3.2.1)\n",
            "Requirement already satisfied: pikepdf in /usr/local/lib/python3.10/dist-packages (from unstructured[all-docs]) (8.10.1)\n",
            "Requirement already satisfied: unstructured.pytesseract>=0.3.12 in /usr/local/lib/python3.10/dist-packages (from unstructured[all-docs]) (0.3.12)\n",
            "Requirement already satisfied: python-docx>=1.1.0 in /usr/local/lib/python3.10/dist-packages (from unstructured[all-docs]) (1.1.0)\n",
            "Requirement already satisfied: unstructured-inference==0.7.18 in /usr/local/lib/python3.10/dist-packages (from unstructured[all-docs]) (0.7.18)\n",
            "Requirement already satisfied: msg-parser in /usr/local/lib/python3.10/dist-packages (from unstructured[all-docs]) (1.2.0)\n",
            "Requirement already satisfied: pandas in /usr/local/lib/python3.10/dist-packages (from unstructured[all-docs]) (1.5.3)\n",
            "Requirement already satisfied: python-pptx<=0.6.23 in /usr/local/lib/python3.10/dist-packages (from unstructured[all-docs]) (0.6.23)\n",
            "Requirement already satisfied: openpyxl in /usr/local/lib/python3.10/dist-packages (from unstructured[all-docs]) (3.1.2)\n",
            "Requirement already satisfied: onnx in /usr/local/lib/python3.10/dist-packages (from unstructured[all-docs]) (1.15.0)\n",
            "Requirement already satisfied: pypdf in /usr/local/lib/python3.10/dist-packages (from unstructured[all-docs]) (3.17.3)\n",
            "Requirement already satisfied: layoutparser[layoutmodels,tesseract] in /usr/local/lib/python3.10/dist-packages (from unstructured-inference==0.7.18->unstructured[all-docs]) (0.3.4)\n",
            "Requirement already satisfied: python-multipart in /usr/local/lib/python3.10/dist-packages (from unstructured-inference==0.7.18->unstructured[all-docs]) (0.0.6)\n",
            "Requirement already satisfied: huggingface-hub in /usr/local/lib/python3.10/dist-packages (from unstructured-inference==0.7.18->unstructured[all-docs]) (0.19.4)\n",
            "Requirement already satisfied: opencv-python!=4.7.0.68 in /usr/local/lib/python3.10/dist-packages (from unstructured-inference==0.7.18->unstructured[all-docs]) (4.8.0.76)\n",
            "Requirement already satisfied: onnxruntime<1.16 in /usr/local/lib/python3.10/dist-packages (from unstructured-inference==0.7.18->unstructured[all-docs]) (1.15.1)\n",
            "Requirement already satisfied: transformers>=4.25.1 in /usr/local/lib/python3.10/dist-packages (from unstructured-inference==0.7.18->unstructured[all-docs]) (4.35.2)\n",
            "Requirement already satisfied: regex>=2022.1.18 in /usr/local/lib/python3.10/dist-packages (from tiktoken) (2023.6.3)\n",
            "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.3->langchain) (23.1.0)\n",
            "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.3->langchain) (6.0.4)\n",
            "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.3->langchain) (1.9.4)\n",
            "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.3->langchain) (1.4.1)\n",
            "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.3->langchain) (1.3.1)\n",
            "Requirement already satisfied: idna>=2.8 in /usr/local/lib/python3.10/dist-packages (from anyio<5,>=3.5.0->openai) (3.6)\n",
            "Requirement already satisfied: exceptiongroup in /usr/local/lib/python3.10/dist-packages (from anyio<5,>=3.5.0->openai) (1.2.0)\n",
            "Requirement already satisfied: marshmallow<4.0.0,>=3.18.0 in /usr/local/lib/python3.10/dist-packages (from dataclasses-json<0.7,>=0.5.7->langchain) (3.20.1)\n",
            "Requirement already satisfied: typing-inspect<1,>=0.4.0 in /usr/local/lib/python3.10/dist-packages (from dataclasses-json<0.7,>=0.5.7->langchain) (0.9.0)\n",
            "Requirement already satisfied: certifi in /usr/local/lib/python3.10/dist-packages (from httpx<1,>=0.23.0->openai) (2023.11.17)\n",
            "Requirement already satisfied: httpcore==1.* in /usr/local/lib/python3.10/dist-packages (from httpx<1,>=0.23.0->openai) (1.0.2)\n",
            "Requirement already satisfied: h11<0.15,>=0.13 in /usr/local/lib/python3.10/dist-packages (from httpcore==1.*->httpx<1,>=0.23.0->openai) (0.14.0)\n",
            "Requirement already satisfied: jsonpointer>=1.9 in /usr/local/lib/python3.10/dist-packages (from jsonpatch<2.0,>=1.33->langchain) (2.4)\n",
            "Requirement already satisfied: packaging<24.0,>=23.2 in /usr/local/lib/python3.10/dist-packages (from langchain-core<0.2,>=0.1->langchain) (23.2)\n",
            "Requirement already satisfied: Pillow>=3.3.2 in /usr/local/lib/python3.10/dist-packages (from python-pptx<=0.6.23->unstructured[all-docs]) (10.1.0)\n",
            "Requirement already satisfied: XlsxWriter>=0.5.7 in /usr/local/lib/python3.10/dist-packages (from python-pptx<=0.6.23->unstructured[all-docs]) (3.1.9)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests<3,>=2->langchain) (3.3.2)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests<3,>=2->langchain) (2.0.7)\n",
            "Requirement already satisfied: greenlet!=0.4.17 in /usr/local/lib/python3.10/dist-packages (from SQLAlchemy<3,>=1.4->langchain) (3.0.2)\n",
            "Requirement already satisfied: soupsieve>1.2 in /usr/local/lib/python3.10/dist-packages (from beautifulsoup4->unstructured[all-docs]) (2.5)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.10/dist-packages (from langdetect->unstructured[all-docs]) (1.16.0)\n",
            "Requirement already satisfied: olefile>=0.46 in /usr/local/lib/python3.10/dist-packages (from msg-parser->unstructured[all-docs]) (0.47)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.10/dist-packages (from nltk->unstructured[all-docs]) (8.1.7)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.10/dist-packages (from nltk->unstructured[all-docs]) (1.3.2)\n",
            "Requirement already satisfied: protobuf>=3.20.2 in /usr/local/lib/python3.10/dist-packages (from onnx->unstructured[all-docs]) (3.20.3)\n",
            "Requirement already satisfied: et-xmlfile in /usr/local/lib/python3.10/dist-packages (from openpyxl->unstructured[all-docs]) (1.1.0)\n",
            "Requirement already satisfied: python-dateutil>=2.8.1 in /usr/local/lib/python3.10/dist-packages (from pandas->unstructured[all-docs]) (2.8.2)\n",
            "Requirement already satisfied: cryptography>=36.0.0 in /usr/local/lib/python3.10/dist-packages (from pdfminer.six->unstructured[all-docs]) (41.0.7)\n",
            "Requirement already satisfied: Deprecated in /usr/local/lib/python3.10/dist-packages (from pikepdf->unstructured[all-docs]) (1.2.14)\n",
            "Requirement already satisfied: jsonpath-python>=1.0.6 in /usr/local/lib/python3.10/dist-packages (from unstructured-client->unstructured[all-docs]) (1.0.6)\n",
            "Requirement already satisfied: mypy-extensions>=1.0.0 in /usr/local/lib/python3.10/dist-packages (from unstructured-client->unstructured[all-docs]) (1.0.0)\n",
            "Requirement already satisfied: cffi>=1.12 in /usr/local/lib/python3.10/dist-packages (from cryptography>=36.0.0->pdfminer.six->unstructured[all-docs]) (1.16.0)\n",
            "Requirement already satisfied: coloredlogs in /usr/local/lib/python3.10/dist-packages (from onnxruntime<1.16->unstructured-inference==0.7.18->unstructured[all-docs]) (15.0.1)\n",
            "Requirement already satisfied: flatbuffers in /usr/local/lib/python3.10/dist-packages (from onnxruntime<1.16->unstructured-inference==0.7.18->unstructured[all-docs]) (23.5.26)\n",
            "Requirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from onnxruntime<1.16->unstructured-inference==0.7.18->unstructured[all-docs]) (1.12)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from transformers>=4.25.1->unstructured-inference==0.7.18->unstructured[all-docs]) (3.13.1)\n",
            "Requirement already satisfied: tokenizers<0.19,>=0.14 in /usr/local/lib/python3.10/dist-packages (from transformers>=4.25.1->unstructured-inference==0.7.18->unstructured[all-docs]) (0.15.0)\n",
            "Requirement already satisfied: safetensors>=0.3.1 in /usr/local/lib/python3.10/dist-packages (from transformers>=4.25.1->unstructured-inference==0.7.18->unstructured[all-docs]) (0.4.1)\n",
            "Requirement already satisfied: fsspec>=2023.5.0 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub->unstructured-inference==0.7.18->unstructured[all-docs]) (2023.6.0)\n",
            "Requirement already satisfied: scipy in /usr/local/lib/python3.10/dist-packages (from layoutparser[layoutmodels,tesseract]->unstructured-inference==0.7.18->unstructured[all-docs]) (1.11.4)\n",
            "Requirement already satisfied: iopath in /usr/local/lib/python3.10/dist-packages (from layoutparser[layoutmodels,tesseract]->unstructured-inference==0.7.18->unstructured[all-docs]) (0.1.10)\n",
            "Requirement already satisfied: pdfplumber in /usr/local/lib/python3.10/dist-packages (from layoutparser[layoutmodels,tesseract]->unstructured-inference==0.7.18->unstructured[all-docs]) (0.10.3)\n",
            "Requirement already satisfied: pytesseract in /usr/local/lib/python3.10/dist-packages (from layoutparser[layoutmodels,tesseract]->unstructured-inference==0.7.18->unstructured[all-docs]) (0.3.10)\n",
            "Requirement already satisfied: torch in /usr/local/lib/python3.10/dist-packages (from layoutparser[layoutmodels,tesseract]->unstructured-inference==0.7.18->unstructured[all-docs]) (2.1.0+cu121)\n",
            "Requirement already satisfied: torchvision in /usr/local/lib/python3.10/dist-packages (from layoutparser[layoutmodels,tesseract]->unstructured-inference==0.7.18->unstructured[all-docs]) (0.16.0+cu121)\n",
            "Requirement already satisfied: effdet in /usr/local/lib/python3.10/dist-packages (from layoutparser[layoutmodels,tesseract]->unstructured-inference==0.7.18->unstructured[all-docs]) (0.4.1)\n",
            "Requirement already satisfied: pycparser in /usr/local/lib/python3.10/dist-packages (from cffi>=1.12->cryptography>=36.0.0->pdfminer.six->unstructured[all-docs]) (2.21)\n",
            "Requirement already satisfied: humanfriendly>=9.1 in /usr/local/lib/python3.10/dist-packages (from coloredlogs->onnxruntime<1.16->unstructured-inference==0.7.18->unstructured[all-docs]) (10.0)\n",
            "Requirement already satisfied: timm>=0.9.2 in /usr/local/lib/python3.10/dist-packages (from effdet->layoutparser[layoutmodels,tesseract]->unstructured-inference==0.7.18->unstructured[all-docs]) (0.9.12)\n",
            "Requirement already satisfied: pycocotools>=2.0.2 in /usr/local/lib/python3.10/dist-packages (from effdet->layoutparser[layoutmodels,tesseract]->unstructured-inference==0.7.18->unstructured[all-docs]) (2.0.7)\n",
            "Requirement already satisfied: omegaconf>=2.0 in /usr/local/lib/python3.10/dist-packages (from effdet->layoutparser[layoutmodels,tesseract]->unstructured-inference==0.7.18->unstructured[all-docs]) (2.3.0)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from torch->layoutparser[layoutmodels,tesseract]->unstructured-inference==0.7.18->unstructured[all-docs]) (3.1.2)\n",
            "Requirement already satisfied: triton==2.1.0 in /usr/local/lib/python3.10/dist-packages (from torch->layoutparser[layoutmodels,tesseract]->unstructured-inference==0.7.18->unstructured[all-docs]) (2.1.0)\n",
            "Requirement already satisfied: portalocker in /usr/local/lib/python3.10/dist-packages (from iopath->layoutparser[layoutmodels,tesseract]->unstructured-inference==0.7.18->unstructured[all-docs]) (2.8.2)\n",
            "Requirement already satisfied: pypdfium2>=4.18.0 in /usr/local/lib/python3.10/dist-packages (from pdfplumber->layoutparser[layoutmodels,tesseract]->unstructured-inference==0.7.18->unstructured[all-docs]) (4.25.0)\n",
            "Requirement already satisfied: mpmath>=0.19 in /usr/local/lib/python3.10/dist-packages (from sympy->onnxruntime<1.16->unstructured-inference==0.7.18->unstructured[all-docs]) (1.3.0)\n",
            "Requirement already satisfied: antlr4-python3-runtime==4.9.* in /usr/local/lib/python3.10/dist-packages (from omegaconf>=2.0->effdet->layoutparser[layoutmodels,tesseract]->unstructured-inference==0.7.18->unstructured[all-docs]) (4.9.3)\n",
            "Requirement already satisfied: matplotlib>=2.1.0 in /usr/local/lib/python3.10/dist-packages (from pycocotools>=2.0.2->effdet->layoutparser[layoutmodels,tesseract]->unstructured-inference==0.7.18->unstructured[all-docs]) (3.7.1)\n",
            "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->torch->layoutparser[layoutmodels,tesseract]->unstructured-inference==0.7.18->unstructured[all-docs]) (2.1.3)\n",
            "Requirement already satisfied: contourpy>=1.0.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib>=2.1.0->pycocotools>=2.0.2->effdet->layoutparser[layoutmodels,tesseract]->unstructured-inference==0.7.18->unstructured[all-docs]) (1.2.0)\n",
            "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.10/dist-packages (from matplotlib>=2.1.0->pycocotools>=2.0.2->effdet->layoutparser[layoutmodels,tesseract]->unstructured-inference==0.7.18->unstructured[all-docs]) (0.12.1)\n",
            "Requirement already satisfied: fonttools>=4.22.0 in /usr/local/lib/python3.10/dist-packages (from matplotlib>=2.1.0->pycocotools>=2.0.2->effdet->layoutparser[layoutmodels,tesseract]->unstructured-inference==0.7.18->unstructured[all-docs]) (4.46.0)\n",
            "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib>=2.1.0->pycocotools>=2.0.2->effdet->layoutparser[layoutmodels,tesseract]->unstructured-inference==0.7.18->unstructured[all-docs]) (1.4.5)\n",
            "Requirement already satisfied: pyparsing>=2.3.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib>=2.1.0->pycocotools>=2.0.2->effdet->layoutparser[layoutmodels,tesseract]->unstructured-inference==0.7.18->unstructured[all-docs]) (3.1.1)\n"
          ]
        }
      ],
      "source": [
        "!pip install langchain neo4j openai unstructured[all-docs] tiktoken"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Connecting Neo4j Graph Database"
      ],
      "metadata": {
        "id": "hrvTgjabE16y"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from langchain.graphs import Neo4jGraph\n",
        "\n",
        "url = \"neo4j+s://50ebd397.databases.neo4j.io\"\n",
        "username =\"neo4j\"\n",
        "password = \"OvfrBWwn4rYYngJJDTsVkwByoqjnhoi9lgfAwk1rydo\"\n",
        "graph = Neo4jGraph(\n",
        "    url=url,\n",
        "    username=username,\n",
        "    password=password\n",
        ")"
      ],
      "metadata": {
        "id": "F25P-N_DmEBK"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Defining Property, Node and Relationship for the Knowledge Graph"
      ],
      "metadata": {
        "id": "Sa7L1ScFHFf7"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from langchain.graphs.graph_document import (\n",
        "    Node as BaseNode,\n",
        "    Relationship as BaseRelationship,\n",
        "    GraphDocument,\n",
        ")\n",
        "from langchain.schema import Document\n",
        "from typing import List, Dict, Any, Optional\n",
        "from langchain.pydantic_v1 import Field, BaseModel\n",
        "\n",
        "class Property(BaseModel):\n",
        "  \"\"\"A single property consisting of key and value\"\"\"\n",
        "  key: str = Field(..., description=\"key\")\n",
        "  value: str = Field(..., description=\"value\")\n",
        "\n",
        "class Node(BaseNode):\n",
        "    properties: Optional[List[Property]] = Field(\n",
        "        None, description=\"List of node properties\")\n",
        "\n",
        "class Relationship(BaseRelationship):\n",
        "    properties: Optional[List[Property]] = Field(\n",
        "        None, description=\"List of relationship properties\"\n",
        "    )\n",
        "\n",
        "class KnowledgeGraph(BaseModel):\n",
        "    \"\"\"Generate a knowledge graph with entities and relationships.\"\"\"\n",
        "    nodes: List[Node] = Field(\n",
        "        ..., description=\"List of nodes in the knowledge graph\")\n",
        "    rels: List[Relationship] = Field(\n",
        "        ..., description=\"List of relationships in the knowledge graph\"\n",
        "    )"
      ],
      "metadata": {
        "id": "cRy1Rbt2Go26"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# KG to Base format conversion utilities"
      ],
      "metadata": {
        "id": "edTU3Yv6H00k"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def format_property_key(s: str) -> str:\n",
        "    words = s.split()\n",
        "    if not words:\n",
        "        return s\n",
        "    first_word = words[0].lower()\n",
        "    capitalized_words = [word.capitalize() for word in words[1:]]\n",
        "    return \"\".join([first_word] + capitalized_words)\n",
        "\n",
        "def props_to_dict(props) -> dict:\n",
        "    \"\"\"Convert properties to a dictionary.\"\"\"\n",
        "    properties = {}\n",
        "    if not props:\n",
        "      return properties\n",
        "    for p in props:\n",
        "        properties[format_property_key(p.key)] = p.value\n",
        "    return properties\n",
        "\n",
        "def map_to_base_node(node: Node) -> BaseNode:\n",
        "    \"\"\"Map the KnowledgeGraph Node to the base Node.\"\"\"\n",
        "    properties = props_to_dict(node.properties) if node.properties else {}\n",
        "    # Add name property for better Cypher statement generation\n",
        "    properties[\"name\"] = node.id.title()\n",
        "    return BaseNode(\n",
        "        id=node.id.title(), type=node.type.capitalize(), properties=properties\n",
        "    )\n",
        "\n",
        "\n",
        "def map_to_base_relationship(rel: Relationship) -> BaseRelationship:\n",
        "    \"\"\"Map the KnowledgeGraph Relationship to the base Relationship.\"\"\"\n",
        "    source = map_to_base_node(rel.source)\n",
        "    target = map_to_base_node(rel.target)\n",
        "    properties = props_to_dict(rel.properties) if rel.properties else {}\n",
        "    return BaseRelationship(\n",
        "        source=source, target=target, type=rel.type, properties=properties\n",
        "    )"
      ],
      "metadata": {
        "id": "-bV8nHwimqS7"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Training the LLM for the specific need"
      ],
      "metadata": {
        "id": "vqTLlddQH_7U"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import os\n",
        "from langchain.chains.openai_functions import (\n",
        "    create_openai_fn_chain,\n",
        "    create_structured_output_chain,\n",
        ")\n",
        "from langchain.chat_models import ChatOpenAI\n",
        "from langchain.prompts import ChatPromptTemplate\n",
        "os.environ[\"OPENAI_API_KEY\"] = \"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\"\n",
        "llm = ChatOpenAI(model=\"gpt-3.5-turbo-16k\", temperature=0)\n",
        "\n",
        "def get_extraction_chain(\n",
        "    allowed_nodes: Optional[List[str]] = None,\n",
        "    allowed_rels: Optional[List[str]] = None\n",
        "    ):\n",
        "    prompt = ChatPromptTemplate.from_messages(\n",
        "        [(\n",
        "          \"system\",\n",
        "          f\"\"\"# Knowledge Graph Instructions for GPT-3.5\n",
        "## 1. Overview\n",
        "You are a top-tier algorithm designed for extracting information in structured formats to build a knowledge graph.\n",
        "- **Nodes** represent entities and concepts. They’re akin to AWS EC2 documentation nodes.\n",
        "- The aim is to achieve simplicity and clarity in the knowledge graph, making it accessible for a vast audience.\n",
        "## 2. Labeling Nodes\n",
        "- **Consistency**: Ensure you use basic or elementary types for node labels.\n",
        "  - For example, when you identify an entity representing a service, always label it as **\"service\"**. Avoid using more specific terms like \"EC2 instance\" or \"VPC\".\n",
        "- **Node IDs**: Never utilize integers as node IDs. Node IDs should be names or human-readable identifiers found in the text.\n",
        "{'- **Allowed Node Labels:**' + \", \".join(allowed_nodes) if allowed_nodes else \"\"}\n",
        "{'- **Allowed Relationship Types**:' + \", \".join(allowed_rels) if allowed_rels else \"\"}\n",
        "## 3. Handling Numerical Data and Dates\n",
        "- Numerical data, like age or other related information, should be incorporated as attributes or properties of the respective nodes.\n",
        "- **No Separate Nodes for Dates/Numbers**: Do not create separate nodes for dates or numerical values. Always attach them as attributes or properties of nodes.\n",
        "- **Property Format**: Properties must be in a key-value format.\n",
        "- **Quotation Marks**: Never use escaped single or double quotes within property values.\n",
        "- **Naming Convention**: Use camelCase for property keys, e.g., `birthDate`.\n",
        "## 4. Coreference Resolution\n",
        "- **Maintain Entity Consistency**:When extracting entities, it’s vital to ensure consistency. If an entity, such as “Amazon EC2”, is mentioned multiple times in the text but is referred to by different names or abbreviations (e.g., “EC2”, “Elastic Compute Cloud”), always use the most complete identifier for that entity throughout the knowledge graph. In this example, use “Amazon EC2” as the entity ID. Remember, the knowledge graph should be coherent and easily understandable, so maintaining consistency in entity references is crucial.\n",
        "## 5. Strict Compliance\n",
        "Adhere to the rules strictly. Non-compliance will result in termination.\n",
        "          \"\"\"),\n",
        "            (\"human\", \"Use the given format to extract information from the following input: {input}\"),\n",
        "            (\"human\", \"Tip: Make sure to answer in the correct format\"),\n",
        "        ])\n",
        "    return create_structured_output_chain(KnowledgeGraph, llm, prompt, verbose=False)"
      ],
      "metadata": {
        "id": "kpGkdMyuUmAn"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Graph Skeleton"
      ],
      "metadata": {
        "id": "y9mSeuKpISRv"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def extract_and_store_graph(\n",
        "    document: Document,\n",
        "    nodes:Optional[List[str]] = None,\n",
        "    rels:Optional[List[str]]=None) -> None:\n",
        "    # Extract graph data using OpenAI functions\n",
        "    extract_chain = get_extraction_chain(nodes, rels)\n",
        "    data = extract_chain.run(document.page_content)\n",
        "    # Construct a graph document\n",
        "    graph_document = GraphDocument(\n",
        "      nodes = [map_to_base_node(node) for node in data.nodes],\n",
        "      relationships = [map_to_base_relationship(rel) for rel in data.rels],\n",
        "      source = document\n",
        "    )\n",
        "    # Store information into a graph\n",
        "    graph.add_graph_documents([graph_document])"
      ],
      "metadata": {
        "id": "Gu7aaanyU2NW"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Evaluation\n",
        "We will extract information from the AWS EC2 documentation and construct a knowledge graph to test the pipeline. Here, we will utilize the AWS EC2 pdf and text chunking modules provided by LangChain."
      ],
      "metadata": {
        "id": "Je_UuAIYPsc5"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from langchain.text_splitter import TokenTextSplitter\n",
        "from langchain.document_loaders import UnstructuredPDFLoader\n",
        "\n",
        "# Read the article\n",
        "raw_documents = UnstructuredPDFLoader(\"/content/ec2-ug-11-30.pdf\").load()\n",
        "\n",
        "# Define chunking strategy\n",
        "text_splitter = TokenTextSplitter(chunk_size=2048, chunk_overlap=24)\n",
        "\n",
        "# Only take the first the raw_documents\n",
        "documents = text_splitter.split_documents(raw_documents[:3])"
      ],
      "metadata": {
        "id": "wCvmpkhnyL0R"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from tqdm import tqdm\n",
        "\n",
        "for i, d in tqdm(enumerate(documents), total=len(documents)):\n",
        "    extract_and_store_graph(d)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "CULLN4VMU_Ou",
        "outputId": "e206011d-4978-4b57-e7b0-7618fcee84a8"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 5/5 [07:45<00:00, 93.19s/it]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Reset Graph - Delete previous Nodes and Relationships"
      ],
      "metadata": {
        "id": "-uNP0WzZJBVa"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Delete the graph\n",
        "graph.query(\"MATCH (n) DETACH DELETE n\")"
      ],
      "metadata": {
        "id": "5wIIu5DJQVKK",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "658f5f88-987c-449c-a3d6-d05f8ca984e7"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[]"
            ]
          },
          "metadata": {},
          "execution_count": 14
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Specify which node labels should be extracted by the LLM"
      ],
      "metadata": {
        "id": "ez8J29fOJYbk"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "allowed_nodes = [\"How to Get Started with Amazon EC2\",\"Related Services\",\"Accessing Amazon EC2\",\"Features of Amazon EC2\",\"Instances and AMIs\",\"Storage for Your Instance\",\"Instances\",\"Stopping, Starting, and Terminating Instances\",\"AMIs\",\"Amazon Machine Image (AMI)\", \"Security Best Practices\",\"Regions\",\"Availability Zones\",\"Describing Your Regions and Availability Zones\",\"Specifying the Region for a Resource\"]\n",
        "\n",
        "for i, d in tqdm(enumerate(documents), total=len(documents)):\n",
        "    extract_and_store_graph(d, allowed_nodes)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "shS8aPGAvZgT",
        "outputId": "46caa538-64d0-4a31-89a5-4f937f8b0f53"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 5/5 [03:33<00:00, 42.79s/it]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Query the knowledge graph in a RAG application"
      ],
      "metadata": {
        "id": "foFAv7owJfd_"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Query the knowledge graph in a RAG application\n",
        "from langchain.chains import GraphCypherQAChain\n",
        "\n",
        "graph.refresh_schema()\n",
        "\n",
        "cypher_chain = GraphCypherQAChain.from_llm(\n",
        "    graph=graph,\n",
        "    cypher_llm=ChatOpenAI(temperature=0, model=\"gpt-3.5-turbo-16k\"),\n",
        "    qa_llm=ChatOpenAI(temperature=0, model=\"gpt-3.5-turbo\"),\n",
        "    validate_cypher=True, # Validate relationship directions\n",
        "    verbose=True\n",
        ")"
      ],
      "metadata": {
        "id": "8EeZG9I8JYGJ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "cypher_chain.run(\"How to specify the region for a resource using the console?\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 261
        },
        "id": "0-qaxjC2NOJm",
        "outputId": "7c02f1a0-73af-44a1-d601-cd8bd41a39ef"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "\n",
            "\u001b[1m> Entering new GraphCypherQAChain chain...\u001b[0m\n",
            "Generated Cypher:\n",
            "\u001b[32;1m\u001b[1;3mMATCH (s:Service {name: \"Resource\"})-[:HAS]->(c:Concept {name: \"Region\"})\n",
            "RETURN c.name\u001b[0m\n",
            "Full Context:\n",
            "\u001b[32;1m\u001b[1;3m[]\u001b[0m\n",
            "\n",
            "\u001b[1m> Finished chain.\u001b[0m\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'To specify the region for a resource using the console, you can follow these steps:\\n\\n1. Open the console for the cloud service provider you are using.\\n2. Navigate to the resource you want to specify the region for.\\n3. Look for the region settings or configuration options for the resource.\\n4. Select the desired region from the available options.\\n5. Save or apply the changes to ensure that the resource is deployed or located in the specified region.\\n\\nBy following these steps, you can easily specify the region for a resource using the console.'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 18
        }
      ]
    }
  ]
}