psychic_notebook.ipynb

{
 "cells": [
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {
    "id": "htlUaTMwU6bl"
   },
   "source": [
    "# Psychic\n",
    "This notebook covers how to load documents from the [Psychic python library](https://pypi.org/project/psychicapi/) and use them for question answering. See [here](../../../../ecosystem/psychic.md) for more details.\n",
    "\n",
    "## Prerequisites\n",
    "1. Follow the Quick Start section in [this document](../../../../ecosystem/psychic.md)\n",
    "2. Log into the [Psychic dashboard](https://dashboard.psychic.dev/) and get your secret key\n",
    "3. Install the frontend react library into your web app and have a user authenticate a connection. The connection will be created using the connection id that you specify."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "BTBqjwwVVRO5",
    "outputId": "887501d3-354d-4e1e-fe47-33e50d9b7834"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: psychicapi in /opt/homebrew/lib/python3.10/site-packages (0.6)\n",
      "Requirement already satisfied: requests in /opt/homebrew/lib/python3.10/site-packages (from psychicapi) (2.28.2)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /opt/homebrew/lib/python3.10/site-packages (from requests->psychicapi) (2022.12.7)\n",
      "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /opt/homebrew/lib/python3.10/site-packages (from requests->psychicapi) (1.26.15)\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in /opt/homebrew/lib/python3.10/site-packages (from requests->psychicapi) (3.1.0)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /opt/homebrew/lib/python3.10/site-packages (from requests->psychicapi) (3.4)\n",
      "\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip available: \u001b[0m\u001b[31;49m22.3.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m23.1.2\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpython3.10 -m pip install --upgrade pip\u001b[0m\n",
      "Requirement already satisfied: langchain in /opt/homebrew/lib/python3.10/site-packages (0.0.174)\n",
      "Requirement already satisfied: aiohttp<4.0.0,>=3.8.3 in /opt/homebrew/lib/python3.10/site-packages (from langchain) (3.8.4)\n",
      "Requirement already satisfied: PyYAML>=5.4.1 in /opt/homebrew/lib/python3.10/site-packages (from langchain) (6.0)\n",
      "Requirement already satisfied: pydantic<2,>=1 in /opt/homebrew/lib/python3.10/site-packages (from langchain) (1.10.4)\n",
      "Requirement already satisfied: SQLAlchemy<3,>=1.4 in /opt/homebrew/lib/python3.10/site-packages (from langchain) (2.0.15)\n",
      "Requirement already satisfied: tenacity<9.0.0,>=8.1.0 in /opt/homebrew/lib/python3.10/site-packages (from langchain) (8.2.2)\n",
      "Requirement already satisfied: requests<3,>=2 in /opt/homebrew/lib/python3.10/site-packages (from langchain) (2.28.2)\n",
      "Requirement already satisfied: openapi-schema-pydantic<2.0,>=1.2 in /opt/homebrew/lib/python3.10/site-packages (from langchain) (1.2.4)\n",
      "Requirement already satisfied: async-timeout<5.0.0,>=4.0.0 in /opt/homebrew/lib/python3.10/site-packages (from langchain) (4.0.2)\n",
      "Requirement already satisfied: dataclasses-json<0.6.0,>=0.5.7 in /opt/homebrew/lib/python3.10/site-packages (from langchain) (0.5.7)\n",
      "Requirement already satisfied: numpy<2,>=1 in /opt/homebrew/lib/python3.10/site-packages (from langchain) (1.24.1)\n",
      "Requirement already satisfied: numexpr<3.0.0,>=2.8.4 in /opt/homebrew/lib/python3.10/site-packages (from langchain) (2.8.4)\n",
      "Requirement already satisfied: attrs>=17.3.0 in /opt/homebrew/lib/python3.10/site-packages (from aiohttp<4.0.0,>=3.8.3->langchain) (22.2.0)\n",
      "Requirement already satisfied: multidict<7.0,>=4.5 in /opt/homebrew/lib/python3.10/site-packages (from aiohttp<4.0.0,>=3.8.3->langchain) (6.0.4)\n",
      "Requirement already satisfied: charset-normalizer<4.0,>=2.0 in /opt/homebrew/lib/python3.10/site-packages (from aiohttp<4.0.0,>=3.8.3->langchain) (3.1.0)\n",
      "Requirement already satisfied: frozenlist>=1.1.1 in /opt/homebrew/lib/python3.10/site-packages (from aiohttp<4.0.0,>=3.8.3->langchain) (1.3.3)\n",
      "Requirement already satisfied: aiosignal>=1.1.2 in /opt/homebrew/lib/python3.10/site-packages (from aiohttp<4.0.0,>=3.8.3->langchain) (1.3.1)\n",
      "Requirement already satisfied: yarl<2.0,>=1.0 in /opt/homebrew/lib/python3.10/site-packages (from aiohttp<4.0.0,>=3.8.3->langchain) (1.8.2)\n",
      "Requirement already satisfied: typing-inspect>=0.4.0 in /opt/homebrew/lib/python3.10/site-packages (from dataclasses-json<0.6.0,>=0.5.7->langchain) (0.8.0)\n",
      "Requirement already satisfied: marshmallow-enum<2.0.0,>=1.5.1 in /opt/homebrew/lib/python3.10/site-packages (from dataclasses-json<0.6.0,>=0.5.7->langchain) (1.5.1)\n",
      "Requirement already satisfied: marshmallow<4.0.0,>=3.3.0 in /opt/homebrew/lib/python3.10/site-packages (from dataclasses-json<0.6.0,>=0.5.7->langchain) (3.19.0)\n",
      "Requirement already satisfied: typing-extensions>=4.2.0 in /opt/homebrew/lib/python3.10/site-packages (from pydantic<2,>=1->langchain) (4.5.0)\n",
      "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /opt/homebrew/lib/python3.10/site-packages (from requests<3,>=2->langchain) (1.26.15)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /opt/homebrew/lib/python3.10/site-packages (from requests<3,>=2->langchain) (2022.12.7)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /opt/homebrew/lib/python3.10/site-packages (from requests<3,>=2->langchain) (3.4)\n",
      "Requirement already satisfied: packaging>=17.0 in /opt/homebrew/lib/python3.10/site-packages (from marshmallow<4.0.0,>=3.3.0->dataclasses-json<0.6.0,>=0.5.7->langchain) (23.0)\n",
      "Requirement already satisfied: mypy-extensions>=0.3.0 in /opt/homebrew/lib/python3.10/site-packages (from typing-inspect>=0.4.0->dataclasses-json<0.6.0,>=0.5.7->langchain) (1.0.0)\n",
      "\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip available: \u001b[0m\u001b[31;49m22.3.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m23.1.2\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpython3.10 -m pip install --upgrade pip\u001b[0m\n",
      "Requirement already satisfied: openai in /opt/homebrew/lib/python3.10/site-packages (0.26.4)\n",
      "Requirement already satisfied: aiohttp in /opt/homebrew/lib/python3.10/site-packages (from openai) (3.8.4)\n",
      "Requirement already satisfied: tqdm in /opt/homebrew/lib/python3.10/site-packages (from openai) (4.64.1)\n",
      "Requirement already satisfied: requests>=2.20 in /opt/homebrew/lib/python3.10/site-packages (from openai) (2.28.2)\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in /opt/homebrew/lib/python3.10/site-packages (from requests>=2.20->openai) (3.1.0)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /opt/homebrew/lib/python3.10/site-packages (from requests>=2.20->openai) (3.4)\n",
      "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /opt/homebrew/lib/python3.10/site-packages (from requests>=2.20->openai) (1.26.15)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /opt/homebrew/lib/python3.10/site-packages (from requests>=2.20->openai) (2022.12.7)\n",
      "Requirement already satisfied: aiosignal>=1.1.2 in /opt/homebrew/lib/python3.10/site-packages (from aiohttp->openai) (1.3.1)\n",
      "Requirement already satisfied: frozenlist>=1.1.1 in /opt/homebrew/lib/python3.10/site-packages (from aiohttp->openai) (1.3.3)\n",
      "Requirement already satisfied: attrs>=17.3.0 in /opt/homebrew/lib/python3.10/site-packages (from aiohttp->openai) (22.2.0)\n",
      "Requirement already satisfied: yarl<2.0,>=1.0 in /opt/homebrew/lib/python3.10/site-packages (from aiohttp->openai) (1.8.2)\n",
      "Requirement already satisfied: multidict<7.0,>=4.5 in /opt/homebrew/lib/python3.10/site-packages (from aiohttp->openai) (6.0.4)\n",
      "Requirement already satisfied: async-timeout<5.0,>=4.0.0a3 in /opt/homebrew/lib/python3.10/site-packages (from aiohttp->openai) (4.0.2)\n",
      "\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip available: \u001b[0m\u001b[31;49m22.3.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m23.1.2\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpython3.10 -m pip install --upgrade pip\u001b[0m\n",
      "Collecting pybind11\n",
      "  Using cached pybind11-2.10.4-py3-none-any.whl (222 kB)\n",
      "Installing collected packages: pybind11\n",
      "Successfully installed pybind11-2.10.4\n",
      "\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip available: \u001b[0m\u001b[31;49m22.3.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m23.1.2\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpython3.10 -m pip install --upgrade pip\u001b[0m\n",
      "Collecting chromadb\n",
      "  Using cached chromadb-0.3.25-py3-none-any.whl (86 kB)\n",
      "Requirement already satisfied: numpy>=1.21.6 in /opt/homebrew/lib/python3.10/site-packages (from chromadb) (1.24.1)\n",
      "Requirement already satisfied: typing-extensions>=4.5.0 in /opt/homebrew/lib/python3.10/site-packages (from chromadb) (4.5.0)\n",
      "Collecting uvicorn[standard]>=0.18.3\n",
      "  Using cached uvicorn-0.22.0-py3-none-any.whl (58 kB)\n",
      "Collecting hnswlib>=0.7\n",
      "  Using cached hnswlib-0.7.0.tar.gz (33 kB)\n",
      "  Installing build dependencies ... \u001b[?25ldone\n",
      "\u001b[?25h  Getting requirements to build wheel ... \u001b[?25ldone\n",
      "\u001b[?25h  Preparing metadata (pyproject.toml) ... \u001b[?25ldone\n",
      "\u001b[?25hCollecting tqdm>=4.65.0\n",
      "  Using cached tqdm-4.65.0-py3-none-any.whl (77 kB)\n",
      "Collecting onnxruntime>=1.14.1\n",
      "  Using cached onnxruntime-1.15.0-cp310-cp310-macosx_11_0_arm64.whl (6.1 MB)\n",
      "Requirement already satisfied: tokenizers>=0.13.2 in /opt/homebrew/lib/python3.10/site-packages (from chromadb) (0.13.2)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: requests>=2.28 in /opt/homebrew/lib/python3.10/site-packages (from chromadb) (2.28.2)\n",
      "Collecting overrides>=7.3.1\n",
      "  Using cached overrides-7.3.1-py3-none-any.whl (17 kB)\n",
      "Requirement already satisfied: pydantic>=1.9 in /opt/homebrew/lib/python3.10/site-packages (from chromadb) (1.10.4)\n",
      "Requirement already satisfied: fastapi>=0.85.1 in /opt/homebrew/lib/python3.10/site-packages (from chromadb) (0.95.2)\n",
      "Requirement already satisfied: pandas>=1.3 in /opt/homebrew/lib/python3.10/site-packages (from chromadb) (1.5.3)\n",
      "Collecting clickhouse-connect>=0.5.7\n",
      "  Using cached clickhouse_connect-0.5.25-cp310-cp310-macosx_11_0_arm64.whl (227 kB)\n",
      "Collecting duckdb>=0.7.1\n",
      "  Using cached duckdb-0.8.0-cp310-cp310-macosx_11_0_arm64.whl (12.6 MB)\n",
      "Collecting posthog>=2.4.0\n",
      "  Using cached posthog-3.0.1-py2.py3-none-any.whl (37 kB)\n",
      "Collecting zstandard\n",
      "  Downloading zstandard-0.21.0-cp310-cp310-macosx_11_0_arm64.whl (364 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m364.7/364.7 kB\u001b[0m \u001b[31m5.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0ma \u001b[36m0:00:01\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: pytz in /opt/homebrew/lib/python3.10/site-packages (from clickhouse-connect>=0.5.7->chromadb) (2022.7.1)\n",
      "Requirement already satisfied: urllib3>=1.26 in /opt/homebrew/lib/python3.10/site-packages (from clickhouse-connect>=0.5.7->chromadb) (1.26.15)\n",
      "Requirement already satisfied: certifi in /opt/homebrew/lib/python3.10/site-packages (from clickhouse-connect>=0.5.7->chromadb) (2022.12.7)\n",
      "Collecting lz4\n",
      "  Downloading lz4-4.3.2-cp310-cp310-macosx_11_0_arm64.whl (212 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m212.3/212.3 kB\u001b[0m \u001b[31m11.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: starlette<0.28.0,>=0.27.0 in /opt/homebrew/lib/python3.10/site-packages (from fastapi>=0.85.1->chromadb) (0.27.0)\n",
      "Collecting coloredlogs\n",
      "  Downloading coloredlogs-15.0.1-py2.py3-none-any.whl (46 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m46.0/46.0 kB\u001b[0m \u001b[31m5.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: protobuf in /opt/homebrew/lib/python3.10/site-packages (from onnxruntime>=1.14.1->chromadb) (4.22.0)\n",
      "Collecting flatbuffers\n",
      "  Downloading flatbuffers-23.5.26-py2.py3-none-any.whl (26 kB)\n",
      "Collecting sympy\n",
      "  Downloading sympy-1.12-py3-none-any.whl (5.7 MB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m5.7/5.7 MB\u001b[0m \u001b[31m17.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: packaging in /opt/homebrew/lib/python3.10/site-packages (from onnxruntime>=1.14.1->chromadb) (23.0)\n",
      "Requirement already satisfied: python-dateutil>=2.8.1 in /opt/homebrew/lib/python3.10/site-packages (from pandas>=1.3->chromadb) (2.8.2)\n",
      "Requirement already satisfied: six>=1.5 in /opt/homebrew/lib/python3.10/site-packages (from posthog>=2.4.0->chromadb) (1.16.0)\n",
      "Requirement already satisfied: backoff>=1.10.0 in /opt/homebrew/lib/python3.10/site-packages (from posthog>=2.4.0->chromadb) (2.2.1)\n",
      "Requirement already satisfied: monotonic>=1.5 in /opt/homebrew/lib/python3.10/site-packages (from posthog>=2.4.0->chromadb) (1.6)\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in /opt/homebrew/lib/python3.10/site-packages (from requests>=2.28->chromadb) (3.1.0)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /opt/homebrew/lib/python3.10/site-packages (from requests>=2.28->chromadb) (3.4)\n",
      "Requirement already satisfied: click>=7.0 in /opt/homebrew/lib/python3.10/site-packages (from uvicorn[standard]>=0.18.3->chromadb) (8.1.3)\n",
      "Collecting h11>=0.8\n",
      "  Downloading h11-0.14.0-py3-none-any.whl (58 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m58.3/58.3 kB\u001b[0m \u001b[31m3.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: python-dotenv>=0.13 in /opt/homebrew/lib/python3.10/site-packages (from uvicorn[standard]>=0.18.3->chromadb) (1.0.0)\n",
      "Collecting websockets>=10.4\n",
      "  Downloading websockets-11.0.3-cp310-cp310-macosx_11_0_arm64.whl (121 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m121.0/121.0 kB\u001b[0m \u001b[31m10.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: pyyaml>=5.1 in /opt/homebrew/lib/python3.10/site-packages (from uvicorn[standard]>=0.18.3->chromadb) (6.0)\n",
      "Collecting httptools>=0.5.0\n",
      "  Downloading httptools-0.5.0-cp310-cp310-macosx_10_9_universal2.whl (228 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m228.3/228.3 kB\u001b[0m \u001b[31m11.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hCollecting watchfiles>=0.13\n",
      "  Downloading watchfiles-0.19.0-cp37-abi3-macosx_11_0_arm64.whl (388 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m388.0/388.0 kB\u001b[0m \u001b[31m19.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hCollecting uvloop!=0.15.0,!=0.15.1,>=0.14.0\n",
      "  Downloading uvloop-0.17.0-cp310-cp310-macosx_10_9_universal2.whl (2.1 MB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.1/2.1 MB\u001b[0m \u001b[31m13.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: anyio<5,>=3.4.0 in /opt/homebrew/lib/python3.10/site-packages (from starlette<0.28.0,>=0.27.0->fastapi>=0.85.1->chromadb) (3.6.2)\n",
      "Collecting humanfriendly>=9.1\n",
      "  Downloading humanfriendly-10.0-py2.py3-none-any.whl (86 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m86.8/86.8 kB\u001b[0m \u001b[31m8.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hCollecting mpmath>=0.19\n",
      "  Downloading mpmath-1.3.0-py3-none-any.whl (536 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m536.2/536.2 kB\u001b[0m \u001b[31m17.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: sniffio>=1.1 in /opt/homebrew/lib/python3.10/site-packages (from anyio<5,>=3.4.0->starlette<0.28.0,>=0.27.0->fastapi>=0.85.1->chromadb) (1.3.0)\n",
      "Building wheels for collected packages: hnswlib\n",
      "  Building wheel for hnswlib (pyproject.toml) ... \u001b[?25ldone\n",
      "\u001b[?25h  Created wheel for hnswlib: filename=hnswlib-0.7.0-cp310-cp310-macosx_13_0_arm64.whl size=185148 sha256=f85a1ed3b279eb31778188280c8550d39f08a9ddcab962f72d3b3a013cfeb0cf\n",
      "  Stored in directory: /Users/jasonfan/Library/Caches/pip/wheels/36/5e/06/bba7cbc79ef454a94746ba051381601575597e1b324b1221da\n",
      "Successfully built hnswlib\n",
      "Installing collected packages: mpmath, flatbuffers, duckdb, zstandard, websockets, uvloop, tqdm, sympy, overrides, lz4, humanfriendly, httptools, hnswlib, h11, watchfiles, uvicorn, posthog, coloredlogs, clickhouse-connect, onnxruntime, chromadb\n",
      "  Attempting uninstall: tqdm\n",
      "    Found existing installation: tqdm 4.64.1\n",
      "    Uninstalling tqdm-4.64.1:\n",
      "      Successfully uninstalled tqdm-4.64.1\n",
      "  Attempting uninstall: posthog\n",
      "    Found existing installation: posthog 2.3.1\n",
      "    Uninstalling posthog-2.3.1:\n",
      "      Successfully uninstalled posthog-2.3.1\n",
      "Successfully installed chromadb-0.3.25 clickhouse-connect-0.5.25 coloredlogs-15.0.1 duckdb-0.8.0 flatbuffers-23.5.26 h11-0.14.0 hnswlib-0.7.0 httptools-0.5.0 humanfriendly-10.0 lz4-4.3.2 mpmath-1.3.0 onnxruntime-1.15.0 overrides-7.3.1 posthog-3.0.1 sympy-1.12 tqdm-4.65.0 uvicorn-0.22.0 uvloop-0.17.0 watchfiles-0.19.0 websockets-11.0.3 zstandard-0.21.0\n",
      "\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip available: \u001b[0m\u001b[31;49m22.3.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m23.1.2\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpython3.10 -m pip install --upgrade pip\u001b[0m\n",
      "Requirement already satisfied: tiktoken in /opt/homebrew/lib/python3.10/site-packages (0.2.0)\n",
      "Requirement already satisfied: regex>=2022.1.18 in /opt/homebrew/lib/python3.10/site-packages (from tiktoken) (2022.10.31)\n",
      "Requirement already satisfied: requests>=2.26.0 in /opt/homebrew/lib/python3.10/site-packages (from tiktoken) (2.28.2)\n",
      "Requirement already satisfied: blobfile>=2 in /opt/homebrew/lib/python3.10/site-packages (from tiktoken) (2.0.1)\n",
      "Requirement already satisfied: filelock~=3.0 in /opt/homebrew/lib/python3.10/site-packages (from blobfile>=2->tiktoken) (3.10.7)\n",
      "Requirement already satisfied: pycryptodomex~=3.8 in /opt/homebrew/lib/python3.10/site-packages (from blobfile>=2->tiktoken) (3.17)\n",
      "Requirement already satisfied: urllib3<3,>=1.25.3 in /opt/homebrew/lib/python3.10/site-packages (from blobfile>=2->tiktoken) (1.26.15)\n",
      "Requirement already satisfied: lxml~=4.9 in /opt/homebrew/lib/python3.10/site-packages (from blobfile>=2->tiktoken) (4.9.2)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /opt/homebrew/lib/python3.10/site-packages (from requests>=2.26.0->tiktoken) (2022.12.7)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /opt/homebrew/lib/python3.10/site-packages (from requests>=2.26.0->tiktoken) (3.4)\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in /opt/homebrew/lib/python3.10/site-packages (from requests>=2.26.0->tiktoken) (3.1.0)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip available: \u001b[0m\u001b[31;49m22.3.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m23.1.2\u001b[0m\r\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpython3.10 -m pip install --upgrade pip\u001b[0m\r\n"
     ]
    }
   ],
   "source": [
    "!pip install psychicapi\n",
    "!pip install langchain\n",
    "!pip install openai \n",
    "!pip install chromadb\n",
    "!pip install tiktoken"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "id": "NlrVWJfLY7wb"
   },
   "outputs": [],
   "source": [
    "import os"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {
    "id": "RgICg0xTWdNS"
   },
   "source": [
    "## Loading documents\n",
    "\n",
    "Use the `get_documents` function to load in documents from a connection. Each connection has a connector id (corresponding to the SaaS app that was connected) and a connection id (which you passed in to the frontend library)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "Yj4ZOXHCVWi5",
    "outputId": "ab4d7e46-899b-47a1-e343-24d1de215217"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Document(page_content='(05/31/23:12:17) jason: nvm adding it now\\n(05/31/23:10:53) jason: i.e. add get_conversations\\n(05/31/23:10:53) jason: did you already make the changes to the python package for slack? I want to make a post about it today', metadata={'source': 'C04DWRUG83T', 'source_name': 'engineering'})]\n"
     ]
    }
   ],
   "source": [
    "from dotenv import load_dotenv\n",
    "load_dotenv()\n",
    "from langchain.docstore.document import Document\n",
    "from psychicapi import Psychic, ConnectorId\n",
    "from bs4 import BeautifulSoup\n",
    "import datetime\n",
    "\n",
    "# Create a document loader for google drive. We can also load from other connectors by setting the connector_id to the appropriate value e.g. ConnectorId.notion.value\n",
    "# This loader uses our test credentials\n",
    "psychic = Psychic(secret_key=os.getenv(\"PSYCHIC_SECRET_KEY\"))\n",
    "\n",
    "raw_messages = psychic.get_conversations(connector_id=ConnectorId.slack, account_id=\"slack_test\")\n",
    "crammed_messages = []\n",
    "channel_id = None\n",
    "channel_name = None\n",
    "for message in raw_messages:\n",
    "    # Message content is provided as HTML in order to preserve markup like links, tables, images, etc.\n",
    "    channel_id = message[\"channel\"][\"id\"]\n",
    "    channel_name = message[\"channel\"][\"name\"]\n",
    "    content = BeautifulSoup(message[\"content\"]).get_text()\n",
    "    time = datetime.datetime.fromtimestamp(float(message[\"timestamp\"])).strftime(\"%m/%d/%y:%H:%M\")\n",
    "    crammed_messages.append(\"({}) \".format(time) + message[\"sender\"][\"name\"] + \": \" + content)\n",
    "# print(crammed_messages)\n",
    "\n",
    "docs = [Document(page_content=\"\\n\".join(crammed_messages), metadata={\"source\": channel_id, \"source_name\": channel_name})]\n",
    "print(docs)\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {
    "id": "21GMyhgaWlVn"
   },
   "source": [
    "## Converting the docs to embeddings \n",
    "\n",
    "We can now convert these documents into embeddings and store them in a vector database like Chroma"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "6usIGR64Wm4Z",
    "outputId": "a29c63e6-f027-4006-f5b4-c6d594f043e3"
   },
   "outputs": [],
   "source": [
    "from langchain.embeddings.openai import OpenAIEmbeddings\n",
    "from langchain.vectorstores import Chroma\n",
    "from langchain.text_splitter import CharacterTextSplitter\n",
    "from langchain.llms import OpenAI\n",
    "from langchain.chains import RetrievalQAWithSourcesChain\n",
    "\n",
    "text_splitter = CharacterTextSplitter(chunk_size=1000, chunk_overlap=0)\n",
    "texts = text_splitter.split_documents(docs)\n",
    "\n",
    "\n",
    "embeddings = OpenAIEmbeddings()\n",
    "docsearch = Chroma.from_documents(texts, embeddings)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {
    "id": "K9snKtfFWx32"
   },
   "source": [
    "## Question answering\n",
    "\n",
    "Now we use OpenAI to ask questions over the"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "d7zlKUVYWpIF",
    "outputId": "203e6140-a5b1-44bd-d341-e5f1c7cf1a96"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Number of requested results 4 is greater than number of elements in index 1, updating n_results = 1\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'answer': ' Jason wanted to add the get_conversations feature to the Python package for Slack.\\n',\n",
       " 'sources': 'C04DWRUG83T'}"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "chain = RetrievalQAWithSourcesChain.from_chain_type(OpenAI(temperature=0), chain_type=\"stuff\", retriever=docsearch.as_retriever())\n",
    "chain({\"question\": \"what did jason want to do on 05/31/2023?\"}, return_only_outputs=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "colab": {
   "provenance": [],
   "toc_visible": true
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}