Merge pull request #13 from mongodb-developer/rag_local_load

Local dataset loading in RAG lab

Author: ajosh0504

notebooks/ai-rag-lab.ipynb

@@ -51,7 +51,7 @@
       "cell_type": "markdown",
       "metadata": {},
       "source": [
-        "# Step 2: Download the dataset"
+        "# Step 2: Load the dataset"
       ]
     },
     {
@@ -60,9 +60,7 @@
       "metadata": {},
       "outputs": [],
       "source": [
-        "# You may see a warning upon running this cell. You can ignore it.\n",
-        "import pandas as pd\n",
-        "from datasets import load_dataset"
+        "import json"
       ]
     },
     {
@@ -71,10 +69,10 @@
       "metadata": {},
       "outputs": [],
       "source": [
-        "# Download the `mongodb-docs` dataset from Hugging Face\n",
-        "data = load_dataset(\"mongodb/mongodb-docs\", split=\"train\")\n",
-        "# Convert the dataset into a dataframe first, then into a list of Python objects/dictionaries\n",
-        "docs = pd.DataFrame(data).to_dict(\"records\")"
+        "with open(\"../data/mongodb_docs.json\", \"r\") as data_file:\n",
+        "    json_data = data_file.read()\n",
+        "\n",
+        "docs = json.loads(json_data)"
       ]
     },
     {
@@ -177,7 +175,7 @@
         "    chunked_data = []\n",
         "    for chunk in chunks:\n",
         "        temp = doc.copy()\n",
-        "        temp[text_field]=chunk\n",
+        "        temp[text_field] = chunk\n",
         "        chunked_data.append(temp)\n",
         "\n",
         "    return chunked_data"

requirements.txt

@@ -1,5 +1,4 @@
 pymongo==4.11.3
-datasets==3.6.0
 langchain==0.3.25
 langchain-aws==0.2.22
 langchain-google-genai==2.1.4