google_maps_scraper (1).ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "5eb1f0e9",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Import webdriver\n",
    "from selenium import webdriver\n",
    "from selenium.webdriver.common.keys import Keys\n",
    "from selenium.webdriver.chrome.options import Options\n",
    "from selenium.webdriver.support.ui import WebDriverWait\n",
    "from selenium.webdriver.common.by import By\n",
    "from selenium.webdriver.support import expected_conditions as EC\n",
    "from webdriver_manager.chrome import ChromeDriverManager\n",
    "from selenium.webdriver.chrome.service import Service\n",
    "import time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "d04c1168",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<selenium.webdriver.remote.webelement.WebElement (session=\"59ee3622f19952027624c71624f2dcaf\", element=\"f.9ACF1A8F82F7691A78EB0518BFB16FF5.d.F4FFA8E8EE54C83190022049F9612B38.e.42\")>"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "chrome_options = Options()\n",
    "chrome_options.add_argument(\"--start-maximized\")\n",
    "service = Service(ChromeDriverManager().install())\n",
    "driver = webdriver.Chrome(service=service, options=chrome_options)\n",
    "\n",
    "# Initialize the webdriver\n",
    "count = 0\n",
    "url_path = r\"https://www.google.com/maps\"\n",
    "driver.get(url_path)\n",
    "\n",
    "# initiate search\n",
    "search_input = input(\"Enter the search key word/s for Google Maps : \")\n",
    "xpath_searchbox = r'//*[@id=\"searchboxinput\"]'\n",
    "enter = driver.find_element(By.XPATH,xpath_searchbox)\n",
    "enter.send_keys(search_input)\n",
    "enter.send_keys(Keys.ENTER)\n",
    "\n",
    "# Wait for the page to load\n",
    "wait = WebDriverWait(driver, 10)\n",
    "wait.until(EC.presence_of_element_located((By.ID, \"pane\")))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "aa53d599",
   "metadata": {},
   "outputs": [],
   "source": [
    "def scroll_down(driver,pixels=500):\n",
    "    arg = \"window.scrollTo(0, );\"\n",
    "    arg = arg[:-3] + str(pixels) + arg[-3+1:]\n",
    "    driver.execute_script(arg,\"\")\n",
    "    time.sleep(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "94d756f3",
   "metadata": {},
   "outputs": [],
   "source": [
    "# get the height of the page\n",
    "last_height = driver.execute_script(\"return document.body.scrollHeight\")\n",
    "\n",
    "# loop to scroll down the page\n",
    "while True:\n",
    "    # scroll down to the bottom of the page\n",
    "    driver.execute_script(\"window.scrollTo(0, document.body.scrollHeight);\")\n",
    "\n",
    "    # wait for the page to load\n",
    "    time.sleep(2)\n",
    "\n",
    "    # calculate the new height of the page\n",
    "    new_height = driver.execute_script(\"return document.body.scrollHeight\")\n",
    "\n",
    "    # check if the page height has changed\n",
    "    if new_height == last_height:\n",
    "        break\n",
    "\n",
    "    # update the last height\n",
    "    last_height = new_height"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "d03cf58c",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "xpath_objects = r'//*[@id=\"QA0Szd\"]/div/div/div[1]/div[2]/div/div[1]/div/div/div[2]/div[1]/div[3]/div/a'\n",
    "odd = 3\n",
    "names = []\n",
    "#addresses = []\n",
    "urls = []\n",
    "status = []\n",
    "address = []\n",
    "wait = WebDriverWait(driver, 10)\n",
    "screen_height = driver.execute_script(\"return window.screen.height;\")\n",
    "count = 0\n",
    "pixels = 50\n",
    "driver.maximize_window()\n",
    "time.sleep(2)\n",
    "while True:\n",
    "    pixels += 2500\n",
    "    scroll_down(driver,pixels)\n",
    "    time.sleep(1)\n",
    "    xpath_objects = r'//*[@id=\"QA0Szd\"]/div/div/div[1]/div[2]/div/div[1]/div/div/div[2]/div[1]/div[3]/div/a'\n",
    "    xpath_objects = xpath_objects[:-8] + str(odd) + xpath_objects[-8+1:]\n",
    "    try:\n",
    "        wait.until(EC.presence_of_element_located((By.XPATH, xpath_objects)))\n",
    "        driver.find_element(By.XPATH, xpath_objects).click()\n",
    "    except:\n",
    "        pixels += 2500\n",
    "        scroll_down(driver,pixels)\n",
    "        try:\n",
    "            wait.until(EC.presence_of_element_located((By.XPATH, xpath_objects)))\n",
    "            driver.find_element(By.XPATH, xpath_objects).click()\n",
    "        except:\n",
    "            break\n",
    "    scroll_down(driver)\n",
    "    try:\n",
    "        wait.until(EC.presence_of_element_located((By.XPATH,r'//*[@id=\"QA0Szd\"]/div/div/div[1]/div[3]/div/div[1]/div/div/div[2]/div[7]/div[3]/button/div[1]/div[2]/div[1]')))\n",
    "        address.append(driver.find_element(By.XPATH,r'//*[@id=\"QA0Szd\"]/div/div/div[1]/div[3]/div/div[1]/div/div/div[2]/div[7]/div[3]/button/div[1]/div[2]/div[1]').text)\n",
    "    except:\n",
    "        address.append(\"N/A\")\n",
    "    try:\n",
    "        wait.until(EC.presence_of_element_located((By.CLASS_NAME,\"aSftqf\")))\n",
    "        status.append(driver.find_element(By.CLASS_NAME,\"aSftqf\").text)\n",
    "    except:\n",
    "        status.append(\"Open\")\n",
    "    urls.append(driver.current_url)\n",
    "    time.sleep(1)\n",
    "    #driver.execute_script(\"window.history.go(-1)\")\n",
    "    wait.until(EC.presence_of_element_located((By.XPATH,r'//*[@id=\"QA0Szd\"]/div/div/div[1]/div[3]/div/div[1]/div/div/div[1]/div/div/div[3]/span/button')))\n",
    "    driver.find_element(By.XPATH,r'//*[@id=\"QA0Szd\"]/div/div/div[1]/div[3]/div/div[1]/div/div/div[1]/div/div/div[3]/span/button').click()\n",
    "    count += 1\n",
    "    odd += 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "8e8b3d5c",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\Users\\Fatima Azfar\\anaconda3\\Lib\\site-packages\\pandas\\core\\arrays\\masked.py:60: UserWarning: Pandas requires version '1.3.6' or newer of 'bottleneck' (version '1.3.5' currently installed).\n",
      "  from pandas.core import (\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Name</th>\n",
       "      <th>Status</th>\n",
       "      <th>Latitude</th>\n",
       "      <th>Longitude</th>\n",
       "      <th>Address</th>\n",
       "      <th>URL</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [Name, Status, Latitude, Longitude, Address, URL]\n",
       "Index: []"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "df = pd.DataFrame()\n",
    "lats = []\n",
    "longs = []\n",
    "names = []\n",
    "for url in urls:\n",
    "    # Extract the substring after the \"@\" character\n",
    "    coordinates = url.split(\"@\")[1].split(\",\")\n",
    "    # Extract the latitude and longitude\n",
    "    latitude = float(coordinates[0])\n",
    "    longitude = float(coordinates[1].split(\"z\")[0])\n",
    "    lats.append(latitude)\n",
    "    longs.append(longitude)\n",
    "    # to extract the names\n",
    "    # Extract the substring after \"place/\" and before \"/\"\n",
    "    name = url.split(\"place/\")[1].split(\"/\")[0]\n",
    "    # Replace \"+\" with \" \"\n",
    "    names.append(name.replace(\"+\", \" \"))\n",
    "\n",
    "df[\"Name\"] = names\n",
    "df[\"Status\"] = status\n",
    "df[\"Latitude\"] = lats\n",
    "df[\"Longitude\"] = longs\n",
    "df[\"Address\"] = address\n",
    "df[\"URL\"] = urls\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "66fdf0ef",
   "metadata": {},
   "outputs": [],
   "source": [
    "path = 'C:/Users/Nadeem/Desktop/PFA/Google_Maps_Data' \n",
    "path = path + \"/\" + search_input + \".csv\"\n",
    "df.to_csv(path)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}