WebScraper(from InShorts).ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "49e6ce64",
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "from bs4 import BeautifulSoup\n",
    "import pandas as pd\n",
    "from selenium import webdriver\n",
    "from selenium.webdriver.common.keys import Keys\n",
    "from selenium.webdriver.common.by import By\n",
    "from selenium.webdriver.support.ui import WebDriverWait\n",
    "from selenium.webdriver.support import expected_conditions as EC\n",
    "from selenium.webdriver.firefox.options import Options\n",
    "import time\n",
    "import json\n",
    "from webdriver_manager.chrome import ChromeDriverManager\n",
    "from tqdm.notebook import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "ab74dd1b",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "7e7a3d7e",
   "metadata": {},
   "outputs": [],
   "source": [
    "base_url = \"https://www.inshorts.com/hi/read/\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "827ed57b",
   "metadata": {},
   "outputs": [],
   "source": [
    "def find_content(category):\n",
    "    url = base_url + category\n",
    "    fire_opts = webdriver.ChromeOptions()\n",
    "    headers = {'User-Agent': 'Mozilla/5.0'}\n",
    "    fire_opts.add_argument(\"user-agent=Mozilla/5.0\")\n",
    "    driver = webdriver.Chrome(ChromeDriverManager().install(), options=fire_opts)\n",
    "    driver.get(url)\n",
    "    times = 100\n",
    "    for i in tqdm(range(times)):\n",
    "        load_more = WebDriverWait(driver, 10).until(EC.presence_of_element_located((By.CLASS_NAME, 'QMXJlc3R5MMJjDGSV4Jd')))\n",
    "        load_more.click()\n",
    "    divs = WebDriverWait(driver, 10).until(EC.presence_of_element_located((By.TAG_NAME, 'div')))\n",
    "    content = divs.find_elements_by_xpath('./child::*')[2]\n",
    "    cards = content.find_elements_by_xpath('./child::*')\n",
    "    data = []\n",
    "    for card in tqdm(cards):\n",
    "        try:\n",
    "            title = card.find_element_by_class_name('S2DdZEgzkqC9bYeTJUGw').text\n",
    "            text = card.find_element_by_class_name('Hxtmf9GvkV8Ti6V0GUSn').text\n",
    "            obj = {}\n",
    "            obj['title'] = title\n",
    "            obj['content'] = text\n",
    "            obj['label'] = category\n",
    "            data.append(obj)\n",
    "        except:\n",
    "            continue\n",
    "    df = pd.DataFrame(data)\n",
    "    driver.quit()\n",
    "    return df\n",
    "        \n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3e8fd3a4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3cc9d66852a74e7e8a9a9b217f5b53a6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/100 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "df = find_content(\"world\")\n",
    "df.to_csv('world.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e37a9454",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}