Basic Data Scraping .ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Taking all libraries which are useful to do [web scraping,create dataframe] \n",
    "import pandas as pd # used to create DataFrame\n",
    "import numpy as np # used to create array of a single column\n",
    "\n",
    "import re # used to Regular Expressions\n",
    "import requests # Used to get the page\n",
    "from bs4 import BeautifulSoup # used for creating a html doc in to text\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "url='https://www.bankbazaar.com/reviews.html'\n",
    "\n",
    "requests.get(url) # requesting to get text from all pages , find Number of Pages\n",
    "\n",
    "page=requests.get(url)\n",
    "\n",
    "page.text # text of All Pages \n",
    "\n",
    "soup= BeautifulSoup(page.text, 'html.parser')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "#read review text\n",
    "\n",
    "review_text=[]\n",
    "review_text_elem= soup.find_all(class_='text_here review-desc-more')\n",
    "\n",
    "for item in review_text_elem:\n",
    "    review_text.append(item.text)\n",
    "    \n",
    "#review_text    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"User_Name = []\\nname = soup.find_all(class_ = 'js-author-name')\\n\\nfor j in range(len(name)): \\n        if name[j]:\\n            n = name[j].text\\n        else:\\n            n=np.nan  \\n        User_Name.append(n)\\n        \\n        \\nUser_Name   \""
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#read User Name \n",
    "\n",
    "user_name = []\n",
    "user_name_elem= soup.find_all(class_='js-author-name')\n",
    "\n",
    "for item in user_name_elem:\n",
    "    user_name.append(item.text)\n",
    "    \n",
    "#user_name    \n",
    "\n",
    "\n",
    "'''User_Name = []\n",
    "name = soup.find_all(class_ = 'js-author-name')\n",
    "\n",
    "for j in range(len(name)): \n",
    "        if name[j]:\n",
    "            n = name[j].text\n",
    "        else:\n",
    "            n=np.nan  \n",
    "        User_Name.append(n)\n",
    "        \n",
    "        \n",
    "User_Name   '''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "#read Bank Name\n",
    "\n",
    "bank_name=[]\n",
    "bank_name_elem =soup.find_all('div', class_='review-bank-title')\n",
    "\n",
    "for item in bank_name_elem:\n",
    "    bank_name.append(item.find('img').get('alt'))\n",
    "    \n",
    "#bank_name"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#read Bank type\n",
    "\n",
    "bank_type=[]\n",
    "bank_type_elem = soup.find_all('div', class_='review-bank-title')\n",
    "\n",
    "for item in bank_type_elem:\n",
    "    bank_type.append(item.text)\n",
    "         \n",
    "#for item in  bank_type_elem:\n",
    "    #bank_type.append(re.findall(r'[LTCSPO]\\w+\\s\\w+', x)[0])\n",
    "    \n",
    "#bank_type"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#read Bank Rating\n",
    "\n",
    "Bank_Rating = []\n",
    "rating=soup.find_all('div' ,class_='rating-section review-user-score')  \n",
    "\n",
    "for item in rating:\n",
    "    Bank_Rating.append(item.text)\n",
    "\n",
    "#Bank_Rating"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#read bank caption \n",
    "\n",
    "bank_caption=[]\n",
    "bank_caption_elem = soup.find_all('a',{'data-actionloc':'reviews:ReviewCards'})\n",
    "\n",
    "\n",
    "for item in bank_caption_elem:\n",
    "        bank_caption.append(item.text)\n",
    "        \n",
    "#bank_caption"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#read city \n",
    "\n",
    "Location= []\n",
    "city=soup.find_all('div', class_='reviewer-profile')\n",
    "\n",
    "for item in city:\n",
    "        Location.append(item.text)\n",
    "\n",
    "#Location"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "Rating_Comment=[]\n",
    "word = soup.find_all('div',class_='rating-section review-user-score')\n",
    "\n",
    "for k in word:\n",
    "        x = k.text\n",
    "        if len(re.findall(r'\\b[GBPS]\\w+',x)) == 1:\n",
    "            Rating_Comment.append(re.findall(r'\\b[GBPS]\\w+',x)[0])\n",
    "        else:\n",
    "            Rating_Comment.append(np.nan) \n",
    "            \n",
    "#Rating_Comment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "final_array=[]\n",
    "\n",
    "for text,name,bank,model,rating,caption,city,comment in zip(review_text,user_name,bank_name,bank_type,Bank_Rating,bank_caption,Location,Rating_Comment):\n",
    "    final_array.append({'Review_text':text,'User_Name':name,'Bank_Name':bank,'Bank_type':model,'Bank_Rating':rating,'Bank_Caption':caption, 'Location':city, 'Comment':comment})\n",
    "    \n",
    "    \n",
    "df=pd.DataFrame(final_array)\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}