[Py] 다양한 블로그 수집하기(공개용).ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import libraries\n",
    "import os\n",
    "import re\n",
    "import sys\n",
    "import math\n",
    "import pandas as pd\n",
    "from urllib.request import urlopen, Request\n",
    "from urllib.parse import quote_plus\n",
    "from fake_useragent import UserAgent\n",
    "from bs4 import BeautifulSoup\n",
    "from datetime import datetime\n",
    "from datetime import timedelta\n",
    "from random import randint\n",
    "from time import sleep\n",
    "from unicodedata import normalize\n",
    "\n",
    "# set user agent\n",
    "#ua = str(UserAgent().random)\n",
    "ua = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36'\n",
    "\n",
    "# get page numbers\n",
    "def getPageNums(sDate):\n",
    "    main = \"https://search.naver.com/search.naver?where=post\"\n",
    "    sub0 = \"&sm=tab_pge\"\n",
    "    sub1 = \"&query=\" + quote_plus(keywd)\n",
    "    sub2 = \"&st=date\"\n",
    "    sub3 = \"&date_option=8\"\n",
    "    sub4 = \"&date_from=\" + sDate\n",
    "    sub5 = \"&date_to=\" + sDate\n",
    "    sub6 = \"&dup_remove=1\"\n",
    "    sub7 = \"&srchby=all\"\n",
    "    sub8 = \"&ie=utf8\"\n",
    "    sub9 = \"&start=1\"\n",
    "    \n",
    "    URL = main + sub0 + sub1 + sub2 + sub3 + sub4 + sub5 + sub6 + sub7 + sub8 + sub9\n",
    "    #print(URL)\n",
    "    \n",
    "    resp = Request(URL, data=None, headers={'User-Agent':ua})\n",
    "    html = urlopen(resp)\n",
    "    soup = BeautifulSoup(html, 'html.parser')\n",
    "    \n",
    "    # check page number\n",
    "    try:\n",
    "        count = soup.find('span',{'class':'title_num'}).get_text().split('/')[1].strip().replace(\"건\",\"\").replace(\",\",\"\")\n",
    "        count = int(count)\n",
    "    except:\n",
    "        count = 0\n",
    "        pass\n",
    "        \n",
    "    print(\"The number of blogs for\", sDate, \":\", count)\n",
    "    \n",
    "    return(count)\n",
    "\n",
    "\n",
    "# get blog links and summaries\n",
    "def getBlogLinks(sDate, count): \n",
    "    sumLinks = pd.DataFrame()\n",
    "    maxPages = math.ceil(count/10)\n",
    "        \n",
    "    for i in range(maxPages):\n",
    "        page = i*10 + 1\n",
    "        #print(page)\n",
    "        \n",
    "        main = \"https://search.naver.com/search.naver?where=post\"\n",
    "        sub0 = \"&sm=tab_pge\"\n",
    "        sub1 = \"&query=\" + quote_plus(keywd)\n",
    "        sub2 = \"&st=date\"\n",
    "        sub3 = \"&date_option=8\"\n",
    "        sub4 = \"&date_from=\" + sDate\n",
    "        sub5 = \"&date_to=\" + sDate\n",
    "        sub6 = \"&dup_remove=1\"\n",
    "        sub7 = \"&srchby=all\"\n",
    "        sub8 = \"&ie=utf8\"\n",
    "        sub9 = \"&start=\" + str(page)\n",
    "        \n",
    "        URL = main + sub0 + sub1 + sub2 + sub3 + sub4 + sub5 + sub6 + sub7 + sub8 + sub9\n",
    "        #print(URL)\n",
    "        \n",
    "        resp = Request(URL, data=None, headers={'User-Agent':ua})\n",
    "        html = urlopen(resp)\n",
    "        soup = BeautifulSoup(html, 'html.parser')\n",
    "        \n",
    "        blogs = soup.findAll(\"li\",{\"class\":\"sh_blog_top\"})\n",
    "        \n",
    "        title = []\n",
    "        pdate = []\n",
    "        link = []\n",
    "        text = []\n",
    "        #bloger = []\n",
    "        \n",
    "        for blog in blogs:\n",
    "            try:\n",
    "                title.append(blog.find('dt').find('a').get_text())\n",
    "                pdate.append(blog.find('dd',{'class':'txt_inline'}).get_text().split(\" \")[0])\n",
    "                link.append(blog.find('dt').find('a').attrs['href'].replace(\"?Redirect=Log&logNo=\",\"/\"))\n",
    "                text.append(blog.find('dd',{'class':'sh_blog_passage'}).get_text())\n",
    "                #bloger.append(blog.find('a',{'class':'txt84'}).get_text())\n",
    "            except:\n",
    "                pass\n",
    "        \n",
    "        df = pd.DataFrame({\"keyword\":keywd, \"title\":title, \"postdate\":pdate, \"link\":link, \"summary\":text})\n",
    "        sumLinks = sumLinks.append(df)\n",
    "    \n",
    "    return sumLinks\n",
    "\n",
    "\n",
    "# input query and dates\n",
    "keywd = str(input(\"검색어:\"))\n",
    "strDate = str(input(\"검색 시작일(YYYYMMDD):\"))\n",
    "endDate = str(input(\"검색 종료일(YYYYMMDD):\"))\n",
    "print(\"-----------------------------------\")\n",
    "\n",
    "# calculate timedelta\n",
    "dateGap = datetime.strptime(endDate, '%Y%m%d') - datetime.strptime(strDate, '%Y%m%d')\n",
    "dateGap.days + 1\n",
    "\n",
    "# collect blogs information\n",
    "result = pd.DataFrame()\n",
    "\n",
    "for i in range(dateGap.days+1):\n",
    "    sDate = datetime.strptime(strDate, '%Y%m%d') + timedelta(days=i)\n",
    "    sDate = sDate.strftime(\"%Y%m%d\")\n",
    "    \n",
    "    # get blog page numbers\n",
    "    count = getPageNums(sDate)\n",
    "    \n",
    "    # get blog links and summary\n",
    "    summary = getBlogLinks(sDate, count)\n",
    "    \n",
    "    # rearragne columns\n",
    "    #summary = summary[[\"keyword\", \"title\", \"postdate\", \"link\", \"summary\"]]\n",
    "    \n",
    "    # add by rows\n",
    "    result = result.append(summary)\n",
    "    \n",
    "    \n",
    "# check blog numbers before removing duplicated\n",
    "print(\"\\n\")\n",
    "print(\"The total number of blogs :\", len(result))\n",
    "\n",
    "# remove duplicated\n",
    "result = result.reset_index(drop=True)\n",
    "result = result.drop_duplicates(['link'], keep='last')\n",
    "print(\"After removing duplicated :\", len(result))\n",
    "print(\"\\n\")\n",
    "\n",
    "\n",
    "# get full text from various blogs\n",
    "def getTextFromNaver(url):\n",
    "    try:\n",
    "        resp = Request(url, data=None, headers={'User-Agent':ua})\n",
    "        html = urlopen(resp)\n",
    "        soup = BeautifulSoup(html, 'html.parser')\n",
    "        \n",
    "        try:\n",
    "            url1 = soup.find('frame',{'id':'screenFrame'}).attrs['src'].split(\"?\")[0]\n",
    "            url1\n",
    "        except:\n",
    "            url1 = url\n",
    "        \n",
    "        postNo = url1.split(\"/\")[-1]\n",
    "        \n",
    "        html = urlopen(url1)\n",
    "        soup = BeautifulSoup(html, 'html.parser')\n",
    "        soup.find(\"frame\",{\"id\":\"mainFrame\"}).attrs[\"src\"]\n",
    "        url2 = \"http://blog.naver.com\" + soup.find(\"frame\",{\"id\":\"mainFrame\"}).attrs[\"src\"]\n",
    "        \n",
    "        html = urlopen(url2)\n",
    "        soup = BeautifulSoup(html, 'html.parser')\n",
    "        \n",
    "        text = soup.find('div',{'id':'post-view' + postNo}).get_text()\n",
    "        full = re.compile('[^ㄱ-ㅣ가-힣]+').sub(' ', text).strip()\n",
    "        return(full)\n",
    "    except:\n",
    "        pass\n",
    "\n",
    "\n",
    "def getTextFromEgloos(url):\n",
    "    try:\n",
    "        resp = Request(url, data=None, headers={'User-Agent':ua})\n",
    "        html = urlopen(resp)\n",
    "        soup = BeautifulSoup(html, 'html.parser')\n",
    "        \n",
    "        text = soup.find('div',{'class':'hentry'}).get_text()\n",
    "        full = re.compile('[^ㄱ-ㅣ가-힣]+').sub(' ', text).strip()\n",
    "        return(full)\n",
    "    except:\n",
    "        pass\n",
    "\n",
    "\n",
    "def getTextFromTistory(url):\n",
    "    try:\n",
    "        resp = Request(url, data=None, headers={'User-Agent':ua})\n",
    "        html = urlopen(resp)\n",
    "        soup = BeautifulSoup(html, 'html.parser')\n",
    "        \n",
    "        try:\n",
    "            text = soup.find('div',{'class':'article'}).get_text()\n",
    "        except:\n",
    "            text = soup.find('div',{'class':'area_view'}).get_text()\n",
    "        full = re.compile('[^ㄱ-ㅣ가-힣]+').sub(' ', text).strip()\n",
    "        return(full)\n",
    "    except:\n",
    "        pass\n",
    "\n",
    "\n",
    "def getTextFromDaum(url):\n",
    "    try:\n",
    "        resp = Request(url, data=None, headers={'User-Agent':ua})\n",
    "        html = urlopen(resp)\n",
    "        soup = BeautifulSoup(html, 'html.parser')\n",
    "        \n",
    "        url1 = 'http://blog.daum.net' + soup.find('frame',{'name':'BlogMain'}).attrs['src']\n",
    "        html = urlopen(url1)\n",
    "        soup = BeautifulSoup(html, 'html.parser')\n",
    "        \n",
    "        url2 = 'http://blog.daum.net' + soup.find('div',{'id':'cContentBody'}).find('iframe').attrs['src']\n",
    "        html = urlopen(url2)\n",
    "        soup = BeautifulSoup(html, 'html.parser')\n",
    "        \n",
    "        text = soup.find('div',{'id':'contentDiv'}).get_text()\n",
    "        full = re.compile(\"[^ㄱ-ㅣ가-힣]+\").sub(' ', text).strip()\n",
    "        return(full)\n",
    "    except:\n",
    "        pass\n",
    "\n",
    "\n",
    "# get full bloger titles\n",
    "def getBlogerFromTistory(url):\n",
    "    try:\n",
    "        resp = Request(url, data=None, headers={'User-Agent':ua})\n",
    "        html = urlopen(resp)\n",
    "        soup = BeautifulSoup(html, 'html.parser')\n",
    "        bloger = soup.find(\"meta\",{\"property\":\"og:site_name\"}).attrs[\"content\"]\n",
    "    except:\n",
    "        pass\n",
    "    \n",
    "    return(bloger)\n",
    "\n",
    "\n",
    "def getBlogerFromOthers(url):\n",
    "    try:\n",
    "        resp = Request(url, data=None, headers={'User-Agent':ua})\n",
    "        html = urlopen(resp)\n",
    "        soup = BeautifulSoup(html, 'html.parser')\n",
    "        bloger = soup.find(\"title\").get_text().split(\":\")[0].strip()\n",
    "    except:\n",
    "        pass\n",
    "    \n",
    "    return(bloger)\n",
    "\n",
    "\n",
    "# create user defined function for checking the domain of a blog\n",
    "def getBlogDomain(url):\n",
    "    try:\n",
    "        resp = Request(url, data=None, headers={'User-Agent':ua})\n",
    "        html = urlopen(resp)\n",
    "        soup = BeautifulSoup(html, 'html.parser')\n",
    "    except:\n",
    "        pass\n",
    "    \n",
    "    domain = ''\n",
    "    \n",
    "    # for egloos or tistory    \n",
    "    try:\n",
    "        domain = soup.find('link',{'rel':'shortcut icon'}).attrs['href']\n",
    "    except:\n",
    "        pass\n",
    "    \n",
    "    # for daum\n",
    "    try:\n",
    "        domain = soup.find('link',{'rel':'icon'}).attrs['href']\n",
    "    except:\n",
    "        pass\n",
    "    \n",
    "    # for naver or blog.me\n",
    "    try:\n",
    "        domain = soup.find('link',{'rel':'wlwmanifest'}).attrs['href']\n",
    "    except:\n",
    "        pass\n",
    "    \n",
    "    return(domain)\n",
    "\n",
    "\n",
    "# set urls\n",
    "urls = result[\"link\"]\n",
    "\n",
    "# set default dataframes\n",
    "df = pd.DataFrame()\n",
    "fulltext = pd.DataFrame()\n",
    "i = 0\n",
    "\n",
    "# fetch full text from blogs\n",
    "for url in urls:\n",
    "    i += 1\n",
    "    print(i, url)\n",
    "    \n",
    "    # set time sleep randomly\n",
    "    sleep(randint(1,3))\n",
    "\n",
    "    domain = getBlogDomain(url)\n",
    "    \n",
    "    if 'naver' in domain:\n",
    "        print(\"naver blog!\")\n",
    "        full = getTextFromNaver(url)\n",
    "        bloger = getBlogerFromOthers(url)\n",
    "        \n",
    "    elif 'blog.me' in domain:\n",
    "        print(\"naver blog!\")\n",
    "        full = getTextFromNaver(url)\n",
    "        bloger = getBlogerFromOthers(url)\n",
    "        \n",
    "    elif 'tistory' in domain:\n",
    "        print(\"tistory blog!\")\n",
    "        full = getTextFromTistory(url)\n",
    "        bloger = getBlogerFromTistory(url)\n",
    "        \n",
    "    elif 'egloos' in domain:\n",
    "        print(\"egloos blog!\")\n",
    "        full = getTextFromEgloos(url)\n",
    "        bloger = getBlogerFromOthers(url)\n",
    "        \n",
    "    elif 'daum' in domain:\n",
    "        print(\"daum blog!\")\n",
    "        full = getTextFromDaum(url)\n",
    "        bloger = getBlogerFromOthers(url)\n",
    "    \n",
    "    else:\n",
    "        print(\"Next!\")\n",
    "        full = ''\n",
    "    \n",
    "    df = pd.DataFrame({\"link\":[url], \"fulltext\":[full], \"bloger\":[bloger]})\n",
    "    fulltext = fulltext.append(df)\n",
    "\n",
    "print(\"It's done!\")\n",
    "\n",
    "\n",
    "# create the final dataframe \n",
    "final = pd.merge(result, fulltext, how=\"left\", on=\"link\", left_index=True)\n",
    "final = final.reset_index(drop=True)\n",
    "final = final[[\"keyword\", \"title\", \"postdate\", \"link\", \"summary\", \"bloger\", \"fulltext\"]]\n",
    "print(\"The number of blogs:\", len(final))\n",
    "\n",
    "# download it as an Excel file\n",
    "wdir = \"./Data/\"\n",
    "if not os.path.exists(wdir):\n",
    "    os.makedirs(wdir)\n",
    "\n",
    "wtime = datetime.today().strftime(\"%Y%m%d\")\n",
    "fname = wdir + \"Blog_\" + normalize(\"NFC\", keywd) + \"_\" + strDate + \"_\" + endDate + \".xlsx\"\n",
    "final.to_excel(fname, index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}