YNWA.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 1. Libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "### Computational Libraries\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "### HTML Servlet related\n",
    "import requests\n",
    "import markdown2\n",
    "from xml.dom.minidom import parseString\n",
    "\n",
    "\n",
    "### News Summary Library\n",
    "import newspaper\n",
    "\n",
    "### NLTK\n",
    "from nltk.tokenize import sent_tokenize,word_tokenize\n",
    "from nltk.corpus import stopwords\n",
    "\n",
    "### String Manipulation\n",
    "from collections import defaultdict\n",
    "from string import punctuation\n",
    "from heapq import nlargest\n",
    "\n",
    "from IPython.display import Markdown, display\n",
    "def printmd(string):\n",
    "    display(Markdown(string))\n",
    "\n",
    "#### From Markdown to HTML\n",
    "\n",
    "from webbrowser import open_new_tab\n",
    "\n",
    "\n",
    "### Local HTML to PDF\n",
    "\n",
    "import pdfcrowd\n",
    "import sys\n",
    "\n",
    "## Date for convention\n",
    "\n",
    "from datetime import date\n",
    "import calendar"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 2. Defined Classes"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2.1 Class 'FrequencySummarizer' for Summarizing a Text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "class FrequencySummarizer:\n",
    "  def __init__(self, min_cut=0.5, max_cut=0.95):\n",
    "    \"\"\" Words that have a frequency term lower than min_cut or higer than max_cut will be ignored. \"\"\"\n",
    "    self._min_cut = min_cut\n",
    "    self._max_cut = max_cut \n",
    "    self._stopwords = set(stopwords.words('english') + list(punctuation))\n",
    "\n",
    "  def _compute_frequencies(self, word_sent):\n",
    "    \"\"\" \n",
    "      Input: a list of sentences already tokenized.\n",
    "      Output:  a dictionary where freq[w] is the frequency of w.\n",
    "    \"\"\"\n",
    "    freq = defaultdict(int)\n",
    "    for s in word_sent:\n",
    "      for word in s:\n",
    "        if word not in self._stopwords:\n",
    "          freq[word] += 1\n",
    "    # frequencies normalization and fitering\n",
    "    m = float(max(freq.values()))\n",
    "    for w in list(freq):\n",
    "      freq[w] = freq[w]/m\n",
    "      if freq[w] >= self._max_cut or freq[w] <= self._min_cut:\n",
    "        del freq[w]\n",
    "    return freq\n",
    "\n",
    "  def summarize(self, text, n):\n",
    "    \"\"\"\"\n",
    "    Return a list of n sentences which represent the summary of text.\n",
    "    \"\"\"\n",
    "    sents = sent_tokenize(text)\n",
    "    assert n <= len(sents)\n",
    "    word_sent = [word_tokenize(s.lower()) for s in sents]\n",
    "    self._freq = self._compute_frequencies(word_sent)\n",
    "    ranking = defaultdict(int)\n",
    "    for i,sent in enumerate(word_sent):\n",
    "      for w in sent:\n",
    "        if w in self._freq:\n",
    "          ranking[i] += self._freq[w]\n",
    "    sents_idx = self._rank(ranking, n)    \n",
    "    return [sents[j] for j in sents_idx]\n",
    "\n",
    "  def _rank(self, ranking, n):\n",
    "    \"\"\" return the first n sentences with highest ranking \"\"\"\n",
    "    return nlargest(n, ranking, key=ranking.get)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 3. Web Search\n",
    "\n",
    "*  The Data will be collected from the following News Sources only via Google-News:\n",
    "    - goal.com  [HomePage](https://www.goal.com/en)\n",
    "    - skysports.com  [HomePage](https://www.skysports.com/football)\n",
    "    - espn.in  [HomePage](http://www.espn.in/football/?src=com)\n",
    "    - theguardian.com  [HomePage](https://www.theguardian.com/football)\n",
    "\n",
    "\n",
    "*  The News window is set to past 7 days.\n",
    "\n",
    "- This is not an exhaustive list. You can customize this list by adding/removing websites. You can also filter particluar websited that you don't want in your search list."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- News wrt **Liverpool** and **English Premier League** will be fetched.\n",
    "- News wrt **Liverpool** and **Champions League** will also be fetched separately.\n",
    "- Then, I will combine both the News sources to only look for unique News."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 4. News Compilation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "urls = 0    ####Initialisation of the urls variable\n",
    "\n",
    "sites = ['goal.com', 'skysports.com', 'espn.in', 'theguardian.com/football']    #### List to store the Websites"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 4.1 Function to fetch the URL and Title."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def URL_Decorator( func):\n",
    "    \"\"\"\n",
    "    *  The function searches for the news as entered in the term 'search_string'\n",
    "    *  The function also returns the title of the News for the fetched URL - retured by 'return_title' \"\"\"\n",
    "    def wrapper(string, days, urls):\n",
    "        \"\"\" A Function Wrapper \"\"\"\n",
    "        term, count = func(string,days, urls)\n",
    "        results = []\n",
    "        results_title = []\n",
    "        results_url = []\n",
    "        obj = parseString( requests.get('https://news.google.com/rss/search?q={term}&hl=en-IN&gl=IN&ceid=IN:en'.format(term=term)).text )\n",
    "        items = obj.getElementsByTagName('item') # Get each item\n",
    "        for item in items[:count]:\n",
    "            t,l = '', ''\n",
    "            for node in item.childNodes:\n",
    "                if node.nodeName == 'title':\n",
    "                    t = node.childNodes[0].data\n",
    "                elif node.nodeName == 'link':\n",
    "                    l = node.childNodes[0].data\n",
    "            results.append( (t,l) )\n",
    "            results_title.append(t)\n",
    "            results_url.append(l)\n",
    "        return results_title, results_url\n",
    "        \n",
    "    return wrapper\n",
    "    \n",
    "\n",
    "@URL_Decorator\n",
    "def search_strings(search_string, days, url):\n",
    "    \"\"\"\n",
    "    A Function to write automated script as an input for 'get_google_new_results_test' function.\n",
    "    \n",
    "    search_string = Value that we need to search\n",
    "    days = No of Days in the past\n",
    "    urls = No of URL's to fetch\n",
    "    \"\"\"\n",
    "    urls = int(url)\n",
    "    for site in sites:\n",
    "        if site == sites[0]:\n",
    "            string = str('{search_string} {site}').format(search_string=search_string, site=site)\n",
    "        elif site != sites[-1]:\n",
    "            string = str(string) + str(' OR ') + str('{site}').format(site= site)\n",
    "        else:\n",
    "            string = string + str(' when:{day}d').format(day = days)\n",
    "    return string, urls"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 7. Scraping Data into Markdown File"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Defining the Class Object ##\n",
    "fs = FrequencySummarizer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "### Adding Naming Convension - based on Date\n",
    "today_date = date.today()\n",
    "today_date\n",
    "\n",
    "day_file_name = str(today_date.day)+'/'+today_date.strftime('%m')+'/'+str(today_date.year)   ## In case Publish Date is absent\n",
    "file2 = str(today_date.day)+'_'+today_date.strftime('%m')+'_'+str(today_date.year)\n",
    "file_name = file2 + '_Report.pdf'\n",
    "\n",
    "final_file_name = str(file_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%capture output\n",
    "\n",
    "### Setting search term\n",
    "search_term = ['Liverpool AND English Premier League', 'Liverpool AND Champions League']\n",
    "headings = ['English Premier League', 'Champions League']    #### Setting Headings\n",
    "\n",
    "total_urls = []\n",
    "\n",
    "for term, heading in zip(search_term,headings):\n",
    "    \n",
    "    title, urls = search_strings(('{0}').format(term), 20, 3)\n",
    "    urls =  list(set(urls)^set(list(set(urls).intersection(total_urls))))  ## Only Unique URL's with respect to that fruit\n",
    "    total_urls.extend(urls)\n",
    "    \n",
    "    print('<center><h3>' + str('{heading}  Overview').format(heading = str(heading))+'</h3></center>')\n",
    "\n",
    "    for url in urls:\n",
    "        try:\n",
    "            common_stop_words = stopwords.words('english')  # Variable to store common English Stop words\n",
    "            article = newspaper.Article(url)\n",
    "            article.download()\n",
    "            article.parse()\n",
    "            article.nlp()\n",
    "\n",
    "            title_url = article.title   # News Title\n",
    "            text_text = article.text    # News Text\n",
    "            title_genre =  article.keywords[0:5]\n",
    "            #words = [ss('english').stem(word) for word in title_genre if word not in common_stop_words]\n",
    "            words = [word for word in title_genre if word not in common_stop_words]\n",
    "\n",
    "            summary = sent_tokenize(article.summary)\n",
    "                \n",
    "        except:\n",
    "            continue\n",
    "\n",
    "        ####################### Publish Date ####################################\n",
    "        \n",
    "        try:\n",
    "            if(int(article.publish_date.month)>10):\n",
    "\n",
    "                publish_date = str(article.publish_date.day)+'/' + str(article.publish_date.month)+'/' + str(article.publish_date.year)\n",
    "                publish_date = str(publish_date)\n",
    "            else:\n",
    "                publish_date = str(article.publish_date.day)+'/0' + str(article.publish_date.month)+'/' + str(article.publish_date.year)\n",
    "                publish_date = str(publish_date)\n",
    "        except AttributeError:\n",
    "            publish_date = str(day_file_name)\n",
    "        ####################### Publish Date Ends ###############################\n",
    "\n",
    "\n",
    "        ######################### Title #########################################\n",
    "        print('<table>')\n",
    "\n",
    "        print('<tr>')\n",
    "        print('<th>')\n",
    "\n",
    "        \n",
    "        print('<a style= \"color:#333399;font-family: \"Times New Roman\", Times, serif;font-size:110%;\" href = \"{url} \">'.format(url = url)+ str(title_url) +'</a>')\n",
    "        \n",
    "\n",
    "        print('</th>')\n",
    "        print('</tr>')\n",
    "        ######################### Share Widgets begins ######################################\n",
    "        \n",
    "       \n",
    "        print('<a href=\"mailto:?subject={subject_title}&amp;body= {article_url} \" title=\"Share by Email\"> <img src=\"https://github.com/S-B-Iqbal/Test/raw/master/Images/outlook.png\" alt=\"Outlook\" style=\"float:right;height:20px;width:20px\" hspace = \"20\"> </a>'.format(subject_title = title_url, article_url = str(url)))\n",
    "        \n",
    "        print('<a href=\"https://wa.me/send?text ={share_url}\" ><img src=\"https://github.com/S-B-Iqbal/Test/raw/master/Images/whatsapp.png\" alt=\"Whatsapp\" style=\"float:right; height:20px;width:20px\" hspace = \"40\"></a>'.format(share_url = str(url)))\n",
    "        \n",
    "        ######################### Share Widgets Ends ###################################### \n",
    "        \n",
    "        \n",
    "        print('</table>')              \n",
    "        \n",
    "        #print('    ')\n",
    "        #print('    ')\n",
    "        print('<article><strong>'+format(publish_date) + '</article>')\n",
    "        \n",
    "        ######################### Title Ends ######################################\n",
    "        \n",
    "        #print('\\n {0}'.format(title_url))\n",
    "        \n",
    "        \n",
    "        try:\n",
    "            print(' ')\n",
    "            print('<article>' + '<b style= \"color:grey\">Keywords:</b> <i>' + ' | '.join(words) + '</i> </article>')\n",
    "            print('    ')\n",
    "            print('------------------------------------------------------------------')\n",
    "            \"\"\" Printing the Summary as bullet points.\"\"\"\n",
    "            print('<ul style=\"color:#000000; font-family:calibri\">')\n",
    "            for s in summary:\n",
    "                print('*  '+ str(s))\n",
    "            print('    ')\n",
    "            \n",
    "            print('    ')\n",
    "            print('</ul>')\n",
    "            print('------------------------------------------------------------------')\n",
    "            print('    ')\n",
    "            \n",
    "            print('------------------------------------------------------------------')\n",
    "            print('    ')\n",
    "            print('    ')\n",
    "            \n",
    "        except AssertionError:\n",
    "            continue\n",
    "            \n",
    "del total_urls"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [],
   "source": [
    "### Getting Text of the Output\n",
    "Summary = output.stdout"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/markdown": [
       "<center><h3>English Premier League  Overview</h3></center>\n",
       "<table>\n",
       "<tr>\n",
       "<th>\n",
       "<a style= \"color:#333399;font-family: \"Times New Roman\", Times, serif;font-size:110%;\" href = \"https://www.goal.com/en/news/sturridge-offered-to-several-a-league-clubs/adodhuf0hnkg1a658ll2gj3bb \">Liverpool striker Daniel Sturridge reportedly offered to 'several' A-League clubs</a>\n",
       "</th>\n",
       "</tr>\n",
       "<a href=\"mailto:?subject=Liverpool striker Daniel Sturridge reportedly offered to 'several' A-League clubs&amp;body= https://www.goal.com/en/news/sturridge-offered-to-several-a-league-clubs/adodhuf0hnkg1a658ll2gj3bb \" title=\"Share by Email\"> <img src=\"https://github.com/S-B-Iqbal/Test/raw/master/Images/outlook.png\" alt=\"Outlook\" style=\"float:right;height:20px;width:20px\" hspace = \"20\"> </a>\n",
       "<a href=\"https://wa.me/send?text =https://www.goal.com/en/news/sturridge-offered-to-several-a-league-clubs/adodhuf0hnkg1a658ll2gj3bb\" ><img src=\"https://github.com/S-B-Iqbal/Test/raw/master/Images/whatsapp.png\" alt=\"Whatsapp\" style=\"float:right; height:20px;width:20px\" hspace = \"40\"></a>\n",
       "</table>\n",
       "<article><strong>25/04/2019</article>\n",
       " \n",
       "<article><b style= \"color:grey\">Keywords:</b> <i>liverpool | aleague | come | daniel | sturridge</i> </article>\n",
       "    \n",
       "------------------------------------------------------------------\n",
       "<ul style=\"color:#000000; font-family:calibri\">\n",
       "*  Sturridge offered to 'several' A-League clubsThe English striker could be heading Down UnderCurrent player Daniel Sturridge has emerged as a surprise marquee candidate for the A-League with Brisbane Roar shaping as his most likely destination.\n",
       "*  \"One name I will throw up, and I know this player has already been offered to several A-League clubs already, is Daniel Sturridge,\" Shaw said on the Fox Football Podcast .\n",
       "*  \"I wouldn’t be at all surprised if the phone call is made from Robbie Fowler to bring someone like Daniel Sturridge down as a marquee next season.\"\n",
       "*  While now at Liverpool, Sturridge began his career with City before moving to where he won the Premier League in 2010 and in 2012.\n",
       "*  Sturridge spent some of last season on loan with and is off contract with Liverpool at season's end.\n",
       "    \n",
       "    \n",
       "</ul>\n",
       "------------------------------------------------------------------\n",
       "    \n",
       "------------------------------------------------------------------\n",
       "    \n",
       "    \n",
       "<table>\n",
       "<tr>\n",
       "<th>\n",
       "<a style= \"color:#333399;font-family: \"Times New Roman\", Times, serif;font-size:110%;\" href = \"https://www.goal.com/en-in/news/premier-league-should-do-more-to-help-english-teams-in/1qcmwazazud6e1jui73hndvajm \">Liverpool News: 'Premier League should do more to help English clubs in Europe' - Klopp</a>\n",
       "</th>\n",
       "</tr>\n",
       "<a href=\"mailto:?subject=Liverpool News: 'Premier League should do more to help English clubs in Europe' - Klopp&amp;body= https://www.goal.com/en-in/news/premier-league-should-do-more-to-help-english-teams-in/1qcmwazazud6e1jui73hndvajm \" title=\"Share by Email\"> <img src=\"https://github.com/S-B-Iqbal/Test/raw/master/Images/outlook.png\" alt=\"Outlook\" style=\"float:right;height:20px;width:20px\" hspace = \"20\"> </a>\n",
       "<a href=\"https://wa.me/send?text =https://www.goal.com/en-in/news/premier-league-should-do-more-to-help-english-teams-in/1qcmwazazud6e1jui73hndvajm\" ><img src=\"https://github.com/S-B-Iqbal/Test/raw/master/Images/whatsapp.png\" alt=\"Whatsapp\" style=\"float:right; height:20px;width:20px\" hspace = \"40\"></a>\n",
       "</table>\n",
       "<article><strong>20/04/2019</article>\n",
       " \n",
       "<article><b style= \"color:grey\">Keywords:</b> <i>dont | liverpool | premier | klopp | europe</i> </article>\n",
       "    \n",
       "------------------------------------------------------------------\n",
       "<ul style=\"color:#000000; font-family:calibri\">\n",
       "*  Premier League should do more to help English teams in Europe - KloppThe Reds boss believes top-flight could do more to aid their clubs on the continent, as the Dutch authorities have done for Ajaxmanager Jurgen Klopp has called on the Premier League to help English teams in European competitions.\n",
       "*  In contrast, Liverpool must play vital games in the Premier League run in before their own semi-final with .\n",
       "*  “The Dutch league had a problem because for the last two match days, everyone plays at the same time,” Klopp told a press conference.\n",
       "*  No, I don’t think it would happen here or in , to be 100 per cent honest.\n",
       "*  Klopp said that the television deal for the Premier League made it difficult for the governing body to move the schedule as they have in the .\n",
       "    \n",
       "    \n",
       "</ul>\n",
       "------------------------------------------------------------------\n",
       "    \n",
       "------------------------------------------------------------------\n",
       "    \n",
       "    \n",
       "<table>\n",
       "<tr>\n",
       "<th>\n",
       "<a style= \"color:#333399;font-family: \"Times New Roman\", Times, serif;font-size:110%;\" href = \"https://www.goal.com/en-us/news/why-most-man-utd-fans-want-man-city-to-win-premier-league/181ycus3aw0gj1mhvgsa5fir7h \">Why do Man Utd fans want Man City to win Premier League instead of Liverpool?</a>\n",
       "</th>\n",
       "</tr>\n",
       "<a href=\"mailto:?subject=Why do Man Utd fans want Man City to win Premier League instead of Liverpool?&amp;body= https://www.goal.com/en-us/news/why-most-man-utd-fans-want-man-city-to-win-premier-league/181ycus3aw0gj1mhvgsa5fir7h \" title=\"Share by Email\"> <img src=\"https://github.com/S-B-Iqbal/Test/raw/master/Images/outlook.png\" alt=\"Outlook\" style=\"float:right;height:20px;width:20px\" hspace = \"20\"> </a>\n",
       "<a href=\"https://wa.me/send?text =https://www.goal.com/en-us/news/why-most-man-utd-fans-want-man-city-to-win-premier-league/181ycus3aw0gj1mhvgsa5fir7h\" ><img src=\"https://github.com/S-B-Iqbal/Test/raw/master/Images/whatsapp.png\" alt=\"Whatsapp\" style=\"float:right; height:20px;width:20px\" hspace = \"40\"></a>\n",
       "</table>\n",
       "<article><strong>24/04/2019</article>\n",
       " \n",
       "<article><b style= \"color:grey\">Keywords:</b> <i>united | liverpool | majority | win | utd</i> </article>\n",
       "    \n",
       "------------------------------------------------------------------\n",
       "<ul style=\"color:#000000; font-family:calibri\">\n",
       "*  Wind forward to 2019 and Liverpool are attempting to finally win a 19th league title 29 years and 13 United crowns on from their last.\n",
       "*  Article continues belowFor United fans, this might be the worst-case scenario.\n",
       "*  They have no hope of a title themselves, and it is City and Liverpool who are battling for the crown.\n",
       "*  “And whilst that might change in the future, at the moment the majority of United fans I’d say would consider Liverpool to be our biggest rivals.\n",
       "*  But, whatever the result at Old Trafford, don’t be too surprised if you see United fans siding with City in the remainder of the run-in.\n",
       "    \n",
       "    \n",
       "</ul>\n",
       "------------------------------------------------------------------\n",
       "    \n",
       "------------------------------------------------------------------\n",
       "    \n",
       "    \n",
       "<center><h3>Champions League  Overview</h3></center>\n",
       "<table>\n",
       "<tr>\n",
       "<th>\n",
       "<a style= \"color:#333399;font-family: \"Times New Roman\", Times, serif;font-size:110%;\" href = \"https://www.goal.com/en/news/vidal-stakes-claim-for-liverpool-spot-as-barca-turn-focus/1ele7763c56611gsv0lqgs77x8 \">Vidal stakes claim for Liverpool spot as Barca turn focus towards Champions League</a>\n",
       "</th>\n",
       "</tr>\n",
       "<a href=\"mailto:?subject=Vidal stakes claim for Liverpool spot as Barca turn focus towards Champions League&amp;body= https://www.goal.com/en/news/vidal-stakes-claim-for-liverpool-spot-as-barca-turn-focus/1ele7763c56611gsv0lqgs77x8 \" title=\"Share by Email\"> <img src=\"https://github.com/S-B-Iqbal/Test/raw/master/Images/outlook.png\" alt=\"Outlook\" style=\"float:right;height:20px;width:20px\" hspace = \"20\"> </a>\n",
       "<a href=\"https://wa.me/send?text =https://www.goal.com/en/news/vidal-stakes-claim-for-liverpool-spot-as-barca-turn-focus/1ele7763c56611gsv0lqgs77x8\" ><img src=\"https://github.com/S-B-Iqbal/Test/raw/master/Images/whatsapp.png\" alt=\"Whatsapp\" style=\"float:right; height:20px;width:20px\" hspace = \"40\"></a>\n",
       "</table>\n",
       "<article><strong>23/04/2019</article>\n",
       " \n",
       "<article><b style= \"color:grey\">Keywords:</b> <i>focus | best | stakes | liverpool | alaves</i> </article>\n",
       "    \n",
       "------------------------------------------------------------------\n",
       "<ul style=\"color:#000000; font-family:calibri\">\n",
       "*  Vidal stakes claim for Liverpool spot as Barca turn focus towards Champions LeagueThe Chilean could join the lineup as the club edges closer int heir pursuit of a trebleearned a comfortable 2-0 win over on Tuesday which means they might be Spanish champions before they even play their next match, at on Saturday.\n",
       "*  Th e treble is Barcelona’s dream and having won it in 2009 and 2015, hopes and expectations are rising.\n",
       "*  Perhap s the player outside what is considered Valverde’s best XI with the best chance of breaking into the side is Arturo Vidal .\n",
       "*  Liverpool’ s workhorses might look to strangle Arthur, Barcelona’s passmaster , out of the game, but Vidal would offer some means of protection .\n",
       "*  Th e game was done and dusted by that point, with Luis Suarez slamming home from the penalty spot .\n",
       "    \n",
       "    \n",
       "</ul>\n",
       "------------------------------------------------------------------\n",
       "    \n",
       "------------------------------------------------------------------\n",
       "    \n",
       "    \n",
       "<table>\n",
       "<tr>\n",
       "<th>\n",
       "<a style= \"color:#333399;font-family: \"Times New Roman\", Times, serif;font-size:110%;\" href = \"https://www.goal.com/en-bh/news/valverde-wary-of-powerful-liverpool-ahead-of-champions/ghz9od8lyiel15vz6yijas5en \">Barcelona news: Valverde wary of 'powerful' Liverpool ahead of Champions League semis</a>\n",
       "</th>\n",
       "</tr>\n",
       "<a href=\"mailto:?subject=Barcelona news: Valverde wary of 'powerful' Liverpool ahead of Champions League semis&amp;body= https://www.goal.com/en-bh/news/valverde-wary-of-powerful-liverpool-ahead-of-champions/ghz9od8lyiel15vz6yijas5en \" title=\"Share by Email\"> <img src=\"https://github.com/S-B-Iqbal/Test/raw/master/Images/outlook.png\" alt=\"Outlook\" style=\"float:right;height:20px;width:20px\" hspace = \"20\"> </a>\n",
       "<a href=\"https://wa.me/send?text =https://www.goal.com/en-bh/news/valverde-wary-of-powerful-liverpool-ahead-of-champions/ghz9od8lyiel15vz6yijas5en\" ><img src=\"https://github.com/S-B-Iqbal/Test/raw/master/Images/whatsapp.png\" alt=\"Whatsapp\" style=\"float:right; height:20px;width:20px\" hspace = \"40\"></a>\n",
       "</table>\n",
       "<article><strong>20/04/2019</article>\n",
       " \n",
       "<article><b style= \"color:grey\">Keywords:</b> <i>liverpool | barcelona | premier</i> </article>\n",
       "    \n",
       "------------------------------------------------------------------\n",
       "<ul style=\"color:#000000; font-family:calibri\">\n",
       "*  Valverde wary of 'powerful' Liverpool ahead of Champions League semisBarcelona head coach Ernesto Valverde expects a tough but attractive Champions League semi-final against the Premier League outfitErnesto Valverde has said he is wary of the \"very powerful\" frontline ahead of 's semi-final against the Premier League giants.\n",
       "*  Barca are in control of ’s title race, while Liverpool are fighting for the Premier League crown and were runners-up in Europe's top club competition last season.\n",
       "*  They are also fighting in the Premier League.\n",
       "*  Between Barca and Liverpool, the two European powerhouses have won 10 Champions Leagues/European Cups, with only and winning the jug-eared trophy on more occasions.\n",
       "*  Overall, the sides have met four times in the Champions League, with Liverpool eliminating the Catalan side over two legs in their last encounter, a last-16 tie in 2007.\n",
       "    \n",
       "    \n",
       "</ul>\n",
       "------------------------------------------------------------------\n",
       "    \n",
       "------------------------------------------------------------------\n",
       "    \n",
       "    \n",
       "<table>\n",
       "<tr>\n",
       "<th>\n",
       "<a style= \"color:#333399;font-family: \"Times New Roman\", Times, serif;font-size:110%;\" href = \"https://www.goal.com/en-us/news/champions-league-semi-finals-fixtures-live-streams-how-to/1so7lxwo5fcws11wxzdwupxied \">Champions League semi-finals: Fixtures, live streams & how to watch</a>\n",
       "</th>\n",
       "</tr>\n",
       "<a href=\"mailto:?subject=Champions League semi-finals: Fixtures, live streams & how to watch&amp;body= https://www.goal.com/en-us/news/champions-league-semi-finals-fixtures-live-streams-how-to/1so7lxwo5fcws11wxzdwupxied \" title=\"Share by Email\"> <img src=\"https://github.com/S-B-Iqbal/Test/raw/master/Images/outlook.png\" alt=\"Outlook\" style=\"float:right;height:20px;width:20px\" hspace = \"20\"> </a>\n",
       "<a href=\"https://wa.me/send?text =https://www.goal.com/en-us/news/champions-league-semi-finals-fixtures-live-streams-how-to/1so7lxwo5fcws11wxzdwupxied\" ><img src=\"https://github.com/S-B-Iqbal/Test/raw/master/Images/whatsapp.png\" alt=\"Whatsapp\" style=\"float:right; height:20px;width:20px\" hspace = \"40\"></a>\n",
       "</table>\n",
       "<article><strong>22/04/2019</article>\n",
       " \n",
       "<article><b style= \"color:grey\">Keywords:</b> <i>watch | streams | live | champions | sport</i> </article>\n",
       "    \n",
       "------------------------------------------------------------------\n",
       "<ul style=\"color:#000000; font-family:calibri\">\n",
       "*  Champions League semi-finals: Fixtures, live streams & how to watchEverything you need to know about the Champions League as it gets set to whittle the number of teams down to two, including how to watch every gameThe quest for continental glory is heating up with the moving into the semi-final phase.\n",
       "*  Champions League semi-finals fixtures & resultsThis season's Champions League semi-finals will kick off on April 30, 2019 when Tottenham host Ajax at home and May 1, 2019 when Liverpool travel to Barcelona.\n",
       "*  TV Channel, Live Stream & How To WatchIn the UK, Champions League semi-final matches will be available to watch live on TV via BT Sport , who also offer an online streaming service through their app.\n",
       "*  In the US, viewers are able to watch the Champions League semi-final games on TV through the Univision network (which includes Univision Deportes, UniMas and Galavision) or TNT USA .\n",
       "*  Semi-final One: Ajax vs TottenhamAjax vs Tottenham Semi-final Two: Barcelona vs LiverpoolWinners of the semi-finals will contest the 2019 Champions League final on June 1, 2019 .\n",
       "    \n",
       "    \n",
       "</ul>\n",
       "------------------------------------------------------------------\n",
       "    \n",
       "------------------------------------------------------------------\n",
       "    \n",
       "    \n"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "execution_count": 113,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## Direct Markdown conversion of Summary in 'test_summary'\n",
    "test_summary = Markdown(data= Summary)\n",
    "test_summary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Conversion of Markdown to HTML"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {},
   "outputs": [],
   "source": [
    "html = markdown2.markdown(str(Summary), html4tags=True, tab_width=5,  )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Cleaning the HTML File"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {},
   "outputs": [],
   "source": [
    "### Rectify Out of Bound Paras\n",
    "html = html.replace(\"<p>\",\"<ul><li><p>\")\n",
    "\n",
    "html = html.replace('</p>', '</p></li></ul>')\n",
    "\n",
    "#### Rectify Headlines Link\n",
    "html = html.replace(\"<ul><li><p><a href\",\"<p><a href\")\n",
    "\n",
    "html = html.replace('/a></p></li></ul>', '/a></p>')\n",
    "\n",
    "###  Rectify <centre>\n",
    "html = html.replace(\"<ul><li><p><center></p></li></ul>\",\"<p><center></p>\")\n",
    "\n",
    "html = html.replace('<ul><li><p></center></p></li></ul>', '<p></center></p>')\n",
    "\n",
    "\n",
    "### Rectify *\n",
    "\n",
    "html = html.replace(\"*\",\"</li><li>\")\n",
    "\n",
    "### Rectify Image\n",
    "\n",
    "html = html.replace(\"<li><img \",\"<lily imagine \")\n",
    "\n",
    "#html = html.replace(\"</h1></center></p></li></ul>\",\"</h1></center></p>\")\n",
    "\n",
    "## Rectify Title Bullet point\n",
    "\n",
    "html = html.replace(\"<ul><li><p><center><h3>\",\"<p><center><h3>\")\n",
    "\n",
    "html = html.replace(\"</h3></center></p></li></ul>\",\"</h3></center></p>\")\n",
    "\n",
    "## Open in new tab\n",
    "\n",
    "html = html.replace(\"<a href\", '<a rel=\"noopener noreferrer\" target=\"_blank\" href')\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Checking the sample Webpage"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 116,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "filename = 'File.html'\n",
    "\n",
    "f = open(filename,'w', encoding='utf-16')\n",
    "\n",
    "wrapper = \"\"\"<!DOCTYPE html>\n",
    "<html lang=\"en\">\n",
    "\n",
    "\n",
    "\n",
    "<head>\n",
    "\n",
    "<style>\n",
    ".center {\n",
    "  text-align: center;\n",
    "  \n",
    "}\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "</style>\n",
    "\n",
    "\n",
    "<meta charset=\"ISO-8859-1\" >\n",
    "<div class=\"page center\">\n",
    "   <a href=\"https://www.liverpoolfc.com/welcome-to-liverpool-fc\"><img class=\"auto-size__target postpone-load--fade-in widge-figure__image\" data-src=\"https://e1.365dm.com/18/06/768x432/skysports-liverpool-fixtures_4325753.jpg?20180601143103\" alt=\"\" itemprop=\"image\" src=\"https://e1.365dm.com/18/06/768x432/skysports-liverpool-fixtures_4325753.jpg?20180601143103\" style=\"PADDING-RIGHT: 70px\"></a>\n",
    "</div>\n",
    "\n",
    "<title>%s</title>\n",
    "\n",
    "</head>\n",
    "\n",
    "<body>\n",
    "    \n",
    "    %s\n",
    "</body>\n",
    "\n",
    "\n",
    "\n",
    "</html>\"\"\"\n",
    "\n",
    "title = str('Liverpool News')\n",
    "\n",
    "whole = wrapper % (title, html)\n",
    "\n",
    "#whole = wrapper % (html)\n",
    "\n",
    "f.write(whole)\n",
    "\n",
    "f.close()\n",
    "\n",
    "open_new_tab(filename)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Conversion to PDF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [],
   "source": [
    "try:\n",
    "    # create the API client instance\n",
    "    client = pdfcrowd.HtmlToPdfClient('<user_name>', '<API_Key>')\n",
    "\n",
    "    # run the conversion and write the result to a file\n",
    "    client.convertFileToFile('File.html', final_file_name )\n",
    "except pdfcrowd.Error as why:\n",
    "    # report the error\n",
    "    sys.stderr.write('Pdfcrowd Error: {}\\n'.format(why))\n",
    "\n",
    "    # handle the exception here or rethrow and handle it at a higher level\n",
    "    raise"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}