notebooks/Final Model - Classification_NEWS.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Final model - preprocessing and classification for NEWS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import re\n",
    "import string\n",
    "import unicodedata\n",
    "\n",
    "from transformers import pipeline\n",
    "from nltk.tokenize import word_tokenize\n",
    "from googletrans import Translator\n",
    "\n",
    "#pd.set_option('display.max_rows', None)\n",
    "#pd.set_option('display.max_columns', None)\n",
    "#pd.set_option('display.width', None)\n",
    "#pd.set_option('display.max_colwidth', None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [],
   "source": [
    "import httpx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Timeout(timeout=5)"
      ]
     },
     "execution_count": 80,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "httpx.Timeout(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "publico_path = '../extra/notícias/news_sample_1to1000_updated.csv'\n",
    "observador_path = '../extra/notícias/noticias_observador_fev_a_julho.csv'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_publico = pd.read_csv(publico_path, index_col='Unnamed: 0')\n",
    "df_observador = pd.read_csv(observador_path, index_col='Unnamed: 0')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Cleaning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Removendo \"tipo\" quizz\n",
    "df_publico = df_publico[df_publico[\"tipo\"]!=\"QUIZZ\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# colunas essenciais\n",
    "df_publico = df_publico[[\"data\", \"titulo\", \"descricao\"]]\n",
    "df_observador = df_observador[[\"Data\", \"Titulo\", \"Desc\"]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# rename de colunas do observador para serem iguais às do público\n",
    "df_observador.rename(columns={\"Data\": \"data\",\n",
    "                              \"Titulo\": \"titulo\",\n",
    "                              \"Desc\": \"descricao\"}, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 9668 entries, 0 to 9669\n",
      "Data columns (total 3 columns):\n",
      " #   Column     Non-Null Count  Dtype \n",
      "---  ------     --------------  ----- \n",
      " 0   data       9668 non-null   object\n",
      " 1   titulo     9668 non-null   object\n",
      " 2   descricao  9287 non-null   object\n",
      "dtypes: object(3)\n",
      "memory usage: 302.1+ KB\n"
     ]
    }
   ],
   "source": [
    "df_publico.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 28735 entries, 0 to 28734\n",
      "Data columns (total 3 columns):\n",
      " #   Column     Non-Null Count  Dtype \n",
      "---  ------     --------------  ----- \n",
      " 0   data       28735 non-null  object\n",
      " 1   titulo     28735 non-null  object\n",
      " 2   descricao  28735 non-null  object\n",
      "dtypes: object(3)\n",
      "memory usage: 898.0+ KB\n"
     ]
    }
   ],
   "source": [
    "df_observador.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "# change data to date type\n",
    "df_publico[\"data\"] = pd.to_datetime(df_publico[\"data\"]).dt.normalize()\n",
    "df_observador[\"data\"] = pd.to_datetime(df_observador[\"data\"]).dt.normalize()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "# lets consider dates between 01/01/2020 and 30/06/2020\n",
    "start_date = \"2020-01-01\"\n",
    "end_date = \"2020-06-30\"\n",
    "\n",
    "df_publico = df_publico.loc[(df_publico[\"data\"] >= start_date) & \n",
    "                            (df_publico[\"data\"] <= end_date)]\n",
    "df_observador = df_observador.loc[(df_observador[\"data\"] >= start_date) & \n",
    "                            (df_observador[\"data\"] <= end_date)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 8858 entries, 810 to 9669\n",
      "Data columns (total 3 columns):\n",
      " #   Column     Non-Null Count  Dtype         \n",
      "---  ------     --------------  -----         \n",
      " 0   data       8858 non-null   datetime64[ns]\n",
      " 1   titulo     8858 non-null   object        \n",
      " 2   descricao  8506 non-null   object        \n",
      "dtypes: datetime64[ns](1), object(2)\n",
      "memory usage: 276.8+ KB\n"
     ]
    }
   ],
   "source": [
    "df_publico.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 28735 entries, 0 to 28734\n",
      "Data columns (total 3 columns):\n",
      " #   Column     Non-Null Count  Dtype         \n",
      "---  ------     --------------  -----         \n",
      " 0   data       28735 non-null  datetime64[ns]\n",
      " 1   titulo     28735 non-null  object        \n",
      " 2   descricao  28735 non-null  object        \n",
      "dtypes: datetime64[ns](1), object(2)\n",
      "memory usage: 898.0+ KB\n"
     ]
    }
   ],
   "source": [
    "df_observador.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<ipython-input-32-6534b548169f>:1: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df_publico['source'] = 'publico'\n"
     ]
    }
   ],
   "source": [
    "df_publico['source'] = 'publico'\n",
    "df_observador['source'] = 'observador'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "# concatenate both dataframes\n",
    "df_news = pd.concat([df_publico, df_observador], ignore_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 37593 entries, 0 to 37592\n",
      "Data columns (total 4 columns):\n",
      " #   Column     Non-Null Count  Dtype         \n",
      "---  ------     --------------  -----         \n",
      " 0   data       37593 non-null  datetime64[ns]\n",
      " 1   titulo     37593 non-null  object        \n",
      " 2   descricao  37241 non-null  object        \n",
      " 3   source     37593 non-null  object        \n",
      "dtypes: datetime64[ns](1), object(3)\n",
      "memory usage: 1.1+ MB\n"
     ]
    }
   ],
   "source": [
    "df_news.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>data</th>\n",
       "      <th>titulo</th>\n",
       "      <th>descricao</th>\n",
       "      <th>source</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2020-06-30</td>\n",
       "      <td>Ex-ministro da Saúde Correia de Campos alerta ...</td>\n",
       "      <td>“Preparemo-nos para o que aí vem”, avisou o an...</td>\n",
       "      <td>publico</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2020-06-30</td>\n",
       "      <td>Estados Unidos compram quase todo o &lt;i&gt;stock&lt;/...</td>\n",
       "      <td>A compra das mais de 500 mil doses do medicame...</td>\n",
       "      <td>publico</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2020-06-30</td>\n",
       "      <td>Governo admite quarentena para hotéis com cont...</td>\n",
       "      <td>Hóspedes e funcionários também poderão ficar d...</td>\n",
       "      <td>publico</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2020-06-30</td>\n",
       "      <td>Covid-19: quem quer ajudar os cientistas a seg...</td>\n",
       "      <td>A perda de olfacto é um dos sintomas comuns as...</td>\n",
       "      <td>publico</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2020-06-30</td>\n",
       "      <td>Profissionais do SNS “não deram o litro, deram...</td>\n",
       "      <td>O documento foi apresentado no Infarmed, em Li...</td>\n",
       "      <td>publico</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        data                                             titulo  \\\n",
       "0 2020-06-30  Ex-ministro da Saúde Correia de Campos alerta ...   \n",
       "1 2020-06-30  Estados Unidos compram quase todo o <i>stock</...   \n",
       "2 2020-06-30  Governo admite quarentena para hotéis com cont...   \n",
       "3 2020-06-30  Covid-19: quem quer ajudar os cientistas a seg...   \n",
       "4 2020-06-30  Profissionais do SNS “não deram o litro, deram...   \n",
       "\n",
       "                                           descricao   source  \n",
       "0  “Preparemo-nos para o que aí vem”, avisou o an...  publico  \n",
       "1  A compra das mais de 500 mil doses do medicame...  publico  \n",
       "2  Hóspedes e funcionários também poderão ficar d...  publico  \n",
       "3  A perda de olfacto é um dos sintomas comuns as...  publico  \n",
       "4  O documento foi apresentado no Infarmed, em Li...  publico  "
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_news.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Preprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 144,
   "metadata": {},
   "outputs": [],
   "source": [
    "stopwords = {\"coronavirus\",\"covid19\", \"covid19pt\",\"covid\",\"CoronaVirus\",\"FiqueEmCasa\", \"fiqueemcasa\", \"Covid19\",\"StayAtHome\",\n",
    "            \"Coronavirus.\",\"COVID2019\", \"COVID19\" \"Coronavirus\", \"STOPCOVID19\", \"ficaemcasa\"}\n",
    "\n",
    "# Function to clean Portuguese text and tokenize it\n",
    "def preProcessing(text):\n",
    "    if isinstance(text, str):\n",
    "        #text = text.lower() # convert text to lower-case\n",
    "        text = re.sub('https?:\\/\\/\\S+', '', text) # Removing hyperlink\n",
    "        text = re.sub('@[A-Za-z0–9]+', '', text) #Removing @mentions\n",
    "        text = re.sub('RT[\\s]+', '', text) # Removing RT\n",
    "        text = re.sub(r'#([^\\s]+)', r'\\1', text) # remove the # in #hashtag\n",
    "        text = re.sub(r'<.*?>', '', text) # remove HTML formatting\n",
    "        text = word_tokenize(text) # remove repeated characters\n",
    "        word_list = [word for word in text if word not in stopwords]\n",
    "        text = ' '.join(word_list)\n",
    "    else:\n",
    "        text = ''\n",
    "    return text\n",
    "\n",
    "translator = Translator(timeout=httpx.Timeout(5))\n",
    "\n",
    "# Function to translate the current text to english\n",
    "def Translation(text):\n",
    "    if  translator.detect(text) != 'en':\n",
    "        traducao = translator.translate(text, dest='en')\n",
    "        return traducao.text\n",
    "    else:\n",
    "        return text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<ipython-input-44-8670de5afc49>:1: FutureWarning: Treating datetime data as categorical rather than numeric in `.describe` is deprecated and will be removed in a future version of pandas. Specify `datetime_is_numeric=True` to silence this warning and adopt the future behavior now.\n",
      "  df_news.describe()\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>data</th>\n",
       "      <th>titulo</th>\n",
       "      <th>descricao</th>\n",
       "      <th>source</th>\n",
       "      <th>titulo_clean</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>37593</td>\n",
       "      <td>37593</td>\n",
       "      <td>37241</td>\n",
       "      <td>37593</td>\n",
       "      <td>37593</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unique</th>\n",
       "      <td>162</td>\n",
       "      <td>21333</td>\n",
       "      <td>20778</td>\n",
       "      <td>2</td>\n",
       "      <td>21318</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>top</th>\n",
       "      <td>2020-03-26 00:00:00</td>\n",
       "      <td>O que vai acontecer aos portugueses de ...</td>\n",
       "      <td>As notícias no Observador. Aconteça o que acon...</td>\n",
       "      <td>observador</td>\n",
       "      <td>Wuhan Coronavírus : afinal , qual o risco ?</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>freq</th>\n",
       "      <td>465</td>\n",
       "      <td>31</td>\n",
       "      <td>192</td>\n",
       "      <td>28735</td>\n",
       "      <td>31</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>first</th>\n",
       "      <td>2020-01-21 00:00:00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>last</th>\n",
       "      <td>2020-06-30 00:00:00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                       data  \\\n",
       "count                 37593   \n",
       "unique                  162   \n",
       "top     2020-03-26 00:00:00   \n",
       "freq                    465   \n",
       "first   2020-01-21 00:00:00   \n",
       "last    2020-06-30 00:00:00   \n",
       "\n",
       "                                                   titulo  \\\n",
       "count                                               37593   \n",
       "unique                                              21333   \n",
       "top            O que vai acontecer aos portugueses de ...   \n",
       "freq                                                   31   \n",
       "first                                                 NaN   \n",
       "last                                                  NaN   \n",
       "\n",
       "                                                descricao      source  \\\n",
       "count                                               37241       37593   \n",
       "unique                                              20778           2   \n",
       "top     As notícias no Observador. Aconteça o que acon...  observador   \n",
       "freq                                                  192       28735   \n",
       "first                                                 NaN         NaN   \n",
       "last                                                  NaN         NaN   \n",
       "\n",
       "                                       titulo_clean  \n",
       "count                                         37593  \n",
       "unique                                        21318  \n",
       "top     Wuhan Coronavírus : afinal , qual o risco ?  \n",
       "freq                                             31  \n",
       "first                                           NaN  \n",
       "last                                            NaN  "
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_news.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_news['titulo_clean'] = df_news['titulo'].apply(preProcessing)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 145,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_news['descricao_clean'] = df_news['descricao'].apply(preProcessing)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [],
   "source": [
    "titles = list(df_news[\"titulo_clean\"].to_numpy())\n",
    "\n",
    "translations = translator.translate(titles, dest='en')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 146,
   "metadata": {},
   "outputs": [],
   "source": [
    "desc = list(df_news['descricao_clean'].to_numpy())\n",
    "\n",
    "translations = translator.translate(desc, dest='en')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 152,
   "metadata": {},
   "outputs": [],
   "source": [
    "translations_list = []\n",
    "\n",
    "for translation in translations:\n",
    "    translations_list.append(translation.text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 153,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_news['descricao_en'] = translations_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 154,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>descricao</th>\n",
       "      <th>descricao_en</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>“Preparemo-nos para o que aí vem”, avisou o an...</td>\n",
       "      <td>“Let us prepare for what is coming”, warned th...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>A compra das mais de 500 mil doses do medicame...</td>\n",
       "      <td>The purchase of more than 500 thousand doses o...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Hóspedes e funcionários também poderão ficar d...</td>\n",
       "      <td>Guests and staff may also be quarantined in th...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>A perda de olfacto é um dos sintomas comuns as...</td>\n",
       "      <td>Loss of smell is one of the common symptoms as...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>O documento foi apresentado no Infarmed, em Li...</td>\n",
       "      <td>The document was presented at Infarmed, in Lis...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Joe Biden apresentou um plano para combater a ...</td>\n",
       "      <td>Joe Biden presented a plan to combat covid-19 ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Proposta do PSD prevê que inquilinos possam re...</td>\n",
       "      <td>PSD's proposal provides that tenants can redee...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Durante duas semanas de confinamento, um grupo...</td>\n",
       "      <td>During two weeks of confinement, a group of re...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>No final das votações vai saber-se qual o part...</td>\n",
       "      <td>At the end of the polls, it will be known whic...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Propostas do BE e PCP foram rejeitadas com vot...</td>\n",
       "      <td>BE and PCP proposals were rejected with votes ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                           descricao  \\\n",
       "0  “Preparemo-nos para o que aí vem”, avisou o an...   \n",
       "1  A compra das mais de 500 mil doses do medicame...   \n",
       "2  Hóspedes e funcionários também poderão ficar d...   \n",
       "3  A perda de olfacto é um dos sintomas comuns as...   \n",
       "4  O documento foi apresentado no Infarmed, em Li...   \n",
       "5  Joe Biden apresentou um plano para combater a ...   \n",
       "6  Proposta do PSD prevê que inquilinos possam re...   \n",
       "7  Durante duas semanas de confinamento, um grupo...   \n",
       "8  No final das votações vai saber-se qual o part...   \n",
       "9  Propostas do BE e PCP foram rejeitadas com vot...   \n",
       "\n",
       "                                        descricao_en  \n",
       "0  “Let us prepare for what is coming”, warned th...  \n",
       "1  The purchase of more than 500 thousand doses o...  \n",
       "2  Guests and staff may also be quarantined in th...  \n",
       "3  Loss of smell is one of the common symptoms as...  \n",
       "4  The document was presented at Infarmed, in Lis...  \n",
       "5  Joe Biden presented a plan to combat covid-19 ...  \n",
       "6  PSD's proposal provides that tenants can redee...  \n",
       "7  During two weeks of confinement, a group of re...  \n",
       "8  At the end of the polls, it will be known whic...  \n",
       "9  BE and PCP proposals were rejected with votes ...  "
      ]
     },
     "execution_count": 154,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_news[['descricao', 'descricao_en']].head(10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Classification"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 155,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Allocate a pipeline for sentiment-analysis\n",
    "nlp = pipeline('sentiment-analysis')\n",
    "\n",
    "def getSentiment(sentence):\n",
    "    sentiment = nlp(sentence)[0]['label']\n",
    "    return sentiment\n",
    "\n",
    "def getScore(sentence):\n",
    "    score = nlp(sentence)[0]['score']\n",
    "    return score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 156,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_news['descricao_classification'] = df_news['descricao_en'].apply(getSentiment)\n",
    "df_news['descricao_score'] = df_news['descricao_en'].apply(getScore)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 157,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>data</th>\n",
       "      <th>titulo</th>\n",
       "      <th>descricao</th>\n",
       "      <th>source</th>\n",
       "      <th>titulo_clean</th>\n",
       "      <th>titulo_en</th>\n",
       "      <th>classification</th>\n",
       "      <th>score</th>\n",
       "      <th>descricao_clean</th>\n",
       "      <th>descricao_en</th>\n",
       "      <th>descricao_classification</th>\n",
       "      <th>descricao_score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2020-06-30</td>\n",
       "      <td>Ex-ministro da Saúde Correia de Campos alerta ...</td>\n",
       "      <td>“Preparemo-nos para o que aí vem”, avisou o an...</td>\n",
       "      <td>publico</td>\n",
       "      <td>Ex-ministro da Saúde Correia de Campos alerta ...</td>\n",
       "      <td>Former Health Minister Correia de Campos warns...</td>\n",
       "      <td>NEGATIVE</td>\n",
       "      <td>0.947701</td>\n",
       "      <td>“ Preparemo-nos para o que aí vem ” , avisou o...</td>\n",
       "      <td>“Let us prepare for what is coming”, warned th...</td>\n",
       "      <td>NEGATIVE</td>\n",
       "      <td>0.937429</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2020-06-30</td>\n",
       "      <td>Estados Unidos compram quase todo o &lt;i&gt;stock&lt;/...</td>\n",
       "      <td>A compra das mais de 500 mil doses do medicame...</td>\n",
       "      <td>publico</td>\n",
       "      <td>Estados Unidos compram quase todo o stock de r...</td>\n",
       "      <td>United States buys almost all stock of remdesivir</td>\n",
       "      <td>NEGATIVE</td>\n",
       "      <td>0.852459</td>\n",
       "      <td>A compra das mais de 500 mil doses do medicame...</td>\n",
       "      <td>The purchase of more than 500 thousand doses o...</td>\n",
       "      <td>POSITIVE</td>\n",
       "      <td>0.849041</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2020-06-30</td>\n",
       "      <td>Governo admite quarentena para hotéis com cont...</td>\n",
       "      <td>Hóspedes e funcionários também poderão ficar d...</td>\n",
       "      <td>publico</td>\n",
       "      <td>Governo admite quarentena para hotéis com cont...</td>\n",
       "      <td>Government admits quarantine for hotels with c...</td>\n",
       "      <td>NEGATIVE</td>\n",
       "      <td>0.983680</td>\n",
       "      <td>Hóspedes e funcionários também poderão ficar d...</td>\n",
       "      <td>Guests and staff may also be quarantined in th...</td>\n",
       "      <td>NEGATIVE</td>\n",
       "      <td>0.960382</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2020-06-30</td>\n",
       "      <td>Covid-19: quem quer ajudar os cientistas a seg...</td>\n",
       "      <td>A perda de olfacto é um dos sintomas comuns as...</td>\n",
       "      <td>publico</td>\n",
       "      <td>Covid-19 : quem quer ajudar os cientistas a se...</td>\n",
       "      <td>Covid-19: who wants to help scientists track t...</td>\n",
       "      <td>NEGATIVE</td>\n",
       "      <td>0.998983</td>\n",
       "      <td>A perda de olfacto é um dos sintomas comuns as...</td>\n",
       "      <td>Loss of smell is one of the common symptoms as...</td>\n",
       "      <td>NEGATIVE</td>\n",
       "      <td>0.992243</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2020-06-30</td>\n",
       "      <td>Profissionais do SNS “não deram o litro, deram...</td>\n",
       "      <td>O documento foi apresentado no Infarmed, em Li...</td>\n",
       "      <td>publico</td>\n",
       "      <td>Profissionais do SNS “ não deram o litro , der...</td>\n",
       "      <td>SNS professionals “did not give the liter, the...</td>\n",
       "      <td>NEGATIVE</td>\n",
       "      <td>0.962007</td>\n",
       "      <td>O documento foi apresentado no Infarmed , em L...</td>\n",
       "      <td>The document was presented at Infarmed, in Lis...</td>\n",
       "      <td>NEGATIVE</td>\n",
       "      <td>0.998898</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        data                                             titulo  \\\n",
       "0 2020-06-30  Ex-ministro da Saúde Correia de Campos alerta ...   \n",
       "1 2020-06-30  Estados Unidos compram quase todo o <i>stock</...   \n",
       "2 2020-06-30  Governo admite quarentena para hotéis com cont...   \n",
       "3 2020-06-30  Covid-19: quem quer ajudar os cientistas a seg...   \n",
       "4 2020-06-30  Profissionais do SNS “não deram o litro, deram...   \n",
       "\n",
       "                                           descricao   source  \\\n",
       "0  “Preparemo-nos para o que aí vem”, avisou o an...  publico   \n",
       "1  A compra das mais de 500 mil doses do medicame...  publico   \n",
       "2  Hóspedes e funcionários também poderão ficar d...  publico   \n",
       "3  A perda de olfacto é um dos sintomas comuns as...  publico   \n",
       "4  O documento foi apresentado no Infarmed, em Li...  publico   \n",
       "\n",
       "                                        titulo_clean  \\\n",
       "0  Ex-ministro da Saúde Correia de Campos alerta ...   \n",
       "1  Estados Unidos compram quase todo o stock de r...   \n",
       "2  Governo admite quarentena para hotéis com cont...   \n",
       "3  Covid-19 : quem quer ajudar os cientistas a se...   \n",
       "4  Profissionais do SNS “ não deram o litro , der...   \n",
       "\n",
       "                                           titulo_en classification     score  \\\n",
       "0  Former Health Minister Correia de Campos warns...       NEGATIVE  0.947701   \n",
       "1  United States buys almost all stock of remdesivir       NEGATIVE  0.852459   \n",
       "2  Government admits quarantine for hotels with c...       NEGATIVE  0.983680   \n",
       "3  Covid-19: who wants to help scientists track t...       NEGATIVE  0.998983   \n",
       "4  SNS professionals “did not give the liter, the...       NEGATIVE  0.962007   \n",
       "\n",
       "                                     descricao_clean  \\\n",
       "0  “ Preparemo-nos para o que aí vem ” , avisou o...   \n",
       "1  A compra das mais de 500 mil doses do medicame...   \n",
       "2  Hóspedes e funcionários também poderão ficar d...   \n",
       "3  A perda de olfacto é um dos sintomas comuns as...   \n",
       "4  O documento foi apresentado no Infarmed , em L...   \n",
       "\n",
       "                                        descricao_en descricao_classification  \\\n",
       "0  “Let us prepare for what is coming”, warned th...                 NEGATIVE   \n",
       "1  The purchase of more than 500 thousand doses o...                 POSITIVE   \n",
       "2  Guests and staff may also be quarantined in th...                 NEGATIVE   \n",
       "3  Loss of smell is one of the common symptoms as...                 NEGATIVE   \n",
       "4  The document was presented at Infarmed, in Lis...                 NEGATIVE   \n",
       "\n",
       "   descricao_score  \n",
       "0         0.937429  \n",
       "1         0.849041  \n",
       "2         0.960382  \n",
       "3         0.992243  \n",
       "4         0.998898  "
      ]
     },
     "execution_count": 157,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_news.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "NEGATIVE    27360\n",
       "POSITIVE    10233\n",
       "Name: classification, dtype: int64"
      ]
     },
     "execution_count": 101,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_news['classification'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 158,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "NEGATIVE    28239\n",
       "POSITIVE     9354\n",
       "Name: descricao_classification, dtype: int64"
      ]
     },
     "execution_count": 158,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_news['descricao_classification'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 159,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_news.to_csv('noticias_en_classification_2.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}