news_scraping.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "b945e3f1-6779-45ee-b828-aacf407c0b9e",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import pandas as pd\n",
    "from pathlib import Path\n",
    "\n",
    "#os.system('pip3 install pandas')\n",
    "\n",
    "import requests\n",
    "import json\n",
    "\n",
    "from bs4 import BeautifulSoup\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "3b5b79fd-8144-4dfb-913d-88ef2553392d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "                                                     0\n",
      "0                                   Asalin hoton, NYSC\n",
      "1    Masu yi wa ƙasa hidima a Najeriya na ci gaba d...\n",
      "2    A ranar Laraba ne gwamnatin ƙasar ta sanar da ...\n",
      "3    Wata sanarwa da kakakin hukumar NYSC, Caroline...\n",
      "4    Sanawar ta ƙara da cewa an yi ƙarin alawus ɗin...\n",
      "..                                                 ...\n",
      "142  Annobar kwalara ta kashe mutum fiye da 400 a S...\n",
      "143  Hare-haren ƴanbindiga a Sahel sun jefa shugaba...\n",
      "144  Ambaliya na haifar da cikas a harkokin koyo da...\n",
      "145  Abin da ya sa nake so gwamnatin Najeriya ta da...\n",
      "146  Har yanzu ambaliyar Maiduguri na hana ni barci...\n",
      "\n",
      "[147 rows x 1 columns]\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "                                                     0\n",
      "0                                   Asalin hoton, NYSC\n",
      "1    Masu yi wa ƙasa hidima a Najeriya na ci gaba d...\n",
      "2    A ranar Laraba ne gwamnatin ƙasar ta sanar da ...\n",
      "3    Wata sanarwa da kakakin hukumar NYSC, Caroline...\n",
      "4    Sanawar ta ƙara da cewa an yi ƙarin alawus ɗin...\n",
      "..                                                 ...\n",
      "142  Annobar kwalara ta kashe mutum fiye da 400 a S...\n",
      "143  Hare-haren ƴanbindiga a Sahel sun jefa shugaba...\n",
      "144  Ambaliya na haifar da cikas a harkokin koyo da...\n",
      "145  Abin da ya sa nake so gwamnatin Najeriya ta da...\n",
      "146  Har yanzu ambaliyar Maiduguri na hana ni barci...\n",
      "\n",
      "[147 rows x 1 columns]\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "                                                     0\n",
      "0                                   Asalin hoton, NYSC\n",
      "1    Masu yi wa ƙasa hidima a Najeriya na ci gaba d...\n",
      "2    A ranar Laraba ne gwamnatin ƙasar ta sanar da ...\n",
      "3    Wata sanarwa da kakakin hukumar NYSC, Caroline...\n",
      "4    Sanawar ta ƙara da cewa an yi ƙarin alawus ɗin...\n",
      "..                                                 ...\n",
      "142  Annobar kwalara ta kashe mutum fiye da 400 a S...\n",
      "143  Hare-haren ƴanbindiga a Sahel sun jefa shugaba...\n",
      "144  Ambaliya na haifar da cikas a harkokin koyo da...\n",
      "145  Abin da ya sa nake so gwamnatin Najeriya ta da...\n",
      "146  Har yanzu ambaliyar Maiduguri na hana ni barci...\n",
      "\n",
      "[147 rows x 1 columns]\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "                                                     0\n",
      "0    Mummunar ambaliya ruwa da ta auka wa birnin Ma...\n",
      "1    Ɓallewar madatsar ruwa da Alau da ke wajen bir...\n",
      "2    Gidaje da dama da motocin hawa sun nutse cikin...\n",
      "3    Ambaliyar ta shafe maƙabartu da dama inda ake ...\n",
      "4    Wakilin BBC da ke birnin Maiduguri ya ce ya ga...\n",
      "..                                                 ...\n",
      "285  Annobar kwalara ta kashe mutum fiye da 400 a S...\n",
      "286  Hare-haren ƴanbindiga a Sahel sun jefa shugaba...\n",
      "287  Ambaliya na haifar da cikas a harkokin koyo da...\n",
      "288  Abin da ya sa nake so gwamnatin Najeriya ta da...\n",
      "289  Har yanzu ambaliyar Maiduguri na hana ni barci...\n",
      "\n",
      "[290 rows x 1 columns]\n"
     ]
    }
   ],
   "source": [
    "#create a easy functions to automate our flow\n",
    "\n",
    "def getSoupDataFromLink(url):\n",
    "    html = requests.get(url).text\n",
    "    soup = BeautifulSoup(requests.get(url).content, \"html.parser\")\n",
    "    \n",
    "    return soup\n",
    "\n",
    "def getWordsFromSoup(data):\n",
    "\n",
    "    extracted_words = []\n",
    "    \n",
    "    mainBody = data.find('body').find_all('h3')\n",
    "    newsBody = data.find('body').find_all('p')\n",
    "\n",
    "    newsBody_list = [x.text.strip() for x in newsBody]\n",
    "    headlines_list = [x.text.strip() for x in mainBody]\n",
    "    \n",
    "    extracted_words.extend(newsBody_list)\n",
    "    extracted_words.extend(headlines_list)\n",
    "\n",
    "    print()\n",
    "\n",
    "    return extracted_words\n",
    "\n",
    "\n",
    "\n",
    "def getMoreLinksFromArticles(data, link_check):\n",
    "    moreLinks = []\n",
    "    \n",
    "    link_lists = [x['href'] for x in data.find_all('a', href=True)]\n",
    "    for x in link_lists:\n",
    "        if x.find(link_check) != -1:\n",
    "            moreLinks.append(x)\n",
    "                \n",
    "    return moreLinks, len(moreLinks)\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "def turnListOfWordsToDF (data, counting_len, stop_len, file_name):    \n",
    "    df = pd.DataFrame(data)\n",
    "    \n",
    "    sorted(data)\n",
    "\n",
    "    if(counting_len == stop_len):\n",
    "        print(df)\n",
    "        \n",
    "        filepath = Path(file_name)  \n",
    "        filepath.parent.mkdir(parents=True, exist_ok=True)\n",
    "        \n",
    "        df['lang'] = [\"hausa\"] * df[df.columns[0]].count()\n",
    "        \n",
    "        df.to_csv (filepath, index = None, header=True)\n",
    "\n",
    "\n",
    "\n",
    "def extractFromSetLinkList(links_to_check):\n",
    "\n",
    "    extractedListOfWords = []\n",
    "\n",
    "    soup_data = getSoupDataFromLink(links_to_check)\n",
    "    more_links_gotten = getMoreLinksFromArticles(soup_data)[0]\n",
    "\n",
    "    a = 0\n",
    "\n",
    "    for datax in more_links_gotten:\n",
    "        a = a + 1\n",
    "        filename = 'sabifolder/newsData/extractedNewsData00' + str(a) + \".csv\"\n",
    "\n",
    "        soup = getSoupDataFromLink(datax, \"https://www.bbc.com/hausa/articles\")\n",
    "        words = getWordsFromSoup(soup)\n",
    "        extractedListOfWords.extend(words)\n",
    "        turnListOfWordsToDF(extractedListOfWords, a, len(more_links_gotten), filename)\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "links_to_test = [\"https://www.bbc.com/hausa/articles/c33v1pp0v5ko\", \n",
    "                 \"https://www.bbc.com/hausa/articles/c78d7499w94o\",\n",
    "                 \"https://www.bbc.com/hausa/articles/cwyl0r1v403o\",\n",
    "                 \"https://www.bbc.com/hausa/articles/cn03yl0nwz9o\"]\n",
    "\n",
    "single_to_test = [\"https://www.bbc.com/yoruba/articles/c99vxzpj722o\"]\n",
    "\n",
    "\n",
    "\n",
    "for l in links_to_test:\n",
    "    extractFromSetLinkList(l)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2b2b371a-56b3-40ad-a1bf-d33a671a7221",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}