Scrapping_PC.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Import necessary files and libs:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "import pandas as pd\n",
    "import re\n",
    "from bs4 import BeautifulSoup\n",
    "from files.lol_regions import regions\n",
    "from files.lol_genders import genders\n",
    "from files.lol_names import correction_roles\n",
    "from files.wiki_names import wiki_tags"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### __Champion Voices:__"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Get champions list from API:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "version = \"12.12.1\"\n",
    "response = requests.get(f\"http://ddragon.leagueoflegends.com/cdn/{version}/data/en_US/champion.json\").json()\n",
    "champions_all_info = list(response['data'].items())\n",
    "champions = list(response['data'].keys())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Apply search correction in names:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "lol_champions = list(map(wiki_tags.get, champions, champions))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def remove_tag_voices(v_line): # \"text 123\" --> text 123\n",
    "    tag_removed = str(v_line)[3:-4]\n",
    "    if ((tag_removed[0] == '\"') and (tag_removed[-1] == '\"')):\n",
    "        return(tag_removed[1:-1],True)\n",
    "    else:\n",
    "        return(tag_removed,False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def scrap_voices(dataframe, champ_name):\n",
    "    champ = requests.get(f\"https://leagueoflegends.fandom.com/wiki/{champ_name}/LoL/Audio\")\n",
    "    if champ.status_code == 404:\n",
    "        print(i + \" \" + str(champ.status_code))\n",
    "    no_trivia = re.sub(r'<span class=\"mw-headline\" id=\"Trivia\">.*', '', champ.text, flags=re.DOTALL).strip()\n",
    "    soup = BeautifulSoup(no_trivia, 'html.parser')\n",
    "    unsorted_lines = list(soup.find('div', class_=\"mw-parser-output\").find_all('i'))\n",
    "    \n",
    "    sorted_lines = list(map(remove_tag_voices, unsorted_lines))\n",
    "    formatted_lines = pd.DataFrame([[champ_name,line[0],line[1]] for line in sorted_lines],\n",
    "                                   columns=['champion', 'voice_line', 'is_spoken'])\n",
    "    dataframe = pd.concat([dataframe,formatted_lines])\n",
    "    return dataframe"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Run multi-page scrapping: (150+ Pages)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "unfiltered_voices = pd.DataFrame(columns=['champion', 'voice_line', 'is_spoken'])\n",
    "for champion in lol_champions:\n",
    "    unfiltered_voices = scrap_voices(unfiltered_voices, champion)\n",
    "unfiltered_voices.to_csv(\"files/unfiltered_voices.csv\", index=False, encoding='utf-8')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "unfiltered_voices"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Create function to Clean Html tags:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "CLEANR = re.compile('<.*?>|&([a-z0-9]+|#[0-9]{1,6}|#x[0-9a-f]{1,6});')\n",
    "\n",
    "def cleanhtml(row):\n",
    "    cleantext = re.sub(CLEANR, '', row)\n",
    "    return cleantext"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Organize and drop audio files:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "voice_df = pd.DataFrame(pd.read_csv(\"files/unfiltered_voices.csv\"))\n",
    "voice_df = voice_df.dropna() # Remove empty rows\n",
    "voice_df.drop(voice_df[voice_df.voice_line.str.endswith(\".ogg\")].index, inplace = True) # Remove audios\n",
    "voice_df.drop_duplicates(subset =\"voice_line\", keep = \"first\", inplace = True)\n",
    "voice_df.voice_line = voice_df.voice_line.map(cleanhtml) # Remove Html tags\n",
    "voice_df.sort_values(by=\"champion\",ignore_index=True, inplace=True)\n",
    "voice_df.to_csv(\"files/voices.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "voice_df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### __Champion gender:__"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Set main dataframe:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "lol_df = pd.DataFrame(voice_df)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Add gender to respective champions:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "for k,v in genders.items():\n",
    "    lol_df.loc[lol_df[\"champion\"] == k, \"gender\"] = v"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "lol_df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### __Champion roles:__"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Create list from API keys:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "champions_keys = []\n",
    "for info_tuple in champions_all_info:\n",
    "    champions_keys.append(info_tuple[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "champions_keys"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Storage information about champíons:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "champions_tuple = tuple(champions_all_info)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Create list with champions:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "list_champions = []\n",
    "for i in champions_tuple:\n",
    "    list_champions.append(i[1][\"name\"])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Create list with champions and tags:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "list_tags = []\n",
    "for k,c in zip(champions_keys, list_champions):\n",
    "    tag_champion = response['data'][k]['tags']\n",
    "    try:\n",
    "        list_tags.append([c, f\"{tag_champion[0]}, {tag_champion[1]}\"])\n",
    "    except:\n",
    "        list_tags.append([c, f\"{tag_champion[0]}\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "list_tags"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Create dataframe:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "tags_df = pd.DataFrame(list_tags, columns=[\"champion\", \"role\"])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Apply name correction in dataframe::"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "for key, value in correction_roles.items():\n",
    "    lol_df.loc[lol_df[\"champion\"] == key, \"champion\"] = value"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Merge dataframes:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "lol_df = lol_df.merge(tags_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "lol_df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### __Champion Stats with Selenium:__"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Note:**\n",
    "- Selenium will emulate an hidden browser in your PC."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "from webdriver_manager.chrome import ChromeDriverManager\n",
    "from selenium.webdriver.common.by import By\n",
    "from selenium import webdriver"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "options = webdriver.ChromeOptions()\n",
    "options.headless = True\n",
    "\n",
    "url = \"https://na.op.gg/statistics/champions?hl=en_US&region=global\"\n",
    "driver = webdriver.Chrome(ChromeDriverManager().install(), chrome_options=options)\n",
    "\n",
    "driver.get(url)\n",
    "driver.implicitly_wait(7)\n",
    "driver.find_element(By.XPATH, '//*[@id=\"content-container\"]/div[2]/table/tbody/tr[118]/td[2]')\n",
    "html = driver.page_source\n",
    "driver.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Transform html table into dataframe:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "stats_df = pd.read_html(html)\n",
    "stats_df = stats_df[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "stats_df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Clean and organize column names:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "stats_df.drop([\"#\", \"CS\", \"Gold\", \"Games played\", \"KDA\"], inplace=True, axis=1, errors='ignore')\n",
    "stats_df.rename(\n",
    "inplace=True,\n",
    "columns= {\"Champion\": \"champion\", \n",
    "          \"Win rate\": \"win_rate\",\n",
    "          \"Pick ratio per game\": \"pick_rate\",\n",
    "          \"Ban ratio per game\": \"ban_rate\"})\n",
    "          \n",
    "stats_df.sort_values(\"champion\", inplace=True, axis=0, ignore_index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Merge dataframes:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "lol_df = lol_df.merge(stats_df, sort=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "stats_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "lol_df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### __Champion Region:__"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Requests method - **Fast**:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_json_and_scrap(reg):\n",
    "    url = f'https://universe-meeps.leagueoflegends.com/v1/en_gb/factions/{reg}/index.json'\n",
    "    response = requests.get(url).json()\n",
    "\n",
    "    region_name = response['faction']['name']\n",
    "    region_members = response['associated-champions']\n",
    "\n",
    "    for i in region_members:\n",
    "        champ_name = i['title']\n",
    "        champs_with_region.append([champ_name, region_name])\n",
    "    return"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Run multi-page scrapping: (10+ Pages)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "champs_with_region = []\n",
    "for region in regions:\n",
    "    get_json_and_scrap(region)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Selenium method - **Slow** (Optional):"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_html_and_scrap(reg):\n",
    "    options = webdriver.ChromeOptions()\n",
    "    options.headless = True\n",
    "    \n",
    "    driver = webdriver.Chrome(ChromeDriverManager().install(), chrome_options=options)\n",
    "    url = f\"https://universe.leagueoflegends.com/en_US/region/{reg}/\"\n",
    "\n",
    "    driver.get(url)\n",
    "    driver.implicitly_wait(5)\n",
    "    driver.find_element(By.ID, 'champions-of-faction')\n",
    "    html = driver.page_source\n",
    "    driver.close()\n",
    "\n",
    "    soup = BeautifulSoup(html, \"html.parser\")\n",
    "    regions_champ = soup.find_all(\"div\", class_ = \"copy_xxN7\")\n",
    "    for champ in regions_champ:\n",
    "        champs_with_region.append([champ.h1.text, champ.h2.text])\n",
    "    return"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Run multi-page scrapping (Optional):"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "champs_with_region = []\n",
    "for region in regions:  #   ~3 minutes\n",
    "    get_html_and_scrap(region)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### __Let's continue:__"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Create Dataframe:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "scrap_region_df = pd.DataFrame(champs_with_region, columns=[\"champion\",\"region\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "scrap_region_df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Create an function to organize dataframes:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "def drop_and_sort_rows(dframe):\n",
    "    dframe.drop_duplicates(subset=\"champion\", keep=\"first\", inplace=True)\n",
    "    dframe.sort_values(by=\"champion\",ignore_index=True, inplace=True)\n",
    "    return"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Organize and apply name correction:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "scrap_region_df.champion.replace(\"’\",\"'\", regex=True, inplace=True)\n",
    "drop_and_sort_rows(scrap_region_df)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Create template with organized champions:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "main_champions = pd.DataFrame({\"champion\":lol_df.champion.unique()})\n",
    "drop_and_sort_rows(main_champions)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Merge data to region Dataframe:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "full_region_df = main_champions.merge(scrap_region_df, how=\"left\", sort=True)\n",
    "drop_and_sort_rows(full_region_df)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Add Runeterra for champions without region:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "full_region_df.region.loc[full_region_df.region.isnull()] = \"Runeterra\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Merge and organize dataframes:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "lol_df = lol_df.merge(full_region_df, sort=True)\n",
    "\n",
    "lol_info_df = lol_df.drop([\"voice_line\", \"is_spoken\"], axis=1)\n",
    "drop_and_sort_rows(lol_info_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Voices Dataframe:\n",
    "voice_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Complete Dataframe with voices:\n",
    "lol_df.to_csv('files/lol_df.csv')\n",
    "lol_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Complete dataframe without voices:\n",
    "lol_info_df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.8.10 ('venv': venv)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "4dcd7147f286eef54aced7eba66d3ffe2df35a9c088b12a18e3e58b25dfa18cc"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}