02.Steam_Data_Cleaning.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "e8518882",
   "metadata": {},
   "source": [
    "# Steam Data Cleaning\n",
    "In this part of the project I will be cleaning the dataset for better visualizations on the data.\n",
    "Data pre-processing is performed using the pandas library --> and then it is again changed into sql database and deployed on heroku using docker images."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "323a3391",
   "metadata": {},
   "source": [
    "### Import the necessary libraries "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "4411a862",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import json\n",
    "from ast import literal_eval"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "3b03333b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>type</th>\n",
       "      <th>name</th>\n",
       "      <th>steam_appid</th>\n",
       "      <th>required_age</th>\n",
       "      <th>is_free</th>\n",
       "      <th>detailed_description</th>\n",
       "      <th>about_the_game</th>\n",
       "      <th>short_description</th>\n",
       "      <th>supported_languages</th>\n",
       "      <th>header_image</th>\n",
       "      <th>...</th>\n",
       "      <th>total_reviews</th>\n",
       "      <th>controller_support</th>\n",
       "      <th>dlc</th>\n",
       "      <th>demos</th>\n",
       "      <th>recommendations</th>\n",
       "      <th>achievements</th>\n",
       "      <th>reviews</th>\n",
       "      <th>ext_user_account_notice</th>\n",
       "      <th>metacritic</th>\n",
       "      <th>drm_notice</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1198490</th>\n",
       "      <td>game</td>\n",
       "      <td>All That Remains: Part 1</td>\n",
       "      <td>1198490</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>“Duncan Price is paranoid” they used to say. H...</td>\n",
       "      <td>“Duncan Price is paranoid” they used to say. H...</td>\n",
       "      <td>“Duncan Price is paranoid” they used to say. H...</td>\n",
       "      <td>English&lt;strong&gt;*&lt;/strong&gt;, French, Italian, Ge...</td>\n",
       "      <td>https://cdn.akamai.steamstatic.com/steam/apps/...</td>\n",
       "      <td>...</td>\n",
       "      <td>4</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1198510</th>\n",
       "      <td>game</td>\n",
       "      <td>Paperball</td>\n",
       "      <td>1198510</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>It's time to get rolling!&lt;br&gt;&lt;br&gt;Navigate tric...</td>\n",
       "      <td>It's time to get rolling!&lt;br&gt;&lt;br&gt;Navigate tric...</td>\n",
       "      <td>It's time to get rolling! Navigate tricky pass...</td>\n",
       "      <td>English&lt;strong&gt;*&lt;/strong&gt;, German, Japanese, D...</td>\n",
       "      <td>https://cdn.akamai.steamstatic.com/steam/apps/...</td>\n",
       "      <td>...</td>\n",
       "      <td>134</td>\n",
       "      <td>full</td>\n",
       "      <td>[1209620, 1209621, 1285150]</td>\n",
       "      <td>[{'appid': 1219680, 'description': ''}]</td>\n",
       "      <td>{'total': 143}</td>\n",
       "      <td>{'total': 37, 'highlighted': [{'name': 'Ready ...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1198550</th>\n",
       "      <td>game</td>\n",
       "      <td>Moderium</td>\n",
       "      <td>1198550</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>Moderium is a Wave-Based Isometric Action Role...</td>\n",
       "      <td>Moderium is a Wave-Based Isometric Action Role...</td>\n",
       "      <td>Conquer waves of encroaching enemies in this A...</td>\n",
       "      <td>English&lt;strong&gt;*&lt;/strong&gt;&lt;br&gt;&lt;strong&gt;*&lt;/strong...</td>\n",
       "      <td>https://cdn.akamai.steamstatic.com/steam/apps/...</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[{'appid': 1220740, 'description': ''}]</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1198600</th>\n",
       "      <td>game</td>\n",
       "      <td>Pizza Time Explosion</td>\n",
       "      <td>1198600</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>&lt;img src=\"https://cdn.akamai.steamstatic.com/s...</td>\n",
       "      <td>&lt;img src=\"https://cdn.akamai.steamstatic.com/s...</td>\n",
       "      <td>Welcome to the Pizza Dimension!</td>\n",
       "      <td>English</td>\n",
       "      <td>https://cdn.akamai.steamstatic.com/steam/apps/...</td>\n",
       "      <td>...</td>\n",
       "      <td>18</td>\n",
       "      <td>full</td>\n",
       "      <td>[1220660]</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{'total': 32, 'highlighted': [{'name': 'Pizza ...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1198630</th>\n",
       "      <td>game</td>\n",
       "      <td>Customer Cums First!</td>\n",
       "      <td>1198630</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>&lt;img src=\"https://cdn.akamai.steamstatic.com/s...</td>\n",
       "      <td>&lt;img src=\"https://cdn.akamai.steamstatic.com/s...</td>\n",
       "      <td>It's just like any other shopping district in ...</td>\n",
       "      <td>English</td>\n",
       "      <td>https://cdn.akamai.steamstatic.com/steam/apps/...</td>\n",
       "      <td>...</td>\n",
       "      <td>10</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{'total': 18, 'highlighted': [{'name': 'Now th...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 44 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         type                      name steam_appid required_age is_free  \\\n",
       "1198490  game  All That Remains: Part 1     1198490            0   False   \n",
       "1198510  game                 Paperball     1198510            0   False   \n",
       "1198550  game                  Moderium     1198550            0   False   \n",
       "1198600  game      Pizza Time Explosion     1198600            0   False   \n",
       "1198630  game      Customer Cums First!     1198630            0   False   \n",
       "\n",
       "                                      detailed_description  \\\n",
       "1198490  “Duncan Price is paranoid” they used to say. H...   \n",
       "1198510  It's time to get rolling!<br><br>Navigate tric...   \n",
       "1198550  Moderium is a Wave-Based Isometric Action Role...   \n",
       "1198600  <img src=\"https://cdn.akamai.steamstatic.com/s...   \n",
       "1198630  <img src=\"https://cdn.akamai.steamstatic.com/s...   \n",
       "\n",
       "                                            about_the_game  \\\n",
       "1198490  “Duncan Price is paranoid” they used to say. H...   \n",
       "1198510  It's time to get rolling!<br><br>Navigate tric...   \n",
       "1198550  Moderium is a Wave-Based Isometric Action Role...   \n",
       "1198600  <img src=\"https://cdn.akamai.steamstatic.com/s...   \n",
       "1198630  <img src=\"https://cdn.akamai.steamstatic.com/s...   \n",
       "\n",
       "                                         short_description  \\\n",
       "1198490  “Duncan Price is paranoid” they used to say. H...   \n",
       "1198510  It's time to get rolling! Navigate tricky pass...   \n",
       "1198550  Conquer waves of encroaching enemies in this A...   \n",
       "1198600                    Welcome to the Pizza Dimension!   \n",
       "1198630  It's just like any other shopping district in ...   \n",
       "\n",
       "                                       supported_languages  \\\n",
       "1198490  English<strong>*</strong>, French, Italian, Ge...   \n",
       "1198510  English<strong>*</strong>, German, Japanese, D...   \n",
       "1198550  English<strong>*</strong><br><strong>*</strong...   \n",
       "1198600                                            English   \n",
       "1198630                                            English   \n",
       "\n",
       "                                              header_image  ... total_reviews  \\\n",
       "1198490  https://cdn.akamai.steamstatic.com/steam/apps/...  ...             4   \n",
       "1198510  https://cdn.akamai.steamstatic.com/steam/apps/...  ...           134   \n",
       "1198550  https://cdn.akamai.steamstatic.com/steam/apps/...  ...             1   \n",
       "1198600  https://cdn.akamai.steamstatic.com/steam/apps/...  ...            18   \n",
       "1198630  https://cdn.akamai.steamstatic.com/steam/apps/...  ...            10   \n",
       "\n",
       "        controller_support                          dlc  \\\n",
       "1198490                NaN                          NaN   \n",
       "1198510               full  [1209620, 1209621, 1285150]   \n",
       "1198550                NaN                          NaN   \n",
       "1198600               full                    [1220660]   \n",
       "1198630                NaN                          NaN   \n",
       "\n",
       "                                           demos recommendations  \\\n",
       "1198490                                      NaN             NaN   \n",
       "1198510  [{'appid': 1219680, 'description': ''}]  {'total': 143}   \n",
       "1198550  [{'appid': 1220740, 'description': ''}]             NaN   \n",
       "1198600                                      NaN             NaN   \n",
       "1198630                                      NaN             NaN   \n",
       "\n",
       "                                              achievements reviews  \\\n",
       "1198490                                                NaN     NaN   \n",
       "1198510  {'total': 37, 'highlighted': [{'name': 'Ready ...     NaN   \n",
       "1198550                                                NaN     NaN   \n",
       "1198600  {'total': 32, 'highlighted': [{'name': 'Pizza ...     NaN   \n",
       "1198630  {'total': 18, 'highlighted': [{'name': 'Now th...     NaN   \n",
       "\n",
       "        ext_user_account_notice metacritic drm_notice  \n",
       "1198490                     NaN        NaN        NaN  \n",
       "1198510                     NaN        NaN        NaN  \n",
       "1198550                     NaN        NaN        NaN  \n",
       "1198600                     NaN        NaN        NaN  \n",
       "1198630                     NaN        NaN        NaN  \n",
       "\n",
       "[5 rows x 44 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# open JSON\n",
    "with open(\"Dataset/database.json\") as f:\n",
    "    data = json.load(f)\n",
    "    \n",
    "# convert the JSON to a dataframe\n",
    "df = pd.DataFrame(data)\n",
    "df1 = df.T # Take the transpose of the dataframe.\n",
    "df1.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "2e72b360",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read it into a csv file.\n",
    "df1.to_csv('steam.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "a44628b1",
   "metadata": {},
   "outputs": [],
   "source": [
    "steam = pd.read_csv(\"steam.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "226c7f7c",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Unnamed: 0                    0\n",
       "type                          0\n",
       "name                          0\n",
       "steam_appid                   0\n",
       "required_age                  0\n",
       "is_free                       0\n",
       "detailed_description          1\n",
       "about_the_game                1\n",
       "short_description             2\n",
       "supported_languages           0\n",
       "header_image                  0\n",
       "website                    1673\n",
       "pc_requirements               0\n",
       "mac_requirements              0\n",
       "linux_requirements            0\n",
       "legal_notice               3056\n",
       "developers                    3\n",
       "publishers                    0\n",
       "price_overview              372\n",
       "packages                    336\n",
       "package_groups                0\n",
       "platforms                     0\n",
       "categories                   69\n",
       "genres                        3\n",
       "screenshots                   2\n",
       "movies                       79\n",
       "release_date                  0\n",
       "support_info                  0\n",
       "background                    2\n",
       "content_descriptors           0\n",
       "num_reviews                   0\n",
       "review_score                  0\n",
       "review_score_desc             0\n",
       "total_positive                0\n",
       "total_negative                0\n",
       "total_reviews                 0\n",
       "controller_support         3462\n",
       "dlc                        3658\n",
       "demos                      4231\n",
       "recommendations            3491\n",
       "achievements               1753\n",
       "reviews                    3742\n",
       "ext_user_account_notice    4561\n",
       "metacritic                 4347\n",
       "drm_notice                 4610\n",
       "dtype: int64"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# To check the number of null values in each column. \n",
    "null_count = steam.isna().sum()\n",
    "null_count"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "decd9b0c",
   "metadata": {},
   "source": [
    "## Data Pre-Processing\n",
    "### Drop columns with null values."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "781724a4",
   "metadata": {},
   "source": [
    "These columns contain more than 50% null values so we will drop them --> `'controller_support', 'dlc', 'demos', 'recommendations', 'reviews', 'ext_user_account_notice', 'metacritic', 'drm_notice'` and `'legal_notice'`"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ca7696d0",
   "metadata": {},
   "source": [
    "The first function will remove the columns with more than 50% missing values, taking care of the columns with high null counts. We can do this by running a filter on the dataframe, as seen below."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "20193ffa",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Columns to drop: ['legal_notice', 'controller_support', 'dlc', 'demos', 'recommendations', 'reviews', 'ext_user_account_notice', 'metacritic', 'drm_notice']\n"
     ]
    }
   ],
   "source": [
    "threshold = steam.shape[0] // 2\n",
    "\n",
    "drop_col = steam.columns[null_count > threshold]\n",
    "\n",
    "print('Columns to drop: {}'.format(list(drop_col)))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "757dd1cc",
   "metadata": {},
   "outputs": [],
   "source": [
    "steam.drop(columns=drop_col, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "9fdc0699",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Also drop the Unnamed: 0 column which is the appid\n",
    "steam.drop(columns = ['Unnamed: 0'], inplace=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "298b81ab",
   "metadata": {},
   "source": [
    "### Delete the duplicate rows if they have same `steam_appid`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "f12c1895",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Duplicate rows to remove: 3\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>type</th>\n",
       "      <th>name</th>\n",
       "      <th>steam_appid</th>\n",
       "      <th>required_age</th>\n",
       "      <th>is_free</th>\n",
       "      <th>detailed_description</th>\n",
       "      <th>about_the_game</th>\n",
       "      <th>short_description</th>\n",
       "      <th>supported_languages</th>\n",
       "      <th>header_image</th>\n",
       "      <th>...</th>\n",
       "      <th>support_info</th>\n",
       "      <th>background</th>\n",
       "      <th>content_descriptors</th>\n",
       "      <th>num_reviews</th>\n",
       "      <th>review_score</th>\n",
       "      <th>review_score_desc</th>\n",
       "      <th>total_positive</th>\n",
       "      <th>total_negative</th>\n",
       "      <th>total_reviews</th>\n",
       "      <th>achievements</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1864</th>\n",
       "      <td>game</td>\n",
       "      <td>Tom Clancy's Rainbow Six® Siege</td>\n",
       "      <td>359550</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>&lt;h1&gt;Edition Comparison&lt;/h1&gt;&lt;p&gt;&lt;img src=\"https:...</td>\n",
       "      <td>Master the art of destruction and gadgetry in ...</td>\n",
       "      <td>Tom Clancy's Rainbow Six Siege is the latest i...</td>\n",
       "      <td>English&lt;strong&gt;*&lt;/strong&gt;, French&lt;strong&gt;*&lt;/st...</td>\n",
       "      <td>https://cdn.akamai.steamstatic.com/steam/apps/...</td>\n",
       "      <td>...</td>\n",
       "      <td>{'url': 'http://support.ubi.com', 'email': ''}</td>\n",
       "      <td>https://cdn.akamai.steamstatic.com/steam/apps/...</td>\n",
       "      <td>{'ids': [], 'notes': None}</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>2 user reviews</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1865</th>\n",
       "      <td>game</td>\n",
       "      <td>Tom Clancy's Rainbow Six® Siege</td>\n",
       "      <td>359550</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>&lt;h1&gt;Edition Comparison&lt;/h1&gt;&lt;p&gt;&lt;img src=\"https:...</td>\n",
       "      <td>Master the art of destruction and gadgetry in ...</td>\n",
       "      <td>Tom Clancy's Rainbow Six Siege is the latest i...</td>\n",
       "      <td>English&lt;strong&gt;*&lt;/strong&gt;, French&lt;strong&gt;*&lt;/st...</td>\n",
       "      <td>https://cdn.akamai.steamstatic.com/steam/apps/...</td>\n",
       "      <td>...</td>\n",
       "      <td>{'url': 'http://support.ubi.com', 'email': ''}</td>\n",
       "      <td>https://cdn.akamai.steamstatic.com/steam/apps/...</td>\n",
       "      <td>{'ids': [], 'notes': None}</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>2 user reviews</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1866</th>\n",
       "      <td>game</td>\n",
       "      <td>Tom Clancy's Rainbow Six® Siege</td>\n",
       "      <td>359550</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>&lt;h1&gt;Edition Comparison&lt;/h1&gt;&lt;p&gt;&lt;img src=\"https:...</td>\n",
       "      <td>Master the art of destruction and gadgetry in ...</td>\n",
       "      <td>Tom Clancy's Rainbow Six Siege is the latest i...</td>\n",
       "      <td>English&lt;strong&gt;*&lt;/strong&gt;, French&lt;strong&gt;*&lt;/st...</td>\n",
       "      <td>https://cdn.akamai.steamstatic.com/steam/apps/...</td>\n",
       "      <td>...</td>\n",
       "      <td>{'url': 'http://support.ubi.com', 'email': ''}</td>\n",
       "      <td>https://cdn.akamai.steamstatic.com/steam/apps/...</td>\n",
       "      <td>{'ids': [], 'notes': None}</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>2 user reviews</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3 rows × 35 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      type                             name  steam_appid  required_age  \\\n",
       "1864  game  Tom Clancy's Rainbow Six® Siege       359550             0   \n",
       "1865  game  Tom Clancy's Rainbow Six® Siege       359550             0   \n",
       "1866  game  Tom Clancy's Rainbow Six® Siege       359550             0   \n",
       "\n",
       "      is_free                               detailed_description  \\\n",
       "1864    False  <h1>Edition Comparison</h1><p><img src=\"https:...   \n",
       "1865    False  <h1>Edition Comparison</h1><p><img src=\"https:...   \n",
       "1866    False  <h1>Edition Comparison</h1><p><img src=\"https:...   \n",
       "\n",
       "                                         about_the_game  \\\n",
       "1864  Master the art of destruction and gadgetry in ...   \n",
       "1865  Master the art of destruction and gadgetry in ...   \n",
       "1866  Master the art of destruction and gadgetry in ...   \n",
       "\n",
       "                                      short_description  \\\n",
       "1864  Tom Clancy's Rainbow Six Siege is the latest i...   \n",
       "1865  Tom Clancy's Rainbow Six Siege is the latest i...   \n",
       "1866  Tom Clancy's Rainbow Six Siege is the latest i...   \n",
       "\n",
       "                                    supported_languages  \\\n",
       "1864  English<strong>*</strong>, French<strong>*</st...   \n",
       "1865  English<strong>*</strong>, French<strong>*</st...   \n",
       "1866  English<strong>*</strong>, French<strong>*</st...   \n",
       "\n",
       "                                           header_image  ...  \\\n",
       "1864  https://cdn.akamai.steamstatic.com/steam/apps/...  ...   \n",
       "1865  https://cdn.akamai.steamstatic.com/steam/apps/...  ...   \n",
       "1866  https://cdn.akamai.steamstatic.com/steam/apps/...  ...   \n",
       "\n",
       "                                        support_info  \\\n",
       "1864  {'url': 'http://support.ubi.com', 'email': ''}   \n",
       "1865  {'url': 'http://support.ubi.com', 'email': ''}   \n",
       "1866  {'url': 'http://support.ubi.com', 'email': ''}   \n",
       "\n",
       "                                             background  \\\n",
       "1864  https://cdn.akamai.steamstatic.com/steam/apps/...   \n",
       "1865  https://cdn.akamai.steamstatic.com/steam/apps/...   \n",
       "1866  https://cdn.akamai.steamstatic.com/steam/apps/...   \n",
       "\n",
       "             content_descriptors num_reviews review_score review_score_desc  \\\n",
       "1864  {'ids': [], 'notes': None}           2            0    2 user reviews   \n",
       "1865  {'ids': [], 'notes': None}           2            0    2 user reviews   \n",
       "1866  {'ids': [], 'notes': None}           2            0    2 user reviews   \n",
       "\n",
       "     total_positive total_negative total_reviews achievements  \n",
       "1864              1              1             2          NaN  \n",
       "1865              2              0             2          NaN  \n",
       "1866              1              1             2          NaN  \n",
       "\n",
       "[3 rows x 35 columns]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Delete the duplicate rows in the dataset where each game has a unique steam_appid.\n",
    "duplicate_rows = steam[steam.duplicated(subset='steam_appid')] \n",
    "\n",
    "print('Duplicate rows to remove:', duplicate_rows.shape[0])\n",
    "\n",
    "duplicate_rows.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "bcb639ac",
   "metadata": {},
   "outputs": [],
   "source": [
    "steam.drop_duplicates(subset =\"steam_appid\",keep = 'first', inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "1baf8d61",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['type', 'name', 'steam_appid', 'required_age', 'is_free',\n",
       "       'detailed_description', 'about_the_game', 'short_description',\n",
       "       'supported_languages', 'header_image', 'website', 'pc_requirements',\n",
       "       'mac_requirements', 'linux_requirements', 'developers', 'publishers',\n",
       "       'price_overview', 'packages', 'package_groups', 'platforms',\n",
       "       'categories', 'genres', 'screenshots', 'movies', 'release_date',\n",
       "       'support_info', 'background', 'content_descriptors', 'num_reviews',\n",
       "       'review_score', 'review_score_desc', 'total_positive', 'total_negative',\n",
       "       'total_reviews', 'achievements'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "steam.columns"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9ededa9e",
   "metadata": {},
   "source": [
    "### Processing the `price_overview` column to extract the price of each game.\n",
    "\n",
    "The column `is_free` and `price_overview` are linked with respect to the price of each game. \n",
    "If `is_free` is true and `price_overview` has a null value then it means that the price of that game is €0.0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "76cae49b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "372"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# check how many null values are in price_overview\n",
    "steam['price_overview'].isnull().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "585c196b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "276"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "price_zero = steam[(steam['is_free']) & (steam['price_overview'].isnull())]\n",
    "len(price_zero)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "baaa0c04",
   "metadata": {},
   "source": [
    "This shows that there are 276 games have a price of €0.0\n",
    "\n",
    "Let's set the final price value for these columns as 0.0\n",
    "\n",
    "Also there are 96 games whose price value is not stated and it states that these games are not free.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "3c1413e8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"{'currency': 'EUR', 'initial': 239, 'final': 239, 'discount_percent': 0, 'initial_formatted': '', 'final_formatted': '2,39€'}\""
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "steam['price_overview'][0]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e2c86078",
   "metadata": {},
   "source": [
    "##### Lets create separate columns `Final_price` and `Discount` and extract from `price_overview`.\n",
    "We know that the currency of the price is in EURO."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "a7327c4f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>name</th>\n",
       "      <th>Discount</th>\n",
       "      <th>Price</th>\n",
       "      <th>Currency</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>All That Remains: Part 1</td>\n",
       "      <td>0</td>\n",
       "      <td>239</td>\n",
       "      <td>EUR</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Paperball</td>\n",
       "      <td>0</td>\n",
       "      <td>999</td>\n",
       "      <td>EUR</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Moderium</td>\n",
       "      <td>0</td>\n",
       "      <td>79</td>\n",
       "      <td>EUR</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Pizza Time Explosion</td>\n",
       "      <td>0</td>\n",
       "      <td>499</td>\n",
       "      <td>EUR</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Customer Cums First!</td>\n",
       "      <td>0</td>\n",
       "      <td>999</td>\n",
       "      <td>EUR</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                       name  Discount  Price Currency\n",
       "0  All That Remains: Part 1         0    239      EUR\n",
       "1                 Paperball         0    999      EUR\n",
       "2                  Moderium         0     79      EUR\n",
       "3      Pizza Time Explosion         0    499      EUR\n",
       "4      Customer Cums First!         0    999      EUR"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def process_price(steam):\n",
    "    steam = steam.copy()\n",
    "        \n",
    "    def parse_price(x):\n",
    "        if x is not np.nan:\n",
    "            return literal_eval(x)\n",
    "        else:\n",
    "            return {'discount_percent':0, 'initial': -1,'currency':'EUR'}\n",
    "    \n",
    "    # evaluate as dictionary and set to -1 if missing\n",
    "    steam['price_overview'] = steam['price_overview'].apply(parse_price)\n",
    "    \n",
    "    # Create columns from currency and initial values\n",
    "    steam['Discount'] = steam['price_overview'].apply(lambda x: x['discount_percent'])\n",
    "    steam['Price'] = steam['price_overview'].apply(lambda x: x['initial'])\n",
    "    steam['Currency'] = steam['price_overview'].apply(lambda x: x['currency'])\n",
    "    \n",
    "    # Set price of free games to 0\n",
    "    steam.loc[steam['is_free'], 'Price'] = 0\n",
    "    \n",
    "    return steam\n",
    "\n",
    "price_data = process_price(steam)[['name', 'Discount', 'Price','Currency']]\n",
    "price_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "f39428b7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>name</th>\n",
       "      <th>Discount</th>\n",
       "      <th>Price</th>\n",
       "      <th>Currency</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>95</th>\n",
       "      <td>Mokoko</td>\n",
       "      <td>0</td>\n",
       "      <td>499</td>\n",
       "      <td>USD</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>202</th>\n",
       "      <td>DEATH STRANDING</td>\n",
       "      <td>0</td>\n",
       "      <td>7999</td>\n",
       "      <td>CAD</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>428</th>\n",
       "      <td>DeadShotZ</td>\n",
       "      <td>0</td>\n",
       "      <td>199</td>\n",
       "      <td>USD</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>914</th>\n",
       "      <td>PANDARA</td>\n",
       "      <td>0</td>\n",
       "      <td>400</td>\n",
       "      <td>CHF</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>937</th>\n",
       "      <td>Sakura Fox Adventure</td>\n",
       "      <td>0</td>\n",
       "      <td>999</td>\n",
       "      <td>USD</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>948</th>\n",
       "      <td>Three Of a Fish</td>\n",
       "      <td>0</td>\n",
       "      <td>99</td>\n",
       "      <td>USD</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1023</th>\n",
       "      <td>Night Road</td>\n",
       "      <td>0</td>\n",
       "      <td>289</td>\n",
       "      <td>GBP</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1096</th>\n",
       "      <td>Prison Princess</td>\n",
       "      <td>0</td>\n",
       "      <td>1549</td>\n",
       "      <td>GBP</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1730</th>\n",
       "      <td>Battle Knights</td>\n",
       "      <td>0</td>\n",
       "      <td>299</td>\n",
       "      <td>USD</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1843</th>\n",
       "      <td>The Elder Scrolls V: Skyrim Special Edition</td>\n",
       "      <td>60</td>\n",
       "      <td>4616000</td>\n",
       "      <td>KRW</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1845</th>\n",
       "      <td>Edge of Twilight – Return To Glory</td>\n",
       "      <td>0</td>\n",
       "      <td>1499</td>\n",
       "      <td>USD</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2202</th>\n",
       "      <td>UNO</td>\n",
       "      <td>0</td>\n",
       "      <td>999</td>\n",
       "      <td>USD</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3554</th>\n",
       "      <td>Aseprite</td>\n",
       "      <td>0</td>\n",
       "      <td>7199</td>\n",
       "      <td>PLN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3739</th>\n",
       "      <td>Little Nightmares</td>\n",
       "      <td>0</td>\n",
       "      <td>9900</td>\n",
       "      <td>MYR</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4139</th>\n",
       "      <td>Snapshot VR</td>\n",
       "      <td>0</td>\n",
       "      <td>1999</td>\n",
       "      <td>USD</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                             name  Discount    Price Currency\n",
       "95                                         Mokoko         0      499      USD\n",
       "202                               DEATH STRANDING         0     7999      CAD\n",
       "428                                     DeadShotZ         0      199      USD\n",
       "914                                       PANDARA         0      400      CHF\n",
       "937                          Sakura Fox Adventure         0      999      USD\n",
       "948                               Three Of a Fish         0       99      USD\n",
       "1023                                   Night Road         0      289      GBP\n",
       "1096                              Prison Princess         0     1549      GBP\n",
       "1730                               Battle Knights         0      299      USD\n",
       "1843  The Elder Scrolls V: Skyrim Special Edition        60  4616000      KRW\n",
       "1845           Edge of Twilight – Return To Glory         0     1499      USD\n",
       "2202                                          UNO         0      999      USD\n",
       "3554                                     Aseprite         0     7199      PLN\n",
       "3739                            Little Nightmares         0     9900      MYR\n",
       "4139                                  Snapshot VR         0     1999      USD"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "price_data[price_data['Currency'] != 'EUR']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "567074f1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>name</th>\n",
       "      <th>price</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>All That Remains: Part 1</td>\n",
       "      <td>2.39</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Paperball</td>\n",
       "      <td>9.99</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Moderium</td>\n",
       "      <td>0.79</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Pizza Time Explosion</td>\n",
       "      <td>4.99</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Customer Cums First!</td>\n",
       "      <td>9.99</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4620</th>\n",
       "      <td>Lunch A Palooza</td>\n",
       "      <td>10.79</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4621</th>\n",
       "      <td>Uncharted Tides: Port Royal</td>\n",
       "      <td>14.99</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4622</th>\n",
       "      <td>Persona 4 Golden</td>\n",
       "      <td>19.99</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4623</th>\n",
       "      <td>Magic Farm 3: The Ice Danger</td>\n",
       "      <td>16.79</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4624</th>\n",
       "      <td>The Stranger: Interactive Film</td>\n",
       "      <td>6.59</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>4607 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                name  price\n",
       "0           All That Remains: Part 1   2.39\n",
       "1                          Paperball   9.99\n",
       "2                           Moderium   0.79\n",
       "3               Pizza Time Explosion   4.99\n",
       "4               Customer Cums First!   9.99\n",
       "...                              ...    ...\n",
       "4620                 Lunch A Palooza  10.79\n",
       "4621     Uncharted Tides: Port Royal  14.99\n",
       "4622                Persona 4 Golden  19.99\n",
       "4623    Magic Farm 3: The Ice Danger  16.79\n",
       "4624  The Stranger: Interactive Film   6.59\n",
       "\n",
       "[4607 rows x 2 columns]"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def process_price(steam):\n",
    "    \"\"\"Process price_overview column into formatted price column.\"\"\"\n",
    "    steam = steam.copy()\n",
    "    \n",
    "    def parse_price(x):\n",
    "        if x is not np.nan:\n",
    "            return literal_eval(x)\n",
    "        else:\n",
    "            return {'discount_percent':0, 'currency': 'EUR', 'initial': -1}\n",
    "    \n",
    "    # evaluate as dictionary and set to -1 if missing\n",
    "    steam['price_overview'] = steam['price_overview'].apply(parse_price)\n",
    "    \n",
    "    # create columns from currency and initial values\n",
    "    steam['discount'] = steam['price_overview'].apply(lambda x: x['discount_percent'])\n",
    "    steam['price'] = steam['price_overview'].apply(lambda x: x['initial'])\n",
    "    steam['currency'] = steam['price_overview'].apply(lambda x: x['currency'])\n",
    "    \n",
    "    # set price of free games to 0\n",
    "    steam.loc[steam['is_free'], 'price'] = 0\n",
    "    \n",
    "    # remove non-EUR rows\n",
    "    steam = steam[steam['currency'] == 'EUR']\n",
    "    \n",
    "    # change price to display in pounds (only applying to rows with a value greater than 0)\n",
    "    steam.loc[steam['price'] > 0, 'price'] /= 100\n",
    "    \n",
    "    # remove columns no longer needed\n",
    "    steam = steam.drop(['is_free', 'currency', 'price_overview'], axis=1)\n",
    "    \n",
    "    return steam\n",
    "\n",
    "\n",
    "price_steam = process_price(steam)\n",
    "price_steam[['name', 'price']]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c892553c",
   "metadata": {},
   "source": [
    "There are 15 games with different currency. Let's convert them to EURO with today's conversion rate and get the price."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "id": "1f65a2b8",
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"def price(price_data):\n",
    "    price_data.loc[price_data['Currency'] == 'USD', 'Currency'] = 'EUR'\n",
    "    price_data.loc[price_data['Currency']=='USD', 'Price'] *= 0.89\n",
    "        \n",
    "\n",
    "dfprice = price(price_data)\n",
    "dfprice   \"\"\""
   ]
  },
  {
   "cell_type": "markdown",
   "id": "66f91416",
   "metadata": {},
   "source": [
    "### Processing the `genres` column to extract the GENRE of each game.\n",
    "Let's have a look at one of the value of genre column."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "772021d7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"[{'id': '25', 'description': 'Adventure'}, {'id': '23', 'description': 'Indie'}]\""
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Let's have a look at one of the genres value\n",
    "steam['genres'][50]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "35af74b9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(4622, 35)"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "steam.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "de26bf82",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(4539, 35)"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "price_steam = price_steam.dropna(subset=['categories'])\n",
    "price_steam.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "80fc63a1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(4537, 35)"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "price_steam = price_steam.dropna(subset=['genres'])\n",
    "price_steam.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "75633fa0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>steam_appid</th>\n",
       "      <th>categories</th>\n",
       "      <th>genres</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1198490</td>\n",
       "      <td>Single-player;Steam Cloud</td>\n",
       "      <td>Adventure</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1198510</td>\n",
       "      <td>Single-player;Multi-player;PvP;Shared/Split Sc...</td>\n",
       "      <td>Action;Indie</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1198550</td>\n",
       "      <td>Single-player</td>\n",
       "      <td>Action;Casual;Indie;RPG</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1198600</td>\n",
       "      <td>Single-player;Steam Achievements;Full controll...</td>\n",
       "      <td>Action;Indie</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1198630</td>\n",
       "      <td>Single-player;Steam Achievements</td>\n",
       "      <td>Casual</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   steam_appid                                         categories  \\\n",
       "0      1198490                          Single-player;Steam Cloud   \n",
       "1      1198510  Single-player;Multi-player;PvP;Shared/Split Sc...   \n",
       "2      1198550                                      Single-player   \n",
       "3      1198600  Single-player;Steam Achievements;Full controll...   \n",
       "4      1198630                   Single-player;Steam Achievements   \n",
       "\n",
       "                    genres  \n",
       "0                Adventure  \n",
       "1             Action;Indie  \n",
       "2  Action;Casual;Indie;RPG  \n",
       "3             Action;Indie  \n",
       "4                   Casual  "
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def process_categories_and_genres(steam):\n",
    "    steam = steam.copy()\n",
    "    steam = steam[(steam['categories'].notnull()) & (steam['genres'].notnull())]\n",
    "    \n",
    "    for col in ['categories', 'genres']:\n",
    "        steam[col] = steam[col].apply(lambda x: ';'.join(item['description'] for item in literal_eval(x)))\n",
    "    \n",
    "    return steam\n",
    "\n",
    "cat_gen_steam = process_categories_and_genres(price_steam)\n",
    "cat_gen_steam[['steam_appid', 'categories', 'genres']].head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "9b435a1f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(4537, 35)"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cat_gen_steam.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ce444f28",
   "metadata": {},
   "source": [
    "### Processing `developers` and `publishers` column."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "ffd629e4",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Developers null counts: 3\n",
      "Developers empty list counts: 0\n",
      "\n",
      "Publishers null counts: 0\n",
      "Publishers empty list counts: 31\n"
     ]
    }
   ],
   "source": [
    "print('Developers null counts:', cat_gen_steam['developers'].isnull().sum())\n",
    "print('Developers empty list counts:', cat_gen_steam[cat_gen_steam['developers'] == \"['']\"].shape[0])\n",
    "\n",
    "print('\\nPublishers null counts:', cat_gen_steam['publishers'].isnull().sum())\n",
    "print('Publishers empty list counts:', cat_gen_steam[cat_gen_steam['publishers'] == \"['']\"].shape[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "563fb174",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"['Volrest']\""
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "steam['developers'][50]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "id": "12be0915",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>developers</th>\n",
       "      <th>publishers</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>['Turnfollow']</td>\n",
       "      <td>['Tender Claws']</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>['Flight School Studio']</td>\n",
       "      <td>['MWM Interactive']</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>['Elephant Games']</td>\n",
       "      <td>['Big Fish Games']</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>['Domini Games']</td>\n",
       "      <td>['Big Fish Games']</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                  developers           publishers\n",
       "24            ['Turnfollow']     ['Tender Claws']\n",
       "25  ['Flight School Studio']  ['MWM Interactive']\n",
       "26        ['Elephant Games']   ['Big Fish Games']\n",
       "27          ['Domini Games']   ['Big Fish Games']"
      ]
     },
     "execution_count": 125,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "steam[['developers', 'publishers']].iloc[24:28]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "1bc1e8a2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rows in developers column with multiple values: 296\n",
      "Rows in publishers column with multiple values: 148\n"
     ]
    }
   ],
   "source": [
    "def process_developers_and_publishers(steam):\n",
    "    # remove rows with missing data\n",
    "    steam = steam[(steam['developers'].notnull()) & (steam['publishers'] != \"['']\")].copy()\n",
    "    \n",
    "    for col in ['developers', 'publishers']:\n",
    "        steam[col] = steam[col].apply(lambda x: literal_eval(x))\n",
    "        \n",
    "        # filter dataframe to rows with lists longer than 1, and store the number of rows\n",
    "        num_rows = steam[steam[col].str.len() > 1].shape[0]\n",
    "        \n",
    "        print('Rows in {} column with multiple values:'.format(col), num_rows)\n",
    "\n",
    "process_developers_and_publishers(cat_gen_steam)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "id": "ea91d9cd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3"
      ]
     },
     "execution_count": 128,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "steam[(steam['publishers'] == \"['NA']\") | (steam['developers'] == \"['N/A']\")].shape[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "c3cf5c5e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>name</th>\n",
       "      <th>steam_appid</th>\n",
       "      <th>developer</th>\n",
       "      <th>publisher</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>All That Remains: Part 1</td>\n",
       "      <td>1198490</td>\n",
       "      <td>Glitch Games</td>\n",
       "      <td>Glitch Games</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Paperball</td>\n",
       "      <td>1198510</td>\n",
       "      <td>Cliax Games</td>\n",
       "      <td>Cliax Games</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Moderium</td>\n",
       "      <td>1198550</td>\n",
       "      <td>All Saints Gaming</td>\n",
       "      <td>All Saints Gaming</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Pizza Time Explosion</td>\n",
       "      <td>1198600</td>\n",
       "      <td>Electric Prune Juice</td>\n",
       "      <td>Electric Prune Juice</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Customer Cums First!</td>\n",
       "      <td>1198630</td>\n",
       "      <td>Miel</td>\n",
       "      <td>Cherry Kiss Games</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                       name  steam_appid             developer  \\\n",
       "0  All That Remains: Part 1      1198490          Glitch Games   \n",
       "1                 Paperball      1198510           Cliax Games   \n",
       "2                  Moderium      1198550     All Saints Gaming   \n",
       "3      Pizza Time Explosion      1198600  Electric Prune Juice   \n",
       "4      Customer Cums First!      1198630                  Miel   \n",
       "\n",
       "              publisher  \n",
       "0          Glitch Games  \n",
       "1           Cliax Games  \n",
       "2     All Saints Gaming  \n",
       "3  Electric Prune Juice  \n",
       "4     Cherry Kiss Games  "
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def process_developers_and_publishers(steam):\n",
    "    \"\"\"Parse columns as semicolon-separated string.\"\"\"\n",
    "    # remove rows with missing data (~ means not)\n",
    "    steam = steam[(steam['developers'].notnull()) & (steam['publishers'] != \"['']\")].copy()\n",
    "    steam = steam[~(steam['developers'].str.contains(';')) & ~(steam['publishers'].str.contains(';'))]\n",
    "    steam = steam[(steam['publishers'] != \"['NA']\") & (steam['publishers'] != \"['N/A']\")]\n",
    "    \n",
    "    # create list for each\n",
    "    steam['developer'] = steam['developers'].apply(lambda x: ';'.join(literal_eval(x)))\n",
    "    steam['publisher'] = steam['publishers'].apply(lambda x: ';'.join(literal_eval(x)))\n",
    "\n",
    "    steam = steam.drop(['developers', 'publishers'], axis=1)\n",
    "    \n",
    "    return steam\n",
    "\n",
    "dev_pub_steam = process_developers_and_publishers(cat_gen_steam)\n",
    "dev_pub_steam[['name', 'steam_appid', 'developer', 'publisher']].head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "ec6980b8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(4500, 35)"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dev_pub_steam.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "a7df7038",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0     1713\n",
       "10     139\n",
       "12     102\n",
       "8       93\n",
       "13      87\n",
       "Name: achievements, dtype: int64"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def process_achievements(steam):\n",
    "    \"\"\"Parse as total number of achievements.\"\"\"\n",
    "    steam = steam.copy()\n",
    "    \n",
    "    steam = steam.drop('content_descriptors', axis=1)\n",
    "    \n",
    "    def parse_achievements(x):\n",
    "        if x is np.nan:\n",
    "            # missing data, assume has no achievements\n",
    "            return 0\n",
    "        else:\n",
    "            # else has data, so can extract and return number under total\n",
    "            return literal_eval(x)['total']\n",
    "        \n",
    "    steam['achievements'] = steam['achievements'].apply(parse_achievements)\n",
    "    \n",
    "    return steam\n",
    "\n",
    "achiev_steam = process_achievements(dev_pub_steam)\n",
    "achiev_steam['achievements'].value_counts().head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a27f798f",
   "metadata": {},
   "source": [
    "### Processing the `platforms` column."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "b71acad6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'windows': True, 'mac': False, 'linux': False}    3223\n",
       "{'windows': True, 'mac': True, 'linux': True}       694\n",
       "{'windows': True, 'mac': True, 'linux': False}      562\n",
       "{'windows': True, 'mac': False, 'linux': True}      143\n",
       "Name: platforms, dtype: int64"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "steam['platforms'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "0ac72b44",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "steam['platforms'].isnull().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "99ec677b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "windows              3131\n",
       "windows;mac;linux     679\n",
       "windows;mac           550\n",
       "windows;linux         140\n",
       "Name: platforms, dtype: int64"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def process_platforms(steam):\n",
    "    \"\"\"Split platforms column into separate boolean columns for each platform.\"\"\"\n",
    "    # evaluate values in platforms column, so can index into dictionaries\n",
    "    steam = steam.copy()\n",
    "    \n",
    "    def parse_platforms(x):\n",
    "        \n",
    "        d = literal_eval(x)\n",
    "        \n",
    "        return ';'.join(platform for platform in d.keys() if d[platform])\n",
    "    \n",
    "    steam['platforms'] = steam['platforms'].apply(parse_platforms)\n",
    "    \n",
    "    return steam\n",
    "\n",
    "\n",
    "steam1 = process_platforms(achiev_steam)\n",
    "steam1['platforms'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "3e6639aa",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(4500, 34)"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "steam1.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "11fd4980",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "type                       0\n",
       "name                       0\n",
       "steam_appid                0\n",
       "required_age               0\n",
       "detailed_description       1\n",
       "about_the_game             1\n",
       "short_description          2\n",
       "supported_languages        0\n",
       "header_image               0\n",
       "website                 1638\n",
       "pc_requirements            0\n",
       "mac_requirements           0\n",
       "linux_requirements         0\n",
       "packages                 315\n",
       "package_groups             0\n",
       "platforms                  0\n",
       "categories                 0\n",
       "genres                     0\n",
       "screenshots                2\n",
       "movies                    76\n",
       "release_date               0\n",
       "support_info               0\n",
       "background                 2\n",
       "num_reviews                0\n",
       "review_score               0\n",
       "review_score_desc          0\n",
       "total_positive             0\n",
       "total_negative             0\n",
       "total_reviews              0\n",
       "achievements               0\n",
       "discount                   0\n",
       "price                      0\n",
       "developer                  0\n",
       "publisher                  0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "steam1.isnull().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "ad81adfc",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "detailed_description    1\n",
       "about_the_game          1\n",
       "short_description       2\n",
       "dtype: int64"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "steam1[['detailed_description', 'about_the_game', 'short_description']].isnull().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "id": "6a515749",
   "metadata": {},
   "outputs": [],
   "source": [
    "def export_data(df, filename):\n",
    "    \"\"\"Export dataframe to csv file, filename prepended with 'steam_'.\n",
    "    \n",
    "    filename : str without file extension\n",
    "    \"\"\"\n",
    "    filepath = 'steam_' + filename + '.csv'\n",
    "    \n",
    "    df.to_csv(filepath, index=False)\n",
    "    \n",
    "    print_name = filename.replace('_', ' ')\n",
    "    print(\"Exported {} to '{}'\".format(print_name, filepath))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "c436aa06",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Exported description data to 'steam_description_data.csv'\n"
     ]
    }
   ],
   "source": [
    "def process_descriptions(steam, export=False):\n",
    "    \"\"\"Export descriptions to external csv file then remove these columns.\"\"\"\n",
    "    # remove rows with missing description data\n",
    "    steam = steam[steam['detailed_description'].notnull()].copy()\n",
    "    \n",
    "    # remove rows with unusually small description\n",
    "    steam = steam[steam['detailed_description'].str.len() > 20]\n",
    "    \n",
    "    # by default we don't export, useful if calling function later\n",
    "    if export:\n",
    "        # create dataframe of description columns\n",
    "        description_data = steam[['steam_appid', 'detailed_description', 'about_the_game', 'short_description']]\n",
    "        \n",
    "        export_data(description_data, filename='description_data')\n",
    "    \n",
    "    # drop description columns from main dataframe\n",
    "    steam = steam.drop(['detailed_description', 'about_the_game', 'short_description'], axis=1)\n",
    "    \n",
    "    return steam\n",
    "\n",
    "desc_steam = process_descriptions(steam1, export=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "id": "9eeee6ee",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(4500, 34)"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "steam1.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "id": "0bc8b30d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "header_image: 0\n",
      "screenshots: 1\n",
      "background: 1\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>header_image</th>\n",
       "      <th>screenshots</th>\n",
       "      <th>background</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>https://cdn.akamai.steamstatic.com/steam/apps/...</td>\n",
       "      <td>[{'id': 0, 'path_thumbnail': 'https://cdn.akam...</td>\n",
       "      <td>https://cdn.akamai.steamstatic.com/steam/apps/...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>https://cdn.akamai.steamstatic.com/steam/apps/...</td>\n",
       "      <td>[{'id': 0, 'path_thumbnail': 'https://cdn.akam...</td>\n",
       "      <td>https://cdn.akamai.steamstatic.com/steam/apps/...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>https://cdn.akamai.steamstatic.com/steam/apps/...</td>\n",
       "      <td>[{'id': 0, 'path_thumbnail': 'https://cdn.akam...</td>\n",
       "      <td>https://cdn.akamai.steamstatic.com/steam/apps/...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>https://cdn.akamai.steamstatic.com/steam/apps/...</td>\n",
       "      <td>[{'id': 0, 'path_thumbnail': 'https://cdn.akam...</td>\n",
       "      <td>https://cdn.akamai.steamstatic.com/steam/apps/...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>https://cdn.akamai.steamstatic.com/steam/apps/...</td>\n",
       "      <td>[{'id': 0, 'path_thumbnail': 'https://cdn.akam...</td>\n",
       "      <td>https://cdn.akamai.steamstatic.com/steam/apps/...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                        header_image  \\\n",
       "0  https://cdn.akamai.steamstatic.com/steam/apps/...   \n",
       "1  https://cdn.akamai.steamstatic.com/steam/apps/...   \n",
       "2  https://cdn.akamai.steamstatic.com/steam/apps/...   \n",
       "3  https://cdn.akamai.steamstatic.com/steam/apps/...   \n",
       "4  https://cdn.akamai.steamstatic.com/steam/apps/...   \n",
       "\n",
       "                                         screenshots  \\\n",
       "0  [{'id': 0, 'path_thumbnail': 'https://cdn.akam...   \n",
       "1  [{'id': 0, 'path_thumbnail': 'https://cdn.akam...   \n",
       "2  [{'id': 0, 'path_thumbnail': 'https://cdn.akam...   \n",
       "3  [{'id': 0, 'path_thumbnail': 'https://cdn.akam...   \n",
       "4  [{'id': 0, 'path_thumbnail': 'https://cdn.akam...   \n",
       "\n",
       "                                          background  \n",
       "0  https://cdn.akamai.steamstatic.com/steam/apps/...  \n",
       "1  https://cdn.akamai.steamstatic.com/steam/apps/...  \n",
       "2  https://cdn.akamai.steamstatic.com/steam/apps/...  \n",
       "3  https://cdn.akamai.steamstatic.com/steam/apps/...  \n",
       "4  https://cdn.akamai.steamstatic.com/steam/apps/...  "
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "image_cols = ['header_image', 'screenshots', 'background']\n",
    "\n",
    "for col in image_cols:\n",
    "    print(col+':', desc_steam[col].isnull().sum())\n",
    "\n",
    "desc_steam[image_cols].head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "id": "b07183bf",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Exported media data to 'steam_media_data.csv'\n"
     ]
    }
   ],
   "source": [
    "def process_media(steam, export=False):\n",
    "    \"\"\"Remove media columns from dataframe, optionally exporting them to csv first.\"\"\"\n",
    "    steam = steam[steam['screenshots'].notnull()].copy()\n",
    "    \n",
    "    if export:\n",
    "        media_data = steam[['steam_appid', 'header_image', 'screenshots', 'background', 'movies']]\n",
    "        \n",
    "        export_data(media_data, 'media_data')\n",
    "        \n",
    "    steam = steam.drop(['header_image', 'screenshots', 'background', 'movies'], axis=1)\n",
    "    \n",
    "    return steam\n",
    "\n",
    "media_steam = process_media(desc_steam, export=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "id": "27c11f20",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "website null counts: 1637\n",
      "support_info null counts: 0\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>name</th>\n",
       "      <th>website</th>\n",
       "      <th>support_info</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>75</th>\n",
       "      <td>The Pale City</td>\n",
       "      <td>https://thepalecity.com/</td>\n",
       "      <td>{'url': 'https://thepalecity.com/contact/', 'email': ''}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>76</th>\n",
       "      <td>Nexomon: Extinction</td>\n",
       "      <td>http://www.nexomongame.com</td>\n",
       "      <td>{'url': 'https://www.facebook.com/NexomonGame/', 'email': 'support@vewointeractive.com'}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>77</th>\n",
       "      <td>OctaFight</td>\n",
       "      <td>http://octafight.com/</td>\n",
       "      <td>{'url': '', 'email': 'support@octafight.com'}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>78</th>\n",
       "      <td>Dead Dreams</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{'url': '', 'email': 'aiazmarx@gmail.com'}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>79</th>\n",
       "      <td>Solar Panic: Utter Distress</td>\n",
       "      <td>http://www.barracudadisaster.com/solarpanic/</td>\n",
       "      <td>{'url': '', 'email': 'contact@barracudadisaster.com'}</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                           name                                       website  \\\n",
       "75                The Pale City                      https://thepalecity.com/   \n",
       "76          Nexomon: Extinction                    http://www.nexomongame.com   \n",
       "77                    OctaFight                         http://octafight.com/   \n",
       "78                  Dead Dreams                                           NaN   \n",
       "79  Solar Panic: Utter Distress  http://www.barracudadisaster.com/solarpanic/   \n",
       "\n",
       "                                                                                support_info  \n",
       "75                                  {'url': 'https://thepalecity.com/contact/', 'email': ''}  \n",
       "76  {'url': 'https://www.facebook.com/NexomonGame/', 'email': 'support@vewointeractive.com'}  \n",
       "77                                             {'url': '', 'email': 'support@octafight.com'}  \n",
       "78                                                {'url': '', 'email': 'aiazmarx@gmail.com'}  \n",
       "79                                     {'url': '', 'email': 'contact@barracudadisaster.com'}  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "print('website null counts:', media_steam['website'].isnull().sum())\n",
    "print('support_info null counts:', media_steam['support_info'].isnull().sum())\n",
    "\n",
    "with pd.option_context(\"display.max_colwidth\", 100): # ensures strings not cut short\n",
    "    display(media_steam[['name', 'website', 'support_info']][75:80])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "id": "3b262001",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Exported support info to 'steam_support_info.csv'\n"
     ]
    }
   ],
   "source": [
    "def process_info(steam, export=False):\n",
    "    \"\"\"Drop support information from dataframe, optionally exporting beforehand.\"\"\"\n",
    "    if export:\n",
    "        support_info = steam[['steam_appid', 'website', 'support_info']].copy()\n",
    "        \n",
    "        support_info['support_info'] = support_info['support_info'].apply(lambda x: literal_eval(x))\n",
    "        support_info['support_url'] = support_info['support_info'].apply(lambda x: x['url'])\n",
    "        support_info['support_email'] = support_info['support_info'].apply(lambda x: x['email'])\n",
    "        \n",
    "        support_info = support_info.drop('support_info', axis=1)\n",
    "        \n",
    "        # only keep rows with at least one piece of information\n",
    "        support_info = support_info[(support_info['website'].notnull()) | (support_info['support_url'] != '') | (support_info['support_email'] != '')]\n",
    "\n",
    "        export_data(support_info, 'support_info')\n",
    "    \n",
    "    steam = steam.drop(['website', 'support_info'], axis=1)\n",
    "    \n",
    "    return steam\n",
    "\n",
    "\n",
    "info_steam = process_info(media_steam, export=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "id": "e404ab51",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Exported description data to 'steam_description_data.csv'\n",
      "Exported media data to 'steam_media_data.csv'\n",
      "Exported support info to 'steam_support_info.csv'\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>type</th>\n",
       "      <th>name</th>\n",
       "      <th>steam_appid</th>\n",
       "      <th>required_age</th>\n",
       "      <th>supported_languages</th>\n",
       "      <th>pc_requirements</th>\n",
       "      <th>mac_requirements</th>\n",
       "      <th>linux_requirements</th>\n",
       "      <th>packages</th>\n",
       "      <th>package_groups</th>\n",
       "      <th>...</th>\n",
       "      <th>review_score</th>\n",
       "      <th>review_score_desc</th>\n",
       "      <th>total_positive</th>\n",
       "      <th>total_negative</th>\n",
       "      <th>total_reviews</th>\n",
       "      <th>achievements</th>\n",
       "      <th>discount</th>\n",
       "      <th>price</th>\n",
       "      <th>developer</th>\n",
       "      <th>publisher</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>game</td>\n",
       "      <td>All That Remains: Part 1</td>\n",
       "      <td>1198490</td>\n",
       "      <td>0</td>\n",
       "      <td>English&lt;strong&gt;*&lt;/strong&gt;, French, Italian, Ge...</td>\n",
       "      <td>{'minimum': '&lt;strong&gt;Minimum:&lt;/strong&gt;&lt;br&gt;&lt;ul ...</td>\n",
       "      <td>{'minimum': '&lt;strong&gt;Minimum:&lt;/strong&gt;&lt;br&gt;&lt;ul ...</td>\n",
       "      <td>[]</td>\n",
       "      <td>[413222]</td>\n",
       "      <td>[{'name': 'default', 'title': 'Buy All That Re...</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>4 user reviews</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2.39</td>\n",
       "      <td>Glitch Games</td>\n",
       "      <td>Glitch Games</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>game</td>\n",
       "      <td>Paperball</td>\n",
       "      <td>1198510</td>\n",
       "      <td>0</td>\n",
       "      <td>English&lt;strong&gt;*&lt;/strong&gt;, German, Japanese, D...</td>\n",
       "      <td>{'minimum': '&lt;strong&gt;Minimum:&lt;/strong&gt;&lt;br&gt;&lt;ul ...</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[413229]</td>\n",
       "      <td>[{'name': 'default', 'title': 'Buy Paperball',...</td>\n",
       "      <td>...</td>\n",
       "      <td>8</td>\n",
       "      <td>Very Positive</td>\n",
       "      <td>130</td>\n",
       "      <td>4</td>\n",
       "      <td>134</td>\n",
       "      <td>37</td>\n",
       "      <td>0</td>\n",
       "      <td>9.99</td>\n",
       "      <td>Cliax Games</td>\n",
       "      <td>Cliax Games</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>game</td>\n",
       "      <td>Moderium</td>\n",
       "      <td>1198550</td>\n",
       "      <td>0</td>\n",
       "      <td>English&lt;strong&gt;*&lt;/strong&gt;&lt;br&gt;&lt;strong&gt;*&lt;/strong...</td>\n",
       "      <td>{'minimum': '&lt;strong&gt;Minimum:&lt;/strong&gt;&lt;br&gt;&lt;ul ...</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[413245]</td>\n",
       "      <td>[{'name': 'default', 'title': 'Buy Moderium', ...</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>1 user reviews</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.79</td>\n",
       "      <td>All Saints Gaming</td>\n",
       "      <td>All Saints Gaming</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>game</td>\n",
       "      <td>Pizza Time Explosion</td>\n",
       "      <td>1198600</td>\n",
       "      <td>0</td>\n",
       "      <td>English</td>\n",
       "      <td>{'minimum': '&lt;strong&gt;Minimum:&lt;/strong&gt;&lt;br&gt;&lt;ul ...</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[413271]</td>\n",
       "      <td>[{'name': 'default', 'title': 'Buy Pizza Time ...</td>\n",
       "      <td>...</td>\n",
       "      <td>7</td>\n",
       "      <td>Positive</td>\n",
       "      <td>17</td>\n",
       "      <td>1</td>\n",
       "      <td>18</td>\n",
       "      <td>32</td>\n",
       "      <td>0</td>\n",
       "      <td>4.99</td>\n",
       "      <td>Electric Prune Juice</td>\n",
       "      <td>Electric Prune Juice</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>game</td>\n",
       "      <td>Customer Cums First!</td>\n",
       "      <td>1198630</td>\n",
       "      <td>0</td>\n",
       "      <td>English</td>\n",
       "      <td>{'minimum': '&lt;strong&gt;Minimum:&lt;/strong&gt;&lt;br&gt;&lt;ul ...</td>\n",
       "      <td>{'minimum': '&lt;strong&gt;Minimum:&lt;/strong&gt;&lt;br&gt;&lt;ul ...</td>\n",
       "      <td>{'minimum': '&lt;strong&gt;Minimum:&lt;/strong&gt;&lt;br&gt;&lt;ul ...</td>\n",
       "      <td>[413283]</td>\n",
       "      <td>[{'name': 'default', 'title': 'Buy Customer Cu...</td>\n",
       "      <td>...</td>\n",
       "      <td>7</td>\n",
       "      <td>Positive</td>\n",
       "      <td>8</td>\n",
       "      <td>2</td>\n",
       "      <td>10</td>\n",
       "      <td>18</td>\n",
       "      <td>0</td>\n",
       "      <td>9.99</td>\n",
       "      <td>Miel</td>\n",
       "      <td>Cherry Kiss Games</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 25 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   type                      name  steam_appid  required_age  \\\n",
       "0  game  All That Remains: Part 1      1198490             0   \n",
       "1  game                 Paperball      1198510             0   \n",
       "2  game                  Moderium      1198550             0   \n",
       "3  game      Pizza Time Explosion      1198600             0   \n",
       "4  game      Customer Cums First!      1198630             0   \n",
       "\n",
       "                                 supported_languages  \\\n",
       "0  English<strong>*</strong>, French, Italian, Ge...   \n",
       "1  English<strong>*</strong>, German, Japanese, D...   \n",
       "2  English<strong>*</strong><br><strong>*</strong...   \n",
       "3                                            English   \n",
       "4                                            English   \n",
       "\n",
       "                                     pc_requirements  \\\n",
       "0  {'minimum': '<strong>Minimum:</strong><br><ul ...   \n",
       "1  {'minimum': '<strong>Minimum:</strong><br><ul ...   \n",
       "2  {'minimum': '<strong>Minimum:</strong><br><ul ...   \n",
       "3  {'minimum': '<strong>Minimum:</strong><br><ul ...   \n",
       "4  {'minimum': '<strong>Minimum:</strong><br><ul ...   \n",
       "\n",
       "                                    mac_requirements  \\\n",
       "0  {'minimum': '<strong>Minimum:</strong><br><ul ...   \n",
       "1                                                 []   \n",
       "2                                                 []   \n",
       "3                                                 []   \n",
       "4  {'minimum': '<strong>Minimum:</strong><br><ul ...   \n",
       "\n",
       "                                  linux_requirements  packages  \\\n",
       "0                                                 []  [413222]   \n",
       "1                                                 []  [413229]   \n",
       "2                                                 []  [413245]   \n",
       "3                                                 []  [413271]   \n",
       "4  {'minimum': '<strong>Minimum:</strong><br><ul ...  [413283]   \n",
       "\n",
       "                                      package_groups  ... review_score  \\\n",
       "0  [{'name': 'default', 'title': 'Buy All That Re...  ...            0   \n",
       "1  [{'name': 'default', 'title': 'Buy Paperball',...  ...            8   \n",
       "2  [{'name': 'default', 'title': 'Buy Moderium', ...  ...            0   \n",
       "3  [{'name': 'default', 'title': 'Buy Pizza Time ...  ...            7   \n",
       "4  [{'name': 'default', 'title': 'Buy Customer Cu...  ...            7   \n",
       "\n",
       "  review_score_desc total_positive total_negative  total_reviews  \\\n",
       "0    4 user reviews              1              3              4   \n",
       "1     Very Positive            130              4            134   \n",
       "2    1 user reviews              1              0              1   \n",
       "3          Positive             17              1             18   \n",
       "4          Positive              8              2             10   \n",
       "\n",
       "   achievements discount  price             developer             publisher  \n",
       "0             0        0   2.39          Glitch Games          Glitch Games  \n",
       "1            37        0   9.99           Cliax Games           Cliax Games  \n",
       "2             0        0   0.79     All Saints Gaming     All Saints Gaming  \n",
       "3            32        0   4.99  Electric Prune Juice  Electric Prune Juice  \n",
       "4            18        0   9.99                  Miel     Cherry Kiss Games  \n",
       "\n",
       "[5 rows x 25 columns]"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def process(steam):\n",
    "    \"\"\"Process data set. Will eventually contain calls to all functions we write.\"\"\"\n",
    "    \n",
    "    # Copy the input dataframe to avoid accidentally modifying original data\n",
    "    steam = steam.copy()\n",
    "    steam = process_descriptions(steam, export=True)\n",
    "    steam = process_media(steam, export=True)\n",
    "    steam = process_info(steam, export=True)\n",
    "    #steam = process_requirements(steam, export=True)\n",
    "    return steam\n",
    "\n",
    "steam_data = process(steam1)\n",
    "steam_data.head()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "id": "092a7bf2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "type                     0\n",
       "name                     0\n",
       "steam_appid              0\n",
       "required_age             0\n",
       "supported_languages      0\n",
       "pc_requirements          0\n",
       "mac_requirements         0\n",
       "linux_requirements       0\n",
       "packages               313\n",
       "package_groups           0\n",
       "platforms                0\n",
       "categories               0\n",
       "genres                   0\n",
       "release_date             0\n",
       "num_reviews              0\n",
       "review_score             0\n",
       "review_score_desc        0\n",
       "total_positive           0\n",
       "total_negative           0\n",
       "total_reviews            0\n",
       "achievements             0\n",
       "discount                 0\n",
       "price                    0\n",
       "developer                0\n",
       "publisher                0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "steam_data.isnull().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "id": "6a5c493b",
   "metadata": {},
   "outputs": [],
   "source": [
    "steam_data.drop(columns = ['packages'], inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "id": "2adf11cc",
   "metadata": {},
   "outputs": [],
   "source": [
    "steam_data.to_csv('clea_data.csv', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bec7e57e",
   "metadata": {},
   "source": [
    "### Processing Release date"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "id": "f65bbc1e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dec 7, 2020\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>type</th>\n",
       "      <th>name</th>\n",
       "      <th>steam_appid</th>\n",
       "      <th>required_age</th>\n",
       "      <th>supported_languages</th>\n",
       "      <th>pc_requirements</th>\n",
       "      <th>mac_requirements</th>\n",
       "      <th>linux_requirements</th>\n",
       "      <th>package_groups</th>\n",
       "      <th>platforms</th>\n",
       "      <th>...</th>\n",
       "      <th>review_score</th>\n",
       "      <th>review_score_desc</th>\n",
       "      <th>total_positive</th>\n",
       "      <th>total_negative</th>\n",
       "      <th>total_reviews</th>\n",
       "      <th>achievements</th>\n",
       "      <th>discount</th>\n",
       "      <th>price</th>\n",
       "      <th>developer</th>\n",
       "      <th>publisher</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>game</td>\n",
       "      <td>All That Remains: Part 1</td>\n",
       "      <td>1198490</td>\n",
       "      <td>0</td>\n",
       "      <td>English&lt;strong&gt;*&lt;/strong&gt;, French, Italian, Ge...</td>\n",
       "      <td>{'minimum': '&lt;strong&gt;Minimum:&lt;/strong&gt;&lt;br&gt;&lt;ul ...</td>\n",
       "      <td>{'minimum': '&lt;strong&gt;Minimum:&lt;/strong&gt;&lt;br&gt;&lt;ul ...</td>\n",
       "      <td>[]</td>\n",
       "      <td>[{'name': 'default', 'title': 'Buy All That Re...</td>\n",
       "      <td>windows;mac</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>4 user reviews</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2.39</td>\n",
       "      <td>Glitch Games</td>\n",
       "      <td>Glitch Games</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>game</td>\n",
       "      <td>Paperball</td>\n",
       "      <td>1198510</td>\n",
       "      <td>0</td>\n",
       "      <td>English&lt;strong&gt;*&lt;/strong&gt;, German, Japanese, D...</td>\n",
       "      <td>{'minimum': '&lt;strong&gt;Minimum:&lt;/strong&gt;&lt;br&gt;&lt;ul ...</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[{'name': 'default', 'title': 'Buy Paperball',...</td>\n",
       "      <td>windows</td>\n",
       "      <td>...</td>\n",
       "      <td>8</td>\n",
       "      <td>Very Positive</td>\n",
       "      <td>130</td>\n",
       "      <td>4</td>\n",
       "      <td>134</td>\n",
       "      <td>37</td>\n",
       "      <td>0</td>\n",
       "      <td>9.99</td>\n",
       "      <td>Cliax Games</td>\n",
       "      <td>Cliax Games</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>game</td>\n",
       "      <td>Moderium</td>\n",
       "      <td>1198550</td>\n",
       "      <td>0</td>\n",
       "      <td>English&lt;strong&gt;*&lt;/strong&gt;&lt;br&gt;&lt;strong&gt;*&lt;/strong...</td>\n",
       "      <td>{'minimum': '&lt;strong&gt;Minimum:&lt;/strong&gt;&lt;br&gt;&lt;ul ...</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[{'name': 'default', 'title': 'Buy Moderium', ...</td>\n",
       "      <td>windows</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>1 user reviews</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.79</td>\n",
       "      <td>All Saints Gaming</td>\n",
       "      <td>All Saints Gaming</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>game</td>\n",
       "      <td>Pizza Time Explosion</td>\n",
       "      <td>1198600</td>\n",
       "      <td>0</td>\n",
       "      <td>English</td>\n",
       "      <td>{'minimum': '&lt;strong&gt;Minimum:&lt;/strong&gt;&lt;br&gt;&lt;ul ...</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[{'name': 'default', 'title': 'Buy Pizza Time ...</td>\n",
       "      <td>windows</td>\n",
       "      <td>...</td>\n",
       "      <td>7</td>\n",
       "      <td>Positive</td>\n",
       "      <td>17</td>\n",
       "      <td>1</td>\n",
       "      <td>18</td>\n",
       "      <td>32</td>\n",
       "      <td>0</td>\n",
       "      <td>4.99</td>\n",
       "      <td>Electric Prune Juice</td>\n",
       "      <td>Electric Prune Juice</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>game</td>\n",
       "      <td>Customer Cums First!</td>\n",
       "      <td>1198630</td>\n",
       "      <td>0</td>\n",
       "      <td>English</td>\n",
       "      <td>{'minimum': '&lt;strong&gt;Minimum:&lt;/strong&gt;&lt;br&gt;&lt;ul ...</td>\n",
       "      <td>{'minimum': '&lt;strong&gt;Minimum:&lt;/strong&gt;&lt;br&gt;&lt;ul ...</td>\n",
       "      <td>{'minimum': '&lt;strong&gt;Minimum:&lt;/strong&gt;&lt;br&gt;&lt;ul ...</td>\n",
       "      <td>[{'name': 'default', 'title': 'Buy Customer Cu...</td>\n",
       "      <td>windows;mac;linux</td>\n",
       "      <td>...</td>\n",
       "      <td>7</td>\n",
       "      <td>Positive</td>\n",
       "      <td>8</td>\n",
       "      <td>2</td>\n",
       "      <td>10</td>\n",
       "      <td>18</td>\n",
       "      <td>0</td>\n",
       "      <td>9.99</td>\n",
       "      <td>Miel</td>\n",
       "      <td>Cherry Kiss Games</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 24 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   type                      name  steam_appid  required_age  \\\n",
       "0  game  All That Remains: Part 1      1198490             0   \n",
       "1  game                 Paperball      1198510             0   \n",
       "2  game                  Moderium      1198550             0   \n",
       "3  game      Pizza Time Explosion      1198600             0   \n",
       "4  game      Customer Cums First!      1198630             0   \n",
       "\n",
       "                                 supported_languages  \\\n",
       "0  English<strong>*</strong>, French, Italian, Ge...   \n",
       "1  English<strong>*</strong>, German, Japanese, D...   \n",
       "2  English<strong>*</strong><br><strong>*</strong...   \n",
       "3                                            English   \n",
       "4                                            English   \n",
       "\n",
       "                                     pc_requirements  \\\n",
       "0  {'minimum': '<strong>Minimum:</strong><br><ul ...   \n",
       "1  {'minimum': '<strong>Minimum:</strong><br><ul ...   \n",
       "2  {'minimum': '<strong>Minimum:</strong><br><ul ...   \n",
       "3  {'minimum': '<strong>Minimum:</strong><br><ul ...   \n",
       "4  {'minimum': '<strong>Minimum:</strong><br><ul ...   \n",
       "\n",
       "                                    mac_requirements  \\\n",
       "0  {'minimum': '<strong>Minimum:</strong><br><ul ...   \n",
       "1                                                 []   \n",
       "2                                                 []   \n",
       "3                                                 []   \n",
       "4  {'minimum': '<strong>Minimum:</strong><br><ul ...   \n",
       "\n",
       "                                  linux_requirements  \\\n",
       "0                                                 []   \n",
       "1                                                 []   \n",
       "2                                                 []   \n",
       "3                                                 []   \n",
       "4  {'minimum': '<strong>Minimum:</strong><br><ul ...   \n",
       "\n",
       "                                      package_groups          platforms  ...  \\\n",
       "0  [{'name': 'default', 'title': 'Buy All That Re...        windows;mac  ...   \n",
       "1  [{'name': 'default', 'title': 'Buy Paperball',...            windows  ...   \n",
       "2  [{'name': 'default', 'title': 'Buy Moderium', ...            windows  ...   \n",
       "3  [{'name': 'default', 'title': 'Buy Pizza Time ...            windows  ...   \n",
       "4  [{'name': 'default', 'title': 'Buy Customer Cu...  windows;mac;linux  ...   \n",
       "\n",
       "  review_score review_score_desc total_positive  total_negative  \\\n",
       "0            0    4 user reviews              1               3   \n",
       "1            8     Very Positive            130               4   \n",
       "2            0    1 user reviews              1               0   \n",
       "3            7          Positive             17               1   \n",
       "4            7          Positive              8               2   \n",
       "\n",
       "   total_reviews achievements  discount  price             developer  \\\n",
       "0              4            0         0   2.39          Glitch Games   \n",
       "1            134           37         0   9.99           Cliax Games   \n",
       "2              1            0         0   0.79     All Saints Gaming   \n",
       "3             18           32         0   4.99  Electric Prune Juice   \n",
       "4             10           18         0   9.99                  Miel   \n",
       "\n",
       "              publisher  \n",
       "0          Glitch Games  \n",
       "1           Cliax Games  \n",
       "2     All Saints Gaming  \n",
       "3  Electric Prune Juice  \n",
       "4     Cherry Kiss Games  \n",
       "\n",
       "[5 rows x 24 columns]"
      ]
     },
     "execution_count": 70,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import re\n",
    "def process_release_date(df):\n",
    "    df = df.copy()\n",
    "    \n",
    "    def eval_date(x):\n",
    "        x = literal_eval(x)\n",
    "        if x['coming_soon']:\n",
    "            return '' # return blank string so can drop missing at end\n",
    "        else:\n",
    "            return x['date']\n",
    "    \n",
    "    df['release_date'] = df['release_date'].apply(eval_date)\n",
    "    \n",
    "    def parse_date(x):\n",
    "        if re.search(r'[\\d]{1,2} [A-Za-z]{3}, [\\d]{4}', x):\n",
    "            return x.replace(',', '')\n",
    "        elif re.search(r'[A-Za-z]{3} [\\d]{4}', x):\n",
    "            return '1 ' + x\n",
    "        elif x == '':\n",
    "            return np.nan\n",
    "        else:\n",
    "            # Should be everything, print out anything left just in case\n",
    "            print(x)\n",
    "            \n",
    "    df['release_date'] = df['release_date'].apply(parse_date)\n",
    "    df['release_date'] = pd.to_datetime(df['release_date'], format='%d %b %Y', errors='coerce')\n",
    "    \n",
    "    df = df[df['release_date'].notnull()]\n",
    "    \n",
    "    return df\n",
    "\n",
    "steam_new = process_release_date(steam_data)\n",
    "steam_new.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "id": "dcf37452",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Timestamp('2020-01-30 00:00:00')"
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "steam_new['release_date'][50]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "id": "79e2aa21",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>type</th>\n",
       "      <th>name</th>\n",
       "      <th>steam_appid</th>\n",
       "      <th>required_age</th>\n",
       "      <th>supported_languages</th>\n",
       "      <th>pc_requirements</th>\n",
       "      <th>mac_requirements</th>\n",
       "      <th>linux_requirements</th>\n",
       "      <th>package_groups</th>\n",
       "      <th>platforms</th>\n",
       "      <th>...</th>\n",
       "      <th>review_score</th>\n",
       "      <th>review_score_desc</th>\n",
       "      <th>total_positive</th>\n",
       "      <th>total_negative</th>\n",
       "      <th>total_reviews</th>\n",
       "      <th>achievements</th>\n",
       "      <th>discount</th>\n",
       "      <th>price</th>\n",
       "      <th>developer</th>\n",
       "      <th>publisher</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>game</td>\n",
       "      <td>All That Remains: Part 1</td>\n",
       "      <td>1198490</td>\n",
       "      <td>0</td>\n",
       "      <td>English&lt;strong&gt;*&lt;/strong&gt;, French, Italian, Ge...</td>\n",
       "      <td>{'minimum': '&lt;strong&gt;Minimum:&lt;/strong&gt;&lt;br&gt;&lt;ul ...</td>\n",
       "      <td>{'minimum': '&lt;strong&gt;Minimum:&lt;/strong&gt;&lt;br&gt;&lt;ul ...</td>\n",
       "      <td>[]</td>\n",
       "      <td>[{'name': 'default', 'title': 'Buy All That Re...</td>\n",
       "      <td>windows;mac</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>4 user reviews</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2.39</td>\n",
       "      <td>Glitch Games</td>\n",
       "      <td>Glitch Games</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>game</td>\n",
       "      <td>Paperball</td>\n",
       "      <td>1198510</td>\n",
       "      <td>0</td>\n",
       "      <td>English&lt;strong&gt;*&lt;/strong&gt;, German, Japanese, D...</td>\n",
       "      <td>{'minimum': '&lt;strong&gt;Minimum:&lt;/strong&gt;&lt;br&gt;&lt;ul ...</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[{'name': 'default', 'title': 'Buy Paperball',...</td>\n",
       "      <td>windows</td>\n",
       "      <td>...</td>\n",
       "      <td>8</td>\n",
       "      <td>Very Positive</td>\n",
       "      <td>130</td>\n",
       "      <td>4</td>\n",
       "      <td>134</td>\n",
       "      <td>37</td>\n",
       "      <td>0</td>\n",
       "      <td>9.99</td>\n",
       "      <td>Cliax Games</td>\n",
       "      <td>Cliax Games</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>game</td>\n",
       "      <td>Moderium</td>\n",
       "      <td>1198550</td>\n",
       "      <td>0</td>\n",
       "      <td>English&lt;strong&gt;*&lt;/strong&gt;&lt;br&gt;&lt;strong&gt;*&lt;/strong...</td>\n",
       "      <td>{'minimum': '&lt;strong&gt;Minimum:&lt;/strong&gt;&lt;br&gt;&lt;ul ...</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[{'name': 'default', 'title': 'Buy Moderium', ...</td>\n",
       "      <td>windows</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>1 user reviews</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.79</td>\n",
       "      <td>All Saints Gaming</td>\n",
       "      <td>All Saints Gaming</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>game</td>\n",
       "      <td>Pizza Time Explosion</td>\n",
       "      <td>1198600</td>\n",
       "      <td>0</td>\n",
       "      <td>English</td>\n",
       "      <td>{'minimum': '&lt;strong&gt;Minimum:&lt;/strong&gt;&lt;br&gt;&lt;ul ...</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[{'name': 'default', 'title': 'Buy Pizza Time ...</td>\n",
       "      <td>windows</td>\n",
       "      <td>...</td>\n",
       "      <td>7</td>\n",
       "      <td>Positive</td>\n",
       "      <td>17</td>\n",
       "      <td>1</td>\n",
       "      <td>18</td>\n",
       "      <td>32</td>\n",
       "      <td>0</td>\n",
       "      <td>4.99</td>\n",
       "      <td>Electric Prune Juice</td>\n",
       "      <td>Electric Prune Juice</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>game</td>\n",
       "      <td>Customer Cums First!</td>\n",
       "      <td>1198630</td>\n",
       "      <td>0</td>\n",
       "      <td>English</td>\n",
       "      <td>{'minimum': '&lt;strong&gt;Minimum:&lt;/strong&gt;&lt;br&gt;&lt;ul ...</td>\n",
       "      <td>{'minimum': '&lt;strong&gt;Minimum:&lt;/strong&gt;&lt;br&gt;&lt;ul ...</td>\n",
       "      <td>{'minimum': '&lt;strong&gt;Minimum:&lt;/strong&gt;&lt;br&gt;&lt;ul ...</td>\n",
       "      <td>[{'name': 'default', 'title': 'Buy Customer Cu...</td>\n",
       "      <td>windows;mac;linux</td>\n",
       "      <td>...</td>\n",
       "      <td>7</td>\n",
       "      <td>Positive</td>\n",
       "      <td>8</td>\n",
       "      <td>2</td>\n",
       "      <td>10</td>\n",
       "      <td>18</td>\n",
       "      <td>0</td>\n",
       "      <td>9.99</td>\n",
       "      <td>Miel</td>\n",
       "      <td>Cherry Kiss Games</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4620</th>\n",
       "      <td>game</td>\n",
       "      <td>Lunch A Palooza</td>\n",
       "      <td>1113770</td>\n",
       "      <td>0</td>\n",
       "      <td>English, French, Italian, German, Spanish - Sp...</td>\n",
       "      <td>{'minimum': '&lt;strong&gt;Minimum:&lt;/strong&gt;&lt;br&gt;&lt;ul ...</td>\n",
       "      <td>{'minimum': '&lt;strong&gt;Minimum:&lt;/strong&gt;&lt;br&gt;&lt;ul ...</td>\n",
       "      <td>[]</td>\n",
       "      <td>[{'name': 'default', 'title': 'Buy Lunch A Pal...</td>\n",
       "      <td>windows;mac</td>\n",
       "      <td>...</td>\n",
       "      <td>7</td>\n",
       "      <td>Positive</td>\n",
       "      <td>12</td>\n",
       "      <td>1</td>\n",
       "      <td>13</td>\n",
       "      <td>21</td>\n",
       "      <td>0</td>\n",
       "      <td>10.79</td>\n",
       "      <td>Seashell Studio</td>\n",
       "      <td>Alternative Software Ltd</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4621</th>\n",
       "      <td>game</td>\n",
       "      <td>Uncharted Tides: Port Royal</td>\n",
       "      <td>1113780</td>\n",
       "      <td>0</td>\n",
       "      <td>English&lt;strong&gt;*&lt;/strong&gt;, French, Italian, Ge...</td>\n",
       "      <td>{'minimum': '&lt;strong&gt;Minimum:&lt;/strong&gt;&lt;br&gt;&lt;ul ...</td>\n",
       "      <td>{'minimum': '&lt;strong&gt;Minimum:&lt;/strong&gt;&lt;br&gt;&lt;ul ...</td>\n",
       "      <td>{'minimum': '&lt;strong&gt;Minimum:&lt;/strong&gt;&lt;br&gt;&lt;ul ...</td>\n",
       "      <td>[{'name': 'default', 'title': 'Buy Uncharted T...</td>\n",
       "      <td>windows;mac;linux</td>\n",
       "      <td>...</td>\n",
       "      <td>7</td>\n",
       "      <td>Positive</td>\n",
       "      <td>26</td>\n",
       "      <td>4</td>\n",
       "      <td>30</td>\n",
       "      <td>22</td>\n",
       "      <td>0</td>\n",
       "      <td>14.99</td>\n",
       "      <td>Cordelia Games</td>\n",
       "      <td>Artifex Mundi</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4622</th>\n",
       "      <td>game</td>\n",
       "      <td>Persona 4 Golden</td>\n",
       "      <td>1113000</td>\n",
       "      <td>0</td>\n",
       "      <td>English&lt;strong&gt;*&lt;/strong&gt;, Japanese&lt;strong&gt;*&lt;/...</td>\n",
       "      <td>{'minimum': '&lt;strong&gt;Minimum:&lt;/strong&gt;&lt;br&gt;&lt;ul ...</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[{'name': 'default', 'title': 'Buy Persona 4 G...</td>\n",
       "      <td>windows</td>\n",
       "      <td>...</td>\n",
       "      <td>9</td>\n",
       "      <td>Overwhelmingly Positive</td>\n",
       "      <td>29369</td>\n",
       "      <td>629</td>\n",
       "      <td>29998</td>\n",
       "      <td>50</td>\n",
       "      <td>0</td>\n",
       "      <td>19.99</td>\n",
       "      <td>ATLUS</td>\n",
       "      <td>SEGA</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4623</th>\n",
       "      <td>game</td>\n",
       "      <td>Magic Farm 3: The Ice Danger</td>\n",
       "      <td>1113060</td>\n",
       "      <td>0</td>\n",
       "      <td>English</td>\n",
       "      <td>{'minimum': '&lt;strong&gt;Minimum:&lt;/strong&gt;&lt;br&gt;&lt;ul ...</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[{'name': 'default', 'title': 'Buy Magic Farm ...</td>\n",
       "      <td>windows</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>4 user reviews</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>27</td>\n",
       "      <td>0</td>\n",
       "      <td>16.79</td>\n",
       "      <td>Meridian'93</td>\n",
       "      <td>Meridian'93</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4624</th>\n",
       "      <td>game</td>\n",
       "      <td>The Stranger: Interactive Film</td>\n",
       "      <td>1113110</td>\n",
       "      <td>0</td>\n",
       "      <td>English&lt;strong&gt;*&lt;/strong&gt;&lt;br&gt;&lt;strong&gt;*&lt;/strong...</td>\n",
       "      <td>{'minimum': '&lt;strong&gt;Minimum:&lt;/strong&gt;&lt;br&gt;&lt;ul ...</td>\n",
       "      <td>{'minimum': '&lt;strong&gt;Minimum:&lt;/strong&gt;&lt;br&gt;&lt;ul ...</td>\n",
       "      <td>[]</td>\n",
       "      <td>[{'name': 'default', 'title': 'Buy The Strange...</td>\n",
       "      <td>windows;mac</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>2 user reviews</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>6.59</td>\n",
       "      <td>Giovanni Altrui</td>\n",
       "      <td>Giovanni Altrui</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2055 rows × 24 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      type                            name  steam_appid  required_age  \\\n",
       "0     game        All That Remains: Part 1      1198490             0   \n",
       "1     game                       Paperball      1198510             0   \n",
       "2     game                        Moderium      1198550             0   \n",
       "3     game            Pizza Time Explosion      1198600             0   \n",
       "4     game            Customer Cums First!      1198630             0   \n",
       "...    ...                             ...          ...           ...   \n",
       "4620  game                 Lunch A Palooza      1113770             0   \n",
       "4621  game     Uncharted Tides: Port Royal      1113780             0   \n",
       "4622  game                Persona 4 Golden      1113000             0   \n",
       "4623  game    Magic Farm 3: The Ice Danger      1113060             0   \n",
       "4624  game  The Stranger: Interactive Film      1113110             0   \n",
       "\n",
       "                                    supported_languages  \\\n",
       "0     English<strong>*</strong>, French, Italian, Ge...   \n",
       "1     English<strong>*</strong>, German, Japanese, D...   \n",
       "2     English<strong>*</strong><br><strong>*</strong...   \n",
       "3                                               English   \n",
       "4                                               English   \n",
       "...                                                 ...   \n",
       "4620  English, French, Italian, German, Spanish - Sp...   \n",
       "4621  English<strong>*</strong>, French, Italian, Ge...   \n",
       "4622  English<strong>*</strong>, Japanese<strong>*</...   \n",
       "4623                                            English   \n",
       "4624  English<strong>*</strong><br><strong>*</strong...   \n",
       "\n",
       "                                        pc_requirements  \\\n",
       "0     {'minimum': '<strong>Minimum:</strong><br><ul ...   \n",
       "1     {'minimum': '<strong>Minimum:</strong><br><ul ...   \n",
       "2     {'minimum': '<strong>Minimum:</strong><br><ul ...   \n",
       "3     {'minimum': '<strong>Minimum:</strong><br><ul ...   \n",
       "4     {'minimum': '<strong>Minimum:</strong><br><ul ...   \n",
       "...                                                 ...   \n",
       "4620  {'minimum': '<strong>Minimum:</strong><br><ul ...   \n",
       "4621  {'minimum': '<strong>Minimum:</strong><br><ul ...   \n",
       "4622  {'minimum': '<strong>Minimum:</strong><br><ul ...   \n",
       "4623  {'minimum': '<strong>Minimum:</strong><br><ul ...   \n",
       "4624  {'minimum': '<strong>Minimum:</strong><br><ul ...   \n",
       "\n",
       "                                       mac_requirements  \\\n",
       "0     {'minimum': '<strong>Minimum:</strong><br><ul ...   \n",
       "1                                                    []   \n",
       "2                                                    []   \n",
       "3                                                    []   \n",
       "4     {'minimum': '<strong>Minimum:</strong><br><ul ...   \n",
       "...                                                 ...   \n",
       "4620  {'minimum': '<strong>Minimum:</strong><br><ul ...   \n",
       "4621  {'minimum': '<strong>Minimum:</strong><br><ul ...   \n",
       "4622                                                 []   \n",
       "4623                                                 []   \n",
       "4624  {'minimum': '<strong>Minimum:</strong><br><ul ...   \n",
       "\n",
       "                                     linux_requirements  \\\n",
       "0                                                    []   \n",
       "1                                                    []   \n",
       "2                                                    []   \n",
       "3                                                    []   \n",
       "4     {'minimum': '<strong>Minimum:</strong><br><ul ...   \n",
       "...                                                 ...   \n",
       "4620                                                 []   \n",
       "4621  {'minimum': '<strong>Minimum:</strong><br><ul ...   \n",
       "4622                                                 []   \n",
       "4623                                                 []   \n",
       "4624                                                 []   \n",
       "\n",
       "                                         package_groups          platforms  \\\n",
       "0     [{'name': 'default', 'title': 'Buy All That Re...        windows;mac   \n",
       "1     [{'name': 'default', 'title': 'Buy Paperball',...            windows   \n",
       "2     [{'name': 'default', 'title': 'Buy Moderium', ...            windows   \n",
       "3     [{'name': 'default', 'title': 'Buy Pizza Time ...            windows   \n",
       "4     [{'name': 'default', 'title': 'Buy Customer Cu...  windows;mac;linux   \n",
       "...                                                 ...                ...   \n",
       "4620  [{'name': 'default', 'title': 'Buy Lunch A Pal...        windows;mac   \n",
       "4621  [{'name': 'default', 'title': 'Buy Uncharted T...  windows;mac;linux   \n",
       "4622  [{'name': 'default', 'title': 'Buy Persona 4 G...            windows   \n",
       "4623  [{'name': 'default', 'title': 'Buy Magic Farm ...            windows   \n",
       "4624  [{'name': 'default', 'title': 'Buy The Strange...        windows;mac   \n",
       "\n",
       "      ... review_score        review_score_desc total_positive  \\\n",
       "0     ...            0           4 user reviews              1   \n",
       "1     ...            8            Very Positive            130   \n",
       "2     ...            0           1 user reviews              1   \n",
       "3     ...            7                 Positive             17   \n",
       "4     ...            7                 Positive              8   \n",
       "...   ...          ...                      ...            ...   \n",
       "4620  ...            7                 Positive             12   \n",
       "4621  ...            7                 Positive             26   \n",
       "4622  ...            9  Overwhelmingly Positive          29369   \n",
       "4623  ...            0           4 user reviews              4   \n",
       "4624  ...            0           2 user reviews              1   \n",
       "\n",
       "      total_negative  total_reviews achievements  discount  price  \\\n",
       "0                  3              4            0         0   2.39   \n",
       "1                  4            134           37         0   9.99   \n",
       "2                  0              1            0         0   0.79   \n",
       "3                  1             18           32         0   4.99   \n",
       "4                  2             10           18         0   9.99   \n",
       "...              ...            ...          ...       ...    ...   \n",
       "4620               1             13           21         0  10.79   \n",
       "4621               4             30           22         0  14.99   \n",
       "4622             629          29998           50         0  19.99   \n",
       "4623               0              4           27         0  16.79   \n",
       "4624               1              2            0         0   6.59   \n",
       "\n",
       "                 developer                 publisher  \n",
       "0             Glitch Games              Glitch Games  \n",
       "1              Cliax Games               Cliax Games  \n",
       "2        All Saints Gaming         All Saints Gaming  \n",
       "3     Electric Prune Juice      Electric Prune Juice  \n",
       "4                     Miel         Cherry Kiss Games  \n",
       "...                    ...                       ...  \n",
       "4620       Seashell Studio  Alternative Software Ltd  \n",
       "4621        Cordelia Games             Artifex Mundi  \n",
       "4622                 ATLUS                      SEGA  \n",
       "4623           Meridian'93               Meridian'93  \n",
       "4624       Giovanni Altrui           Giovanni Altrui  \n",
       "\n",
       "[2055 rows x 24 columns]"
      ]
     },
     "execution_count": 73,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "steam_new[steam_new['release_date'] > '2019-05-01']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "id": "828abd0b",
   "metadata": {},
   "outputs": [],
   "source": [
    "steam_new.drop(columns = ['supported_languages','pc_requirements','mac_requirements','linux_requirements'], inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "id": "a3c6b044",
   "metadata": {},
   "outputs": [],
   "source": [
    "steam_new.to_csv('clean_data.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "id": "165489ea",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "type                 0\n",
       "name                 0\n",
       "steam_appid          0\n",
       "required_age         0\n",
       "package_groups       0\n",
       "platforms            0\n",
       "categories           0\n",
       "genres               0\n",
       "release_date         0\n",
       "num_reviews          0\n",
       "review_score         0\n",
       "review_score_desc    0\n",
       "total_positive       0\n",
       "total_negative       0\n",
       "total_reviews        0\n",
       "achievements         0\n",
       "discount             0\n",
       "price                0\n",
       "developer            0\n",
       "publisher            0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 77,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "steam_new.isnull().sum()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "62b2677b",
   "metadata": {},
   "source": [
    "# Boxlieter method"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e98f9858",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}