IMDB_DATA_CLEANING.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "366f0411",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting ftfy\n",
      "  Downloading ftfy-6.1.1-py3-none-any.whl (53 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m53.1/53.1 KB\u001b[0m \u001b[31m313.7 kB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0ma \u001b[36m0:00:01\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: wcwidth>=0.2.5 in /opt/anaconda3/lib/python3.7/site-packages (from ftfy) (0.2.5)\n",
      "Installing collected packages: ftfy\n",
      "Successfully installed ftfy-6.1.1\n",
      "\u001b[33mWARNING: You are using pip version 22.0.4; however, version 23.0.1 is available.\n",
      "You should consider upgrading via the '/opt/anaconda3/bin/python -m pip install --upgrade pip' command.\u001b[0m\u001b[33m\n",
      "\u001b[0mNote: you may need to restart the kernel to use updated packages.\n"
     ]
    }
   ],
   "source": [
    "pip install ftfy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "325fc4d6",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import ftfy\n",
    "from scipy import stats"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "ddc5705d",
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_data(filename):\n",
    "    \"\"\"Given a filename of a csv load data into a Pandas dataframe.\n",
    "\n",
    "        filename - string\n",
    "\n",
    "        return Pandas dataframe\n",
    "    \"\"\"\n",
    "    return pd.read_csv(filename, encoding='utf-8')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "6f68bb1d",
   "metadata": {},
   "outputs": [],
   "source": [
    "def remove_unnecessary_columns(imdb):\n",
    "    return imdb.drop('budget', axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "730b32be",
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "def null_val(imdb):\n",
    "    return imdb.isnull().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "61c8f553",
   "metadata": {},
   "outputs": [],
   "source": [
    "def fill_missing_values(imdb):\n",
    "    mean_value=imdb['gross'].mean()\n",
    "    return imdb['gross'].fillna(value=mean_value, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "73540342",
   "metadata": {},
   "outputs": [],
   "source": [
    "def update_country_names(imdb):\n",
    "    imdb[\"Country\"]= imdb[\"Country\"].str.upper()\n",
    "    imdb['Country'] = np.where((imdb.Country == 'UNITED STATES'),'USA',imdb.Country)\n",
    "    return \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "cdc600e0",
   "metadata": {},
   "outputs": [],
   "source": [
    "def fix_director_values(imdb):\n",
    "    imdb.director_name = imdb.director_name.fillna('', inplace=True)\n",
    "    return"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "c18f7a03",
   "metadata": {},
   "outputs": [],
   "source": [
    "def fix_unicode_movie_title(imdb):\n",
    "    imdb[\"movie_title\"] = ftfy.fix_text(imdb[\"Les Mis√©rables\"])\n",
    "   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "ff5679de",
   "metadata": {},
   "outputs": [],
   "source": [
    "def fix_outliers(imdb):\n",
    "    Z=np.abs(stats.zscore(imdb['imdb_score']))\n",
    "    threshold = 3 \n",
    "    # Position of the outlier\n",
    "    return (np.where(Z > 3) &imdb.drop(imdb[imdb['title_year'] < 2010].index, inplace = true))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "5fd0f8b2",
   "metadata": {},
   "outputs": [],
   "source": [
    "def main():\n",
    "    \"\"\"Clean up the imdb dataset\n",
    "    \"\"\"\n",
    "    # 3. Rename filename.csv to the data filename.\n",
    "    imdb = load_data('imdb.csv')\n",
    "    imdb.dtypes\n",
    "    #Attributes and types are given below for action 2.\n",
    "    \"\"\"color                    object\n",
    "       director_name            object\n",
    "       duration                  int64\n",
    "       gross                   float64\n",
    "       genres                   object\n",
    "       movie_title              object\n",
    "       title_year                int64\n",
    "       language                 object\n",
    "       country                  object\n",
    "       budget                  float64\n",
    "       imdb_score              float64\n",
    "       actors                   object\n",
    "       movie_facebook_likes      int64\n",
    "       dtype: object\"\"\"\n",
    "    # 4. Using Pandas dataframe drop function get rid of unnecessary columns (set\n",
    "    # inplace=True)\n",
    "    # **5. How many columns did you remove?** We removed ONE column - budget because it was highly correlative to gross attribute\n",
    "    remove_unnecersary_columns(imdb)\n",
    "\n",
    "    # 6. How many missing values are there within each column?**\n",
    "    # hint: use isnull and the sum function\n",
    "    null_val(imdb)\n",
    "\n",
    "     \n",
    "    fill_missing_values(imdb)\n",
    "\n",
    "    # 9. Uppercase all of the country values (hint: str.upper())\n",
    "    # 9a. replace any reference to United States to USA\n",
    "    update_country_names(imdb)\n",
    "    imdb[\"Country\"]= imdb[\"Country\"].str.upper()\n",
    "\n",
    "    # 10. Replace N/A, Nan, Null with an empty string\n",
    "    imdb = fix_director_values(imdb)\n",
    "\n",
    "    # 11. Fix unicode in 'movie_title' column with import ftfy\n",
    "    imdb = fix_unicode_movie_title(imdb)\n",
    "\n",
    "    # 12. Assume a movie cannot be < 10 mins or > 300 mins. If a movie is outside those\n",
    "    # bounds set the value to 0.\n",
    "    imdb['duration']=imdb['duration'].where(imdb['duration'] <10, 0)\n",
    "\n",
    "    imdb['duration']=imdb['duration'].where(imdb['duration'] >300, 0)\n",
    "    \n",
    "    # **13. What would be considered an outlier for imdb_score?**\n",
    "    \"\"\"We calculated z score\n",
    "       In z score, any value which is greater than 3 is considered as an outlier. \n",
    "       Because 99.6% of the data is under z-score 3. Therefore anything greater than z score 3 is an outlier\"\"\"\n",
    "    # 14. Fix imdb_score and title_year (no year prior to 2010) outliers.\n",
    "    imdb = fix_outliers(imdb)\n",
    "\n",
    "    # 15. output the cleaned up file onto a new csv called clean_imdb.csv\n",
    "    imdb.to_csv('clean_imdb.csv',index=False)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5802de91",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}