Data_Collection/Feather_Data/amex_generate_features.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "b3e3d008",
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5",
    "execution": {
     "iopub.execute_input": "2023-02-08T04:51:04.948430Z",
     "iopub.status.busy": "2023-02-08T04:51:04.947629Z",
     "iopub.status.idle": "2023-02-08T04:51:06.586147Z",
     "shell.execute_reply": "2023-02-08T04:51:06.585006Z"
    },
    "papermill": {
     "duration": 1.649488,
     "end_time": "2023-02-08T04:51:06.589455",
     "exception": false,
     "start_time": "2023-02-08T04:51:04.939967",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import gc\n",
    "import pandas as pd \n",
    "import numpy as np\n",
    "from sklearn.preprocessing import OneHotEncoder\n",
    "from sklearn.impute import SimpleImputer"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e1822301",
   "metadata": {
    "papermill": {
     "duration": 0.002739,
     "end_time": "2023-02-08T04:51:06.595800",
     "exception": false,
     "start_time": "2023-02-08T04:51:06.593061",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Helper functions to impute, encode, and generate features/targets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "e01a29ee",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-02-08T04:51:06.604916Z",
     "iopub.status.busy": "2023-02-08T04:51:06.603636Z",
     "iopub.status.idle": "2023-02-08T04:51:06.613343Z",
     "shell.execute_reply": "2023-02-08T04:51:06.611908Z"
    },
    "papermill": {
     "duration": 0.017718,
     "end_time": "2023-02-08T04:51:06.616552",
     "exception": false,
     "start_time": "2023-02-08T04:51:06.598834",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def one_hot_encode_categorical(cat_features, cat_names):\n",
    "    \"\"\"\n",
    "    One-hot encodes categorical features using scikit-learn OneHotEncoder\n",
    "\n",
    "    Parameters\n",
    "    ----------\n",
    "    cat_features : pd.DataFrame\n",
    "        DataFrame, with index, that has only the categorical columns to one-hot encode\n",
    "    cat_names : list\n",
    "        list of categorical column names \n",
    "\n",
    "    Returns\n",
    "    -------\n",
    "    pd.DataFrame\n",
    "        DataFrame that holds each of the one-hot encoded columns \n",
    "    \"\"\"\n",
    "    \n",
    "    enc = OneHotEncoder(sparse=False)\n",
    "    encoded_df = pd.DataFrame(enc.fit_transform(cat_features), columns=enc.get_feature_names(cat_names), index=cat_features.index)\n",
    "    return encoded_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "8e19fba1",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-02-08T04:51:06.625128Z",
     "iopub.status.busy": "2023-02-08T04:51:06.624592Z",
     "iopub.status.idle": "2023-02-08T04:51:06.633032Z",
     "shell.execute_reply": "2023-02-08T04:51:06.631772Z"
    },
    "papermill": {
     "duration": 0.016288,
     "end_time": "2023-02-08T04:51:06.636013",
     "exception": false,
     "start_time": "2023-02-08T04:51:06.619725",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def simple_impute_numerical(numeric_features, numeric_names):\n",
    "    \"\"\"\n",
    "    Imputes numerical columns with scikit-learn SimpleImputer()\n",
    "\n",
    "    Parameters\n",
    "    ----------\n",
    "    numeric_features : pd.DataFrame\n",
    "        DataFrame, with index, that has only the numerical columns to impute\n",
    "    numeric_features : list\n",
    "        list of numerical column names \n",
    "\n",
    "    Returns\n",
    "    -------\n",
    "    pd.DataFrame\n",
    "        DataFrame that holds each of the imputed numerical columns\n",
    "    \"\"\"\n",
    "    \n",
    "    # current numeric columns are float16, and they will not work when computing mean()\n",
    "    # need to convert to float32\n",
    "    for column in numeric_features.columns:\n",
    "        numeric_features[column] = numeric_features[column].astype(np.float32)\n",
    "\n",
    "    # impute columns using the mean\n",
    "    imp_mean = SimpleImputer(missing_values=np.nan, strategy='mean')\n",
    "    numeric_df = pd.DataFrame(imp_mean.fit_transform(numeric_features), columns=numeric_names, index=numeric_features.index)\n",
    "    \n",
    "    # convert back to float16 for lighter load\n",
    "    for column in numeric_df.columns:\n",
    "        numeric_df[column] = numeric_df[column].astype(np.float16)\n",
    "    \n",
    "    return numeric_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "c969c05a",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-02-08T04:51:06.644525Z",
     "iopub.status.busy": "2023-02-08T04:51:06.643998Z",
     "iopub.status.idle": "2023-02-08T04:51:06.655843Z",
     "shell.execute_reply": "2023-02-08T04:51:06.654527Z"
    },
    "papermill": {
     "duration": 0.01957,
     "end_time": "2023-02-08T04:51:06.658794",
     "exception": false,
     "start_time": "2023-02-08T04:51:06.639224",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def generate_x_y(df_file_path, test=False):\n",
    "    \"\"\"\n",
    "    Returns the features (X) and targets (y) for the given data file\n",
    "\n",
    "    Parameters\n",
    "    ----------\n",
    "    df_file_path : string\n",
    "        File path to generate DataFrame from \n",
    "    test : boolean\n",
    "        Whether or not the provided data file is the test set\n",
    "        False = training set \n",
    "        True = test set \n",
    "\n",
    "    Returns\n",
    "    -------\n",
    "    pd.DataFrame\n",
    "        If it is the test dataset it will return only the features (X)\n",
    "        \n",
    "    OR \n",
    "    \n",
    "    Tuple(pd.DataFrame, pd.DataFrame)\n",
    "        If it is the training set it will return the features and targets in a tuple (X, y)\n",
    "    \"\"\"\n",
    "    \n",
    "    # read in data and set index to customer ID\n",
    "    df = pd.read_feather(df_file_path)\n",
    "    df = df.set_index('customer_ID')\n",
    "    \n",
    "    # get X and y; drop dates from X \n",
    "    X = df.drop('S_2', axis=1) if test else df.drop(['S_2', 'target'], axis=1)\n",
    "    y = None if test else df['target']\n",
    "    \n",
    "    # delete original dataframe from memory \n",
    "    del df\n",
    "    gc.collect()\n",
    "    \n",
    "    # encode categorical features\n",
    "    cat_features = ['B_30', 'B_38', 'D_114', 'D_116', 'D_117', 'D_120', 'D_126', 'D_63', 'D_64', 'D_66', 'D_68']\n",
    "    encoded_df = one_hot_encode_categorical(X[cat_features], cat_features)\n",
    "    \n",
    "    # simple impute numerical columns with mean()\n",
    "    X = X.drop(cat_features, axis=1)\n",
    "    X = simple_impute_numerical(X, list(X.columns))\n",
    "    \n",
    "    # get final encoded and imputed features\n",
    "    X = pd.concat([X, encoded_df], axis=1)\n",
    "\n",
    "    if test: \n",
    "        return X\n",
    "    else: \n",
    "        return (X, y)\n",
    "    \n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "13c80336",
   "metadata": {
    "papermill": {
     "duration": 0.002777,
     "end_time": "2023-02-08T04:51:06.664795",
     "exception": false,
     "start_time": "2023-02-08T04:51:06.662018",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Generate the features and targets, and then save to .ftr file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "e0af88be",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-02-08T04:51:06.673261Z",
     "iopub.status.busy": "2023-02-08T04:51:06.672365Z",
     "iopub.status.idle": "2023-02-08T04:58:55.517663Z",
     "shell.execute_reply": "2023-02-08T04:58:55.516187Z"
    },
    "papermill": {
     "duration": 468.858968,
     "end_time": "2023-02-08T04:58:55.526777",
     "exception": false,
     "start_time": "2023-02-08T04:51:06.667809",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/sklearn/utils/deprecation.py:87: FutureWarning: Function get_feature_names is deprecated; get_feature_names is deprecated in 1.0 and will be removed in 1.2. Please use get_feature_names_out instead.\n",
      "  warnings.warn(msg, category=FutureWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train, y_train = generate_x_y('../input/amexfeather/train_data.ftr')\n",
    "\n",
    "# sort columns for matching with test set \n",
    "X_train = X_train.reindex(sorted(X_train.columns), axis=1)\n",
    "\n",
    "# feather files do not support indexing\n",
    "X_train = X_train.reset_index()\n",
    "y_train = y_train.reset_index()\n",
    "X_train.to_feather('X_train.ftr')\n",
    "y_train.to_feather('y_train.ftr')\n",
    "\n",
    "del X_train, y_train\n",
    "gc.collect()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.12"
  },
  "papermill": {
   "default_parameters": {},
   "duration": 482.314248,
   "end_time": "2023-02-08T04:58:56.863548",
   "environment_variables": {},
   "exception": null,
   "input_path": "__notebook__.ipynb",
   "output_path": "__notebook__.ipynb",
   "parameters": {},
   "start_time": "2023-02-08T04:50:54.549300",
   "version": "2.3.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}