Untitled1.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARN: OMP_NUM_THREADS=None =>\n",
      "... If you are using openblas if you are using openblas set OMP_NUM_THREADS=1 or risk subprocess calls hanging indefinitely\n"
     ]
    }
   ],
   "source": [
    "import pickle\n",
    "from mlxtend.classifier import StackingClassifier\n",
    "import pandas as pd\n",
    "import featuretools as ft\n",
    "from sklearn.preprocessing import Imputer, StandardScaler\n",
    "from sklearn.pipeline import Pipeline, make_pipeline\n",
    "from hpsklearn import HyperoptEstimator, any_classifier\n",
    "import numpy as np\n",
    "from collections import defaultdict\n",
    "import seaborn as sns\n",
    "from sklearn.calibration import CalibratedClassifierCV\n",
    "from xgboost import XGBClassifier\n",
    "from sklearn.model_selection import cross_val_score\n",
    "from sklearn.base import BaseEstimator, TransformerMixin\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "pickle_files = ['0.162_100_ada_boost_titanic_dfs_ensemble_pipeline_cv10_estimator.pickle',\n",
    "                '0.158_100_extra_trees_titanic_dfs_ensemble_pipeline_cv10_estimator.pickle',\n",
    "                '0.155_100_xgboost_classification_titanic_dfs_ensemble_pipeline_cv10_estimator.pickle',\n",
    "                '0.148_100_random_forest_titanic_dfs_ensemble_pipeline_cv10_estimator.pickle',\n",
    "                '0.172_100_knn_titanic_dfs_ensemble_pipeline_cv10_estimator.pickle',\n",
    "                '0.18_100_sgd_titanic_dfs_ensemble_pipeline_cv10_estimator.pickle'\n",
    "               ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "class FeaturetoolsTransformer(BaseEstimator, TransformerMixin):\n",
    "    \n",
    "    def __init__(self, categorical_features, index, filepath, warm_start):\n",
    "        self.categorical_features = categorical_features\n",
    "        self.index = index\n",
    "        self.filepath = filepath\n",
    "        self.warm_start = warm_start\n",
    "        \n",
    "    def fit(self, X, y=None):\n",
    "        return self\n",
    "\n",
    "    def transform(self, X, y=None):\n",
    "        variable_types = {variable: ft.variable_types.Categorical for variable in self.categorical_features}\n",
    "        es = ft.EntitySet(id=\"id\")\n",
    "        es.entity_from_dataframe(entity_id=\"id\",\n",
    "                                 dataframe=X,\n",
    "                                 index=self.index,\n",
    "                                 variable_types=variable_types)\n",
    "        for variable in self.categorical_features:\n",
    "            es.normalize_entity(base_entity_id=\"id\",\n",
    "                            new_entity_id=variable,\n",
    "                            index=variable)\n",
    "        self.es = es\n",
    "        if self.warm_start:\n",
    "            self.features = ft.load_features(self.filepath, self.es)\n",
    "            matrix = ft.calculate_feature_matrix(self.features)\n",
    "            return matrix\n",
    "        else:\n",
    "            matrix, self.features = ft.dfs(entityset=self.es,\n",
    "                                      target_entity=\"id\",\n",
    "                                      save_progress=\"results/\",\n",
    "                                      verbose=False)\n",
    "            \n",
    "            matrix, self.features = ft.encode_features(matrix, self.features)\n",
    "            self.warm_start = True\n",
    "            ft.save_features(self.features, self.filepath)\n",
    "            return matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(f\"results/models/0.148_100_random_forest_titanic_dfs_ensemble_pipeline_cv10_estimator.pickle\", \"rb\") as model_file:\n",
    "    hyperopt_model = pickle.load(model_file)\n",
    "    model = hyperopt_model.best_model()['learner']\n",
    "    prepro = hyperopt_model.best_model()['preprocs']\n",
    "\n",
    "pipeline = make_pipeline(FeaturetoolsTransformer(\n",
    "        categorical_features=[\"Pclass\", \"Sex\", \"Embarked\", \"CabinClass\", \n",
    "                          \"LastName\", \"Honorific\", \"TicketPrefix\"],\n",
    "    index = \"PassengerId\",\n",
    "    filepath=\"results/sklearn_encoded_features.pkl\",\n",
    "    warm_start=True), Imputer(strategy='median'),\n",
    "                         StandardScaler(),\n",
    "                         *prepro, model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/deadhead/miniconda3/envs/automl-talk/lib/python3.6/site-packages/ipykernel_launcher.py:5: FutureWarning: currently extract(expand=None) means expand=False (return Index/Series/DataFrame) but in a future version of pandas this will be changed to expand=True (return DataFrame)\n",
      "  \"\"\"\n",
      "/home/deadhead/miniconda3/envs/automl-talk/lib/python3.6/site-packages/ipykernel_launcher.py:6: FutureWarning: currently extract(expand=None) means expand=False (return Index/Series/DataFrame) but in a future version of pandas this will be changed to expand=True (return DataFrame)\n",
      "  \n",
      "/home/deadhead/miniconda3/envs/automl-talk/lib/python3.6/site-packages/ipykernel_launcher.py:7: FutureWarning: currently extract(expand=None) means expand=False (return Index/Series/DataFrame) but in a future version of pandas this will be changed to expand=True (return DataFrame)\n",
      "  import sys\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "Pipeline(memory=None,\n",
       "     steps=[('featuretoolstransformer', FeaturetoolsTransformer(categorical_features=['Pclass', 'Sex', 'Embarked', 'CabinClass', 'LastName', 'Honorific', 'TicketPrefix'],\n",
       "            filepath='results/sklearn_encoded_features.pkl',\n",
       "            index='PassengerId', warm_start=True)), ('imputer', Imputer(a...mators=22, n_jobs=1, oob_score=False, random_state=3,\n",
       "            verbose=False, warm_start=False))])"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df = pd.read_csv(\"data/train.csv\")\n",
    "train_df[\"Pclass\"] = train_df[\"Pclass\"].astype(\"category\")\n",
    "train_df['CabinClass'] = train_df[\"Cabin\"].str.get(0)\n",
    "train_df['LastName'] = train_df['Name'].str.split(\", \").apply(lambda x: x[0])\n",
    "train_df['Honorific'] = train_df['Name'].str.extract(\" ([a-zA-z]+)\")\n",
    "train_df['TicketPrefix'] = train_df['Ticket'].str.extract(\"(.+) \")\n",
    "train_df['TicketNumber'] = train_df['Ticket'].str.extract(\"([0-9]+)$\").astype(float)\n",
    "train_df = train_df.drop(columns=[\"Name\", \"Cabin\", \"Ticket\"])\n",
    "train_features = train_df.drop(columns=[\"Survived\"])\n",
    "train_target = train_df[\"Survived\"]\n",
    "pipeline.fit(train_features, train_target)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "prepro_pipeline = make_pipeline(FeaturetoolsTransformer(\n",
    "        categorical_features=[\"Pclass\", \"Sex\", \"Embarked\", \"CabinClass\", \n",
    "                          \"LastName\", \"Honorific\", \"TicketPrefix\"],\n",
    "        index = \"PassengerId\",\n",
    "        filepath=\"results/sklearn_encoded_features2.pkl\",\n",
    "        warm_start=False), Imputer(strategy='median'),\n",
    "                             StandardScaler())\n",
    "prepro_features = prepro_pipeline.fit_transform(train_features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tqdm import tnrange, tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 83%|████████▎ | 5/6 [10:54<02:10, 130.87s/it]/home/deadhead/miniconda3/envs/automl-talk/lib/python3.6/site-packages/sklearn/linear_model/stochastic_gradient.py:117: DeprecationWarning: n_iter parameter is deprecated in 0.19 and will be removed in 0.21. Use max_iter and tol instead.\n",
      "  DeprecationWarning)\n",
      "/home/deadhead/miniconda3/envs/automl-talk/lib/python3.6/site-packages/sklearn/linear_model/stochastic_gradient.py:117: DeprecationWarning: n_iter parameter is deprecated in 0.19 and will be removed in 0.21. Use max_iter and tol instead.\n",
      "  DeprecationWarning)\n",
      "/home/deadhead/miniconda3/envs/automl-talk/lib/python3.6/site-packages/sklearn/linear_model/stochastic_gradient.py:117: DeprecationWarning: n_iter parameter is deprecated in 0.19 and will be removed in 0.21. Use max_iter and tol instead.\n",
      "  DeprecationWarning)\n",
      "/home/deadhead/miniconda3/envs/automl-talk/lib/python3.6/site-packages/sklearn/linear_model/stochastic_gradient.py:117: DeprecationWarning: n_iter parameter is deprecated in 0.19 and will be removed in 0.21. Use max_iter and tol instead.\n",
      "  DeprecationWarning)\n",
      "/home/deadhead/miniconda3/envs/automl-talk/lib/python3.6/site-packages/sklearn/linear_model/stochastic_gradient.py:117: DeprecationWarning: n_iter parameter is deprecated in 0.19 and will be removed in 0.21. Use max_iter and tol instead.\n",
      "  DeprecationWarning)\n",
      "100%|██████████| 6/6 [10:54<00:00, 109.13s/it]\n"
     ]
    }
   ],
   "source": [
    "calibrated_estimators = []\n",
    "for pickle_file in tqdm(pickle_files, total=len(pickle_files)):\n",
    "    with open(f\"results/models/{pickle_file}\", \"rb\") as model_file:\n",
    "        hyperopt_model = pickle.load(model_file)\n",
    "        model = hyperopt_model.best_model()['learner']\n",
    "        prepro = hyperopt_model.best_model()['preprocs']\n",
    "        pipeline =  make_pipeline(*prepro, model)\n",
    "        calibrator = CalibratedClassifierCV(model, cv=5, method=\"isotonic\")\n",
    "        calibrator.fit(prepro_features, train_target)\n",
    "        calibrated_estimators.append(calibrator)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "stack = StackingClassifier(classifiers=calibrated_estimators, \n",
    "                   meta_classifier=XGBClassifier(),\n",
    "                   use_probas=True,\n",
    "                   average_probas=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/deadhead/miniconda3/envs/automl-talk/lib/python3.6/site-packages/sklearn/linear_model/stochastic_gradient.py:117: DeprecationWarning: n_iter parameter is deprecated in 0.19 and will be removed in 0.21. Use max_iter and tol instead.\n",
      "  DeprecationWarning)\n",
      "/home/deadhead/miniconda3/envs/automl-talk/lib/python3.6/site-packages/sklearn/linear_model/stochastic_gradient.py:117: DeprecationWarning: n_iter parameter is deprecated in 0.19 and will be removed in 0.21. Use max_iter and tol instead.\n",
      "  DeprecationWarning)\n",
      "/home/deadhead/miniconda3/envs/automl-talk/lib/python3.6/site-packages/sklearn/linear_model/stochastic_gradient.py:117: DeprecationWarning: n_iter parameter is deprecated in 0.19 and will be removed in 0.21. Use max_iter and tol instead.\n",
      "  DeprecationWarning)\n",
      "/home/deadhead/miniconda3/envs/automl-talk/lib/python3.6/site-packages/sklearn/linear_model/stochastic_gradient.py:117: DeprecationWarning: n_iter parameter is deprecated in 0.19 and will be removed in 0.21. Use max_iter and tol instead.\n",
      "  DeprecationWarning)\n",
      "/home/deadhead/miniconda3/envs/automl-talk/lib/python3.6/site-packages/sklearn/linear_model/stochastic_gradient.py:117: DeprecationWarning: n_iter parameter is deprecated in 0.19 and will be removed in 0.21. Use max_iter and tol instead.\n",
      "  DeprecationWarning)\n",
      "/home/deadhead/miniconda3/envs/automl-talk/lib/python3.6/site-packages/sklearn/preprocessing/label.py:151: DeprecationWarning: The truth value of an empty array is ambiguous. Returning False, but in future this will result in an error. Use `array.size > 0` to check that an array is not empty.\n",
      "  if diff:\n",
      "/home/deadhead/miniconda3/envs/automl-talk/lib/python3.6/site-packages/sklearn/linear_model/stochastic_gradient.py:117: DeprecationWarning: n_iter parameter is deprecated in 0.19 and will be removed in 0.21. Use max_iter and tol instead.\n",
      "  DeprecationWarning)\n",
      "/home/deadhead/miniconda3/envs/automl-talk/lib/python3.6/site-packages/sklearn/linear_model/stochastic_gradient.py:117: DeprecationWarning: n_iter parameter is deprecated in 0.19 and will be removed in 0.21. Use max_iter and tol instead.\n",
      "  DeprecationWarning)\n",
      "/home/deadhead/miniconda3/envs/automl-talk/lib/python3.6/site-packages/sklearn/linear_model/stochastic_gradient.py:117: DeprecationWarning: n_iter parameter is deprecated in 0.19 and will be removed in 0.21. Use max_iter and tol instead.\n",
      "  DeprecationWarning)\n",
      "/home/deadhead/miniconda3/envs/automl-talk/lib/python3.6/site-packages/sklearn/linear_model/stochastic_gradient.py:117: DeprecationWarning: n_iter parameter is deprecated in 0.19 and will be removed in 0.21. Use max_iter and tol instead.\n",
      "  DeprecationWarning)\n",
      "/home/deadhead/miniconda3/envs/automl-talk/lib/python3.6/site-packages/sklearn/linear_model/stochastic_gradient.py:117: DeprecationWarning: n_iter parameter is deprecated in 0.19 and will be removed in 0.21. Use max_iter and tol instead.\n",
      "  DeprecationWarning)\n",
      "/home/deadhead/miniconda3/envs/automl-talk/lib/python3.6/site-packages/sklearn/preprocessing/label.py:151: DeprecationWarning: The truth value of an empty array is ambiguous. Returning False, but in future this will result in an error. Use `array.size > 0` to check that an array is not empty.\n",
      "  if diff:\n",
      "/home/deadhead/miniconda3/envs/automl-talk/lib/python3.6/site-packages/sklearn/linear_model/stochastic_gradient.py:117: DeprecationWarning: n_iter parameter is deprecated in 0.19 and will be removed in 0.21. Use max_iter and tol instead.\n",
      "  DeprecationWarning)\n",
      "/home/deadhead/miniconda3/envs/automl-talk/lib/python3.6/site-packages/sklearn/linear_model/stochastic_gradient.py:117: DeprecationWarning: n_iter parameter is deprecated in 0.19 and will be removed in 0.21. Use max_iter and tol instead.\n",
      "  DeprecationWarning)\n",
      "/home/deadhead/miniconda3/envs/automl-talk/lib/python3.6/site-packages/sklearn/linear_model/stochastic_gradient.py:117: DeprecationWarning: n_iter parameter is deprecated in 0.19 and will be removed in 0.21. Use max_iter and tol instead.\n",
      "  DeprecationWarning)\n",
      "/home/deadhead/miniconda3/envs/automl-talk/lib/python3.6/site-packages/sklearn/linear_model/stochastic_gradient.py:117: DeprecationWarning: n_iter parameter is deprecated in 0.19 and will be removed in 0.21. Use max_iter and tol instead.\n",
      "  DeprecationWarning)\n",
      "/home/deadhead/miniconda3/envs/automl-talk/lib/python3.6/site-packages/sklearn/linear_model/stochastic_gradient.py:117: DeprecationWarning: n_iter parameter is deprecated in 0.19 and will be removed in 0.21. Use max_iter and tol instead.\n",
      "  DeprecationWarning)\n",
      "/home/deadhead/miniconda3/envs/automl-talk/lib/python3.6/site-packages/sklearn/preprocessing/label.py:151: DeprecationWarning: The truth value of an empty array is ambiguous. Returning False, but in future this will result in an error. Use `array.size > 0` to check that an array is not empty.\n",
      "  if diff:\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([0.8047138 , 0.85521886, 0.8047138 ])"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cross_val_score(stack, prepro_features, train_target)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/deadhead/miniconda3/envs/automl-talk/lib/python3.6/site-packages/sklearn/linear_model/stochastic_gradient.py:117: DeprecationWarning: n_iter parameter is deprecated in 0.19 and will be removed in 0.21. Use max_iter and tol instead.\n",
      "  DeprecationWarning)\n",
      "/home/deadhead/miniconda3/envs/automl-talk/lib/python3.6/site-packages/sklearn/linear_model/stochastic_gradient.py:117: DeprecationWarning: n_iter parameter is deprecated in 0.19 and will be removed in 0.21. Use max_iter and tol instead.\n",
      "  DeprecationWarning)\n",
      "/home/deadhead/miniconda3/envs/automl-talk/lib/python3.6/site-packages/sklearn/linear_model/stochastic_gradient.py:117: DeprecationWarning: n_iter parameter is deprecated in 0.19 and will be removed in 0.21. Use max_iter and tol instead.\n",
      "  DeprecationWarning)\n",
      "/home/deadhead/miniconda3/envs/automl-talk/lib/python3.6/site-packages/sklearn/linear_model/stochastic_gradient.py:117: DeprecationWarning: n_iter parameter is deprecated in 0.19 and will be removed in 0.21. Use max_iter and tol instead.\n",
      "  DeprecationWarning)\n",
      "/home/deadhead/miniconda3/envs/automl-talk/lib/python3.6/site-packages/sklearn/linear_model/stochastic_gradient.py:117: DeprecationWarning: n_iter parameter is deprecated in 0.19 and will be removed in 0.21. Use max_iter and tol instead.\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "StackingClassifier(average_probas=False,\n",
       "          classifiers=[CalibratedClassifierCV(base_estimator=AdaBoostClassifier(algorithm='SAMME', base_estimator=None,\n",
       "          learning_rate=0.2969585196792592, n_estimators=866,\n",
       "          random_state=1),\n",
       "            cv=5, method='isotonic'), CalibratedClassifierCV(base_estimator=ExtraTreesClassifier(boot...     shuffle=True, tol=None, verbose=False, warm_start=False),\n",
       "            cv=5, method='isotonic')],\n",
       "          meta_classifier=XGBClassifier(base_score=0.5, booster='gbtree', colsample_bylevel=1,\n",
       "       colsample_bytree=1, gamma=0, learning_rate=0.1, max_delta_step=0,\n",
       "       max_depth=3, min_child_weight=1, missing=None, n_estimators=100,\n",
       "       n_jobs=1, nthread=None, objective='binary:logistic', random_state=0,\n",
       "       reg_alpha=0, reg_lambda=1, scale_pos_weight=1, seed=None,\n",
       "       silent=True, subsample=1),\n",
       "          refit=True, store_train_meta_features=False,\n",
       "          use_features_in_secondary=False, use_probas=True, verbose=0)"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "stack.fit(prepro_features, train_target)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/deadhead/miniconda3/envs/automl-talk/lib/python3.6/site-packages/ipykernel_launcher.py:5: FutureWarning: currently extract(expand=None) means expand=False (return Index/Series/DataFrame) but in a future version of pandas this will be changed to expand=True (return DataFrame)\n",
      "  \"\"\"\n",
      "/home/deadhead/miniconda3/envs/automl-talk/lib/python3.6/site-packages/ipykernel_launcher.py:6: FutureWarning: currently extract(expand=None) means expand=False (return Index/Series/DataFrame) but in a future version of pandas this will be changed to expand=True (return DataFrame)\n",
      "  \n",
      "/home/deadhead/miniconda3/envs/automl-talk/lib/python3.6/site-packages/ipykernel_launcher.py:7: FutureWarning: currently extract(expand=None) means expand=False (return Index/Series/DataFrame) but in a future version of pandas this will be changed to expand=True (return DataFrame)\n",
      "  import sys\n"
     ]
    }
   ],
   "source": [
    "test_df = pd.read_csv(\"data/test.csv\")\n",
    "test_df[\"Pclass\"] = test_df[\"Pclass\"].astype(\"category\")\n",
    "test_df['CabinClass'] = test_df[\"Cabin\"].str.get(0)\n",
    "test_df['LastName'] = test_df['Name'].str.split(\", \").apply(lambda x: x[0])\n",
    "test_df['Honorific'] = test_df['Name'].str.extract(\" ([a-zA-z]+)\")\n",
    "test_df['TicketPrefix'] = test_df['Ticket'].str.extract(\"(.+) \")\n",
    "test_df['TicketNumber'] = test_df['Ticket'].str.extract(\"([0-9]+)$\").astype(float)\n",
    "test_df = test_df.drop(columns=[\"Name\", \"Cabin\", \"Ticket\"])\n",
    "test_features = prepro_pipeline.transform(test_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/deadhead/miniconda3/envs/automl-talk/lib/python3.6/site-packages/sklearn/preprocessing/label.py:151: DeprecationWarning: The truth value of an empty array is ambiguous. Returning False, but in future this will result in an error. Use `array.size > 0` to check that an array is not empty.\n",
      "  if diff:\n"
     ]
    }
   ],
   "source": [
    "submission_df = pd.DataFrame({'PassengerId': test_df['PassengerId'],\n",
    "              'Survived': stack.predict(test_features)})\n",
    "submission_df.to_csv(\"results/ft_hpsklearn_submission_stacked_pipeline.csv\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(f\"results/models/0.0_100_sgd_titanic_dfs_ensemble_pipeline_estimator.pickle\", \"rb\") as model_file:\n",
    "    hyperopt_model = pickle.load(model_file)\n",
    "    model = hyperopt_model.best_model()['learner']\n",
    "    prepro = hyperopt_model.best_model()['preprocs']\n",
    "    pipeline = make_pipeline(*prepro, model)\n",
    "    featuretools = FeaturetoolsTransformer(\n",
    "        categorical_features=[\"Pclass\", \"Sex\", \"Embarked\", \"CabinClass\", \n",
    "                          \"LastName\", \"Honorific\", \"TicketPrefix\"],\n",
    "    index = \"PassengerId\",\n",
    "    filepath=\"results/sklearn_encoded_features_sgd.pkl\",\n",
    "    warm_start=False)\n",
    "    pipeline = make_pipeline(featuretools, Imputer(strategy='median'), StandardScaler(), *prepro, model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "class FeaturetoolsTransformer(BaseEstimator, TransformerMixin):\n",
    "    \n",
    "    def __init__(self, categorical_features, index, filepath, warm_start):\n",
    "        self.categorical_features = categorical_features\n",
    "        self.index = index\n",
    "        self.filepath = filepath\n",
    "        self._warm_start = warm_start\n",
    "        \n",
    "    def fit(self, X, y=None):\n",
    "        return self\n",
    "\n",
    "    def transform(self, X, y=None):\n",
    "        variable_types = {variable: ft.variable_types.Categorical for variable in self.categorical_features}\n",
    "        es = ft.EntitySet(id=\"id\")\n",
    "        es.entity_from_dataframe(entity_id=\"id\",\n",
    "                                 dataframe=X,\n",
    "                                 index=self.index,\n",
    "                                 variable_types=variable_types)\n",
    "        for variable in self.categorical_features:\n",
    "            es.normalize_entity(base_entity_id=\"id\",\n",
    "                            new_entity_id=variable,\n",
    "                            index=variable)\n",
    "        self.es = es\n",
    "        if self._warm_start:\n",
    "            self.features = ft.load_features(self.filepath, self.es)\n",
    "            matrix = ft.calculate_feature_matrix(self.features)\n",
    "            return matrix\n",
    "        else:\n",
    "            matrix, self.features = ft.dfs(entityset=self.es,\n",
    "                                      target_entity=\"id\",\n",
    "                                      save_progress=\"results/\",\n",
    "                                      verbose=False)\n",
    "            \n",
    "            matrix, self.features = ft.encode_features(matrix, self.features)\n",
    "            self._warm_start = True\n",
    "            ft.save_features(self.features, self.filepath)\n",
    "            return matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/deadhead/miniconda3/envs/automl-talk/lib/python3.6/site-packages/sklearn/linear_model/stochastic_gradient.py:117: DeprecationWarning: n_iter parameter is deprecated in 0.19 and will be removed in 0.21. Use max_iter and tol instead.\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "Pipeline(memory=None,\n",
       "     steps=[('featuretoolstransformer', FeaturetoolsTransformer(categorical_features=['Pclass', 'Sex', 'Embarked', 'CabinClass', 'LastName', 'Honorific', 'TicketPrefix'],\n",
       "            filepath='results/sklearn_encoded_features_sgd.pkl',\n",
       "            index='PassengerId', warm_start=None)), ('imputer', Imput....6816036005988287, random_state=2, shuffle=True, tol=None,\n",
       "       verbose=False, warm_start=False))])"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pipeline.fit(train_features, train_target)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "pipeline.named_steps['featuretoolstransformer'].warm_start = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [],
   "source": [
    "submission_df = pd.DataFrame({'PassengerId': test_df['PassengerId'],\n",
    "              'Survived': pipeline.predict(test_df)})\n",
    "submission_df.to_csv(\"results/overfitting_sgd.csv\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[-0.15508855, -0.01567737,  0.        ,  0.14310383,  0.        ,\n",
       "        -0.00723776,  0.        ,  0.        ,  0.        , -0.03647884,\n",
       "         0.03647884,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "         0.        ,  0.        , -0.0181046 ,  0.        ,  0.        ,\n",
       "         0.        , -0.01299327, -0.13416503, -0.13416503,  0.01383824,\n",
       "         0.        ,  0.01383824, -0.18036973,  0.        , -0.0181046 ,\n",
       "         0.14817187,  0.        ,  0.        ,  0.0121081 ,  0.04367097,\n",
       "        -0.01299327,  0.        ,  0.        , -0.0733633 , -0.0733633 ,\n",
       "         0.        ,  0.        ,  0.        ,  0.        ,  0.07720188,\n",
       "         0.        , -0.04358718,  0.        ,  0.        ,  0.        ,\n",
       "        -0.01299327,  0.        ,  0.        ,  0.        , -0.00400306,\n",
       "        -0.00468909,  0.        , -0.00922274,  0.01196244, -0.00668621,\n",
       "        -0.00655651,  0.        , -0.01010861,  0.        , -0.00723776,\n",
       "         0.        ,  0.        , -0.00893621, -0.00243609,  0.        ,\n",
       "         0.        ,  0.        ,  0.00575646,  0.        ,  0.        ,\n",
       "         0.        ,  0.        ,  0.        , -0.00411419,  0.        ,\n",
       "         0.        , -0.01224454, -0.00206868,  0.        ,  0.        ,\n",
       "         0.        , -0.00148894,  0.        ,  0.        ,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "         0.        , -0.00723776,  0.        ,  0.        ,  0.        ,\n",
       "         0.        ,  0.        , -0.00723776,  0.        ,  0.        ,\n",
       "         0.        , -0.03647884, -0.03647884,  0.03647884, -0.03647884,\n",
       "        -0.03647884, -0.03647884,  0.03647884,  0.03647884,  0.03647884,\n",
       "        -0.03647884, -0.03647884,  0.        ,  0.03647884,  0.        ,\n",
       "        -0.03647884, -0.03647884, -0.03647884, -0.03647884,  0.08967056,\n",
       "         0.03647884,  0.        ,  0.        ,  0.03647884,  0.        ,\n",
       "        -0.03647884,  0.03647884,  0.03647884,  0.03647884, -0.03647884,\n",
       "        -0.03647884,  0.        ,  0.        ,  0.        , -0.03647884,\n",
       "        -0.03647884, -0.03647884,  0.        ,  0.        ,  0.        ,\n",
       "         0.        , -0.03647884,  0.03647884,  0.        , -0.03647884,\n",
       "         0.03647884,  0.        , -0.03647884,  0.03647884,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "         0.10478097,  0.        ,  0.        , -0.0181046 ,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "         0.        ,  0.        ,  0.04827402,  0.08004694,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ,  0.01486999,  0.        ,\n",
       "        -0.0181046 ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "         0.        ,  0.        ,  0.        , -0.0181046 ,  0.        ,\n",
       "         0.        ,  0.        , -0.0181046 ,  0.        ,  0.03286065,\n",
       "         0.        , -0.0181046 ,  0.        ,  0.        , -0.19445308,\n",
       "        -0.03359677,  0.        , -0.09371632,  0.        ,  0.        ,\n",
       "         0.        ,  0.00377641,  0.        ,  0.        ,  0.        ,\n",
       "         0.        ,  0.09904491,  0.        , -0.12544621,  0.        ,\n",
       "         0.        ,  0.01309384,  0.        ,  0.        ,  0.        ,\n",
       "         0.        ,  0.01173586,  0.        ,  0.        ,  0.        ,\n",
       "         0.        ,  0.06745916,  0.        , -0.04010494,  0.        ,\n",
       "        -0.13304579,  0.06534972,  0.        ,  0.        ,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "         0.        ,  0.        , -0.11250147,  0.        ,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "         0.        ,  0.        , -0.0181046 ,  0.        ,  0.        ,\n",
       "         0.        , -0.10787219,  0.08048832,  0.        ,  0.        ,\n",
       "         0.        ,  0.        , -0.0733633 , -0.0733633 ,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        -0.04358718,  0.        ,  0.        , -0.01299327,  0.        ,\n",
       "        -0.01299327,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ,  0.        ,  0.01367384,\n",
       "         0.        ,  0.        , -0.0086633 ,  0.        ,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ,  0.        , -0.07668566,\n",
       "         0.        ,  0.        ,  0.        ,  0.11224168,  0.04173663,\n",
       "         0.0313023 ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "         0.        ,  0.        ,  0.        , -0.05476634,  0.05476634,\n",
       "         0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "         0.        ,  0.04367097,  0.06943706, -0.08645735,  0.        ,\n",
       "         0.        ,  0.0121081 ,  0.04367097, -0.01299327,  0.        ,\n",
       "         0.        , -0.0733633 , -0.0733633 , -0.00605213,  0.        ,\n",
       "         0.        ,  0.04367097,  0.        , -0.14230402, -0.14230402,\n",
       "         0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ,  0.0568205 ,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ,  0.        , -0.00849966,\n",
       "        -0.02487856,  0.        ,  0.02123393,  0.        , -0.01433613,\n",
       "         0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "         0.        , -0.24794944,  0.        ,  0.        ,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ,  0.07720188,  0.        ,\n",
       "        -0.04358718,  0.        ,  0.        , -0.01299327,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ,  0.        , -0.06832177,\n",
       "         0.        ,  0.        ,  0.        ,  0.        ]])"
      ]
     },
     "execution_count": 73,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pipeline.named_steps['sgdclassifier'].coef_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}