phase2_mlstm_fcn_with_validation.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "9ef176be",
   "metadata": {},
   "source": [
    "# Train sktime-dl LSTM-FCN Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "4365e471",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2' # Only print warnings, ignore info and error \n",
    "os.environ['CUDA_VISIBLE_DEVICES'] = '-1' # Disable GPU\n",
    "\n",
    "# os.environ['TF_FORCE_GPU_ALLOW_GROWTH'] = 'true' \n",
    "# import tensorflow as tf\n",
    "# tf.config.experimental.set_memory_growth(tf.config.list_physical_devices('GPU')[0], True)\n",
    "\n",
    "import time\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "from tensorflow.keras import callbacks\n",
    "\n",
    "from sktime.classification.deep_learning.mlstmfcn import MLSTMFCNClassifier\n",
    "\n",
    "from sklearn.model_selection import StratifiedShuffleSplit\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.metrics import f1_score, make_scorer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "cf079b49",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1000000, 3, 46)\n",
      "Training data:\n",
      "(950000, 3, 46)\n",
      "(950000,)\n",
      "\n",
      "Validation data:\n",
      "(50000, 3, 46)\n",
      "(50000,)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# you can use a larger subset, if you like, there are:\n",
    "# - SITS-train-phase2-subset-1000000.csv.gz\n",
    "# - SITS-train-phase2-subset-100000.csv.gz\n",
    "# - SITS-train-phase2-subset-10000.csv.gz\n",
    "DATA_TRAIN = \"./dataset/SITS-train-phase2-subset-1000000.csv.gz\"\n",
    "use_univariate = False\n",
    "\n",
    "# Set univariate=False, if you use a classifier with multivariate capabilities\n",
    "def read_data_sktime(DATA, univariate=False):\n",
    "    data = pd.read_csv(DATA, delimiter=\",\" , \n",
    "                       na_values=['?'], dtype='float', \n",
    "                       index_col=\"id\", compression='gzip')\n",
    "\n",
    "    # Fill NaN values\n",
    "    # We use the most basic way with bfill and ffill to carry on the last values\n",
    "    data.fillna(method='bfill', inplace=True, axis=1)\n",
    "    data.fillna(method='ffill', inplace=True, axis=1)\n",
    "\n",
    "    # Extract Data and Labels\n",
    "    X = np.array(data.iloc[:,1:].values)\n",
    "    y = np.array(data.iloc[:,0].astype(int))\n",
    "\n",
    "\n",
    "    if univariate:\n",
    "        X = X.reshape(X.shape[0], 1, X.shape[1])\n",
    "    else:\n",
    "        X = X.reshape(X.shape[0], 3, X.shape[1]//3)\n",
    "        \n",
    "        \n",
    "    print(X.shape)\n",
    "    return X, y\n",
    "\n",
    "X, y = read_data_sktime(DATA_TRAIN, univariate=use_univariate)\n",
    "\n",
    "for (train_ix, test_ix)  in StratifiedShuffleSplit(n_splits=1, test_size=0.05, random_state=42).split(X, y):\n",
    "    X_train, y_train = X[train_ix], y[train_ix]\n",
    "    X_val, y_val = X[test_ix], y[test_ix]\n",
    "\n",
    "\n",
    "    print(\"Training data:\")\n",
    "    print(X_train.shape)\n",
    "    print(y_train.shape)\n",
    "    print()\n",
    "\n",
    "    print(\"Validation data:\")\n",
    "    print(X_val.shape)\n",
    "    print(y_val.shape)\n",
    "    print()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "b861481f",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define callbacks\n",
    "\n",
    "reduce_lr = callbacks.ReduceLROnPlateau(monitor=\"loss\", factor=0.7, patience=10, min_lr=0.0001)\n",
    "early_stopping = callbacks.EarlyStopping(monitor='val_loss', patience=50, restore_best_weights=True)\n",
    "tensorboard = callbacks.TensorBoard(log_dir=\"./tensorboard/phase_2_mlstm_full_dataset_tuned_and_fixed_multivariate_with_dilation\", histogram_freq=1)\n",
    "# model_save = callbacks.ModelCheckpoint('best_model_exploring.h5', monitor='val_loss', mode='min', save_best_only=True)\n",
    "\n",
    "callbacks_ = [early_stopping, reduce_lr, tensorboard]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "ebf156fd",
   "metadata": {},
   "outputs": [],
   "source": [
    "class MLSTMFCNWithValidation(MLSTMFCNClassifier):\n",
    "\n",
    "    def fit(self, X, y, X_val=None, y_val=None, **kwargs):\n",
    "        self.reset()\n",
    "\n",
    "        start = int(round(time.time() * 1000))\n",
    "        # convenience conversions to allow user flexibility:\n",
    "        # if X is 2D array, convert to 3D, if y is Series, convert to numpy\n",
    "        X, y = self._internal_convert(X, y)\n",
    "        X_metadata = self._check_classifier_input(X, y)\n",
    "        missing = X_metadata[\"has_nans\"]\n",
    "        multivariate = not X_metadata[\"is_univariate\"]\n",
    "        unequal = not X_metadata[\"is_equal_length\"]\n",
    "        self._X_metadata = X_metadata\n",
    "\n",
    "        # Check this classifier can handle characteristics\n",
    "        self._check_capabilities(missing, multivariate, unequal)\n",
    "\n",
    "        # remember class labels\n",
    "        self.classes_ = np.unique(y)\n",
    "        self.n_classes_ = self.classes_.shape[0]\n",
    "        self._class_dictionary = {}\n",
    "        for index, class_val in enumerate(self.classes_):\n",
    "            self._class_dictionary[class_val] = index\n",
    "\n",
    "        # escape early and do not fit if only one class label has been seen\n",
    "        #   in this case, we later predict the single class label seen\n",
    "        if len(self.classes_) == 1:\n",
    "            self.fit_time_ = int(round(time.time() * 1000)) - start\n",
    "            self._is_fitted = True\n",
    "            return self\n",
    "\n",
    "        # Convert data as dictated by the classifier tags\n",
    "        X = self._convert_X(X)\n",
    "        multithread = self.get_tag(\"capability:multithreading\")\n",
    "        if multithread:\n",
    "            try:\n",
    "                self._threads_to_use = check_n_jobs(self.n_jobs)\n",
    "            except NameError:\n",
    "                raise AttributeError(\n",
    "                    \"self.n_jobs must be set if capability:multithreading is True\"\n",
    "                )\n",
    "\n",
    "        # pass coerced and checked data to inner _fit\n",
    "        self._fit(X, y, X_val, y_val, **kwargs)\n",
    "        self.fit_time_ = int(round(time.time() * 1000)) - start\n",
    "\n",
    "        # this should happen last\n",
    "        self._is_fitted = True\n",
    "        return self\n",
    "\n",
    "    def _fit(self, X, y, X_val=None, y_val=None, **kwargs):\n",
    "        from sklearn.utils.validation import check_random_state\n",
    "        \n",
    "        self.random_state = check_random_state(self.random_state)\n",
    "        y_onehot = self.convert_y_to_keras(y)\n",
    "        \n",
    "        if y_val is not None:\n",
    "            y_val = self.label_encoder.transform(y_val)\n",
    "            y_val = y_val.reshape(-1, 1)\n",
    "            y_val = self.onehot_encoder.transform(y_val)\n",
    "\n",
    "        # Transpose to conform to Keras input style.\n",
    "        X = X.transpose(0, 2, 1)\n",
    "        \n",
    "        if X_val is not None:\n",
    "             X_val = X_val.transpose(0, 2, 1)\n",
    "             \n",
    "        validation_data = (X_val, y_val) if X_val is not None and y_val is not None else None\n",
    "\n",
    "        # ignore the number of instances, X.shape[0],\n",
    "        # just want the shape of each instance\n",
    "        self.input_shape = X.shape[1:]\n",
    "\n",
    "        self.model_ = self.build_model(self.input_shape, self.n_classes_)\n",
    "\n",
    "        if self.verbose:\n",
    "            self.model_.summary()\n",
    "\n",
    "        self.history = self.model_.fit(\n",
    "            X,\n",
    "            y_onehot,\n",
    "            batch_size=self.batch_size,\n",
    "            epochs=self.n_epochs,\n",
    "            verbose=self.verbose,\n",
    "            callbacks=self.callbacks,\n",
    "            validation_data=validation_data,\n",
    "            **kwargs\n",
    "        )\n",
    "\n",
    "        self._is_fitted = True\n",
    "\n",
    "        return self"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "c96bbf05",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"model_1\"\n",
      "__________________________________________________________________________________________________\n",
      " Layer (type)                   Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      " input_2 (InputLayer)           [(None, 46, 3)]      0           []                               \n",
      "                                                                                                  \n",
      " conv1d_3 (Conv1D)              (None, 46, 64)       1024        ['input_2[0][0]']                \n",
      "                                                                                                  \n",
      " batch_normalization_3 (BatchNo  (None, 46, 64)      256         ['conv1d_3[0][0]']               \n",
      " rmalization)                                                                                     \n",
      "                                                                                                  \n",
      " activation_3 (Activation)      (None, 46, 64)       0           ['batch_normalization_3[0][0]']  \n",
      "                                                                                                  \n",
      " global_average_pooling1d_3 (Gl  (None, 64)          0           ['activation_3[0][0]']           \n",
      " obalAveragePooling1D)                                                                            \n",
      "                                                                                                  \n",
      " reshape_2 (Reshape)            (None, 1, 64)        0           ['global_average_pooling1d_3[0][0\n",
      "                                                                 ]']                              \n",
      "                                                                                                  \n",
      " dense_5 (Dense)                (None, 1, 4)         256         ['reshape_2[0][0]']              \n",
      "                                                                                                  \n",
      " dense_6 (Dense)                (None, 1, 64)        256         ['dense_5[0][0]']                \n",
      "                                                                                                  \n",
      " multiply_2 (Multiply)          (None, 46, 64)       0           ['activation_3[0][0]',           \n",
      "                                                                  'dense_6[0][0]']                \n",
      "                                                                                                  \n",
      " conv1d_4 (Conv1D)              (None, 46, 128)      24704       ['multiply_2[0][0]']             \n",
      "                                                                                                  \n",
      " batch_normalization_4 (BatchNo  (None, 46, 128)     512         ['conv1d_4[0][0]']               \n",
      " rmalization)                                                                                     \n",
      "                                                                                                  \n",
      " activation_4 (Activation)      (None, 46, 128)      0           ['batch_normalization_4[0][0]']  \n",
      "                                                                                                  \n",
      " global_average_pooling1d_4 (Gl  (None, 128)         0           ['activation_4[0][0]']           \n",
      " obalAveragePooling1D)                                                                            \n",
      "                                                                                                  \n",
      " reshape_3 (Reshape)            (None, 1, 128)       0           ['global_average_pooling1d_4[0][0\n",
      "                                                                 ]']                              \n",
      "                                                                                                  \n",
      " dense_7 (Dense)                (None, 1, 8)         1024        ['reshape_3[0][0]']              \n",
      "                                                                                                  \n",
      " dense_8 (Dense)                (None, 1, 128)       1024        ['dense_7[0][0]']                \n",
      "                                                                                                  \n",
      " multiply_3 (Multiply)          (None, 46, 128)      0           ['activation_4[0][0]',           \n",
      "                                                                  'dense_8[0][0]']                \n",
      "                                                                                                  \n",
      " conv1d_5 (Conv1D)              (None, 46, 64)       8256        ['multiply_3[0][0]']             \n",
      "                                                                                                  \n",
      " permute_1 (Permute)            (None, 3, 46)        0           ['input_2[0][0]']                \n",
      "                                                                                                  \n",
      " batch_normalization_5 (BatchNo  (None, 46, 64)      256         ['conv1d_5[0][0]']               \n",
      " rmalization)                                                                                     \n",
      "                                                                                                  \n",
      " lstm_1 (LSTM)                  (None, 3)            600         ['permute_1[0][0]']              \n",
      "                                                                                                  \n",
      " activation_5 (Activation)      (None, 46, 64)       0           ['batch_normalization_5[0][0]']  \n",
      "                                                                                                  \n",
      " dropout_1 (Dropout)            (None, 3)            0           ['lstm_1[0][0]']                 \n",
      "                                                                                                  \n",
      " global_average_pooling1d_5 (Gl  (None, 64)          0           ['activation_5[0][0]']           \n",
      " obalAveragePooling1D)                                                                            \n",
      "                                                                                                  \n",
      " concatenate_1 (Concatenate)    (None, 67)           0           ['dropout_1[0][0]',              \n",
      "                                                                  'global_average_pooling1d_5[0][0\n",
      "                                                                 ]']                              \n",
      "                                                                                                  \n",
      " dense_9 (Dense)                (None, 24)           1632        ['concatenate_1[0][0]']          \n",
      "                                                                                                  \n",
      "==================================================================================================\n",
      "Total params: 39,800\n",
      "Trainable params: 39,288\n",
      "Non-trainable params: 512\n",
      "__________________________________________________________________________________________________\n",
      " 1704/29688 [>.............................] - ETA: 2:07 - loss: 1.1733 - accuracy: 0.6110"
     ]
    }
   ],
   "source": [
    "clf = MLSTMFCNWithValidation(\n",
    "    n_epochs=1,\n",
    "    attention=False,\n",
    "    batch_size= 32, \n",
    "    dilation_rate= 2, \n",
    "    filter_sizes= (64, 128, 64),\n",
    "    kernel_sizes= (5, 3, 1),\n",
    "    lstm_size= 3,\n",
    "    callbacks=callbacks_,\n",
    "    verbose=True\n",
    ")\n",
    "\n",
    "clf.fit(X_train, y_train, X_val, y_val)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6213f87e",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "DATA_TEST = \"./dataset/SITS-test-data-phase2-nolabel.csv.gz\"\n",
    "\n",
    "X_test, _ = read_data_sktime(DATA_TEST, univariate=use_univariate)\n",
    "# X_test, _ = read_data_sktime(DATA_TEST, univariate=True)\n",
    "\n",
    "\n",
    "# Make a prediction\n",
    "predictions = clf.predict(X_test)\n",
    "\n",
    "\n",
    "# Create a submission file for kaggle\n",
    "submission = pd.DataFrame({'PREDICTED': predictions})\n",
    "submission.index.name=\"ID\"\n",
    "\n",
    "filename = 'baseline_mlstm_submission_multivariate_new_params_phase2.csv'\n",
    "submission.to_csv(filename,index=True)\n",
    "print('Saved file: ' + filename)\n",
    "\n",
    "#Visualize the first 5 rows\n",
    "submission.head()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.7.5 64-bit ('se-algo-timeseries')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.5"
  },
  "vscode": {
   "interpreter": {
    "hash": "0c5b454c6344d901b60c31445b6f796b4db34a2188a0e869ba4b24428762ebb0"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}