approach_2.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Skin Lesion Classifier - Approach 2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Google Colab"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Run this cell to mount Google Drive for Colab\n",
    "from google.colab import drive\n",
    "drive.mount('/content/drive/')\n",
    "# !ls '/content/drive/My Drive/Colab Notebooks'\n",
    "\n",
    "import os\n",
    "os.chdir('/content/drive/My Drive/Colab Notebooks/isic-2019')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# !cp '/content/drive/My Drive/Colab Notebooks/ISIC_2019_Training_Input.zip' '/home/ISIC_2019_Training_Input.zip'\n",
    "# !cp '/content/drive/My Drive/Colab Notebooks/ISIC_2019_Training_GroundTruth.csv' '/home/ISIC_2019_Training_GroundTruth.csv'\n",
    "# !cp '/content/drive/My Drive/Colab Notebooks/Out_Distribution.zip' '/home/Out_Distribution.zip'\n",
    "# !unzip -qq '/home/ISIC_2019_Training_Input.zip' -d '/home'\n",
    "# !unzip -qq '/home/Out_Distribution.zip' -d '/home'\n",
    "# !cp '/content/drive/My Drive/Colab Notebooks/ISIC_2019_Test_Input.zip' '/home/ISIC_2019_Test_Input.zip'\n",
    "# !unzip -qq '/home/ISIC_2019_Test_Input.zip' -d '/home'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # Ref https://docs.fast.ai/performance.html\n",
    "# !pip3 uninstall -y pillow pil jpeg libtiff libjpeg-turbo\n",
    "# !CFLAGS=\"${CFLAGS} -mavx2\" pip3 install --upgrade --no-cache-dir --force-reinstall --no-binary :all: --compile pillow-simd"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Common Parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import matplotlib.pyplot as plt\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "%matplotlib inline\n",
    "\n",
    "data_folder = 'C:\\ISIC_2019'\n",
    "# data_folder = '/home'\n",
    "# data_folder = '/home/jupyter'\n",
    "\n",
    "model_folder = 'models_2'\n",
    "history_folder = 'history_2'\n",
    "pred_result_folder_val = 'val_predict_results_2'\n",
    "\n",
    "# How to handle SVG fonts\n",
    "plt.rcParams['svg.fonttype'] = 'none'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Import Training Data and Out-of-Distribution Dataset\n",
    "\n",
    "* All images of the out-of-distribution dataset are regarded as the unknown category.\n",
    "* Two duplicate images (ISIC_0069013, ISIC_0067980) of the training data are removed."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import Counter\n",
    "from data import load_isic_training_and_out_dist_data\n",
    "from visuals import autolabel\n",
    "\n",
    "training_image_folder = os.path.join(data_folder, 'ISIC_2019_Training_Input')\n",
    "ground_truth_file = os.path.join(data_folder, 'ISIC_2019_Training_GroundTruth_DuplicateRemoved.csv')\n",
    "out_dist_image_folder = os.path.join(data_folder, 'Out_Distribution')\n",
    "\n",
    "df_ground_truth, category_names = load_isic_training_and_out_dist_data(training_image_folder, ground_truth_file, out_dist_image_folder)\n",
    "\n",
    "category_num = len(category_names)\n",
    "print(\"Number of categories: {}\".format(category_num))\n",
    "print(category_names, '\\n')\n",
    "\n",
    "# mapping from category to index\n",
    "print('Category to Index:')\n",
    "category_to_index = dict((c, i) for i, c in enumerate(category_names))\n",
    "print(category_to_index, '\\n')\n",
    "\n",
    "count_per_category = Counter(df_ground_truth['category'])\n",
    "total_sample_count = sum(count_per_category.values())\n",
    "print(\"Original training data has {} samples.\".format(total_sample_count))\n",
    "for i, c in enumerate(category_names):\n",
    "    print(\"'%s':\\t%d\\t(%.2f%%)\" % (c, count_per_category[i], count_per_category[i]*100/total_sample_count))\n",
    "\n",
    "# Create a bar chart\n",
    "fig, ax = plt.subplots(figsize=(8, 5))\n",
    "fig.patch.set_facecolor('white')\n",
    "ax.set(xlabel='Category', ylabel='Number of Images')\n",
    "rects = plt.bar(category_names, [count_per_category[i] for i in range(category_num)])\n",
    "autolabel(ax, rects)\n",
    "fig.tight_layout()\n",
    "\n",
    "df_ground_truth.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Shuffle and Split Original Training Data into Training  and Validation Sets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from data import train_validation_split\n",
    "from visuals import plot_grouped_2bars\n",
    "\n",
    "df_train, df_val = train_validation_split(df_ground_truth)\n",
    "\n",
    "# Training Set\n",
    "sample_count_train = df_train.shape[0]\n",
    "print(\"Training set has {} samples.\".format(sample_count_train))\n",
    "count_per_category_train = Counter(df_train['category'])\n",
    "for i, c in enumerate(category_names):\n",
    "    print(\"'%s':\\t%d\\t(%.2f%%)\" % (c, count_per_category_train[i], count_per_category_train[i]*100/sample_count_train))\n",
    "\n",
    "# Validation Set\n",
    "sample_count_val = df_val.shape[0]\n",
    "print(\"\\nValidation set has {} samples.\".format(sample_count_val))\n",
    "count_per_category_val = Counter(df_val['category'])\n",
    "for i, c in enumerate(category_names):\n",
    "    print(\"'%s':\\t%d\\t(%.2f%%)\" % (c, count_per_category_val[i], count_per_category_val[i]*100/sample_count_val))\n",
    "\n",
    "plot_grouped_2bars(\n",
    "    scalars=[[count_per_category_train[i] for i in range(category_num)],\n",
    "             [count_per_category_val[i] for i in range(category_num)]],\n",
    "    scalarlabels=['Training', 'Validation'],\n",
    "    xticklabels=category_names,\n",
    "    xlabel='Category',\n",
    "    ylabel='Number of Images',\n",
    "    title='Distribution of Training and Validation Sets'\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Class Weights based on the Traning Set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from data import compute_class_weight_dict\n",
    "\n",
    "class_weight_dict, class_weights = compute_class_weight_dict(df_train)\n",
    "print('Class Weights Dictionary:')\n",
    "print(class_weight_dict)\n",
    "\n",
    "# Create a bar chart\n",
    "fig, ax = plt.subplots(figsize=(8, 5))\n",
    "fig.patch.set_facecolor('white')\n",
    "ax.set_title('Class Weights')\n",
    "ax.set(xlabel='Category', ylabel='Weight')\n",
    "plt.bar(category_names, [class_weight_dict[i] for i in range(category_num)]);"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Per-channel Mean and Standard Deviation over the Training Set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from utils import calculate_mean_std\n",
    "\n",
    "### Uncomment below codes to calculate per-channel mean and standard deviation over the training set\n",
    "rgb_mean, rgb_std = calculate_mean_std(df_train['path'])\n",
    "print(\"Mean:{}\\nSTD:{}\".format(rgb_mean, rgb_std))\n",
    "\n",
    "# Output was:\n",
    "# Mean:[0.6296238064420809, 0.5202302775509949, 0.5032952297664738]\n",
    "# STD:[0.24130893564897463, 0.22150225707876617, 0.2297057828857888]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Transfer Learning"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Train Models by Transfer Learning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "!python3 main.py /home --approach 2 --modelfolder models_2 --training --epoch 100 --batchsize 32 --maxqueuesize 10 --model DenseNet201 Xception ResNeXt50"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Complexity Graph of Transfer Learning Models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "from visuals import *\n",
    "\n",
    "model_names = ['DenseNet201', 'Xception', 'ResNeXt50']\n",
    "feature_extract_epochs = 3\n",
    "\n",
    "for model_name in model_names:\n",
    "    file_path = os.path.join(history_folder, \"{}.training.csv\".format(model_name))\n",
    "    if os.path.exists(file_path):\n",
    "        fig = plot_complexity_graph(csv_file=file_path,\n",
    "                              title=\"Complexity Graph of {}\".format(model_name),\n",
    "                              figsize=(14, 10),\n",
    "                              feature_extract_epochs=feature_extract_epochs)\n",
    "        fig.savefig(os.path.join(history_folder, \"{}.training.svg\".format(model_name)), format='svg',\n",
    "                    bbox_inches='tight', pad_inches=0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Predict Validation Set"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Predict Validation Set by Different Models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# !python3 main.py /home --approach 2 --modelfolder models_2 --predval --predvalresultfolder predict_results_2 --model Xception DenseNet201 ResNeXt50\n",
    "!python main.py C:\\ISIC_2019 --approach 2 --modelfolder models_2 --predval --predvalresultfolder predict_results_2 --model Xception DenseNet201 ResNeXt50"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Ensemble Models' Predictions on Validation Set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from utils import ensemble_predictions\n",
    "\n",
    "ensemble_predictions(pred_result_folder_val, category_names)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load Prediction Results on Validation Set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from sklearn.metrics import balanced_accuracy_score, recall_score\n",
    "from visuals import plot_confusion_matrix\n",
    "from keras.utils import np_utils\n",
    "from keras_numpy_backend import categorical_crossentropy\n",
    "\n",
    "model_names = ['DenseNet201', 'Xception', 'ResNeXt50', 'Ensemble']\n",
    "postfix = 'best_balanced_acc'\n",
    "print('Model selection criteria: ', postfix)\n",
    "\n",
    "for model_name in model_names:\n",
    "    # Load predicted results\n",
    "    file_path = os.path.join(pred_result_folder_val, \"{}_{}.csv\".format(model_name, postfix))\n",
    "    if not os.path.exists(file_path):\n",
    "        continue\n",
    "\n",
    "    print(\"========== {} ==========\".format(model_name))\n",
    "    df = pd.read_csv(file_path)\n",
    "    y_true = df['category']\n",
    "    y_pred = df['pred_category']\n",
    "\n",
    "    # Compute Balanced Accuracy\n",
    "    print('balanced_accuracy_score: ', balanced_accuracy_score(y_true, y_pred))\n",
    "    print('macro recall_score: ', recall_score(y_true, y_pred, average='macro'))\n",
    "\n",
    "    # Compute categorical_crossentropy\n",
    "    y_true_onehot = np_utils.to_categorical(df['category'], num_classes=category_num)\n",
    "    y_pred_onehot = np.array(df.iloc[:,1:1+category_num])\n",
    "    print('categorical_crossentropy: ',\n",
    "          np.average(categorical_crossentropy(y_true_onehot, y_pred_onehot)))\n",
    "\n",
    "    # Compute weighted categorical_crossentropy\n",
    "    print('weighted categorical_crossentropy: ',\n",
    "          np.average(categorical_crossentropy(y_true_onehot, y_pred_onehot, class_weights=class_weights)))\n",
    "\n",
    "    # Confusion Matrix\n",
    "    fig = plot_confusion_matrix(y_true, y_pred, category_names, normalize=True,\n",
    "                                title=\"Confusion Matrix of {}\".format(model_name),\n",
    "                                figsize=(8, 6))\n",
    "    fig.savefig(os.path.join(pred_result_folder_val, \"{}_{}.svg\".format(model_name, postfix)), format='svg',\n",
    "                bbox_inches='tight', pad_inches=0)\n",
    "    print('')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "from visuals import plot_grouped_2bars\n",
    "\n",
    "sample_count_val = y_true.shape[0]\n",
    "print(\"Validation set has {} samples.\\n\".format(sample_count_val))\n",
    "\n",
    "print('========== Ground Truth ==========')\n",
    "count_true = Counter(y_true)\n",
    "for i, c in enumerate(category_names):\n",
    "    print(\"'%s':\\t%d\\t(%.2f%%)\" % (c, count_true[i], count_true[i]*100/sample_count_val))\n",
    "\n",
    "for model_name in model_names:\n",
    "    # Load predicted results\n",
    "    file_path = os.path.join(pred_result_folder_val, \"{}_{}.csv\".format(model_name, postfix))\n",
    "    if not os.path.exists(file_path):\n",
    "        continue\n",
    "\n",
    "    print(\"\\n========== {} Prediction ==========\".format(model_name))\n",
    "    df = pd.read_csv(file_path)\n",
    "    y_pred = df['pred_category']\n",
    "    \n",
    "    count_pred = Counter(y_pred)\n",
    "    for i, c in enumerate(category_names):\n",
    "        print(\"'%s':\\t%d\\t(%.2f%%)\" % (c, count_pred[i], count_pred[i]*100/sample_count_val))\n",
    "\n",
    "    # Plot Prediction Distribution\n",
    "    plot_grouped_2bars(\n",
    "        scalars=[[count_true[i] for i in range(category_num)],\n",
    "                 [count_pred[i] for i in range(category_num)]],\n",
    "        scalarlabels=['Ground Truth', 'Prediction'],\n",
    "        xticklabels=category_names,\n",
    "        xlabel='Category',\n",
    "        ylabel='Number of Images',\n",
    "        title=\"Prediction Distribution of {}\".format(model_name)\n",
    "    )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Test Data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Predict Test Data by Different Models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!python3 main.py /home --approach 2 --modelfolder models_2 --predtest --predtestresultfolder test_predict_results_2 --model DenseNet201 Xception ResNeXt50"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}