Automated_covid_detector_validation.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Automated_covid_detector_validation.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/mr7495/COVID-CT-Code/blob/master/Automated_covid_detector_validation.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6JmhSAzXPUaX",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!nvidia-smi #show the allocated GPU"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5yqhPi-FS60r",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#Connect your Google Drive \n",
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7UAp6XlqPeou",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#upgrade and install essential libraries\n",
        "#Install essential libraries\n",
        "!pip install zipfile36\n",
        "!pip install git+https://github.com/mr7495/RetinaNet"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zw0BhQ7nPihU",
        "colab_type": "code",
        "outputId": "c7b67f86-7616-4b00-ff1f-7dc088bf2215",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "import keras\n",
        "import numpy as np\n",
        "import cv2\n",
        "import os\n",
        "import random\n",
        "import shutil\n",
        "import pandas as pd\n",
        "import csv\n",
        "import zipfile\n",
        "from keras import optimizers\n",
        "from keras.models import Sequential,Model\n",
        "from keras.layers import Dropout, Flatten, Dense,Input\n",
        "from keras.applications.resnet_v2 import ResNet50V2\n",
        "from keras.applications.xception import Xception\n",
        "from keras.applications.resnet50 import ResNet50\n",
        "from keras.applications.vgg16 import VGG16\n",
        "from keras.callbacks import ModelCheckpoint\n",
        "from keras.applications.imagenet_utils import preprocess_input\n",
        "from keras import backend as K\n",
        "from keras.preprocessing.image import ImageDataGenerator\n",
        "from keras.initializers import RandomNormal\n",
        "import keras.backend as k\n",
        "from sklearn.utils import shuffle\n",
        "import io\n",
        "from PIL import Image as pil_image\n",
        "from keras_retinanet import layers\n",
        "import keras.backend as k\n",
        "import keras_retinanet"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Using TensorFlow backend.\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LD5t4fhoziGf",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Add the COVID-CTset to your drive through this link:\n",
        "#https://drive.google.com/drive/folders/1xdk-mCkxCDNwsMAk2SGv203rY1mrbnPB?usp=sharing"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7x0sa1zuRPI9",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "archive = zipfile.ZipFile('drive/My Drive/COVID-CTset/COVID-CTset.zip') #Path to the shared dataset\n",
        "for file in archive.namelist():\n",
        "     archive.extract(file, './data') #Extract the data"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "89hm-PDB0SSb",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "fold_num=1 #Select Fold Number (You can predict and evaluate the patients in every folds)\n",
        "shape=(512,512,1) #shape of the dataset images (in TIFF format)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tpjAiyH3bKIu",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#Create the reader\n",
        "test_df = pd.read_csv('drive/My Drive/COVID-CTset/CSV/test{}.csv'.format(fold_num))#raed test csv file (For evaluating the final version of the trained network)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rRXGewOAMJ1q",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#The fully automated patient classification validation\n",
        "Path='Path to the trained models' # You can use this trained model : 'drive/My Drive/COVID-CTset/trained_networks/FPN-fold1.hdf5'\n",
        "\n",
        "full_reports=[]\n",
        "k.clear_session() #clear keras backend\n",
        "custom_object={'UpsampleLike': keras_retinanet.layers._misc.UpsampleLike} #Upsampling layer that was used in feature pyramid network\n",
        "model_name=Path\n",
        "net=keras.models.load_model(Path, custom_objects=custom_object)\n",
        "\n",
        "\n",
        "patients={'normal':{},'covid':{}} #Dictionary for refering to each patient images path\n",
        "for r,d,f in os.walk('data'): \n",
        "    for file in f:\n",
        "        if'.tif' in file:\n",
        "            full_add=os.path.join(r,file)\n",
        "            if 'SR_2' in full_add: #Select only SR_2 folder\n",
        "              index1=full_add.index('patient')\n",
        "              index2=full_add.index('SR_')\n",
        "              if 'covid' in full_add:\n",
        "                if full_add[index1:index2-1] not in patients['covid']: #full_add[index1:index2-1] shows the patient id\n",
        "                    patients['covid'][full_add[index1:index2-1]]=[]\n",
        "                patients['covid'][full_add[index1:index2-1]].append(full_add) #add the images of a patient\n",
        "              elif 'normal' in full_add:\n",
        "                if full_add[index1:index2-1] not in patients['normal']:\n",
        "                    patients['normal'][full_add[index1:index2-1]]=[]\n",
        "                patients['normal'][full_add[index1:index2-1]].append(full_add)\n",
        "\n",
        "\n",
        "tp=0 #All True Positives\n",
        "fp=0 #All False Positives\n",
        "ctp=0 #COVID True Postivies\n",
        "cfp=0 #COVID False Postivies\n",
        "ntp=0 #Normal True Postivies\n",
        "nfp=0 #Normal False Postivies\n",
        "covid_label=0 # index of COViD-19 class \n",
        "normal_label=1  # index of normal class \n",
        "results={}\n",
        "for patientid in patients['normal']:\n",
        "  results[patientid]={'covid-detect':0,'normal-detect':0,'gt':'normal','decision':0}\n",
        "  for img_name in patients['normal'][patientid]: #read images of a normal person CT scans\n",
        "    img=cv2.imread(img_name,cv2.IMREAD_UNCHANGED)\n",
        "    pred_ind=np.argmax(net.predict(np.expand_dims(np.expand_dims(img,axis=0),axis=3))[0]) #predicted label\n",
        "    if pred_ind==normal_label:\n",
        "      results[patientid]['normal-detect']+=1\n",
        "    else:\n",
        "      results[patientid]['covid-detect']+=1\n",
        "  all_imgs_num=len(patients['normal'][patientid]) \n",
        "  normal_detect_num=results[patientid]['normal-detect']\n",
        "  covid_detect_num=results[patientid]['covid-detect']\n",
        "  if covid_detect_num>=0.1*all_imgs_num: #if at least 10% of a patient CT scans be classified as covid\n",
        "    results[patientid]['decision']='covid'\n",
        "    fp+=1 \n",
        "    cfp+=1\n",
        "  else:\n",
        "    results[patientid]['decision']='normal'\n",
        "    tp+=1\n",
        "    ntp+=1\n",
        "###########################################\n",
        "for patientid in patients['covid']: #read images of a normal person CT scans\n",
        "  results[patientid]={'covid-detect':0,'normal-detect':0,'gt':'covid','decision':0}\n",
        "  for img_name in patients['covid'][patientid]:\n",
        "    img=cv2.imread(img_name,cv2.IMREAD_UNCHANGED)\n",
        "    pred_ind=np.argmax(net.predict(np.expand_dims(np.expand_dims(img,axis=0),axis=3))[0]) #predicted label\n",
        "    if pred_ind==covid_label:\n",
        "      results[patientid]['covid-detect']+=1\n",
        "    else:\n",
        "      results[patientid]['normal-detect']+=1\n",
        "  all_imgs_num=len(patients['covid'][patientid])\n",
        "  normal_detect_num=results[patientid]['normal-detect']\n",
        "  covid_detect_num=results[patientid]['covid-detect']\n",
        "  if covid_detect_num>=0.1*all_imgs_num: #if at least 10% of a patient CT scans be classified as covid\n",
        "    results[patientid]['decision']='covid'\n",
        "    tp+=1\n",
        "    ctp+=1\n",
        "  else:\n",
        "    results[patientid]['decision']='normal'\n",
        "    fp+=1\n",
        "    nfp+=1\n",
        "full_reports.append([model_name,tp,fp,ctp,cfp,ntp,nfp])\n",
        "print('tp: ',tp,'fp: ',fp) \n",
        "with open('{}-results.csv'.format(model_name), mode='w',newline='') as csv_file: #write the results\n",
        "    csvwriter = csv.writer(csv_file, delimiter=',', quotechar='\"',quoting=csv.QUOTE_MINIMAL)\n",
        "    csvwriter.writerow(['model_name','tp','fp','ctp','cfp','ntp','nfp'])\n",
        "    for row in full_reports:\n",
        "        csvwriter.writerow(row)  "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kA55vI0wgOWD",
        "colab_type": "code",
        "outputId": "b8acd2e3-5cfa-4dd9-b0d3-970292b05ff3",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "# Use this code to see which patient has been detected wrong\n",
        "for key in results:\n",
        "  if results[key]['gt']!=results[key]['decision']:\n",
        "    print(key)\n",
        "    print(results[key])\n"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "\"\\nfor key in results:\\n  if results[key]['gt']!=results[key]['decision']:\\n    print(key)\\n    print(results[key])\\n\""
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 14
        }
      ]
    }
  ]
}