chest-xray-detection-kaggle.ipynb

{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"source":"<a href=\"https://www.kaggle.com/code/divyam6969/chest-xray-classifier?scriptVersionId=125933446\" target=\"_blank\"><img align=\"left\" alt=\"Kaggle\" title=\"Open in Kaggle\" src=\"https://kaggle.com/static/images/open-in-kaggle.svg\"></a>","metadata":{},"cell_type":"markdown"},{"cell_type":"markdown","source":"# **This model will classify a chest xray into Pneumonial or Normal Category**","metadata":{}},{"cell_type":"markdown","source":"Pneumonia is a common and potentially life-threatening lung infection that affects millions of people worldwide. Early detection and treatment are critical for successful management of this disease. Artificial Intelligence (AI) and Machine Learning (ML) have shown tremendous promise in improving the accuracy and speed of medical diagnoses, including pneumonia detection. In this project, we will explore the application of AI and ML techniques to detect pneumonia from chest X-ray images. By developing a pneumonia detection model, we aim to assist healthcare professionals in providing accurate and timely diagnosis to patients, ultimately improving outcomes and saving lives.\n\nUsing this Pneumonia Detection Model, we can help doctors and medical professionals make faster and more accurate diagnoses of pneumonia, which can lead to better patient outcomes.\n\n\n[Github Link for similar project that will classify image into three categories](https://github.com/Divyam6969/Pneumonia-Detection-using-FastAI)\n\n","metadata":{}},{"cell_type":"markdown","source":"\n\n# Preprocessing","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19"}},{"cell_type":"code","source":"!pip install gradio","metadata":{"execution":{"iopub.status.busy":"2023-04-12T16:39:59.780298Z","iopub.execute_input":"2023-04-12T16:39:59.780652Z","iopub.status.idle":"2023-04-12T16:40:08.844034Z","shell.execute_reply.started":"2023-04-12T16:39:59.78062Z","shell.execute_reply":"2023-04-12T16:40:08.842672Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Importing Libraries\n(Will import kera library for image classification)","metadata":{}},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport seaborn as sns\nimport keras\nfrom keras.models import Sequential\nfrom keras.layers import Dense, Conv2D , MaxPool2D , Flatten , Dropout , BatchNormalization\nfrom keras.preprocessing.image import ImageDataGenerator\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import classification_report,confusion_matrix\nfrom keras.callbacks import ReduceLROnPlateau\nimport cv2\nimport os\nimport numpy as np\nimport pandas as pd\nimport gradio","metadata":{"execution":{"iopub.status.busy":"2023-04-12T16:40:08.84688Z","iopub.execute_input":"2023-04-12T16:40:08.847317Z","iopub.status.idle":"2023-04-12T16:40:08.854394Z","shell.execute_reply.started":"2023-04-12T16:40:08.847278Z","shell.execute_reply":"2023-04-12T16:40:08.853417Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data preprocessing\n(Getting training, validation and testing data set)","metadata":{}},{"cell_type":"code","source":"import cv2\nimport os\nimport numpy as np\n\nlabels = ['PNEUMONIA', 'NORMAL']\nimg_size = 150\n\ndef get_data(data_dir):\n    data = [] \n    for label in labels: \n        path = os.path.join(data_dir, label)\n        class_num = labels.index(label)\n        for img in os.listdir(path):\n            try:\n                img_arr = cv2.imread(os.path.join(path, img), cv2.IMREAD_GRAYSCALE)\n                resized_arr = cv2.resize(img_arr, (img_size, img_size)) # Reshaping images to preferred size\n                data.append([resized_arr, class_num])\n            except Exception as e:\n                print(e)\n    return np.array(data, dtype=object)\n\ntrain = get_data('/kaggle/input/chest-xray-pneumonia/chest_xray/train')\ntest = get_data('/kaggle/input/chest-xray-pneumonia/chest_xray/test')\nval = get_data('/kaggle/input/chest-xray-pneumonia/chest_xray/val')","metadata":{"execution":{"iopub.status.busy":"2023-04-12T16:44:31.222121Z","iopub.execute_input":"2023-04-12T16:44:31.223135Z","iopub.status.idle":"2023-04-12T16:45:17.126806Z","shell.execute_reply.started":"2023-04-12T16:44:31.223105Z","shell.execute_reply":"2023-04-12T16:45:17.125503Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x_train = []\ny_train = []\n\nx_val = []\ny_val = []\n\nx_test = []\ny_test = []\n\nfor feature, label in train:\n    x_train.append(feature)\n    y_train.append(label)\n\nfor feature, label in test:\n    x_test.append(feature)\n    y_test.append(label)\n    \nfor feature, label in val:\n    x_val.append(feature)\n    y_val.append(label)","metadata":{"execution":{"iopub.status.busy":"2023-04-12T16:43:38.10844Z","iopub.execute_input":"2023-04-12T16:43:38.108833Z","iopub.status.idle":"2023-04-12T16:43:38.123015Z","shell.execute_reply.started":"2023-04-12T16:43:38.108801Z","shell.execute_reply":"2023-04-12T16:43:38.12199Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"positives=[]\nnegatives=[]\nfor i in range(len(y_train)):\n    if y_train[i]:\n        positives.append(x_train[i])\n    else:\n        negatives.append(x_train[i])","metadata":{"execution":{"iopub.status.busy":"2023-04-12T16:43:35.704506Z","iopub.execute_input":"2023-04-12T16:43:35.704845Z","iopub.status.idle":"2023-04-12T16:43:35.713521Z","shell.execute_reply.started":"2023-04-12T16:43:35.704816Z","shell.execute_reply":"2023-04-12T16:43:35.712417Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.bar(labels, [len(negatives), len(positives)], color=[\"green\", \"blue\"])\nplt.title(\"Cases count in training data set\")\nplt.ylabel(\"count\")\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2023-04-12T16:43:33.901784Z","iopub.execute_input":"2023-04-12T16:43:33.902132Z","iopub.status.idle":"2023-04-12T16:43:34.019813Z","shell.execute_reply.started":"2023-04-12T16:43:33.902102Z","shell.execute_reply":"2023-04-12T16:43:34.018632Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.imshow(positives[0])\nplt.title(\"Pneumonia\")\nplt.show()\n\nplt.imshow(negatives[4], cmap=\"gray\")\nplt.title(\"Normal\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-04-12T16:43:31.484706Z","iopub.execute_input":"2023-04-12T16:43:31.485068Z","iopub.status.idle":"2023-04-12T16:43:31.882863Z","shell.execute_reply.started":"2023-04-12T16:43:31.485036Z","shell.execute_reply":"2023-04-12T16:43:31.882126Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Performing a grayscale normalization to reduce the effect of illumination's differences","metadata":{}},{"cell_type":"code","source":"# Normalize the data\nx_train = np.array(x_train) / 255\nx_val = np.array(x_val) / 255\nx_test = np.array(x_test) / 255","metadata":{"execution":{"iopub.status.busy":"2023-04-12T16:43:29.528783Z","iopub.execute_input":"2023-04-12T16:43:29.529135Z","iopub.status.idle":"2023-04-12T16:43:29.721947Z","shell.execute_reply.started":"2023-04-12T16:43:29.529104Z","shell.execute_reply":"2023-04-12T16:43:29.72096Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Resizing the data for CNN training and testing","metadata":{}},{"cell_type":"code","source":"# resize data for deep learning \nx_train = x_train.reshape(-1, img_size, img_size, 1)\ny_train = np.array(y_train)\n\nx_val = x_val.reshape(-1, img_size, img_size, 1)\ny_val = np.array(y_val)\n\nx_test = x_test.reshape(-1, img_size, img_size, 1)\ny_test = np.array(y_test)","metadata":{"execution":{"iopub.status.busy":"2023-04-12T16:43:27.335211Z","iopub.execute_input":"2023-04-12T16:43:27.335519Z","iopub.status.idle":"2023-04-12T16:43:27.341923Z","shell.execute_reply.started":"2023-04-12T16:43:27.335492Z","shell.execute_reply":"2023-04-12T16:43:27.340651Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x_test[0].shape\n","metadata":{"execution":{"iopub.status.busy":"2023-04-12T16:43:25.100586Z","iopub.execute_input":"2023-04-12T16:43:25.101554Z","iopub.status.idle":"2023-04-12T16:43:25.107256Z","shell.execute_reply.started":"2023-04-12T16:43:25.101521Z","shell.execute_reply":"2023-04-12T16:43:25.106324Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_train = y_train.reshape(-1,1)\ny_test = y_test.reshape(-1,1)\ny_val = y_val.reshape(-1,1)","metadata":{"execution":{"iopub.status.busy":"2023-04-12T16:43:22.713318Z","iopub.execute_input":"2023-04-12T16:43:22.713646Z","iopub.status.idle":"2023-04-12T16:43:22.718842Z","shell.execute_reply.started":"2023-04-12T16:43:22.713617Z","shell.execute_reply":"2023-04-12T16:43:22.717822Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data augmentation \n(Now I will make a model to classify these X-Rays)","metadata":{}},{"cell_type":"code","source":"datagen = ImageDataGenerator(\n        featurewise_center=False,\n        samplewise_center=False,\n        featurewise_std_normalization=False,\n        samplewise_std_normalization=False,\n        zca_whitening=False,\n        rotation_range = 30,  # randomly rotate images in the range (degrees, 0 to 180)\n        zoom_range = 0.2, # Randomly zoom image \n        width_shift_range=0.1,  # randomly shift images horizontally (fraction of total width)\n        height_shift_range=0.1,  # randomly shift images vertically (fraction of total height)\n        horizontal_flip = True,  # randomly flip images\n        vertical_flip=False)  # randomly flip images\n\n\ndatagen.fit(x_train)","metadata":{"execution":{"iopub.status.busy":"2023-04-12T16:43:18.682559Z","iopub.execute_input":"2023-04-12T16:43:18.682918Z","iopub.status.idle":"2023-04-12T16:43:19.090634Z","shell.execute_reply.started":"2023-04-12T16:43:18.682888Z","shell.execute_reply":"2023-04-12T16:43:19.089538Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import keras\nfrom keras.models import Sequential\nfrom keras.layers import Conv2D,MaxPooling2D,Flatten,Dropout,Dense,BatchNormalization,SpatialDropout2D","metadata":{"execution":{"iopub.status.busy":"2023-04-12T17:28:45.664363Z","iopub.execute_input":"2023-04-12T17:28:45.66482Z","iopub.status.idle":"2023-04-12T17:28:55.680631Z","shell.execute_reply.started":"2023-04-12T17:28:45.664775Z","shell.execute_reply":"2023-04-12T17:28:55.67912Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model=Sequential()\n\nmodel.add(Conv2D(32,(3,3),padding='same',activation='relu',input_shape=(128,128,3)))\nmodel.add(MaxPooling2D(2,2))\nmodel.add(SpatialDropout2D(0.1))\n\nmodel.add(Conv2D(32,(3,3),padding='same',activation='relu'))\nmodel.add(MaxPooling2D(2,2))\nmodel.add(BatchNormalization())\nmodel.add(SpatialDropout2D(0.2))\n\nmodel.add(Conv2D(32,(3,3),padding='same',activation='relu'))\nmodel.add(MaxPooling2D(2,2))\nmodel.add(BatchNormalization())\nmodel.add(SpatialDropout2D(0.2))\n\nmodel.add(Conv2D(32,(3,3),padding='same',activation='relu'))\nmodel.add(MaxPooling2D(2,2))\nmodel.add(BatchNormalization())\nmodel.add(SpatialDropout2D(0.3))\n\nmodel.add(Conv2D(32,(3,3),padding='same',activation='relu'))\nmodel.add(MaxPooling2D(2,2))\nmodel.add(BatchNormalization())\nmodel.add(SpatialDropout2D(0.3))\n\nmodel.add(Conv2D(32,(3,3),padding='same',activation='relu'))\nmodel.add(MaxPooling2D(2,2))\nmodel.add(BatchNormalization())\nmodel.add(SpatialDropout2D(0.3))\n\nmodel.add(Conv2D(32,(3,3),padding='same',activation='relu'))\nmodel.add(MaxPooling2D(2,2))\nmodel.add(BatchNormalization())\nmodel.add(SpatialDropout2D(0.5))\n\nmodel.add(Flatten())\nmodel.add(Dropout(0.5))\nmodel.add(Dense(128,activation='relu'))\nmodel.add(Dropout(0.5))\nmodel.add(Dense(128,activation='relu'))\nmodel.add(Dropout(0.5))\nmodel.add(Dense(1,activation='sigmoid'))\n\nmodel.compile(optimizer = \"rmsprop\" , loss = 'binary_crossentropy' , metrics = ['accuracy'])\nmodel.summary()\nmodel.compile(optimizer = \"rmsprop\" , \n            loss = 'binary_crossentropy' , \n            metrics = ['accuracy'])\n# model.summary()\nadam=keras.optimizers.Adam(learning_rate=0.001)\nmodel.compile(optimizer=adam,loss='binary_crossentropy',metrics=['accuracy'])\nmodel.summary()\nmodel.save_weights('kaggle/saved_model_ai/pneumoniadetection')","metadata":{"execution":{"iopub.status.busy":"2023-04-12T17:30:11.857647Z","iopub.execute_input":"2023-04-12T17:30:11.858069Z","iopub.status.idle":"2023-04-12T17:30:12.551105Z","shell.execute_reply.started":"2023-04-12T17:30:11.858035Z","shell.execute_reply":"2023-04-12T17:30:12.549416Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from keras.preprocessing.image import ImageDataGenerator\ntrain_datagen = ImageDataGenerator(\n        rescale=1./255,\n        shear_range=0.2,\n        zoom_range=0.2,\n        horizontal_flip=True)\n\ntest_datagen = ImageDataGenerator(rescale=1./255)\n\ntraining_set = train_datagen.flow_from_directory(\n        '/kaggle/input/chest-xray-pneumonia/chest_xray/train',\n        target_size=(128,128),\n        batch_size=16 ,\n        class_mode='binary')\n\nval_set = test_datagen.flow_from_directory(\n        '/kaggle/input/chest-xray-pneumonia/chest_xray/test',\n        target_size=(128,128),\n        batch_size=16,\n        class_mode='binary')\n\nmodel.fit_generator(\n        training_set,\n        steps_per_epoch=326,\n        epochs=128,\n        validation_data=val_set,\n        validation_steps=39)\n\n","metadata":{"execution":{"iopub.status.busy":"2023-04-12T17:30:59.574154Z","iopub.execute_input":"2023-04-12T17:30:59.574576Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Evaluation Matrix","metadata":{}},{"cell_type":"code","source":"print(\"Loss of the model is - \" , model.evaluate(x_test,y_test)[0])\nprint(\"Accuracy of the model is - \" , model.evaluate(x_test,y_test)[1]*100 , \"%\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"epochs = list(range(10))\nfig , ax = plt.subplots(1,2)\ntrain_acc = history.history['accuracy']\ntrain_loss = history.history['loss']\nval_acc = history.history['val_accuracy']\nval_loss = history.history['val_loss']\nfig.set_size_inches(20,10)\n\nax[0].plot(epochs , train_acc , 'go-' , label = 'Training Accuracy')\nax[0].plot(epochs , val_acc , 'ro-' , label = 'Validation Accuracy')\nax[0].set_title('Training & Validation Accuracy')\nax[0].legend()\nax[0].set_xlabel(\"Epochs\")\nax[0].set_ylabel(\"Accuracy\")\n\nax[1].plot(epochs , train_loss , 'g-o' , label = 'Training Loss')\nax[1].plot(epochs , val_loss , 'r-o' , label = 'Validation Loss')\nax[1].set_title('Testing Accuracy & Loss')\nax[1].legend()\nax[1].set_xlabel(\"Epochs\")\nax[1].set_ylabel(\"Training & Validation Loss\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-04-12T16:42:51.236638Z","iopub.execute_input":"2023-04-12T16:42:51.237549Z","iopub.status.idle":"2023-04-12T16:42:51.592672Z","shell.execute_reply.started":"2023-04-12T16:42:51.237509Z","shell.execute_reply":"2023-04-12T16:42:51.591806Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predictions = model.predict(x_test) \nfor i in range(len(predictions)): \n    predictions[i] = 1 if predictions[i]>0.5 else 0","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(classification_report(y_test, \n                            predictions, \n                            target_names = ['Pneumonia (Class 0)','Normal (Class 1)']))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Confusion Matrix","metadata":{}},{"cell_type":"code","source":"cm = confusion_matrix(y_test,predictions)\ncm = pd.DataFrame(cm , index = ['0','1'] , columns = ['0','1'])\ncm","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.heatmap(cm, cmap=\"Blues\", annot=True, xticklabels = labels,yticklabels = labels)\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def pneumoniaPrediction(img):\n    if img is not None:\n        img = np.array(img)/255\n        img = np.array(img)/255\n        img = img.reshape(-1, 150, 150, 1)\n        isPneumonic = model.predict(img)[0]\n        imgClass = \"Normal\" if isPneumonic<0.5 else \"Pneumonic\"\n        return imgClass\n        # perform prediction and return label\n    else:\n        return None\n    ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pr = model.predict(x_test)\nfor i in range(len(pr)):\n    if pr[i]>0.5:\n        pr[i]=1\n    else:\n        pr[i]=0","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"img = gradio.inputs.Image(shape=(150, 150))\nlabel = gradio.outputs.Label(num_top_classes=1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"interface = gradio.Interface(fn = pneumoniaPrediction,\n                            title = \"Pneumonia Detection using Chest X-Ray\",\n                            inputs = img,\n                            outputs = label,\n                            interpretation = \"default\")\ninterface.launch(debug=True, share=True)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\n#Checking for individual images\n\ntest_image = image.load_img('enter image name', target_size = (128, 128))\n\ntest_image\n\ntest_image = image.img_to_array(test_image)\n\ntest_image = np.expand_dims(test_image, axis = 0)\n\nresult = model.predict_classes(test_image)\n\nprint(result)\n'''\n","metadata":{},"execution_count":null,"outputs":[]}]}