Setup.ipynb

{
 "cells": [
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\Users\\Dinesh\\anaconda3\\lib\\site-packages\\paramiko\\transport.py:219: CryptographyDeprecationWarning: Blowfish has been deprecated\n",
      "  \"class\": algorithms.Blowfish,\n"
     ]
    }
   ],
   "source": [
    "# Utils\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import json\n",
    "import os\n",
    "import subprocess\n",
    "import IPython.display as ipd\n",
    "from pathlib import Path\n",
    "from collections import Counter\n",
    "\n",
    "# Library for audio processing\n",
    "import librosa\n",
    "import soundfile as sf\n",
    "\n",
    "# Visualizations and plots\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "# Options\n",
    "pd.set_option(\"display.max_columns\", None)\n",
    "pd.options.mode.chained_assignment = None"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Data Loading"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "60217\n"
     ]
    }
   ],
   "source": [
    "print(len(os.listdir('public_dataset/public_dataset')))"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Dataset Cleaning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>uuid</th>\n",
       "      <th>datetime</th>\n",
       "      <th>cough_detected</th>\n",
       "      <th>latitude</th>\n",
       "      <th>longitude</th>\n",
       "      <th>age</th>\n",
       "      <th>gender</th>\n",
       "      <th>respiratory_condition</th>\n",
       "      <th>fever_muscle_pain</th>\n",
       "      <th>status</th>\n",
       "      <th>quality_1</th>\n",
       "      <th>cough_type_1</th>\n",
       "      <th>dyspnea_1</th>\n",
       "      <th>wheezing_1</th>\n",
       "      <th>stridor_1</th>\n",
       "      <th>choking_1</th>\n",
       "      <th>congestion_1</th>\n",
       "      <th>nothing_1</th>\n",
       "      <th>diagnosis_1</th>\n",
       "      <th>severity_1</th>\n",
       "      <th>quality_2</th>\n",
       "      <th>cough_type_2</th>\n",
       "      <th>dyspnea_2</th>\n",
       "      <th>wheezing_2</th>\n",
       "      <th>stridor_2</th>\n",
       "      <th>choking_2</th>\n",
       "      <th>congestion_2</th>\n",
       "      <th>nothing_2</th>\n",
       "      <th>diagnosis_2</th>\n",
       "      <th>severity_2</th>\n",
       "      <th>quality_3</th>\n",
       "      <th>dyspnea_3</th>\n",
       "      <th>wheezing_3</th>\n",
       "      <th>stridor_3</th>\n",
       "      <th>choking_3</th>\n",
       "      <th>congestion_3</th>\n",
       "      <th>nothing_3</th>\n",
       "      <th>cough_type_3</th>\n",
       "      <th>diagnosis_3</th>\n",
       "      <th>severity_3</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>00039425-7f3a-42aa-ac13-834aaa2b6b92</td>\n",
       "      <td>2020-04-13T21:30:59.801831+00:00</td>\n",
       "      <td>0.9609</td>\n",
       "      <td>31.3</td>\n",
       "      <td>34.8</td>\n",
       "      <td>15.0</td>\n",
       "      <td>male</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>healthy</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0009eb28-d8be-4dc1-92bb-907e53bc5c7a</td>\n",
       "      <td>2020-04-12T04:02:18.159383+00:00</td>\n",
       "      <td>0.9301</td>\n",
       "      <td>40.0</td>\n",
       "      <td>-75.1</td>\n",
       "      <td>34.0</td>\n",
       "      <td>male</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>healthy</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0012c608-33d0-4ef7-bde3-75a0b1a0024e</td>\n",
       "      <td>2020-04-15T01:03:59.029326+00:00</td>\n",
       "      <td>0.0482</td>\n",
       "      <td>-16.5</td>\n",
       "      <td>-71.5</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>001328dc-ea5d-4847-9ccf-c5aa2a3f2d0f</td>\n",
       "      <td>2020-04-13T22:23:06.997578+00:00</td>\n",
       "      <td>0.9968</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>21.0</td>\n",
       "      <td>male</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>healthy</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>001c85a8-cc4d-4921-9297-848be52d4715</td>\n",
       "      <td>2020-04-17T15:24:35.822355+00:00</td>\n",
       "      <td>0.0735</td>\n",
       "      <td>40.6</td>\n",
       "      <td>-3.6</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                   uuid                          datetime  \\\n",
       "0  00039425-7f3a-42aa-ac13-834aaa2b6b92  2020-04-13T21:30:59.801831+00:00   \n",
       "1  0009eb28-d8be-4dc1-92bb-907e53bc5c7a  2020-04-12T04:02:18.159383+00:00   \n",
       "2  0012c608-33d0-4ef7-bde3-75a0b1a0024e  2020-04-15T01:03:59.029326+00:00   \n",
       "3  001328dc-ea5d-4847-9ccf-c5aa2a3f2d0f  2020-04-13T22:23:06.997578+00:00   \n",
       "4  001c85a8-cc4d-4921-9297-848be52d4715  2020-04-17T15:24:35.822355+00:00   \n",
       "\n",
       "   cough_detected  latitude  longitude   age gender respiratory_condition  \\\n",
       "0          0.9609      31.3       34.8  15.0   male                 False   \n",
       "1          0.9301      40.0      -75.1  34.0   male                  True   \n",
       "2          0.0482     -16.5      -71.5   NaN    NaN                   NaN   \n",
       "3          0.9968       NaN        NaN  21.0   male                 False   \n",
       "4          0.0735      40.6       -3.6   NaN    NaN                   NaN   \n",
       "\n",
       "  fever_muscle_pain   status quality_1 cough_type_1 dyspnea_1 wheezing_1  \\\n",
       "0             False  healthy       NaN          NaN       NaN        NaN   \n",
       "1             False  healthy       NaN          NaN       NaN        NaN   \n",
       "2               NaN      NaN       NaN          NaN       NaN        NaN   \n",
       "3             False  healthy       NaN          NaN       NaN        NaN   \n",
       "4               NaN      NaN       NaN          NaN       NaN        NaN   \n",
       "\n",
       "  stridor_1 choking_1 congestion_1 nothing_1 diagnosis_1 severity_1 quality_2  \\\n",
       "0       NaN       NaN          NaN       NaN         NaN        NaN       NaN   \n",
       "1       NaN       NaN          NaN       NaN         NaN        NaN       NaN   \n",
       "2       NaN       NaN          NaN       NaN         NaN        NaN       NaN   \n",
       "3       NaN       NaN          NaN       NaN         NaN        NaN       NaN   \n",
       "4       NaN       NaN          NaN       NaN         NaN        NaN       NaN   \n",
       "\n",
       "  cough_type_2 dyspnea_2 wheezing_2 stridor_2 choking_2 congestion_2  \\\n",
       "0          NaN       NaN        NaN       NaN       NaN          NaN   \n",
       "1          NaN       NaN        NaN       NaN       NaN          NaN   \n",
       "2          NaN       NaN        NaN       NaN       NaN          NaN   \n",
       "3          NaN       NaN        NaN       NaN       NaN          NaN   \n",
       "4          NaN       NaN        NaN       NaN       NaN          NaN   \n",
       "\n",
       "  nothing_2 diagnosis_2 severity_2 quality_3 dyspnea_3 wheezing_3 stridor_3  \\\n",
       "0       NaN         NaN        NaN       NaN       NaN        NaN       NaN   \n",
       "1       NaN         NaN        NaN       NaN       NaN        NaN       NaN   \n",
       "2       NaN         NaN        NaN       NaN       NaN        NaN       NaN   \n",
       "3       NaN         NaN        NaN       NaN       NaN        NaN       NaN   \n",
       "4       NaN         NaN        NaN       NaN       NaN        NaN       NaN   \n",
       "\n",
       "  choking_3 congestion_3 nothing_3 cough_type_3 diagnosis_3 severity_3  \n",
       "0       NaN          NaN       NaN          NaN         NaN        NaN  \n",
       "1       NaN          NaN       NaN          NaN         NaN        NaN  \n",
       "2       NaN          NaN       NaN          NaN         NaN        NaN  \n",
       "3       NaN          NaN       NaN          NaN         NaN        NaN  \n",
       "4       NaN          NaN       NaN          NaN         NaN        NaN  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "metadata = pd.read_csv('public_dataset/public_dataset/metadata_compiled.csv')\n",
    "metadata.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(20072, 40)\n",
      "Index(['uuid', 'datetime', 'cough_detected', 'latitude', 'longitude', 'age',\n",
      "       'gender', 'respiratory_condition', 'fever_muscle_pain', 'status',\n",
      "       'quality_1', 'cough_type_1', 'dyspnea_1', 'wheezing_1', 'stridor_1',\n",
      "       'choking_1', 'congestion_1', 'nothing_1', 'diagnosis_1', 'severity_1',\n",
      "       'quality_2', 'cough_type_2', 'dyspnea_2', 'wheezing_2', 'stridor_2',\n",
      "       'choking_2', 'congestion_2', 'nothing_2', 'diagnosis_2', 'severity_2',\n",
      "       'quality_3', 'dyspnea_3', 'wheezing_3', 'stridor_3', 'choking_3',\n",
      "       'congestion_3', 'nothing_3', 'cough_type_3', 'diagnosis_3',\n",
      "       'severity_3'],\n",
      "      dtype='object')\n"
     ]
    }
   ],
   "source": [
    "print(metadata.shape)\n",
    "print(metadata.columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "metadata = metadata.drop(['datetime', 'latitude', 'longitude', 'quality_1', 'cough_type_1', 'dyspnea_1', \n",
    "                                  'wheezing_1', 'stridor_1', 'choking_1', 'congestion_1', 'nothing_1', 'diagnosis_1', 'severity_1', \n",
    "                                  'quality_2', 'cough_type_2', 'dyspnea_2', 'wheezing_2', 'stridor_2', 'choking_2', 'congestion_2', \n",
    "                                  'nothing_2', 'diagnosis_2', 'severity_2', 'quality_3', 'cough_type_3', 'dyspnea_3', 'wheezing_3', \n",
    "                                  'stridor_3', 'choking_3', 'congestion_3', 'nothing_3', 'diagnosis_3', 'severity_3'], axis = 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "uuid                        0\n",
       "cough_detected              0\n",
       "age                      9604\n",
       "gender                   8758\n",
       "respiratory_condition    8758\n",
       "fever_muscle_pain        8758\n",
       "status                   8758\n",
       "dtype: int64"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "metadata.isna().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(11314, 7)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "metadata = metadata.dropna(subset=['status'])\n",
    "metadata.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(10468, 7)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "metadata = metadata.dropna(subset=['age'])\n",
    "metadata.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(6879, 7)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cough_tracks = metadata[metadata['cough_detected'] >= 0.8]\n",
    "cough_tracks.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "status\n",
       "COVID-19        468\n",
       "healthy        5309\n",
       "symptomatic    1102\n",
       "dtype: int64"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cough_tracks.groupby('status').size()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Saving the dataframe\n",
    "with open('public_dataset/public_dataset//metadata_coughtracks.csv', 'w', encoding = 'utf-8-sig') as f:\n",
    "  cough_tracks.to_csv(f)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "base",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}