1.b) Data Extraction - Final Step_(NOT COLAB).ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "gew0QNYTa5Na"
   },
   "source": [
    "# Import Needed Modules"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "#!pip install ee\n",
    "#!pip install earthengine-api"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "id": "kAU3u37XcRhr"
   },
   "outputs": [],
   "source": [
    "import ee  # Import the Earth Engine library\n",
    "import numpy as np  # Import NumPy for numerical operations\n",
    "import pandas as pd  # Import Pandas for data handling\n",
    "\n",
    "from sklearn.model_selection import train_test_split  # Import train-test split function\n",
    "from sklearn.metrics import accuracy_score  # Import accuracy_score for model evaluation\n",
    "from sklearn.ensemble import RandomForestClassifier  # Import RandomForestClassifier from scikit-learn\n",
    "import random  # Import the random library for generating random values\n",
    "import matplotlib.pyplot as plt  # Import Matplotlib for plotting\n",
    "\n",
    "# Set seed for reproducability\n",
    "SEED = 2023\n",
    "random.seed(SEED)\n",
    "np.random.seed(SEED)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Authenticate and initialize Earth Engine."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "svpe_-b2dSzV",
    "outputId": "91afc165-56e5-4544-9872-149c35dfadfe"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<p>To authorize access needed by Earth Engine, open the following\n",
       "        URL in a web browser and follow the instructions:</p>\n",
       "        <p><a href=https://code.earthengine.google.com/client-auth?scopes=https%3A//www.googleapis.com/auth/earthengine%20https%3A//www.googleapis.com/auth/devstorage.full_control&request_id=WZjvV1zU3DCFM7fJXrsYBwxMkqzUUo67zzthkbDhtuI&tc=CFmea9n2VmPD-fmrxzQoYsrtQChampsNd0GQWefHuPI&cc=xmgCLIZHkxf-GDBXA_LT6oIdCztzDfwL9qbTOt6MyEU>https://code.earthengine.google.com/client-auth?scopes=https%3A//www.googleapis.com/auth/earthengine%20https%3A//www.googleapis.com/auth/devstorage.full_control&request_id=WZjvV1zU3DCFM7fJXrsYBwxMkqzUUo67zzthkbDhtuI&tc=CFmea9n2VmPD-fmrxzQoYsrtQChampsNd0GQWefHuPI&cc=xmgCLIZHkxf-GDBXA_LT6oIdCztzDfwL9qbTOt6MyEU</a></p>\n",
       "        <p>The authorization workflow will generate a code, which you should paste in the box below.</p>\n",
       "        "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Enter verification code: 4/1AfJohXk11pH9ooJ1jFbMuGvqCNZLkKJ9uw-Ks1oA-zbD8HeDtME9Ka8uD9g\n",
      "\n",
      "Successfully saved authorization token.\n"
     ]
    }
   ],
   "source": [
    "# Get authetication token and sign in to Google Earth Engine\n",
    "ee.Authenticate()\n",
    "ee.Initialize()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "CCsnR8lSdumy"
   },
   "source": [
    "# Extract the bands"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Afghanistan_train_afghan.csv saved.\n",
      "Afghanistan_test_afghan.csv saved.\n",
      "Afghanistan_additional.csv saved.\n",
      "Iran_train_iran.csv saved.\n",
      "Iran_test_iran.csv saved.\n",
      "Sudan_train_sudan.csv saved.\n",
      "Sudan_test_sudan.csv saved.\n",
      "Wall time: 2min 38s\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>lat</th>\n",
       "      <th>lon</th>\n",
       "      <th>B1</th>\n",
       "      <th>B2</th>\n",
       "      <th>B3</th>\n",
       "      <th>B4</th>\n",
       "      <th>B5</th>\n",
       "      <th>B6</th>\n",
       "      <th>B7</th>\n",
       "      <th>B8</th>\n",
       "      <th>B8A</th>\n",
       "      <th>B9</th>\n",
       "      <th>B11</th>\n",
       "      <th>B12</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ID_SOYSG7W04UH3</td>\n",
       "      <td>14.431884</td>\n",
       "      <td>33.399991</td>\n",
       "      <td>618.928571</td>\n",
       "      <td>778.793651</td>\n",
       "      <td>1072.063492</td>\n",
       "      <td>1291.865079</td>\n",
       "      <td>1565.880952</td>\n",
       "      <td>1985.888889</td>\n",
       "      <td>2175.746032</td>\n",
       "      <td>2072.095238</td>\n",
       "      <td>2302.619048</td>\n",
       "      <td>2702.007937</td>\n",
       "      <td>1959.325397</td>\n",
       "      <td>1551.007937</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ID_EAP7EXXV8ZDE</td>\n",
       "      <td>14.281866</td>\n",
       "      <td>33.441224</td>\n",
       "      <td>541.193548</td>\n",
       "      <td>691.467742</td>\n",
       "      <td>957.580645</td>\n",
       "      <td>1101.645161</td>\n",
       "      <td>1381.838710</td>\n",
       "      <td>1869.854839</td>\n",
       "      <td>2032.129032</td>\n",
       "      <td>2118.274194</td>\n",
       "      <td>2150.822581</td>\n",
       "      <td>2553.629032</td>\n",
       "      <td>1784.435484</td>\n",
       "      <td>1408.661290</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ID_QPRX1TUQVGHU</td>\n",
       "      <td>14.399365</td>\n",
       "      <td>33.109566</td>\n",
       "      <td>884.450820</td>\n",
       "      <td>1002.942623</td>\n",
       "      <td>1321.467213</td>\n",
       "      <td>1722.245902</td>\n",
       "      <td>2080.459016</td>\n",
       "      <td>2217.959016</td>\n",
       "      <td>2369.434426</td>\n",
       "      <td>2361.122951</td>\n",
       "      <td>2512.713115</td>\n",
       "      <td>2963.934426</td>\n",
       "      <td>2905.065574</td>\n",
       "      <td>2575.368852</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ID_C78YQ32G1KO9</td>\n",
       "      <td>14.196346</td>\n",
       "      <td>33.507160</td>\n",
       "      <td>616.580645</td>\n",
       "      <td>703.838710</td>\n",
       "      <td>1006.983871</td>\n",
       "      <td>1080.596774</td>\n",
       "      <td>1430.758065</td>\n",
       "      <td>1800.725806</td>\n",
       "      <td>1995.725806</td>\n",
       "      <td>2262.096774</td>\n",
       "      <td>2000.935484</td>\n",
       "      <td>2179.258065</td>\n",
       "      <td>1345.370968</td>\n",
       "      <td>1013.806452</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ID_M5X39UIEM64N</td>\n",
       "      <td>14.348251</td>\n",
       "      <td>33.252488</td>\n",
       "      <td>476.050847</td>\n",
       "      <td>613.661017</td>\n",
       "      <td>880.949153</td>\n",
       "      <td>1094.271186</td>\n",
       "      <td>1357.288136</td>\n",
       "      <td>1875.169492</td>\n",
       "      <td>2161.457627</td>\n",
       "      <td>2244.728814</td>\n",
       "      <td>2261.830508</td>\n",
       "      <td>2458.830508</td>\n",
       "      <td>1692.254237</td>\n",
       "      <td>1336.779661</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                id        lat        lon          B1           B2  \\\n",
       "0  ID_SOYSG7W04UH3  14.431884  33.399991  618.928571   778.793651   \n",
       "1  ID_EAP7EXXV8ZDE  14.281866  33.441224  541.193548   691.467742   \n",
       "2  ID_QPRX1TUQVGHU  14.399365  33.109566  884.450820  1002.942623   \n",
       "3  ID_C78YQ32G1KO9  14.196346  33.507160  616.580645   703.838710   \n",
       "4  ID_M5X39UIEM64N  14.348251  33.252488  476.050847   613.661017   \n",
       "\n",
       "            B3           B4           B5           B6           B7  \\\n",
       "0  1072.063492  1291.865079  1565.880952  1985.888889  2175.746032   \n",
       "1   957.580645  1101.645161  1381.838710  1869.854839  2032.129032   \n",
       "2  1321.467213  1722.245902  2080.459016  2217.959016  2369.434426   \n",
       "3  1006.983871  1080.596774  1430.758065  1800.725806  1995.725806   \n",
       "4   880.949153  1094.271186  1357.288136  1875.169492  2161.457627   \n",
       "\n",
       "            B8          B8A           B9          B11          B12  \n",
       "0  2072.095238  2302.619048  2702.007937  1959.325397  1551.007937  \n",
       "1  2118.274194  2150.822581  2553.629032  1784.435484  1408.661290  \n",
       "2  2361.122951  2512.713115  2963.934426  2905.065574  2575.368852  \n",
       "3  2262.096774  2000.935484  2179.258065  1345.370968  1013.806452  \n",
       "4  2244.728814  2261.830508  2458.830508  1692.254237  1336.779661  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "# Authenticate to the Earth Engine servers\n",
    "ee.Initialize()\n",
    "\n",
    "# Function to mask out cloudy pixels\n",
    "def mask_clouds(image):\n",
    "    # Select the QA60 band from the image\n",
    "    QA60 = image.select(['QA60'])\n",
    "\n",
    "    # Create a cloud mask where cloudy pixels are set to 0 and others to 1\n",
    "    cloud_mask = QA60.bitwiseAnd(1 << 10).eq(0)\n",
    "\n",
    "    # Update the image mask with the cloud mask\n",
    "    return image.updateMask(cloud_mask)\n",
    "\n",
    "# Define the time_maps dictionary for your regions\n",
    "time_maps = {\n",
    "    'Afghanistan': {\n",
    "        'start': '2022-04-01',\n",
    "        'end': '2022-04-30'\n",
    "    },\n",
    "    'Iran': {\n",
    "        'start': '2019-07-01',\n",
    "        'end': '2020-06-30'\n",
    "    },\n",
    "    'Sudan': {\n",
    "        'start': '2019-07-01',\n",
    "        'end': '2020-06-30'\n",
    "    }\n",
    "}\n",
    "\n",
    "BANDS = ['B1', 'B2', 'B3', 'B4', 'B5', 'B6', 'B7', 'B8', 'B8A', 'B9', 'B11', 'B12']\n",
    "\n",
    "# List of countries and files\n",
    "country_files = {\n",
    "    'Afghanistan': ['train_afghan.csv', 'test_afghan.csv','additional.csv'],\n",
    "    'Iran': ['train_iran.csv', 'test_iran.csv'],\n",
    "    'Sudan': ['train_sudan.csv', 'test_sudan.csv']\n",
    "}\n",
    "\n",
    "# Iterate through countries and their corresponding files\n",
    "for country, files in country_files.items():\n",
    "    # Iterate through the files for the current country\n",
    "    for file in files:\n",
    "        # Read the CSV table\n",
    "        csv_data = pd.read_csv(file)\n",
    "        \n",
    "        # Define the tileScale parameter\n",
    "        tile_scale = 2  # You can adjust this value as needed\n",
    "        \n",
    "        # Extract latitude and longitude from the CSV data\n",
    "        point_geometries = [ee.Geometry.Point(lon, lat) for lon, lat in zip(csv_data['lon'], csv_data['lat'])]\n",
    "        \n",
    "        # Create an ImageCollection based on the country and time period\n",
    "        collection = (ee.ImageCollection('COPERNICUS/S2_SR_HARMONIZED')\n",
    "                      .filterDate(time_maps[country]['start'], time_maps[country]['end']))\n",
    "        \n",
    "        # Apply the cloud masking function to the collection\n",
    "        masked_collection = collection.map(mask_clouds)\n",
    "        \n",
    "        # Calculate mean values for the specified bands\n",
    "        mean_values = (masked_collection\n",
    "                       .select(BANDS)\n",
    "                       .filterBounds(ee.FeatureCollection(point_geometries))\n",
    "                       .mean()  \n",
    "                       .reduceRegions(collection=ee.FeatureCollection(point_geometries), \n",
    "                                      reducer=ee.Reducer.mean(),  \n",
    "                                      scale=10, \n",
    "                                      tileScale=tile_scale))\n",
    "        \n",
    "        # Extract the mean values for the specified bands\n",
    "        results = []\n",
    "        for feature in mean_values.getInfo()['features']:\n",
    "            values = [feature['properties'][band] for band in BANDS]\n",
    "            results.append(values)\n",
    "        \n",
    "        # Assign the extracted values to the CSV data\n",
    "        for i, band in enumerate(BANDS):\n",
    "            csv_data[band] = [values[i] for values in results]\n",
    "        \n",
    "        # Save the updated CSV data with the corresponding file name\n",
    "        output_file = f'{country}_{file}'\n",
    "        csv_data.to_csv(output_file, index=False)\n",
    "        print(f'{output_file} saved.')\n",
    "\n",
    "csv_data.head()"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}