two_stream_fusion_validate_full_video.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "qQfDlpGcg20T"
   },
   "source": [
    "# Setup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "id": "cH5Gz3XSg20g"
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import os\n",
    "import random\n",
    "import cv2\n",
    "import tensorflow as tf\n",
    "from tensorflow import keras\n",
    "from tensorflow.keras import layers\n",
    "from keras.models import Sequential, load_model, Model\n",
    "from keras.layers import Input, average, concatenate, GlobalAveragePooling2D\n",
    "from keras.layers import TimeDistributed, GlobalAveragePooling1D\n",
    "from keras.layers.core import Dense, Dropout, Activation, Flatten\n",
    "from keras.models import Sequential, load_model, Model\n",
    "from pathlib import Path\n",
    "import os\n",
    "import re\n",
    "\n",
    "from keras.layers.core import Dense,Flatten\n",
    "from tensorflow.keras.models import Sequential\n",
    "from tensorflow.keras.optimizers import Adam"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "L0QDC18tKULs"
   },
   "source": [
    "# Models fusion"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "Ue5GsMmIg20w"
   },
   "outputs": [],
   "source": [
    "def two_stream_fuse(spatial_model, temporal_model):\n",
    "    # spatial stream (frozen)\n",
    "    cnn_spatial = spatial_model\n",
    "\n",
    "    # temporal stream (frozen)\n",
    "    cnn_temporal = temporal_model\n",
    "\n",
    "    # fused by taking average\n",
    "    outputs = average([cnn_spatial.output, cnn_temporal.output])\n",
    "\n",
    "    model = Model(inputs = [cnn_spatial.input, temporal_model.input], outputs = outputs)\n",
    "    #model = Model(tf.keras.layers.Concatenate(axis = -1)(cnn_spatial, cnn_temporal))\n",
    "    \n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "id": "5ldGaOAjg201"
   },
   "outputs": [],
   "source": [
    "# Importazione dei modelli\n",
    "spatial_model = load_model('/content/gdrive/MyDrive/Deep learning project Pasinato Carbone Scuri/spatial_model_finetuned_resnet.hdf5')\n",
    "temporal_model = load_model('/content/gdrive/MyDrive/Deep learning project Pasinato Carbone Scuri/temporal_model_cnn.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "dnF6w76hrHxO"
   },
   "outputs": [],
   "source": [
    "spatial_model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "CctQE6p5g205",
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "two_stream_model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "ip610d0fg202"
   },
   "outputs": [],
   "source": [
    "# Model\n",
    "two_stream_model = two_stream_fuse(spatial_model, temporal_model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "kBEKX4X3g204"
   },
   "outputs": [],
   "source": [
    "# Compiling\n",
    "optimizer = Adam()\n",
    "two_stream_model.compile(loss=keras.losses.sparse_categorical_crossentropy, \n",
    "                   metrics=['sparse_categorical_accuracy','sparse_top_k_categorical_accuracy'], \n",
    "                   optimizer=optimizer)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "4mGvlC8FJARR"
   },
   "source": [
    "#### Dictionaries label name - label number and vice-versa"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "id": "VIkg-dSOqZKr"
   },
   "outputs": [],
   "source": [
    "labels = dict()\n",
    "count = 0\n",
    "for action in sorted(os.listdir(\"/content/data/hmdb51/rawframes\")):\n",
    "  labels[count] = action \n",
    "  count += 1\n",
    "\n",
    "labels_rev = {v: k for k, v in labels.items()}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "LNWZm-r-KCxY"
   },
   "source": [
    "Test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "S6vHVLPFpTmM"
   },
   "outputs": [],
   "source": [
    "for i in range (batch_size):\n",
    "  top5 = list()\n",
    "  res = two_stream_model([np.expand_dims(img[0][0][i], axis=0), np.expand_dims(img[0][1][i], axis=0)])\n",
    "  pred = np.argmax(res)\n",
    "  label = img[1][i]\n",
    "  top5 = [labels[n] for n in np.argsort(res, axis=1)[:,-5:].tolist()[0]]\n",
    "\n",
    "  res_spat = spatial_model(np.expand_dims(img[0][0][i], axis=0))\n",
    "  res_temp = temporal_model(np.expand_dims(img[0][1][i], axis=0))\n",
    "\n",
    "  top5_spat = [labels[n] for n in np.argsort(res_spat, axis=1)[:,-5:].tolist()[0]]\n",
    "  top5_temp = [labels[n] for n in np.argsort(res_temp, axis=1)[:,-5:].tolist()[0]]\n",
    "\n",
    "  # pred_spat = np.argmax(res_spat)\n",
    "  # pred_temp = np.argmax(res_temp)\n",
    "  \n",
    "  print(f\"correct label: {labels[label]}, predicted: {labels[pred]},     top5_fused: {top5} \\nspat: {top5_spat},      temp: {top5_temp}\\n\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "pgkbumPzDov-"
   },
   "source": [
    "# Predict a video"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "RCUczWMM4aSL"
   },
   "source": [
    "#### Spatial Model prediction over all frames"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "2yw8hJXi5cOf"
   },
   "outputs": [],
   "source": [
    "video_path = \"/content/data/hmdb51/rawframes/climb/Amazing_Wall_Climber_(Must_be_Seen_to_Be_Believed!)_climb_f_cm_np1_ba_bad_1\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "id": "0zzu84EU4dB7"
   },
   "outputs": [],
   "source": [
    "class_ind = pd.read_csv(\"./data/hmdb51/annotations/classInd.txt\", sep=\" \", header=None)\n",
    "class_ind.columns = [\"class_ind\", \"class\"]\n",
    "class_ind['class_ind'] = class_ind['class_ind'] - 1\n",
    "class_ind.set_index('class_ind', inplace=True)\n",
    "class_ind.head(10)\n",
    "class_ind_copy = class_ind.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "id": "BynExUh04fSl"
   },
   "outputs": [],
   "source": [
    "resnet_model = keras.models.load_model('/content/gdrive/MyDrive/Deep learning project Pasinato Carbone Scuri/spatial_model_finetuned_resnet.hdf5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "id": "mvp3sdL74xEh"
   },
   "outputs": [],
   "source": [
    "def find_paths(partition, type_frame):\n",
    "    if partition == 'train':\n",
    "        video_list = pd.read_csv(f'{path_annotations}/hmdb51_train_split_1_rawframes.txt', sep=\" \", header=None) #train\n",
    "        video_list.columns = [\"path\", \"num_frames_tot\", \"class\"]\n",
    "    elif partition == 'val':\n",
    "        video_list = pd.read_csv(f'{path_annotations}/hmdb51_val_split_1_rawframes.txt', sep=\" \", header=None) #test\n",
    "        video_list.columns = [\"path\", \"num_frames_tot\", \"class\"]\n",
    "    else:\n",
    "        raise Exception(\"invalid partition\")\n",
    "\n",
    "\n",
    "    paths = []\n",
    "    classes = []\n",
    "    for index, row in video_list.iterrows():\n",
    "        temp_path = row['path'].replace(\"\\\\\",\"/\")\n",
    "        frame_list = os.listdir(os.path.join(f'./{temp_path}'))\n",
    "\n",
    "        frame_list_type = [i for i in frame_list if i.startswith(f'{type_frame}')]\n",
    "\n",
    "        filename = frame_list_type\n",
    "\n",
    "        paths.append([os.path.join('./', temp_path, file) for file in filename])\n",
    "        temp = [row['class']] * len(filename)\n",
    "        classes.append(temp)\n",
    "\n",
    "    return(paths, classes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "id": "mhqwIyOR44Vs"
   },
   "outputs": [],
   "source": [
    "filenames_img, labels = find_paths(partition='val', type_frame='img')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "id": "2fvrfWSc6UdX"
   },
   "outputs": [],
   "source": [
    "video_path = \"/content/data/hmdb51/rawframes/golf/Natalie_Gulbis_1_golf_f_cm_np1_le_med_0\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "id": "v9mGM4nHYcEO"
   },
   "outputs": [],
   "source": [
    "video_path = \"/content/data/hmdb51/rawframes/ride_bike/Justin_lernt_Fahrrad_fahren_ride_bike_f_cm_np1_fr_med_0\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {
    "id": "5MGPPcS55EPi"
   },
   "outputs": [],
   "source": [
    "# ------ all in one (for loop)\n",
    "#random_video_frames_path = random.choice(filenames_img)\n",
    "\n",
    "from imgaug import augmenters as iaa\n",
    "\n",
    "random.shuffle(filenames_img)\n",
    "top1 = 0\n",
    "top5 = 0\n",
    "miss = 0\n",
    "\n",
    "contatore = 0\n",
    "\n",
    "#for count, video_frames_path in enumerate(filenames_img[:200]):\n",
    "count = 1\n",
    "list_frames = os.listdir(video_path)\n",
    "video_frames_path = [img for img in list_frames if img.startswith(\"img\")]\n",
    "\n",
    "#print(len(video_frames_path))\n",
    "class_ind = class_ind_copy.copy()\n",
    "\n",
    "original_rgb_frames = []\n",
    "\n",
    "for frame in video_frames_path:\n",
    "    original_rgb_frames.append(cv2.imread(f\"{video_path}/{frame}\", cv2.IMREAD_UNCHANGED))\n",
    "\n",
    "# generate spatial batch as done in the dataloader\n",
    "spatial_batch_temp = []\n",
    "for image in original_rgb_frames:\n",
    "    spatial_batch_temp.append(cv2.cvtColor(image, cv2.COLOR_BGR2RGB))\n",
    "    #spatial_batch_temp.append(image) - [103.939, 116.779, 123.68]) #peggiora se non si fa BGR2RGB e se si toglie la media\n",
    "\n",
    "# image resize augmenter to be fed into the network\n",
    "\n",
    "augmenter = iaa.Sequential([\n",
    "    iaa.Resize((img_height, img_width))\n",
    "    #iaa.CropToFixedSize(img_height, img_width)\n",
    "])\n",
    "spatial_batch = np.array(augmenter.augment_images(spatial_batch_temp), dtype=np.float32)# /255.0\n",
    "\n",
    "#true_class = video_frames_path[0].split('/')[4]\n",
    "#print(true_class)\n",
    "\n",
    "# predict spatial stream output\n",
    "#print(spatial_batch.shape)\n",
    "try:\n",
    "    spatial_pred = resnet_model.predict(spatial_batch)\n",
    "except:\n",
    "    print('saltatoooooo')\n",
    "spatial_classes = np.argsort(spatial_pred,axis=1)#[:,:-6:-1]\n",
    "spatial_scores = np.sort(spatial_pred,axis=1)#[:,:-6:-1]\n",
    "\n",
    "spatial_sorted = []\n",
    "\n",
    "for spatial_class, spacial_score in zip(spatial_classes, spatial_scores):\n",
    "    zipped = zip(spatial_class, spacial_score)\n",
    "    spatial_sorted.append(sorted(zipped, key=lambda x: x[0]))\n",
    "\n",
    "results_spat = np.average(spatial_sorted, axis=0)       #avg_scores\n",
    "# class_ind['percentage'] = round(pd.Series(avg_scores[:,1])*100, 1)\n",
    "# class_ind.sort_values('percentage', ascending=False, inplace=True)\n",
    "# class_ind = class_ind.head(5)\n",
    "# class_ind.reset_index('class_ind', inplace = True)\n",
    "\n",
    "results_spat = np.expand_dims(results_spat[:,1], axis=0)\n",
    "\n",
    "# if class_ind.loc[0]['class'] == true_class:\n",
    "#     #print('top 1')\n",
    "#     top1 += 1\n",
    "# elif true_class in class_ind['class'].tolist():\n",
    "#     #print('top 5')\n",
    "#     top5 += 1\n",
    "# else:\n",
    "#     #print('niente')\n",
    "#     miss += 1\n",
    "# print(count,'/',len(filenames_img))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "aDjsOhVm4fu7"
   },
   "source": [
    "#### Temporal model prediction over all frames"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "id": "vDoQnfsQY-Z5"
   },
   "outputs": [],
   "source": [
    "# Given a video this function creates for each frame a stacked optical flow (224,224,20) and gives it to the temporal model\n",
    "# The overall prediction is an average of all the predictions\n",
    "\n",
    "def predict_video_temp(video_path):                 \n",
    "  list_frames = os.listdir(video_path)\n",
    "  n_frames = len([img for img in list_frames if img.startswith(\"img\")])\n",
    "  results = np.zeros(shape = (1,51))\n",
    "  # label = \n",
    "  count = 0\n",
    "  for n_frame in range(1, n_frames - 10):                     \n",
    "    opt_flow_stack = list()\n",
    "    count += 1\n",
    "    for i in range(10):\n",
    "      for lettera in [\"x\",\"y\"]:      \n",
    "        img = None # reset to be safe\n",
    "        img = cv2.imread(os.path.join(f'{video_path}' + f'/flow_{lettera}_' + str(\"%05d\"%(n_frame + i)) + '.jpg'), 0)\n",
    "        #print(os.path.join(f'.\\\\{temp_path}' + '\\\\flow_x_' + str(\"%05d\"%(i_frame)) + '.jpg'))\n",
    "        #print(img.shape)\n",
    "        if img is None:\n",
    "          continue\n",
    "\n",
    "        img = np.array(img)\n",
    "        # mean substraction \n",
    "        img = img - np.mean(img)\n",
    "        #img = img[top : bottom, left : right]\n",
    "        img = img / 255. # normalize pixels \n",
    "        img = cv2.resize(img, (224,224))\n",
    "        #print(img.shape)\n",
    "        opt_flow_stack.append(img)\n",
    "\n",
    "    opt_flow_stack = np.array(opt_flow_stack)\n",
    "    opt_flow_stack = np.swapaxes(opt_flow_stack, 0, 1)\n",
    "    opt_flow_stack = np.swapaxes(opt_flow_stack, 1, 2)\n",
    "    if opt_flow_stack.shape[2] != 20:\n",
    "      break\n",
    "    opt_flow_stack = np.expand_dims(opt_flow_stack, axis=0)\n",
    "    results += temporal_model.predict(opt_flow_stack)\n",
    "    #np.argsort(results / count, axis=1)[:,-5:]\n",
    "\n",
    "  return results/count"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {
    "id": "412LWjAftSXH"
   },
   "outputs": [],
   "source": [
    "results_temp = predict_video_temp(video_path)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 35
    },
    "id": "O8D8p1Nv7EfN",
    "outputId": "6ccbbe92-e6ae-49fc-c94d-142f5e6db628"
   },
   "outputs": [
    {
     "data": {
      "application/vnd.google.colaboratory.intrinsic+json": {
       "type": "string"
      },
      "text/plain": [
       "'golf'"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pred = np.argmax((results_temp + results_spat) / 2 )\n",
    "labels[pred]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "Gzg7F6xNIdje",
    "outputId": "b1d3e3e5-748b-4057-ba0c-8f83443676cb"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "spat: handstand: 0.000320975007933783\n",
      "spat: kick_ball: 0.0025262375589692157\n",
      "spat: catch: 0.004053326523492693\n",
      "spat: shoot_bow: 0.008194903623927845\n",
      "spat: golf: 0.9838962849275565\n",
      "temp: pick: 0.020356225293861437\n",
      "temp: walk: 0.02042131956706206\n",
      "temp: handstand: 0.050235423372526254\n",
      "temp: climb: 0.06163918194350637\n",
      "temp: golf: 0.6632648305169173\n"
     ]
    }
   ],
   "source": [
    "for elemento in [results_spat, results_temp]:\n",
    "  for i in np.argsort(elemento, axis = 1)[:,-5:][0]:\n",
    "    if elemento is results_spat:\n",
    "      stringa = \"spat\"\n",
    "    else:\n",
    "      stringa = \"temp\"\n",
    "    print(f\"{stringa}: {labels[i]}: {elemento[0][i]}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 35
    },
    "id": "n-Dy3sESYrT3",
    "outputId": "bdc97a57-56db-4950-f217-77cff6261e27"
   },
   "outputs": [
    {
     "data": {
      "application/vnd.google.colaboratory.intrinsic+json": {
       "type": "string"
      },
      "text/plain": [
       "'ride_bike'"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pred = np.argmax((results_temp + results_spat) / 2 )\n",
    "labels[pred]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "NXkEfJDZYpZT",
    "outputId": "0ee89593-e0f0-48ca-fefe-9ad7c3165a6f"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "spat: dive: 0.007335681733763621\n",
      "spat: ride_horse: 0.009148698536668175\n",
      "spat: draw_sword: 0.011017633753251132\n",
      "spat: push: 0.17185946898280235\n",
      "spat: ride_bike: 0.7562838335521519\n",
      "temp: climb: 0.05316095326122814\n",
      "temp: ride_horse: 0.05339219504832358\n",
      "temp: somersault: 0.06613920255145733\n",
      "temp: cartwheel: 0.06840309912365848\n",
      "temp: catch: 0.19249734054387047\n"
     ]
    }
   ],
   "source": [
    "for elemento in [results_spat, results_temp]:\n",
    "  for i in np.argsort(elemento, axis = 1)[:,-5:][0]:\n",
    "    if elemento is results_spat:\n",
    "      stringa = \"spat\"\n",
    "    else:\n",
    "      stringa = \"temp\"\n",
    "    print(f\"{stringa}: {labels[i]}: {elemento[0][i]}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "9P5ER1mUOqUd"
   },
   "source": [
    "Finding the label in the given path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "FObOkERqJuJX"
   },
   "outputs": [],
   "source": [
    "key = \"\"\n",
    "count = 0\n",
    "init = video_path.find(\"rawframes\")\n",
    "while True:\n",
    "  if (video_path[init + len(\"rawframes/\") + count] == \"/\") or (video_path[init + len(\"rawframes/\") + count] == \"\\\\\"):\n",
    "    break\n",
    "  key += video_path[21 + len(\"rawframes/\") + count]\n",
    "  count += 1\n",
    "label = labels_rev[key]"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "provenance": []
  },
  "gpuClass": "standard",
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  },
  "vscode": {
   "interpreter": {
    "hash": "5f73b1806a9ec290aebb7f16dd5394fd2c9a0b8bec90a8791426b7ef5a3d9604"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}