test_exported_model.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import os\n",
    "import tensorflow as tf\n",
    "import cv2\n",
    "import imutils\n",
    "import matplotlib.patches as patches\n",
    "\n",
    "from matplotlib import pyplot as plt\n",
    "\n",
    "from collections import namedtuple\n",
    "from collections import defaultdict\n",
    "\n",
    "from skimage import segmentation\n",
    "from skimage import measure\n",
    "from imutils import perspective\n",
    "from pylab import array, plot, show, axis, arange, figure, uint8 \n",
    "\n",
    "from utils import ops as utils_ops\n",
    "from utils import label_map_util\n",
    "from utils import visualization_utils as vis_util\n",
    "\n",
    "tf.debugging.set_log_device_placement(True)\n",
    "\n",
    "print ('Tensorflow version:', tf.__version__)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "MODEL_FOLDER = 'exported_model_directory'\n",
    "\n",
    "# Change it to load different models from MODEL_FOLDER\n",
    "PATH_TO_PLATE_DETECTOR = MODEL_FOLDER + '/plate_detector_ssd_inception_v2/frozen_inference_graph.pb'\n",
    "\n",
    "PATH_TO_PLATE_LABELS = os.path.join('label_map', 'plate_label_map.pbtxt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load a (frozen) Tensorflow model into memory\n",
    "plate_detection_graph = tf.Graph()\n",
    "with plate_detection_graph.as_default():\n",
    "    od_graph_def1 = tf.GraphDef()\n",
    "    with tf.gfile.GFile(PATH_TO_PLATE_DETECTOR, 'rb') as fid:\n",
    "        serialized_graph = fid.read()\n",
    "        od_graph_def1.ParseFromString(serialized_graph)\n",
    "        tf.import_graph_def(od_graph_def1, name='')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Loading label map\n",
    "plate_category_index = label_map_util.create_category_index_from_labelmap(PATH_TO_PLATE_LABELS, use_display_name=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Helper code\n",
    "def load_image_into_numpy_array(image):\n",
    "    (im_width, im_height) = image.size\n",
    "    return np.array(image.getdata()).reshape((im_height, im_width, 3)).astype(np.uint8)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Loading the test images\n",
    "PATH_TO_TEST_IMAGES_DIR = 'mongol_alpr_images/'\n",
    "\n",
    "TEST_IMAGE_PATHS = []\n",
    "for file_name in os.listdir(PATH_TO_TEST_IMAGES_DIR):\n",
    "    fformat = file_name.split('.')[1]\n",
    "    if fformat == 'jpg' or fformat == 'png':\n",
    "        TEST_IMAGE_PATHS.append(PATH_TO_TEST_IMAGES_DIR + file_name)\n",
    "\n",
    "# Size, in inches, of the output images.\n",
    "IMAGE_SIZE = (9, 6)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def run_inference_for_images(images, graph):\n",
    "    with graph.as_default():\n",
    "        with tf.Session() as sess:\n",
    "            output_dicts = []\n",
    "            for i, image in enumerate(images):\n",
    "                # Get handles to input and output tensors\n",
    "                ops = tf.get_default_graph().get_operations()\n",
    "                all_tensor_names = {output.name for op in ops for output in op.outputs}\n",
    "                tensor_dict = {}\n",
    "                for key in [\n",
    "                    'num_detections', 'detection_boxes', 'detection_scores',\n",
    "                    'detection_classes', 'detection_masks'\n",
    "                ]:\n",
    "                    tensor_name = key + ':0'\n",
    "                    if tensor_name in all_tensor_names:\n",
    "                        tensor_dict[key] = tf.get_default_graph().get_tensor_by_name(tensor_name)\n",
    "                if 'detection_masks' in tensor_dict:\n",
    "                    # The following processing is only for single image\n",
    "                    detection_boxes = tf.squeeze(tensor_dict['detection_boxes'], [0])\n",
    "                    detection_masks = tf.squeeze(tensor_dict['detection_masks'], [0])\n",
    "                    # Reframe is required to translate mask from box coordinates to image coordinates and fit the image size.\n",
    "                    real_num_detection = tf.cast(tensor_dict['num_detections'][0], tf.int32)\n",
    "                    detection_boxes = tf.slice(detection_boxes, [0, 0], [real_num_detection, -1])\n",
    "                    detection_masks = tf.slice(detection_masks, [0, 0, 0], [real_num_detection, -1, -1])\n",
    "                    detection_masks_reframed = utils_ops.reframe_box_masks_to_image_masks(\n",
    "                        detection_masks, detection_boxes, image.shape[1], image.shape[2])\n",
    "                    detection_masks_reframed = tf.cast(tf.greater(detection_masks_reframed, 0.5), tf.uint8)\n",
    "                    # Follow the convention by adding back the batch dimension\n",
    "                    tensor_dict['detection_masks'] = tf.expand_dims(detection_masks_reframed, 0)\n",
    "                image_tensor = tf.get_default_graph().get_tensor_by_name('image_tensor:0')\n",
    "\n",
    "                # Run inference\n",
    "                output_dict = sess.run(tensor_dict, feed_dict={image_tensor: image})\n",
    "\n",
    "                # all outputs are float32 numpy arrays, so convert types as appropriate\n",
    "                output_dict['num_detections'] = int(output_dict['num_detections'][0])\n",
    "                output_dict['detection_classes'] = output_dict['detection_classes'][0].astype(np.int64)\n",
    "                output_dict['detection_boxes'] = output_dict['detection_boxes'][0]\n",
    "                output_dict['detection_scores'] = output_dict['detection_scores'][0]\n",
    "                if 'detection_masks' in output_dict:\n",
    "                    output_dict['detection_masks'] = output_dict['detection_masks'][0]\n",
    "                output_dicts.append(output_dict)\n",
    "                \n",
    "                print (str(int(100*(i/len(images)))) + '%', end='\\r')\n",
    "            return output_dicts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def char_segment(im):\n",
    "    # Removing shadow from an image\n",
    "    # https://stackoverflow.com/questions/44752240/how-to-remove-shadow-from-scanned-images-using-opencv\n",
    "    rgb_planes = cv2.split(im)\n",
    "    result_planes = []\n",
    "    result_norm_planes = []\n",
    "    for plane in rgb_planes:\n",
    "        dilated_img = cv2.dilate(plane, np.ones((3,3), np.uint8))\n",
    "        bg_img = cv2.medianBlur(dilated_img, 31)\n",
    "        diff_img = 255 - cv2.absdiff(plane, bg_img)\n",
    "        norm_img = cv2.normalize(diff_img,None, alpha=0, beta=255, norm_type=cv2.NORM_MINMAX, dtype=cv2.CV_8UC1)\n",
    "        result_planes.append(diff_img)\n",
    "        result_norm_planes.append(norm_img)\n",
    "\n",
    "    result = cv2.merge(result_planes)\n",
    "    result_norm = cv2.merge(result_norm_planes)\n",
    "    \n",
    "    # Create a CLAHE object (Arguments are optional).\n",
    "    clahe = cv2.createCLAHE(clipLimit=2.0, tileGridSize=(8,8))\n",
    "    # Applying a contrast limited adaptive histogram equlization\n",
    "    im = clahe.apply(im)\n",
    "    \n",
    "    # Turning the image into binary image\n",
    "    ret, im = cv2.threshold(im,0,255,cv2.THRESH_BINARY_INV+cv2.THRESH_OTSU)\n",
    "    im = segmentation.clear_border(im)\n",
    "    \n",
    "    # Finding the bounding boxes of the label masks\n",
    "    bboxes = []\n",
    "    \n",
    "    labels = measure.label(im, neighbors=8, background=0)\n",
    "    charCandidates = np.zeros(im.shape, dtype=\"uint8\")\n",
    "\n",
    "    for label in np.unique(labels):\n",
    "        # if this is the background label, ignore it\n",
    "        if label == 0:\n",
    "            continue\n",
    "\n",
    "        # otherwise, construct the label mask to display only connected components for the\n",
    "        # current label, then find contours in the label mask\n",
    "        labelMask = np.zeros(im.shape, dtype=\"uint8\")\n",
    "        labelMask[labels == label] = 255\n",
    "        cnts = cv2.findContours(labelMask, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)\n",
    "        \n",
    "#         cnts = cnts[0] if imutils.is_cv2() else cnts[1]\n",
    "        cnts = cnts[0] if len(cnts) == 2 else cnts[1]\n",
    "\n",
    "        # grab the largest contour which corresponds to the component in the mask, then\n",
    "        c = max(cnts, key=cv2.contourArea)\n",
    "            \n",
    "        # grab the bounding box for the contour\n",
    "        (boxX, boxY, boxW, boxH) = cv2.boundingRect(c)\n",
    "\n",
    "        # compute the aspect ratio, solidity, and height ratio for the component\n",
    "        aspectRatio = boxW / float(boxH)\n",
    "        solidity = cv2.contourArea(c) / float(boxW * boxH)\n",
    "        areaRatio = float(boxW * boxH) / float(im.shape[0] * im.shape[1])\n",
    "        heightRatio = boxH / float(im.shape[0])\n",
    "\n",
    "        bboxes.append([boxX-1, boxY-1, boxW+1, boxH+1, aspectRatio, areaRatio])\n",
    "    \n",
    "    return bboxes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "LP_labels = {\n",
    "    0: '0',\n",
    "    1: '1',\n",
    "    2: '2',\n",
    "    3: '3',\n",
    "    4: '4',\n",
    "    5: '5',\n",
    "    6: '6',\n",
    "    7: '7',\n",
    "    8: '8',\n",
    "    9: '9',\n",
    "    10: 'a',\n",
    "    11: 'b',\n",
    "    12: 'ch',\n",
    "    13: 'd',\n",
    "    14: 'e',\n",
    "    15: 'g',\n",
    "    16: 'h',\n",
    "    17: 'i',\n",
    "    18: 'k',\n",
    "    19: 'l',\n",
    "    20: 'm',\n",
    "    21: 'n',\n",
    "    22: 'p',\n",
    "    23: 'q',\n",
    "    24: 'r',\n",
    "    25: 's',\n",
    "    26: 't',\n",
    "    27: 'ts',\n",
    "    28: 'u',\n",
    "    29: 'uu',\n",
    "    30: 'v',\n",
    "    31: 'ya',\n",
    "    32: 'ye',\n",
    "    33: 'z'\n",
    "}\n",
    "# returns (Character string, probability)\n",
    "def decode_output(output):\n",
    "    output = list(output)\n",
    "    the_max = max(output)\n",
    "    return (LP_labels[output.index(the_max)], the_max)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "char_binary_model = tf.keras.models.load_model('char_recognition_models/plate_char_binary_recognizer.h5')\n",
    "char_model = tf.keras.models.load_model('char_recognition_models/plate_char_recognizer.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# get frames from video\n",
    "def get_frames_from_vid(video_path):\n",
    "    frames = []\n",
    "    vidcap = cv2.VideoCapture(video_path)\n",
    "    success,image = vidcap.read()\n",
    "    frames.append(image)\n",
    "    count = 0\n",
    "    while success:\n",
    "        success,image = vidcap.read()\n",
    "        if success:\n",
    "            frames.append(image)\n",
    "        count += 1\n",
    "    print (frames[0].shape)\n",
    "    return frames"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Loading images\n",
    "images_expanded = []\n",
    "images = []\n",
    "\n",
    "for image_path in TEST_IMAGE_PATHS:\n",
    "    image_np = cv2.imread(image_path, 0)\n",
    "    \n",
    "#     Enhancing image\n",
    "    clahe = cv2.createCLAHE(clipLimit=2.0, tileGridSize=(8,8))\n",
    "    image_np = clahe.apply(image_np)\n",
    "    image_np = cv2.cvtColor(image_np, cv2.COLOR_GRAY2BGR)\n",
    "    \n",
    "#     Expand dimensions since the model expects images to have a shape: [1, None, None, 3]\n",
    "    image_np_expanded = np.expand_dims(image_np, axis=0)\n",
    "    \n",
    "    images.append(image_np)\n",
    "    images_expanded.append(image_np_expanded)\n",
    "\n",
    "print (len(images), 'image(s) found')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Detection of license plates.\n",
    "# This might take some resources to initialize the tf graph\n",
    "# After it's done it will start detecting plates from given images\n",
    "# will print the progress in percentage\n",
    "\n",
    "plate_output_dicts = run_inference_for_images(images_expanded, plate_detection_graph)\n",
    "print ('done')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# This is needed to display the images.\n",
    "%matplotlib inline\n",
    "print (len(plate_output_dicts))\n",
    "\n",
    "font = cv2.FONT_HERSHEY_SIMPLEX \n",
    "fontScale = 1\n",
    "color = (0, 0, 255) \n",
    "thickness = 2\n",
    "\n",
    "for i, plate_output_dict in enumerate(plate_output_dicts):\n",
    "    img_height, img_width, img_channel = images[i].shape\n",
    "    absolute_coord = []\n",
    "    \n",
    "    # ADJUST THE THRESHOLD FOR PLATE DETECTION\n",
    "    THRESHOLD = 0.5\n",
    "    \n",
    "    N = len(plate_output_dict['detection_boxes'])\n",
    "    for j in range(N):\n",
    "        if plate_output_dict['detection_scores'][j] > THRESHOLD:\n",
    "            box = plate_output_dict['detection_boxes'][j]\n",
    "            ymin, xmin, ymax, xmax = box\n",
    "            x_up = int(xmin*img_width)\n",
    "            y_up = int(ymin*img_height)\n",
    "            x_down = int(xmax*img_width)\n",
    "            y_down = int(ymax*img_height)\n",
    "            absolute_coord.append((x_up,y_up,x_down,y_down))\n",
    "\n",
    "#     Visualization of the results of a detection.\n",
    "    vis_util.visualize_boxes_and_labels_on_image_array(\n",
    "        images[i],\n",
    "        plate_output_dict['detection_boxes'],\n",
    "        plate_output_dict['detection_classes'],\n",
    "        plate_output_dict['detection_scores'],\n",
    "        plate_category_index,\n",
    "        min_score_thresh = THRESHOLD,\n",
    "        instance_masks=plate_output_dict.get('detection_masks'),\n",
    "        use_normalized_coordinates=True,\n",
    "        line_thickness=16)\n",
    "    \n",
    "    for c in absolute_coord:\n",
    "        # cropped plate image\n",
    "        plate = images[i][c[1]:c[3], c[0]:c[2],:]\n",
    "        plate = cv2.cvtColor(plate, cv2.COLOR_BGR2GRAY)\n",
    "        # bounding boxes for characters\n",
    "        bboxes = char_segment(plate)\n",
    "        bboxes = [bbox for bbox in bboxes if bbox[4] < 1 and bbox[4] > 0.20 and bbox[5] > 0.01 and bbox[5] < 0.15]\n",
    "        \n",
    "        bboxes_with_chars = []\n",
    "        # recognizing characters inside bbox nad putting it in \n",
    "        for bbox in bboxes:\n",
    "            # cropping potential character image\n",
    "            char_img = images[i][c[1]+bbox[1]:c[1]+bbox[1]+bbox[3], c[0]+bbox[0]:c[0]+bbox[0]+bbox[2]]\n",
    "            \n",
    "            # padding the image by 10%\n",
    "            top = int(0.1 * char_img.shape[0])\n",
    "            bottom = top\n",
    "            left = int(0.1 * char_img.shape[1])\n",
    "            right = left\n",
    "            char_img = cv2.copyMakeBorder(char_img, top, bottom, left, right, cv2.BORDER_REPLICATE)\n",
    "            \n",
    "            # resizing and reshaping to fit it into CNN\n",
    "            char_img = cv2.resize(char_img, (20,36), interpolation = cv2.INTER_AREA)\n",
    "            char_img = cv2.cvtColor(char_img, cv2.COLOR_BGR2GRAY)\n",
    "            char_img = char_img.reshape(1, char_img.shape[0], char_img.shape[1], 1)\n",
    "            \n",
    "            # predicting character from a prepared character image\n",
    "            the_char = char_model.predict(char_img)\n",
    "            \n",
    "            # decoding output vector into a string character\n",
    "            the_char, score = decode_output(the_char[0])\n",
    "            \n",
    "            # drawing bbox and a character into an original image (with cars)\n",
    "            images[i] = cv2.putText(images[i], str(the_char), (c[0]+bbox[0],c[1]+bbox[1]),\n",
    "                                    font, fontScale, color, thickness, cv2.LINE_AA)\n",
    "            \n",
    "            images[i] = cv2.rectangle(images[i], (c[0]+bbox[0],c[1]+bbox[1]),\n",
    "                                      (c[0]+bbox[0]+bbox[2],c[1]+bbox[1]+bbox[3]), (255,0,0), 2)\n",
    "            \n",
    "            bboxes_with_chars.append([bbox, the_char])\n",
    "    cv2.imwrite('mongol_alpr_output_test/'+'{}.jpg'.format(i), images[i])\n",
    "    print (str(int(100*(i/len(plate_output_dicts)))) + '%', end='\\r')\n",
    "print ('done')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "tf1",
   "language": "python",
   "name": "tf1"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}