Lecture 7.3 - Mini Project # 7 - Live Face Swaps.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Mini Project # 7 -  Live Face Swaps "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#!/usr/bin/python\n",
    "\n",
    "# Copyright (c) 2015 Matthew Earl\n",
    "# \n",
    "# Permission is hereby granted, free of charge, to any person obtaining a copy\n",
    "# of this software and associated documentation files (the \"Software\"), to deal\n",
    "# in the Software without restriction, including without limitation the rights\n",
    "# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n",
    "# copies of the Software, and to permit persons to whom the Software is\n",
    "# furnished to do so, subject to the following conditions:\n",
    "# \n",
    "#     The above copyright notice and this permission notice shall be included\n",
    "#     in all copies or substantial portions of the Software.\n",
    "# \n",
    "#     THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS\n",
    "#     OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF\n",
    "#     MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN\n",
    "#     NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,\n",
    "#     DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR\n",
    "#     OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE\n",
    "#     USE OR OTHER DEALINGS IN THE SOFTWARE.\n",
    "\n",
    "\"\"\"\n",
    "This is the code behind the Switching Eds blog post:\n",
    "    http://matthewearl.github.io/2015/07/28/switching-eds-with-python/\n",
    "See the above for an explanation of the code below.\n",
    "To run the script you'll need to install dlib (http://dlib.net) including its\n",
    "Python bindings, and OpenCV. You'll also need to obtain the trained model from\n",
    "sourceforge:\n",
    "    http://sourceforge.net/projects/dclib/files/dlib/v18.10/shape_predictor_68_face_landmarks.dat.bz2\n",
    "Unzip with `bunzip2` and change `PREDICTOR_PATH` to refer to this file. The\n",
    "script is run like so:\n",
    "    ./faceswap.py <head image> <face image>\n",
    "If successful, a file `output.jpg` will be produced with the facial features\n",
    "from `<head image>` replaced with the facial features from `<face image>`.\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Anaconda2\\lib\\site-packages\\ipykernel\\__main__.py:173: FutureWarning: elementwise comparison failed; returning scalar instead, but in the future will perform elementwise comparison\n"
     ]
    }
   ],
   "source": [
    "import cv2\n",
    "import dlib\n",
    "import numpy\n",
    "from time import sleep\n",
    "import sys\n",
    "\n",
    "## Our pretrained model that predicts the rectangles that correspond to the facial features of a face\n",
    "PREDICTOR_PATH = \"shape_predictor_68_face_landmarks.dat\"\n",
    "SCALE_FACTOR = 1 \n",
    "FEATHER_AMOUNT = 11\n",
    "\n",
    "FACE_POINTS = list(range(17, 68))\n",
    "MOUTH_POINTS = list(range(48, 61))\n",
    "RIGHT_BROW_POINTS = list(range(17, 22))\n",
    "LEFT_BROW_POINTS = list(range(22, 27))\n",
    "RIGHT_EYE_POINTS = list(range(36, 42))\n",
    "LEFT_EYE_POINTS = list(range(42, 48))\n",
    "NOSE_POINTS = list(range(27, 35))\n",
    "JAW_POINTS = list(range(0, 17))\n",
    "\n",
    "# Points used to line up the images.\n",
    "ALIGN_POINTS = (LEFT_BROW_POINTS + RIGHT_EYE_POINTS + LEFT_EYE_POINTS +\n",
    "                               RIGHT_BROW_POINTS + NOSE_POINTS + MOUTH_POINTS)\n",
    "\n",
    "# Points from the second image to overlay on the first. The convex hull of each\n",
    "# element will be overlaid.\n",
    "OVERLAY_POINTS = [\n",
    "    LEFT_EYE_POINTS + RIGHT_EYE_POINTS + LEFT_BROW_POINTS + RIGHT_BROW_POINTS,\n",
    "    NOSE_POINTS + MOUTH_POINTS,\n",
    "]\n",
    "\n",
    "# Amount of blur to use during colour correction, as a fraction of the\n",
    "# pupillary distance.\n",
    "COLOUR_CORRECT_BLUR_FRAC = 0.6\n",
    "cascade_path='Haarcascades/haarcascade_frontalface_default.xml'\n",
    "cascade = cv2.CascadeClassifier(cascade_path)\n",
    "detector = dlib.get_frontal_face_detector()\n",
    "predictor = dlib.shape_predictor(PREDICTOR_PATH)\n",
    "\n",
    "\n",
    "def get_landmarks(im, dlibOn):\n",
    "    \n",
    "    if (dlibOn == True):\n",
    "        rects = detector(im, 1)\n",
    "        if len(rects) > 1:\n",
    "            return \"error\"\n",
    "        if len(rects) == 0:\n",
    "            return \"error\"\n",
    "        return numpy.matrix([[p.x, p.y] for p in predictor(im, rects[0]).parts()])\n",
    "    \n",
    "    else:\n",
    "        rects = cascade.detectMultiScale(im, 1.3,5)\n",
    "        if len(rects) > 1:\n",
    "            return \"error\"\n",
    "        if len(rects) == 0:\n",
    "            return \"error\"\n",
    "        x,y,w,h =rects[0]\n",
    "        rect=dlib.rectangle(x,y,x+w,y+h)\n",
    "        return numpy.matrix([[p.x, p.y] for p in predictor(im, rect).parts()])\n",
    "\n",
    "    \n",
    "def annotate_landmarks(im, landmarks):\n",
    "    im = im.copy()\n",
    "    for idx, point in enumerate(landmarks):\n",
    "        pos = (point[0, 0], point[0, 1])\n",
    "        cv2.putText(im, str(idx), pos,\n",
    "                    fontFace=cv2.FONT_HERSHEY_SCRIPT_SIMPLEX,\n",
    "                    fontScale=0.4,\n",
    "                    color=(0, 0, 255))\n",
    "        cv2.circle(im, pos, 3, color=(0, 255, 255))\n",
    "    return im\n",
    "\n",
    "\n",
    "def draw_convex_hull(im, points, color):\n",
    "    points = cv2.convexHull(points)\n",
    "    cv2.fillConvexPoly(im, points, color=color)\n",
    "\n",
    "    \n",
    "def get_face_mask(im, landmarks):\n",
    "    im = numpy.zeros(im.shape[:2], dtype=numpy.float64)\n",
    "\n",
    "    for group in OVERLAY_POINTS:\n",
    "        draw_convex_hull(im,\n",
    "                         landmarks[group],\n",
    "                         color=1)\n",
    "\n",
    "    im = numpy.array([im, im, im]).transpose((1, 2, 0))\n",
    "\n",
    "    im = (cv2.GaussianBlur(im, (FEATHER_AMOUNT, FEATHER_AMOUNT), 0) > 0) * 1.0\n",
    "    im = cv2.GaussianBlur(im, (FEATHER_AMOUNT, FEATHER_AMOUNT), 0)\n",
    "\n",
    "    return im\n",
    "    \n",
    "    \n",
    "def transformation_from_points(points1, points2):\n",
    "    \"\"\"\n",
    "    Return an affine transformation [s * R | T] such that:\n",
    "        sum ||s*R*p1,i + T - p2,i||^2\n",
    "    is minimized.\n",
    "    \"\"\"\n",
    "    # Solve the procrustes problem by subtracting centroids, scaling by the\n",
    "    # standard deviation, and then using the SVD to calculate the rotation. See\n",
    "    # the following for more details:\n",
    "    #   https://en.wikipedia.org/wiki/Orthogonal_Procrustes_problem\n",
    "\n",
    "    points1 = points1.astype(numpy.float64)\n",
    "    points2 = points2.astype(numpy.float64)\n",
    "\n",
    "    c1 = numpy.mean(points1, axis=0)\n",
    "    c2 = numpy.mean(points2, axis=0)\n",
    "    points1 -= c1\n",
    "    points2 -= c2\n",
    "\n",
    "    s1 = numpy.std(points1)\n",
    "    s2 = numpy.std(points2)\n",
    "    points1 /= s1\n",
    "    points2 /= s2\n",
    "\n",
    "    U, S, Vt = numpy.linalg.svd(points1.T * points2)\n",
    "\n",
    "    # The R we seek is in fact the transpose of the one given by U * Vt. This\n",
    "    # is because the above formulation assumes the matrix goes on the right\n",
    "    # (with row vectors) where as our solution requires the matrix to be on the\n",
    "    # left (with column vectors).\n",
    "    R = (U * Vt).T\n",
    "\n",
    "    return numpy.vstack([numpy.hstack(((s2 / s1) * R,\n",
    "                                       c2.T - (s2 / s1) * R * c1.T)),\n",
    "                         numpy.matrix([0., 0., 1.])])\n",
    "\n",
    "\n",
    "def read_im_and_landmarks(fname):\n",
    "    im = cv2.imread(fname, cv2.IMREAD_COLOR)\n",
    "    im = cv2.resize(im,None,fx=0.35, fy=0.35, interpolation = cv2.INTER_LINEAR)\n",
    "    im = cv2.resize(im, (im.shape[1] * SCALE_FACTOR,\n",
    "                         im.shape[0] * SCALE_FACTOR))\n",
    "    s = get_landmarks(im,dlibOn)\n",
    "\n",
    "    return im, s\n",
    "\n",
    "\n",
    "def warp_im(im, M, dshape):\n",
    "    output_im = numpy.zeros(dshape, dtype=im.dtype)\n",
    "    cv2.warpAffine(im,\n",
    "                   M[:2],\n",
    "                   (dshape[1], dshape[0]),\n",
    "                   dst=output_im,\n",
    "                   borderMode=cv2.BORDER_TRANSPARENT,\n",
    "                   flags=cv2.WARP_INVERSE_MAP)\n",
    "    return output_im\n",
    "\n",
    "\n",
    "def correct_colours(im1, im2, landmarks1):\n",
    "    blur_amount = COLOUR_CORRECT_BLUR_FRAC * numpy.linalg.norm(\n",
    "                              numpy.mean(landmarks1[LEFT_EYE_POINTS], axis=0) -\n",
    "                              numpy.mean(landmarks1[RIGHT_EYE_POINTS], axis=0))\n",
    "    blur_amount = int(blur_amount)\n",
    "    if blur_amount % 2 == 0:\n",
    "        blur_amount += 1\n",
    "    im1_blur = cv2.GaussianBlur(im1, (blur_amount, blur_amount), 0)\n",
    "    im2_blur = cv2.GaussianBlur(im2, (blur_amount, blur_amount), 0)\n",
    "\n",
    "    # Avoid divide-by-zero errors.\n",
    "    im2_blur += (128 * (im2_blur <= 1.0)).astype(im2_blur.dtype)\n",
    "\n",
    "    return (im2.astype(numpy.float64) * im1_blur.astype(numpy.float64) /\n",
    "                                                im2_blur.astype(numpy.float64))\n",
    "\n",
    "def face_swap(img,name):\n",
    "\n",
    "    s = get_landmarks(img,True)\n",
    "    \n",
    "    if (s == \"error\"):\n",
    "        print \"No or too many faces\"\n",
    "        return img\n",
    "       \n",
    "    im1, landmarks1 = img, s\n",
    "    im2, landmarks2 = read_im_and_landmarks(name)\n",
    "\n",
    "    M = transformation_from_points(landmarks1[ALIGN_POINTS],\n",
    "                                   landmarks2[ALIGN_POINTS])\n",
    "\n",
    "    mask = get_face_mask(im2, landmarks2)\n",
    "    warped_mask = warp_im(mask, M, im1.shape)\n",
    "    combined_mask = numpy.max([get_face_mask(im1, landmarks1), warped_mask],\n",
    "                              axis=0)\n",
    "\n",
    "    warped_im2 = warp_im(im2, M, im1.shape)\n",
    "    \n",
    "    warped_corrected_im2 = correct_colours(im1, warped_im2, landmarks1)\n",
    "\n",
    "    output_im = im1 * (1.0 - combined_mask) + warped_corrected_im2 * combined_mask\n",
    "    \n",
    "    #output_im is no longer in the expected OpenCV format so we use openCV \n",
    "    #to write the image to diks and then reload it\n",
    "    cv2.imwrite('output.jpg', output_im)\n",
    "    image = cv2.imread('output.jpg')\n",
    "    \n",
    "    frame = cv2.resize(image,None,fx=1.5, fy=1.5, interpolation = cv2.INTER_LINEAR)\n",
    "    \n",
    "    return image  \n",
    "    \n",
    "\n",
    "cap = cv2.VideoCapture(0)\n",
    "\n",
    "\n",
    "# Name is the image we want to swap onto ours\n",
    "# dlibOn controls if use dlib's facial landmark detector (better) \n",
    "# or use HAAR Cascade Classifiers (faster)\n",
    "\n",
    "filter_image =  \"images/Trump.jpg\" ### Put your image here!\n",
    "dlibOn = False\n",
    "\n",
    "while True:   \n",
    "    ret, frame = cap.read()   \n",
    "    \n",
    "    #Reduce image size by 75% to reduce processing time and improve framerates\n",
    "    frame = cv2.resize(frame, None, fx=0.75, fy=0.75, interpolation = cv2.INTER_LINEAR)\n",
    "    \n",
    "    # flip image so that it's more mirror like\n",
    "    frame = cv2.flip(frame, 1)\n",
    "    \n",
    "    cv2.imshow('Our Amazing Face Swapper', face_swap(frame, filter_image))\n",
    "    \n",
    "    if cv2.waitKey(1) == 13: #13 is the Enter Key\n",
    "        break\n",
    "\n",
    "cap.release()\n",
    "cv2.destroyAllWindows()  "
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}