neuralstyletransfer.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "p7OjzRsJREOf"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "from keras.preprocessing.image import load_img,img_to_array\n",
    "\n",
    "target_image_path = \"C:\\\\Users\\\\shivu\\\\bopro\\\\deeplearning\\\\style transfer\\\\results\\\\content.jpeg\"\n",
    "style_reference_img_path = \"C:\\\\Users\\\\shivu\\\\bopro\\\\deeplearning\\\\style transfer\\\\results\\\\style2.jpg\"\n",
    "\n",
    "width,height = load_img(target_image_path).size\n",
    "img_height = 400\n",
    "img_width = int(width*img_height / height)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "N9TkCb79VNIu",
    "outputId": "998e2aee-5e87-498a-8efd-cd0528373c7e"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "400"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "img_width"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "lEjlczbBVO1h"
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from keras.applications import vgg19\n",
    "\n",
    "def preprocess_image(image_path):\n",
    "  img = load_img(image_path,target_size=(img_height,img_width))\n",
    "  img = img_to_array(img)\n",
    "  img = np.expand_dims(img,axis=0)\n",
    "  img = vgg19.preprocess_input(img)\n",
    "  return img\n",
    "  \n",
    "  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "xurgWZEsVxmJ",
    "outputId": "5db5c7c1-7eb4-40bb-b207-07fc4549816f"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1, 400, 400, 3)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "preprocess_image(target_image_path).shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "SV6TdeykV4wW"
   },
   "outputs": [],
   "source": [
    "def deprocess_image(x):\n",
    "  x[:,:,0]+=103.939\n",
    "  x[:,:,1]+=116.779\n",
    "  x[:,:,2]+=123.68\n",
    "  \n",
    "  x = x[:,:,::-1]\n",
    "  \n",
    "  x = np.clip(x,0,255).astype('uint8')\n",
    "  \n",
    "  return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Y0WQIloCW0js"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "MwMuIcH9Wixq",
    "outputId": "f8db0362-9d8d-4356-b15c-eeb71eab8d70"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model loaded\n"
     ]
    }
   ],
   "source": [
    "from keras import backend as K\n",
    "\n",
    "target_image = K.constant(preprocess_image(target_image_path))\n",
    "style_reference_image = K.constant(preprocess_image(style_reference_img_path))\n",
    "combination_image = K.placeholder((1,img_height,img_width,3))\n",
    "\n",
    "input_tensor = K.concatenate([target_image,style_reference_image,combination_image],axis=0)\n",
    "\n",
    "model = vgg19.VGG19(input_tensor=input_tensor,weights=\"imagenet\",include_top=False)\n",
    "\n",
    "print(\"Model loaded\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "OHoIkeC7W3yM"
   },
   "outputs": [],
   "source": [
    "def content_loss(base,combination):\n",
    "  return K.sum(K.square(base-combination))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "1_m8PmrrX0Iv"
   },
   "outputs": [],
   "source": [
    "def gram_matrix(x):\n",
    "  features = K.batch_flatten(K.permute_dimensions(x,(2,0,1)))\n",
    "  gram = K.dot(features,K.transpose(features))\n",
    "  return gram"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "y8SKG-X5YU98"
   },
   "outputs": [],
   "source": [
    "def style_loss(style,combination):\n",
    "  S = gram_matrix(style)\n",
    "  C = gram_matrix(combination)\n",
    "  \n",
    "  channels = 3\n",
    "  size = img_height * img_width\n",
    "  \n",
    "  return K.sum(K.square(S-C)) / (4. * (channels ** 2) * (size ** 2))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "WMPUi_4gY6-U"
   },
   "outputs": [],
   "source": [
    "def total_variation_loss(x):\n",
    "  a = K.square(\n",
    "x[:, :img_height - 1, :img_width - 1, :] -\n",
    "x[:, 1:, :img_width - 1, :])\n",
    "  b = K.square(\n",
    "x[:, :img_height - 1, :img_width - 1, :] -\n",
    "x[:, :img_height - 1, 1:, :])\n",
    "  return K.sum(K.pow(a + b, 1.25))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "jdQg5JT6ZMzx"
   },
   "outputs": [],
   "source": [
    "output_dict = dict([(layer.name,layer.output) for layer in model.layers])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 391
    },
    "colab_type": "code",
    "id": "9yHfTpZnZpY6",
    "outputId": "abb728aa-8f72-447f-a77a-7148401c7110"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'input_1': <tf.Tensor 'concat:0' shape=(3, 400, 400, 3) dtype=float32>,\n",
       " 'block1_conv1': <tf.Tensor 'block1_conv1/Relu:0' shape=(3, 400, 400, 64) dtype=float32>,\n",
       " 'block1_conv2': <tf.Tensor 'block1_conv2/Relu:0' shape=(3, 400, 400, 64) dtype=float32>,\n",
       " 'block1_pool': <tf.Tensor 'block1_pool/MaxPool:0' shape=(3, 200, 200, 64) dtype=float32>,\n",
       " 'block2_conv1': <tf.Tensor 'block2_conv1/Relu:0' shape=(3, 200, 200, 128) dtype=float32>,\n",
       " 'block2_conv2': <tf.Tensor 'block2_conv2/Relu:0' shape=(3, 200, 200, 128) dtype=float32>,\n",
       " 'block2_pool': <tf.Tensor 'block2_pool/MaxPool:0' shape=(3, 100, 100, 128) dtype=float32>,\n",
       " 'block3_conv1': <tf.Tensor 'block3_conv1/Relu:0' shape=(3, 100, 100, 256) dtype=float32>,\n",
       " 'block3_conv2': <tf.Tensor 'block3_conv2/Relu:0' shape=(3, 100, 100, 256) dtype=float32>,\n",
       " 'block3_conv3': <tf.Tensor 'block3_conv3/Relu:0' shape=(3, 100, 100, 256) dtype=float32>,\n",
       " 'block3_conv4': <tf.Tensor 'block3_conv4/Relu:0' shape=(3, 100, 100, 256) dtype=float32>,\n",
       " 'block3_pool': <tf.Tensor 'block3_pool/MaxPool:0' shape=(3, 50, 50, 256) dtype=float32>,\n",
       " 'block4_conv1': <tf.Tensor 'block4_conv1/Relu:0' shape=(3, 50, 50, 512) dtype=float32>,\n",
       " 'block4_conv2': <tf.Tensor 'block4_conv2/Relu:0' shape=(3, 50, 50, 512) dtype=float32>,\n",
       " 'block4_conv3': <tf.Tensor 'block4_conv3/Relu:0' shape=(3, 50, 50, 512) dtype=float32>,\n",
       " 'block4_conv4': <tf.Tensor 'block4_conv4/Relu:0' shape=(3, 50, 50, 512) dtype=float32>,\n",
       " 'block4_pool': <tf.Tensor 'block4_pool/MaxPool:0' shape=(3, 25, 25, 512) dtype=float32>,\n",
       " 'block5_conv1': <tf.Tensor 'block5_conv1/Relu:0' shape=(3, 25, 25, 512) dtype=float32>,\n",
       " 'block5_conv2': <tf.Tensor 'block5_conv2/Relu:0' shape=(3, 25, 25, 512) dtype=float32>,\n",
       " 'block5_conv3': <tf.Tensor 'block5_conv3/Relu:0' shape=(3, 25, 25, 512) dtype=float32>,\n",
       " 'block5_conv4': <tf.Tensor 'block5_conv4/Relu:0' shape=(3, 25, 25, 512) dtype=float32>,\n",
       " 'block5_pool': <tf.Tensor 'block5_pool/MaxPool:0' shape=(3, 12, 12, 512) dtype=float32>}"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "output_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "kpdWHSdxZq-7"
   },
   "outputs": [],
   "source": [
    "content_layer = \"block5_conv2\"\n",
    "style_layers = ['block1_conv1',\n",
    "                'block2_conv1',\n",
    "                'block3_conv1',\n",
    "                'block4_conv1',\n",
    "                'block5_conv1']\n",
    "\n",
    "total_variation_weights = 1e-4\n",
    "style_weight = 1.\n",
    "content_weight = 0.025\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "22qIElKAaOMT"
   },
   "outputs": [],
   "source": [
    "loss = K.variable(0.)\n",
    "layer_features = output_dict[content_layer]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "QPHohzCcasn1",
    "outputId": "7a2c544a-c97e-450c-84a7-10bb800bba31"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor 'block5_conv2/Relu:0' shape=(3, 25, 25, 512) dtype=float32>"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "layer_features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "mAGkeyprauDX"
   },
   "outputs": [],
   "source": [
    "target_image_features = layer_features[0,:,:,:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "WV5Mtrimaxk5",
    "outputId": "339dd103-2bd3-4e42-95e2-d39f5beccc5d"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor 'strided_slice:0' shape=(25, 25, 512) dtype=float32>"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "target_image_features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 54
    },
    "colab_type": "code",
    "id": "I1B15_1wazLx",
    "outputId": "1881e946-9718-40cf-844d-5d3a32d971da"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:Variable += will be deprecated. Use variable.assign_add if you want assignment to the variable value or 'x = x + y' if you want a new python Tensor object.\n"
     ]
    }
   ],
   "source": [
    "combination_features  = layer_features[2,:,:,:]\n",
    "\n",
    "loss += content_weight*content_loss(target_image_features,combination_features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "aE4BToIVbMgp",
    "outputId": "c08f3ea2-0b6b-4080-a9e3-8c10c19afb12"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor 'add:0' shape=() dtype=float32>"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "jDUJtkpkbPEP"
   },
   "outputs": [],
   "source": [
    "for layer_name in style_layers:\n",
    "  layer_features = output_dict[layer_name]\n",
    "  style_reference_features = layer_features[1,:,:,:]\n",
    "  combination_features = layer_features[2,:,:,:]\n",
    "  \n",
    "  sl = style_loss(style_reference_features,combination_features)\n",
    "  \n",
    "  loss+=(style_weight/len(style_layers)) * sl\n",
    "  \n",
    "loss+=total_variation_weights * total_variation_loss(combination_image)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "Cx1tMc_McI2x",
    "outputId": "706669cf-3f92-458a-a7b8-e1e2718b8014"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor 'add_7:0' shape=() dtype=float32>"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "nl8QT644cVNa"
   },
   "outputs": [],
   "source": [
    "grads = K.gradients(loss, combination_image)[0]\n",
    "\n",
    "fetch_loss_and_grads = K.function([combination_image], [loss, grads])\n",
    "\n",
    "class Evaluator(object):\n",
    "  \n",
    "  def __init__(self):\n",
    "    self.loss_value = None\n",
    "    self.grads_values = None\n",
    "  def loss(self, x):\n",
    "    assert self.loss_value is None\n",
    "    x = x.reshape((1, img_height, img_width, 3))\n",
    "    outs = fetch_loss_and_grads([x])\n",
    "    loss_value = outs[0]\n",
    "    grad_values = outs[1].flatten().astype('float64')\n",
    "    self.loss_value = loss_value\n",
    "    self.grad_values = grad_values\n",
    "    return self.loss_value\n",
    "  def grads(self, x):\n",
    "    assert self.loss_value is not None\n",
    "    grad_values = np.copy(self.grad_values)\n",
    "    self.loss_value = None\n",
    "    self.grad_values = None\n",
    "    return grad_values\n",
    "evaluator = Evaluator()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "aObMFMT2dTEK"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Start of iteration 0\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-27-6e8ed26afdf9>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m     10\u001b[0m   \u001b[0mprint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'Start of iteration'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mi\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     11\u001b[0m   \u001b[0mstart_time\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtime\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtime\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 12\u001b[1;33m   \u001b[0mx\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmin_val\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0minfo\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mfmin_l_bfgs_b\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mevaluator\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mloss\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mx\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mfprime\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mevaluator\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mgrads\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mmaxfun\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m20\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     13\u001b[0m   \u001b[0mprint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'Current loss value:'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmin_val\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     14\u001b[0m   \u001b[0mimg\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mx\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcopy\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mreshape\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mimg_height\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mimg_width\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;36m3\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\shivu\\downloads\\python notes\\lib\\site-packages\\scipy\\optimize\\lbfgsb.py\u001b[0m in \u001b[0;36mfmin_l_bfgs_b\u001b[1;34m(func, x0, fprime, args, approx_grad, bounds, m, factr, pgtol, epsilon, iprint, maxfun, maxiter, disp, callback, maxls)\u001b[0m\n\u001b[0;32m    197\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    198\u001b[0m     res = _minimize_lbfgsb(fun, x0, args=args, jac=jac, bounds=bounds,\n\u001b[1;32m--> 199\u001b[1;33m                            **opts)\n\u001b[0m\u001b[0;32m    200\u001b[0m     d = {'grad': res['jac'],\n\u001b[0;32m    201\u001b[0m          \u001b[1;34m'task'\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0mres\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'message'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\shivu\\downloads\\python notes\\lib\\site-packages\\scipy\\optimize\\lbfgsb.py\u001b[0m in \u001b[0;36m_minimize_lbfgsb\u001b[1;34m(fun, x0, args, jac, bounds, disp, maxcor, ftol, gtol, eps, maxfun, maxiter, iprint, callback, maxls, **unknown_options)\u001b[0m\n\u001b[0;32m    333\u001b[0m             \u001b[1;31m# until the completion of the current minimization iteration.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    334\u001b[0m             \u001b[1;31m# Overwrite f and g:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 335\u001b[1;33m             \u001b[0mf\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mg\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mfunc_and_grad\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mx\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    336\u001b[0m         \u001b[1;32melif\u001b[0m \u001b[0mtask_str\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mstartswith\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34mb'NEW_X'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    337\u001b[0m             \u001b[1;31m# new iteration\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\shivu\\downloads\\python notes\\lib\\site-packages\\scipy\\optimize\\lbfgsb.py\u001b[0m in \u001b[0;36mfunc_and_grad\u001b[1;34m(x)\u001b[0m\n\u001b[0;32m    283\u001b[0m     \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    284\u001b[0m         \u001b[1;32mdef\u001b[0m \u001b[0mfunc_and_grad\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mx\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 285\u001b[1;33m             \u001b[0mf\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mfun\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mx\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    286\u001b[0m             \u001b[0mg\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mjac\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mx\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    287\u001b[0m             \u001b[1;32mreturn\u001b[0m \u001b[0mf\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mg\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\shivu\\downloads\\python notes\\lib\\site-packages\\scipy\\optimize\\optimize.py\u001b[0m in \u001b[0;36mfunction_wrapper\u001b[1;34m(*wrapper_args)\u001b[0m\n\u001b[0;32m    290\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mfunction_wrapper\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0mwrapper_args\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    291\u001b[0m         \u001b[0mncalls\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m+=\u001b[0m \u001b[1;36m1\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 292\u001b[1;33m         \u001b[1;32mreturn\u001b[0m \u001b[0mfunction\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mwrapper_args\u001b[0m \u001b[1;33m+\u001b[0m \u001b[0margs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    293\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    294\u001b[0m     \u001b[1;32mreturn\u001b[0m \u001b[0mncalls\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfunction_wrapper\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m<ipython-input-26-96ed98249e4e>\u001b[0m in \u001b[0;36mloss\u001b[1;34m(self, x)\u001b[0m\n\u001b[0;32m     11\u001b[0m     \u001b[1;32massert\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mloss_value\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     12\u001b[0m     \u001b[0mx\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mx\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mreshape\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mimg_height\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mimg_width\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;36m3\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 13\u001b[1;33m     \u001b[0mouts\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mfetch_loss_and_grads\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mx\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     14\u001b[0m     \u001b[0mloss_value\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mouts\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     15\u001b[0m     \u001b[0mgrad_values\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mouts\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mflatten\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mastype\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'float64'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\shivu\\downloads\\python notes\\lib\\site-packages\\keras\\backend\\tensorflow_backend.py\u001b[0m in \u001b[0;36m__call__\u001b[1;34m(self, inputs)\u001b[0m\n\u001b[0;32m   2719\u001b[0m                     \u001b[1;34m'In order to feed symbolic tensors to a Keras model '\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2720\u001b[0m                     'in TensorFlow, you need tensorflow 1.8 or higher.')\n\u001b[1;32m-> 2721\u001b[1;33m             \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_legacy_call\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   2722\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2723\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\shivu\\downloads\\python notes\\lib\\site-packages\\keras\\backend\\tensorflow_backend.py\u001b[0m in \u001b[0;36m_legacy_call\u001b[1;34m(self, inputs)\u001b[0m\n\u001b[0;32m   2691\u001b[0m         \u001b[0msession\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mget_session\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2692\u001b[0m         updated = session.run(fetches=fetches, feed_dict=feed_dict,\n\u001b[1;32m-> 2693\u001b[1;33m                               **self.session_kwargs)\n\u001b[0m\u001b[0;32m   2694\u001b[0m         \u001b[1;32mreturn\u001b[0m \u001b[0mupdated\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m:\u001b[0m\u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0moutputs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2695\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\shivu\\downloads\\python notes\\lib\\site-packages\\tensorflow\\python\\client\\session.py\u001b[0m in \u001b[0;36mrun\u001b[1;34m(self, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[0;32m    903\u001b[0m     \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    904\u001b[0m       result = self._run(None, fetches, feed_dict, options_ptr,\n\u001b[1;32m--> 905\u001b[1;33m                          run_metadata_ptr)\n\u001b[0m\u001b[0;32m    906\u001b[0m       \u001b[1;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    907\u001b[0m         \u001b[0mproto_data\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\shivu\\downloads\\python notes\\lib\\site-packages\\tensorflow\\python\\client\\session.py\u001b[0m in \u001b[0;36m_run\u001b[1;34m(self, handle, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[0;32m   1135\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0mfinal_fetches\u001b[0m \u001b[1;32mor\u001b[0m \u001b[0mfinal_targets\u001b[0m \u001b[1;32mor\u001b[0m \u001b[1;33m(\u001b[0m\u001b[0mhandle\u001b[0m \u001b[1;32mand\u001b[0m \u001b[0mfeed_dict_tensor\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1136\u001b[0m       results = self._do_run(handle, final_targets, final_fetches,\n\u001b[1;32m-> 1137\u001b[1;33m                              feed_dict_tensor, options, run_metadata)\n\u001b[0m\u001b[0;32m   1138\u001b[0m     \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1139\u001b[0m       \u001b[0mresults\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\shivu\\downloads\\python notes\\lib\\site-packages\\tensorflow\\python\\client\\session.py\u001b[0m in \u001b[0;36m_do_run\u001b[1;34m(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)\u001b[0m\n\u001b[0;32m   1353\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0mhandle\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1354\u001b[0m       return self._do_call(_run_fn, self._session, feeds, fetches, targets,\n\u001b[1;32m-> 1355\u001b[1;33m                            options, run_metadata)\n\u001b[0m\u001b[0;32m   1356\u001b[0m     \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1357\u001b[0m       \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_do_call\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0m_prun_fn\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_session\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mhandle\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfeeds\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfetches\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\shivu\\downloads\\python notes\\lib\\site-packages\\tensorflow\\python\\client\\session.py\u001b[0m in \u001b[0;36m_do_call\u001b[1;34m(self, fn, *args)\u001b[0m\n\u001b[0;32m   1359\u001b[0m   \u001b[1;32mdef\u001b[0m \u001b[0m_do_call\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfn\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1360\u001b[0m     \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1361\u001b[1;33m       \u001b[1;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1362\u001b[0m     \u001b[1;32mexcept\u001b[0m \u001b[0merrors\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mOpError\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1363\u001b[0m       \u001b[0mmessage\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mcompat\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mas_text\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0me\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmessage\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\shivu\\downloads\\python notes\\lib\\site-packages\\tensorflow\\python\\client\\session.py\u001b[0m in \u001b[0;36m_run_fn\u001b[1;34m(session, feed_dict, fetch_list, target_list, options, run_metadata)\u001b[0m\n\u001b[0;32m   1338\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1339\u001b[0m           return tf_session.TF_Run(session, options, feed_dict, fetch_list,\n\u001b[1;32m-> 1340\u001b[1;33m                                    target_list, status, run_metadata)\n\u001b[0m\u001b[0;32m   1341\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1342\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0m_prun_fn\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0msession\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mhandle\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "from scipy.optimize import fmin_l_bfgs_b\n",
    "import matplotlib.pyplot as plt\n",
    "import time\n",
    "result_prefix = 'my_result'\n",
    "iterations = 20\n",
    "x = preprocess_image(target_image_path)\n",
    "x = x.flatten()\n",
    "\n",
    "for i in range(iterations):\n",
    "  print('Start of iteration', i)\n",
    "  start_time = time.time()\n",
    "  x, min_val, info = fmin_l_bfgs_b(evaluator.loss,x,fprime=evaluator.grads,maxfun=20)\n",
    "  print('Current loss value:', min_val)\n",
    "  img = x.copy().reshape((img_height, img_width, 3))\n",
    "  img = deprocess_image(img)\n",
    "  fname = result_prefix + '_at_iteration_%d.png' % i\n",
    "  plt.imsave(fname, img)\n",
    "  print('Image saved as', fname)\n",
    "  end_time = time.time()\n",
    "  print('Iteration %d completed in %ds' % (i, end_time - start_time))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "import cv2\n",
    "import matplotlib.pyplot as plt\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "878TzsaYmfXB",
    "outputId": "56798a9a-ae4b-4488-e1e1-0d9c82b3d6cf"
   },
   "outputs": [],
   "source": [
    "from imutils.video import VideoStream\n",
    "from imutils.video import WebcamVideoStream\n",
    "from imutils.video import FPS\n",
    "vs = WebcamVideoStream(src=0).start()\n",
    "fps = FPS().start()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "n_A_ToWbd-vY",
    "outputId": "1e35a64b-eb83-4430-f110-71f92bbad999"
   },
   "outputs": [],
   "source": [
    "import imutils\n",
    "import cv2\n",
    "while True:\n",
    "   # grab the frame from the threaded video stream and resize it\n",
    "    # to have a maximum width of 400 pixels\n",
    "    frame = vs.read()\n",
    "    frame = imutils.resize(frame, width=500)\n",
    " \n",
    "    # grab the frame dimensions and convert it to a blob\n",
    "    (h, w) = frame.shape[:2]\n",
    "    blob = cv2.dnn.blobFromImage(cv2.resize(frame, (300, 300)), 1.0,\n",
    "        (300, 300), (104.0, 177.0, 123.0))\n",
    "    \n",
    "#     frame = cv2.cvtColor(frame,cv2.COLOR_BGR2RGB)\n",
    "#     cv2.namedWindow(\"preview\")\n",
    "    cv2.imshow(\"preview\", frame)\n",
    "    key = cv2.waitKey(3) & 0xFF\n",
    "    fps.update()\n",
    " \n",
    "    # if the `q` key was pressed, break from the loop\n",
    "    if key == ord(\"q\"):\n",
    "                WebcamVideoStream(src=0).stop()\n",
    "                vs.stream.release()\n",
    "                break\n",
    "    \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 51
    },
    "colab_type": "code",
    "id": "pEgyUg9uesPU",
    "outputId": "6b0f5d36-4c3f-4b3b-a79c-c800103252ab"
   },
   "outputs": [],
   "source": [
    "vs.stop()\n",
    "WebcamVideoStream(src=0).stop()\n",
    "fps.stop()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "vs.stream.release()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "jobkrLaTe0bB"
   },
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "2Sysd39xfSWd",
    "outputId": "099fa261-1a77-4336-c3a6-e40ce6b489af"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saved model to disk\n"
     ]
    }
   ],
   "source": [
    "from keras.models import model_from_json\n",
    "network_json=model.to_json()\n",
    "with open(\"style.json\", \"w\") as json_file:\n",
    "    json_file.write(network_json)\n",
    "    \n",
    "model.save_weights(\"styletransfer.h5\")\n",
    "print(\"Saved model to disk\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "pxOm8Tjcjlyb"
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "name": "neuralstyletransfer.ipynb",
   "provenance": [],
   "version": "0.3.2"
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}