From 18072df9c1cefd5924b2a24bf0d8484e65bf5097 Mon Sep 17 00:00:00 2001 From: Aditya M Date: Sat, 9 Jan 2021 20:08:51 +0530 Subject: [PATCH] DCGAN --- .ipynb_checkpoints/DCGAN-checkpoint.ipynb | 473 ++++++++++++++++++++++ DCGAN.ipynb | 321 ++------------- 2 files changed, 517 insertions(+), 277 deletions(-) create mode 100644 .ipynb_checkpoints/DCGAN-checkpoint.ipynb diff --git a/.ipynb_checkpoints/DCGAN-checkpoint.ipynb b/.ipynb_checkpoints/DCGAN-checkpoint.ipynb new file mode 100644 index 0000000..b80aa2b --- /dev/null +++ b/.ipynb_checkpoints/DCGAN-checkpoint.ipynb @@ -0,0 +1,473 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "1czVdIlqnImH" + }, + "source": [ + "# Deep Convolutional GAN (DCGAN)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "1KD3ZgLs80vY" + }, + "source": [ + "### Goal\n", + "In this notebook, you're going to create another GAN using the MNIST dataset." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "wU8DDM6l9rZb" + }, + "source": [ + "## Getting Started\n", + "\n", + "#### DCGAN\n", + " \n", + "\n", + "\n", + "\n", + "\n", + "* Use convolutions without any pooling layers\n", + "* Use batchnorm in both the generator and the discriminator\n", + "* Don't use fully connected hidden layers\n", + "* Use ReLU activation in the generator for all layers except for the output, which uses a Tanh activation.\n", + "* Use LeakyReLU activation in the discriminator for all layers except for the output, which does not use an activation\n" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "colab": {}, + "colab_type": "code", + "id": "JfkorNJrnmNO" + }, + "outputs": [], + "source": [ + "import torch\n", + "from torch import nn\n", + "from tqdm.auto import tqdm\n", + "from torchvision import transforms\n", + "from torchvision.datasets import MNIST\n", + "from torchvision.utils import make_grid\n", + "from torch.utils.data import DataLoader\n", + "import matplotlib.pyplot as plt\n", + "torch.manual_seed(0) # Set for testing purposes, please do not change!\n", + "\n", + "\n", + "def show_tensor_images(image_tensor, num_images=25, size=(1, 28, 28)):\n", + " '''\n", + " Function for visualizing images: Given a tensor of images, number of images, and\n", + " size per image, plots and prints the images in an uniform grid.\n", + " '''\n", + " image_tensor = (image_tensor + 1) / 2\n", + " image_unflat = image_tensor.detach().cpu()\n", + " image_grid = make_grid(image_unflat[:num_images], nrow=5)\n", + " plt.imshow(image_grid.permute(1, 2, 0).squeeze())\n", + " plt.show()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "P1A1M6kpnfxw" + }, + "source": [ + "## Generator\n", + "The first component you will make is the generator. You may notice that instead of passing in the image dimension, you will pass the number of image channels to the generator. This is because with DCGAN, you use convolutions which don’t depend on the number of pixels on an image. However, the number of channels is important to determine the size of the filters.\n", + "\n", + "You will build a generator using 4 layers (3 hidden layers + 1 output layer). As before, you will need to write a function to create a single block for the generator's neural network.\n", + " \n", + "Since in DCGAN the activation function will be different for the output layer, you will need to check what layer is being created. You are supplied with some tests following the code cell so you can see if you're on the right track!\n", + "\n", + "At the end of the generator class, you are given a forward pass function that takes in a noise vector and generates an image of the output dimension using your neural network. You are also given a function to create a noise vector. These functions are the same as the ones from the last assignment.\n", + "\n", + "
\n", + "\n", + "\n", + "Optional hint for make_gen_block\n", + "\n", + "\n", + "\n", + "1. You'll find [nn.ConvTranspose2d](https://pytorch.org/docs/master/generated/torch.nn.ConvTranspose2d.html) and [nn.BatchNorm2d](https://pytorch.org/docs/master/generated/torch.nn.BatchNorm2d.html) useful!\n", + "
" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": { + "colab": {}, + "colab_type": "code", + "id": "EvO7h0LYnEJZ" + }, + "outputs": [], + "source": [ + "# UNQ_C1 (UNIQUE CELL IDENTIFIER, DO NOT EDIT)\n", + "# GRADED FUNCTION: Generator\n", + "class Generator(nn.Module):\n", + " def __init__(self, z_dim=10, im_chan=1, hidden_dim=64):\n", + " super(Generator, self).__init__()\n", + " self.z_dim = z_dim\n", + " # Build the neural network\n", + " self.gen = nn.Sequential(\n", + " self.make_gen_block(z_dim, hidden_dim * 4),\n", + " self.make_gen_block(hidden_dim * 4, hidden_dim * 2, kernel_size=4, stride=1),\n", + " self.make_gen_block(hidden_dim * 2, hidden_dim),\n", + " self.make_gen_block(hidden_dim, im_chan, kernel_size=4, final_layer=True),\n", + " )\n", + "\n", + " def make_gen_block(self, input_channels, output_channels, kernel_size=3, stride=2, final_layer=False):\n", + " if not final_layer:\n", + " return nn.Sequential(\n", + " nn.ConvTranspose2d(input_channels, output_channels, kernel_size, stride),\n", + " nn.BatchNorm2d(output_channels),\n", + " nn.ReLU(inplace=True)\n", + " )\n", + " else:\n", + " return nn.Sequential(\n", + " nn.ConvTranspose2d(input_channels, output_channels, kernel_size, stride),\n", + " nn.Tanh()\n", + " )\n", + "\n", + " def unsqueeze_noise(self, noise):\n", + " return noise.view(len(noise), self.z_dim, 1, 1)\n", + "\n", + " def forward(self, noise):\n", + " x = self.unsqueeze_noise(noise)\n", + " return self.gen(x)\n", + "\n", + "def get_noise(n_samples, z_dim, device='cpu'):\n", + " return torch.randn(n_samples, z_dim, device=device)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "vBnOVbTpzW2M" + }, + "source": [ + "Here's the test for your generator block:" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "r9fScH98nkYH" + }, + "source": [ + "## Discriminator\n", + "The second component you need to create is the discriminator.\n", + "\n", + "You will use 3 layers in your discriminator's neural network. Like with the generator, you will need create the function to create a single neural network block for the discriminator.\n", + "" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": { + "colab": {}, + "colab_type": "code", + "id": "aA4AxGnmpuPq" + }, + "outputs": [], + "source": [ + "class Discriminator(nn.Module):\n", + " def __init__(self, im_chan=1, hidden_dim=16):\n", + " super(Discriminator, self).__init__()\n", + " self.disc = nn.Sequential(\n", + " self.make_disc_block(im_chan, hidden_dim),\n", + " self.make_disc_block(hidden_dim, hidden_dim * 2),\n", + " self.make_disc_block(hidden_dim * 2, 1, final_layer=True),\n", + " )\n", + " def make_disc_block(self, input_channels, output_channels, kernel_size=4, stride=2, final_layer=False):\n", + " if not final_layer:\n", + " return nn.Sequential(\n", + " nn.Conv2d(input_channels, output_channels, kernel_size, stride),\n", + " nn.BatchNorm2d(output_channels),\n", + " nn.LeakyReLU(0.2, inplace=True)\n", + " )\n", + " else:\n", + " return nn.Sequential(\n", + " nn.Conv2d(input_channels, output_channels, kernel_size, stride)\n", + " )\n", + " def forward(self, image):\n", + " disc_pred = self.disc(image)\n", + " return disc_pred.view(len(disc_pred), -1)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "tsOvZwjIzQ0F" + }, + "source": [ + "Here's a test for your discriminator block:" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "qRk_8azSq3tF" + }, + "source": [ + "## Training\n", + "Now you can put it all together!\n", + "Remember that these are your parameters:\n", + " * criterion: the loss function\n", + " * n_epochs: the number of times you iterate through the entire dataset when training\n", + " * z_dim: the dimension of the noise vector\n", + " * display_step: how often to display/visualize the images\n", + " * batch_size: the number of images per forward/backward pass\n", + " * lr: the learning rate\n", + " * beta_1, beta_2: the momentum term\n", + " * device: the device type\n", + "\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": { + "colab": {}, + "colab_type": "code", + "id": "IFLQ039u-qdu" + }, + "outputs": [], + "source": [ + "criterion = nn.BCEWithLogitsLoss()\n", + "z_dim = 64\n", + "display_step = 500\n", + "batch_size = 128\n", + "lr = 0.0002\n", + "\n", + "beta_1 = 0.5 \n", + "beta_2 = 0.999\n", + "device = 'cpu'\n", + "\n", + "# You can tranform the image values to be between -1 and 1 (the range of the tanh activation)\n", + "transform = transforms.Compose([\n", + " transforms.ToTensor(),\n", + " transforms.Normalize((0.5,), (0.5,)),\n", + "])\n", + "\n", + "dataloader = DataLoader(\n", + " MNIST('.', download=False, transform=transform),\n", + " batch_size=batch_size,\n", + " shuffle=True)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "24Var22i_Ccs" + }, + "source": [ + "Then, you can initialize your generator, discriminator, and optimizers." + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": { + "colab": {}, + "colab_type": "code", + "id": "sDFRZ8tg_Y57" + }, + "outputs": [], + "source": [ + "gen = Generator(z_dim).to(device)\n", + "gen_opt = torch.optim.Adam(gen.parameters(), lr=lr, betas=(beta_1, beta_2))\n", + "disc = Discriminator().to(device) \n", + "disc_opt = torch.optim.Adam(disc.parameters(), lr=lr, betas=(beta_1, beta_2))\n", + "\n", + "# You initialize the weights to the normal distribution\n", + "# with mean 0 and standard deviation 0.02\n", + "def weights_init(m):\n", + " if isinstance(m, nn.Conv2d) or isinstance(m, nn.ConvTranspose2d):\n", + " torch.nn.init.normal_(m.weight, 0.0, 0.02)\n", + " if isinstance(m, nn.BatchNorm2d):\n", + " torch.nn.init.normal_(m.weight, 0.0, 0.02)\n", + " torch.nn.init.constant_(m.bias, 0)\n", + "gen = gen.apply(weights_init)\n", + "disc = disc.apply(weights_init)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "7iCTg3w4_Zw6" + }, + "source": [ + "Finally, you can train your GAN!\n", + "For each epoch, you will process the entire dataset in batches. For every batch, you will update the discriminator and generator. Then, you can see DCGAN's results!" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "-5dhXMXLvt7l" + }, + "source": [ + "Here's roughly the progression you should be expecting. On GPU this takes about 30 seconds per thousand steps. On CPU, this can take about 8 hours per thousand steps. You might notice that in the image of Step 5000, the generator is disproprotionately producing things that look like ones. If the discriminator didn't learn to detect this imbalance quickly enough, then the generator could just produce more ones. As a result, it may have ended up tricking the discriminator so well that there would be no more improvement, known as mode collapse: \n", + "![MNIST Digits Progression](https://drive.google.com/uc?id=1SvEV3Xz5n1kC48_VG7iehRXdL4g4l1a8&export=view)\n" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": { + "colab": {}, + "colab_type": "code", + "id": "UXptQZcwrBrq" + }, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "0b08aac3d2314fdabb8aaebefe77a3dd", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + " 0%| | 0/469 [00:00\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[1;32m 22\u001b[0m \u001b[0mmean_discriminator_loss\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0mdisc_loss\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mitem\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m/\u001b[0m \u001b[0mdisplay_step\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 23\u001b[0m \u001b[0;31m# Update gradients\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 24\u001b[0;31m \u001b[0mdisc_loss\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mretain_graph\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 25\u001b[0m \u001b[0;31m# Update optimizer\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 26\u001b[0m \u001b[0mdisc_opt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/opt/miniconda3/envs/myenv/lib/python3.7/site-packages/torch/tensor.py\u001b[0m in \u001b[0;36mbackward\u001b[0;34m(self, gradient, retain_graph, create_graph)\u001b[0m\n\u001b[1;32m 193\u001b[0m \u001b[0mproducts\u001b[0m\u001b[0;34m.\u001b[0m \u001b[0mDefaults\u001b[0m \u001b[0mto\u001b[0m\u001b[0;31m \u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 194\u001b[0m \"\"\"\n\u001b[0;32m--> 195\u001b[0;31m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mautograd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgradient\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mretain_graph\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcreate_graph\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 196\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 197\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mregister_hook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/opt/miniconda3/envs/myenv/lib/python3.7/site-packages/torch/autograd/__init__.py\u001b[0m in \u001b[0;36mbackward\u001b[0;34m(tensors, grad_tensors, retain_graph, create_graph, grad_variables)\u001b[0m\n\u001b[1;32m 97\u001b[0m Variable._execution_engine.run_backward(\n\u001b[1;32m 98\u001b[0m \u001b[0mtensors\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgrad_tensors\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mretain_graph\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcreate_graph\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 99\u001b[0;31m allow_unreachable=True) # allow_unreachable flag\n\u001b[0m\u001b[1;32m 100\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 101\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;31mKeyboardInterrupt\u001b[0m: " + ] + } + ], + "source": [ + "n_epochs = 50\n", + "cur_step = 0\n", + "mean_generator_loss = 0\n", + "mean_discriminator_loss = 0\n", + "for epoch in range(n_epochs):\n", + " # Dataloader returns the batches\n", + " for real, _ in tqdm(dataloader):\n", + " cur_batch_size = len(real)\n", + " real = real.to(device)\n", + " \n", + " ## Update discriminator ##\n", + " disc_opt.zero_grad()\n", + " fake_noise = get_noise(cur_batch_size, z_dim, device=device)\n", + " fake = gen(fake_noise)\n", + " disc_fake_pred = disc(fake.detach())\n", + " disc_fake_loss = criterion(disc_fake_pred, torch.zeros_like(disc_fake_pred))\n", + " disc_real_pred = disc(real)\n", + " disc_real_loss = criterion(disc_real_pred, torch.ones_like(disc_real_pred))\n", + " disc_loss = (disc_fake_loss + disc_real_loss) / 2\n", + "\n", + " # Keep track of the average discriminator loss\n", + " mean_discriminator_loss += disc_loss.item() / display_step\n", + " # Update gradients\n", + " disc_loss.backward(retain_graph=True)\n", + " # Update optimizer\n", + " disc_opt.step()\n", + "\n", + " ## Update generator ##\n", + " gen_opt.zero_grad()\n", + " fake_noise_2 = get_noise(cur_batch_size, z_dim, device=device)\n", + " fake_2 = gen(fake_noise_2)\n", + " disc_fake_pred = disc(fake_2)\n", + " gen_loss = criterion(disc_fake_pred, torch.ones_like(disc_fake_pred))\n", + " gen_loss.backward()\n", + " gen_opt.step()\n", + "\n", + " # Keep track of the average generator loss\n", + " mean_generator_loss += gen_loss.item() / display_step\n", + "\n", + " ## Visualization code ##\n", + " if cur_step % display_step == 0 and cur_step > 0:\n", + " print(f\"Step {cur_step}: Generator loss: {mean_generator_loss}, discriminator loss: {mean_discriminator_loss}\")\n", + " show_tensor_images(fake)\n", + " show_tensor_images(real)\n", + " mean_generator_loss = 0\n", + " mean_discriminator_loss = 0\n", + " cur_step += 1\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "accelerator": "GPU", + "colab": { + "collapsed_sections": [], + "name": "C1W2_3: Deep Convolutional GAN (DCGAN) (Student).ipynb", + "provenance": [] + }, + "coursera": { + "schema_names": [ + "GANSC1-2A" + ] + }, + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.9" + } + }, + "nbformat": 4, + "nbformat_minor": 1 +} diff --git a/DCGAN.ipynb b/DCGAN.ipynb index 66545e3..b80aa2b 100644 --- a/DCGAN.ipynb +++ b/DCGAN.ipynb @@ -18,18 +18,7 @@ }, "source": [ "### Goal\n", - "In this notebook, you're going to create another GAN using the MNIST dataset. You will implement a Deep Convolutional GAN (DCGAN), a very successful and influential GAN model developed in 2015.\n", - "\n", - "*Note: [here](https://arxiv.org/pdf/1511.06434v1.pdf) is the paper if you are interested! It might look dense now, but soon you'll be able to understand many parts of it :)*\n", - "\n", - "### Learning Objectives\n", - "1. Get hands-on experience making a widely used GAN: Deep Convolutional GAN (DCGAN).\n", - "2. Train a powerful generative model.\n", - "\n", - "\n", - "![Generator architecture](https://drive.google.com/uc?id=1kcGhcT--EuR6gzZJoD6_xClaaLIncJjt&export=view)\n", - "\n", - "Figure: Architectural drawing of a generator from DCGAN from [Radford et al (2016)](https://arxiv.org/pdf/1511.06434v1.pdf)." + "In this notebook, you're going to create another GAN using the MNIST dataset." ] }, { @@ -42,7 +31,7 @@ "## Getting Started\n", "\n", "#### DCGAN\n", - "Here are the main features of DCGAN (don't worry about memorizing these, you will be guided through the implementation!): \n", + " \n", "\n", "\n", - "There are also tests at the end for you to use.\n", - "
\n", - "\n", - "\n", - "Optional hint for make_disc_block\n", - "\n", - "\n", - "\n", - "1. You'll find [nn.Conv2d](https://pytorch.org/docs/master/generated/torch.nn.Conv2d.html), [nn.BatchNorm2d](https://pytorch.org/docs/master/generated/torch.nn.BatchNorm2d.html), and [nn.LeakyReLU](https://pytorch.org/docs/master/generated/torch.nn.LeakyReLU.html) useful!\n", - "
" + "" ] }, { @@ -340,16 +193,7 @@ }, "outputs": [], "source": [ - "# UNQ_C3 (UNIQUE CELL IDENTIFIER, DO NOT EDIT)\n", - "# GRADED FUNCTION: Discriminator\n", "class Discriminator(nn.Module):\n", - " '''\n", - " Discriminator Class\n", - " Values:\n", - " im_chan: the number of channels of the output image, a scalar\n", - " (MNIST is black-and-white, so 1 channel is your default)\n", - " hidden_dim: the inner dimension, a scalar\n", - " '''\n", " def __init__(self, im_chan=1, hidden_dim=16):\n", " super(Discriminator, self).__init__()\n", " self.disc = nn.Sequential(\n", @@ -357,79 +201,22 @@ " self.make_disc_block(hidden_dim, hidden_dim * 2),\n", " self.make_disc_block(hidden_dim * 2, 1, final_layer=True),\n", " )\n", - "\n", " def make_disc_block(self, input_channels, output_channels, kernel_size=4, stride=2, final_layer=False):\n", - " '''\n", - " Function to return a sequence of operations corresponding to a discriminator block of DCGAN, \n", - " corresponding to a convolution, a batchnorm (except for in the last layer), and an activation.\n", - " Parameters:\n", - " input_channels: how many channels the input feature representation has\n", - " output_channels: how many channels the output feature representation should have\n", - " kernel_size: the size of each convolutional filter, equivalent to (kernel_size, kernel_size)\n", - " stride: the stride of the convolution\n", - " final_layer: a boolean, true if it is the final layer and false otherwise \n", - " (affects activation and batchnorm)\n", - " '''\n", - " # Steps:\n", - " # 1) Add a convolutional layer using the given parameters.\n", - " # 2) Do a batchnorm, except for the last layer.\n", - " # 3) Follow each batchnorm with a LeakyReLU activation with slope 0.2.\n", - " \n", - " # Build the neural block\n", " if not final_layer:\n", " return nn.Sequential(\n", - " #### START CODE HERE ####\n", " nn.Conv2d(input_channels, output_channels, kernel_size, stride),\n", " nn.BatchNorm2d(output_channels),\n", " nn.LeakyReLU(0.2, inplace=True)\n", - " #### END CODE HERE ####\n", " )\n", - " else: # Final Layer\n", + " else:\n", " return nn.Sequential(\n", - " #### START CODE HERE ####\n", " nn.Conv2d(input_channels, output_channels, kernel_size, stride)\n", - " #### END CODE HERE ####\n", " )\n", - "\n", - " '''\n", - " Function for completing a forward pass of the discriminator: Given an image tensor, \n", - " returns a 1-dimension tensor representing fake/real.\n", - " Parameters:\n", - " image: a flattened image tensor with dimension (im_dim)\n", - " '''\n", " def forward(self, image):\n", " disc_pred = self.disc(image)\n", " return disc_pred.view(len(disc_pred), -1)" ] }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [], - "source": [ - "# UNQ_C4 (UNIQUE CELL IDENTIFIER, DO NOT EDIT)\n", - "'''\n", - "Test your make_disc_block() function\n", - "'''\n", - "num_test = 100\n", - "\n", - "gen = Generator()\n", - "disc = Discriminator()\n", - "test_images = gen(get_noise(num_test, gen.z_dim))\n", - "\n", - "# Test the hidden block\n", - "test_hidden_block = disc.make_disc_block(1, 5, kernel_size=6, stride=3)\n", - "hidden_output = test_hidden_block(test_images)\n", - "\n", - "# Test the final block\n", - "test_final_block = disc.make_disc_block(1, 10, kernel_size=2, stride=5, final_layer=True)\n", - "final_output = test_final_block(test_images)\n", - "\n", - "# Test the whole thing:\n", - "disc_output = disc(test_images)" - ] - }, { "cell_type": "markdown", "metadata": { @@ -440,51 +227,6 @@ "Here's a test for your discriminator block:" ] }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "GemvBkChn0_k" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Success!\n" - ] - } - ], - "source": [ - "try:\n", - " # Test the hidden block\n", - " assert tuple(hidden_output.shape) == (num_test, 5, 8, 8)\n", - " # Because of the LeakyReLU slope\n", - " assert -hidden_output.min() / hidden_output.max() > 0.15\n", - " assert -hidden_output.min() / hidden_output.max() < 0.25\n", - " assert hidden_output.std() > 0.5\n", - " assert hidden_output.std() < 1\n", - "\n", - " # Test the final block\n", - "\n", - " assert tuple(final_output.shape) == (num_test, 10, 6, 6)\n", - " assert final_output.max() > 1.0\n", - " assert final_output.min() < -1.0\n", - " assert final_output.std() > 0.3\n", - " assert final_output.std() < 0.6\n", - "\n", - " # Test the whole thing:\n", - "\n", - " assert tuple(disc_output.shape) == (num_test, 1)\n", - " assert disc_output.std() > 0.25\n", - " assert disc_output.std() < 0.5\n", - " print(\"Success!\")\n", - "except:\n", - " print(\"Some of asserts have fail\")" - ] - }, { "cell_type": "markdown", "metadata": { @@ -512,7 +254,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 6, "metadata": { "colab": {}, "colab_type": "code", @@ -524,14 +266,11 @@ "z_dim = 64\n", "display_step = 500\n", "batch_size = 128\n", - "# A learning rate of 0.0002 works well on DCGAN\n", "lr = 0.0002\n", "\n", - "# These parameters control the optimizer's momentum, which you can read more about here:\n", - "# https://distill.pub/2017/momentum/ but you don’t need to worry about it for this course!\n", "beta_1 = 0.5 \n", "beta_2 = 0.999\n", - "device = 'cuda'\n", + "device = 'cpu'\n", "\n", "# You can tranform the image values to be between -1 and 1 (the range of the tanh activation)\n", "transform = transforms.Compose([\n", @@ -557,7 +296,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 7, "metadata": { "colab": {}, "colab_type": "code", @@ -606,13 +345,41 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 8, "metadata": { "colab": {}, "colab_type": "code", "id": "UXptQZcwrBrq" }, - "outputs": [], + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "0b08aac3d2314fdabb8aaebefe77a3dd", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + " 0%| | 0/469 [00:00\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[1;32m 22\u001b[0m \u001b[0mmean_discriminator_loss\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0mdisc_loss\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mitem\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m/\u001b[0m \u001b[0mdisplay_step\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 23\u001b[0m \u001b[0;31m# Update gradients\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 24\u001b[0;31m \u001b[0mdisc_loss\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mretain_graph\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 25\u001b[0m \u001b[0;31m# Update optimizer\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 26\u001b[0m \u001b[0mdisc_opt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/opt/miniconda3/envs/myenv/lib/python3.7/site-packages/torch/tensor.py\u001b[0m in \u001b[0;36mbackward\u001b[0;34m(self, gradient, retain_graph, create_graph)\u001b[0m\n\u001b[1;32m 193\u001b[0m \u001b[0mproducts\u001b[0m\u001b[0;34m.\u001b[0m \u001b[0mDefaults\u001b[0m \u001b[0mto\u001b[0m\u001b[0;31m \u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 194\u001b[0m \"\"\"\n\u001b[0;32m--> 195\u001b[0;31m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mautograd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgradient\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mretain_graph\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcreate_graph\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 196\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 197\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mregister_hook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/opt/miniconda3/envs/myenv/lib/python3.7/site-packages/torch/autograd/__init__.py\u001b[0m in \u001b[0;36mbackward\u001b[0;34m(tensors, grad_tensors, retain_graph, create_graph, grad_variables)\u001b[0m\n\u001b[1;32m 97\u001b[0m Variable._execution_engine.run_backward(\n\u001b[1;32m 98\u001b[0m \u001b[0mtensors\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgrad_tensors\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mretain_graph\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcreate_graph\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 99\u001b[0;31m allow_unreachable=True) # allow_unreachable flag\n\u001b[0m\u001b[1;32m 100\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 101\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;31mKeyboardInterrupt\u001b[0m: " + ] + } + ], "source": [ "n_epochs = 50\n", "cur_step = 0\n", @@ -623,7 +390,7 @@ " for real, _ in tqdm(dataloader):\n", " cur_batch_size = len(real)\n", " real = real.to(device)\n", - "\n", + " \n", " ## Update discriminator ##\n", " disc_opt.zero_grad()\n", " fake_noise = get_noise(cur_batch_size, z_dim, device=device)\n", @@ -698,7 +465,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.6.9" + "version": "3.7.9" } }, "nbformat": 4,