GAN-Copy2.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "from __future__ import print_function\n",
    "import numpy as np\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.parallel\n",
    "import torch.optim as optim\n",
    "import torch.utils.data\n",
    "import torchvision.datasets as dset\n",
    "import torchvision.transforms as transforms\n",
    "import torchvision.utils as vutils\n",
    "from torch.autograd import Variable\n",
    "from torch.distributions.normal import Normal\n",
    "from torchvision.utils import save_image\n",
    "from torch.nn import functional as F\n",
    "\n",
    "batchSize = 64 \n",
    "image_size = 32\n",
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Files already downloaded and verified\n"
     ]
    }
   ],
   "source": [
    "transform = transforms.Compose([\n",
    "                               transforms.Resize(image_size),\n",
    "                               transforms.CenterCrop(image_size),\n",
    "                               transforms.ToTensor(),\n",
    "                               transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)),\n",
    "                                ])\n",
    "dataset = dset.CIFAR10(root = './data', download = True, transform = transform) \n",
    "dataloader = torch.utils.data.DataLoader(dataset, batch_size = batchSize, shuffle = True, num_workers = 2) \n",
    "\n",
    "def weights_init(m):\n",
    "    classname = m.__class__.__name__\n",
    "    if classname.find('Conv') != -1:\n",
    "        nn.init.normal_(m.weight.data, 0.0, 0.02)\n",
    "    elif classname.find('BatchNorm') != -1:\n",
    "        nn.init.normal_(m.weight.data, 1.0, 0.02)\n",
    "        nn.init.constant_(m.bias.data, 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Generator(nn.Module):\n",
    "    # initializers\n",
    "    def __init__(self, d=128):\n",
    "        super(Generator, self).__init__()\n",
    "        self.deconv1_1 = nn.ConvTranspose2d(100, d*2, 4, 1, 0)\n",
    "        self.deconv1_1_bn = nn.BatchNorm2d(d*2)\n",
    "        self.deconv1_2 = nn.ConvTranspose2d(10, d*2, 4, 1, 0)\n",
    "        self.deconv1_2_bn = nn.BatchNorm2d(d*2)\n",
    "        self.deconv2 = nn.ConvTranspose2d(d*4, d*2, 4, 2, 1)\n",
    "        self.deconv2_bn = nn.BatchNorm2d(d*2)\n",
    "        self.deconv3 = nn.ConvTranspose2d(d*2, d, 4, 2, 1)\n",
    "        self.deconv3_bn = nn.BatchNorm2d(d)\n",
    "        self.deconv4 = nn.ConvTranspose2d(d, 3, 4, 2, 1)\n",
    "\n",
    "    # weight_init\n",
    "    def weight_init(self, mean, std):\n",
    "        for m in self._modules:\n",
    "            normal_init(self._modules[m], mean, std)\n",
    "\n",
    "    # forward method\n",
    "    def forward(self, input, label):\n",
    "        x = F.relu(self.deconv1_1_bn(self.deconv1_1(input)))\n",
    "        y = F.relu(self.deconv1_2_bn(self.deconv1_2(label)))\n",
    "        #print(x.shape)\n",
    "        #print(y.shape)\n",
    "        x = torch.cat([x, y], 1)\n",
    "        x = F.relu(self.deconv2_bn(self.deconv2(x)))\n",
    "        x = F.relu(self.deconv3_bn(self.deconv3(x)))\n",
    "        x = F.tanh(self.deconv4(x))\n",
    "        # x = F.relu(self.deconv4_bn(self.deconv4(x)))\n",
    "        # x = F.tanh(self.deconv5(x))\n",
    "\n",
    "        return x\n",
    "\n",
    "class Discriminator(nn.Module):\n",
    "    # initializers\n",
    "    def __init__(self, d=128):\n",
    "        super(Discriminator, self).__init__()\n",
    "        self.conv1_1 = nn.Conv2d(3, d, 4, 2, 1)\n",
    "        self.conv1_2 = nn.Conv2d(10, d, 4, 2, 1)\n",
    "        self.conv2 = nn.Conv2d(d*2, d*2, 4, 2, 1)\n",
    "        self.conv2_bn = nn.BatchNorm2d(d*2)\n",
    "        self.conv3 = nn.Conv2d(d*2, d*4, 4, 2, 1)\n",
    "        self.conv3_bn = nn.BatchNorm2d(d*4)\n",
    "        self.conv4 = nn.Conv2d(d * 4, 1, 4, 1, 0)\n",
    "\n",
    "    # weight_init\n",
    "    def weight_init(self, mean, std):\n",
    "        for m in self._modules:\n",
    "            normal_init(self._modules[m], mean, std)\n",
    "\n",
    "    # forward method\n",
    "    def forward(self, input, label):\n",
    "        #print(input.shape)\n",
    "        x = F.leaky_relu(self.conv1_1(input), 0.2)\n",
    "        y = F.leaky_relu(self.conv1_2(label), 0.2)\n",
    "        x = torch.cat([x, y], 1)\n",
    "        #print(x.shape)\n",
    "        x = F.leaky_relu(self.conv2_bn(self.conv2(x)), 0.2)\n",
    "        x = F.leaky_relu(self.conv3_bn(self.conv3(x)), 0.2)\n",
    "        x = F.sigmoid(self.conv4(x))\n",
    "\n",
    "        return x\n",
    "\n",
    "def normal_init(m, mean, std):\n",
    "    if isinstance(m, nn.ConvTranspose2d) or isinstance(m, nn.Conv2d):\n",
    "        m.weight.data.normal_(mean, std)\n",
    "        m.bias.data.zero_()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/CS/pvp0001/.conda/envs/praveen/lib/python3.6/site-packages/ipykernel_launcher.py:14: UserWarning: volatile was removed and now has no effect. Use `with torch.no_grad():` instead.\n",
      "  \n"
     ]
    }
   ],
   "source": [
    "# fixed noise & label\n",
    "temp_z_ = torch.randn(10, 100)\n",
    "fixed_z_ = temp_z_\n",
    "fixed_y_ = torch.zeros(10, 1)\n",
    "for i in range(9):\n",
    "    fixed_z_ = torch.cat([fixed_z_, temp_z_], 0)\n",
    "    temp = torch.ones(10, 1) + i\n",
    "    fixed_y_ = torch.cat([fixed_y_, temp], 0)\n",
    "\n",
    "fixed_z_ = fixed_z_.view(-1, 100, 1, 1)\n",
    "fixed_y_label_ = torch.zeros(100, 10)\n",
    "fixed_y_label_.scatter_(1, fixed_y_.type(torch.LongTensor), 1)\n",
    "fixed_y_label_ = fixed_y_label_.view(-1, 10, 1, 1)\n",
    "fixed_z_, fixed_y_label_ = Variable(fixed_z_.cuda(), volatile=True), Variable(fixed_y_label_.cuda(), volatile=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "discriminator(\n",
       "  (conv1_1): Conv2d(3, 128, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1))\n",
       "  (conv1_2): Conv2d(10, 128, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1))\n",
       "  (conv2): Conv2d(256, 256, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1))\n",
       "  (conv2_bn): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "  (conv3): Conv2d(256, 512, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1))\n",
       "  (conv3_bn): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "  (conv4): Conv2d(512, 1, kernel_size=(4, 4), stride=(1, 1))\n",
       ")"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "G = Generator(128)\n",
    "D = Discriminator(128)\n",
    "G.weight_init(mean=0.0, std=0.02)\n",
    "D.weight_init(mean=0.0, std=0.02)\n",
    "G.cuda()\n",
    "D.cuda()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "criterion = nn.BCELoss()\n",
    "\n",
    "#fixed_noise = np.random.normal(loc=0.0,scale=1, size = (batchSize, 100, 1, 1))\n",
    "#print(fixed_noise)\n",
    "#fixed_noise = torch.FloatTensor(fixed_noise).to(device)\n",
    "\n",
    "\n",
    "fixed_noise = torch.randn(batchSize, 100, 1, 1).to(device)\n",
    "#print(fixed_noise)\n",
    "real_label = 1\n",
    "fake_label = 0\n",
    "\n",
    "\n",
    "optimizerD = optim.Adam(D.parameters(), lr=0.0002, betas=(0.5, 0.999))\n",
    "optimizerG = optim.Adam(G.parameters(), lr=0.0002, betas=(0.5, 0.999))\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "# label preprocess\n",
    "onehot = torch.zeros(10, 10)\n",
    "onehot = onehot.scatter_(1, torch.LongTensor([0, 1, 2, 3, 4, 5, 6, 7, 8, 9]).view(10,1), 1).view(10, 10, 1, 1)\n",
    "fill = torch.zeros([10, 10, image_size, image_size])\n",
    "for i in range(10):\n",
    "    fill[i, i, :, :] = 1\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_model(dataloader,netD,netG,criterion, optimizerD, optimizerG):\n",
    "    #D_x = 0\n",
    "    #D_G_z1 = 0\n",
    "    #D_G_z2 = 0\n",
    "    #corrects = 0\n",
    "    saved_model_G = netG\n",
    "    saved_model_D = netD\n",
    "    for epoch in range(25):\n",
    "        D_x = 0\n",
    "        D_G_z1 = 0\n",
    "        D_G_z2 = 0\n",
    "        corrects = 0\n",
    "        for i, (images,labels) in enumerate(dataloader, 0):\n",
    "            real_cpu = images.to(device)\n",
    "            batch_size = real_cpu.size(0)\n",
    "            labels = labels.to(device)\n",
    "            # labels(will only be used in loss function)\n",
    "            y_real_ = (torch.ones(batch_size)).to(device)\n",
    "            y_fake_ = (torch.zeros(batch_size)).to(device)\n",
    "            \n",
    "            ############################\n",
    "            # (1) Update D network: maximize log(D(x)) + log(1 - D(G(z)))\n",
    "            ###########################\n",
    "            # train with real\n",
    "            netD.zero_grad()\n",
    "            \n",
    "            mod_labels = fill[labels].to(device)\n",
    "            output = netD(real_cpu, mod_labels)\n",
    "            \n",
    "            errD_real = criterion(output, y_real_)\n",
    "            errD_real.backward()\n",
    "            D_x += output.mean().item()\n",
    "            \n",
    "            \n",
    "            \n",
    "\n",
    "            # train with fake\n",
    "            noise = torch.randn(batch_size, 100, 1, 1).to(device)\n",
    "            y_ = (torch.rand(batch_size, 1) * 10).type(torch.LongTensor).squeeze()\n",
    "            y_label_ = onehot[y_].to(device)\n",
    "            fake = netG(noise, y_label_)\n",
    "            \n",
    "            y_fill_ = fill[y_].to(device)\n",
    "            output = netD(fake.detach(), y_fill_)\n",
    "            \n",
    "            errD_fake = criterion(output, y_fake_)\n",
    "            errD_fake.backward()\n",
    "            D_G_z1 += output.mean().item()\n",
    "            errD = errD_real + errD_fake\n",
    "            optimizerD.step()\n",
    "\n",
    "            ############################\n",
    "            # (2) Update G network: maximize log(D(G(z)))\n",
    "            ###########################\n",
    "            netG.zero_grad()\n",
    "            #label.fill_(real_label).to(device)  # fake labels are real for generator cost\n",
    "            output = netD(fake, y_fill_)\n",
    "            #corrects += (output == torch.FloatTensor(labels)).sum().item()\n",
    "            \n",
    "            errG = criterion(output, y_real_)\n",
    "            errG.backward()\n",
    "            D_G_z2 += output.mean().item()\n",
    "            optimizerG.step()\n",
    "\n",
    "            \n",
    "\n",
    "            if i % len(dataloader) == 0:\n",
    "                save_image(real_cpu,\n",
    "                        '../results/val/real_samples{}_{}.png'.format(epoch,i),\n",
    "                        normalize=True)\n",
    "                #print(\"hmm\")\n",
    "                fake = netG(noise, y_label_)\n",
    "                save_image(fake.detach(),\n",
    "                        '../results/val/fake_samples_epoch{}_{}.png'.format(epoch, i),\n",
    "                        normalize=True)\n",
    "        \n",
    "            if(round(errD.item(),1) == round(errG.item(),1)):\n",
    "                print(epoch)\n",
    "                print(errD.item(),    round(errD.item(),1),  errG.item(),   round(errG.item(),1))\n",
    "                saved_model_G = netG\n",
    "                saved_model_D = netD\n",
    "            \n",
    "        print('[%d/%d] Loss_D: %.4f Loss_G: %.4f D(x): %.4f D(G(z)): %.4f / %.4f'\n",
    "                  % (epoch, 25,\n",
    "                     errD.item(), errG.item(), D_x/len(dataloader), D_G_z1/len(dataloader), D_G_z2/len(dataloader)))\n",
    "    return saved_model_G,saved_model_D"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0/25] Loss_D: 0.4538 Loss_G: 4.0499 D(x): 0.8080 D(G(z)): 0.1869 / 0.0523\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/CS/pvp0001/.conda/envs/praveen/lib/python3.6/site-packages/torch/nn/modules/loss.py:512: UserWarning: Using a target size (torch.Size([16])) that is different to the input size (torch.Size([16, 1, 1, 1])) is deprecated. Please ensure they have the same size.\n",
      "  return F.binary_cross_entropy(input, target, weight=self.weight, reduction=self.reduction)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1\n",
      "1.1773673295974731 1.2 1.249106526374817 1.2\n",
      "1\n",
      "1.249679446220398 1.2 1.1887118816375732 1.2\n",
      "1\n",
      "1.2380160093307495 1.2 1.2494089603424072 1.2\n",
      "1\n",
      "1.1872479915618896 1.2 1.1932779550552368 1.2\n",
      "[1/25] Loss_D: 1.0930 Loss_G: 5.1519 D(x): 0.7671 D(G(z)): 0.2309 / 0.0873\n",
      "[2/25] Loss_D: 0.4309 Loss_G: 3.1673 D(x): 0.7951 D(G(z)): 0.2029 / 0.0837\n",
      "3\n",
      "0.8409655690193176 0.8 0.8085958957672119 0.8\n",
      "[3/25] Loss_D: 0.2985 Loss_G: 3.9847 D(x): 0.8031 D(G(z)): 0.1962 / 0.0854\n",
      "[4/25] Loss_D: 0.3141 Loss_G: 4.6108 D(x): 0.8069 D(G(z)): 0.1915 / 0.0879\n",
      "[5/25] Loss_D: 0.3666 Loss_G: 4.8858 D(x): 0.7960 D(G(z)): 0.2027 / 0.0950\n",
      "6\n",
      "1.0633553266525269 1.1 1.0673272609710693 1.1\n",
      "6\n",
      "1.1936947107315063 1.2 1.168980360031128 1.2\n",
      "6\n",
      "0.9267253875732422 0.9 0.8709442615509033 0.9\n",
      "[6/25] Loss_D: 1.0706 Loss_G: 5.3575 D(x): 0.7520 D(G(z)): 0.2468 / 0.1351\n",
      "7\n",
      "1.1575927734375 1.2 1.155476689338684 1.2\n",
      "7\n",
      "0.9099842309951782 0.9 0.9391071796417236 0.9\n",
      "7\n",
      "0.9707447290420532 1.0 0.9877685904502869 1.0\n",
      "7\n",
      "1.126444935798645 1.1 1.1086095571517944 1.1\n",
      "7\n",
      "1.0098634958267212 1.0 0.9542313814163208 1.0\n",
      "[7/25] Loss_D: 1.2425 Loss_G: 0.8358 D(x): 0.7466 D(G(z)): 0.2513 / 0.1449\n",
      "8\n",
      "0.9749988317489624 1.0 1.0036200284957886 1.0\n",
      "8\n",
      "1.1532275676727295 1.2 1.164175271987915 1.2\n",
      "8\n",
      "0.9890576004981995 1.0 0.9903829097747803 1.0\n",
      "8\n",
      "1.0228146314620972 1.0 0.9761626720428467 1.0\n",
      "[8/25] Loss_D: 0.5683 Loss_G: 3.2085 D(x): 0.7620 D(G(z)): 0.2376 / 0.1369\n",
      "9\n",
      "1.089461326599121 1.1 1.0973050594329834 1.1\n",
      "9\n",
      "1.019848108291626 1.0 0.9896130561828613 1.0\n",
      "9\n",
      "1.1834466457366943 1.2 1.2016491889953613 1.2\n",
      "[9/25] Loss_D: 0.4527 Loss_G: 4.0354 D(x): 0.7727 D(G(z)): 0.2266 / 0.1335\n",
      "10\n",
      "1.0748748779296875 1.1 1.1133092641830444 1.1\n",
      "10\n",
      "1.0050301551818848 1.0 1.0168813467025757 1.0\n",
      "10\n",
      "1.0499670505523682 1.0 0.9819079041481018 1.0\n",
      "10\n",
      "1.2589720487594604 1.3 1.3330527544021606 1.3\n",
      "10\n",
      "1.2119064331054688 1.2 1.209395408630371 1.2\n",
      "[10/25] Loss_D: 0.2559 Loss_G: 3.3902 D(x): 0.7703 D(G(z)): 0.2289 / 0.1349\n",
      "[11/25] Loss_D: 0.6720 Loss_G: 2.2281 D(x): 0.7735 D(G(z)): 0.2259 / 0.1349\n",
      "12\n",
      "1.1638492345809937 1.2 1.2223224639892578 1.2\n",
      "12\n",
      "1.0203067064285278 1.0 0.95911705493927 1.0\n",
      "[12/25] Loss_D: 0.9448 Loss_G: 2.3466 D(x): 0.7777 D(G(z)): 0.2218 / 0.1298\n",
      "13\n",
      "0.641776442527771 0.6 0.5815947651863098 0.6\n",
      "13\n",
      "0.8527896404266357 0.9 0.8585232496261597 0.9\n",
      "13\n",
      "1.351815104484558 1.4 1.4190384149551392 1.4\n",
      "13\n",
      "0.8741512298583984 0.9 0.856208324432373 0.9\n",
      "13\n",
      "1.0489152669906616 1.0 1.0302952527999878 1.0\n",
      "13\n",
      "0.9466625452041626 0.9 0.8634400367736816 0.9\n",
      "[13/25] Loss_D: 0.3708 Loss_G: 3.2910 D(x): 0.7821 D(G(z)): 0.2177 / 0.1285\n",
      "14\n",
      "1.1217607259750366 1.1 1.1320805549621582 1.1\n",
      "[14/25] Loss_D: 0.5793 Loss_G: 5.2579 D(x): 0.7906 D(G(z)): 0.2093 / 0.1234\n",
      "15\n",
      "1.0414055585861206 1.0 1.0416632890701294 1.0\n",
      "[15/25] Loss_D: 0.6446 Loss_G: 3.6921 D(x): 0.7945 D(G(z)): 0.2049 / 0.1204\n",
      "16\n",
      "1.5009809732437134 1.5 1.5483731031417847 1.5\n",
      "[16/25] Loss_D: 0.3929 Loss_G: 3.7389 D(x): 0.7993 D(G(z)): 0.2005 / 0.1187\n",
      "17\n",
      "1.226417899131775 1.2 1.2339732646942139 1.2\n",
      "[17/25] Loss_D: 1.6426 Loss_G: 9.8689 D(x): 0.8080 D(G(z)): 0.1922 / 0.1102\n",
      "18\n",
      "1.0171843767166138 1.0 0.996172308921814 1.0\n",
      "[18/25] Loss_D: 0.3308 Loss_G: 4.9727 D(x): 0.8074 D(G(z)): 0.1919 / 0.1116\n",
      "[19/25] Loss_D: 0.5930 Loss_G: 7.8641 D(x): 0.8128 D(G(z)): 0.1874 / 0.1098\n",
      "20\n",
      "0.7618198990821838 0.8 0.8481922149658203 0.8\n",
      "[20/25] Loss_D: 0.7494 Loss_G: 8.5176 D(x): 0.8205 D(G(z)): 0.1792 / 0.1037\n",
      "21\n",
      "0.765735387802124 0.8 0.8057770729064941 0.8\n",
      "21\n",
      "0.5811717510223389 0.6 0.5693551301956177 0.6\n",
      "[21/25] Loss_D: 2.2303 Loss_G: 2.1111 D(x): 0.8201 D(G(z)): 0.1790 / 0.1044\n",
      "[22/25] Loss_D: 0.4078 Loss_G: 5.8816 D(x): 0.8294 D(G(z)): 0.1707 / 0.0946\n",
      "23\n",
      "0.8333792090415955 0.8 0.8475278615951538 0.8\n",
      "[23/25] Loss_D: 0.6775 Loss_G: 9.2756 D(x): 0.8297 D(G(z)): 0.1704 / 0.0960\n",
      "24\n",
      "1.4087375402450562 1.4 1.4224897623062134 1.4\n",
      "24\n",
      "0.8625616431236267 0.9 0.9192192554473877 0.9\n",
      "24\n",
      "1.1447300910949707 1.1 1.0970938205718994 1.1\n",
      "[24/25] Loss_D: 1.1447 Loss_G: 1.0971 D(x): 0.8364 D(G(z)): 0.1625 / 0.0916\n"
     ]
    }
   ],
   "source": [
    "model_G, model_D = train_model(dataloader,D,G,criterion, optimizerD, optimizerG)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}