lab-09_5_mnist_nn_dropout.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Lab 10 MNIST and softmax\n",
    "import torch\n",
    "import torchvision.datasets as dsets\n",
    "import torchvision.transforms as transforms\n",
    "import random"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "device = 'cuda' if torch.cuda.is_available() else 'cpu'\n",
    "\n",
    "# for reproducibility\n",
    "random.seed(777)\n",
    "torch.manual_seed(777)\n",
    "if device == 'cuda':\n",
    "    torch.cuda.manual_seed_all(777)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# parameters\n",
    "learning_rate = 0.001\n",
    "training_epochs = 15\n",
    "batch_size = 100\n",
    "drop_prob = 0.3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# MNIST dataset\n",
    "mnist_train = dsets.MNIST(root='MNIST_data/',\n",
    "                          train=True,\n",
    "                          transform=transforms.ToTensor(),\n",
    "                          download=True)\n",
    "\n",
    "mnist_test = dsets.MNIST(root='MNIST_data/',\n",
    "                         train=False,\n",
    "                         transform=transforms.ToTensor(),\n",
    "                         download=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# dataset loader\n",
    "data_loader = torch.utils.data.DataLoader(dataset=mnist_train,\n",
    "                                          batch_size=batch_size,\n",
    "                                          shuffle=True,\n",
    "                                          drop_last=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# nn layers\n",
    "linear1 = torch.nn.Linear(784, 512, bias=True)\n",
    "linear2 = torch.nn.Linear(512, 512, bias=True)\n",
    "linear3 = torch.nn.Linear(512, 512, bias=True)\n",
    "linear4 = torch.nn.Linear(512, 512, bias=True)\n",
    "linear5 = torch.nn.Linear(512, 10, bias=True)\n",
    "relu = torch.nn.ReLU()\n",
    "dropout = torch.nn.Dropout(p=drop_prob)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Parameter containing:\n",
       "tensor([[-0.0565,  0.0423, -0.0155,  ...,  0.1012,  0.0459, -0.0191],\n",
       "        [ 0.0772,  0.0452, -0.0638,  ...,  0.0476, -0.0638,  0.0528],\n",
       "        [ 0.0311, -0.1023, -0.0701,  ...,  0.0412, -0.1004,  0.0738],\n",
       "        ...,\n",
       "        [ 0.0334,  0.0187, -0.1021,  ...,  0.0280, -0.0583, -0.1018],\n",
       "        [-0.0506, -0.0939, -0.0467,  ..., -0.0554, -0.0325,  0.0640],\n",
       "        [-0.0183, -0.0123,  0.1025,  ..., -0.0214,  0.0220, -0.0741]],\n",
       "       requires_grad=True)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# xavier initialization\n",
    "torch.nn.init.xavier_uniform_(linear1.weight)\n",
    "torch.nn.init.xavier_uniform_(linear2.weight)\n",
    "torch.nn.init.xavier_uniform_(linear3.weight)\n",
    "torch.nn.init.xavier_uniform_(linear4.weight)\n",
    "torch.nn.init.xavier_uniform_(linear5.weight)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# model\n",
    "model = torch.nn.Sequential(linear1, relu, dropout,\n",
    "                            linear2, relu, dropout,\n",
    "                            linear3, relu, dropout,\n",
    "                            linear4, relu, dropout,\n",
    "                            linear5).to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# define cost/loss & optimizer\n",
    "criterion = torch.nn.CrossEntropyLoss().to(device)    # Softmax is internally computed.\n",
    "optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 0001 cost = 0.309925616\n",
      "Epoch: 0002 cost = 0.143516496\n",
      "Epoch: 0003 cost = 0.113396436\n",
      "Epoch: 0004 cost = 0.092770174\n",
      "Epoch: 0005 cost = 0.081650071\n",
      "Epoch: 0006 cost = 0.073365353\n",
      "Epoch: 0007 cost = 0.070349611\n",
      "Epoch: 0008 cost = 0.061270669\n",
      "Epoch: 0009 cost = 0.060892191\n",
      "Epoch: 0010 cost = 0.054064836\n",
      "Epoch: 0011 cost = 0.051594462\n",
      "Epoch: 0012 cost = 0.048855171\n",
      "Epoch: 0013 cost = 0.043751985\n",
      "Epoch: 0014 cost = 0.044706535\n",
      "Epoch: 0015 cost = 0.044633854\n",
      "Learning finished\n"
     ]
    }
   ],
   "source": [
    "total_batch = len(data_loader)\n",
    "model.train()    # set the model to train mode (dropout=True)\n",
    "for epoch in range(training_epochs):\n",
    "    avg_cost = 0\n",
    "\n",
    "    for X, Y in data_loader:\n",
    "        # reshape input image into [batch_size by 784]\n",
    "        # label is not one-hot encoded\n",
    "        X = X.view(-1, 28 * 28).to(device)\n",
    "        Y = Y.to(device)\n",
    "\n",
    "        optimizer.zero_grad()\n",
    "        hypothesis = model(X)\n",
    "        cost = criterion(hypothesis, Y)\n",
    "        cost.backward()\n",
    "        optimizer.step()\n",
    "\n",
    "        avg_cost += cost / total_batch\n",
    "\n",
    "    print('Epoch:', '%04d' % (epoch + 1), 'cost =', '{:.9f}'.format(avg_cost))\n",
    "\n",
    "print('Learning finished')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.9771999716758728\n",
      "Label:  8\n",
      "Prediction:  8\n"
     ]
    }
   ],
   "source": [
    "# Test model and check accuracy\n",
    "with torch.no_grad():\n",
    "    model.eval()    # set the model to evaluation mode (dropout=False)\n",
    "\n",
    "    # Test the model using test sets\n",
    "    X_test = mnist_test.test_data.view(-1, 28 * 28).float().to(device)\n",
    "    Y_test = mnist_test.test_labels.to(device)\n",
    "\n",
    "    prediction = model(X_test)\n",
    "    correct_prediction = torch.argmax(prediction, 1) == Y_test\n",
    "    accuracy = correct_prediction.float().mean()\n",
    "    print('Accuracy:', accuracy.item())\n",
    "\n",
    "    # Get one and predict\n",
    "    r = random.randint(0, len(mnist_test) - 1)\n",
    "    X_single_data = mnist_test.test_data[r:r + 1].view(-1, 28 * 28).float().to(device)\n",
    "    Y_single_data = mnist_test.test_labels[r:r + 1].to(device)\n",
    "\n",
    "    print('Label: ', Y_single_data.item())\n",
    "    single_prediction = model(X_single_data)\n",
    "    print('Prediction: ', torch.argmax(single_prediction, 1).item())"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}