lab-10-4-mnist_nn_batchnorm.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# import"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "from tensorflow.keras.utils import to_categorical\n",
    "from tensorflow.keras.datasets import mnist\n",
    "from time import time\n",
    "import os"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Eager mode on"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "tf.enable_eager_execution()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Checkpoint function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load(model, checkpoint_dir):\n",
    "    print(\" [*] Reading checkpoints...\")\n",
    "\n",
    "    ckpt = tf.train.get_checkpoint_state(checkpoint_dir)\n",
    "    if ckpt :\n",
    "        ckpt_name = os.path.basename(ckpt.model_checkpoint_path)\n",
    "        checkpoint = tf.train.Checkpoint(dnn=model)\n",
    "        checkpoint.restore(save_path=os.path.join(checkpoint_dir, ckpt_name))\n",
    "        counter = int(ckpt_name.split('-')[1])\n",
    "        print(\" [*] Success to read {}\".format(ckpt_name))\n",
    "        return True, counter\n",
    "    else:\n",
    "        print(\" [*] Failed to find a checkpoint\")\n",
    "        return False, 0\n",
    "\n",
    "def check_folder(dir):\n",
    "    if not os.path.exists(dir):\n",
    "        os.makedirs(dir)\n",
    "    return dir"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data load & pre-processing function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_mnist() :\n",
    "    (train_data, train_labels), (test_data, test_labels) = mnist.load_data()\n",
    "    train_data = np.expand_dims(train_data, axis=-1) # [N, 28, 28] -> [N, 28, 28, 1]\n",
    "    test_data = np.expand_dims(test_data, axis=-1) # [N, 28, 28] -> [N, 28, 28, 1]\n",
    "\n",
    "    train_data, test_data = normalize(train_data, test_data)\n",
    "\n",
    "    train_labels = to_categorical(train_labels, 10) # [N,] -> [N, 10]\n",
    "    test_labels = to_categorical(test_labels, 10) # [N,] -> [N, 10]\n",
    "\n",
    "    return train_data, train_labels, test_data, test_labels\n",
    "\n",
    "def normalize(train_data, test_data):\n",
    "    train_data = train_data.astype(np.float32) / 255.0\n",
    "    test_data = test_data.astype(np.float32) / 255.0\n",
    "\n",
    "    return train_data, test_data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Performance function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def loss_fn(model, images, labels):\n",
    "    logits = model(images, training=True)\n",
    "    loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits_v2(logits=logits, labels=labels))\n",
    "    return loss\n",
    "\n",
    "def accuracy_fn(model, images, labels):\n",
    "    logits = model(images, training=False)\n",
    "    prediction = tf.equal(tf.argmax(logits, -1), tf.argmax(labels, -1))\n",
    "    accuracy = tf.reduce_mean(tf.cast(prediction, tf.float32))\n",
    "    return accuracy\n",
    "\n",
    "def grad(model, images, labels):\n",
    "    with tf.GradientTape() as tape:\n",
    "        loss = loss_fn(model, images, labels)\n",
    "    return tape.gradient(loss, model.variables)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Model function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def flatten() :\n",
    "    return tf.keras.layers.Flatten()\n",
    "\n",
    "def dense(label_dim, weight_init) :\n",
    "    return tf.keras.layers.Dense(units=label_dim, use_bias=True, kernel_initializer=weight_init)\n",
    "\n",
    "def relu() :\n",
    "    return tf.keras.layers.Activation(tf.keras.activations.relu)\n",
    "\n",
    "def batch_norm() :\n",
    "    return tf.keras.layers.BatchNormalization()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Create model (class version)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "class create_model_class(tf.keras.Model):\n",
    "    def __init__(self, label_dim):\n",
    "        super(create_model_class, self).__init__()\n",
    "        weight_init = tf.keras.initializers.glorot_uniform()\n",
    "\n",
    "        self.model = tf.keras.Sequential()\n",
    "        self.model.add(flatten())\n",
    "\n",
    "        for i in range(4):\n",
    "            self.model.add(dense(512, weight_init))\n",
    "            self.model.add(batch_norm())\n",
    "            self.model.add(relu())\n",
    "\n",
    "        self.model.add(dense(label_dim, weight_init))\n",
    "\n",
    "    def call(self, x, training=None, mask=None):\n",
    "\n",
    "        x = self.model(x)\n",
    "\n",
    "        return x"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Create model (function version)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_model_function(label_dim) :\n",
    "    weight_init = tf.keras.initializers.glorot_uniform()\n",
    "\n",
    "    model = tf.keras.Sequential()\n",
    "    model.add(flatten())\n",
    "\n",
    "    for i in range(4) :\n",
    "        model.add(dense(512, weight_init))\n",
    "        model.add(batch_norm())\n",
    "        model.add(relu())\n",
    "\n",
    "    model.add(dense(label_dim, weight_init))\n",
    "\n",
    "    return model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Define data & hyper-parameter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\" dataset \"\"\"\n",
    "train_x, train_y, test_x, test_y = load_mnist()\n",
    "\n",
    "\"\"\" parameters \"\"\"\n",
    "learning_rate = 0.001\n",
    "batch_size = 128\n",
    "\n",
    "training_epochs = 1\n",
    "training_iterations = len(train_x) // batch_size\n",
    "\n",
    "label_dim = 10\n",
    "\n",
    "train_flag = True\n",
    "\n",
    "\"\"\" Graph Input using Dataset API \"\"\"\n",
    "train_dataset = tf.data.Dataset.from_tensor_slices((train_x, train_y)).\\\n",
    "    shuffle(buffer_size=100000).\\\n",
    "    prefetch(buffer_size=batch_size).\\\n",
    "    batch(batch_size).\\\n",
    "    repeat()\n",
    "\n",
    "test_dataset = tf.data.Dataset.from_tensor_slices((test_x, test_y)).\\\n",
    "    shuffle(buffer_size=100000).\\\n",
    "    prefetch(buffer_size=len(test_x)).\\\n",
    "    batch(len(test_x)).\\\n",
    "    repeat()\n",
    "\n",
    "train_iterator = train_dataset.make_one_shot_iterator()\n",
    "test_iterator = test_dataset.make_one_shot_iterator()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Define model & optimizer & writer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\" Model \"\"\"\n",
    "network = create_model_function(label_dim)\n",
    "\n",
    "\"\"\" Training \"\"\"\n",
    "optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate)\n",
    "\n",
    "\"\"\" Writer \"\"\"\n",
    "checkpoint_dir = 'checkpoints'\n",
    "logs_dir = 'logs'\n",
    "\n",
    "model_dir = 'nn_batchnorm'\n",
    "\n",
    "checkpoint_dir = os.path.join(checkpoint_dir, model_dir)\n",
    "check_folder(checkpoint_dir)\n",
    "checkpoint_prefix = os.path.join(checkpoint_dir, model_dir)\n",
    "logs_dir = os.path.join(logs_dir, model_dir)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Restore checkpoint & start train or test phase"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " [*] Reading checkpoints...\n",
      " [*] Failed to find a checkpoint\n",
      " [!] Load failed...\n",
      "Epoch: [ 0] [    0/  468] time: 0.4975, train_loss: 0.33036286, train_accuracy: 0.3359, test_Accuracy: 0.2291\n",
      "Epoch: [ 0] [    1/  468] time: 0.7914, train_loss: 0.29299241, train_accuracy: 0.3672, test_Accuracy: 0.3296\n",
      "Epoch: [ 0] [    2/  468] time: 1.0807, train_loss: 0.29535800, train_accuracy: 0.4297, test_Accuracy: 0.3945\n",
      "Epoch: [ 0] [    3/  468] time: 1.3561, train_loss: 0.30906296, train_accuracy: 0.5078, test_Accuracy: 0.4767\n",
      "Epoch: [ 0] [    4/  468] time: 1.6425, train_loss: 0.26806700, train_accuracy: 0.6250, test_Accuracy: 0.5505\n",
      "Epoch: [ 0] [    5/  468] time: 1.9350, train_loss: 0.32235762, train_accuracy: 0.5781, test_Accuracy: 0.5983\n",
      "Epoch: [ 0] [    6/  468] time: 2.2008, train_loss: 0.26554614, train_accuracy: 0.6641, test_Accuracy: 0.6494\n",
      "Epoch: [ 0] [    7/  468] time: 2.4638, train_loss: 0.20839277, train_accuracy: 0.7500, test_Accuracy: 0.6980\n",
      "Epoch: [ 0] [    8/  468] time: 2.7304, train_loss: 0.24621534, train_accuracy: 0.8203, test_Accuracy: 0.7361\n",
      "Epoch: [ 0] [    9/  468] time: 3.0195, train_loss: 0.30940843, train_accuracy: 0.7500, test_Accuracy: 0.7723\n",
      "Epoch: [ 0] [   10/  468] time: 3.3082, train_loss: 0.23286557, train_accuracy: 0.7422, test_Accuracy: 0.7954\n",
      "Epoch: [ 0] [   11/  468] time: 3.6021, train_loss: 0.37185064, train_accuracy: 0.7422, test_Accuracy: 0.8212\n",
      "Epoch: [ 0] [   12/  468] time: 3.8878, train_loss: 0.20489557, train_accuracy: 0.8594, test_Accuracy: 0.8423\n",
      "Epoch: [ 0] [   13/  468] time: 4.1745, train_loss: 0.14178658, train_accuracy: 0.8750, test_Accuracy: 0.8579\n",
      "Epoch: [ 0] [   14/  468] time: 4.4688, train_loss: 0.18831021, train_accuracy: 0.8906, test_Accuracy: 0.8664\n",
      "Epoch: [ 0] [   15/  468] time: 4.7581, train_loss: 0.22178584, train_accuracy: 0.8516, test_Accuracy: 0.8686\n",
      "Epoch: [ 0] [   16/  468] time: 5.0499, train_loss: 0.15011916, train_accuracy: 0.8906, test_Accuracy: 0.8705\n",
      "Epoch: [ 0] [   17/  468] time: 5.3366, train_loss: 0.11185556, train_accuracy: 0.8906, test_Accuracy: 0.8712\n",
      "Epoch: [ 0] [   18/  468] time: 5.6250, train_loss: 0.18777201, train_accuracy: 0.8750, test_Accuracy: 0.8737\n",
      "Epoch: [ 0] [   19/  468] time: 5.9215, train_loss: 0.24554151, train_accuracy: 0.8984, test_Accuracy: 0.8709\n",
      "Epoch: [ 0] [   20/  468] time: 6.2004, train_loss: 0.12950307, train_accuracy: 0.9141, test_Accuracy: 0.8627\n",
      "Epoch: [ 0] [   21/  468] time: 6.4901, train_loss: 0.30777186, train_accuracy: 0.8359, test_Accuracy: 0.8610\n",
      "Epoch: [ 0] [   22/  468] time: 6.7930, train_loss: 0.16224533, train_accuracy: 0.8516, test_Accuracy: 0.8616\n",
      "Epoch: [ 0] [   23/  468] time: 7.0895, train_loss: 0.17987403, train_accuracy: 0.8281, test_Accuracy: 0.8669\n",
      "Epoch: [ 0] [   24/  468] time: 7.3713, train_loss: 0.14167723, train_accuracy: 0.9062, test_Accuracy: 0.8668\n",
      "Epoch: [ 0] [   25/  468] time: 7.6630, train_loss: 0.23095348, train_accuracy: 0.8828, test_Accuracy: 0.8683\n",
      "Epoch: [ 0] [   26/  468] time: 7.9532, train_loss: 0.15378220, train_accuracy: 0.8750, test_Accuracy: 0.8665\n",
      "Epoch: [ 0] [   27/  468] time: 8.2432, train_loss: 0.18093681, train_accuracy: 0.8828, test_Accuracy: 0.8635\n",
      "Epoch: [ 0] [   28/  468] time: 8.5337, train_loss: 0.16689263, train_accuracy: 0.8984, test_Accuracy: 0.8606\n",
      "Epoch: [ 0] [   29/  468] time: 8.7974, train_loss: 0.16516356, train_accuracy: 0.8359, test_Accuracy: 0.8598\n",
      "Epoch: [ 0] [   30/  468] time: 9.0920, train_loss: 0.16654181, train_accuracy: 0.8672, test_Accuracy: 0.8638\n",
      "Epoch: [ 0] [   31/  468] time: 9.3805, train_loss: 0.23453392, train_accuracy: 0.8906, test_Accuracy: 0.8688\n",
      "Epoch: [ 0] [   32/  468] time: 9.6560, train_loss: 0.16296065, train_accuracy: 0.8828, test_Accuracy: 0.8730\n",
      "Epoch: [ 0] [   33/  468] time: 9.9461, train_loss: 0.19446757, train_accuracy: 0.8906, test_Accuracy: 0.8762\n",
      "Epoch: [ 0] [   34/  468] time: 10.2198, train_loss: 0.21550670, train_accuracy: 0.8984, test_Accuracy: 0.8809\n",
      "Epoch: [ 0] [   35/  468] time: 10.5128, train_loss: 0.18052366, train_accuracy: 0.9141, test_Accuracy: 0.8874\n",
      "Epoch: [ 0] [   36/  468] time: 10.8006, train_loss: 0.14563078, train_accuracy: 0.9141, test_Accuracy: 0.8918\n",
      "Epoch: [ 0] [   37/  468] time: 11.0921, train_loss: 0.19181316, train_accuracy: 0.9141, test_Accuracy: 0.8956\n",
      "Epoch: [ 0] [   38/  468] time: 11.3841, train_loss: 0.18485656, train_accuracy: 0.9297, test_Accuracy: 0.9015\n",
      "Epoch: [ 0] [   39/  468] time: 11.6669, train_loss: 0.21941191, train_accuracy: 0.9141, test_Accuracy: 0.9056\n",
      "Epoch: [ 0] [   40/  468] time: 11.9537, train_loss: 0.14315601, train_accuracy: 0.9453, test_Accuracy: 0.9075\n",
      "Epoch: [ 0] [   41/  468] time: 12.2379, train_loss: 0.11220451, train_accuracy: 0.9219, test_Accuracy: 0.9064\n",
      "Epoch: [ 0] [   42/  468] time: 12.5294, train_loss: 0.08643550, train_accuracy: 0.9688, test_Accuracy: 0.9033\n",
      "Epoch: [ 0] [   43/  468] time: 12.7989, train_loss: 0.16712800, train_accuracy: 0.8906, test_Accuracy: 0.9028\n",
      "Epoch: [ 0] [   44/  468] time: 13.0906, train_loss: 0.15595710, train_accuracy: 0.9141, test_Accuracy: 0.9045\n",
      "Epoch: [ 0] [   45/  468] time: 13.3590, train_loss: 0.20998164, train_accuracy: 0.9141, test_Accuracy: 0.9059\n",
      "Epoch: [ 0] [   46/  468] time: 13.6519, train_loss: 0.10192706, train_accuracy: 0.8984, test_Accuracy: 0.9097\n",
      "Epoch: [ 0] [   47/  468] time: 13.9382, train_loss: 0.10565534, train_accuracy: 0.9531, test_Accuracy: 0.9162\n",
      "Epoch: [ 0] [   48/  468] time: 14.2218, train_loss: 0.14988501, train_accuracy: 0.9453, test_Accuracy: 0.9177\n",
      "Epoch: [ 0] [   49/  468] time: 14.5139, train_loss: 0.14903027, train_accuracy: 0.9219, test_Accuracy: 0.9172\n",
      "Epoch: [ 0] [   50/  468] time: 14.7953, train_loss: 0.15871224, train_accuracy: 0.9297, test_Accuracy: 0.9171\n",
      "Epoch: [ 0] [   51/  468] time: 15.0769, train_loss: 0.24307902, train_accuracy: 0.8984, test_Accuracy: 0.9154\n",
      "Epoch: [ 0] [   52/  468] time: 15.3702, train_loss: 0.18263561, train_accuracy: 0.9219, test_Accuracy: 0.9141\n",
      "Epoch: [ 0] [   53/  468] time: 15.6631, train_loss: 0.14420322, train_accuracy: 0.9375, test_Accuracy: 0.9139\n",
      "Epoch: [ 0] [   54/  468] time: 15.9534, train_loss: 0.15244173, train_accuracy: 0.9062, test_Accuracy: 0.9150\n",
      "Epoch: [ 0] [   55/  468] time: 16.2415, train_loss: 0.10869100, train_accuracy: 0.9531, test_Accuracy: 0.9162\n",
      "Epoch: [ 0] [   56/  468] time: 16.5322, train_loss: 0.14378312, train_accuracy: 0.9375, test_Accuracy: 0.9161\n",
      "Epoch: [ 0] [   57/  468] time: 16.8042, train_loss: 0.07503418, train_accuracy: 0.9688, test_Accuracy: 0.9207\n",
      "Epoch: [ 0] [   58/  468] time: 17.0842, train_loss: 0.07075039, train_accuracy: 0.9531, test_Accuracy: 0.9236\n",
      "Epoch: [ 0] [   59/  468] time: 17.3789, train_loss: 0.15796611, train_accuracy: 0.9531, test_Accuracy: 0.9243\n",
      "Epoch: [ 0] [   60/  468] time: 17.6723, train_loss: 0.15890257, train_accuracy: 0.9531, test_Accuracy: 0.9263\n",
      "Epoch: [ 0] [   61/  468] time: 17.9480, train_loss: 0.07769872, train_accuracy: 0.9609, test_Accuracy: 0.9268\n",
      "Epoch: [ 0] [   62/  468] time: 18.2334, train_loss: 0.18216269, train_accuracy: 0.9062, test_Accuracy: 0.9271\n",
      "Epoch: [ 0] [   63/  468] time: 18.5226, train_loss: 0.15232329, train_accuracy: 0.8984, test_Accuracy: 0.9282\n",
      "Epoch: [ 0] [   64/  468] time: 18.8084, train_loss: 0.13572270, train_accuracy: 0.9688, test_Accuracy: 0.9302\n",
      "Epoch: [ 0] [   65/  468] time: 19.0933, train_loss: 0.18152100, train_accuracy: 0.9375, test_Accuracy: 0.9335\n",
      "Epoch: [ 0] [   66/  468] time: 19.3762, train_loss: 0.17971778, train_accuracy: 0.9297, test_Accuracy: 0.9348\n",
      "Epoch: [ 0] [   67/  468] time: 19.6686, train_loss: 0.08616480, train_accuracy: 0.9609, test_Accuracy: 0.9348\n",
      "Epoch: [ 0] [   68/  468] time: 19.9560, train_loss: 0.08750863, train_accuracy: 0.9375, test_Accuracy: 0.9345\n",
      "Epoch: [ 0] [   69/  468] time: 20.2475, train_loss: 0.19714886, train_accuracy: 0.9453, test_Accuracy: 0.9342\n",
      "Epoch: [ 0] [   70/  468] time: 20.5411, train_loss: 0.13898247, train_accuracy: 0.9688, test_Accuracy: 0.9354\n",
      "Epoch: [ 0] [   71/  468] time: 20.8265, train_loss: 0.19265622, train_accuracy: 0.9297, test_Accuracy: 0.9355\n",
      "Epoch: [ 0] [   72/  468] time: 21.1151, train_loss: 0.13917142, train_accuracy: 0.9609, test_Accuracy: 0.9375\n",
      "Epoch: [ 0] [   73/  468] time: 21.3939, train_loss: 0.15555823, train_accuracy: 0.8984, test_Accuracy: 0.9377\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: [ 0] [   74/  468] time: 21.6645, train_loss: 0.17417696, train_accuracy: 0.9297, test_Accuracy: 0.9382\n",
      "Epoch: [ 0] [   75/  468] time: 21.9596, train_loss: 0.19491014, train_accuracy: 0.9531, test_Accuracy: 0.9366\n",
      "Epoch: [ 0] [   76/  468] time: 22.2544, train_loss: 0.26912385, train_accuracy: 0.9141, test_Accuracy: 0.9390\n",
      "Epoch: [ 0] [   77/  468] time: 22.5362, train_loss: 0.10218343, train_accuracy: 0.9609, test_Accuracy: 0.9426\n",
      "Epoch: [ 0] [   78/  468] time: 22.8188, train_loss: 0.14223242, train_accuracy: 0.9531, test_Accuracy: 0.9461\n",
      "Epoch: [ 0] [   79/  468] time: 23.0969, train_loss: 0.10368903, train_accuracy: 0.9609, test_Accuracy: 0.9480\n",
      "Epoch: [ 0] [   80/  468] time: 23.3720, train_loss: 0.11104535, train_accuracy: 0.9453, test_Accuracy: 0.9463\n",
      "Epoch: [ 0] [   81/  468] time: 23.6597, train_loss: 0.13129853, train_accuracy: 0.9531, test_Accuracy: 0.9456\n",
      "Epoch: [ 0] [   82/  468] time: 23.9430, train_loss: 0.14899245, train_accuracy: 0.9453, test_Accuracy: 0.9447\n",
      "Epoch: [ 0] [   83/  468] time: 24.2228, train_loss: 0.11205100, train_accuracy: 0.9375, test_Accuracy: 0.9443\n",
      "Epoch: [ 0] [   84/  468] time: 24.5048, train_loss: 0.11709529, train_accuracy: 0.9453, test_Accuracy: 0.9437\n",
      "Epoch: [ 0] [   85/  468] time: 24.7968, train_loss: 0.11966251, train_accuracy: 0.9766, test_Accuracy: 0.9451\n",
      "Epoch: [ 0] [   86/  468] time: 25.0782, train_loss: 0.11464158, train_accuracy: 0.9375, test_Accuracy: 0.9460\n",
      "Epoch: [ 0] [   87/  468] time: 25.3723, train_loss: 0.17129403, train_accuracy: 0.9375, test_Accuracy: 0.9467\n",
      "Epoch: [ 0] [   88/  468] time: 25.6574, train_loss: 0.12412425, train_accuracy: 0.9453, test_Accuracy: 0.9492\n",
      "Epoch: [ 0] [   89/  468] time: 25.9481, train_loss: 0.13608775, train_accuracy: 0.9688, test_Accuracy: 0.9497\n",
      "Epoch: [ 0] [   90/  468] time: 26.2238, train_loss: 0.14563996, train_accuracy: 0.9453, test_Accuracy: 0.9509\n",
      "Epoch: [ 0] [   91/  468] time: 26.5165, train_loss: 0.06729016, train_accuracy: 0.9844, test_Accuracy: 0.9510\n",
      "Epoch: [ 0] [   92/  468] time: 26.7967, train_loss: 0.07057892, train_accuracy: 0.9766, test_Accuracy: 0.9498\n",
      "Epoch: [ 0] [   93/  468] time: 27.0774, train_loss: 0.11142360, train_accuracy: 0.9453, test_Accuracy: 0.9516\n",
      "Epoch: [ 0] [   94/  468] time: 27.3521, train_loss: 0.18045920, train_accuracy: 0.9375, test_Accuracy: 0.9519\n",
      "Epoch: [ 0] [   95/  468] time: 27.6458, train_loss: 0.12146823, train_accuracy: 0.9375, test_Accuracy: 0.9517\n",
      "Epoch: [ 0] [   96/  468] time: 27.9257, train_loss: 0.06490819, train_accuracy: 0.9844, test_Accuracy: 0.9537\n",
      "Epoch: [ 0] [   97/  468] time: 28.2050, train_loss: 0.10942525, train_accuracy: 0.9375, test_Accuracy: 0.9549\n",
      "Epoch: [ 0] [   98/  468] time: 28.4950, train_loss: 0.07602119, train_accuracy: 0.9766, test_Accuracy: 0.9545\n",
      "Epoch: [ 0] [   99/  468] time: 28.7873, train_loss: 0.16845354, train_accuracy: 0.9297, test_Accuracy: 0.9533\n",
      "Epoch: [ 0] [  100/  468] time: 29.0717, train_loss: 0.16593015, train_accuracy: 0.9609, test_Accuracy: 0.9532\n",
      "Epoch: [ 0] [  101/  468] time: 29.3624, train_loss: 0.05200160, train_accuracy: 0.9766, test_Accuracy: 0.9542\n",
      "Epoch: [ 0] [  102/  468] time: 29.6454, train_loss: 0.16458914, train_accuracy: 0.9531, test_Accuracy: 0.9542\n",
      "Epoch: [ 0] [  103/  468] time: 29.9293, train_loss: 0.09548508, train_accuracy: 0.9688, test_Accuracy: 0.9533\n",
      "Epoch: [ 0] [  104/  468] time: 30.2145, train_loss: 0.10995573, train_accuracy: 0.9766, test_Accuracy: 0.9537\n",
      "Epoch: [ 0] [  105/  468] time: 30.5087, train_loss: 0.16463488, train_accuracy: 0.9453, test_Accuracy: 0.9541\n",
      "Epoch: [ 0] [  106/  468] time: 30.7941, train_loss: 0.09830833, train_accuracy: 0.9609, test_Accuracy: 0.9538\n",
      "Epoch: [ 0] [  107/  468] time: 31.0792, train_loss: 0.10504629, train_accuracy: 0.9531, test_Accuracy: 0.9539\n",
      "Epoch: [ 0] [  108/  468] time: 31.3577, train_loss: 0.15387638, train_accuracy: 0.9531, test_Accuracy: 0.9531\n",
      "Epoch: [ 0] [  109/  468] time: 31.6348, train_loss: 0.09508120, train_accuracy: 0.9531, test_Accuracy: 0.9523\n",
      "Epoch: [ 0] [  110/  468] time: 31.9216, train_loss: 0.07436610, train_accuracy: 0.9688, test_Accuracy: 0.9511\n",
      "Epoch: [ 0] [  111/  468] time: 32.2114, train_loss: 0.17996854, train_accuracy: 0.9609, test_Accuracy: 0.9528\n",
      "Epoch: [ 0] [  112/  468] time: 32.5028, train_loss: 0.13726750, train_accuracy: 0.9453, test_Accuracy: 0.9547\n",
      "Epoch: [ 0] [  113/  468] time: 32.7882, train_loss: 0.11781720, train_accuracy: 0.9766, test_Accuracy: 0.9552\n",
      "Epoch: [ 0] [  114/  468] time: 33.0772, train_loss: 0.07572955, train_accuracy: 0.9688, test_Accuracy: 0.9546\n",
      "Epoch: [ 0] [  115/  468] time: 33.3581, train_loss: 0.10422419, train_accuracy: 0.9609, test_Accuracy: 0.9545\n",
      "Epoch: [ 0] [  116/  468] time: 33.6403, train_loss: 0.08660074, train_accuracy: 0.9766, test_Accuracy: 0.9556\n",
      "Epoch: [ 0] [  117/  468] time: 33.9152, train_loss: 0.10347768, train_accuracy: 0.9766, test_Accuracy: 0.9549\n",
      "Epoch: [ 0] [  118/  468] time: 34.1936, train_loss: 0.11523935, train_accuracy: 0.9688, test_Accuracy: 0.9535\n",
      "Epoch: [ 0] [  119/  468] time: 34.4707, train_loss: 0.10760069, train_accuracy: 0.9531, test_Accuracy: 0.9513\n",
      "Epoch: [ 0] [  120/  468] time: 34.7471, train_loss: 0.14149761, train_accuracy: 0.9531, test_Accuracy: 0.9480\n",
      "Epoch: [ 0] [  121/  468] time: 35.0266, train_loss: 0.13897499, train_accuracy: 0.9453, test_Accuracy: 0.9478\n",
      "Epoch: [ 0] [  122/  468] time: 35.2971, train_loss: 0.12558100, train_accuracy: 0.9531, test_Accuracy: 0.9492\n",
      "Epoch: [ 0] [  123/  468] time: 35.5660, train_loss: 0.11518922, train_accuracy: 0.9375, test_Accuracy: 0.9504\n",
      "Epoch: [ 0] [  124/  468] time: 35.8444, train_loss: 0.15384050, train_accuracy: 0.9453, test_Accuracy: 0.9529\n",
      "Epoch: [ 0] [  125/  468] time: 36.1355, train_loss: 0.15022758, train_accuracy: 0.9609, test_Accuracy: 0.9552\n",
      "Epoch: [ 0] [  126/  468] time: 36.4077, train_loss: 0.11108676, train_accuracy: 0.9688, test_Accuracy: 0.9570\n",
      "Epoch: [ 0] [  127/  468] time: 36.6866, train_loss: 0.08629279, train_accuracy: 0.9766, test_Accuracy: 0.9570\n",
      "Epoch: [ 0] [  128/  468] time: 36.9639, train_loss: 0.11659998, train_accuracy: 0.9766, test_Accuracy: 0.9568\n",
      "Epoch: [ 0] [  129/  468] time: 37.2328, train_loss: 0.16933802, train_accuracy: 0.9688, test_Accuracy: 0.9562\n",
      "Epoch: [ 0] [  130/  468] time: 37.5031, train_loss: 0.05969649, train_accuracy: 0.9922, test_Accuracy: 0.9544\n",
      "Epoch: [ 0] [  131/  468] time: 37.7796, train_loss: 0.06872123, train_accuracy: 0.9609, test_Accuracy: 0.9537\n",
      "Epoch: [ 0] [  132/  468] time: 38.0583, train_loss: 0.11029512, train_accuracy: 0.9688, test_Accuracy: 0.9547\n",
      "Epoch: [ 0] [  133/  468] time: 38.3421, train_loss: 0.10490000, train_accuracy: 0.9688, test_Accuracy: 0.9556\n",
      "Epoch: [ 0] [  134/  468] time: 38.6265, train_loss: 0.05430870, train_accuracy: 0.9844, test_Accuracy: 0.9576\n",
      "Epoch: [ 0] [  135/  468] time: 38.9141, train_loss: 0.06412394, train_accuracy: 0.9766, test_Accuracy: 0.9575\n",
      "Epoch: [ 0] [  136/  468] time: 39.1889, train_loss: 0.10039046, train_accuracy: 0.9688, test_Accuracy: 0.9572\n",
      "Epoch: [ 0] [  137/  468] time: 39.4814, train_loss: 0.08632743, train_accuracy: 0.9844, test_Accuracy: 0.9572\n",
      "Epoch: [ 0] [  138/  468] time: 39.7685, train_loss: 0.16068777, train_accuracy: 0.9531, test_Accuracy: 0.9573\n",
      "Epoch: [ 0] [  139/  468] time: 40.0501, train_loss: 0.12677762, train_accuracy: 0.9531, test_Accuracy: 0.9575\n",
      "Epoch: [ 0] [  140/  468] time: 40.3299, train_loss: 0.06328069, train_accuracy: 0.9766, test_Accuracy: 0.9566\n",
      "Epoch: [ 0] [  141/  468] time: 40.6109, train_loss: 0.15664023, train_accuracy: 0.9453, test_Accuracy: 0.9570\n",
      "Epoch: [ 0] [  142/  468] time: 40.8875, train_loss: 0.04195043, train_accuracy: 0.9844, test_Accuracy: 0.9560\n",
      "Epoch: [ 0] [  143/  468] time: 41.1625, train_loss: 0.14272642, train_accuracy: 0.9453, test_Accuracy: 0.9554\n",
      "Epoch: [ 0] [  144/  468] time: 41.4455, train_loss: 0.09094274, train_accuracy: 0.9688, test_Accuracy: 0.9545\n",
      "Epoch: [ 0] [  145/  468] time: 41.7236, train_loss: 0.04038617, train_accuracy: 0.9844, test_Accuracy: 0.9529\n",
      "Epoch: [ 0] [  146/  468] time: 42.0082, train_loss: 0.06265198, train_accuracy: 0.9922, test_Accuracy: 0.9509\n",
      "Epoch: [ 0] [  147/  468] time: 42.2959, train_loss: 0.12798062, train_accuracy: 0.9844, test_Accuracy: 0.9493\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: [ 0] [  148/  468] time: 42.5741, train_loss: 0.21957263, train_accuracy: 0.9062, test_Accuracy: 0.9512\n",
      "Epoch: [ 0] [  149/  468] time: 42.8479, train_loss: 0.07872219, train_accuracy: 0.9844, test_Accuracy: 0.9515\n",
      "Epoch: [ 0] [  150/  468] time: 43.1197, train_loss: 0.06321964, train_accuracy: 0.9531, test_Accuracy: 0.9535\n",
      "Epoch: [ 0] [  151/  468] time: 43.3962, train_loss: 0.07186382, train_accuracy: 0.9766, test_Accuracy: 0.9552\n",
      "Epoch: [ 0] [  152/  468] time: 43.6747, train_loss: 0.03968059, train_accuracy: 0.9922, test_Accuracy: 0.9566\n",
      "Epoch: [ 0] [  153/  468] time: 43.9511, train_loss: 0.10695499, train_accuracy: 0.9688, test_Accuracy: 0.9570\n",
      "Epoch: [ 0] [  154/  468] time: 44.2276, train_loss: 0.09737165, train_accuracy: 0.9688, test_Accuracy: 0.9585\n",
      "Epoch: [ 0] [  155/  468] time: 44.5021, train_loss: 0.05561279, train_accuracy: 0.9844, test_Accuracy: 0.9588\n",
      "Epoch: [ 0] [  156/  468] time: 44.7784, train_loss: 0.08986782, train_accuracy: 0.9609, test_Accuracy: 0.9595\n",
      "Epoch: [ 0] [  157/  468] time: 45.0534, train_loss: 0.09963607, train_accuracy: 0.9609, test_Accuracy: 0.9599\n",
      "Epoch: [ 0] [  158/  468] time: 45.3229, train_loss: 0.09294099, train_accuracy: 0.9609, test_Accuracy: 0.9598\n",
      "Epoch: [ 0] [  159/  468] time: 45.5955, train_loss: 0.06942757, train_accuracy: 0.9922, test_Accuracy: 0.9587\n",
      "Epoch: [ 0] [  160/  468] time: 45.8704, train_loss: 0.07577991, train_accuracy: 0.9766, test_Accuracy: 0.9584\n",
      "Epoch: [ 0] [  161/  468] time: 46.1422, train_loss: 0.08554939, train_accuracy: 0.9609, test_Accuracy: 0.9588\n",
      "Epoch: [ 0] [  162/  468] time: 46.4124, train_loss: 0.05963989, train_accuracy: 0.9766, test_Accuracy: 0.9585\n",
      "Epoch: [ 0] [  163/  468] time: 46.6932, train_loss: 0.06812583, train_accuracy: 0.9844, test_Accuracy: 0.9575\n",
      "Epoch: [ 0] [  164/  468] time: 46.9710, train_loss: 0.02800195, train_accuracy: 0.9922, test_Accuracy: 0.9578\n",
      "Epoch: [ 0] [  165/  468] time: 47.2458, train_loss: 0.08831602, train_accuracy: 0.9766, test_Accuracy: 0.9577\n",
      "Epoch: [ 0] [  166/  468] time: 47.5213, train_loss: 0.06744145, train_accuracy: 0.9766, test_Accuracy: 0.9558\n",
      "Epoch: [ 0] [  167/  468] time: 47.8002, train_loss: 0.10970145, train_accuracy: 0.9609, test_Accuracy: 0.9558\n",
      "Epoch: [ 0] [  168/  468] time: 48.0797, train_loss: 0.12387045, train_accuracy: 0.9531, test_Accuracy: 0.9560\n",
      "Epoch: [ 0] [  169/  468] time: 48.3525, train_loss: 0.05811469, train_accuracy: 0.9766, test_Accuracy: 0.9565\n",
      "Epoch: [ 0] [  170/  468] time: 48.6322, train_loss: 0.07130429, train_accuracy: 0.9688, test_Accuracy: 0.9575\n",
      "Epoch: [ 0] [  171/  468] time: 48.9079, train_loss: 0.07273603, train_accuracy: 0.9609, test_Accuracy: 0.9580\n",
      "Epoch: [ 0] [  172/  468] time: 49.1845, train_loss: 0.07671751, train_accuracy: 0.9844, test_Accuracy: 0.9585\n",
      "Epoch: [ 0] [  173/  468] time: 49.4635, train_loss: 0.08122802, train_accuracy: 0.9531, test_Accuracy: 0.9590\n",
      "Epoch: [ 0] [  174/  468] time: 49.7507, train_loss: 0.08260790, train_accuracy: 0.9844, test_Accuracy: 0.9594\n",
      "Epoch: [ 0] [  175/  468] time: 50.0302, train_loss: 0.11819116, train_accuracy: 0.9609, test_Accuracy: 0.9602\n",
      "Epoch: [ 0] [  176/  468] time: 50.3124, train_loss: 0.04399809, train_accuracy: 0.9844, test_Accuracy: 0.9597\n",
      "Epoch: [ 0] [  177/  468] time: 50.5922, train_loss: 0.07825488, train_accuracy: 0.9609, test_Accuracy: 0.9589\n",
      "Epoch: [ 0] [  178/  468] time: 50.8706, train_loss: 0.06946987, train_accuracy: 0.9688, test_Accuracy: 0.9592\n",
      "Epoch: [ 0] [  179/  468] time: 51.1568, train_loss: 0.12197845, train_accuracy: 0.9453, test_Accuracy: 0.9594\n",
      "Epoch: [ 0] [  180/  468] time: 51.4392, train_loss: 0.06101618, train_accuracy: 0.9844, test_Accuracy: 0.9586\n",
      "Epoch: [ 0] [  181/  468] time: 51.7311, train_loss: 0.09107557, train_accuracy: 0.9609, test_Accuracy: 0.9566\n",
      "Epoch: [ 0] [  182/  468] time: 52.0206, train_loss: 0.10251088, train_accuracy: 0.9688, test_Accuracy: 0.9557\n",
      "Epoch: [ 0] [  183/  468] time: 52.3115, train_loss: 0.08218132, train_accuracy: 0.9688, test_Accuracy: 0.9544\n",
      "Epoch: [ 0] [  184/  468] time: 52.5932, train_loss: 0.05174072, train_accuracy: 0.9844, test_Accuracy: 0.9548\n",
      "Epoch: [ 0] [  185/  468] time: 52.8727, train_loss: 0.12143288, train_accuracy: 0.9688, test_Accuracy: 0.9544\n",
      "Epoch: [ 0] [  186/  468] time: 53.1432, train_loss: 0.16068789, train_accuracy: 0.9531, test_Accuracy: 0.9544\n",
      "Epoch: [ 0] [  187/  468] time: 53.4220, train_loss: 0.04978611, train_accuracy: 0.9531, test_Accuracy: 0.9550\n",
      "Epoch: [ 0] [  188/  468] time: 53.7151, train_loss: 0.06694262, train_accuracy: 0.9766, test_Accuracy: 0.9554\n",
      "Epoch: [ 0] [  189/  468] time: 53.9936, train_loss: 0.06999680, train_accuracy: 0.9531, test_Accuracy: 0.9572\n",
      "Epoch: [ 0] [  190/  468] time: 54.2723, train_loss: 0.03524306, train_accuracy: 0.9922, test_Accuracy: 0.9593\n",
      "Epoch: [ 0] [  191/  468] time: 54.5499, train_loss: 0.14963050, train_accuracy: 0.9531, test_Accuracy: 0.9599\n",
      "Epoch: [ 0] [  192/  468] time: 54.8321, train_loss: 0.14891225, train_accuracy: 0.9141, test_Accuracy: 0.9576\n",
      "Epoch: [ 0] [  193/  468] time: 55.1011, train_loss: 0.07175447, train_accuracy: 0.9609, test_Accuracy: 0.9556\n",
      "Epoch: [ 0] [  194/  468] time: 55.3791, train_loss: 0.05370807, train_accuracy: 0.9688, test_Accuracy: 0.9538\n",
      "Epoch: [ 0] [  195/  468] time: 55.6635, train_loss: 0.04225980, train_accuracy: 0.9844, test_Accuracy: 0.9525\n",
      "Epoch: [ 0] [  196/  468] time: 55.9498, train_loss: 0.04621601, train_accuracy: 0.9531, test_Accuracy: 0.9520\n",
      "Epoch: [ 0] [  197/  468] time: 56.2308, train_loss: 0.05708799, train_accuracy: 0.9766, test_Accuracy: 0.9522\n",
      "Epoch: [ 0] [  198/  468] time: 56.5218, train_loss: 0.04533594, train_accuracy: 0.9609, test_Accuracy: 0.9526\n",
      "Epoch: [ 0] [  199/  468] time: 56.8091, train_loss: 0.05754239, train_accuracy: 0.9844, test_Accuracy: 0.9538\n",
      "Epoch: [ 0] [  200/  468] time: 57.0933, train_loss: 0.11793024, train_accuracy: 0.9766, test_Accuracy: 0.9542\n",
      "Epoch: [ 0] [  201/  468] time: 57.3759, train_loss: 0.05186168, train_accuracy: 0.9766, test_Accuracy: 0.9547\n",
      "Epoch: [ 0] [  202/  468] time: 57.6558, train_loss: 0.09045486, train_accuracy: 0.9609, test_Accuracy: 0.9543\n",
      "Epoch: [ 0] [  203/  468] time: 57.9503, train_loss: 0.13246112, train_accuracy: 0.9531, test_Accuracy: 0.9547\n",
      "Epoch: [ 0] [  204/  468] time: 58.2297, train_loss: 0.06015445, train_accuracy: 0.9766, test_Accuracy: 0.9544\n",
      "Epoch: [ 0] [  205/  468] time: 58.5108, train_loss: 0.12857375, train_accuracy: 0.9609, test_Accuracy: 0.9543\n",
      "Epoch: [ 0] [  206/  468] time: 58.7871, train_loss: 0.07586975, train_accuracy: 0.9688, test_Accuracy: 0.9539\n",
      "Epoch: [ 0] [  207/  468] time: 59.0726, train_loss: 0.07531638, train_accuracy: 0.9688, test_Accuracy: 0.9550\n",
      "Epoch: [ 0] [  208/  468] time: 59.3606, train_loss: 0.02517870, train_accuracy: 0.9766, test_Accuracy: 0.9550\n",
      "Epoch: [ 0] [  209/  468] time: 59.6451, train_loss: 0.09541184, train_accuracy: 0.9688, test_Accuracy: 0.9547\n",
      "Epoch: [ 0] [  210/  468] time: 59.9285, train_loss: 0.04788002, train_accuracy: 1.0000, test_Accuracy: 0.9546\n",
      "Epoch: [ 0] [  211/  468] time: 60.2088, train_loss: 0.06888250, train_accuracy: 0.9688, test_Accuracy: 0.9549\n",
      "Epoch: [ 0] [  212/  468] time: 60.4943, train_loss: 0.14295773, train_accuracy: 0.9688, test_Accuracy: 0.9546\n",
      "Epoch: [ 0] [  213/  468] time: 60.7742, train_loss: 0.03590230, train_accuracy: 0.9766, test_Accuracy: 0.9540\n",
      "Epoch: [ 0] [  214/  468] time: 61.0464, train_loss: 0.06747992, train_accuracy: 0.9844, test_Accuracy: 0.9548\n",
      "Epoch: [ 0] [  215/  468] time: 61.3198, train_loss: 0.09480851, train_accuracy: 0.9688, test_Accuracy: 0.9557\n",
      "Epoch: [ 0] [  216/  468] time: 61.5930, train_loss: 0.04776166, train_accuracy: 0.9922, test_Accuracy: 0.9569\n",
      "Epoch: [ 0] [  217/  468] time: 61.8660, train_loss: 0.08187498, train_accuracy: 0.9609, test_Accuracy: 0.9567\n",
      "Epoch: [ 0] [  218/  468] time: 62.1410, train_loss: 0.07451142, train_accuracy: 0.9766, test_Accuracy: 0.9579\n",
      "Epoch: [ 0] [  219/  468] time: 62.4153, train_loss: 0.06011660, train_accuracy: 0.9844, test_Accuracy: 0.9590\n",
      "Epoch: [ 0] [  220/  468] time: 62.6910, train_loss: 0.05186208, train_accuracy: 0.9531, test_Accuracy: 0.9601\n",
      "Epoch: [ 0] [  221/  468] time: 62.9613, train_loss: 0.08667990, train_accuracy: 0.9844, test_Accuracy: 0.9593\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: [ 0] [  222/  468] time: 63.2361, train_loss: 0.10099342, train_accuracy: 0.9219, test_Accuracy: 0.9602\n",
      "Epoch: [ 0] [  223/  468] time: 63.5120, train_loss: 0.08429043, train_accuracy: 0.9375, test_Accuracy: 0.9602\n",
      "Epoch: [ 0] [  224/  468] time: 63.7831, train_loss: 0.04090427, train_accuracy: 0.9922, test_Accuracy: 0.9602\n",
      "Epoch: [ 0] [  225/  468] time: 64.0724, train_loss: 0.06599039, train_accuracy: 0.9844, test_Accuracy: 0.9592\n",
      "Epoch: [ 0] [  226/  468] time: 64.3587, train_loss: 0.06344657, train_accuracy: 0.9766, test_Accuracy: 0.9578\n",
      "Epoch: [ 0] [  227/  468] time: 64.6332, train_loss: 0.13878784, train_accuracy: 0.9453, test_Accuracy: 0.9571\n",
      "Epoch: [ 0] [  228/  468] time: 64.9136, train_loss: 0.16873755, train_accuracy: 0.9453, test_Accuracy: 0.9558\n",
      "Epoch: [ 0] [  229/  468] time: 65.1940, train_loss: 0.11658402, train_accuracy: 0.9609, test_Accuracy: 0.9554\n",
      "Epoch: [ 0] [  230/  468] time: 65.4774, train_loss: 0.08872084, train_accuracy: 0.9688, test_Accuracy: 0.9551\n",
      "Epoch: [ 0] [  231/  468] time: 65.7615, train_loss: 0.06042529, train_accuracy: 0.9766, test_Accuracy: 0.9558\n",
      "Epoch: [ 0] [  232/  468] time: 66.0385, train_loss: 0.11606461, train_accuracy: 0.9844, test_Accuracy: 0.9553\n",
      "Epoch: [ 0] [  233/  468] time: 66.3143, train_loss: 0.08119136, train_accuracy: 0.9609, test_Accuracy: 0.9554\n",
      "Epoch: [ 0] [  234/  468] time: 66.5888, train_loss: 0.09045775, train_accuracy: 0.9531, test_Accuracy: 0.9554\n",
      "Epoch: [ 0] [  235/  468] time: 66.8729, train_loss: 0.09845643, train_accuracy: 0.9688, test_Accuracy: 0.9570\n",
      "Epoch: [ 0] [  236/  468] time: 67.1548, train_loss: 0.04137938, train_accuracy: 0.9766, test_Accuracy: 0.9567\n",
      "Epoch: [ 0] [  237/  468] time: 67.4341, train_loss: 0.17766431, train_accuracy: 0.9531, test_Accuracy: 0.9559\n",
      "Epoch: [ 0] [  238/  468] time: 67.7169, train_loss: 0.04526299, train_accuracy: 0.9766, test_Accuracy: 0.9538\n",
      "Epoch: [ 0] [  239/  468] time: 67.9975, train_loss: 0.17767009, train_accuracy: 0.9375, test_Accuracy: 0.9535\n",
      "Epoch: [ 0] [  240/  468] time: 68.2750, train_loss: 0.08231448, train_accuracy: 0.9688, test_Accuracy: 0.9539\n",
      "Epoch: [ 0] [  241/  468] time: 68.5567, train_loss: 0.16108523, train_accuracy: 0.9219, test_Accuracy: 0.9535\n",
      "Epoch: [ 0] [  242/  468] time: 68.8336, train_loss: 0.07305056, train_accuracy: 0.9766, test_Accuracy: 0.9539\n",
      "Epoch: [ 0] [  243/  468] time: 69.1092, train_loss: 0.07867154, train_accuracy: 0.9766, test_Accuracy: 0.9541\n",
      "Epoch: [ 0] [  244/  468] time: 69.3841, train_loss: 0.07458910, train_accuracy: 0.9688, test_Accuracy: 0.9565\n",
      "Epoch: [ 0] [  245/  468] time: 69.6748, train_loss: 0.13586414, train_accuracy: 0.9688, test_Accuracy: 0.9576\n",
      "Epoch: [ 0] [  246/  468] time: 69.9557, train_loss: 0.05520124, train_accuracy: 0.9844, test_Accuracy: 0.9586\n",
      "Epoch: [ 0] [  247/  468] time: 70.2367, train_loss: 0.06289545, train_accuracy: 0.9688, test_Accuracy: 0.9589\n",
      "Epoch: [ 0] [  248/  468] time: 70.5077, train_loss: 0.09783977, train_accuracy: 0.9297, test_Accuracy: 0.9591\n",
      "Epoch: [ 0] [  249/  468] time: 70.7844, train_loss: 0.06601326, train_accuracy: 0.9688, test_Accuracy: 0.9595\n",
      "Epoch: [ 0] [  250/  468] time: 71.0635, train_loss: 0.04346649, train_accuracy: 1.0000, test_Accuracy: 0.9591\n",
      "Epoch: [ 0] [  251/  468] time: 71.3356, train_loss: 0.10327163, train_accuracy: 0.9844, test_Accuracy: 0.9573\n",
      "Epoch: [ 0] [  252/  468] time: 71.6126, train_loss: 0.05983162, train_accuracy: 0.9688, test_Accuracy: 0.9565\n",
      "Epoch: [ 0] [  253/  468] time: 71.8832, train_loss: 0.11599654, train_accuracy: 0.9766, test_Accuracy: 0.9545\n",
      "Epoch: [ 0] [  254/  468] time: 72.1517, train_loss: 0.07555944, train_accuracy: 0.9688, test_Accuracy: 0.9545\n",
      "Epoch: [ 0] [  255/  468] time: 72.4239, train_loss: 0.13130604, train_accuracy: 0.9531, test_Accuracy: 0.9521\n",
      "Epoch: [ 0] [  256/  468] time: 72.6956, train_loss: 0.10935214, train_accuracy: 0.9609, test_Accuracy: 0.9497\n",
      "Epoch: [ 0] [  257/  468] time: 72.9741, train_loss: 0.13996246, train_accuracy: 0.9531, test_Accuracy: 0.9479\n",
      "Epoch: [ 0] [  258/  468] time: 73.2455, train_loss: 0.09004553, train_accuracy: 0.9609, test_Accuracy: 0.9469\n",
      "Epoch: [ 0] [  259/  468] time: 73.5259, train_loss: 0.09032995, train_accuracy: 0.9688, test_Accuracy: 0.9466\n",
      "Epoch: [ 0] [  260/  468] time: 73.7989, train_loss: 0.19297078, train_accuracy: 0.9531, test_Accuracy: 0.9478\n",
      "Epoch: [ 0] [  261/  468] time: 74.0774, train_loss: 0.13936679, train_accuracy: 0.9453, test_Accuracy: 0.9494\n",
      "Epoch: [ 0] [  262/  468] time: 74.3517, train_loss: 0.11778077, train_accuracy: 0.9219, test_Accuracy: 0.9516\n",
      "Epoch: [ 0] [  263/  468] time: 74.6240, train_loss: 0.18482627, train_accuracy: 0.9453, test_Accuracy: 0.9543\n",
      "Epoch: [ 0] [  264/  468] time: 74.8994, train_loss: 0.07373697, train_accuracy: 0.9531, test_Accuracy: 0.9556\n",
      "Epoch: [ 0] [  265/  468] time: 75.1771, train_loss: 0.12000050, train_accuracy: 0.9688, test_Accuracy: 0.9574\n",
      "Epoch: [ 0] [  266/  468] time: 75.4463, train_loss: 0.07903376, train_accuracy: 0.9766, test_Accuracy: 0.9586\n",
      "Epoch: [ 0] [  267/  468] time: 75.7203, train_loss: 0.08256460, train_accuracy: 0.9688, test_Accuracy: 0.9592\n",
      "Epoch: [ 0] [  268/  468] time: 75.9963, train_loss: 0.08699257, train_accuracy: 0.9453, test_Accuracy: 0.9593\n",
      "Epoch: [ 0] [  269/  468] time: 76.2698, train_loss: 0.07253741, train_accuracy: 0.9375, test_Accuracy: 0.9584\n",
      "Epoch: [ 0] [  270/  468] time: 76.5450, train_loss: 0.05226640, train_accuracy: 0.9609, test_Accuracy: 0.9597\n",
      "Epoch: [ 0] [  271/  468] time: 76.8183, train_loss: 0.10806883, train_accuracy: 0.9531, test_Accuracy: 0.9601\n",
      "Epoch: [ 0] [  272/  468] time: 77.0919, train_loss: 0.04165569, train_accuracy: 0.9766, test_Accuracy: 0.9607\n",
      "Epoch: [ 0] [  273/  468] time: 77.3664, train_loss: 0.09192020, train_accuracy: 0.9766, test_Accuracy: 0.9607\n",
      "Epoch: [ 0] [  274/  468] time: 77.6424, train_loss: 0.20256916, train_accuracy: 0.9375, test_Accuracy: 0.9605\n",
      "Epoch: [ 0] [  275/  468] time: 77.9187, train_loss: 0.12148400, train_accuracy: 0.9531, test_Accuracy: 0.9601\n",
      "Epoch: [ 0] [  276/  468] time: 78.1941, train_loss: 0.08279279, train_accuracy: 0.9688, test_Accuracy: 0.9603\n",
      "Epoch: [ 0] [  277/  468] time: 78.4742, train_loss: 0.21166040, train_accuracy: 0.9453, test_Accuracy: 0.9606\n",
      "Epoch: [ 0] [  278/  468] time: 78.7730, train_loss: 0.07328310, train_accuracy: 0.9844, test_Accuracy: 0.9599\n",
      "Epoch: [ 0] [  279/  468] time: 79.0498, train_loss: 0.07411470, train_accuracy: 0.9844, test_Accuracy: 0.9590\n",
      "Epoch: [ 0] [  280/  468] time: 79.3294, train_loss: 0.06328518, train_accuracy: 0.9844, test_Accuracy: 0.9589\n",
      "Epoch: [ 0] [  281/  468] time: 79.6043, train_loss: 0.15273486, train_accuracy: 0.9297, test_Accuracy: 0.9606\n",
      "Epoch: [ 0] [  282/  468] time: 79.8812, train_loss: 0.14043367, train_accuracy: 0.9531, test_Accuracy: 0.9612\n",
      "Epoch: [ 0] [  283/  468] time: 80.1616, train_loss: 0.06198511, train_accuracy: 0.9766, test_Accuracy: 0.9609\n",
      "Epoch: [ 0] [  284/  468] time: 80.4435, train_loss: 0.05007407, train_accuracy: 0.9766, test_Accuracy: 0.9612\n",
      "Epoch: [ 0] [  285/  468] time: 80.7274, train_loss: 0.08383221, train_accuracy: 0.9688, test_Accuracy: 0.9603\n",
      "Epoch: [ 0] [  286/  468] time: 81.0084, train_loss: 0.07288417, train_accuracy: 0.9766, test_Accuracy: 0.9612\n",
      "Epoch: [ 0] [  287/  468] time: 81.2835, train_loss: 0.10600029, train_accuracy: 0.9766, test_Accuracy: 0.9605\n",
      "Epoch: [ 0] [  288/  468] time: 81.5615, train_loss: 0.14933440, train_accuracy: 0.9609, test_Accuracy: 0.9591\n",
      "Epoch: [ 0] [  289/  468] time: 81.8501, train_loss: 0.10751983, train_accuracy: 0.9609, test_Accuracy: 0.9581\n",
      "Epoch: [ 0] [  290/  468] time: 82.1324, train_loss: 0.12598583, train_accuracy: 0.9609, test_Accuracy: 0.9580\n",
      "Epoch: [ 0] [  291/  468] time: 82.4231, train_loss: 0.07427339, train_accuracy: 0.9766, test_Accuracy: 0.9595\n",
      "Epoch: [ 0] [  292/  468] time: 82.7045, train_loss: 0.06379092, train_accuracy: 0.9844, test_Accuracy: 0.9603\n",
      "Epoch: [ 0] [  293/  468] time: 82.9940, train_loss: 0.10817492, train_accuracy: 0.9766, test_Accuracy: 0.9620\n",
      "Epoch: [ 0] [  294/  468] time: 83.2801, train_loss: 0.10686071, train_accuracy: 0.9688, test_Accuracy: 0.9639\n",
      "Epoch: [ 0] [  295/  468] time: 83.5646, train_loss: 0.03919438, train_accuracy: 0.9844, test_Accuracy: 0.9632\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: [ 0] [  296/  468] time: 83.8589, train_loss: 0.05977339, train_accuracy: 0.9844, test_Accuracy: 0.9626\n",
      "Epoch: [ 0] [  297/  468] time: 84.1454, train_loss: 0.07566610, train_accuracy: 0.9688, test_Accuracy: 0.9615\n",
      "Epoch: [ 0] [  298/  468] time: 84.4224, train_loss: 0.12614591, train_accuracy: 0.9688, test_Accuracy: 0.9578\n",
      "Epoch: [ 0] [  299/  468] time: 84.7060, train_loss: 0.14288053, train_accuracy: 0.9688, test_Accuracy: 0.9551\n",
      "Epoch: [ 0] [  300/  468] time: 84.9845, train_loss: 0.05065518, train_accuracy: 0.9766, test_Accuracy: 0.9531\n",
      "Epoch: [ 0] [  301/  468] time: 85.2610, train_loss: 0.08209319, train_accuracy: 0.9609, test_Accuracy: 0.9519\n",
      "Epoch: [ 0] [  302/  468] time: 85.5373, train_loss: 0.10234933, train_accuracy: 0.9531, test_Accuracy: 0.9523\n",
      "Epoch: [ 0] [  303/  468] time: 85.8163, train_loss: 0.12575126, train_accuracy: 0.9609, test_Accuracy: 0.9522\n",
      "Epoch: [ 0] [  304/  468] time: 86.0927, train_loss: 0.14946353, train_accuracy: 0.9609, test_Accuracy: 0.9549\n",
      "Epoch: [ 0] [  305/  468] time: 86.3714, train_loss: 0.06971928, train_accuracy: 0.9922, test_Accuracy: 0.9578\n",
      "Epoch: [ 0] [  306/  468] time: 86.6563, train_loss: 0.06350295, train_accuracy: 0.9688, test_Accuracy: 0.9585\n",
      "Epoch: [ 0] [  307/  468] time: 86.9378, train_loss: 0.08994578, train_accuracy: 0.9609, test_Accuracy: 0.9586\n",
      "Epoch: [ 0] [  308/  468] time: 87.2179, train_loss: 0.09164275, train_accuracy: 0.9766, test_Accuracy: 0.9588\n",
      "Epoch: [ 0] [  309/  468] time: 87.4939, train_loss: 0.04060182, train_accuracy: 0.9844, test_Accuracy: 0.9598\n",
      "Epoch: [ 0] [  310/  468] time: 87.7691, train_loss: 0.05587260, train_accuracy: 0.9766, test_Accuracy: 0.9600\n",
      "Epoch: [ 0] [  311/  468] time: 88.0514, train_loss: 0.08977218, train_accuracy: 0.9688, test_Accuracy: 0.9600\n",
      "Epoch: [ 0] [  312/  468] time: 88.3185, train_loss: 0.09420986, train_accuracy: 0.9609, test_Accuracy: 0.9603\n",
      "Epoch: [ 0] [  313/  468] time: 88.5916, train_loss: 0.05792616, train_accuracy: 0.9609, test_Accuracy: 0.9611\n",
      "Epoch: [ 0] [  314/  468] time: 88.8737, train_loss: 0.03682083, train_accuracy: 1.0000, test_Accuracy: 0.9599\n",
      "Epoch: [ 0] [  315/  468] time: 89.1538, train_loss: 0.06416598, train_accuracy: 0.9688, test_Accuracy: 0.9603\n",
      "Epoch: [ 0] [  316/  468] time: 89.4304, train_loss: 0.06838025, train_accuracy: 0.9844, test_Accuracy: 0.9603\n",
      "Epoch: [ 0] [  317/  468] time: 89.6996, train_loss: 0.03510253, train_accuracy: 1.0000, test_Accuracy: 0.9605\n",
      "Epoch: [ 0] [  318/  468] time: 89.9687, train_loss: 0.10747174, train_accuracy: 0.9609, test_Accuracy: 0.9608\n",
      "Epoch: [ 0] [  319/  468] time: 90.2404, train_loss: 0.11375832, train_accuracy: 0.9766, test_Accuracy: 0.9604\n",
      "Epoch: [ 0] [  320/  468] time: 90.5104, train_loss: 0.06713366, train_accuracy: 0.9609, test_Accuracy: 0.9613\n",
      "Epoch: [ 0] [  321/  468] time: 90.7873, train_loss: 0.03779683, train_accuracy: 0.9922, test_Accuracy: 0.9622\n",
      "Epoch: [ 0] [  322/  468] time: 91.0573, train_loss: 0.13922530, train_accuracy: 0.9531, test_Accuracy: 0.9634\n",
      "Epoch: [ 0] [  323/  468] time: 91.3275, train_loss: 0.03638251, train_accuracy: 0.9922, test_Accuracy: 0.9635\n",
      "Epoch: [ 0] [  324/  468] time: 91.5989, train_loss: 0.08403267, train_accuracy: 0.9766, test_Accuracy: 0.9649\n",
      "Epoch: [ 0] [  325/  468] time: 91.8762, train_loss: 0.06987871, train_accuracy: 0.9531, test_Accuracy: 0.9656\n",
      "Epoch: [ 0] [  326/  468] time: 92.1455, train_loss: 0.12012964, train_accuracy: 0.9141, test_Accuracy: 0.9661\n",
      "Epoch: [ 0] [  327/  468] time: 92.4203, train_loss: 0.06707145, train_accuracy: 0.9688, test_Accuracy: 0.9673\n",
      "Epoch: [ 0] [  328/  468] time: 92.6951, train_loss: 0.12017716, train_accuracy: 0.9609, test_Accuracy: 0.9684\n",
      "Epoch: [ 0] [  329/  468] time: 92.9691, train_loss: 0.14182222, train_accuracy: 0.9688, test_Accuracy: 0.9683\n",
      "Epoch: [ 0] [  330/  468] time: 93.2393, train_loss: 0.02451390, train_accuracy: 0.9922, test_Accuracy: 0.9684\n",
      "Epoch: [ 0] [  331/  468] time: 93.5154, train_loss: 0.03686274, train_accuracy: 0.9766, test_Accuracy: 0.9683\n",
      "Epoch: [ 0] [  332/  468] time: 93.7861, train_loss: 0.10521655, train_accuracy: 0.9609, test_Accuracy: 0.9685\n",
      "Epoch: [ 0] [  333/  468] time: 94.0604, train_loss: 0.10226198, train_accuracy: 0.9766, test_Accuracy: 0.9678\n",
      "Epoch: [ 0] [  334/  468] time: 94.3291, train_loss: 0.05412788, train_accuracy: 0.9766, test_Accuracy: 0.9676\n",
      "Epoch: [ 0] [  335/  468] time: 94.6035, train_loss: 0.08111814, train_accuracy: 0.9844, test_Accuracy: 0.9673\n",
      "Epoch: [ 0] [  336/  468] time: 94.8763, train_loss: 0.10668784, train_accuracy: 0.9766, test_Accuracy: 0.9667\n",
      "Epoch: [ 0] [  337/  468] time: 95.1476, train_loss: 0.19532707, train_accuracy: 0.9766, test_Accuracy: 0.9668\n",
      "Epoch: [ 0] [  338/  468] time: 95.4189, train_loss: 0.07980758, train_accuracy: 0.9609, test_Accuracy: 0.9660\n",
      "Epoch: [ 0] [  339/  468] time: 95.6942, train_loss: 0.04012486, train_accuracy: 1.0000, test_Accuracy: 0.9660\n",
      "Epoch: [ 0] [  340/  468] time: 95.9662, train_loss: 0.04059280, train_accuracy: 0.9844, test_Accuracy: 0.9656\n",
      "Epoch: [ 0] [  341/  468] time: 96.2425, train_loss: 0.09391157, train_accuracy: 0.9609, test_Accuracy: 0.9646\n",
      "Epoch: [ 0] [  342/  468] time: 96.5238, train_loss: 0.07963171, train_accuracy: 0.9688, test_Accuracy: 0.9633\n",
      "Epoch: [ 0] [  343/  468] time: 96.7980, train_loss: 0.13190109, train_accuracy: 0.9609, test_Accuracy: 0.9633\n",
      "Epoch: [ 0] [  344/  468] time: 97.0741, train_loss: 0.08744772, train_accuracy: 0.9766, test_Accuracy: 0.9628\n",
      "Epoch: [ 0] [  345/  468] time: 97.3590, train_loss: 0.06385061, train_accuracy: 0.9766, test_Accuracy: 0.9626\n",
      "Epoch: [ 0] [  346/  468] time: 97.6360, train_loss: 0.12266198, train_accuracy: 0.9766, test_Accuracy: 0.9617\n",
      "Epoch: [ 0] [  347/  468] time: 97.9188, train_loss: 0.10622233, train_accuracy: 0.9609, test_Accuracy: 0.9615\n",
      "Epoch: [ 0] [  348/  468] time: 98.2022, train_loss: 0.10214111, train_accuracy: 0.9766, test_Accuracy: 0.9596\n",
      "Epoch: [ 0] [  349/  468] time: 98.4783, train_loss: 0.04109959, train_accuracy: 0.9688, test_Accuracy: 0.9581\n",
      "Epoch: [ 0] [  350/  468] time: 98.7510, train_loss: 0.07192951, train_accuracy: 0.9609, test_Accuracy: 0.9564\n",
      "Epoch: [ 0] [  351/  468] time: 99.0399, train_loss: 0.10732453, train_accuracy: 0.9531, test_Accuracy: 0.9567\n",
      "Epoch: [ 0] [  352/  468] time: 99.3177, train_loss: 0.04918836, train_accuracy: 0.9922, test_Accuracy: 0.9570\n",
      "Epoch: [ 0] [  353/  468] time: 99.6017, train_loss: 0.11083402, train_accuracy: 0.9688, test_Accuracy: 0.9595\n",
      "Epoch: [ 0] [  354/  468] time: 99.8797, train_loss: 0.08825064, train_accuracy: 0.9688, test_Accuracy: 0.9592\n",
      "Epoch: [ 0] [  355/  468] time: 100.1582, train_loss: 0.03336693, train_accuracy: 0.9609, test_Accuracy: 0.9591\n",
      "Epoch: [ 0] [  356/  468] time: 100.4313, train_loss: 0.05680642, train_accuracy: 0.9688, test_Accuracy: 0.9587\n",
      "Epoch: [ 0] [  357/  468] time: 100.7048, train_loss: 0.10224360, train_accuracy: 0.9844, test_Accuracy: 0.9603\n",
      "Epoch: [ 0] [  358/  468] time: 100.9751, train_loss: 0.05752976, train_accuracy: 0.9766, test_Accuracy: 0.9614\n",
      "Epoch: [ 0] [  359/  468] time: 101.2457, train_loss: 0.04389186, train_accuracy: 0.9766, test_Accuracy: 0.9621\n",
      "Epoch: [ 0] [  360/  468] time: 101.5210, train_loss: 0.10672540, train_accuracy: 0.9453, test_Accuracy: 0.9630\n",
      "Epoch: [ 0] [  361/  468] time: 101.7986, train_loss: 0.08774728, train_accuracy: 0.9375, test_Accuracy: 0.9637\n",
      "Epoch: [ 0] [  362/  468] time: 102.0678, train_loss: 0.08537506, train_accuracy: 0.9609, test_Accuracy: 0.9642\n",
      "Epoch: [ 0] [  363/  468] time: 102.3403, train_loss: 0.06402623, train_accuracy: 0.9922, test_Accuracy: 0.9655\n",
      "Epoch: [ 0] [  364/  468] time: 102.6103, train_loss: 0.10970350, train_accuracy: 0.9688, test_Accuracy: 0.9655\n",
      "Epoch: [ 0] [  365/  468] time: 102.8819, train_loss: 0.06640990, train_accuracy: 0.9844, test_Accuracy: 0.9657\n",
      "Epoch: [ 0] [  366/  468] time: 103.1570, train_loss: 0.07647148, train_accuracy: 0.9766, test_Accuracy: 0.9666\n",
      "Epoch: [ 0] [  367/  468] time: 103.4331, train_loss: 0.06761818, train_accuracy: 0.9688, test_Accuracy: 0.9672\n",
      "Epoch: [ 0] [  368/  468] time: 103.7019, train_loss: 0.05172930, train_accuracy: 0.9922, test_Accuracy: 0.9669\n",
      "Epoch: [ 0] [  369/  468] time: 103.9765, train_loss: 0.07137258, train_accuracy: 0.9688, test_Accuracy: 0.9677\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: [ 0] [  370/  468] time: 104.2466, train_loss: 0.07196055, train_accuracy: 0.9688, test_Accuracy: 0.9675\n",
      "Epoch: [ 0] [  371/  468] time: 104.5191, train_loss: 0.07451864, train_accuracy: 0.9688, test_Accuracy: 0.9679\n",
      "Epoch: [ 0] [  372/  468] time: 104.7896, train_loss: 0.09573793, train_accuracy: 0.9766, test_Accuracy: 0.9670\n",
      "Epoch: [ 0] [  373/  468] time: 105.0653, train_loss: 0.07326534, train_accuracy: 0.9609, test_Accuracy: 0.9675\n",
      "Epoch: [ 0] [  374/  468] time: 105.3373, train_loss: 0.12505338, train_accuracy: 0.9766, test_Accuracy: 0.9670\n",
      "Epoch: [ 0] [  375/  468] time: 105.6107, train_loss: 0.13861583, train_accuracy: 0.9609, test_Accuracy: 0.9648\n",
      "Epoch: [ 0] [  376/  468] time: 105.8843, train_loss: 0.05679125, train_accuracy: 0.9844, test_Accuracy: 0.9636\n",
      "Epoch: [ 0] [  377/  468] time: 106.1599, train_loss: 0.04744723, train_accuracy: 0.9766, test_Accuracy: 0.9639\n",
      "Epoch: [ 0] [  378/  468] time: 106.4317, train_loss: 0.05268546, train_accuracy: 0.9766, test_Accuracy: 0.9622\n",
      "Epoch: [ 0] [  379/  468] time: 106.7114, train_loss: 0.06168382, train_accuracy: 0.9766, test_Accuracy: 0.9616\n",
      "Epoch: [ 0] [  380/  468] time: 106.9873, train_loss: 0.13276374, train_accuracy: 0.9609, test_Accuracy: 0.9607\n",
      "Epoch: [ 0] [  381/  468] time: 107.2618, train_loss: 0.11481471, train_accuracy: 0.9453, test_Accuracy: 0.9606\n",
      "Epoch: [ 0] [  382/  468] time: 107.5348, train_loss: 0.06801298, train_accuracy: 0.9609, test_Accuracy: 0.9608\n",
      "Epoch: [ 0] [  383/  468] time: 107.8107, train_loss: 0.11362356, train_accuracy: 0.9453, test_Accuracy: 0.9617\n",
      "Epoch: [ 0] [  384/  468] time: 108.0891, train_loss: 0.09086045, train_accuracy: 0.9688, test_Accuracy: 0.9626\n",
      "Epoch: [ 0] [  385/  468] time: 108.3621, train_loss: 0.12973855, train_accuracy: 0.9688, test_Accuracy: 0.9636\n",
      "Epoch: [ 0] [  386/  468] time: 108.6362, train_loss: 0.05483389, train_accuracy: 0.9844, test_Accuracy: 0.9643\n",
      "Epoch: [ 0] [  387/  468] time: 108.9143, train_loss: 0.05093582, train_accuracy: 0.9844, test_Accuracy: 0.9645\n",
      "Epoch: [ 0] [  388/  468] time: 109.1860, train_loss: 0.12514561, train_accuracy: 0.9766, test_Accuracy: 0.9642\n",
      "Epoch: [ 0] [  389/  468] time: 109.4622, train_loss: 0.04488120, train_accuracy: 0.9844, test_Accuracy: 0.9642\n",
      "Epoch: [ 0] [  390/  468] time: 109.7359, train_loss: 0.07844785, train_accuracy: 0.9844, test_Accuracy: 0.9648\n",
      "Epoch: [ 0] [  391/  468] time: 110.0121, train_loss: 0.03005332, train_accuracy: 1.0000, test_Accuracy: 0.9643\n",
      "Epoch: [ 0] [  392/  468] time: 110.2873, train_loss: 0.11281857, train_accuracy: 0.9766, test_Accuracy: 0.9646\n",
      "Epoch: [ 0] [  393/  468] time: 110.5647, train_loss: 0.05506925, train_accuracy: 0.9922, test_Accuracy: 0.9658\n",
      "Epoch: [ 0] [  394/  468] time: 110.8382, train_loss: 0.05781502, train_accuracy: 0.9922, test_Accuracy: 0.9667\n",
      "Epoch: [ 0] [  395/  468] time: 111.1198, train_loss: 0.07646196, train_accuracy: 0.9688, test_Accuracy: 0.9677\n",
      "Epoch: [ 0] [  396/  468] time: 111.3957, train_loss: 0.08497742, train_accuracy: 0.9688, test_Accuracy: 0.9678\n",
      "Epoch: [ 0] [  397/  468] time: 111.6763, train_loss: 0.02395377, train_accuracy: 0.9922, test_Accuracy: 0.9684\n",
      "Epoch: [ 0] [  398/  468] time: 111.9520, train_loss: 0.05425565, train_accuracy: 0.9844, test_Accuracy: 0.9678\n",
      "Epoch: [ 0] [  399/  468] time: 112.2379, train_loss: 0.06788430, train_accuracy: 0.9922, test_Accuracy: 0.9685\n",
      "Epoch: [ 0] [  400/  468] time: 112.5216, train_loss: 0.08179754, train_accuracy: 0.9766, test_Accuracy: 0.9674\n",
      "Epoch: [ 0] [  401/  468] time: 112.8053, train_loss: 0.06286477, train_accuracy: 1.0000, test_Accuracy: 0.9676\n",
      "Epoch: [ 0] [  402/  468] time: 113.0863, train_loss: 0.04528212, train_accuracy: 0.9609, test_Accuracy: 0.9677\n",
      "Epoch: [ 0] [  403/  468] time: 113.3597, train_loss: 0.11551024, train_accuracy: 0.9531, test_Accuracy: 0.9677\n",
      "Epoch: [ 0] [  404/  468] time: 113.6399, train_loss: 0.06776837, train_accuracy: 0.9688, test_Accuracy: 0.9672\n",
      "Epoch: [ 0] [  405/  468] time: 113.9159, train_loss: 0.04432866, train_accuracy: 0.9922, test_Accuracy: 0.9672\n",
      "Epoch: [ 0] [  406/  468] time: 114.2022, train_loss: 0.04255662, train_accuracy: 1.0000, test_Accuracy: 0.9668\n",
      "Epoch: [ 0] [  407/  468] time: 114.4893, train_loss: 0.08487280, train_accuracy: 0.9688, test_Accuracy: 0.9667\n",
      "Epoch: [ 0] [  408/  468] time: 114.7734, train_loss: 0.03581078, train_accuracy: 0.9922, test_Accuracy: 0.9662\n",
      "Epoch: [ 0] [  409/  468] time: 115.0499, train_loss: 0.02377347, train_accuracy: 1.0000, test_Accuracy: 0.9656\n",
      "Epoch: [ 0] [  410/  468] time: 115.3388, train_loss: 0.08597452, train_accuracy: 0.9688, test_Accuracy: 0.9651\n",
      "Epoch: [ 0] [  411/  468] time: 115.6187, train_loss: 0.06086750, train_accuracy: 0.9688, test_Accuracy: 0.9649\n",
      "Epoch: [ 0] [  412/  468] time: 115.9059, train_loss: 0.05947427, train_accuracy: 0.9766, test_Accuracy: 0.9654\n",
      "Epoch: [ 0] [  413/  468] time: 116.1871, train_loss: 0.14364804, train_accuracy: 0.9531, test_Accuracy: 0.9650\n",
      "Epoch: [ 0] [  414/  468] time: 116.4699, train_loss: 0.14255790, train_accuracy: 0.9531, test_Accuracy: 0.9654\n",
      "Epoch: [ 0] [  415/  468] time: 116.7619, train_loss: 0.15623891, train_accuracy: 0.9453, test_Accuracy: 0.9665\n",
      "Epoch: [ 0] [  416/  468] time: 117.0394, train_loss: 0.06397348, train_accuracy: 0.9844, test_Accuracy: 0.9662\n",
      "Epoch: [ 0] [  417/  468] time: 117.3206, train_loss: 0.04354159, train_accuracy: 0.9766, test_Accuracy: 0.9661\n",
      "Epoch: [ 0] [  418/  468] time: 117.6046, train_loss: 0.16735369, train_accuracy: 0.9609, test_Accuracy: 0.9674\n",
      "Epoch: [ 0] [  419/  468] time: 117.8821, train_loss: 0.04083662, train_accuracy: 0.9844, test_Accuracy: 0.9684\n",
      "Epoch: [ 0] [  420/  468] time: 118.1619, train_loss: 0.09119068, train_accuracy: 0.9844, test_Accuracy: 0.9689\n",
      "Epoch: [ 0] [  421/  468] time: 118.4370, train_loss: 0.05534330, train_accuracy: 0.9766, test_Accuracy: 0.9685\n",
      "Epoch: [ 0] [  422/  468] time: 118.7203, train_loss: 0.07439876, train_accuracy: 0.9766, test_Accuracy: 0.9690\n",
      "Epoch: [ 0] [  423/  468] time: 118.9944, train_loss: 0.05504909, train_accuracy: 0.9922, test_Accuracy: 0.9692\n",
      "Epoch: [ 0] [  424/  468] time: 119.2791, train_loss: 0.06466702, train_accuracy: 0.9766, test_Accuracy: 0.9692\n",
      "Epoch: [ 0] [  425/  468] time: 119.5594, train_loss: 0.08699164, train_accuracy: 0.9688, test_Accuracy: 0.9688\n",
      "Epoch: [ 0] [  426/  468] time: 119.8396, train_loss: 0.07600092, train_accuracy: 0.9766, test_Accuracy: 0.9687\n",
      "Epoch: [ 0] [  427/  468] time: 120.1195, train_loss: 0.08780071, train_accuracy: 0.9766, test_Accuracy: 0.9686\n",
      "Epoch: [ 0] [  428/  468] time: 120.3960, train_loss: 0.11962836, train_accuracy: 0.9609, test_Accuracy: 0.9679\n",
      "Epoch: [ 0] [  429/  468] time: 120.6730, train_loss: 0.10566874, train_accuracy: 0.9609, test_Accuracy: 0.9682\n",
      "Epoch: [ 0] [  430/  468] time: 120.9582, train_loss: 0.09882703, train_accuracy: 0.9766, test_Accuracy: 0.9672\n",
      "Epoch: [ 0] [  431/  468] time: 121.2341, train_loss: 0.11905998, train_accuracy: 0.9609, test_Accuracy: 0.9670\n",
      "Epoch: [ 0] [  432/  468] time: 121.5075, train_loss: 0.07744007, train_accuracy: 0.9688, test_Accuracy: 0.9667\n",
      "Epoch: [ 0] [  433/  468] time: 121.7826, train_loss: 0.16874829, train_accuracy: 0.9609, test_Accuracy: 0.9675\n",
      "Epoch: [ 0] [  434/  468] time: 122.0607, train_loss: 0.15265706, train_accuracy: 0.9766, test_Accuracy: 0.9689\n",
      "Epoch: [ 0] [  435/  468] time: 122.3394, train_loss: 0.09362063, train_accuracy: 0.9766, test_Accuracy: 0.9691\n",
      "Epoch: [ 0] [  436/  468] time: 122.6190, train_loss: 0.05701040, train_accuracy: 0.9844, test_Accuracy: 0.9700\n",
      "Epoch: [ 0] [  437/  468] time: 122.9020, train_loss: 0.08446917, train_accuracy: 0.9609, test_Accuracy: 0.9701\n",
      "Epoch: [ 0] [  438/  468] time: 123.1771, train_loss: 0.04116307, train_accuracy: 0.9922, test_Accuracy: 0.9708\n",
      "Epoch: [ 0] [  439/  468] time: 123.4556, train_loss: 0.05500548, train_accuracy: 0.9844, test_Accuracy: 0.9708\n",
      "Epoch: [ 0] [  440/  468] time: 123.7356, train_loss: 0.08892813, train_accuracy: 0.9766, test_Accuracy: 0.9717\n",
      "Epoch: [ 0] [  441/  468] time: 124.0134, train_loss: 0.06729247, train_accuracy: 0.9688, test_Accuracy: 0.9719\n",
      "Epoch: [ 0] [  442/  468] time: 124.2954, train_loss: 0.06463272, train_accuracy: 0.9766, test_Accuracy: 0.9723\n",
      "Epoch: [ 0] [  443/  468] time: 124.5712, train_loss: 0.07042312, train_accuracy: 0.9609, test_Accuracy: 0.9723\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: [ 0] [  444/  468] time: 124.8562, train_loss: 0.04957449, train_accuracy: 0.9688, test_Accuracy: 0.9721\n",
      "Epoch: [ 0] [  445/  468] time: 125.1316, train_loss: 0.14170647, train_accuracy: 0.9531, test_Accuracy: 0.9723\n",
      "Epoch: [ 0] [  446/  468] time: 125.4117, train_loss: 0.04400287, train_accuracy: 0.9766, test_Accuracy: 0.9717\n",
      "Epoch: [ 0] [  447/  468] time: 125.6919, train_loss: 0.05544932, train_accuracy: 0.9844, test_Accuracy: 0.9718\n",
      "Epoch: [ 0] [  448/  468] time: 125.9700, train_loss: 0.08386404, train_accuracy: 0.9609, test_Accuracy: 0.9715\n",
      "Epoch: [ 0] [  449/  468] time: 126.2484, train_loss: 0.09472851, train_accuracy: 0.9609, test_Accuracy: 0.9709\n",
      "Epoch: [ 0] [  450/  468] time: 126.5273, train_loss: 0.06384026, train_accuracy: 0.9922, test_Accuracy: 0.9706\n",
      "Epoch: [ 0] [  451/  468] time: 126.8147, train_loss: 0.03368580, train_accuracy: 0.9844, test_Accuracy: 0.9703\n",
      "Epoch: [ 0] [  452/  468] time: 127.0962, train_loss: 0.10920897, train_accuracy: 0.9766, test_Accuracy: 0.9694\n",
      "Epoch: [ 0] [  453/  468] time: 127.3796, train_loss: 0.10738675, train_accuracy: 0.9844, test_Accuracy: 0.9687\n",
      "Epoch: [ 0] [  454/  468] time: 127.6584, train_loss: 0.05605371, train_accuracy: 0.9844, test_Accuracy: 0.9683\n",
      "Epoch: [ 0] [  455/  468] time: 127.9428, train_loss: 0.08430134, train_accuracy: 0.9922, test_Accuracy: 0.9681\n",
      "Epoch: [ 0] [  456/  468] time: 128.2236, train_loss: 0.05011282, train_accuracy: 0.9688, test_Accuracy: 0.9684\n",
      "Epoch: [ 0] [  457/  468] time: 128.5106, train_loss: 0.06256937, train_accuracy: 0.9688, test_Accuracy: 0.9692\n",
      "Epoch: [ 0] [  458/  468] time: 128.7864, train_loss: 0.06745856, train_accuracy: 0.9922, test_Accuracy: 0.9703\n",
      "Epoch: [ 0] [  459/  468] time: 129.0722, train_loss: 0.13526490, train_accuracy: 0.9609, test_Accuracy: 0.9715\n",
      "Epoch: [ 0] [  460/  468] time: 129.3535, train_loss: 0.05344047, train_accuracy: 0.9766, test_Accuracy: 0.9729\n",
      "Epoch: [ 0] [  461/  468] time: 129.6380, train_loss: 0.05183939, train_accuracy: 0.9922, test_Accuracy: 0.9735\n",
      "Epoch: [ 0] [  462/  468] time: 129.9270, train_loss: 0.05679365, train_accuracy: 0.9922, test_Accuracy: 0.9741\n",
      "Epoch: [ 0] [  463/  468] time: 130.2190, train_loss: 0.04974241, train_accuracy: 0.9844, test_Accuracy: 0.9741\n",
      "Epoch: [ 0] [  464/  468] time: 130.4944, train_loss: 0.07346866, train_accuracy: 0.9844, test_Accuracy: 0.9743\n",
      "Epoch: [ 0] [  465/  468] time: 130.7829, train_loss: 0.04353916, train_accuracy: 0.9922, test_Accuracy: 0.9738\n",
      "Epoch: [ 0] [  466/  468] time: 131.0678, train_loss: 0.07985547, train_accuracy: 0.9688, test_Accuracy: 0.9739\n",
      "Epoch: [ 0] [  467/  468] time: 131.3470, train_loss: 0.04797248, train_accuracy: 0.9922, test_Accuracy: 0.9751\n"
     ]
    }
   ],
   "source": [
    "if train_flag :\n",
    "\n",
    "    checkpoint = tf.train.Checkpoint(dnn=network)\n",
    "\n",
    "    # create writer for tensorboard\n",
    "    summary_writer = tf.contrib.summary.create_file_writer(logdir=logs_dir)\n",
    "    start_time = time()\n",
    "\n",
    "    # restore check-point if it exits\n",
    "    could_load, checkpoint_counter = load(network, checkpoint_dir)\n",
    "    global_step = tf.train.create_global_step()\n",
    "\n",
    "    if could_load:\n",
    "        start_epoch = (int)(checkpoint_counter / training_iterations)\n",
    "        start_iteration = checkpoint_counter - start_epoch * training_iterations\n",
    "        counter = checkpoint_counter\n",
    "        global_step.assign(checkpoint_counter)\n",
    "        print(\" [*] Load SUCCESS\")\n",
    "    else:\n",
    "        start_epoch = 0\n",
    "        start_iteration = 0\n",
    "        counter = 0\n",
    "        print(\" [!] Load failed...\")\n",
    "    \n",
    "    # train phase\n",
    "    with summary_writer.as_default(), tf.contrib.summary.always_record_summaries():  # for tensorboard\n",
    "        for epoch in range(start_epoch, training_epochs):\n",
    "            for idx in range(start_iteration, training_iterations):\n",
    "                train_input, train_label = train_iterator.get_next()\n",
    "                grads = grad(network, train_input, train_label)\n",
    "                optimizer.apply_gradients(grads_and_vars=zip(grads, network.variables), global_step=global_step)\n",
    "\n",
    "                train_loss = loss_fn(network, train_input, train_label)\n",
    "                train_accuracy = accuracy_fn(network, train_input, train_label)\n",
    "\n",
    "                test_input, test_label = test_iterator.get_next()\n",
    "                test_accuracy = accuracy_fn(network, test_input, test_label)\n",
    "\n",
    "                tf.contrib.summary.scalar(name='train_loss', tensor=train_loss)\n",
    "                tf.contrib.summary.scalar(name='train_accuracy', tensor=train_accuracy)\n",
    "                tf.contrib.summary.scalar(name='test_accuracy', tensor=test_accuracy)\n",
    "\n",
    "                print(\n",
    "                    \"Epoch: [%2d] [%5d/%5d] time: %4.4f, train_loss: %.8f, train_accuracy: %.4f, test_Accuracy: %.4f\" \\\n",
    "                    % (epoch, idx, training_iterations, time() - start_time, train_loss, train_accuracy,\n",
    "                       test_accuracy))\n",
    "                counter += 1\n",
    "        checkpoint.save(file_prefix=checkpoint_prefix + '-{}'.format(counter))\n",
    "        \n",
    "# test phase      \n",
    "else :\n",
    "    _, _ = load(network, checkpoint_dir)\n",
    "    test_input, test_label = test_iterator.get_next()\n",
    "    test_accuracy = accuracy_fn(network, test_input, test_label)\n",
    "\n",
    "    print(\"test_Accuracy: %.4f\" % (test_accuracy))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Test accuracy : 97.51 %"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}