Autoencoders_conv.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/hpathak/anaconda3/lib/python3.6/site-packages/h5py/__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n"
     ]
    }
   ],
   "source": [
    "%matplotlib inline\n",
    "\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From <ipython-input-2-8c515496db61>:2: read_data_sets (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use alternatives such as official/mnist/dataset.py from tensorflow/models.\n",
      "WARNING:tensorflow:From /Users/hpathak/anaconda3/lib/python3.6/site-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:260: maybe_download (from tensorflow.contrib.learn.python.learn.datasets.base) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please write your own downloading logic.\n",
      "WARNING:tensorflow:From /Users/hpathak/anaconda3/lib/python3.6/site-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:262: extract_images (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use tf.data to implement this functionality.\n",
      "Extracting MNIST_data/train-images-idx3-ubyte.gz\n",
      "WARNING:tensorflow:From /Users/hpathak/anaconda3/lib/python3.6/site-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:267: extract_labels (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use tf.data to implement this functionality.\n",
      "Extracting MNIST_data/train-labels-idx1-ubyte.gz\n",
      "Extracting MNIST_data/t10k-images-idx3-ubyte.gz\n",
      "Extracting MNIST_data/t10k-labels-idx1-ubyte.gz\n",
      "WARNING:tensorflow:From /Users/hpathak/anaconda3/lib/python3.6/site-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:290: DataSet.__init__ (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use alternatives such as official/mnist/dataset.py from tensorflow/models.\n"
     ]
    }
   ],
   "source": [
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "mnist = input_data.read_data_sets('MNIST_data', validation_size=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "epochs =1 #change the number\n",
    "batch_size=200\n",
    "learning_rate = 0.001\n",
    "pr_type = 32\n",
    "max_pool = True\n",
    "if pr_type == 32:\n",
    "    dType = tf.float32\n",
    "else:\n",
    "    dType = tf.float16"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### CNN AUTOENCODER NO NORM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "tf.reset_default_graph()\n",
    "g1 = tf.Graph()\n",
    "\n",
    "with g1.as_default():\n",
    "    # Input and target placeholders\n",
    "    inputs_ = tf.placeholder(dtype=dType, shape=(None, 28,28,1), name= 'inputs')\n",
    "    targets_ = tf.placeholder(dtype=dType, shape=(None, 28,28,1) , name='targets')\n",
    "\n",
    "\n",
    "    w1 = tf.get_variable(name='W1',shape=[3,3,1,16],dtype=dType,initializer=tf.contrib.layers.xavier_initializer())    \n",
    "    w2 = tf.get_variable(name='W2',shape=[3,3,16,8],dtype=dType,initializer=tf.contrib.layers.xavier_initializer())    \n",
    "    w3 = tf.get_variable(name='W3',shape=[3,3,8,8],dtype=dType,initializer=tf.contrib.layers.xavier_initializer())    \n",
    "    w4 = tf.get_variable(name='W4',shape=[3,3,8,8],dtype=dType,initializer=tf.contrib.layers.xavier_initializer())    \n",
    "    w5 = tf.get_variable(name='W5',shape=[3,3,8,8],dtype=dType,initializer=tf.contrib.layers.xavier_initializer())    \n",
    "    w6 = tf.get_variable(name='W6',shape=[3,3,8,16],dtype=dType,initializer=tf.contrib.layers.xavier_initializer()) \n",
    "    logit_w = tf.get_variable(name='logit_w',shape=[3,3,16,1],dtype=dType,initializer=tf.contrib.layers.xavier_initializer())    \n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "    conv1 = tf.nn.conv2d(input=inputs_,filter=w1,strides=[1,2,2,1], padding='SAME', name='conv1')\n",
    "    conv1 = tf.nn.relu(conv1)\n",
    "    \n",
    "\n",
    "\n",
    "    max_pool1 = tf.nn.max_pool(value=conv1,ksize=[1,2,2,1], strides=[1,2,2,1],padding='SAME', name='maxpool1')\n",
    "    conv2 = tf.nn.conv2d(input=max_pool1, filter=w2, strides=[1,2,2,1], padding='SAME', name='conv2')\n",
    "    conv2 = tf.nn.relu(conv2)\n",
    "    \n",
    "    max_pool2 = tf.nn.max_pool(value=conv2,ksize=[1,2,2,1], strides=[1,2,2,1],padding='SAME', name='maxpool2')\n",
    "    conv3 = tf.nn.conv2d(input=max_pool2, filter=w3, strides=[1,2,2,1], padding='SAME', name='conv3')\n",
    "    conv3 = tf.nn.relu(conv3)\n",
    "\n",
    "    encoded = tf.nn.max_pool(value=conv3, ksize=[1,2,2,1],strides=[1,2,2,1],padding='SAME',name='encoded')\n",
    "\n",
    "\n",
    "    upsample1 = tf.image.resize_nearest_neighbor(encoded, size=(7,7), name='upsample1')\n",
    "    conv4 = tf.nn.conv2d(input=upsample1, filter=w4,strides=[1,1,1,1] ,  padding='SAME', name='conv4')\n",
    "    conv4 = tf.nn.relu(conv4)\n",
    "\n",
    "    upsample2 = tf.image.resize_nearest_neighbor(conv4, size=(14,14), name='upsample2')\n",
    "    conv5 = tf.nn.conv2d(input=upsample2, filter=w5,strides=[1,1,1,1], padding='SAME', name = 'conv5')\n",
    "    conv5 = tf.nn.relu(conv5)\n",
    "\n",
    "    upsample3 = tf.image.resize_nearest_neighbor(conv5, size=(28,28), name='upsample3')\n",
    "    conv6 = tf.nn.conv2d(input=upsample3, filter=w6, strides=[1,1,1,1], padding='SAME', name='conv5')\n",
    "    conv6 = tf.nn.relu(conv6)\n",
    "\n",
    "    logits = tf.nn.conv2d(input=conv6, filter=logit_w,strides=[1,1,1,1] ,padding='SAME')\n",
    "\n",
    "    decoded = tf.nn.sigmoid(logits,name='decoded')\n",
    "\n",
    "    loss = tf.nn.sigmoid_cross_entropy_with_logits(labels=targets_, logits=logits)\n",
    "    cost = tf.reduce_mean(loss)\n",
    "    opt = tf.train.AdamOptimizer(learning_rate).minimize(cost)\n",
    "    init = tf.global_variables_initializer()\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tf.reset_default_graph()\n",
    "sess = tf.Session(graph=g1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "\n",
    "\n",
    "sess.run(init)\n",
    "for e in range(epochs):\n",
    "    for ii in range(mnist.train.num_examples//batch_size):\n",
    "        batch = mnist.train.next_batch(batch_size)\n",
    "        imgs = batch[0].reshape((-1, 28, 28, 1))\n",
    "        batch_cost, _ = sess.run([cost, opt], feed_dict={inputs_: imgs,\n",
    "                                                         targets_: imgs})\n",
    "        if (ii % 20) ==0 :\n",
    "            print(\"Epoch: {}/{}...\".format(e+1, epochs),\n",
    "              \"Training loss: {:.4f}\".format(batch_cost))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "fig, axes = plt.subplots(nrows=2, ncols=10, sharex=True, sharey=True, figsize=(20,4))\n",
    "in_imgs = mnist.test.images[:10]\n",
    "reconstructed = sess.run(decoded, feed_dict={inputs_: in_imgs.reshape((10, 28, 28, 1))})\n",
    "\n",
    "for images, row in zip([in_imgs, reconstructed], axes):\n",
    "    for img, ax in zip(images, row):\n",
    "        ax.imshow(img.reshape((28, 28)), cmap='Greys_r')\n",
    "        ax.get_xaxis().set_visible(False)\n",
    "        ax.get_yaxis().set_visible(False)\n",
    "\n",
    "\n",
    "fig.tight_layout(pad=0.1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Weight Normalized Auto encoder\n",
    "#### Only Weights - data independent\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_normed_weights(shape, axis=None, name=None, return_all=True,\n",
    "                       reuse=None,\n",
    "                       init=tf.random_normal_initializer(stddev=0.05)):\n",
    "    \"\"\"\n",
    "    Returns a normalised tensor of the given shape.\n",
    "    Args:\n",
    "      shape: the shape of the desired weights. At the moment we assume\n",
    "        this is [num_inputs x num_outputs] and we have a gain/scale per\n",
    "        output.\n",
    "      axis: the axis or axes over which to normalise. If None (default), then\n",
    "        each element is divided by the norm of the entire tensor.\n",
    "      scope: scope in which to get the variables required. Defaults to None,\n",
    "        which means `weightnorm` will be used.\n",
    "      return_all: if true, returns the allocated trainable variable as well as\n",
    "        the resulting weights.\n",
    "      reuse: whether or not to attempt to reuse variables. Default is False.\n",
    "      init: the initializer to use to initialise the variables. Defaults to the\n",
    "        values from the paper, ie. normally distributed with mean 0 and\n",
    "        standard deviation 0.05.\n",
    "    Returns:\n",
    "      - if `return_all` is true it will return `(w, g, v)` where `w` is the\n",
    "          required weights, `g` and `v` are the scale and the unnormalised\n",
    "          weights respectively.\n",
    "      - otherwise, just return `w`.\n",
    "    \"\"\"\n",
    "    v = tf.get_variable(name=name, shape=shape, dtype=tf.float32, initializer=init)\n",
    "    g = tf.get_variable(name=name+'g', shape=shape[-1], initializer=tf.constant_initializer(1),trainable=False)\n",
    "    \n",
    "    inv_norm = tf.rsqrt(tf.reduce_sum(tf.square(v), reduction_indices=axis))\n",
    "    w = v * g * inv_norm\n",
    "    return w\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "learning_rate = 0.001\n",
    "pr_type = 32\n",
    "max_pool = True\n",
    "\n",
    "if pr_type == 32:\n",
    "    dType = tf.float32\n",
    "else:\n",
    "    dType = tf.float16\n",
    "\n",
    "tf.reset_default_graph()\n",
    "g2 = tf.Graph()\n",
    "\n",
    "\n",
    "with g2.as_default():\n",
    "    # Input and target placeholders\n",
    "    inputs_ = tf.placeholder(dtype=dType, shape=(None, 28,28,1), name= 'inputs')\n",
    "    targets_ = tf.placeholder(dtype=dType, shape=(None, 28,28,1) , name='targets')\n",
    "\n",
    "\n",
    "    #w1 = tf.get_variable(name='W1',shape=,dtype=dType,initializer=tf.contrib.layers.xavier_initializer())    \n",
    "    w1 = get_normed_weights(shape=[3,3,1,16], axis=[0,1,2],name='W1', init=tf.contrib.layers.xavier_initializer())\n",
    "    \n",
    "    #w2 = tf.get_variable(name='W2',shape=[3,3,16,8],dtype=dType,)    \n",
    "    w2 = get_normed_weights(shape=[3,3,16,8], axis=[0,1,2],name='W2', init=tf.contrib.layers.xavier_initializer())\n",
    "    \n",
    "    #w3 = tf.get_variable(name='W3',shape=[3,3,8,8],dtype=dType,initializer=tf.contrib.layers.xavier_initializer())    \n",
    "    w3 = get_normed_weights(shape=[3,3,8,8], axis=[0,1,2],name='W3', init=tf.contrib.layers.xavier_initializer())\n",
    "    \n",
    "    #w4 = tf.get_variable(name='W4',shape=[3,3,8,8],dtype=dType,initializer=tf.contrib.layers.xavier_initializer())    \n",
    "    w4 = get_normed_weights(shape=[3,3,8,8], axis=[0,1,2],name='W4', init=tf.contrib.layers.xavier_initializer())\n",
    "    \n",
    "    #w5 = tf.get_variable(name='W5',shape=[3,3,8,8],dtype=dType,initializer=tf.contrib.layers.xavier_initializer())    \n",
    "    w5 = get_normed_weights(shape=[3,3,8,8], axis=[0,1,2],name='W5', init=tf.contrib.layers.xavier_initializer())\n",
    "    \n",
    "    #w6 = tf.get_variable(name='W6',shape=[3,3,8,16],dtype=dType,initializer=tf.contrib.layers.xavier_initializer()) \n",
    "    w6 = get_normed_weights(shape=[3,3,8,16], axis=[0,1,2],name='W6', init=tf.contrib.layers.xavier_initializer())\n",
    "    \n",
    "    logit_w = tf.get_variable(name='logit_w',shape=[3,3,16,1],dtype=dType,initializer=tf.contrib.layers.xavier_initializer())    \n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "    conv1 = tf.nn.conv2d(input=inputs_,filter=w1,strides=[1,2,2,1], padding='SAME', name='conv1')\n",
    "    \n",
    "    #conv1 = wnconv(x=inputs_, filter_size=[3,3,1,16], strides=[1,2,2,1], padding=\"SAME\", name='conv1', init=True)\n",
    "    conv1 = tf.nn.relu(conv1)\n",
    "    \n",
    "\n",
    "\n",
    "    max_pool1 = tf.nn.max_pool(value=conv1,ksize=[1,2,2,1], strides=[1,2,2,1],padding='SAME', name='maxpool1')\n",
    "    conv2 = tf.nn.conv2d(input=max_pool1, filter=w2, strides=[1,2,2,1], padding='SAME', name='conv2')\n",
    "    conv2 = tf.nn.relu(conv2)\n",
    "    \n",
    "    max_pool2 = tf.nn.max_pool(value=conv2,ksize=[1,2,2,1], strides=[1,2,2,1],padding='SAME', name='maxpool2')\n",
    "    conv3 = tf.nn.conv2d(input=max_pool2, filter=w3, strides=[1,2,2,1], padding='SAME', name='conv3')\n",
    "    conv3 = tf.nn.relu(conv3)\n",
    "\n",
    "    encoded = tf.nn.max_pool(value=conv3, ksize=[1,2,2,1],strides=[1,2,2,1],padding='SAME',name='encoded')\n",
    "\n",
    "\n",
    "    upsample1 = tf.image.resize_nearest_neighbor(encoded, size=(7,7), name='upsample1')\n",
    "    conv4 = tf.nn.conv2d(input=upsample1, filter=w4,strides=[1,1,1,1] ,  padding='SAME', name='conv4')\n",
    "    conv4 = tf.nn.relu(conv4)\n",
    "\n",
    "    upsample2 = tf.image.resize_nearest_neighbor(conv4, size=(14,14), name='upsample2')\n",
    "    conv5 = tf.nn.conv2d(input=upsample2, filter=w5,strides=[1,1,1,1], padding='SAME', name = 'conv5')\n",
    "    conv5 = tf.nn.relu(conv5)\n",
    "\n",
    "    upsample3 = tf.image.resize_nearest_neighbor(conv5, size=(28,28), name='upsample3')\n",
    "    conv6 = tf.nn.conv2d(input=upsample3, filter=w6, strides=[1,1,1,1], padding='SAME', name='conv5')\n",
    "    conv6 = tf.nn.relu(conv6)\n",
    "\n",
    "    logits = tf.nn.conv2d(input=conv6, filter=logit_w,strides=[1,1,1,1] ,padding='SAME')\n",
    "\n",
    "    decoded = tf.nn.sigmoid(logits,name='decoded')\n",
    "\n",
    "    loss = tf.nn.sigmoid_cross_entropy_with_logits(labels=targets_, logits=logits)\n",
    "    cost = tf.reduce_mean(loss)\n",
    "    opt = tf.train.AdamOptimizer(learning_rate).minimize(cost)\n",
    "    init = tf.global_variables_initializer()\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Data Dependet Parameter initialization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "# def get_var_maybe_avg(var_name, ema, **kwargs):\n",
    "#     ''' utility for retrieving polyak averaged params '''\n",
    "#     v = tf.get_variable(var_name, **kwargs)\n",
    "#     if ema is not None:\n",
    "#         v = ema.average(v)\n",
    "#     return v\n",
    "\n",
    "# def get_vars_maybe_avg(var_names, ema, **kwargs):\n",
    "#     ''' utility for retrieving polyak averaged params '''\n",
    "#     vars = []\n",
    "#     for vn in var_names:\n",
    "#         vars.append(get_var_maybe_avg(vn, ema, **kwargs))\n",
    "#     return vars\n",
    "\n",
    "# def get_name(layer_name, counters):\n",
    "#     ''' utlity for keeping track of layer names '''\n",
    "#     if not layer_name in counters:\n",
    "#         counters[layer_name] = 0\n",
    "#     name = layer_name + '_' + str(counters[layer_name])\n",
    "#     counters[layer_name] += 1\n",
    "#     return name\n",
    "#\n",
    "#input=inputs_,filter=w1,strides=[1,2,2,1], padding='SAME', name='conv1'\n",
    "#\n",
    "def wnconv2d(x, filter_size, name ,strides=[1,2,2,1], padding='SAME', init_scale=1.0, init=True):\n",
    "    ''' convolutional layer '''\n",
    "    #name = get_name('conv2d', counters)\n",
    "    with tf.variable_scope(name):\n",
    "        if init:\n",
    "            # data based initialization of parameters\n",
    "            V = tf.get_variable(name=name, shape=filter_size, dtype=tf.float32, initializer=tf.contrib.layers.xavier_initializer(), trainable=True)\n",
    "            V_norm = tf.nn.l2_normalize(x=V.initialized_value(),axis=[0,1,2])\n",
    "            x_init = tf.nn.conv2d(input=x, filter=V_norm,strides=strides , padding=padding)\n",
    "            m_init, v_init = tf.nn.moments(x_init, [0,1,2])\n",
    "            scale_init = init_scale/tf.sqrt(v_init + 1e-8)\n",
    "            \n",
    "            #g = tf.get_variable(name+'g', dtype=tf.float32, initializer=scale_init, trainable=True)\n",
    "            #b = tf.get_variable(name+'b', dtype=tf.float32, initializer=-m_init*scale_init, trainable=True)\n",
    "            \n",
    "            x_init = tf.reshape(scale_init,[1,1,1,filter_size[-1]])*(x_init-tf.reshape(m_init,[1,1,1,filter_size[-1]]))\n",
    "            return tf.cast(x_init, tf.float32)\n",
    "\n",
    "#         else:\n",
    "#             V, g, b = get_vars_maybe_avg(['V', 'g', 'b'], ema)\n",
    "#             tf.assert_variables_initialized([V,g,b])\n",
    "\n",
    "#             # use weight normalization (Salimans & Kingma, 2016)\n",
    "#             W = tf.reshape(g,[1,1,1,num_filters])*tf.nn.l2_normalize(V,[0,1,2])\n",
    "\n",
    "#             # calculate convolutional layer output\n",
    "#             x = tf.nn.bias_add(tf.nn.conv2d(x, W, [1]+stride+[1], pad), b)\n",
    "\n",
    "        \n",
    "#             return x\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "g3 = tf.Graph()\n",
    "\n",
    "\n",
    "with g3.as_default():\n",
    "    # Input and target placeholders\n",
    "    \n",
    "    inputs_ = tf.placeholder(dtype=tf.float32, shape=(None,28,28,1), name='inputs_')\n",
    "    targets_ = tf.placeholder(dtype=tf.float32, shape=(None, 28,28,1) , name='targets_')\n",
    "\n",
    "\n",
    "    logit_w = tf.get_variable(name='logit_w',shape=[3,3,16,1],dtype=tf.float32,initializer=tf.contrib.layers.xavier_initializer())    \n",
    "\n",
    "\n",
    "    \n",
    "    conv1 = wnconv2d(x=inputs_, filter_size=[3,3,1,16], name='conv1',strides=[1,2,2,1], padding=\"SAME\",  init=True)\n",
    "    conv1 = tf.nn.relu(conv1)\n",
    "    \n",
    "    max_pool1 = tf.nn.max_pool(value=conv1,ksize=[1,2,2,1], strides=[1,2,2,1],padding='SAME', name='maxpool1')\n",
    "    conv2 = wnconv2d(x=max_pool1, filter_size=[3,3,16,8], name='conv2',strides=[1,2,2,1], padding=\"SAME\",  init=True)\n",
    "    conv2 = tf.nn.relu(conv2)\n",
    "    \n",
    "    max_pool2 = tf.nn.max_pool(value=conv2,ksize=[1,2,2,1], strides=[1,2,2,1],padding='SAME', name='maxpool2')\n",
    "    conv3 = wnconv2d(x=max_pool2, filter_size=[3,3,8,8], name='conv3',strides=[1,2,2,1], padding=\"SAME\",  init=True)\n",
    "    conv3 = tf.nn.relu(conv3)\n",
    "\n",
    "    encoded = tf.nn.max_pool(value=conv3, ksize=[1,2,2,1],strides=[1,2,2,1],padding='SAME',name='encoded')\n",
    "\n",
    "\n",
    "    upsample1 = tf.image.resize_nearest_neighbor(encoded, size=(7,7), name='upsample1')\n",
    "    conv4 = wnconv2d(x=encoded, filter_size=[3,3,8,8], name='conv4',strides=[1,2,2,1], padding=\"SAME\",  init=True)\n",
    "    conv4 = tf.nn.relu(conv4)\n",
    "\n",
    "    upsample2 = tf.image.resize_nearest_neighbor(conv4, size=(14,14), name='upsample2')\n",
    "    conv5 = wnconv2d(x=upsample2, filter_size=[3,3,8,8], name='conv5',strides=[1,1,1,1], padding=\"SAME\",  init=True)\n",
    "    conv5 = tf.nn.relu(conv5)\n",
    "\n",
    "    upsample3 = tf.image.resize_nearest_neighbor(conv5, size=(28,28), name='upsample3')\n",
    "    conv6 = wnconv2d(x=upsample3, filter_size=[3,3,8,16], name='conv6',strides=[1,1,1,1], padding=\"SAME\",  init=True)\n",
    "    conv6 = tf.nn.relu(conv6)\n",
    "\n",
    "    logits = tf.nn.conv2d(input=conv6, filter=logit_w,strides=[1,1,1,1] ,padding='SAME', name='logits')\n",
    "\n",
    "    decoded = tf.nn.sigmoid(logits,name='decoded')\n",
    "\n",
    "    loss = tf.nn.sigmoid_cross_entropy_with_logits(labels=targets_, logits=logits)\n",
    "    cost = tf.reduce_mean(loss)\n",
    "    opt = tf.train.AdamOptimizer(learning_rate).minimize(cost)\n",
    "    init_ = tf.global_variables_initializer()\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "tf.reset_default_graph()\n",
    "sess = tf.Session(graph=g3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 1/5... Training loss: 0.8073\n",
      "Epoch: 1/5... Training loss: 0.3477\n",
      "Epoch: 1/5... Training loss: 0.3198\n",
      "Epoch: 1/5... Training loss: 0.3126\n",
      "Epoch: 1/5... Training loss: 0.3012\n",
      "Epoch: 1/5... Training loss: 0.2999\n",
      "Epoch: 2/5... Training loss: 0.3038\n",
      "Epoch: 2/5... Training loss: 0.3024\n",
      "Epoch: 2/5... Training loss: 0.2904\n",
      "Epoch: 2/5... Training loss: 0.2923\n",
      "Epoch: 2/5... Training loss: 0.2996\n",
      "Epoch: 2/5... Training loss: 0.2960\n",
      "Epoch: 3/5... Training loss: 0.2925\n",
      "Epoch: 3/5... Training loss: 0.2936\n",
      "Epoch: 3/5... Training loss: 0.2981\n",
      "Epoch: 3/5... Training loss: 0.2883\n",
      "Epoch: 3/5... Training loss: 0.2964\n",
      "Epoch: 3/5... Training loss: 0.2977\n",
      "Epoch: 4/5... Training loss: 0.2936\n",
      "Epoch: 4/5... Training loss: 0.2924\n",
      "Epoch: 4/5... Training loss: 0.2959\n",
      "Epoch: 4/5... Training loss: 0.2956\n",
      "Epoch: 4/5... Training loss: 0.2966\n",
      "Epoch: 4/5... Training loss: 0.2912\n",
      "Epoch: 5/5... Training loss: 0.2952\n",
      "Epoch: 5/5... Training loss: 0.2934\n",
      "Epoch: 5/5... Training loss: 0.2919\n",
      "Epoch: 5/5... Training loss: 0.2881\n",
      "Epoch: 5/5... Training loss: 0.2894\n",
      "Epoch: 5/5... Training loss: 0.2890\n",
      "CPU times: user 18min 56s, sys: 9min 28s, total: 28min 25s\n",
      "Wall time: 6min 22s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "epochs =5\n",
    "batch_size=200\n",
    "sess.run(init_)\n",
    "\n",
    "\n",
    "for e in range(epochs):\n",
    "    for ii in range(mnist.train.num_examples//batch_size):\n",
    "        batch = mnist.train.next_batch(batch_size)\n",
    "        imgs = batch[0].reshape((-1, 28, 28, 1))\n",
    "        batch_cost, _ = sess.run([cost, opt], feed_dict={inputs_: imgs ,targets_: imgs})\n",
    "        if (ii % 50) ==0 :\n",
    "            print(\"Epoch: {}/{}...\".format(e+1, epochs),\n",
    "              \"Training loss: {:.4f}\".format(batch_cost))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAABawAAAEsCAYAAAAvofT2AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzt3Xu0VWW5OOC52YDcxBuiZlxKy9ShhqKCooBpx07KKD2CQ0HTVPKCpeWFUryEldejmae0zNuQ00hMJT1yGiac1DIQ78WJBBVEURFB5M5mnz9+p/E7c34T1trr+u29nue/9+Vbc73G51xzva6+t6m1tTUBAAAAAIB661TvAgAAAAAAIEk0rAEAAAAAiISGNQAAAAAAUdCwBgAAAAAgChrWAAAAAABEQcMaAAAAAIAoaFgDAAAAABAFDWsAAAAAAKKgYQ0AAAAAQBQ6t2Vxnz59WgcOHFilUmjv5syZs7S1tXXHzf25/cPm2DuUw/6hHPYP5bB/KIf9QznsH8ph/1AO+4dyFNo//9CmhvXAgQOT5557rvSq6NCampre3NKf2z9sjr1DOewfymH/UA77h3LYP5TD/qEc9g/lsH8oR6H98w+OBAEAAAAAIApt+oX1/9XU1FTJOminWltbS3qd/UOS2D+Ux/6hHKXsH3uHJHHvoTz2D+WwfyiH/UM57B/KUcr+8QtrAAAAAACioGENAAAAAEAUNKwBAAAAAIiChjUAAAAAAFHQsAYAAAAAIAoa1gAAAAAAREHDGgAAAACAKGhYAwAAAAAQBQ1rAAAAAACioGENAAAAAEAUNKwBAAAAAIiChjUAAAAAAFHQsAYAAAAAIAqd610AtCfXXnttkOvRo0eQGzx4cCoeMmRIUdefNm1aKp4xY0aw5uabby7qWgAAAADQ3viFNQAAAAAAUdCwBgAAAAAgChrWAAAAAABEQcMaAAAAAIAoGLoIW/DMM8+k4qFDh5Z0ndbW1qLWHXvssan40EMPDdZkBzMmSZIsWLCgpLro2PbZZ58g99JLLwW573//+6n4iiuuqFpNVF+vXr1S8f333x+syd5rkiRJFi5cmIq/8IUvBGvmz59fZnUAANAYdthhhyC3xx57tPk6//3f/x3krrnmmiCX/a738ssvB2v++Mc/tvn9oR78whoAAAAAgChoWAMAAAAAEAUNawAAAAAAouAMa/hf2fOqk6T0M6vfe++9VDxjxoxgze677x7kDjjggFS8/fbbB2smTJgQ5C644IK2lkgDOOyww4Jc3nnqixYtqkU51MjAgQNT8THHHBOsydsH/fv3T8Vjx44N1lx11VXlFUddHH744UEubx7CtttuW4tyNuvEE08Mcn/+859T8euvv16rcqiTU089NcjdfffdQe7KK69MxZMnTw7WtLS0VKosirTLLruk4pkzZwZrnn766SD3ox/9KBX//e9/r2hdlbDddtsFuVGjRgW5KVOmpOINGzZUrSagfsaNG5eK855jDjrooCCXd651IUuXLg1yec9tnTsXbvF16uR3q7QPdioAAAAAAFHQsAYAAAAAIAoa1gAAAAAAREHDGgAAAACAKBi6SEMaOXJkkDv44IMLvm7JkiVBbvjw4QXXrVy5MljTtWvXIDd//vxUvOuuuwZr+vbtW7BOSJIkOfDAA4Nc3uCfX/ziF7UohyrYeeedg9wjjzxSh0qI2Ve+8pUg19zcXIdKtmzMmDFB7rzzzkvFw4YNq1U51Ej2uebWW28t6nXZoYvXX399sGb16tUl10VheYPDXnvttVS81VZbBWvyhoe1hyGL2X+2JEmSnj17Brk5c+ak4ldffbWyhTW4vEFz2cGse+21V7Bm7733DnIGYpIkSbLnnnum4kmTJgVrjjvuuCCXHXDY1NRU2cL+jz59+lTt2hArv7AGAAAAACAKGtYAAAAAAERBwxoAAAAAgCi0mzOszzrrrFQ8YcKEYM27774b5LJn191xxx3BmgULFgS5v/71r20tkXakf//+QS7vzKnsWdR551wvWrSopBquvfbaIJd3Hm3Wb37zm5Lej44vuz9POumkYM306dNrVQ4VdvXVVwe5E044IcgNHDiwIu/3xS9+Mch16hT+d+7nn38+FTtDu/6yZyoee+yxdaqkbZ5++ukg9+1vfzsV9+rVK1jz8ccfV60mqi+7P7feeuuiXvfUU0+l4jVr1lSsJkI77bRTkJs5c2aQ6969eyp+6KGHgjXHH398xeqqpux56tkzrZMkSSZOnBjknFldOeeff36Qy3se6t27d8Fr5f39vffee6UVRoeyxx57pOK8mRq1lt2beT0r4pR3hn6/fv2CXPa7et5stE2bNgW5n/zkJ6n4d7/7XbCmo3wO+YU1AAAAAABR0LAGAAAAACAKGtYAAAAAAERBwxoAAAAAgCi0m6GL2QF122yzTbBm7733LnidY445JsitX78+yC1evLgN1dVGdqjk9773vWDNjBkzalVOu3bPPfcEubxhTytWrEjFS5curVgNo0ePDnLNzc0Vuz6NZ7/99kvFXbp0CdbcddddtSqHCrvsssuCXGtra9Xeb8iQIUXlli9fnorzhmnlDeaierJ/B5/+9KeDNXfffXeNqilenz59glx20Juhi+1bt27dgtwVV1xR0rVuv/32VFzN+yFJMnLkyCCXHVSW59xzz61GORU3ePDgIJcdiDVr1qxgzc9+9rOq1dSIsoOjf/jDHwZrsoM9izV16tQgd9xxx6XiSn7Xo7ryBsFOnjw5Fef1RqZMmRLk1q5dm4rXrVsXrMnrGXXt2jUVz5kzJ1iTHU6eJEnyzDPPpOK85+RVq1alYs86cTj44IODXPY72hFHHBGsKfW+leeGG25IxXmDGd9///1UPHv27GDNv/zLvwS5vH1eT35hDQAAAABAFDSsAQAAAACIgoY1AAAAAABR0LAGAAAAACAK7Wbo4llnnZWK999//2DNK6+8EuT22WefVDx06NBgzaBBg4Lcpz71qVT80UcfBWt69+6dX2wBeYeir169OhXnDRXK1nTGGWcEawxdLN38+fOrdu3rrrsuyPXt27fg615//fUgN3369IrURMfz3e9+NxVnh4YmSZI88cQTtSqHMr344oupuKmpqarvt2bNmlScN3Qjb+Dxdtttl4qffPLJYE2nTv77eLXkDX/JDlddtmxZsOZb3/pW1WoqVXb4FR3PIYccEuT69etX8HV5z873339/RWoi3y677JKKx40bV9TrLrroolS8ZMmSitVUSdkhi8V8h/r3f//3IJf3rEXpst+ZKjmobNiwYUFu0aJFqfiWW24J1kyaNCnIxTaYrKPL640899xzQW7XXXdNxdnhhpuT/X697777Bmv+/ve/B7nsUOs33ngjWJP3+UWcssPlL7/88mBN3kDFrbbaquC1V65cGeReeumlVDxv3rxgzWmnnRbkFi5cmIoHDBgQrOnZs2cqPvzww4M1F198cZDLDi6tN98gAQAAAACIgoY1AAAAAABR0LAGAAAAACAK7eYM6wceeGCLcTl22GGHIDdy5MhUnHfu61FHHVXS+2XPq06SJJkzZ04qXrBgQbCmW7duqfhvf/tbSe9P9Z1yyimp+IILLgjWNDc3B7lVq1al4m9/+9sF19CYPvOZzwS5/v37p+KlS5cGaz7++OOq1UTpvvKVrwS57N9na2trsCYvV4yHH344yE2bNi0VL1++PFjzT//0T0Fu/PjxBd8vewbc97///YKvoTg33nhjkOvSpUsqHjNmTLAm7yy9WuvTp08q/uxnPxusKXWPE6diz0HOevnllytcCYVkz2sePnx4sCZ7/m+SJMntt99etZoq6eijj07F2fM+kyRJfv/736fivPONKd1uu+0W5EaNGlXwde+8806Qy85q2HvvvYuqIXv27LnnnhusufXWW4Pc4sWLi7o+penatWsqnjlzZrAme151kiTJnXfemYpL7RnlnVedJ69nQ/vw2GOPBbkRI0ak4mLP0J87d24qzntmOf3004Ncdn5Qnryz90888cRU/OCDDwZrsvNB8npIV199dZD7xS9+kYrrPYfCL6wBAAAAAIiChjUAAAAAAFHQsAYAAAAAIAoa1gAAAAAARKHdDF2spg8++CDITZ06teDrKjn48cwzz0zF2QGLSRIOmPi3f/u3ir0/lTVkyJBUnDdgMc/jjz+eivMGo0GSJMmxxx5bcM2KFStqUAltlTcw87777gtyPXr0KOn62WGJjz76aLDmnHPOCXLFDHR99dVXg1x2iFpe3ZdddlkqzhticsUVVwS5DRs2FKypkZx11llBbvDgwUEuO3D1ySefrFpN5fjxj3+civMGLGYHTOc9s9F+HH744QXXtLS0BLnzzjuvGuWwBdl/H/P+/Xz//feD3Lp166pWUzHyPoNuvvnmIDd27NiC1zrqqKMqUhP58u4H2WF7r732WrAmb0Bv9rki755x6aWXBrntttsuFffq1StY88wzzwS57Gdv3qBzirP11lsHuX/9139Nxfvvv3+wZvXq1UHu4osvTsXFPNvS8WTvB9dff32w5ktf+lLB6+TtsXvvvTfIZffdxx9/XPDaxerdu3eQ69w53cb93ve+F6yZMmVKKt5mm20qVlMt+YU1AAAAAABR0LAGAAAAACAKGtYAAAAAAERBwxoAAAAAgCgYulgHu+yyS5DLDhZoamoK1lx55ZWp2HCHOMyePTvI7bfffgVflzcE6+tf/3pFaqLjO+CAAwqumTx5cg0qoa222mqrIFfqgMXsQLokSZKRI0em4nfffbeka+eZP39+kLvppptScXbAYpIkSZcuXVLxJZdcEqzJGzw5d+7ctpbYoZ166qlBLvu/bZIkyU9/+tNalNMmecNGR40alYo3bdoUrLn88stTsUGc7UfeQKNPf/rTBV+X93ecN/SM+hs0aFCQe+WVV1LxRx99FKzJfm6U48gjj0zF2c/AJEmST33qUwWv86c//aliNVGcbt26FVzzox/9qKhrrVmzJhXnDVk7+eSTg1x26GLecNG1a9cGuXoPF+1ITj/99IK5vEHyefefDz/8sHKF0W599atfTcVnnnlmUa/LDks87rjjgjVPPPFE6YVlNDc3p+K8Z6S870fZGoq5l+b1F2fOnBnkYhtu7hfWAAAAAABEQcMaAAAAAIAoaFgDAAAAABAFZ1jXwaRJk4Jc9vzSvLOyXnrpparVRHH69esX5Pbaa68g17lz+l+t1atXB2smTJgQ5FauXFlGdXRURx99dJDLns2VJEny1ltvpeJf//rXVauJ2lu4cGGQO+aYY4JcJc+sLsa9996bik855ZRgzYABA2pVToeSPVtz7733Lup1V199dTXKKcull14a5Lp3756K33vvvWDN1KlTq1YT1XXIIYeU9Lr777+/wpVQiquuuioVT5s2LVjTq1evIPfZz3624LWnTJlSemEVkj3r9owzzqhTJY3rtNNOK7jmhBNOCHK//OUvS3q/vFkKxcg739x3tso54ogjCq6ZN29ekHvjjTeqUA0dQfZs6LwZKXlaWlpS8WGHHRasyfueU8zzeV5/LztfYaeddgrW5PWRevbsWfD9slatWhXkzj///CAX26wYv7AGAAAAACAKGtYAAAAAAERBwxoAAAAAgChoWAMAAAAAEAVDF6vsy1/+cpA788wzC77uxBNPDHKzZs2qSE2UbubMmUEuOzQqT96gmrlz51aiJBrAP//zPwe5vH33+uuvp+I1a9ZUrSYqq6mpqeCagQMHVr+QEnTqlP5v33n/LMX88/3sZz8LcsOHDy+9sA6gW7duqXjrrbcO1jz99NO1Kqcsn/vc5wquee2112pQCbVy+OGHF7UuO4ho8uTJ1SiHNso+82aHQyVJkowYMSLIjRo1KhWPGzcuWJM3ROrBBx9sW4H/67bbbkvFzz77bFGvyw6z91xee3fddVeQGzx4cCred999gzWf//zng9yQIUNS8UknnRSsyX6mJkl4/8lbM2bMmCD3k5/8JBXPmTMnWENxjjzyyIJrBg0aFOSy/+4nSZL86le/SsVPPfVU6YXRbmU/TyZMmBCs2W+//YLcNttsk4onTZoUrGltbS34/nlrivkulKeYAYt575ftHY4ePTpYs2jRopJqqiW/sAYAAAAAIAoa1gAAAAAAREHDGgAAAACAKGhYAwAAAAAQBUMXq+yrX/1qkMsOqEqScNDHf/zHf1StJor3ta99LRX379+/qNf97W9/S8Xjx4+vVEk0oAMPPDDI5Q1XuPfee2tRDmWaOHFikCtmgEesxo4dm4r79esXrMn+8+X9837jG9+obGEdwIoVK1Lx4sWLgzW77757kOvTp08qXrp0aWULK2CXXXYJckOHDi34uieeeKIa5VAjxxxzTCo+7LDDinrdunXrUvEbb7xRqZKooA8++CDI5Q1KzOZOPfXUqtWUJMUNdM27d+YN5aO2HnjggSB30003peK8z5Pnn3++pPf7y1/+EuSyAxWzw0aTJPxMTZIkufLKK1PxscceW1JNJEmPHj2CXPY5sXPnsG119tlnB7nss+TDDz8crPmv//qvIJcdbD5v3rxgzezZs4NcVt53tunTpwc5n3PVlR3se9BBBwVrtt9++yCXvf8ceuihwZrly5cHuTfffDMVd+/ePViz1157BbkBAwYEuVI8+uijQe60005LxcuWLavIe9WaX1gDAAAAABAFDWsAAAAAAKKgYQ0AAAAAQBScYV1h2TOYvvjFLwZrWlpagtx3vvOdVLxhw4bKFkZBffv2DXJXXHFFKm5ubi7qWi+88EIqXrlyZemF0XB23XXXVLzPPvsEa/LOpL3zzjurVhOVk/e5EKOdd945yA0ZMiTIXXjhhW2+dvZsuSQJz7El/N9p0aJFwZq8v5NZs2al4uuuu65iNe23335BLnsu3yc+8YlgTTHntLfns9xJkh133DEVNzU1FfW6P/3pT9UohwZx2223FVyT/Z6VJEmyZMmSapRDG+Q9y2bPPL/nnnuCNd26dQty2c+PvPPVTznllCC3Zs2aVPzb3/42WJM9CzZJkmTYsGGpeM899wzWZGdUke/+++8PcqWeMZ/93MmbJ5aXq6a8Z94XX3wxFWf3E9WXd6Zzdn5ZJc2YMSPIFXOG9fr164PcpEmTUvGNN94YrMnrObZHfmENAAAAAEAUNKwBAAAAAIiChjUAAAAAAFHQsAYAAAAAIAqGLlZYdrDRJz/5yWDNyy+/HOQef/zxqtVEcX74wx8GuWIOws8Ot0qSJBk/fnxFaqIxZYfYZYe5JkmSPPvss7Uqhwb14x//OMgdf/zxJV1r+fLlqThvqMmCBQtKunYjOe+884Jc3sCxwYMHF1xTquyAqiQJh13l3bOKccMNN5T0OuJQzLCitWvXBrnrr7++CtXQEX3jG98IciNHjkzFeQOq3nnnnarVRGX9+te/LrjmzDPPDHLZAY5nnXVWsCbv8ytrwoQJQS5v+Hkxn7NHHHFEwfcjHLSZJEnyy1/+MhXn7Yvm5uYg17t371Rc7PDfasp7Jho6dGgqznvmPv/886tWE9WV91xz2GGHlXStiy66KMjdeuutJV2rPfILawAAAAAAoqBhDQAAAABAFDSsAQAAAACIgoY1AAAAAABRMHSxDOPGjQtyZ599dipet25dsObSSy+tWk2U7pRTTinpdSeccEKQW7lyZbnl0MA+85nPFFzz/vvv16ASGsmLL76Yivv371+xa7/55pupeNq0aRW7diN54YUXgtwhhxwS5LKDXfbcc8+K1XDHHXcUXPPkk08GueHDhxd83erVq0uqidobOHBgkCtmoFB2AGuS5O8XyFPM4N8///nPQe4Pf/hDNcqhBvKG7RUzmLFUeZ9D99xzT5DLDl084IADgjV9+vRJxdnBkPw/LS0tQS77uZD933Jzst/Lu3TpEqy55pprgtyAAQOKun6lZIdBDhkypKbvT2VdcsklqThveGunToV/K/zuu+8GuZ///OelF9YB+IU1AAAAAABR0LAGAAAAACAKGtYAAAAAAETBGdZF6tu3b5C75ZZbglz2PKLZs2cHa6ZPn165wqi7nXbaKcitX7++ItdetmxZkNuwYUOQy57Ptf322xe89o477hjk8s70KsbGjRuDXPZM8FWrVpV07UY0YsSIgmsefPDB6hdCVWQ/JzaXyzr55JOLuv5Pf/rTVNyrV6+S6mptbS3qdcUYNGhQxa5FYU899dQW42qbO3dukCvmDOuDDz44yOWdR0v9felLXwpyxdzHHn300WqUQ4PIO+c1+1x8+eWX16ocGkT2uSpJkmTMmDGpeNiwYcGaK6+8MhWfd955Fa2L0AMPPFBwTd554xdccEEq3rRpU7Dm8ccfD3I33nhjKr7qqquCNcXMd6D9OPLII4Nc9u+9a9euRV0r2zM644wzgjVr165tQ3Udj19YAwAAAAAQBQ1rAAAAAACioGENAAAAAEAUNKwBAAAAAIiCoYub0dzcnIrzhiduu+22Qe7DDz9MxePHj69sYURn1qxZVbv2H//4xyD31ltvBblPfOITqThv8Eet/eAHP0jF3/zmN+tUSdxGjRoV5Hr27FmHSqiVO+64I8hdcsklBV933333BbliBiOWOjyx1Nc9/PDDJb2OjqPUwaIGLLYfffr0Kbhm9erVQe6yyy6rRjl0QHl7Je/5KLvP/vCHP1StJhpT3gC+iRMnpuIZM2YEa84555xUfPvttwdrXnnllTKro60eeeSRIJcdutipU/i7zi9/+ctBbrfddkvFe+yxR0k1LV68uKTXUXujR48OcsUMWcwOCE6SJDnppJNS8WOPPVZ6YR2UX1gDAAAAABAFDWsAAAAAAKKgYQ0AAAAAQBScYb0Ze+21Vyru169fUa+78MILU/HcuXMrVhPV9fzzzwe5Aw88sA6V/H+HHHJIxa6VPX+t2PNps2d0P/PMM0W97sknnyyusAZ34oknBrnsWa9555Y/9NBDVauJ6rrzzjuD3IQJE4Jcjx49alHOZuWdP5u3F4877rhUvHDhwqrVRPuQ9/lS6pnoxClv/kLWBx98EOSWLVtWjXLogM4+++yi1uXNe8naZpttgtwOO+yQihcsWFBcYZCE34duuummYM3FF1+cin/+858Ha4444oggl/f8ReU899xzQS7793nooYcWda3Pfe5zBdfknYGe7TuMGzeuqPejtvI+O04//fSSrvW73/0uyP3mN78p6VqNxC+sAQAAAACIgoY1AAAAAABR0LAGAAAAACAKGtYAAAAAAETB0MUkSXbbbbcg99RTTxV83XXXXRfk7r333orURO0dfPDBQe76669PxV27di3p2oMGDQpyw4YNK+la//mf/xnk5s2bV/B1d999dyp+4YUXSnp/StezZ88gd+SRRxZ83dSpU4NcS0tLRWqi9ubPnx/kxo4dG+SyAznHjBlTtZry3HDDDUHuqquuqmkNtE/FDgzduHFjlSuhErp06RLkPvnJTxZ83YYNG4rKQTmy95Hzzz8/WPOd73wnyL322mupOG/4HRTr5ptvDnJnnHFGKj7ooIOCNfvuu2+Qe/bZZytXGIG8oZbZZ+zHHnssWLP77rsHuex3u+XLlwdrfvWrXwW5c845p2Cd1N7WW2+dihctWhSs6dSp8G9+33nnnSA3evTo0gtrYH5hDQAAAABAFDSsAQAAAACIgoY1AAAAAABR0LAGAAAAACAKhi4mSTJx4sQg17t374Kvyxt+19raWpGaiMNFF11U7xLoQNavXx/kVq5cGeTefPPNVHz55ZdXrSbi8MgjjxTM/fa3vw3WfPOb3wxygwcPTsWzZ88O1txyyy1BrqmpKRUb+kOpTjjhhCC3bt26IHfjjTfWohzKtGnTpiD3l7/8JcjtvPPOqTj7WQbVcPTRR28xTpIkmT59epA799xzq1YTjWfJkiVBLjtkMTvoM0mS5Nprrw1yw4cPr1xhFOXtt99OxYMGDQrWfOtb3wpyI0aMSMVnn312sCZvAB9xOv7441NxdghjkhTX78v7frZmzZrSC2tgfmENAAAAAEAUNKwBAAAAAIiChjUAAAAAAFFouDOsR40aFeTGjh1bh0qARrNhw4Ygt9tuu9WhEtqjKVOmFJWDeps3b16Q+8EPfhDkpk6dWotyKFNLS0uQO/3004PcnXfemYqffvrpqtVEx5d3Fmzeeb8zZsxIxZMnTw7WLF26NMjlzRWBSlqwYEEq/utf/xqsGTJkSJA74IADUvGcOXMqWxglufnmm4vK0X5dc801qbjY+XT33XdfKvZ8Wzl+YQ0AAAAAQBQ0rAEAAAAAiIKGNQAAAAAAUdCwBgAAAAAgCg03dHHEiBFBrmvXrgVf9+GHHxaVAwBoZPvvv3+9S6DKFi5cGOSOOuqoOlRCRzVt2rSictBeDBs2LMi9/vrrQW6fffZJxYYuQm306tUrFTc1NQVrVq1aFeQuu+yyqtXU6PzCGgAAAACAKGhYAwAAAAAQBQ1rAAAAAACioGENAAAAAEAUGm7oYrHefvvtVPz5z38+WLN06dJalQMAAAC0Q8uXLw9y2223XR0qAfLcdtttqXjixInBmhtuuCHILVq0qGo1NTq/sAYAAAAAIAoa1gAAAAAAREHDGgAAAACAKDTcGdYXXnhhUTkAAAAAoGP77ne/u8WY2vMLawAAAAAAoqBhDQAAAABAFDSsAQAAAACIgoY1AAAAAABRKHnoYmtrayXroMHYP5TD/qEc9g+lsncoh/1DOewfymH/UA77h3LYP5TKL6wBAAAAAIiChjUAAAAAAFFoasvP85uamt5PkuTN6pVDOzegtbV1x839of3DFtg7lMP+oRz2D+WwfyiH/UM57B/KYf9QDvuHcmxx//xDmxrWAAAAAABQLY4EAQAAAAAgChrWAAAAAABEQcMaAAAAAIAoaFgDAAAAABAFDWsAAAAAAKKgYQ0AAAAAQBQ6t2Vxnz59WgcOHFilUmjv5syZs7S1tXXHzf25/cPm2DuUw/6hHPYP5bB/KIf9QznsH8ph/1AO+4dyFNo//9CmhvXAgQOT5557rvSq6NCampre3NKf2z9sjr1DOewfymH/UA77h3LYP5TD/qEc9g/lsH8oR6H98w9talhn3qDUl9KBtLbRPazHAAAJiklEQVS2lvQ6+6fx5P2db9q0qaRrdeoUnmZUzF7Me529WFstLS0lva6S+8ffOUlS2udXly5dgtzGjRsLvs6eq79Sn1cqea3OncPH7kreE6meSj7v+uyiWPYPlVbqvazU7160X+4/VFop9wxnWAMAAAAAEAUNawAAAAAAoqBhDQAAAABAFDSsAQAAAACIQslDFwHaIoaBV3mDIvKGiBAfg12IQTEDFvPYvyRJ6QMW89hT7YO/J8ph/xALe7Hx+DsnBjo1AAAAAABEQcMaAAAAAIAoaFgDAAAAABAFZ1jT7m277bb1LqGhrFixIsi15zOumpub611CQ8k7R7w9cwZ6bdV7/zQ1NQW5Uu9/edeietrz51Qe+6e2Otr+AQCInW/aAAAAAABEQcMaAAAAAIAoaFgDAAAAABAFDWsAAAAAAKJg6CLt3ujRo+tdQkO56667gtzGjRuDXHsZUNS3b996l9BQFi9eXO8SKsrQ19r68MMPg1x7uddk9ezZs94lNJSPP/643iVUlP1TWx1t/wDUi4HltVXvgeWVZuh0bdX7e5a7BQAAAAAAUdCwBgAAAAAgChrWAAAAAABEQcMaAAAAAIAo1HzoYufO4Vtu2LCh1mU0tLxBPWvWrAly9T5gPU/eIfu33357HSppXA888ECQW7FiRZCLcf80NzcHubfeeqsOlTSuvPvP2rVrg1x2QEgMAza6dOkS5D744IM6VNK4dthhhyCXd/9paWmpyvvnDQrK7tW8Nd27dw9yK1eurFxhFNS1a9cgV+uBwXl7I/t+eWt69OgR5D766KPKFUZBW221VZDL+/5S72efvM/KvGcf371qK29Ac94gz2p9diVJ/t4o5v6Td+/M+95I9eQ9Q6xbty7I1fv+kyfv/pP32Uv17LrrrkHu/fffD3LV/Fwo9f6T99m7evXqyhVGQXnf3Wv5d+AX1gAAAAAAREHDGgAAAACAKGhYAwAAAAAQBQ1rAAAAAACioGENAAAAAEAUNKwBAAAAAIiChjUAAAAAAFHQsAYAAAAAIAoa1gAAAAAAREHDGgAAAACAKGhYAwAAAAAQBQ1rAAAAAACioGENAAAAAEAUNKwBAAAAAIiChjUAAAAAAFHQsAYAAAAAIAoa1gAAAAAAREHDGgAAAACAKGhYAwAAAAAQBQ1rAAAAAACioGENAAAAAEAUNKwBAAAAAIiChjUAAAAAAFHQsAYAAAAAIAoa1gAAAAAAREHDGgAAAACAKGhYAwAAAAAQBQ1rAAAAAACioGENAAAAAEAUNKwBAAAAAIiChjUAAAAAAFHQsAYAAAAAIAoa1gAAAAAAREHDGgAAAACAKGhYAwAAAAAQBQ1rAAAAAACioGENAAAAAEAUNKwBAAAAAIiChjUAAAAAAFHQsAYAAAAAIAoa1gAAAAAAREHDGgAAAACAKGhYAwAAAAAQBQ1rAAAAAACioGENAAAAAEAUNKwBAAAAAIiChjUAAAAAAFHQsAYAAAAAIAoa1gAAAAAAREHDGgAAAACAKGhYAwAAAAAQBQ1rAAAAAACioGENAAAAAEAUNKwBAAAAAIiChjUAAAAAAFHQsAYAAAAAIAoa1gAAAAAAREHDGgAAAACAKGhYAwAAAAAQBQ1rAAAAAACioGENAAAAAEAUNKwBAAAAAIiChjUAAAAAAFHQsAYAAAAAIAoa1gAAAAAAREHDGgAAAACAKGhYAwAAAAAQBQ1rAAAAAACioGENAAAAAEAUNKwBAAAAAIiChjUAAAAAAFHQsAYAAAAAIAoa1gAAAAAAREHDGgAAAACAKGhYAwAAAAAQBQ1rAAAAAACioGENAAAAAEAUNKwBAAAAAIiChjUAAAAAAFHQsAYAAAAAIAoa1gAAAAAAREHDGgAAAACAKGhYAwAAAAAQBQ1rAAAAAACioGENAAAAAEAUNKwBAAAAAIiChjUAAAAAAFHQsAYAAAAAIAoa1gAAAAAAREHDGgAAAACAKGhYAwAAAAAQhc61fsPW1tYgt2TJklqX0dA2bdpU7xJKlrd/mpqa6lAJ7VFLS0uQ22mnnepQSeNavXp1Sa/L+3e/1jZs2BDkunTpUodKGtfGjRvr+v55n5/ZvZl3n1m1alWQGzp0aOUKo6C8vVPr+0re+5W6f04++eTKFUZBMeyfYuTVlLenfv/739eiHP7X+vXr611CUfs17zMu79lnzpw5FamJ4rTn7+559x/PzrVV72fnYuXdo/Jqf/XVV2tRDluQ13+r1jORX1gDAAAAABAFDWsAAAAAAKKgYQ0AAAAAQBQ0rAEAAAAAiELNhy7m6d27d71LaCh5gw7WrVsX5AwzpBG899579S6Bdqy9DDKhMkodKJL3ulmzZpVbDm0Qw4C8UmvIG7j10EMPlVsObdCeh57l7buvf/3rdaikca1duzbIxXBPKkbe0Lxzzz23DpU0rvYy9LVYnp0bTzH7tdihi+PHj69ITRSn3p9ffmENAAAAAEAUNKwBAAAAAIiChjUAAAAAAFHQsAYAAAAAIAolD13MG9yXHcqQN6Akb3DD1772tVLLoAR5B6fnyR6mbggjscjbi+15+AjQPlTy3tOeh7hRmubm5iCX91xcjLxh2XRsnTqFvzMq9T7y9ttvl1sObRDDM2re/Se7f4qt85VXXqlITRQnhv0D5Sj1/pOXe+GFFypXGAXV+/7jF9YAAAAAAERBwxoAAAAAgChoWAMAAAAAEIWSz7Du1atXkFu5cmUqLvYcmkcffbTUMijBhg0bilqXPauze/fu1SgH2swZ1o2nW7duFbtW3gyGjRs3puJi91PemaJUT6nnteadnVeKvL2T/Ux1L+pY8v7OK3mtUs+Qtc/ah0o+O1dylkyx3wWor0o+Y+Tdf0o9C3/NmjXllkMblHq/94xKLPLuP+vXr0/Fxe7zYuexURn1vv+4iwEAAAAAEAUNawAAAAAAoqBhDQAAAABAFDSsAQAAAACIQslDF5ctW1bJOmgwhgVRjpaWlnqXQDuWHfIBxSp1QBUkiUFllCc7HBjawv2HcvjuTjncfyiVX1gDAAAAABAFDWsAAAAAAKLQ1Jb/e0dTU9P7SZK8Wb1yaOcGtLa27ri5P7R/2AJ7h3LYP5TD/qEc9g/lsH8oh/1DOewfymH/UI4t7p9/aFPDGgAAAAAAqsWRIAAAAAAAREHDGgAAAACAKGhYAwAAAAAQBQ1rAAAAAACioGENAAAAAEAUNKwBAAAAAIiChjUAAAAAAFHQsAYAAAAAIAoa1gAAAAAAROF/AKP9SbB/prZZAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1c2e8555f8>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "\n",
    "fig, axes = plt.subplots(nrows=2, ncols=10, sharex=True, sharey=True, figsize=(20,4))\n",
    "in_imgs = mnist.test.images[:10]\n",
    "reconstructed = sess.run(decoded, feed_dict={inputs_: in_imgs.reshape((10, 28, 28, 1))})\n",
    "\n",
    "for images, row in zip([in_imgs, reconstructed], axes):\n",
    "    for img, ax in zip(images, row):\n",
    "        ax.imshow(img.reshape((28, 28)), cmap='Greys_r')\n",
    "        ax.get_xaxis().set_visible(False)\n",
    "        ax.get_yaxis().set_visible(False)\n",
    "\n",
    "\n",
    "fig.tight_layout(pad=0.1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'1.8.0'"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf.__version__\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Benchmarking\n",
    "\n",
    "| Initializer | optimizer |presion type | epochs | Wall time | loss | Norm\n",
    "|---|---|---|---|\n",
    "|xavier|ADAM|float32|5 epochs|5.50 mins| 0.2372 | No\n",
    "|xavier|ADAM|float32|5 epochs|5.55 mins| 0.2350 | weight\n",
    "|xavier|ADAM|float32|5 epochs|6.22 mins| 0.2850 | wn data dependent\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}