quadratic_feasibility.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 166,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import random\n",
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import time\n",
    "import IPython"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 167,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "\n",
    "def generate_trace_zero_constraints(dim, num_constraints):\n",
    "    constraints = []\n",
    "    for _ in range(num_constraints):\n",
    "        M = np.random.randn(dim,dim)\n",
    "        M = (np.transpose(M)+M)/2\n",
    "        average = 0\n",
    "        for i in range(dim):\n",
    "            average += M[i,i]\n",
    "        for i in range(dim):\n",
    "            M[i,i] = M[i,i]-average/dim           \n",
    "        constraints.append(M)\n",
    "    x = np.random.randn(dim,1)\n",
    "\n",
    "    percent_feasible = 0\n",
    "    for i in range(len(constraints)):\n",
    "        if np.matmul(np.transpose(x),np.matmul(constraints[i],x)) < 0:\n",
    "            constraints[i] = -1*constraints[i]\n",
    "        elif np.matmul(np.transpose(x),np.matmul(constraints[i],x)) ==0:\n",
    "            print('Oh no! Something evaluates to 0!')\n",
    "    \n",
    "    return constraints, percent_feasible"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 163,
   "metadata": {},
   "outputs": [],
   "source": [
    "def find_feasible_point(dim, num_constraints, scale, batch_size, initial_step, decay_rate):\n",
    "    #get matrices\n",
    "    constraints, percent_feasible = generate_trace_zero_constraints(dim, num_constraints)     \n",
    "    \n",
    "    #learned feasible point\n",
    "    X = tf.Variable(tf.random_normal([dim, 1], 0., scale))\n",
    "    stacked_matrices = tf.placeholder(tf.float32, [None, dim])\n",
    "    exact_batch_size = tf.placeholder(tf.int32)\n",
    "\n",
    "    #tf.reduce sum is for inner product x & Px\n",
    "    \n",
    "    quadratic = tf.reduce_sum(tf.reshape(tf.multiply(tf.transpose(tf.tile(tf.transpose(X), [1,exact_batch_size])), tf.matmul(stacked_matrices, X)),[exact_batch_size,dim]),axis=1)\n",
    "    \n",
    "    #loss = tf.reduce_mean(tf.maximum(0., 1.-quadratic))#does coordinatewise?\n",
    "    loss = tf.reduce_mean(tf.log(1+tf.exp(-1*quadratic)))\n",
    "    sess = tf.InteractiveSession()\n",
    "\n",
    "    global_step = tf.Variable(1.0, trainable=False)\n",
    "    learning_rate = tf.train.exponential_decay(initial_step, global_step, num_constraints, decay_rate)\n",
    "    #learning_rate = tf.train.inverse_time_decay(initial_step, global_step, num_constraints, decay_rate, staircase=True)\n",
    "    train_step = tf.train.AdamOptimizer(learning_rate).minimize(loss, global_step=global_step)\n",
    "    var_grad = tf.gradients(loss,X)[0]   \n",
    "    correct = tf.less(tf.zeros([1,exact_batch_size]),quadratic)\n",
    "    accuracy = tf.reduce_mean(tf.cast(correct, tf.float32))\n",
    "    \n",
    "    #for shuffling\n",
    "    indices = np.random.permutation(num_constraints)\n",
    "    \n",
    "    tf.global_variables_initializer().run() \n",
    "    iteration = 0\n",
    "\n",
    "    initials = sess.run([accuracy,loss],\n",
    "                          feed_dict={stacked_matrices: np.concatenate([constraints[i] for i in range(num_constraints)]),\n",
    "                                    exact_batch_size:num_constraints})\n",
    "    old_loss = []\n",
    "    old_accuracy = []\n",
    "    old_loss.append(initials[1])\n",
    "    old_accuracy.append(initials[0])\n",
    "    for _ in range(8000):\n",
    "        #print('iter {}'.format(_))\n",
    "        x = sess.run([accuracy, loss, var_grad, learning_rate,X],\n",
    "                              feed_dict={stacked_matrices: np.concatenate([constraints[i] for i in range(num_constraints)]),\n",
    "                                        exact_batch_size:num_constraints})\n",
    "        #print('accuracy', x[0], 'loss', x[1], 'gradient', np.linalg.norm(x[2],ord='fro'), 'learning_rate', x[3])\n",
    "        \n",
    "        iteration += 1\n",
    "        IPython.display.clear_output(True)\n",
    "        plt.plot(np.log(old_loss))\n",
    "        angle = np.matmul(x[4].T,x[2])/(np.linalg.norm(x[4])*np.linalg.norm(x[2]))\n",
    "        plt.title('accuracy:{:2f} loss:{:2f} grad:{:2f} step:{:2f} norm:{:2f} angle:{:2f}'.format(x[0], x[1], np.linalg.norm(x[2],ord='fro'), x[3], np.linalg.norm(x[4]), angle[0][0]))\n",
    "        plt.show()\n",
    "        curr_x = x[4]\n",
    "        gg = (np.dot(sum([-2/(1+np.exp(curr_x.T@Pi@curr_x))*Pi for Pi in constraints]),curr_x)/num_constraints)\n",
    "        #gg = gg/np.linalg.norm(gg)\n",
    "        print([gg])#, x[4]/np.linalg.norm(x[4])])\n",
    "        #print(x[4])\n",
    "        print(percent_feasible)\n",
    "        \n",
    "        old_loss.append(x[1])\n",
    "        old_accuracy.append(x[0])\n",
    "\n",
    "#         if x[0] == 1 or x[1]==0 :\n",
    "#             break\n",
    "\n",
    "\n",
    "        for i in range(1, int(np.round(num_constraints/batch_size))):\n",
    "            if len(constraints[batch_size*(i-1): batch_size*i]) > 0:\n",
    "                sess.run([train_step],\n",
    "                          feed_dict={stacked_matrices: np.concatenate([constraints[j] for j in indices[batch_size*(i-1): batch_size*i]]),\n",
    "                                    exact_batch_size: batch_size})\n",
    "            else:\n",
    "                sess.run([train_step],\n",
    "                          feed_dict={stacked_matrices: np.concatenate([constraints[j] for j in indices[batch_size*(i-1):]]),\n",
    "                                    exact_batch_size: len(indicies[batch_size*(i-1):])})\n",
    "        indices = np.random.permutation(num_constraints)\n",
    "        #shuffle samples        \n",
    "    x = sess.run([X,correct],\n",
    "                          feed_dict={stacked_matrices: np.concatenate([constraints[i] for i in range(num_constraints)]),\n",
    "                                    exact_batch_size:num_constraints})\n",
    "\n",
    "    return old_loss, old_accuracy, iteration"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 168,
   "metadata": {},
   "outputs": [
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-168-1ea5c81fcb1a>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0mt\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtime\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0m_\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 4\u001b[0;31m     \u001b[0;34m[\u001b[0m\u001b[0mmatrices\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpoint\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfind_feasible_point\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m100\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m1e-3\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m10\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m.001\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m.99\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      5\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtime\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0mt\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-163-dfb1f60b113c>\u001b[0m in \u001b[0;36mfind_feasible_point\u001b[0;34m(dim, num_constraints, scale, batch_size, initial_step, decay_rate)\u001b[0m\n\u001b[1;32m     46\u001b[0m         \u001b[0miteration\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     47\u001b[0m         \u001b[0mIPython\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdisplay\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mclear_output\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 48\u001b[0;31m         \u001b[0mplt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mplot\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlog\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mold_loss\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     49\u001b[0m         \u001b[0mangle\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmatmul\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m4\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mT\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m/\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlinalg\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnorm\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m4\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlinalg\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnorm\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     50\u001b[0m         \u001b[0mplt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtitle\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'accuracy:{:2f} loss:{:2f} grad:{:2f} step:{:2f} norm:{:2f} angle:{:2f}'\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlinalg\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnorm\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mord\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'fro'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m3\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlinalg\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnorm\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m4\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mangle\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda/envs/tensorflow/lib/python3.5/site-packages/matplotlib/pyplot.py\u001b[0m in \u001b[0;36mplot\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m   3305\u001b[0m \u001b[0;34m@\u001b[0m\u001b[0m_autogen_docstring\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mAxes\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mplot\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3306\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mplot\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 3307\u001b[0;31m     \u001b[0max\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mgca\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   3308\u001b[0m     \u001b[0;31m# Deprecated: allow callers to override the hold state\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3309\u001b[0m     \u001b[0;31m# by passing hold=True|False\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda/envs/tensorflow/lib/python3.5/site-packages/matplotlib/pyplot.py\u001b[0m in \u001b[0;36mgca\u001b[0;34m(**kwargs)\u001b[0m\n\u001b[1;32m    948\u001b[0m     \u001b[0mmatplotlib\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfigure\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mFigure\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgca\u001b[0m \u001b[0;34m:\u001b[0m \u001b[0mThe\u001b[0m \u001b[0mfigure\u001b[0m\u001b[0;31m'\u001b[0m\u001b[0ms\u001b[0m \u001b[0mgca\u001b[0m \u001b[0mmethod\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    949\u001b[0m     \"\"\"\n\u001b[0;32m--> 950\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0mgcf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgca\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    951\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    952\u001b[0m \u001b[0;31m# More ways of creating axes:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda/envs/tensorflow/lib/python3.5/site-packages/matplotlib/figure.py\u001b[0m in \u001b[0;36mgca\u001b[0;34m(self, **kwargs)\u001b[0m\n\u001b[1;32m   1366\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1367\u001b[0m         \u001b[0;31m# no axes found, so create one which spans the figure\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1368\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0madd_subplot\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1369\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1370\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0msca\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0ma\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda/envs/tensorflow/lib/python3.5/site-packages/matplotlib/figure.py\u001b[0m in \u001b[0;36madd_subplot\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1018\u001b[0m                     \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_axstack\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mremove\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0max\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1019\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1020\u001b[0;31m             \u001b[0ma\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msubplot_class_factory\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mprojection_class\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1021\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1022\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_axstack\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0madd\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0ma\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda/envs/tensorflow/lib/python3.5/site-packages/matplotlib/axes/_subplots.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, fig, *args, **kwargs)\u001b[0m\n\u001b[1;32m     71\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     72\u001b[0m         \u001b[0;31m# _axes_class is set in the subplot_class_factory\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 73\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_axes_class\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__init__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfig\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfigbox\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     74\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     75\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__reduce__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda/envs/tensorflow/lib/python3.5/site-packages/matplotlib/axes/_base.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, fig, rect, facecolor, frameon, sharex, sharey, label, xscale, yscale, axisbg, **kwargs)\u001b[0m\n\u001b[1;32m    527\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    528\u001b[0m         \u001b[0;31m# this call may differ for non-sep axes, e.g., polar\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 529\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_init_axis\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    530\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0maxisbg\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mfacecolor\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    531\u001b[0m             raise TypeError('Both axisbg and facecolor are not None. '\n",
      "\u001b[0;32m/anaconda/envs/tensorflow/lib/python3.5/site-packages/matplotlib/axes/_base.py\u001b[0m in \u001b[0;36m_init_axis\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    620\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_init_axis\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    621\u001b[0m         \u001b[0;34m\"move this out of __init__ because non-separable axes don't use it\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 622\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mxaxis\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmaxis\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mXAxis\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    623\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mspines\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'bottom'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mregister_axis\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mxaxis\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    624\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mspines\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'top'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mregister_axis\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mxaxis\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda/envs/tensorflow/lib/python3.5/site-packages/matplotlib/axis.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, axes, pickradius)\u001b[0m\n\u001b[1;32m    674\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_minor_tick_kw\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    675\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 676\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcla\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    677\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_set_scale\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'linear'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    678\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda/envs/tensorflow/lib/python3.5/site-packages/matplotlib/axis.py\u001b[0m in \u001b[0;36mcla\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    758\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_set_artist_props\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlabel\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    759\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 760\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreset_ticks\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    761\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    762\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconverter\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda/envs/tensorflow/lib/python3.5/site-packages/matplotlib/axis.py\u001b[0m in \u001b[0;36mreset_ticks\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    772\u001b[0m         \u001b[0mcbook\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpopall\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mminorTicks\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    773\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 774\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmajorTicks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mextend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_get_tick\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmajor\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    775\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mminorTicks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mextend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_get_tick\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmajor\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    776\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_lastNumMajorTicks\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda/envs/tensorflow/lib/python3.5/site-packages/matplotlib/axis.py\u001b[0m in \u001b[0;36m_get_tick\u001b[0;34m(self, major)\u001b[0m\n\u001b[1;32m   1727\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1728\u001b[0m             \u001b[0mtick_kw\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_minor_tick_kw\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1729\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mXTick\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0maxes\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m''\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmajor\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mmajor\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mtick_kw\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1730\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1731\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_get_label\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda/envs/tensorflow/lib/python3.5/site-packages/matplotlib/axis.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, axes, loc, label, size, width, color, tickdir, pad, labelsize, labelcolor, zorder, gridOn, tick1On, tick2On, label1On, label2On, major)\u001b[0m\n\u001b[1;32m    148\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mapply_tickdir\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtickdir\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    149\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 150\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtick1line\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_get_tick1line\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    151\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtick2line\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_get_tick2line\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    152\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgridline\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_get_gridline\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda/envs/tensorflow/lib/python3.5/site-packages/matplotlib/axis.py\u001b[0m in \u001b[0;36m_get_tick1line\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    419\u001b[0m                           \u001b[0mmarkersize\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_size\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    420\u001b[0m                           markeredgewidth=self._width, zorder=self._zorder)\n\u001b[0;32m--> 421\u001b[0;31m         \u001b[0ml\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mset_transform\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0maxes\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_xaxis_transform\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mwhich\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'tick1'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    422\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_set_artist_props\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ml\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    423\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0ml\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda/envs/tensorflow/lib/python3.5/site-packages/matplotlib/lines.py\u001b[0m in \u001b[0;36mset_transform\u001b[0;34m(self, t)\u001b[0m\n\u001b[1;32m    757\u001b[0m         \u001b[0mACCEPTS\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0ma\u001b[0m \u001b[0;34m:\u001b[0m\u001b[0;32mclass\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;31m`\u001b[0m\u001b[0mmatplotlib\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtransforms\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTransform\u001b[0m\u001b[0;31m`\u001b[0m \u001b[0minstance\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    758\u001b[0m         \"\"\"\n\u001b[0;32m--> 759\u001b[0;31m         \u001b[0mArtist\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mset_transform\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mt\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    760\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_invalidx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    761\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_invalidy\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda/envs/tensorflow/lib/python3.5/site-packages/matplotlib/artist.py\u001b[0m in \u001b[0;36mset_transform\u001b[0;34m(self, t)\u001b[0m\n\u001b[1;32m    348\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_transform\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mt\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    349\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_transformSet\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 350\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpchanged\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    351\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstale\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    352\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda/envs/tensorflow/lib/python3.5/site-packages/matplotlib/artist.py\u001b[0m in \u001b[0;36mpchanged\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    329\u001b[0m         \u001b[0mregistered\u001b[0m \u001b[0mcallbacks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    330\u001b[0m         \"\"\"\n\u001b[0;32m--> 331\u001b[0;31m         \u001b[0;32mfor\u001b[0m \u001b[0moid\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfunc\u001b[0m \u001b[0;32min\u001b[0m \u001b[0msix\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0miteritems\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_propobservers\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    332\u001b[0m             \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    333\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda/envs/tensorflow/lib/python3.5/site-packages/six.py\u001b[0m in \u001b[0;36miteritems\u001b[0;34m(d, **kw)\u001b[0m\n\u001b[1;32m    579\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    580\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0miteritems\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0md\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkw\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 581\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0miter\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0md\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mitems\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m**\u001b[0m\u001b[0mkw\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    582\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    583\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0miterlists\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0md\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkw\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "#find_feasible_point          (dim, num_constraints, scale, batch_size, initial_step, decay_rate)\n",
    "t = time.time()\n",
    "for _ in range(1):\n",
    "    [matrices, point] = find_feasible_point(5, 100, 1e-3, 10,.001,.99)\n",
    "print(time.time()-t)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "num_exp = 8001\n",
    "average_loss =np.zeros(num_exp)\n",
    "average_accuracy= np.zeros(num_exp)\n",
    "n = 20.0\n",
    "std_dev_loss = np.zeros(num_exp)\n",
    "std_dev_accuracy = np.zeros(num_exp)\n",
    "\n",
    "for loss in losses:\n",
    "    for i in range(num_exp):\n",
    "        if i < len(loss):\n",
    "            average_loss[i] += np.log(loss[i])\n",
    "        else:\n",
    "            average_loss[i]+= np.log(loss[len(loss)-1])\n",
    "\n",
    "average_loss = average_loss / n\n",
    "\n",
    "for loss in losses:\n",
    "    for i in range(num_exp):\n",
    "        if i < len(loss):\n",
    "            std_dev = (np.log(loss[i])-average_loss[i])*(np.log(loss[i])-average_loss[i])\n",
    "            std_dev_loss[i]+=std_dev\n",
    "        else:\n",
    "            std_dev = (np.log(loss[len(loss)-1])-average_loss[i])*(np.log(loss[len(loss)-1])-average_loss[i])\n",
    "            std_dev_loss[i]+=std_dev\n",
    "\n",
    "\n",
    "for accuracy in accuracies:\n",
    "    for i in range(num_exp):\n",
    "        if i < len(accuracy):\n",
    "            average_accuracy[i] += accuracy[i]\n",
    "        else:\n",
    "            average_accuracy[i] += accuracy[len(accuracy)-1]\n",
    "\n",
    "average_accuracy = average_accuracy / n\n",
    "\n",
    "for accuracy in accuracies:\n",
    "    for i in range(len(accuracy)):\n",
    "        std_dev= (accuracy[i]-average_accuracy[i])*(accuracy[i]-average_accuracy[i])\n",
    "        std_dev_accuracy[i]+=std_dev\n",
    "    for i in range(num_exp-len(accuracy)):\n",
    "        std_dev = (accuracy[len(accuracy)-1]-average_accuracy[len(accuracy)+i])*(accuracy[len(accuracy)-1]-average_accuracy[len(accuracy)+i])\n",
    "        std_dev_accuracy[i+len(accuracy)]+=std_dev\n",
    "\n",
    "for i in range(len(std_dev_loss)):\n",
    "    std_dev_loss[i] = np.sqrt(std_dev_loss[i]/n)\n",
    "    std_dev_accuracy[i] = np.sqrt(std_dev_accuracy[i]/n)\n",
    "    \n",
    "#plt.errorbar([i for i in range(len(average_accuracy[50:1431]))],average_accuracy[50:1431], yerr=std_dev_accuracy[50:1431], fmt='o')\n",
    "lower = 50\n",
    "step = 50\n",
    "upper = 700\n",
    "x = [i for i in range(len(average_accuracy))]\n",
    "plt.plot(x[lower:upper] , average_accuracy[lower:upper])\n",
    "plt.errorbar(x[lower:upper:step], average_accuracy[lower:upper:step], yerr=std_dev_accuracy[lower:upper:step],linestyle = 'None')\n",
    "\n",
    "\n",
    "#plt.plot(average_accuracy[100:527])\n",
    "plt.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}