HW2.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "HW2.ipynb",
      "version": "0.3.2",
      "provenance": [],
      "collapsed_sections": [
        "SVIZ21hy7B0J",
        "a3kRKMrm66Tp",
        "o6_Nn7BV3jFN",
        "R9pnxEU_3vYI",
        "FPIEUKcQACOo",
        "6iPE7HEgouD9"
      ]
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "metadata": {
        "id": "SVIZ21hy7B0J",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "# Kernel setup (RAM boost + install Keras 2.1.0)"
      ]
    },
    {
      "metadata": {
        "id": "VKfqHi9x9it6",
        "colab_type": "code",
        "outputId": "16c738a7-5929-47d3-8faf-9d82ca094d6b",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 102
        }
      },
      "cell_type": "code",
      "source": [
        "!pip install  keras==2.1.0"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: keras==2.1.0 in /usr/local/lib/python3.6/dist-packages (2.1.0)\n",
            "Requirement already satisfied: scipy>=0.14 in /usr/local/lib/python3.6/dist-packages (from keras==2.1.0) (1.1.0)\n",
            "Requirement already satisfied: six>=1.9.0 in /usr/local/lib/python3.6/dist-packages (from keras==2.1.0) (1.11.0)\n",
            "Requirement already satisfied: pyyaml in /usr/local/lib/python3.6/dist-packages (from keras==2.1.0) (3.13)\n",
            "Requirement already satisfied: numpy>=1.9.1 in /usr/local/lib/python3.6/dist-packages (from keras==2.1.0) (1.14.6)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "OLA9jOEo9k0s",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "e48f8e6f-8032-4baa-c43e-4c2608d0b42f"
      },
      "cell_type": "code",
      "source": [
        "import keras"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Using TensorFlow backend.\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "metadata": {
        "id": "emi9UmE19o33",
        "colab_type": "code",
        "outputId": "5d85d304-5480-48b9-9226-c2ff777c8fb4",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "cell_type": "code",
      "source": [
        "keras.__version__"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'2.1.0'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 3
        }
      ]
    },
    {
      "metadata": {
        "id": "Los2kYBFFPxt",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "#this will clear all the GPU RAM\n",
        "#!pkill -9 -f ipykernel_launcher"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "4dLuLmu5Ew00",
        "colab_type": "code",
        "outputId": "b1eee38e-9cbd-47a5-fa6a-c0ac762ee139",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 357
        }
      },
      "cell_type": "code",
      "source": [
        "!ln -sf /opt/bin/nvidia-smi /usr/bin/nvidia-smi\n",
        "!pip install gputil\n",
        "!pip install psutil\n",
        "!pip install humanize\n",
        "import psutil\n",
        "import humanize\n",
        "import os\n",
        "import GPUtil as GPU\n",
        "GPUs = GPU.getGPUs()\n",
        "# XXX: only one GPU on Colab and isn’t guaranteed\n",
        "gpu = GPUs[0]\n",
        "def printm():\n",
        "  process = psutil.Process(os.getpid())\n",
        "  print(\"Gen RAM Free: \" + humanize.naturalsize( psutil.virtual_memory().available ), \" | Proc size: \" + humanize.naturalsize( process.memory_info().rss))\n",
        "  print(\"GPU RAM Free: {0:.0f}MB | Used: {1:.0f}MB | Util {2:3.0f}% | Total {3:.0f}MB\".format(gpu.memoryFree, gpu.memoryUsed, gpu.memoryUtil*100, gpu.memoryTotal))\n",
        "printm()"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting gputil\n",
            "  Downloading https://files.pythonhosted.org/packages/45/99/837428d26b47ebd6b66d6e1b180e98ec4a557767a93a81a02ea9d6242611/GPUtil-1.3.0.tar.gz\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from gputil) (1.14.6)\n",
            "Building wheels for collected packages: gputil\n",
            "  Running setup.py bdist_wheel for gputil ... \u001b[?25l-\b \bdone\n",
            "\u001b[?25h  Stored in directory: /root/.cache/pip/wheels/17/0f/04/b79c006972335e35472c0b835ed52bfc0815258d409f560108\n",
            "Successfully built gputil\n",
            "Installing collected packages: gputil\n",
            "Successfully installed gputil-1.3.0\n",
            "Requirement already satisfied: psutil in /usr/local/lib/python3.6/dist-packages (5.4.8)\n",
            "Collecting humanize\n",
            "  Downloading https://files.pythonhosted.org/packages/8c/e0/e512e4ac6d091fc990bbe13f9e0378f34cf6eecd1c6c268c9e598dcf5bb9/humanize-0.5.1.tar.gz\n",
            "Building wheels for collected packages: humanize\n",
            "  Running setup.py bdist_wheel for humanize ... \u001b[?25l-\b \bdone\n",
            "\u001b[?25h  Stored in directory: /root/.cache/pip/wheels/69/86/6c/f8b8593bc273ec4b0c653d3827f7482bb2001a2781a73b7f44\n",
            "Successfully built humanize\n",
            "Installing collected packages: humanize\n",
            "Successfully installed humanize-0.5.1\n",
            "Gen RAM Free: 12.9 GB  | Proc size: 298.9 MB\n",
            "GPU RAM Free: 11441MB | Used: 0MB | Util   0% | Total 11441MB\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "a3kRKMrm66Tp",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "# 1. Autoencoder"
      ]
    },
    {
      "metadata": {
        "id": "Kv6ul_T-9r1N",
        "colab_type": "code",
        "outputId": "d560348d-6964-4719-ea6f-43e7ad645607",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "cell_type": "code",
      "source": [
        "from keras.utils.conv_utils import normalize_data_format\n",
        "from keras.datasets import mnist\n",
        "import numpy as np\n",
        "from keras.layers import Input, Dense, Conv2D, MaxPooling2D, UpSampling2D\n",
        "from keras.models import Model\n",
        "from keras import backend as K\n",
        "from keras.callbacks import TensorBoard\n",
        "import io, gzip, requests, gc\n",
        "from keras.utils import np_utils\n",
        "from keras.applications import VGG16\n",
        "from keras.preprocessing import image\n",
        "import numpy as np\n",
        "from keras.datasets import mnist\n",
        "from keras.models import Sequential\n",
        "from keras.layers import Dense, Flatten, Conv2D, MaxPooling2D, Dropout\n",
        "from keras.utils import np_utils\n",
        "import matplotlib.pyplot as plt\n",
        "import io, gzip, requests, gc\n",
        "from keras.applications.vgg16 import preprocess_input\n",
        "from keras.preprocessing.image import img_to_array, array_to_img\n",
        "plt.viridis()\n",
        "from keras import models\n",
        "from keras.models import Model\n",
        "from keras import layers\n",
        "from keras import optimizers\n",
        "from keras import callbacks\n",
        "from keras.layers.advanced_activations import LeakyReLU"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<matplotlib.figure.Figure at 0x7f075bd494e0>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "metadata": {
        "id": "WqvMN7Mt3QqU",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "#Prepare your data:\n",
        "\n",
        "#For MINIST\n",
        "#(x_train, _), (x_test, _) = mnist.load_data()\n",
        "#x_train = x_train.astype('float32') / 255.\n",
        "#x_test = x_test.astype('float32') / 255.\n",
        "#x_train = np.reshape(x_train, (len(x_train), 28, 28, 1))  # adapt this if using `channels_first` image data format\n",
        "#x_test = np.reshape(x_test, (len(x_test), 28, 28, 1))\n",
        "\n",
        "train_image_url = \"http://fashion-mnist.s3-website.eu-central-1.amazonaws.com/train-images-idx3-ubyte.gz\"\n",
        "train_label_url = \"http://fashion-mnist.s3-website.eu-central-1.amazonaws.com/train-labels-idx1-ubyte.gz\"\n",
        "test_image_url = \"http://fashion-mnist.s3-website.eu-central-1.amazonaws.com/t10k-images-idx3-ubyte.gz\"\n",
        "test_label_url = \"http://fashion-mnist.s3-website.eu-central-1.amazonaws.com/t10k-labels-idx1-ubyte.gz\"\n",
        "\n",
        "def readRemoteGZipFile(url, isLabel=True):\n",
        "    response=requests.get(url, stream=True)\n",
        "    gzip_content = response.content\n",
        "    fObj = io.BytesIO(gzip_content)\n",
        "    content = gzip.GzipFile(fileobj=fObj).read()\n",
        "    if isLabel:\n",
        "        offset=8\n",
        "    else:\n",
        "        offset=16\n",
        "    result = np.frombuffer(content, dtype=np.uint8, offset=offset)    \n",
        "    return(result)\n",
        "\n",
        "train_labels = readRemoteGZipFile(train_label_url, isLabel=True)\n",
        "train_images_raw = readRemoteGZipFile(train_image_url, isLabel=False)\n",
        "test_labels = readRemoteGZipFile(test_label_url, isLabel=True)\n",
        "test_images_raw = readRemoteGZipFile(test_image_url, isLabel=False)\n",
        "\n",
        "train_images = train_images_raw.reshape(len(train_labels), 784)\n",
        "test_images = test_images_raw.reshape(len(test_labels), 784)\n",
        "Y_train = train_labels\n",
        "Y_test = test_labels\n",
        "\n",
        "X_train = train_images.reshape(train_images.shape[0], 28, 28, 1).astype('float32')\n",
        "X_test = test_images.reshape(test_images.shape[0], 28, 28, 1).astype('float32')\n",
        "\n",
        "X_train /= 255\n",
        "X_test /= 255\n",
        "\n",
        "X_train *= 2.\n",
        "X_test *= 2.\n",
        "\n",
        "Y_train = np_utils.to_categorical(Y_train, 10)\n",
        "Y_test = np_utils.to_categorical(Y_test, 10)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "J5BqfiXn3RYj",
        "colab_type": "code",
        "outputId": "dfe1c1ba-5e4d-4cd1-cfd5-c4e78728efa0",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 88
        }
      },
      "cell_type": "code",
      "source": [
        "input_img = Input(shape=(28, 28, 1))  # adapt this if using `channels_first` image data format\n",
        "\n",
        "x = Conv2D(16, (3, 3), activation='relu', padding='same')(input_img)\n",
        "x = MaxPooling2D((2, 2), padding='same')(x)\n",
        "x = Conv2D(8, (3, 3), activation='relu', padding='same')(x)\n",
        "x = MaxPooling2D((2, 2), padding='same')(x)\n",
        "x = Conv2D(8, (3, 3), activation='relu', padding='same')(x)\n",
        "encoded = MaxPooling2D((2, 2), padding='same')(x)\n",
        "\n",
        "# at this point the representation is (4, 4, 8) i.e. 128-dimensional\n",
        "\n",
        "x = Conv2D(8, (3, 3), activation='relu', padding='same')(encoded)\n",
        "x = UpSampling2D((2, 2))(x)\n",
        "x = Conv2D(8, (3, 3), activation='relu', padding='same')(x)\n",
        "x = UpSampling2D((2, 2))(x)\n",
        "x = Conv2D(16, (3, 3), activation='relu')(x)\n",
        "x = UpSampling2D((2, 2))(x)\n",
        "decoded = Conv2D(1, (3, 3), activation='sigmoid', padding='same')(x)\n",
        "\n",
        "autoencoder = Model(input_img, decoded)\n",
        "autoencoder.compile(optimizer='adadelta', loss='binary_crossentropy')"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:1344: calling reduce_mean (from tensorflow.python.ops.math_ops) with keep_dims is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "keep_dims is deprecated, use keepdims instead\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "t1O2twb83Rb4",
        "colab_type": "code",
        "outputId": "0421fe3a-9caa-42bd-b280-4c01c46ebb77",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1751
        }
      },
      "cell_type": "code",
      "source": [
        "autoencoder.fit(X_train, X_train,\n",
        "                epochs=50,\n",
        "                batch_size=128,\n",
        "                shuffle=True,\n",
        "                validation_data=(X_test, X_test),\n",
        "                callbacks=[TensorBoard(log_dir='/tmp/autoencoder')],\n",
        "                verbose = 1)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Train on 60000 samples, validate on 10000 samples\n",
            "Epoch 1/50\n",
            "60000/60000 [==============================] - 10s 160us/step - loss: -1.3777 - val_loss: -1.7851\n",
            "Epoch 2/50\n",
            "60000/60000 [==============================] - 7s 114us/step - loss: -1.8505 - val_loss: -1.9135\n",
            "Epoch 3/50\n",
            "60000/60000 [==============================] - 7s 114us/step - loss: -1.9502 - val_loss: -1.9863\n",
            "Epoch 4/50\n",
            "60000/60000 [==============================] - 7s 114us/step - loss: -2.0057 - val_loss: -1.9980\n",
            "Epoch 5/50\n",
            "60000/60000 [==============================] - 7s 114us/step - loss: -2.0423 - val_loss: -2.0494\n",
            "Epoch 6/50\n",
            "60000/60000 [==============================] - 7s 114us/step - loss: -2.0664 - val_loss: -2.0379\n",
            "Epoch 7/50\n",
            "60000/60000 [==============================] - 7s 114us/step - loss: -2.0813 - val_loss: -2.0812\n",
            "Epoch 8/50\n",
            "60000/60000 [==============================] - 7s 115us/step - loss: -2.0956 - val_loss: -2.0989\n",
            "Epoch 9/50\n",
            "60000/60000 [==============================] - 7s 114us/step - loss: -2.1065 - val_loss: -2.0952\n",
            "Epoch 10/50\n",
            "60000/60000 [==============================] - 7s 115us/step - loss: -2.1138 - val_loss: -2.0873\n",
            "Epoch 11/50\n",
            "60000/60000 [==============================] - 7s 114us/step - loss: -2.1198 - val_loss: -2.1052\n",
            "Epoch 12/50\n",
            "60000/60000 [==============================] - 7s 114us/step - loss: -2.1274 - val_loss: -2.1189\n",
            "Epoch 13/50\n",
            "60000/60000 [==============================] - 7s 114us/step - loss: -2.1350 - val_loss: -2.1266\n",
            "Epoch 14/50\n",
            "60000/60000 [==============================] - 7s 114us/step - loss: -2.1403 - val_loss: -2.1259\n",
            "Epoch 15/50\n",
            "60000/60000 [==============================] - 7s 114us/step - loss: -2.1440 - val_loss: -2.1219\n",
            "Epoch 16/50\n",
            "60000/60000 [==============================] - 7s 114us/step - loss: -2.1510 - val_loss: -2.1357\n",
            "Epoch 17/50\n",
            "60000/60000 [==============================] - 7s 114us/step - loss: -2.1525 - val_loss: -2.1502\n",
            "Epoch 18/50\n",
            "60000/60000 [==============================] - 7s 114us/step - loss: -2.1560 - val_loss: -2.1228\n",
            "Epoch 19/50\n",
            "60000/60000 [==============================] - 7s 114us/step - loss: -2.1586 - val_loss: -2.1494\n",
            "Epoch 20/50\n",
            "60000/60000 [==============================] - 7s 114us/step - loss: -2.1642 - val_loss: -2.1459\n",
            "Epoch 21/50\n",
            "60000/60000 [==============================] - 7s 114us/step - loss: -2.1681 - val_loss: -2.1472\n",
            "Epoch 22/50\n",
            "60000/60000 [==============================] - 7s 114us/step - loss: -2.1693 - val_loss: -2.1635\n",
            "Epoch 23/50\n",
            "60000/60000 [==============================] - 7s 114us/step - loss: -2.1738 - val_loss: -2.1653\n",
            "Epoch 24/50\n",
            "60000/60000 [==============================] - 7s 114us/step - loss: -2.1755 - val_loss: -2.1607\n",
            "Epoch 25/50\n",
            "60000/60000 [==============================] - 7s 114us/step - loss: -2.1786 - val_loss: -2.1748\n",
            "Epoch 26/50\n",
            "60000/60000 [==============================] - 7s 114us/step - loss: -2.1800 - val_loss: -2.1759\n",
            "Epoch 27/50\n",
            "60000/60000 [==============================] - 7s 114us/step - loss: -2.1807 - val_loss: -2.1668\n",
            "Epoch 28/50\n",
            "60000/60000 [==============================] - 7s 114us/step - loss: -2.1831 - val_loss: -2.1677\n",
            "Epoch 29/50\n",
            "60000/60000 [==============================] - 7s 114us/step - loss: -2.1859 - val_loss: -2.1828\n",
            "Epoch 30/50\n",
            "60000/60000 [==============================] - 7s 114us/step - loss: -2.1873 - val_loss: -2.1669\n",
            "Epoch 31/50\n",
            "60000/60000 [==============================] - 7s 114us/step - loss: -2.1872 - val_loss: -2.1828\n",
            "Epoch 32/50\n",
            "60000/60000 [==============================] - 7s 115us/step - loss: -2.1888 - val_loss: -2.1874\n",
            "Epoch 33/50\n",
            "60000/60000 [==============================] - 7s 114us/step - loss: -2.1918 - val_loss: -2.1744\n",
            "Epoch 34/50\n",
            "60000/60000 [==============================] - 7s 115us/step - loss: -2.1945 - val_loss: -2.1917\n",
            "Epoch 35/50\n",
            "60000/60000 [==============================] - 7s 116us/step - loss: -2.1947 - val_loss: -2.1556\n",
            "Epoch 36/50\n",
            "60000/60000 [==============================] - 7s 114us/step - loss: -2.1995 - val_loss: -2.1857\n",
            "Epoch 37/50\n",
            "60000/60000 [==============================] - 7s 116us/step - loss: -2.1959 - val_loss: -2.1763\n",
            "Epoch 38/50\n",
            "60000/60000 [==============================] - 7s 117us/step - loss: -2.1991 - val_loss: -2.1825\n",
            "Epoch 39/50\n",
            "60000/60000 [==============================] - 7s 115us/step - loss: -2.2016 - val_loss: -2.1972\n",
            "Epoch 40/50\n",
            "60000/60000 [==============================] - 7s 114us/step - loss: -2.2041 - val_loss: -2.1995\n",
            "Epoch 41/50\n",
            "60000/60000 [==============================] - 7s 115us/step - loss: -2.2050 - val_loss: -2.1874\n",
            "Epoch 42/50\n",
            "60000/60000 [==============================] - 7s 116us/step - loss: -2.2037 - val_loss: -2.1964\n",
            "Epoch 43/50\n",
            "60000/60000 [==============================] - 7s 116us/step - loss: -2.2068 - val_loss: -2.2027\n",
            "Epoch 44/50\n",
            "60000/60000 [==============================] - 7s 116us/step - loss: -2.2126 - val_loss: -2.1915\n",
            "Epoch 45/50\n",
            "60000/60000 [==============================] - 7s 116us/step - loss: -2.2118 - val_loss: -2.2013\n",
            "Epoch 46/50\n",
            "60000/60000 [==============================] - 7s 116us/step - loss: -2.2124 - val_loss: -2.2021\n",
            "Epoch 47/50\n",
            "60000/60000 [==============================] - 7s 116us/step - loss: -2.2132 - val_loss: -2.2144\n",
            "Epoch 48/50\n",
            "60000/60000 [==============================] - 7s 116us/step - loss: -2.2178 - val_loss: -2.1881\n",
            "Epoch 49/50\n",
            "60000/60000 [==============================] - 7s 115us/step - loss: -2.2167 - val_loss: -2.2118\n",
            "Epoch 50/50\n",
            "60000/60000 [==============================] - 7s 115us/step - loss: -2.2169 - val_loss: -2.2185\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<keras.callbacks.History at 0x7fba3d06ddd8>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 29
        }
      ]
    },
    {
      "metadata": {
        "id": "ZWiX6YTh3Rec",
        "colab_type": "code",
        "outputId": "4f9884a7-fe74-4da2-ea74-73823e7209ba",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 244
        }
      },
      "cell_type": "code",
      "source": [
        "decoded_imgs = autoencoder.predict(X_test)\n",
        "\n",
        "n = 10\n",
        "plt.figure(figsize=(20, 4))\n",
        "for i in range(n):\n",
        "    # display original\n",
        "    ax = plt.subplot(2, n, i + 1)\n",
        "    plt.imshow(X_test[i].reshape(28, 28))\n",
        "    plt.gray()\n",
        "    ax.get_xaxis().set_visible(False)\n",
        "    ax.get_yaxis().set_visible(False)\n",
        "\n",
        "    # display reconstruction\n",
        "    ax = plt.subplot(2, n, i + 1 + n)\n",
        "    plt.imshow(decoded_imgs[i].reshape(28, 28))\n",
        "    plt.gray()\n",
        "    ax.get_xaxis().set_visible(False)\n",
        "    ax.get_yaxis().set_visible(False)\n",
        "plt.show()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAABGoAAADjCAYAAADdR/IFAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzt3Xn8XdO9//HltqXGEAkRkTkSRcwk\nQlRi7iU01PAoitLx6qja3t7b0unqgFavPrguVffRoi4tQqm0ERISIiRIZJJJZBJi7Oz3R3/Wfa+3\n71729+R8v999znk9//qc7P3dZ5+991p7n5P1WZ+N3nzzzTcDAAAAAAAAutw/dfUOAAAAAAAA4B/4\noQYAAAAAAKAi+KEGAAAAAACgIvihBgAAAAAAoCL4oQYAAAAAAKAi+KEGAAAAAACgIt6dW7jRRht1\n1n7A1LNqelXO45Zbbhnj/fffP1k2ceLEdm9v7733Tl6/+uqrMZ43b167t9cR6nUeO/Mc+nvpZxg7\ndmyMzz///GS9xx9/PMa9evWK8YIFC5L1tthiixhvs802ybK//OUvMR44cGCMTzjhhFL73hGapS32\n7Nkzxuedd16M169fn6z3xhtvtPn3vp4el3e9613Jso033jjGq1evjvGkSZOS9f785z+/w17XT2e3\nxX/6p/T/Qf7+97+X2kYt+zlixIjk9eabbx5jPRd+ntQmm2ySvF6zZk2MJ0+e3O596gjN0haVtom/\n/vWvybI//elPMX7ve98b48WLFyfr6bLtt98+Wab3RT3/fn1+4AMfKL/TG6jZ7ovK72kvvvhijAcN\nGhTjHj16JOv97W9/i/Ef//jHZNmTTz7Zvp3tBI3aFv261/fWc+DOOOOMGI8cOTLG7353+jVKz/ec\nOXOSZdddd12b2y57beX+rtbz0YhtEalGbYtIFZ1HRtQAAAAAAABUxEZvZn6K45e1rtNIv5Dq/+Z9\n9rOfTZadeuqpMdb/adL/3Q8hhNdffz3G3bt3L/W+/r9OOhJA/2fk/vvvT9a75pprYvzb3/621HvV\nqhH/tyI3EuCBBx6I8UEHHVRqey+//HLyerPNNoux/2+UXge63rHHHpusd+edd5Z673popLaY84lP\nfCLGl112WYzXrVuXrPf888/HWEc1LV++PFlv/vz5Md5ll12SZdo277vvvhjPmjUrWe+GG24ote/1\n0Nltsex6uf3SUYghhDBmzJgY64jCo48+OlnvmWeeaXP7OpothBC23XbbGK9duzZZtummm8ZYR2Lc\ncccdyXq33357jJcuXdrGp6ifZmmLW221VYwXLlwYYx195rQ/9D5a25uPCtA+VUdN+XvpaMmO1ij3\nRb3u/bjqe+vIp/e85z3Jenr8tU299NJLyXr6dz6y6r/+679i/KUvfanUvne0ZmmLRYYPH568fuKJ\nJ2I8derUGOvzUQjpufNnJH1Wzo3eqcdImbIapS2iWLO3xVbBiBoAAAAAAICK44caAAAAAACAiuCH\nGgAAAAAAgIrIVn0C2nLJJZckr7WCjM+poPPGaOzzYmjudlGVihDSKjGa+x1Cmrevufj//M//nKw3\nbty4GD/00EPJstGjR4dW5znXas8994yxn0Od4yI3D80LL7wQY8/F1/zYwYMHx3jYsGHJep05R02z\n2G677WKsVWNyufI6X423RZ3fROfcCCGdl6h3794xnjt3bvkdbnCeb1x23gHtT3feeedkmZ4DPZY3\n3XRTsp62U50/w9uizmXjc0lp/6pzivXr1y9Z79JLL23zb0II4ctf/nKMV6xYEfAPOleFXgt+fvR+\np7FWlgkhvS68Ler2td0XVXfD/8n1jSeffHKML7744hj73CYnnnhijH/wgx/EeK+99krWO+yww2Ks\n83qFEMKVV14ZY71GcvfPjp7bpFnos4VWTFu1alWy3gEHHBDjiy66KMbe3rQP/OhHP5os0+dLnb/G\nn6k7sxoigGpjRA0AAAAAAEBF8EMNAAAAAABARZD6hFJ0OL6Xh1y5cmWMNW0pZ+ONN05ea3lRjX34\nrqbleBnMou35Pulw5gMPPDBZpqVnvSQ00vK+Xs5XhwBrGpqmXoSQDtPXFLW21n3LTjvt1P6dRUJT\nldasWRNjLcEdQprSpqmM3o623nrrGHtJR/07bbOzZ89u7243LD8mRakIWjY9hPQ8aapKCCH85S9/\nibG2MS+1fP/998f4hBNOiLH21SGk7c33T8+Vlv+eN29est769etj7GlR3/rWt2J89tlnB/zD+PHj\nY9y9e/cYL1u2LFlP01xyfaou07Qq30a3bt1ivMMOOyTr7bPPPjGeMWNG/gMgSTt67rnnYqzXfAgh\n3HXXXTE+6qijYjxgwIDCbXuf4P1AEdKd2qbX9vHHH58s03YwZcqUGOv9LYQ0ZVtTRjWlOIQ09UlL\neoeQPvdqqqk/U0+aNCnGni7sz10AmhsjagAAAAAAACqCH2oAAAAAAAAqgh9qAAAAAAAAKoI5alDK\nN7/5zRh7GVedg8LLi/bq1avN7Xl5Ud2G5n5vvvnmyXqaf685wyGkc5/oPDQ+D4rOHeElGLV8Yo8e\nPWLcynnBWrJS6XwZIaT58Tpngpd11vPrpcB1G3qdeR442m/JkiUx3mOPPWLs50Bfa769lwzVc+xz\nn+i8G7peK5Xnzs1Ro3Mu9e3bN1lv0aJFMdY5odxrr70WY2+jCxcubHN7Q4YMSdbTPnT69OnJMu0L\ndQ4OnwNl0003jbGXfNb+//TTT4/xDTfckKzXamWFzznnnBg///zzMda5o0JI+z3tN/v06ZOsp+3U\n27PO16bb8Gtm//33j3Gzz1FTdL353Hl77713jH3OEn2uGDx4cIx33XXXZL1jjjkmxi+99FKM9byH\nEMLOO+9cuL9Dhw5t83295L3O2+fPNn5dNDMvdz1x4sQY+7Oc3pOeeuqpGPfv3z9Z74wzzoixtg+d\nryaEtH887rjjkmX33HNPjOfMmRPjESNGJOsdfvjhMR45cmSy7LbbbovxggULAoDmxogaAAAAAACA\niuCHGgAAAAAAgIog9QmlaFnPXGlQT3W68sorY3z11VfH2IdW6zBgHdb9yiuvJOstXbo0xp4Oo6kZ\nWnJx+fLlyXq6/1pSOoR0GL+WLW7l1KfddtutzX/31Cc9dpp6pnEI6fXiNE1Kz5OmoaE2OvR91qxZ\nMdYUmhDStIBBgwbFeJtttilcb/78+YXvq6k3mnrR7HKpBpoq4cdE00e9JLqmPWhbyZVO1/LA3/nO\nd5L1NFXJ01b1taZReDqq9qGeOqJteK+99oqxpz61QrqT0lQWvRdqHxpCmsqi/aa3WT/uSsuna+zX\nZ+/evd9pt5tG0fX2vve9L3m93377xdhTXLTP0zLMnpa25ZZbxlhLQ8+cOTNZT+9xfh3o+d52221j\nrP1ICOk92e/Pzf4Mo88pnnJ04YUXxthLnWv/q/cqX0/vf9ddd12M9TkxhPTc7bnnnsmyadOmxXiz\nzTaLsaewaaqpb+MLX/hCjL2MO4Dmw4gaAAAAAACAiuCHGgAAAAAAgIog9Qml6JB7rSIRwturm6iv\nfvWrMdZh114JSIeBTpo0KcaHHnpo4baffvrp5PUuu+wSYx2Of/755yfrfetb34qxV9nQ4eWjRo2K\nsVdEaSXDhw+PsaaX+XWg51CvF08vW7duXeF76bWk2/Ch/mg/He6v6YDejtSJJ54YYx1yH0Ja3WTy\n5MnJMk3n0GHcnqKh1WpaiR47b0depU5pOyiqchdC2uY0rfTee+9N1tNh/74NrSii7dLTWzVFyitC\nKU0jaTWaihtCesxWr14dY0/n1Tarfa9WDQshvYY8DU7Tp/R9/brzlOZW5Omd2gY85U/Pm7Y3r0ap\nKUf77rtvjLXKVgghPPnkkzHu2bNnskzTp7Ripr+XprN5+lSz02N71FFHJcvOOuusGGv6WQjp+dEK\nUJqeGEKaTqXn26tDaRv2Sl56zegyTTEOIT2vfn+eMGFCANA6GFEDAAAAAABQEfxQAwAAAAAAUBH8\nUAMAAAAAAFARzFGT4fOoaP5vrpyozjHged9aTlHzn6uoqOSnl/XMzanw85//PMbjxo0rXK979+4x\n1nlpLr744mS9l19+Ocannnpq4Tb69u0b45tuuilZT+eo8VLROk+DlpNtZZpLr+de56QJIZ3vQsu5\nP/bYY8l6Wm5S8+1DSNuLbn/ZsmXt3W2YOXPmxHjs2LFt/nsI6TnQ/Hifp+mqq66KsZ8fnQNHz7GW\ng25lWsJX5+4KId+f6hwH2j68tLbOZ6Lz4WhZ9hDSPtNLxGq5Zi33vf322yfr6Rw4+l4hhPDss8/G\nWOem8nuL7m8z8mNWNOeWz/em7UXniHr00UeT9bQ0sc+l8sorr8RY73deFt7nrGkVW2yxRYx1LpgQ\n0jbhzy+zZ8+OcW5uJp0zSOcL8jlktJy2P5fo86bO6+VzfGmf4PfnZjdmzJgYa78TQlo+XZ8hQ0jP\nj84T1K9fv2Q97ecmTpwYYy+Rrud49913T5bpnIjaJ6xatSpZz/tzpfcOLene7OXXgVbFiBoAAAAA\nAICK4IcaAAAAAACAimiq1CcdNuxDiDVlY8cdd0yWjRw5MsZ33313jGstCZwrczl+/PgYX3LJJTVt\nv7Po0HflqU+5MpB+rIucdNJJbf67pk6FkA7P9tQ0Hd6q5VC9XGlZQ4YMqenvmo2WPdfh2X4d6BBy\nHSY8YsSIZD0dxu1DvPW1Dv/NlfRGOToUXvs2L7fs6Whv8eHYmqLj51HbqaZYeIpAK5UE9vSXt2i7\nCSEtEeypStr+vP9T2jb1GHv5YU1B8numnm/tT/2c6TY0RcrpNTJ8+PBkmafyNBsv9attIvecoX2l\nngNPt5g5c2aMvSTw0qVLY6zXj5djb6W2qPSa9bRDTUnx9qtlmPUc5lLKNA1Nz0UIafvz+52m8mg7\n8n63qBR7CPmU/GagJbO9fL32L/psEkJ6XF566aUYe1+m51WnLdA07xDSdEVvi7qPep/1tMP7778/\nxvqdIYS07Ws6JKlPaCZlv8/XavTo0TGePHnyBm+vLE9NLvM7AyNqAAAAAAAAKoIfagAAAAAAACqC\nH2oAAAAAAAAqoqnmqFG5HLaDDz44eX3AAQfEWOdl+fGPf1zTe2vu8pFHHpks89KAVaal/3I0L9rz\nrnWOGs+nVpqTq+65557k9cCBA2P8wgsvJMuOOeaYGP/hD3+Isc5dE0I6Z43vk+Yh+9wdrUpzsPX4\n5OaoufXWW0tt2+fZ8HkT3lJUKh7laS6szlfj51H7QJ3nQOfBCCGdP8PnqdI+Qc+x9w+tZMCAATHW\nPsjnxdAcZj3GIaTltPUY58oDax/n7UvPfc+ePQu3ofvoc1/otaRzcPjfad+hxyKE5p+jZtiwYclr\nbYt6vn1+E50XJTcHxcMPPxzjPfbYI1mm51jPh19bzV4ivYje3/wY6JwlPneXHkudU8afKfT4azvV\nuUxCSOcp8fas/auW5NZnzRDStrl+/fpkmc6PomWim4WeK5+L6+ijj46xtyM9tjonkfdR/fv3bzPW\nOfxCSJ9L9Xk1hBD++7//O8Z6n/U2e8ghh8T4wAMPTJbp+fd7B9As9P7k96oi/p29b9++MX7ggQeS\nZWPHjo2xzgG2bNmy0vuo/a3fu9UFF1wQY5+PdcyYMe/4PoyoAQAAAAAAqAh+qAEAAAAAAKiIpkp9\n0iH2Pgxp3333jbEPVdThjlqS+bbbbkvW0+GtPtR/yZIlMdaSeTrcNIQQli9fXvwBKqZPnz5t/ruX\nSlM6LDOENH1IhwD7NrR86X/8x3/EeNCgQYXvNWfOnOS1Di/v169fjD/5yU8m62k5di+DqUOfy5YW\nb3Y6vFrPb2444i9/+cvCZVoaVFM5Qnh7OttbNL0CtdFzp20xV75elz3++OOF63l/qMP49Xy3cuqT\nDsPV45NLCdW/CSG9z2hf5SmE+lrPod8Xdfu+DV1Xz6GnPmnZaO//9Xxr7GVrm52X09a0FE3r9POj\n6RE/+9nPCrevKRUf//jHk2VFZdz9vYrSTpud9l2e+qTHxPs4TQ1fvXp1jP2+WHSf9OOv58n7BG1/\n+nde1jl3Dn3dZjNjxowYX3/99ckyTR/ylCZ9Xte+zNOnNLVbS3dvueWWyXp6Hn36AH2m1u8aXrJX\n01A9LVRTvPz5Fag67dvK9pVOUwqnT58eY//e8dhjj8XY+0b9rnHFFVfE+Pjjjy+1DyEUpzudfvrp\nyeuTTz45xt5feFp0WxhRAwAAAAAAUBH8UAMAAAAAAFARDZ/6pMOodBiSDyXUmZZ1GGkI6Qz7OizJ\n03P0vXzZrrvuGmOdNdorBfiw8SorqgLiVWJ0qKcPs9Zh99/+9rdjrBVLQgjhiCOOiLHOgL/bbrsl\n6+n58SFjmjJ10003xXjPPfds41O0vb/62XwfW5WmHen5zF3LWnXLPfTQQzHWNLQQiofpF6VEoTy9\ntjUNJTf8NJcWpVVLvCqXVrXRfrlV0ytCSNNY9Dh4JUCt5OGps3oOtf35cdV2pOfTq4Toel6xSYf+\na9qEp4Do/vtQfx2mr/fPXJ/cjPw8atvR8+N9qt6DLr/88sLta3qE35+LnpFyaT6tRJ///Bjoda8V\nuEJI24emsmkqTQjFqW3+Xno+cudQ25tWBwohrczn/XouZb1R6fPhKaecEmNPgdDP7s91eu70fudV\ns/Q8apx7TvTnFu0Py14Lv/3tb5NlOp3AoYceGuMbbrihcD+agbYrT1/T73uazjt79uxkvY997GMx\n1uO1YsWKZD099/4dTmm7zFUadno9lk33qRrvT/Rz5D5f7jhpu9LrXFOYQgjhRz/6UYy/973vxXjW\nrFnJelqdzX8TePrpp2N8+OGHx9jTCb/73e/G2KdD0TY8atSoGPt0G7qeVyF+7rnnwjthRA0AAAAA\nAEBF8EMNAAAAAABARfBDDQAAAAAAQEV06oQpuZw2L0eoyzT2OSyK8qq9ROXKlStj7GUKNY9N85W1\nbLe/t+fZ6ZwMml/quek6R4DnzOk2qkBLFapc/nQu//erX/1q4Xvpenrc3/e+9xX+jZ7TENI5dXKl\nKHPXU1H+ZNnrrpX4uS4q5+sWL14c44MOOihZVpRH7/niaL+1a9fGONf3ap5wrh1pPr+fN/07zcFt\nTx53s9HyrnqP8Bx4zbH/zW9+U7gNPYde9lzvMxp7m9W/8/lR9F6o582viblz58b4uOOOS5bpPupn\n1m23Aj8/eq/XY6TzgYWQ3uMWLVpU6r18Xgxtm3qt+XxCrXZO3qLtw8vL67HzZzk9N1quOTcng14H\nuWdev16K5oM78cQTk9fz5s2Lsc+70YznV/tDndPiIx/5SLLeMcccE+OLLrooWabHTJ89va/ccccd\nY6zz7Pmz4Jo1a2Ls810sWLCgzfW8FLjOhbHLLrsky3QORy1J3tVz1OTmQMrNw6LP1nosdf6dEEI4\n//zzYzxo0KBkmfabep9ZuHBhsp5eI/fff3+MP/3pTyfrHXbYYTH2e9rDDz8c47Lzrfh8YI06L43K\nfYbcMn/mV9o2tf/66Ec/mqyn19pOO+0U4/33379w2z63nm5jwoQJMfbvGvpbwllnnZUs02dgnUNJ\n56kNIe0vfB/9vtIWRtQAAAAAAABUBD/UAAAAAAAAVESHpD4VlebKDYfKDSErGhrnTj311BjrELcQ\n0vJePqRRh63qsGEftqhDhbVMtO+j8uGtOkRvyJAhybLHH3+8zW10laLy3E6H9U2cODFZNnr06Bgv\nX748xn4edZigDvP1krHKz6MORdZhvr4NHdrmZWKLykBrelwIbx9S2Sq0DfvxL3tM9DrIDf9GfT3/\n/PMx9nLaSvuoXOlRbaeetqklZIv6xlajKRZanllTBkNI759aQjKEEA4++OAY50qna/+q9zdPs9L2\n5vuh6Re5Ye2aOuCpO/p3mg6p+9QK/FmiqF1pKkcIby/NW4anBOuzVS7dolXbqfaF3j70+h06dGiy\nTFMANfY2UHRcc2nXfl8sausnnHBC8vqHP/xhjD3dwq+tZqD9o6bW33vvvcl6et2PHz8+WabPg/ps\n4t9JTjvttBhrGuLAgQOT9Xr37h1j7a9DSK8vTdnw7xPaL991113Jsj/84Q8x9vtDVfj1m/t+p/eq\nvffeO8af+9znkvWeeeaZGN90003JskcffTTGej415S2EEEaOHBljTafx9qUpcLfeemuy7Nlnn43x\nJZdcEuPbb789Wc/bXysZPHhwjP1er9/Thw0bliz71re+FWOdGsS/z+syfQ71tGztY/2a1O+Ieg+4\n+eabk/X0vPo9QFPwli5dGmP/HvzSSy/F+EMf+lCyLDdNRNz3d1wDAAAAAAAAnYIfagAAAAAAACqi\nQ1KfitIXfOiRvvZUGN1GLt1JZ2HWYUk+67KmLfkwbp0NWiuU+HBEHb7n1QF0GFVR6pc78sgjk9dV\nS30qGp7uQ2h1uOj111+fLNOhh37MlF4LevyKqh2E8PZjq8PJNc3Ah/Rfd911MfbUpyJeIaNVU580\nHcKrlj355JOltqEzrH/pS19KlnkfgfrR9qexpy3pOejevXvh9vTvtL2FkA5BLUonbHbed+nw2lya\nibYxr9xSlILkFQ30nqnt1M9FLjW5KPXJ933+/Pkx9rQPvZb0eHjfofeUXEpXo/L0W0070uPi1Uy+\n8IUvtLm9XGqBDs0PIa1Wo5Xf/Prs06dPm+/VSjRlM4S0XxswYEDhuvr859WVtB3pefJ0kNxzrvbX\n2hY9zU3P9axZs5JlzXhv1ekDdt555xj7sdxuu+1i7P2XvtZ+1LehqUpajdSrMuk14/21PqNqdT+/\nzz711FMx9uqz+pmHDx8eYz/fna3sd7YcrWKlVXRCeHv6aBn+fcRfv8WnNvja174WY/+OoN8Lv/KV\nr8TY+wdNNffzq9eFtku/NnU9T5f9/e9/HzqCpi1pmlIIIaxevTrG3s/pcdF99fM2adKkGGvKWghp\nRSTtH71f1u90evx8ug5NmfLvrdrWtc36s5Q+52r6XQghPPjggzHWtEY/35qi6sdjt912C++k+Xpu\nAAAAAACABsUPNQAAAAAAABXBDzUAAAAAAAAVUfMcNbl8V81V1Bw7z8nNlWxTWu7ugx/8YLJM88k0\nV97z0TQHzXMftYya7rvn2yvPwdQSW7rM53/Qzzxq1KjC7VeB5tnljouWPvTylkqPs+db1lKW2f9G\n8zt1mZcinjZtWqltagndXHnaVpKbW8PnRiiiudR+borK1no7Qvtpv6TzgHhfrnNXaNt22t96Xq+e\nV89lbhU+r1XR3GU+V4j2k75MX2uets8RpHnQOr+Fty89b5p/HkJ6vei++71Pc/FzcxNof+p9t+aS\nL1iwoHAbjcpLtWqb0GcVb4tF5XdzpZ11fosQ0rkTNNff8/lz9+5mo8dfj7mXSt1qq60Kt6HzLGkb\ny5W5137R5+zTtu3tQ68fnYdmhx12SNbLzTPU7HPU6Lxo3s9pSdwvf/nLyTJtL1pG14+Xnq9f/OIX\nMd5rr72S9XQ/fN6Su+++O8YPPfRQjH1Oi8suu6xw+/r8rdeWzympn6Uz6LWn5yWEtO/XOIT0nnH5\n5ZfH2O9pBx54YIy7deuWLCsqtezn8IADDoix3nN8DhSdi+S+++5Llulzj87PefzxxyfraWn2snOs\n+P1el/n5feSRR0JH+OQnPxljnQMphHwpaT2PWiLd7zN67vyZQ59Lte34PC7aB+px8edQvS5y853q\n5/LrU+dN2m+//ZJln/70p2Osn9/v27n5m8o87zRfzw0AAAAAANCg+KEGAAAAAACgIrKpTzq81ofr\nlE1byqW06JCofv36JcuGDRsWYx3e6UOIdUiZDoHyIatFpZtDSD+L7ocPn9ShhDrk0Lehw9V8GJUe\nUy/Zueuuu4Yq0eOpQ8M8lUGHq3mpQqXXkKe8qLJpUJ6OVFRq1ocM5rZfVDbPh++1Eh3eqcNu/Th6\nKeEiPjRcFaVWkfpUX9q3aangENIhorl0CB3e6UPutf/1If6twvsdPa46PN7XW7ZsWYz9HqHpFlqa\nNzdkWvtd77tz5Wi1ner2Pa1YX/tQZr0v6jZ8SLqWz23G1CcvnatlSPV5RIfVh/D28stvyT1/TZgw\nIXn9L//yLzHW62f77bdP1vPS7c2s6D7jKQqewqH02U6fj/za1vahbSxXYj2Xjvrcc8/FOFe62en9\nOvds30j22WefGGu6p09vMHTo0Bj788ehhx4a43nz5sXY+7lDDjkkxjNnzoyxlgUPIe3PfT8mT54c\n45EjR8bYv9csXbo0xp76pOdf02s91bazU5/0e4DfZ/R7lT8r6PWn/eQ555xT+F7+HU6Pn+6H349u\nvvnmGGuqvqbv1uqqq65KXuszkF9zRWk4/p0mN+VCR53fW265JcZeSlpL1Ptzo5bn1u/ses8JIS2F\nrilSIaTpTvp3fvyK0qz8+7ZeT1oWPIS0beqUKkcccUQoSz9zbqoU/f7ibd37mbYwogYAAAAAAKAi\n+KEGAAAAAACgIvihBgAAAAAAoCKyc9Tkclc1v9nnl9HcMo29dJbmo3l+l84Bkyslq6W+dPue06bb\n9zkTNL9Y507xvEV9L99fnctBc848j09z1bQ8XAhvz2ftakXlrp2Wshs0aFDheroNP49FJd1zcuW5\n9Zx6KT/PWy3ahu6H5/+2Es2D1/Pref6eq13EczRVUZ+Ty/9E+2lf4/NiHHPMMTH2vGv12GOPxVjn\n3AghndcoV9K9mXn/pPcx7Z+83cydO7fNvwmheH4nP8Y6B5Huh86NE0Ka0+3zChSV8/VSsnpPmz17\ndrJMc7j1HulzrJTJ025kOjdCCCGcffbZMdY+z+fWGzNmTIzvvffeGOfukXo/DiFti7l5UHKlqJuZ\nHhN/NtQ5ULw967r67OnXtj7L5p6ptW37NorOt8/dpnOxuKIywI08R83UqVNjPG3atBh7Od8HH3ww\nxj7vmq6r/aa3j6L5C/2a0fkMc3MN6Xv5M5H2+94udd4NXbZmzZrQlXTOFC1D3qp8vqtGoeXqlyxZ\nkizLzeWjzyD6LDFw4MBkPZ2P7uijj06W/exnP4uxXuc+f1ruO0Qt7rjjjhgfddRRybInnngixt4v\na9+pbdb7a/0dxL/3+xw+bWE8IdOPAAAgAElEQVREDQAAAAAAQEXwQw0AAAAAAEBFZFOf1GGHHZa8\n7t27d4y9VLUObdKhf7khoV6GVIdC61AhH1KkZdp0SGOuRKIPE9fho7ofXjpMP1dOboi3DpH1EtW5\nssVdoezwWC1pOHr06FLbc3peNS5bSjuE9JznjqUOBdc4hOL0Mx3C32oeeeSRGGv5dU3fCCGEPfbY\nY4Pfy8suFr0XNoyWGvV0RR2Oevrppxdu48knn4yxp8N8+tOfjrEOYZ0xY0b7d7ZBeV+ifZLeB7w8\ntx4vHUYfQnE/5H2rtiO9v3k/rvcnT03W+6Teq/2e1rdv3xgvXLgwWXbggQe2uX1N7wqh+dNu/Ljr\n8dRnE79vafvT1Kfc/W3t2rXJ66I0dU8387S4ZlaU4uLPG9qv+fOGDnXX4ev+XKftSIfs556H/HrR\ntqPXjqcE5Lap7bZsennVaelq7Xv23HPPZD0taa2lg0NIy0WvXLkyxt7Xaj+nZYp1Cgffnvep2hZ1\ne34e9Zlar9UQ0s+i59unWfDvL0AZet34fXns2LEx9r5G+yVNg9PnxBDSNvGTn/wkWbZo0aIYaz/q\nU08UPQd5e9MULP9NQO+h2qb8u8bBBx8cY02DCiFtm7o9/42haDqYEN5eAr0tjKgBAAAAAACoCH6o\nAQAAAAAAqIhs6tMRRxwR43POOSdZpkOXfSZone26aNinL3OagqRDoHxIqA7N0uGcPgRKh336UEJN\nrdKhibvuumuynv5dbt91qLlXq9HhxT5jf64aUVfQiiC51Cc9tsOGDUuW6XC4oioi7ZEbpqz7kdvf\nwYMHx1iHuoaQXgt6vbZy1aHJkyfH+Kyzzoqxpzzuvffe7d62n6eidtXIlSmqQvtHPc5DhgxJ1luw\nYEGMc+kQOtTTK6sdcMABMfb+tlV4e9B7ksZ6zwkhTZ3dd999k2VaYUT7O09HKrrvevvS194/6xBg\njT3tRlMefbh9UVUpH/6rn/OWW24JzU7TjvRa8Pbm1dRqocddr0lP0fFnpmamn13bgF/bekz8eUOv\ndW3Dvp6ea31fX09fe5+pbVOfr729aUqO089Wj2exKvjABz4QY72/feYzn0nWu+eee2Ls6bfad2ol\nQz+W06dPj7FWxvFjqefE00M0dUJTlbwSlU6zcOmllybLtLLXjjvuGOPvfve7yXqLFy8OwIZYtmxZ\n9rXS71V6z9F/DyG97r0P1Gtb07e9n9PvHroN/06t7crbovYX+r3Pq6fp81MuZdTT/5X22Z7q5Oni\nbWmO3hoAAAAAAKAJ8EMNAAAAAABARfBDDQAAAAAAQEVk56jRnMwRI0Yky3bfffcYjxo1qnAbmhfr\nJbg1V8vztjQnTfN6PUdMS6BqfpvPKaJz2XhenObYa2lUz/HUEuVeRriojLTnPGsZMM1bC+Ht5TK7\nms4LkpuTR3P/vCStzqmQ20aRXHlup7nGufcaN25cjP0ca7lH3Z6XPmwlU6dOjbHOoeDXdi1zLHmf\nUJQDWsu1g5S2Je1TfW6KsqXQdR4Fz//VOWtyJWObmedLa962zi3gpSYff/zxGHuZWS17mZs3S9uR\n3qu8HWkf7/urudna1n0+nP79+8f49ttvT5Zde+21Mb755psL38vnuWt2U6ZMifFpp50WYy/TqyWg\na7VkyZIYax69z1HTLPOWlKHtI/eMoaWcde4u/zttR35f1Ne5Z6rc/Afe5t4yZ86c5LU+A7tmnKPm\ni1/8YowffvjhGPuztM4DsfXWWyfL9P6kzzfa14aQzmeoz/F+LPWa8bnb9JrR+T703hBC2javueaa\nZNmDDz7Y5nvrvwOdzfvHIl6uG++sOXprAAAAAACAJsAPNQAAAAAAABWRHZOuQ/8uvvjiwvV8mKGW\nZt15551jfOCBBybr6ZDp4cOHJ8u0fGdumKoOCdX0qdmzZyfr/e53v4vx3XffnSzLlaBVOqy7b9++\nybK1a9fGWNM5PLVDh596isH8+fNL7Udn0WG6PjRT7bLLLjH24dT6GXWIqQ/lLRr26/9edshyLlVG\nrztNdQshhBNPPLHNv2nVEsMhpEPnNV3P0//0Ghk4cGCMFy1aVLhtL/FdlCZD6lN9aVqLpoWG8Pa0\nlCJFQ/pDSNuLDhlvJdddd13hMr1nalsJIW0v48ePT5ZpuUndhg+/13t3jx49Yuz9WC4tqqg0sZev\n1LToq666KlnWs2fPGGsaT9l7brP6yU9+EmO95/h9UdM0yvapTp9BNM3Oz7eXCG5m+hxRlFYUQvqc\nt3z58sJt6L3P72m6TNupP7/klhWVTvfnS71/5tIcmyUdddCgQTHWZ03/7M8880yMx44dmyz74Ac/\nGON99tknxr17907WO/PMM2Os7dK/C+jzsD83a1qUptl7aV/9vqJ9aAhpKXh93vY0K++nATQmRtQA\nAAAAAABUBD/UAAAAAAAAVAQ/1AAAAAAAAFREXRJVvYTkxIkT24x/+tOf1uPtusxxxx3X1bvQqXQe\ni1zpSC1d7bnUuo1cLnjRMs/V1te+TPdRYy31HkIII0eOjPG8efMK90m3X5Qj3mpyc1povnTZ+RS8\nLK/OH6RzTjVLOdGqeOONN2LsefRl5w/J9Q96vnzOBqT3TJ8nS+cR2XbbbZNl2iZ0nolVq1Yl62l/\npdvw86Tn0PtTbeu5ku1aJnyPPfZIlvl8cPgHLe+r8wnp3HwhpH3q/vvvH+P2zFGj507v1T6fnM85\n1ir8OChtRz6HoM75kusz9dlG21jufX3OryKvv/568lr3V9tlCOmcYrn3biTaXnQuF5/X5dFHH43x\nY489lizTZ8ApU6bE2OfN1HvmTTfdFONdd901WU+3788tv/zlL2M8Y8aMGPscNb/97W8Lt6+fWecp\n8/MNoDnw7QcAAAAAAKAi+KEGAAAAAACgIpqjRh86hKYs6LBPL8f+wx/+MMZe+lCH4pYdzls2vclp\nKo6+l5cfnjRpUozvvPPOZNnXv/71NrfRLEOFy/BjrOfgtttui/Fpp52WrKfDfA866KAY33fffYXv\nlSsFrfuh6QHYcL169Yqxp7CVTTPT9B1PXdRtat/RyvR61mPs/aK2nVzamB5XP2eDBw+O8bPPPlu4\nDS316u1eU+I0xcLPp6bxHHLIIckyTX3S7Xu/3uxyfeq9994bYy3VHUKamjZu3LgY33jjjaXfW/tY\nvU78msndW5uNXtu55xJNxZ06dWqybMCAATHWssueBqVlz3Pls3XZe97znsJlytuilmj27WvqU7PQ\nNNE+ffrEWPu/ENL+68gjj0yW6XHS46znNIQQ5syZE2Ntv55+pqmsWj48hPQ5ZvXq1THWftjf20uw\n9+vXL8b6LO4pzACaAyNqAAAAAAAAKoIfagAAAAAAACqC1CcU0lnkdXiwD8fXtKC1a9cmy4YMGRLj\nhQsXxrhsekVuOLYv0/QLHebrM+rrkFPfX6WfWYebNrvcMP3f/OY3MT7jjDOS9fS6GD9+fIy/8Y1v\nFL6XD+kuSnsrW4kI5WiVoO222y5ZVnaIvA7p9/QBrSCj7a2V6fWcS7cYOnRojL1infa1uo2dd945\nWW/x4sUx1tSX3r17J+vpcHnvkzVtVfsETcfx15pS5/Tz5/qYZuTHVs/dXXfdFeOTTjopWU9TWzS1\noz30GtLrRyuIhfD2CmPNTO87em/xdCFtH1o5KITiNuHnWittaVv0NlBUzSeEtH3oPnkFo5UrV8bY\nrxetbuSpVY1q9uzZMX744YdjrH1oCOmziaZL+TJNHRsxYkSynj4rHn744TH2Sm1ake2AAw5Ilv3u\nd7+LsZ4fTbELIT1XkydPTpa9733vi/HLL78cY32+BtA8GFEDAAAAAABQEfxQAwAAAAAAUBH8UAMA\nAAAAAFARzFGDQlqOcuTIkTH2+UI0n9bnSqi6gQMHJq+1FKLOs/HII4902j51Nc+x17l/tNyuzlES\nQnq8vFxzkSeffDJ5vfvuu8dY52fwuTWwYXRejH333TdZVvbcaVvRXPkQ0nkUdL4U/IPOheHz1eh8\nWDqnSAghzJ8/P8Z6np555plkPZ1/ROc08HOrc1X4fuj5LZrnJIS03eu8Zr7sT3/6U4xbbY6aXJua\nMmVKjLXUeQjpnBk6/88ee+yRrPfEE08Ubl/bpp4fn4vK+/NmVjQXmt9n9Fq/5ZZbOn7H/r8XXnih\n1Ho+b47OlzJ27Nhkmd5rfV6VRrVkyZIYjxkzJsZ9+/ZN1tP2521nxYoVMdb2oeXXQyiek83n+9Ft\neMlsnR9Hz8FOO+2UrKf9o/abIaSlvLW/aKX2C7QSRtQAAAAAAABUBD/UAAAAAAAAVASpTyg0ffr0\nGOtwTi/PWjZVoop82KoO1ddhz6+++mqn7VNXy5UOVkuXLk1eazlLHdZ74IEHJutpSl2uHKqemx49\nepTaJ5Sj6Ys+PLvs+VdayjmE9Px7OgfyqT5f/epXY3zBBRcky44++ugYb7311jF+9tlnk/W05Kye\nmzVr1iTraelgL1vbvXv3GOtwey8ZrmVrr7jiimSZD9t/SyPfM2pRNrXL+9Rjjz02xpqqpOWBQ8in\nPul59Xaq9Bw3O02N0fQyjUMI4Zvf/Gan7VM9/PjHP46x9wmaOqfpzY2cMqPpXOeff36M99tvv8K/\n+fnPf5681ucWvfd5iXRNR9OUeU8h1GdlTzHTfk+fL/0czJ07N8bDhw9Plml6uKYVN3v6KNCqGFED\nAAAAAABQEfxQAwAAAAAAUBH8UAMAAAAAAFARzFGDQsuXL4/xY489FmMvz/3aa68VbuPd7/6/S0zz\nf708a0fy99L9WLBgQbJswoQJMdZ89YcffriD9q56yuY6X3311clrzau+8cYbY6xz0rgbbrghea3H\nXMsDP/DAA6X2CeXocT/44IOTZVqCvazbb7+9cNns2bPbvb1ml5ujRcvSX3zxxYXr6TwbWoI7hHS+\nka222irGOjeF87nHdO4FnTtFy0mH0Frzd3W0b3/728nrlStXxljPz6RJk0pv86abborxqlWrYvzS\nSy8l602cOLH0NhudPrPoXCF6zwmh/HHWZ4yunCvkf//3f2Ps7dnng2sG2kfdeuutMX7++ecL/0bn\ntWnr9Vuuvfba5PWMGTNirHOF+RxsOm+M78fTTz/d5np33HFH4f7q+4aQ3juWLVsWY+aoAZoTI2oA\nAAAAAAAqgh9qAAAAAAAAKmKjNxkvBwAAAAAAUAmMqAEAAAAAAKgIfqgBAAAAAACoCH6oAQAAAAAA\nqAh+qAEAAAAAAKgIfqgBAAAAAACoCH6oAQAAAAAAqAh+qAEAAAAAAKgIfqgBAAAAAACoCH6oAQAA\nAAAAqAh+qAEAAAAAAKgIfqgBAAAAAACoCH6oAQAAAAAAqAh+qAEAAAAAAKgIfqgBAAAAAACoCH6o\nAQAAAAAAqAh+qAEAAAAAAKgIfqgBAAAAAACoCH6oAQAAAAAAqAh+qAEAAAAAAKgIfqgBAAAAAACo\nCH6oAQAAAAAAqAh+qAEAAAAAAKgIfqgBAAAAAACoiHfnFm600UYx/qd/Sn/T+fvf/94xe4QQQghv\nvvlm3bal57Gj+XWy5ZZbxnjYsGEx3mSTTZL1PvjBD8Z44sSJMe7WrVuy3rJly2J81VVXJct69+4d\n41/84hcx/slPfpKsN3fu3Bj/9a9/beNT1E+9zmNnnsMcPebnnXdep72vX1dlj6set9zf5Pq3Rm2L\n/l4f+chHYnzttdeW2oa2D28rG2+8cYz9+Km1a9fGeKeddkqW/fGPfyy1H/VQpbao23j3u9PbsB7L\nzTbbLFn26quvxrhfv34xvvrqq5P1Dj300Hbvk9/Tv/KVr8T4+uuvj/Hrr7+erKev9ZoIIYQ///nP\nMdbP9Ze//KXd+xdC47bFnJdffjnGer/saO9617uS1535TFeltljWpptuWrisb9++Mf73f//3ZNlp\np53W7vfS55wQQhg4cGCM9Tz5Oay1XRVp1Ptibnve3/7tb3+LsT6Xvuc970nWW79+fbv3Q59lQwhh\n3LhxMX7jjTcK3+tPf/pTu9+rVo3YFhtd0XNp7hj6skZpiyiv6DwyogYAAAAAAKAi+KEGAAAAAACg\nIjZ6MzNmiiFQXaeRhrINHjw4xtOmTUuWde/evUPfe0PtsMMOyeuVK1fWdfvNNqy0ntdle+yzzz7J\n68cee6zT3ruR2mJOV507NWDAgOT14sWLO+29q9oWPV1Ih+Z7uoUOzX/88cdj3LNnz7ruU46nyOh7\ne1qUphVo6lyt56JZ2qLqqnbp75tLX+zo965VZ57DLbbYInmtqdYzZswoXK8eNKVJn6k0fSaEtL3l\n0oXrcfyr3BZ9e7qvnvqk/dnmm28eY01JrBdNadLzqOcthDRl1NPbdN0qnceq9KetqMptEeWR+gQA\nAAAAAFBx/FADAAAAAABQEfxQAwAAAAAAUBHZ8twql/OJ1uJlej2HtpE8//zzyWsth+v53+g6OgdA\nCOTRNqp169Z19S50GZ0zQu+fPXr0SNbTPmj77bdPlp1zzjkx7sx5aZTPfbFmzZoY//CHP0yWaRn4\nefPmxbjVnx+8HG9XoA9tH2+nZ555Zox1bpOOoOdq2LBhMZ49e3ayXq50tz+3NTNvX9rf5PrNAw44\noMP2KYS07zz66KNjPHPmzGQ9nSvxve99b7LslVdeiXG9y7EDqB5G1AAAAAAAAFQEP9QAAAAAAABU\nBKlPKEWHzTZyqtM70fKyDA3/h6233rqrdwFNwq+ljiiBWlVahlvLbP/bv/1bst4uu+wS49deey1Z\n9txzz8VYy7R2ZZ+sw/kvuOCCZJm+1iH8Wqa2FWlqA6rD7/narvr06ZMsu/7662O80047xfj000/f\n4P3w5+sbb7wxxqtXr46xl5r2Ms/NTs+PHjO/z2y66aYxHjp0aLJsq622ivHee+8dYy2RHULaf5e1\ndOnS5PXNN9/c5j5pyloIIWy55ZYx3mGHHZJlixcvjvH69etjzHcyoDkxogYAAAAAAKAi+KEGAAAA\nAACgIvihBgAAAAAAoCJKz1HjOZRoLc08L00RLX1YhXKqXcXnnQBq5SVidU6IZsux9/kutCzs/vvv\nH+NRo0Yl63Xr1i3GL730UrJs0aJFMV61alWMe/fuvWE72wluu+22GB9zzDFduCddr3v37l29C2/z\n9a9/PcYXXXRRF+5J19G2F0I6j4jOHRVCCJtttlmM//jHP8bY+7Fa5rrTUva+Td0nLxm+Zs2aGPs8\nUM3Qv/qcPHq+9N6i/WsIIWy77bYx9rbXq1evGA8aNCjG3vdut912Mc4dS/2u9MILLyTLfH6ct/h8\nOPre/fr1S5bp59TrTmMAzYMRNQAAAAAAABXBDzUAAAAAAAAVsdGbmTF8zVSeOPdZqjgktJ77VOt5\n1L9r9dS3Wo9hvc5jV7bFKraPzjweVWiL9aClW7WkckfT47fFFlsky3S4dkf3MZ3dFv0Y77XXXjH+\n9a9/HWMv+6v76cPv77zzzhhrCsScOXOS9c4999wYa9qml63tqpTWru5PN2Qf6qGKfarq6GPT1fdF\nbZuaTqPlmUNI+6v99tsvWda/f/8Ya+qhpjiGkKZQK093WbJkSYxfe+21ZNl9990X4yeffDLGnvo0\nf/78GGvp5hDqX7q7s9qinistpR1Cmhak95KjjjoqWU+Pkx93TWnafPPNY/zKK68k6w0bNizG2vd6\nOq9eM56O9MQTT4S2+D7pZ9Z9CiGE+++/P8ZTpkyJsZbtbo+uboutqN4p381yX9xQZfuRENLnzarc\nj4v2gxE1AAAAAAAAFcEPNQAAAAAAABVRuupTo2nPECgdblQUv9M2axku5tuvyvCrKtNj1MhD9IDO\npkO0N954405732nTpsXYh983c5+X69+9uozSfs2rzWl6hKYx6TEOIa1k8sgjj8R4zz33TNY76aST\nYuxVYnbcccc29wnt15mphnhnmvKnqU+eZvLe9743xt53rVu3LsabbLJJjL3Sj6bCaPz6668n6+k1\n4ulSmlql6ZB9+/ZN1tOqT566U+/Up66gKUchpMdd+y9PJdJj/cYbbyTLXn311RhripSnLWkqmR5L\nT33SPtvPsV5reg16JSrdvlYXCyE950WpVCE0d0XFemvP/Y1jWV258+gV4/Q8ahuu4vdynh4AAAAA\nAAAqgh9qAAAAAAAAKoIfagAAAAAAACqiaeeo8Xw0LcXlea6as6p5wj4/gOYNe1lTfb9cGVz9O8+V\n9VzXrqalDzuTHgcv2av53zvssEOn7RPQ6P71X/81xt///vc77X2vueaaGHsfV4X8386ic75sueWW\npf7G72Pq5ZdfjvH++++fLNP5Gg488MAY58qj+7nQ+1huP2qhc3/4fjSjeh8/bBg9H9tuu22MfY6a\n0aNHx3jQoEHJMp2jRp8p/ZlP26LON+LzKTz33HMx9rlNtN1qOWl/PtLt+zNqbh6GKtPj5MdM+yid\nu8WfDfUce+nzF198McY6583222+frFf03cC/J/hrpedH30v78hDSc+zXU58+fWKs8xAxj9g7KzpG\nfox1PW9juWVlNVL7q6KiPsG/22vfq/P2+d/pd3uf30rn/eoqjKgBAAAAAACoCH6oAQAAAAAAqIiG\nH4+rw5d0+JqXtNMhUD6EdcWKFTHu2bNnjPfaa69kvcmTJ8dYhy2GkA4z1XJ/22yzTbKeDvmeN29e\nsszLKXa1Qw45pEveNzdMnHSnxuMpfj48EfWTG/784x//OMa1pD7dd999yevDDjus1N/98pe/jHGt\nQ4WbgQ+pLcPLqGsZVz2WmoYRQjqcX+8rmgIQQppK6iWBfahwPWm54RBCWLRoUYe9VxVUYfj0O2ml\ntqkpTpri7e1jwIABMfb71sqVK2Osz41OyzpriounN2mp5a222ipZpmWj9TnXU7U0RSqXut9Ipbo1\nTcT7Qz1Omlrqz//K01yUtgE/P/odQp/x/Z6r14mWUg8hhFWrVrW5j/5e2s/rufdl+l5+vhs11a0W\nZdO+9Bjp3/h1lSuxrvSe3p7+U78v6t81+3mqlZ9fP19F/65/N2LEiGSZtp21a9fGeNasWYXb8L6j\n6Nzl9sNTUv37UVsYUQMAAAAAAFAR/FADAAAAAABQEfxQAwAAAAAAUBENP0eNzvmiJU+HDBmSrLd4\n8eIYf/nLX06WHX744W1uY+bMmcl6Tz31VIyXLl2aLNPcNc0pPfnkk5P1dt999xj/4Ac/SJZNnz49\nVEmzl0xF5/CSwI2UH99oNE/W83prmSPlV7/6VYw//OEPJ8s0Zz8nl4vfbHK58tr3l6VzzYSQ5j7n\n2pGeGy0z63PD6BxqPsdBR5aUvuuuu5LXw4YN67D3qgKfc2RD/ehHP0pef+Yzn9ngbebm7mh0/tm2\n3nrrGOuzYq4NeAll7cuef/75GPs8Ezr/gT5f+pxQOldit27dkmU6t4nu40477ZSsp3Mh+RyIzVC+\n2edz0POjz90+N6SeEz/u+h1Cef+q763b97/Xa02vM9+GLvN7qc63o2XHQwhh5513jvHpp58e48su\nuyxZz+ctazS569Xbs77WOUB8Gzqnk84z6u1Nt6dtO4T0utB5gJzOX+L7q/uYm5em2ees8fOjr30e\nWKXnKzcH39ChQ2P8ne98J1mm9+QvfelLMV6+fHmynp6Dst9dvM1OmDChcB/79Onzjttr3jszAAAA\nAABAg+GHGgAAAAAAgIpo+NQnHUKmQ+y9bOEpp5wS4w996EPJsqIhvwcccEDy+g9/+EOMvYSXDinP\nDWvT4YieilC1YW777LNPV+8CmkArlX1tNpqe6UPGy/JyhM0s14drOd+y/NjpMHu9f+TKQWq6hcYh\npMNwc0P4623w4MEdtu1W8LnPfS55XY/Up1zapL5uhv686Nr29CYtG++pgPosp0Pi/bnuhRdeiLE+\no/o+aBqFln92mvrkKRu6j7n0kEai16Wn4xel0vr1q+fHr199rfc4T0cqSnnx7Wk/6vcDPXea2uHX\njKaLe6lx3aaW6vZj0+jt1O99ek79XqX3P73H+XqaZjJw4MDC99Lz+8QTTyTLNJVRz4Wfa01z8+tR\n26K22Vwqlava98Va+HHXNqFpnV7m/lOf+lSMP//5z8fY20pZ1157bYw9ZVSnJPE2pedA22y9004b\ns+cGAAAAAABoQvxQAwAAAAAAUBENl/rkwzd1pncdQqazPYfw9gpLtdBZ2kePHl3TNnS4nQ7ZCiEd\nzlUF5513XlfvAhpU2YpAZTXDMM/OVo9jpv1trcM3G3XIfS30GPnxHzNmTLu358Oudai/Dpf3IdM6\nxF7vW16FZNNNN41xZ6aotVI6XEfw6kQ333xzjD21u6xc+262/leHsOtzo9+3dJlX69C0Ck2D8hQU\nPXbaZnP9og/h1/aSq+Kmz5dTp05NlulnqaUCYBX4daj9l6ZKDB8+PFlP+0ev4jNnzpwYa7vKpf9p\neo1Xp9Fz58s0pU23oZ8jhLTqk6e1anqe9uftSZupEj2uGnsb0PbiVfT69+8fY532wtMV9bhq2/Hq\nik8//XSMvXqWpj5pqpynWZVNQ9TYU/k03S6XzldluX7OpyjZd999Y/y1r30txt6eNeWv3h588MHC\nZd63L1u2LMZeabqsE0444R3XaZ0naAAAAAAAgIrjhxoAAAAAAICK4IcaAAAAAACAimiIOWo0x83z\nADV3V0t41VqmqzONHz8+ef2xj32si/akbV76ESjr3HPPLVyWKwNb5m/QNWo9B9qP+PwArWTQoEHt\n/pvvf//7yetTTjklxnrv83Oj7UrzwH1umFwJ16rMI7OhpS2bxZIlS2KscyOEEMJPf/rTGNc6R41e\nQz6vQKPMh1DE24cev1WrVsVY558IIZ0zwkvb69wVug2f50bfS/tCb3tF80+FkJZ+1bkvfP4S3b6f\ns2a4h/pn0jkjdL4WPy56rH0+GJ2XRo9tbo6aHF3P25Hul/avPg9K0bwtIaRz0QwYMKDUPnWG3Pxs\nOfqdTu9Ve+yxR7Kefhj3vsIAABcQSURBVKfTuUlDSOd/07bo+9GzZ88YDxs2LMY+l5DOgaNzAoUQ\nws9+9rMYP/vsszFesWJFst6aNWsKt6+fRdv2ggULkvX0vf2afuWVV0JHqPU86rWu1/YOO+yQrNer\nV68Y77rrrsmyq666KsY+D1sV+O8Ptc5Lo6644op3XIcRNQAAAAAAABXBDzUAAAAAAAAVUZnUp1z5\nMh2G5uW8dKiilvb67ne/W+9drDsv9eWvUR0+nFzLoeLtHnjggcJl8+bNi/HQoUNLba9Ry4k2Oh3O\nW+vQ+eOOOy7Gc+fO3eB96mq1Dg1evHhxjEePHl24nqZKzJgxI1l2/PHHt/nePkxYX+u91NMtdPi0\nl3fV4eWtVGK9PfRa8FQxPZ651LSyfvWrXxUuu//++2Ps57jsucuVxtWywo2YQuP7rOlJq1evjrF+\nzhBCOPLII2O83XbbJcumTZsWY02n8XOrbVFjb2+5dqrvrWWovWS4phL4MP3XXnstNDo/tprKsuee\ne8bYUxL0WD/33HPJsmeeeSbGeg68tHNRaqC3r7LnWD+Lpr+EkJ473w9dpqlufg/IfZ/oiHRS3S9P\n/9P24de2XrN6Dvfbb79kPT0O3hb1OtAUIW/Puo/6vp6alEu7ufzyy2Os18Sdd96ZrHf11VfH2J9f\nd9lllxhrepMfG/07/+67aNGiwn3cEFr62vdHj6en3+ox1HOgqWIhhHDIIYfEmOeKtBT8wQcf3OY6\nHCUAAAAAAICK4IcaAAAAAACAiujU1CcfbqfDxnwm9m222SbG/fr1i3Hfvn2T9XSImg41a4RKEf6Z\ncxU40LU+/OEPJ69JfcrzGeqVzoZfNvVJK2Kg89SjT9ptt91inLsHNIpc6pMO5fUKJWWH+U6ePDnG\nL774YrJMt1m2epr+je+TDs33Yfr6Otee60GHdTdDiobz4+7pDGXkqnDptaBDqUNI219Zmm4eQnpO\nGrHNOj0f2sa8kooeB08H0zQKrTjk9yp9L/0bT4fQ8+vtTVNj9L28ooqmH/g59Oo1jSLX3+prrcrl\n7UuPrR6/ENK+J9dH63vp+fFzpe+duz9o7KlPuQpsel41FaWrv/NoyoxW4A0h7YO8L9RlRxxxRIw9\n1eeNN96Isaf1aZqXLvOUJr1G9BjX+pyjx3z33XdPlp111lkx9pRvTdVauHBhjD2VUfk17Wld9TJ8\n+PAYT5gwoXAf/P31u7h+t61K9ciqmjlzZoxJfQIAAAAAAKg4fqgBAAAAAACoCH6oAQAAAAAAqIhs\nonSuXGDZPH3lJc90m55PW1Ri7aGHHkrW++xnP1u4/arzY+rlzuoll7vaDPnmneHwww/v6l1oKJ6j\nrE444YQY63w1OZTx23C1HMN69A+5MsWNSHOu2zNHjZbnzpk1a1aMPRdfX+t7eTlUnS9A74t+L9C8\nf78f+f53pD59+sRYy+WG0PVzL5SVu7ZrnQNB50+59dZbS/3NmWeembz2Eu9lNOM8QUrnh1m/fn2M\ne/funazXv3//wm1oP6BtzOey0ffyuUhUbg6OYcOGtRlrO/e/07LjIby9j2gU2v59vgs9FnrufD4P\nPQdLly5Nlum1XlRKPYS0Des9zfsnPc5+z9XX2l/4vEY6n5CXotb90uupo+cReydaNn7AgAHJstNO\nOy3GPreJHn+dc2ndunXJejpnjR/zhx9+OMa5Odl69OgR49zcKTpPi5ZADyE9v9o/e/+vc+/8/ve/\nT5bpPGI777xzjL09jxkzJsbedzz77LOF+78hLr300hj7PKpK55FF7a688soYn3/++W2uw7cfAAAA\nAACAiuCHGgAAAAAAgIrIpj7pEDsv3Zkb/l1Umis3THOvvfZKll144YUx1pJlXo6wkXVWqlYzpBt0\nNS+libzckEkvOVwGx3/D1VIm0fv9WpxyyikxPvvsszd4e10t15/m7os67LosH8Kvw/ZzKU26TEuS\n+jWQG6bfmalPOvzbU5+qnPZYtkS6pzaUTVPQvrJfv34xnjp1auF+LFiwoNS2c2opH95I9NrWVAzv\n73LnVFNScs/Kun195vXnP03B8ran7VRLiPft2zdZT9v6yJEjk2V6zej1WPXnQ23/uX4ud81q+WZP\nffJUtaLt6dQEeo5z6U25ZfpZvIx0LdMg5J65OsOSJUti7MdOy1N7CWpN4Xn88cdj7OXkNfXJz9kD\nDzwQY70mPN1Pj7PeZ/w8abroUUcdlSz7n//5nxg/+eSTMfZ2r5/lhRdeSJbp/mv6v5ck1xQs38ey\nqdTtde6558b4kUce6ZD3aBY6xcPnP//5GPtzlvbL06dPT5b5805bqvsEBAAAAAAA0GL4oQYAAAAA\nAKAi+KEGAAAAAACgIrKJyFoy2/NYt9566xh7PuXHP/7xNv/uiCOOSNbbfffdY+z5uo1SkrOePHez\nI/hx1fNTyxwWrSJXVhNv5zm5qpZStR09R02uXTQS/Rz+GWrpU/fee+8Y33333TXtU1fnztebHkef\na2T77bePsc87oOU6c7RMr5cH1nuy9kme267zZ2h+tM8doNeI58DrsrJzsdTq6quvjnGjzkPXEcel\nV69eMS47V089yjD7fAuN2h8W0c+jx6s9155uQ/sBf1bQe5fP06h0G7m5pNauXdvm37ibb745eb3v\nvvvG2Et3V5n2Wd7PDRkyJMa59qHz/8yePTtZpmWl9Xj6dxJ9b439XOn8I94n6BxFukz3L4R0nim9\np/j76WfWfr4r6NxJbsqUKTHWeVdCCGHhwoUx1s+j90Ff5udGt6HH2L+bajuaP39+jH1OKJ0b7Ior\nrkiWadvRZ1nvI/W9/TrQ/nXWrFkx9rl3Zs6cGWP/fjhv3rzQEZ566qkYa/sKIZ0Xx58ldC7CQYMG\nxVjPRwghHHnkkTH2+4y+tz433nLLLcl62k8vWrQoxqtWrUrW03mOli1blizTsvB6fvz7Se7Zp2iZ\nr6fbrGXOPUbUAAAAAAAAVAQ/1AAAAAAAAFRENvVJh09/4xvfSJaNGDEixjo8N4TmL+3YUTRlrCv4\nMDf8n1ZMxdsQuTS6WobRN1v6TKOoJU2t2em17elN22yzTYy1nHII5VNbteSuDxPX99Ph9926dUvW\nKxp+X3bobgjpsOSOTn3S7fvzQ5XLc6vcfmr52BBCOPbYY0ttU1NId9xxxxjnzkE9UpibLdXJFfVr\n/iybo21MUzG8T9Dy3LrM90HTbjzFR9NafFkRb0djxoyJsab/VP1c67H1a9tTVopoX+apgbp9PQee\nhqLvrekcvk+6DW+nuqxsv+bnpyhdtau/dxWVLw8hTeXytHhtH5q64ml9ej48nUbTpLT0vKdP+X69\nxc+TXiNa2t23kWs7el349vW61c/vKf7Tpk2LsadNlr3220uPraZZhpBeY+9///uTZfp7gaYye381\nderUGGsqdwhpCpu2S09X1HOg58ePie5vri3qerln3mHDhiWv9VrWaQKWLl2arFf0uUIIYd26dYXv\n95bGeAICAAAAAABoAfxQAwAAAAAAUBHZsXI6c/OJJ57Y4TvT6nTIXkfJDdV7/vnnO/z9G1Urp4Cs\nWbMmxj179iz1N5oC4hUNauGpHWhbrupTLUPc61FBppl5moMOXfah8zNmzIjx0KFDY+x9S58+fWJ8\n6qmnJst0mKymUnn70GHXel/xYbd6fvV+39b+d6RPfepTMfZh7VVPzXiL76emJVx22WXJsrKpTzqc\nXCtu5Pg5rsW4ceOS1zfccMMGb7NKtH1omx07dmzpbRSlA3p71vanKbw+TF+r5ngbKKo00x5ayaZq\ncql82sd6f6t9VC49U59hnnvuuWSZpizo+fF2VLZdaVqG74emueg16Ok1up6n6xTd4zu6MuY70evZ\n+0L9DH796t9pKolXT9Pjr8cnhPS86bn29fSYa7qL76/uox9X3YZ+Lj9P2v+XTRf2PuGee+6JcY8e\nPZJlWnm5nrRSUi696sEHH0xeP/roozHWvsxTA/V7Zi7dWpd59a6i9L9ctSVPDSw6Jz7Ngu6TV47S\n56c77rijzf0LIT2O/r5lnrMYUQMAAAAAAFAR/FADAAAAAABQEfxQAwAAAAAAUBHZOWpWrFgRY8+5\nolxx/ZXNQe8o9chtb1aNUiK2I3z/+9+P8fe+971Sf1PvOX06eo6gRpkHY0PUUs6xqJxlCGnecNm+\no9byqlWix8Tz13WZz/n161//OsZ77rlnjP2YaCleLy/qJUvf4nMc6P35pZdeirHnQ+t17+fQ54Oo\nJ2/PWqbT99Hz06sqt59akrQ99HwvX748xrn+ynPsa6HXYAj5ua+ayT777FN6XZ1DQ9tfrs/U6z43\nV4c/b+gx1/7Ct6F/5+dpyZIlpbZRNbn9K3v/yJXp9fmA3uLtWf+u7LwYuefGou2FkJ47vwfk5oLp\nSrn90vahc8iEUNxv+ufWeU9y8/Hoe+W+t+ox92ssd1zLHvPctVk0/4rfx5XP9dhR9+ey/cFrr72W\nvPb5gIrkfjvoyPtM2TmcfK7YsvtR63p+nbeldb99AgAAAAAAVAw/1AAAAAAAAFRENvVJU3H69u2b\nLDvzzDNjfPbZZyfLBg4cWIdd2zA+nG7t2rUxnjdvXrLsyiuvjLEOwf7whz+crDdq1KgYe6mvevjN\nb35T9222h5bGQ6rZU/30evah25dffnmMy6Y+DRkyJMZLly7dwL17e3oI2pYb0l7LUNIFCxYULtPU\nHr8/FNFyw76NRqHtQ4dj+zItcxlCOhRa29T++++frHfyySfH2PsdTWPSIfueEqXvpUOSfZi/DgfW\ne1+9jBgxIsYnnXRSjC+99NJkvdx1UJTuVTXevrRPzZUXLTsUfL/99ovxww8/nKynbX233XYrucfF\nNM3K96MZFD2/eXpBjrYdjbVkawjFbdaH4mvag6cVzJ8/v83teYnegw46KMbeN+l7V+185vZHU1Ry\nJZBzVq1aFWMvU679S1Hp5RDSVJPc80gu5UXPuW7f0y10+wsXLkyWaVsfMGBAjPv06ZOsN3fu3ML9\n6Ai55wvdZ+8Li46XP7/o67Lphbl9LIo7Qy3v58fJr5l6qXeqT61yaVAdmSKV2573U/V47zKpZoyo\nAQAAAAAAqAh+qAEAAAAAAKgIfqgBAAAAAACoiOxEK5rL6flx3/72t9uMm8k111xTuGzHHXcsXKbl\n53K56UBV5HK9aymPW+95JVq5PHp71Lt/yZWLrOW6KFsisVH4fVFLLfpn1fzme++9N8Y+h9P48eNj\n7CWztSSwzmOQK9Wp14TPzbHNNtu0uX+10rkgQghh+vTpMdY5M3wukNx124gl3J1/vnXr1sV42223\nLbWNXr16lVrvQx/6UPkdE7qPuXbfDIrK9Lan/yyaJ8HnqNF+INfGdN4N71t1vqtu3boV7q++1rls\nQqj2HDW5+7s+S/h6RaW1XdlzoHLr5eY3yc2fovuhn8WvGf3Mvj2d00L3ccsttyx8366mx6js3Hn+\n79om6vGcw3exrudtrJa5Z+oxX03ZOeO66prh2w8AAAAAAEBF8EMNAAAAAABARWRTn8oOOe6IklUd\nWX6rHlavXp281uGJVdzfMhp1v7Hh6p2ScsQRR8T47rvv3uDt1ZJm04pypf5qGe7uqTdqxowZMR40\naFCp7WnKRzPSe6bfP/X4r1ixonA97Yc33XTTwvfSNKbcudW0KL8+6l32/sILLyzc/tChQ2Ps5Z9z\nctdgleVSIC644IIYX3vttaW2N3bs2Bh//etfL3yvc889t+wuJhYvXhzjKVOmJMua7dlAryltH3fe\neWey3siRI2Ps51OPl5bK9TRETfnTdCRPd9FteIqPluvWUvZe1r6oZHgIIey1114xnjBhQqiS3PN+\nrjy3HtuilKAQ0hLXni5VlBblpc/1da7v1bRB77t0XY232mqrZD29FnIlyXXfy6aBdbUy5Yjb0mx9\nEPKpm+35u3r+TRWvM0bUAAAAAAAAVAQ/1AAAAAAAAFRENvWpLB8+rUO5ax1GVMvM7Dok0PdJhwjW\nY2iTb1+Hu2qahg9vBVrBBz7wgRh/7nOfS5aVrXKC9qv3sM1cNSFNZSmrisNKO4t+dr1HakWXENL7\nmFdpqiV9Tf+m3qlOLpfm+PTTT8dYUxbeSTNWILrttttiXDb1qXfv3jH2qnr1SF3Vftm3r6kYzUA/\nn6anrF27Nlkvl6qk17DGPXv2TNbT1JXcs7Huk6exaOqT/p1WmQshXwVIK8ZVrepTbpoFPX6bbbZZ\nskz7hlx6dK7qk55jTTX186jHU/to3572596OtMperlKYfmbv//R61b9rT9Wnqp1/tI5WfgasFSNq\nAAAAAAAAKoIfagAAAAAAACqCH2oAAAAAAAAqoi5z1OTKUNaqbAk3zXfTuCP2STVj3nyttKTlscce\nmyzTXF49P567q/nFeu59joZcLvPmm2/e5vb8b3Q9zU+ul2bO/9Xyyt27dy9cr2/fvjH24/Hiiy+2\n+32b+ZjWUy7/t5bcYJ3XwJ133nkxnjZtWqnt+XkkXzmdfyKEtPTyj370o2RZ1dvBJz7xieT1f/7n\nf8Z49erVMW7PeW/Ga6SW+47et7RUdwgh3HPPPRu8T3qv1rk0Qkjn3ai1vG6V6HPF+9///hjvtttu\nyXrPPPNMjH1+lCVLlsRY54rxZ0OdR0TnSvHjqCWa/flI+wjdhpeQzs1vpfPodPRcVe2Va+N6bP1Z\nbtasWTFesGBBjP0c6PHzuWe0XWm84447JusVlb/2fdc5ZHLfQ/T8r1y5Mlmmn2XAgAHJMj3/emy2\n2267wvcC0LgYUQMAAAAAAFAR/FADAAAAAABQEXVJfaqHWofE69/lhoU3wvDpqg9rL3LiiScWLisa\nJl22bHl7UtheffXVNv9dhwOHkA7j9qG0tQwJ9r9phGutVttvv32Mc+Uwdei2p0jVUup1zZo17f6b\nVqR9iF+HtVyXZ599dowvvPDCZNmoUaPavb2cVk2L8s/5u9/9Lsbef3pfVjVa4jmENF3ESx/jH/Qe\n5+kqStvHYYcdlizTa6ZWem31798/WbZq1aoYF91nG4kec/1s8+fPT9bTlBQvf6xpN9pOtcRzCOn9\nTtP/vL/T1Bp/LtFtampNUTpOCG9/dtL7ddVSn8o+u3s5cj0/mi6kaV4hpOfYU6/1nOizive9Rc+y\n/u96nP0ZaeHChTHW9KwHHnggWU8/1957750s03M+ePDgGHvfm3sWqPp9BMD/obUCAAAAAABUBD/U\nAAAAAAAAVAQ/1AAAAAAAAFREZeaoqXU+gqLy3Cgnl8eqefD33XdfjD1Hvex8M10ld10cdNBByevP\nf/7zMR4/fnyMPU+47Pabjea961w/IaTlRT/5yU/GeP369YXbKKve86E0K53jwo9zUa6/z6mgy7S8\nsue1//SnP43xpZdeWrhPOl+RtxXtfzbZZJNkWa6UbTPx+Rl0foIbb7wxWaZt7owzzohxroy6loK+\n5JJLkmU6T8JFF12ULNNStStWrIixX1dnnXVWjB966KFkmc6F4SVz8Q/aby5evDhZpm1u3LhxMdbS\n0CGk86B88YtfTJb94Ac/aPN9582bl7wePXp0jP0en5uPrCuVnYfD+y59rfPSPPvss8l62gf5NrRd\n6Xt76XXtu7T9ervX+Zz83tqrV68Ya7ucNGlSsp6WnvZ5i+68884Y+xw4XS13HrXf8Dmcpk+fHuOn\nnnoqxn6/0D7L+y/t2/Re+PTTTxfuh14XGocQwgsvvBBjP8czZ86MsbapdevWJevptfbrX/86Wab3\nU517x/te3YYfj9xcWACqhRE1AAAAAAAAFcEPNQAAAAAAABWx0ZutlLcBAAAAAABQYYyoAQAAAAAA\nqAh+qAEAAAAAAKgIfqgBAAAAAACoCH6oAQAAAAAAqAh+qAEAAAAAAKgIfqgBAAAAAACoiP8H1PPm\nB4ENlmsAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<matplotlib.figure.Figure at 0x7fba42043860>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "metadata": {
        "id": "FcUNungS3fYz",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "# 2. Image Classification"
      ]
    },
    {
      "metadata": {
        "id": "o6_Nn7BV3jFN",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "## 2.1    Deep CNN"
      ]
    },
    {
      "metadata": {
        "id": "YT_3wLyM3RhS",
        "colab_type": "code",
        "outputId": "6dcb5bd5-cdd4-4bef-a330-450f68a2f013",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "cell_type": "code",
      "source": [
        "from keras.applications import VGG16\n",
        "from keras.preprocessing import image\n",
        "import numpy as np\n",
        "from keras.datasets import mnist\n",
        "from keras.models import Sequential\n",
        "from keras.layers import Dense, Flatten, Conv2D, MaxPooling2D, Dropout\n",
        "from keras.utils import np_utils\n",
        "import matplotlib.pyplot as plt\n",
        "import io, gzip, requests, gc\n",
        "plt.viridis()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<matplotlib.figure.Figure at 0x7fa0a8d8eeb8>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "metadata": {
        "id": "MVUZGyjG3lkA",
        "colab_type": "code",
        "outputId": "40eaa540-80d3-4f95-a262-9362fd38807f",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 996
        }
      },
      "cell_type": "code",
      "source": [
        "model = Sequential()\n",
        "model.add(Conv2D(filters = 32,kernel_size=(3, 3), \n",
        "                 activation='relu', strides=(1, 1), \n",
        "                 padding='valid',\n",
        "                 input_shape=(28,28,1)))\n",
        "model.add(Conv2D(filters = 32,kernel_size=(3, 3), \n",
        "                 activation='relu', strides=(1, 1), \n",
        "                 padding='valid'))\n",
        "model.add(MaxPooling2D(pool_size=(2,2)))\n",
        "model.add(Dropout(0.25))\n",
        "model.add(Flatten())\n",
        "model.add(Dense(64, activation='relu'))\n",
        "model.add(Dropout(0.5))\n",
        "model.add(Dense(10, activation='softmax'))\n",
        "\n",
        "model.compile(loss='categorical_crossentropy',\n",
        "              optimizer='adam',\n",
        "              metrics=['accuracy'])\n",
        "\n",
        "model_training = model.fit(X_train, Y_train,\n",
        "          epochs=15,\n",
        "          batch_size=128,\n",
        "         verbose = 1)\n",
        "\n",
        "plt.plot(model_training.history[\"acc\"])"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:1259: calling reduce_prod (from tensorflow.python.ops.math_ops) with keep_dims is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "keep_dims is deprecated, use keepdims instead\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:2880: calling reduce_sum (from tensorflow.python.ops.math_ops) with keep_dims is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "keep_dims is deprecated, use keepdims instead\n",
            "Epoch 1/15\n",
            "60000/60000 [==============================] - 8s 127us/step - loss: 0.5987 - acc: 0.7908\n",
            "Epoch 2/15\n",
            "60000/60000 [==============================] - 6s 108us/step - loss: 0.4019 - acc: 0.8570\n",
            "Epoch 3/15\n",
            "60000/60000 [==============================] - 6s 108us/step - loss: 0.3447 - acc: 0.8780\n",
            "Epoch 4/15\n",
            "60000/60000 [==============================] - 7s 108us/step - loss: 0.3149 - acc: 0.8872\n",
            "Epoch 5/15\n",
            "60000/60000 [==============================] - 6s 108us/step - loss: 0.2897 - acc: 0.8960\n",
            "Epoch 6/15\n",
            "60000/60000 [==============================] - 7s 109us/step - loss: 0.2738 - acc: 0.9001\n",
            "Epoch 7/15\n",
            "60000/60000 [==============================] - 7s 109us/step - loss: 0.2557 - acc: 0.9067\n",
            "Epoch 8/15\n",
            "60000/60000 [==============================] - 7s 109us/step - loss: 0.2464 - acc: 0.9098\n",
            "Epoch 9/15\n",
            "60000/60000 [==============================] - 7s 109us/step - loss: 0.2325 - acc: 0.9144\n",
            "Epoch 10/15\n",
            "60000/60000 [==============================] - 6s 108us/step - loss: 0.2225 - acc: 0.9175\n",
            "Epoch 11/15\n",
            "60000/60000 [==============================] - 6s 108us/step - loss: 0.2163 - acc: 0.9197\n",
            "Epoch 12/15\n",
            "60000/60000 [==============================] - 7s 109us/step - loss: 0.2076 - acc: 0.9237\n",
            "Epoch 13/15\n",
            "60000/60000 [==============================] - 6s 108us/step - loss: 0.2014 - acc: 0.9243\n",
            "Epoch 14/15\n",
            "60000/60000 [==============================] - 7s 109us/step - loss: 0.1934 - acc: 0.9275\n",
            "Epoch 15/15\n",
            "60000/60000 [==============================] - 6s 108us/step - loss: 0.1875 - acc: 0.9294\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[<matplotlib.lines.Line2D at 0x7fba320ce518>]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 32
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAecAAAFKCAYAAAAnj5dkAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzt3X10lOWd//H3ZGYyyWQmDxNmIAmB\nhPAkYbHQSFXU1TbR+qO17baV1CJaqW537dntnuPpauhZ9hwPVPytu3XbrrqVbvtrQeMitmzbBUSx\n2hWlK4INyNNAAiSBzJCHyeQ5M/fvj8AACoSHmdyTyed1jiczmYd8v8Tkk/u67vu6LIZhGIiIiEjS\nSDO7ABERETmXwllERCTJKJxFRESSjMJZREQkySicRUREkozCWUREJMnYzC7gtECgM67vl5fnpK2t\nO67vmYzUZ2pRn6lFfaaWePfp9bov+FjKHjnbbFazSxgR6jO1qM/Uoj5Ty0j2mbLhLCIiMlopnEVE\nRJKMwllERCTJKJxFRESSjMJZREQkySicRUREkozCWUREJMkonEVERJKMwllERCTJKJxFRESSTNKs\nrS0iIpJM+gYiBDt6OdnRw8mOXipmF+JOH5ljWoWziIiMSYORKK2dfQTbewh29BI49THY3kOgo5dQ\nV/85z//waAd//YXyEalN4SwiIikpahh0hPtPhW4PwfZeAqc+Bjt6ae3sxTA+/jprmgVPtoNrJufh\nzc1gXE4m43IzuOWTk+jv6f/4CxJA4SwiIqOSYRiEewbOPeo968j3ZEcvg5Hox15nAXLdDqYW5TAu\nJzMWwKc/5rkdpKVZPva6HJeDgMJZRERkSN9AhIbjnRxqCnGoOcTxk10EOnrp64+c9/muTDvFvqyh\no96cDMblZuI99TE/OwO7LbnPh1Y4i4hIUokaBs3BLg41hTjcHOJQU4hjgS6iZ41BO9KtQ2F7asjZ\ne9bH/JwMMh2jO95Gd/UiIjLqtYf7zgniw80hes86Irbb0phSmH3mv4Js8nMysFg+PvScKhTOIiIy\nYvr6IzScODU83dTBoeYQraG+c55TkO9kSsHpMM6hyJuFzZrcw9DxpnAWEZGEiEYNmk92xeaJDzWF\naPzI8HS2084npo6j9NQRcWmBG2eG3cSqk4PCWURE4uL08PTpo+L6450fH54uyj5zVDwGhqevlMJZ\nRETOETUM+voj9A1EYh97z7rfG/vcIH0DEVrD/Xx4uJW2Tg1Px4vCWUQkRXSE+wiGhi4v6uuP0HtW\nmPYPnHt/KGgHY5+LBe9AhP6Bj18bPJxzhqcLsymdkI0zQxFzpfQvJyIySvX0DbLvaDt7Dreyp6GN\npmDXZb9HmsVCRroVR7qVrEw7edkZZNiH7mekW3Gcuu2wn3s/I902dNuexrTScVgGBzU8HUcKZxGR\nUWIwEuVQU4g99UNhfLgpRCQ6dHJVuj2N2VM8TPS6yEi3xgI2FqZ2K45021m3hx6zWS1XHapej5NA\noDMeLcopCmcRkSRlGAaNwS721Lexp76VfUfbYytiWSwwpSCba0o8lJfkMaUwJ+lXvZJLp3AWEUki\nraFePmwYCuM99W10nLUz0gSPk1kleZSXeJgxKVeXHKWwSwrnlStXsmvXLiwWCzU1NcyZMyf22JYt\nW3jmmWdIT09n4cKFLF68GIAnn3yS9957j8HBQf7yL/+S22+/PTEdiIiMYt29g+w70jZ0dNzQSvPJ\n7thj2VnpXF8+nlmTPcwqycOTnWFipTKShg3n7du309DQQG1tLX6/n5qaGmprawGIRqM8/vjjvPLK\nK+Tm5vLggw9SWVlJfX09Bw4coLa2lra2Nr70pS8pnEVEGJo39jd2xIaqDzd3xhblcNitzCnLZ1bJ\nUBgXjcvSSVZj1LDhvG3bNiorKwEoKyujo6ODcDiMy+Wira2N7OxsPB4PANdffz1vv/02X/jCF2JH\n19nZ2fT09BCJRLBarQlsRUQk+UQNg8ZAV2yYet/RttilSmkWC1MKs5lVksesEg9TCrN1HbAAlxDO\nwWCQ8vLy2H2Px0MgEMDlcuHxeOjq6qK+vp6ioiLeffdd5s+fj9Vqxel0ArBu3TpuueUWBbOIpLxo\n1KA11Mvxtm5OtPZwNNjFzn0thLoHYs8pHJfFrMlDYTxjUu6o3z1JEuOy/68wzloT1WKx8MQTT1BT\nU4Pb7WbixInnPHfLli2sW7eOn/70p8O+b16eE5stvgHu9brj+n7JSn2mFvWZ3AxjKICbgl00BcI0\nBbpoDIRpCnbRHOxiMHLuAh6ebAe3fXIin5ju5dppXvJzMk2qPLFG6/fzco1Un8OGs8/nIxgMxu63\ntLTg9Xpj9+fPn8/atWsBeOqppygqKgLgrbfe4tlnn+X555/H7R6+mba27mGfczm8XveYuO5OfaYW\n9Zk8wj0DnGjt5nhrNyfaejjR2s2JtqHbfWetF31apsNGsS+L8XlOxnucjM/L5NqZ48lIIzZvHO0f\nTPq+r8Ro+H7GQ7z7vFjQDxvOCxYs4Ic//CHV1dXs3r0bn8+Hy+WKPf7Nb36TVatWkZmZydatW/nG\nN75BZ2cnTz75JD/72c/Izc2NTxciInHW0zdIS1sPJ9pOhXDr0O0Trd109Q5+7PnptjR8eU4meDJP\nBbCT8aduuzPtHzt5a6yElsTfsOE8b948ysvLqa6uxmKxsHz5ctavX4/b7aaqqoq7776bBx54AIvF\nwkMPPYTH44mdpf2d73wn9j6rVq2isLAwoc2IiHyUYRi0tPXQGOyKHf0ePxXCHeH+jz3fmmbBm5vJ\ntIm5+PIymXDqKHi8x0mu20Gazp6WEWAxzp5ENlG8/7ocK3+xqs/Uoj6vXkdXP4dP7R98uDlEfXPo\nY0fBFiA/J4PxHicT8pz4PGdCOD8nA2tafM6Y1vcztSTVsLaISLLq7R+k4XjnUBA3DYXxydC52xb6\n8jL5syn5FPtcTPA48Xmc+HIzsMf5BFSReFI4i8ioMBiJ0hjo4nDzmaPipmAXZ4/9ZTvtXFuWP7Rt\nYUE2JQXZuDK1xKWMPgpnEUk6hmHQ0t7D4eYQh5s6OdwcouFEJwODZy5TctitTJ+Ye1YQu8nPztCK\nWpISFM4iYrpQV/85Q9OHPzJPnGaxMNGXxZSCbEoLsiktzKYwP4u0NAWxpCaFs4iMqL6BCHX+IO9/\neCIWyCdDvec8x5ebyewp+ZQWDB0VF4934bBrjljGDoWziCRcqLufXQeDvL8/yO761nOGp12ZduaU\n5Q8dFRdmUzLBjduZbmK1IuZTOItIQrS0dfP+gSDv7w9woLEjduJW4bgsrps1gYK8DKYUZJOfo3li\nkY9SOItIXBiGQf3xzqFAPhCgMdAFDF1TXDYxh3nTvMydNo7xHueYuS5W5EopnEXkig1Gouw70s77\nBwK8fyBIW+fQNcY2axrXluUzd7qXa6eOIydLw9Qil0PhLCKXpadvkD8dOsnOA0F2+U/S0zd0VnVW\nho0byicwb/o4yks9ZKTr14vIldJPj4gMqz3cx84DQd4/EOTDhlYGI0MTyPnZDhbMnsDc6V6mTczB\nZo3PspciY53CWUTOq/lkFzv2B9h5IIi/KRT7/CSfi09MG8e86V6KfS6dzCWSAApnEQEgahgcagrx\n/v4AOw4EOdE6tMd6msXCzEm5zD11Qte43EyTKxVJfQpnkTGsfyDC3iNt7NgfZOfBIKGuoS0U0+1p\nzJs+FMbXTh2n9alFRpjCWWSMiEYNmk52nVoes5PDTSGOBcJEokPzx26nnZvnFDB3mpdZJXmka0Uu\nEdMonEVSkGEYnAz1xkL4cHOI+uOd9A1EYs+xWdMomeBm2sRcPjFtHFOLcrRWtUiSUDiLpIBwz8Cp\nHZzObKfY2T0Qe9zC0MpcpzeNKC1wM9Hr0tnVIklK4SwyyvQNRGg43hnbvelwc4hA+7kbR+RnZ1Ax\nM4/SAjdTCrKZNN5NpkM/7iKjhX5aRZJYJBqlMdAVC+FDTZ00BbuInl6omqGNI2ZP8ZzZTrEgm2yt\nyCUyqimcRZKEYRg0B7v43z3HqW/u5FBziCPHO+k/awendFsaZUVnQri0MBuvNo4QSTkKZxGT9Q1E\n+J8/NfPqH49yoq0n9vk0i4Ui79A88ZRTWykWebOwpmmeWCTVKZxFTNIR7uO1HcfYuqORrt5BbNY0\nFlxbSHG+k9LCoXlihy5nEhmTFM4iI6wxEGbTH4/yzu7jDEYMXJl27lpQwm3zJjK1JF9bKYqIwllk\nJBiGwZ6GNjZtP0LdoVYAxnuc3HFdMTfOnqAFP0TkHApnkQQajER5d88JNm0/yrFAGIDpxbncMb+Y\na6eOI00nconIeSicRRKgq3eAN95v5LX3jtEe7ifNYmH+NT7umD+J0oJss8sTkSSncBaJo0B7D6/+\n8ShvfdBM30CEjHQrt19XTGXFRMblaDcnEbk0CmeROPA3drBp+xHe2x/AMCDP7eALN5Vyy7WFODP0\nYyYil0e/NUSuUDRq8P6BIJu2H+FgYwcAk8e7uWN+MRUzfVq3WkSumMJZ5DL19Uf4w6lFQ1rahxYN\nubYsnzvmT2LGpFyt1iUiV+2SwnnlypXs2rULi8VCTU0Nc+bMiT22ZcsWnnnmGdLT01m4cCGLFy8e\n9jUio1F7uI/X3jvGG++fWTTklmsLuf26YgrHZZldnoikkGHDefv27TQ0NFBbW4vf76empoba2loA\notEojz/+OK+88gq5ubk8+OCDVFZWcuTIkQu+RmS0ORYIs3n7Ud7Zc+6iIZ+eN1EbTIhIQgwbztu2\nbaOyshKAsrIyOjo6CIfDuFwu2trayM7OxuPxAHD99dfz9ttvc/To0Qu+RmQ0MAyDPfWnFg05fNai\nIfOLubFci4aISGING87BYJDy8vLYfY/HQyAQwOVy4fF46Orqor6+nqKiIt59913mz59/0deIJKue\nvkE+bGhj9+FW/nToJMGOoT2SZxTncsf8ScyZmq9FQ0RkRFz2CWHGWfvIWiwWnnjiCWpqanC73Uyc\nOHHY11xIXp4Tmy2+RyNerzuu75es1OeViUYNDh5r5/39Lby/L8De+lYi0aH/V50ZNm6dN5G7bpnC\ntOK8uH7d4ej7mVrUZ2oZqT6HDWefz0cwGIzdb2lpwev1xu7Pnz+ftWvXAvDUU09RVFREX1/fRV9z\nPm1t3Zdd/MV4ve4xsYGA+rw8bZ191B0+ye7DreypbyPcMwCABSgtzKa8xMPsKR6mFGbHtmYcyX9f\nfT9Ti/pMLfHu82JBP2w4L1iwgB/+8IdUV1eze/dufD7fOcPT3/zmN1m1ahWZmZls3bqVb3zjGxQU\nFFz0NSIjpX8gwv5j7dQdamV3fSuNga7YY3luBzfNKWB2qYdZJR5cmXYTKxUROWPYcJ43bx7l5eVU\nV1djsVhYvnw569evx+12U1VVxd13380DDzyAxWLhoYcewuPx4PF4PvYakZFgGAZNwS7qDrey+3Ar\n+462MzAYBcBuS2P2FA+zSzyUT8mnMN+pa5JFJClZjEuZEB4B8R4S0TBLarlYn+GeAfbUt8aOjts6\n+2KPTfRmUV7qYXZpPtOLc7DH+byGeNP3M7Woz9SSVMPaIslmMBLlUFModnRc3xzi9F+Yrkw7n5o1\nnvISD+WlHvLcDlNrFRG5EgpnGRWOn+zizfcbqTt0kr1H2ujpiwBgTbMwrTj31NGxh8kT3LrcSURG\nPYWzJLXDzSFe2HIgtrEEgC83k+tnDYXxzMl5ZDr0v7GIpBb9VpOkFO4Z4OXf+3lzZxMGMG+Gj1mT\nc5ld6sGX5zS7PBGRhFI4S1KJRg1+v6uJ9b/309U7SOG4LL5eOY1brps8Jk44EREBhbMkkYONHazZ\nvJ+GE51kpFtZ9OmpfOaTE7UvsoiMOQpnMV2oq591b/j5w5+aAbihfAJfva2MXJfOtBaRsUnhLKaJ\nRKO8vqORX711mJ6+QSZ6XSy+fTrTi3PNLk1ExFQKZzHFviNtrHl1P8cCXTgdNr5eNZ1b5xbG1rMW\nERnLFM4yoto6+/jPrQd5Z88JAG6eU8CX/7yM7Kx0kysTEUkeCmcZEYORKFv+9xi//p/D9PVHmDzB\nzeLbp1NWmGN2aSIiSUfhLAm3u76Vta/up/lkN1kZNhZ9dga3zCkkLU0reYmInI/CWRLmZEcvta8f\n4H/3BbAAt80t4ku3TNHWjCIiw1A4S9wNDEbZtP0Iv9lWT/9AlLKibBZXzWDyhAvvwCIiImconCWu\nPvCfZO2W/bS09ZDttHPv7TO4YfYEbUYhInIZFM4SFy3tPby45QA7DwZJs1iorJjIF2+agjND/4uJ\niFwu/eaUq9I/EOF37zTwu3eOMBiJMr04l8VV05noc5ldmojIqKVwlitiGAbvHwjy4msHCHb0kutK\n5+5PT+VT14zHoiFsEZGronCWy3a8tZu1W/ZTd6gVa5qFz35qEp+/sUT7KouIxIl+m8ol6xuI8Ju3\n69n47hEiUYNZJXl8vWo6BflZZpcmIpJSFM5ySf506CS/2LSPYEcvnmwH1Z+exidneDWELSKSAApn\nuai2zj5efO0Af9zbQpplaAj7CwtKcaRbzS5NRCRlKZzlvKJRg63vN7L+TT89fRHKirJZcsdMinUW\ntohIwimc5WMajnfy8417qT/eidNhY8lnZ3DLtYVaSEREZIQonCWmp2+QV946xGvvHcMw4Pry8Sz6\n9DRytJ2jiMiIUjgLhmGwY3+AtVsO0NbZx/i8TBbfMYPyEo/ZpYmIjEkK5zEu2N7Dmlf3s8t/EpvV\nwl0LSlh4w2TsNp3wJSJiFoXzGDUYifLqH4/y6/85TP9AlGsm57H4dl2zLCKSDBTOY9DBYx38fNNe\nGgNduJ127rtjJteXa9lNEZFkoXAeQ8I9A6x7w8+bu5oAuOXaQr5yaxmuTLvJlYmIyNkUzmOAYRi8\ns/sEL75+gM7uAYq8WSy5YwbTJuaaXZqIiJzHJYXzypUr2bVrFxaLhZqaGubMmRN7bM2aNWzYsIG0\ntDRmz57NsmXLOHHiBDU1NfT39xONRnnssceYPXt2wpqQCzve2s0vNu3jw4Y20m1pfPXWMqquK8Zm\nTTO7NBERuYBhw3n79u00NDRQW1uL3++npqaG2tpaAMLhMKtXr2bz5s3YbDYeeOABdu7cyaZNm6iq\nqqK6upodO3bwL//yL6xevTrhzcgZA4MRfrutgd+908BgxGBOWT6Lq6YzLjfT7NJERGQYw4bztm3b\nqKysBKCsrIyOjg7C4TAulwu73Y7dbqe7uxun00lPTw85OTnk5eXR3t4OQCgUIi8vL7FdyDn21Lfy\ni037ONHWQ57bwT2V05g3XZtUiIiMFsOGczAYpLy8PHbf4/EQCARwuVw4HA4efvhhKisrcTgcLFy4\nkNLSUu6//36+8pWv8Ktf/YpwOMwLL7wwbCF5eU5scb621ut1x/X9ktXpPts7+1i9oY43dhwjzQJ3\n3TyFr392Js6M1Djha6x9P1Od+kwt6jO+LvuEMMMwYrfD4TDPPfccGzduxOVycd9997F3715ef/11\n7rzzTv7qr/6KrVu3smrVKn70ox9d9H3b2rovv/qL8HrdBAKdcX3PZOT1ujnREuLNXU2s2+qnu2+Q\nkglu7vvsTCZPcNPV2UtXZ6/ZZV61sfT9VJ+pQ32mlnj3ebGgH/asIJ/PRzAYjN1vaWnB6/UC4Pf7\nKS4uxuPxkJ6eTkVFBXV1dezYsYObb74ZgAULFlBXV3e1PcgF1DeH+P4v3+P/bdxH1DD4etV0vrek\ngskTxsZfsSIiqWjYcF6wYAGbNm0CYPfu3fh8PlyuoW0Di4qK8Pv99PYOHZnV1dVRUlLC5MmT2bVr\nFwAffPABkydPTlT9Y9qm7Uf4239+A39jiOtm+ljx4PV85pMTSUvT3LKIyGg27LD2vHnzKC8vp7q6\nGovFwvLly1m/fj1ut5uqqiqWLl3KkiVLsFqtzJ07l4qKCiZNmsSyZcvYuHEjAMuWLUt4I2PN3oY2\nXnr9IJ6cDJbcMYM/m5JvdkkiIhInFuPsSWQTxXu+IpXnQLp7B/iHn26nvbOf//s3N5OXmfpryaTy\n9/Ns6jO1qM/UklRzzpJ8frl5P62hPu5aUML0SbpMTUQk1SicR5l39hznnT0nKCvMZuGNmssXEUlF\nCudR5GRHL7/YtB+H3cqDn5+FNU3fPhGRVKTf7qNE1DBY/ds99PQN8rXKafjynGaXJCIiCaJwHiU2\nbz/K3iPtzJ02jpvnFJhdjoiIJJDCeRQ4cqKTl3/vJycrnfvvnKk1skVEUpzCOckNDEb4yX/tIRI1\neGDhNbid6WaXJCIiCaZwTnLr3jhEY7CLz8ybqIVGRETGCIVzEtt9uJVX//coBflOvnJbmdnliIjI\nCFE4J6lwzwCrf7sHa5qFhz5fjsMe3+00RUQkeSmck5BhGPx8417aw/188eZS7TAlIjLGKJyT0Nt1\nx3lvX4DpE3O481NaBUxEZKxROCeZQHsPa17dT0a6lW9+bpa2fxQRGYMUzkkkGjX4yW/20NsfYfHt\n0xmXm2l2SSIiYgKFcxL53TsNHDzWwXUzfdxQPsHsckRExCQK5yRxuDnEr/9wmDy3g3vvmKFVwERE\nxjCFcxLoGzizCtjShdfgyrSbXZKIiJhI4ZwEXtp6kOOt3dx+XTGzSjxmlyMiIiZTOJvsA3+QrTsa\nKfJm8eU/n2J2OSIikgQUziYKdfXz099+iM06tAqY3aZVwEREROFsGsMw+Nl/7yXUPcCX/7yMYp/L\n7JJERCRJKJxN8uauJnYeDHLN5Dyqris2uxwREUkiCmcTnGjt5oXXDuB02Fi68BrSdNmUiIicReE8\nwgYjUf79v/bQPxBlyWdn4MnOMLskERFJMgrnEfabt+s53BzihvLxzL9mvNnliIhIElI4j6CDjR38\n5u0G8rMdfL1qhtnliIhIklI4j5De/kGe/689GIbBNz83C2eGzeySREQkSSmcR8iLrx2gpb2Hz14/\niRmT8swuR0REkpjCeQTs2B/gzV3NTPK5+NLNWgVMREQuTuGcYO3hPn7233ux29J48K5ybFb9k4uI\nyMVd0sTnypUr2bVrFxaLhZqaGubMmRN7bM2aNWzYsIG0tDRmz57NsmXLAFi9ejUbNmzAZrOxfPny\nc14zVhiGwU9/9yHhngG+XjWdonFZZpckIiKjwLDhvH37dhoaGqitrcXv91NTU0NtbS0A4XCY1atX\ns3nzZmw2Gw888AA7d+4kKyuL3/72t7z88svs27eP1157bUyG8+s7Gqk71MrsUg+fnldkdjkiIjJK\nDBvO27Zto7KyEoCysjI6OjoIh8O4XC7sdjt2u53u7m6cTic9PT3k5OTw6quvcuedd2Kz2SgvL6e8\nvDzhjSSbpmAXL209iCvTzjf+zzVYtAqYiIhcomHDORgMnhOuHo+HQCCAy+XC4XDw8MMPU1lZicPh\nYOHChZSWltLY2IjVamXp0qUMDg7y2GOPMXPmzIt+nbw8J7Y478rk9brj+n6XamAwyopfvsfAYJRH\nvv5Jpk8Zl9CvZ1afI019phb1mVrUZ3xd9sW2hmHEbofDYZ577jk2btyIy+XivvvuY+/evRiGQSQS\n4fnnn+e9995j2bJlvPzyyxd937a27suv/iK8XjeBQGdc3/NSvfx7P/5jHdw0p4BpBYmtw8w+R5L6\nTC3qM7Wozyt/vwsZNpx9Ph/BYDB2v6WlBa/XC4Df76e4uBiPxwNARUUFdXV1jBs3jilTpmCxWKio\nqKCxsfFqexg19h9t53fbGvDmZvC1z0wzuxwRERmFhr2uZ8GCBWzatAmA3bt34/P5cLmG9h4uKirC\n7/fT29sLQF1dHSUlJdxyyy384Q9/AIYCvKCgIFH1J5Xu3kF+8l97wAIPfr6cTIdWARMRkcs3bHrM\nmzeP8vJyqqursVgsLF++nPXr1+N2u6mqqmLp0qUsWbIEq9XK3LlzqaioAODNN99k0aJFAPzDP/xD\nYrtIEmu37OdkqJfP31jC1KIcs8sREZFRymKcPYlsonjPV4z0HMgf97bwzK/qKC1w89jiT47YYiOa\n60kt6jO1qM/UMpJzzlquKk5+83Y9NquFBz+vVcBEROTqKEXioKdvkGOBMFMKc5jgcZpdjoiIjHIK\n5zg41BzCMKCsKNvsUkREJAUonOPA39gBwNRCnQQmIiJXT+EcB/7GEABlOkNbRETiQOF8laKGwaGm\nDny5mWRnpZtdjoiIpACF81U60dpNV++g5ptFRCRuFM5X6eCp+WYNaYuISLwonK/S6ZPBynQymIiI\nxInC+Sr5G0M47FYm+rLMLkVERFKEwvkqdPcO0hTsorTAjTVN/5QiIhIfSpSrcKi5AwPNN4uISHwp\nnK+Crm8WEZFEUDhfhTMng+kyKhERiR+F8xWKGgb+phDj8zJxO7X4iIiIxI/C+Qo1B7vo6RvUkLaI\niMSdwvkK+ZuG5punKpxFRCTOFM5XSCuDiYhIoiicr5C/sYOMdCtF47T4iIiIxJfC+Qp09Q7QfLKb\n0oJs0tIsZpcjIiIpRuF8BXR9s4iIJJLC+Qqcvr55qraJFBGRBFA4XwF/01A4T9FOVCIikgAK58sU\njRocagpRkO/ElWk3uxwREUlBCufL1BTsorc/ov2bRUQkYRTOl+lg0+nrmzXfLCIiiaFwvkz+Y1p8\nREREEkvhfJkONoXIdFgp1OIjIiKSIArnyxDuGeBEazdTCnNIs2jxERERSQyF82XQ/s0iIjISLimc\nV65cyaJFi6iuruaDDz4457E1a9awaNEivva1r7FixYpzHgsGg1x33XW8++678avYRKevb9ZOVCIi\nkkjDhvP27dtpaGigtraWFStWnBPA4XCY1atXs2bNGl544QX8fj87d+6MPf7kk09SXFycmMpNcPDY\n6cVHdOQsIiKJM2w4b9u2jcrKSgDKysro6OggHA4DYLfbsdvtdHd3Mzg4SE9PDzk5ObHXZWVlMX36\n9ASWP3Ii0SiHmzspHJeFM0OLj4iISOIMG87BYJC8vLzYfY/HQyAQAMDhcPDwww9TWVnJbbfdxrXX\nXktpaSn9/f38+Mc/5u/+7u8SV/kIawx00TcQ0XyziIgknO1yX2AYRux2OBzmueeeY+PGjbhcLu67\n7z727t3Lli1b+OpXv0p29qVRXqD1AAARBElEQVQHWV6eE5vNernlXJTX647be/3xQBCAuTPHx/V9\n4yHZ6kkU9Zla1GdqUZ/xNWw4+3w+gsFg7H5LSwterxcAv99PcXExHo8HgIqKCurq6vjDH/5ANBpl\nzZo1HDlyhA8++ICnn36aadOmXfDrtLV1X20v5/B63QQCnXF7v517WwDwZTvi+r5XK959Jiv1mVrU\nZ2pRn1f+fhcy7LD2ggUL2LRpEwC7d+/G5/PhcrkAKCoqwu/309vbC0BdXR0lJSW8+OKLvPTSS7z0\n0kvceuutLF++/KLBPBr4mzpwOmxMyHeaXYqIiKS4YY+c582bR3l5OdXV1VgsFpYvX8769etxu91U\nVVWxdOlSlixZgtVqZe7cuVRUVIxE3SMq1NVPS1sPs6d4tPiIiIgk3CXNOT/yyCPn3J85c2bsdnV1\nNdXV1Rd87RNPPHGFpSWP2PXN2olKRERGgFYIuwT+xhCgzS5ERGRkKJwvgb+xAwtafEREREaGwnkY\nkWiUw8dDFHmzyHRc9pVnIiIil03hPIxjLV30D0Q1pC0iIiNG4TyMg7GdqBTOIiIyMhTOw4htE1mk\n+WYRERkZCudhHGzsICvDxgSPFh8REZGRoXC+iI6ufoIdvZQV5WDR4iMiIjJCFM4XcWZIW/PNIiIy\nchTOF3E6nKfq+mYRERlBCueLONjYgcUCpQpnEREZQQrnCxiMRKk/3slEr4uMdC0+IiIiI0fhfAFH\nW8IMDGrxERERGXkK5ws4vfjIVF3fLCIiI0zhfAE6U1tERMyicL4Af2MIV6YdX26m2aWIiMgYo3A+\nj7bOPk6GepmqxUdERMQECufz0HraIiJiJoXzefibtBOViIiYR+F8Hv7GEGkWC6UFOnIWEZGRp3D+\niNOLjxT7XDjSrWaXIyIiY5DC+SMaTnQyGIlqvllEREyjcP4I/zFd3ywiIuZSOH/EwaYQoHAWERHz\nKJw/wt/YQbbTjjcnw+xSRERkjFI4n6U11EtbZx9lWnxERERMpHA+i//UkPZUDWmLiIiJFM5n0WYX\nIiKSDBTOZznY2IE1zULJBLfZpYiIyBimcD5lYDBCw6nFR9LtWnxERETMY7uUJ61cuZJdu3ZhsVio\nqalhzpw5scfWrFnDhg0bSEtLY/bs2SxbtozBwUGWLVvGkSNHiEQifPe736WioiJhTcRDw/Ewkaih\nIW0RETHdsOG8fft2GhoaqK2txe/3U1NTQ21tLQDhcJjVq1ezefNmbDYbDzzwADt37sTv95OZmckL\nL7zAgQMHeOyxx1i3bl3Cm7kaB0/NN+tkMBERMduw4bxt2zYqKysBKCsro6Ojg3A4jMvlwm63Y7fb\n6e7uxul00tPTQ05ODnfddRef+9znAPB4PLS3tye2iziI7USlZTtFRMRkw4ZzMBikvLw8dt/j8RAI\nBHC5XDgcDh5++GEqKytxOBwsXLiQ0tLSc17/85//PBbUycowDPyNHeS40snP1uIjIiJirkuacz6b\nYRix2+FwmOeee46NGzficrm477772Lt3LzNnzgSG5qN3797Ns88+O+z75uU5sdnieyKW13tpZ123\ntHbTHu7nhj8rwOcbfUfOl9rnaKc+U4v6TC3qM76GDWefz0cwGIzdb2lpwev1AuD3+ykuLsbj8QBQ\nUVFBXV0dM2fO5D//8z95/fXX+bd/+zfsdvuwhbS1dV9pD+fl9boJBDov6bnb95wAoHhc1iW/Jllc\nTp+jmfpMLeoztajPK3+/Cxn2UqoFCxawadMmAHbv3o3P58PlcgFQVFSE3++nt7cXgLq6OkpKSjh6\n9CgvvvgiP/rRj3A4HPHoIaHOLD4y+o6aRUQk9Qx75Dxv3jzKy8uprq7GYrGwfPly1q9fj9vtpqqq\niqVLl7JkyRKsVitz586loqKCf/7nf6a9vZ2HHnoo9j6rV68mPT09oc1cKX+TFh8REZHkYTHOnkQ2\nUbyHRC51+KF/IMLD//Imkye4+d6S5L4W+3w0nJRa1GdqUZ+pJamGtVNd/fHOocVHCnV9s4iIJIcx\nH86abxYRkWQz5sNZK4OJiEiyGdPhbBgG/qYQeW4HHi0+IiIiSWJMh3Owo5dQV782uxARkaQypsP5\n9Hzz1ELNN4uISPIY0+F8MHYymI6cRUQkeYzpcPY3hrBZLUwar8VHREQkeYzZcO7rj3C0JczkCW7s\ntjH7zyAiIklozKZS/fEQUUOLj4iISPIZs+Gs65tFRCRZjdlw9jeGAJ0MJiIiyWdMhrNhGBxs7CA/\n20GeO/m3tBQRkbFlTIZzS3sP4Z4BHTWLiEhSGpPhHNvsQieDiYhIEhqj4az5ZhERSV5jNJw7sNvS\nmDTeZXYpIiIiHzPmwrm3f5CjgTAlE9zYrGOufRERGQXGXDodbgphGBrSFhGR5DXmwvlg06n5Zp0M\nJiIiSWrMhXNsm8gibRMpIiLJaUyFs2EYHGoKMS4ngxyXFh8REZHkNKbC+UTb0OIjWk9bRESS2ZgK\n59jiIwpnERFJYmMqnA/GwlnzzSIikrzGVDj7GztIt6Ux0avFR0REJHmNmXDu6RukMdBFSUG2Fh8R\nEZGkNmZS6lBzCAMNaYuISPIbM+F85vpmnQwmIiLJbQyFs1YGExGR0WFMhHPUMPA3duDLzSQ7K93s\nckRERC7KdilPWrlyJbt27cJisVBTU8OcOXNij61Zs4YNGzaQlpbG7NmzWbZsGQMDAzz66KM0NTVh\ntVr5/ve/T3FxccKaGM7xk9109w1y7dR802oQERG5VMMeOW/fvp2GhgZqa2tZsWIFK1asiD0WDodZ\nvXo1a9as4YUXXsDv97Nz505+85vfkJ2dzQsvvMC3vvUtnnrqqYQ2MRwtPiIiIqPJsOG8bds2Kisr\nASgrK6Ojo4NwOAyA3W7HbrfT3d3N4OAgPT095OTksG3bNqqqqgC48cYb2bFjRwJbGJ6/6VQ4a75Z\nRERGgWGHtYPBIOXl5bH7Ho+HQCCAy+XC4XDw8MMPU1lZicPhYOHChZSWlhIMBvF4PACkpaVhsVjo\n7+8nPf3C8715eU5sNmscWjrD63UDUH8iTEa6lbmzJmBNwWucT/eZ6tRnalGfqUV9xtclzTmfzTCM\n2O1wOMxzzz3Hxo0bcblc3Hfffezdu/eir7mQtrbuyy3lorxeN4FAJ929Axw93smMSbm0tnbF9Wsk\ng9N9pjr1mVrUZ2pRn1f+fhcy7GGkz+cjGAzG7re0tOD1egHw+/0UFxfj8XhIT0+noqKCuro6fD4f\ngUAAgIGBAQzDuOhRcyIdajq9+IiGtEVEZHQYNpwXLFjApk2bANi9ezc+nw+Xa2ht6qKiIvx+P729\nvQDU1dVRUlLCggUL2LhxIwBbt27lU5/6VKLqH9ZBnQwmIiKjzLDD2vPmzaO8vJzq6mosFgvLly9n\n/fr1uN1uqqqqWLp0KUuWLMFqtTJ37lwqKiqIRCK8/fbbfO1rXyM9PZ0nnnhiJHo5L3/T6cVHtGyn\niIiMDpc05/zII4+cc3/mzJmx29XV1VRXV5/z+Olrm80WNQwONYUYn5eJ26nFR0REZHRIvVOXz9Ic\n7KKnb1DraYuIyKiS0uGs+WYRERmNUjqcY5tdKJxFRGQUSe1wbuogI91K0bgss0sRERG5ZCkbzp3d\n/TSf7Ka0IJu0NIvZ5YiIiFyylA3nfQ1tgIa0RURk9EnZcN7b0AqgM7VFRGTUSd1wrh8K57IiLT4i\nIiKjS0qGczRqsP9IGwX5TrIy7GaXIyIicllSMpwbg1309EW0f7OIiIxKKRnO/tjiIxrSFhGR0SfF\nw1lHziIiMvqkZDjb7VaKvFkUavEREREZhS5pV6rRZvHt0/GOc3PyZNjsUkRERC5bSh45p1ksWhVM\nRERGrZQMZxERkdFM4SwiIpJkFM4iIiJJRuEsIiKSZBTOIiIiSUbhLCIikmQUziIiIklG4SwiIpJk\nFM4iIiJJRuEsIiKSZBTOIiIiScZiGIZhdhEiIiJyho6cRUREkozCWUREJMkonEVERJKMwllERCTJ\nKJxFRESSjMJZREQkyaRkOK9cuZJFixZRXV3NBx98YHY5CfPkk0+yaNEivvzlL7N582azy0mo3t5e\nKisrWb9+vdmlJMyGDRu46667+Iu/+AveeOMNs8tJiK6uLr797W9z7733Ul1dzVtvvWV2SXG1f/9+\nKisr+eUvfwlAc3Mz9957L/fccw9/+7d/S39/v8kVxsf5+rz//vtZvHgx999/P4FAwOQK4+OjfZ72\n1ltvMWPGjIR+7ZQL5+3bt9PQ0EBtbS0rVqxgxYoVZpeUEO+88w4HDhygtraW559/npUrV5pdUkI9\n88wz5OTkmF1GwrS1tfHjH/+YtWvX8uyzz/Laa6+ZXVJCvPLKK5SWlvKLX/yCp59+OqV+Pru7u3n8\n8ce54YYbYp/713/9V+655x7Wrl3L5MmTWbdunYkVxsf5+vzBD37A3XffzS9/+Uuqqqr4j//4DxMr\njI/z9QnQ19fHv//7v+P1ehP69VMunLdt20ZlZSUAZWVldHR0EA6HTa4q/q677jqefvppALKzs+np\n6SESiZhcVWL4/X4OHjzIrbfeanYpCbNt2zZuuOEGXC4XPp+Pxx9/3OySEiIvL4/29nYAQqEQeXl5\nJlcUP+np6fzkJz/B5/PFPvfuu+/ymc98BoDbbruNbdu2mVVe3Jyvz+XLl3PHHXcA536PR7Pz9Qnw\n7LPPcs8995Cenp7Qr59y4RwMBs/5gfd4PCkzxHI2q9WK0+kEYN26ddxyyy1YrVaTq0qMVatW8eij\nj5pdRkIdO3aM3t5evvWtb3HPPfekxC/x81m4cCFNTU1UVVWxePFi/v7v/97skuLGZrORkZFxzud6\nenpiv8Tz8/NT4nfR+fp0Op1YrVYikQhr167l85//vEnVxc/5+jx8+DB79+7lzjvvTPzXT/hXMFmq\nr066ZcsW1q1bx09/+lOzS0mIX/3qV3ziE5+guLjY7FISrr29nR/96Ec0NTWxZMkStm7disViMbus\nuPr1r39NYWEhq1evZu/evdTU1KT0eQRnS/XfRZFIhO9+97tcf/31HxsKThXf//73+d73vjciXyvl\nwtnn8xEMBmP3W1paEj43YJa33nqLZ599lueffx632212OQnxxhtvcPToUd544w2OHz9Oeno6EyZM\n4MYbbzS7tLjKz89n7ty52Gw2Jk2aRFZWFq2treTn55tdWlzt2LGDm266CYCZM2fS0tJCJBJJ2VEf\np9NJb28vGRkZnDhx4mNDpKnkscceY/LkyXz72982u5SEOHHiBIcOHeKRRx4BhrJl8eLFHztZLF5S\nblh7wYIFbNq0CYDdu3fj8/lwuVwmVxV/nZ2dPPnkkzz33HPk5uaaXU7C/OAHP+Dll1/mpZde4qtf\n/Sp//dd/nXLBDHDTTTfxzjvvEI1GaWtro7u7O6XmY0+bPHkyu3btAqCxsZGsrKyUDWaAG2+8Mfb7\naPPmzdx8880mV5QYGzZswG638zd/8zdml5Iw48ePZ8uWLbz00ku89NJL+Hy+hAUzpOCR87x58ygv\nL6e6uhqLxcLy5cvNLikhfve739HW1sZ3vvOd2OdWrVpFYWGhiVXJlRo/fjx33HEHd999NwDf+973\nSEtLub+dWbRoETU1NSxevJjBwUH+8R//0eyS4qauro5Vq1bR2NiIzWZj06ZN/NM//ROPPvootbW1\nFBYW8sUvftHsMq/a+fo8efIkDoeDe++9Fxg6GXe0f2/P1+cPf/jDETsY0paRIiIiSSb1/jQXEREZ\n5RTOIiIiSUbhLCIikmQUziIiIklG4SwiIpJkFM4iIiJJRuEsIiKSZBTOIiIiSeb/A25We7bH2wmo\nAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<matplotlib.figure.Figure at 0x7fba325d5438>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "metadata": {
        "id": "yr6N_GJ03lms",
        "colab_type": "code",
        "outputId": "4eeb86f5-b371-4455-e7cc-fc222c4bfd7c",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 68
        }
      },
      "cell_type": "code",
      "source": [
        "score = model.evaluate(X_test, Y_test, verbose=1)\n",
        "print('Test loss:', score[0])\n",
        "print('Test accuracy:', score[1])"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "10000/10000 [==============================] - 1s 111us/step\n",
            "Test loss: 0.22695973647534848\n",
            "Test accuracy: 0.9231\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "R9pnxEU_3vYI",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "## 2.2 Transfer Learning\n",
        "\n",
        "Reference: https://www.kaggle.com/anandad/classify-fashion-mnist-with-vgg16"
      ]
    },
    {
      "metadata": {
        "id": "NiXsqIpn87Se",
        "colab_type": "code",
        "outputId": "36ab8d97-f682-4a3c-f9c2-2d846700f283",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1346
        }
      },
      "cell_type": "code",
      "source": [
        "\n",
        "#from skimage.transform import resize\n",
        "import cv2\n",
        "from keras.applications import VGG19\n",
        "\n",
        "import tensorflow as tf\n",
        "fashion_mnist = tf.keras.datasets.fashion_mnist\n",
        "\n",
        "# Prepare the train and test dataset for the pretrained VGG19 model\n",
        "(x_train_rgb, y_train_rgb), (x_test_rgb, y_test_rgb) = fashion_mnist.load_data()\n",
        "# convert the gray-scale dataset to RGB with size 48*48*3\n",
        "x_train_rgb = [cv2.cvtColor(cv2.resize(i, (48,48)), cv2.COLOR_GRAY2BGR) for i in x_train_rgb]\n",
        "x_train_rgb = np.concatenate([arr[np.newaxis] for arr in x_train_rgb]).astype('float32')/255\n",
        "\n",
        "x_test_rgb = [cv2.cvtColor(cv2.resize(i, (48,48)), cv2.COLOR_GRAY2BGR) for i in x_test_rgb]\n",
        "x_test_rgb = np.concatenate([arr[np.newaxis] for arr in x_test_rgb]).astype('float32')/255\n",
        "\n",
        "\n",
        "Y_train_rgb = np_utils.to_categorical(y_train_rgb, 10)\n",
        "Y_test_rgb = np_utils.to_categorical(y_test_rgb, 10)\n",
        "print(x_train_rgb.shape)\n",
        "print(x_test_rgb.shape)\n",
        "print(Y_train_rgb.shape)\n",
        "print(Y_test_rgb.shape)\n",
        "\n",
        "\n",
        "conv_base = VGG19(weights='imagenet', include_top = False, input_shape=(x_train_rgb.shape[1:]))\n",
        "conv_base.trainable = False\n",
        "\n",
        "transfer_model = Sequential()\n",
        "transfer_model.add(conv_base)\n",
        "transfer_model.add(Flatten())\n",
        "transfer_model.add(Dense(256, activation = 'relu'))\n",
        "transfer_model.add(Dropout(0.25))\n",
        "transfer_model.add(Dense(128, activation = 'relu'))\n",
        "transfer_model.add(Dropout(0.25))\n",
        "transfer_model.add(Dense(10, activation = 'softmax'))\n",
        "transfer_model.summary()\n",
        "\n",
        "transfer_model.compile(loss = 'categorical_crossentropy',\n",
        "             optimizer = 'adam',\n",
        "             metrics = ['accuracy'])\n",
        "\n",
        "transfer_model.fit(x_train_rgb, Y_train_rgb,\n",
        "         epochs = 20,\n",
        "         batch_size = 128)\n",
        "\n",
        "\n",
        "\n"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "(60000, 48, 48, 3)\n",
            "(10000, 48, 48, 3)\n",
            "(60000, 10)\n",
            "(10000, 10)\n",
            "Downloading data from https://github.com/fchollet/deep-learning-models/releases/download/v0.1/vgg19_weights_tf_dim_ordering_tf_kernels_notop.h5\n",
            "80142336/80134624 [==============================] - 3s 0us/step\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:1259: calling reduce_prod (from tensorflow.python.ops.math_ops) with keep_dims is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "keep_dims is deprecated, use keepdims instead\n",
            "_________________________________________________________________\n",
            "Layer (type)                 Output Shape              Param #   \n",
            "=================================================================\n",
            "vgg19 (Model)                (None, 1, 1, 512)         20024384  \n",
            "_________________________________________________________________\n",
            "flatten_1 (Flatten)          (None, 512)               0         \n",
            "_________________________________________________________________\n",
            "dense_1 (Dense)              (None, 256)               131328    \n",
            "_________________________________________________________________\n",
            "dropout_1 (Dropout)          (None, 256)               0         \n",
            "_________________________________________________________________\n",
            "dense_2 (Dense)              (None, 128)               32896     \n",
            "_________________________________________________________________\n",
            "dropout_2 (Dropout)          (None, 128)               0         \n",
            "_________________________________________________________________\n",
            "dense_3 (Dense)              (None, 10)                1290      \n",
            "=================================================================\n",
            "Total params: 20,189,898\n",
            "Trainable params: 165,514\n",
            "Non-trainable params: 20,024,384\n",
            "_________________________________________________________________\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:2880: calling reduce_sum (from tensorflow.python.ops.math_ops) with keep_dims is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "keep_dims is deprecated, use keepdims instead\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:1344: calling reduce_mean (from tensorflow.python.ops.math_ops) with keep_dims is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "keep_dims is deprecated, use keepdims instead\n",
            "Epoch 1/20\n",
            "60000/60000 [==============================] - 47s 790us/step - loss: 0.6399 - acc: 0.7733\n",
            "Epoch 2/20\n",
            "60000/60000 [==============================] - 43s 718us/step - loss: 0.4458 - acc: 0.8402\n",
            "Epoch 3/20\n",
            "60000/60000 [==============================] - 43s 720us/step - loss: 0.4005 - acc: 0.8539\n",
            "Epoch 4/20\n",
            "60000/60000 [==============================] - 43s 721us/step - loss: 0.3781 - acc: 0.8618\n",
            "Epoch 5/20\n",
            "60000/60000 [==============================] - 43s 721us/step - loss: 0.3598 - acc: 0.8693\n",
            "Epoch 6/20\n",
            "60000/60000 [==============================] - 43s 720us/step - loss: 0.3470 - acc: 0.8724\n",
            "Epoch 7/20\n",
            "60000/60000 [==============================] - 43s 720us/step - loss: 0.3350 - acc: 0.8769\n",
            "Epoch 8/20\n",
            "60000/60000 [==============================] - 43s 720us/step - loss: 0.3261 - acc: 0.8793\n",
            "Epoch 9/20\n",
            "60000/60000 [==============================] - 43s 719us/step - loss: 0.3171 - acc: 0.8831\n",
            "Epoch 10/20\n",
            "60000/60000 [==============================] - 43s 720us/step - loss: 0.3094 - acc: 0.8859\n",
            "Epoch 11/20\n",
            "60000/60000 [==============================] - 43s 720us/step - loss: 0.3035 - acc: 0.8877\n",
            "Epoch 12/20\n",
            "60000/60000 [==============================] - 43s 721us/step - loss: 0.2977 - acc: 0.8897\n",
            "Epoch 13/20\n",
            "60000/60000 [==============================] - 43s 720us/step - loss: 0.2910 - acc: 0.8907\n",
            "Epoch 14/20\n",
            "60000/60000 [==============================] - 43s 719us/step - loss: 0.2863 - acc: 0.8948\n",
            "Epoch 15/20\n",
            "60000/60000 [==============================] - 43s 721us/step - loss: 0.2828 - acc: 0.8953\n",
            "Epoch 16/20\n",
            "60000/60000 [==============================] - 43s 718us/step - loss: 0.2757 - acc: 0.8965\n",
            "Epoch 17/20\n",
            "60000/60000 [==============================] - 43s 722us/step - loss: 0.2701 - acc: 0.8982\n",
            "Epoch 18/20\n",
            "60000/60000 [==============================] - 43s 718us/step - loss: 0.2698 - acc: 0.8992\n",
            "Epoch 19/20\n",
            "60000/60000 [==============================] - 43s 721us/step - loss: 0.2642 - acc: 0.9008\n",
            "Epoch 20/20\n",
            "60000/60000 [==============================] - 43s 718us/step - loss: 0.2579 - acc: 0.9025\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<keras.callbacks.History at 0x7f075023f048>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 6
        }
      ]
    },
    {
      "metadata": {
        "id": "R6iuooHfVCRN",
        "colab_type": "code",
        "outputId": "c6aef8df-f00f-4e76-ac81-7f6f4c6f337a",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        }
      },
      "cell_type": "code",
      "source": [
        "transfer_model.evaluate(x_test_rgb, Y_test_rgb)"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "10000/10000 [==============================] - 11s 1ms/step\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[0.3458766212105751, 0.8824]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 7
        }
      ]
    },
    {
      "metadata": {
        "id": "NOYI2BW74bTt",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "The accuracy of transfer learning using VGG19 is 88% on test dataset. The model achieved a moderate result and can be improved by more feature engineering. "
      ]
    },
    {
      "metadata": {
        "id": "j2A9Bvap4Mer",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "# 3. Text Classification\n",
        "\n",
        "Inspiration: https://github.com/incertum/cyber-matrix-ai/tree/master/Malicious-URL-Detection-Deep-Learning\n",
        "\n",
        "About how to use Keras's Tokenizer funciton: https://github.com/rstudio/keras/issues/336\n",
        "\n",
        "About how to use Keras's pad_sequence function: https://machinelearningmastery.com/data-preparation-variable-length-input-sequences-sequence-prediction/\n",
        "\n",
        "About 1D CNN: https://machinelearningmastery.com/cnn-models-for-human-activity-recognition-time-series-classification/"
      ]
    },
    {
      "metadata": {
        "id": "FPIEUKcQACOo",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "##  3.1    RNN"
      ]
    },
    {
      "metadata": {
        "id": "ZTTVazUMH9c8",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "benign_url = \"https://s3.amazonaws.com/anly-590/url-classification/benign-urls.txt\"\n",
        "benign = requests.get(benign_url).content.decode('utf-8').split('\\n')\n",
        "\n",
        "bad_url = \"https://s3.amazonaws.com/anly-590/url-classification/malicious-urls.txt\"\n",
        "bad = requests.get(bad_url).content.decode('utf-8').split('\\n')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "Sa04VxWkFAUW",
        "colab_type": "code",
        "outputId": "dba2ea0c-6849-4ed0-a368-b40c74ba252a",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "cell_type": "code",
      "source": [
        "len(benign)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "66057"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 8
        }
      ]
    },
    {
      "metadata": {
        "id": "TIbreFq2Zmme",
        "colab_type": "code",
        "outputId": "2090e259-bc36-4658-d145-b4a54a8b4735",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "cell_type": "code",
      "source": [
        "len(bad)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "1297"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 9
        }
      ]
    },
    {
      "metadata": {
        "id": "S5PKDoOxt2v0",
        "colab_type": "code",
        "outputId": "1715710d-8780-4fd6-dec8-619f5769fa1f",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 187
        }
      },
      "cell_type": "code",
      "source": [
        "bad[:10]"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['.1337x.pl',\n",
              " '.1link.io',\n",
              " '.1n.pm',\n",
              " '.22apple.com',\n",
              " '.22find.com',\n",
              " '.2s5m19yk.ru',\n",
              " '.301.li',\n",
              " '.3utilities.com',\n",
              " '.44d.info',\n",
              " '.4everproxy.co']"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 11
        }
      ]
    },
    {
      "metadata": {
        "id": "xQWbh3yvpvg2",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "This is a very imbalanced dataset. Down sample Benign url set to keep same length with bad url set."
      ]
    },
    {
      "metadata": {
        "id": "LDfB59dgakkM",
        "colab_type": "code",
        "outputId": "c298b6f4-3e99-47e4-e085-1417b613e270",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "cell_type": "code",
      "source": [
        "import random\n",
        "random.seed(1120)  \n",
        "benign_train = random.sample(benign, len(bad))\n",
        "bad_train = bad\n",
        "train = benign_train + bad_train\n",
        "lable = np.array([1] * len(bad) + [0] * len(bad))\n",
        "print(len(train), len(lable))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "2594 2594\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "KfPi7qtVz7fc",
        "colab_type": "code",
        "outputId": "7ec35927-beb6-437c-d1f8-e4c139eefec6",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 398
        }
      },
      "cell_type": "code",
      "source": [
        "plt.hist([len(i) for i in train])"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(array([126., 938., 531., 696., 185.,  91.,  22.,   1.,   2.,   2.]),\n",
              " array([ 0. ,  8.6, 17.2, 25.8, 34.4, 43. , 51.6, 60.2, 68.8, 77.4, 86. ]),\n",
              " <a list of 10 Patch objects>)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 174
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAeQAAAFKCAYAAADMuCxnAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAE4pJREFUeJzt3WFslfXd//FPbWkIWIQ2p0swSpxL\nMDEIEk0GAyciDuP+GcuEGYIu0Qc6O3UJCbiGOYzRIe5vRCRqgjhjQuxWp3InZhBnMD4oLNqFmCUm\napZFYZZ2KYJAVaT/R//ebjram7vt+XF8vR611znnOr/z9cR3r+u0F3WDg4ODAQCq6qxqLwAAEGQA\nKIIgA0ABBBkACiDIAFAAQQaAAjRU88l7e4+M6v6mTZuU/v5jo7pPRsbsq8Pcq8Pcq+dMn32l0vQf\nb6upI+SGhvpqL+Fry+yrw9yrw9yrp5ZnX1NBBoAzlSADQAEEGQAKIMgAUABBBoACCDIAFECQAaAA\nggwABRBkACiAIANAAQQZAAogyABQgKr+a09fRzdveLXaSxjWtruvqvYSAL52HCEDQAEEGQAKIMgA\nUABBBoACCDIAFECQAaAAggwABRBkACiAIANAAQQZAAogyABQAEEGgAIIMgAUQJABoACCDAAFEGQA\nKIAgA0ABBBkACiDIAFAAQQaAAggyABRAkAGgAIIMAAUQZAAogCADQAEEGQAKIMgAUABBBoACCDIA\nFECQAaAAggwABRBkACiAIANAAQQZAAogyABQAEEGgAIIMgAUoGG4Oxw9ejRr167NRx99lM8++yxt\nbW2pVCpZv359kmTmzJm59957kyRbt27NH//4x9TV1eVnP/tZvvvd747p4gGgVgwb5BdeeCEXXHBB\nVq9enZ6envzkJz9JpVJJe3t7LrnkkqxevTqvvfZavvnNb+bll1/Oc889l48//jgrV67MggULUl9f\nPx6vAwDOaMOesp42bVoOHTqUJDl8+HCmTp2a/fv355JLLkmSLFq0KF1dXdm7d28WLlyYxsbGNDc3\n59xzz8277747tqsHgBoxbJCvu+66HDhwIEuWLMmqVauyZs2aTJkyZej2lpaW9Pb2pq+vL83NzUPb\nm5ub09vbOzarBoAaM+wp65deeinTp0/PU089lbfffjttbW1pamoaun1wcPArH/eftn/RtGmT0tAw\nuqe0K5Wm4e/EKZ3uDM2+Osy9Osy9emp19sMGubu7OwsWLEiSXHTRRfnkk09y4sSJodt7enrS2tqa\n1tbW/O1vf/vS9lPp7z92uuv+SpVKU3p7j4zqPr+OTmeGZl8d5l4d5l49Z/rsT/XDxLCnrGfMmJF9\n+/YlSfbv35/JkyfnwgsvzBtvvJEk2bVrVxYuXJhvf/vb2b17dz799NP09PTk4MGD+da3vjVKLwEA\natuwR8g//vGP097enlWrVuXEiRNZv359KpVK7rnnnpw8eTKzZ8/O/PnzkyQrVqzIqlWrUldXl/Xr\n1+ess/yZMwCMxLBBnjx5cjZt2vSl7du3b//SthtvvDE33njj6KwMAL5GHMICQAEEGQAKIMgAUABB\nBoACCDIAFECQAaAAggwABRBkACiAIANAAQQZAAogyABQAEEGgAIIMgAUQJABoACCDAAFEGQAKIAg\nA0ABBBkACiDIAFAAQQaAAggyABRAkAGgAIIMAAUQZAAogCADQAEEGQAKIMgAUICGai8A/qdu3vBq\ntZcwrG13X1XtJQBnGEfIAFAAQQaAAggyABRAkAGgAIIMAAUQZAAogCADQAEEGQAKIMgAUABBBoAC\nCDIAFECQAaAAggwABRBkACiAIANAAQQZAAogyABQAEEGgAIIMgAUQJABoACCDAAFEGQAKIAgA0AB\nBBkACtAwkjvt2LEjW7duTUNDQ+68887MnDkza9asyeeff55KpZKHHnoojY2N2bFjR5555pmcddZZ\nWbFiRZYvXz7W6weAmjBskPv7+7Nly5Y8//zzOXbsWDZv3pydO3dm5cqVufbaa/Pwww+ns7Mzy5Yt\ny5YtW9LZ2ZkJEybk+uuvz5IlSzJ16tTxeB0AcEYb9pR1V1dX5s2bl7PPPjutra257777snfv3ixe\nvDhJsmjRonR1dWXfvn2ZNWtWmpqaMnHixMydOzfd3d1j/gIAoBYMe4T8wQcfZGBgILfddlsOHz6c\nO+64I8ePH09jY2OSpKWlJb29venr60tzc/PQ45qbm9Pb2zt2KweAGjKiz5APHTqUxx57LAcOHMhN\nN92UwcHBodu++PUX/aftXzRt2qQ0NNSPcKkjU6k0jer+vo5Od4Zm/9/GcxbmXh3mXj21Ovthg9zS\n0pJLL700DQ0NOf/88zN58uTU19dnYGAgEydOTE9PT1pbW9Pa2pq+vr6hxx08eDBz5sw55b77+4/9\n71/BF1QqTentPTKq+/w6Op0Zmv2/Gq9ZmHt1mHv1nOmzP9UPE8N+hrxgwYLs2bMnJ0+eTH9/f44d\nO5b58+dn586dSZJdu3Zl4cKFmT17dt56660cPnw4R48eTXd3dy677LLRexUAUMOGPUL+xje+ke99\n73tZsWJFkmTdunWZNWtW1q5dm46OjkyfPj3Lli3LhAkTsnr16txyyy2pq6tLW1tbmppq87QCAIy2\nEX2GfMMNN+SGG274l21PP/30l+63dOnSLF26dHRWBgBfI67UBQAFEGQAKIAgA0ABBBkACiDIAFAA\nQQaAAggyABRAkAGgAIIMAAUQZAAogCADQAEEGQAKIMgAUABBBoACCDIAFECQAaAADdVeAOW5ecOr\n1V4CwNeOI2QAKIAgA0ABBBkACiDIAFAAQQaAAggyABRAkAGgAIIMAAUQZAAogCADQAEEGQAKIMgA\nUABBBoACCDIAFECQAaAAggwABRBkACiAIANAAQQZAAogyABQAEEGgAIIMgAUQJABoACCDAAFEGQA\nKIAgA0ABBBkACiDIAFAAQQaAAggyABRAkAGgAIIMAAUQZAAogCADQAEEGQAKMKIgDwwM5Oqrr84f\n/vCH/OMf/8iNN96YlStX5q677sqnn36aJNmxY0d+9KMfZfny5fn9738/posGgFozoiA//vjjOeec\nc5Ikjz76aFauXJnt27dnxowZ6ezszLFjx7Jly5b89re/zbPPPptnnnkmhw4dGtOFA0AtGTbI7733\nXt59991ceeWVSZK9e/dm8eLFSZJFixalq6sr+/bty6xZs9LU1JSJEydm7ty56e7uHtOFA0AtaRju\nDg8++GB++ctf5sUXX0ySHD9+PI2NjUmSlpaW9Pb2pq+vL83NzUOPaW5uTm9v77BPPm3apDQ01J/u\n2r9SpdI0qvuD0zGe70Pv+eow9+qp1dmfMsgvvvhi5syZk/POO+8rbx8cHPwfbf93/f3HRnS/kapU\nmtLbe2RU9wmnY7zeh97z1WHu1XOmz/5UP0ycMsi7d+/O+++/n927d+fDDz9MY2NjJk2alIGBgUyc\nODE9PT1pbW1Na2tr+vr6hh538ODBzJkzZ/ReAQDUuFMG+ZFHHhn6evPmzTn33HPzl7/8JTt37swP\nfvCD7Nq1KwsXLszs2bOzbt26HD58OPX19enu7k57e/uYLx4AasWwnyH/uzvuuCNr165NR0dHpk+f\nnmXLlmXChAlZvXp1brnlltTV1aWtrS1NTbV5jh8AxsKIg3zHHXcMff30009/6falS5dm6dKlo7Mq\nAPiacaUuACiAIANAAQQZAAogyABQAEEGgAIIMgAUQJABoACCDAAFEGQAKIAgA0ABBBkACiDIAFAA\nQQaAAggyABRAkAGgAIIMAAUQZAAogCADQAEEGQAKIMgAUABBBoACCDIAFECQAaAAggwABRBkACiA\nIANAAQQZAAogyABQAEEGgAIIMgAUQJABoACCDAAFEGQAKIAgA0ABBBkACiDIAFAAQQaAAggyABRA\nkAGgAIIMAAUQZAAogCADQAEEGQAKIMgAUABBBoACCDIAFECQAaAAggwABRBkACiAIANAAQQZAArQ\nMJI7bdy4MW+++WZOnDiRW2+9NbNmzcqaNWvy+eefp1Kp5KGHHkpjY2N27NiRZ555JmeddVZWrFiR\n5cuXj/X6AaAmDBvkPXv25J133klHR0f6+/vzwx/+MPPmzcvKlStz7bXX5uGHH05nZ2eWLVuWLVu2\npLOzMxMmTMj111+fJUuWZOrUqePxOgDgjDbsKevLL788mzZtSpJMmTIlx48fz969e7N48eIkyaJF\ni9LV1ZV9+/Zl1qxZaWpqysSJEzN37tx0d3eP7eoBoEYMG+T6+vpMmjQpSdLZ2Zkrrrgix48fT2Nj\nY5KkpaUlvb296evrS3Nz89Djmpub09vbO0bLBoDaMqLPkJPklVdeSWdnZ7Zt25ZrrrlmaPvg4OBX\n3v8/bf+iadMmpaGhfqRLGJFKpWlU9wenYzzfh97z1WHu1VOrsx9RkF9//fU88cQT2bp1a5qamjJp\n0qQMDAxk4sSJ6enpSWtra1pbW9PX1zf0mIMHD2bOnDmn3G9//7H/3er/TaXSlN7eI6O6Tzgd4/U+\n9J6vDnOvnjN99qf6YWLYU9ZHjhzJxo0b8+STTw79gtb8+fOzc+fOJMmuXbuycOHCzJ49O2+99VYO\nHz6co0ePpru7O5dddtkovQQAqG3DHiG//PLL6e/vz89//vOhbRs2bMi6devS0dGR6dOnZ9myZZkw\nYUJWr16dW265JXV1dWlra0tTU22eVgCA0VY3OJIPe8fIaJ92OBNOZdy84dVqL4FxsO3uq8blec6E\n93wtMvfqOdNnf6pT1iP+pS5g5Er/wWu8fmAARs6lMwGgAIIMAAUQZAAogCADQAEEGQAKIMgAUABB\nBoACCDIAFECQAaAAggwABRBkACiAIANAAQQZAAogyABQAEEGgAIIMgAUQJABoAAN1V7AaPo/q1+q\n9hIA4LQ4QgaAAggyABRAkAGgAIIMAAUQZAAogCADQAEEGQAKIMgAUABBBoAC1NSVuoCRuXnDq9Ve\nwrC23X1VtZcA48oRMgAUQJABoACCDAAFEGQAKIAgA0ABBBkACiDIAFAAQQaAAggyABRAkAGgAIIM\nAAUQZAAogCADQAEEGQAKIMgAUABBBoACCDIAFECQAaAAggwABRBkAChAQ7UXAPBVbt7warWXcErb\n7r6q2kugxjhCBoACCDIAFGDUT1k/8MAD2bdvX+rq6tLe3p5LLrlktJ8CAGrOqAb5z3/+c/7+97+n\no6Mj7733Xtrb29PR0TGaTwEANWlUT1l3dXXl6quvTpJceOGF+eijj/Lxxx+P5lMAQE0a1SPkvr6+\nXHzxxUPfNzc3p7e3N2efffZoPg1A1ZX+W+CMjvH8bfox/bOnwcHBU95eqTSN6vP91//9wajuDwDG\ny6iesm5tbU1fX9/Q9wcPHkylUhnNpwCAmjSqQf7Od76TnTt3Jkn++te/prW11elqABiBUT1lPXfu\n3Fx88cW54YYbUldXl1/96lejuXsAqFl1g8N90AsAjDlX6gKAAggyABSgZv61J5fsHF8bN27Mm2++\nmRMnTuTWW2/NrFmzsmbNmnz++eepVCp56KGH0tjYWO1l1qSBgYF8//vfz+2335558+aZ+zjYsWNH\ntm7dmoaGhtx5552ZOXOmuY+Do0ePZu3atfnoo4/y2Wefpa2tLZVKJevXr0+SzJw5M/fee291FzmK\nauII+YuX7Lz//vtz//33V3tJNW3Pnj1555130tHRka1bt+aBBx7Io48+mpUrV2b79u2ZMWNGOjs7\nq73MmvX444/nnHPOSRJzHwf9/f3ZsmVLtm/fnieeeCJ/+tOfzH2cvPDCC7ngggvy7LPPZtOmTUP/\nf29vb89zzz2Xjz/+OK+99lq1lzlqaiLILtk5vi6//PJs2rQpSTJlypQcP348e/fuzeLFi5MkixYt\nSldXVzWXWLPee++9vPvuu7nyyiuTxNzHQVdXV+bNm5ezzz47ra2tue+++8x9nEybNi2HDh1Kkhw+\nfDhTp07N/v37h86A1trsayLIfX19mTZt2tD3//+SnYyN+vr6TJo0KUnS2dmZK664IsePHx86ZdfS\n0mL+Y+TBBx/M3XffPfS9uY+9Dz74IAMDA7ntttuycuXKdHV1mfs4ue6663LgwIEsWbIkq1atypo1\nazJlypSh22tt9jXzGfIX+Uuu8fHKK6+ks7Mz27ZtyzXXXDO03fzHxosvvpg5c+bkvPPO+8rbzX3s\nHDp0KI899lgOHDiQm2666V9mbe5j56WXXsr06dPz1FNP5e23305bW1uamv77ksu1NvuaCLJLdo6/\n119/PU888US2bt2apqamTJo0KQMDA5k4cWJ6enrS2tpa7SXWnN27d+f999/P7t278+GHH6axsdHc\nx0FLS0suvfTSNDQ05Pzzz8/kyZNTX19v7uOgu7s7CxYsSJJcdNFF+eSTT3LixImh22tt9jVxytol\nO8fXkSNHsnHjxjz55JOZOnVqkmT+/PlD/w127dqVhQsXVnOJNemRRx7J888/n9/97ndZvnx5br/9\ndnMfBwsWLMiePXty8uTJ9Pf359ixY+Y+TmbMmJF9+/YlSfbv35/JkyfnwgsvzBtvvJGk9mZfM1fq\n+s1vfpM33nhj6JKdF110UbWXVLM6OjqyefPmXHDBBUPbNmzYkHXr1uWTTz7J9OnT8+tf/zoTJkyo\n4ipr2+bNm3PuuedmwYIFWbt2rbmPseeee27oN6l/+tOfZtasWeY+Do4ePZr29vb885//zIkTJ3LX\nXXelUqnknnvuycmTJzN79uz84he/qPYyR03NBBkAzmQ1ccoaAM50ggwABRBkACiAIANAAQQZAAog\nyABQAEEGgAIIMgAU4P8BC94IpcF0cQgAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<matplotlib.figure.Figure at 0x7fba30610550>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "metadata": {
        "id": "cwXIETC7cAdL",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "from keras.preprocessing.sequence import pad_sequences\n",
        "from keras.preprocessing.text import Tokenizer\n",
        "tokenizer = Tokenizer(char_level=True)\n",
        "tokenizer.fit_on_texts(train)\n",
        "sequences = tokenizer.texts_to_sequences(train)\n",
        "maxlen = max([len(i) for i in train])\n",
        "train_padseq = pad_sequences(sequences, maxlen = maxlen) "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "I5oLjy-Hc5Rb",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "import numpy as np\n",
        "from sklearn.model_selection import train_test_split\n",
        "X_train, X_test, y_train, y_test = train_test_split(train_padseq, lable, test_size=0.2, random_state=42)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "y4odYg0TdPbe",
        "colab_type": "code",
        "outputId": "9db9d858-43b7-4380-d1d5-9db633491df9",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "cell_type": "code",
      "source": [
        "X_train.shape, y_train.shape, X_test.shape, y_test.shape"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "((2075, 86), (2075,), (519, 86), (519,))"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 177
        }
      ]
    },
    {
      "metadata": {
        "id": "Ux8NF0rZP_wy",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "num_char = len(tokenizer.word_index)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "vxsQgxXWdRjw",
        "colab_type": "code",
        "outputId": "4811cf55-dcf1-47fd-d74e-9090d6e06a71",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1734
        }
      },
      "cell_type": "code",
      "source": [
        "from keras.preprocessing import sequence\n",
        "from keras.layers import Dense, SimpleRNN, LSTM, Embedding\n",
        "rnn_model = Sequential()\n",
        "rnn_model.add(Embedding(50, 64))\n",
        "rnn_model.add(SimpleRNN(32))\n",
        "rnn_model.add(Dense(1, activation='sigmoid'))\n",
        "\n",
        "rnn_model.compile(optimizer=\"rmsprop\", loss=\"binary_crossentropy\", metrics=[\"acc\"])\n",
        "\n",
        "rnn_model.fit(X_train, y_train, epochs=50, batch_size=256, verbose = 1)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch 1/50\n",
            "2075/2075 [==============================] - 1s 462us/step - loss: 0.6155 - acc: 0.6848\n",
            "Epoch 2/50\n",
            "2075/2075 [==============================] - 0s 219us/step - loss: 0.5118 - acc: 0.8048\n",
            "Epoch 3/50\n",
            "2075/2075 [==============================] - 1s 249us/step - loss: 0.4512 - acc: 0.8260\n",
            "Epoch 4/50\n",
            "2075/2075 [==============================] - 1s 254us/step - loss: 0.4282 - acc: 0.8270\n",
            "Epoch 5/50\n",
            "2075/2075 [==============================] - 1s 243us/step - loss: 0.3750 - acc: 0.8636\n",
            "Epoch 6/50\n",
            "2075/2075 [==============================] - 1s 250us/step - loss: 0.3452 - acc: 0.8766\n",
            "Epoch 7/50\n",
            "2075/2075 [==============================] - 1s 251us/step - loss: 0.3193 - acc: 0.8853\n",
            "Epoch 8/50\n",
            "2075/2075 [==============================] - 1s 252us/step - loss: 0.3015 - acc: 0.8867\n",
            "Epoch 9/50\n",
            "2075/2075 [==============================] - 1s 247us/step - loss: 0.3118 - acc: 0.8795\n",
            "Epoch 10/50\n",
            "2075/2075 [==============================] - 1s 250us/step - loss: 0.2724 - acc: 0.9099\n",
            "Epoch 11/50\n",
            "2075/2075 [==============================] - 1s 249us/step - loss: 0.2484 - acc: 0.9157\n",
            "Epoch 12/50\n",
            "2075/2075 [==============================] - 1s 252us/step - loss: 0.2325 - acc: 0.9219\n",
            "Epoch 13/50\n",
            "2075/2075 [==============================] - 1s 247us/step - loss: 0.2387 - acc: 0.9224\n",
            "Epoch 14/50\n",
            "2075/2075 [==============================] - 1s 253us/step - loss: 0.2242 - acc: 0.9267\n",
            "Epoch 15/50\n",
            "2075/2075 [==============================] - 1s 250us/step - loss: 0.1973 - acc: 0.9412\n",
            "Epoch 16/50\n",
            "2075/2075 [==============================] - 1s 255us/step - loss: 0.1874 - acc: 0.9398\n",
            "Epoch 17/50\n",
            "2075/2075 [==============================] - 1s 248us/step - loss: 0.2290 - acc: 0.9094\n",
            "Epoch 18/50\n",
            "2075/2075 [==============================] - 1s 247us/step - loss: 0.1774 - acc: 0.9451\n",
            "Epoch 19/50\n",
            "2075/2075 [==============================] - 1s 247us/step - loss: 0.1665 - acc: 0.9484\n",
            "Epoch 20/50\n",
            "2075/2075 [==============================] - 1s 253us/step - loss: 0.1677 - acc: 0.9489\n",
            "Epoch 21/50\n",
            "2075/2075 [==============================] - 1s 249us/step - loss: 0.1618 - acc: 0.9484\n",
            "Epoch 22/50\n",
            "2075/2075 [==============================] - 1s 253us/step - loss: 0.1625 - acc: 0.9465\n",
            "Epoch 23/50\n",
            "2075/2075 [==============================] - 1s 247us/step - loss: 0.1440 - acc: 0.9590\n",
            "Epoch 24/50\n",
            "2075/2075 [==============================] - 1s 257us/step - loss: 0.1297 - acc: 0.9653\n",
            "Epoch 25/50\n",
            "2075/2075 [==============================] - 1s 248us/step - loss: 0.1337 - acc: 0.9619\n",
            "Epoch 26/50\n",
            "2075/2075 [==============================] - 1s 250us/step - loss: 0.1635 - acc: 0.9412\n",
            "Epoch 27/50\n",
            "2075/2075 [==============================] - 1s 249us/step - loss: 0.1160 - acc: 0.9692\n",
            "Epoch 28/50\n",
            "2075/2075 [==============================] - 1s 248us/step - loss: 0.1293 - acc: 0.9663\n",
            "Epoch 29/50\n",
            "2075/2075 [==============================] - 1s 247us/step - loss: 0.1107 - acc: 0.9730\n",
            "Epoch 30/50\n",
            "2075/2075 [==============================] - 1s 247us/step - loss: 0.1267 - acc: 0.9658\n",
            "Epoch 31/50\n",
            "2075/2075 [==============================] - 1s 246us/step - loss: 0.1174 - acc: 0.9672\n",
            "Epoch 32/50\n",
            "2075/2075 [==============================] - 1s 248us/step - loss: 0.0935 - acc: 0.9793\n",
            "Epoch 33/50\n",
            "2075/2075 [==============================] - 1s 249us/step - loss: 0.0906 - acc: 0.9783\n",
            "Epoch 34/50\n",
            "2075/2075 [==============================] - 1s 251us/step - loss: 0.1952 - acc: 0.9248\n",
            "Epoch 35/50\n",
            "2075/2075 [==============================] - 1s 247us/step - loss: 0.0901 - acc: 0.9783\n",
            "Epoch 36/50\n",
            "2075/2075 [==============================] - 1s 252us/step - loss: 0.0990 - acc: 0.9740\n",
            "Epoch 37/50\n",
            "2075/2075 [==============================] - 1s 250us/step - loss: 0.0798 - acc: 0.9831\n",
            "Epoch 38/50\n",
            "2075/2075 [==============================] - 1s 253us/step - loss: 0.1445 - acc: 0.9513\n",
            "Epoch 39/50\n",
            "2075/2075 [==============================] - 1s 252us/step - loss: 0.0757 - acc: 0.9836\n",
            "Epoch 40/50\n",
            "2075/2075 [==============================] - 1s 258us/step - loss: 0.0752 - acc: 0.9841\n",
            "Epoch 41/50\n",
            "2075/2075 [==============================] - 1s 250us/step - loss: 0.0767 - acc: 0.9855\n",
            "Epoch 42/50\n",
            "2075/2075 [==============================] - 1s 254us/step - loss: 0.0697 - acc: 0.9860\n",
            "Epoch 43/50\n",
            "2075/2075 [==============================] - 1s 249us/step - loss: 0.0662 - acc: 0.9870\n",
            "Epoch 44/50\n",
            "2075/2075 [==============================] - 1s 252us/step - loss: 0.1048 - acc: 0.9716\n",
            "Epoch 45/50\n",
            "2075/2075 [==============================] - 1s 248us/step - loss: 0.0675 - acc: 0.9846\n",
            "Epoch 46/50\n",
            "2075/2075 [==============================] - 1s 256us/step - loss: 0.0687 - acc: 0.9831\n",
            "Epoch 47/50\n",
            "2075/2075 [==============================] - 1s 250us/step - loss: 0.0647 - acc: 0.9865\n",
            "Epoch 48/50\n",
            "2075/2075 [==============================] - 1s 254us/step - loss: 0.0512 - acc: 0.9894\n",
            "Epoch 49/50\n",
            "2075/2075 [==============================] - 1s 247us/step - loss: 0.1198 - acc: 0.9614\n",
            "Epoch 50/50\n",
            "2075/2075 [==============================] - 1s 255us/step - loss: 0.0628 - acc: 0.9851\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<keras.callbacks.History at 0x7fba30b8beb8>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 179
        }
      ]
    },
    {
      "metadata": {
        "id": "EQvv99v4dxs_",
        "colab_type": "code",
        "outputId": "90c24fa2-f94e-4084-ce3e-dd6ac266c6ed",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        }
      },
      "cell_type": "code",
      "source": [
        "from sklearn.metrics import roc_curve\n",
        "rnn_model.evaluate(X_test, y_test) #98% accuracy, pretty good. "
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "519/519 [==============================] - 0s 841us/step\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[0.07764389887021456, 0.9865125241996235]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 180
        }
      ]
    },
    {
      "metadata": {
        "id": "yCD17OdwkXln",
        "colab_type": "code",
        "outputId": "253ed6b0-9b55-4905-97c6-820d6907d95b",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 347
        }
      },
      "cell_type": "code",
      "source": [
        "pred_rnn = rnn_model.predict(X_test)\n",
        "fpr, tpr, thresholds = roc_curve(y_test, pred_rnn)\n",
        "plt.plot(fpr, tpr)\n",
        "plt.show() "
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAeEAAAFKCAYAAAAqkecjAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAFRNJREFUeJzt3WFsk4edx/GfYwM9Gl9k39kDElij\nTFU0TyAiWh0XRAZypq7q3iHsqC2oq9qrjm5qVzS12VSjrU6pDvZiLS8qtNsLQDSosqqdVDWTNtBN\nYBbWbaCkQiWRGiUUxTakOdzAVOC5F9U8UoKd9rH9j53v51WePubxX39BvvgxdTyO4zgCAABV12A9\nAAAAixURBgDACBEGAMAIEQYAwAgRBgDACBEGAMCIr9pPmM1eLev1AoHlmpqaKes1FyP26B47dI8d\nuscO3avEDkMh/5z/veZfCft8XusR6gJ7dI8duscO3WOH7lVzhzUfYQAAahURBgDACBEGAMAIEQYA\nwAgRBgDACBEGAMAIEQYAwAgRBgDAyLwi/OGHHyoajerw4cN3nDt16pS2bdumWCymAwcOlH1AAADq\nVckIz8zM6Oc//7k2btw45/lXXnlFr7/+uo4ePaqTJ09qZGSk7EMCAFCPSn529NKlS3Xw4EEdPHjw\njnPj4+NqamrSypUrJUldXV1Kp9P6xje+Uf5JAQBVd+z3IzpzPmM9RlVt7mjR9/5tTVWeq2SEfT6f\nfL65H5bNZhUMBgvHwWBQ4+PjRa8XCCwv++dy3u2DsfHlsEf32KF79bjD//6fYZ08e9F6jK8kM3VN\nkhQO/JPxJNVVrd+HVf8pSpX4yRTl/slMixF7dI8dulevO/zfP09o6urfFPAvq/hzeb0e3bzplO16\n//LP9+iB9rC2b108dzgr8fvwblF3FeFwOKxcLlc4npycVDgcdnNJoCoqcYut3N/8FqN63eHfA/xf\n//nvFX+uev2LTL1yFeGWlhbl83lNTExoxYoVOn78uPbt21eu2WpOLb93Uq/f/O7m8v9dl/T53/KB\nSgv4l+mBdl6g4E4lIzw0NKTXXntNFy9elM/n08DAgLZu3aqWlhZ1d3drz549euGFFyRJDz/8sFpb\nWys+9JdVrTjyjb12VOIWG69A3GOHWGw8juNU9eVPOf+AHfv9iP58IVvyFVw141ir753wzc89duge\nO3SPHbpXM+8JWztzPqOp/N8UaCz+jx0W4z8sAAAsfDUdYUn616Z7tPc/5v4gEQAAFjI+OxoAACNE\nGAAAI0QYAAAjRBgAACNEGAAAI0QYAAAjRBgAACNEGAAAI0QYAAAjRBgAACNEGAAAI0QYAAAjRBgA\nACNEGAAAI0QYAAAjRBgAACNEGAAAI0QYAAAjRBgAACNEGAAAI0QYAAAjRBgAACNEGAAAI0QYAAAj\nRBgAACNEGAAAI0QYAAAjRBgAACNEGAAAI0QYAAAjRBgAACNEGAAAI0QYAAAjRBgAACNEGAAAI0QY\nAAAjRBgAACNEGAAAI0QYAAAjRBgAACNEGAAAI0QYAAAjRBgAACO++Tyor69PZ8+elcfjUW9vr9au\nXVs4d+TIEf3mN79RQ0ODvvWtb+knP/lJxYYFAKCelHwlPDg4qLGxMfX39yuZTCqZTBbO5fN5/epX\nv9KRI0d09OhRjY6O6q9//WtFBwYAoF6UjHA6nVY0GpUktbW1aXp6Wvl8XpK0ZMkSLVmyRDMzM7px\n44auXbumpqamyk4MAECdKBnhXC6nQCBQOA4Gg8pms5KkZcuWadeuXYpGo9qyZYvWrVun1tbWyk0L\nAEAdmdd7wrdzHKfwdT6f15tvvqn33ntPjY2N2rlzp86fP6/29va7/vpAYLl8Pu9Xm/YLvF6PJCkU\n8pfleosde3SPHbrHDt1jh+5Va4clIxwOh5XL5QrHmUxGoVBIkjQ6OqrVq1crGAxKkjZs2KChoaGi\nEZ6amnE7c8HNm468Xo+y2atlu+ZiFQr52aNL7NA9dugeO3SvEju8W9RL3o7u7OzUwMCAJGl4eFjh\ncFiNjY2SpObmZo2Ojur69euSpKGhId13331lGhkAgPpW8pVwR0eHIpGI4vG4PB6PEomEUqmU/H6/\nuru79eSTT2rHjh3yer1av369NmzYUI25AQCoefN6T3j37t2zjm+/3RyPxxWPx8s7FQAAiwCfmAUA\ngBEiDACAESIMAIARIgwAgBEiDACAESIMAIARIgwAgBEiDACAESIMAIARIgwAgBEiDACAESIMAIAR\nIgwAgBEiDACAESIMAIARIgwAgBEiDACAESIMAIARIgwAgBEiDACAESIMAIARIgwAgBEiDACAESIM\nAIARIgwAgBEiDACAESIMAIARIgwAgBEiDACAESIMAIARIgwAgBEiDACAESIMAIARIgwAgBEiDACA\nESIMAIARIgwAgBEiDACAESIMAIARIgwAgBEiDACAESIMAIARIgwAgBEiDACAESIMAIARIgwAgBHf\nfB7U19ens2fPyuPxqLe3V2vXri2cu3Tpkn70ox/ps88+0ze/+U397Gc/q9iwAADUk5KvhAcHBzU2\nNqb+/n4lk0klk8lZ5/fu3avvf//7evvtt+X1evXxxx9XbFgAAOpJyQin02lFo1FJUltbm6anp5XP\n5yVJt27d0vvvv6+tW7dKkhKJhFatWlXBcQEAqB8lI5zL5RQIBArHwWBQ2WxWknTlyhXde++9evXV\nV9XT06P9+/dXblIAAOrMvN4Tvp3jOLO+npyc1I4dO9Tc3Kynn35aJ06c0Le//e27/vpAYLl8Pu9X\nGvaLvF6PJCkU8pfleosde3SPHbrHDt1jh+5Va4clIxwOh5XL5QrHmUxGoVBIkhQIBLRq1SqtWbNG\nkrRx40ZduHChaISnpmZcjvwPN2868no9ymavlu2ai1Uo5GePLrFD99ihe+zQvUrs8G5RL3k7urOz\nUwMDA5Kk4eFhhcNhNTY2SpJ8Pp9Wr16tjz76qHC+tbW1TCMDAFDfSr4S7ujoUCQSUTwel8fjUSKR\nUCqVkt/vV3d3t3p7e/Xiiy/KcRzdf//9hX+kBQAAipvXe8K7d++eddze3l74+utf/7qOHj1a3qkA\nAFgE+MQsAACMEGEAAIwQYQAAjBBhAACMEGEAAIwQYQAAjBBhAACMEGEAAIwQYQAAjBBhAACMEGEA\nAIwQYQAAjBBhAACMEGEAAIwQYQAAjBBhAACMEGEAAIwQYQAAjBBhAACMEGEAAIwQYQAAjBBhAACM\nEGEAAIwQYQAAjBBhAACMEGEAAIwQYQAAjBBhAACMEGEAAIwQYQAAjBBhAACMEGEAAIwQYQAAjBBh\nAACMEGEAAIwQYQAAjBBhAACMEGEAAIwQYQAAjBBhAACMEGEAAIwQYQAAjBBhAACMEGEAAIwQYQAA\njBBhAACMzCvCfX19isViisfjOnfu3JyP2b9/vx5//PGyDgcAQD0rGeHBwUGNjY2pv79fyWRSyWTy\njseMjIzozJkzFRkQAIB6VTLC6XRa0WhUktTW1qbp6Wnl8/lZj9m7d6+ef/75ykwIAECd8pV6QC6X\nUyQSKRwHg0Fls1k1NjZKklKplB588EE1NzfP6wkDgeXy+bxfcdzZvF6PJCkU8pfleosde3SPHbrH\nDt1jh+5Va4clI/xFjuMUvv7kk0+USqX061//WpOTk/P69VNTM1/2Ke/q5k1HXq9H2ezVsl1zsQqF\n/OzRJXboHjt0jx26V4kd3i3qJW9Hh8Nh5XK5wnEmk1EoFJIknT59WleuXNGjjz6qZ599VsPDw+rr\n6yvTyAAA1LeSEe7s7NTAwIAkaXh4WOFwuHAr+qGHHtK7776rY8eO6Y033lAkElFvb29lJwYAoE6U\nvB3d0dGhSCSieDwuj8ejRCKhVColv9+v7u7uaswIAEBdmtd7wrt375513N7efsdjWlpadOjQofJM\nBQDAIsAnZgEAYIQIAwBghAgDAGCECAMAYIQIAwBghAgDAGCECAMAYIQIAwBghAgDAGCECAMAYIQI\nAwBghAgDAGCECAMAYIQIAwBghAgDAGCECAMAYIQIAwBghAgDAGCECAMAYIQIAwBghAgDAGCECAMA\nYIQIAwBghAgDAGCECAMAYIQIAwBghAgDAGCECAMAYIQIAwBghAgDAGCECAMAYIQIAwBghAgDAGCE\nCAMAYIQIAwBghAgDAGCECAMAYIQIAwBghAgDAGCECAMAYIQIAwBghAgDAGCECAMAYIQIAwBghAgD\nAGDEN58H9fX16ezZs/J4POrt7dXatWsL506fPq1f/OIXamhoUGtrq5LJpBoaaDsAAKWUrOXg4KDG\nxsbU39+vZDKpZDI56/zLL7+sX/7yl3rrrbf06aef6g9/+EPFhgUAoJ6UjHA6nVY0GpUktbW1aXp6\nWvl8vnA+lUppxYoVkqRgMKipqakKjQoAQH0pGeFcLqdAIFA4DgaDymazhePGxkZJUiaT0cmTJ9XV\n1VWBMQEAqD/zek/4do7j3PHfLl++rGeeeUaJRGJWsOcSCCyXz+f9sk87J6/XI0kKhfxlud5ixx7d\nY4fusUP32KF71dphyQiHw2HlcrnCcSaTUSgUKhzn83k99dRTeu6557Rp06aSTzg1NfMVR73TzZuO\nvF6PstmrZbvmYhUK+dmjS+zQPXboHjt0rxI7vFvUS96O7uzs1MDAgCRpeHhY4XC4cAtakvbu3aud\nO3dq8+bNZRoVAIDFoeQr4Y6ODkUiEcXjcXk8HiUSCaVSKfn9fm3atEnvvPOOxsbG9Pbbb0uSHnnk\nEcVisYoPDgBArZvXe8K7d++eddze3l74emhoqLwTAQCwSPCpGgAAGCHCAAAYIcIAABghwgAAGCHC\nAAAYIcIAABghwgAAGCHCAAAYIcIAABghwgAAGCHCAAAYIcIAABghwgAAGCHCAAAYIcIAABghwgAA\nGCHCAAAYIcIAABghwgAAGCHCAAAYIcIAABghwgAAGCHCAAAYIcIAABghwgAAGCHCAAAYIcIAABgh\nwgAAGCHCAAAYIcIAABghwgAAGCHCAAAYIcIAABghwgAAGCHCAAAYIcIAABghwgAAGCHCAAAYIcIA\nABghwgAAGCHCAAAYIcIAABghwgAAGCHCAAAYIcIAABghwgAAGJlXhPv6+hSLxRSPx3Xu3LlZ506d\nOqVt27YpFovpwIEDFRkSAIB6VDLCg4ODGhsbU39/v5LJpJLJ5Kzzr7zyil5//XUdPXpUJ0+e1MjI\nSMWGBQCgnpSMcDqdVjQalSS1tbVpenpa+XxekjQ+Pq6mpiatXLlSDQ0N6urqUjqdruzEt3mgPazO\ndc1Vez4AAMrJV+oBuVxOkUikcBwMBpXNZtXY2KhsNqtgMDjr3Pj4eNHrBQLL5fN5XYz8D7ti68ty\nHXwuFPJbj1Dz2KF77NA9duhetXZYMsJf5DiOqyecmppx9eu/KBTyK5u9WtZrLkbs0T126B47dI8d\nuleJHd4t6iVvR4fDYeVyucJxJpNRKBSa89zk5KTC4bDbWQEAWBRKRrizs1MDAwOSpOHhYYXDYTU2\nNkqSWlpalM/nNTExoRs3buj48ePq7Oys7MQAANSJkrejOzo6FIlEFI/H5fF4lEgklEql5Pf71d3d\nrT179uiFF16QJD388MNqbW2t+NAAANQDj+P2Td4vqRL32Xn/wz326B47dI8duscO3VtQ7wkDAIDK\nIMIAABghwgAAGCHCAAAYIcIAABghwgAAGCHCAAAYqfr/JwwAAD7HK2EAAIwQYQAAjBBhAACMEGEA\nAIwQYQAAjBBhAACM1FSE+/r6FIvFFI/Hde7cuVnnTp06pW3btikWi+nAgQNGEy58xXZ4+vRpbd++\nXfF4XC+99JJu3bplNOXCVmyHf7d//349/vjjVZ6sdhTb4aVLl9TT06Nt27bp5ZdfNpqwNhTb45Ej\nRxSLxdTT06NkMmk04cL34YcfKhqN6vDhw3ecq0pXnBrxxz/+0Xn66acdx3GckZERZ/v27bPOf/e7\n33U+/vhj5+bNm05PT49z4cIFizEXtFI77O7udi5duuQ4juP84Ac/cE6cOFH1GRe6Ujt0HMe5cOGC\nE4vFnMcee6za49WEUjv84Q9/6Pz2t791HMdx9uzZ41y8eLHqM9aCYnu8evWqs2XLFuezzz5zHMdx\nnnjiCecvf/mLyZwL2aeffuo89thjzk9/+lPn0KFDd5yvRldq5pVwOp1WNBqVJLW1tWl6elr5fF6S\nND4+rqamJq1cuVINDQ3q6upSOp22HHdBKrZDSUqlUlqxYoUkKRgMampqymTOhazUDiVp7969ev75\n5y3GqwnFdnjr1i29//772rp1qyQpkUho1apVZrMuZMX2uGTJEi1ZskQzMzO6ceOGrl27pqamJstx\nF6SlS5fq4MGDCofDd5yrVldqJsK5XE6BQKBwHAwGlc1mJUnZbFbBYHDOc/iHYjuUpMbGRklSJpPR\nyZMn1dXVVfUZF7pSO0ylUnrwwQfV3NxsMV5NKLbDK1eu6N5779Wrr76qnp4e7d+/32rMBa/YHpct\nW6Zdu3YpGo1qy5YtWrdunVpbW61GXbB8Pp/uueeeOc9Vqys1E+Evcvi0Tdfm2uHly5f1zDPPKJFI\nzPoDjrndvsNPPvlEqVRKTzzxhOFEtef2HTqOo8nJSe3YsUOHDx/WBx98oBMnTtgNV0Nu32M+n9eb\nb76p9957T7/73e909uxZnT9/3nA63E3NRDgcDiuXyxWOM5mMQqHQnOcmJyfnvL2w2BXbofT5H9yn\nnnpKzz33nDZt2mQx4oJXbIenT5/WlStX9Oijj+rZZ5/V8PCw+vr6rEZdsIrtMBAIaNWqVVqzZo28\nXq82btyoCxcuWI26oBXb4+joqFavXq1gMKilS5dqw4YNGhoashq1JlWrKzUT4c7OTg0MDEiShoeH\nFQ6HC7dPW1palM/nNTExoRs3buj48ePq7Oy0HHdBKrZD6fP3Mnfu3KnNmzdbjbjgFdvhQw89pHff\nfVfHjh3TG2+8oUgkot7eXstxF6RiO/T5fFq9erU++uijwnluo86t2B6bm5s1Ojqq69evS5KGhoZ0\n3333WY1ak6rVlZr6KUr79u3Tn/70J3k8HiUSCX3wwQfy+/3q7u7WmTNntG/fPknSd77zHT355JPG\n0y5Md9vhpk2b9MADD2j9+vWFxz7yyCOKxWKG0y5MxX4f/t3ExIReeuklHTp0yHDShavYDsfGxvTi\niy/KcRzdf//92rNnjxoaaub1QlUV2+Nbb72lVColr9er9evX68c//rH1uAvO0NCQXnvtNV28eFE+\nn09f+9rXtHXrVrW0tFStKzUVYQAA6gl/vQQAwAgRBgDACBEGAMAIEQYAwAgRBgDACBEGAMAIEQYA\nwAgRBgDAyP8D2bbylK4z+dUAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<matplotlib.figure.Figure at 0x7fba30b59da0>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "metadata": {
        "id": "ZSuohC1_4KBs",
        "colab_type": "code",
        "outputId": "39013b19-eed9-42c6-a0e6-2b2d52b7d304",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "cell_type": "code",
      "source": [
        "from sklearn import metrics\n",
        "print(\"For RNN the AUC is: \" + str(round(metrics.auc(fpr, tpr),2)))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "For RNN the AUC is: 0.99\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "9TWhXOrC7wJK",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "Very good performance on RNN."
      ]
    },
    {
      "metadata": {
        "id": "6iPE7HEgouD9",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "## 3.2 CNN 1D\n",
        "\n",
        "Data Prep into 3d for CNN: http://debajyotidatta.github.io/nlp/deep/learning/word-embeddings/2016/11/27/Understanding-Convolutions-In-Text/\n",
        "\n",
        "Use method in paper: https://richliao.github.io/supervised/classification/2016/11/26/textclassifier-convolutional/"
      ]
    },
    {
      "metadata": {
        "id": "bncy6CEvnRDH",
        "colab_type": "code",
        "outputId": "feb1ac05-9389-4683-bc44-b3fc5b200f86",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        }
      },
      "cell_type": "code",
      "source": [
        "#our data:\n",
        "train[:10], lable[:10]"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(['.r8---sn-q4f7snly.googlevideo.com',\n",
              "  '.ca-systest6045.us.oracle.com',\n",
              "  '.r13---sn-4g57kned.googlevideo.com',\n",
              "  '.mts.google.com',\n",
              "  '.baymsg1010823.gateway.edge.messenger.live.com',\n",
              "  '.sharepointppe.partners.extranet.microsoft.com',\n",
              "  '.accoun.microsoft.com',\n",
              "  '.gcaapp.iad2.secureserver.net',\n",
              "  '.b2bapisit2.partners.extranet.microsoft.com',\n",
              "  '.cgbusvn.us.oracle.com'],\n",
              " array([1, 1, 1, 1, 1, 1, 1, 1, 1, 1]))"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 119
        }
      ]
    },
    {
      "metadata": {
        "id": "nODSAe-0ecZU",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "Since CNN1D only adapt 3d input, we have to convert our input into a 3d tensor, unlike with RNN a 2d matrix\n",
        "\n",
        "Model input shape:\n",
        "\n",
        "RNN: num_url x padded_maxlen\n",
        "\n",
        "CNN1D: num_url x padded_maxlen x num_chars (This is just the one-hot encoding by all unique characters in all training urls.)"
      ]
    },
    {
      "metadata": {
        "id": "nHAtiMhjYngf",
        "colab_type": "code",
        "outputId": "b4247ab4-61d8-433c-e0c5-0db3df792182",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "cell_type": "code",
      "source": [
        "txt = ''\n",
        "for doc in train: #concatinate all benign/bad urls into one string\n",
        "    for s in doc:\n",
        "        txt += s\n",
        "chars = set(txt)\n",
        "vocab_size = len(chars)\n",
        "print('total chars:', len(chars))\n",
        "char_indices = dict((c, i) for i, c in enumerate(chars))\n",
        "indices_char = dict((i, c) for i, c in enumerate(chars))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "total chars: 63\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "i_hfvTfwYnjd",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "from keras.utils import to_categorical\n",
        "def vectorize_sentences(train, char_indices):\n",
        "    X = []\n",
        "    for url in train:\n",
        "        x = [char_indices[w] for w in url]\n",
        "        x2 = np.eye(len(char_indices))[x]\n",
        "        X.append(x2)\n",
        "    return (pad_sequences(X))\n",
        "\n",
        "train_data = vectorize_sentences(train, char_indices)\n",
        "X_train_cnn1d, X_test_cnn1d, y_train_cnn1d, y_test_cnn1d = train_test_split(train_data, lable, test_size=0.2, random_state=42)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "KiwMt-Z7Ynmn",
        "colab_type": "code",
        "outputId": "4c945d95-35f7-4560-e078-7fd827eee7f5",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "cell_type": "code",
      "source": [
        "X_train_cnn1d.shape #looking good!"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(2075, 86, 63)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 149
        }
      ]
    },
    {
      "metadata": {
        "id": "fQJ6YPYqb-oU",
        "colab_type": "code",
        "outputId": "a7ba82ff-c75f-45cc-e9c2-49d65c12e492",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "cell_type": "code",
      "source": [
        "y_train_cnn1d.shape"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(2075,)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 150
        }
      ]
    },
    {
      "metadata": {
        "id": "MDQcuIwQYnxa",
        "colab_type": "code",
        "outputId": "2a8b4085-21eb-43b6-a251-bc40487e237c",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1734
        }
      },
      "cell_type": "code",
      "source": [
        "from keras.layers import Conv1D, MaxPooling1D, Dropout, Embedding\n",
        "n_timesteps, n_features, n_outputs = X_train.shape[0], X_train.shape[1], y_train.shape[0]\n",
        "\n",
        "cnn1d_model = Sequential()\n",
        "cnn1d_model.add(Conv1D(filters=64, kernel_size=3, activation='relu', input_shape=(X_train_cnn1d.shape[1], X_train_cnn1d.shape[2])))\n",
        "cnn1d_model.add(Dropout(0.5))\n",
        "cnn1d_model.add(MaxPooling1D(pool_size = 2))\n",
        "cnn1d_model.add(Flatten())\n",
        "cnn1d_model.add(Dense(1, activation = 'sigmoid'))\n",
        "cnn1d_model.compile(optimizer = \"rmsprop\", loss='binary_crossentropy', metrics = [\"acc\"])\n",
        "#cnn1d_model.summary()\n",
        "cnn1d_model.fit(X_train_cnn1d, y_train_cnn1d, epochs = 50, batch_size = 256, verbose = 1)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch 1/50\n",
            "2075/2075 [==============================] - 1s 537us/step - loss: 0.6252 - acc: 0.6178\n",
            "Epoch 2/50\n",
            "2075/2075 [==============================] - 0s 55us/step - loss: 0.5259 - acc: 0.7981\n",
            "Epoch 3/50\n",
            "2075/2075 [==============================] - 0s 52us/step - loss: 0.4657 - acc: 0.8096\n",
            "Epoch 4/50\n",
            "2075/2075 [==============================] - 0s 51us/step - loss: 0.4218 - acc: 0.8227\n",
            "Epoch 5/50\n",
            "2075/2075 [==============================] - 0s 53us/step - loss: 0.3928 - acc: 0.8313\n",
            "Epoch 6/50\n",
            "2075/2075 [==============================] - 0s 55us/step - loss: 0.3683 - acc: 0.8419\n",
            "Epoch 7/50\n",
            "2075/2075 [==============================] - 0s 54us/step - loss: 0.3514 - acc: 0.8578\n",
            "Epoch 8/50\n",
            "2075/2075 [==============================] - 0s 51us/step - loss: 0.3366 - acc: 0.8612\n",
            "Epoch 9/50\n",
            "2075/2075 [==============================] - 0s 50us/step - loss: 0.3193 - acc: 0.8684\n",
            "Epoch 10/50\n",
            "2075/2075 [==============================] - 0s 51us/step - loss: 0.3016 - acc: 0.8752\n",
            "Epoch 11/50\n",
            "2075/2075 [==============================] - 0s 52us/step - loss: 0.2897 - acc: 0.8810\n",
            "Epoch 12/50\n",
            "2075/2075 [==============================] - 0s 53us/step - loss: 0.2766 - acc: 0.8882\n",
            "Epoch 13/50\n",
            "2075/2075 [==============================] - 0s 50us/step - loss: 0.2632 - acc: 0.8973\n",
            "Epoch 14/50\n",
            "2075/2075 [==============================] - 0s 51us/step - loss: 0.2549 - acc: 0.8969\n",
            "Epoch 15/50\n",
            "2075/2075 [==============================] - 0s 52us/step - loss: 0.2422 - acc: 0.9060\n",
            "Epoch 16/50\n",
            "2075/2075 [==============================] - 0s 55us/step - loss: 0.2290 - acc: 0.9147\n",
            "Epoch 17/50\n",
            "2075/2075 [==============================] - 0s 50us/step - loss: 0.2153 - acc: 0.9239\n",
            "Epoch 18/50\n",
            "2075/2075 [==============================] - 0s 52us/step - loss: 0.2110 - acc: 0.9239\n",
            "Epoch 19/50\n",
            "2075/2075 [==============================] - 0s 51us/step - loss: 0.1996 - acc: 0.9354\n",
            "Epoch 20/50\n",
            "2075/2075 [==============================] - 0s 52us/step - loss: 0.1909 - acc: 0.9325\n",
            "Epoch 21/50\n",
            "2075/2075 [==============================] - 0s 52us/step - loss: 0.1845 - acc: 0.9340\n",
            "Epoch 22/50\n",
            "2075/2075 [==============================] - 0s 51us/step - loss: 0.1729 - acc: 0.9417\n",
            "Epoch 23/50\n",
            "2075/2075 [==============================] - 0s 53us/step - loss: 0.1639 - acc: 0.9470\n",
            "Epoch 24/50\n",
            "2075/2075 [==============================] - 0s 52us/step - loss: 0.1560 - acc: 0.9518\n",
            "Epoch 25/50\n",
            "2075/2075 [==============================] - 0s 56us/step - loss: 0.1545 - acc: 0.9547\n",
            "Epoch 26/50\n",
            "2075/2075 [==============================] - 0s 56us/step - loss: 0.1416 - acc: 0.9634\n",
            "Epoch 27/50\n",
            "2075/2075 [==============================] - 0s 54us/step - loss: 0.1431 - acc: 0.9600\n",
            "Epoch 28/50\n",
            "2075/2075 [==============================] - 0s 54us/step - loss: 0.1345 - acc: 0.9629\n",
            "Epoch 29/50\n",
            "2075/2075 [==============================] - 0s 54us/step - loss: 0.1291 - acc: 0.9687\n",
            "Epoch 30/50\n",
            "2075/2075 [==============================] - 0s 55us/step - loss: 0.1231 - acc: 0.9682\n",
            "Epoch 31/50\n",
            "2075/2075 [==============================] - 0s 54us/step - loss: 0.1203 - acc: 0.9692\n",
            "Epoch 32/50\n",
            "2075/2075 [==============================] - 0s 54us/step - loss: 0.1160 - acc: 0.9706\n",
            "Epoch 33/50\n",
            "2075/2075 [==============================] - 0s 55us/step - loss: 0.1081 - acc: 0.9735\n",
            "Epoch 34/50\n",
            "2075/2075 [==============================] - 0s 59us/step - loss: 0.1133 - acc: 0.9658\n",
            "Epoch 35/50\n",
            "2075/2075 [==============================] - 0s 54us/step - loss: 0.1026 - acc: 0.9764\n",
            "Epoch 36/50\n",
            "2075/2075 [==============================] - 0s 54us/step - loss: 0.0990 - acc: 0.9769\n",
            "Epoch 37/50\n",
            "2075/2075 [==============================] - 0s 54us/step - loss: 0.0974 - acc: 0.9778\n",
            "Epoch 38/50\n",
            "2075/2075 [==============================] - 0s 54us/step - loss: 0.0913 - acc: 0.9817\n",
            "Epoch 39/50\n",
            "2075/2075 [==============================] - 0s 67us/step - loss: 0.0920 - acc: 0.9773\n",
            "Epoch 40/50\n",
            "2075/2075 [==============================] - 0s 55us/step - loss: 0.0933 - acc: 0.9783\n",
            "Epoch 41/50\n",
            "2075/2075 [==============================] - 0s 57us/step - loss: 0.0848 - acc: 0.9827\n",
            "Epoch 42/50\n",
            "2075/2075 [==============================] - 0s 55us/step - loss: 0.0814 - acc: 0.9831\n",
            "Epoch 43/50\n",
            "2075/2075 [==============================] - 0s 62us/step - loss: 0.0791 - acc: 0.9846\n",
            "Epoch 44/50\n",
            "2075/2075 [==============================] - 0s 56us/step - loss: 0.0750 - acc: 0.9846\n",
            "Epoch 45/50\n",
            "2075/2075 [==============================] - 0s 57us/step - loss: 0.0779 - acc: 0.9827\n",
            "Epoch 46/50\n",
            "2075/2075 [==============================] - 0s 55us/step - loss: 0.0745 - acc: 0.9841\n",
            "Epoch 47/50\n",
            "2075/2075 [==============================] - 0s 55us/step - loss: 0.0733 - acc: 0.9855\n",
            "Epoch 48/50\n",
            "2075/2075 [==============================] - 0s 56us/step - loss: 0.0704 - acc: 0.9855\n",
            "Epoch 49/50\n",
            "2075/2075 [==============================] - 0s 55us/step - loss: 0.0702 - acc: 0.9827\n",
            "Epoch 50/50\n",
            "2075/2075 [==============================] - 0s 56us/step - loss: 0.0679 - acc: 0.9851\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<keras.callbacks.History at 0x7fba3f7de630>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 151
        }
      ]
    },
    {
      "metadata": {
        "id": "m8mjnYybYn2m",
        "colab_type": "code",
        "outputId": "3c248e6c-119d-4642-ccc8-3c6ed8a360ef",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        }
      },
      "cell_type": "code",
      "source": [
        "cnn1d_model.evaluate(X_test_cnn1d, y_test_cnn1d) #lower than RNN"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "519/519 [==============================] - 0s 511us/step\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[0.11250215504670188, 0.9807321773788144]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 152
        }
      ]
    },
    {
      "metadata": {
        "id": "8IkbgwXBYn5f",
        "colab_type": "code",
        "outputId": "3999279f-ed30-42b6-b0e5-c322f6e75d45",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 347
        }
      },
      "cell_type": "code",
      "source": [
        "from sklearn.metrics import roc_curve\n",
        "pred_cnn1d = cnn1d_model.predict(X_test_cnn1d)\n",
        "fpr_cnn1d, tpr_cnn1d, ___ = roc_curve(y_test_cnn1d, pred_cnn1d)\n",
        "plt.plot(fpr_cnn1d, tpr_cnn1d)\n",
        "plt.show() "
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAeEAAAFKCAYAAAAqkecjAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAFUxJREFUeJzt3W9s04edx/GPYwMdjRfZO3tAAmuU\nqoqWiooI0HFBMCJnYlXvGSKO2oK6qr3q2J92RVObTTXa6pTqoA/W9kGFdnsAiAZVVrWTKjJpA90E\nZmHdliqpUEmkpkBRbEOaww1sBX73oKoHJbFDf7a/tvN+PWl+/Bznq69A7/hn1/Y4juMIAACUXZ31\nAAAAzFdEGAAAI0QYAAAjRBgAACNEGAAAI0QYAAAjvnL/wHT6clHvLxBYrMnJ6aLe53zEHt1jh+6x\nQ/fYoXul2GEo5J/xz6v+kbDP57UeoSawR/fYoXvs0D126F45d1j1EQYAoFoRYQAAjBBhAACMEGEA\nAIwQYQAAjBBhAACMEGEAAIwQYQAAjMwpwh988IEikYgOHDhw27kTJ05oy5Yt6u7u1uuvv170AQEA\nqFUFIzw9Pa1f/vKXWrdu3YznX3zxRb366qs6dOiQjh8/rtHR0aIPCQBALSr43tELFy7Uvn37tG/f\nvtvOnT17Vg0NDVq6dKkkaePGjUomk7r33nuLPykAYF44/IdRnTqdMvv5G9qb9O//uqIsP6tghH0+\nn3y+mW+WTqcVDAZzx8FgUGfPns17f4HA4qK/L+dsb4yNO8Me3WOH7rHDufvv/xnR8aHz1mMUXWry\niiQpHPia2Qzl+ntY9k9RKsUnUxT7k5nmI/boHjt0jx3emf/9yzlNXv67Av5FuT/zej26ft0xnMq9\nb3z9Lq1pDWtrp81V1VL8PZwt6q4iHA6HlclkcscTExMKh8Nu7rImWV9amYta+IdrjR26xw7vzBcB\n/q///Lfcn/GLTHVxFeGmpiZls1mdO3dOS5Ys0dGjR7Vnz55izVZUliG8+H9XJX3+2x0AFEvAv0hr\nWnngU80KRnh4eFgvv/yyzp8/L5/Pp4GBAXV2dqqpqUldXV3atWuXnn32WUnSgw8+qObm5pIPfacO\n/2FURwY/kmQTQutLK3PBb8/usUP32CHmm4IRvv/++7V///5Zz69Zs0b9/f1FHarYvngEvHntiooO\nIQBgfpk375j1ja/fRYABABWl7K+OLrWZnvv98qsHAQCoBDUR4ZvDO9OLoHjxAgCgEtVEhE+dTuUe\n7VbDi6AAAJBqJMKSbvt/5QAAqHTz5oVZAABUGiIMAIARIgwAgBEiDACAESIMAIARIgwAgBEiDACA\nESIMAIARIgwAgBEiDACAESIMAIARIgwAgJGq/gCHw38Y1V/OpPm8YABAVarqCJ86ndJk9u98XjAA\noCpVdYQl6V8a7tLu/1hnPQYAAHeM54QBADBChAEAMEKEAQAwQoQBADBChAEAMEKEAQAwQoQBADBC\nhAEAMEKEAQAwQoQBADBChAEAMEKEAQAwQoQBADBChAEAMEKEAQAwQoQBADBChAEAMEKEAQAwQoQB\nADBChAEAMEKEAQAwQoQBADBChAEAMEKEAQAwQoQBADBChAEAMOKby436+vo0NDQkj8ej3t5erVy5\nMnfu4MGD+u1vf6u6ujrdf//9+tnPflayYQEAqCUFHwkPDg5qfHxc/f39isfjisfjuXPZbFa//vWv\ndfDgQR06dEhjY2P629/+VtKBAQCoFQUjnEwmFYlEJEktLS2amppSNpuVJC1YsEALFizQ9PS0rl27\npitXrqihoaG0EwMAUCMKRjiTySgQCOSOg8Gg0um0JGnRokXasWOHIpGINm3apAceeEDNzc2lmxYA\ngBoyp+eEb+Y4Tu7rbDarN954Q0eOHFF9fb22b9+u06dPq7W1ddbvDwQWy+fzfrVpv8Tr9UiSQiF/\nUe5vvmOP7rFD99ihe+zQvXLtsGCEw+GwMplM7jiVSikUCkmSxsbGtHz5cgWDQUnS6tWrNTw8nDfC\nk5PTbmfOuX7dkdfrUTp9uWj3OV+FQn726BI7dI8duscO3SvFDmeLesHL0R0dHRoYGJAkjYyMKBwO\nq76+XpLU2NiosbExXb16VZI0PDyse+65p0gjAwBQ2wo+Em5vb1dbW5ui0ag8Ho9isZgSiYT8fr+6\nurr0+OOPa9u2bfJ6vVq1apVWr15djrkBAKh6c3pOeOfOnbcc33y5ORqNKhqNFncqAADmAd4xCwAA\nI0QYAAAjRBgAACNEGAAAI0QYAAAjRBgAACNEGAAAI0QYAAAjRBgAACNEGAAAI0QYAAAjRBgAACNE\nGAAAI0QYAAAjRBgAACNEGAAAI0QYAAAjRBgAACNEGAAAI0QYAAAjRBgAACNEGAAAI0QYAAAjRBgA\nACNEGAAAI0QYAAAjRBgAACNEGAAAI0QYAAAjRBgAACNEGAAAI0QYAAAjRBgAACNEGAAAI0QYAAAj\nRBgAACNEGAAAI0QYAAAjRBgAACNEGAAAI0QYAAAjRBgAACNEGAAAI0QYAAAjRBgAACNEGAAAI765\n3Kivr09DQ0PyeDzq7e3VypUrc+cuXLign/zkJ/rss8/07W9/W7/4xS9KNiwAALWk4CPhwcFBjY+P\nq7+/X/F4XPF4/Jbzu3fv1ve//3299dZb8nq9+vjjj0s2LAAAtaRghJPJpCKRiCSppaVFU1NTymaz\nkqQbN27o3XffVWdnpyQpFotp2bJlJRwXAIDaUTDCmUxGgUAgdxwMBpVOpyVJly5d0t13362XXnpJ\nPT092rt3b+kmBQCgxszpOeGbOY5zy9cTExPatm2bGhsb9eSTT+rYsWP6zne+M+v3BwKL5fN5v9Kw\nX+b1eiRJoZC/KPc337FH99ihe+zQPXboXrl2WDDC4XBYmUwmd5xKpRQKhSRJgUBAy5Yt04oVKyRJ\n69at05kzZ/JGeHJy2uXI/3T9uiOv16N0+nLR7nO+CoX87NEldugeO3SPHbpXih3OFvWCl6M7Ojo0\nMDAgSRoZGVE4HFZ9fb0kyefzafny5frwww9z55ubm4s0MgAAta3gI+H29na1tbUpGo3K4/EoFosp\nkUjI7/erq6tLvb29eu655+Q4ju67777ci7QAAEB+c3pOeOfOnbcct7a25r7+1re+pUOHDhV3KgAA\n5gHeMQsAACNEGAAAI0QYAAAjRBgAACNEGAAAI0QYAAAjRBgAACNEGAAAI0QYAAAjRBgAACNEGAAA\nI0QYAAAjRBgAACNEGAAAI0QYAAAjRBgAACNEGAAAI0QYAAAjRBgAACNEGAAAI0QYAAAjRBgAACNE\nGAAAI0QYAAAjRBgAACNEGAAAI0QYAAAjRBgAACNEGAAAI0QYAAAjRBgAACNEGAAAI0QYAAAjRBgA\nACNEGAAAI0QYAAAjRBgAACNEGAAAI0QYAAAjRBgAACNEGAAAI0QYAAAjRBgAACNEGAAAI0QYAAAj\nRBgAACNzinBfX5+6u7sVjUb13nvvzXibvXv36tFHHy3qcAAA1LKCER4cHNT4+Lj6+/sVj8cVj8dv\nu83o6KhOnTpVkgEBAKhVBSOcTCYViUQkSS0tLZqamlI2m73lNrt379YzzzxTmgkBAKhRBSOcyWQU\nCARyx8FgUOl0OnecSCS0du1aNTY2lmZCAABqlO9Ov8FxnNzXn3zyiRKJhH7zm99oYmJiTt8fCCyW\nz+e90x87I6/XI0kKhfxFub/5jj26xw7dY4fusUP3yrXDghEOh8PKZDK541QqpVAoJEk6efKkLl26\npIcfflj/+Mc/9NFHH6mvr0+9vb2z3t/k5HQRxv7c9euOvF6P0unLRbvP+SoU8rNHl9ihe+zQPXbo\nXil2OFvUC16O7ujo0MDAgCRpZGRE4XBY9fX1kqTNmzfrnXfe0eHDh/Xaa6+pra0tb4ABAMA/FXwk\n3N7erra2NkWjUXk8HsViMSUSCfn9fnV1dZVjRgAAatKcnhPeuXPnLcetra233aapqUn79+8vzlQA\nAMwDvGMWAABGiDAAAEaIMAAARogwAABGiDAAAEaIMAAARogwAABGiDAAAEaIMAAARogwAABGiDAA\nAEaIMAAARogwAABGiDAAAEaIMAAARogwAABGiDAAAEaIMAAARogwAABGiDAAAEaIMAAARogwAABG\niDAAAEaIMAAARogwAABGiDAAAEaIMAAARogwAABGiDAAAEaIMAAARogwAABGiDAAAEaIMAAARogw\nAABGiDAAAEaIMAAARogwAABGiDAAAEaIMAAARogwAABGiDAAAEaIMAAARogwAABGiDAAAEaIMAAA\nRogwAABGiDAAAEZ8c7lRX1+fhoaG5PF41Nvbq5UrV+bOnTx5Uq+88orq6urU3NyseDyuujraDgBA\nIQVrOTg4qPHxcfX39ysejysej99y/oUXXtCvfvUrvfnmm/r000/1xz/+sWTDAgBQSwpGOJlMKhKJ\nSJJaWlo0NTWlbDabO59IJLRkyRJJUjAY1OTkZIlGBQCgthS8HJ3JZNTW1pY7DgaDSqfTqq+vl6Tc\nf1OplI4fP64f//jHee8vEFgsn8/rZuYcr9cjSQqF/EW5v/mOPbrHDt1jh+6xQ/fKtcM5PSd8M8dx\nbvuzixcv6qmnnlIsFlMgEMj7/ZOT03f6I2d1/bojr9ejdPpy0e5zvgqF/OzRJXboHjt0jx26V4od\nzhb1gpejw+GwMplM7jiVSikUCuWOs9msnnjiCT399NNav359EUYFAGB+KBjhjo4ODQwMSJJGRkYU\nDodzl6Alaffu3dq+fbs2bNhQuikBAKhBBS9Ht7e3q62tTdFoVB6PR7FYTIlEQn6/X+vXr9fbb7+t\n8fFxvfXWW5Kkhx56SN3d3SUfHACAajen54R37tx5y3Fra2vu6+Hh4eJOBADAPMG7agAAYIQIAwBg\nhAgDAGCECAMAYIQIAwBghAgDAGCECAMAYIQIAwBghAgDAGCECAMAYIQIAwBghAgDAGCECAMAYIQI\nAwBghAgDAGCECAMAYIQIAwBghAgDAGCECAMAYIQIAwBghAgDAGCECAMAYIQIAwBghAgDAGCECAMA\nYIQIAwBghAgDAGCECAMAYIQIAwBghAgDAGCECAMAYIQIAwBghAgDAGCECAMAYIQIAwBghAgDAGCE\nCAMAYIQIAwBghAgDAGCECAMAYIQIAwBghAgDAGCECAMAYIQIAwBghAgDAGDEZz2AG2taw/ra4oXW\nYwAA8JXMKcJ9fX0aGhqSx+NRb2+vVq5cmTt34sQJvfLKK/J6vdqwYYN27NhRsmG/bGvnvQqF/Eqn\nL5ftZwIAUCwFL0cPDg5qfHxc/f39isfjisfjt5x/8cUX9eqrr+rQoUM6fvy4RkdHSzYsAAC1pGCE\nk8mkIpGIJKmlpUVTU1PKZrOSpLNnz6qhoUFLly5VXV2dNm7cqGQyWdqJAQCoEQUjnMlkFAgEcsfB\nYFDpdFqSlE6nFQwGZzwHAADyu+MXZjmO4+oHBgKL5fN5Xd3Hl4VC/qLe33zFHt1jh+6xQ/fYoXvl\n2mHBCIfDYWUymdxxKpVSKBSa8dzExITC4XDe+5ucnP6qs86IF2YVB3t0jx26xw7dY4fulWKHs0W9\n4OXojo4ODQwMSJJGRkYUDodVX18vSWpqalI2m9W5c+d07do1HT16VB0dHUUcGwCA2lXwkXB7e7va\n2toUjUbl8XgUi8WUSCTk9/vV1dWlXbt26dlnn5UkPfjgg2pubi750AAA1AKP4/ZJ3jtUiof4XHpx\njz26xw7dY4fusUP3KupyNAAAKA0iDACAESIMAIARIgwAgJGyvzALAAB8jkfCAAAYIcIAABghwgAA\nGCHCAAAYIcIAABghwgAAGKmqCPf19am7u1vRaFTvvffeLedOnDihLVu2qLu7W6+//rrRhJUv3w5P\nnjyprVu3KhqN6vnnn9eNGzeMpqxs+Xb4hb179+rRRx8t82TVI98OL1y4oJ6eHm3ZskUvvPCC0YTV\nId8eDx48qO7ubvX09CgejxtNWPk++OADRSIRHThw4LZzZemKUyX+9Kc/OU8++aTjOI4zOjrqbN26\n9Zbz3/ve95yPP/7YuX79utPT0+OcOXPGYsyKVmiHXV1dzoULFxzHcZwf/vCHzrFjx8o+Y6UrtEPH\ncZwzZ8443d3dziOPPFLu8apCoR3+6Ec/cn73u985juM4u3btcs6fP1/2GatBvj1evnzZ2bRpk/PZ\nZ585juM4jz32mPPXv/7VZM5K9umnnzqPPPKI8/Of/9zZv3//befL0ZWqeSScTCYViUQkSS0tLZqa\nmlI2m5UknT17Vg0NDVq6dKnq6uq0ceNGJZNJy3ErUr4dSlIikdCSJUskScFgUJOTkyZzVrJCO5Sk\n3bt365lnnrEYryrk2+GNGzf07rvvqrOzU5IUi8W0bNkys1krWb49LliwQAsWLND09LSuXbumK1eu\nqKGhwXLcirRw4ULt27dP4XD4tnPl6krVRDiTySgQCOSOg8Gg0um0JCmdTisYDM54Dv+Ub4eSVF9f\nL0lKpVI6fvy4Nm7cWPYZK12hHSYSCa1du1aNjY0W41WFfDu8dOmS7r77br300kvq6enR3r17rcas\nePn2uGjRIu3YsUORSESbNm3SAw88wGe9z8Dn8+muu+6a8Vy5ulI1Ef4yh3fbdG2mHV68eFFPPfWU\nYrHYLf/AMbObd/jJJ58okUjoscceM5yo+ty8Q8dxNDExoW3btunAgQN6//33dezYMbvhqsjNe8xm\ns3rjjTd05MgR/f73v9fQ0JBOnz5tOB1mUzURDofDymQyueNUKqVQKDTjuYmJiRkvL8x3+XYoff4P\n94knntDTTz+t9evXW4xY8fLt8OTJk7p06ZIefvhh/eAHP9DIyIj6+vqsRq1Y+XYYCAS0bNkyrVix\nQl6vV+vWrdOZM2esRq1o+fY4Njam5cuXKxgMauHChVq9erWGh4etRq1K5epK1US4o6NDAwMDkqSR\nkRGFw+Hc5dOmpiZls1mdO3dO165d09GjR9XR0WE5bkXKt0Pp8+cyt2/frg0bNliNWPHy7XDz5s16\n5513dPjwYb322mtqa2tTb2+v5bgVKd8OfT6fli9frg8//DB3nsuoM8u3x8bGRo2Njenq1auSpOHh\nYd1zzz1Wo1alcnWlqj5Fac+ePfrzn/8sj8ejWCym999/X36/X11dXTp16pT27NkjSfrud7+rxx9/\n3HjayjTbDtevX681a9Zo1apVuds+9NBD6u7uNpy2MuX7e/iFc+fO6fnnn9f+/fsNJ61c+XY4Pj6u\n5557To7j6L777tOuXbtUV1c1jxfKKt8e33zzTSUSCXm9Xq1atUo//elPrcetOMPDw3r55Zd1/vx5\n+Xw+ffOb31RnZ6eamprK1pWqijAAALWEXy8BADBChAEAMEKEAQAwQoQBADBChAEAMEKEAQAwQoQB\nADBChAEAMPL/mmn/w0oioT4AAAAASUVORK5CYII=\n",
            "text/plain": [
              "<matplotlib.figure.Figure at 0x7fba3209ceb8>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "metadata": {
        "id": "DvUno2oP3OeB",
        "colab_type": "code",
        "outputId": "823e9d93-7631-4da7-f478-c05a563f5a23",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        }
      },
      "cell_type": "code",
      "source": [
        "from sklearn import metrics\n",
        "print(\"For RNN the AUC is: \" + str(round(metrics.auc(fpr, tpr),2)))\n",
        "print(\"For CNN1D the AUC is: \" + str(round(metrics.auc(fpr_cnn1d, tpr_cnn1d),2)))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "For RNN the AUC is: 0.99\n",
            "For CNN1D the AUC is: 0.99\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "DH0gV9TU28vO",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "To compare both method for character-wise text sequence classificastion for malicioius url detection, we have to refer to the area under the ROC curve. The AUC for both method is 0.99, indicsating that they both achieve similarly high result. "
      ]
    },
    {
      "metadata": {
        "id": "OqPId9alULYi",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}