12_custom_models_and_training_with_tensorflow.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Chapter 12 – Custom Models and Training with TensorFlow**"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "_This notebook contains all the sample code in chapter 12._"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Setup"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "First, let's import a few common modules, ensure MatplotLib plots figures inline and prepare a function to save the figures. We also check that Python 3.5 or later is installed (although Python 2.x may work, it is deprecated so we strongly recommend you use Python 3 instead), as well as Scikit-Learn ≥0.20 and TensorFlow ≥2.0-preview."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Python ≥3.5 is required\n",
    "import sys\n",
    "assert sys.version_info >= (3, 5)\n",
    "\n",
    "# Scikit-Learn ≥0.20 is required\n",
    "import sklearn\n",
    "assert sklearn.__version__ >= \"0.20\"\n",
    "\n",
    "# TensorFlow ≥2.0-preview is required\n",
    "import tensorflow as tf\n",
    "from tensorflow import keras\n",
    "assert tf.__version__ >= \"2.0\"\n",
    "\n",
    "# Common imports\n",
    "import numpy as np\n",
    "import os\n",
    "\n",
    "# to make this notebook's output stable across runs\n",
    "np.random.seed(42)\n",
    "\n",
    "# To plot pretty figures\n",
    "%matplotlib inline\n",
    "import matplotlib as mpl\n",
    "import matplotlib.pyplot as plt\n",
    "mpl.rc('axes', labelsize=14)\n",
    "mpl.rc('xtick', labelsize=12)\n",
    "mpl.rc('ytick', labelsize=12)\n",
    "\n",
    "# Where to save the figures\n",
    "PROJECT_ROOT_DIR = \".\"\n",
    "CHAPTER_ID = \"deep\"\n",
    "IMAGES_PATH = os.path.join(PROJECT_ROOT_DIR, \"images\", CHAPTER_ID)\n",
    "os.makedirs(IMAGES_PATH, exist_ok=True)\n",
    "\n",
    "def save_fig(fig_id, tight_layout=True, fig_extension=\"png\", resolution=300):\n",
    "    path = os.path.join(IMAGES_PATH, fig_id + \".\" + fig_extension)\n",
    "    print(\"Saving figure\", fig_id)\n",
    "    if tight_layout:\n",
    "        plt.tight_layout()\n",
    "    plt.savefig(path, format=fig_extension, dpi=resolution)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Tensors and operations"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Tensors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: id=0, shape=(2, 3), dtype=float32, numpy=\n",
       "array([[1., 2., 3.],\n",
       "       [4., 5., 6.]], dtype=float32)>"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf.constant([[1., 2., 3.], [4., 5., 6.]]) # matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: id=2, shape=(), dtype=int32, numpy=42>"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf.constant(42) # scalar"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: id=4, shape=(2, 3), dtype=float32, numpy=\n",
       "array([[1., 2., 3.],\n",
       "       [4., 5., 6.]], dtype=float32)>"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "t = tf.constant([[1., 2., 3.], [4., 5., 6.]])\n",
    "t"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "TensorShape([2, 3])"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "t.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tf.float32"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "t.dtype"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Indexing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: id=9, shape=(2, 2), dtype=float32, numpy=\n",
       "array([[2., 3.],\n",
       "       [5., 6.]], dtype=float32)>"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "t[:, 1:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: id=14, shape=(2, 1), dtype=float32, numpy=\n",
       "array([[2.],\n",
       "       [5.]], dtype=float32)>"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "t[..., 1, tf.newaxis]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Ops"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: id=17, shape=(2, 3), dtype=float32, numpy=\n",
       "array([[11., 12., 13.],\n",
       "       [14., 15., 16.]], dtype=float32)>"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "t + 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: id=19, shape=(2, 3), dtype=float32, numpy=\n",
       "array([[ 1.,  4.,  9.],\n",
       "       [16., 25., 36.]], dtype=float32)>"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf.square(t)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: id=23, shape=(2, 2), dtype=float32, numpy=\n",
       "array([[14., 32.],\n",
       "       [32., 77.]], dtype=float32)>"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "t @ tf.transpose(t)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Using `keras.backend`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: id=29, shape=(3, 2), dtype=float32, numpy=\n",
       "array([[11., 26.],\n",
       "       [14., 35.],\n",
       "       [19., 46.]], dtype=float32)>"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from tensorflow import keras\n",
    "K = keras.backend\n",
    "K.square(K.transpose(t)) + 10"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### From/To NumPy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: id=31, shape=(3,), dtype=float64, numpy=array([2., 4., 5.])>"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a = np.array([2., 4., 5.])\n",
    "tf.constant(a)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[1., 2., 3.],\n",
       "       [4., 5., 6.]], dtype=float32)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "t.numpy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[1., 2., 3.],\n",
       "       [4., 5., 6.]], dtype=float32)"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.array(t)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: id=36, shape=(3,), dtype=float64, numpy=array([ 4., 16., 25.])>"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf.square(a)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 1.,  4.,  9.],\n",
       "       [16., 25., 36.]], dtype=float32)"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.square(t)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Conflicting Types"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cannot compute Add as input #1(zero-based) was expected to be a float tensor but is a int32 tensor [Op:Add] name: add/\n"
     ]
    }
   ],
   "source": [
    "try:\n",
    "    tf.constant(2.0) + tf.constant(40)\n",
    "except tf.errors.InvalidArgumentError as ex:\n",
    "    print(ex)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cannot compute Add as input #1(zero-based) was expected to be a float tensor but is a double tensor [Op:Add] name: add/\n"
     ]
    }
   ],
   "source": [
    "try:\n",
    "    tf.constant(2.0) + tf.constant(40., dtype=tf.float64)\n",
    "except tf.errors.InvalidArgumentError as ex:\n",
    "    print(ex)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: id=46, shape=(), dtype=float32, numpy=42.0>"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "t2 = tf.constant(40., dtype=tf.float64)\n",
    "tf.constant(2.0) + tf.cast(t2, tf.float32)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Strings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: id=48, shape=(), dtype=string, numpy=b'hello world'>"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf.constant(b\"hello world\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: id=50, shape=(), dtype=string, numpy=b'caf\\xc3\\xa9'>"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf.constant(\"café\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: id=52, shape=(4,), dtype=int32, numpy=array([ 99,  97, 102, 233], dtype=int32)>"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "u = tf.constant([ord(c) for c in \"café\"])\n",
    "u"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: id=64, shape=(), dtype=int32, numpy=4>"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "b = tf.strings.unicode_encode(u, \"UTF-8\")\n",
    "tf.strings.length(b, unit=\"UTF8_CHAR\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: id=69, shape=(4,), dtype=int32, numpy=array([ 99,  97, 102, 233], dtype=int32)>"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf.strings.unicode_decode(b, \"UTF-8\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### String arrays"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "p = tf.constant([\"Café\", \"Coffee\", \"caffè\", \"咖啡\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: id=72, shape=(4,), dtype=int32, numpy=array([4, 6, 5, 2], dtype=int32)>"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf.strings.length(p, unit=\"UTF8_CHAR\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tf.RaggedTensor(values=tf.Tensor(\n",
       "[   67    97   102   233    67   111   102   102   101   101    99    97\n",
       "   102   102   232 21654 21857], shape=(17,), dtype=int32), row_splits=tf.Tensor([ 0  4 10 15 17], shape=(5,), dtype=int64))"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "r = tf.strings.unicode_decode(p, \"UTF8\")\n",
    "r"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<tf.RaggedTensor [[67, 97, 102, 233], [67, 111, 102, 102, 101, 101], [99, 97, 102, 102, 232], [21654, 21857]]>\n"
     ]
    }
   ],
   "source": [
    "print(r)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Ragged tensors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tf.Tensor([ 67 111 102 102 101 101], shape=(6,), dtype=int32)\n"
     ]
    }
   ],
   "source": [
    "print(r[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<tf.RaggedTensor [[67, 111, 102, 102, 101, 101], [99, 97, 102, 102, 232]]>\n"
     ]
    }
   ],
   "source": [
    "print(r[1:3])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<tf.RaggedTensor [[67, 97, 102, 233], [67, 111, 102, 102, 101, 101], [99, 97, 102, 102, 232], [21654, 21857], [65, 66], [], [67]]>\n"
     ]
    }
   ],
   "source": [
    "r2 = tf.ragged.constant([[65, 66], [], [67]])\n",
    "print(tf.concat([r, r2], axis=0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<tf.RaggedTensor [[67, 97, 102, 233, 68, 69, 70], [67, 111, 102, 102, 101, 101, 71], [99, 97, 102, 102, 232], [21654, 21857, 72, 73]]>\n"
     ]
    }
   ],
   "source": [
    "r3 = tf.ragged.constant([[68, 69, 70], [71], [], [72, 73]])\n",
    "print(tf.concat([r, r3], axis=1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: id=235, shape=(4,), dtype=string, numpy=array([b'DEF', b'G', b'', b'HI'], dtype=object)>"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf.strings.unicode_encode(r3, \"UTF-8\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: id=301, shape=(4, 6), dtype=int32, numpy=\n",
       "array([[   67,    97,   102,   233,     0,     0],\n",
       "       [   67,   111,   102,   102,   101,   101],\n",
       "       [   99,    97,   102,   102,   232,     0],\n",
       "       [21654, 21857,     0,     0,     0,     0]], dtype=int32)>"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "r.to_tensor()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Sparse tensors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "s = tf.SparseTensor(indices=[[0, 1], [1, 0], [2, 3]],\n",
    "                    values=[1., 2., 3.],\n",
    "                    dense_shape=[3, 4])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SparseTensor(indices=tf.Tensor(\n",
      "[[0 1]\n",
      " [1 0]\n",
      " [2 3]], shape=(3, 2), dtype=int64), values=tf.Tensor([1. 2. 3.], shape=(3,), dtype=float32), dense_shape=tf.Tensor([3 4], shape=(2,), dtype=int64))\n"
     ]
    }
   ],
   "source": [
    "print(s)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: id=310, shape=(3, 4), dtype=float32, numpy=\n",
       "array([[0., 1., 0., 0.],\n",
       "       [2., 0., 0., 0.],\n",
       "       [0., 0., 0., 3.]], dtype=float32)>"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf.sparse.to_dense(s)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "s2 = s * 2.0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "unsupported operand type(s) for +: 'SparseTensor' and 'float'\n"
     ]
    }
   ],
   "source": [
    "try:\n",
    "    s3 = s + 1.\n",
    "except TypeError as ex:\n",
    "    print(ex)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: id=315, shape=(3, 2), dtype=float32, numpy=\n",
       "array([[ 30.,  40.],\n",
       "       [ 20.,  40.],\n",
       "       [210., 240.]], dtype=float32)>"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "s4 = tf.constant([[10., 20.], [30., 40.], [50., 60.], [70., 80.]])\n",
    "tf.sparse.sparse_dense_matmul(s, s4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SparseTensor(indices=tf.Tensor(\n",
      "[[0 2]\n",
      " [0 1]], shape=(2, 2), dtype=int64), values=tf.Tensor([1. 2.], shape=(2,), dtype=float32), dense_shape=tf.Tensor([3 4], shape=(2,), dtype=int64))\n"
     ]
    }
   ],
   "source": [
    "s5 = tf.SparseTensor(indices=[[0, 2], [0, 1]],\n",
    "                     values=[1., 2.],\n",
    "                     dense_shape=[3, 4])\n",
    "print(s5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "indices[1] = [0,1] is out of order [Op:SparseToDense]\n"
     ]
    }
   ],
   "source": [
    "try:\n",
    "    tf.sparse.to_dense(s5)\n",
    "except tf.errors.InvalidArgumentError as ex:\n",
    "    print(ex)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: id=330, shape=(3, 4), dtype=float32, numpy=\n",
       "array([[0., 2., 1., 0.],\n",
       "       [0., 0., 0., 0.],\n",
       "       [0., 0., 0., 0.]], dtype=float32)>"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "s6 = tf.sparse.reorder(s5)\n",
    "tf.sparse.to_dense(s6)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Sets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: id=338, shape=(2, 6), dtype=int32, numpy=\n",
       "array([[ 2,  3,  4,  5,  6,  7],\n",
       "       [ 0,  7,  9, 10,  0,  0]], dtype=int32)>"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "set1 = tf.constant([[2, 3, 5, 7], [7, 9, 0, 0]])\n",
    "set2 = tf.constant([[4, 5, 6], [9, 10, 0]])\n",
    "tf.sparse.to_dense(tf.sets.union(set1, set2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: id=344, shape=(2, 3), dtype=int32, numpy=\n",
       "array([[2, 3, 7],\n",
       "       [7, 0, 0]], dtype=int32)>"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf.sparse.to_dense(tf.sets.difference(set1, set2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: id=350, shape=(2, 2), dtype=int32, numpy=\n",
       "array([[5, 0],\n",
       "       [0, 9]], dtype=int32)>"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf.sparse.to_dense(tf.sets.intersection(set1, set2))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "v = tf.Variable([[1., 2., 3.], [4., 5., 6.]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Variable 'UnreadVariable' shape=(2, 3) dtype=float32, numpy=\n",
       "array([[ 2.,  4.,  6.],\n",
       "       [ 8., 10., 12.]], dtype=float32)>"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "v.assign(2 * v)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Variable 'UnreadVariable' shape=(2, 3) dtype=float32, numpy=\n",
       "array([[ 2., 42.,  6.],\n",
       "       [ 8., 10., 12.]], dtype=float32)>"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "v[0, 1].assign(42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Variable 'UnreadVariable' shape=(2, 3) dtype=float32, numpy=\n",
       "array([[ 2., 42.,  0.],\n",
       "       [ 8., 10.,  1.]], dtype=float32)>"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "v[:, 2].assign([0., 1.])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "'ResourceVariable' object does not support item assignment\n"
     ]
    }
   ],
   "source": [
    "try:\n",
    "    v[1] = [7., 8., 9.]\n",
    "except TypeError as ex:\n",
    "    print(ex)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Variable 'UnreadVariable' shape=(2, 3) dtype=float32, numpy=\n",
       "array([[100.,  42.,   0.],\n",
       "       [  8.,  10., 200.]], dtype=float32)>"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "v.scatter_nd_update(indices=[[0, 0], [1, 2]],\n",
    "                    updates=[100., 200.])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Variable 'UnreadVariable' shape=(2, 3) dtype=float32, numpy=\n",
       "array([[4., 5., 6.],\n",
       "       [1., 2., 3.]], dtype=float32)>"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sparse_delta = tf.IndexedSlices(values=[[1., 2., 3.], [4., 5., 6.]],\n",
    "                                indices=[1, 0])\n",
    "v.scatter_update(sparse_delta)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Tensor Arrays"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "array = tf.TensorArray(dtype=tf.float32, size=3)\n",
    "array = array.write(0, tf.constant([1., 2.]))\n",
    "array = array.write(1, tf.constant([3., 10.]))\n",
    "array = array.write(2, tf.constant([5., 7.]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: id=391, shape=(2,), dtype=float32, numpy=array([ 3., 10.], dtype=float32)>"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "array.read(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: id=397, shape=(3, 2), dtype=float32, numpy=\n",
       "array([[1., 2.],\n",
       "       [0., 0.],\n",
       "       [5., 7.]], dtype=float32)>"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "array.stack()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: id=406, shape=(2,), dtype=float32, numpy=array([2., 3.], dtype=float32)>"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mean, variance = tf.nn.moments(array.stack(), axes=0)\n",
    "mean"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: id=407, shape=(2,), dtype=float32, numpy=array([4.6666665, 8.666667 ], dtype=float32)>"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "variance"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Custom loss function"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's start by loading and preparing the California housing dataset. We first load it, then split it into a training set, a validation set and a test set, and finally we scale it:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.datasets import fetch_california_housing\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "\n",
    "housing = fetch_california_housing()\n",
    "X_train_full, X_test, y_train_full, y_test = train_test_split(\n",
    "    housing.data, housing.target.reshape(-1, 1), random_state=42)\n",
    "X_train, X_valid, y_train, y_valid = train_test_split(\n",
    "    X_train_full, y_train_full, random_state=42)\n",
    "\n",
    "scaler = StandardScaler()\n",
    "X_train_scaled = scaler.fit_transform(X_train)\n",
    "X_valid_scaled = scaler.transform(X_valid)\n",
    "X_test_scaled = scaler.transform(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "def huber_fn(y_true, y_pred):\n",
    "    error = y_true - y_pred\n",
    "    is_small_error = tf.abs(error) < 1\n",
    "    squared_loss = tf.square(error) / 2\n",
    "    linear_loss  = tf.abs(error) - 0.5\n",
    "    return tf.where(is_small_error, squared_loss, linear_loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAd8AAAEDCAYAAAB0/A4MAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3XmcjeX7wPHPPWNihrFl3yIZIlsqSTKJqCTVV3uoRKpfllSU1i8VUiJEG1KWShJZIiNkKcVXKbLvuzEzZjHL/fvjmmEsY87MnHOes1zv1+u8ZubMM+e5njkz5zrPc1/3dRtrLUoppZTynhCnA1BKKaWCjSZfpZRSyss0+SqllFJepslXKaWU8jJNvkoppZSXafJVSimlvEyTr1I+xBgTbYyxxpgyXtpfV2NMgjf2pZQ6TZOvUgVkjJlgjJl9nvuvykyk1b0flVLKl2nyVSoIGGMucjoGpdRpmnyV8pLzXVI2xlTPvO+qsza/1hiz1hiTbIxZY4xpctZjXWeMWWKMSTTG7DHGjDXGFM/2/ZjM+94xxhwCluchzh7GmM3GmJOZHx8/z/c3ZcZ22Bgz3xhTKPN79Y0xi4wxccaYBGPMOmPMjXn5PSkVDDT5KuWb3gFeAK4CtgKzjTERIAkOWADMAhoCdwGNgE/PeoyHAAO0ADq7slNjzJ3AB8AI4ArgfWCMMeb2zO9fBYwGXgdqAzcB87I9xJfAPuCazJheA5JdPmqlgkQhpwNQKkC0O0/hUkHe3P7XWjsfwBjzCLAbeAD4GHgOmGatHZ61sTGmJ/CHMaactfZg5t3brLXP5nG//YDPrbUfZH69KfOs+wXge6AacAKYZa2NB3YA67L9/CXAO9bafzK/3pzH/SsVFPTMVyn3+Bk508t+e6AAj7ci6xNrbQKwHqibeVcT4KHMy7oJmUk/67JyzWyPsSYf+72ccy9RL8u27x+RhLvNGPOFMaaLMSYy27bvAh8bY34yxrxkjKmTjxiUCniafJVyj0Rr7ebsN+RsNbuMzI8m231h+dhXCHIGnD3RNwRqAWuzbXciH4+dEwuQebZ7JXAPsBMYAPxjjKmU+f3XkEQ9E7gO+J8x5lE3xqFUQNDkq5T3HMr8WDHbfY1y2PbarE+MMUWR8de/M+/6Hah3drLPvCUVMMa/geZn3Xc9sCHrC2ttmrX2J2vtAKABUBRon+37/1prR1prbwM+AboVMCalAo6O+SrlPZuBXcBrxpj+QHVgYA7bDsysUt4LvAKcRIqZAIYAK40xHwLjgHigDnC7tbZHAWMcBnxljFmDFHW1Ax5EirowxrRHLm3/DBwFbgQigb+NMeFIodhXwHagPJK4VxUwJqUCjiZfpbzEWptqjLkPGIMUKa0FXgTOadAB9AeGIxXFfwHtrbUnMh/nf8aYG4BBwBIgFKmI/tYNMc40xvwfUng1AhnffdJa+33mJrFAR+QNQQSwBehmrV2aOZe4FDABObs/knls/Qoal1KBxlhrnY5BKaWUCio65quUUkp5WZ6SrzGmVmZXm8meCkgppZQKdHk98x0N/OqJQJRSSqlg4XLyzSwUiQUWeS4cpZRSKvC5lHwzG7a/AfT1bDhKKaVU4HN1qtF/gU+stbuNMTluZIzpDnQHKFKkSJNq1aoVPEIflZGRQUhIzu9drIUjRwpTpkyKF6Nyj9yOzd8F8vHt2rULay3B/L/n7/zx+JKTQylcOANjLjx7xh+PLS82bdp02Fpb1pVtc02+xphGQGugcW7bWmvHA+MBateubTdu3OhKDH4pJiaG6OjoXLdLTISICM/H406uHpu/CuTji46OJjY2lrVr1+a+sZ8K5OcP/O/4Vq6EevUgMjL3bf3t2PLKGLPD1W1deQsSjXTi2WmM2Y9MmL/bGPN7vqILIseOQYMGkJbmdCRKKeUZEyfC3r1OR+F/XLnsPB6Ymu3rfkgy7umJgAJJqVLw559QSPuIKaUC1NixTkfgn3I987XWJlpr92fdgAQg2Vp7KLefVZCaCi++KGPASikVSO64A/76y+ko/FOez8kylwxTLipWDKpWlUvPYflZPE4ppXzUe+9BANf2eVTglp35CGOgZ0/Yv9/pSJRSyn2++QZKltRhtfzS5OsFSUlw223yUSmlAsGqVTqcVhD6nsULwsNh3To5C1ZKKX+XmgpDhzodhX/TM18vSU+He++Veb9KKeWvrIXGjWHPHqcj8W965uslhQpBt246PqKU8m/GwC+/QPHiTkfi3/TM14tat4bVq3WcRCnlv956S1/D3EGTr5e99x4c0hnSSik/lJ4OF10ERYs6HYn/04ugXmSMlOcrpZQ/OnwYnn3W6SgCg575epm10KYN7NzpdCRKKeW6lBRo2RISEpyOJDDoma+XGQOjR0OVKk5HopRSritcGDZsgABeEdCr9NfogKgo+OornXaklPIPJ09Cly4yv1e5hyZfh2zYAAcPOh2FUkq55q675OxXuYdednbI66/LYgvWaucrpZRvW78eOnRwOorAome+DrrtNlizxukolFIqZ0ePyrKoGRlORxJY9MzXQdOmyaogSinlq0qXhvnznY4i8OiZr4NKloSPP4aNG52ORCmlzrVrF9xyi3a08gSPJd/YWF053hWlSmnpvlLKN1WqBO+8o3Uprvjss7xt77GX/YMHi9Crl7QjUzm7+26oUAHi4pyORCmlTjt+HGbNgnr1nI7Et2VkwIAB8Oijefs5j55zjRwJd9wB8fGe3Iv/GzgQ5s1zOgqllDrtwAGZEqlylpgoS8W+/TaEhubtZz2WfKtWTaR0aZgzB1q0kLEDdX4jRsA99zgdhVJKifR0qFkTXnrJ6Uh81/79cOON8PXXsrziDz/k7ec9lnzDw9NZtUq6Oa1bB02b6rSanBgDkybBF184HYlSSsGCBdC5s9NR+K4//5Sctno1VK8u6xvffHPeHsOjl50vuwxWrIDoaNi3T86AZ8705B7919VXw3XXOR2FUkpJhfPYsU5H4ZvmzZPX6p074dprYeXK/I2Le7zONmuOWNeukJQkLcreeUdL1892+eWnG5crpZRTVq2SHgTFizsdie8ZM0aaI8XHy1DhTz9B+fL5eyyvTHK56CL49FN4801Jus89Bz16aJPusy1fDjExTkehlApmEREyBVKdlp4OffrAU09JdfPAgTBlCoSH5/8xvdbhyhgpx77sMhlL+Ogj2LpVBqu1y5Po1Ek+ar9npZQTDh+WQqv69Z2OxHckJMD998Ps2RAWJrmrS5eCP67X2zt06iRnd+XKwaJF0KyZJGElfvgBevZ0OgqlVDD66it47z2no/Adu3dLrdLs2XI14Mcf3ZN4waHezk2byrhC+/bw11/y9XffacERyBN97bVOR6GUCkY9e2o9Tpbff4fbb4e9e6FWLUnAUVHue3zHGhtWry5jnG3byqWOVq3kGnqwi4yUyxzffON0JEqpYDJypJwE6ZCX/B5atJDEe8MNMmvHnYkXHF5YoUQJeTfRsyekpMADD8Abb+g7r/R0udyhlFLe0q4dNGrkdBTOshbefRfuvFO6V3XuLHOeL77Y/ftyvKV/oUIwerSMMxgDr74qB5yS4nRkzqlRA3r1kt6qSinlaatXy5jmJZc4HYlzUlPlRPDZZyUJDxoEEybIFFBPcDz5giTd3r3lVL9oUZg8GVq3lsvRwWrbNrkUH+xXAZRSnjd/Pvz7r9NROOf4calBGjdOku3UqdJa05OX4H0i+Wa5/XZYuhQqV4Zly6TwKFjXuq1RQ8YZdPxFKeVJGRnw8svBW/C6fbsc+4IFULYsLF4siyV4mk8lX4DGjaUSunFj2LJFEvDixU5H5YyTJ+Hxx3VZRqWU59x0k8w6CUYrV8psmw0boG5dyT3Nmnln3z6XfEHOfJcuhQ4dIDZWGlbndaHiQFC0qBRBZGQ4HYlSKlBNmyaJJ9hMny6rEh08CG3ayOybGjW8t3+fTL4giWfGDBn8TkuThYoHDAiuRGSM9MJes0bHfpVS7jd0qIxxBtPwlrXS6vjeeyE5Gbp3l6Vvvd1p0WeTL8jixO+8Ax9+KJ+//bb8whITnY7Mu4YOlbUjlVLKXdLSJBEVLep0JN5z8iQ88sjpYqrhwyW/hIV5PxafTr5ZevSAuXNllY2vv5ZLBcGSjIyRKwAVKjgdiVIqkBw8CC+8INM9g8HRozKEOXGiLB4xYwb07evcWb9fJF+Qa/IrVkhnrNWrZZB8/Xqno/Kee+6B335zOgqlVCA4elTW7E1LczoS7/j3XyneXbIEKlaEn3+Gjh2djcml5GuMmWyM2WeMiTPGbDLGdPN0YOdTt65Up117rSxk3Ly5LGwcDIYNgyuvdDoKpVQgKF0a1q4NjrPen3+WnPHvv9CwoZy8NWnidFSun/m+BVS31hYHOgCDjDGOhF++vCxgfO+9sqDxbbfJAseBrnp1WfEoWOc9K6XcY+tW6NYtOIqsJk2Shk1Hj0oTjWXLoEoVp6MSLiVfa+1f1tqsho8281bTY1HlIjwcvvxSFjTOyJAFjnv3Dvz5sMePyxsOpZTKrwoV4LHHnI7Cs7Iah3TpIm0je/WCmTOhWDGnIzvNWBfnsBhjxgBdgXDgD+AGa23CWdt0B7oDlC1btsn06dPdGuz5zJ9fnnfeqU1aWgjNmh1m4MC/iYjwfBZOSEigmAPPZHq6ISUlxKPH6NSxeUsgH1/v3r1JT09n1KhRTofiMYH8/IFnj+/YsTD27Qunbt04jzx+brzx3J08GcLbb9dh8eJyhIRYnn76X+68c69H95nlxhtvXGOtvcqlja21Lt+AUOB6YCAQdqFto6KirLcsWWJt6dLWgrUNG1q7a5fn97l48WLP7+Q8Bg2ydvhwz+7DqWPzlkA+vpYtW9qGDRs6HYZHBfLzZ61nj2/FCnkNcYqnn7sDB6xt1kxyQWSktXPnenR35wB+sy7m0zxVO1tr0621y4AqQM88vSXwoBtukEKsWrVg3Tq45hppTBGInn9eyuOVUiov0tKk8Oill5yOxDM2bJBZMCtWQLVq0rGqXTuno8pZfqcaFcLBMd/zqVVLEnDLlrBvnyTkmTOdjsr9wsLkj2rIEKcjUUr5k/ffl0ZFgejHH6Un8/btcPXV0qO5fn2no7qwXJOvMaacMeY+Y0wxY0yoMaYtcD+wyPPh5U3p0rIyRZcu0gXrrrukg0mgtWa89FKIjnY6CqWUP+nVS9arDTQffSRzluPi4O67ISbGP5oSuXLma5FLzLuBY8A7QG9r7SxPBpZfF10kizAMHixJt18/eOIJqXgLFBUryru6pUudjkQp5Q8mToT//Q9KlHA6EvdJT4fnnpPezOnp0L+/LJYQEeF0ZK7JdYq1tfYQ0NILsbiNMfDii3DZZXIWPH68zG376ivvN8/2lCNH4NNPoUULpyNRSvm60qUDK/GeOAEPPSRDi4UKSX9mf5s+5TftJfPjnntkLeBy5WDhQlkweds2p6Nyj6pV5Qw/0Oc2K6UKZuNGaUZU06eqdPJv716p7Zk5U06m5s/3v8QLAZ58Qar7Vq2CevXg779PV8MFgqQkuOKK4FvlSSnlumefhS1bnI7CPdatk9fwNWuk9mXFCmjVyumo8ifgky9Ia8bly2VFi0OHZFWkqVOdjqrgwsNl3NdfxjiUUt43e7bMBvF3c+ZIP//du+XjqlVQp47TUeVfUCRfkPGOOXOk+ColBe6/HwYN8v9K6DJl5DiOHnU6EqWUL0lMhOuvD4wrYyNHQocOMtb7wAMyjFimjNNRFUzQJF+QgfkxY+C996QoK6v3Z0pK7j/ry6pVC56lwZRSromIkGJTf74ylpYGTz8t06QyMuC112DyZChSxOnICi6oki9I0u3dWwbrixaFzz+XtYKPHHE6svzr3Fn+MI8fdzoSpZQvOHECPvlElmH1V3FxcrY7erRMIZ08GV59NXBWYwq65JulQwcZL61UST5ee61/L9f35pvwyy9OR6GU8gVHj8Lhw05HkX87d8ol87lz5fLyokXw4INOR+VeQZt8ARo3loWVGzeGzZulPVlMjNNR5c/IkdLlRSkV3JKT4eKL4YUXnI4kf379Vfrzr18PtWtL2+Drr3c6KvcL6uQLULky/PyznAkfOyYV0Z995nRU+TNpEgwd6nQUSiknLVgg46T+6JtvZA7vgQMyK2XFisCZn3y2oE++IAssz5ghqwWlpsKjj0qHrIwMpyPLm9atJXalVPDq0AHGjXM6iryxVk4c/vMf6V/w2GMwbx6UKuV0ZJ6jyTdTaKgswjB2rHz+1ltw773yh+AvKlWSyu2vvnI6EqWUE0aOhFmzZPUzf3HyJDz++OnL5EOGyGIJF13kbFyelmtv52DzxBPSOaVTJ/j6axn4nzULypd3OjLXZGTIslpKqeDTtq1/TcM5dkzOdn/6SeKePFlWJgoGeuZ7HjffLJXDl1wiBVlNm8KffzodlWuqVpWVPg4ccDoSpZQ3zZ8vJwmXXOJ0JK7ZskWKXH/6SeJesiR4Ei9o8s1RvXrSvqxpU9ixQxZlmD/f6ahck5AAN90kVY9KqeCwYIHMjfUHy5efnt5Zv76c5FxzjdNReZcm3wsoX15WRbrnHoiPl5VBxo51OqrcFSsma3f60+UnpVT+nTghNSvVqjkdSe6+/FIWQzh8GNq1g2XL/CNud9Pkm4vwcJgyBV56SZbve/JJ6NPH95fyCwmBrl39u3GIUip38fHQqJHvX+myFiZOvIQHH5Qiq6eegu+/h+LFnY7MGZp8XRASIosXTJggVYQjRsArr1xBQoLTkV3Y00/Lik5KqcAVGSlL7fnyla6UFHj4YZgwoQYhIfD++/DBB9JvP1hp8s2DLl3gxx9l7tkvv5ShRQtZ3spXXXWVrHv5999OR6KU8oRNm6B/f99ePOHwYelB8MUXUKRIOt99B88843RUztPkm0ctW0q7sypVElm7Vgqyfv/d6ahytnUr7N3rdBRKKU8oU0ZmZ/iqf/6R18hly6BKFRg16g/at3c6Kt+gyTcfoqLggw9+54YbJLG1aAHffed0VOf30ENS3KArHikVWDZvltXYWrVyOpLz++knmUq0dStceaXMHrnsMh8fq/MiTb75VKJEGgsWyKXoxES48054910pKvA1M2fCs886HYVSyp3WrvXdhWA+/VQafsTGwh13SP/8SpWcjsq3BPFwd8EVLiyLMNSqBQMHSoLbtAlGjfKt9m4dOshNKRUYTp6UzlC+JiND+uIPGSJf9+sHb78tLXvVmfTMt4CMkWlI06ZJMh43TuYD+9Jl3tBQKXp44AH/WyxCKXWu++/3vbPexETpiTBkiLzmjBsHw4Zp4s2JJl83uece+WcoW1Yqoq+7DrZtczqq08qVg27d5M2CUsq/TZwotSa+Yv9+iI6WJQGLF4e5c6F7d6ej8m2afN3o2mulqKBuXdiwQar8VqxwOiphjBRmTJ/uXys1KaVOS0uDHj3kc185o1y/Xl7rfv1V+gqsWAFt2jgdle/T5OtmNWrIogxt2sChQ7Ig9LRpTkd12p9/6qILSvmzNm2gaFGnoxDz5kHz5rL6W/aTD5U7Tb4eUKIEzJkj71BTUuC++6RDli9UQv/3v1J1GB/vdCRKqbw4cULe2P/nP74xfDRmjNS3xMfL2uc//STDW8o1mnw9JCxMFmEYPlz+UV5+WXotp6Q4HZkk4MmTnY5CKZUXW7dKL2SnpadD797SmzkjQ2Z6fPml9MFXrtOpRh5kDPTtCzVrSqXxpEmy0P2MGXDxxc7F9eqrwd1TVSl/k5wMV1wh1cNOSkiQSuvZs+UE4+OPoXNnZ2PyV3rm6wV33AFLl8rl3p9/lrGRTZuci6dQIWmJ+dRTzsWglHLdO+/Igi5O2r1bKqxnz4bSpWHhQk28BaHJ10uy2qs1aiRt4Zo1gyVLnIunTh2ZeqSU8n0DBpyucnbCmjWy2P3atdJUaOVKuOEG5+IJBJp8vahKFTkDvv12OHpUqhYnTnQmlogIqFdPLhv5QiGYUur8nnkGduxwbuWi776TRLtvn3xcsUISsCoYTb5eVqwYfPst9OkDqalShDVwoDOdp0JD5fJ3YqL3962Ucs2tt8obd2+zVgpG77xTXiOyllR1sl4lkGjydUBoqCzCMGaMfD54sBQxeLv5RWgoDB0q+z150rv7VkpdWFqaNMVp2xYuusi7+05NhZ49pTeztTJV8rPPvB9HINPk66CePWU+cGSk/JPdeKMzDTB694bly72/X6VUzg4dkrFVbzt+XObvjhsn/eqnTZP+9b4wtziQaPJ1WNu2MnH+kkukIKtpU/jrL+/GMGmSJH6llG9ISIBSpeQKmTeT3rZt0pf+xx+lT31MjPStV+6nydcHXHHF6cS7Y4f88c+f7739h4TIGXjfvt7bp1IqZ9Ony3x8b1qxQl6DNmyQFpGrVsm0SOUZuSZfY0xhY8wnxpgdxph4Y8xaY8wt3ggumJQvD4sXQ6dOEBcnl30+/NB7+7/+ehnfUUo579FHZZzVW6ZNk6tfhw7JLIxffpE+9cpzXDnzLQTsAloCJYCBwHRjTHXPhRWcwsNh6lRZjDo9XcaE+/aVzz2tRAkZe37zTZ16pJSThg6tzdq10kHK06yVgs/77pPWtz16yFWwEiU8v+9gl2vytdaesNa+Zq3dbq3NsNbOBrYBTTwfXvAJCZF/hs8+k3++996Du+6SMSBPi4iQAou0NK2sUMopd9+92ysrA6WknJ7qaIxMKxo71jtJX+VjzNcYUx6IArxcFhRcunaFBQuk6GLWLGnrtnu3Z/cZGgrPPgv79oXr3F+lvCwlRRJg9eonPD6l58gRuPlmKbaMiJDeA337akWzN+Wpvb4xJgz4Aphorf3nPN/vDnQHKFu2LDExMe6I0SclJCR45fjefz+cAQPqs3ZtBI0bp/Dmm+upVcuzp8FTptQgIWENdesG5rqD3nrunBAbG0t6enrAHh8E7vMXF1eILVsqULu2Z49v9255Tdm9O4IyZVIYPHg9JUok4I1faaA+d/lirXXphpwlTwV+AMJy2z4qKsoGssWLF3ttX4cOWduihbVgbUSEtd9959n9ZR1bSopn9+MUbz533tayZUvbsGFDp8PwqEB8/vbvt/bAAfnck8cXE2Nt6dLyWtKwobW7dnlsV+cViM9ddsBv1sWc6tJlZ2OMAT4BygN3W2tTPfReQJ1HmTIy7+7hh6XNW8eOMhbsycKomTPhySc99/hKqdMWLYKPPvLsPiZNkkrmo0ehfXtYtsyZtpVKuHrZeSxwOdDaWuvlJogKpBBq4kSIioKXX5bxmU2bYNQoz6zNe+utMiaklPKstDRZ79tTMjJkznDW1KXevWWJwtBQz+1T5c6Veb6XAD2ARsB+Y0xC5u1Bj0enzmCMVCZOmSLJ+MMPZT7w8ePu31dWwceDD3q/57RSwaRtW1i/3jOPnZQkiX3QIJlJMXq0XDXTxOu8XM+ZrLU7AK2B8yH33SftKO+4QyqimzeXBa6rV3fvfiIi5FK3NlNXynOmTpWhJXc7eFBeI1auPN0/vl079+9H5Y+2l/RTzZpJ+7fLL5de0E2beqYJe7t20t9161b3P7ZSwWzXLnjqKUm87p7is2HD6deEatVk4RRNvL5Fk68fq1FD2sC1aSPvcm+8Ud7dutv27c6stqRUICtdWtbKdXfi/fFHeXO+fTtcfbW8Sa9f3737UAWnydfPlSwp7eC6d4fkZLj3Xve3iHzsMXkXvW2b+x5TqWC2bBns3AmtW7v3ccePh1tukf7w//mPXLWqUMG9+1Duock3AISFSfHV8OHyLvqll+CRR+DkSfftY+1aXXhBKXfZuRP27XPf46Wny/9njx7yef/+slhCRIT79qHcywOTVJQTjJHpR5deKhXKEyfKZadvvoGLLy744195JXz9tfxja6WkUvm3bZt7pxadOCH/8999J9MOx42TVZGUb9Mz3wDTsSMsXQqVKsGSJTL28++/7nv8Vq1kfrFSKu9OnpTF6WNj3fN4e/fCDTdI4i1ZUmY/aOL1D5p8A9CVV0qRRcOGknivvVYScUEZIwVdUVEFfyylgk1GhpyZrl4tibKg1q6Fa66B33+HmjVhxQopulT+QZNvgKpSRYo62reXdnJt2kh7uYIqXx7mzZPJ+kop102fLquGuaO6efZsuP562LNH5vmvXAl16hT8cZX3aPINYMWKSY/m3r0hNRW6dJEOWRkZBXvcOnXkH18p5bpOnaQYsiCshfffl+YZWWO9ixZ5pkmH8ixNvgEuNFTayY0eLZ8PHizFHgVpGVm9ujT3+Phjzy7uoFQgsFaaaWzbVrAkmZYG//d/8mY6IwNefx0+/1xazSr/o8k3SDz5pFyqioyUKQitWkljjvwKCYHNm2WVJaVUzoyRZhrVquX/MeLi4Pbb5U30RRfBF1/AK6+4v0GH8h5NvkGkXTtpM1etmowRNW0qrSnzo1AhePttSEiA/fvdG6dSgeLIEZn217p1/nuk79gh47rz5smZ808/eXYVJOUdmnyDTP36Ugl9zTUyD/i666QdXX5NnFiwn1cqkMXGyllrfv36q7xJ/vNPqbVYuVISsfJ/mnyDUIUKsHixtJ+Li5N2dOPG5e+xnn9eVj46ccK9MSrl79avl/n2//d/+fv5b76Bli2lr3qrVtLHvWZN98aonKPJN0hFRMjY74AB0rXqiSdkGkR6et4f6+BBmUuclub+OJXyVx99BH/8kfefsxaGDJE3x0lJ0lt93jwoVcr9MSrnaHvJIBYSIosw1KolCzO8+y5s2QI9euTtPVm5cnI5rJD+NSkFyOXmkSPz/nMnT8I779Tmhx/k6yFD4LnntLAqEOmZr+KRR6QtXalS0qauV6/G7NmTt8coWlSqL2fM8EyMSvmLjRuluU1ep+EdOyZFkT/8UJHwcLns/PzzmngDlSZfBUhbuhUrZEzp338jado075fMunSBtm09E59S/sBaqF1bGl/kJWlu2SJ92BcvhtKlU1iyBO66y3NxKudp8lWn1K4tl48bNIhlzx5o0QK+/971n69ZU6YePf+8Nt9QwenJJ2H+/Lw1vli2TCqaN26U2QhjxvzO1Vd7LkblGzT5qjOUKQPDhq3joYekgvmOO2DECNeTael9S763AAAdhUlEQVTS0KiRZ2NUyle9/LKsMuSqL76Am26S+cC33CKJuHz5FM8FqHyGJl91josuskyaBG+8IUm3Tx9pj+dKNXNYmDQAWLhQOmApFQx274aePaFiRQgPz317a+G11+Chh6TI6umnYdYsKF7c46EqH6HJV52XMfIu/ssv5RLa2LFSRHL8uGs/v3s3HDrk2RiV8hUXXywtJF0Z501OlqT7+usy42DkSBg1SmcLBBtNvuqC7r9f2tmVLStjWc2bS2es3DzyiIxj/fqrx0NUylFffgm7dsHNN+e+7aFD0mryyy9l1bFZs/LfhEP5N8fea8XFxXHw4EFSU1OdCqFASpQowd9//+10GB5x9rGVKxfG0qXluPPO4vz1lyTVWbPk44UcPAiDBsn0o9BQDwetlEOSk2W4JTf//AO33QZbt8p627NnQ8OGno9P+SZHkm9cXBwHDhygcuXKhIeHY/xwIlt8fDyRkZFOh+ER2Y/NWktSUhJ79uxh4ULo0qU4CxdCdDRMmiRrlOakQgWZN5yQIGNcAfrrUkEqOVkWKnn00dy3/eknuPtuab7RpInMIqhY0fMxKt/lyGXngwcPUrlyZSIiIvwy8QYTYwwRERFUrlyZxMSD/PADPP64vPDccw+89VbuldBDhsD06d6JVylv2bFDzl5z88knMv89NhY6doQlSzTxKoeSb2pqKuGulAQqnxEeHk5qaiphYbIIwzvvSHHJiy/KO/+TJ3P+2ddek/60F9pGKX+yaZO0ZX3vvZy3yciA/v2hWzeZKfDcc9K1qmhR78WpfJdjBVd6xutfsj9fxsgiDDNmyAINEyZIscnRo+f/2dBQufTcuLGufqQCw0svyTJ/OUlMlCtDQ4bI3//48TB0qFQ3KwVa7awKoGNH+PlnuYS2ZIm0x/v33/NvW6yYNBAoWlS7Xyn/lZYmy3BOnw4NGpx/m/37pSbim2+gRAlZkejxx70apvIDmnxVgTRpAqtXS9Xmpk2ytODSpefftlQpmWLxxhvejVEpd/nhB+jbN+f5vOvXn55iV6OGrMHburV3Y1T+QZOvKrAqVSTh3nabXHq+6Sb4/PPzb9uunawdrJS/SUuDDh1gzJjzf3/uXJkHv3OnXAVauRLq1vVujMp/aPLNo+joaJ5++mnHHyM3x44do3z58mzZsiXXbTt16sTw4cMLtL/IyKzlCCE1FTp3liUGz77EXLq0rP/7yCNSLaqUP0hLg6uvhsOH4aKLzv3+6NHSAS4+Hu67T6YWlSvn/TiV/9DkG6DefPNNbr31VmrWrJnrtq+88gqDBw/muKu9I3MQGiqLMHzwgRSW/Pe/0uc5OfnM7YyR5FupUoF2p5RXWCutH+fPl4VHsktPh969pTdzRoa0ZP3iCyhSxJlYlf/Q5BtATmbO5UlMTOTjjz/msccec+nn6tevz6WXXsrkyZPdEsdTT8n8x8hImDoVWrWSblfZ3XCDnPm++aZbdqmUxwweLBX9Z5/JxsfLql/vvy8drrIWI9GKZuUK/TPJh4yMDF5//XXKlClDuXLl6NevHxkZGcD5Lyl37dqV9u3bn3FfWloavXr1olSpUpQqVYrnnnvu1GOAdJYaOnQoNWvWJDw8nPr165+THKOjo+nZsyf9+vWjbNmyNG/eHIAffvgBY8yprwGGDh2KMeac2yuvvAJAhw4dmDJlitt+R7fcIt1/qlWDFSukCGXDhjO3KVtW5koq5cueeELGerPbvVvWu54zR4ZSFi6Ehx92Jj7ln3wi+RrjzC2/vvjiC0JDQ/nll1/44IMPGDFiBNOmTcvzY2RkZLBixQrGjRvH+PHjGTFixKnvDxw4kE8++YTRo0ezYcMGBgwYQI8ePZgzZ84ZjzN58mSstSxdupRJkyYBsHTpUpo0aXLG3NyePXuyb9++U7dnn32WChUq0LlzZwCuueYaVq9eTVJSUn5/LeeoXx9WrZKxsu3bpQjlxx9Pf79ECWlPOWMGrF3rtt0q5RZbt8KDD8qKRaVLn75/zRq45hpYtw6ioqSwKi9r+CoFDi6s4M/q1q3LwIEDiYyMJCoqio8++ohFixZx//33u/wYFStWZOTIkRhjqFOnDps2beLdd9+lb9++nDhxgnfffZcFCxbQokULAGrUqMHq1asZPXo0t91226nHqVGjxjnFUjt27KDSWQOqkZGRp/o1DxkyhClTphATE8Nll10GQKVKlUhNTWXv3r2Uc2OlSIUKEBMjBVjffCNnxGPGQPfup7cJDdW5v8r3VKsmBYTZ36jPnCkJOTHx9Fze7IlZKVf5xJmvtc7c8qvBWbPrK1WqxMGzBzVzce21155xZtqsWTP27NlDXFwcGzZsIDk5mXbt2lGsWLFTt7Fjx55TvdykSZNzHjspKYkiOVR8vPXWW4waNYrFixdTu3btU/dntft055lvlogIaUrQv78UqPToAf36yecg42YNGsBHH0lVqVJOslbm8u7YIWe4WfcNHw533SWJt0sXKcDSxKvyy6UzX2PM00BXoD4wxVrb1YMx+byws9YPM8acGq8NCQnBnpXZ87psYtZjff/991SrVu2C+y56nkaxZcqU4dixY+fcP2jQID788MMzznizHM3sDVm2bNk8xeqqkBBZhKFWLUm+w4fDli0wefLprlfbt0v7yRIlPBKCUi4xBtq0gcqV5evUVKlmHj9evn7zTXkjqR1yVUG4eua7FxgEfOrBWAJC2bJl2bdv3xn3rVu37pztVq1adUaSXrlyJZUqVaJ48eLUrVuXwoULs2PHDi677LIzbpdcckmuMTRu3JgNZ1U3vfHGG4wfP54lS5ack3gB/vzzTypXrkz58uVdPdR8efRRWLAASpaUS3g33AB798pUjsGD5cx30SKPhqBUjmbOlBqEW26R6UKxsXDrrZJ4ixSRKzgDBmjiVQXnUvK11s6w1s4Ejng4Hr/XqlUr5s6dy6xZs9i4cSN9+/Zl165d52y3d+9eevfuzcaNG/n6668ZNmwYffr0AWR8tl+/fvTr149PP/2UzZs3s3btWj788EPGZ739voC2bdvy999/c+SIPF2DBg1i5MiRTJ06laJFi7J//372799PcrYJuEuXLqVt27Zu+i1c2I03SpFKzZrw++9yaS+r4GrPHukBrZQTLr0UqleXz7dtk45VCxfKNKOYmAuvX61UXri14MoY0x3oDnIGGBMTc97tSpQoQXx8vDt37TXp6emcPHmS9PT0U8eQmppKWloa8fHxdOrUid9++41HHnkEgMcff5z27dtz5MiRU9unp6dzzz33kJSURNOmTTHG8PDDD9OtW7dT2zz//POUKFGCoUOH0rNnTyIjI2nQoAG9evU643FOnjx5zu+yevXqNGnShAkTJvD4448zbNgw4uLizph6BDBr1iyio6NJTk7m22+/ZcaMGcTHx59xbNklJyfn+Jzmx/DhYbz8cj3Wry9Js2bpvPLKBpo1O0LLlvDxx8UoXjyVcuVS3La/LAkJCW49Dl8SGxtLenp6wB4feOb5i40N47vvKtG58w6MgdGjizNw4BXExl5E9eoneOut9SQlJeONX2sg/30G8rHlmbXW5Rty6XmCK9tGRUXZnGzYsCHH7/mLuLg4p0O4oLlz59qoqCiblpaW67YffPCBbdOmzamvczo2TzxvycnWPvSQlMCFhFg7YoS1GRnWjhxp7dy5bt+dtdbaxYsXe+aBfUDLli1tw4YNnQ7Dozzx/B09au3EifL51KnWFi4sf5M332xtbKzbd3dBgfz3GcjHZq21wG/WxXzqE9XOyv3atWvHU089xe7du3PdNiwsjFGjRnkhqnMVLiydgV5/XdrzZbXq69lTFmFYtOh0VbRS7matFAJaK00yBg2S3swpKdJcY84cLQBUnqHzfAPYM88849J23bNPunWAMbIIQ61a0LWrzAPeuhWmTJFq6Dp1TleeKuVO1kobVGPkb2/SJPn83XfPneOrlDu5OtWoUOa2oUCoMaYIkGat1VmZym3uv18aG3TsKAuQt2ghPaLLl5dil+hopyNUgeSLL+Dyy+XvrmNH+PlnmZM+Zcq57SSVcjdXLzsPBJKA/sBDmZ8P9FRQKng1by4tKevUgT//lJ7Qc+dKY3vtgqXcqVgxWfDj2msl8VaqJOtSa+JV3uDqVKPXrLXmrNtrHo5NBalLL5XFGG66CQ4cgHvukbVSDx2CX391Ojrl7/73P7m8XLKktIrcvBkaN4bVq+HKK52OTgULLbhSPqlkSTnj7dZN1gPu1AleegkWL3Y6MuXvihSRxRHatIGjR+H22+XMV+sKlDdp8lU+KyxMOgsNHSqFLx9/DP/8I004XCjiVuoM+/fDiy/CxIkwcqS0jezTB779Vi5BK+VNmnyVTzMGnntOVo8JD4fPPoNHHoHffnM6MuVvChWSKydvvikraY0ZI1XNoaFOR6aCkSZf5RfuvFMuDVaoIGN0L7wgL5zHjzsdmfJ1KSmypGW7dtLWNDJS5u/27Ol0ZCqYafJVfuOqq6QopkED2LQJBg6EH390Oirl6/79V6aurVkDl1wCv/wCXmpjrlSONPkqv1K1qoz53norJCXBAw9INbQHliFWAeCWW2Qq0aFDMm1t1Sq44gqno1JKk2+e3XnnnZQqVYqHH37Y6VCCVmQkfPcdPPOMFM189ZW0BdR5wCqLtTBunCxfeeKEVMsvXiwNW5TyBZp886hXr15MmjQpzz+3a9cuoqOjqVu3Lg0aNOCrr77yQHTBo1AheP99GDUKQkKkiKZOHXmhVcEtPV3m6z7xhPQLf/FFmDpVCvaU8hWafPMoOjqayMjIPP9coUKFGDFiBBs2bGDBggX07t2bE5opCuzpp+H776FoURkHvvlmucSogtPx43DXXbI+dKFCUh0/eLC8QVPKl+ifpJdUrFiRRo0aAVChQgXKlCnD0aNHHY4qMNx6qxTRVK0qH2vUgL//djoq5W179kiv5lmzoFQpKcbr2tXpqJQ6P02+DlizZg3p6elUrVrV6VACRoMGUkxz1VVy6blZM5g/3+molLesWCHP/b59ULOmTCnShTiUL9Pk62VHjx6lc+fOjB8/3ulQAk7FirBkCdx9t1x+vOUW+Ogjp6NSnvb995Jo9++XlbBWroSoKKejUurCNPm60dChQzHGnHN75ZVXAEhJSaFjx47079+f6667zuFoA1NEBEyfDs8/LxWv3bvDk09K4Y0KLNZKlXuHDnDyJDz0kFxqLlPG6ciUyp0m3zxq3bo1nTp1YsGCBVSpUoUVK1ac+l7Pnj3Zt2/fqduzzz5LhQoV6Ny5M9ZaunbtSqtWrXSakoeFhMCQIdILOiQExo6Vs2GtbwscaWlSbPfyy/L1f/8rKxUVLuxsXEq5qpDTAfibhQsXAhAfH39O1XNkZOSp+4YMGcKUKVOIiYnhsssuY9myZUybNo0GDRowc+ZMAD7//HPq16/v3QMIIo89JsVXd90FM2fKmOCiRU5HpQrqxIlQoqNh+XJJtp99Bvff73RUSuWNJl8PeOuttxg9ejSLFy8mKnPw6frrrydDr316XatWMgbYqpWsiNS0Kbz6alEtxvFTO3bA0083Zvt2KF5clp3UERzlj3zmsvNrr8kNpFhi0ybpxdqkidz37LMwfLh8XqkS7N0LMTGnKxq7d5fl50A6IMXHSyHG7bfLfQ88AF9+KZ8bk78Ys4/jFi9e/JyxXYBBgwYxevRoYmJiTiVe5aw6dWQB9ebNZSnCJ5+8kjlznI5K5dXq1dCoEWzfXozLL4c//tDEq/yYtdYjt6ioKJuTDRs25Pg9X7Zz507bsmVLe/nll9t69erZ6dOnn/H9119/3VatWtVu3rzZoQjdIy4u7rz3++vzliUpydrHbt5pq7DTGmPtu+86HZH7tWzZ0jZs2NDpMNxuwgRrCxe2tgo77U1Rf9pjx5yOyHMWL17sdAgeE8jHZq21wG/WxRzpM2e+/iB7l6qZM2ee0aVq0KBBjBw5kqlTp1K0aFH279/P/v37SU5OdjhqlaVIEfhoXlVad03DWujbV4p20tKcjkzlJC1NnqeuXWVpwFu7V6X/6MOULOl0ZEoVjCbfPMjepap8+fKnulRZaxk2bBhHjhyhefPmVKxY8dRt+fLlDketsjPTp9Gv6mdMmgRhYTB6tKx6s2+f05Gpsx0+DNdfD++9J60iR4+Gca2mUfFnrZpT/k8LrvLpjz/+OKNL1XFd1d0/jB1L5dhY6q19g5o1ZY7omjXQuDFMmwYtWzodoAL4/XeZHpZVWDVnjiRiouX54403nA5RqQLRM998OHr0KD169NAuVX7uuuvgr7/gxhvhwAH5OHSoLk3oJGtltaqrr5bEe/XV8hxdf73TkSnlXpp88yirS1WfPn20S1UAKF9e1nzt00de+F94QdpSHjzodGTB58gRaN0aeveWjmSPPQY//wxVqjgdmVLup8k3D2y2LlX366z+gFGoELz7Lnz3nVzinD8frrgCnY7kRUuXyjSin36S5SG/+UY6lBUp4nRkSnmGJt88WL58OdOmTWPmzJk0b96cRo0asX79eqfDUm7SoQOsXy/N+Q8dgvbt5ewrMdHpyAJXUpLM4b/hBpmD3awZ/PmndCVTKpBpwVUeZO9Sdb72ksoPfP01fy1fTvMcvl2tmjRvGTYMXnwRPv1ULn1+9pmOO7rbL7/Agw/K2K4xkoTffFOq0HOUy/OnlL/QM18VXMqUIbVEiQtuEhIiY7+//w716sHmzXI2/MQTEBvrpTgDWGIi9OsnHce2b4fatWUt5mHDckm84NLzp5Q/0OSrgsuECVSYN8+lTRs2hN9+gwEDZFx43Di49FIZj9SK6LyzVsbVa9WSVrHGyJuctWulqtkleXj+lPJlmnxVcMnji3eRInIpdO1aaNAAjh2D//xHeor//bfnwgw0mzfLHOqOHaUv+2WXyYIXb7+dx6IqTb4qQGjyVcoF9epJI/+xY6Ua9+efoX59aU95+LDT0fmu48fh+eehbl2paA4Ph5Ej5Y3LNdc4HZ1SztHkq5SLQkJk3HfrVujRQ+aijh4NNWvKZdSkJKcj9B1JSfI7ufRSGctNTYUuXWDbNvi//5PL+EoFM8eSr9VBM7+iz9dp5crBhx/CunVSiBUXJwVEVarAiBHBnYRTU+GTT+QNSb9+cPSoFFb98gtMmCBNTZRSDiXfsLAwkoL5FcoPJSUlEZZrKWpwqV8fliyRZhy1akmi6dMHKlSQph2ZC14FhYQEeeNRuTJ06yYLVURFyWL3S5fK/F2l1GmOJN9y5cqxZ88eEhMT9YzKx1lrSUxMZM+ePZQrV87pcAruhx/439tvu+3hjIFbb4WNG2HWLBnbjIuTOasVKkCvXjKdJlAdOAADB8oZbZ8+0pykalWYMkXGddu1k9+R27j5+VPKKY6MvBQvXhyAvXv3kpqa6kQIBZacnEyRAO19d/axhYWFUb58+VPPm1+LiCDDA8+bMXD77dIVa84cGDxYqnlHjoRRo6BVK+jfXz6G+HmlRUYGLFwoxzZvHqSny/1XXQUvvyy/A48do4eeP6W8zbGyh+LFi/v1i3lMTAyNGzd2OgyPCORjY8wYKm3aJHOFPMAYST7t28Pq1ZJ4p0yBRYvkVrYsdO4si8NfcYVHQvCYzZth6lRJuocOyX3GwB13wHPPydiux3n4+VPKW7TmUAWX6dMp56U2VddcA59/LtW+48dLIdLOnVIFPHy4zHXt0kUuWzdu7ObLs26yYQN8/bW8gfjnn9P3V6oEPXvCI4/IOK/XePH5U8qTXLo4ZIwpbYz51hhzwhizwxjzgKcDUypQVKgAr7wi02yWLpVpSsWKyZnkyy9DkyZQsaIs4vDVV9KEwin790uifewxeXNQrx68+qok3iJFpBfz/Pmwa5eM9Xo18SoVQFw98x0NnATKA42AOcaYddbavzwWmVIBJiREFme4/npZMH7+fJg9W84sDxyQRRw+/VS2LVdOLuO2bCmdterUkSTuzrPjI0dkutQff0gHr8WLYc+eM7eJjJSx7AcfhJtugsKF3bd/pYJZrsnXGFMUuBu4wlqbACwzxswCHgb6ezg+pQJS4cKyhGGHDtIz+n//k2rphQslER48CN9+K7csRYtCjRoyVlyypMwrLl9expHDw6VxxfHjcOJEIZYtg/j407fjx6U5yO7dsGMHbNki958tPFzeHERHQ5s2cjlcG2Io5X4mt6k+xpjGwHJrbUS2+/oBLa21t+f0cxEREfaaAO4fFxsbS8mSJZ0OwyMC+dhYu5a0tDQKXXWV05HkyFpZ+ScuTpJmQgKkpEBamis/vTbzY6NctwwJgYgISerFi8ul8MhI3xx7PsUPnr+CCuT/v0A+NoAlS5assda69MfpSvJtAXxlra2Q7b7HgQettdFnbdsd6J755RXAn3mI29+UAQK1q28gHxvo8fk7PT7/FcjHBlDbWuvSQu+uXFBKAM6eE1QcOOeilbV2PDAewBjzm6vvAPxRIB9fIB8b6PH5Oz0+/xXIxwZyfK5u60q18yagkDGmVrb7GgJabKWUUkrlQ67J11p7ApgBvGGMKWqMaQ7cAXzu6eCUUkqpQORqE7gngXDgIDAF6OnCNKPxBQnMDwTy8QXysYEen7/T4/NfgXxskIfjy7XgSimllFLu5ect3pVSSin/o8lXKaWU8jKvJF9jTC1jTLIxZrI39uctxpjJxph9xpg4Y8wmY0w3p2NyF2NMYWPMJ5m9vOONMWuNMbc4HZc7GWOeNsb8ZoxJMcZMcDqeggrkHuyB9lydLdD/3wL5tTK7vOQ6b535jgZ+9dK+vOktoLq1tjjQARhkjGnicEzuUgjYBbQESgADgenGmOoOxuRue4FBwKdOB+Im2XuwPwiMNcbUczYktwm05+psgf7/Fsivldm5nOs8nnyNMfcBscAiT+/L26y1f1lrU7K+zLzVdDAkt7HWnrDWvmat3W6tzbDWzga2AQHzD2OtnWGtnQkccTqWgsrWg/1la22CtXYZkNWD3e8F0nN1PoH+/xbIr5VZ8prrPJp8jTHFgTeAvp7cj5OMMWOMMYnAP8A+4AeHQ/IIY0x5IAptruKrooA0a+2mbPetAwLlzDeoBOL/WyC/VuYn13n6zPe/wCfW2t0e3o9jrLVPApFAC6QZScqFf8L/GGPCgC+Aidbaf3LbXjmiGBB31n3Hkb9N5UcC9f8twF8r85zr8p18jTExxhibw22ZMaYR0Bp4L7/7cFJux5d9W2tteuZlvipAT2cizhtXj88YE4J0MzsJPO1YwHmUl+cvQLjcg135Ln/9f3OVP75W5ia/uS7fK3WevaLReQLqDVQHdhpZo6wYEGqMqWutvTK/+/WW3I4vB4Xwk3EMV47PyBP3CVLAc6u1NtXTcblLPp8/f3aqB7u19t/M+7QHux/x5/+3fPCb10oXRJOPXOfJy87jkV9uo8zbh8AcoK0H9+k1xphyxpj7jDHFjDGhxpi2wP0EVmHZWOBy4HZrbZLTwbibMaaQMaYIEIr8sxQxxvjl0vGB3oM9kJ6rCwjI/7cgeK3MV67zWPK11iZaa/dn3ZDLYsnW2kOe2qeXWeSyyW7gGPAO0NtaO8vRqNzEGHMJ0AP5Y9pvjEnIvD3ocGjuNBBIAvoDD2V+PtDRiAomPz3Y/UWgPVdnCPD/t4B+rcxvrtPezkoppZSXaXtJpZRSyss0+SqllFJepslXKaWU8jJNvkoppZSXafJVSimlvEyTr1JKKeVlmnyVUkopL9Pkq5RSSnmZJl+llFLKyzT5KhUAjDHP57CC0xtOx6aUOpe2l1QqABhjIoGi2e7qBzwItLDWbnYmKqVUTjT5KhVgjDEvAM8Aray1G52ORyl1rkBbkkupoGaMGQA8Bdxord3kdDxKqfPT5KtUgDDGDASeAKL1UrNSvk2Tr1IBwBjzCtANaGmt3eJ0PEqpC9Pkq5SfyzzjfQboAJwwxlTI/FastTbZuciUUjnRgiul/JgxxgCxQPHzfLu1tXaRl0NSSrlAk69SSinlZdpkQymllPIyTb5KKaWUl2nyVUoppbxMk69SSinlZZp8lVJKKS/T5KuUUkp5mSZfpZRSyss0+SqllFJepslXKaWU8rL/BzJNtiuZCyhkAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 576x252 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(figsize=(8, 3.5))\n",
    "z = np.linspace(-4, 4, 200)\n",
    "plt.plot(z, huber_fn(0, z), \"b-\", linewidth=2, label=\"huber($z$)\")\n",
    "plt.plot(z, z**2 / 2, \"b:\", linewidth=1, label=r\"$\\frac{1}{2}z^2$\")\n",
    "plt.plot([-1, -1], [0, huber_fn(0., -1.)], \"r--\")\n",
    "plt.plot([1, 1], [0, huber_fn(0., 1.)], \"r--\")\n",
    "plt.gca().axhline(y=0, color='k')\n",
    "plt.gca().axvline(x=0, color='k')\n",
    "plt.axis([-4, 4, 0, 4])\n",
    "plt.grid(True)\n",
    "plt.xlabel(\"$z$\")\n",
    "plt.legend(fontsize=14)\n",
    "plt.title(\"Huber loss\", fontsize=14)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "input_shape = X_train.shape[1:]\n",
    "\n",
    "model = keras.models.Sequential([\n",
    "    keras.layers.Dense(30, activation=\"selu\", kernel_initializer=\"lecun_normal\",\n",
    "                       input_shape=input_shape),\n",
    "    keras.layers.Dense(1),\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.compile(loss=huber_fn, optimizer=\"nadam\", metrics=[\"mae\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 11610 samples, validate on 3870 samples\n",
      "Epoch 1/2\n",
      "11610/11610 [==============================] - 1s 71us/sample - loss: 0.6223 - mae: 0.9741 - val_loss: 0.2200 - val_mae: 0.5236\n",
      "Epoch 2/2\n",
      "11610/11610 [==============================] - 1s 48us/sample - loss: 0.2160 - mae: 0.5158 - val_loss: 0.2038 - val_mae: 0.4928\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.callbacks.History at 0x12e4aaac8>"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit(X_train_scaled, y_train, epochs=2,\n",
    "          validation_data=(X_valid_scaled, y_valid))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Saving/Loading Models with Custom Objects"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.save(\"my_model_with_a_custom_loss.h5\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = keras.models.load_model(\"my_model_with_a_custom_loss.h5\",\n",
    "                                custom_objects={\"huber_fn\": huber_fn})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 11610 samples, validate on 3870 samples\n",
      "Epoch 1/2\n",
      "11610/11610 [==============================] - 1s 72us/sample - loss: 0.2056 - mae: 0.4990 - val_loss: 0.1890 - val_mae: 0.4743\n",
      "Epoch 2/2\n",
      "11610/11610 [==============================] - 1s 48us/sample - loss: 0.2004 - mae: 0.4911 - val_loss: 0.1876 - val_mae: 0.4724\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.callbacks.History at 0x1384e1908>"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit(X_train_scaled, y_train, epochs=2,\n",
    "          validation_data=(X_valid_scaled, y_valid))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_huber(threshold=1.0):\n",
    "    def huber_fn(y_true, y_pred):\n",
    "        error = y_true - y_pred\n",
    "        is_small_error = tf.abs(error) < threshold\n",
    "        squared_loss = tf.square(error) / 2\n",
    "        linear_loss  = threshold * tf.abs(error) - threshold**2 / 2\n",
    "        return tf.where(is_small_error, squared_loss, linear_loss)\n",
    "    return huber_fn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.compile(loss=create_huber(2.0), optimizer=\"nadam\", metrics=[\"mae\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 11610 samples, validate on 3870 samples\n",
      "Epoch 1/2\n",
      "11610/11610 [==============================] - 1s 73us/sample - loss: 0.2212 - mae: 0.4882 - val_loss: 0.2103 - val_mae: 0.4669\n",
      "Epoch 2/2\n",
      "11610/11610 [==============================] - 1s 49us/sample - loss: 0.2174 - mae: 0.4836 - val_loss: 0.2020 - val_mae: 0.4647\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.callbacks.History at 0x138432208>"
      ]
     },
     "execution_count": 71,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit(X_train_scaled, y_train, epochs=2,\n",
    "          validation_data=(X_valid_scaled, y_valid))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.save(\"my_model_with_a_custom_loss_threshold_2.h5\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = keras.models.load_model(\"my_model_with_a_custom_loss_threshold_2.h5\",\n",
    "                                custom_objects={\"huber_fn\": create_huber(2.0)})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 11610 samples, validate on 3870 samples\n",
      "Epoch 1/2\n",
      "11610/11610 [==============================] - 1s 74us/sample - loss: 0.2141 - mae: 0.4789 - val_loss: 0.1983 - val_mae: 0.4616\n",
      "Epoch 2/2\n",
      "11610/11610 [==============================] - 1s 49us/sample - loss: 0.2112 - mae: 0.4767 - val_loss: 0.2140 - val_mae: 0.4645\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.callbacks.History at 0x139320978>"
      ]
     },
     "execution_count": 74,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit(X_train_scaled, y_train, epochs=2,\n",
    "          validation_data=(X_valid_scaled, y_valid))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [],
   "source": [
    "class HuberLoss(keras.losses.Loss):\n",
    "    def __init__(self, threshold=1.0, **kwargs):\n",
    "        self.threshold = threshold\n",
    "        super().__init__(**kwargs)\n",
    "    def call(self, y_true, y_pred):\n",
    "        error = y_true - y_pred\n",
    "        is_small_error = tf.abs(error) < self.threshold\n",
    "        squared_loss = tf.square(error) / 2\n",
    "        linear_loss  = self.threshold * tf.abs(error) - self.threshold**2 / 2\n",
    "        return tf.where(is_small_error, squared_loss, linear_loss)\n",
    "    def get_config(self):\n",
    "        base_config = super().get_config()\n",
    "        return {**base_config, \"threshold\": self.threshold}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = keras.models.Sequential([\n",
    "    keras.layers.Dense(30, activation=\"selu\", kernel_initializer=\"lecun_normal\",\n",
    "                       input_shape=input_shape),\n",
    "    keras.layers.Dense(1),\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.compile(loss=HuberLoss(2.), optimizer=\"nadam\", metrics=[\"mae\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 11610 samples, validate on 3870 samples\n",
      "Epoch 1/2\n",
      "11610/11610 [==============================] - 1s 75us/sample - loss: 0.8046 - mae: 0.9679 - val_loss: 0.4891 - val_mae: 0.6449\n",
      "Epoch 2/2\n",
      "11610/11610 [==============================] - 1s 50us/sample - loss: 0.2531 - mae: 0.5239 - val_loss: 0.4233 - val_mae: 0.5946\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.callbacks.History at 0x13aa2b240>"
      ]
     },
     "execution_count": 78,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit(X_train_scaled, y_train, epochs=2,\n",
    "          validation_data=(X_valid_scaled, y_valid))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.save(\"my_model_with_a_custom_loss_class.h5\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [],
   "source": [
    "#model = keras.models.load_model(\"my_model_with_a_custom_loss_class.h5\", # TODO: check PR #25956\n",
    "#                                custom_objects={\"HuberLoss\": HuberLoss})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 11610 samples, validate on 3870 samples\n",
      "Epoch 1/2\n",
      "11610/11610 [==============================] - 1s 49us/sample - loss: 0.2399 - mae: 0.5080 - val_loss: 0.3400 - val_mae: 0.5462\n",
      "Epoch 2/2\n",
      "11610/11610 [==============================] - 1s 50us/sample - loss: 0.2300 - mae: 0.4977 - val_loss: 0.2762 - val_mae: 0.5069\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.callbacks.History at 0x13b4d6080>"
      ]
     },
     "execution_count": 81,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit(X_train_scaled, y_train, epochs=2,\n",
    "          validation_data=(X_valid_scaled, y_valid))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [],
   "source": [
    "#model = keras.models.load_model(\"my_model_with_a_custom_loss_class.h5\",  # TODO: check PR #25956\n",
    "#                                custom_objects={\"HuberLoss\": HuberLoss})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2.0"
      ]
     },
     "execution_count": 83,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.loss.threshold"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Other Custom Functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [],
   "source": [
    "def my_softplus(z): # return value is just tf.nn.softplus(z)\n",
    "    return tf.math.log(tf.exp(z) + 1.0)\n",
    "\n",
    "def my_glorot_initializer(shape, dtype=tf.float32):\n",
    "    stddev = tf.sqrt(2. / (shape[0] + shape[1]))\n",
    "    return tf.random.normal(shape, stddev=stddev, dtype=dtype)\n",
    "\n",
    "def my_l1_regularizer(weights):\n",
    "    return tf.reduce_sum(tf.abs(0.01 * weights))\n",
    "\n",
    "def my_positive_weights(weights): # return value is just tf.nn.relu(weights)\n",
    "    return tf.where(weights < 0., tf.zeros_like(weights), weights)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [],
   "source": [
    "layer = keras.layers.Dense(1, activation=my_softplus,\n",
    "                           kernel_initializer=my_glorot_initializer,\n",
    "                           kernel_regularizer=my_l1_regularizer,\n",
    "                           kernel_constraint=my_positive_weights)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = keras.models.Sequential([\n",
    "    keras.layers.Dense(30, activation=\"selu\", kernel_initializer=\"lecun_normal\",\n",
    "                       input_shape=input_shape),\n",
    "    keras.layers.Dense(1, activation=my_softplus,\n",
    "                       kernel_regularizer=my_l1_regularizer,\n",
    "                       kernel_constraint=my_positive_weights,\n",
    "                       kernel_initializer=my_glorot_initializer),\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.compile(loss=\"mse\", optimizer=\"nadam\", metrics=[\"mae\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 11610 samples, validate on 3870 samples\n",
      "Epoch 1/2\n",
      "11610/11610 [==============================] - 1s 77us/sample - loss: 1.9031 - mae: 0.8792 - val_loss: inf - val_mae: inf\n",
      "Epoch 2/2\n",
      "11610/11610 [==============================] - 1s 49us/sample - loss: 0.7626 - mae: 0.5320 - val_loss: inf - val_mae: inf\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.callbacks.History at 0x138432128>"
      ]
     },
     "execution_count": 88,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit(X_train_scaled, y_train, epochs=2,\n",
    "          validation_data=(X_valid_scaled, y_valid))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.save(\"my_model_with_many_custom_parts.h5\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'\\nmodel = keras.models.load_model(\\n    \"my_model_with_many_custom_parts.h5\",\\n    custom_objects={\\n       \"my_l1_regularizer\": my_l1_regularizer(0.01),\\n       \"my_positive_weights\": my_positive_weights,\\n       \"my_glorot_initializer\": my_glorot_initializer,\\n       \"my_softplus\": my_softplus,\\n    })\\n'"
      ]
     },
     "execution_count": 90,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# TODO: \n",
    "\"\"\"\n",
    "model = keras.models.load_model(\n",
    "    \"my_model_with_many_custom_parts.h5\",\n",
    "    custom_objects={\n",
    "       \"my_l1_regularizer\": my_l1_regularizer(0.01),\n",
    "       \"my_positive_weights\": my_positive_weights,\n",
    "       \"my_glorot_initializer\": my_glorot_initializer,\n",
    "       \"my_softplus\": my_softplus,\n",
    "    })\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [],
   "source": [
    "class MyL1Regularizer(keras.regularizers.Regularizer):\n",
    "    def __init__(self, factor):\n",
    "        self.factor = factor\n",
    "    def __call__(self, weights):\n",
    "        return tf.reduce_sum(tf.abs(self.factor * weights))\n",
    "    def get_config(self):\n",
    "        return {\"factor\": self.factor}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = keras.models.Sequential([\n",
    "    keras.layers.Dense(30, activation=\"selu\", kernel_initializer=\"lecun_normal\",\n",
    "                       input_shape=input_shape),\n",
    "    keras.layers.Dense(1, activation=my_softplus,\n",
    "                       kernel_regularizer=MyL1Regularizer(0.01),\n",
    "                       kernel_constraint=my_positive_weights,\n",
    "                       kernel_initializer=my_glorot_initializer),\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.compile(loss=\"mse\", optimizer=\"nadam\", metrics=[\"mae\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 11610 samples, validate on 3870 samples\n",
      "Epoch 1/2\n",
      "11610/11610 [==============================] - 1s 75us/sample - loss: 2.3877 - mae: 0.9873 - val_loss: inf - val_mae: inf\n",
      "Epoch 2/2\n",
      "11610/11610 [==============================] - 1s 49us/sample - loss: 0.9491 - mae: 0.5495 - val_loss: inf - val_mae: inf\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.callbacks.History at 0x13be91278>"
      ]
     },
     "execution_count": 94,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit(X_train_scaled, y_train, epochs=2,\n",
    "          validation_data=(X_valid_scaled, y_valid))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.save(\"my_model_with_many_custom_parts.h5\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'\\nmodel = keras.models.load_model(\\n    \"my_model_with_many_custom_parts.h5\",\\n    custom_objects={\\n       \"MyL1Regularizer\": MyL1Regularizer,\\n       \"my_positive_weights\": my_positive_weights,\\n       \"my_glorot_initializer\": my_glorot_initializer,\\n       \"my_softplus\": my_softplus,\\n    })\\n'"
      ]
     },
     "execution_count": 96,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# TODO: check https://github.com/tensorflow/tensorflow/issues/26061\n",
    "\"\"\"\n",
    "model = keras.models.load_model(\n",
    "    \"my_model_with_many_custom_parts.h5\",\n",
    "    custom_objects={\n",
    "       \"MyL1Regularizer\": MyL1Regularizer,\n",
    "       \"my_positive_weights\": my_positive_weights,\n",
    "       \"my_glorot_initializer\": my_glorot_initializer,\n",
    "       \"my_softplus\": my_softplus,\n",
    "    })\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Custom Metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = keras.models.Sequential([\n",
    "    keras.layers.Dense(30, activation=\"selu\", kernel_initializer=\"lecun_normal\",\n",
    "                       input_shape=input_shape),\n",
    "    keras.layers.Dense(1),\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.compile(loss=\"mse\", optimizer=\"nadam\", metrics=[create_huber(2.0)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/2\n",
      "11610/11610 [==============================] - 1s 56us/sample - loss: 2.0490 - huber_fn: 0.8245\n",
      "Epoch 2/2\n",
      "11610/11610 [==============================] - 0s 35us/sample - loss: 0.6203 - huber_fn: 0.2507\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.callbacks.History at 0x13ca48dd8>"
      ]
     },
     "execution_count": 99,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit(X_train_scaled, y_train, epochs=2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Warning**: if you use the same function as the loss and a metric, you may be surprised to see different results. This is generally just due to floating point precision errors: even though the mathematical equations are equivalent, the operations are not run in the same order, which can lead to small differences. Moreover, when using sample weights, there's more than just precision errors:\n",
    "* the loss since the start of the epoch is the mean of all batch losses seen so far. Each batch loss is the sum of the weighted instance losses divided by the _batch size_ (not the sum of weights, so the batch loss is _not_ the weighted mean of the losses).\n",
    "* the metric since the start of the epoch is equal to the sum of weighted instance losses divided by sum of all weights seen so far. In other words, it is the weighted mean of all the instance losses. Not the same thing.\n",
    "\n",
    "If you do the math, you will find that metric = loss * mean of sample weights (plus some floating point precision error)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.compile(loss=create_huber(2.0), optimizer=\"nadam\", metrics=[create_huber(2.0)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/2\n",
      "11610/11610 [==============================] - 1s 60us/sample - loss: 0.1167 - huber_fn: 0.2309\n",
      "Epoch 2/2\n",
      "11610/11610 [==============================] - 1s 44us/sample - loss: 0.1115 - huber_fn: 0.2218\n"
     ]
    }
   ],
   "source": [
    "sample_weight = np.random.rand(len(y_train))\n",
    "history = model.fit(X_train_scaled, y_train, epochs=2, sample_weight=sample_weight)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.11666121805001702, 0.11494729649264307)"
      ]
     },
     "execution_count": 102,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "history.history[\"loss\"][0], history.history[\"huber_fn\"][0] * sample_weight.mean()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Streaming metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: id=84595, shape=(), dtype=float32, numpy=0.8>"
      ]
     },
     "execution_count": 103,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "precision = keras.metrics.Precision()\n",
    "precision([0, 1, 1, 1, 0, 1, 0, 1], [1, 1, 0, 1, 0, 1, 0, 1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: id=84645, shape=(), dtype=float32, numpy=0.5>"
      ]
     },
     "execution_count": 104,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "precision([0, 1, 0, 0, 1, 0, 1, 1], [1, 0, 1, 1, 0, 0, 0, 0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: id=84655, shape=(), dtype=float32, numpy=0.5>"
      ]
     },
     "execution_count": 105,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "precision.result()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<tf.Variable 'true_positives:0' shape=(1,) dtype=float32, numpy=array([4.], dtype=float32)>,\n",
       " <tf.Variable 'false_positives:0' shape=(1,) dtype=float32, numpy=array([4.], dtype=float32)>]"
      ]
     },
     "execution_count": 106,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "precision.variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [],
   "source": [
    "precision.reset_states()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Creating a streaming metric:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [],
   "source": [
    "class HuberMetric(keras.metrics.Metric):\n",
    "    def __init__(self, threshold=1.0, **kwargs):\n",
    "        super().__init__(**kwargs) # handles base args (e.g., dtype)\n",
    "        self.threshold = threshold\n",
    "        self.huber_fn = create_huber(threshold)\n",
    "        self.total = self.add_weight(\"total\", initializer=\"zeros\")\n",
    "        self.count = self.add_weight(\"count\", initializer=\"zeros\")\n",
    "    def update_state(self, y_true, y_pred, sample_weight=None):\n",
    "        metric = self.huber_fn(y_true, y_pred)\n",
    "        self.total.assign_add(tf.reduce_sum(metric))\n",
    "        self.count.assign_add(tf.cast(tf.size(y_true), tf.float32))\n",
    "    def result(self):\n",
    "        return self.total / self.count\n",
    "    def get_config(self):\n",
    "        base_config = super().get_config()\n",
    "        return {**base_config, \"threshold\": self.threshold}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: id=84709, shape=(), dtype=float32, numpy=14.0>"
      ]
     },
     "execution_count": 109,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "m = HuberMetric(2.)\n",
    "\n",
    "# total = 2 * |10 - 2| - 2²/2 = 14\n",
    "# count = 1\n",
    "# result = 14 / 1 = 14\n",
    "m(tf.constant([[2.]]), tf.constant([[10.]])) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: id=84745, shape=(), dtype=float32, numpy=7.0>"
      ]
     },
     "execution_count": 110,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# total = total + (|1 - 0|² / 2) + (2 * |9.25 - 5| - 2² / 2) = 14 + 7 = 21\n",
    "# count = count + 2 = 3\n",
    "# result = total / count = 21 / 3 = 7\n",
    "m(tf.constant([[0.], [5.]]), tf.constant([[1.], [9.25]]))\n",
    "\n",
    "m.result()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<tf.Variable 'total:0' shape=() dtype=float32, numpy=21.0>,\n",
       " <tf.Variable 'count:0' shape=() dtype=float32, numpy=3.0>]"
      ]
     },
     "execution_count": 111,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "m.variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<tf.Variable 'total:0' shape=() dtype=float32, numpy=0.0>,\n",
       " <tf.Variable 'count:0' shape=() dtype=float32, numpy=0.0>]"
      ]
     },
     "execution_count": 112,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "m.reset_states()\n",
    "m.variables"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's check that the `HuberMetric` class works well:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = keras.models.Sequential([\n",
    "    keras.layers.Dense(30, activation=\"selu\", kernel_initializer=\"lecun_normal\",\n",
    "                       input_shape=input_shape),\n",
    "    keras.layers.Dense(1),\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.compile(loss=create_huber(2.0), optimizer=\"nadam\", metrics=[HuberMetric(2.0)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/2\n",
      "11610/11610 [==============================] - 1s 63us/sample - loss: 0.8135 - huber_metric_1: 0.8135\n",
      "Epoch 2/2\n",
      "11610/11610 [==============================] - 1s 46us/sample - loss: 0.2471 - huber_metric_1: 0.2471\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.callbacks.History at 0x13e15b7f0>"
      ]
     },
     "execution_count": 115,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit(X_train_scaled, y_train, epochs=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.save(\"my_model_with_a_custom_metric.h5\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {},
   "outputs": [],
   "source": [
    "#model = keras.models.load_model(\"my_model_with_a_custom_metric.h5\",           # TODO: check PR #25956\n",
    "#                                custom_objects={\"huber_fn\": create_huber(2.0),\n",
    "#                                                \"HuberMetric\": HuberMetric})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/2\n",
      "11610/11610 [==============================] - 0s 41us/sample - loss: 0.2324 - huber_metric_1: 0.2324\n",
      "Epoch 2/2\n",
      "11610/11610 [==============================] - 0s 38us/sample - loss: 0.2236 - huber_metric_1: 0.2236\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.callbacks.History at 0x13b683ac8>"
      ]
     },
     "execution_count": 118,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit(X_train_scaled, y_train, epochs=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2.0"
      ]
     },
     "execution_count": 119,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.metrics[0].threshold"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Looks like it works fine! More simply, we could have created the class like this:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {},
   "outputs": [],
   "source": [
    "class HuberMetric(keras.metrics.Mean):\n",
    "    def __init__(self, threshold=1.0, name='HuberMetric', dtype=None):\n",
    "        self.threshold = threshold\n",
    "        self.huber_fn = create_huber(threshold)\n",
    "        super().__init__(name=name, dtype=dtype)\n",
    "    def update_state(self, y_true, y_pred, sample_weight=None):\n",
    "        metric = self.huber_fn(y_true, y_pred)\n",
    "        super(HuberMetric, self).update_state(metric, sample_weight)\n",
    "    def get_config(self):\n",
    "        base_config = super().get_config()\n",
    "        return {**base_config, \"threshold\": self.threshold}        "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This class handles shapes better, and it also supports sample weights."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = keras.models.Sequential([\n",
    "    keras.layers.Dense(30, activation=\"selu\", kernel_initializer=\"lecun_normal\",\n",
    "                       input_shape=input_shape),\n",
    "    keras.layers.Dense(1),\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.compile(loss=keras.losses.Huber(2.0), optimizer=\"nadam\", weighted_metrics=[HuberMetric(2.0)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/2\n",
      "11610/11610 [==============================] - 1s 84us/sample - loss: 0.4076 - HuberMetric: 0.8146\n",
      "Epoch 2/2\n",
      "11610/11610 [==============================] - 1s 52us/sample - loss: 0.1188 - HuberMetric: 0.2374\n"
     ]
    }
   ],
   "source": [
    "sample_weight = np.random.rand(len(y_train))\n",
    "history = model.fit(X_train_scaled, y_train, epochs=2, sample_weight=sample_weight)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.407584022901933, 0.4075840845370592)"
      ]
     },
     "execution_count": 124,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "history.history[\"loss\"][0], history.history[\"HuberMetric\"][0] * sample_weight.mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.save(\"my_model_with_a_custom_metric_v2.h5\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {},
   "outputs": [],
   "source": [
    "#model = keras.models.load_model(\"my_model_with_a_custom_metric_v2.h5\",        # TODO: check PR #25956\n",
    "#                                custom_objects={\"HuberMetric\": HuberMetric})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/2\n",
      "11610/11610 [==============================] - 1s 45us/sample - loss: 0.2269 - HuberMetric: 0.2269\n",
      "Epoch 2/2\n",
      "11610/11610 [==============================] - 0s 41us/sample - loss: 0.2193 - HuberMetric: 0.2193\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.callbacks.History at 0x13f1e97b8>"
      ]
     },
     "execution_count": 127,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit(X_train_scaled, y_train, epochs=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2.0"
      ]
     },
     "execution_count": 128,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.metrics[0].threshold"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Custom Layers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {},
   "outputs": [],
   "source": [
    "exponential_layer = keras.layers.Lambda(lambda x: tf.exp(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: id=110600, shape=(3,), dtype=float32, numpy=array([0.36787945, 1.        , 2.7182817 ], dtype=float32)>"
      ]
     },
     "execution_count": 130,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "exponential_layer([-1., 0., 1.])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Adding an exponential layer at the output of a regression model can be useful if the values to predict are positive and with very different scales (e.g., 0.001, 10., 10000):"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 11610 samples, validate on 3870 samples\n",
      "Epoch 1/5\n",
      "11610/11610 [==============================] - 1s 81us/sample - loss: 17455295763.6862 - val_loss: 5.2690\n",
      "Epoch 2/5\n",
      "11610/11610 [==============================] - 1s 50us/sample - loss: 34.5006 - val_loss: 1.6420\n",
      "Epoch 3/5\n",
      "11610/11610 [==============================] - 1s 50us/sample - loss: 2.1489 - val_loss: 1.1421\n",
      "Epoch 4/5\n",
      "11610/11610 [==============================] - 1s 49us/sample - loss: 1.1264 - val_loss: 0.7558\n",
      "Epoch 5/5\n",
      "11610/11610 [==============================] - 1s 48us/sample - loss: 0.6549 - val_loss: 0.6205\n",
      "5160/5160 [==============================] - 0s 19us/sample - loss: 29118.6526\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "29118.652586295255"
      ]
     },
     "execution_count": 131,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model = keras.models.Sequential([\n",
    "    keras.layers.Dense(30, activation=\"relu\", input_shape=input_shape),\n",
    "    keras.layers.Dense(1),\n",
    "    exponential_layer\n",
    "])\n",
    "model.compile(loss=\"mse\", optimizer=\"nadam\")\n",
    "model.fit(X_train_scaled, y_train, epochs=5,\n",
    "          validation_data=(X_valid_scaled, y_valid))\n",
    "model.evaluate(X_test_scaled, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "metadata": {},
   "outputs": [],
   "source": [
    "class MyDense(keras.layers.Layer):\n",
    "    def __init__(self, units, activation=None, **kwargs):\n",
    "        super().__init__(**kwargs)\n",
    "        self.units = units\n",
    "        self.activation = keras.activations.get(activation)\n",
    "\n",
    "    def build(self, batch_input_shape):\n",
    "        self.kernel = self.add_weight(\n",
    "            name=\"kernel\", shape=[batch_input_shape[-1], self.units],\n",
    "            initializer=\"glorot_normal\")\n",
    "        self.bias = self.add_weight(\n",
    "            name=\"bias\", shape=[self.units], initializer=\"zeros\")\n",
    "        super().build(batch_input_shape) # must be at the end\n",
    "\n",
    "    def call(self, X):\n",
    "        return self.activation(X @ self.kernel + self.bias)\n",
    "\n",
    "    def compute_output_shape(self, batch_input_shape):\n",
    "        return tf.TensorShape(batch_input_shape.as_list()[:-1] + [self.units])\n",
    "\n",
    "    def get_config(self):\n",
    "        base_config = super().get_config()\n",
    "        return {**base_config, \"units\": self.units,\n",
    "                \"activation\": keras.activations.serialize(self.activation)}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = keras.models.Sequential([\n",
    "    MyDense(30, activation=\"relu\", input_shape=input_shape),\n",
    "    MyDense(1)\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 11610 samples, validate on 3870 samples\n",
      "Epoch 1/2\n",
      "11610/11610 [==============================] - 1s 70us/sample - loss: 1.4813 - val_loss: 22.7766\n",
      "Epoch 2/2\n",
      "11610/11610 [==============================] - 1s 44us/sample - loss: 0.7190 - val_loss: 15.0291\n",
      "5160/5160 [==============================] - 0s 18us/sample - loss: 0.5322\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.5321827204652535"
      ]
     },
     "execution_count": 134,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.compile(loss=\"mse\", optimizer=\"nadam\")\n",
    "model.fit(X_train_scaled, y_train, epochs=2,\n",
    "          validation_data=(X_valid_scaled, y_valid))\n",
    "model.evaluate(X_test_scaled, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.save(\"my_model_with_a_custom_layer.h5\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = keras.models.load_model(\"my_model_with_a_custom_layer.h5\",\n",
    "                                custom_objects={\"MyDense\": MyDense})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 137,
   "metadata": {},
   "outputs": [],
   "source": [
    "class MyMultiLayer(keras.layers.Layer):\n",
    "    def call(self, X):\n",
    "        X1, X2 = X\n",
    "        return X1 + X2, X1 * X2\n",
    "\n",
    "    def compute_output_shape(self, batch_input_shape):\n",
    "        batch_input_shape1, batch_input_shape2 = batch_input_shape\n",
    "        return [batch_input_shape1, batch_input_shape2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "metadata": {},
   "outputs": [],
   "source": [
    "inputs1 = keras.layers.Input(shape=[2])\n",
    "inputs2 = keras.layers.Input(shape=[2])\n",
    "outputs1, outputs2 = MyMultiLayer()((inputs1, inputs2))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's create a layer with a different behavior during training and testing:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 139,
   "metadata": {},
   "outputs": [],
   "source": [
    "class AddGaussianNoise(keras.layers.Layer):\n",
    "    def __init__(self, stddev, **kwargs):\n",
    "        super().__init__(**kwargs)\n",
    "        self.stddev = stddev\n",
    "\n",
    "    def call(self, X, training=None):\n",
    "        if training:\n",
    "            noise = tf.random.normal(tf.shape(X), stddev=self.stddev)\n",
    "            return X + noise\n",
    "        else:\n",
    "            return X\n",
    "\n",
    "    def compute_output_shape(self, batch_input_shape):\n",
    "        return batch_input_shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 140,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 11610 samples, validate on 3870 samples\n",
      "Epoch 1/2\n",
      "11610/11610 [==============================] - 1s 70us/sample - loss: 0.5817 - val_loss: 12.9048\n",
      "Epoch 2/2\n",
      "11610/11610 [==============================] - 1s 48us/sample - loss: 0.5087 - val_loss: 8.0876\n",
      "5160/5160 [==============================] - 0s 17us/sample - loss: 0.4285\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.4284568049648935"
      ]
     },
     "execution_count": 140,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.compile(loss=\"mse\", optimizer=\"nadam\")\n",
    "model.fit(X_train_scaled, y_train, epochs=2,\n",
    "          validation_data=(X_valid_scaled, y_valid))\n",
    "model.evaluate(X_test_scaled, y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Custom Models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 141,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_new_scaled = X_test_scaled"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 142,
   "metadata": {},
   "outputs": [],
   "source": [
    "class ResidualBlock(keras.layers.Layer):\n",
    "    def __init__(self, n_layers, n_neurons, **kwargs):\n",
    "        super().__init__(**kwargs)\n",
    "        self.n_layers = n_layers                                     # not shown in the book\n",
    "        self.n_neurons = n_neurons                                   # not shown\n",
    "        self.hidden = [keras.layers.Dense(n_neurons, activation=\"elu\",\n",
    "                                          kernel_initializer=\"he_normal\")\n",
    "                       for _ in range(n_layers)]\n",
    "\n",
    "    def call(self, inputs):\n",
    "        Z = inputs\n",
    "        for layer in self.hidden:\n",
    "            Z = layer(Z)\n",
    "        return inputs + Z\n",
    "    \n",
    "    def get_config(self):                                            # not shown\n",
    "        base_config = super().get_config()                           # not shown\n",
    "        return {**base_config,                                       # not shown\n",
    "                \"n_layers\": self.n_layers, \"n_neurons\": n_neurons}   # not shown"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 143,
   "metadata": {},
   "outputs": [],
   "source": [
    "class ResidualRegressor(keras.models.Model):\n",
    "    def __init__(self, output_dim, **kwargs):\n",
    "        super().__init__(**kwargs)\n",
    "        self.output_dim = output_dim                                 # not shown in the book\n",
    "        self.hidden1 = keras.layers.Dense(30, activation=\"elu\",\n",
    "                                          kernel_initializer=\"he_normal\")\n",
    "        self.block1 = ResidualBlock(2, 30)\n",
    "        self.block2 = ResidualBlock(2, 30)\n",
    "        self.out = keras.layers.Dense(output_dim)\n",
    "\n",
    "    def call(self, inputs):\n",
    "        Z = self.hidden1(inputs)\n",
    "        for _ in range(1 + 3):\n",
    "            Z = self.block1(Z)\n",
    "        Z = self.block2(Z)\n",
    "        return self.out(Z)\n",
    "\n",
    "    def get_config(self):                                            # not shown\n",
    "        base_config = super().get_config()                           # not shown\n",
    "        return {**base_config,                                       # not shown\n",
    "                \"output_dim\": self.output_dim}                       # not shown"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 144,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/5\n",
      "11610/11610 [==============================] - 1s 118us/sample - loss: 2.7279\n",
      "Epoch 2/5\n",
      "11610/11610 [==============================] - 1s 69us/sample - loss: 1.0324\n",
      "Epoch 3/5\n",
      "11610/11610 [==============================] - 1s 66us/sample - loss: 1.0542\n",
      "Epoch 4/5\n",
      "11610/11610 [==============================] - 1s 64us/sample - loss: 0.5402\n",
      "Epoch 5/5\n",
      "11610/11610 [==============================] - 1s 65us/sample - loss: 0.6903\n",
      "5160/5160 [==============================] - 0s 34us/sample - loss: 1.2116\n"
     ]
    }
   ],
   "source": [
    "model = ResidualRegressor(1)\n",
    "model.compile(loss=\"mse\", optimizer=\"nadam\")\n",
    "history = model.fit(X_train_scaled, y_train, epochs=5)\n",
    "score = model.evaluate(X_test_scaled, y_test)\n",
    "y_pred = model.predict(X_new_scaled)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 145,
   "metadata": {},
   "outputs": [],
   "source": [
    "#TODO: check that persistence ends up working in TF2\n",
    "#model.save(\"my_custom_model.h5\")\n",
    "#model = keras.models.load_model(\"my_custom_model.h5\",\n",
    "#                                custom_objects={\n",
    "#                                    \"ResidualBlock\": ResidualBlock,\n",
    "#                                    \"ResidualRegressor\": ResidualRegressor\n",
    "#                                })"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We could have defined the model using the sequential API instead:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 146,
   "metadata": {},
   "outputs": [],
   "source": [
    "block1 = ResidualBlock(2, 30)\n",
    "model = keras.models.Sequential([\n",
    "    keras.layers.Dense(30, activation=\"elu\", kernel_initializer=\"he_normal\"),\n",
    "    block1, block1, block1, block1,\n",
    "    ResidualBlock(2, 30),\n",
    "    keras.layers.Dense(1)\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 147,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/5\n",
      "11610/11610 [==============================] - 2s 163us/sample - loss: 1.5833\n",
      "Epoch 2/5\n",
      "11610/11610 [==============================] - 1s 97us/sample - loss: 0.6670\n",
      "Epoch 3/5\n",
      "11610/11610 [==============================] - 1s 96us/sample - loss: 2.2517\n",
      "Epoch 4/5\n",
      "11610/11610 [==============================] - 1s 94us/sample - loss: 0.5019\n",
      "Epoch 5/5\n",
      "11610/11610 [==============================] - 1s 94us/sample - loss: 0.5117\n",
      "5160/5160 [==============================] - 0s 28us/sample - loss: 1.0046\n"
     ]
    }
   ],
   "source": [
    "model.compile(loss=\"mse\", optimizer=\"nadam\")\n",
    "history = model.fit(X_train_scaled, y_train, epochs=5)\n",
    "score = model.evaluate(X_test_scaled, y_test)\n",
    "y_pred = model.predict(X_new_scaled)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Losses and Metrics Based on Model Internals"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "TODO: check https://github.com/tensorflow/tensorflow/issues/26260\n",
    "```python\n",
    "class ReconstructingRegressor(keras.models.Model):\n",
    "    def __init__(self, output_dim, **kwargs):\n",
    "        super().__init__(**kwargs)\n",
    "        self.hidden = [keras.layers.Dense(30, activation=\"selu\",\n",
    "                                          kernel_initializer=\"lecun_normal\")\n",
    "                       for _ in range(5)]\n",
    "        self.out = keras.layers.Dense(output_dim)\n",
    "        self.reconstruction_mean = keras.metrics.Mean(name=\"reconstruction_error\")\n",
    "\n",
    "    def build(self, batch_input_shape):\n",
    "        n_inputs = batch_input_shape[-1]\n",
    "        self.reconstruct = keras.layers.Dense(n_inputs)\n",
    "        super().build(batch_input_shape)\n",
    "\n",
    "    @tf.function\n",
    "    def call(self, inputs, training=None):\n",
    "        Z = inputs\n",
    "        for layer in self.hidden:\n",
    "            Z = layer(Z)\n",
    "        reconstruction = self.reconstruct(Z)\n",
    "        recon_loss = tf.reduce_mean(tf.square(reconstruction - inputs))\n",
    "        self.add_loss(0.05 * reconstruction_loss)\n",
    "        if training:\n",
    "            result = self.reconstruction_mean(recon_loss)\n",
    "            self.add_metric(result)\n",
    "        return self.out(Z)\n",
    "\n",
    "model = ReconstructingRegressor(1)\n",
    "model.build(tf.TensorShape([None, 8]))       # <= Fails if this line is removed\n",
    "model.compile(loss=\"mse\", optimizer=\"nadam\")\n",
    "history = model.fit(X, y, epochs=2)\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 148,
   "metadata": {},
   "outputs": [],
   "source": [
    "class ReconstructingRegressor(keras.models.Model):\n",
    "    def __init__(self, output_dim, **kwargs):\n",
    "        super().__init__(**kwargs)\n",
    "        self.hidden = [keras.layers.Dense(30, activation=\"selu\",\n",
    "                                          kernel_initializer=\"lecun_normal\")\n",
    "                       for _ in range(5)]\n",
    "        self.out = keras.layers.Dense(output_dim)\n",
    "\n",
    "    def build(self, batch_input_shape):\n",
    "        n_inputs = batch_input_shape[-1]\n",
    "        self.reconstruct = keras.layers.Dense(n_inputs)\n",
    "        super().build(batch_input_shape)\n",
    "\n",
    "    def call(self, inputs):\n",
    "        Z = inputs\n",
    "        for layer in self.hidden:\n",
    "            Z = layer(Z)\n",
    "        reconstruction = self.reconstruct(Z)\n",
    "        recon_loss = tf.reduce_mean(tf.square(reconstruction - inputs))\n",
    "        self.add_loss(0.05 * recon_loss)\n",
    "        return self.out(Z)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/2\n",
      "11610/11610 [==============================] - 2s 141us/sample - loss: 0.6859\n",
      "Epoch 2/2\n",
      "11610/11610 [==============================] - 1s 83us/sample - loss: 0.4133\n"
     ]
    }
   ],
   "source": [
    "model = ReconstructingRegressor(1)\n",
    "model.build(tf.TensorShape([None, 8])) # TODO: check https://github.com/tensorflow/tensorflow/issues/26274\n",
    "model.compile(loss=\"mse\", optimizer=\"nadam\")\n",
    "history = model.fit(X_train_scaled, y_train, epochs=2)\n",
    "y_pred = model.predict(X_test_scaled)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Computing Gradients Using Autodiff"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 150,
   "metadata": {},
   "outputs": [],
   "source": [
    "def f(w1, w2):\n",
    "    return 3 * w1 ** 2 + 2 * w1 * w2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 151,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "36.000003007075065"
      ]
     },
     "execution_count": 151,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "w1, w2 = 5, 3\n",
    "eps = 1e-6\n",
    "(f(w1 + eps, w2) - f(w1, w2)) / eps"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 152,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "10.000000003174137"
      ]
     },
     "execution_count": 152,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(f(w1, w2 + eps) - f(w1, w2)) / eps"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 153,
   "metadata": {},
   "outputs": [],
   "source": [
    "w1, w2 = tf.Variable(5.), tf.Variable(3.)\n",
    "with tf.GradientTape() as tape:\n",
    "    z = f(w1, w2)\n",
    "\n",
    "gradients = tape.gradient(z, [w1, w2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 154,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<tf.Tensor: id=180500, shape=(), dtype=float32, numpy=36.0>,\n",
       " <tf.Tensor: id=180471, shape=(), dtype=float32, numpy=10.0>]"
      ]
     },
     "execution_count": 154,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gradients"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 155,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GradientTape.gradient can only be called once on non-persistent tapes.\n"
     ]
    }
   ],
   "source": [
    "with tf.GradientTape() as tape:\n",
    "    z = f(w1, w2)\n",
    "\n",
    "dz_dw1 = tape.gradient(z, w1)\n",
    "try:\n",
    "    dz_dw2 = tape.gradient(z, w2)\n",
    "except RuntimeError as ex:\n",
    "    print(ex)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 156,
   "metadata": {},
   "outputs": [],
   "source": [
    "with tf.GradientTape(persistent=True) as tape:\n",
    "    z = f(w1, w2)\n",
    "\n",
    "dz_dw1 = tape.gradient(z, w1)\n",
    "dz_dw2 = tape.gradient(z, w2) # works now!\n",
    "del tape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 157,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(<tf.Tensor: id=180588, shape=(), dtype=float32, numpy=36.0>,\n",
       " <tf.Tensor: id=180591, shape=(), dtype=float32, numpy=10.0>)"
      ]
     },
     "execution_count": 157,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dz_dw1, dz_dw2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 158,
   "metadata": {},
   "outputs": [],
   "source": [
    "c1, c2 = tf.constant(5.), tf.constant(3.)\n",
    "with tf.GradientTape() as tape:\n",
    "    z = f(c1, c2)\n",
    "\n",
    "gradients = tape.gradient(z, [c1, c2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 159,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[None, None]"
      ]
     },
     "execution_count": 159,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gradients"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 160,
   "metadata": {},
   "outputs": [],
   "source": [
    "with tf.GradientTape() as tape:\n",
    "    tape.watch(c1)\n",
    "    tape.watch(c2)\n",
    "    z = f(c1, c2)\n",
    "\n",
    "gradients = tape.gradient(z, [c1, c2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 161,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<tf.Tensor: id=180671, shape=(), dtype=float32, numpy=36.0>,\n",
       " <tf.Tensor: id=180642, shape=(), dtype=float32, numpy=10.0>]"
      ]
     },
     "execution_count": 161,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gradients"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 162,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<tf.Tensor: id=180806, shape=(), dtype=float32, numpy=136.0>,\n",
       " <tf.Tensor: id=180807, shape=(), dtype=float32, numpy=30.0>]"
      ]
     },
     "execution_count": 162,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "with tf.GradientTape() as tape:\n",
    "    z1 = f(w1, w2 + 2.)\n",
    "    z2 = f(w1, w2 + 5.)\n",
    "    z3 = f(w1, w2 + 7.)\n",
    "\n",
    "tape.gradient([z1, z2, z3], [w1, w2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 163,
   "metadata": {},
   "outputs": [],
   "source": [
    "with tf.GradientTape(persistent=True) as tape:\n",
    "    z1 = f(w1, w2 + 2.)\n",
    "    z2 = f(w1, w2 + 5.)\n",
    "    z3 = f(w1, w2 + 7.)\n",
    "\n",
    "tf.reduce_sum(tf.stack([tape.gradient(z, [w1, w2]) for z in (z1, z2, z3)]), axis=0)\n",
    "del tape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 164,
   "metadata": {},
   "outputs": [],
   "source": [
    "with tf.GradientTape(persistent=True) as hessian_tape:\n",
    "    with tf.GradientTape() as jacobian_tape:\n",
    "        z = f(w1, w2)\n",
    "    jacobians = jacobian_tape.gradient(z, [w1, w2])\n",
    "hessians = [hessian_tape.gradient(jacobian, [w1, w2])\n",
    "            for jacobian in jacobians]\n",
    "del hessian_tape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 165,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<tf.Tensor: id=180993, shape=(), dtype=float32, numpy=36.0>,\n",
       " <tf.Tensor: id=180964, shape=(), dtype=float32, numpy=10.0>]"
      ]
     },
     "execution_count": 165,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "jacobians"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 166,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[[<tf.Tensor: id=181017, shape=(), dtype=float32, numpy=6.0>,\n",
       "  <tf.Tensor: id=181034, shape=(), dtype=float32, numpy=2.0>],\n",
       " [<tf.Tensor: id=181039, shape=(), dtype=float32, numpy=2.0>, None]]"
      ]
     },
     "execution_count": 166,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hessians"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 167,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<tf.Tensor: id=181073, shape=(), dtype=float32, numpy=30.0>, None]"
      ]
     },
     "execution_count": 167,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def f(w1, w2):\n",
    "    return 3 * w1 ** 2 + tf.stop_gradient(2 * w1 * w2)\n",
    "\n",
    "with tf.GradientTape() as tape:\n",
    "    z = f(w1, w2)\n",
    "\n",
    "tape.gradient(z, [w1, w2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 168,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<tf.Tensor: id=181103, shape=(), dtype=float32, numpy=nan>]"
      ]
     },
     "execution_count": 168,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x = tf.Variable(100.)\n",
    "with tf.GradientTape() as tape:\n",
    "    z = my_softplus(x)\n",
    "\n",
    "tape.gradient(z, [x])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 169,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: id=181109, shape=(), dtype=float32, numpy=30.0>"
      ]
     },
     "execution_count": 169,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf.math.log(tf.exp(tf.constant(30., dtype=tf.float32)) + 1.)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 170,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<tf.Tensor: id=181136, shape=(1,), dtype=float32, numpy=array([nan], dtype=float32)>]"
      ]
     },
     "execution_count": 170,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x = tf.Variable([100.])\n",
    "with tf.GradientTape() as tape:\n",
    "    z = my_softplus(x)\n",
    "\n",
    "tape.gradient(z, [x])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 171,
   "metadata": {},
   "outputs": [],
   "source": [
    "@tf.custom_gradient\n",
    "def my_better_softplus(z):\n",
    "    exp = tf.exp(z)\n",
    "    def my_softplus_gradients(grad):\n",
    "        return grad / (1 + 1 / exp)\n",
    "    return tf.math.log(exp + 1), my_softplus_gradients"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 172,
   "metadata": {},
   "outputs": [],
   "source": [
    "def my_better_softplus(z):\n",
    "    return tf.where(z > 30., z, tf.math.log(tf.exp(z) + 1.))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 173,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(<tf.Tensor: id=181154, shape=(1,), dtype=float32, numpy=array([1000.], dtype=float32)>,\n",
       " [<tf.Tensor: id=181178, shape=(1,), dtype=float32, numpy=array([nan], dtype=float32)>])"
      ]
     },
     "execution_count": 173,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x = tf.Variable([1000.])\n",
    "with tf.GradientTape() as tape:\n",
    "    z = my_better_softplus(x)\n",
    "\n",
    "z, tape.gradient(z, [x])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Computing Gradients Using Autodiff"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 174,
   "metadata": {},
   "outputs": [],
   "source": [
    "l2_reg = keras.regularizers.l2(0.05)\n",
    "model = keras.models.Sequential([\n",
    "    keras.layers.Dense(30, activation=\"elu\", kernel_initializer=\"he_normal\",\n",
    "                       kernel_regularizer=l2_reg),\n",
    "    keras.layers.Dense(1, kernel_regularizer=l2_reg)\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 175,
   "metadata": {},
   "outputs": [],
   "source": [
    "def random_batch(X, y, batch_size=32):\n",
    "    idx = np.random.randint(len(X), size=batch_size)\n",
    "    return X[idx], y[idx]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 176,
   "metadata": {},
   "outputs": [],
   "source": [
    "def print_status_bar(iteration, total, loss, metrics=None):\n",
    "    metrics = \" - \".join([\"{}: {:.4f}\".format(m.name, m.result())\n",
    "                         for m in [loss] + (metrics or [])])\n",
    "    end = \"\" if iteration < total else \"\\n\"\n",
    "    print(\"\\r{}/{} - \".format(iteration, total) + metrics,\n",
    "          end=end)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 177,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "50/50 - loss: 0.0900 - mean_square: 858.5000\n"
     ]
    }
   ],
   "source": [
    "import time\n",
    "\n",
    "mean_loss = keras.metrics.Mean(name=\"loss\")\n",
    "mean_square = keras.metrics.Mean(name=\"mean_square\")\n",
    "for i in range(1, 50 + 1):\n",
    "    loss = 1 / i\n",
    "    mean_loss(loss)\n",
    "    mean_square(i ** 2)\n",
    "    print_status_bar(i, 50, mean_loss, [mean_square])\n",
    "    time.sleep(0.05)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "A fancier version with a progress bar:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 178,
   "metadata": {},
   "outputs": [],
   "source": [
    "def progress_bar(iteration, total, size=30):\n",
    "    running = iteration < total\n",
    "    c = \">\" if running else \"=\"\n",
    "    p = (size - 1) * iteration // total\n",
    "    fmt = \"{{:-{}d}}/{{}} [{{}}]\".format(len(str(total)))\n",
    "    params = [iteration, total, \"=\" * p + c + \".\" * (size - p - 1)]\n",
    "    return fmt.format(*params)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 179,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "' 3500/10000 [=>....]'"
      ]
     },
     "execution_count": 179,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "progress_bar(3500, 10000, size=6)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 180,
   "metadata": {},
   "outputs": [],
   "source": [
    "def print_status_bar(iteration, total, loss, metrics=None, size=30):\n",
    "    metrics = \" - \".join([\"{}: {:.4f}\".format(m.name, m.result())\n",
    "                         for m in [loss] + (metrics or [])])\n",
    "    end = \"\" if iteration < total else \"\\n\"\n",
    "    print(\"\\r{} - {}\".format(progress_bar(iteration, total), metrics), end=end)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 181,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "50/50 [==============================] - loss: 0.0900 - mean_square: 858.5000\n"
     ]
    }
   ],
   "source": [
    "mean_loss = keras.metrics.Mean(name=\"loss\")\n",
    "mean_square = keras.metrics.Mean(name=\"mean_square\")\n",
    "for i in range(1, 50 + 1):\n",
    "    loss = 1 / i\n",
    "    mean_loss(loss)\n",
    "    mean_square(i ** 2)\n",
    "    print_status_bar(i, 50, mean_loss, [mean_square])\n",
    "    time.sleep(0.05)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 182,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_epochs = 5\n",
    "batch_size = 32\n",
    "n_steps = len(X_train) // batch_size\n",
    "optimizer = keras.optimizers.Nadam(lr=0.01)\n",
    "loss_fn = keras.losses.mean_squared_error\n",
    "mean_loss = keras.metrics.Mean()\n",
    "metrics = [keras.metrics.MeanAbsoluteError()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 183,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/5\n",
      "11610/11610 [==============================] - mean: 1.5850 - mean_absolute_error: 0.57446\n",
      "Epoch 2/5\n",
      "11610/11610 [==============================] - mean: 0.7521 - mean_absolute_error: 0.5266\n",
      "Epoch 3/5\n",
      "11610/11610 [==============================] - mean: 0.7095 - mean_absolute_error: 0.5347\n",
      "Epoch 4/5\n",
      "11610/11610 [==============================] - mean: 0.6629 - mean_absolute_error: 0.5236\n",
      "Epoch 5/5\n",
      "11610/11610 [==============================] - mean: 0.6421 - mean_absolute_error: 0.5150\n"
     ]
    }
   ],
   "source": [
    "for epoch in range(1, n_epochs + 1):\n",
    "    print(\"Epoch {}/{}\".format(epoch, n_epochs))\n",
    "    for step in range(1, n_steps + 1):\n",
    "        X_batch, y_batch = random_batch(X_train_scaled, y_train)\n",
    "        with tf.GradientTape() as tape:\n",
    "            y_pred = model(X_batch)\n",
    "            main_loss = tf.reduce_mean(loss_fn(y_batch, y_pred))\n",
    "            loss = tf.add_n([main_loss] + model.losses)\n",
    "        gradients = tape.gradient(loss, model.trainable_variables)\n",
    "        optimizer.apply_gradients(zip(gradients, model.trainable_variables))\n",
    "        for variable in model.variables:\n",
    "            if variable.constraint is not None:\n",
    "                variable.assign(variable.constraint(variable))\n",
    "        mean_loss(loss)\n",
    "        for metric in metrics:\n",
    "            metric(y_batch, y_pred)\n",
    "        print_status_bar(step * batch_size, len(y_train), mean_loss, metrics)\n",
    "    print_status_bar(len(y_train), len(y_train), mean_loss, metrics)\n",
    "    for metric in [mean_loss] + metrics:\n",
    "        metric.reset_states()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 184,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c9a587e82fec425197076e93cf608b1a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='All epochs', max=5, style=ProgressStyle(description_width='in…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5e9e39fb31e7403184b45fa2645b2743",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch 1/5', max=362, style=ProgressStyle(description_width='i…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "05d9c331e500444391f22b7ec9b85241",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch 2/5', max=362, style=ProgressStyle(description_width='i…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "178d9338311c427cb553371bc0e98df7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch 3/5', max=362, style=ProgressStyle(description_width='i…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8b3c03c2904e4e8ab0117595c77773c1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch 4/5', max=362, style=ProgressStyle(description_width='i…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1bbee5ec0e904aff8245d58fb174fed8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch 5/5', max=362, style=ProgressStyle(description_width='i…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "try:\n",
    "    from tqdm import tnrange\n",
    "    from collections import OrderedDict\n",
    "    with tnrange(1, n_epochs + 1, desc=\"All epochs\") as epochs:\n",
    "        for epoch in epochs:\n",
    "            with tnrange(1, n_steps + 1, desc=\"Epoch {}/{}\".format(epoch, n_epochs)) as steps:\n",
    "                for step in steps:\n",
    "                    X_batch, y_batch = random_batch(X_train_scaled, y_train)\n",
    "                    with tf.GradientTape() as tape:\n",
    "                        y_pred = model(X_batch)\n",
    "                        main_loss = tf.reduce_mean(loss_fn(y_batch, y_pred))\n",
    "                        loss = tf.add_n([main_loss] + model.losses)\n",
    "                    gradients = tape.gradient(loss, model.trainable_variables)\n",
    "                    optimizer.apply_gradients(zip(gradients, model.trainable_variables))\n",
    "                    for variable in model.variables:\n",
    "                        if variable.constraint is not None:\n",
    "                            variable.assign(variable.constraint(variable))                    \n",
    "                    status = OrderedDict()\n",
    "                    mean_loss(loss)\n",
    "                    status[\"loss\"] = mean_loss.result().numpy()\n",
    "                    for metric in metrics:\n",
    "                        metric(y_batch, y_pred)\n",
    "                        status[metric.name] = metric.result().numpy()\n",
    "                    steps.set_postfix(status)\n",
    "            for metric in [mean_loss] + metrics:\n",
    "                metric.reset_states()\n",
    "except ImportError as ex:\n",
    "    print(\"To run this cell, please install tqdm, ipywidgets and restart Jupyter\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## TensorFlow Functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 185,
   "metadata": {},
   "outputs": [],
   "source": [
    "def cube(x):\n",
    "    return x ** 3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 186,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "8"
      ]
     },
     "execution_count": 186,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cube(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 187,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: id=1508933, shape=(), dtype=float32, numpy=8.0>"
      ]
     },
     "execution_count": 187,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cube(tf.constant(2.0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 188,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.eager.def_function.Function at 0x1421df198>"
      ]
     },
     "execution_count": 188,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf_cube = tf.function(cube)\n",
    "tf_cube"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 189,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: id=1508940, shape=(), dtype=int32, numpy=8>"
      ]
     },
     "execution_count": 189,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf_cube(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 190,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: id=1508950, shape=(), dtype=float32, numpy=8.0>"
      ]
     },
     "execution_count": 190,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf_cube(tf.constant(2.0))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### TF Functions and Concrete Functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 191,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.framework.func_graph.FuncGraph at 0x14226dba8>"
      ]
     },
     "execution_count": 191,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "concrete_function = tf_cube.get_concrete_function(tf.constant(2.0))\n",
    "concrete_function.graph"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 192,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: id=1508955, shape=(), dtype=float32, numpy=8.0>"
      ]
     },
     "execution_count": 192,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "concrete_function(tf.constant(2.0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 193,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 193,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "concrete_function is tf_cube.get_concrete_function(tf.constant(2.0))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Exploring Function Definitions and Graphs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 194,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.framework.func_graph.FuncGraph at 0x14226dba8>"
      ]
     },
     "execution_count": 194,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "concrete_function.graph"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 195,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<tf.Operation 'x' type=Placeholder>,\n",
       " <tf.Operation 'pow/y' type=Const>,\n",
       " <tf.Operation 'pow' type=Pow>,\n",
       " <tf.Operation 'Identity' type=Identity>]"
      ]
     },
     "execution_count": 195,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ops = concrete_function.graph.get_operations()\n",
    "ops"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 196,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<tf.Tensor 'x:0' shape=() dtype=float32>,\n",
       " <tf.Tensor 'pow/y:0' shape=() dtype=float32>]"
      ]
     },
     "execution_count": 196,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pow_op = ops[2]\n",
    "list(pow_op.inputs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 197,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<tf.Tensor 'pow:0' shape=() dtype=float32>]"
      ]
     },
     "execution_count": 197,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pow_op.outputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 198,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Operation 'x' type=Placeholder>"
      ]
     },
     "execution_count": 198,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "concrete_function.graph.get_operation_by_name('x')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 199,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor 'Identity:0' shape=() dtype=float32>"
      ]
     },
     "execution_count": 199,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "concrete_function.graph.get_tensor_by_name('Identity:0')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 200,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "name: \"__inference_cube_1508949\"\n",
       "input_arg {\n",
       "  name: \"x\"\n",
       "  type: DT_FLOAT\n",
       "}\n",
       "output_arg {\n",
       "  name: \"identity\"\n",
       "  type: DT_FLOAT\n",
       "}"
      ]
     },
     "execution_count": 200,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "concrete_function.function_def.signature"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### How TF Functions Trace Python Functions to Extract Their Computation Graphs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 201,
   "metadata": {},
   "outputs": [],
   "source": [
    "@tf.function\n",
    "def tf_cube(x):\n",
    "    print(\"print:\", x)\n",
    "    return x ** 3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 202,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "print: Tensor(\"x:0\", shape=(), dtype=float32)\n"
     ]
    }
   ],
   "source": [
    "result = tf_cube(tf.constant(2.0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 203,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: id=1508968, shape=(), dtype=float32, numpy=8.0>"
      ]
     },
     "execution_count": 203,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 204,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "print: 2\n",
      "print: 3\n",
      "print: Tensor(\"x:0\", shape=(1, 2), dtype=float32)\n",
      "print: Tensor(\"x:0\", shape=(None, 2), dtype=float32)\n"
     ]
    }
   ],
   "source": [
    "result = tf_cube(2)\n",
    "result = tf_cube(3)\n",
    "result = tf_cube(tf.constant([[1., 2.]])) # New shape: trace!\n",
    "result = tf_cube(tf.constant([[3., 4.], [5., 6.]])) # New shape: trace!\n",
    "result = tf_cube(tf.constant([[7., 8.], [9., 10.], [11., 12.]])) # no trace"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "It is also possible to specify a particular input signature:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 205,
   "metadata": {},
   "outputs": [],
   "source": [
    "@tf.function(input_signature=[tf.TensorSpec([None, 28, 28], tf.float32)])\n",
    "def shrink(images):\n",
    "    print(\"Tracing\", images)\n",
    "    return images[:, ::2, ::2] # drop half the rows and columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 206,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tracing Tensor(\"images:0\", shape=(None, 28, 28), dtype=float32)\n"
     ]
    }
   ],
   "source": [
    "img_batch_1 = tf.random.uniform(shape=[100, 28, 28])\n",
    "img_batch_2 = tf.random.uniform(shape=[50, 28, 28])\n",
    "preprocessed_images = shrink(img_batch_1) # Traces the function.\n",
    "preprocessed_images = shrink(img_batch_2) # Reuses the same concrete function."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 207,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Python inputs incompatible with input_signature: inputs ((<tf.Tensor: id=1509032, shape=(2, 2, 2), dtype=float32, numpy=\n",
      "array([[[0.888957  , 0.60685956],\n",
      "        [0.01694834, 0.01444435]],\n",
      "\n",
      "       [[0.7288147 , 0.3239665 ],\n",
      "        [0.45528448, 0.43155968]]], dtype=float32)>,)), input_signature ((TensorSpec(shape=(None, 28, 28), dtype=tf.float32, name=None),))\n"
     ]
    }
   ],
   "source": [
    "img_batch_3 = tf.random.uniform(shape=[2, 2, 2])\n",
    "try:\n",
    "    preprocessed_images = shrink(img_batch_3)  # rejects unexpected types or shapes\n",
    "except ValueError as ex:\n",
    "    print(ex)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Using Autograph To Capture Control Flow"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "A \"static\" `for` loop using `range()`:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 208,
   "metadata": {},
   "outputs": [],
   "source": [
    "@tf.function\n",
    "def add_10(x):\n",
    "    for i in range(10):\n",
    "        x += 1\n",
    "    return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 209,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: id=1509061, shape=(), dtype=int32, numpy=15>"
      ]
     },
     "execution_count": 209,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "add_10(tf.constant(5))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 210,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<tf.Operation 'x' type=Placeholder>,\n",
       " <tf.Operation 'add/y' type=Const>,\n",
       " <tf.Operation 'add' type=Add>,\n",
       " <tf.Operation 'add_1/y' type=Const>,\n",
       " <tf.Operation 'add_1' type=Add>,\n",
       " <tf.Operation 'add_2/y' type=Const>,\n",
       " <tf.Operation 'add_2' type=Add>,\n",
       " <tf.Operation 'add_3/y' type=Const>,\n",
       " <tf.Operation 'add_3' type=Add>,\n",
       " <tf.Operation 'add_4/y' type=Const>,\n",
       " <tf.Operation 'add_4' type=Add>,\n",
       " <tf.Operation 'add_5/y' type=Const>,\n",
       " <tf.Operation 'add_5' type=Add>,\n",
       " <tf.Operation 'add_6/y' type=Const>,\n",
       " <tf.Operation 'add_6' type=Add>,\n",
       " <tf.Operation 'add_7/y' type=Const>,\n",
       " <tf.Operation 'add_7' type=Add>,\n",
       " <tf.Operation 'add_8/y' type=Const>,\n",
       " <tf.Operation 'add_8' type=Add>,\n",
       " <tf.Operation 'add_9/y' type=Const>,\n",
       " <tf.Operation 'add_9' type=Add>,\n",
       " <tf.Operation 'Identity' type=Identity>]"
      ]
     },
     "execution_count": 210,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "add_10.get_concrete_function(tf.constant(5)).graph.get_operations()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "A \"dynamic\" loop using `tf.while_loop()`:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 211,
   "metadata": {},
   "outputs": [],
   "source": [
    "@tf.function\n",
    "def add_10(x):\n",
    "    condition = lambda i, x: tf.less(i, 10)\n",
    "    body = lambda i, x: (tf.add(i, 1), tf.add(x, 1))\n",
    "    final_i, final_x = tf.while_loop(condition, body, [tf.constant(0), x])\n",
    "    return final_x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 212,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: id=1509107, shape=(), dtype=int32, numpy=15>"
      ]
     },
     "execution_count": 212,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "add_10(tf.constant(5))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 213,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<tf.Operation 'x' type=Placeholder>,\n",
       " <tf.Operation 'Const' type=Const>,\n",
       " <tf.Operation 'while/maximum_iterations' type=Const>,\n",
       " <tf.Operation 'while/loop_counter' type=Const>,\n",
       " <tf.Operation 'while' type=While>,\n",
       " <tf.Operation 'while/Identity' type=Identity>,\n",
       " <tf.Operation 'while/Identity_1' type=Identity>,\n",
       " <tf.Operation 'while/Identity_2' type=Identity>,\n",
       " <tf.Operation 'while/Identity_3' type=Identity>,\n",
       " <tf.Operation 'Identity' type=Identity>]"
      ]
     },
     "execution_count": 213,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "add_10.get_concrete_function(tf.constant(5)).graph.get_operations()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "A \"dynamic\" `for` loop using `tf.range()` (captured by autograph):"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 214,
   "metadata": {},
   "outputs": [],
   "source": [
    "@tf.function\n",
    "def add_10(x):\n",
    "    for i in tf.range(10):\n",
    "        x = x + 1\n",
    "    return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 215,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<tf.Operation 'x' type=Placeholder>,\n",
       " <tf.Operation 'range/start' type=Const>,\n",
       " <tf.Operation 'range/limit' type=Const>,\n",
       " <tf.Operation 'range/delta' type=Const>,\n",
       " <tf.Operation 'range' type=Range>,\n",
       " <tf.Operation 'Const' type=Const>,\n",
       " <tf.Operation 'while/maximum_iterations' type=Const>,\n",
       " <tf.Operation 'while/loop_counter' type=Const>,\n",
       " <tf.Operation 'while' type=While>,\n",
       " <tf.Operation 'while/Identity' type=Identity>,\n",
       " <tf.Operation 'while/Identity_1' type=Identity>,\n",
       " <tf.Operation 'while/Identity_2' type=Identity>,\n",
       " <tf.Operation 'while/Identity_3' type=Identity>,\n",
       " <tf.Operation 'while/Identity_4' type=Identity>,\n",
       " <tf.Operation 'Identity' type=Identity>]"
      ]
     },
     "execution_count": 215,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "add_10.get_concrete_function(tf.constant(0)).graph.get_operations()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Handling Variables and Other Resources in TF Functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 216,
   "metadata": {},
   "outputs": [],
   "source": [
    "counter = tf.Variable(0)\n",
    "\n",
    "@tf.function\n",
    "def increment(counter, c=1):\n",
    "    return counter.assign_add(c)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 217,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: id=1509189, shape=(), dtype=int32, numpy=2>"
      ]
     },
     "execution_count": 217,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "increment(counter)\n",
    "increment(counter)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 218,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "name: \"counter\"\n",
       "type: DT_RESOURCE"
      ]
     },
     "execution_count": 218,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "function_def = increment.get_concrete_function(counter).function_def\n",
    "function_def.signature.input_arg[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 219,
   "metadata": {},
   "outputs": [],
   "source": [
    "counter = tf.Variable(0)\n",
    "\n",
    "@tf.function\n",
    "def increment(c=1):\n",
    "    return counter.assign_add(c)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 220,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: id=1509207, shape=(), dtype=int32, numpy=2>"
      ]
     },
     "execution_count": 220,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "increment()\n",
    "increment()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 221,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "name: \"assignaddvariableop_resource\"\n",
       "type: DT_RESOURCE"
      ]
     },
     "execution_count": 221,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "function_def = increment.get_concrete_function().function_def\n",
    "function_def.signature.input_arg[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 222,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Counter:\n",
    "    def __init__(self):\n",
    "        self.counter = tf.Variable(0)\n",
    "\n",
    "    @tf.function\n",
    "    def increment(self, c=1):\n",
    "        return self.counter.assign_add(c)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 223,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: id=1509225, shape=(), dtype=int32, numpy=2>"
      ]
     },
     "execution_count": 223,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "c = Counter()\n",
    "c.increment()\n",
    "c.increment()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 224,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"from __future__ import print_function\\n\\ndef tf__add_10(x):\\n  do_return = False\\n  retval_ = None\\n\\n  def loop_body(loop_vars, x_1):\\n    i = loop_vars\\n    x_1 += 1\\n    return x_1,\\n  x, = ag__.for_stmt(ag__.converted_call('range', tf, ag__.ConversionOptions(recursive=True, verbose=0, strip_decorators=(ag__.convert, ag__.do_not_convert, ag__.converted_call), force_conversion=False, optional_features=(), internal_convert_user_code=True), (10,), {}), None, loop_body, (x,))\\n  do_return = True\\n  retval_ = x\\n  return retval_\\n\\n\\n\\ntf__add_10.autograph_info__ = {}\\n\""
      ]
     },
     "execution_count": 224,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "@tf.function\n",
    "def add_10(x):\n",
    "    for i in tf.range(10):\n",
    "        x += 1\n",
    "    return x\n",
    "\n",
    "tf.autograph.to_code(add_10.python_function, experimental_optional_features=None)\n",
    "# TODO: experimental_optional_features is needed to have the same behavior as @tf.function,\n",
    "#       check that this is not needed when TF2 is released"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 225,
   "metadata": {},
   "outputs": [],
   "source": [
    "def display_tf_code(func, experimental_optional_features=None):\n",
    "    from IPython.display import display, Markdown\n",
    "    if hasattr(func, \"python_function\"):\n",
    "        func = func.python_function\n",
    "    code = tf.autograph.to_code(func, experimental_optional_features=experimental_optional_features)\n",
    "    display(Markdown('```python\\n{}\\n```'.format(code)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 226,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/markdown": [
       "```python\n",
       "from __future__ import print_function\n",
       "\n",
       "def tf__add_10(x):\n",
       "  do_return = False\n",
       "  retval_ = None\n",
       "\n",
       "  def loop_body(loop_vars, x_1):\n",
       "    i = loop_vars\n",
       "    x_1 += 1\n",
       "    return x_1,\n",
       "  x, = ag__.for_stmt(ag__.converted_call('range', tf, ag__.ConversionOptions(recursive=True, verbose=0, strip_decorators=(ag__.convert, ag__.do_not_convert, ag__.converted_call), force_conversion=False, optional_features=(), internal_convert_user_code=True), (10,), {}), None, loop_body, (x,))\n",
       "  do_return = True\n",
       "  retval_ = x\n",
       "  return retval_\n",
       "\n",
       "\n",
       "\n",
       "tf__add_10.autograph_info__ = {}\n",
       "\n",
       "```"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "display_tf_code(add_10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Using TF Functions with tf.keras (or Not)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "By default, tf.keras will automatically convert your custom code into TF Functions, no need to use\n",
    "`tf.function()`:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 227,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Custom loss function\n",
    "def my_mse(y_true, y_pred):\n",
    "    print(\"Tracing loss my_mse()\")\n",
    "    return tf.reduce_mean(tf.square(y_pred - y_true))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 228,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Custom metric function\n",
    "def my_mae(y_true, y_pred):\n",
    "    print(\"Tracing metric my_mae()\")\n",
    "    return tf.reduce_mean(tf.abs(y_pred - y_true))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 229,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Custom layer\n",
    "class MyDense(keras.layers.Layer):\n",
    "    def __init__(self, units, activation=None, **kwargs):\n",
    "        super().__init__(**kwargs)\n",
    "        self.units = units\n",
    "        self.activation = keras.activations.get(activation)\n",
    "\n",
    "    def build(self, input_shape):\n",
    "        self.kernel = self.add_weight(name='kernel', \n",
    "                                      shape=(input_shape[1], self.units),\n",
    "                                      initializer='uniform',\n",
    "                                      trainable=True)\n",
    "        self.biases = self.add_weight(name='bias', \n",
    "                                      shape=(self.units,),\n",
    "                                      initializer='zeros',\n",
    "                                      trainable=True)\n",
    "        super().build(input_shape)\n",
    "\n",
    "    def call(self, X):\n",
    "        print(\"Tracing MyDense.call()\")\n",
    "        return self.activation(X @ self.kernel + self.biases)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 230,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Custom model\n",
    "class MyModel(keras.models.Model):\n",
    "    def __init__(self, **kwargs):\n",
    "        super().__init__(**kwargs)\n",
    "        self.hidden1 = MyDense(30, activation=\"relu\")\n",
    "        self.hidden2 = MyDense(30, activation=\"relu\")\n",
    "        self.output_ = MyDense(1)\n",
    "\n",
    "    def call(self, input):\n",
    "        print(\"Tracing MyModel.call()\")\n",
    "        hidden1 = self.hidden1(input)\n",
    "        hidden2 = self.hidden2(hidden1)\n",
    "        concat = keras.layers.concatenate([input, hidden2])\n",
    "        output = self.output_(concat)\n",
    "        return output\n",
    "\n",
    "model = MyModel()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 231,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.compile(loss=my_mse, optimizer=\"nadam\", metrics=[my_mae])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 232,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tracing MyModel.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing metric my_mae()\n",
      "Tracing metric my_mae()\n",
      "Tracing loss my_mse()\n",
      "Train on 11610 samples, validate on 3870 samples\n",
      "Epoch 1/2\n",
      "11610/11610 [==============================] - 1s 77us/sample - loss: 1.4017 - my_mae: 0.8108 - val_loss: 0.4857 - val_my_mae: 0.4775\n",
      "Epoch 2/2\n",
      "11610/11610 [==============================] - 1s 50us/sample - loss: 0.4487 - my_mae: 0.4756 - val_loss: 1.0410 - val_my_mae: 0.4636\n",
      "5160/5160 [==============================] - 0s 15us/sample - loss: 0.4326 - my_mae: 0.4597\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[0.43256516364193703, 0.45965773]"
      ]
     },
     "execution_count": 232,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit(X_train_scaled, y_train, epochs=2,\n",
    "          validation_data=(X_valid_scaled, y_valid))\n",
    "model.evaluate(X_test_scaled, y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "You can turn this off by creating the model with `dynamic=True` (or calling `super().__init__(dynamic=True, **kwargs)` in the model's constructor):"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 233,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = MyModel(dynamic=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 234,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.compile(loss=my_mse, optimizer=\"nadam\", metrics=[my_mae])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Not the custom code will be called at each iteration. Let's fit, validate and evaluate with tiny datasets to avoid getting too much output:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 235,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tracing MyModel.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing loss my_mse()\n",
      "Tracing metric my_mae()\n",
      "Tracing MyModel.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing loss my_mse()\n",
      "Tracing metric my_mae()\n",
      "Tracing MyModel.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing loss my_mse()\n",
      "Tracing metric my_mae()\n",
      "Tracing MyModel.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing loss my_mse()\n",
      "Tracing metric my_mae()\n",
      "Tracing MyModel.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing loss my_mse()\n",
      "Tracing metric my_mae()\n",
      "Tracing MyModel.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing loss my_mse()\n",
      "Tracing metric my_mae()\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[5.544255495071411, 2.0656278]"
      ]
     },
     "execution_count": 235,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit(X_train_scaled[:64], y_train[:64], epochs=1,\n",
    "          validation_data=(X_valid_scaled[:64], y_valid[:64]), verbose=0)\n",
    "model.evaluate(X_test_scaled[:64], y_test[:64], verbose=0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Alternatively, you can compile a model with `run_eagerly=True`:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 236,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = MyModel()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 237,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.compile(loss=my_mse, optimizer=\"nadam\", metrics=[my_mae], run_eagerly=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 238,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tracing MyModel.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing MyModel.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing loss my_mse()\n",
      "Tracing metric my_mae()\n",
      "Tracing MyModel.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing loss my_mse()\n",
      "Tracing metric my_mae()\n",
      "Tracing MyModel.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing loss my_mse()\n",
      "Tracing metric my_mae()\n",
      "Tracing MyModel.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing loss my_mse()\n",
      "Tracing metric my_mae()\n",
      "Tracing MyModel.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing loss my_mse()\n",
      "Tracing metric my_mae()\n",
      "Tracing MyModel.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing loss my_mse()\n",
      "Tracing metric my_mae()\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[5.633018493652344, 2.0630658]"
      ]
     },
     "execution_count": 238,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit(X_train_scaled[:64], y_train[:64], epochs=1,\n",
    "          validation_data=(X_valid_scaled[:64], y_valid[:64]), verbose=0)\n",
    "model.evaluate(X_test_scaled[:64], y_test[:64], verbose=0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Custom Optimizers"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Defining custom optimizers is not very common, but in case you are one of the happy few who gets to write one, here is an example:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 239,
   "metadata": {},
   "outputs": [],
   "source": [
    "class MyMomentumOptimizer(keras.optimizers.Optimizer):\n",
    "    def __init__(self, learning_rate=0.001, momentum=0.9, name=\"MyMomentumOptimizer\", **kwargs):\n",
    "        \"\"\"Call super().__init__() and use _set_hyper() to store hyperparameters\"\"\"\n",
    "        super().__init__(name, **kwargs)\n",
    "        self._set_hyper(\"learning_rate\", kwargs.get(\"lr\", learning_rate)) # handle lr=learning_rate\n",
    "        self._set_hyper(\"decay\", self._initial_decay) # \n",
    "        self._set_hyper(\"momentum\", momentum)\n",
    "    \n",
    "    def _create_slots(self, var_list):\n",
    "        \"\"\"For each model variable, create the optimizer variable associated with it.\n",
    "        TensorFlow calls these optimizer variables \"slots\".\n",
    "        For momentum optimization, we need one momentum slot per model variable.\n",
    "        \"\"\"\n",
    "        for var in var_list:\n",
    "            self.add_slot(var, \"momentum\")\n",
    "\n",
    "    @tf.function\n",
    "    def _resource_apply_dense(self, grad, var):\n",
    "        \"\"\"Update the slots and perform one optimization step for one model variable\n",
    "        \"\"\"\n",
    "        var_dtype = var.dtype.base_dtype\n",
    "        lr_t = self._decayed_lr(var_dtype) # handle learning rate decay\n",
    "        momentum_var = self.get_slot(var, \"momentum\")\n",
    "        momentum_hyper = self._get_hyper(\"momentum\", var_dtype)\n",
    "        momentum_var.assign(momentum_var * momentum_hyper - (1. - momentum_hyper)* grad)\n",
    "        var.assign_add(momentum_var * lr_t)\n",
    "\n",
    "    def _resource_apply_sparse(self, grad, var):\n",
    "        raise NotImplementedError\n",
    "\n",
    "    def get_config(self):\n",
    "        base_config = super().get_config()\n",
    "        return {\n",
    "            **base_config,\n",
    "            \"learning_rate\": self._serialize_hyperparameter(\"learning_rate\"),\n",
    "            \"decay\": self._serialize_hyperparameter(\"decay\"),\n",
    "            \"momentum\": self._serialize_hyperparameter(\"momentum\"),\n",
    "        }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 240,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/5\n",
      "11610/11610 [==============================] - 0s 33us/sample - loss: 4.6747\n",
      "Epoch 2/5\n",
      "11610/11610 [==============================] - 0s 26us/sample - loss: 1.4888\n",
      "Epoch 3/5\n",
      "11610/11610 [==============================] - 0s 25us/sample - loss: 0.8565\n",
      "Epoch 4/5\n",
      "11610/11610 [==============================] - 0s 25us/sample - loss: 0.7086\n",
      "Epoch 5/5\n",
      "11610/11610 [==============================] - 0s 26us/sample - loss: 0.6652\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.callbacks.History at 0x145154e48>"
      ]
     },
     "execution_count": 240,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model = keras.models.Sequential([keras.layers.Dense(1, input_shape=[8])])\n",
    "model.compile(loss=\"mse\", optimizer=MyMomentumOptimizer())\n",
    "model.fit(X_train_scaled, y_train, epochs=5)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}