train.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "id": "CgQ1RcxD0Rc-"
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import pandas as pd\n",
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "import tqdm\n",
    "from joblib import Parallel, delayed\n",
    "import string\n",
    "import pickle\n",
    "import random"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'2.6.0'"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf.__version__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "id": "K4TaetdodqOV"
   },
   "outputs": [],
   "source": [
    "random.seed(25)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "id": "SzQcuklDXfJD"
   },
   "outputs": [],
   "source": [
    "AUTOTUNE = tf.data.AUTOTUNE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "id": "BgI8GViL0qfh"
   },
   "outputs": [],
   "source": [
    "def load_keep_probs(keep_prob_path):\n",
    "    #sorted from most freq to least\n",
    "    with open(keep_prob_path, 'rb') as f:\n",
    "        keep_probs = pickle.load(f)\n",
    "    keep_probs = dict(sorted(keep_probs.items(), key=lambda item: item[1]))\n",
    "    return keep_probs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Num GPUs Available:  0\n"
     ]
    }
   ],
   "source": [
    "print(\"Num GPUs Available: \", len(tf.config.list_physical_devices('GPU')))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "id": "OU_0wTC-0AqA"
   },
   "outputs": [],
   "source": [
    "def get_gen(lines_path, keep_prob_path, char_set, max_word_len, window_size=2, neg_sample_size = 5):\n",
    "\n",
    "    with open(lines_path, 'rb') as f:\n",
    "        lines = pickle.load(f)\n",
    "    keep_probs = load_keep_probs(keep_prob_path)\n",
    "    words = list(keep_probs.keys())\n",
    "    word_count = len(words)\n",
    "    \n",
    "    random.shuffle(lines)\n",
    "    lines = lines[:int(len(lines) * 0.50)]\n",
    "    print(\"Number of lines {}\".format(len(lines)))\n",
    "\n",
    "    def to_idx(word, char_set=char_set, max_word_len=max_word_len):\n",
    "        if(len(word) < max_word_len):\n",
    "            res = [char_set.index(c) if c in char_set else -1 for c in word]\n",
    "            res += [-1 for _ in range(max_word_len-len(word))]\n",
    "        else:\n",
    "            res = [char_set.index(c) if c in char_set else -1 for c in word[:max_word_len]]\n",
    "        return res\n",
    "        #return tf.one_hot(res,len(char_set)).numpy()\n",
    "  \n",
    "    def gen():\n",
    "      counter = 0\n",
    "      targets = []\n",
    "      contexts = []\n",
    "      labels = [] \n",
    "      for line in lines:\n",
    "          \n",
    "          line = line.split(\" \")\n",
    "          for i,word in enumerate(line):\n",
    "              word = word.rstrip()\n",
    "              for j in range(i - window_size, i + window_size+1):\n",
    "                  if j==i or j<0 or j>=len(line):\n",
    "                      continue\n",
    "                \n",
    "                  flag=True\n",
    "                  try:\n",
    "                      prob = keep_probs[word]\n",
    "                  except:\n",
    "                      continue\n",
    "                  flag = np.random.rand() < prob\n",
    "                  if flag == False:\n",
    "                      continue\n",
    "                \n",
    "                  targets.append(to_idx(word))\n",
    "                  contexts.append(to_idx(line[j]))\n",
    "                  labels.append(1)\n",
    "\n",
    "              for _ in range(neg_sample_size):\n",
    "                  if flag == False:\n",
    "                      continue\n",
    "                  idx = int(abs(np.random.rand() - 0.25) * word_count)\n",
    "                  targets.append(to_idx(word))\n",
    "                  contexts.append(to_idx(words[idx]))\n",
    "                  labels.append(0)\n",
    "\n",
    "          counter += 1\n",
    "          if(counter % 200 == 0):\n",
    "            counter=0\n",
    "            t, c, l = np.array(targets, dtype=np.int32), np.array(contexts, dtype=np.int32), np.expand_dims(np.array(labels, dtype=np.int32), axis=-1)\n",
    "            \n",
    "            targets = []\n",
    "            contexts = []\n",
    "            labels = []\n",
    "            yield (t, c), l \n",
    "          else:\n",
    "            continue\n",
    "        \n",
    "    generator = tf.data.Dataset.from_generator(gen,\n",
    "                                               output_signature=(\n",
    "                                                                (tf.TensorSpec(shape=(None,max_word_len), dtype=tf.int32),\n",
    "                                                                 tf.TensorSpec(shape=(None,max_word_len), dtype=tf.int32)),\n",
    "                                                                 tf.TensorSpec(shape=(None,1), dtype=tf.int32),\n",
    "                                                                  ))\n",
    "    generator = generator.prefetch(AUTOTUNE)\n",
    "    return generator\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "line_path = \"data/lines.pickle\"\n",
    "keep_prob_path = \"data/keep_probs.pickle\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "VWqXyN--0GVo",
    "outputId": "0f4f9bd3-fb9f-4da2-994b-fddacb5ed85d"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of lines 4575670\n"
     ]
    }
   ],
   "source": [
    "data_gen = get_gen(line_path, keep_prob_path, char_set=string.printable+\"üÜiİöÖğĞşŞçÇ\",max_word_len=15, window_size=1, neg_sample_size = 5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "id": "TOSRz1LpuKOU"
   },
   "outputs": [],
   "source": [
    "#RUN UNTIL HERE and GET num lines"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "id": "PYdt7PVjfjSs"
   },
   "outputs": [],
   "source": [
    "char_set=string.printable+\"üÜiİöÖğĞşŞçÇ\"\n",
    "char_size = len(char_set)\n",
    "num_lines = 4575670"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "id": "W7ITE1nRz8xI"
   },
   "outputs": [],
   "source": [
    "class Submodel(tf.keras.Model):\n",
    "  def __init__(self, input_shape, **kwargs):\n",
    "    super(Submodel, self).__init__(**kwargs)\n",
    "    self.LSTM = tf.keras.layers.LSTM(100,input_shape=input_shape)\n",
    "  \n",
    "  def call(self,inputs):\n",
    "    return self.LSTM(inputs)    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "id": "I7QgkAa-6yHN"
   },
   "outputs": [],
   "source": [
    "def get_submodel():\n",
    "  input = tf.keras.layers.Input(shape=(15,len(char_set)),dtype=tf.float32)\n",
    "  output = tf.keras.layers.LSTM(100)(input)\n",
    "  model =  tf.keras.Model(inputs=input, outputs=output)\n",
    "  return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "id": "e62GwWpClO5e"
   },
   "outputs": [],
   "source": [
    "target_net = get_submodel()\n",
    "context_net = get_submodel()\n",
    "\n",
    "def create_model():\n",
    "  target_input = tf.keras.layers.Input(shape=(15,), name=\"target\",dtype=tf.int32)\n",
    "  context_input = tf.keras.layers.Input(shape=(15,), name=\"context\",dtype=tf.int32)\n",
    "\n",
    "  target = tf.one_hot(target_input,char_size)\n",
    "  context = tf.one_hot(context_input,char_size)\n",
    "\n",
    "  target = target_net(target)\n",
    "  context = context_net(context)\n",
    "\n",
    "  output = tf.keras.layers.Dot(axes=-1)([target,context])\n",
    "  model =  tf.keras.Model(inputs=[target_input, context_input], outputs=output)\n",
    "  model.compile(optimizer='adam',\n",
    "                 loss=tf.keras.losses.BinaryCrossentropy(from_logits=True),\n",
    "                 metrics=['accuracy'])\n",
    "  \n",
    "  return model\n",
    "\n",
    "model = create_model()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "49eFizjgonfC",
    "outputId": "7469741f-57b2-437f-bc20-11723e00fc36"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"model_2\"\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "target (InputLayer)             [(None, 15)]         0                                            \n",
      "__________________________________________________________________________________________________\n",
      "context (InputLayer)            [(None, 15)]         0                                            \n",
      "__________________________________________________________________________________________________\n",
      "tf.one_hot (TFOpLambda)         (None, 15, 112)      0           target[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "tf.one_hot_1 (TFOpLambda)       (None, 15, 112)      0           context[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "model (Functional)              (None, 100)          85200       tf.one_hot[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "model_1 (Functional)            (None, 100)          85200       tf.one_hot_1[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "dot (Dot)                       (None, 1)            0           model[0][0]                      \n",
      "                                                                 model_1[0][0]                    \n",
      "==================================================================================================\n",
      "Total params: 170,400\n",
      "Trainable params: 170,400\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 320
    },
    "id": "yLlxi9ACin_I",
    "outputId": "2706ba89-9677-4aaf-997e-cb69044817ae"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "    3/22878 [..............................] - ETA: 19:51:27 - loss: 0.6927 - accuracy: 0.7221"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-20-0fbe93af32e2>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0mmodel\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdata_gen\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mepochs\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0msteps_per_epoch\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mnum_lines\u001b[0m\u001b[1;33m/\u001b[0m\u001b[1;36m200\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32mD:\\anaconda3\\envs\\tf\\lib\\site-packages\\keras\\engine\\training.py\u001b[0m in \u001b[0;36mfit\u001b[1;34m(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, validation_batch_size, validation_freq, max_queue_size, workers, use_multiprocessing)\u001b[0m\n\u001b[0;32m   1182\u001b[0m                 _r=1):\n\u001b[0;32m   1183\u001b[0m               \u001b[0mcallbacks\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mon_train_batch_begin\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mstep\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1184\u001b[1;33m               \u001b[0mtmp_logs\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtrain_function\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0miterator\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1185\u001b[0m               \u001b[1;32mif\u001b[0m \u001b[0mdata_handler\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mshould_sync\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1186\u001b[0m                 \u001b[0mcontext\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0masync_wait\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\anaconda3\\envs\\tf\\lib\\site-packages\\tensorflow\\python\\eager\\def_function.py\u001b[0m in \u001b[0;36m__call__\u001b[1;34m(self, *args, **kwds)\u001b[0m\n\u001b[0;32m    883\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    884\u001b[0m       \u001b[1;32mwith\u001b[0m \u001b[0mOptionalXlaContext\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_jit_compile\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 885\u001b[1;33m         \u001b[0mresult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_call\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    886\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    887\u001b[0m       \u001b[0mnew_tracing_count\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mexperimental_get_tracing_count\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\anaconda3\\envs\\tf\\lib\\site-packages\\tensorflow\\python\\eager\\def_function.py\u001b[0m in \u001b[0;36m_call\u001b[1;34m(self, *args, **kwds)\u001b[0m\n\u001b[0;32m    915\u001b[0m       \u001b[1;31m# In this case we have created variables on the first call, so we run the\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    916\u001b[0m       \u001b[1;31m# defunned version which is guaranteed to never create variables.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 917\u001b[1;33m       \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_stateless_fn\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[1;33m)\u001b[0m  \u001b[1;31m# pylint: disable=not-callable\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    918\u001b[0m     \u001b[1;32melif\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_stateful_fn\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    919\u001b[0m       \u001b[1;31m# Release the lock early so that multiple threads can perform the call\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\anaconda3\\envs\\tf\\lib\\site-packages\\tensorflow\\python\\eager\\function.py\u001b[0m in \u001b[0;36m__call__\u001b[1;34m(self, *args, **kwargs)\u001b[0m\n\u001b[0;32m   3037\u001b[0m       (graph_function,\n\u001b[0;32m   3038\u001b[0m        filtered_flat_args) = self._maybe_define_function(args, kwargs)\n\u001b[1;32m-> 3039\u001b[1;33m     return graph_function._call_flat(\n\u001b[0m\u001b[0;32m   3040\u001b[0m         filtered_flat_args, captured_inputs=graph_function.captured_inputs)  # pylint: disable=protected-access\n\u001b[0;32m   3041\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\anaconda3\\envs\\tf\\lib\\site-packages\\tensorflow\\python\\eager\\function.py\u001b[0m in \u001b[0;36m_call_flat\u001b[1;34m(self, args, captured_inputs, cancellation_manager)\u001b[0m\n\u001b[0;32m   1961\u001b[0m         and executing_eagerly):\n\u001b[0;32m   1962\u001b[0m       \u001b[1;31m# No tape is watching; skip to running the function.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1963\u001b[1;33m       return self._build_call_outputs(self._inference_function.call(\n\u001b[0m\u001b[0;32m   1964\u001b[0m           ctx, args, cancellation_manager=cancellation_manager))\n\u001b[0;32m   1965\u001b[0m     forward_backward = self._select_forward_and_backward_functions(\n",
      "\u001b[1;32mD:\\anaconda3\\envs\\tf\\lib\\site-packages\\tensorflow\\python\\eager\\function.py\u001b[0m in \u001b[0;36mcall\u001b[1;34m(self, ctx, args, cancellation_manager)\u001b[0m\n\u001b[0;32m    589\u001b[0m       \u001b[1;32mwith\u001b[0m \u001b[0m_InterpolateFunctionError\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    590\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mcancellation_manager\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 591\u001b[1;33m           outputs = execute.execute(\n\u001b[0m\u001b[0;32m    592\u001b[0m               \u001b[0mstr\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msignature\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mname\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    593\u001b[0m               \u001b[0mnum_outputs\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_num_outputs\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\anaconda3\\envs\\tf\\lib\\site-packages\\tensorflow\\python\\eager\\execute.py\u001b[0m in \u001b[0;36mquick_execute\u001b[1;34m(op_name, num_outputs, inputs, attrs, ctx, name)\u001b[0m\n\u001b[0;32m     57\u001b[0m   \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     58\u001b[0m     \u001b[0mctx\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mensure_initialized\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 59\u001b[1;33m     tensors = pywrap_tfe.TFE_Py_Execute(ctx._handle, device_name, op_name,\n\u001b[0m\u001b[0;32m     60\u001b[0m                                         inputs, attrs, num_outputs)\n\u001b[0;32m     61\u001b[0m   \u001b[1;32mexcept\u001b[0m \u001b[0mcore\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_NotOkStatusException\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "model.fit(data_gen, epochs=1, steps_per_epoch=int(num_lines/200)) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "PPtuKxpB7m54"
   },
   "outputs": [],
   "source": [
    "target_net.save('target.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "2y7igivMhg2K"
   },
   "outputs": [],
   "source": [
    "context_net.save('context.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "0BKkO8mthhAZ"
   },
   "outputs": [],
   "source": [
    "model.save('model.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "yPQIFpmw7sIa"
   },
   "outputs": [],
   "source": [
    "from google.colab import files\n",
    "files.download('target.h5')\n",
    "files.download('context.h5')\n",
    "files.download('model.h5')"
   ]
  }
 ],
 "metadata": {
  "accelerator": "TPU",
  "colab": {
   "collapsed_sections": [],
   "name": "train.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}