Untitled.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "8a286230-4eaa-49c7-b2da-875514b66e61",
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "import pandas as pd\n",
    "import argparse\n",
    "from typing import List\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "id": "9ce04d9d-14a5-428c-b105-d50cd8c77993",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 소수 두째자리까지 round\n",
    "def rounder(num):\n",
    "    return round(num, 2)\n",
    "\n",
    "def post_process_metric(x, not_hundred_metrics):\n",
    "    keys = x.keys()\n",
    "    for k in keys:\n",
    "        if max([hm in k for hm in not_hundred_metrics]):\n",
    "            x[k] = x[k] * 100\n",
    "        x[k] = rounder(x[k])\n",
    "    return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 170,
   "id": "ca2bdefe-ebc0-4d78-8714-cb0ef19af72e",
   "metadata": {},
   "outputs": [],
   "source": [
    "def parse_logfile(log_folder_name):\n",
    "    if not log_folder_name.endswith('.json'):\n",
    "        latest_checkpoint = sorted([d for d in os.listdir(path) if 'checkpoint' in d and not 'ipynb' in d],\n",
    "                                    key=lambda x : int(x.split('-')[1]))[-1]\n",
    "        file = os.path.join(log_folder_name, latest_checkpoint, 'trainer_state.json')\n",
    "    else:\n",
    "        file = log_folder_name\n",
    "    logs = pd.read_json(file).log_history\n",
    "    \n",
    "    # aggregate logs\n",
    "    eval_seen = [l for l in logs if 'eval_seen_ppl' in l.keys()]\n",
    "    eval_unseen = [l for l in logs if 'eval_unseen_ppl' in l.keys()]\n",
    "    test_seen = [l for l in logs if 'test_seen_ppl' in l.keys()]\n",
    "    test_unseen = [l for l in logs if 'test_unseen_ppl' in l.keys()]\n",
    "    \n",
    "    assert len(eval_seen) == len(eval_unseen) == len(test_seen) == len(test_unseen)\n",
    "    for i in range(len(eval_seen)):\n",
    "        eval_seen[i].update(eval_unseen[i])\n",
    "        eval_seen[i].update(test_seen[i])\n",
    "        eval_seen[i].update(test_unseen[i])\n",
    "    \n",
    "    log_pd = pd.DataFrame(eval_seen)\n",
    "    keys = log_pd.keys()\n",
    "    \n",
    "    metrics = sorted([s for s in set([k.split('seen')[-1].strip('_') for k in keys])\n",
    "                      if not max([i in s for i in ['num', 'runtime', 'second', 'epoch', 'step']])])\n",
    "    not_hundred_metrics = [m for m in metrics if not max([i in m for i in ['know_acc', 'loss', 'ppl']])]\n",
    "    \n",
    "    new_keys = ['epoch', 'step']\n",
    "    for m in metrics:\n",
    "        new_keys.append('eval_seen_'+m)\n",
    "        new_keys.append('eval_unseen_'+m)\n",
    "        new_keys.append('test_seen_'+m)\n",
    "        new_keys.append('test_unseen_'+m)\n",
    "        \n",
    "    log_pd_processed = log_pd.apply(lambda x : post_process_metric(x, not_hundred_metrics), axis=1)\n",
    "    return log_pd_processed[new_keys]   \n",
    "    # return log_pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 160,
   "id": "1419092f-f540-4709-b2c9-237fd0567e17",
   "metadata": {},
   "outputs": [],
   "source": [
    "a = parse_logfile('output/wowtest3/trainer_state.json')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 162,
   "id": "a77f3142-3c14-4ae7-a0e5-cb39660a9a56",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>epoch</th>\n",
       "      <th>step</th>\n",
       "      <th>eval_seen_BLEU1</th>\n",
       "      <th>eval_unseen_BLEU1</th>\n",
       "      <th>test_seen_BLEU1</th>\n",
       "      <th>test_unseen_BLEU1</th>\n",
       "      <th>eval_seen_BLEU2</th>\n",
       "      <th>eval_unseen_BLEU2</th>\n",
       "      <th>test_seen_BLEU2</th>\n",
       "      <th>test_unseen_BLEU2</th>\n",
       "      <th>...</th>\n",
       "      <th>test_seen_rougeL</th>\n",
       "      <th>test_unseen_rougeL</th>\n",
       "      <th>eval_seen_token_loss</th>\n",
       "      <th>eval_unseen_token_loss</th>\n",
       "      <th>test_seen_token_loss</th>\n",
       "      <th>test_unseen_token_loss</th>\n",
       "      <th>eval_seen_total_loss</th>\n",
       "      <th>eval_unseen_total_loss</th>\n",
       "      <th>test_seen_total_loss</th>\n",
       "      <th>test_unseen_total_loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.0</td>\n",
       "      <td>2303.0</td>\n",
       "      <td>5.85</td>\n",
       "      <td>5.61</td>\n",
       "      <td>5.57</td>\n",
       "      <td>5.63</td>\n",
       "      <td>1.29</td>\n",
       "      <td>1.14</td>\n",
       "      <td>1.19</td>\n",
       "      <td>1.18</td>\n",
       "      <td>...</td>\n",
       "      <td>8.39</td>\n",
       "      <td>8.49</td>\n",
       "      <td>5.87</td>\n",
       "      <td>6.10</td>\n",
       "      <td>5.90</td>\n",
       "      <td>6.16</td>\n",
       "      <td>9.75</td>\n",
       "      <td>11.12</td>\n",
       "      <td>9.90</td>\n",
       "      <td>11.01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2.0</td>\n",
       "      <td>4606.0</td>\n",
       "      <td>7.85</td>\n",
       "      <td>8.25</td>\n",
       "      <td>7.92</td>\n",
       "      <td>7.02</td>\n",
       "      <td>1.96</td>\n",
       "      <td>1.90</td>\n",
       "      <td>1.95</td>\n",
       "      <td>1.74</td>\n",
       "      <td>...</td>\n",
       "      <td>10.77</td>\n",
       "      <td>9.67</td>\n",
       "      <td>5.42</td>\n",
       "      <td>5.77</td>\n",
       "      <td>5.46</td>\n",
       "      <td>5.88</td>\n",
       "      <td>9.21</td>\n",
       "      <td>10.87</td>\n",
       "      <td>9.40</td>\n",
       "      <td>11.06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3.0</td>\n",
       "      <td>6909.0</td>\n",
       "      <td>9.56</td>\n",
       "      <td>9.05</td>\n",
       "      <td>9.37</td>\n",
       "      <td>8.43</td>\n",
       "      <td>2.67</td>\n",
       "      <td>2.26</td>\n",
       "      <td>2.57</td>\n",
       "      <td>2.19</td>\n",
       "      <td>...</td>\n",
       "      <td>12.38</td>\n",
       "      <td>11.01</td>\n",
       "      <td>5.14</td>\n",
       "      <td>5.59</td>\n",
       "      <td>5.19</td>\n",
       "      <td>5.72</td>\n",
       "      <td>8.95</td>\n",
       "      <td>10.85</td>\n",
       "      <td>9.15</td>\n",
       "      <td>11.04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4.0</td>\n",
       "      <td>9212.0</td>\n",
       "      <td>10.66</td>\n",
       "      <td>10.09</td>\n",
       "      <td>10.39</td>\n",
       "      <td>9.35</td>\n",
       "      <td>3.02</td>\n",
       "      <td>2.55</td>\n",
       "      <td>2.86</td>\n",
       "      <td>2.16</td>\n",
       "      <td>...</td>\n",
       "      <td>13.61</td>\n",
       "      <td>11.73</td>\n",
       "      <td>4.94</td>\n",
       "      <td>5.47</td>\n",
       "      <td>5.00</td>\n",
       "      <td>5.62</td>\n",
       "      <td>8.78</td>\n",
       "      <td>10.83</td>\n",
       "      <td>9.00</td>\n",
       "      <td>11.13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5.0</td>\n",
       "      <td>11515.0</td>\n",
       "      <td>11.16</td>\n",
       "      <td>10.31</td>\n",
       "      <td>10.98</td>\n",
       "      <td>9.53</td>\n",
       "      <td>3.52</td>\n",
       "      <td>2.72</td>\n",
       "      <td>3.36</td>\n",
       "      <td>2.32</td>\n",
       "      <td>...</td>\n",
       "      <td>13.55</td>\n",
       "      <td>11.52</td>\n",
       "      <td>4.81</td>\n",
       "      <td>5.40</td>\n",
       "      <td>4.87</td>\n",
       "      <td>5.55</td>\n",
       "      <td>8.80</td>\n",
       "      <td>10.95</td>\n",
       "      <td>9.04</td>\n",
       "      <td>11.27</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>6.0</td>\n",
       "      <td>13818.0</td>\n",
       "      <td>11.14</td>\n",
       "      <td>10.12</td>\n",
       "      <td>11.04</td>\n",
       "      <td>9.32</td>\n",
       "      <td>3.40</td>\n",
       "      <td>2.55</td>\n",
       "      <td>3.40</td>\n",
       "      <td>2.20</td>\n",
       "      <td>...</td>\n",
       "      <td>14.52</td>\n",
       "      <td>12.05</td>\n",
       "      <td>4.72</td>\n",
       "      <td>5.39</td>\n",
       "      <td>4.78</td>\n",
       "      <td>5.53</td>\n",
       "      <td>8.89</td>\n",
       "      <td>11.34</td>\n",
       "      <td>9.14</td>\n",
       "      <td>11.59</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>7.0</td>\n",
       "      <td>16121.0</td>\n",
       "      <td>12.02</td>\n",
       "      <td>10.79</td>\n",
       "      <td>11.72</td>\n",
       "      <td>10.02</td>\n",
       "      <td>3.99</td>\n",
       "      <td>2.89</td>\n",
       "      <td>3.89</td>\n",
       "      <td>2.46</td>\n",
       "      <td>...</td>\n",
       "      <td>14.83</td>\n",
       "      <td>12.24</td>\n",
       "      <td>4.64</td>\n",
       "      <td>5.38</td>\n",
       "      <td>4.70</td>\n",
       "      <td>5.53</td>\n",
       "      <td>9.01</td>\n",
       "      <td>11.77</td>\n",
       "      <td>9.31</td>\n",
       "      <td>12.05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>8.0</td>\n",
       "      <td>18424.0</td>\n",
       "      <td>12.01</td>\n",
       "      <td>10.14</td>\n",
       "      <td>11.54</td>\n",
       "      <td>9.40</td>\n",
       "      <td>4.09</td>\n",
       "      <td>2.67</td>\n",
       "      <td>3.92</td>\n",
       "      <td>2.32</td>\n",
       "      <td>...</td>\n",
       "      <td>14.91</td>\n",
       "      <td>11.94</td>\n",
       "      <td>4.58</td>\n",
       "      <td>5.35</td>\n",
       "      <td>4.64</td>\n",
       "      <td>5.53</td>\n",
       "      <td>9.06</td>\n",
       "      <td>11.87</td>\n",
       "      <td>9.36</td>\n",
       "      <td>12.26</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>9.0</td>\n",
       "      <td>20727.0</td>\n",
       "      <td>12.62</td>\n",
       "      <td>11.04</td>\n",
       "      <td>12.78</td>\n",
       "      <td>10.85</td>\n",
       "      <td>4.50</td>\n",
       "      <td>3.11</td>\n",
       "      <td>4.54</td>\n",
       "      <td>2.77</td>\n",
       "      <td>...</td>\n",
       "      <td>15.65</td>\n",
       "      <td>12.79</td>\n",
       "      <td>4.52</td>\n",
       "      <td>5.32</td>\n",
       "      <td>4.58</td>\n",
       "      <td>5.47</td>\n",
       "      <td>9.05</td>\n",
       "      <td>11.85</td>\n",
       "      <td>9.36</td>\n",
       "      <td>12.12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>10.0</td>\n",
       "      <td>23030.0</td>\n",
       "      <td>12.92</td>\n",
       "      <td>10.86</td>\n",
       "      <td>12.39</td>\n",
       "      <td>10.27</td>\n",
       "      <td>4.65</td>\n",
       "      <td>2.98</td>\n",
       "      <td>4.43</td>\n",
       "      <td>2.53</td>\n",
       "      <td>...</td>\n",
       "      <td>15.16</td>\n",
       "      <td>12.36</td>\n",
       "      <td>4.52</td>\n",
       "      <td>5.37</td>\n",
       "      <td>4.59</td>\n",
       "      <td>5.54</td>\n",
       "      <td>9.39</td>\n",
       "      <td>12.45</td>\n",
       "      <td>9.75</td>\n",
       "      <td>12.77</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>11.0</td>\n",
       "      <td>25333.0</td>\n",
       "      <td>12.55</td>\n",
       "      <td>10.85</td>\n",
       "      <td>12.44</td>\n",
       "      <td>10.15</td>\n",
       "      <td>4.56</td>\n",
       "      <td>3.00</td>\n",
       "      <td>4.57</td>\n",
       "      <td>2.52</td>\n",
       "      <td>...</td>\n",
       "      <td>15.42</td>\n",
       "      <td>12.29</td>\n",
       "      <td>4.48</td>\n",
       "      <td>5.38</td>\n",
       "      <td>4.55</td>\n",
       "      <td>5.53</td>\n",
       "      <td>9.42</td>\n",
       "      <td>12.63</td>\n",
       "      <td>9.76</td>\n",
       "      <td>12.84</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>12.0</td>\n",
       "      <td>27636.0</td>\n",
       "      <td>13.00</td>\n",
       "      <td>11.17</td>\n",
       "      <td>12.96</td>\n",
       "      <td>10.48</td>\n",
       "      <td>4.92</td>\n",
       "      <td>3.02</td>\n",
       "      <td>4.88</td>\n",
       "      <td>2.86</td>\n",
       "      <td>...</td>\n",
       "      <td>15.79</td>\n",
       "      <td>12.60</td>\n",
       "      <td>4.48</td>\n",
       "      <td>5.37</td>\n",
       "      <td>4.54</td>\n",
       "      <td>5.52</td>\n",
       "      <td>9.59</td>\n",
       "      <td>12.68</td>\n",
       "      <td>9.91</td>\n",
       "      <td>12.87</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>13.0</td>\n",
       "      <td>29939.0</td>\n",
       "      <td>13.14</td>\n",
       "      <td>11.20</td>\n",
       "      <td>13.01</td>\n",
       "      <td>10.77</td>\n",
       "      <td>4.93</td>\n",
       "      <td>3.15</td>\n",
       "      <td>4.93</td>\n",
       "      <td>2.92</td>\n",
       "      <td>...</td>\n",
       "      <td>15.91</td>\n",
       "      <td>12.95</td>\n",
       "      <td>4.47</td>\n",
       "      <td>5.39</td>\n",
       "      <td>4.54</td>\n",
       "      <td>5.54</td>\n",
       "      <td>9.69</td>\n",
       "      <td>12.85</td>\n",
       "      <td>10.02</td>\n",
       "      <td>13.01</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>13 rows × 62 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    epoch     step  eval_seen_BLEU1  eval_unseen_BLEU1  test_seen_BLEU1  \\\n",
       "0     1.0   2303.0             5.85               5.61             5.57   \n",
       "1     2.0   4606.0             7.85               8.25             7.92   \n",
       "2     3.0   6909.0             9.56               9.05             9.37   \n",
       "3     4.0   9212.0            10.66              10.09            10.39   \n",
       "4     5.0  11515.0            11.16              10.31            10.98   \n",
       "5     6.0  13818.0            11.14              10.12            11.04   \n",
       "6     7.0  16121.0            12.02              10.79            11.72   \n",
       "7     8.0  18424.0            12.01              10.14            11.54   \n",
       "8     9.0  20727.0            12.62              11.04            12.78   \n",
       "9    10.0  23030.0            12.92              10.86            12.39   \n",
       "10   11.0  25333.0            12.55              10.85            12.44   \n",
       "11   12.0  27636.0            13.00              11.17            12.96   \n",
       "12   13.0  29939.0            13.14              11.20            13.01   \n",
       "\n",
       "    test_unseen_BLEU1  eval_seen_BLEU2  eval_unseen_BLEU2  test_seen_BLEU2  \\\n",
       "0                5.63             1.29               1.14             1.19   \n",
       "1                7.02             1.96               1.90             1.95   \n",
       "2                8.43             2.67               2.26             2.57   \n",
       "3                9.35             3.02               2.55             2.86   \n",
       "4                9.53             3.52               2.72             3.36   \n",
       "5                9.32             3.40               2.55             3.40   \n",
       "6               10.02             3.99               2.89             3.89   \n",
       "7                9.40             4.09               2.67             3.92   \n",
       "8               10.85             4.50               3.11             4.54   \n",
       "9               10.27             4.65               2.98             4.43   \n",
       "10              10.15             4.56               3.00             4.57   \n",
       "11              10.48             4.92               3.02             4.88   \n",
       "12              10.77             4.93               3.15             4.93   \n",
       "\n",
       "    test_unseen_BLEU2  ...  test_seen_rougeL  test_unseen_rougeL  \\\n",
       "0                1.18  ...              8.39                8.49   \n",
       "1                1.74  ...             10.77                9.67   \n",
       "2                2.19  ...             12.38               11.01   \n",
       "3                2.16  ...             13.61               11.73   \n",
       "4                2.32  ...             13.55               11.52   \n",
       "5                2.20  ...             14.52               12.05   \n",
       "6                2.46  ...             14.83               12.24   \n",
       "7                2.32  ...             14.91               11.94   \n",
       "8                2.77  ...             15.65               12.79   \n",
       "9                2.53  ...             15.16               12.36   \n",
       "10               2.52  ...             15.42               12.29   \n",
       "11               2.86  ...             15.79               12.60   \n",
       "12               2.92  ...             15.91               12.95   \n",
       "\n",
       "    eval_seen_token_loss  eval_unseen_token_loss  test_seen_token_loss  \\\n",
       "0                   5.87                    6.10                  5.90   \n",
       "1                   5.42                    5.77                  5.46   \n",
       "2                   5.14                    5.59                  5.19   \n",
       "3                   4.94                    5.47                  5.00   \n",
       "4                   4.81                    5.40                  4.87   \n",
       "5                   4.72                    5.39                  4.78   \n",
       "6                   4.64                    5.38                  4.70   \n",
       "7                   4.58                    5.35                  4.64   \n",
       "8                   4.52                    5.32                  4.58   \n",
       "9                   4.52                    5.37                  4.59   \n",
       "10                  4.48                    5.38                  4.55   \n",
       "11                  4.48                    5.37                  4.54   \n",
       "12                  4.47                    5.39                  4.54   \n",
       "\n",
       "    test_unseen_token_loss  eval_seen_total_loss  eval_unseen_total_loss  \\\n",
       "0                     6.16                  9.75                   11.12   \n",
       "1                     5.88                  9.21                   10.87   \n",
       "2                     5.72                  8.95                   10.85   \n",
       "3                     5.62                  8.78                   10.83   \n",
       "4                     5.55                  8.80                   10.95   \n",
       "5                     5.53                  8.89                   11.34   \n",
       "6                     5.53                  9.01                   11.77   \n",
       "7                     5.53                  9.06                   11.87   \n",
       "8                     5.47                  9.05                   11.85   \n",
       "9                     5.54                  9.39                   12.45   \n",
       "10                    5.53                  9.42                   12.63   \n",
       "11                    5.52                  9.59                   12.68   \n",
       "12                    5.54                  9.69                   12.85   \n",
       "\n",
       "    test_seen_total_loss  test_unseen_total_loss  \n",
       "0                   9.90                   11.01  \n",
       "1                   9.40                   11.06  \n",
       "2                   9.15                   11.04  \n",
       "3                   9.00                   11.13  \n",
       "4                   9.04                   11.27  \n",
       "5                   9.14                   11.59  \n",
       "6                   9.31                   12.05  \n",
       "7                   9.36                   12.26  \n",
       "8                   9.36                   12.12  \n",
       "9                   9.75                   12.77  \n",
       "10                  9.76                   12.84  \n",
       "11                  9.91                   12.87  \n",
       "12                 10.02                   13.01  \n",
       "\n",
       "[13 rows x 62 columns]"
      ]
     },
     "execution_count": 162,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 164,
   "id": "9c197c7a-408b-495e-a741-ec080c4d5575",
   "metadata": {},
   "outputs": [],
   "source": [
    "def write_folder_tocsv(write_file_name,\n",
    "                       log_file_names : List,\n",
    "                       ):\n",
    "    for log_file_name in log_file_names:\n",
    "        parsed_log = parse_logfile(log_file_name)\n",
    "        \n",
    "        # write file as csv\n",
    "        write_csv_file_name = write_file_name+'.csv'\n",
    "        parsed_log.to_csv(write_csv_file_name, mode='a')\n",
    "        with open(write_csv_file_name, 'a') as f:\n",
    "            f.write(log_file_name)\n",
    "            f.write('\\n\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 175,
   "id": "18e01545-7f7b-4764-8a4f-54e42d42f07b",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_log_file_names(log_folder):\n",
    "    log_file_names = [os.path.join(log_folder,log_file)\n",
    "                      for log_file in os.listdir(log_folder)]\n",
    "    return log_file_names"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "54e3b1b7-fcd6-4832-a563-d3085d68873b",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b165bc47-6a61-4763-8191-31a05ab46781",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e5611e4c-2a2b-428d-b94d-943a5f7f2c29",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3ca41323-e5aa-42b0-87fa-6805a8186c06",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d2c664b4-11a7-4ffc-88a0-73c4043f8adc",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c3d8dd37-7393-43aa-8752-6766b4158797",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0bc68ff8-2f26-4c07-95fd-620433b38ec7",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ef0ebb2c-fa83-496e-81ac-280bb751dccd",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "b6bfbe33-c60a-42e6-b155-c7d919101041",
   "metadata": {},
   "outputs": [],
   "source": [
    "path = 'output/wowtest3'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "13d07681-2598-4c73-b13a-3749f2b977c7",
   "metadata": {},
   "outputs": [],
   "source": [
    "latest_checkpoint = sorted([d for d in os.listdir(path) if 'checkpoint' in d], key=lambda x : int(x.split('-')[1]))[-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "dff4d03a-e90c-48b0-a60a-626524402f76",
   "metadata": {},
   "outputs": [],
   "source": [
    "file = os.path.join(path, latest_checkpoint, 'trainer_state.json')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ccdebb2f-b27d-409f-93ac-cb8e4ddcef0c",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "id": "fa8739e9-6834-49e5-a14b-e7084e56495d",
   "metadata": {},
   "outputs": [],
   "source": [
    "file = 'output/wowtest3/trainer_state.json'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "id": "2cffc92c-9e87-4ae8-ac8a-b297cf420f62",
   "metadata": {},
   "outputs": [],
   "source": [
    "a = pd.read_json(file).log_history"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "id": "18149022-822e-4945-b926-0b637e061f9d",
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "eval_seen = [aa for aa in a if 'eval_seen_ppl' in aa.keys()]\n",
    "eval_unseen = [aa for aa in a if 'eval_unseen_ppl' in aa.keys()]\n",
    "test_seen = [aa for aa in a if 'test_seen_ppl' in aa.keys()]\n",
    "test_unseen = [aa for aa in a if 'test_unseen_ppl' in aa.keys()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "id": "d5e2e12d-1835-4826-a258-bab2252f2337",
   "metadata": {},
   "outputs": [],
   "source": [
    "assert len(eval_seen) == len(eval_unseen) == len(test_seen) == len(test_unseen)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "id": "b5350b99-b23e-446a-99e0-11c40f708dc1",
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in range(len(eval_seen)):\n",
    "    eval_seen[i].update(eval_unseen[i])\n",
    "    eval_seen[i].update(test_seen[i])\n",
    "    eval_seen[i].update(test_unseen[i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "id": "f20d0d1f-6705-424e-ba2e-f77203881970",
   "metadata": {},
   "outputs": [],
   "source": [
    "d = pd.DataFrame(eval_seen)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "id": "e72e6137-064b-4bbb-8375-db84c837384a",
   "metadata": {},
   "outputs": [],
   "source": [
    "keys = pd.DataFrame(eval_seen).keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "id": "ff42df85-ca4b-436c-ba77-5a46c51002fc",
   "metadata": {},
   "outputs": [],
   "source": [
    "metrics = sorted([s for s in set([k.split('seen')[-1].strip('_') for k in keys]) if not max([i in s for i in ['num', 'runtime', 'second', 'epoch', 'step']])])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "id": "f0c6a021-a303-40bb-ae04-b234972e2808",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['BLEU1',\n",
       " 'BLEU2',\n",
       " 'BLEU3',\n",
       " 'BLEU4',\n",
       " 'Distinct-1',\n",
       " 'Distinct-2',\n",
       " 'F1',\n",
       " 'know_acc',\n",
       " 'knowledge_loss',\n",
       " 'ppl',\n",
       " 'rouge1',\n",
       " 'rouge2',\n",
       " 'rougeL',\n",
       " 'token_loss',\n",
       " 'total_loss']"
      ]
     },
     "execution_count": 111,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "id": "e6e020cd-d3aa-413b-a364-9705e52dfe59",
   "metadata": {},
   "outputs": [],
   "source": [
    "not_hundred_metrics = [m for m in metrics if not max([i in m for i in ['know_acc', 'loss', 'ppl']])]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "id": "a5c3e985-b5c0-4aa8-9a79-40e7cca732b6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['BLEU1',\n",
       " 'BLEU2',\n",
       " 'BLEU3',\n",
       " 'BLEU4',\n",
       " 'Distinct-1',\n",
       " 'Distinct-2',\n",
       " 'F1',\n",
       " 'rouge1',\n",
       " 'rouge2',\n",
       " 'rougeL']"
      ]
     },
     "execution_count": 122,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "not_hundred_metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "id": "2d70a2ef-96e0-4abd-9ce0-b9a42c4de964",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>epoch</th>\n",
       "      <th>eval_seen_BLEU1</th>\n",
       "      <th>eval_seen_BLEU2</th>\n",
       "      <th>eval_seen_BLEU3</th>\n",
       "      <th>eval_seen_BLEU4</th>\n",
       "      <th>eval_seen_Distinct-1</th>\n",
       "      <th>eval_seen_Distinct-2</th>\n",
       "      <th>eval_seen_F1</th>\n",
       "      <th>eval_seen_know_acc</th>\n",
       "      <th>eval_seen_knowledge_loss</th>\n",
       "      <th>...</th>\n",
       "      <th>test_unseen_rouge1</th>\n",
       "      <th>test_unseen_rouge2</th>\n",
       "      <th>test_unseen_rougeL</th>\n",
       "      <th>test_unseen_runtime</th>\n",
       "      <th>test_unseen_samples_per_second</th>\n",
       "      <th>test_unseen_steps_per_second</th>\n",
       "      <th>test_unseen_token_loss</th>\n",
       "      <th>test_unseen_total_episode_num</th>\n",
       "      <th>test_unseen_total_loss</th>\n",
       "      <th>test_unseen_total_sentence_num</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.058510</td>\n",
       "      <td>0.012894</td>\n",
       "      <td>0.001982</td>\n",
       "      <td>0.000722</td>\n",
       "      <td>0.001195</td>\n",
       "      <td>0.004463</td>\n",
       "      <td>0.094001</td>\n",
       "      <td>25.539477</td>\n",
       "      <td>3.882978</td>\n",
       "      <td>...</td>\n",
       "      <td>0.090370</td>\n",
       "      <td>0.012079</td>\n",
       "      <td>0.084882</td>\n",
       "      <td>714.8713</td>\n",
       "      <td>1.354</td>\n",
       "      <td>0.677</td>\n",
       "      <td>6.158024</td>\n",
       "      <td>968</td>\n",
       "      <td>11.013030</td>\n",
       "      <td>3922</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2.0</td>\n",
       "      <td>0.078471</td>\n",
       "      <td>0.019601</td>\n",
       "      <td>0.003888</td>\n",
       "      <td>0.001077</td>\n",
       "      <td>0.003541</td>\n",
       "      <td>0.017700</td>\n",
       "      <td>0.117044</td>\n",
       "      <td>27.011932</td>\n",
       "      <td>3.788660</td>\n",
       "      <td>...</td>\n",
       "      <td>0.104146</td>\n",
       "      <td>0.017117</td>\n",
       "      <td>0.096719</td>\n",
       "      <td>734.1567</td>\n",
       "      <td>1.319</td>\n",
       "      <td>0.659</td>\n",
       "      <td>5.878667</td>\n",
       "      <td>968</td>\n",
       "      <td>11.056437</td>\n",
       "      <td>3922</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3.0</td>\n",
       "      <td>0.095602</td>\n",
       "      <td>0.026743</td>\n",
       "      <td>0.006384</td>\n",
       "      <td>0.001681</td>\n",
       "      <td>0.005833</td>\n",
       "      <td>0.033122</td>\n",
       "      <td>0.137919</td>\n",
       "      <td>27.113481</td>\n",
       "      <td>3.810125</td>\n",
       "      <td>...</td>\n",
       "      <td>0.119839</td>\n",
       "      <td>0.020590</td>\n",
       "      <td>0.110090</td>\n",
       "      <td>725.4196</td>\n",
       "      <td>1.334</td>\n",
       "      <td>0.667</td>\n",
       "      <td>5.715868</td>\n",
       "      <td>968</td>\n",
       "      <td>11.039643</td>\n",
       "      <td>3922</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4.0</td>\n",
       "      <td>0.106576</td>\n",
       "      <td>0.030195</td>\n",
       "      <td>0.007743</td>\n",
       "      <td>0.002867</td>\n",
       "      <td>0.009856</td>\n",
       "      <td>0.054993</td>\n",
       "      <td>0.153579</td>\n",
       "      <td>27.189642</td>\n",
       "      <td>3.840566</td>\n",
       "      <td>...</td>\n",
       "      <td>0.130343</td>\n",
       "      <td>0.019265</td>\n",
       "      <td>0.117314</td>\n",
       "      <td>705.4067</td>\n",
       "      <td>1.372</td>\n",
       "      <td>0.686</td>\n",
       "      <td>5.617607</td>\n",
       "      <td>968</td>\n",
       "      <td>11.125307</td>\n",
       "      <td>3922</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5.0</td>\n",
       "      <td>0.111576</td>\n",
       "      <td>0.035153</td>\n",
       "      <td>0.010178</td>\n",
       "      <td>0.003580</td>\n",
       "      <td>0.012999</td>\n",
       "      <td>0.071559</td>\n",
       "      <td>0.151464</td>\n",
       "      <td>26.732673</td>\n",
       "      <td>3.987263</td>\n",
       "      <td>...</td>\n",
       "      <td>0.128677</td>\n",
       "      <td>0.019991</td>\n",
       "      <td>0.115154</td>\n",
       "      <td>718.6471</td>\n",
       "      <td>1.347</td>\n",
       "      <td>0.673</td>\n",
       "      <td>5.549252</td>\n",
       "      <td>968</td>\n",
       "      <td>11.266858</td>\n",
       "      <td>3922</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>6.0</td>\n",
       "      <td>0.111393</td>\n",
       "      <td>0.033960</td>\n",
       "      <td>0.009280</td>\n",
       "      <td>0.003222</td>\n",
       "      <td>0.020326</td>\n",
       "      <td>0.108535</td>\n",
       "      <td>0.161270</td>\n",
       "      <td>26.199543</td>\n",
       "      <td>4.169706</td>\n",
       "      <td>...</td>\n",
       "      <td>0.133620</td>\n",
       "      <td>0.019927</td>\n",
       "      <td>0.120454</td>\n",
       "      <td>688.6639</td>\n",
       "      <td>1.406</td>\n",
       "      <td>0.703</td>\n",
       "      <td>5.529822</td>\n",
       "      <td>968</td>\n",
       "      <td>11.594268</td>\n",
       "      <td>3922</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>7.0</td>\n",
       "      <td>0.120231</td>\n",
       "      <td>0.039894</td>\n",
       "      <td>0.012193</td>\n",
       "      <td>0.005344</td>\n",
       "      <td>0.024203</td>\n",
       "      <td>0.130281</td>\n",
       "      <td>0.167913</td>\n",
       "      <td>25.818736</td>\n",
       "      <td>4.371843</td>\n",
       "      <td>...</td>\n",
       "      <td>0.136948</td>\n",
       "      <td>0.021217</td>\n",
       "      <td>0.122412</td>\n",
       "      <td>703.5591</td>\n",
       "      <td>1.376</td>\n",
       "      <td>0.688</td>\n",
       "      <td>5.534378</td>\n",
       "      <td>968</td>\n",
       "      <td>12.051908</td>\n",
       "      <td>3922</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>8.0</td>\n",
       "      <td>0.120127</td>\n",
       "      <td>0.040927</td>\n",
       "      <td>0.012756</td>\n",
       "      <td>0.005785</td>\n",
       "      <td>0.028507</td>\n",
       "      <td>0.152181</td>\n",
       "      <td>0.169608</td>\n",
       "      <td>25.260218</td>\n",
       "      <td>4.475227</td>\n",
       "      <td>...</td>\n",
       "      <td>0.132672</td>\n",
       "      <td>0.021628</td>\n",
       "      <td>0.119412</td>\n",
       "      <td>684.9259</td>\n",
       "      <td>1.413</td>\n",
       "      <td>0.707</td>\n",
       "      <td>5.526098</td>\n",
       "      <td>968</td>\n",
       "      <td>12.260585</td>\n",
       "      <td>3922</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>9.0</td>\n",
       "      <td>0.126193</td>\n",
       "      <td>0.044992</td>\n",
       "      <td>0.015258</td>\n",
       "      <td>0.007090</td>\n",
       "      <td>0.030407</td>\n",
       "      <td>0.155902</td>\n",
       "      <td>0.172531</td>\n",
       "      <td>25.260218</td>\n",
       "      <td>4.531729</td>\n",
       "      <td>...</td>\n",
       "      <td>0.143993</td>\n",
       "      <td>0.023550</td>\n",
       "      <td>0.127861</td>\n",
       "      <td>701.4569</td>\n",
       "      <td>1.380</td>\n",
       "      <td>0.690</td>\n",
       "      <td>5.467641</td>\n",
       "      <td>968</td>\n",
       "      <td>12.122010</td>\n",
       "      <td>3922</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>10.0</td>\n",
       "      <td>0.129229</td>\n",
       "      <td>0.046544</td>\n",
       "      <td>0.015429</td>\n",
       "      <td>0.007091</td>\n",
       "      <td>0.035082</td>\n",
       "      <td>0.173668</td>\n",
       "      <td>0.174417</td>\n",
       "      <td>24.727088</td>\n",
       "      <td>4.870489</td>\n",
       "      <td>...</td>\n",
       "      <td>0.137606</td>\n",
       "      <td>0.021793</td>\n",
       "      <td>0.123608</td>\n",
       "      <td>724.5852</td>\n",
       "      <td>1.336</td>\n",
       "      <td>0.668</td>\n",
       "      <td>5.538153</td>\n",
       "      <td>968</td>\n",
       "      <td>12.772947</td>\n",
       "      <td>3922</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>11.0</td>\n",
       "      <td>0.125470</td>\n",
       "      <td>0.045596</td>\n",
       "      <td>0.016496</td>\n",
       "      <td>0.007863</td>\n",
       "      <td>0.039726</td>\n",
       "      <td>0.197031</td>\n",
       "      <td>0.172876</td>\n",
       "      <td>24.676314</td>\n",
       "      <td>4.940875</td>\n",
       "      <td>...</td>\n",
       "      <td>0.137540</td>\n",
       "      <td>0.021529</td>\n",
       "      <td>0.122889</td>\n",
       "      <td>700.5157</td>\n",
       "      <td>1.382</td>\n",
       "      <td>0.691</td>\n",
       "      <td>5.526578</td>\n",
       "      <td>968</td>\n",
       "      <td>12.836178</td>\n",
       "      <td>3922</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>12.0</td>\n",
       "      <td>0.130016</td>\n",
       "      <td>0.049173</td>\n",
       "      <td>0.018261</td>\n",
       "      <td>0.008411</td>\n",
       "      <td>0.042578</td>\n",
       "      <td>0.206655</td>\n",
       "      <td>0.175754</td>\n",
       "      <td>23.635440</td>\n",
       "      <td>5.108103</td>\n",
       "      <td>...</td>\n",
       "      <td>0.140970</td>\n",
       "      <td>0.024655</td>\n",
       "      <td>0.126004</td>\n",
       "      <td>704.3580</td>\n",
       "      <td>1.374</td>\n",
       "      <td>0.687</td>\n",
       "      <td>5.521423</td>\n",
       "      <td>968</td>\n",
       "      <td>12.870567</td>\n",
       "      <td>3922</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>13.0</td>\n",
       "      <td>0.131383</td>\n",
       "      <td>0.049296</td>\n",
       "      <td>0.018434</td>\n",
       "      <td>0.009232</td>\n",
       "      <td>0.047493</td>\n",
       "      <td>0.222134</td>\n",
       "      <td>0.179372</td>\n",
       "      <td>23.889312</td>\n",
       "      <td>5.210962</td>\n",
       "      <td>...</td>\n",
       "      <td>0.145504</td>\n",
       "      <td>0.025532</td>\n",
       "      <td>0.129451</td>\n",
       "      <td>696.3415</td>\n",
       "      <td>1.390</td>\n",
       "      <td>0.695</td>\n",
       "      <td>5.541563</td>\n",
       "      <td>968</td>\n",
       "      <td>13.013042</td>\n",
       "      <td>3922</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>13 rows × 82 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    epoch  eval_seen_BLEU1  eval_seen_BLEU2  eval_seen_BLEU3  eval_seen_BLEU4  \\\n",
       "0     1.0         0.058510         0.012894         0.001982         0.000722   \n",
       "1     2.0         0.078471         0.019601         0.003888         0.001077   \n",
       "2     3.0         0.095602         0.026743         0.006384         0.001681   \n",
       "3     4.0         0.106576         0.030195         0.007743         0.002867   \n",
       "4     5.0         0.111576         0.035153         0.010178         0.003580   \n",
       "5     6.0         0.111393         0.033960         0.009280         0.003222   \n",
       "6     7.0         0.120231         0.039894         0.012193         0.005344   \n",
       "7     8.0         0.120127         0.040927         0.012756         0.005785   \n",
       "8     9.0         0.126193         0.044992         0.015258         0.007090   \n",
       "9    10.0         0.129229         0.046544         0.015429         0.007091   \n",
       "10   11.0         0.125470         0.045596         0.016496         0.007863   \n",
       "11   12.0         0.130016         0.049173         0.018261         0.008411   \n",
       "12   13.0         0.131383         0.049296         0.018434         0.009232   \n",
       "\n",
       "    eval_seen_Distinct-1  eval_seen_Distinct-2  eval_seen_F1  \\\n",
       "0               0.001195              0.004463      0.094001   \n",
       "1               0.003541              0.017700      0.117044   \n",
       "2               0.005833              0.033122      0.137919   \n",
       "3               0.009856              0.054993      0.153579   \n",
       "4               0.012999              0.071559      0.151464   \n",
       "5               0.020326              0.108535      0.161270   \n",
       "6               0.024203              0.130281      0.167913   \n",
       "7               0.028507              0.152181      0.169608   \n",
       "8               0.030407              0.155902      0.172531   \n",
       "9               0.035082              0.173668      0.174417   \n",
       "10              0.039726              0.197031      0.172876   \n",
       "11              0.042578              0.206655      0.175754   \n",
       "12              0.047493              0.222134      0.179372   \n",
       "\n",
       "    eval_seen_know_acc  eval_seen_knowledge_loss  ...  test_unseen_rouge1  \\\n",
       "0            25.539477                  3.882978  ...            0.090370   \n",
       "1            27.011932                  3.788660  ...            0.104146   \n",
       "2            27.113481                  3.810125  ...            0.119839   \n",
       "3            27.189642                  3.840566  ...            0.130343   \n",
       "4            26.732673                  3.987263  ...            0.128677   \n",
       "5            26.199543                  4.169706  ...            0.133620   \n",
       "6            25.818736                  4.371843  ...            0.136948   \n",
       "7            25.260218                  4.475227  ...            0.132672   \n",
       "8            25.260218                  4.531729  ...            0.143993   \n",
       "9            24.727088                  4.870489  ...            0.137606   \n",
       "10           24.676314                  4.940875  ...            0.137540   \n",
       "11           23.635440                  5.108103  ...            0.140970   \n",
       "12           23.889312                  5.210962  ...            0.145504   \n",
       "\n",
       "    test_unseen_rouge2  test_unseen_rougeL  test_unseen_runtime  \\\n",
       "0             0.012079            0.084882             714.8713   \n",
       "1             0.017117            0.096719             734.1567   \n",
       "2             0.020590            0.110090             725.4196   \n",
       "3             0.019265            0.117314             705.4067   \n",
       "4             0.019991            0.115154             718.6471   \n",
       "5             0.019927            0.120454             688.6639   \n",
       "6             0.021217            0.122412             703.5591   \n",
       "7             0.021628            0.119412             684.9259   \n",
       "8             0.023550            0.127861             701.4569   \n",
       "9             0.021793            0.123608             724.5852   \n",
       "10            0.021529            0.122889             700.5157   \n",
       "11            0.024655            0.126004             704.3580   \n",
       "12            0.025532            0.129451             696.3415   \n",
       "\n",
       "    test_unseen_samples_per_second  test_unseen_steps_per_second  \\\n",
       "0                            1.354                         0.677   \n",
       "1                            1.319                         0.659   \n",
       "2                            1.334                         0.667   \n",
       "3                            1.372                         0.686   \n",
       "4                            1.347                         0.673   \n",
       "5                            1.406                         0.703   \n",
       "6                            1.376                         0.688   \n",
       "7                            1.413                         0.707   \n",
       "8                            1.380                         0.690   \n",
       "9                            1.336                         0.668   \n",
       "10                           1.382                         0.691   \n",
       "11                           1.374                         0.687   \n",
       "12                           1.390                         0.695   \n",
       "\n",
       "    test_unseen_token_loss  test_unseen_total_episode_num  \\\n",
       "0                 6.158024                            968   \n",
       "1                 5.878667                            968   \n",
       "2                 5.715868                            968   \n",
       "3                 5.617607                            968   \n",
       "4                 5.549252                            968   \n",
       "5                 5.529822                            968   \n",
       "6                 5.534378                            968   \n",
       "7                 5.526098                            968   \n",
       "8                 5.467641                            968   \n",
       "9                 5.538153                            968   \n",
       "10                5.526578                            968   \n",
       "11                5.521423                            968   \n",
       "12                5.541563                            968   \n",
       "\n",
       "    test_unseen_total_loss  test_unseen_total_sentence_num  \n",
       "0                11.013030                            3922  \n",
       "1                11.056437                            3922  \n",
       "2                11.039643                            3922  \n",
       "3                11.125307                            3922  \n",
       "4                11.266858                            3922  \n",
       "5                11.594268                            3922  \n",
       "6                12.051908                            3922  \n",
       "7                12.260585                            3922  \n",
       "8                12.122010                            3922  \n",
       "9                12.772947                            3922  \n",
       "10               12.836178                            3922  \n",
       "11               12.870567                            3922  \n",
       "12               13.013042                            3922  \n",
       "\n",
       "[13 rows x 82 columns]"
      ]
     },
     "execution_count": 123,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "d.apply(lambda x : )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "id": "27fa5f6e-8d3a-450e-9df7-efa524c4a5b7",
   "metadata": {},
   "outputs": [],
   "source": [
    "def post_process_metric(x, not_hundred_metrics):\n",
    "    keys = x.keys()\n",
    "    for k in keys:\n",
    "        if max([hm in k for hm in not_hundred_metrics]):\n",
    "            x[k] = x[k] * 100\n",
    "        x[k] = rounder(x[k])\n",
    "    return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "id": "b455f365-cae1-4bfb-9746-edafa7d95045",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>epoch</th>\n",
       "      <th>eval_seen_BLEU1</th>\n",
       "      <th>eval_seen_BLEU2</th>\n",
       "      <th>eval_seen_BLEU3</th>\n",
       "      <th>eval_seen_BLEU4</th>\n",
       "      <th>eval_seen_Distinct-1</th>\n",
       "      <th>eval_seen_Distinct-2</th>\n",
       "      <th>eval_seen_F1</th>\n",
       "      <th>eval_seen_know_acc</th>\n",
       "      <th>eval_seen_knowledge_loss</th>\n",
       "      <th>...</th>\n",
       "      <th>test_unseen_rouge1</th>\n",
       "      <th>test_unseen_rouge2</th>\n",
       "      <th>test_unseen_rougeL</th>\n",
       "      <th>test_unseen_runtime</th>\n",
       "      <th>test_unseen_samples_per_second</th>\n",
       "      <th>test_unseen_steps_per_second</th>\n",
       "      <th>test_unseen_token_loss</th>\n",
       "      <th>test_unseen_total_episode_num</th>\n",
       "      <th>test_unseen_total_loss</th>\n",
       "      <th>test_unseen_total_sentence_num</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.0</td>\n",
       "      <td>5.85</td>\n",
       "      <td>1.29</td>\n",
       "      <td>0.20</td>\n",
       "      <td>0.07</td>\n",
       "      <td>0.12</td>\n",
       "      <td>0.45</td>\n",
       "      <td>9.40</td>\n",
       "      <td>25.54</td>\n",
       "      <td>3.88</td>\n",
       "      <td>...</td>\n",
       "      <td>9.04</td>\n",
       "      <td>1.21</td>\n",
       "      <td>8.49</td>\n",
       "      <td>714.87</td>\n",
       "      <td>1.35</td>\n",
       "      <td>0.68</td>\n",
       "      <td>6.16</td>\n",
       "      <td>968.0</td>\n",
       "      <td>11.01</td>\n",
       "      <td>3922.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2.0</td>\n",
       "      <td>7.85</td>\n",
       "      <td>1.96</td>\n",
       "      <td>0.39</td>\n",
       "      <td>0.11</td>\n",
       "      <td>0.35</td>\n",
       "      <td>1.77</td>\n",
       "      <td>11.70</td>\n",
       "      <td>27.01</td>\n",
       "      <td>3.79</td>\n",
       "      <td>...</td>\n",
       "      <td>10.41</td>\n",
       "      <td>1.71</td>\n",
       "      <td>9.67</td>\n",
       "      <td>734.16</td>\n",
       "      <td>1.32</td>\n",
       "      <td>0.66</td>\n",
       "      <td>5.88</td>\n",
       "      <td>968.0</td>\n",
       "      <td>11.06</td>\n",
       "      <td>3922.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3.0</td>\n",
       "      <td>9.56</td>\n",
       "      <td>2.67</td>\n",
       "      <td>0.64</td>\n",
       "      <td>0.17</td>\n",
       "      <td>0.58</td>\n",
       "      <td>3.31</td>\n",
       "      <td>13.79</td>\n",
       "      <td>27.11</td>\n",
       "      <td>3.81</td>\n",
       "      <td>...</td>\n",
       "      <td>11.98</td>\n",
       "      <td>2.06</td>\n",
       "      <td>11.01</td>\n",
       "      <td>725.42</td>\n",
       "      <td>1.33</td>\n",
       "      <td>0.67</td>\n",
       "      <td>5.72</td>\n",
       "      <td>968.0</td>\n",
       "      <td>11.04</td>\n",
       "      <td>3922.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4.0</td>\n",
       "      <td>10.66</td>\n",
       "      <td>3.02</td>\n",
       "      <td>0.77</td>\n",
       "      <td>0.29</td>\n",
       "      <td>0.99</td>\n",
       "      <td>5.50</td>\n",
       "      <td>15.36</td>\n",
       "      <td>27.19</td>\n",
       "      <td>3.84</td>\n",
       "      <td>...</td>\n",
       "      <td>13.03</td>\n",
       "      <td>1.93</td>\n",
       "      <td>11.73</td>\n",
       "      <td>705.41</td>\n",
       "      <td>1.37</td>\n",
       "      <td>0.69</td>\n",
       "      <td>5.62</td>\n",
       "      <td>968.0</td>\n",
       "      <td>11.13</td>\n",
       "      <td>3922.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5.0</td>\n",
       "      <td>11.16</td>\n",
       "      <td>3.52</td>\n",
       "      <td>1.02</td>\n",
       "      <td>0.36</td>\n",
       "      <td>1.30</td>\n",
       "      <td>7.16</td>\n",
       "      <td>15.15</td>\n",
       "      <td>26.73</td>\n",
       "      <td>3.99</td>\n",
       "      <td>...</td>\n",
       "      <td>12.87</td>\n",
       "      <td>2.00</td>\n",
       "      <td>11.52</td>\n",
       "      <td>718.65</td>\n",
       "      <td>1.35</td>\n",
       "      <td>0.67</td>\n",
       "      <td>5.55</td>\n",
       "      <td>968.0</td>\n",
       "      <td>11.27</td>\n",
       "      <td>3922.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>6.0</td>\n",
       "      <td>11.14</td>\n",
       "      <td>3.40</td>\n",
       "      <td>0.93</td>\n",
       "      <td>0.32</td>\n",
       "      <td>2.03</td>\n",
       "      <td>10.85</td>\n",
       "      <td>16.13</td>\n",
       "      <td>26.20</td>\n",
       "      <td>4.17</td>\n",
       "      <td>...</td>\n",
       "      <td>13.36</td>\n",
       "      <td>1.99</td>\n",
       "      <td>12.05</td>\n",
       "      <td>688.66</td>\n",
       "      <td>1.41</td>\n",
       "      <td>0.70</td>\n",
       "      <td>5.53</td>\n",
       "      <td>968.0</td>\n",
       "      <td>11.59</td>\n",
       "      <td>3922.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>7.0</td>\n",
       "      <td>12.02</td>\n",
       "      <td>3.99</td>\n",
       "      <td>1.22</td>\n",
       "      <td>0.53</td>\n",
       "      <td>2.42</td>\n",
       "      <td>13.03</td>\n",
       "      <td>16.79</td>\n",
       "      <td>25.82</td>\n",
       "      <td>4.37</td>\n",
       "      <td>...</td>\n",
       "      <td>13.69</td>\n",
       "      <td>2.12</td>\n",
       "      <td>12.24</td>\n",
       "      <td>703.56</td>\n",
       "      <td>1.38</td>\n",
       "      <td>0.69</td>\n",
       "      <td>5.53</td>\n",
       "      <td>968.0</td>\n",
       "      <td>12.05</td>\n",
       "      <td>3922.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>8.0</td>\n",
       "      <td>12.01</td>\n",
       "      <td>4.09</td>\n",
       "      <td>1.28</td>\n",
       "      <td>0.58</td>\n",
       "      <td>2.85</td>\n",
       "      <td>15.22</td>\n",
       "      <td>16.96</td>\n",
       "      <td>25.26</td>\n",
       "      <td>4.48</td>\n",
       "      <td>...</td>\n",
       "      <td>13.27</td>\n",
       "      <td>2.16</td>\n",
       "      <td>11.94</td>\n",
       "      <td>684.93</td>\n",
       "      <td>1.41</td>\n",
       "      <td>0.71</td>\n",
       "      <td>5.53</td>\n",
       "      <td>968.0</td>\n",
       "      <td>12.26</td>\n",
       "      <td>3922.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>9.0</td>\n",
       "      <td>12.62</td>\n",
       "      <td>4.50</td>\n",
       "      <td>1.53</td>\n",
       "      <td>0.71</td>\n",
       "      <td>3.04</td>\n",
       "      <td>15.59</td>\n",
       "      <td>17.25</td>\n",
       "      <td>25.26</td>\n",
       "      <td>4.53</td>\n",
       "      <td>...</td>\n",
       "      <td>14.40</td>\n",
       "      <td>2.35</td>\n",
       "      <td>12.79</td>\n",
       "      <td>701.46</td>\n",
       "      <td>1.38</td>\n",
       "      <td>0.69</td>\n",
       "      <td>5.47</td>\n",
       "      <td>968.0</td>\n",
       "      <td>12.12</td>\n",
       "      <td>3922.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>10.0</td>\n",
       "      <td>12.92</td>\n",
       "      <td>4.65</td>\n",
       "      <td>1.54</td>\n",
       "      <td>0.71</td>\n",
       "      <td>3.51</td>\n",
       "      <td>17.37</td>\n",
       "      <td>17.44</td>\n",
       "      <td>24.73</td>\n",
       "      <td>4.87</td>\n",
       "      <td>...</td>\n",
       "      <td>13.76</td>\n",
       "      <td>2.18</td>\n",
       "      <td>12.36</td>\n",
       "      <td>724.59</td>\n",
       "      <td>1.34</td>\n",
       "      <td>0.67</td>\n",
       "      <td>5.54</td>\n",
       "      <td>968.0</td>\n",
       "      <td>12.77</td>\n",
       "      <td>3922.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>11.0</td>\n",
       "      <td>12.55</td>\n",
       "      <td>4.56</td>\n",
       "      <td>1.65</td>\n",
       "      <td>0.79</td>\n",
       "      <td>3.97</td>\n",
       "      <td>19.70</td>\n",
       "      <td>17.29</td>\n",
       "      <td>24.68</td>\n",
       "      <td>4.94</td>\n",
       "      <td>...</td>\n",
       "      <td>13.75</td>\n",
       "      <td>2.15</td>\n",
       "      <td>12.29</td>\n",
       "      <td>700.52</td>\n",
       "      <td>1.38</td>\n",
       "      <td>0.69</td>\n",
       "      <td>5.53</td>\n",
       "      <td>968.0</td>\n",
       "      <td>12.84</td>\n",
       "      <td>3922.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>12.0</td>\n",
       "      <td>13.00</td>\n",
       "      <td>4.92</td>\n",
       "      <td>1.83</td>\n",
       "      <td>0.84</td>\n",
       "      <td>4.26</td>\n",
       "      <td>20.67</td>\n",
       "      <td>17.58</td>\n",
       "      <td>23.64</td>\n",
       "      <td>5.11</td>\n",
       "      <td>...</td>\n",
       "      <td>14.10</td>\n",
       "      <td>2.47</td>\n",
       "      <td>12.60</td>\n",
       "      <td>704.36</td>\n",
       "      <td>1.37</td>\n",
       "      <td>0.69</td>\n",
       "      <td>5.52</td>\n",
       "      <td>968.0</td>\n",
       "      <td>12.87</td>\n",
       "      <td>3922.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>13.0</td>\n",
       "      <td>13.14</td>\n",
       "      <td>4.93</td>\n",
       "      <td>1.84</td>\n",
       "      <td>0.92</td>\n",
       "      <td>4.75</td>\n",
       "      <td>22.21</td>\n",
       "      <td>17.94</td>\n",
       "      <td>23.89</td>\n",
       "      <td>5.21</td>\n",
       "      <td>...</td>\n",
       "      <td>14.55</td>\n",
       "      <td>2.55</td>\n",
       "      <td>12.95</td>\n",
       "      <td>696.34</td>\n",
       "      <td>1.39</td>\n",
       "      <td>0.70</td>\n",
       "      <td>5.54</td>\n",
       "      <td>968.0</td>\n",
       "      <td>13.01</td>\n",
       "      <td>3922.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>13 rows × 82 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    epoch  eval_seen_BLEU1  eval_seen_BLEU2  eval_seen_BLEU3  eval_seen_BLEU4  \\\n",
       "0     1.0             5.85             1.29             0.20             0.07   \n",
       "1     2.0             7.85             1.96             0.39             0.11   \n",
       "2     3.0             9.56             2.67             0.64             0.17   \n",
       "3     4.0            10.66             3.02             0.77             0.29   \n",
       "4     5.0            11.16             3.52             1.02             0.36   \n",
       "5     6.0            11.14             3.40             0.93             0.32   \n",
       "6     7.0            12.02             3.99             1.22             0.53   \n",
       "7     8.0            12.01             4.09             1.28             0.58   \n",
       "8     9.0            12.62             4.50             1.53             0.71   \n",
       "9    10.0            12.92             4.65             1.54             0.71   \n",
       "10   11.0            12.55             4.56             1.65             0.79   \n",
       "11   12.0            13.00             4.92             1.83             0.84   \n",
       "12   13.0            13.14             4.93             1.84             0.92   \n",
       "\n",
       "    eval_seen_Distinct-1  eval_seen_Distinct-2  eval_seen_F1  \\\n",
       "0                   0.12                  0.45          9.40   \n",
       "1                   0.35                  1.77         11.70   \n",
       "2                   0.58                  3.31         13.79   \n",
       "3                   0.99                  5.50         15.36   \n",
       "4                   1.30                  7.16         15.15   \n",
       "5                   2.03                 10.85         16.13   \n",
       "6                   2.42                 13.03         16.79   \n",
       "7                   2.85                 15.22         16.96   \n",
       "8                   3.04                 15.59         17.25   \n",
       "9                   3.51                 17.37         17.44   \n",
       "10                  3.97                 19.70         17.29   \n",
       "11                  4.26                 20.67         17.58   \n",
       "12                  4.75                 22.21         17.94   \n",
       "\n",
       "    eval_seen_know_acc  eval_seen_knowledge_loss  ...  test_unseen_rouge1  \\\n",
       "0                25.54                      3.88  ...                9.04   \n",
       "1                27.01                      3.79  ...               10.41   \n",
       "2                27.11                      3.81  ...               11.98   \n",
       "3                27.19                      3.84  ...               13.03   \n",
       "4                26.73                      3.99  ...               12.87   \n",
       "5                26.20                      4.17  ...               13.36   \n",
       "6                25.82                      4.37  ...               13.69   \n",
       "7                25.26                      4.48  ...               13.27   \n",
       "8                25.26                      4.53  ...               14.40   \n",
       "9                24.73                      4.87  ...               13.76   \n",
       "10               24.68                      4.94  ...               13.75   \n",
       "11               23.64                      5.11  ...               14.10   \n",
       "12               23.89                      5.21  ...               14.55   \n",
       "\n",
       "    test_unseen_rouge2  test_unseen_rougeL  test_unseen_runtime  \\\n",
       "0                 1.21                8.49               714.87   \n",
       "1                 1.71                9.67               734.16   \n",
       "2                 2.06               11.01               725.42   \n",
       "3                 1.93               11.73               705.41   \n",
       "4                 2.00               11.52               718.65   \n",
       "5                 1.99               12.05               688.66   \n",
       "6                 2.12               12.24               703.56   \n",
       "7                 2.16               11.94               684.93   \n",
       "8                 2.35               12.79               701.46   \n",
       "9                 2.18               12.36               724.59   \n",
       "10                2.15               12.29               700.52   \n",
       "11                2.47               12.60               704.36   \n",
       "12                2.55               12.95               696.34   \n",
       "\n",
       "    test_unseen_samples_per_second  test_unseen_steps_per_second  \\\n",
       "0                             1.35                          0.68   \n",
       "1                             1.32                          0.66   \n",
       "2                             1.33                          0.67   \n",
       "3                             1.37                          0.69   \n",
       "4                             1.35                          0.67   \n",
       "5                             1.41                          0.70   \n",
       "6                             1.38                          0.69   \n",
       "7                             1.41                          0.71   \n",
       "8                             1.38                          0.69   \n",
       "9                             1.34                          0.67   \n",
       "10                            1.38                          0.69   \n",
       "11                            1.37                          0.69   \n",
       "12                            1.39                          0.70   \n",
       "\n",
       "    test_unseen_token_loss  test_unseen_total_episode_num  \\\n",
       "0                     6.16                          968.0   \n",
       "1                     5.88                          968.0   \n",
       "2                     5.72                          968.0   \n",
       "3                     5.62                          968.0   \n",
       "4                     5.55                          968.0   \n",
       "5                     5.53                          968.0   \n",
       "6                     5.53                          968.0   \n",
       "7                     5.53                          968.0   \n",
       "8                     5.47                          968.0   \n",
       "9                     5.54                          968.0   \n",
       "10                    5.53                          968.0   \n",
       "11                    5.52                          968.0   \n",
       "12                    5.54                          968.0   \n",
       "\n",
       "    test_unseen_total_loss  test_unseen_total_sentence_num  \n",
       "0                    11.01                          3922.0  \n",
       "1                    11.06                          3922.0  \n",
       "2                    11.04                          3922.0  \n",
       "3                    11.13                          3922.0  \n",
       "4                    11.27                          3922.0  \n",
       "5                    11.59                          3922.0  \n",
       "6                    12.05                          3922.0  \n",
       "7                    12.26                          3922.0  \n",
       "8                    12.12                          3922.0  \n",
       "9                    12.77                          3922.0  \n",
       "10                   12.84                          3922.0  \n",
       "11                   12.87                          3922.0  \n",
       "12                   13.01                          3922.0  \n",
       "\n",
       "[13 rows x 82 columns]"
      ]
     },
     "execution_count": 129,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "d.apply(lambda x : post_process_metric(x, not_hundred_metrics), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "89e93cdf-b208-4294-8c47-de7cb7ca208e",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9f4fcbb0-d41d-40b3-bf02-8e25c92052d0",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "88bae7f5-13a7-47d8-804e-9ab9335e2225",
   "metadata": {},
   "outputs": [],
   "source": [
    "rounder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "id": "77912d2b-2516-4de2-b381-27e0f35c6582",
   "metadata": {},
   "outputs": [],
   "source": [
    "new_keys = ['epoch', 'step']\n",
    "for m in metrics:\n",
    "    new_keys.append('eval_seen_'+m)\n",
    "    new_keys.append('eval_unseen_'+m)\n",
    "    new_keys.append('test_seen_'+m)\n",
    "    new_keys.append('test_unseen_'+m)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "id": "b8eb62fc-5b20-48cf-95f1-c34effa4b766",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0     352.583104\n",
       "1     225.446893\n",
       "2     170.018567\n",
       "3     139.856059\n",
       "4     122.500312\n",
       "5     112.119942\n",
       "6     103.284017\n",
       "7      97.636196\n",
       "8      91.913577\n",
       "9      91.955135\n",
       "10     88.348304\n",
       "11     88.151702\n",
       "12     87.734454\n",
       "Name: eval_seen_ppl, dtype: float64"
      ]
     },
     "execution_count": 119,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "d[new_keys]['eval_seen_ppl']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c97df438-03a9-4be7-9dcb-41f6089d8939",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}