squad_huggingface_experiment_with_Trainer_TPU.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "squad huggingface experiment with Trainer / TPU",
      "provenance": [],
      "collapsed_sections": [],
      "toc_visible": true,
      "authorship_tag": "ABX9TyP9WeylDTSkd3HplFN1kpd4",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "8602662e6a4746ffa8806f598f778bfe": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_639c4ab8ec4b4cdc9ca1bcd1c3d87486",
              "IPY_MODEL_98bb1ef6391045d6a3899b936ae794b1"
            ],
            "layout": "IPY_MODEL_396a0c99a36b47a7b4bad3a18bcb3197"
          }
        },
        "639c4ab8ec4b4cdc9ca1bcd1c3d87486": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "Epoch: 100%",
            "description_tooltip": null,
            "layout": "IPY_MODEL_9c4706a6eff341f2a0665826819dd016",
            "max": 3,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_3cd6c2f25dd74d5595bdff24345259a9",
            "value": 3
          }
        },
        "98bb1ef6391045d6a3899b936ae794b1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_8a81c36807a94b27b604bef11df924f6",
            "placeholder": "​",
            "style": "IPY_MODEL_6271c2ff48e94521854d31adcd1708ca",
            "value": " 3/3 [2:07:33&lt;00:00, 2551.29s/it]"
          }
        },
        "396a0c99a36b47a7b4bad3a18bcb3197": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "9c4706a6eff341f2a0665826819dd016": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "3cd6c2f25dd74d5595bdff24345259a9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": "initial"
          }
        },
        "8a81c36807a94b27b604bef11df924f6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "6271c2ff48e94521854d31adcd1708ca": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "ef3b2b8e6cbc4e5da17dcafe6f22b247": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_97078c3f52d745f3bd791728fd3fac06",
              "IPY_MODEL_d2b8ed2982c2484a86ce904ad71f2301"
            ],
            "layout": "IPY_MODEL_fa60acccced543daa8ffcf62bb8dea08"
          }
        },
        "97078c3f52d745f3bd791728fd3fac06": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "Iteration: 100%",
            "description_tooltip": null,
            "layout": "IPY_MODEL_181b91946e284f2f95bcca13e9d594d2",
            "max": 1473,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_704cc452a05b415e88a353222b44f49d",
            "value": 1473
          }
        },
        "d2b8ed2982c2484a86ce904ad71f2301": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_43479ab8eab44eb4b1f0c6163b78f99a",
            "placeholder": "​",
            "style": "IPY_MODEL_3c5514e0d5144031a47a2d767eb7e8f1",
            "value": " 1473/1473 [2:07:33&lt;00:00,  5.20s/it]"
          }
        },
        "fa60acccced543daa8ffcf62bb8dea08": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "181b91946e284f2f95bcca13e9d594d2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "704cc452a05b415e88a353222b44f49d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": "initial"
          }
        },
        "43479ab8eab44eb4b1f0c6163b78f99a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "3c5514e0d5144031a47a2d767eb7e8f1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "03ac45eeea894b1683b5215a824577e4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_c95e3307f3184b34bda83487d4fa8ad7",
              "IPY_MODEL_4f0ae3c9c29d4c7fb0114fb26e9e408a"
            ],
            "layout": "IPY_MODEL_7307b235fbbc46ebbddb4297c6b5c3d5"
          }
        },
        "c95e3307f3184b34bda83487d4fa8ad7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "Iteration: 100%",
            "description_tooltip": null,
            "layout": "IPY_MODEL_5521d35d381f43f9bd09a86eff22aa92",
            "max": 1473,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_7e263b6143a343febb94e9b5159d8861",
            "value": 1473
          }
        },
        "4f0ae3c9c29d4c7fb0114fb26e9e408a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_7daa4e54b39e4efc94c42684439da21a",
            "placeholder": "​",
            "style": "IPY_MODEL_1ddf2773f8e54d89946209719fe4200d",
            "value": " 1473/1473 [1:25:46&lt;00:00,  3.49s/it]"
          }
        },
        "7307b235fbbc46ebbddb4297c6b5c3d5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "5521d35d381f43f9bd09a86eff22aa92": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "7e263b6143a343febb94e9b5159d8861": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": "initial"
          }
        },
        "7daa4e54b39e4efc94c42684439da21a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "1ddf2773f8e54d89946209719fe4200d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "182d2dbccbcc41c592660f0a95704a82": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_7c5a95be09514e71a6151f5d7b4cd940",
              "IPY_MODEL_a164153b960c4064bfb31dcb19f26c18"
            ],
            "layout": "IPY_MODEL_b0354c28ad9340cea8e3f75074ad9f1f"
          }
        },
        "7c5a95be09514e71a6151f5d7b4cd940": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "Iteration: 100%",
            "description_tooltip": null,
            "layout": "IPY_MODEL_e5ac4d6688e14a35a166f8a36cb10746",
            "max": 1473,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_9fe16545f9ef4c109e4e73ab2c8cfdb2",
            "value": 1473
          }
        },
        "a164153b960c4064bfb31dcb19f26c18": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_db268c5c8f734f199b839e3192c2a1f1",
            "placeholder": "​",
            "style": "IPY_MODEL_54c7770c9e1d4dcca3769019e234f26c",
            "value": " 1473/1473 [44:06&lt;00:00,  1.80s/it]"
          }
        },
        "b0354c28ad9340cea8e3f75074ad9f1f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e5ac4d6688e14a35a166f8a36cb10746": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "9fe16545f9ef4c109e4e73ab2c8cfdb2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": "initial"
          }
        },
        "db268c5c8f734f199b839e3192c2a1f1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "54c7770c9e1d4dcca3769019e234f26c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "020fe134868042a3a87276555a8be882": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_34e40d10ba95405db7114080946ca83e",
              "IPY_MODEL_71601cd26c2c414ab748c6219fad3301"
            ],
            "layout": "IPY_MODEL_d57cb24343b64006983ee7747b34037e"
          }
        },
        "34e40d10ba95405db7114080946ca83e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "Evaluating: 100%",
            "description_tooltip": null,
            "layout": "IPY_MODEL_20a54f8bf0ad4601a6ed454dff553c90",
            "max": 833,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_80152d8f3cca4a789516e65685fc0e00",
            "value": 833
          }
        },
        "71601cd26c2c414ab748c6219fad3301": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_b01ef7910deb40f3bbccd723d148816c",
            "placeholder": "​",
            "style": "IPY_MODEL_8842bc486d9c4a088b45e69540afc441",
            "value": " 833/833 [02:28&lt;00:00,  5.62it/s]"
          }
        },
        "d57cb24343b64006983ee7747b34037e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "20a54f8bf0ad4601a6ed454dff553c90": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "80152d8f3cca4a789516e65685fc0e00": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": "initial"
          }
        },
        "b01ef7910deb40f3bbccd723d148816c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "8842bc486d9c4a088b45e69540afc441": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "eb143cd09f8d499a906f7f1445c89490": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_b86f75683b0941c8a1bfc2ddf0a23f91",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_02bb4bff0df94b5fbe1f330bfd7da658",
              "IPY_MODEL_a5b676acb5ff47fba258494b84dc3e24"
            ]
          }
        },
        "b86f75683b0941c8a1bfc2ddf0a23f91": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "02bb4bff0df94b5fbe1f330bfd7da658": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_8f2600ab266a4db5bf1f16ae16906360",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 442,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 442,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_43e35e5df12d441a82228353818a90cc"
          }
        },
        "a5b676acb5ff47fba258494b84dc3e24": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_bea0fbcc65234b13973273509dcc436e",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 442/442 [00:00&lt;00:00, 827B/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_ae26f3c488484454854992cafb18b0d3"
          }
        },
        "8f2600ab266a4db5bf1f16ae16906360": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "43e35e5df12d441a82228353818a90cc": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "bea0fbcc65234b13973273509dcc436e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "ae26f3c488484454854992cafb18b0d3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "077013e2538247a19c777baf28d2d622": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_3b2d332b84244f98991d126a5761a6fd",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_33dd4e91fd9c4a2897f6641880c797dd",
              "IPY_MODEL_341822c58bc542d596126a2df8fc4843"
            ]
          }
        },
        "3b2d332b84244f98991d126a5761a6fd": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "33dd4e91fd9c4a2897f6641880c797dd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_a92b2164f6f143089a2bdff8f6b8abf5",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 267967963,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 267967963,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_8f91c018472743ffa0a15d8f97c6d18d"
          }
        },
        "341822c58bc542d596126a2df8fc4843": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_63db2c5931104986a36696d34f3850f7",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 268M/268M [00:05&lt;00:00, 50.5MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_4a92e09d8f894868abb4653d2e1f9246"
          }
        },
        "a92b2164f6f143089a2bdff8f6b8abf5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "8f91c018472743ffa0a15d8f97c6d18d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "63db2c5931104986a36696d34f3850f7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "4a92e09d8f894868abb4653d2e1f9246": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/manishiitg/ML_Experiments/blob/master/squad_huggingface_experiment_with_Trainer_TPU.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "d-fafWWx8YUz",
        "colab_type": "code",
        "outputId": "cddc75ec-5aff-4bad-8f39-433cb51ad797",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 372
        }
      },
      "source": [
        "!nvidia-smi"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Sun Jun 14 06:27:17 2020       \r\n",
            "+-----------------------------------------------------------------------------+\r\n",
            "| NVIDIA-SMI 418.87.01    Driver Version: 418.87.01    CUDA Version: 10.1     |\r\n",
            "|-------------------------------+----------------------+----------------------+\r\n",
            "| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |\r\n",
            "| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |\r\n",
            "|===============================+======================+======================|\r\n",
            "|   0  Tesla T4            Off  | 00000000:00:04.0 Off |                    0 |\r\n",
            "| N/A   69C    P0    32W /  70W |      0MiB / 15079MiB |      0%      Default |\r\n",
            "+-------------------------------+----------------------+----------------------+\n",
            "|   1  Tesla T4            Off  | 00000000:00:05.0 Off |                    0 |\n",
            "| N/A   72C    P0    34W /  70W |      0MiB / 15079MiB |      0%      Default |\n",
            "+-------------------------------+----------------------+----------------------+\n",
            "                                                                               \n",
            "+-----------------------------------------------------------------------------+\n",
            "| Processes:                                                       GPU Memory |\n",
            "|  GPU       PID   Type   Process name                             Usage      |\n",
            "|=============================================================================|\n",
            "|  No running processes found                                                 |\n",
            "+-----------------------------------------------------------------------------+\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "krs17ANQChyY",
        "colab_type": "code",
        "outputId": "5c504b04-2abb-4909-ea84-5f727acae970",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 730
        }
      },
      "source": [
        "!git clone https://github.com/huggingface/transformers.git\n",
        "!pip install -U ./transformers"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Cloning into 'transformers'...\n",
            "remote: Enumerating objects: 29234, done.\u001b[K\n",
            "remote: Total 29234 (delta 0), reused 0 (delta 0), pack-reused 29234\u001b[K\n",
            "Receiving objects: 100% (29234/29234), 26.53 MiB | 36.52 MiB/s, done.\n",
            "Resolving deltas: 100% (20256/20256), done.\n",
            "Processing ./transformers\n",
            "Requirement already satisfied, skipping upgrade: numpy in /opt/conda/lib/python3.7/site-packages (from transformers==2.11.0) (1.18.1)\n",
            "Collecting tokenizers==0.7.0\n",
            "  Downloading tokenizers-0.7.0-cp37-cp37m-manylinux1_x86_64.whl (5.6 MB)\n",
            "\u001b[K     |████████████████████████████████| 5.6 MB 3.0 MB/s \n",
            "\u001b[?25hRequirement already satisfied, skipping upgrade: packaging in /opt/conda/lib/python3.7/site-packages (from transformers==2.11.0) (20.4)\n",
            "Requirement already satisfied, skipping upgrade: filelock in /opt/conda/lib/python3.7/site-packages (from transformers==2.11.0) (3.0.12)\n",
            "Requirement already satisfied, skipping upgrade: requests in /opt/conda/lib/python3.7/site-packages (from transformers==2.11.0) (2.23.0)\n",
            "Requirement already satisfied, skipping upgrade: tqdm>=4.27 in /opt/conda/lib/python3.7/site-packages (from transformers==2.11.0) (4.46.1)\n",
            "Requirement already satisfied, skipping upgrade: regex!=2019.12.17 in /opt/conda/lib/python3.7/site-packages (from transformers==2.11.0) (2019.8.19)\n",
            "Collecting sentencepiece\n",
            "  Downloading sentencepiece-0.1.91-cp37-cp37m-manylinux1_x86_64.whl (1.1 MB)\n",
            "\u001b[K     |████████████████████████████████| 1.1 MB 22.3 MB/s \n",
            "\u001b[?25hCollecting sacremoses\n",
            "  Downloading sacremoses-0.0.43.tar.gz (883 kB)\n",
            "\u001b[K     |████████████████████████████████| 883 kB 23.8 MB/s \n",
            "\u001b[?25hRequirement already satisfied, skipping upgrade: six in /opt/conda/lib/python3.7/site-packages (from packaging->transformers==2.11.0) (1.15.0)\n",
            "Requirement already satisfied, skipping upgrade: pyparsing>=2.0.2 in /opt/conda/lib/python3.7/site-packages (from packaging->transformers==2.11.0) (2.4.7)\n",
            "Requirement already satisfied, skipping upgrade: idna<3,>=2.5 in /opt/conda/lib/python3.7/site-packages (from requests->transformers==2.11.0) (2.9)\n",
            "Requirement already satisfied, skipping upgrade: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /opt/conda/lib/python3.7/site-packages (from requests->transformers==2.11.0) (1.25.9)\n",
            "Requirement already satisfied, skipping upgrade: chardet<4,>=3.0.2 in /opt/conda/lib/python3.7/site-packages (from requests->transformers==2.11.0) (3.0.4)\n",
            "Requirement already satisfied, skipping upgrade: certifi>=2017.4.17 in /opt/conda/lib/python3.7/site-packages (from requests->transformers==2.11.0) (2020.4.5.1)\n",
            "Requirement already satisfied, skipping upgrade: click in /opt/conda/lib/python3.7/site-packages (from sacremoses->transformers==2.11.0) (7.1.2)\n",
            "Requirement already satisfied, skipping upgrade: joblib in /opt/conda/lib/python3.7/site-packages (from sacremoses->transformers==2.11.0) (0.15.1)\n",
            "Building wheels for collected packages: transformers, sacremoses\n",
            "  Building wheel for transformers (setup.py) ... \u001b[?25l-\b \b\\\b \b|\b \bdone\n",
            "\u001b[?25h  Created wheel for transformers: filename=transformers-2.11.0-py3-none-any.whl size=688546 sha256=70176578663bfb087387b7225bf30896ad88f7b1432bcf0f6ef44aecf0e917fd\n",
            "  Stored in directory: /tmp/pip-ephem-wheel-cache-zy2ctdd4/wheels/59/62/2f/3bb155e7afddd1a07de7f4cba65229c4ca1cd8367d38098280\n",
            "  Building wheel for sacremoses (setup.py) ... \u001b[?25l-\b \b\\\b \bdone\n",
            "\u001b[?25h  Created wheel for sacremoses: filename=sacremoses-0.0.43-py3-none-any.whl size=893259 sha256=8802d5ce3a487da0094644762bd803a313c82c91492f6615a1b3b0c0201c50a6\n",
            "  Stored in directory: /home/jupyter/.cache/pip/wheels/69/09/d1/bf058f7d6fa0ecba2ce7c66be3b8d012beb4bf61a6e0c101c0\n",
            "Successfully built transformers sacremoses\n",
            "Installing collected packages: tokenizers, sentencepiece, sacremoses, transformers\n",
            "Successfully installed sacremoses-0.0.43 sentencepiece-0.1.91 tokenizers-0.7.0 transformers-2.11.0\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "R-H--ge54jmZ",
        "colab_type": "text"
      },
      "source": [
        "# Download SQUAD v2 Dataset"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "x9Wgemvc-ESj",
        "colab_type": "code",
        "outputId": "c28a2026-ffad-41ef-ccaf-b8cc3a94eece",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 408
        }
      },
      "source": [
        "!wget https://rajpurkar.github.io/SQuAD-explorer/dataset/train-v2.0.json\n",
        "!wget https://rajpurkar.github.io/SQuAD-explorer/dataset/dev-v2.0.json"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "--2020-06-14 05:50:51--  https://rajpurkar.github.io/SQuAD-explorer/dataset/train-v2.0.json\r\n",
            "Resolving rajpurkar.github.io (rajpurkar.github.io)... 185.199.111.153, 185.199.108.153, 185.199.110.153, ...\n",
            "Connecting to rajpurkar.github.io (rajpurkar.github.io)|185.199.111.153|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 42123633 (40M) [application/json]\n",
            "Saving to: ‘train-v2.0.json’\n",
            "\n",
            "train-v2.0.json     100%[===================>]  40.17M  18.8MB/s    in 2.1s    \n",
            "\n",
            "2020-06-14 05:50:54 (18.8 MB/s) - ‘train-v2.0.json’ saved [42123633/42123633]\n",
            "\n",
            "--2020-06-14 05:50:54--  https://rajpurkar.github.io/SQuAD-explorer/dataset/dev-v2.0.json\n",
            "Resolving rajpurkar.github.io (rajpurkar.github.io)... 185.199.111.153, 185.199.108.153, 185.199.110.153, ...\n",
            "Connecting to rajpurkar.github.io (rajpurkar.github.io)|185.199.111.153|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 4370528 (4.2M) [application/json]\n",
            "Saving to: ‘dev-v2.0.json’\n",
            "\n",
            "dev-v2.0.json       100%[===================>]   4.17M  6.53MB/s    in 0.6s    \n",
            "\n",
            "2020-06-14 05:50:54 (6.53 MB/s) - ‘dev-v2.0.json’ saved [4370528/4370528]\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ztAOwL_E63OD",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import dataclasses\n",
        "import logging\n",
        "import os\n",
        "import sys\n",
        "from dataclasses import dataclass, field\n",
        "from typing import Dict, Optional\n",
        "\n",
        "import numpy as np\n",
        "\n",
        "from transformers import (\n",
        "    HfArgumentParser,\n",
        "    Trainer,\n",
        "    TrainingArguments,\n",
        "    glue_compute_metrics,\n",
        "    glue_output_modes,\n",
        "    glue_tasks_num_labels,\n",
        "    set_seed,\n",
        ")\n",
        "\n",
        "\n",
        "from transformers import (\n",
        "    MODEL_FOR_QUESTION_ANSWERING_MAPPING,\n",
        "    WEIGHTS_NAME,\n",
        "    AdamW,\n",
        "    AutoConfig,\n",
        "    AutoModelForQuestionAnswering,\n",
        "    AutoTokenizer,\n",
        "    get_linear_schedule_with_warmup,\n",
        "    squad_convert_examples_to_features,\n",
        "    PreTrainedTokenizer,\n",
        "    DataCollator\n",
        ")\n",
        "from transformers.data.metrics.squad_metrics import (\n",
        "    compute_predictions_log_probs,\n",
        "    compute_predictions_logits,\n",
        "    squad_evaluate,\n",
        ")\n",
        "from transformers.data.processors.squad import SquadResult, SquadV1Processor, SquadV2Processor, SquadExample, SquadFeatures\n",
        "\n",
        "from torch.utils.data.dataset import Dataset\n",
        "from typing import List, Optional, Union\n",
        "from enum import Enum\n",
        "from filelock import FileLock\n",
        "import time\n",
        "import torch\n",
        "\n",
        "from torch.utils.data import DataLoader, RandomSampler, SequentialSampler\n",
        "from tqdm.auto import tqdm\n",
        "import timeit\n",
        "\n",
        "\n",
        "class Split(Enum):\n",
        "    train = \"train\"\n",
        "    dev = \"dev\"\n",
        "    test = \"test\"\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rFyWJbp87Rf5",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "logger = logging.getLogger(__name__)\n",
        "\n",
        "@dataclass\n",
        "class ModelArguments:\n",
        "    \"\"\"\n",
        "    Arguments pertaining to which model/config/tokenizer we are going to fine-tune from.\n",
        "    \"\"\"\n",
        "    model_type: str = field(\n",
        "        metadata={\"help\": \"type of model\"}\n",
        "    )\n",
        "    model_name_or_path: str = field(\n",
        "        metadata={\"help\": \"Path to pretrained model or model identifier from huggingface.co/models\"}\n",
        "    )\n",
        "    config_name: Optional[str] = field(\n",
        "        default=None, metadata={\"help\": \"Pretrained config name or path if not the same as model_name\"}\n",
        "    )\n",
        "    tokenizer_name: Optional[str] = field(\n",
        "        default=None, metadata={\"help\": \"Pretrained tokenizer name or path if not the same as model_name\"}\n",
        "    )\n",
        "    cache_dir: Optional[str] = field(\n",
        "        default=None, metadata={\"help\": \"Where do you want to store the pretrained models downloaded from s3\"}\n",
        "    )\n",
        "    model_checkpoint: Optional[str] = field(\n",
        "        default=None, metadata={\"help\": \"Which checkout do you want to resume check point\"}\n",
        "    )\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "QrP442h54rDe",
        "colab_type": "text"
      },
      "source": [
        "# Copy your check points to google drive.\n",
        "\n",
        "Useful if your training stops in between in colab and you need to resume again"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RDuSwUNp7A7e",
        "colab_type": "code",
        "cellView": "both",
        "colab": {}
      },
      "source": [
        "# !rm -rf cache\n",
        "# !rm -rf output\n",
        "\n",
        "# from google.colab import drive\n",
        "# drive.mount('/content/drive')\n",
        "\n",
        "# !cp -rf output/distilbert/distilbert-base-uncased /content/drive/My\\ Drive/\n",
        "\n",
        "# !cp -rf output/distilbert//content/output/distilbert/distilbert-base-uncased/checkpoint-52500/* /content/drive/My\\ Drive/distilbert-base-uncased\n",
        "\n",
        "# !mkdir output\n",
        "# !mkdir output/distilbert\n",
        "# !cp -rf /content/drive/My\\ Drive/distilbert-base-uncased output/distilbert/"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "tVQO4hJeAXGg",
        "colab_type": "text"
      },
      "source": [
        "# Define your training configuration"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iGOUVOpM7W5R",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# model_name = 'bert'\n",
        "# model_name_or_path = 'bert-base-uncased'\n",
        "\n",
        "# model_name = 'distilbert'\n",
        "# model_name_or_path = 'distilbert-base-uncased' \n",
        "\n",
        "model_name = 'roberta'\n",
        "model_name_or_path = 'distilroberta-base'\n",
        "\n",
        "model_checkpoint = './output/' + model_name + '/' + model_name_or_path + '/checkpoint-52500'\n",
        "\n",
        "\n",
        "base_dir = \"./drive/\" # on google gcp instance else set this to \"./\" on colab\n",
        "\n",
        "config = {\n",
        "    \"model_type\" : model_name,\n",
        "    \"model_name_or_path\" : model_name_or_path,\n",
        "    \"output_dir\": base_dir + \"output/\" + model_name + \"/\" + model_name_or_path,\n",
        "    \"model_checkpoint\" : base_dir + model_checkpoint,\n",
        "    \"do_train\" : True,\n",
        "    \"do_eval\" : True,\n",
        "    \"do_predict\" : False,\n",
        "    \"data_dir\" : \"\",\n",
        "    \"overwrite_output_dir\" : True,\n",
        "    \"overwrite_cache\" : False,\n",
        "    \"cache_dir\" : base_dir + \"cache\",\n",
        "    \"limit_length\": None,\n",
        "    \"max_seq_length\": 328,\n",
        "    \"doc_stride\" : 128,\n",
        "    \"train_batch_size\" : 8,\n",
        "    \"per_device_train_batch_size\" : 48,\n",
        "    \"save_steps\": 5000,\n",
        "    \"save_total_limit\": 2,\n",
        "}\n",
        "\n",
        "import json\n",
        "with open('args.json', 'w') as f:\n",
        "    json.dump(config, f)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "IdeLeWuQCiSd",
        "colab_type": "text"
      },
      "source": [
        "# Data Training Arguments"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_hiSgMk_GKiZ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "@dataclass\n",
        "class SquadDataTrainingArguments:\n",
        "    \"\"\"\n",
        "    Arguments pertaining to what data we are going to input our model for training and eval.\n",
        "    Using `HfArgumentParser` we can turn this class\n",
        "    into argparse arguments to be able to specify them on\n",
        "    the command line.\n",
        "    \"\"\"\n",
        "\n",
        "\n",
        "    data_dir: str = field(\n",
        "        metadata={\"help\": \"The input data dir. Should contain the .tsv files (or other data files) for the task.\"}\n",
        "    )\n",
        "    max_seq_length: int = field(\n",
        "        default=328,\n",
        "        metadata={\n",
        "            \"help\": \"The maximum total input sequence length after tokenization. Sequences longer \"\n",
        "            \"than this will be truncated, sequences shorter will be padded.\"\n",
        "        },\n",
        "    )\n",
        "    doc_stride: int = field(\n",
        "        default=128,\n",
        "        metadata={\n",
        "            \"help\": \"When splitting up a long document into chunks, how much stride to take between chunks.\"\n",
        "        },\n",
        "    )\n",
        "    max_query_length: int = field(\n",
        "        default=64,\n",
        "        metadata={\n",
        "            \"help\": \"The maximum number of tokens for the question. Questions longer than this will \"\n",
        "        \"be truncated to this length.\"\n",
        "        },\n",
        "    )\n",
        "    overwrite_cache: bool = field(\n",
        "        default=False, metadata={\"help\": \"Overwrite the cached training and evaluation sets.\"}\n",
        "    )\n",
        "    threads: int = field(\n",
        "        default=1, metadata={\"help\": \"multiple threads for converting example to features.\"}\n",
        "    )\n",
        "    version_2_with_negative : bool = field(\n",
        "        default=True, metadata={\"help\": \"If true, the SQuAD examples contain some that do not have an answer.\"}\n",
        "    )\n",
        "    train_file: str = field(\n",
        "        default='train-v2.0.json',\n",
        "        metadata={\"help\": \"name of training file\"}\n",
        "    )\n",
        "    predict_file: str = field(\n",
        "        default='dev-v2.0.json',\n",
        "        metadata={\"help\": \"name of dev file\"}\n",
        "    )\n",
        "    do_lower_case: bool = field(\n",
        "        default=False, metadata={\"help\": \"do lower case\"}\n",
        "    )\n",
        "    limit_length: int = field(\n",
        "        default=None, metadata={\"help\": \"length of dataset to process optional\"}\n",
        "    )\n",
        "    n_best_size: int = field(\n",
        "        default=20,  metadata={\"help\": \"The total number of n-best predictions to generate in the nbest_predictions.json output file.\"}\n",
        "    )\n",
        "    max_answer_length:int = field(\n",
        "        default=30, metadata={\"help\": \"The maximum length of an answer that can be generated. This is needed because the start \"\n",
        "        \"and end predictions are not conditioned on one another.\"}\n",
        "    )\n",
        "    verbose_logging: bool = field(\n",
        "        default=False, metadata={\"help\": \"If true, all of the warnings related to data processing will be printed.\"}\n",
        "    )\n",
        "    lang_id:int = field(\n",
        "        default=0, metadata={\"help\": \"language id of input for language-specific xlm models (see tokenization_xlm.PRETRAINED_INIT_CONFIGURATION)\"}\n",
        "    )\n",
        "    null_score_diff_threshold: float = field(\n",
        "        default=0.0, metadata={\"help\": \"If null_score - best_non_null is greater than the threshold predict null.\"}\n",
        "    )\n",
        "\n",
        "    def __post_init__(self):\n",
        "        self.task_name = \"squad\""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "piFn_xRCCl_b",
        "colab_type": "text"
      },
      "source": [
        "# Squad Custom Dataset"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WElcx5gNHLUL",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from torch.utils.data import TensorDataset\n",
        "\n",
        "class SquadCustomDataset(Dataset):\n",
        "    \"\"\"\n",
        "    This will be superseded by a framework-agnostic approach\n",
        "    soon.\n",
        "    \"\"\"\n",
        "\n",
        "    args: SquadDataTrainingArguments\n",
        "    output_mode: str\n",
        "    features: List[SquadFeatures]\n",
        "    examples: List[SquadExample]\n",
        "    dataset: TensorDataset\n",
        "\n",
        "    def __init__(\n",
        "        self,\n",
        "        args: SquadDataTrainingArguments,\n",
        "        tokenizer: PreTrainedTokenizer,\n",
        "        limit_length: Optional[int] = None,\n",
        "        mode: Union[str, Split] = Split.train,\n",
        "    ):\n",
        "        self.args = args\n",
        "        self.processor = SquadV2Processor() if args.version_2_with_negative else SquadV1Processor()\n",
        "        self.output_mode = \"\" #need to see for this\n",
        "        if isinstance(mode, str):\n",
        "            try:\n",
        "                mode = Split[mode]\n",
        "            except KeyError:\n",
        "                raise KeyError(\"mode is not a valid split name\")\n",
        "        # Load data features from cache or dataset file\n",
        "        cached_features_file = os.path.join(\n",
        "            args.data_dir,\n",
        "            \"cached_{}_{}_{}_{}_{}\".format(\n",
        "                mode.value, tokenizer.__class__.__name__, str(args.max_seq_length), args.task_name,limit_length\n",
        "            ),\n",
        "        )\n",
        "        \n",
        "        # Make sure only the first process in distributed training processes the dataset,\n",
        "        # and the others will use the cache.\n",
        "        lock_path = cached_features_file + \".lock\"\n",
        "        with FileLock(lock_path):\n",
        "\n",
        "            if os.path.exists(cached_features_file) and not args.overwrite_cache:\n",
        "                start = time.time()\n",
        "                obj = torch.load(cached_features_file)\n",
        "                self.features = obj[\"features\"]\n",
        "                self.examples = obj[\"examples\"]\n",
        "                # self.dataset = obj[\"dataset\"]\n",
        "\n",
        "\n",
        "                logger.info(\n",
        "                    f\"Loading features from cached file {cached_features_file} [took %.3f s]\", time.time() - start\n",
        "                )\n",
        "            else:\n",
        "                logger.info(f\"Creating features from dataset file at {args.data_dir}\")\n",
        "\n",
        "                if mode == Split.dev:\n",
        "                    examples = self.processor.get_dev_examples(args.data_dir, filename=args.predict_file)\n",
        "                elif mode == Split.test:\n",
        "                    examples = self.processor.get_test_examples(args.data_dir, filename=args.predict_file)\n",
        "                else:\n",
        "                    examples = self.processor.get_train_examples(args.data_dir, filename=args.train_file)\n",
        "\n",
        "                if limit_length is not None:\n",
        "                    logger.info(\n",
        "                        \"limit data to length %s \", limit_length\n",
        "                    )\n",
        "                    examples = examples[:limit_length]\n",
        "\n",
        "                self.examples = examples\n",
        "\n",
        "                self.features, self.dataset = squad_convert_examples_to_features(\n",
        "                    examples=examples,\n",
        "                    tokenizer=tokenizer,\n",
        "                    max_seq_length=args.max_seq_length,\n",
        "                    doc_stride=args.doc_stride,\n",
        "                    max_query_length=args.max_query_length,\n",
        "                    is_training= mode == Split.test,\n",
        "                    return_dataset=\"pt\",\n",
        "                    threads=args.threads,\n",
        "                )\n",
        "                \n",
        "                start = time.time()\n",
        "                torch.save({\n",
        "                    \"examples\" : self.examples,\n",
        "                    \"features\" : self.features,\n",
        "                    # \"dataset\" : self.dataset\n",
        "                }, cached_features_file)\n",
        "                # ^ This seems to take a lot of time so I want to investigate why and how we can improve.\n",
        "                logger.info(\n",
        "                    \"Saving features into cached file %s [took %.3f s]\", cached_features_file, time.time() - start\n",
        "                )\n",
        "\n",
        "    def __len__(self):\n",
        "        return len(self.features)\n",
        "\n",
        "    def __getitem__(self, i) -> SquadFeatures:\n",
        "        return self.features[i]\n",
        "\n",
        "    def get_features(self):\n",
        "      return self.features\n",
        "\n",
        "    def get_examples(self):\n",
        "      return self.examples"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "K2DxelmNQk11",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from torch import nn\n",
        "\n",
        "# not used\n",
        "@dataclass\n",
        "class PassDataCollator(DataCollator):\n",
        "  def collate_batch(self, batch: List) -> Dict[str, torch.Tensor]:\n",
        "    return batch \n",
        "\n",
        "@dataclass\n",
        "class SquadDataCollator(DataCollator):\n",
        "\n",
        "    model_args: ModelArguments\n",
        "\n",
        "    def __init__(\n",
        "        self,\n",
        "        model_args: ModelArguments):\n",
        "      \n",
        "      self.model_args = model_args\n",
        "\n",
        "\n",
        "    def collate_batch(self, features: List) -> Dict[str, torch.Tensor]:\n",
        "        # taken from https://github.com/huggingface/transformers/blob/5daca95dddf940139d749b1ca42c59ebc5191979/src/transformers/data/processors/squad.py#L325\n",
        "        # and https://github.com/huggingface/transformers/blob/5daca95dddf940139d749b1ca42c59ebc5191979/src/transformers/data/processors/squad.py#L325\n",
        "        # first = features[0]\n",
        "\n",
        "        \n",
        "\n",
        "        # for f in features:\n",
        "        #   logger.warning(\"unique id collator %s\", f.unique_id)\n",
        "        #   logger.warning(\"example_index collator %s\", f.example_index)\n",
        "\n",
        "        unique_ids = torch.tensor([f.unique_id for f in features])\n",
        "\n",
        "        all_input_ids = torch.tensor([f.input_ids for f in features], dtype=torch.long)\n",
        "        all_attention_masks = torch.tensor([f.attention_mask for f in features], dtype=torch.long)\n",
        "        all_token_type_ids = torch.tensor([f.token_type_ids for f in features], dtype=torch.long)\n",
        "        all_cls_index = torch.tensor([f.cls_index for f in features], dtype=torch.long)\n",
        "        all_p_mask = torch.tensor([f.p_mask for f in features], dtype=torch.float)\n",
        "\n",
        "        \n",
        "\n",
        "        # if hasattr(first, \"start_position\") and first.start_position is not None:\n",
        "        all_start_positions = torch.tensor([f.start_position for f in features], dtype=torch.long)\n",
        "        all_end_positions = torch.tensor([f.end_position for f in features], dtype=torch.long)\n",
        "        all_example_index = torch.arange(all_input_ids.size(0), dtype=torch.long)\n",
        "\n",
        "        inputs = {\n",
        "            \"input_ids\" : all_input_ids,\n",
        "            \"attention_mask\" : all_attention_masks,\n",
        "            \"token_type_ids\" : all_token_type_ids,\n",
        "            \"start_positions\" : all_start_positions,\n",
        "            \"end_positions\" : all_end_positions,\n",
        "            \"cls_index\": all_cls_index,\n",
        "            \"p_mask\": all_p_mask,\n",
        "            \"example_index\" : all_example_index,\n",
        "            \"unique_ids\" : unique_ids\n",
        "        }\n",
        "\n",
        "        # if self.model_args.model_type in [\"xlm\", \"roberta\", \"distilbert\", \"camembert\"]:\n",
        "        #     del inputs[\"token_type_ids\"]\n",
        "\n",
        "        # if self.model_args.model_type in [\"xlnet\", \"xlm\"]:\n",
        "        #     # inputs.update({\"cls_index\": batch[5], \"p_mask\": batch[6]})\n",
        "        #     # if args.version_2_with_negative:\n",
        "        #     #     inputs.update({\"is_impossible\": batch[7]})\n",
        "        #     # if hasattr(model, \"config\") and hasattr(model.config, \"lang2id\"):\n",
        "        #     #     inputs.update(\n",
        "        #     #         {\"langs\": (torch.ones(batch[0].shape, dtype=torch.int64) * args.lang_id).to(args.device)}\n",
        "        #     #     )\n",
        "        # else:\n",
        "        #   del inputs[\"cls_index\"]\n",
        "        #   del inputs[\"p_mask\"]\n",
        "\n",
        "        return inputs\n",
        "        \n",
        "class SquadTrainer(Trainer):\n",
        "    model_args: ModelArguments\n",
        "    def __init__(self, model_args, **kwargs):\n",
        "        self.model_args = model_args\n",
        "        super().__init__(**kwargs)\n",
        "\n",
        "    def _training_step(\n",
        "        self, model: nn.Module, inputs: Dict[str, torch.Tensor], optimizer: torch.optim.Optimizer\n",
        "    ) -> float:\n",
        "        model.train()\n",
        "        for k, v in inputs.items():\n",
        "          inputs[k] = v.to(self.args.device)\n",
        "\n",
        "        del inputs[\"unique_ids\"]\n",
        "        del inputs[\"example_index\"]\n",
        "\n",
        "        # this can be handled at dataset level as well. \n",
        "        # no need to extend SquadTrainer\n",
        "\n",
        "        if self.model_args.model_type in [\"xlm\", \"roberta\", \"distilbert\", \"camembert\"]:\n",
        "            del inputs[\"token_type_ids\"]\n",
        "\n",
        "        if self.model_args.model_type in [\"xlnet\", \"xlm\"]:\n",
        "            inputs.update({\"cls_index\": batch[5], \"p_mask\": batch[6]})\n",
        "            if args.version_2_with_negative:\n",
        "                inputs.update({\"is_impossible\": batch[7]})\n",
        "            if hasattr(model, \"config\") and hasattr(model.config, \"lang2id\"):\n",
        "                inputs.update(\n",
        "                    {\"langs\": (torch.ones(batch[0].shape, dtype=torch.int64) * args.lang_id).to(args.device)}\n",
        "                )\n",
        "        else:\n",
        "          del inputs[\"cls_index\"]\n",
        "          del inputs[\"p_mask\"]\n",
        "        \n",
        "\n",
        "        outputs = model(**inputs)\n",
        "        loss = outputs[0]  # model outputs are always tuple in transformers (see doc)\n",
        "\n",
        "        if self.args.n_gpu > 1:\n",
        "            loss = loss.mean()  # mean() to average on multi-gpu parallel training\n",
        "        if self.args.gradient_accumulation_steps > 1:\n",
        "            loss = loss / self.args.gradient_accumulation_steps\n",
        "\n",
        "        if self.args.fp16:\n",
        "            with amp.scale_loss(loss, optimizer) as scaled_loss:\n",
        "                scaled_loss.backward()\n",
        "        else:\n",
        "            loss.backward()\n",
        "\n",
        "        return loss.item()\n",
        "\n",
        "\n",
        "def to_list(tensor):\n",
        "    return tensor.detach().cpu().tolist()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "NtIFV9TVCzTD",
        "colab_type": "text"
      },
      "source": [
        "# main()"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2Hmud2_CJDna",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def main():\n",
        "    # See all possible arguments in src/transformers/training_args.py\n",
        "    # or by passing the --help flag to this script.\n",
        "    # We now keep distinct sets of args, for a cleaner separation of concerns.\n",
        "\n",
        "\n",
        "    parser = HfArgumentParser((ModelArguments, SquadDataTrainingArguments, TrainingArguments))\n",
        "    model_args, data_args, training_args = parser.parse_json_file(json_file=\"args.json\")\n",
        "\n",
        "    if (\n",
        "        os.path.exists(training_args.output_dir)\n",
        "        and os.listdir(training_args.output_dir)\n",
        "        and training_args.do_train\n",
        "        and not training_args.overwrite_output_dir\n",
        "    ):\n",
        "        raise ValueError(\n",
        "            f\"Output directory ({training_args.output_dir}) already exists and is not empty. Use --overwrite_output_dir to overcome.\"\n",
        "        )\n",
        "\n",
        "    # Setup logging\n",
        "    logging.basicConfig(\n",
        "        format=\"%(asctime)s - %(levelname)s - %(name)s -   %(message)s\",\n",
        "        datefmt=\"%m/%d/%Y %H:%M:%S\",\n",
        "        level=logging.INFO if training_args.local_rank in [-1, 0] else logging.WARN,\n",
        "    )\n",
        "    logger.warning(\n",
        "        \"Process rank: %s, device: %s, n_gpu: %s, distributed training: %s, 16-bits training: %s\",\n",
        "        training_args.local_rank,\n",
        "        training_args.device,\n",
        "        training_args.n_gpu,\n",
        "        bool(training_args.local_rank != -1),\n",
        "        training_args.fp16,\n",
        "    )\n",
        "\n",
        "    if data_args.doc_stride >= data_args.max_seq_length - data_args.max_query_length:\n",
        "        logger.warning(\n",
        "            \"WARNING - You've set a doc stride which may be superior to the document length in some \"\n",
        "            \"examples. This could result in errors when building features from the examples. Please reduce the doc \"\n",
        "            \"stride or increase the maximum length to ensure the features are correctly built.\"\n",
        "        )\n",
        "\n",
        "    logger.info(\"Training/evaluation parameters %s\", training_args)\n",
        "\n",
        "    # Set seed\n",
        "    set_seed(training_args.seed)\n",
        "\n",
        "    config = AutoConfig.from_pretrained(\n",
        "        model_args.config_name if model_args.config_name else model_args.model_name_or_path,\n",
        "        cache_dir=model_args.cache_dir if model_args.cache_dir else None,\n",
        "    )\n",
        "    tokenizer = AutoTokenizer.from_pretrained(\n",
        "        model_args.tokenizer_name if model_args.tokenizer_name else model_args.model_name_or_path,\n",
        "        do_lower_case=data_args.do_lower_case,\n",
        "        cache_dir=model_args.cache_dir if model_args.cache_dir else None,\n",
        "        use_fast=False\n",
        "    )\n",
        "    model = AutoModelForQuestionAnswering.from_pretrained(\n",
        "        model_args.model_name_or_path,\n",
        "        from_tf=bool(\".ckpt\" in model_args.model_name_or_path),\n",
        "        config=config,\n",
        "        cache_dir=model_args.cache_dir if model_args.cache_dir else None,\n",
        "    )\n",
        "\n",
        "    # Get datasets\n",
        "    train_dataset = SquadCustomDataset(data_args, tokenizer=tokenizer, limit_length=data_args.limit_length) if training_args.do_train else None\n",
        "    eval_dataset = SquadCustomDataset(data_args, tokenizer=tokenizer, mode=\"dev\", limit_length=data_args.limit_length) if training_args.do_eval else None\n",
        "    \n",
        "    # Initialize our Trainer\n",
        "    trainer = SquadTrainer(\n",
        "        model_args=model_args,\n",
        "        model=model,\n",
        "        args=training_args,\n",
        "        train_dataset=train_dataset,\n",
        "        eval_dataset=eval_dataset,\n",
        "        compute_metrics=None,\n",
        "        data_collator=SquadDataCollator(model_args),\n",
        "    )\n",
        "\n",
        "    # Training\n",
        "    if training_args.do_train:\n",
        "        trainer.train(\n",
        "            model_path=model_args.model_checkpoint if os.path.isdir(model_args.model_checkpoint) else None\n",
        "        )\n",
        "        trainer.save_model()\n",
        "        # For convenience, we also re-save the tokenizer to the same directory,\n",
        "        # so that you can share your model easily on huggingface.co/models =)\n",
        "        if trainer.is_world_master():\n",
        "            tokenizer.save_pretrained(training_args.output_dir)\n",
        "\n",
        "    # Evaluation\n",
        "\n",
        "    ## unable to figure out evalution with Trainer due to distrbuted eval. \n",
        "    ## will come back to it later on\n",
        "    eval_results = {}\n",
        "    all_results = []\n",
        "    results = {}\n",
        "    start_time = timeit.default_timer()\n",
        "    prefix = ''\n",
        "    features = eval_dataset.get_features()\n",
        "    if training_args.do_eval:\n",
        "        logger.warning(\"*** Evaluate ***\")\n",
        "\n",
        "        eval_sampler = SequentialSampler(eval_dataset)\n",
        "        eval_dataloader = DataLoader(eval_dataset, sampler=eval_sampler, batch_size=training_args.eval_batch_size, collate_fn=SquadDataCollator(model_args).collate_batch)\n",
        "\n",
        "        for inputs in tqdm(eval_dataloader, desc=\"Evaluating\"):\n",
        "          model.eval()\n",
        "\n",
        "          with torch.no_grad():\n",
        "              for k, v in inputs.items():\n",
        "                inputs[k] = v.to(training_args.device)\n",
        "\n",
        "              if model_args.model_type in [\"xlm\", \"roberta\", \"distilbert\", \"camembert\"]:\n",
        "                  del inputs[\"token_type_ids\"]\n",
        "\n",
        "              feature_indices = inputs[\"example_index\"]\n",
        "              unique_ids = inputs[\"unique_ids\"]\n",
        "\n",
        "              del inputs[\"example_index\"]\n",
        "              del inputs[\"start_positions\"]\n",
        "              del inputs['end_positions']\n",
        "              del inputs['unique_ids']\n",
        "\n",
        "              # XLNet and XLM use more arguments for their predictions\n",
        "              if model_args.model_type in [\"xlnet\", \"xlm\"]:\n",
        "                  # inputs.update({\"cls_index\": inputs[\"cls_index\"], \"p_mask\": inputs[\"p_mask\"]})\n",
        "                  # for lang_id-sensitive xlm models\n",
        "                  if hasattr(model, \"config\") and hasattr(model.config, \"lang2id\"):\n",
        "                      inputs.update(\n",
        "                          {\"langs\": (torch.ones(batch[0].shape, dtype=torch.int64) * model_args.lang_id).to(model_args.device)}\n",
        "                      )\n",
        "              else:\n",
        "                del inputs[\"cls_index\"]\n",
        "                del inputs[\"p_mask\"]\n",
        "\n",
        "              outputs = model(**inputs)\n",
        "\n",
        "              unique_ids = unique_ids.cpu().numpy()\n",
        "              for i,unique_id in enumerate(unique_ids):\n",
        "                  # TODO: i and feature_index are the same number! Simplify by removing enumerate?\n",
        "                  # logger.warning(\"feature index %s\" , feature_index.item())\n",
        "                  # eval_feature = features[feature_index.item()]\n",
        "                  # unique_id = int(eval_feature.unique_id)\n",
        "                  # logger.warn(\"unique id %s\" ,unique_id)\n",
        "                  output = [to_list(output[i]) for output in outputs]\n",
        "\n",
        "                  \n",
        "\n",
        "                  # Some models (XLNet, XLM) use 5 arguments for their predictions, while the other \"simpler\"\n",
        "                  # models only use two.\n",
        "                  if len(output) >= 5:\n",
        "                      start_logits = output[0]\n",
        "                      start_top_index = output[1]\n",
        "                      end_logits = output[2]\n",
        "                      end_top_index = output[3]\n",
        "                      cls_logits = output[4]\n",
        "\n",
        "                      result = SquadResult(\n",
        "                          unique_id,\n",
        "                          start_logits,\n",
        "                          end_logits,\n",
        "                          start_top_index=start_top_index,\n",
        "                          end_top_index=end_top_index,\n",
        "                          cls_logits=cls_logits,\n",
        "                      )\n",
        "\n",
        "                  else:\n",
        "                      start_logits, end_logits = output\n",
        "                      result = SquadResult(unique_id, start_logits, end_logits)\n",
        "\n",
        "                  all_results.append(result)\n",
        "\n",
        "        evalTime = timeit.default_timer() - start_time\n",
        "        logger.info(\"  Evaluation done in total %f secs (%f sec per example)\", evalTime, evalTime / len(eval_dataset))\n",
        "\n",
        "        # Compute predictions\n",
        "        output_prediction_file = os.path.join(training_args.output_dir, \"predictions_{}.json\".format(prefix))\n",
        "        output_nbest_file = os.path.join(training_args.output_dir, \"nbest_predictions_{}.json\".format(prefix))\n",
        "\n",
        "        if data_args.version_2_with_negative:\n",
        "            output_null_log_odds_file = os.path.join(training_args.output_dir, \"null_odds_{}.json\".format(prefix))\n",
        "        else:\n",
        "            output_null_log_odds_file = None\n",
        "\n",
        "\n",
        "        examples = eval_dataset.get_examples()\n",
        "        \n",
        "        # XLNet and XLM use a more complex post-processing procedure\n",
        "        if model_args.model_type in [\"xlnet\", \"xlm\"]:\n",
        "            start_n_top = model.config.start_n_top if hasattr(model, \"config\") else model.module.config.start_n_top\n",
        "            end_n_top = model.config.end_n_top if hasattr(model, \"config\") else model.module.config.end_n_top\n",
        "\n",
        "            predictions = compute_predictions_log_probs(\n",
        "                examples,\n",
        "                features,\n",
        "                all_results,\n",
        "                data_args.n_best_size,\n",
        "                data_args.max_answer_length,\n",
        "                output_prediction_file,\n",
        "                output_nbest_file,\n",
        "                output_null_log_odds_file,\n",
        "                start_n_top,\n",
        "                end_n_top,\n",
        "                data_args.version_2_with_negative,\n",
        "                tokenizer,\n",
        "                data_args.verbose_logging,\n",
        "            )\n",
        "        else:\n",
        "            predictions = compute_predictions_logits(\n",
        "                examples,\n",
        "                features,\n",
        "                all_results,\n",
        "                data_args.n_best_size,\n",
        "                data_args.max_answer_length,\n",
        "                data_args.do_lower_case,\n",
        "                output_prediction_file,\n",
        "                output_nbest_file,\n",
        "                output_null_log_odds_file,\n",
        "                data_args.verbose_logging,\n",
        "                data_args.version_2_with_negative,\n",
        "                data_args.null_score_diff_threshold,\n",
        "                tokenizer,\n",
        "            )\n",
        "\n",
        "        # Compute the F1 and exact scores.\n",
        "        result = squad_evaluate(examples, predictions)\n",
        "\n",
        "        logger.info(result)\n",
        "\n",
        "    return eval_results\n",
        "\n",
        "def _mp_fn(index):\n",
        "    # For xla_spawn (TPUs)\n",
        "    main()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aoYrbKqHj9Jl",
        "colab_type": "code",
        "outputId": "c4886843-b497-4b32-9344-a8136fb1bb14",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000,
          "referenced_widgets": [
            "8602662e6a4746ffa8806f598f778bfe",
            "639c4ab8ec4b4cdc9ca1bcd1c3d87486",
            "98bb1ef6391045d6a3899b936ae794b1",
            "396a0c99a36b47a7b4bad3a18bcb3197",
            "9c4706a6eff341f2a0665826819dd016",
            "3cd6c2f25dd74d5595bdff24345259a9",
            "8a81c36807a94b27b604bef11df924f6",
            "6271c2ff48e94521854d31adcd1708ca",
            "ef3b2b8e6cbc4e5da17dcafe6f22b247",
            "97078c3f52d745f3bd791728fd3fac06",
            "d2b8ed2982c2484a86ce904ad71f2301",
            "fa60acccced543daa8ffcf62bb8dea08",
            "181b91946e284f2f95bcca13e9d594d2",
            "704cc452a05b415e88a353222b44f49d",
            "43479ab8eab44eb4b1f0c6163b78f99a",
            "3c5514e0d5144031a47a2d767eb7e8f1",
            "03ac45eeea894b1683b5215a824577e4",
            "c95e3307f3184b34bda83487d4fa8ad7",
            "4f0ae3c9c29d4c7fb0114fb26e9e408a",
            "7307b235fbbc46ebbddb4297c6b5c3d5",
            "5521d35d381f43f9bd09a86eff22aa92",
            "7e263b6143a343febb94e9b5159d8861",
            "7daa4e54b39e4efc94c42684439da21a",
            "1ddf2773f8e54d89946209719fe4200d",
            "182d2dbccbcc41c592660f0a95704a82",
            "7c5a95be09514e71a6151f5d7b4cd940",
            "a164153b960c4064bfb31dcb19f26c18",
            "b0354c28ad9340cea8e3f75074ad9f1f",
            "e5ac4d6688e14a35a166f8a36cb10746",
            "9fe16545f9ef4c109e4e73ab2c8cfdb2",
            "db268c5c8f734f199b839e3192c2a1f1",
            "54c7770c9e1d4dcca3769019e234f26c",
            "020fe134868042a3a87276555a8be882",
            "34e40d10ba95405db7114080946ca83e",
            "71601cd26c2c414ab748c6219fad3301",
            "d57cb24343b64006983ee7747b34037e",
            "20a54f8bf0ad4601a6ed454dff553c90",
            "80152d8f3cca4a789516e65685fc0e00",
            "b01ef7910deb40f3bbccd723d148816c",
            "8842bc486d9c4a088b45e69540afc441"
          ]
        }
      },
      "source": [
        "main()   #via gpu or cpu"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "06/14/2020 06:29:07 - INFO - transformers.training_args -   PyTorch: setting up devices\n",
            "06/14/2020 06:29:08 - WARNING - __main__ -   Process rank: -1, device: cuda, n_gpu: 2, distributed training: False, 16-bits training: False\n",
            "06/14/2020 06:29:08 - INFO - __main__ -   Training/evaluation parameters TrainingArguments(output_dir='./drive/output/roberta/distilroberta-base', overwrite_output_dir=True, do_train=True, do_eval=True, do_predict=False, evaluate_during_training=False, per_device_train_batch_size=48, per_device_eval_batch_size=8, per_gpu_train_batch_size=None, per_gpu_eval_batch_size=None, gradient_accumulation_steps=1, learning_rate=5e-05, weight_decay=0.0, adam_epsilon=1e-08, max_grad_norm=1.0, num_train_epochs=3.0, max_steps=-1, warmup_steps=0, logging_dir='runs/Jun14_06-29-07_torchvm3', logging_first_step=False, logging_steps=500, save_steps=5000, save_total_limit=2, no_cuda=False, seed=42, fp16=False, fp16_opt_level='O1', local_rank=-1, tpu_num_cores=None, tpu_metrics_debug=False, dataloader_drop_last=False)\n",
            "06/14/2020 06:29:09 - INFO - transformers.configuration_utils -   loading configuration file https://s3.amazonaws.com/models.huggingface.co/bert/distilroberta-base-config.json from cache at ./drive/cache/d52ced8fd31ba6aa311b6eeeae65178cca00ddd6333c087be4601dc46c20bd96.1221e000e415a518ec3c28f32c14c1dd3baa36dc0537db4848b7236b38f50313\n",
            "06/14/2020 06:29:09 - INFO - transformers.configuration_utils -   Model config RobertaConfig {\n",
            "  \"architectures\": [\n",
            "    \"RobertaForMaskedLM\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"eos_token_id\": 2,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"layer_norm_eps\": 1e-05,\n",
            "  \"max_position_embeddings\": 514,\n",
            "  \"model_type\": \"roberta\",\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_hidden_layers\": 6,\n",
            "  \"pad_token_id\": 1,\n",
            "  \"type_vocab_size\": 1,\n",
            "  \"vocab_size\": 50265\n",
            "}\n",
            "\n",
            "06/14/2020 06:29:09 - INFO - transformers.configuration_utils -   loading configuration file https://s3.amazonaws.com/models.huggingface.co/bert/distilroberta-base-config.json from cache at ./drive/cache/d52ced8fd31ba6aa311b6eeeae65178cca00ddd6333c087be4601dc46c20bd96.1221e000e415a518ec3c28f32c14c1dd3baa36dc0537db4848b7236b38f50313\n",
            "06/14/2020 06:29:09 - INFO - transformers.configuration_utils -   Model config RobertaConfig {\n",
            "  \"architectures\": [\n",
            "    \"RobertaForMaskedLM\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"eos_token_id\": 2,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"layer_norm_eps\": 1e-05,\n",
            "  \"max_position_embeddings\": 514,\n",
            "  \"model_type\": \"roberta\",\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_hidden_layers\": 6,\n",
            "  \"pad_token_id\": 1,\n",
            "  \"type_vocab_size\": 1,\n",
            "  \"vocab_size\": 50265\n",
            "}\n",
            "\n",
            "06/14/2020 06:29:09 - INFO - transformers.tokenization_utils -   loading file https://s3.amazonaws.com/models.huggingface.co/bert/distilroberta-base-vocab.json from cache at ./drive/cache/5f11352d3c3e932888f3ba75bc24579eacb5d1596d39ce56166aeae8fd363df8.ef00af9e673c7160b4d41cfda1f48c5f4cba57d5142754525572a846a1ab1b9b\n",
            "06/14/2020 06:29:09 - INFO - transformers.tokenization_utils -   loading file https://s3.amazonaws.com/models.huggingface.co/bert/distilroberta-base-merges.txt from cache at ./drive/cache/01f63a14ad93494c050af2090c59930fb787bdfb347c4cad7ce9063e1a5fe140.70bec105b4158ed9a1747fea67a43f5dee97855c64d62b6ec3742f4cfdb5feda\n",
            "06/14/2020 06:29:09 - INFO - transformers.modeling_utils -   loading weights file https://cdn.huggingface.co/distilroberta-base-pytorch_model.bin from cache at ./drive/cache/5aab0d7dfa1db7d97ead13a37479db888b133a51a05ae4ab62ff5c8d1fcabb65.52b6ec356fb91985b3940e086d1b2ebf8cd40f8df0ba1cabf4cac27769dee241\n",
            "06/14/2020 06:29:12 - INFO - transformers.modeling_utils -   Weights of RobertaForQuestionAnswering not initialized from pretrained model: ['qa_outputs.weight', 'qa_outputs.bias']\n",
            "06/14/2020 06:29:12 - INFO - transformers.modeling_utils -   Weights from pretrained model not used in RobertaForQuestionAnswering: ['lm_head.bias', 'lm_head.dense.weight', 'lm_head.dense.bias', 'lm_head.layer_norm.weight', 'lm_head.layer_norm.bias', 'lm_head.decoder.weight']\n",
            "06/14/2020 06:29:12 - INFO - filelock -   Lock 140576270991632 acquired on cached_train_RobertaTokenizer_328_squad_None.lock\n",
            "06/14/2020 06:29:40 - INFO - __main__ -   Loading features from cached file cached_train_RobertaTokenizer_328_squad_None [took 27.933 s]\n",
            "06/14/2020 06:29:40 - INFO - filelock -   Lock 140576270991632 released on cached_train_RobertaTokenizer_328_squad_None.lock\n",
            "06/14/2020 06:29:40 - INFO - filelock -   Lock 140574396069200 acquired on cached_dev_RobertaTokenizer_328_squad_None.lock\n",
            "06/14/2020 06:29:42 - INFO - __main__ -   Loading features from cached file cached_dev_RobertaTokenizer_328_squad_None [took 1.687 s]\n",
            "06/14/2020 06:29:42 - INFO - filelock -   Lock 140574396069200 released on cached_dev_RobertaTokenizer_328_squad_None.lock\n",
            "06/14/2020 06:29:45 - WARNING - transformers.trainer -   You are instantiating a Trainer but Tensorboard is not installed. You should consider installing it.\n",
            "06/14/2020 06:29:45 - INFO - transformers.trainer -   You are instantiating a Trainer but W&B is not installed. To use wandb logging, run `pip install wandb; wandb login` see https://docs.wandb.com/huggingface.\n",
            "06/14/2020 06:29:45 - INFO - transformers.trainer -   ***** Running training *****\n",
            "06/14/2020 06:29:45 - INFO - transformers.trainer -     Num examples = 141316\n",
            "06/14/2020 06:29:45 - INFO - transformers.trainer -     Num Epochs = 3\n",
            "06/14/2020 06:29:45 - INFO - transformers.trainer -     Instantaneous batch size per device = 48\n",
            "06/14/2020 06:29:45 - INFO - transformers.trainer -     Total train batch size (w. parallel, distributed & accumulation) = 96\n",
            "06/14/2020 06:29:45 - INFO - transformers.trainer -     Gradient Accumulation steps = 1\n",
            "06/14/2020 06:29:45 - INFO - transformers.trainer -     Total optimization steps = 4419\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Epoch', max=3.0, style=ProgressStyle(description_width='i…"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "8602662e6a4746ffa8806f598f778bfe"
            }
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Iteration', max=1473.0, style=ProgressStyle(description_w…"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "ef3b2b8e6cbc4e5da17dcafe6f22b247"
            }
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "/opt/conda/lib/python3.7/site-packages/torch/nn/parallel/_functions.py:61: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
            "  warnings.warn('Was asked to gather along dimension 0, but all '\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "{\"loss\": 0.10346589727760147, \"learning_rate\": 4.434261145055443e-05, \"epoch\": 0.3394433129667346, \"step\": 500}\n",
            "{\"loss\": 3.783510135690449e-05, \"learning_rate\": 3.8685222901108846e-05, \"epoch\": 0.6788866259334692, \"step\": 1000}\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Iteration', max=1473.0, style=ProgressStyle(description_w…"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "03ac45eeea894b1683b5215a824577e4"
            }
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "{\"loss\": 2.2400965321139666e-05, \"learning_rate\": 3.3027834351663274e-05, \"epoch\": 1.0183299389002036, \"step\": 1500}\n",
            "{\"loss\": 1.4963840394557337e-05, \"learning_rate\": 2.73704458022177e-05, \"epoch\": 1.3577732518669383, \"step\": 2000}\n",
            "{\"loss\": 1.1106098689197097e-05, \"learning_rate\": 2.171305725277212e-05, \"epoch\": 1.6972165648336728, \"step\": 2500}\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Iteration', max=1473.0, style=ProgressStyle(description_w…"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "182d2dbccbcc41c592660f0a95704a82"
            }
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "{\"loss\": 8.760819851886481e-06, \"learning_rate\": 1.6055668703326546e-05, \"epoch\": 2.0366598778004072, \"step\": 3000}\n",
            "{\"loss\": 7.3004911382668065e-06, \"learning_rate\": 1.0398280153880969e-05, \"epoch\": 2.3761031907671417, \"step\": 3500}\n",
            "{\"loss\": 6.398871559213149e-06, \"learning_rate\": 4.740891604435393e-06, \"epoch\": 2.7155465037338766, \"step\": 4000}\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "06/14/2020 08:34:50 - INFO - transformers.trainer -   \n",
            "\n",
            "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
            "\n",
            "\n",
            "06/14/2020 08:34:50 - INFO - transformers.trainer -   Saving model checkpoint to ./drive/output/roberta/distilroberta-base\n",
            "06/14/2020 08:34:50 - INFO - transformers.configuration_utils -   Configuration saved in ./drive/output/roberta/distilroberta-base/config.json\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "06/14/2020 08:34:51 - INFO - transformers.modeling_utils -   Model weights saved in ./drive/output/roberta/distilroberta-base/pytorch_model.bin\n",
            "06/14/2020 08:34:51 - WARNING - __main__ -   *** Evaluate ***\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Evaluating', max=833.0, style=ProgressStyle(description_w…"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "020fe134868042a3a87276555a8be882"
            }
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "06/14/2020 08:37:14 - INFO - __main__ -     Evaluation done in total 142.852460 secs (0.010724 sec per example)\n",
            "06/14/2020 08:37:14 - INFO - transformers.data.metrics.squad_metrics -   Writing predictions to: ./drive/output/roberta/distilroberta-base/predictions_.json\n",
            "06/14/2020 08:37:14 - INFO - transformers.data.metrics.squad_metrics -   Writing nbest to: ./drive/output/roberta/distilroberta-base/nbest_predictions_.json\n",
            "06/14/2020 08:37:14 - INFO - transformers.data.metrics.squad_metrics -   Writing null_log_odds to: ./drive/output/roberta/distilroberta-base/null_odds_.json\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "06/14/2020 08:37:37 - INFO - __main__ -   OrderedDict([('exact', 50.07159100480081), ('f1', 50.07159100480081), ('total', 11873), ('HasAns_exact', 0.0), ('HasAns_f1', 0.0), ('HasAns_total', 5928), ('NoAns_exact', 100.0), ('NoAns_f1', 100.0), ('NoAns_total', 5945), ('best_exact', 50.07159100480081), ('best_exact_thresh', 0.0), ('best_f1', 50.07159100480081), ('best_f1_thresh', 0.0)])\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{}"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ealJH-adC8CI",
        "colab_type": "text"
      },
      "source": [
        "# Publish Your Model To Huggingface"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6DKJAkDaWnmb",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!transformers-cli login"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rhqQgEA9W7XB",
        "colab_type": "code",
        "outputId": "c8a466dc-b91f-4a36-db11-fec2eba845c6",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 570
        }
      },
      "source": [
        "!transformers-cli upload ./drive/output/roberta/distilroberta-base"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "2020-06-13 15:50:22.603803: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1\n",
            "About to upload file \u001b[1m/content/distilbert-squad-256seq-8batch-test/vocab.txt\u001b[0m to S3 under filename \u001b[1mdistilbert-squad-256seq-8batch-test/vocab.txt\u001b[0m and namespace \u001b[1mmanishiitg\u001b[0m\n",
            "About to upload file \u001b[1m/content/distilbert-squad-256seq-8batch-test/predictions_.json\u001b[0m to S3 under filename \u001b[1mdistilbert-squad-256seq-8batch-test/predictions_.json\u001b[0m and namespace \u001b[1mmanishiitg\u001b[0m\n",
            "About to upload file \u001b[1m/content/distilbert-squad-256seq-8batch-test/null_odds_.json\u001b[0m to S3 under filename \u001b[1mdistilbert-squad-256seq-8batch-test/null_odds_.json\u001b[0m and namespace \u001b[1mmanishiitg\u001b[0m\n",
            "About to upload file \u001b[1m/content/distilbert-squad-256seq-8batch-test/tokenizer_config.json\u001b[0m to S3 under filename \u001b[1mdistilbert-squad-256seq-8batch-test/tokenizer_config.json\u001b[0m and namespace \u001b[1mmanishiitg\u001b[0m\n",
            "About to upload file \u001b[1m/content/distilbert-squad-256seq-8batch-test/pytorch_model.bin\u001b[0m to S3 under filename \u001b[1mdistilbert-squad-256seq-8batch-test/pytorch_model.bin\u001b[0m and namespace \u001b[1mmanishiitg\u001b[0m\n",
            "About to upload file \u001b[1m/content/distilbert-squad-256seq-8batch-test/special_tokens_map.json\u001b[0m to S3 under filename \u001b[1mdistilbert-squad-256seq-8batch-test/special_tokens_map.json\u001b[0m and namespace \u001b[1mmanishiitg\u001b[0m\n",
            "About to upload file \u001b[1m/content/distilbert-squad-256seq-8batch-test/training_args.bin\u001b[0m to S3 under filename \u001b[1mdistilbert-squad-256seq-8batch-test/training_args.bin\u001b[0m and namespace \u001b[1mmanishiitg\u001b[0m\n",
            "About to upload file \u001b[1m/content/distilbert-squad-256seq-8batch-test/nbest_predictions_.json\u001b[0m to S3 under filename \u001b[1mdistilbert-squad-256seq-8batch-test/nbest_predictions_.json\u001b[0m and namespace \u001b[1mmanishiitg\u001b[0m\n",
            "About to upload file \u001b[1m/content/distilbert-squad-256seq-8batch-test/config.json\u001b[0m to S3 under filename \u001b[1mdistilbert-squad-256seq-8batch-test/config.json\u001b[0m and namespace \u001b[1mmanishiitg\u001b[0m\n",
            "Proceed? [Y/n] Y\n",
            "\u001b[1mUploading... This might take a while if files are large\u001b[0m\n",
            "Your file now lives at:\n",
            "https://s3.amazonaws.com/models.huggingface.co/bert/manishiitg/distilbert-squad-256seq-8batch-test/vocab.txt\n",
            "Your file now lives at:\n",
            "https://s3.amazonaws.com/models.huggingface.co/bert/manishiitg/distilbert-squad-256seq-8batch-test/predictions_.json\n",
            "Your file now lives at:\n",
            "https://s3.amazonaws.com/models.huggingface.co/bert/manishiitg/distilbert-squad-256seq-8batch-test/null_odds_.json\n",
            "Your file now lives at:\n",
            "https://s3.amazonaws.com/models.huggingface.co/bert/manishiitg/distilbert-squad-256seq-8batch-test/tokenizer_config.json\n",
            "Your file now lives at:\n",
            "https://s3.amazonaws.com/models.huggingface.co/bert/manishiitg/distilbert-squad-256seq-8batch-test/pytorch_model.bin\n",
            "Your file now lives at:\n",
            "https://s3.amazonaws.com/models.huggingface.co/bert/manishiitg/distilbert-squad-256seq-8batch-test/special_tokens_map.json\n",
            "Your file now lives at:\n",
            "https://s3.amazonaws.com/models.huggingface.co/bert/manishiitg/distilbert-squad-256seq-8batch-test/training_args.bin\n",
            "Your file now lives at:\n",
            "https://s3.amazonaws.com/models.huggingface.co/bert/manishiitg/distilbert-squad-256seq-8batch-test/nbest_predictions_.json\n",
            "Your file now lives at:\n",
            "https://s3.amazonaws.com/models.huggingface.co/bert/manishiitg/distilbert-squad-256seq-8batch-test/config.json\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Cib8yf4aDDrG",
        "colab_type": "text"
      },
      "source": [
        "# Do Prediction"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fMPpaikDVOuK",
        "colab_type": "code",
        "outputId": "e52acd57-b106-4007-d576-90754bab5f39",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "tokenizer = AutoTokenizer.from_pretrained('output/roberta/distilroberta-base')\n",
        "model = AutoModelForQuestionAnswering.from_pretrained('output/roberta/distilroberta-base')\n",
        "\n",
        "model.eval()\n"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "06/14/2020 08:38:57 - INFO - transformers.configuration_utils -   loading configuration file output/roberta/distilroberta-base/config.json\n",
            "06/14/2020 08:38:57 - INFO - transformers.configuration_utils -   Model config RobertaConfig {\n",
            "  \"architectures\": [\n",
            "    \"RobertaForQuestionAnswering\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"eos_token_id\": 2,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"layer_norm_eps\": 1e-05,\n",
            "  \"max_position_embeddings\": 514,\n",
            "  \"model_type\": \"roberta\",\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_hidden_layers\": 6,\n",
            "  \"pad_token_id\": 1,\n",
            "  \"type_vocab_size\": 1,\n",
            "  \"vocab_size\": 50265\n",
            "}\n",
            "\n",
            "06/14/2020 08:38:57 - INFO - transformers.tokenization_utils -   Model name 'output/roberta/distilroberta-base' not found in model shortcut name list (roberta-base, roberta-large, roberta-large-mnli, distilroberta-base, roberta-base-openai-detector, roberta-large-openai-detector). Assuming 'output/roberta/distilroberta-base' is a path, a model identifier, or url to a directory containing tokenizer files.\n",
            "06/14/2020 08:38:57 - INFO - transformers.tokenization_utils -   Didn't find file output/roberta/distilroberta-base/added_tokens.json. We won't load it.\n",
            "06/14/2020 08:38:57 - INFO - transformers.tokenization_utils -   loading file output/roberta/distilroberta-base/vocab.json\n",
            "06/14/2020 08:38:57 - INFO - transformers.tokenization_utils -   loading file output/roberta/distilroberta-base/merges.txt\n",
            "06/14/2020 08:38:57 - INFO - transformers.tokenization_utils -   loading file None\n",
            "06/14/2020 08:38:57 - INFO - transformers.tokenization_utils -   loading file output/roberta/distilroberta-base/special_tokens_map.json\n",
            "06/14/2020 08:38:57 - INFO - transformers.tokenization_utils -   loading file output/roberta/distilroberta-base/tokenizer_config.json\n",
            "06/14/2020 08:38:57 - INFO - transformers.configuration_utils -   loading configuration file output/roberta/distilroberta-base/config.json\n",
            "06/14/2020 08:38:57 - INFO - transformers.configuration_utils -   Model config RobertaConfig {\n",
            "  \"architectures\": [\n",
            "    \"RobertaForQuestionAnswering\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"eos_token_id\": 2,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"layer_norm_eps\": 1e-05,\n",
            "  \"max_position_embeddings\": 514,\n",
            "  \"model_type\": \"roberta\",\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_hidden_layers\": 6,\n",
            "  \"pad_token_id\": 1,\n",
            "  \"type_vocab_size\": 1,\n",
            "  \"vocab_size\": 50265\n",
            "}\n",
            "\n",
            "06/14/2020 08:38:57 - INFO - transformers.modeling_utils -   loading weights file output/roberta/distilroberta-base/pytorch_model.bin\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "RobertaForQuestionAnswering(\n",
              "  (roberta): RobertaModel(\n",
              "    (embeddings): RobertaEmbeddings(\n",
              "      (word_embeddings): Embedding(50265, 768, padding_idx=1)\n",
              "      (position_embeddings): Embedding(514, 768, padding_idx=1)\n",
              "      (token_type_embeddings): Embedding(1, 768)\n",
              "      (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "      (dropout): Dropout(p=0.1, inplace=False)\n",
              "    )\n",
              "    (encoder): BertEncoder(\n",
              "      (layer): ModuleList(\n",
              "        (0): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (1): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (2): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (3): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (4): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (5): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "    )\n",
              "    (pooler): BertPooler(\n",
              "      (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "      (activation): Tanh()\n",
              "    )\n",
              "  )\n",
              "  (qa_outputs): Linear(in_features=768, out_features=2, bias=True)\n",
              ")"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "T4iRLvCBZnXZ",
        "colab_type": "code",
        "outputId": "5eb9104f-a7cf-473e-e96c-cd1a93d8c5f6",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "with torch.no_grad():\n",
        "  input_ids = torch.tensor(tokenizer.encode(\"how much work experiance?\", \"I have been working with excellence technologies for 10 years now and before that i worked for 2years with headstrong ltd\"))  # Batch size 1\n",
        "  outputs = model(input_ids.unsqueeze(0))\n",
        "  start_logits = outputs[0]\n",
        "  end_logits = outputs[1]\n",
        "  \n",
        "  # start_idx = torch.argmax(start_logits)\n",
        "  # end_idx = torch.argmax(end_logits) + 1\n",
        "  # print(start_idx)\n",
        "  # print(end_idx)\n",
        "\n",
        "  all_tokens = tokenizer.convert_ids_to_tokens(input_ids)\n",
        "  answer = ' '.join(all_tokens[torch.argmax(start_logits) : torch.argmax(end_logits)+1])\n",
        "  print(answer)\n"
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "<s>\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nIA8F6-jcmAW",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Crash on purpose to get more ram : doens't work anymore \n",
        "# import torch\n",
        "# torch.tensor([10.]*10000000000)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nDhUivjrVo74",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "VERSION = \"nightly\"  #@param [\"1.5\" , \"20200325\", \"nightly\"]\n",
        "!curl https://raw.githubusercontent.com/pytorch/xla/master/contrib/scripts/env-setup.py -o pytorch-xla-env-setup.py\n",
        "!python pytorch-xla-env-setup.py --version $VERSION"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HL-XvIgyfo4-",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!pip install tf-nightly"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "s_xSTdmWkwMu",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import tensorflow as tf\n",
        "\n",
        "import os\n",
        "import tensorflow_datasets as tfds\n",
        "\n",
        "resolver = tf.distribute.cluster_resolver.TPUClusterResolver(tpu='grpc://' + os.environ['COLAB_TPU_ADDR'])\n",
        "tf.config.experimental_connect_to_cluster(resolver)\n",
        "# This is the TPU initialization code that has to be at the beginning.\n",
        "tf.tpu.experimental.initialize_tpu_system(resolver)\n",
        "print(\"All devices: \", tf.config.list_logical_devices('TPU'))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CgqGXvn3VxjW",
        "colab_type": "code",
        "outputId": "26c6890a-885b-4d76-8bc3-31f0a63a58bb",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 863,
          "referenced_widgets": [
            "eb143cd09f8d499a906f7f1445c89490",
            "b86f75683b0941c8a1bfc2ddf0a23f91",
            "02bb4bff0df94b5fbe1f330bfd7da658",
            "a5b676acb5ff47fba258494b84dc3e24",
            "8f2600ab266a4db5bf1f16ae16906360",
            "43e35e5df12d441a82228353818a90cc",
            "bea0fbcc65234b13973273509dcc436e",
            "ae26f3c488484454854992cafb18b0d3",
            "077013e2538247a19c777baf28d2d622",
            "3b2d332b84244f98991d126a5761a6fd",
            "33dd4e91fd9c4a2897f6641880c797dd",
            "341822c58bc542d596126a2df8fc4843",
            "a92b2164f6f143089a2bdff8f6b8abf5",
            "8f91c018472743ffa0a15d8f97c6d18d",
            "63db2c5931104986a36696d34f3850f7",
            "4a92e09d8f894868abb4653d2e1f9246"
          ]
        }
      },
      "source": [
        "import torch_xla.distributed.xla_multiprocessing as xmp\n",
        "xmp.spawn(_mp_fn, args=(), nprocs=8, start_method='fork')"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "WARNING:__main__:Process rank: -1, device: xla:1, n_gpu: 0, distributed training: False, 16-bits training: False\n",
            "WARNING:__main__:WARNING - You've set a doc stride which may be superior to the document length in some examples. This could result in errors when building features from the examples. Please reduce the doc stride or increase the maximum length to ensure the features are correctly built.\n",
            "WARNING:__main__:Process rank: -1, device: xla:0, n_gpu: 0, distributed training: False, 16-bits training: False\n",
            "WARNING:__main__:WARNING - You've set a doc stride which may be superior to the document length in some examples. This could result in errors when building features from the examples. Please reduce the doc stride or increase the maximum length to ensure the features are correctly built.\n",
            "WARNING:__main__:Process rank: -1, device: xla:0, n_gpu: 0, distributed training: False, 16-bits training: False\n",
            "WARNING:__main__:WARNING - You've set a doc stride which may be superior to the document length in some examples. This could result in errors when building features from the examples. Please reduce the doc stride or increase the maximum length to ensure the features are correctly built.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "eb143cd09f8d499a906f7f1445c89490",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=442.0, style=ProgressStyle(description_…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "077013e2538247a19c777baf28d2d622",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=267967963.0, style=ProgressStyle(descri…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:__main__:Process rank: -1, device: xla:0, n_gpu: 0, distributed training: False, 16-bits training: False\n",
            "WARNING:__main__:WARNING - You've set a doc stride which may be superior to the document length in some examples. This could result in errors when building features from the examples. Please reduce the doc stride or increase the maximum length to ensure the features are correctly built.\n",
            "WARNING:__main__:Process rank: -1, device: xla:0, n_gpu: 0, distributed training: False, 16-bits training: False\n",
            "WARNING:__main__:WARNING - You've set a doc stride which may be superior to the document length in some examples. This could result in errors when building features from the examples. Please reduce the doc stride or increase the maximum length to ensure the features are correctly built.\n",
            "WARNING:__main__:Process rank: -1, device: xla:0, n_gpu: 0, distributed training: False, 16-bits training: False\n",
            "WARNING:__main__:WARNING - You've set a doc stride which may be superior to the document length in some examples. This could result in errors when building features from the examples. Please reduce the doc stride or increase the maximum length to ensure the features are correctly built.\n",
            "WARNING:__main__:Process rank: -1, device: xla:0, n_gpu: 0, distributed training: False, 16-bits training: False\n",
            "WARNING:__main__:WARNING - You've set a doc stride which may be superior to the document length in some examples. This could result in errors when building features from the examples. Please reduce the doc stride or increase the maximum length to ensure the features are correctly built.\n",
            "WARNING:__main__:Process rank: -1, device: xla:0, n_gpu: 0, distributed training: False, 16-bits training: False\n",
            "WARNING:__main__:WARNING - You've set a doc stride which may be superior to the document length in some examples. This could result in errors when building features from the examples. Please reduce the doc stride or increase the maximum length to ensure the features are correctly built.\n",
            "100%|██████████| 442/442 [00:50<00:00,  8.76it/s]\n",
            "convert squad examples to features: 100%|██████████| 10000/10000 [01:05<00:00, 152.35it/s]\n",
            "add example index and unique id: 100%|██████████| 10000/10000 [00:00<00:00, 479809.65it/s]\n",
            "100%|██████████| 35/35 [00:08<00:00,  4.08it/s]\n",
            "convert squad examples to features: 100%|██████████| 10000/10000 [01:46<00:00, 93.87it/s]\n",
            "add example index and unique id: 100%|██████████| 10000/10000 [00:00<00:00, 442432.46it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "error",
          "ename": "Exception",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mException\u001b[0m                                 Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-19-e5964fb04bef>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      8\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      9\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mtorch_xla\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdistributed\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mxla_multiprocessing\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mxmp\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 10\u001b[0;31m \u001b[0mxmp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mspawn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0m_mp_fn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnprocs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m8\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstart_method\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'fork'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/torch_xla/distributed/xla_multiprocessing.py\u001b[0m in \u001b[0;36mspawn\u001b[0;34m(fn, args, nprocs, join, daemon, start_method)\u001b[0m\n\u001b[1;32m    294\u001b[0m         \u001b[0mjoin\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mjoin\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    295\u001b[0m         \u001b[0mdaemon\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdaemon\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 296\u001b[0;31m         start_method=start_method)\n\u001b[0m\u001b[1;32m    297\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    298\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/torch/multiprocessing/spawn.py\u001b[0m in \u001b[0;36mstart_processes\u001b[0;34m(fn, args, nprocs, join, daemon, start_method)\u001b[0m\n\u001b[1;32m    156\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    157\u001b[0m     \u001b[0;31m# Loop on join until it returns True or raises an exception.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 158\u001b[0;31m     \u001b[0;32mwhile\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mcontext\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mjoin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    159\u001b[0m         \u001b[0;32mpass\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    160\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/torch/multiprocessing/spawn.py\u001b[0m in \u001b[0;36mjoin\u001b[0;34m(self, timeout)\u001b[0m\n\u001b[1;32m    106\u001b[0m                 raise Exception(\n\u001b[1;32m    107\u001b[0m                     \u001b[0;34m\"process %d terminated with signal %s\"\u001b[0m \u001b[0;34m%\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 108\u001b[0;31m                     \u001b[0;34m(\u001b[0m\u001b[0merror_index\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    109\u001b[0m                 )\n\u001b[1;32m    110\u001b[0m             \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mException\u001b[0m: process 3 terminated with signal SIGKILL"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QsbKAsfzisxb",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}