{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "pretrain_transformer.ipynb",
      "provenance": [],
      "collapsed_sections": [
        "qSuUpkj1UuUa"
      ],
      "toc_visible": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "5b22f6305d4d4be4b5ff4cd55a37a793": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_13d461b4dc884a7c95a2c971a1fe1ff7",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_14e396ff5ecf41c7b2ad1c0a88f8f44d",
              "IPY_MODEL_93568a1211bb4ec6831b2a7a1706d45c"
            ]
          }
        },
        "13d461b4dc884a7c95a2c971a1fe1ff7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "14e396ff5ecf41c7b2ad1c0a88f8f44d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_8ccce63db8f740c4acf30a5984524887",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 433,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 433,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_b0c769e363454565acbcff91e66bfa8b"
          }
        },
        "93568a1211bb4ec6831b2a7a1706d45c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_3e3dade805e14bcd950bc8dbe61919c9",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 433/433 [00:01&lt;00:00, 234B/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_67e3b557ed944846ba9f4ccb814a7cea"
          }
        },
        "8ccce63db8f740c4acf30a5984524887": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "b0c769e363454565acbcff91e66bfa8b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "3e3dade805e14bcd950bc8dbe61919c9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "67e3b557ed944846ba9f4ccb814a7cea": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "d19a2a7d0b0b406bb8c6242581756faa": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_cefebaecd4ab4b559f9eb277be8a87de",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_1872e7aa64e449639234c5619ff401d4",
              "IPY_MODEL_a42f68f5d3964ccbb24a58c436c5ab66"
            ]
          }
        },
        "cefebaecd4ab4b559f9eb277be8a87de": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "1872e7aa64e449639234c5619ff401d4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_75b2d7819d1a49409cc028c2ec171185",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 213450,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 213450,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_95eca277baa44ef1ada751b2135e435c"
          }
        },
        "a42f68f5d3964ccbb24a58c436c5ab66": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_8900998ecc724614b55437211b661438",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 213k/213k [00:00&lt;00:00, 302kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_6f9e6fe87db24bc6a4007f38b4ef8d36"
          }
        },
        "75b2d7819d1a49409cc028c2ec171185": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "95eca277baa44ef1ada751b2135e435c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "8900998ecc724614b55437211b661438": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "6f9e6fe87db24bc6a4007f38b4ef8d36": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "6c5c1fdbcfba4584adeae63600fd1eef": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_f6a5137958aa4f6cad1d1ff259c22433",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_2170da5ca9874a799bc1efce37a76ee2",
              "IPY_MODEL_90c6fcdbbdfe4029af1d42927d5be8bf"
            ]
          }
        },
        "f6a5137958aa4f6cad1d1ff259c22433": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "2170da5ca9874a799bc1efce37a76ee2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_7adb596bed44419ea65d7c3fa343ce3d",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 435779157,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 435779157,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_479276e74d4a49bc9b81a0b4e1d0daad"
          }
        },
        "90c6fcdbbdfe4029af1d42927d5be8bf": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_bfe3a93243f04b93ad4537e98a90d5d2",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 436M/436M [00:07&lt;00:00, 62.1MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_d85b01ae45e74878b119db827f79ec92"
          }
        },
        "7adb596bed44419ea65d7c3fa343ce3d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "479276e74d4a49bc9b81a0b4e1d0daad": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "bfe3a93243f04b93ad4537e98a90d5d2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "d85b01ae45e74878b119db827f79ec92": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "7c7be961fe3c4fbebd6842599c6e4cee": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_b7534c3a6486497fb29fb1ff79187666",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_96e5b4e416184710a7ac656e36258612",
              "IPY_MODEL_28b801cb34e547b89f0c25ac6d8b6671"
            ]
          }
        },
        "b7534c3a6486497fb29fb1ff79187666": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "96e5b4e416184710a7ac656e36258612": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_d57e431fa76646d49af035782eff6c6e",
            "_dom_classes": [],
            "description": "Epoch: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 3,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 3,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_3dd6aada08e947699cfc3890c4f14990"
          }
        },
        "28b801cb34e547b89f0c25ac6d8b6671": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_0f82fac089484ff3802dd5bca6c1f342",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 3/3 [03:17&lt;00:00, 65.91s/it]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_c1a90eae6dba4800a25dd6e60ab6d04f"
          }
        },
        "d57e431fa76646d49af035782eff6c6e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "3dd6aada08e947699cfc3890c4f14990": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "0f82fac089484ff3802dd5bca6c1f342": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "c1a90eae6dba4800a25dd6e60ab6d04f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "48bcf3b1343b48c48cda53021f1bddb4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_58547604efc742bdba897eb5703150e7",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_fa31d116e3cf4e2e9eecb9785944dc7f",
              "IPY_MODEL_551a97d96d744bffab7f6a570ee9cf23"
            ]
          }
        },
        "58547604efc742bdba897eb5703150e7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "fa31d116e3cf4e2e9eecb9785944dc7f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_7347d675b7794bfd989e6643fe5da3d4",
            "_dom_classes": [],
            "description": "Iteration: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 61,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 61,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_721312449a344a46bdbe101a0c059e45"
          }
        },
        "551a97d96d744bffab7f6a570ee9cf23": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_74398199f8144a3dbd5611812b7a6f04",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 61/61 [03:17&lt;00:00,  3.24s/it]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_06e9cd380db84d02bbb74e77c85f2314"
          }
        },
        "7347d675b7794bfd989e6643fe5da3d4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "721312449a344a46bdbe101a0c059e45": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "74398199f8144a3dbd5611812b7a6f04": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "06e9cd380db84d02bbb74e77c85f2314": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "ce438e273ddc46bc90f039a3794c700b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_5e87074a2bef465fa158c4b335af7124",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_ee4c91c46fd54609a5eee3d9620aa660",
              "IPY_MODEL_313fa40c7e24405f9906be3eca09d424"
            ]
          }
        },
        "5e87074a2bef465fa158c4b335af7124": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "ee4c91c46fd54609a5eee3d9620aa660": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_222d8e29cc264224bf43b67bf3fbaca4",
            "_dom_classes": [],
            "description": "Iteration: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 61,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 61,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_881feace35c249358b16c2c219b7ea2b"
          }
        },
        "313fa40c7e24405f9906be3eca09d424": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_b052b6cdac6248bc84f9ad09e3a4ee9f",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 61/61 [02:12&lt;00:00,  2.18s/it]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_0976f817303e4b9dac0110902b45cd69"
          }
        },
        "222d8e29cc264224bf43b67bf3fbaca4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "881feace35c249358b16c2c219b7ea2b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "b052b6cdac6248bc84f9ad09e3a4ee9f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "0976f817303e4b9dac0110902b45cd69": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "d3abee198a4847f1b2abc06f8fadeb87": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_8343ec37744445f6a46ecf2a565fec5d",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_f261f14c20514175856a617d1b688ff4",
              "IPY_MODEL_9fb431b6108e4752a72f62054be54812"
            ]
          }
        },
        "8343ec37744445f6a46ecf2a565fec5d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "f261f14c20514175856a617d1b688ff4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_49032bd7531143fcaae08b455112d5e3",
            "_dom_classes": [],
            "description": "Iteration: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 61,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 61,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_ed227ed0cf6d49ad93d5939db1ad5c84"
          }
        },
        "9fb431b6108e4752a72f62054be54812": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_f57fdf2386c94eab838090229fc32a33",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 61/61 [01:07&lt;00:00,  1.11s/it]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_a6b7a950a0f144b09166a32b0838c463"
          }
        },
        "49032bd7531143fcaae08b455112d5e3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "ed227ed0cf6d49ad93d5939db1ad5c84": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "f57fdf2386c94eab838090229fc32a33": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "a6b7a950a0f144b09166a32b0838c463": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "50da31b49a7040cbb682a005ed0f54e4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_9feff57e5f7041faa3dbc554a6fbbfd9",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_55c1ebbe8d7841d18751b6ea5a8af243",
              "IPY_MODEL_c925ba070f6244509ba64471c9b2843d"
            ]
          }
        },
        "9feff57e5f7041faa3dbc554a6fbbfd9": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "55c1ebbe8d7841d18751b6ea5a8af243": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_13ad8cf62bd140de9e83047ae726e611",
            "_dom_classes": [],
            "description": "Evaluation: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 69,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 69,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_dcc1331c5f4e459cbed4b5b181950cd9"
          }
        },
        "c925ba070f6244509ba64471c9b2843d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_5098ed899de84a8ebbcb1a33ea0f08dc",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 69/69 [00:26&lt;00:00,  2.61it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_dc74ab056eb74a499c478d4bb4e96c65"
          }
        },
        "13ad8cf62bd140de9e83047ae726e611": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "dcc1331c5f4e459cbed4b5b181950cd9": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "5098ed899de84a8ebbcb1a33ea0f08dc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "dc74ab056eb74a499c478d4bb4e96c65": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qSuUpkj1UuUa",
        "colab_type": "text"
      },
      "source": [
        "### © Copyright 2020 [George Mihaila](https://github.com/gmihaila)."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FkXPZsPbT2aV",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n",
        "# you may not use this file except in compliance with the License.\n",
        "# You may obtain a copy of the License at\n",
        "#\n",
        "# https://www.apache.org/licenses/LICENSE-2.0\n",
        "#\n",
        "# Unless required by applicable law or agreed to in writing, software\n",
        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
        "# See the License for the specific language governing permissions and\n",
        "# limitations under the License."
      ],
      "execution_count": 1,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "aP1zAHX4S70e",
        "colab_type": "text"
      },
      "source": [
        "# Info\n",
        "\n",
        "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/drive/14KCDms4YLrE7Ekxl9VtrdT229UTDyim3#offline=true&sandboxMode=true)\n",
        "[![Generic badge](https://img.shields.io/badge/GitHub-Source-greensvg)](https://github.com/gmihaila/machine_learning_things/blob/master/tutorial_notebooks/pretrain_transformer.ipynb)\n",
        "\n",
        "\n",
        "\n",
        "This notebook is used to pretrain transformers models using [Huggingface](https://huggingface.co/transformers/). This notebooks is part of my trusty notebooks for Machine Learning. Check out more similar content on my website [gmihaila.github.io/useful/useful/](https://gmihaila.github.io/useful/useful/) where I post useful notebooks like this one.\n",
        "\n",
        "This notebook is **heavily inspired** from the Huggingface script used for training language models: [transformers/tree/master/examples/language-modeling](https://github.com/huggingface/transformers/tree/master/examples/language-modeling).\n",
        "\n",
        "'Fine-tuning the library models for language modeling on a text file (GPT, GPT-2, CTRL, BERT, RoBERTa, XLNet).\n",
        "GPT, GPT-2 and CTRL are fine-tuned using a causal language modeling (CLM) loss. BERT and RoBERTa are fine-tuned\n",
        "using a masked language modeling (MLM) loss. XLNet is fine-tuned using a permutation language modeling (PLM) loss.'\n",
        "\n",
        "<br>\n",
        "\n",
        "## How to use this notebook? \n",
        "\n",
        "This notebooks is a code adaptation of the [run_language_modeling.py](https://github.com/huggingface/transformers/blob/master/examples/language-modeling/run_language_modeling.py). \n",
        "\n",
        "**Models that are guarantee to work:** [GPT](https://huggingface.co/transformers/model_summary.html#original-gpt), [GPT-2](https://huggingface.co/transformers/model_summary.html#gpt-2), [BERT](https://huggingface.co/transformers/model_summary.html#bert), [DistilBERT](https://huggingface.co/transformers/model_summary.html#distilbert), [RoBERTa](https://huggingface.co/transformers/model_summary.html#roberta) and [XLNet](https://huggingface.co/transformers/model_summary.html#xlnet). \n",
        "\n",
        "Parse the arguments needed that are split in TrainingArguments, ModelArguments and DataTrainingArguments. The only variables that need configuration depending on your needs are `model_args`, `data_args` and `training_args` in **Parameters**:\n",
        "\n",
        "* `model_args` of type **ModelArguments**: These are the arguments for the model that you want to use such as the model_name_or_path, tokenizer_name etc. You'll need these to load the model and tokenizer.\n",
        "\n",
        "  Minimum setup:\n",
        "\n",
        "  ```python\n",
        "  model_args = ModelArguments(model_name_or_path, \n",
        "                            model_type,\n",
        "                            tokenizer_name,\n",
        "                            )\n",
        "  ```\n",
        "\n",
        "  * `model_name_or_path` path to existing transformers model or name of transformer model to be used: *bert-base-cased*, *roberta-base*, *gpt2* etc. More details [here](https://huggingface.co/transformers/pretrained_models.html).\n",
        "\n",
        "  * `model_type` type of model used: *bert*, *roberta*, *gpt2*. More details [here](https://huggingface.co/transformers/pretrained_models.html).\n",
        "\n",
        "  * `tokenizer_name` [tokenizer](https://huggingface.co/transformers/main_classes/tokenizer.html#tokenizer) used to process data for training the model. It usually has same name as `model_name_or_path`: *bert-base-cased*, *roberta-base*, *gpt2* etc.\n",
        "\n",
        "\n",
        "* `data_args` of type **DataTrainingArguments**: These are as the name suggests arguments needed for the dataset. Such as the directory name where your files are stored etc. You'll need these to load/process the dataset.\n",
        "\n",
        "  Minimum setup:\n",
        "\n",
        "  ```python\n",
        "  data_args = DataArgs(train_data_file,\n",
        "                     eval_data_file,\n",
        "                     mlm,\n",
        "                     )\n",
        "  ```\n",
        "  \n",
        "  * `train_data_file` path to your dataset. This is a plain file that contains all your text data to train a model. Use each line to separate examples: i.e. if you have a dataset composed of multiple  text documents, create a single file with each line in the file associated to a text document.\n",
        "\n",
        "  * `eval_data_file` same story as `train_data_file`. This file is used to evaluate the model performance\n",
        "\n",
        "  * `mlm` is a flag that changes loss function depending on model architecture. This variable needs to be set to **True** when working with masked language models like *bert* or *roberta*.\n",
        "\n",
        "\n",
        "\n",
        "* `training_args` of type **TrainingArguments**: These are the training hyper-parameters such as learning rate, batch size, weight decay, gradient accumulation steps etc. See all possible arguments [here](https://github.com/huggingface/transformers/blob/master/src/transformers/training_args.py). These are used by the Trainer.\n",
        "\n",
        "  Minimum setup:\n",
        "\n",
        "* `model_args`\n",
        "  ```python\n",
        "  training_args = TrainingArguments(output_dir, \n",
        "                                  do_train, \n",
        "                                  do_eval,\n",
        "                                  )\n",
        "  ```\n",
        "\n",
        "  * `output_dir` path where to save the pre-trained model.\n",
        "  * `do_train` variable to signal if you're using train data or not. Set it to **True** if you mentioned `train_data_file`.\n",
        "  * `do_eval` variable to signal if you're using evaluate data or not. Set it to **True** if you mentioned `eval_data_file`.\n",
        "\n",
        "<br>\n",
        "\n",
        "## Example:\n",
        "\n",
        "### Pre-train Bert\n",
        "\n",
        "In the **Parameters** section use arguments:\n",
        "\n",
        "```python\n",
        "# process model arguments. Check Info - Notes for more details\n",
        "model_args = ModelArguments(model_name_or_path='bert-base-cased', \n",
        "                            model_type='bert',\n",
        "                            tokenizer_name='bert-base-cased',\n",
        "                            )\n",
        "\n",
        "# process data arguments. Check Info - Notes for more details\n",
        "data_args = DataArgs(train_data_file='/content/your_train_data',\n",
        "                     eval_data_file='/content/your_test_data,\n",
        "                     mlm=True,\n",
        "                     )\n",
        "\n",
        "# process training arguments. Check Info - Notes for more details\n",
        "training_args = TrainingArguments(output_dir='/content/pretrained_bert', \n",
        "                                  do_train=True, \n",
        "                                  do_eval=False)\n",
        "```\n",
        "\n",
        "\n",
        "<br>\n",
        "\n",
        "## Notes:\n",
        "* Parameters details got from [here](https://github.com/patil-suraj/exploring-T5/blob/master/T5_on_TPU.ipynb).\n",
        "\n",
        "* **Models that are guarantee to work:** [GPT](https://huggingface.co/transformers/model_summary.html#original-gpt), [GPT-2](https://huggingface.co/transformers/model_summary.html#gpt-2), [BERT](https://huggingface.co/transformers/model_summary.html#bert), [DistilBERT](https://huggingface.co/transformers/model_summary.html#distilbert), [RoBERTa](https://huggingface.co/transformers/model_summary.html#roberta) and [XLNet](https://huggingface.co/transformers/model_summary.html#xlnet). I plan on testing more models in the future.\n",
        "* I used the [The WikiText Long Term Dependency Language Modeling Dataset](https://blog.einstein.ai/the-wikitext-long-term-dependency-language-modeling-dataset/) as an example. **To reduce training time I used the evaluate split as training and test split as evaluation!**.\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pmIFMjnxXdI9",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 357
        },
        "outputId": "8d7e23ce-59ad-4d58-df43-7993a3d3c894"
      },
      "source": [
        "# check GPU alocation\n",
        "!nvidia-smi"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Sun Aug  9 14:42:51 2020       \n",
            "+-----------------------------------------------------------------------------+\n",
            "| NVIDIA-SMI 450.57       Driver Version: 418.67       CUDA Version: 10.1     |\n",
            "|-------------------------------+----------------------+----------------------+\n",
            "| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |\n",
            "| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |\n",
            "|                               |                      |               MIG M. |\n",
            "|===============================+======================+======================|\n",
            "|   0  Tesla T4            Off  | 00000000:00:04.0 Off |                    0 |\n",
            "| N/A   63C    P8    11W /  70W |      0MiB / 15079MiB |      0%      Default |\n",
            "|                               |                      |                 ERR! |\n",
            "+-------------------------------+----------------------+----------------------+\n",
            "                                                                               \n",
            "+-----------------------------------------------------------------------------+\n",
            "| Processes:                                                                  |\n",
            "|  GPU   GI   CI        PID   Type   Process name                  GPU Memory |\n",
            "|        ID   ID                                                   Usage      |\n",
            "|=============================================================================|\n",
            "|  No running processes found                                                 |\n",
            "+-----------------------------------------------------------------------------+\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8ppW60cUXZQK",
        "colab_type": "text"
      },
      "source": [
        "# Download"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6l_gehghXapy",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# download any dataset or manualy upload it. I will use the wikitext raw:\n",
        "!wget -q -nc https://s3.amazonaws.com/research.metamind.io/wikitext/wikitext-2-raw-v1.zip\n",
        "!unzip -q -n /content/wikitext-2-raw-v1.zip"
      ],
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "UCLtm5BiXona",
        "colab_type": "text"
      },
      "source": [
        "# Installs"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1JQhmThRXp7b",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 102
        },
        "outputId": "d85af845-8faf-437f-f8ac-15ca0b884015"
      },
      "source": [
        "# install latest version of Tranformers from GitHub\n",
        "!pip install -q git+https://github.com/huggingface/transformers"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\u001b[K     |████████████████████████████████| 3.0MB 15.8MB/s \n",
            "\u001b[K     |████████████████████████████████| 1.1MB 40.8MB/s \n",
            "\u001b[K     |████████████████████████████████| 890kB 44.9MB/s \n",
            "\u001b[?25h  Building wheel for transformers (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Building wheel for sacremoses (setup.py) ... \u001b[?25l\u001b[?25hdone\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "X5IO8-xrXvWY",
        "colab_type": "text"
      },
      "source": [
        "# Imports"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "J42h802BXwTe",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import logging\n",
        "import math\n",
        "import os\n",
        "from dataclasses import dataclass, field\n",
        "from typing import Optional\n",
        "from transformers import (\n",
        "                          CONFIG_MAPPING,\n",
        "                          MODEL_WITH_LM_HEAD_MAPPING,\n",
        "                          AutoConfig,\n",
        "                          AutoModelWithLMHead,\n",
        "                          AutoTokenizer,\n",
        "                          DataCollatorForLanguageModeling,\n",
        "                          DataCollatorForPermutationLanguageModeling,\n",
        "                          HfArgumentParser,\n",
        "                          LineByLineTextDataset,\n",
        "                          PreTrainedTokenizer,\n",
        "                          TextDataset,\n",
        "                          Trainer,\n",
        "                          TrainingArguments,\n",
        "                          set_seed,\n",
        "                          )\n",
        "\n",
        "# setup logger\n",
        "logger = logging.getLogger(__name__)\n",
        "# get names of models with language model heads\n",
        "MODEL_CONFIG_CLASSES = list(MODEL_WITH_LM_HEAD_MAPPING.keys())\n",
        "# get all model types\n",
        "MODEL_TYPES = tuple(conf.model_type for conf in MODEL_CONFIG_CLASSES)"
      ],
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "exr7h9hNYL8v",
        "colab_type": "text"
      },
      "source": [
        "# Helper Functions"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Cccuf4frXz0h",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class ModelArguments:\n",
        "  \"\"\"Class to define model arguments.\n",
        "  These are the arguments for the model that you want to use such as the \n",
        "  model_name_or_path, tokenizer_name etc. \n",
        "  You'll need these to load the model and tokenizer.\n",
        "\n",
        "  Arguments:\n",
        "  \n",
        "    model_name_or_path: path to existing transformers model or name of \n",
        "      transformer model to be used: bert-base-cased, roberta-base, gpt2 etc. \n",
        "      More details: https://huggingface.co/transformers/pretrained_models.html\n",
        "\n",
        "    model_type: type of model used: bert, roberta, gpt2. \n",
        "      More details: https://huggingface.co/transformers/pretrained_models.html\n",
        "\n",
        "    tokenizer_name: tokenizer used to process data for training the model. \n",
        "      It usually has same name as model_name_or_path: bert-base-cased, \n",
        "      roberta-base, gpt2 etc.\n",
        "    \n",
        "    cache_dir: path to cache files to save time when re-running code.\n",
        "  \"\"\"\n",
        "  def __init__(self, model_name_or_path=None, model_type=None, config_name=None, \n",
        "               tokenizer_name=None, cache_dir=None):\n",
        "    self.model_name_or_path = model_name_or_path\n",
        "    self.model_type = model_type\n",
        "    self.config_name = config_name\n",
        "    self.tokenizer_name = tokenizer_name\n",
        "    self.cache_dir = cache_dir\n",
        "\n",
        "    return\n",
        "\n",
        "\n",
        "class DataArguments:\n",
        "  \"\"\"Class to define data arguments.\n",
        "  Arguments needed for the dataset such as the directory name where your \n",
        "  files are stored etc. You'll need these to load/process the dataset.\n",
        "\n",
        "  Arguments:\n",
        "  \n",
        "    train_data_file: path to your dataset. This is a plain file that \n",
        "      contains all your text data to train a model. Use each line to separate \n",
        "      examples: i.e. if you have a dataset composed of multiple text documents, \n",
        "      create a single file with each line in the file associated \n",
        "      to each text document.\n",
        "\n",
        "    eval_data_file: same story as train_data_file. This file is used to evaluate \n",
        "      the model performance\n",
        "\n",
        "    line_by_line: if each line is associated to a specific example in \n",
        "      your dataset.\n",
        "\n",
        "    mlm: is a flag that changes loss function depending on model architecture. \n",
        "      This variable needs to be set to True when working with masked language \n",
        "      models like bert or roberta.\n",
        "\n",
        "    mlm_probability: used when training masked language models. \n",
        "      Needs to have mlm set to True. \n",
        "      It represents the probability of masking tokens when training model.\n",
        "\n",
        "    plm_probability: flag to define the ratio of length of a span of masked \n",
        "      tokens to surrounding context length for permutation language modeling.\n",
        "      Used for XLNet.\n",
        "\n",
        "    max_span_length: flag may also be used to limit the length of a span of \n",
        "      masked tokens used for permutation language modeling.\n",
        "      Used for XLNet.\n",
        "\n",
        "    block_size: it refers to block size of data. Set to -1 to use all data.\n",
        "\n",
        "    overwrite_cache: if there are any cached files, overwrite them.\n",
        "  \"\"\"\n",
        "\n",
        "  def __init__(self, train_data_file=None, eval_data_file=None, line_by_line=False, \n",
        "               mlm=False, mlm_probability=0.15, plm_probability=float(1/6), max_span_length=5,\n",
        "               block_size=-1, overwrite_cache=False):\n",
        "    self.train_data_file = train_data_file\n",
        "    self.eval_data_file = eval_data_file\n",
        "    self.line_by_line = line_by_line\n",
        "    self.mlm = mlm\n",
        "    self.mlm_probability = mlm_probability\n",
        "    self.plm_probability = plm_probability\n",
        "    self.max_span_length = max_span_length\n",
        "    self.block_size = block_size\n",
        "    self.overwrite_cache = overwrite_cache\n",
        "    return\n",
        "\n",
        "\n",
        "def get_dataset(args: DataArguments, tokenizer: PreTrainedTokenizer, evaluate=False):\n",
        "  \"\"\"Process dataset file\n",
        "  \"\"\"\n",
        "  file_path = args.eval_data_file if evaluate else args.train_data_file\n",
        "  if args.line_by_line:\n",
        "    return LineByLineTextDataset(tokenizer=tokenizer, file_path=file_path, block_size=args.block_size)\n",
        "  else:\n",
        "    return TextDataset(\n",
        "  tokenizer=tokenizer, file_path=file_path, block_size=args.block_size, overwrite_cache=args.overwrite_cache\n",
        "    )\n"
      ],
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "LVFOs6cSYvKT",
        "colab_type": "text"
      },
      "source": [
        "# Parameters"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RoIC40FJYwJm",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 88
        },
        "outputId": "a8587533-ee50-4a2a-81ff-20bdb42e7734"
      },
      "source": [
        "# process model arguments. Check Info - Notes for more details\n",
        "model_args = ModelArguments(model_name_or_path='bert-base-cased', \n",
        "                            model_type='bert',\n",
        "                            tokenizer_name='bert-base-cased',\n",
        "                            )\n",
        "\n",
        "# process data arguments. Check Info - Notes for more details\n",
        "data_args = DataArguments(train_data_file='/content/wikitext-2-raw/wiki.valid.raw',\n",
        "                          eval_data_file='/content/wikitext-2-raw/wiki.test.raw',\n",
        "                          line_by_line=False,\n",
        "                          mlm=True,\n",
        "                          )\n",
        "\n",
        "# process training arguments. Check Info - Notes for more details\n",
        "training_args = TrainingArguments(output_dir='pretrain_bert', \n",
        "                                  do_train=True, \n",
        "                                  do_eval=True,\n",
        "                                  overwrite_output_dir=True)\n",
        "\n",
        "# check arguments\n",
        "if data_args.eval_data_file is None and training_args.do_eval:\n",
        "  # make sure do_eval is set to False if no evaluate data\n",
        "  raise ValueError(\"Cannot do evaluation without an evaluation data file. \\\n",
        "  Either supply a file to --eval_data_file \\\n",
        "  or remove the --do_eval argument.\"\n",
        "  )\n",
        "\n",
        "if (\n",
        "    os.path.exists(training_args.output_dir)\n",
        "    and os.listdir(training_args.output_dir)\n",
        "    and training_args.do_train\n",
        "    and not training_args.overwrite_output_dir):\n",
        "    # make sure we set overwrite correct if fileas already exist in path\n",
        "    raise ValueError(f\"Output directory ({training_args.output_dir}) already \\\n",
        "    exists and is not empty. Use --overwrite_output_dir to overcome.\"\n",
        "    )\n",
        "\n",
        "# setup logger\n",
        "logging.basicConfig(\n",
        "    format=\"%(asctime)s - %(levelname)s - %(name)s -   %(message)s\",\n",
        "    datefmt=\"%m/%d/%Y %H:%M:%S\",\n",
        "    level=logging.INFO if training_args.local_rank in [-1, 0] else logging.WARN,\n",
        ")\n",
        "logger.warning(\n",
        "    \"Process rank: %s, device: %s, n_gpu: %s, distributed training: %s, 16-bits training: %s\",\n",
        "    training_args.local_rank,\n",
        "    training_args.device,\n",
        "    training_args.n_gpu,\n",
        "    bool(training_args.local_rank != -1),\n",
        "    training_args.fp16,\n",
        ")\n",
        "logger.info(\"Training/evaluation parameters %s\", training_args)\n",
        "\n",
        "# setup seed\n",
        "set_seed(training_args.seed)"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "08/09/2020 14:43:26 - INFO - transformers.training_args -   PyTorch: setting up devices\n",
            "08/09/2020 14:43:26 - WARNING - __main__ -   Process rank: -1, device: cuda:0, n_gpu: 1, distributed training: False, 16-bits training: False\n",
            "08/09/2020 14:43:26 - INFO - __main__ -   Training/evaluation parameters TrainingArguments(output_dir='pretrain_bert', overwrite_output_dir=True, do_train=True, do_eval=True, do_predict=False, evaluate_during_training=False, per_device_train_batch_size=8, per_device_eval_batch_size=8, per_gpu_train_batch_size=None, per_gpu_eval_batch_size=None, gradient_accumulation_steps=1, learning_rate=5e-05, weight_decay=0.0, adam_beta1=0.9, adam_beta2=0.999, adam_epsilon=1e-08, max_grad_norm=1.0, num_train_epochs=3.0, max_steps=-1, warmup_steps=0, logging_dir='runs/Aug09_14-43-26_431ba4944b5f', logging_first_step=False, logging_steps=500, save_steps=500, save_total_limit=None, no_cuda=False, seed=42, fp16=False, fp16_opt_level='O1', local_rank=-1, tpu_num_cores=None, tpu_metrics_debug=False, debug=False, dataloader_drop_last=False, eval_steps=1000, past_index=-1, run_name=None)\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "CQ8F7gwbbNio",
        "colab_type": "text"
      },
      "source": [
        "# Load Model and Tokenizer"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "H-cZPvObbPPr",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000,
          "referenced_widgets": [
            "5b22f6305d4d4be4b5ff4cd55a37a793",
            "13d461b4dc884a7c95a2c971a1fe1ff7",
            "14e396ff5ecf41c7b2ad1c0a88f8f44d",
            "93568a1211bb4ec6831b2a7a1706d45c",
            "8ccce63db8f740c4acf30a5984524887",
            "b0c769e363454565acbcff91e66bfa8b",
            "3e3dade805e14bcd950bc8dbe61919c9",
            "67e3b557ed944846ba9f4ccb814a7cea",
            "d19a2a7d0b0b406bb8c6242581756faa",
            "cefebaecd4ab4b559f9eb277be8a87de",
            "1872e7aa64e449639234c5619ff401d4",
            "a42f68f5d3964ccbb24a58c436c5ab66",
            "75b2d7819d1a49409cc028c2ec171185",
            "95eca277baa44ef1ada751b2135e435c",
            "8900998ecc724614b55437211b661438",
            "6f9e6fe87db24bc6a4007f38b4ef8d36",
            "6c5c1fdbcfba4584adeae63600fd1eef",
            "f6a5137958aa4f6cad1d1ff259c22433",
            "2170da5ca9874a799bc1efce37a76ee2",
            "90c6fcdbbdfe4029af1d42927d5be8bf",
            "7adb596bed44419ea65d7c3fa343ce3d",
            "479276e74d4a49bc9b81a0b4e1d0daad",
            "bfe3a93243f04b93ad4537e98a90d5d2",
            "d85b01ae45e74878b119db827f79ec92"
          ]
        },
        "outputId": "09bbcedc-1bf4-47f3-c786-3888ab316a24"
      },
      "source": [
        "# Load pretrained model and tokenizer\n",
        "#\n",
        "# Distributed training:\n",
        "# The .from_pretrained methods guarantee that only one local process can concurrently\n",
        "# download model & vocab.\n",
        "\n",
        "# check model configuration\n",
        "if model_args.config_name:\n",
        "  # use configure name if defined\n",
        "  config = AutoConfig.from_pretrained(model_args.config_name, cache_dir=model_args.cache_dir)\n",
        "elif model_args.model_name_or_path:\n",
        "  # use model name or path if defined\n",
        "  config = AutoConfig.from_pretrained(model_args.model_name_or_path, cache_dir=model_args.cache_dir)\n",
        "else:\n",
        "  # use config mapping\n",
        "  config = CONFIG_MAPPING[model_args.model_type]()\n",
        "  logger.warning(\"You are instantiating a new config instance from scratch.\")\n",
        "\n",
        "# check tokenizer configuraiton\n",
        "if model_args.tokenizer_name:\n",
        "  # use tokenizer name if define\n",
        "  tokenizer = AutoTokenizer.from_pretrained(model_args.tokenizer_name, cache_dir=model_args.cache_dir)\n",
        "elif model_args.model_name_or_path:\n",
        "  # use tokenizer name of path if defined\n",
        "  tokenizer = AutoTokenizer.from_pretrained(model_args.model_name_or_path, cache_dir=model_args.cache_dir)\n",
        "else:\n",
        "  # tokenizer configuration incorrect\n",
        "  raise ValueError(\"You are instantiating a new tokenizer from scratch. \\\n",
        "      This is not supported, but you can do it from another script, save it, \\\n",
        "      and load it from here, using --tokenizer_name\"\n",
        "      )\n",
        "\n",
        "# check if using pre-trained model or train from scratch\n",
        "if model_args.model_name_or_path:\n",
        "  # use pre-trained model\n",
        "  model = AutoModelWithLMHead.from_pretrained(\n",
        "      model_args.model_name_or_path,\n",
        "      from_tf=bool(\".ckpt\" in model_args.model_name_or_path),\n",
        "      config=config,\n",
        "      cache_dir=model_args.cache_dir,\n",
        "      )\n",
        "else:\n",
        "  # use model from configuration - train from scratch\n",
        "    logger.info(\"Training new model from scratch\")\n",
        "    model = AutoModelWithLMHead.from_config(config)\n",
        "\n",
        "# resize model to fit all tokens in tokenizer\n",
        "model.resize_token_embeddings(len(tokenizer))\n",
        "\n",
        "# make sure `--mlm` flag si set for masked language models\n",
        "if config.model_type in [\"bert\", \"roberta\", \"distilbert\", \"camembert\"] and not data_args.mlm:\n",
        "  raise ValueError(\"BERT and RoBERTa-like models do not have LM heads but \\\n",
        "  masked LM heads. They must be run using the --mlm flag \\\n",
        "  (masked language modeling).\"\n",
        "  )\n",
        "\n",
        "# setp data block size\n",
        "if data_args.block_size <= 0:\n",
        "  # set block size to maximum length of tokenizer\n",
        "  # input block size will be the max possible for the model\n",
        "  data_args.block_size = tokenizer.max_len\n",
        "else:\n",
        "  # never go beyond tokenzier maximum length\n",
        "  data_args.block_size = min(data_args.block_size, tokenizer.max_len)"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "08/09/2020 14:43:27 - INFO - filelock -   Lock 139780907161584 acquired on /root/.cache/torch/transformers/b945b69218e98b3e2c95acf911789741307dec43c698d35fad11c1ae28bda352.9da767be51e1327499df13488672789394e2ca38b877837e52618a67d7002391.lock\n",
            "08/09/2020 14:43:27 - INFO - transformers.file_utils -   https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-cased-config.json not found in cache or force_download set to True, downloading to /root/.cache/torch/transformers/tmp_g5qdb2_\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "5b22f6305d4d4be4b5ff4cd55a37a793",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=433.0, style=ProgressStyle(description_…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "08/09/2020 14:43:28 - INFO - transformers.file_utils -   storing https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-cased-config.json in cache at /root/.cache/torch/transformers/b945b69218e98b3e2c95acf911789741307dec43c698d35fad11c1ae28bda352.9da767be51e1327499df13488672789394e2ca38b877837e52618a67d7002391\n",
            "08/09/2020 14:43:28 - INFO - transformers.file_utils -   creating metadata file for /root/.cache/torch/transformers/b945b69218e98b3e2c95acf911789741307dec43c698d35fad11c1ae28bda352.9da767be51e1327499df13488672789394e2ca38b877837e52618a67d7002391\n",
            "08/09/2020 14:43:28 - INFO - filelock -   Lock 139780907161584 released on /root/.cache/torch/transformers/b945b69218e98b3e2c95acf911789741307dec43c698d35fad11c1ae28bda352.9da767be51e1327499df13488672789394e2ca38b877837e52618a67d7002391.lock\n",
            "08/09/2020 14:43:28 - INFO - transformers.configuration_utils -   loading configuration file https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-cased-config.json from cache at /root/.cache/torch/transformers/b945b69218e98b3e2c95acf911789741307dec43c698d35fad11c1ae28bda352.9da767be51e1327499df13488672789394e2ca38b877837e52618a67d7002391\n",
            "08/09/2020 14:43:28 - INFO - transformers.configuration_utils -   Model config BertConfig {\n",
            "  \"architectures\": [\n",
            "    \"BertForMaskedLM\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"gradient_checkpointing\": false,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"layer_norm_eps\": 1e-12,\n",
            "  \"max_position_embeddings\": 512,\n",
            "  \"model_type\": \"bert\",\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_hidden_layers\": 12,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"type_vocab_size\": 2,\n",
            "  \"vocab_size\": 28996\n",
            "}\n",
            "\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "08/09/2020 14:43:29 - INFO - transformers.configuration_utils -   loading configuration file https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-cased-config.json from cache at /root/.cache/torch/transformers/b945b69218e98b3e2c95acf911789741307dec43c698d35fad11c1ae28bda352.9da767be51e1327499df13488672789394e2ca38b877837e52618a67d7002391\n",
            "08/09/2020 14:43:29 - INFO - transformers.configuration_utils -   Model config BertConfig {\n",
            "  \"architectures\": [\n",
            "    \"BertForMaskedLM\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"gradient_checkpointing\": false,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"layer_norm_eps\": 1e-12,\n",
            "  \"max_position_embeddings\": 512,\n",
            "  \"model_type\": \"bert\",\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_hidden_layers\": 12,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"type_vocab_size\": 2,\n",
            "  \"vocab_size\": 28996\n",
            "}\n",
            "\n",
            "08/09/2020 14:43:30 - INFO - filelock -   Lock 139780907289400 acquired on /root/.cache/torch/transformers/5e8a2b4893d13790ed4150ca1906be5f7a03d6c4ddf62296c383f6db42814db2.e13dbb970cb325137104fb2e5f36fe865f27746c6b526f6352861b1980eb80b1.lock\n",
            "08/09/2020 14:43:30 - INFO - transformers.file_utils -   https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-cased-vocab.txt not found in cache or force_download set to True, downloading to /root/.cache/torch/transformers/tmpilekh8wo\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "d19a2a7d0b0b406bb8c6242581756faa",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=213450.0, style=ProgressStyle(descripti…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "08/09/2020 14:43:31 - INFO - transformers.file_utils -   storing https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-cased-vocab.txt in cache at /root/.cache/torch/transformers/5e8a2b4893d13790ed4150ca1906be5f7a03d6c4ddf62296c383f6db42814db2.e13dbb970cb325137104fb2e5f36fe865f27746c6b526f6352861b1980eb80b1\n",
            "08/09/2020 14:43:31 - INFO - transformers.file_utils -   creating metadata file for /root/.cache/torch/transformers/5e8a2b4893d13790ed4150ca1906be5f7a03d6c4ddf62296c383f6db42814db2.e13dbb970cb325137104fb2e5f36fe865f27746c6b526f6352861b1980eb80b1\n",
            "08/09/2020 14:43:31 - INFO - filelock -   Lock 139780907289400 released on /root/.cache/torch/transformers/5e8a2b4893d13790ed4150ca1906be5f7a03d6c4ddf62296c383f6db42814db2.e13dbb970cb325137104fb2e5f36fe865f27746c6b526f6352861b1980eb80b1.lock\n",
            "08/09/2020 14:43:31 - INFO - transformers.tokenization_utils_base -   loading file https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-cased-vocab.txt from cache at /root/.cache/torch/transformers/5e8a2b4893d13790ed4150ca1906be5f7a03d6c4ddf62296c383f6db42814db2.e13dbb970cb325137104fb2e5f36fe865f27746c6b526f6352861b1980eb80b1\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/transformers/modeling_auto.py:811: FutureWarning: The class `AutoModelWithLMHead` is deprecated and will be removed in a future version. Please use `AutoModelForCausalLM` for causal language models, `AutoModelForMaskedLM` for masked language models and `AutoModelForSeq2SeqLM` for encoder-decoder models.\n",
            "  FutureWarning,\n",
            "08/09/2020 14:43:32 - INFO - filelock -   Lock 139780907161136 acquired on /root/.cache/torch/transformers/d8f11f061e407be64c4d5d7867ee61d1465263e24085cfa26abf183fdc830569.3fadbea36527ae472139fe84cddaa65454d7429f12d543d80bfc3ad70de55ac2.lock\n",
            "08/09/2020 14:43:32 - INFO - transformers.file_utils -   https://cdn.huggingface.co/bert-base-cased-pytorch_model.bin not found in cache or force_download set to True, downloading to /root/.cache/torch/transformers/tmpfxci_az3\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "6c5c1fdbcfba4584adeae63600fd1eef",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=435779157.0, style=ProgressStyle(descri…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "08/09/2020 14:43:38 - INFO - transformers.file_utils -   storing https://cdn.huggingface.co/bert-base-cased-pytorch_model.bin in cache at /root/.cache/torch/transformers/d8f11f061e407be64c4d5d7867ee61d1465263e24085cfa26abf183fdc830569.3fadbea36527ae472139fe84cddaa65454d7429f12d543d80bfc3ad70de55ac2\n",
            "08/09/2020 14:43:38 - INFO - transformers.file_utils -   creating metadata file for /root/.cache/torch/transformers/d8f11f061e407be64c4d5d7867ee61d1465263e24085cfa26abf183fdc830569.3fadbea36527ae472139fe84cddaa65454d7429f12d543d80bfc3ad70de55ac2\n",
            "08/09/2020 14:43:38 - INFO - filelock -   Lock 139780907161136 released on /root/.cache/torch/transformers/d8f11f061e407be64c4d5d7867ee61d1465263e24085cfa26abf183fdc830569.3fadbea36527ae472139fe84cddaa65454d7429f12d543d80bfc3ad70de55ac2.lock\n",
            "08/09/2020 14:43:38 - INFO - transformers.modeling_utils -   loading weights file https://cdn.huggingface.co/bert-base-cased-pytorch_model.bin from cache at /root/.cache/torch/transformers/d8f11f061e407be64c4d5d7867ee61d1465263e24085cfa26abf183fdc830569.3fadbea36527ae472139fe84cddaa65454d7429f12d543d80bfc3ad70de55ac2\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "08/09/2020 14:43:43 - WARNING - transformers.modeling_utils -   Some weights of the model checkpoint at bert-base-cased were not used when initializing BertForMaskedLM: ['cls.seq_relationship.weight', 'cls.seq_relationship.bias']\n",
            "- This IS expected if you are initializing BertForMaskedLM from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPretraining model).\n",
            "- This IS NOT expected if you are initializing BertForMaskedLM from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "08/09/2020 14:43:43 - WARNING - transformers.modeling_utils -   Some weights of BertForMaskedLM were not initialized from the model checkpoint at bert-base-cased and are newly initialized: ['cls.predictions.decoder.bias']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "/usr/local/lib/python3.6/dist-packages/transformers/tokenization_utils_base.py:1294: FutureWarning: The `max_len` attribute has been deprecated and will be removed in a future version, use `model_max_length` instead.\n",
            "  FutureWarning,\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "znKwkO9oeRC6",
        "colab_type": "text"
      },
      "source": [
        "# Dataset"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "acmLQCeZeUpJ",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 173
        },
        "outputId": "cba4fe2e-3fae-48fe-ce2d-4695ca84597a"
      },
      "source": [
        "# setup train dataset if `do_train` is set\n",
        "train_dataset = get_dataset(data_args, tokenizer=tokenizer) if training_args.do_train else None\n",
        "\n",
        "# setup evaluation dataset if `do_eval` is set\n",
        "eval_dataset = get_dataset(data_args, tokenizer=tokenizer, evaluate=True) if training_args.do_eval else None\n",
        "\n",
        "# special dataset handle depending on model type\n",
        "if config.model_type == \"xlnet\":\n",
        "  # configure data for XLNET\n",
        "  data_collator = DataCollatorForPermutationLanguageModeling(\n",
        "      tokenizer=tokenizer, \n",
        "      plm_probability=data_args.plm_probability, \n",
        "      max_span_length=data_args.max_span_length,\n",
        "      )\n",
        "else:\n",
        "  # configure data for rest of model types\n",
        "  data_collator = DataCollatorForLanguageModeling(\n",
        "      tokenizer=tokenizer, \n",
        "      mlm=data_args.mlm, \n",
        "      mlm_probability=data_args.mlm_probability\n",
        "      )"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "08/09/2020 14:43:43 - INFO - filelock -   Lock 139780907175328 acquired on /content/wikitext-2-raw/cached_lm_BertTokenizer_510_wiki.valid.raw.lock\n",
            "08/09/2020 14:43:43 - INFO - transformers.data.datasets.language_modeling -   Creating features from dataset file at /content/wikitext-2-raw\n",
            "08/09/2020 14:43:45 - INFO - transformers.data.datasets.language_modeling -   Saving features into cached file /content/wikitext-2-raw/cached_lm_BertTokenizer_510_wiki.valid.raw [took 0.008 s]\n",
            "08/09/2020 14:43:45 - INFO - filelock -   Lock 139780907175328 released on /content/wikitext-2-raw/cached_lm_BertTokenizer_510_wiki.valid.raw.lock\n",
            "08/09/2020 14:43:45 - INFO - filelock -   Lock 139783131337168 acquired on /content/wikitext-2-raw/cached_lm_BertTokenizer_510_wiki.test.raw.lock\n",
            "08/09/2020 14:43:45 - INFO - transformers.data.datasets.language_modeling -   Creating features from dataset file at /content/wikitext-2-raw\n",
            "08/09/2020 14:43:47 - INFO - transformers.data.datasets.language_modeling -   Saving features into cached file /content/wikitext-2-raw/cached_lm_BertTokenizer_510_wiki.test.raw [took 0.009 s]\n",
            "08/09/2020 14:43:47 - INFO - filelock -   Lock 139783131337168 released on /content/wikitext-2-raw/cached_lm_BertTokenizer_510_wiki.test.raw.lock\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "pSM9mLfKeT8m",
        "colab_type": "text"
      },
      "source": [
        "# Trainer"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oYiAHslWe_0S",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 556,
          "referenced_widgets": [
            "7c7be961fe3c4fbebd6842599c6e4cee",
            "b7534c3a6486497fb29fb1ff79187666",
            "96e5b4e416184710a7ac656e36258612",
            "28b801cb34e547b89f0c25ac6d8b6671",
            "d57e431fa76646d49af035782eff6c6e",
            "3dd6aada08e947699cfc3890c4f14990",
            "0f82fac089484ff3802dd5bca6c1f342",
            "c1a90eae6dba4800a25dd6e60ab6d04f",
            "48bcf3b1343b48c48cda53021f1bddb4",
            "58547604efc742bdba897eb5703150e7",
            "fa31d116e3cf4e2e9eecb9785944dc7f",
            "551a97d96d744bffab7f6a570ee9cf23",
            "7347d675b7794bfd989e6643fe5da3d4",
            "721312449a344a46bdbe101a0c059e45",
            "74398199f8144a3dbd5611812b7a6f04",
            "06e9cd380db84d02bbb74e77c85f2314",
            "ce438e273ddc46bc90f039a3794c700b",
            "5e87074a2bef465fa158c4b335af7124",
            "ee4c91c46fd54609a5eee3d9620aa660",
            "313fa40c7e24405f9906be3eca09d424",
            "222d8e29cc264224bf43b67bf3fbaca4",
            "881feace35c249358b16c2c219b7ea2b",
            "b052b6cdac6248bc84f9ad09e3a4ee9f",
            "0976f817303e4b9dac0110902b45cd69",
            "d3abee198a4847f1b2abc06f8fadeb87",
            "8343ec37744445f6a46ecf2a565fec5d",
            "f261f14c20514175856a617d1b688ff4",
            "9fb431b6108e4752a72f62054be54812",
            "49032bd7531143fcaae08b455112d5e3",
            "ed227ed0cf6d49ad93d5939db1ad5c84",
            "f57fdf2386c94eab838090229fc32a33",
            "a6b7a950a0f144b09166a32b0838c463"
          ]
        },
        "outputId": "a04e7ef7-2fe6-4a0d-a6c9-2dd95009420e"
      },
      "source": [
        "# initialize Trainer\n",
        "trainer = Trainer(model=model,\n",
        "                  args=training_args,\n",
        "                  data_collator=data_collator,\n",
        "                  train_dataset=train_dataset,\n",
        "                  eval_dataset=eval_dataset,\n",
        "                  prediction_loss_only=True,\n",
        "                  )\n",
        "\n",
        "# check model path to save\n",
        "if training_args.do_train:\n",
        "  # setup model path is already defined\n",
        "  model_path = (model_args.model_name_or_path \n",
        "                if model_args.model_name_or_path is not None and \n",
        "                os.path.isdir(model_args.model_name_or_path) \n",
        "                else None\n",
        "                )\n",
        "  trainer.train(model_path=model_path)\n",
        "  trainer.save_model()\n",
        "  # For convenience, we also re-save the tokenizer to the same directory,\n",
        "  # so that you can share your model easily on huggingface.co/models =)\n",
        "  if trainer.is_world_master():\n",
        "    tokenizer.save_pretrained(training_args.output_dir)"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "08/09/2020 14:44:01 - INFO - transformers.trainer -   You are instantiating a Trainer but W&B is not installed. To use wandb logging, run `pip install wandb; wandb login` see https://docs.wandb.com/huggingface.\n",
            "08/09/2020 14:44:01 - INFO - transformers.trainer -   To use comet_ml logging, run `pip/conda install comet_ml` see https://www.comet.ml/docs/python-sdk/huggingface/\n",
            "08/09/2020 14:44:01 - INFO - transformers.trainer -   ***** Running training *****\n",
            "08/09/2020 14:44:01 - INFO - transformers.trainer -     Num examples = 481\n",
            "08/09/2020 14:44:01 - INFO - transformers.trainer -     Num Epochs = 3\n",
            "08/09/2020 14:44:01 - INFO - transformers.trainer -     Instantaneous batch size per device = 8\n",
            "08/09/2020 14:44:01 - INFO - transformers.trainer -     Total train batch size (w. parallel, distributed & accumulation) = 8\n",
            "08/09/2020 14:44:01 - INFO - transformers.trainer -     Gradient Accumulation steps = 1\n",
            "08/09/2020 14:44:01 - INFO - transformers.trainer -     Total optimization steps = 183\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "7c7be961fe3c4fbebd6842599c6e4cee",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Epoch', max=3.0, style=ProgressStyle(description_width='i…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "48bcf3b1343b48c48cda53021f1bddb4",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Iteration', max=61.0, style=ProgressStyle(description_wid…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "ce438e273ddc46bc90f039a3794c700b",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Iteration', max=61.0, style=ProgressStyle(description_wid…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "d3abee198a4847f1b2abc06f8fadeb87",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Iteration', max=61.0, style=ProgressStyle(description_wid…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "08/09/2020 14:47:17 - INFO - transformers.trainer -   \n",
            "\n",
            "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
            "\n",
            "\n",
            "08/09/2020 14:47:17 - INFO - transformers.trainer -   Saving model checkpoint to pretrain_bert\n",
            "08/09/2020 14:47:17 - INFO - transformers.configuration_utils -   Configuration saved in pretrain_bert/config.json\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "08/09/2020 14:47:19 - INFO - transformers.modeling_utils -   Model weights saved in pretrain_bert/pytorch_model.bin\n",
            "/usr/local/lib/python3.6/dist-packages/transformers/trainer.py:795: FutureWarning: This method is deprecated, use `Trainer.is_world_process_zero()` instead.\n",
            "  warnings.warn(\"This method is deprecated, use `Trainer.is_world_process_zero()` instead.\", FutureWarning)\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "PJuUovebgBkA",
        "colab_type": "text"
      },
      "source": [
        "# Evaluate"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "V4_2kHS2gCg6",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 256,
          "referenced_widgets": [
            "50da31b49a7040cbb682a005ed0f54e4",
            "9feff57e5f7041faa3dbc554a6fbbfd9",
            "55c1ebbe8d7841d18751b6ea5a8af243",
            "c925ba070f6244509ba64471c9b2843d",
            "13ad8cf62bd140de9e83047ae726e611",
            "dcc1331c5f4e459cbed4b5b181950cd9",
            "5098ed899de84a8ebbcb1a33ea0f08dc",
            "dc74ab056eb74a499c478d4bb4e96c65"
          ]
        },
        "outputId": "d8b76a93-1a86-4ba3-c66e-ec3102cae3d4"
      },
      "source": [
        "# save results\n",
        "results = {}\n",
        "\n",
        "# check if `do_eval` flag is set\n",
        "if training_args.do_eval:\n",
        "  # evaluate mode on evaluate data\n",
        "  logger.info(\"*** Evaluate ***\")\n",
        "  # capture output if trainer evaluate\n",
        "  eval_output = trainer.evaluate()\n",
        "  # compute perplexity from model loss\n",
        "  perplexity = math.exp(eval_output[\"eval_loss\"])\n",
        "  # save perplexity of results\n",
        "  result = {\"perplexity\": perplexity}\n",
        "  # set path for output evaluation file\n",
        "  output_eval_file = os.path.join(training_args.output_dir, \"eval_results_lm.txt\")\n",
        "  # dump evaluaiton results to file\n",
        "  if trainer.is_world_master():\n",
        "    with open(output_eval_file, \"w\") as writer:\n",
        "      logger.info(\"***** Eval results *****\")\n",
        "      for key in sorted(result.keys()):\n",
        "        logger.info(\"  %s = %s\", key, str(result[key]))\n",
        "        writer.write(\"%s = %s\\n\" % (key, str(result[key])))\n",
        "  # update results\n",
        "  results.update(result)\n",
        "\n",
        "print(\"results: \",results)"
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "08/09/2020 14:47:19 - INFO - __main__ -   *** Evaluate ***\n",
            "08/09/2020 14:47:19 - INFO - transformers.trainer -   ***** Running Evaluation *****\n",
            "08/09/2020 14:47:19 - INFO - transformers.trainer -     Num examples = 552\n",
            "08/09/2020 14:47:19 - INFO - transformers.trainer -     Batch size = 8\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "50da31b49a7040cbb682a005ed0f54e4",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Evaluation', max=69.0, style=ProgressStyle(description_wi…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/transformers/trainer.py:795: FutureWarning: This method is deprecated, use `Trainer.is_world_process_zero()` instead.\n",
            "  warnings.warn(\"This method is deprecated, use `Trainer.is_world_process_zero()` instead.\", FutureWarning)\n",
            "08/09/2020 14:47:45 - INFO - __main__ -   ***** Eval results *****\n",
            "08/09/2020 14:47:45 - INFO - __main__ -     perplexity = 15.721432898648294\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "{'eval_loss': 2.7550249341605366, 'epoch': 3.0, 'step': 183}\n",
            "results:  {'perplexity': 15.721432898648294}\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}
