{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "pretrain_transformers_pytorch.ipynb",
      "provenance": [],
      "collapsed_sections": [
        "qSuUpkj1UuUa"
      ],
      "toc_visible": true,
      "authorship_tag": "ABX9TyMvOFpb4CvM3bmU6ASVu1bi"
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "e1f3c3cc875f48ddbfe923af092ca99d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_e6205bf4b19240f69f46d40aabd64652",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_ed08b7e3b0304a139c09c51487bba7f1",
              "IPY_MODEL_440cbdd7d1534c7fbd050eda7bc33362"
            ]
          }
        },
        "e6205bf4b19240f69f46d40aabd64652": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "ed08b7e3b0304a139c09c51487bba7f1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_82406385fca142d39cbd778fe8e99f7c",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 433,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 433,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_e4dd96e1d09442debfac1e7d6aeb8a25"
          }
        },
        "440cbdd7d1534c7fbd050eda7bc33362": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_9aafd356ed844b408805c69a921495ec",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 433/433 [00:01&lt;00:00, 285B/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_ad39829093994961b829c57839f15f15"
          }
        },
        "82406385fca142d39cbd778fe8e99f7c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "e4dd96e1d09442debfac1e7d6aeb8a25": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "9aafd356ed844b408805c69a921495ec": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "ad39829093994961b829c57839f15f15": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "130b863b507547fdad72d766ad77e116": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_f87be544429e4cb488437ab28c0d450d",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_f91adcde493d4afca428633712fd3d50",
              "IPY_MODEL_cec990db7c684d53a59a58313254dcaa"
            ]
          }
        },
        "f87be544429e4cb488437ab28c0d450d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "f91adcde493d4afca428633712fd3d50": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_5ca2f0623a71444db224a540232e3059",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 213450,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 213450,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_72663e6053ce40eba183785abc9a8008"
          }
        },
        "cec990db7c684d53a59a58313254dcaa": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_80a531aac29347829de3f48638e2e87f",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 213k/213k [00:00&lt;00:00, 663kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_4e0402a8e583424da0033bdf7b82b07a"
          }
        },
        "5ca2f0623a71444db224a540232e3059": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "72663e6053ce40eba183785abc9a8008": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "80a531aac29347829de3f48638e2e87f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "4e0402a8e583424da0033bdf7b82b07a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "3424afb42d45481fbf959945b7e85082": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_922429f8d3f6467eb73d581117d2fc2c",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_4eb76130d1ad4f5aa606ead0f63a298d",
              "IPY_MODEL_4b94bb215ca549519ab4b9c7083c7c86"
            ]
          }
        },
        "922429f8d3f6467eb73d581117d2fc2c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "4eb76130d1ad4f5aa606ead0f63a298d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_0821f3990f4841dc992c22a4c7b06abe",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 435779157,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 435779157,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_1f202c5015934273b1dbeb6f5fd47162"
          }
        },
        "4b94bb215ca549519ab4b9c7083c7c86": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_d368e63449684840aeb59ff5593d1cdd",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 436M/436M [00:36&lt;00:00, 11.9MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_fa68b3d790394966a03a2b545fa03b92"
          }
        },
        "0821f3990f4841dc992c22a4c7b06abe": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "1f202c5015934273b1dbeb6f5fd47162": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "d368e63449684840aeb59ff5593d1cdd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "fa68b3d790394966a03a2b545fa03b92": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "b255bc3521dd4dc9a7b13f727d966f39": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_48a1f197c13b410a9212b69b207cd30b",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_b94f1b75df744f4894cb9522f7d13389",
              "IPY_MODEL_a512c47cfafa4e9bb23f984c0428027c"
            ]
          }
        },
        "48a1f197c13b410a9212b69b207cd30b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "b94f1b75df744f4894cb9522f7d13389": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_ce58fdcbc43f4393a309177fc09373dd",
            "_dom_classes": [],
            "description": "neg: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 12500,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 12500,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_6a1877f99f7a4cedb25d0a217f540495"
          }
        },
        "a512c47cfafa4e9bb23f984c0428027c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_1200d13dacbe432188a580fc57a48c0a",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 12500/12500 [00:55&lt;00:00, 224.11files/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_dbbe72f10ba64937a9440b99918bfa7a"
          }
        },
        "ce58fdcbc43f4393a309177fc09373dd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "6a1877f99f7a4cedb25d0a217f540495": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "1200d13dacbe432188a580fc57a48c0a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "dbbe72f10ba64937a9440b99918bfa7a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "57adf7d24e37426ebcd6fbf84cdacdd5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_89767f322bc24e65aacf67d84cc3dd55",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_8545893ba7a3438d9da84b02fe012df9",
              "IPY_MODEL_4b13ce2d46e347c9a8cd0fc2d55d0359"
            ]
          }
        },
        "89767f322bc24e65aacf67d84cc3dd55": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "8545893ba7a3438d9da84b02fe012df9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_5ca520d86501455294e4072416e4c228",
            "_dom_classes": [],
            "description": "pos: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 12500,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 12500,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_3ebd47c258a04766bca1ba838fd3bc5f"
          }
        },
        "4b13ce2d46e347c9a8cd0fc2d55d0359": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_e90ae6e058d742359162d6589c9926dd",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 12500/12500 [00:44&lt;00:00, 278.36files/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_b82a00647843451f83d72a13f6434da4"
          }
        },
        "5ca520d86501455294e4072416e4c228": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "3ebd47c258a04766bca1ba838fd3bc5f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "e90ae6e058d742359162d6589c9926dd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "b82a00647843451f83d72a13f6434da4": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "215e2bca7399439f8a2d02bfd32cf3d2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_488cbd647cdc47149f8a02fd1ffbc72a",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_575983162afd4b7da3b1e6439b03a6fe",
              "IPY_MODEL_09232a092df14fec8e9955dbc8d4ee3b"
            ]
          }
        },
        "488cbd647cdc47149f8a02fd1ffbc72a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "575983162afd4b7da3b1e6439b03a6fe": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_4f1af0399c064a2c81780c794a2ca175",
            "_dom_classes": [],
            "description": "neg: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 12500,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 12500,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_b65e403ad42c4292b8e4ac65d1bed9e0"
          }
        },
        "09232a092df14fec8e9955dbc8d4ee3b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_5c714855611e4e7a9819a3646d2ee7a8",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 12500/12500 [00:33&lt;00:00, 375.15files/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_734a6cc541384028a1ca1fd858360088"
          }
        },
        "4f1af0399c064a2c81780c794a2ca175": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "b65e403ad42c4292b8e4ac65d1bed9e0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "5c714855611e4e7a9819a3646d2ee7a8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "734a6cc541384028a1ca1fd858360088": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "4c98b756aa864f0ba4278a88640d1a1b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_c25bf2883da54023aaa2338e02da02f9",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_6b6e758549a84f2f8c20cf2d874caea7",
              "IPY_MODEL_309340d744c242ffb8a9b4e46ab90a76"
            ]
          }
        },
        "c25bf2883da54023aaa2338e02da02f9": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "6b6e758549a84f2f8c20cf2d874caea7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_0061fff440564e02a3d838f0d64df14e",
            "_dom_classes": [],
            "description": "pos: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 12500,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 12500,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_0a9b044241a742f2b5670dc60f5e0c97"
          }
        },
        "309340d744c242ffb8a9b4e46ab90a76": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_b9e57cc1e7bb45f3a399a16ccb55633f",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 12500/12500 [00:22&lt;00:00, 556.08files/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_7dd9fe7aadb74797b4a852aaddfb2a02"
          }
        },
        "0061fff440564e02a3d838f0d64df14e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "0a9b044241a742f2b5670dc60f5e0c97": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "b9e57cc1e7bb45f3a399a16ccb55633f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "7dd9fe7aadb74797b4a852aaddfb2a02": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qSuUpkj1UuUa"
      },
      "source": [
        "### © Copyright 2020 [George Mihaila](https://github.com/gmihaila)."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FkXPZsPbT2aV"
      },
      "source": [
        "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n",
        "# you may not use this file except in compliance with the License.\n",
        "# You may obtain a copy of the License at\n",
        "#\n",
        "# https://www.apache.org/licenses/LICENSE-2.0\n",
        "#\n",
        "# Unless required by applicable law or agreed to in writing, software\n",
        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
        "# See the License for the specific language governing permissions and\n",
        "# limitations under the License."
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "aP1zAHX4S70e"
      },
      "source": [
        "# **Pretrain Transformers Models in PyTorch using Hugging Face Transformers**\n",
        "\n",
        "## **Pretrain 67 transformers models on your custom dataset.**\n",
        "\n",
        "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/gmihaila/ml_things/blob/master/notebooks/pytorch/pretrain_transformers_pytorch.ipynb) &nbsp;\n",
        "[![Generic badge](https://img.shields.io/badge/GitHub-Source-greensvg)](https://github.com/gmihaila/ml_things/blob/master/notebooks/pytorch/pretrain_transformers_pytorch.ipynb)\n",
        "[![Generic badge](https://img.shields.io/badge/Article-Medium-black.svg)](https://medium.com/@gmihaila/fine-tune-transformers-in-pytorch-using-transformers-57b40450635)\n",
        "[![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0)\n",
        "\n",
        "\n",
        "<br>\n",
        "\n",
        "**Disclaimer:** *The format of this tutorial notebook is very similar with my other tutorial notebooks. This is done intentionally in order to keep readers familiar with my format.*\n",
        "\n",
        "<br>\n",
        "\n",
        "This notebook is used to pretrain transformers models using [Huggingface](https://huggingface.co/transformers/) on your own custom dataset.\n",
        "\n",
        "With the AutoClasses functionality we can reuse the code on a large number of transformers models!\n",
        "\n",
        "This notebook is designed to:\n",
        "\n",
        "* **Use an already pretrained transformers model and fine-tune it on your custom dataset.**\n",
        "\n",
        "* **Train a transformer model from scratch on a custom dataset.** This requires an already trained tokenizer. This notebook will use by default the pretrained tokenizer if a already trained tokenizer is no provided.\n",
        "\n",
        "This notebook is **very heavily inspired** from the Hugging Face script used for training language models: [transformers/tree/master/examples/language-modeling](https://github.com/huggingface/transformers/tree/master/examples/language-modeling). I basically adapted that script to work nicely in a notebook with a lot more comments.\n",
        "\n",
        "**Notes from [transformers/tree/master/examples/language-modeling](https://github.com/huggingface/transformers/tree/master/examples/language-modeling):** *Fine-tuning the library models for language modeling on a text file (GPT, GPT-2, CTRL, BERT, RoBERTa, XLNet).\n",
        "GPT, GPT-2 and CTRL are fine-tuned using a causal language modeling (CLM) loss. BERT and RoBERTa are fine-tuned\n",
        "using a masked language modeling (MLM) loss. XLNet is fine-tuned using a permutation language modeling (PLM) loss.*\n",
        "\n",
        "\n",
        "<br>\n",
        "\n",
        "## **What should I know for this notebook?**\n",
        "\n",
        "Since I am using PyTorch to fine-tune our transformers models any knowledge on PyTorch is very useful.\n",
        "\n",
        "Knowing a little bit about the [transformers](https://github.com/huggingface/transformers) library helps too.\n",
        "\n",
        "In this notebook **I am using raw text data to train / fine-tune transformers models** (if I use a pretrained model I like to call this *extended pretraining* since I 'continue' the original training of the model on a custom dataset). There is no need for labeled data since we are not doing classification. The Transformers library handles the text files in same way as the original implementation of each model.\n",
        "\n",
        "<br>\n",
        "\n",
        "## **How to use this notebook?**\n",
        "\n",
        "\n",
        "Like with every project, I built this notebook with reusability in mind. This notebook pulls the custom dataset form `.txt` files. Since the dataset does not come in a single `.txt` file I created a custom function `movie_reviews_to_file` that reads the dataset and creates the `text` file. The way I load the `.txt` files can be easily reused for any other dataset.\n",
        "\n",
        "The only modifications needed to use your own dataset will be in the paths provided to the train `.txt` file and evaluation `.txt` file.\n",
        "\n",
        "All parameters that can be changed are under the **Parameters Setup** section. Each parameter is nicely commented and structured to be as intuitive as possible.\n",
        "\n",
        "<br>\n",
        "\n",
        "\n",
        "## **What transformers models work with this notebook?**\n",
        "\n",
        "A lot of people will probably use it for Bert. When there is a need to run a different transformer model architecture, which one would work with this code?\n",
        "Since the name of the notebooks is **pretrain_transformers** it should work with more than one type of transformers.\n",
        "\n",
        "I ran this notebook across all the pretrained models found on Hugging Face Transformer. This way you know ahead of time if the model you plan to use works with this code without any modifications.\n",
        "\n",
        "The list of pretrained transformers models that work with this notebook can be found [here](https://github.com/gmihaila/ml_things/blob/master/notebooks/pytorch/pretrain_transformers_pytorch_status_models.md). There are **67 models that worked** 😄  and 39 models that failed to work 😢 with this notebook. *Remember these are pretrained models and fine-tuned on custom dataset.*\n",
        "\n",
        "\n",
        "<br>\n",
        "\n",
        "## **Dataset**\n",
        "\n",
        "This notebook will cover pretraining transformers on a custom dataset. I will use the well known movies reviews positive - negative labeled [Large Movie Review Dataset](https://ai.stanford.edu/~amaas/data/sentiment/).\n",
        "\n",
        "The description provided on the Stanford website:\n",
        "\n",
        "*This is a dataset for binary sentiment classification containing substantially more data than previous benchmark datasets. We provide a set of 25,000 highly polar movie reviews for training, and 25,000 for testing. There is additional unlabeled data for use as well. Raw text and already processed bag of words formats are provided. See the README file contained in the release for more details.*\n",
        "\n",
        "**Why this dataset?** I believe is an easy to understand and use dataset for classification. I think sentiment data is always fun to work with.\n",
        "\n",
        "<br>\n",
        "\n",
        "## **Coding**\n",
        "\n",
        "Now let's do some coding! We will go through each coding cell in the notebook and describe what it does, what's the code, and when is relevant - show the output.\n",
        "\n",
        "I made this format to be easy to follow if you decide to run each code cell in your own python notebook.\n",
        "\n",
        "When I learn from a tutorial I always try to replicate the results. I believe it's easy to follow along if you have the code next to the explanations.\n",
        "\n",
        "<br>\n",
        " "
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8ppW60cUXZQK"
      },
      "source": [
        "## **Downloads**\n",
        "\n",
        "Download the *Large Movie Review Dataset* and unzip it locally.\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pcWRle4foNcD"
      },
      "source": [
        "# Download the dataset.\n",
        "!wget -q -nc http://ai.stanford.edu/~amaas/data/sentiment/aclImdb_v1.tar.gz\n",
        "# Unzip the dataset.\n",
        "!tar -zxf /content/aclImdb_v1.tar.gz"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "UCLtm5BiXona"
      },
      "source": [
        "## **Installs**\n",
        "\n",
        "* **[transformers](https://github.com/huggingface/transformers)** library needs to be installed to use all the awesome code from Hugging Face. To get the latest version I will install it straight from GitHub.\n",
        "\n",
        "* **[ml_things](https://github.com/gmihaila/ml_things)** library used for various machine learning related tasks. I created this library to reduce the amount of code I need to write for each machine learning project.\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1JQhmThRXp7b",
        "outputId": "3625e83b-ba95-40a4-d6e5-c4b09461c5f8",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        }
      },
      "source": [
        "# Install transformers library.\n",
        "!pip install -q git+https://github.com/huggingface/transformers.git\n",
        "# Install helper functions.\n",
        "!pip install -q git+https://github.com/gmihaila/ml_things.git"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "  Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n",
            "  Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n",
            "    Preparing wheel metadata ... \u001b[?25l\u001b[?25hdone\n",
            "\u001b[K     |████████████████████████████████| 890kB 8.8MB/s \n",
            "\u001b[K     |████████████████████████████████| 1.1MB 45.7MB/s \n",
            "\u001b[K     |████████████████████████████████| 2.9MB 51.5MB/s \n",
            "\u001b[?25h  Building wheel for transformers (PEP 517) ... \u001b[?25l\u001b[?25hdone\n",
            "  Building wheel for sacremoses (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "\u001b[K     |████████████████████████████████| 71kB 4.8MB/s \n",
            "\u001b[?25h  Building wheel for ml-things (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Building wheel for ftfy (setup.py) ... \u001b[?25l\u001b[?25hdone\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "X5IO8-xrXvWY"
      },
      "source": [
        "## **Imports**\n",
        "\n",
        "Import all needed libraries for this notebook.\n",
        "\n",
        "Declare basic parameters used for this notebook:\n",
        "\n",
        "* `set_seed(123)` - Always good to set a fixed seed for reproducibility.\n",
        "\n",
        "* `device` - Look for gpu to use. I will use cpu by default if no gpu found."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ux2sgMCvq8PE"
      },
      "source": [
        "import io\n",
        "import os\n",
        "import math\n",
        "import torch\n",
        "import warnings\n",
        "from tqdm.notebook import tqdm\n",
        "from ml_things import plot_dict, fix_text\n",
        "from transformers import (\n",
        "                          CONFIG_MAPPING,\n",
        "                          MODEL_FOR_MASKED_LM_MAPPING,\n",
        "                          MODEL_FOR_CAUSAL_LM_MAPPING,\n",
        "                          PreTrainedTokenizer,\n",
        "                          TrainingArguments,\n",
        "                          AutoConfig,\n",
        "                          AutoTokenizer,\n",
        "                          AutoModelWithLMHead,\n",
        "                          AutoModelForCausalLM,\n",
        "                          AutoModelForMaskedLM,\n",
        "                          LineByLineTextDataset,\n",
        "                          TextDataset,\n",
        "                          DataCollatorForLanguageModeling,\n",
        "                          DataCollatorForWholeWordMask,\n",
        "                          DataCollatorForPermutationLanguageModeling,\n",
        "                          PretrainedConfig,\n",
        "                          Trainer,\n",
        "                          set_seed,\n",
        "                          )\n",
        "\n",
        "# Set seed for reproducibility,\n",
        "set_seed(123)\n",
        "\n",
        "# Look for gpu to use. Will use `cpu` by default if no gpu found.\n",
        "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qdDFhC762xRm"
      },
      "source": [
        "## **Helper Functions**\n",
        "\n",
        "I like to keep all Classes and functions that will be used in this notebook under this section to help maintain a clean look of the notebook:\n",
        "\n",
        "**movie_reviews_to_file(path_data: str, path_texts_file: str)**\n",
        "\n",
        "As I mentioned before, we will need `.txt` files to run this notebook. Since the Large Movie Review Dataset comes in multiple files with different labels I created this function to put together all data in a single `.txt` file. Examples are saved on each line of the file. The `path_data` points to the path where data files are present and `path_texts_file` will be the `.txt` file containing all data.\n",
        "\n",
        "<br>\n",
        "\n",
        "**ModelDataArguments**\n",
        "\n",
        "This class follows similar format as the [transformers]((https://github.com/huggingface/transformers) library. The main difference is the way I combined multiple types of arguments into one and used rules to make sure the arguments used are correctly set. Here are all argument detailed (they are also mentioned in the class documentation):\n",
        "\n",
        "* `train_data_file`: \n",
        "  *Path to your `.txt` file dataset. If you have an example on each line of the file make sure to use `line_by_line=True`. If the data file contains all text data without any special grouping use `line_by_line=False` to move a `block_size` window across the text file.*\n",
        "\n",
        "* `eval_data_file`: \n",
        "  *Path to evaluation `.txt` file. It has the same format as `train_data_file`.*\n",
        "\n",
        "* `line_by_line`: \n",
        "  *If the `train_data_file` and `eval_data_file` contains separate examples \n",
        "  on each line set `line_by_line=True`.\n",
        "  If there is no separation between examples and `train_data_file` and \n",
        "  `eval_data_file` contains continuous text then `line_by_line=False` and a \n",
        "  window of `block_size` will be moved across the files to acquire examples.*\n",
        "\n",
        "* `mlm`: \n",
        "  *Is a flag that changes loss function depending on model architecture. \n",
        "  This variable needs to be set to `True` when working with masked language \n",
        "  models like bert or roberta and set to `False` otherwise. There are functions that will raise ValueError if this argument is not set accordingly.*\n",
        "\n",
        "* `whole_word_mask`:\n",
        "  *Used as flag to determine if we decide to use whole word masking or not. Whole word masking means that whole words will be masked during training instead of tokens which can be chunks of words.*\n",
        "\n",
        "* `mlm_probability`: \n",
        "  *Used when training masked language models. Needs to have `mlm=True`. \n",
        "  It represents the probability of masking tokens when training model.*\n",
        "\n",
        "* `plm_probability`: \n",
        "  *Flag to define the ratio of length of a span of masked \n",
        "  tokens to surrounding context length for permutation language modeling. Used for XLNet.*\n",
        "\n",
        "* `max_span_length`: \n",
        "  *Flag may also be used to limit the length of a span of \n",
        "  masked tokens used for permutation language modeling.\n",
        "  Used for XLNet.*\n",
        "\n",
        "* `block_size`: \n",
        "  *It refers to the windows size that is moved across the text file. Set to -1 to use maximum allowed length.*\n",
        "\n",
        "* `overwrite_cache`: \n",
        "  *If there are any cached files, overwrite them.*\n",
        "\n",
        "* `model_type`: \n",
        "  *Type of model used: bert, roberta, gpt2. \n",
        "  More details [here](https://huggingface.co/transformers/pretrained_models.html).*\n",
        "\n",
        "* `model_config_name`:\n",
        "  *Config of model used: bert, roberta, gpt2. \n",
        "  More details [here](https://huggingface.co/transformers/pretrained_models.html).*\n",
        "\n",
        "* `tokenizer_name`:\n",
        "  *Tokenizer used to process data for training the model. \n",
        "  It usually has same name as `model_name_or_path`: bert-base-cased, \n",
        "  roberta-base, gpt2 etc.*\n",
        "\n",
        "* `model_name_or_path`: \n",
        "  *Path to existing transformers model or name of \n",
        "  transformer model to be used: bert-base-cased, roberta-base, gpt2 etc. \n",
        "  More details [here](https://huggingface.co/transformers/pretrained_models.html).*\n",
        "\n",
        "* `model_cache_dir`: \n",
        "  *Path to cache files. It helps to save time when re-running code.*\n",
        "\n",
        "<br>\n",
        "\n",
        "**get_model_config(args: ModelDataArguments)**\n",
        "\n",
        "Get model configuration. Using the ModelDataArguments to return the model configuration. Here are all argument detailed:\n",
        "\n",
        "* `args`: *Model and data configuration arguments needed to perform pretraining.*\n",
        "\n",
        "* Returns: *Model transformers configuration.*\n",
        "\n",
        "* Raises: *ValueError: If `mlm=True` and `model_type` is NOT in [\"bert\", \"roberta\", \"distilbert\", \"camembert\"]. We need to use a masked language model in order to set `mlm=True`.*\n",
        "\n",
        "<br>\n",
        "\n",
        "**get_tokenizer(args: ModelDataArguments)**\n",
        "\n",
        "Get model tokenizer.Using the ModelDataArguments return the model tokenizer and change `block_size` form `args` if needed. Here are all argument detailed:\n",
        "\n",
        "* `args`: *Model and data configuration arugments needed to perform pretraining.*\n",
        "\n",
        "* Returns: *Model transformers tokenizer.*\n",
        "\n",
        "<br>\n",
        "\n",
        "**get_model(args: ModelDataArguments, model_config)**\n",
        "\n",
        "Get model. Using the ModelDataArguments return the actual model. Here are all argument detailed:\n",
        "\n",
        "* `args`: *Model and data configuration arguments needed to perform pretraining.*\n",
        "\n",
        "* `model_config`: *Model transformers configuration.*\n",
        "\n",
        "* Returns: *PyTorch model.*\n",
        "\n",
        "<br>\n",
        "\n",
        "**get_dataset(args: ModelDataArguments, tokenizer: PreTrainedTokenizer, evaluate: bool=False)**\n",
        "\n",
        "Process dataset file into PyTorch Dataset. Using the ModelDataArguments return the actual model. Here are all argument detailed:\n",
        "\n",
        "* `args`: *Model and data configuration arguments needed to perform pretraining.*\n",
        "\n",
        "* `tokenizer`: *Model transformers tokenizer.*\n",
        "\n",
        "* `evaluate`: *If set to `True` the test / validation file is being handled. If set to `False` the train file is being handled.*\n",
        "\n",
        "* Returns: *PyTorch Dataset that contains file's data.*\n",
        "\n",
        "<br>\n",
        "\n",
        "**get_collator(args: ModelDataArguments, model_config: PretrainedConfig, tokenizer: PreTrainedTokenizer)**\n",
        "\n",
        "Get appropriate collator function.\n",
        "Collator function will be used to collate a PyTorch Dataset object. Here are all argument detailed:\n",
        "\n",
        "* `args`: *Model and data configuration arguments needed to perform pretraining.*\n",
        "\n",
        "* `model_config`: *Model transformers configuration.*\n",
        "\n",
        "* `tokenizer`: *Model transformers tokenizer.*\n",
        "\n",
        "* Returns: *Transformers specific data collator.*\n",
        "\n",
        "\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "r2DABVOmwC_J"
      },
      "source": [
        "def movie_reviews_to_file(path_data: str, path_texts_file: str):\n",
        "  r\"\"\"Reading in all data from path and saving it into a single `.txt` file.\n",
        "  \n",
        "  In the pretraining process of our transformers model we require a text file.\n",
        "\n",
        "  This function is designed to work for the Movie Reviews Dataset. \n",
        "  You wil have to create your own function to move all examples into a text \n",
        "  file if you don't already have a text file with all your unlabeled data.\n",
        "\n",
        "  Arguments:\n",
        "\n",
        "      path_data (:obj:`str`):\n",
        "        Path to the Movie Review Dataset partition. We only have `\\train` and \n",
        "        `test` partitions.\n",
        "\n",
        "      path_texts_file (:obj:`str`):\n",
        "        File path of the generated `.txt` file that contains one example / line.\n",
        "\n",
        "  \"\"\"\n",
        "\n",
        "  # Check if path exists.\n",
        "  if not os.path.isdir(path_data):\n",
        "    # Raise error if path is invalid.\n",
        "    raise ValueError('Invalid `path` variable! Needs to be a directory')\n",
        "  # Check max sequence length.\n",
        "  texts = []\n",
        "  print('Reading `%s` partition...' % (os.path.basename(path_data)))\n",
        "  # Since the labels are defined by folders with data we loop \n",
        "  # through each label.\n",
        "  for label  in ['neg', 'pos']:\n",
        "    sentiment_path = os.path.join(path_data, label)\n",
        "\n",
        "    # Get all files from path.\n",
        "    files_names = os.listdir(sentiment_path)#[:30] # SAMPLE FOR DEBUGGING.\n",
        "    # Go through each file and read its content.\n",
        "    for file_name in tqdm(files_names, desc=label, unit='files'):\n",
        "      file_path = os.path.join(sentiment_path, file_name)\n",
        "\n",
        "      # Read content.\n",
        "      content = io.open(file_path, mode='r', encoding='utf-8').read()\n",
        "      # Fix any unicode issues.\n",
        "      content = fix_text(content)\n",
        "      # Save content.\n",
        "      texts.append(content)\n",
        "  # Move list to single string.\n",
        "  all_texts = '\\n'.join(texts)\n",
        "  # Send all texts string to single file.\n",
        "  io.open(file=path_texts_file, mode='w', encoding='utf-8').write(all_texts)\n",
        "  # Print when done.\n",
        "  print('`.txt` file saved in `%s`\\n' % path_texts_file)\n",
        "\n",
        "  return\n",
        "\n",
        "\n",
        "class ModelDataArguments(object):\n",
        "  r\"\"\"Define model and data configuration needed to perform pretraining.\n",
        "\n",
        "  Eve though all arguments are optional there still needs to be a certain \n",
        "  number of arguments that require values attributed.\n",
        "  \n",
        "  Arguments:\n",
        "\n",
        "    train_data_file (:obj:`str`, `optional`): \n",
        "      Path to your .txt file dataset. If you have an example on each line of \n",
        "      the file make sure to use line_by_line=True. If the data file contains \n",
        "      all text data without any special grouping use line_by_line=False to move \n",
        "      a block_size window across the text file.\n",
        "      This argument is optional and it will have a `None` value attributed \n",
        "      inside the function.\n",
        "\n",
        "    eval_data_file (:obj:`str`, `optional`): \n",
        "      Path to evaluation .txt file. It has the same format as train_data_file.\n",
        "      This argument is optional and it will have a `None` value attributed \n",
        "      inside the function.\n",
        "\n",
        "    line_by_line (:obj:`bool`, `optional`, defaults to :obj:`False`): \n",
        "      If the train_data_file and eval_data_file contains separate examples on \n",
        "      each line then line_by_line=True. If there is no separation between \n",
        "      examples and train_data_file and eval_data_file contains continuous text \n",
        "      then line_by_line=False and a window of block_size will be moved across \n",
        "      the files to acquire examples.\n",
        "      This argument is optional and it has a default value.\n",
        "\n",
        "    mlm (:obj:`bool`, `optional`, defaults to :obj:`False`): \n",
        "      Is a flag that changes loss function depending on model architecture. \n",
        "      This variable needs to be set to True when working with masked language \n",
        "      models like bert or roberta and set to False otherwise. There are \n",
        "      functions that will raise ValueError if this argument is \n",
        "      not set accordingly.\n",
        "      This argument is optional and it has a default value.\n",
        "\n",
        "    whole_word_mask (:obj:`bool`, `optional`, defaults to :obj:`False`):\n",
        "      Used as flag to determine if we decide to use whole word masking or not. \n",
        "      Whole word masking means that whole words will be masked during training \n",
        "      instead of tokens which can be chunks of words.\n",
        "      This argument is optional and it has a default value.\n",
        "\n",
        "    mlm_probability(:obj:`float`, `optional`, defaults to :obj:`0.15`): \n",
        "      Used when training masked language models. Needs to have mlm set to True. \n",
        "      It represents the probability of masking tokens when training model.\n",
        "      This argument is optional and it has a default value.\n",
        "\n",
        "    plm_probability (:obj:`float`, `optional`, defaults to :obj:`float(1/6)`): \n",
        "      Flag to define the ratio of length of a span of masked tokens to \n",
        "      surrounding context length for permutation language modeling. \n",
        "      Used for XLNet.\n",
        "      This argument is optional and it has a default value.\n",
        "\n",
        "    max_span_length (:obj:`int`, `optional`, defaults to :obj:`5`): \n",
        "      Flag may also be used to limit the length of a span of masked tokens used \n",
        "      for permutation language modeling. Used for XLNet.\n",
        "      This argument is optional and it has a default value.\n",
        "\n",
        "    block_size (:obj:`int`, `optional`, defaults to :obj:`-1`): \n",
        "      It refers to the windows size that is moved across the text file. \n",
        "      Set to -1 to use maximum allowed length.\n",
        "      This argument is optional and it has a default value.\n",
        "\n",
        "    overwrite_cache (:obj:`bool`, `optional`, defaults to :obj:`False`): \n",
        "      If there are any cached files, overwrite them.\n",
        "      This argument is optional and it has a default value.\n",
        "\n",
        "    model_type (:obj:`str`, `optional`): \n",
        "      Type of model used: bert, roberta, gpt2. \n",
        "      More details: https://huggingface.co/transformers/pretrained_models.html\n",
        "      This argument is optional and it will have a `None` value attributed \n",
        "      inside the function.\n",
        "\n",
        "    model_config_name (:obj:`str`, `optional`):\n",
        "      Config of model used: bert, roberta, gpt2. \n",
        "      More details: https://huggingface.co/transformers/pretrained_models.html\n",
        "      This argument is optional and it will have a `None` value attributed \n",
        "      inside the function.\n",
        "\n",
        "    tokenizer_name: (:obj:`str`, `optional`)\n",
        "      Tokenizer used to process data for training the model. \n",
        "      It usually has same name as model_name_or_path: bert-base-cased, \n",
        "      roberta-base, gpt2 etc.\n",
        "      This argument is optional and it will have a `None` value attributed \n",
        "      inside the function.\n",
        "\n",
        "    model_name_or_path (:obj:`str`, `optional`): \n",
        "      Path to existing transformers model or name of \n",
        "      transformer model to be used: bert-base-cased, roberta-base, gpt2 etc. \n",
        "      More details: https://huggingface.co/transformers/pretrained_models.html\n",
        "      This argument is optional and it will have a `None` value attributed \n",
        "      inside the function.\n",
        "\n",
        "    model_cache_dir (:obj:`str`, `optional`): \n",
        "      Path to cache files to save time when re-running code.\n",
        "      This argument is optional and it will have a `None` value attributed \n",
        "      inside the function.\n",
        "\n",
        "  Raises:\n",
        "\n",
        "        ValueError: If `CONFIG_MAPPING` is not loaded in global variables.\n",
        "\n",
        "        ValueError: If `model_type` is not present in `CONFIG_MAPPING.keys()`.\n",
        "\n",
        "        ValueError: If `model_type`, `model_config_name` and \n",
        "          `model_name_or_path` variables are all `None`. At least one of them \n",
        "          needs to be set.\n",
        "\n",
        "        warnings: If `model_config_name` and `model_name_or_path` are both \n",
        "          `None`, the model will be trained from scratch.\n",
        "\n",
        "        ValueError: If `tokenizer_name` and `model_name_or_path` are both \n",
        "          `None`. We need at least one of them set to load tokenizer.\n",
        "    \n",
        "  \"\"\"\n",
        "\n",
        "  def __init__(self, train_data_file=None, eval_data_file=None, \n",
        "               line_by_line=False, mlm=False, mlm_probability=0.15, \n",
        "               whole_word_mask=False, plm_probability=float(1/6), \n",
        "               max_span_length=5, block_size=-1, overwrite_cache=False, \n",
        "               model_type=None, model_config_name=None, tokenizer_name=None, \n",
        "               model_name_or_path=None, model_cache_dir=None):\n",
        "    \n",
        "    # Make sure CONFIG_MAPPING is imported from transformers module.\n",
        "    if 'CONFIG_MAPPING' not in globals():\n",
        "      raise ValueError('Could not find `CONFIG_MAPPING` imported! Make sure' \\\n",
        "                       ' to import it from `transformers` module!')\n",
        "\n",
        "    # Make sure model_type is valid.\n",
        "    if (model_type is not None) and (model_type not in CONFIG_MAPPING.keys()):\n",
        "      raise ValueError('Invalid `model_type`! Use one of the following: %s' % \n",
        "                       (str(list(CONFIG_MAPPING.keys()))))\n",
        "      \n",
        "    # Make sure that model_type, model_config_name and model_name_or_path \n",
        "    # variables are not all `None`.\n",
        "    if not any([model_type, model_config_name, model_name_or_path]):\n",
        "      raise ValueError('You can`t have all `model_type`, `model_config_name`,' \\\n",
        "                       ' `model_name_or_path` be `None`! You need to have' \\\n",
        "                       'at least one of them set!')\n",
        "    \n",
        "    # Check if a new model will be loaded from scratch.\n",
        "    if not any([model_config_name, model_name_or_path]):\n",
        "      # Setup warning to show pretty. This is an overkill\n",
        "      warnings.formatwarning = lambda message,category,*args,**kwargs: \\\n",
        "                               '%s: %s\\n' % (category.__name__, message)\n",
        "      # Display warning.\n",
        "      warnings.warn('You are planning to train a model from scratch! 🙀')\n",
        "\n",
        "    # Check if a new tokenizer wants to be loaded.\n",
        "    # This feature is not supported!\n",
        "    if not any([tokenizer_name, model_name_or_path]):\n",
        "      # Can't train tokenizer from scratch here! Raise error.\n",
        "      raise ValueError('You want to train tokenizer from scratch! ' \\\n",
        "                    'That is not possible yet! You can train your own ' \\\n",
        "                    'tokenizer separately and use path here to load it!')\n",
        "      \n",
        "    # Set all data related arguments.\n",
        "    self.train_data_file = train_data_file\n",
        "    self.eval_data_file = eval_data_file\n",
        "    self.line_by_line = line_by_line\n",
        "    self.mlm = mlm\n",
        "    self.whole_word_mask = whole_word_mask\n",
        "    self.mlm_probability = mlm_probability\n",
        "    self.plm_probability = plm_probability\n",
        "    self.max_span_length = max_span_length\n",
        "    self.block_size = block_size\n",
        "    self.overwrite_cache = overwrite_cache\n",
        "\n",
        "    # Set all model and tokenizer arguments.\n",
        "    self.model_type = model_type\n",
        "    self.model_config_name = model_config_name\n",
        "    self.tokenizer_name = tokenizer_name\n",
        "    self.model_name_or_path = model_name_or_path\n",
        "    self.model_cache_dir = model_cache_dir\n",
        "    return\n",
        "\n",
        "\n",
        "def get_model_config(args: ModelDataArguments):\n",
        "  r\"\"\"\n",
        "  Get model configuration.\n",
        "\n",
        "  Using the ModelDataArguments return the model configuration.\n",
        "\n",
        "  Arguments:\n",
        "\n",
        "    args (:obj:`ModelDataArguments`):\n",
        "      Model and data configuration arguments needed to perform pretraining.\n",
        "\n",
        "  Returns:\n",
        "\n",
        "    :obj:`PretrainedConfig`: Model transformers configuration.\n",
        "\n",
        "  Raises:\n",
        "\n",
        "    ValueError: If `mlm=True` and `model_type` is NOT in [\"bert\", \n",
        "          \"roberta\", \"distilbert\", \"camembert\"]. We need to use a masked \n",
        "          language model in order to set `mlm=True`.\n",
        "  \"\"\"\n",
        "\n",
        "  # Check model configuration.\n",
        "  if args.model_config_name is not None:\n",
        "    # Use model configure name if defined.\n",
        "    model_config = AutoConfig.from_pretrained(args.model_config_name, \n",
        "                                      cache_dir=args.model_cache_dir)\n",
        "\n",
        "  elif args.model_name_or_path is not None:\n",
        "    # Use model name or path if defined.\n",
        "    model_config = AutoConfig.from_pretrained(args.model_name_or_path, \n",
        "                                      cache_dir=args.model_cache_dir)\n",
        "\n",
        "  else:\n",
        "    # Use config mapping if building model from scratch.\n",
        "    model_config = CONFIG_MAPPING[args.model_type]()\n",
        "\n",
        "  # Make sure `mlm` flag is set for Masked Language Models (MLM).\n",
        "  if (model_config.model_type in [\"bert\", \"roberta\", \"distilbert\", \n",
        "                                  \"camembert\"]) and (args.mlm is False):\n",
        "    raise ValueError('BERT and RoBERTa-like models do not have LM heads ' \\\n",
        "                    'butmasked LM heads. They must be run setting `mlm=True`')\n",
        "  \n",
        "  # Adjust block size for xlnet.\n",
        "  if model_config.model_type == \"xlnet\":\n",
        "    # xlnet used 512 tokens when training.\n",
        "    args.block_size = 512\n",
        "    # setup memory length\n",
        "    model_config.mem_len = 1024\n",
        "  \n",
        "  return model_config\n",
        "\n",
        "\n",
        "def get_tokenizer(args: ModelDataArguments):\n",
        "  r\"\"\"\n",
        "  Get model tokenizer.\n",
        "\n",
        "  Using the ModelDataArguments return the model tokenizer and change \n",
        "  `block_size` form `args` if needed.\n",
        "\n",
        "  Arguments:\n",
        "\n",
        "    args (:obj:`ModelDataArguments`):\n",
        "      Model and data configuration arguments needed to perform pretraining.\n",
        "\n",
        "  Returns:\n",
        "\n",
        "    :obj:`PreTrainedTokenizer`: Model transformers tokenizer.\n",
        "\n",
        "  \"\"\"\n",
        "\n",
        "  # Check tokenizer configuration.\n",
        "  if args.tokenizer_name:\n",
        "    # Use tokenizer name if define.\n",
        "    tokenizer = AutoTokenizer.from_pretrained(args.tokenizer_name, \n",
        "                                              cache_dir=args.model_cache_dir)\n",
        "\n",
        "  elif args.model_name_or_path:\n",
        "    # Use tokenizer name of path if defined.\n",
        "    tokenizer = AutoTokenizer.from_pretrained(args.model_name_or_path, \n",
        "                                              cache_dir=args.model_cache_dir)\n",
        "    \n",
        "  # Setp data block size.\n",
        "  if args.block_size <= 0:\n",
        "    # Set block size to maximum length of tokenizer.\n",
        "    # Input block size will be the max possible for the model.\n",
        "    # Some max lengths are very large and will cause a\n",
        "    args.block_size = tokenizer.model_max_length\n",
        "  else:\n",
        "    # Never go beyond tokenizer maximum length.\n",
        "    args.block_size = min(args.block_size, tokenizer.model_max_length)\n",
        "\n",
        "  return tokenizer\n",
        "  \n",
        "\n",
        "def get_model(args: ModelDataArguments, model_config):\n",
        "  r\"\"\"\n",
        "  Get model.\n",
        "\n",
        "  Using the ModelDataArguments return the actual model.\n",
        "\n",
        "  Arguments:\n",
        "\n",
        "    args (:obj:`ModelDataArguments`):\n",
        "      Model and data configuration arguments needed to perform pretraining.\n",
        "\n",
        "    model_config (:obj:`PretrainedConfig`):\n",
        "      Model transformers configuration.\n",
        "\n",
        "  Returns:\n",
        "\n",
        "    :obj:`torch.nn.Module`: PyTorch model.\n",
        "\n",
        "  \"\"\"\n",
        "\n",
        "  # Make sure MODEL_FOR_MASKED_LM_MAPPING and MODEL_FOR_CAUSAL_LM_MAPPING are \n",
        "  # imported from transformers module.\n",
        "  if ('MODEL_FOR_MASKED_LM_MAPPING' not in globals()) and \\\n",
        "                ('MODEL_FOR_CAUSAL_LM_MAPPING' not in globals()):\n",
        "    raise ValueError('Could not find `MODEL_FOR_MASKED_LM_MAPPING` and' \\\n",
        "                     ' `MODEL_FOR_MASKED_LM_MAPPING` imported! Make sure to' \\\n",
        "                     ' import them from `transformers` module!')\n",
        "    \n",
        "  # Check if using pre-trained model or train from scratch.\n",
        "  if args.model_name_or_path:\n",
        "    # Use pre-trained model.\n",
        "    if type(model_config) in MODEL_FOR_MASKED_LM_MAPPING.keys():\n",
        "      # Masked language modeling head.\n",
        "      return AutoModelForMaskedLM.from_pretrained(\n",
        "                        args.model_name_or_path,\n",
        "                        from_tf=bool(\".ckpt\" in args.model_name_or_path),\n",
        "                        config=model_config,\n",
        "                        cache_dir=args.model_cache_dir,\n",
        "                        )\n",
        "    elif type(model_config) in MODEL_FOR_CAUSAL_LM_MAPPING.keys():\n",
        "      # Causal language modeling head.\n",
        "      return AutoModelForCausalLM.from_pretrained(\n",
        "                                          args.model_name_or_path, \n",
        "                                          from_tf=bool(\".ckpt\" in \n",
        "                                                        args.model_name_or_path),\n",
        "                                          config=model_config, \n",
        "                                          cache_dir=args.model_cache_dir)\n",
        "    else:\n",
        "      raise ValueError(\n",
        "          'Invalid `model_name_or_path`! It should be in %s or %s!' % \n",
        "          (str(MODEL_FOR_MASKED_LM_MAPPING.keys()), \n",
        "           str(MODEL_FOR_CAUSAL_LM_MAPPING.keys())))\n",
        "    \n",
        "  else:\n",
        "    # Use model from configuration - train from scratch.\n",
        "      print(\"Training new model from scratch!\")\n",
        "      return AutoModelWithLMHead.from_config(config)\n",
        "\n",
        "\n",
        "def get_dataset(args: ModelDataArguments, tokenizer: PreTrainedTokenizer, \n",
        "                evaluate: bool=False):\n",
        "  r\"\"\"\n",
        "  Process dataset file into PyTorch Dataset.\n",
        "\n",
        "  Using the ModelDataArguments return the actual model.\n",
        "\n",
        "  Arguments:\n",
        "\n",
        "    args (:obj:`ModelDataArguments`):\n",
        "      Model and data configuration arguments needed to perform pretraining.\n",
        "\n",
        "    tokenizer (:obj:`PreTrainedTokenizer`):\n",
        "      Model transformers tokenizer.\n",
        "\n",
        "    evaluate (:obj:`bool`, `optional`, defaults to :obj:`False`):\n",
        "      If set to `True` the test / validation file is being handled.\n",
        "      If set to `False` the train file is being handled.\n",
        "\n",
        "  Returns:\n",
        "\n",
        "    :obj:`Dataset`: PyTorch Dataset that contains file's data.\n",
        "\n",
        "  \"\"\"\n",
        "\n",
        "  # Get file path for either train or evaluate.\n",
        "  file_path = args.eval_data_file if evaluate else args.train_data_file\n",
        "\n",
        "  # Check if `line_by_line` flag is set to `True`.\n",
        "  if args.line_by_line:\n",
        "    # Each example in data file is on each line.\n",
        "    return LineByLineTextDataset(tokenizer=tokenizer, file_path=file_path, \n",
        "                                 block_size=args.block_size)\n",
        "    \n",
        "  else:\n",
        "    # All data in file is put together without any separation.\n",
        "    return TextDataset(tokenizer=tokenizer, file_path=file_path, \n",
        "                       block_size=args.block_size, \n",
        "                       overwrite_cache=args.overwrite_cache)\n",
        "\n",
        "\n",
        "def get_collator(args: ModelDataArguments, model_config: PretrainedConfig, \n",
        "                 tokenizer: PreTrainedTokenizer):\n",
        "  r\"\"\"\n",
        "  Get appropriate collator function.\n",
        "\n",
        "  Collator function will be used to collate a PyTorch Dataset object.\n",
        "\n",
        "  Arguments:\n",
        "\n",
        "    args (:obj:`ModelDataArguments`):\n",
        "      Model and data configuration arguments needed to perform pretraining.\n",
        "\n",
        "    model_config (:obj:`PretrainedConfig`):\n",
        "      Model transformers configuration.\n",
        "\n",
        "    tokenizer (:obj:`PreTrainedTokenizer`):\n",
        "      Model transformers tokenizer.\n",
        "\n",
        "  Returns:\n",
        "\n",
        "    :obj:`data_collator`: Transformers specific data collator.\n",
        "\n",
        "  \"\"\"\n",
        "\n",
        "  # Special dataset handle depending on model type.\n",
        "  if model_config.model_type == \"xlnet\":\n",
        "    # Configure collator for XLNET.\n",
        "    return DataCollatorForPermutationLanguageModeling(\n",
        "                                          tokenizer=tokenizer,\n",
        "                                          plm_probability=args.plm_probability,\n",
        "                                          max_span_length=args.max_span_length,\n",
        "                                          )\n",
        "  else:\n",
        "    # Configure data for rest of model types.\n",
        "    if args.mlm and args.whole_word_mask:\n",
        "      # Use whole word masking.\n",
        "      return DataCollatorForWholeWordMask(\n",
        "                                          tokenizer=tokenizer, \n",
        "                                          mlm_probability=args.mlm_probability,\n",
        "                                          )\n",
        "    else:\n",
        "      # Regular language modeling.\n",
        "      return DataCollatorForLanguageModeling(\n",
        "                                          tokenizer=tokenizer, \n",
        "                                          mlm=args.mlm, \n",
        "                                          mlm_probability=args.mlm_probability,\n",
        "                                          )\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "E1F-XIQCdOgj"
      },
      "source": [
        "## **Parameters Setup**\n",
        "\n",
        "Declare the rest of the parameters used for this notebook:\n",
        "\n",
        "* `model_data_args` contains all arguments needed to setup dataset, model configuration, model tokenizer and the actual model. This is created using the `ModelDataArguments` class.\n",
        "\n",
        "* `training_args` contain all arguments needed to use the [Trainer]() functionality from Transformers that allows us to train transformers models in PyTorch very easy. You can find the complete documentation [here](https://huggingface.co/transformers/main_classes/trainer.html#trainingarguments). There are a lot of parameters that can be set to allow multiple functionalities. I only used the following parameters (the comments are inspired from the HuggingFace documentation of [TrainingArguments](https://huggingface.co/transformers/main_classes/trainer.html#trainingarguments):\n",
        " \n",
        "  * `output_dir`: *The output directory where the model predictions and checkpoints will be written. I set it up to `pretrained_bert_model` where the model and will be saved.*\n",
        "\n",
        "  * `overwrite_output_dir`: *Overwrite the content of the output directory. I set it to `True` in case I run the notebook multiple times I only care about the last run.*\n",
        "\n",
        "  * `do_train`: *Whether to run training or not. I set this parameter to `True` because I want to train the model on my custom dataset.* \n",
        "\n",
        "  * `do_eval`: *Whether to run evaluation on the evaluation files or not.\n",
        "I set it to `True` since I have test data file and I want to evaluate how well the model trains.*\n",
        "\n",
        "  * `per_device_train_batch_size`: *Batch size GPU/TPU core/CPU training. I set it to `2` for this example. I recommend setting it up as high as your GPU memory allows you.*\n",
        "\n",
        "  * `per_device_eval_batch_size`: *Batch size  GPU/TPU core/CPU for evaluation.I set this value to `100` since it's not dealing with gradients.*\n",
        "\n",
        "  * `evaluation_strategy`: *Evaluation strategy to adopt during training: `no`: No evaluation during training; `steps`: Evaluate every `eval_steps; `epoch`: Evaluate every end of epoch. I set it to 'steps' since I want to evaluate model more often.*\n",
        "\n",
        "  * `logging_steps`: *How often to show logs. I will se this to plot history loss and calculate perplexity. I set this to `20` just as an example. If your evaluate data is large you might not want to run it that often because it will significantly slow down training time.*\n",
        "\n",
        "  * `eval_steps`: *Number of update steps between two evaluations if evaluation_strategy=\"steps\". Will default to the same value as logging_steps if not set. Since I want to evaluate model ever`logging_steps` I will set this to `None` since it will inherit same value as `logging_steps`.*\n",
        "\n",
        "  * `prediction_loss_only`: *Set prediction loss to `True` in order to return loss for perplexity calculation. Since I want to calculate perplexity I set this to `True` since I want to monitor loss and perplexity (which is exp(loss)).*\n",
        "\n",
        "  * `learning_rate`: *The initial learning rate for Adam. Defaults is set to `5e-5`.*\n",
        "\n",
        "  * `weight_decay`: *The weight decay to apply (if not zero)Defaults is set to `0`.*\n",
        "\n",
        "  * `adam_epsilon`: *Epsilon for the Adam optimizer. Defaults to `1e-8`.*\n",
        "\n",
        "  * `max_grad_norm`: *Maximum gradient norm (for gradient clipping). Defaults to `0`.*\n",
        "\n",
        "  * `num_train_epochs`: *Total number of training epochs to perform (if not an integer, will perform the decimal part percents of the last epoch before stopping training). I set it to `2` at most. Since the custom dataset will be a lot smaller than the original dataset the model was trained on we don't want to overfit.*\n",
        "\n",
        "  * `save_steps`: *Number of updates steps before two checkpoint saves. Defaults to `500`.*"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LACnz9HkdQJS"
      },
      "source": [
        "# Define arguments for data, tokenizer and model arguments.\n",
        "# See comments in `ModelDataArguments` class.\n",
        "model_data_args = ModelDataArguments(\n",
        "                                    train_data_file='/content/train.txt', \n",
        "                                    eval_data_file='/content/test.txt', \n",
        "                                    line_by_line=True, \n",
        "                                    mlm=True,\n",
        "                                    whole_word_mask=True,\n",
        "                                    mlm_probability=0.15,\n",
        "                                    plm_probability=float(1/6), \n",
        "                                    max_span_length=5,\n",
        "                                    block_size=50, \n",
        "                                    overwrite_cache=False, \n",
        "                                    model_type='bert', \n",
        "                                    model_config_name='bert-base-cased', \n",
        "                                    tokenizer_name='bert-base-cased', \n",
        "                                    model_name_or_path='bert-base-cased', \n",
        "                                    model_cache_dir=None,\n",
        "                                    )\n",
        "\n",
        "# Define arguments for training\n",
        "# Note: I only used the arguments I care about. `TrainingArguments` contains\n",
        "# a lot more arguments. For more details check the awesome documentation:\n",
        "# https://huggingface.co/transformers/main_classes/trainer.html#trainingarguments\n",
        "training_args = TrainingArguments(\n",
        "                          # The output directory where the model predictions \n",
        "                          # and checkpoints will be written.\n",
        "                          output_dir='pretrain_bert',\n",
        "\n",
        "                          # Overwrite the content of the output directory.\n",
        "                          overwrite_output_dir=True,\n",
        "\n",
        "                          # Whether to run training or not.\n",
        "                          do_train=True, \n",
        "                          \n",
        "                          # Whether to run evaluation on the dev or not.\n",
        "                          do_eval=True,\n",
        "                          \n",
        "                          # Batch size GPU/TPU core/CPU training.\n",
        "                          per_device_train_batch_size=10,\n",
        "                          \n",
        "                          # Batch size  GPU/TPU core/CPU for evaluation.\n",
        "                          per_device_eval_batch_size=100,\n",
        "\n",
        "                          # evaluation strategy to adopt during training\n",
        "                          # `no`: No evaluation during training.\n",
        "                          # `steps`: Evaluate every `eval_steps`.\n",
        "                          # `epoch`: Evaluate every end of epoch.\n",
        "                          evaluation_strategy='steps',\n",
        "\n",
        "                          # How often to show logs. I will se this to \n",
        "                          # plot history loss and calculate perplexity.\n",
        "                          logging_steps=700,\n",
        "\n",
        "                          # Number of update steps between two \n",
        "                          # evaluations if evaluation_strategy=\"steps\".\n",
        "                          # Will default to the same value as l\n",
        "                          # logging_steps if not set.\n",
        "                          eval_steps = None,\n",
        "                          \n",
        "                          # Set prediction loss to `True` in order to \n",
        "                          # return loss for perplexity calculation.\n",
        "                          prediction_loss_only=True,\n",
        "\n",
        "                          # The initial learning rate for Adam. \n",
        "                          # Defaults to 5e-5.\n",
        "                          learning_rate = 5e-5,\n",
        "\n",
        "                          # The weight decay to apply (if not zero).\n",
        "                          weight_decay=0,\n",
        "\n",
        "                          # Epsilon for the Adam optimizer. \n",
        "                          # Defaults to 1e-8\n",
        "                          adam_epsilon = 1e-8,\n",
        "\n",
        "                          # Maximum gradient norm (for gradient \n",
        "                          # clipping). Defaults to 0.\n",
        "                          max_grad_norm = 1.0,\n",
        "                          # Total number of training epochs to perform \n",
        "                          # (if not an integer, will perform the \n",
        "                          # decimal part percents of\n",
        "                          # the last epoch before stopping training).\n",
        "                          num_train_epochs = 2,\n",
        "\n",
        "                          # Number of updates steps before two checkpoint saves. \n",
        "                          # Defaults to 500\n",
        "                          save_steps = -1,\n",
        "                          )"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "OA8S0HqsdpHs"
      },
      "source": [
        "## **Load Configuration, Tokenizer and Model**\n",
        "\n",
        "Loading the three essential parts of the pretrained transformers: configuration, tokenizer and model. \n",
        "\n",
        "Since I use the AutoClass functionality from Hugging Face I only need to worry about the model's name as input and the rest is handled by the transformers library.\n",
        "\n",
        "I will be calling each three functions created in the **Helper Functions** tab that help return `config` of the model, `tokenizer` of the model and the actual PyTorch `model`.\n",
        "\n",
        "After `model` is loaded is always good practice to resize the model depending on the `tokenizer` size. This means that the tokenizer's vocabulary will be aligned with the models embedding layer. This is very useful when we have a different tokenizer that the pretrained one or we train a transformer model from scratch.\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uyUXB374duJs",
        "outputId": "aae0a495-c4f5-49da-ac15-5fe22174f969",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 303,
          "referenced_widgets": [
            "e1f3c3cc875f48ddbfe923af092ca99d",
            "e6205bf4b19240f69f46d40aabd64652",
            "ed08b7e3b0304a139c09c51487bba7f1",
            "440cbdd7d1534c7fbd050eda7bc33362",
            "82406385fca142d39cbd778fe8e99f7c",
            "e4dd96e1d09442debfac1e7d6aeb8a25",
            "9aafd356ed844b408805c69a921495ec",
            "ad39829093994961b829c57839f15f15",
            "130b863b507547fdad72d766ad77e116",
            "f87be544429e4cb488437ab28c0d450d",
            "f91adcde493d4afca428633712fd3d50",
            "cec990db7c684d53a59a58313254dcaa",
            "5ca2f0623a71444db224a540232e3059",
            "72663e6053ce40eba183785abc9a8008",
            "80a531aac29347829de3f48638e2e87f",
            "4e0402a8e583424da0033bdf7b82b07a",
            "3424afb42d45481fbf959945b7e85082",
            "922429f8d3f6467eb73d581117d2fc2c",
            "4eb76130d1ad4f5aa606ead0f63a298d",
            "4b94bb215ca549519ab4b9c7083c7c86",
            "0821f3990f4841dc992c22a4c7b06abe",
            "1f202c5015934273b1dbeb6f5fd47162",
            "d368e63449684840aeb59ff5593d1cdd",
            "fa68b3d790394966a03a2b545fa03b92"
          ]
        }
      },
      "source": [
        "# Load model configuration.\n",
        "print('Loading model configuration...')\n",
        "config = get_model_config(model_data_args)\n",
        "\n",
        "# Load model tokenizer.\n",
        "print('Loading model`s tokenizer...')\n",
        "tokenizer = get_tokenizer(model_data_args)\n",
        "\n",
        "# Loading model.\n",
        "print('Loading actual model...')\n",
        "model = get_model(model_data_args, config)\n",
        "\n",
        "# Resize model to fit all tokens in tokenizer.\n",
        "model.resize_token_embeddings(len(tokenizer))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Loading model configuration...\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "e1f3c3cc875f48ddbfe923af092ca99d",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=433.0, style=ProgressStyle(description_…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Loading model`s tokenizer...\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "130b863b507547fdad72d766ad77e116",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=213450.0, style=ProgressStyle(descripti…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Loading actual model...\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "3424afb42d45481fbf959945b7e85082",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=435779157.0, style=ProgressStyle(descri…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Some weights of the model checkpoint at bert-base-cased were not used when initializing BertForMaskedLM: ['cls.seq_relationship.weight', 'cls.seq_relationship.bias']\n",
            "- This IS expected if you are initializing BertForMaskedLM from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPretraining model).\n",
            "- This IS NOT expected if you are initializing BertForMaskedLM from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Embedding(28996, 768, padding_idx=0)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "VE2MRZZhd5uM"
      },
      "source": [
        "## **Dataset and Collator**\n",
        "\n",
        "This is where I create the PyTorch Dataset and data collator objects that will be used to feed data into our model.\n",
        "\n",
        "This is where I use the MovieReviewsDataset text files created with the `movie_reviews_to_file` function. Since data is partitioned for both train and test I will create two text files: one used for train and one used for evaluation. \n",
        "\n",
        "I strongly recommend to use a validation text file in order to determine how much training is needed in order to avoid overfitting. After you figure out what parameters yield the best results, the validation file can be incorporated in train and run a final train with the whole dataset.\n",
        "\n",
        "The data collator is used to format the PyTorch Dataset outputs to match the output of our specific transformers model: i.e. for Bert it will created the masked tokens needed to train."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4mkU59HSd77l",
        "outputId": "95f24599-9166-475b-dacb-c777eb3f4525",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 349,
          "referenced_widgets": [
            "b255bc3521dd4dc9a7b13f727d966f39",
            "48a1f197c13b410a9212b69b207cd30b",
            "b94f1b75df744f4894cb9522f7d13389",
            "a512c47cfafa4e9bb23f984c0428027c",
            "ce58fdcbc43f4393a309177fc09373dd",
            "6a1877f99f7a4cedb25d0a217f540495",
            "1200d13dacbe432188a580fc57a48c0a",
            "dbbe72f10ba64937a9440b99918bfa7a",
            "57adf7d24e37426ebcd6fbf84cdacdd5",
            "89767f322bc24e65aacf67d84cc3dd55",
            "8545893ba7a3438d9da84b02fe012df9",
            "4b13ce2d46e347c9a8cd0fc2d55d0359",
            "5ca520d86501455294e4072416e4c228",
            "3ebd47c258a04766bca1ba838fd3bc5f",
            "e90ae6e058d742359162d6589c9926dd",
            "b82a00647843451f83d72a13f6434da4",
            "215e2bca7399439f8a2d02bfd32cf3d2",
            "488cbd647cdc47149f8a02fd1ffbc72a",
            "575983162afd4b7da3b1e6439b03a6fe",
            "09232a092df14fec8e9955dbc8d4ee3b",
            "4f1af0399c064a2c81780c794a2ca175",
            "b65e403ad42c4292b8e4ac65d1bed9e0",
            "5c714855611e4e7a9819a3646d2ee7a8",
            "734a6cc541384028a1ca1fd858360088",
            "4c98b756aa864f0ba4278a88640d1a1b",
            "c25bf2883da54023aaa2338e02da02f9",
            "6b6e758549a84f2f8c20cf2d874caea7",
            "309340d744c242ffb8a9b4e46ab90a76",
            "0061fff440564e02a3d838f0d64df14e",
            "0a9b044241a742f2b5670dc60f5e0c97",
            "b9e57cc1e7bb45f3a399a16ccb55633f",
            "7dd9fe7aadb74797b4a852aaddfb2a02"
          ]
        }
      },
      "source": [
        "# Create texts file from train data.\n",
        "movie_reviews_to_file(path_data='/content/aclImdb/train', path_texts_file='/content/train.txt')\n",
        "# Create texts file from test data.\n",
        "movie_reviews_to_file(path_data='/content/aclImdb/test', path_texts_file='/content/test.txt')\n",
        "\n",
        "\n",
        "# Setup train dataset if `do_train` is set.\n",
        "print('Creating train dataset...')\n",
        "train_dataset = get_dataset(model_data_args, tokenizer=tokenizer, evaluate=False) if training_args.do_train else None\n",
        "\n",
        "# Setup evaluation dataset if `do_eval` is set.\n",
        "print('Creating evaluate dataset...')\n",
        "eval_dataset = get_dataset(model_data_args, tokenizer=tokenizer, evaluate=True) if training_args.do_eval else None\n",
        "\n",
        "# Get data collator to modify data format depending on type of model used.\n",
        "data_collator = get_collator(model_data_args, config, tokenizer)\n",
        "\n",
        "# Check how many logging prints you'll have. This is to avoid overflowing the \n",
        "# notebook with a lot of prints. Display warning to user if the logging steps \n",
        "# that will be displayed is larger than 100.\n",
        "if (len(train_dataset) // training_args.per_device_train_batch_size \\\n",
        "    // training_args.logging_steps * training_args.num_train_epochs) > 100:\n",
        "  # Display warning.\n",
        "  warnings.warn('Your `logging_steps` value will will do a lot of printing!' \\\n",
        "                ' Consider increasing `logging_steps` to avoid overflowing' \\\n",
        "                ' the notebook with a lot of prints!')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Reading `train` partition...\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "b255bc3521dd4dc9a7b13f727d966f39",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='neg', max=12500.0, style=ProgressStyle(description_width=…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "57adf7d24e37426ebcd6fbf84cdacdd5",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='pos', max=12500.0, style=ProgressStyle(description_width=…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "`.txt` file saved in `/content/train.txt`\n",
            "\n",
            "Reading `test` partition...\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "215e2bca7399439f8a2d02bfd32cf3d2",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='neg', max=12500.0, style=ProgressStyle(description_width=…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "4c98b756aa864f0ba4278a88640d1a1b",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='pos', max=12500.0, style=ProgressStyle(description_width=…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "`.txt` file saved in `/content/test.txt`\n",
            "\n",
            "Creating train dataset...\n",
            "Creating evaluate dataset...\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "tr-UR5ZZcC0u"
      },
      "source": [
        "## **Train**\n",
        "\n",
        "Hugging Face was very nice to us for creating the `Trainer` class. This helps make PyTorch model training of transformers very easy! We just need to make sure we loaded the proper parameters and everything else is taking care of!\n",
        "\n",
        "At the end of the training the tokenizer is saved along with the model so you can easily re-use it later or even load in on Hugging Face Models.\n",
        "\n",
        "I configured the arguments to display both train and validation loss at every `logging_steps`. It gives us a sense of how well the model is trained."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RWZi6QArWR8I",
        "outputId": "0fa4406b-92f8-4644-930d-f427394205a5",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 326
        }
      },
      "source": [
        "# Initialize Trainer.\n",
        "print('Loading `trainer`...')\n",
        "trainer = Trainer(model=model,\n",
        "                  args=training_args,\n",
        "                  data_collator=data_collator,\n",
        "                  train_dataset=train_dataset,\n",
        "                  eval_dataset=eval_dataset,\n",
        "                  )\n",
        "\n",
        "\n",
        "# Check model path to save.\n",
        "if training_args.do_train:\n",
        "  print('Start training...')\n",
        "\n",
        "  # Setup model path if the model to train loaded from a local path.\n",
        "  model_path = (model_data_args.model_name_or_path \n",
        "                if model_data_args.model_name_or_path is not None and \n",
        "                os.path.isdir(model_data_args.model_name_or_path) \n",
        "                else None\n",
        "                )\n",
        "  # Run training.\n",
        "  trainer.train(model_path=model_path)\n",
        "  # Save model.\n",
        "  trainer.save_model()\n",
        "\n",
        "  # For convenience, we also re-save the tokenizer to the same directory,\n",
        "  # so that you can share your model easily on huggingface.co/models =).\n",
        "  if trainer.is_world_process_zero():\n",
        "    tokenizer.save_pretrained(training_args.output_dir)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Loading `trainer`...\n",
            "Start training...\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "\n",
              "    <div>\n",
              "        <style>\n",
              "            /* Turns off some styling */\n",
              "            progress {\n",
              "                /* gets rid of default border in Firefox and Opera. */\n",
              "                border: none;\n",
              "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
              "                background-size: auto;\n",
              "            }\n",
              "        </style>\n",
              "      \n",
              "      <progress value='5000' max='5000' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [5000/5000 09:43, Epoch 2/2]\n",
              "    </div>\n",
              "    <table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: left;\">\n",
              "      <th>Step</th>\n",
              "      <th>Training Loss</th>\n",
              "      <th>Validation Loss</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>700</td>\n",
              "      <td>2.804672</td>\n",
              "      <td>2.600590</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1400</td>\n",
              "      <td>2.666996</td>\n",
              "      <td>2.548267</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2100</td>\n",
              "      <td>2.625075</td>\n",
              "      <td>2.502431</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2800</td>\n",
              "      <td>2.545872</td>\n",
              "      <td>2.485056</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3500</td>\n",
              "      <td>2.470102</td>\n",
              "      <td>2.444808</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>4200</td>\n",
              "      <td>2.464950</td>\n",
              "      <td>2.420487</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>4900</td>\n",
              "      <td>2.436973</td>\n",
              "      <td>2.410310</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table><p>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "HkVfDQyYsFtp"
      },
      "source": [
        "## **Plot Train**\n",
        "\n",
        "The `Trainer` class is so useful that it will record the log history for us. I use this to access the train and validation losses recorded at each `logging_steps` during training.\n",
        "\n",
        "Since we are training / fine-tuning / extended training or pretraining (depending what terminology you use) a language model, we want to compute the perplexity. \n",
        "\n",
        "This is what [Wikipedia](https://en.wikipedia.org/wiki/Perplexity) says about perplexity: *In information theory, perplexity is a measurement of how well a probability distribution or probability model predicts a sample. It may be used to compare probability models. A low perplexity indicates the probability distribution is good at predicting the sample.*\n",
        "\n",
        "We can look at the perplexity plot in the same way we look at the loss plot: the lower the better and if the validation perplexity starts to increase we are starting to overfit the model.\n",
        "\n",
        "**Note:** It looks from the plots that the train loss is higher than validation loss. That means that our validation data is too easy for the model and we should use a different validation dataset. Since the purpose of this notebook is to show how to train transformers models and provide tools to evaluate such process I will leave the results *as is*.\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0FEY2WDgmCD_",
        "outputId": "d4ff8b5b-e5a3-4912-a432-727acf3aa008",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 949
        }
      },
      "source": [
        "# Keep track of train and evaluate loss.\n",
        "loss_history = {'train_loss':[], 'eval_loss':[]}\n",
        "\n",
        "# Keep track of train and evaluate perplexity.\n",
        "# This is a metric useful to track for language models.\n",
        "perplexity_history = {'train_perplexity':[], 'eval_perplexity':[]}\n",
        "\n",
        "# Loop through each log history.\n",
        "for log_history in trainer.state.log_history:\n",
        "\n",
        "  if 'loss' in log_history.keys():\n",
        "    # Deal with trianing loss.\n",
        "    loss_history['train_loss'].append(log_history['loss'])\n",
        "    perplexity_history['train_perplexity'].append(math.exp(log_history['loss']))\n",
        "    \n",
        "  elif 'eval_loss' in log_history.keys():\n",
        "    # Deal with eval loss.\n",
        "    loss_history['eval_loss'].append(log_history['eval_loss'])\n",
        "    perplexity_history['eval_perplexity'].append(math.exp(log_history['eval_loss']))\n",
        "\n",
        "# Plot Losses.\n",
        "plot_dict(loss_history, start_step=training_args.logging_steps, \n",
        "          step_size=training_args.logging_steps, use_title='Loss', \n",
        "          use_xlabel='Train Steps', use_ylabel='Values', magnify=2)\n",
        "\n",
        "print()\n",
        "\n",
        "# Plot Perplexities.\n",
        "plot_dict(perplexity_history, start_step=training_args.logging_steps, \n",
        "          step_size=training_args.logging_steps, use_title='Perplexity', \n",
        "          use_xlabel='Train Steps', use_ylabel='Values', magnify=2)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "[encoded data, 47436 characters]\n",
            "text/plain": [
              "<Figure size 2592x576 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "[encoded data, 44940 characters]\n",
            "text/plain": [
              "<Figure size 2592x576 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7vzFHOkLXKAp"
      },
      "source": [
        "## **Evaluate**\n",
        "\n",
        "For the final evaluation we can have a separate test set that we use to do our final perplexity evaluation. For simplicity I used the same validation text file for the final evaluation. That is the reason I get the same results as the last validation perplexity plot value."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BHS0IWmpXLA6",
        "outputId": "60c28d26-66ba-4a6c-a360-c9ed3da2f104",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 71
        }
      },
      "source": [
        "# check if `do_eval` flag is set.\n",
        "if training_args.do_eval:\n",
        "  \n",
        "  # capture output if trainer evaluate.\n",
        "  eval_output = trainer.evaluate()\n",
        "  # compute perplexity from model loss.\n",
        "  perplexity = math.exp(eval_output[\"eval_loss\"])\n",
        "  print('\\nEvaluate Perplexity: {:10,.2f}'.format(perplexity))\n",
        "else:\n",
        "  print('No evaluation needed. No evaluation data provided, `do_eval=False`!')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "\n",
              "    <div>\n",
              "        <style>\n",
              "            /* Turns off some styling */\n",
              "            progress {\n",
              "                /* gets rid of default border in Firefox and Opera. */\n",
              "                border: none;\n",
              "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
              "                background-size: auto;\n",
              "            }\n",
              "        </style>\n",
              "      \n",
              "      <progress value='250' max='250' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [250/250 00:25]\n",
              "    </div>\n",
              "    "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Evaluate Perplexity:      11.01\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "GKRFG6qzCPvq"
      },
      "source": [
        "## **Final Note**\n",
        "\n",
        "If you made it this far **Congrats!** 🎊 and **Thank you!** 🙏 for your interest in my tutorial!\n",
        "\n",
        "I've been using this code for a while now and I feel it got to a point where is nicely documented and easy to follow.\n",
        "\n",
        "Of course is easy for me to follow because I built it. That is why any feedback is welcome and it helps me improve my future tutorials!\n",
        "\n",
        "If you see something wrong please let me know by opening an issue on my [ml_things GitHub repository](https://github.com/gmihaila/ml_things/issues)!\n",
        "\n",
        "A lot of tutorials out there are mostly a one-time thing and are not being maintained. I plan on keeping my tutorials up to date as much as I can.\n",
        "\n",
        "## **Contact** 🎣\n",
        "\n",
        "🦊 GitHub: [gmihaila](https://github.com/gmihaila)\n",
        "\n",
        "🌐 Website: [gmihaila.github.io](https://gmihaila.github.io/)\n",
        "\n",
        "👔 LinkedIn: [mihailageorge](https://medium.com/r/?url=https%3A%2F%2Fwww.linkedin.com%2Fin%2Fmihailageorge)\n",
        "\n",
        "📬 Email: [georgemihaila@my.unt.edu.com](mailto:georgemihaila@my.unt.edu.com?subject=GitHub%20Website)"
      ]
    }
  ]
}