{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "pytorch_nn.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "toc_visible": true,
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "e21a8e28cb6b4b58a5fed97b314584cc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_ac48932d4d9c4db2818c70a8b1cba61a",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_93908f84a2e14bf69ba24b4b26d6b8c4",
              "IPY_MODEL_f6835d0b71f2464e8aa1758613198f07"
            ]
          }
        },
        "ac48932d4d9c4db2818c70a8b1cba61a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "93908f84a2e14bf69ba24b4b26d6b8c4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "IntProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_88b55f51bf4e4af1898cb236a3613947",
            "_dom_classes": [],
            "description": "",
            "_model_name": "IntProgressModel",
            "bar_style": "success",
            "max": 1,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_713c849a85e04f91aa6866d2062623fe"
          }
        },
        "f6835d0b71f2464e8aa1758613198f07": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_5d28676e83d54dfda9317879132d2496",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 9920512/? [00:01&lt;00:00, 8214411.84it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_637ea43588a04924ba12b7bcfdbdc7c7"
          }
        },
        "88b55f51bf4e4af1898cb236a3613947": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "713c849a85e04f91aa6866d2062623fe": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "5d28676e83d54dfda9317879132d2496": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "637ea43588a04924ba12b7bcfdbdc7c7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "d36e278feb0240b2a62687373203ae5b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_ea278541e0c04067b30e6df6fa654918",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_ff4b634a71ea4afe8937290e3ec5a2b0",
              "IPY_MODEL_aadf242972974791936099aa604783e2"
            ]
          }
        },
        "ea278541e0c04067b30e6df6fa654918": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "ff4b634a71ea4afe8937290e3ec5a2b0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "IntProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_a00aec413e3d43b3a9cc5d730c9dcc35",
            "_dom_classes": [],
            "description": "",
            "_model_name": "IntProgressModel",
            "bar_style": "success",
            "max": 1,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_fe8ee24f615c488593704d13276873f9"
          }
        },
        "aadf242972974791936099aa604783e2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_b4624253daee49a9b81a66e866f2a3a7",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 32768/? [00:00&lt;00:00, 55917.50it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_5ade06de1af748fd83e020736de088c5"
          }
        },
        "a00aec413e3d43b3a9cc5d730c9dcc35": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "fe8ee24f615c488593704d13276873f9": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "b4624253daee49a9b81a66e866f2a3a7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "5ade06de1af748fd83e020736de088c5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "57dcd54a546247bda8b0a37cd57166fd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_4833a3f0c28b4b72a4f498f3c6522880",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_f6b0ba2ffb9e478a99452947c32504e0",
              "IPY_MODEL_0b365262aeef44e3973bae9baf4ac49e"
            ]
          }
        },
        "4833a3f0c28b4b72a4f498f3c6522880": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "f6b0ba2ffb9e478a99452947c32504e0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "IntProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_5817bb7dc5c0490aaba2572737a5c058",
            "_dom_classes": [],
            "description": "",
            "_model_name": "IntProgressModel",
            "bar_style": "success",
            "max": 1,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_f7ee16f16c184a64a3abb5859cdebd55"
          }
        },
        "0b365262aeef44e3973bae9baf4ac49e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_2e053cc046e1464886305b21db02a99f",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 1654784/? [00:00&lt;00:00, 4259844.67it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_96aee189dc094a7f946cd91fc6b038b9"
          }
        },
        "5817bb7dc5c0490aaba2572737a5c058": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "f7ee16f16c184a64a3abb5859cdebd55": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "2e053cc046e1464886305b21db02a99f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "96aee189dc094a7f946cd91fc6b038b9": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "a75cd5bd1fc04135be339487550205b8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_0b6e18d7cde64a79b7b8fb3aa2014ff0",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_10544987945d4c3b94a97828aae420c8",
              "IPY_MODEL_8d9e8718046848b3b592d390f51cb94c"
            ]
          }
        },
        "0b6e18d7cde64a79b7b8fb3aa2014ff0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "10544987945d4c3b94a97828aae420c8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "IntProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_a9ccddfb188f47a59bad8a4d1d741373",
            "_dom_classes": [],
            "description": "",
            "_model_name": "IntProgressModel",
            "bar_style": "success",
            "max": 1,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_9fbdad8b486a4ef884f868e8226880f1"
          }
        },
        "8d9e8718046848b3b592d390f51cb94c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_afdf59c02b5e4431899e54292d08323a",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 8192/? [00:00&lt;00:00, 46682.52it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_a3c981ad1a194ec3b22a284e3f6677e4"
          }
        },
        "a9ccddfb188f47a59bad8a4d1d741373": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "9fbdad8b486a4ef884f868e8226880f1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "afdf59c02b5e4431899e54292d08323a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "a3c981ad1a194ec3b22a284e3f6677e4": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/gmihaila/machine_learning_things/blob/master/learning_pytorch/pytorch_nn.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Y8f-8KN0CB46",
        "colab_type": "text"
      },
      "source": [
        "### SImple NN\n",
        "\n",
        "1 hiddent layer NN"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-mbu3qmn3Q6I",
        "colab_type": "text"
      },
      "source": [
        "#### Initialize NN"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iddJzHDZCARo",
        "colab_type": "code",
        "outputId": "58696d1b-1356-4232-903d-b743569476c7",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 170
        }
      },
      "source": [
        "import torch\n",
        "\n",
        "n_input, n_hidden, n_output = 5, 3, 1\n",
        "\n",
        "## initialize tensor for inputs, and outputs \n",
        "x = torch.randn((1, n_input))\n",
        "y = torch.rand((1,n_output)) \n",
        "\n",
        "\n",
        "print(x.size())\n",
        "print(y.size())\n",
        "print()\n",
        "\n",
        "## initialize tensor variables for weights \n",
        "w1 = torch.rand((n_input, n_hidden))\n",
        "w2 = torch.rand((n_hidden, n_output))\n",
        "\n",
        "print(w1.size())\n",
        "print(w2.size())\n",
        "print()\n",
        "\n",
        "## initialize tensor variables for bias terms \n",
        "b1 = torch.rand((1,n_hidden))\n",
        "b2 = torch.rand((1,n_output))\n",
        "\n",
        "print(b1.size())\n",
        "print(b2.size())\n",
        "print()\n",
        "\n"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "torch.Size([1, 5])\n",
            "torch.Size([1, 1])\n",
            "\n",
            "torch.Size([5, 3])\n",
            "torch.Size([3, 1])\n",
            "\n",
            "torch.Size([1, 3])\n",
            "torch.Size([1, 1])\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Qsi4mZJN3WJL",
        "colab_type": "text"
      },
      "source": [
        "#### Forward Pass"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "RzKELrKSD-t5",
        "colab_type": "text"
      },
      "source": [
        "1. Forward Propagation\n",
        "2. Loss computation\n",
        "3. Backpropagation\n",
        "4. Updating the parameters"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dHNZuYc-EEWb",
        "colab_type": "code",
        "outputId": "533f2e64-da27-4d29-d562-b035a120cca1",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 136
        }
      },
      "source": [
        "## sigmoid activation function using pytorch\n",
        "def sigmoid_activation(z):\n",
        "  return 1 / (1 + torch.exp(-z))\n",
        "\n",
        "## activation of hidden layer \n",
        "z1 = torch.mm(x,w1) + b1\n",
        "a1 = sigmoid_activation(z1)\n",
        "\n",
        "print(z1)\n",
        "print(a1)\n",
        "print()\n",
        "\n",
        "## activation (output) of final layer \n",
        "z2 = torch.mm(a1, w2) + b2\n",
        "a2 = output = sigmoid_activation(z2)\n",
        "\n",
        "print(z2)\n",
        "print(output)\n",
        "print()\n",
        "\n",
        "loss = y - output\n",
        "\n",
        "print(loss)"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "tensor([[ 3.5044,  3.5523, -0.8083]])\n",
            "tensor([[0.9708, 0.9721, 0.3082]])\n",
            "\n",
            "tensor([[1.3401]])\n",
            "tensor([[0.7925]])\n",
            "\n",
            "tensor([[-0.1339]])\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Y4oRU8O9JTOG",
        "colab_type": "text"
      },
      "source": [
        "#### Backprop\n",
        "\n",
        "* loss gets multiplied by weights to penalize more of the bad weights\n",
        "* some weights contirbute more to the output. If the error is large, their loss will be more"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XbfW0NhaEkM_",
        "colab_type": "code",
        "outputId": "b6d8fe43-075f-4343-d671-7e86b264e2f6",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 119
        }
      },
      "source": [
        "## function to calculate the derivative of activation\n",
        "def sigmoid_delta(x):\n",
        "  return x * (1 - x)\n",
        "\n",
        "## compute derivative of error terms\n",
        "delta_output = sigmoid_delta(output)\n",
        "delta_hidden = sigmoid_delta(a1)\n",
        "\n",
        "print(delta_output)\n",
        "print(delta_hidden)\n",
        "print()\n",
        "\n",
        "\n",
        "## backpass the changes to previous layers \n",
        "d_output = loss * delta_output\n",
        "loss_h = torch.mm(d_output, w2.t())\n",
        "d_hidden = loss_h * delta_hidden\n",
        "\n",
        "print(d_output)\n",
        "print(loss_h)\n",
        "print(d_hidden)"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "tensor([[0.1644]])\n",
            "tensor([[0.0283, 0.0271, 0.2132]])\n",
            "\n",
            "tensor([[-0.0220]])\n",
            "tensor([[-0.0052, -0.0168, -0.0069]])\n",
            "tensor([[-0.0001, -0.0005, -0.0015]])\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "EbWmAwXtJ4xj",
        "colab_type": "text"
      },
      "source": [
        "#### Update Parameters"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YHRwQNWyJ-KG",
        "colab_type": "code",
        "outputId": "effef4a7-321b-4612-a5c5-e8b49b4983f2",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        }
      },
      "source": [
        "learning_rate = 0.1\n",
        "\n",
        "w2 += torch.mm(a1.t(), d_output) * learning_rate\n",
        "w1 += torch.mm(x.t(), d_hidden) * learning_rate\n",
        "\n",
        "\n",
        "print(w2)\n",
        "print(w1)\n",
        "print()\n",
        "\n",
        "b1 += d_output.sum() * learning_rate\n",
        "b2 += d_hidden.sum() * learning_rate\n",
        "\n",
        "print(b1)\n",
        "print(b2)"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "tensor([[0.2323],\n",
            "        [0.7628],\n",
            "        [0.3141]])\n",
            "tensor([[0.7888, 0.4538, 0.7354],\n",
            "        [0.3648, 0.4805, 0.0080],\n",
            "        [0.1933, 0.0877, 0.9228],\n",
            "        [0.7555, 0.7120, 0.0322],\n",
            "        [0.8335, 0.7132, 0.1058]])\n",
            "\n",
            "tensor([[0.8232, 0.9470, 0.4273]])\n",
            "tensor([[0.2717]])\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "vdo5seVgay4f",
        "colab_type": "text"
      },
      "source": [
        "### MNIST Data Loader"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "outputId": "7cf57b4e-0d87-4799-a590-366975450595",
        "id": "ZSRKe2UUm9-K",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 383,
          "referenced_widgets": [
            "e21a8e28cb6b4b58a5fed97b314584cc",
            "ac48932d4d9c4db2818c70a8b1cba61a",
            "93908f84a2e14bf69ba24b4b26d6b8c4",
            "f6835d0b71f2464e8aa1758613198f07",
            "88b55f51bf4e4af1898cb236a3613947",
            "713c849a85e04f91aa6866d2062623fe",
            "5d28676e83d54dfda9317879132d2496",
            "637ea43588a04924ba12b7bcfdbdc7c7",
            "d36e278feb0240b2a62687373203ae5b",
            "ea278541e0c04067b30e6df6fa654918",
            "ff4b634a71ea4afe8937290e3ec5a2b0",
            "aadf242972974791936099aa604783e2",
            "a00aec413e3d43b3a9cc5d730c9dcc35",
            "fe8ee24f615c488593704d13276873f9",
            "b4624253daee49a9b81a66e866f2a3a7",
            "5ade06de1af748fd83e020736de088c5",
            "57dcd54a546247bda8b0a37cd57166fd",
            "4833a3f0c28b4b72a4f498f3c6522880",
            "f6b0ba2ffb9e478a99452947c32504e0",
            "0b365262aeef44e3973bae9baf4ac49e",
            "5817bb7dc5c0490aaba2572737a5c058",
            "f7ee16f16c184a64a3abb5859cdebd55",
            "2e053cc046e1464886305b21db02a99f",
            "96aee189dc094a7f946cd91fc6b038b9",
            "a75cd5bd1fc04135be339487550205b8",
            "0b6e18d7cde64a79b7b8fb3aa2014ff0",
            "10544987945d4c3b94a97828aae420c8",
            "8d9e8718046848b3b592d390f51cb94c",
            "a9ccddfb188f47a59bad8a4d1d741373",
            "9fbdad8b486a4ef884f868e8226880f1",
            "afdf59c02b5e4431899e54292d08323a",
            "a3c981ad1a194ec3b22a284e3f6677e4"
          ]
        }
      },
      "source": [
        "import torch\n",
        "from torch import optim\n",
        "import torch.nn as nn\n",
        "import torch.nn.functional as F\n",
        "from torchvision import transforms\n",
        "from torchvision.datasets import MNIST\n",
        "from torch.utils.data import DataLoader\n",
        "from torch.utils.data.sampler import SubsetRandomSampler\n",
        "import numpy as np\n",
        "\n",
        "# transform the raw dataset into tensors and normalize them in a fixed range\n",
        "_tasks = transforms.Compose([transforms.ToTensor(),transforms.Normalize((0.5,), (0.5,))]) #pass mean 0.5 and std 0.5\n",
        "\n",
        "## Load MNIST Dataset and apply transformations\n",
        "mnist = MNIST(\"data\", download=True, train=True, transform=_tasks)\n",
        "\n",
        "## create training and validation split \n",
        "split = int(0.8 * len(mnist))\n",
        "index_list = list(range(len(mnist)))\n",
        "train_idx, valid_idx = index_list[:split], index_list[split:]\n",
        "\n",
        "## create sampler objects using SubsetRandomSampler\n",
        "tr_sampler = SubsetRandomSampler(train_idx)\n",
        "val_sampler = SubsetRandomSampler(valid_idx)\n",
        "\n",
        "## create iterator objects for train and valid datasets\n",
        "trainloader = DataLoader(mnist, batch_size=256, sampler=tr_sampler)\n",
        "validloader = DataLoader(mnist, batch_size=256, sampler=val_sampler)\n",
        "\n",
        "\n",
        "for data, label in trainloader:\n",
        "  print(np.shape(data))\n",
        "  # Flatten MNIST images into a 784 long vector\n",
        "  # data = data.view(data.shape[0], -1)\n",
        "  # print(data.shape)\n",
        "\n",
        "  data = torch.flatten(data, start_dim=1)\n",
        "  print(data.shape)\n",
        "  break"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Downloading http://yann.lecun.com/exdb/mnist/train-images-idx3-ubyte.gz to data/MNIST/raw/train-images-idx3-ubyte.gz\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "e21a8e28cb6b4b58a5fed97b314584cc",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(IntProgress(value=1, bar_style='info', max=1), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "Extracting data/MNIST/raw/train-images-idx3-ubyte.gz to data/MNIST/raw\n",
            "Downloading http://yann.lecun.com/exdb/mnist/train-labels-idx1-ubyte.gz to data/MNIST/raw/train-labels-idx1-ubyte.gz\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "d36e278feb0240b2a62687373203ae5b",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(IntProgress(value=1, bar_style='info', max=1), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "Extracting data/MNIST/raw/train-labels-idx1-ubyte.gz to data/MNIST/raw\n",
            "Downloading http://yann.lecun.com/exdb/mnist/t10k-images-idx3-ubyte.gz to data/MNIST/raw/t10k-images-idx3-ubyte.gz\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "57dcd54a546247bda8b0a37cd57166fd",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(IntProgress(value=1, bar_style='info', max=1), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "Extracting data/MNIST/raw/t10k-images-idx3-ubyte.gz to data/MNIST/raw\n",
            "Downloading http://yann.lecun.com/exdb/mnist/t10k-labels-idx1-ubyte.gz to data/MNIST/raw/t10k-labels-idx1-ubyte.gz\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "a75cd5bd1fc04135be339487550205b8",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(IntProgress(value=1, bar_style='info', max=1), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "Extracting data/MNIST/raw/t10k-labels-idx1-ubyte.gz to data/MNIST/raw\n",
            "Processing...\n",
            "Done!\n",
            "\n",
            "\n",
            "torch.Size([256, 1, 28, 28])\n",
            "torch.Size([256, 784])\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8RFHs8bda0xx",
        "colab_type": "text"
      },
      "source": [
        "### MNIST - NN"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0coSr0H-a4yR",
        "colab_type": "code",
        "outputId": "6eb1836f-5809-4704-b5e1-e15e000aa3ef",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 221
        }
      },
      "source": [
        "import torch\n",
        "from torch import optim\n",
        "import torch.nn as nn\n",
        "import torch.nn.functional as F\n",
        "from torchvision import transforms\n",
        "from torchvision.datasets import MNIST\n",
        "from torch.utils.data import DataLoader\n",
        "from torch.utils.data.sampler import SubsetRandomSampler\n",
        "import numpy as np\n",
        "\n",
        "# transform the raw dataset into tensors and normalize them in a fixed range\n",
        "_tasks = transforms.Compose([transforms.ToTensor(),transforms.Normalize((0.5,), (0.5,))])\n",
        "\n",
        "## Load MNIST Dataset and apply transformations\n",
        "mnist = MNIST(\"data\", download=True, train=True, transform=_tasks)\n",
        "\n",
        "## create training and validation split \n",
        "split = int(0.8 * len(mnist))\n",
        "index_list = list(range(len(mnist)))\n",
        "train_idx, valid_idx = index_list[:split], index_list[split:]\n",
        "\n",
        "## create sampler objects using SubsetRandomSampler\n",
        "tr_sampler = SubsetRandomSampler(train_idx)\n",
        "val_sampler = SubsetRandomSampler(valid_idx)\n",
        "\n",
        "## create iterator objects for train and valid datasets\n",
        "trainloader = DataLoader(mnist, batch_size=256, sampler=tr_sampler)\n",
        "validloader = DataLoader(mnist, batch_size=256, sampler=val_sampler)\n",
        "\n",
        "## Build class of model\n",
        "class Model(nn.Module):\n",
        "    def __init__(self):\n",
        "        super().__init__()\n",
        "        self.hidden = nn.Linear(784, 128)\n",
        "        self.output = nn.Linear(128, 10)\n",
        "\n",
        "    def forward(self, x):\n",
        "        x = self.hidden(x)\n",
        "        x = torch.sigmoid(x)\n",
        "        x = self.output(x)\n",
        "        return x\n",
        "\n",
        "model = Model()\n",
        "\n",
        "loss_function = nn.CrossEntropyLoss()\n",
        "optimizer = optim.SGD(model.parameters(), lr=0.01, weight_decay=1e-6, momentum=0.9, nesterov=True)\n",
        "\n",
        "for epoch in range(1,11):\n",
        "\n",
        "  train_loss, valid_loss = [], []\n",
        "  model.train() # activates training mod\n",
        "\n",
        "  ## Training on 1 epoch\n",
        "  for data, target in trainloader:\n",
        "\n",
        "    data = torch.flatten(data, start_dim=1)\n",
        "\n",
        "    optimizer.zero_grad() #clears gradients of all optimized classes\n",
        "\n",
        "    ## forward pass\n",
        "    output = model(data)\n",
        "\n",
        "    ## loss calc\n",
        "    loss = loss_function(output, target)\n",
        "\n",
        "    ## backeard propagation\n",
        "    loss.backward()\n",
        "\n",
        "    ## weight optimization\n",
        "    optimizer.step() #performs a single optimization step\n",
        "    train_loss.append(loss.item())\n",
        "\n",
        "  ### Evaluation on 1 epoch\n",
        "  for data, target in validloader:\n",
        "\n",
        "    data = torch.flatten(data, start_dim=1)\n",
        "\n",
        "    output = model(data)\n",
        "    loss = loss_function(output, target)\n",
        "    valid_loss.append(loss.item())\n",
        "  \n",
        "  print(\"Epoch:\", epoch, \"Training Loss: \", np.mean(train_loss), \"Valid Loss: \", np.mean(valid_loss))"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\n",
            "Epoch: 1 Training Loss:  1.4349780979942768 Valid Loss:  0.7062469999840919\n",
            "Epoch: 2 Training Loss:  0.575546939163766 Valid Loss:  0.4442503724960571\n",
            "Epoch: 3 Training Loss:  0.43013869321092646 Valid Loss:  0.3703078912927749\n",
            "Epoch: 4 Training Loss:  0.37567793118192794 Valid Loss:  0.33534921578904414\n",
            "Epoch: 5 Training Loss:  0.3451666477671329 Valid Loss:  0.3142616549070845\n",
            "Epoch: 6 Training Loss:  0.3242472232656276 Valid Loss:  0.30002586036286455\n",
            "Epoch: 7 Training Loss:  0.30901888607347267 Valid Loss:  0.2856987930358724\n",
            "Epoch: 8 Training Loss:  0.2960701195642035 Valid Loss:  0.2761637606519334\n",
            "Epoch: 9 Training Loss:  0.2850086708810735 Valid Loss:  0.26927014361036583\n",
            "Epoch: 10 Training Loss:  0.27550541340036594 Valid Loss:  0.2590240886870851\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "j_5G0MzHewg8",
        "colab_type": "text"
      },
      "source": [
        "#### Evaluation"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "o7DHUA_wbkYI",
        "colab_type": "code",
        "outputId": "38017cf9-da6c-46fd-c288-151df5590519",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 136
        }
      },
      "source": [
        "## dataloader for validation dataset \n",
        "dataiter = iter(validloader)\n",
        "data, labels = dataiter.next()\n",
        "data = torch.flatten(data, start_dim=1)\n",
        "output = model(data)\n",
        "\n",
        "print(output.shape)\n",
        "print(output[0])\n",
        "\n",
        "_, pred_tensor = torch.max(output, 1)\n",
        "\n",
        "print(pred_tensor.shape)\n",
        "print(pred_tensor[0])\n",
        "\n",
        "preds = np.squeeze(pred_tensor.numpy())\n",
        "\n",
        "print(\"Actual: \", labels[:10])\n",
        "print(\"Predic: \", preds[:10])"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "torch.Size([256, 10])\n",
            "tensor([ 1.0437, -1.5937, -0.2724,  1.1894,  1.1126,  5.0138,  2.1918, -2.3319,\n",
            "        -1.1587, -4.2133], grad_fn=<SelectBackward>)\n",
            "torch.Size([256])\n",
            "tensor(5)\n",
            "Actual:  tensor([5, 1, 3, 1, 2, 6, 6, 7, 5, 6])\n",
            "Predic:  [5 1 3 1 2 6 6 7 5 6]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "iY07nwY8rcDg",
        "colab_type": "text"
      },
      "source": [
        "### MNIST - NN [1 GPU]"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ha61G2ngrbiT",
        "colab_type": "code",
        "outputId": "88bc8fa4-055a-432c-adbd-2563207e9b32",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 187
        }
      },
      "source": [
        "import torch\n",
        "from torch import optim\n",
        "import torch.nn as nn\n",
        "import torch.nn.functional as F\n",
        "from torchvision import transforms\n",
        "from torchvision.datasets import MNIST\n",
        "from torch.utils.data import DataLoader\n",
        "from torch.utils.data.sampler import SubsetRandomSampler\n",
        "import numpy as np\n",
        "from torch.backends import cudnn\n",
        "\n",
        "cudnn.benchmark = True\n",
        "\n",
        "# transform the raw dataset into tensors and normalize them in a fixed range\n",
        "_tasks = transforms.Compose([transforms.ToTensor(),transforms.Normalize((0.5,), (0.5,))])\n",
        "\n",
        "## Load MNIST Dataset and apply transformations\n",
        "mnist = MNIST(\"data\", download=True, train=True, transform=_tasks)\n",
        "\n",
        "## create training and validation split \n",
        "split = int(0.8 * len(mnist))\n",
        "index_list = list(range(len(mnist)))\n",
        "train_idx, valid_idx = index_list[:split], index_list[split:]\n",
        "\n",
        "## create sampler objects using SubsetRandomSampler\n",
        "tr_sampler = SubsetRandomSampler(train_idx)\n",
        "val_sampler = SubsetRandomSampler(valid_idx)\n",
        "\n",
        "## create iterator objects for train and valid datasets\n",
        "trainloader = DataLoader(mnist, batch_size=256, sampler=tr_sampler, num_workers=2)\n",
        "validloader = DataLoader(mnist, batch_size=256, sampler=val_sampler, num_workers=2)\n",
        "\n",
        "## GPU\n",
        "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
        "\n",
        "## Build class of model\n",
        "class Model(nn.Module):\n",
        "    def __init__(self):\n",
        "        super().__init__()\n",
        "        self.hidden = nn.Linear(784, 128)\n",
        "        self.output = nn.Linear(128, 10)\n",
        "\n",
        "    def forward(self, x):\n",
        "        x = self.hidden(x)\n",
        "        x = torch.sigmoid(x)\n",
        "        x = self.output(x)\n",
        "        return x\n",
        "\n",
        "model = Model()\n",
        "\n",
        "model.to(device)\n",
        "\n",
        "loss_function = nn.CrossEntropyLoss()\n",
        "optimizer = optim.SGD(model.parameters(), lr=0.01, weight_decay=1e-6, momentum=0.9, nesterov=True)\n",
        "\n",
        "for epoch in range(1,11):\n",
        "\n",
        "  train_loss, valid_loss = [], []\n",
        "  model.train() # activates training mod\n",
        "\n",
        "  ## Training on 1 epoch\n",
        "  for data, target in trainloader:\n",
        "\n",
        "    data = torch.flatten(data.to(device), start_dim=1)\n",
        "\n",
        "    optimizer.zero_grad() #clears gradients of all optimized classes\n",
        "\n",
        "    ## forward pass\n",
        "    output = model(data.to(device))\n",
        "\n",
        "    ## loss calc\n",
        "    loss = loss_function(output.to(device), target.to(device))\n",
        "\n",
        "    ## backeard propagation\n",
        "    loss.backward()\n",
        "\n",
        "    ## weight optimization\n",
        "    optimizer.step() #performs a single optimization step\n",
        "    train_loss.append(loss.item())\n",
        "\n",
        "  ### Evaluation on 1 epoch\n",
        "  for data, target in validloader:\n",
        "\n",
        "    data = torch.flatten(data, start_dim=1)\n",
        "\n",
        "    output = model(data.to(device))\n",
        "    loss = loss_function(output.to(device), target.to(device))\n",
        "    valid_loss.append(loss.item())\n",
        "  \n",
        "  print(\"Epoch:\", epoch, \"Training Loss: \", np.mean(train_loss), \"Valid Loss: \", np.mean(valid_loss))"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch: 1 Training Loss:  1.4226015869607316 Valid Loss:  0.702143872037847\n",
            "Epoch: 2 Training Loss:  0.5734938942688577 Valid Loss:  0.44547347756142314\n",
            "Epoch: 3 Training Loss:  0.4304347908560266 Valid Loss:  0.3725986708985998\n",
            "Epoch: 4 Training Loss:  0.3755895825142556 Valid Loss:  0.3367689743320993\n",
            "Epoch: 5 Training Loss:  0.3452775955517241 Valid Loss:  0.3167601830147682\n",
            "Epoch: 6 Training Loss:  0.3250410721657124 Valid Loss:  0.30036201534119056\n",
            "Epoch: 7 Training Loss:  0.3101884127455823 Valid Loss:  0.2885698393938389\n",
            "Epoch: 8 Training Loss:  0.29760462307232494 Valid Loss:  0.2776079479050129\n",
            "Epoch: 9 Training Loss:  0.2871963589432392 Valid Loss:  0.269358513837165\n",
            "Epoch: 10 Training Loss:  0.2774508301406465 Valid Loss:  0.26319165052251614\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Gb4m3lfy3Dbz",
        "colab_type": "text"
      },
      "source": [
        "#### Evaluation"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2J_sioHK3E9a",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "scUaqYhqvfyP",
        "colab_type": "text"
      },
      "source": [
        "### MNIST - NN [Multy GPU, Core]\n",
        "\n",
        "Specify certain GPUs"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vfdi1HANvi7f",
        "colab_type": "code",
        "outputId": "cdeb2434-2b4f-4c1c-eb74-119ebac6ec03",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 187
        }
      },
      "source": [
        "import os\n",
        "# os.environ[\"CUDA_VISIBLE_DEVICES\"]=\"0,1,2\" # number of gpu devices\n",
        "import torch\n",
        "from torch import optim\n",
        "import torch.nn as nn\n",
        "import torch.nn.functional as F\n",
        "from torchvision import transforms\n",
        "from torchvision.datasets import MNIST\n",
        "from torch.utils.data import DataLoader\n",
        "from torch.utils.data.sampler import SubsetRandomSampler\n",
        "import numpy as np\n",
        "from torch.backends import cudnn\n",
        "import multiprocessing\n",
        "\n",
        "cudnn.benchmark = True\n",
        "\n",
        "n_cores = multiprocessing.cpu_count()\n",
        "\n",
        "# transform the raw dataset into tensors and normalize them in a fixed range\n",
        "_tasks = transforms.Compose([transforms.ToTensor(),transforms.Normalize((0.5,), (0.5,))])\n",
        "\n",
        "## Load MNIST Dataset and apply transformations\n",
        "mnist = MNIST(\"data\", download=True, train=True, transform=_tasks)\n",
        "\n",
        "## create training and validation split \n",
        "split = int(0.8 * len(mnist))\n",
        "index_list = list(range(len(mnist)))\n",
        "train_idx, valid_idx = index_list[:split], index_list[split:]\n",
        "\n",
        "## create sampler objects using SubsetRandomSampler\n",
        "tr_sampler = SubsetRandomSampler(train_idx)\n",
        "val_sampler = SubsetRandomSampler(valid_idx)\n",
        "\n",
        "## create iterator objects for train and valid datasets\n",
        "trainloader = DataLoader(mnist, batch_size=256, sampler=tr_sampler, num_workers=n_cores)\n",
        "validloader = DataLoader(mnist, batch_size=256, sampler=val_sampler, num_workers=n_cores)\n",
        "\n",
        "## GPU\n",
        "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
        "\n",
        "## Build class of model\n",
        "class Model(nn.Module):\n",
        "    def __init__(self):\n",
        "        super().__init__()\n",
        "        self.hidden = nn.Linear(784, 128)\n",
        "        self.output = nn.Linear(128, 10)\n",
        "\n",
        "    def forward(self, x):\n",
        "        x = self.hidden(x)\n",
        "        x = torch.sigmoid(x)\n",
        "        x = self.output(x)\n",
        "        return x\n",
        "\n",
        "model = Model()\n",
        "\n",
        "## Multi GPU\n",
        "if torch.cuda.device_count() > 1:\n",
        "  print(\"We can use\", torch.cuda.device_count(), \"GPUs\")\n",
        "  model = nn.DataParallel(model, device_ids=[1]) # device_ids=[0,1,2] depending on the # of gpus\n",
        "\n",
        "model.to(device)\n",
        "\n",
        "loss_function = nn.CrossEntropyLoss()\n",
        "optimizer = optim.SGD(model.parameters(), lr=0.01, weight_decay=1e-6, momentum=0.9, nesterov=True)\n",
        "\n",
        "for epoch in range(1,11):\n",
        "\n",
        "  train_loss, valid_loss = [], []\n",
        "  model.train() # activates training mod\n",
        "\n",
        "  ## Training on 1 epoch\n",
        "  for data, target in trainloader:\n",
        "\n",
        "    data = torch.flatten(data.to(device), start_dim=1)\n",
        "\n",
        "    optimizer.zero_grad() #clears gradients of all optimized classes\n",
        "\n",
        "    ## forward pass\n",
        "    output = model(data.to(device))\n",
        "\n",
        "    ## loss calc\n",
        "    loss = loss_function(output.to(device), target.to(device))\n",
        "\n",
        "    ## backeard propagation\n",
        "    loss.backward()\n",
        "\n",
        "    ## weight optimization\n",
        "    optimizer.step() #performs a single optimization step\n",
        "    train_loss.append(loss.item())\n",
        "\n",
        "  ### Evaluation on 1 epoch\n",
        "  for data, target in validloader:\n",
        "\n",
        "    data = torch.flatten(data, start_dim=1)\n",
        "\n",
        "    output = model(data.to(device))\n",
        "    loss = loss_function(output.to(device), target.to(device))\n",
        "    valid_loss.append(loss.item())\n",
        "  \n",
        "  print(\"Epoch:\", epoch, \"Training Loss: \", np.mean(train_loss), \"Valid Loss: \", np.mean(valid_loss))"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch: 1 Training Loss:  1.4196174848586955 Valid Loss:  0.7070348668605724\n",
            "Epoch: 2 Training Loss:  0.5785721481797543 Valid Loss:  0.4495881415427999\n",
            "Epoch: 3 Training Loss:  0.4331658460358356 Valid Loss:  0.37312279990378844\n",
            "Epoch: 4 Training Loss:  0.37733632976070364 Valid Loss:  0.33723752802990853\n",
            "Epoch: 5 Training Loss:  0.346332589521053 Valid Loss:  0.31461838109696166\n",
            "Epoch: 6 Training Loss:  0.3250312273331145 Valid Loss:  0.297722431256416\n",
            "Epoch: 7 Training Loss:  0.30921553867928525 Valid Loss:  0.28540977264972445\n",
            "Epoch: 8 Training Loss:  0.29635619277015646 Valid Loss:  0.2772708123034619\n",
            "Epoch: 9 Training Loss:  0.28546011772878627 Valid Loss:  0.2672363671850651\n",
            "Epoch: 10 Training Loss:  0.2761305772719231 Valid Loss:  0.25905618927580243\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "dvTbQmso2_VR",
        "colab_type": "text"
      },
      "source": [
        "#### Evaluation"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dqB5jqTv3BcY",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8Wy3zgHCy7Ot",
        "colab_type": "text"
      },
      "source": [
        "### MNIST CNN [Multy GPU, Core]"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uJUm3qjvBfFe",
        "colab_type": "code",
        "outputId": "c71a5d12-7cd2-425e-f457-e61f56583ce6",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 187
        }
      },
      "source": [
        "import os\n",
        "import torch\n",
        "from torch import optim\n",
        "import torch.nn as nn\n",
        "import torch.nn.functional as F\n",
        "from torchvision import transforms\n",
        "from torchvision.datasets import MNIST\n",
        "from torch.utils.data import DataLoader\n",
        "from torch.utils.data.sampler import SubsetRandomSampler\n",
        "from torch.backends import cudnn\n",
        "import numpy as np\n",
        "import multiprocessing\n",
        "from sklearn.metrics import accuracy_score\n",
        "\n",
        "cudnn.benchmark = True\n",
        "\n",
        "num_cores = multiprocessing.cpu_count()\n",
        "\n",
        "# transform the raw dataset into tensors and normalize them in a fixed range\n",
        "_tasks = transforms.Compose([transforms.ToTensor(), transforms.Normalize((0.5,), (0.5,))])\n",
        "\n",
        "## Load MNIST Dataset and apply transformations\n",
        "mnist = MNIST(\"data\", download=True, train=True, transform=_tasks)\n",
        "\n",
        "## create training and validation split \n",
        "split_train = int(0.7 * len(mnist))\n",
        "split_valid = split_train + int(0.1 * len(mnist))\n",
        "index_list = list(range(len(mnist)))\n",
        "train_idx, valid_idx, test_idx = index_list[:split_train], index_list[split_train:split_valid], index_list[split_valid:]\n",
        "\n",
        "## create sampler objects using SubsetRandomSampler\n",
        "tr_sampler = SubsetRandomSampler(train_idx)\n",
        "val_sampler = SubsetRandomSampler(valid_idx)\n",
        "tes_sampler = SubsetRandomSampler(test_idx)\n",
        "\n",
        "## create iterator objects for train and valid datasets\n",
        "trainloader = DataLoader(mnist, batch_size=256, sampler=tr_sampler, num_workers=num_cores)\n",
        "validloader = DataLoader(mnist, batch_size=256, sampler=val_sampler, num_workers=num_cores)\n",
        "testloader = DataLoader(mnist, batch_size=10, sampler=tes_sampler, num_workers=num_cores)\n",
        "\n",
        "## GPU\n",
        "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
        "\n",
        "## Build class of model\n",
        "class Model(nn.Module):\n",
        "  def __init__(self):\n",
        "    super(Model, self).__init__()\n",
        "    \n",
        "    ## define layers\n",
        "    self.conv1 = nn.Conv2d(1, 16, 3, padding=1)\n",
        "    self.conv2 = nn.Conv2d(16, 32, 3, padding=1)\n",
        "    self.conv3 = nn.Conv2d(32, 64, 3, padding=1)\n",
        "    self.pool = nn.MaxPool2d(kernel_size=2, stride=2)\n",
        "    self.linear1 = nn.Linear(64*3*3, 512)\n",
        "    self.linear2 = nn.Linear(512,10)\n",
        "    \n",
        "    return\n",
        "\n",
        "  def forward(self, x):\n",
        "    x = self.pool(F.relu(self.conv1(x)))\n",
        "    x = self.pool(F.relu(self.conv2(x)))\n",
        "    x = self.pool(F.relu(self.conv3(x)))\n",
        "    x = x.view(-1,64*3*3) #torch.flatten(x, start_dim=1) ## reshaping\n",
        "    x = F.relu(self.linear1(x))\n",
        "    x = self.linear2(x)\n",
        "\n",
        "    return x\n",
        "\n",
        "## create model\n",
        "model = Model()\n",
        "\n",
        "## in case of multi gpu\n",
        "if torch.cuda.device_count() > 1:\n",
        "  print(\"Using\", torch.cuda.device_count(), \"GPUs\")\n",
        "  model = nn.DataParallel(model, device_ids=[1]) # [0,1,2,3]\n",
        "\n",
        "## put model on gpu\n",
        "model.to(device)\n",
        "\n",
        "## loss fucntion\n",
        "loss_function = nn.CrossEntropyLoss()\n",
        "## optimizer\n",
        "optimizer = optim.SGD(model.parameters(), lr=0.01, weight_decay=1e-6, momentum=0.9, nesterov=True)\n",
        "## run for n epochs\n",
        "for epoch in range(1,11):\n",
        "  train_loss , valid_loss = [], []\n",
        "\n",
        "  ## train part\n",
        "  model.train()\n",
        "  for data, target in trainloader:\n",
        "    ## gradients acumulate. need to clear them on each example\n",
        "    optimizer.zero_grad()\n",
        "    output = model(data.to(device))\n",
        "    loss = loss_function(output.to(device), target.to(device))\n",
        "    loss.backward()\n",
        "    optimizer.step()\n",
        "    train_loss.append(loss.item())\n",
        "\n",
        "  ## evaluation part on validation\n",
        "  model.eval() ##set model in evaluation mode\n",
        "  for data, target in validloader:\n",
        "    output = model(data.to(device))\n",
        "    loss = loss_function(output.to(device), target.to(device))\n",
        "    valid_loss.append(loss.item())\n",
        "\n",
        "  print(\"Epoch:\", epoch, \"Training Loss: \", np.mean(train_loss), \"Valid Loss: \", np.mean(valid_loss))"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch: 1 Training Loss:  1.43601565442302 Valid Loss:  0.596405832717816\n",
            "Epoch: 2 Training Loss:  0.16294173417669355 Valid Loss:  0.138090871895353\n",
            "Epoch: 3 Training Loss:  0.0923407681286335 Valid Loss:  0.11756174452602863\n",
            "Epoch: 4 Training Loss:  0.0715204238778714 Valid Loss:  0.3352052476257086\n",
            "Epoch: 5 Training Loss:  0.06112954479952653 Valid Loss:  0.07890327313604455\n",
            "Epoch: 6 Training Loss:  0.0497398627075282 Valid Loss:  0.07062170887365937\n",
            "Epoch: 7 Training Loss:  0.04241228329412865 Valid Loss:  0.06184114965920647\n",
            "Epoch: 8 Training Loss:  0.03664553004006545 Valid Loss:  0.06988479352245729\n",
            "Epoch: 9 Training Loss:  0.03334167885848067 Valid Loss:  0.4880245228608449\n",
            "Epoch: 10 Training Loss:  0.035245017300952565 Valid Loss:  0.06090639275498688\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "hJZJLvwgsUrq",
        "colab_type": "text"
      },
      "source": [
        "#### Evaluation"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eSRiJuEPsW0x",
        "colab_type": "code",
        "outputId": "8706ef72-5464-4421-c667-debaa9bdf569",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "model.eval()\n",
        "\n",
        "y_pred, y_true = [], []\n",
        "\n",
        "for data, target in testloader:\n",
        "  predicted = model(data.to(device))\n",
        "  _, predicted = torch.max(predicted.cpu(), 1)\n",
        "  y_pred += predicted.tolist()\n",
        "  y_true += target.tolist()\n",
        "\n",
        "print(\"Accuracy: \", accuracy_score(y_pred, y_true))"
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Accuracy:  0.9851666666666666\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "pLsU972v4EvB",
        "colab_type": "text"
      },
      "source": [
        "### Sentiment Classification [NOT FINISHED]"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "yuyc1nb2A1Ke",
        "colab_type": "text"
      },
      "source": [
        "#### Download Data"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Cyuwl8oo43OQ",
        "colab_type": "code",
        "outputId": "a7a01b0a-2266-4537-cb52-441543f65bcf",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "from IPython.display import clear_output\n",
        "\n",
        "!wget http://ai.stanford.edu/~amaas/data/sentiment/aclImdb_v1.tar.gz\n",
        "clear_output(wait=True)\n",
        "\n",
        "!gunzip aclImdb_v1.tar.gz\n",
        "clear_output(wait=True)\n",
        "\n",
        "!tar -xvf aclImdb_v1.tar\n",
        "clear_output(wait=True)\n",
        "\n",
        "!ls"
      ],
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "aclImdb  aclImdb_v1.tar  data  sample_data\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "A6yQKeyZBv05",
        "colab_type": "text"
      },
      "source": [
        "#### Import"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HFmoyx-8BxcB",
        "colab_type": "code",
        "outputId": "9a28998c-9191-4178-f121-302e06c22f2d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "## Load TF 2.0\n",
        "try:\n",
        "  # %tensorflow_version only exists in Colab.\n",
        "  %tensorflow_version 2.x\n",
        "except Exception:\n",
        "  pass\n",
        "  \n",
        "import os\n",
        "from os import listdir\n",
        "from os.path import isfile, join\n",
        "import torch\n",
        "from torch import optim\n",
        "import torch.nn as nn\n",
        "import torch.nn.functional as F\n",
        "from torchvision import transforms\n",
        "from torch.utils.data import DataLoader, Dataset\n",
        "from torch.utils.data.sampler import SubsetRandomSampler\n",
        "from torch.backends import cudnn\n",
        "import tensorflow as tf\n",
        "import numpy as np\n",
        "import multiprocessing\n",
        "from keras.preprocessing.text import Tokenizer\n",
        "from keras.preprocessing.sequence import pad_sequences\n",
        "\n",
        "num_cores = multiprocessing.cpu_count()"
      ],
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Using TensorFlow backend.\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zqMBCcSBBxqS",
        "colab_type": "text"
      },
      "source": [
        "#### Data Loader"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OxmgmfNbB00E",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "train_pos_path = \"/content/aclImdb/train/pos/\"\n",
        "train_neg_path = \"/content/aclImdb/train/neg/\"\n",
        "\n",
        "test_pos_path = \"/content/aclImdb/test/pos/\"\n",
        "test_neg_path = \"/content/aclImdb/test/neg/\"\n",
        "\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cqkArXC_Df_s",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class ImdbMovieDataset(Dataset):\n",
        "\n",
        "  def __init__(self, pos_path, neg_path, maxlen=100, text_tokenizer=None):\n",
        "    \"\"\"\n",
        "    Args:\n",
        "\n",
        "    \"\"\"\n",
        "    self.pos_path = pos_path\n",
        "    self.neg_path = neg_path\n",
        "    self.pos_files = [file_name[:-4] for file_name in self.get_files(self.pos_path)]\n",
        "    self.neg_files = [file_name[:-4] for file_name in self.get_files(self.neg_path)]\n",
        "    self.n_pos_files = len(self.pos_files)\n",
        "    self.n_neg_files = len(self.neg_files)\n",
        "    self.maxlen = maxlen\n",
        "    if text_tokenizer:\n",
        "      self.text_tokenizer = text_tokenizer\n",
        "    else:\n",
        "      self.text_tokenizer = self.fit_tokenizer()\n",
        "    return\n",
        "\n",
        "\n",
        "  def __len__(self):\n",
        "    return (self.n_pos_files + self.n_neg_files)\n",
        "\n",
        "\n",
        "  def __getitem__(self, idx):\n",
        "    if idx < self.n_pos_files:\n",
        "      ## positive review\n",
        "      path = self.pos_path + self.pos_files[idx]\n",
        "      y = 1\n",
        "    else:\n",
        "      ## negative review\n",
        "      path = self.neg_path + self.neg_files[idx - self.n_pos_files]\n",
        "      y = 0\n",
        "    review = self.read_file(path)\n",
        "    X = self.text_tokenizer.texts_to_sequences([review])\n",
        "    X = pad_sequences(sequences=X, \n",
        "                      maxlen=self.maxlen, \n",
        "                      padding='post', \n",
        "                      truncating='post')[0]\n",
        "\n",
        "    return torch.tensor(X), torch.tensor(y)\n",
        "\n",
        "  def get_files(self, path):\n",
        "    return [f for f in listdir(path) if isfile(join(path, f))]\n",
        "\n",
        "  def read_file(self, path):\n",
        "    with open(path + '.txt', 'r') as raw_file:\n",
        "      content_file = raw_file.read()\n",
        "    return content_file\n",
        "\n",
        "  def fit_tokenizer(self):\n",
        "    tmp_tokenizer = Tokenizer(num_words=None,\n",
        "                      lower=True,\n",
        "                      oov_token='<UNK>')\n",
        "    print(\"Positive file fit Tokenizer\")\n",
        "    for pos_file in self.pos_files[:50]:\n",
        "      tmp_tokenizer.fit_on_texts([self.read_file(self.pos_path + pos_file)])\n",
        "\n",
        "    print(\"Negative file fit Tokenizer\")\n",
        "    for neg_file in self.neg_files[:50]:\n",
        "      tmp_tokenizer.fit_on_texts([self.read_file(self.neg_path + neg_file)])\n",
        "    return tmp_tokenizer"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hQ5hClFgQO4Y",
        "colab_type": "code",
        "outputId": "c6da2d0d-2c22-43e5-c6ce-c5e0c17696fa",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        }
      },
      "source": [
        "## data generator parameters\n",
        "data_generator_parameters = {'batch_size': 64,\n",
        "                             'shuffle': True,\n",
        "                             'num_workers': num_cores}\n",
        "\n",
        "training_set = ImdbMovieDataset(pos_path=train_pos_path,\n",
        "                              neg_path=train_neg_path)"
      ],
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Positive file fit Tokenizer\n",
            "Negative file fit Tokenizer\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JKZ355GxSPzc",
        "colab_type": "code",
        "outputId": "e9ac5b6b-c1e4-4dc6-82a9-f0d69568248d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 207
        }
      },
      "source": [
        "training_generator = DataLoader(training_set, **data_generator_parameters)\n",
        "\n",
        "for local_batch, local_label in training_generator:\n",
        "  print(local_batch, local_label)\n",
        "  print(np.shape(local_label))\n",
        "  break"
      ],
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "tensor([[  11,  265,    1,  ...,  298,    1,    4],\n",
            "        [  16,   44,   96,  ..., 4186,    1, 4005],\n",
            "        [  12,  824,  195,  ...,    2,   98, 1633],\n",
            "        ...,\n",
            "        [2509, 3712,    4,  ..., 2629,    6,    1],\n",
            "        [ 292,  413,   11,  ...,    0,    0,    0],\n",
            "        [  52,    3,    1,  ..., 4127,   38,  127]], dtype=torch.int32) tensor([1, 1, 1, 0, 0, 0, 0, 1, 0, 1, 1, 0, 1, 0, 0, 1, 0, 1, 0, 1, 0, 0, 0, 1,\n",
            "        0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 1, 1, 1, 1, 1, 0, 0, 0, 1, 0, 0, 0, 1, 1,\n",
            "        1, 1, 1, 1, 1, 1, 0, 0, 0, 1, 1, 1, 1, 1, 0, 1])\n",
            "torch.Size([64])\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4kb5gAfhN0Vm",
        "colab_type": "code",
        "outputId": "d64d2322-50e9-45ba-ed9c-beb31d8d22d1",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "torch.tensor(training_set.text_tokenizer.texts_to_sequences([\"this is me, not you\"]), dtype=torch.long)"
      ],
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([[11,  8, 80, 22, 25]])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 18
        }
      ]
    }
  ]
}