{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "fakenews_george.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "toc_visible": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "f854dbd4fe644a0c8ef4086e6ae586b8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_4d3f19ed08434aa6aa54698880d8d997",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_b0c1c425171643ae9deb8617b9c135af",
              "IPY_MODEL_9f9a02f4c9b34042a48ef0e29370f062"
            ]
          }
        },
        "4d3f19ed08434aa6aa54698880d8d997": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "b0c1c425171643ae9deb8617b9c135af": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_6ba7d4957de14b9d9140bb9045c3f865",
            "_dom_classes": [],
            "description": "100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 1,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_0bc0a2090cae469c811b5d6ef12010d7"
          }
        },
        "9f9a02f4c9b34042a48ef0e29370f062": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_4f050b7cca9644a2a3add840ff6fd3d9",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 1/1 [00:09&lt;00:00,  9.51s/it]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_5ab158d989e14302891212c785393439"
          }
        },
        "6ba7d4957de14b9d9140bb9045c3f865": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "0bc0a2090cae469c811b5d6ef12010d7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "4f050b7cca9644a2a3add840ff6fd3d9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "5ab158d989e14302891212c785393439": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "747ea6942849424a838ade456764972a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_275cd014ca754ebfa3dc85a24d74d57e",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_88ce031e29f54ac2a8c8c3f9476a9a37",
              "IPY_MODEL_91c3dd4e7d584aaea52aae4383a1cff2"
            ]
          }
        },
        "275cd014ca754ebfa3dc85a24d74d57e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "88ce031e29f54ac2a8c8c3f9476a9a37": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_feff3fb7066540d584f98af26cabeb84",
            "_dom_classes": [],
            "description": "100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 1,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_5e74244fc3c842e1a730a10017b3179f"
          }
        },
        "91c3dd4e7d584aaea52aae4383a1cff2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_c764287c354046398677727dd127db79",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 1/1 [00:26&lt;00:00, 26.83s/it]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_11d81797b8304165935c9fa2f2c87709"
          }
        },
        "feff3fb7066540d584f98af26cabeb84": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "5e74244fc3c842e1a730a10017b3179f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "c764287c354046398677727dd127db79": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "11d81797b8304165935c9fa2f2c87709": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "f3a48be3db4345e690d3f8d5c16f355d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_984f8878630148d6964380a714a2f26a",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_8aa3af23b99446e89423737c4c74bb3d",
              "IPY_MODEL_39bb87e159404852859c82d064db1dc9"
            ]
          }
        },
        "984f8878630148d6964380a714a2f26a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "8aa3af23b99446e89423737c4c74bb3d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_4f5edb4201154da4bde2810dd74c3cee",
            "_dom_classes": [],
            "description": "100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 44898,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 44898,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_1d65a93f06514d00a028884eb64ad7cd"
          }
        },
        "39bb87e159404852859c82d064db1dc9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_d3468fc571fc4546959ba07cdefcadf5",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 44898/44898 [02:20&lt;00:00, 320.63it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_049eab7924034dde9e68908ede74af24"
          }
        },
        "4f5edb4201154da4bde2810dd74c3cee": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "1d65a93f06514d00a028884eb64ad7cd": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "d3468fc571fc4546959ba07cdefcadf5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "049eab7924034dde9e68908ede74af24": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "rTy2vS0YWt4-",
        "colab_type": "text"
      },
      "source": [
        "# Info \n",
        "\n",
        "Detect Fake news\n",
        "\n",
        "\n",
        "\n",
        "### [Run in Google Colab](https://colab.research.google.com/drive/1Z8pgm3lAb-LEU2L0CPfHzVjpT_ktEDo2#offline=true&sandboxMode=true)\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "dswkfJ0y87MH",
        "colab_type": "text"
      },
      "source": [
        "# Install"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OjaKVy-J88cs",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# install fix text\n",
        "!pip install -q ftfy"
      ],
      "execution_count": 1,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3ULrf9gm_6a_",
        "colab_type": "text"
      },
      "source": [
        "## Imports"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aiI4KfXUEoya",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "5c15226d-1b95-41db-9310-61df78378583"
      },
      "source": [
        "import os\n",
        "import io\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "import keras\n",
        "import nltk\n",
        "import requests\n",
        "from tqdm.notebook import tqdm\n",
        "import matplotlib.pyplot as plt\n",
        "import traceback\n",
        "import tensorflow as tf\n",
        "from ftfy import fix_text\n",
        "from sklearn.model_selection import train_test_split\n",
        "\n",
        "BUFFER_SIZE = 50000"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Using TensorFlow backend.\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1dK3nGZavrLT",
        "colab_type": "text"
      },
      "source": [
        "# Helper Functions"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "692L4QEivtok",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def text_analysis_dialogues(path, save_tokenizer=False, lowercase=False,\n",
        "                            filters='!\"#$%&()*+,-./:;<=>?@[\\\\]^_`{|}~\\t\\n', use_top=50,\n",
        "                            font_size=22, plot_width=50, plot_height=8, plot_dpi=100,\n",
        "                            label_rotation=45, sentiment_path=\".sentiments\"):\n",
        "    \"\"\"Text analysis statistics: number of tokens, length of sequences, different types of word frequencies.\n",
        "    If you decide to save tokenizer and want to load it:\n",
        "    with open(tokenizer_json) as f:\n",
        "        data = json.load(f)\n",
        "        tokenizer = keras.preprocessing.text.tokenizer_from_json(data)\n",
        "    Args:\n",
        "        path: Path of '.tsv' file or files directory.\n",
        "        save_tokenizer: Save tokenizer or not.\n",
        "        lowercase:\n",
        "        filters:\n",
        "        use_top:\n",
        "        font_size:\n",
        "        plot_width:\n",
        "        plot_height:\n",
        "        plot_dpi:\n",
        "        label_rotation:\n",
        "        sentiment_path:\n",
        "\n",
        "    Returns:\n",
        "        None\n",
        "    \"\"\"\n",
        "    # set plot style\n",
        "    plt.style.use(\"seaborn-dark\")\n",
        "    # variables\n",
        "    n_dialogues = 0\n",
        "    n_tokens_dialogues = []  # n_tokens_context + n_tokens_response\n",
        "    n_tokens_contexts = []\n",
        "    n_tokens_responses = []\n",
        "    tokenizer = keras.preprocessing.text.Tokenizer(num_words=None, filters=filters,\n",
        "                                                   lower=lowercase, split=' ')\n",
        "    # deal with stop words\n",
        "    nltk.download('wordnet')\n",
        "    nltk.download('stopwords')\n",
        "    stop_words = set(nltk.corpus.stopwords.words('english'))\n",
        "    # deal with sentiment words\n",
        "    sentiment_path = \".sentiments\"\n",
        "    # files download urls\n",
        "    positive_url = \"https://raw.githubusercontent.com/gmihaila/machine_learning_things/master/sentiments/positive-words.txt\"\n",
        "    negative_url = \"https://raw.githubusercontent.com/gmihaila/machine_learning_things/master/sentiments/negative-words.txt\"\n",
        "    # build path\n",
        "    os.makedirs(sentiment_path) if not os.path.isdir(sentiment_path) else None\n",
        "    # create file paths\n",
        "    positive_path = os.path.join(sentiment_path, \"positive-words.txt\")\n",
        "    negative_path = os.path.join(sentiment_path, \"negative-words.txt\")\n",
        "    # download files\n",
        "    open(positive_path, 'wb').write(requests.get(positive_url).content) if not os.path.isfile(positive_path) else None\n",
        "    open(negative_path, 'wb').write(requests.get(negative_url).content) if not os.path.isfile(negative_path) else None\n",
        "    # read file\n",
        "    positive_words = io.open(positive_path, encoding='UTF-8').read().strip().split('\\n')\n",
        "    negative_words = io.open(negative_path, encoding='UTF-8').read().strip().split('\\n')\n",
        "\n",
        "    # check if path is folder\n",
        "    if os.path.isdir(path):\n",
        "        # get all file names in path\n",
        "        files = os.listdir(path)\n",
        "        image_path = os.path.join(path, \"plot_%s.png\" % os.path.basename(path))\n",
        "        tokenizer_json_path = os.path.join(path, \"tokenizer_%s.json\" % os.path.basename(path))\n",
        "\n",
        "    # check if path is file\n",
        "    elif os.path.isfile(path):\n",
        "        file_name = path.split('/')[-1]\n",
        "        files = [file_name]\n",
        "        file_id = os.path.basename(path)\n",
        "        file_id = file_id.split(\".\")[0]\n",
        "        output_path = os.path.dirname(path)\n",
        "        image_path = os.path.join(output_path, \"plot_%s.png\" % file_id)\n",
        "        tokenizer_json_path = os.path.join(output_path, \"tokenizer_%s.json\" % file_id)\n",
        "\n",
        "    else:\n",
        "        raise ValueError('ERROR: Invalid path or file.')\n",
        "\n",
        "    print(\"Parsing each file.\")\n",
        "    for file_name in tqdm(files):\n",
        "        if len(files) == 1:\n",
        "            file_path = path\n",
        "        else:\n",
        "            file_path = os.path.join(path, file_name)\n",
        "\n",
        "        # read tsv file\n",
        "        try:\n",
        "            data_frame = pd.read_csv(filepath_or_buffer=file_path)\n",
        "            # get values\n",
        "            inputs = data_frame['title'].astype(str).values\n",
        "            outputs = data_frame['text'].astype(str).values\n",
        "            # fit tokenizer\n",
        "            tokenizer.fit_on_texts(inputs + outputs)\n",
        "            # number of instances\n",
        "            n_dialogues += len(inputs)\n",
        "            # count tokens\n",
        "            n_tokens_contexts += [len(context.split()) for context in inputs]\n",
        "            n_tokens_responses += [len(response.split()) for response in outputs]\n",
        "\n",
        "        except Exception as e:\n",
        "            print(\"ERROR: File/s need to be '.tsv'.\")\n",
        "            print(\"ERROR: \", e, \"\\nTRACE:\", traceback.format_exc())\n",
        "\n",
        "    n_tokens_dialogues += [n_con + n_resp for n_con, n_resp in zip(n_tokens_contexts, n_tokens_responses)]\n",
        "\n",
        "    tokens_details = {'Number of News': len(n_tokens_dialogues),\n",
        "                      'Number of Tokens': sum(n_tokens_dialogues),\n",
        "                      'Vocabulary Size': len(tokenizer.word_counts),\n",
        "                      'Average Tokens per Title-Text Pair': np.average(n_tokens_dialogues),\n",
        "                      'Average Tokens per Ttile': np.average(n_tokens_contexts),\n",
        "                      'Average Tokens per Text': np.average(n_tokens_responses)\n",
        "                      }\n",
        "    print(\"Finished 'tokens_details'.\")\n",
        "\n",
        "    # save tokenizer\n",
        "    if save_tokenizer is True:\n",
        "        print(\"Saving tokenizer to %s\" % tokenizer_json_path)\n",
        "        tokenizer_json = tokenizer.to_json()\n",
        "        with io.open(tokenizer_json_path, 'w', encoding='utf-8') as f:\n",
        "            f.write(json.dumps(tokenizer_json, ensure_ascii=False))\n",
        "\n",
        "    # pos tagging\n",
        "    print(\"POS Tagging.\")\n",
        "    pos_decode = {'n': 'NOUN', 'v': 'VERB', 'a': 'ADJECTIVE', 's': 'ADJECTIVE SATELITE', 'r': 'ADVERB'}\n",
        "    pos_counts = {name: 0 for name in pos_decode.values()}\n",
        "    for word, count in tokenizer.word_counts.items():\n",
        "        pos_tag = nltk.corpus.wordnet.synsets(word)\n",
        "        if len(pos_tag) > 0:\n",
        "            pos_tag = pos_tag[0].pos()\n",
        "            pos_name = pos_decode[pos_tag]\n",
        "            pos_counts[pos_name] += 1\n",
        "\n",
        "    # all words\n",
        "    print(\"All words to DataFrame.\")\n",
        "    words_counts_df = pd.DataFrame(list(tokenizer.word_counts.items()))\n",
        "    words_counts_df.columns = [\"word\", \"count\"]\n",
        "    words_counts_df = words_counts_df.sort_values('count', ascending=False).reset_index(drop=True)\n",
        "\n",
        "    # specific words\n",
        "    print(\"Filter 'only_stop_words_pd'.\")\n",
        "    only_stop_words_pd = words_counts_df[[True if word in stop_words else False for word in\n",
        "                                          words_counts_df[\"word\"]]][:use_top]\n",
        "    print(\"Filter 'without_stop_words_pd'.\")\n",
        "    without_stop_words_pd = words_counts_df[[False if word in stop_words else True for word in\n",
        "                                             words_counts_df[\"word\"]]][:use_top]\n",
        "\n",
        "    print(\"Filter 'using_positive_words_pd'.\")\n",
        "    using_positive_words_pd = words_counts_df[[True if word in positive_words else False for word in\n",
        "                                               words_counts_df[\"word\"]]][:use_top]\n",
        "    print(\"Filter 'using_negative_words_pd'.\")\n",
        "    using_negative_words_pd = words_counts_df[[True if word in negative_words else False for word in\n",
        "                                               words_counts_df[\"word\"]]][:use_top]\n",
        "\n",
        "    words_counts_df = words_counts_df[:use_top]\n",
        "\n",
        "    pos_counts_df = pd.DataFrame(list(pos_counts.items()))\n",
        "    pos_counts_df.columns = [\"word\", \"count\"]\n",
        "    pos_counts_df = pos_counts_df.sort_values('count', ascending=False).reset_index(drop=True)\n",
        "\n",
        "    # plots\n",
        "    print(\"Plotting.\")\n",
        "    plt.rcParams.update({'font.size': font_size})\n",
        "\n",
        "    plots_dict = {\"Data Tokens Details\": tokens_details,\n",
        "                  \"All Words\": words_counts_df,\n",
        "                  \"Without Stop Words\": without_stop_words_pd,\n",
        "                  \"Only Stop Words\": only_stop_words_pd,\n",
        "                  \"Only Positive Words\": using_positive_words_pd,\n",
        "                  \"Only Negative Words\": using_negative_words_pd,\n",
        "                  \"POS Tags Counts\": pos_counts_df}\n",
        "\n",
        "    fig = plt.figure(figsize=(plot_width, plot_height * len(plots_dict) - 1))\n",
        "    fig.subplots_adjust(hspace=1)\n",
        "    fig.suptitle(t='Text Analysis Top %s Tokens' % use_top,\n",
        "                 fontweight='bold',\n",
        "                 fontsize=70)\n",
        "\n",
        "    # loop for each plot\n",
        "    for plot_index, (plot_name, plot_df) in enumerate(plots_dict.items()):\n",
        "        plt.subplot(len(plots_dict), 1, (plot_index + 1))\n",
        "\n",
        "        if plot_name == \"Data Tokens Details\":\n",
        "            # plot token details\n",
        "            text = \"\\n\".join([\"%s : %s\" % (name, f\"{int(value):,d}\") for name, value in plot_df.items()])\n",
        "            # these are matplotlib.patch.Patch properties\n",
        "            props = dict(boxstyle='round', facecolor='w', alpha=0.5)\n",
        "            plt.axis('off')\n",
        "            # place a text box in upper left in axes coords\n",
        "            plt.text(0, 0, text, fontsize=2.5 * font_size,\n",
        "                     horizontalalignment='left',\n",
        "                     verticalalignment='bottom', bbox=props)\n",
        "        else:\n",
        "            # plot box plots\n",
        "            x_values = plot_df['word'].values\n",
        "            y_values = plot_df['count'].values\n",
        "            plt.title(plot_name)\n",
        "            plt.bar(x=x_values, height=y_values)\n",
        "            plt.ylabel('Count')\n",
        "            plt.xticks(rotation=label_rotation)\n",
        "            [plt.text(index - 0.1, value, f\"{int(value):,d}\", rotation=label_rotation) for index, value in\n",
        "             enumerate(y_values)];\n",
        "            plt.grid()\n",
        "    # save figure to image\n",
        "    print(\"Saving figure to %s\" % image_path)\n",
        "    fig.savefig(fname=image_path, dpi=plot_dpi, bbox_inches='tight', pad_inches=0.5)\n",
        "    return\n",
        "\n",
        "\n",
        "\n",
        "def pad_numpy(variable_length_array, fixed_length):\n",
        "    \"\"\"Pad variable length array to a fixed numpy array.\n",
        "    It can handle single arrays [1,2,3] or nested arrays [[1,2],[3]].\n",
        "    Args:\n",
        "        variable_length_array: Single arrays [1,2,3] or nested arrays [[1,2],[3]].\n",
        "        fixed_length: max length of rows for numpy.\n",
        "\n",
        "    Returns:\n",
        "        numpy_array: fixed numpy array shape [len of array X fixed_length].\n",
        "    \"\"\"\n",
        "    # array of arrays\n",
        "    if isinstance(variable_length_array[0], list):\n",
        "        numpy_array = np.zeros((len(variable_length_array), fixed_length))\n",
        "        # verify each row\n",
        "        for numpy_row, array_row in zip(numpy_array, variable_length_array):\n",
        "            # concatenate array row if it is longer\n",
        "            array_row = array_row[:fixed_length]\n",
        "            numpy_row[:len(array_row)] = array_row\n",
        "        return numpy_array\n",
        "    # array of values\n",
        "    elif isinstance(variable_length_array, list):\n",
        "        numpy_array = np.zeros(fixed_length)\n",
        "        variable_length_array = variable_length_array[:fixed_length]\n",
        "        numpy_array[:len(variable_length_array)] = variable_length_array\n",
        "        return numpy_array\n",
        "    else:\n",
        "        print(\"Array is not a valid format.\")\n",
        "        return None"
      ],
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Fi3JOF2yxeOR",
        "colab_type": "text"
      },
      "source": [
        "# Text Analysis"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "O7aKEDJ_v7Aa",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000,
          "referenced_widgets": [
            "f854dbd4fe644a0c8ef4086e6ae586b8",
            "4d3f19ed08434aa6aa54698880d8d997",
            "b0c1c425171643ae9deb8617b9c135af",
            "9f9a02f4c9b34042a48ef0e29370f062",
            "6ba7d4957de14b9d9140bb9045c3f865",
            "0bc0a2090cae469c811b5d6ef12010d7",
            "4f050b7cca9644a2a3add840ff6fd3d9",
            "5ab158d989e14302891212c785393439",
            "747ea6942849424a838ade456764972a",
            "275cd014ca754ebfa3dc85a24d74d57e",
            "88ce031e29f54ac2a8c8c3f9476a9a37",
            "91c3dd4e7d584aaea52aae4383a1cff2",
            "feff3fb7066540d584f98af26cabeb84",
            "5e74244fc3c842e1a730a10017b3179f",
            "c764287c354046398677727dd127db79",
            "11d81797b8304165935c9fa2f2c87709"
          ]
        },
        "outputId": "6fc4fbf4-a3e4-46eb-8bb6-0f686ab69aa5"
      },
      "source": [
        "text_analysis_dialogues(path='/content/True.csv')\n",
        "text_analysis_dialogues(path='/content/Fake.csv')"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[nltk_data] Downloading package wordnet to /root/nltk_data...\n",
            "[nltk_data]   Package wordnet is already up-to-date!\n",
            "[nltk_data] Downloading package stopwords to /root/nltk_data...\n",
            "[nltk_data]   Package stopwords is already up-to-date!\n",
            "Parsing each file.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "f854dbd4fe644a0c8ef4086e6ae586b8",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=1.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Finished 'tokens_details'.\n",
            "POS Tagging.\n",
            "All words to DataFrame.\n",
            "Filter 'only_stop_words_pd'.\n",
            "Filter 'without_stop_words_pd'.\n",
            "Filter 'using_positive_words_pd'.\n",
            "Filter 'using_negative_words_pd'.\n",
            "Plotting.\n",
            "Saving figure to /content/plot_True.png\n",
            "[nltk_data] Downloading package wordnet to /root/nltk_data...\n",
            "[nltk_data]   Package wordnet is already up-to-date!\n",
            "[nltk_data] Downloading package stopwords to /root/nltk_data...\n",
            "[nltk_data]   Package stopwords is already up-to-date!\n",
            "Parsing each file.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "747ea6942849424a838ade456764972a",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=1.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Finished 'tokens_details'.\n",
            "POS Tagging.\n",
            "All words to DataFrame.\n",
            "Filter 'only_stop_words_pd'.\n",
            "Filter 'without_stop_words_pd'.\n",
            "Filter 'using_positive_words_pd'.\n",
            "Filter 'using_negative_words_pd'.\n",
            "Plotting.\n",
            "Saving figure to /content/plot_Fake.png\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "[encoded data, 1716672 characters]\n",
            "text/plain": [
              "<Figure size 3600x3960 with 7 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "[encoded data, 1660284 characters]\n",
            "text/plain": [
              "<Figure size 3600x3960 with 7 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ouKh5AfA4fEr",
        "colab_type": "text"
      },
      "source": [
        "# Concatenated Data"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bD38caUE4duL",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 279
        },
        "outputId": "d6e2a986-14d4-43f6-b7cc-7e9801783a92"
      },
      "source": [
        "fake_data_df = pd.read_csv('/content/Fake.csv')\n",
        "true_data_df = pd.read_csv('/content/True.csv')\n",
        "\n",
        "# add labels '0'-fake and '1' - true\n",
        "fake_data_df['label'] = [0]*len(fake_data_df)\n",
        "true_data_df['label'] = [1] * len(true_data_df)\n",
        "\n",
        "\n",
        "# append both data frames\n",
        "all_news = fake_data_df.append(true_data_df, ignore_index = True)\n",
        "\n",
        "# concatenate title and text\n",
        "all_news['title_text'] = [\"%s %s\"%(title,text) for title, text in \\\n",
        "                          zip(all_news['title'].values, all_news['text'].values)]\n",
        "\n",
        "# re-arrange columsnto make them pretty\n",
        "all_news = all_news[['title_text', 'title', 'text', 'subject', 'date', 'label']]\n",
        "\n",
        "all_news.head()"
      ],
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>title_text</th>\n",
              "      <th>title</th>\n",
              "      <th>text</th>\n",
              "      <th>subject</th>\n",
              "      <th>date</th>\n",
              "      <th>label</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Donald Trump Sends Out Embarrassing New Year’...</td>\n",
              "      <td>Donald Trump Sends Out Embarrassing New Year’...</td>\n",
              "      <td>Donald Trump just couldn t wish all Americans ...</td>\n",
              "      <td>News</td>\n",
              "      <td>December 31, 2017</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Drunk Bragging Trump Staffer Started Russian ...</td>\n",
              "      <td>Drunk Bragging Trump Staffer Started Russian ...</td>\n",
              "      <td>House Intelligence Committee Chairman Devin Nu...</td>\n",
              "      <td>News</td>\n",
              "      <td>December 31, 2017</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>Sheriff David Clarke Becomes An Internet Joke...</td>\n",
              "      <td>Sheriff David Clarke Becomes An Internet Joke...</td>\n",
              "      <td>On Friday, it was revealed that former Milwauk...</td>\n",
              "      <td>News</td>\n",
              "      <td>December 30, 2017</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>Trump Is So Obsessed He Even Has Obama’s Name...</td>\n",
              "      <td>Trump Is So Obsessed He Even Has Obama’s Name...</td>\n",
              "      <td>On Christmas day, Donald Trump announced that ...</td>\n",
              "      <td>News</td>\n",
              "      <td>December 29, 2017</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>Pope Francis Just Called Out Donald Trump Dur...</td>\n",
              "      <td>Pope Francis Just Called Out Donald Trump Dur...</td>\n",
              "      <td>Pope Francis used his annual Christmas Day mes...</td>\n",
              "      <td>News</td>\n",
              "      <td>December 25, 2017</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                          title_text  ... label\n",
              "0   Donald Trump Sends Out Embarrassing New Year’...  ...     0\n",
              "1   Drunk Bragging Trump Staffer Started Russian ...  ...     0\n",
              "2   Sheriff David Clarke Becomes An Internet Joke...  ...     0\n",
              "3   Trump Is So Obsessed He Even Has Obama’s Name...  ...     0\n",
              "4   Pope Francis Just Called Out Donald Trump Dur...  ...     0\n",
              "\n",
              "[5 rows x 6 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 16
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "N_6AY1eMKXbW",
        "colab_type": "text"
      },
      "source": [
        "# Tokenize Data"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SjqJwFJa9cTQ",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 133,
          "referenced_widgets": [
            "f3a48be3db4345e690d3f8d5c16f355d",
            "984f8878630148d6964380a714a2f26a",
            "8aa3af23b99446e89423737c4c74bb3d",
            "39bb87e159404852859c82d064db1dc9",
            "4f5edb4201154da4bde2810dd74c3cee",
            "1d65a93f06514d00a028884eb64ad7cd",
            "d3468fc571fc4546959ba07cdefcadf5",
            "049eab7924034dde9e68908ede74af24"
          ]
        },
        "outputId": "43f922fa-393f-4257-d1fd-665b582b9602"
      },
      "source": [
        "# create tokenizer\n",
        "tokenizer = tf.keras.preprocessing.text.Tokenizer()\n",
        "\n",
        "# make sure all text is in correct format\n",
        "print('Fixing text to make sure formatting is right...')\n",
        "all_text = [fix_text(text) for text in tqdm(all_news['title_text'].values)]\n",
        "all_labels = all_news['label'].values\n",
        "\n",
        "# fit tokenizer to allt text\n",
        "tokenizer.fit_on_texts(all_text)\n",
        "print('Fit tokenizer to all text and titles!')\n",
        "\n",
        "# get vocabulary size\n",
        "vocab_size = len(tokenizer.word_index)\n",
        "\n",
        "# get average length of text\n",
        "indexed_all_text = tokenizer.texts_to_sequences(all_text)\n",
        "avg_len = int(np.average([len(indexed_text) for indexed_text in indexed_all_text]))\n",
        "print('Average text length is: %d words.' % avg_len)\n",
        "\n",
        "# fix length all text\n",
        "numpy_all_text = pad_numpy(indexed_all_text, avg_len)\n",
        "print('Fixed length numpy matrix: ', numpy_all_text.shape)"
      ],
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Fixing text to make sure formatting is right...\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "f3a48be3db4345e690d3f8d5c16f355d",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=44898.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Fit tokenizer to all text and titles!\n",
            "Average text length is: 427 words.\n",
            "Fixed length numpy matrix:  (44898, 427)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Vyx79ymaKWyw",
        "colab_type": "text"
      },
      "source": [
        "# Split Train-Test"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GkpFG9hxRxGC",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "text_train, text_test, labels_train, labels_test = train_test_split(numpy_all_text, all_labels, test_size=0.1, random_state=123)"
      ],
      "execution_count": 18,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "plzfIqzsRvIH",
        "colab_type": "text"
      },
      "source": [
        "# Train -Test Dataset"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "S25xe2JqIbX6",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "BATCH_SIZE = 32\n",
        "\n",
        "# train dataset\n",
        "train_dataset = tf.data.Dataset.from_tensor_slices((text_train, labels_train))\n",
        "train_dataset = train_dataset.cache()\n",
        "train_dataset = train_dataset.shuffle(BUFFER_SIZE, seed=123, reshuffle_each_iteration=True)\n",
        "train_dataset = train_dataset.batch(BATCH_SIZE)\n",
        "\n",
        "# test dataset\n",
        "test_dataset = tf.data.Dataset.from_tensor_slices((text_test, labels_test))\n",
        "test_dataset = test_dataset.cache()\n",
        "test_dataset = test_dataset.batch(BATCH_SIZE)\n"
      ],
      "execution_count": 19,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qzjy0p1xUJST",
        "colab_type": "text"
      },
      "source": [
        "# Build Model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uZXFH8CyQnNE",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 252
        },
        "outputId": "4715cc67-6054-44c6-e4ec-c858d1e49505"
      },
      "source": [
        "model = tf.keras.Sequential([\n",
        "            tf.keras.layers.Embedding(vocab_size, 16),\n",
        "            tf.keras.layers.GlobalAveragePooling1D(),\n",
        "            tf.keras.layers.Dense(1)])\n",
        "\n",
        "model.compile(optimizer='adam',\n",
        "              loss=tf.losses.BinaryCrossentropy(from_logits=True),\n",
        "              metrics=['accuracy'])\n",
        "\n",
        "model.summary()"
      ],
      "execution_count": 21,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Model: \"sequential_1\"\n",
            "_________________________________________________________________\n",
            "Layer (type)                 Output Shape              Param #   \n",
            "=================================================================\n",
            "embedding_1 (Embedding)      (None, None, 16)          2153712   \n",
            "_________________________________________________________________\n",
            "global_average_pooling1d_1 ( (None, 16)                0         \n",
            "_________________________________________________________________\n",
            "dense_1 (Dense)              (None, 1)                 17        \n",
            "=================================================================\n",
            "Total params: 2,153,729\n",
            "Trainable params: 2,153,729\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LOUjfiNsKesh",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 576
        },
        "outputId": "a37562af-3367-4393-dab8-bb111358de90"
      },
      "source": [
        "history = model.fit(train_dataset,\n",
        "                    epochs=2,\n",
        "                    validation_data=test_dataset)"
      ],
      "execution_count": 22,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch 1/2\n",
            " 529/1263 [===========>..................] - ETA: 22s - loss: 0.5711 - accuracy: 0.5996"
          ],
          "name": "stdout"
        },
        {
          "output_type": "error",
          "ename": "InvalidArgumentError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mInvalidArgumentError\u001b[0m                      Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-22-97a68cd00980>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m history = model.fit(train_dataset,\n\u001b[1;32m      2\u001b[0m                     \u001b[0mepochs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m                     validation_data=test_dataset)\n\u001b[0m",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/training.py\u001b[0m in \u001b[0;36m_method_wrapper\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m     64\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m_method_wrapper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     65\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_in_multi_worker_mode\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m  \u001b[0;31m# pylint: disable=protected-access\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 66\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mmethod\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     67\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     68\u001b[0m     \u001b[0;31m# Running inside `run_distribute_coordinator` already.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/training.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, validation_batch_size, validation_freq, max_queue_size, workers, use_multiprocessing)\u001b[0m\n\u001b[1;32m    846\u001b[0m                 batch_size=batch_size):\n\u001b[1;32m    847\u001b[0m               \u001b[0mcallbacks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mon_train_batch_begin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 848\u001b[0;31m               \u001b[0mtmp_logs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtrain_function\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0miterator\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    849\u001b[0m               \u001b[0;31m# Catch OutOfRangeError for Datasets of unknown size.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    850\u001b[0m               \u001b[0;31m# This blocks until the batch has finished executing.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/eager/def_function.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwds)\u001b[0m\n\u001b[1;32m    578\u001b[0m         \u001b[0mxla_context\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mExit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    579\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 580\u001b[0;31m       \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    581\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    582\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mtracing_count\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_get_tracing_count\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/eager/def_function.py\u001b[0m in \u001b[0;36m_call\u001b[0;34m(self, *args, **kwds)\u001b[0m\n\u001b[1;32m    609\u001b[0m       \u001b[0;31m# In this case we have created variables on the first call, so we run the\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    610\u001b[0m       \u001b[0;31m# defunned version which is guaranteed to never create variables.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 611\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_stateless_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# pylint: disable=not-callable\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    612\u001b[0m     \u001b[0;32melif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_stateful_fn\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    613\u001b[0m       \u001b[0;31m# Release the lock early so that multiple threads can perform the call\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/eager/function.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   2418\u001b[0m     \u001b[0;32mwith\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_lock\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2419\u001b[0m       \u001b[0mgraph_function\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwargs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_maybe_define_function\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2420\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0mgraph_function\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_filtered_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# pylint: disable=protected-access\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2421\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2422\u001b[0m   \u001b[0;34m@\u001b[0m\u001b[0mproperty\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/eager/function.py\u001b[0m in \u001b[0;36m_filtered_call\u001b[0;34m(self, args, kwargs)\u001b[0m\n\u001b[1;32m   1663\u001b[0m          if isinstance(t, (ops.Tensor,\n\u001b[1;32m   1664\u001b[0m                            resource_variable_ops.BaseResourceVariable))),\n\u001b[0;32m-> 1665\u001b[0;31m         self.captured_inputs)\n\u001b[0m\u001b[1;32m   1666\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1667\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m_call_flat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcaptured_inputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcancellation_manager\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/eager/function.py\u001b[0m in \u001b[0;36m_call_flat\u001b[0;34m(self, args, captured_inputs, cancellation_manager)\u001b[0m\n\u001b[1;32m   1744\u001b[0m       \u001b[0;31m# No tape is watching; skip to running the function.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1745\u001b[0m       return self._build_call_outputs(self._inference_function.call(\n\u001b[0;32m-> 1746\u001b[0;31m           ctx, args, cancellation_manager=cancellation_manager))\n\u001b[0m\u001b[1;32m   1747\u001b[0m     forward_backward = self._select_forward_and_backward_functions(\n\u001b[1;32m   1748\u001b[0m         \u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/eager/function.py\u001b[0m in \u001b[0;36mcall\u001b[0;34m(self, ctx, args, cancellation_manager)\u001b[0m\n\u001b[1;32m    596\u001b[0m               \u001b[0minputs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    597\u001b[0m               \u001b[0mattrs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mattrs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 598\u001b[0;31m               ctx=ctx)\n\u001b[0m\u001b[1;32m    599\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    600\u001b[0m           outputs = execute.execute_with_cancellation(\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/eager/execute.py\u001b[0m in \u001b[0;36mquick_execute\u001b[0;34m(op_name, num_outputs, inputs, attrs, ctx, name)\u001b[0m\n\u001b[1;32m     58\u001b[0m     \u001b[0mctx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mensure_initialized\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     59\u001b[0m     tensors = pywrap_tfe.TFE_Py_Execute(ctx._handle, device_name, op_name,\n\u001b[0;32m---> 60\u001b[0;31m                                         inputs, attrs, num_outputs)\n\u001b[0m\u001b[1;32m     61\u001b[0m   \u001b[0;32mexcept\u001b[0m \u001b[0mcore\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_NotOkStatusException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     62\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mname\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mInvalidArgumentError\u001b[0m:  indices[22,134] = 134607 is not in [0, 134607)\n\t [[node sequential_1/embedding_1/embedding_lookup (defined at <ipython-input-22-97a68cd00980>:3) ]] [Op:__inference_train_function_1532]\n\nErrors may have originated from an input operation.\nInput Source operations connected to node sequential_1/embedding_1/embedding_lookup:\n sequential_1/embedding_1/embedding_lookup/1277 (defined at /usr/lib/python3.6/contextlib.py:81)\n\nFunction call stack:\ntrain_function\n"
          ]
        }
      ]
    }
  ]
}
