{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "keras_tokenizer_fix.ipynb",
      "version": "0.3.2",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/gmihaila/machine_learning_toolbox/blob/master/keras_tokenizer_fix.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "metadata": {
        "id": "NRdntd0DKAqB",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "## Probably my most usefull tool: How to fix the keras tokenizer to speed things up\n",
        "\n",
        "by GeorgeM"
      ]
    },
    {
      "metadata": {
        "id": "GSmIeoaP7mEE",
        "colab_type": "code",
        "outputId": "1849ed29-8a68-4166-dcd6-89bfac5ea498",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 215
        }
      },
      "cell_type": "code",
      "source": [
        "from keras.preprocessing.text import Tokenizer\n",
        "import string\n",
        "\n",
        "\n",
        "n_words = 3\n",
        "\n",
        "\n",
        "tk = Tokenizer(num_words=(n_words+1), filters=string.punctuation, lower=True, split=' ', char_level=False, oov_token='<unk>')\n",
        "texts = [\"my name is far faraway asdasd\", \"my name is\",\"your name is\"]\n",
        "\n",
        "tk.fit_on_texts([\"my name is far faraway \", \"my name is\",\"your name is\"])\n",
        "tk.fit_on_texts([\"asdasd my name is far faraway asdasd\"])\n",
        "\n",
        "## NOTE: You can fit chunks of data, it will work same way. When having big data don't need to load all files\n",
        "\n",
        "print('Original text: %s\\n'%texts)\n",
        "print('Only use top %s words\\n\\n'%n_words)\n",
        "\n",
        "print(tk.word_index)\n",
        "print('%s  <-WRONG!'%tk.texts_to_sequences(texts))\n",
        "\n",
        "print('\\n')\n",
        "\n",
        "## **Key Step**\n",
        "tk.word_index = {e:i for e,i in tk.word_index.items() if i <= n_words} # <= because tokenizer is 1 indexed\n",
        "tk.word_index[tk.oov_token] = n_words + 1\n",
        "\n",
        "\n",
        "print(tk.word_index)\n",
        "print('%s  <-RIGHT!'%tk.texts_to_sequences(texts))\n"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Original text: ['my name is far faraway asdasd', 'my name is', 'your name is']\n",
            "\n",
            "Only use top 3 words\n",
            "\n",
            "\n",
            "{'<unk>': 1, 'name': 2, 'is': 3, 'my': 4, 'far': 5, 'faraway': 6, 'asdasd': 7, 'your': 8}\n",
            "[[1, 2, 3, 1, 1, 1], [1, 2, 3], [1, 2, 3]]  <-WRONG!\n",
            "\n",
            "\n",
            "{'<unk>': 4, 'name': 2, 'is': 3}\n",
            "[[4, 2, 3, 4, 4, 4], [4, 2, 3], [4, 2, 3]]  <-RIGHT!\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "bchfX1IMME5d",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "Thank you se4u(https://github.com/se4u)\n",
        "Also: https://github.com/keras-team/keras/issues/8092"
      ]
    }
  ]
}