{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "pytorch_transformer_finetune.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "toc_visible": true,
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/gmihaila/machine_learning_things/blob/master/learning_pytorch/pytorch_transformer_finetune.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "v-hqoBOYRA8Y",
        "colab_type": "text"
      },
      "source": [
        "### Links\n",
        "\n",
        "https://modelzoo.co/model/pytorch-pretrained-bert"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "aZ0mvSclJNb5",
        "colab_type": "text"
      },
      "source": [
        "### Installs"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eiaZwVF4EDbG",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from IPython.display import clear_output\n",
        "\n",
        "!pip install pytorch-transformers\n",
        "clear_output()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "o2V3_9hwJQxs",
        "colab_type": "text"
      },
      "source": [
        "### Imports"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mS-PVZ27fpYl",
        "colab_type": "code",
        "outputId": "8f8abfb2-fec5-4d4e-bb35-774e28f3a14d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 255
        }
      },
      "source": [
        "import torch\n",
        "import numpy as np\n",
        "from torchsummary import summary\n",
        "from pytorch_transformers import *\n",
        "from keras.preprocessing.sequence import pad_sequences\n",
        "\n",
        "# PyTorch-Transformers has a unified API\n",
        "# for 8 transformer architectures and 30 pretrained weights.\n",
        "\n",
        "#          Model          | Tokenizer          | Pretrained weights shortcut\n",
        "MODELS = [(BertModel,       BertTokenizer,       BERT_PRETRAINED_MODEL_ARCHIVE_MAP.keys()),\n",
        "          (OpenAIGPTModel,  OpenAIGPTTokenizer,  OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_MAP.keys()),\n",
        "          (GPT2Model,       GPT2Tokenizer,       GPT2_PRETRAINED_MODEL_ARCHIVE_MAP.keys()),\n",
        "          (TransfoXLModel,  TransfoXLTokenizer,  TRANSFO_XL_PRETRAINED_MODEL_ARCHIVE_MAP.keys()),\n",
        "          (XLNetModel,      XLNetTokenizer,      XLNET_PRETRAINED_MODEL_ARCHIVE_MAP.keys()),\n",
        "          (XLMModel,        XLMTokenizer,        XLM_PRETRAINED_MODEL_ARCHIVE_MAP.keys()),\n",
        "          (RobertaModel,    RobertaTokenizer,    ROBERTA_PRETRAINED_MODEL_ARCHIVE_MAP.keys()),\n",
        "          (DistilBertModel, DistilBertTokenizer, DISTILBERT_PRETRAINED_MODEL_ARCHIVE_MAP.keys())]\n",
        "\n",
        "\n",
        "\n",
        "list(BERT_PRETRAINED_MODEL_ARCHIVE_MAP.keys())"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Using TensorFlow backend.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['bert-base-uncased',\n",
              " 'bert-large-uncased',\n",
              " 'bert-base-cased',\n",
              " 'bert-large-cased',\n",
              " 'bert-base-multilingual-uncased',\n",
              " 'bert-base-multilingual-cased',\n",
              " 'bert-base-chinese',\n",
              " 'bert-base-german-cased',\n",
              " 'bert-large-uncased-whole-word-masking',\n",
              " 'bert-large-cased-whole-word-masking',\n",
              " 'bert-large-uncased-whole-word-masking-finetuned-squad',\n",
              " 'bert-large-cased-whole-word-masking-finetuned-squad',\n",
              " 'bert-base-cased-finetuned-mrpc']"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 2
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "SjwxRshSJUPM",
        "colab_type": "text"
      },
      "source": [
        "### Load Model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ni9ckbNqfrEs",
        "colab_type": "code",
        "outputId": "2299caa7-d103-4403-a54c-bb9f46d1f6b8",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 85
        }
      },
      "source": [
        "pretrained_weights = 'bert-base-uncased'\n",
        "use_model = BertForSequenceClassification\n",
        "use_tokenizer = BertTokenizer\n",
        "\n",
        "## tokenizer\n",
        "tokenizer = use_tokenizer.from_pretrained(pretrained_weights,)\n",
        "## model\n",
        "model = use_model.from_pretrained(pretrained_weights,\n",
        "                                  output_hidden_states=True, \n",
        "                                  output_attentions=True, \n",
        "                                  torchscript=True,\n",
        "                                  num_labels=2)\n",
        "\n",
        "print(\"Loaded \", pretrained_weights)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 231508/231508 [00:00<00:00, 1177454.49B/s]\n",
            "100%|██████████| 313/313 [00:00<00:00, 57627.72B/s]\n",
            "100%|██████████| 440473133/440473133 [00:11<00:00, 38319041.81B/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Loaded  bert-base-uncased\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "k0aYqGSYLiNA",
        "colab_type": "code",
        "outputId": "26ff1251-5dac-4a12-ae0c-864e8eaafeae",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "## to check input shape and output shape\n",
        "print(model)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "BertForSequenceClassification(\n",
            "  (bert): BertModel(\n",
            "    (embeddings): BertEmbeddings(\n",
            "      (word_embeddings): Embedding(30522, 768, padding_idx=0)\n",
            "      (position_embeddings): Embedding(512, 768)\n",
            "      (token_type_embeddings): Embedding(2, 768)\n",
            "      (LayerNorm): LayerNorm(torch.Size([768]), eps=1e-12, elementwise_affine=True)\n",
            "      (dropout): Dropout(p=0.1)\n",
            "    )\n",
            "    (encoder): BertEncoder(\n",
            "      (layer): ModuleList(\n",
            "        (0): BertLayer(\n",
            "          (attention): BertAttention(\n",
            "            (self): BertSelfAttention(\n",
            "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
            "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
            "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
            "              (dropout): Dropout(p=0.1)\n",
            "            )\n",
            "            (output): BertSelfOutput(\n",
            "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
            "              (LayerNorm): LayerNorm(torch.Size([768]), eps=1e-12, elementwise_affine=True)\n",
            "              (dropout): Dropout(p=0.1)\n",
            "            )\n",
            "          )\n",
            "          (intermediate): BertIntermediate(\n",
            "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
            "          )\n",
            "          (output): BertOutput(\n",
            "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
            "            (LayerNorm): LayerNorm(torch.Size([768]), eps=1e-12, elementwise_affine=True)\n",
            "            (dropout): Dropout(p=0.1)\n",
            "          )\n",
            "        )\n",
            "        (1): BertLayer(\n",
            "          (attention): BertAttention(\n",
            "            (self): BertSelfAttention(\n",
            "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
            "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
            "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
            "              (dropout): Dropout(p=0.1)\n",
            "            )\n",
            "            (output): BertSelfOutput(\n",
            "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
            "              (LayerNorm): LayerNorm(torch.Size([768]), eps=1e-12, elementwise_affine=True)\n",
            "              (dropout): Dropout(p=0.1)\n",
            "            )\n",
            "          )\n",
            "          (intermediate): BertIntermediate(\n",
            "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
            "          )\n",
            "          (output): BertOutput(\n",
            "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
            "            (LayerNorm): LayerNorm(torch.Size([768]), eps=1e-12, elementwise_affine=True)\n",
            "            (dropout): Dropout(p=0.1)\n",
            "          )\n",
            "        )\n",
            "        (2): BertLayer(\n",
            "          (attention): BertAttention(\n",
            "            (self): BertSelfAttention(\n",
            "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
            "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
            "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
            "              (dropout): Dropout(p=0.1)\n",
            "            )\n",
            "            (output): BertSelfOutput(\n",
            "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
            "              (LayerNorm): LayerNorm(torch.Size([768]), eps=1e-12, elementwise_affine=True)\n",
            "              (dropout): Dropout(p=0.1)\n",
            "            )\n",
            "          )\n",
            "          (intermediate): BertIntermediate(\n",
            "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
            "          )\n",
            "          (output): BertOutput(\n",
            "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
            "            (LayerNorm): LayerNorm(torch.Size([768]), eps=1e-12, elementwise_affine=True)\n",
            "            (dropout): Dropout(p=0.1)\n",
            "          )\n",
            "        )\n",
            "        (3): BertLayer(\n",
            "          (attention): BertAttention(\n",
            "            (self): BertSelfAttention(\n",
            "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
            "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
            "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
            "              (dropout): Dropout(p=0.1)\n",
            "            )\n",
            "            (output): BertSelfOutput(\n",
            "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
            "              (LayerNorm): LayerNorm(torch.Size([768]), eps=1e-12, elementwise_affine=True)\n",
            "              (dropout): Dropout(p=0.1)\n",
            "            )\n",
            "          )\n",
            "          (intermediate): BertIntermediate(\n",
            "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
            "          )\n",
            "          (output): BertOutput(\n",
            "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
            "            (LayerNorm): LayerNorm(torch.Size([768]), eps=1e-12, elementwise_affine=True)\n",
            "            (dropout): Dropout(p=0.1)\n",
            "          )\n",
            "        )\n",
            "        (4): BertLayer(\n",
            "          (attention): BertAttention(\n",
            "            (self): BertSelfAttention(\n",
            "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
            "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
            "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
            "              (dropout): Dropout(p=0.1)\n",
            "            )\n",
            "            (output): BertSelfOutput(\n",
            "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
            "              (LayerNorm): LayerNorm(torch.Size([768]), eps=1e-12, elementwise_affine=True)\n",
            "              (dropout): Dropout(p=0.1)\n",
            "            )\n",
            "          )\n",
            "          (intermediate): BertIntermediate(\n",
            "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
            "          )\n",
            "          (output): BertOutput(\n",
            "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
            "            (LayerNorm): LayerNorm(torch.Size([768]), eps=1e-12, elementwise_affine=True)\n",
            "            (dropout): Dropout(p=0.1)\n",
            "          )\n",
            "        )\n",
            "        (5): BertLayer(\n",
            "          (attention): BertAttention(\n",
            "            (self): BertSelfAttention(\n",
            "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
            "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
            "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
            "              (dropout): Dropout(p=0.1)\n",
            "            )\n",
            "            (output): BertSelfOutput(\n",
            "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
            "              (LayerNorm): LayerNorm(torch.Size([768]), eps=1e-12, elementwise_affine=True)\n",
            "              (dropout): Dropout(p=0.1)\n",
            "            )\n",
            "          )\n",
            "          (intermediate): BertIntermediate(\n",
            "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
            "          )\n",
            "          (output): BertOutput(\n",
            "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
            "            (LayerNorm): LayerNorm(torch.Size([768]), eps=1e-12, elementwise_affine=True)\n",
            "            (dropout): Dropout(p=0.1)\n",
            "          )\n",
            "        )\n",
            "        (6): BertLayer(\n",
            "          (attention): BertAttention(\n",
            "            (self): BertSelfAttention(\n",
            "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
            "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
            "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
            "              (dropout): Dropout(p=0.1)\n",
            "            )\n",
            "            (output): BertSelfOutput(\n",
            "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
            "              (LayerNorm): LayerNorm(torch.Size([768]), eps=1e-12, elementwise_affine=True)\n",
            "              (dropout): Dropout(p=0.1)\n",
            "            )\n",
            "          )\n",
            "          (intermediate): BertIntermediate(\n",
            "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
            "          )\n",
            "          (output): BertOutput(\n",
            "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
            "            (LayerNorm): LayerNorm(torch.Size([768]), eps=1e-12, elementwise_affine=True)\n",
            "            (dropout): Dropout(p=0.1)\n",
            "          )\n",
            "        )\n",
            "        (7): BertLayer(\n",
            "          (attention): BertAttention(\n",
            "            (self): BertSelfAttention(\n",
            "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
            "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
            "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
            "              (dropout): Dropout(p=0.1)\n",
            "            )\n",
            "            (output): BertSelfOutput(\n",
            "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
            "              (LayerNorm): LayerNorm(torch.Size([768]), eps=1e-12, elementwise_affine=True)\n",
            "              (dropout): Dropout(p=0.1)\n",
            "            )\n",
            "          )\n",
            "          (intermediate): BertIntermediate(\n",
            "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
            "          )\n",
            "          (output): BertOutput(\n",
            "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
            "            (LayerNorm): LayerNorm(torch.Size([768]), eps=1e-12, elementwise_affine=True)\n",
            "            (dropout): Dropout(p=0.1)\n",
            "          )\n",
            "        )\n",
            "        (8): BertLayer(\n",
            "          (attention): BertAttention(\n",
            "            (self): BertSelfAttention(\n",
            "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
            "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
            "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
            "              (dropout): Dropout(p=0.1)\n",
            "            )\n",
            "            (output): BertSelfOutput(\n",
            "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
            "              (LayerNorm): LayerNorm(torch.Size([768]), eps=1e-12, elementwise_affine=True)\n",
            "              (dropout): Dropout(p=0.1)\n",
            "            )\n",
            "          )\n",
            "          (intermediate): BertIntermediate(\n",
            "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
            "          )\n",
            "          (output): BertOutput(\n",
            "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
            "            (LayerNorm): LayerNorm(torch.Size([768]), eps=1e-12, elementwise_affine=True)\n",
            "            (dropout): Dropout(p=0.1)\n",
            "          )\n",
            "        )\n",
            "        (9): BertLayer(\n",
            "          (attention): BertAttention(\n",
            "            (self): BertSelfAttention(\n",
            "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
            "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
            "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
            "              (dropout): Dropout(p=0.1)\n",
            "            )\n",
            "            (output): BertSelfOutput(\n",
            "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
            "              (LayerNorm): LayerNorm(torch.Size([768]), eps=1e-12, elementwise_affine=True)\n",
            "              (dropout): Dropout(p=0.1)\n",
            "            )\n",
            "          )\n",
            "          (intermediate): BertIntermediate(\n",
            "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
            "          )\n",
            "          (output): BertOutput(\n",
            "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
            "            (LayerNorm): LayerNorm(torch.Size([768]), eps=1e-12, elementwise_affine=True)\n",
            "            (dropout): Dropout(p=0.1)\n",
            "          )\n",
            "        )\n",
            "        (10): BertLayer(\n",
            "          (attention): BertAttention(\n",
            "            (self): BertSelfAttention(\n",
            "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
            "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
            "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
            "              (dropout): Dropout(p=0.1)\n",
            "            )\n",
            "            (output): BertSelfOutput(\n",
            "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
            "              (LayerNorm): LayerNorm(torch.Size([768]), eps=1e-12, elementwise_affine=True)\n",
            "              (dropout): Dropout(p=0.1)\n",
            "            )\n",
            "          )\n",
            "          (intermediate): BertIntermediate(\n",
            "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
            "          )\n",
            "          (output): BertOutput(\n",
            "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
            "            (LayerNorm): LayerNorm(torch.Size([768]), eps=1e-12, elementwise_affine=True)\n",
            "            (dropout): Dropout(p=0.1)\n",
            "          )\n",
            "        )\n",
            "        (11): BertLayer(\n",
            "          (attention): BertAttention(\n",
            "            (self): BertSelfAttention(\n",
            "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
            "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
            "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
            "              (dropout): Dropout(p=0.1)\n",
            "            )\n",
            "            (output): BertSelfOutput(\n",
            "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
            "              (LayerNorm): LayerNorm(torch.Size([768]), eps=1e-12, elementwise_affine=True)\n",
            "              (dropout): Dropout(p=0.1)\n",
            "            )\n",
            "          )\n",
            "          (intermediate): BertIntermediate(\n",
            "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
            "          )\n",
            "          (output): BertOutput(\n",
            "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
            "            (LayerNorm): LayerNorm(torch.Size([768]), eps=1e-12, elementwise_affine=True)\n",
            "            (dropout): Dropout(p=0.1)\n",
            "          )\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (pooler): BertPooler(\n",
            "      (dense): Linear(in_features=768, out_features=768, bias=True)\n",
            "      (activation): Tanh()\n",
            "    )\n",
            "  )\n",
            "  (dropout): Dropout(p=0.1)\n",
            "  (classifier): Linear(in_features=768, out_features=2, bias=True)\n",
            ")\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ERrQqvof3Kj_",
        "colab_type": "text"
      },
      "source": [
        "### Example Sentence\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EkHIqU2nM3Dn",
        "colab_type": "code",
        "outputId": "c8cc0d7c-a88c-4cbd-fc53-f23f17233e8f",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "sentence = [\"[CLS] This is my first test. [SEP]\"]\n",
        "\n",
        "tokenized_texts = [tokenizer.tokenize(sent) for sent in sentence]\n",
        "\n",
        "tokenized_texts"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[['[CLS]', 'this', 'is', 'my', 'first', 'test', '.', '[SEP]']]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 5
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TYZ9LPaQ37Ou",
        "colab_type": "code",
        "outputId": "4e377466-490e-4464-a823-d993cdb04022",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 323
        }
      },
      "source": [
        "# Set the maximum sequence length. The longest sequence in our training set is 47, but we'll leave room on the end anyway. \n",
        "# In the original paper, the authors used a length of 512.\n",
        "MAX_LEN = 128\n",
        "\n",
        "# Pad our input tokens\n",
        "input_ids = pad_sequences([tokenizer.encode(sen) for sen in sentence], \n",
        "                          maxlen=MAX_LEN,\n",
        "                          dtype='long',\n",
        "                          truncating='post',\n",
        "                          padding='post')\n",
        "\n",
        "# Create attention masks\n",
        "attention_masks = np.where(input_ids>0, 1, input_ids)\n",
        "\n",
        "# Convert all of our data into torch tensors, the required datatype for our model\n",
        "input_ids = torch.tensor(input_ids)\n",
        "attention_masks = torch.tensor(attention_masks)\n",
        "label = torch.tensor([1])\n",
        "\n",
        "input_ids, attention_masks, label"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(tensor([[ 101, 2023, 2003, 2026, 2034, 3231, 1012,  102,    0,    0,    0,    0,\n",
              "             0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
              "             0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
              "             0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
              "             0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
              "             0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
              "             0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
              "             0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
              "             0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
              "             0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
              "             0,    0,    0,    0,    0,    0,    0,    0]]),\n",
              " tensor([[1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
              "          0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
              "          0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
              "          0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
              "          0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
              "          0, 0, 0, 0, 0, 0, 0, 0]]),\n",
              " tensor([1]))"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 25
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "24u59xGx-lCv",
        "colab_type": "text"
      },
      "source": [
        "### Train model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2rnMUdHv-_NU",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "model.train()\n",
        "\n",
        "# Parameters:\n",
        "lr = 1e-3\n",
        "max_grad_norm = 1.0\n",
        "num_total_steps = 1000\n",
        "num_warmup_steps = 100\n",
        "warmup_proportion = float(num_warmup_steps) / float(num_total_steps)  # 0.1\n",
        "\n",
        "### In PyTorch-Transformers, optimizer and schedules are splitted and instantiated like this:\n",
        "optimizer = AdamW(model.parameters(), lr=lr, correct_bias=False)  # To reproduce BertAdam specific behavior set correct_bias=False\n",
        "scheduler = WarmupLinearSchedule(optimizer, warmup_steps=num_warmup_steps, t_total=num_total_steps)  # PyTorch scheduler\n",
        "\n",
        "\n",
        "## train sequence:\n",
        "optimizer.zero_grad()\n",
        "model.train()\n",
        "\n",
        "## for loop batch\n",
        "\n",
        "# model outputs are always tuple in pytorch-transformers (see doc)\n",
        "output = model(input_ids, token_type_ids=None, attention_mask=attention_masks, labels=label)\n",
        "\n",
        "loss = output[0]\n",
        "loss.backward()\n",
        "\n",
        "torch.nn.utils.clip_grad_norm_(model.parameters(), max_grad_norm)  # Gradient clipping is not in AdamW anymore (so you can use amp without issue)\n",
        "\n",
        "optimizer.step()\n",
        "# Update learning rate schedule\n",
        "scheduler.step()\n",
        "optimizer.zero_grad()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "vRolQ6Is-hv4",
        "colab_type": "text"
      },
      "source": [
        "### Predict"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0G6kuzgo4bRr",
        "colab_type": "code",
        "outputId": "3cba22e3-5662-4eeb-d826-2e30c0c98648",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "# Put model in evaluation mode to evaluate loss on the validation set\n",
        "model.eval()\n",
        "\n",
        "with torch.no_grad():\n",
        "  logits = model(input_ids, token_type_ids=None, attention_mask=attention_masks)\n",
        "\n",
        "# Move logits and labels to CPU\n",
        "\n",
        "\n",
        "pred_flat = np.argmax(logits[0], axis=1).flatten().numpy()\n",
        "\n",
        "print(pred_flat)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[0]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UPUfMnK3yv-2",
        "colab_type": "code",
        "outputId": "f744164f-629a-4c98-ac85-d5fe1eb195c4",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "torch.softmax(logits[0],1)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([[0.6654, 0.3346]])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 17
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PrdC_jVVxkBs",
        "colab_type": "code",
        "outputId": "36046587-4713-4f7e-d9a8-aafb9c9c6a62",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "(logits[0])\n"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([[ 0.4153, -0.2721]])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 19
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "VgKM1xtqn2UJ",
        "colab_type": "text"
      },
      "source": [
        "### Evaluate"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "F8_iLXWJ6Gmo",
        "colab_type": "code",
        "outputId": "1a436853-eddf-43ef-86cc-eba6e465761b",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "# Function to calculate the accuracy of our predictions vs labels\n",
        "def flat_accuracy(preds, labels):\n",
        "  pred_flat = preds.numpy()\n",
        "  pred_flat = np.argmax(pred_flat, axis=1).flatten()\n",
        "\n",
        "  labels_flat = labels.flatten()\n",
        "  labels_flat = labels_flat.numpy()\n",
        "\n",
        "  return np.sum(pred_flat == labels_flat) / len(labels_flat)\n",
        "\n",
        "\n",
        "\n",
        "# Put model in evaluation mode to evaluate loss on the validation set\n",
        "model.eval()\n",
        "\n",
        "with torch.no_grad():\n",
        "  logits = model(input_ids, token_type_ids=None, attention_mask=attention_masks)\n",
        "\n",
        "flat_accuracy(logits[0], label)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "1.0"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 9
        }
      ]
    }
  ]
}