train_model.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "xYGgvy-vu9Id",
    "outputId": "7037cb16-2a8d-494a-a5ce-48b54c42d34e"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting sentencepiece\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/f5/99/e0808cb947ba10f575839c43e8fafc9cc44e4a7a2c8f79c60db48220a577/sentencepiece-0.1.95-cp37-cp37m-manylinux2014_x86_64.whl (1.2MB)\n",
      "\r",
      "\u001b[K     |▎                               | 10kB 19.9MB/s eta 0:00:01\r",
      "\u001b[K     |▌                               | 20kB 24.5MB/s eta 0:00:01\r",
      "\u001b[K     |▉                               | 30kB 22.4MB/s eta 0:00:01\r",
      "\u001b[K     |█                               | 40kB 16.9MB/s eta 0:00:01\r",
      "\u001b[K     |█▍                              | 51kB 8.5MB/s eta 0:00:01\r",
      "\u001b[K     |█▋                              | 61kB 7.9MB/s eta 0:00:01\r",
      "\u001b[K     |██                              | 71kB 8.6MB/s eta 0:00:01\r",
      "\u001b[K     |██▏                             | 81kB 9.6MB/s eta 0:00:01\r",
      "\u001b[K     |██▌                             | 92kB 9.9MB/s eta 0:00:01\r",
      "\u001b[K     |██▊                             | 102kB 8.2MB/s eta 0:00:01\r",
      "\u001b[K     |███                             | 112kB 8.2MB/s eta 0:00:01\r",
      "\u001b[K     |███▎                            | 122kB 8.2MB/s eta 0:00:01\r",
      "\u001b[K     |███▌                            | 133kB 8.2MB/s eta 0:00:01\r",
      "\u001b[K     |███▉                            | 143kB 8.2MB/s eta 0:00:01\r",
      "\u001b[K     |████                            | 153kB 8.2MB/s eta 0:00:01\r",
      "\u001b[K     |████▍                           | 163kB 8.2MB/s eta 0:00:01\r",
      "\u001b[K     |████▋                           | 174kB 8.2MB/s eta 0:00:01\r",
      "\u001b[K     |█████                           | 184kB 8.2MB/s eta 0:00:01\r",
      "\u001b[K     |█████▏                          | 194kB 8.2MB/s eta 0:00:01\r",
      "\u001b[K     |█████▌                          | 204kB 8.2MB/s eta 0:00:01\r",
      "\u001b[K     |█████▊                          | 215kB 8.2MB/s eta 0:00:01\r",
      "\u001b[K     |██████                          | 225kB 8.2MB/s eta 0:00:01\r",
      "\u001b[K     |██████▎                         | 235kB 8.2MB/s eta 0:00:01\r",
      "\u001b[K     |██████▌                         | 245kB 8.2MB/s eta 0:00:01\r",
      "\u001b[K     |██████▉                         | 256kB 8.2MB/s eta 0:00:01\r",
      "\u001b[K     |███████                         | 266kB 8.2MB/s eta 0:00:01\r",
      "\u001b[K     |███████▍                        | 276kB 8.2MB/s eta 0:00:01\r",
      "\u001b[K     |███████▋                        | 286kB 8.2MB/s eta 0:00:01\r",
      "\u001b[K     |████████                        | 296kB 8.2MB/s eta 0:00:01\r",
      "\u001b[K     |████████▏                       | 307kB 8.2MB/s eta 0:00:01\r",
      "\u001b[K     |████████▍                       | 317kB 8.2MB/s eta 0:00:01\r",
      "\u001b[K     |████████▊                       | 327kB 8.2MB/s eta 0:00:01\r",
      "\u001b[K     |█████████                       | 337kB 8.2MB/s eta 0:00:01\r",
      "\u001b[K     |█████████▎                      | 348kB 8.2MB/s eta 0:00:01\r",
      "\u001b[K     |█████████▌                      | 358kB 8.2MB/s eta 0:00:01\r",
      "\u001b[K     |█████████▉                      | 368kB 8.2MB/s eta 0:00:01\r",
      "\u001b[K     |██████████                      | 378kB 8.2MB/s eta 0:00:01\r",
      "\u001b[K     |██████████▍                     | 389kB 8.2MB/s eta 0:00:01\r",
      "\u001b[K     |██████████▋                     | 399kB 8.2MB/s eta 0:00:01\r",
      "\u001b[K     |███████████                     | 409kB 8.2MB/s eta 0:00:01\r",
      "\u001b[K     |███████████▏                    | 419kB 8.2MB/s eta 0:00:01\r",
      "\u001b[K     |███████████▍                    | 430kB 8.2MB/s eta 0:00:01\r",
      "\u001b[K     |███████████▊                    | 440kB 8.2MB/s eta 0:00:01\r",
      "\u001b[K     |████████████                    | 450kB 8.2MB/s eta 0:00:01\r",
      "\u001b[K     |████████████▎                   | 460kB 8.2MB/s eta 0:00:01\r",
      "\u001b[K     |████████████▌                   | 471kB 8.2MB/s eta 0:00:01\r",
      "\u001b[K     |████████████▉                   | 481kB 8.2MB/s eta 0:00:01\r",
      "\u001b[K     |█████████████                   | 491kB 8.2MB/s eta 0:00:01\r",
      "\u001b[K     |█████████████▍                  | 501kB 8.2MB/s eta 0:00:01\r",
      "\u001b[K     |█████████████▋                  | 512kB 8.2MB/s eta 0:00:01\r",
      "\u001b[K     |█████████████▉                  | 522kB 8.2MB/s eta 0:00:01\r",
      "\u001b[K     |██████████████▏                 | 532kB 8.2MB/s eta 0:00:01\r",
      "\u001b[K     |██████████████▍                 | 542kB 8.2MB/s eta 0:00:01\r",
      "\u001b[K     |██████████████▊                 | 552kB 8.2MB/s eta 0:00:01\r",
      "\u001b[K     |███████████████                 | 563kB 8.2MB/s eta 0:00:01\r",
      "\u001b[K     |███████████████▎                | 573kB 8.2MB/s eta 0:00:01\r",
      "\u001b[K     |███████████████▌                | 583kB 8.2MB/s eta 0:00:01\r",
      "\u001b[K     |███████████████▉                | 593kB 8.2MB/s eta 0:00:01\r",
      "\u001b[K     |████████████████                | 604kB 8.2MB/s eta 0:00:01\r",
      "\u001b[K     |████████████████▍               | 614kB 8.2MB/s eta 0:00:01\r",
      "\u001b[K     |████████████████▋               | 624kB 8.2MB/s eta 0:00:01\r",
      "\u001b[K     |████████████████▉               | 634kB 8.2MB/s eta 0:00:01\r",
      "\u001b[K     |█████████████████▏              | 645kB 8.2MB/s eta 0:00:01\r",
      "\u001b[K     |█████████████████▍              | 655kB 8.2MB/s eta 0:00:01\r",
      "\u001b[K     |█████████████████▊              | 665kB 8.2MB/s eta 0:00:01\r",
      "\u001b[K     |██████████████████              | 675kB 8.2MB/s eta 0:00:01\r",
      "\u001b[K     |██████████████████▎             | 686kB 8.2MB/s eta 0:00:01\r",
      "\u001b[K     |██████████████████▌             | 696kB 8.2MB/s eta 0:00:01\r",
      "\u001b[K     |██████████████████▉             | 706kB 8.2MB/s eta 0:00:01\r",
      "\u001b[K     |███████████████████             | 716kB 8.2MB/s eta 0:00:01\r",
      "\u001b[K     |███████████████████▎            | 727kB 8.2MB/s eta 0:00:01\r",
      "\u001b[K     |███████████████████▋            | 737kB 8.2MB/s eta 0:00:01\r",
      "\u001b[K     |███████████████████▉            | 747kB 8.2MB/s eta 0:00:01\r",
      "\u001b[K     |████████████████████▏           | 757kB 8.2MB/s eta 0:00:01\r",
      "\u001b[K     |████████████████████▍           | 768kB 8.2MB/s eta 0:00:01\r",
      "\u001b[K     |████████████████████▊           | 778kB 8.2MB/s eta 0:00:01\r",
      "\u001b[K     |█████████████████████           | 788kB 8.2MB/s eta 0:00:01\r",
      "\u001b[K     |█████████████████████▎          | 798kB 8.2MB/s eta 0:00:01\r",
      "\u001b[K     |█████████████████████▌          | 808kB 8.2MB/s eta 0:00:01\r",
      "\u001b[K     |█████████████████████▉          | 819kB 8.2MB/s eta 0:00:01\r",
      "\u001b[K     |██████████████████████          | 829kB 8.2MB/s eta 0:00:01\r",
      "\u001b[K     |██████████████████████▎         | 839kB 8.2MB/s eta 0:00:01\r",
      "\u001b[K     |██████████████████████▋         | 849kB 8.2MB/s eta 0:00:01\r",
      "\u001b[K     |██████████████████████▉         | 860kB 8.2MB/s eta 0:00:01\r",
      "\u001b[K     |███████████████████████▏        | 870kB 8.2MB/s eta 0:00:01\r",
      "\u001b[K     |███████████████████████▍        | 880kB 8.2MB/s eta 0:00:01\r",
      "\u001b[K     |███████████████████████▊        | 890kB 8.2MB/s eta 0:00:01\r",
      "\u001b[K     |████████████████████████        | 901kB 8.2MB/s eta 0:00:01\r",
      "\u001b[K     |████████████████████████▎       | 911kB 8.2MB/s eta 0:00:01\r",
      "\u001b[K     |████████████████████████▌       | 921kB 8.2MB/s eta 0:00:01\r",
      "\u001b[K     |████████████████████████▊       | 931kB 8.2MB/s eta 0:00:01\r",
      "\u001b[K     |█████████████████████████       | 942kB 8.2MB/s eta 0:00:01\r",
      "\u001b[K     |█████████████████████████▎      | 952kB 8.2MB/s eta 0:00:01\r",
      "\u001b[K     |█████████████████████████▋      | 962kB 8.2MB/s eta 0:00:01\r",
      "\u001b[K     |█████████████████████████▉      | 972kB 8.2MB/s eta 0:00:01\r",
      "\u001b[K     |██████████████████████████▏     | 983kB 8.2MB/s eta 0:00:01\r",
      "\u001b[K     |██████████████████████████▍     | 993kB 8.2MB/s eta 0:00:01\r",
      "\u001b[K     |██████████████████████████▊     | 1.0MB 8.2MB/s eta 0:00:01\r",
      "\u001b[K     |███████████████████████████     | 1.0MB 8.2MB/s eta 0:00:01\r",
      "\u001b[K     |███████████████████████████▎    | 1.0MB 8.2MB/s eta 0:00:01\r",
      "\u001b[K     |███████████████████████████▌    | 1.0MB 8.2MB/s eta 0:00:01\r",
      "\u001b[K     |███████████████████████████▊    | 1.0MB 8.2MB/s eta 0:00:01\r",
      "\u001b[K     |████████████████████████████    | 1.1MB 8.2MB/s eta 0:00:01\r",
      "\u001b[K     |████████████████████████████▎   | 1.1MB 8.2MB/s eta 0:00:01\r",
      "\u001b[K     |████████████████████████████▋   | 1.1MB 8.2MB/s eta 0:00:01\r",
      "\u001b[K     |████████████████████████████▉   | 1.1MB 8.2MB/s eta 0:00:01\r",
      "\u001b[K     |█████████████████████████████▏  | 1.1MB 8.2MB/s eta 0:00:01\r",
      "\u001b[K     |█████████████████████████████▍  | 1.1MB 8.2MB/s eta 0:00:01\r",
      "\u001b[K     |█████████████████████████████▊  | 1.1MB 8.2MB/s eta 0:00:01\r",
      "\u001b[K     |██████████████████████████████  | 1.1MB 8.2MB/s eta 0:00:01\r",
      "\u001b[K     |██████████████████████████████▏ | 1.1MB 8.2MB/s eta 0:00:01\r",
      "\u001b[K     |██████████████████████████████▌ | 1.1MB 8.2MB/s eta 0:00:01\r",
      "\u001b[K     |██████████████████████████████▊ | 1.2MB 8.2MB/s eta 0:00:01\r",
      "\u001b[K     |███████████████████████████████ | 1.2MB 8.2MB/s eta 0:00:01\r",
      "\u001b[K     |███████████████████████████████▎| 1.2MB 8.2MB/s eta 0:00:01\r",
      "\u001b[K     |███████████████████████████████▋| 1.2MB 8.2MB/s eta 0:00:01\r",
      "\u001b[K     |███████████████████████████████▉| 1.2MB 8.2MB/s eta 0:00:01\r",
      "\u001b[K     |████████████████████████████████| 1.2MB 8.2MB/s \n",
      "\u001b[?25hInstalling collected packages: sentencepiece\n",
      "Successfully installed sentencepiece-0.1.95\n"
     ]
    }
   ],
   "source": [
    "! pip install sentencepiece"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "b0BU1hYHu0ax",
    "outputId": "67f3912f-a8cf-48e0-f063-cfcb962bc4e1"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: transformers in /usr/local/lib/python3.7/dist-packages (4.6.1)\n",
      "Requirement already satisfied: importlib-metadata; python_version < \"3.8\" in /usr/local/lib/python3.7/dist-packages (from transformers) (4.0.1)\n",
      "Requirement already satisfied: tokenizers<0.11,>=0.10.1 in /usr/local/lib/python3.7/dist-packages (from transformers) (0.10.2)\n",
      "Requirement already satisfied: huggingface-hub==0.0.8 in /usr/local/lib/python3.7/dist-packages (from transformers) (0.0.8)\n",
      "Requirement already satisfied: sacremoses in /usr/local/lib/python3.7/dist-packages (from transformers) (0.0.45)\n",
      "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (1.19.5)\n",
      "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (2019.12.20)\n",
      "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.7/dist-packages (from transformers) (4.41.1)\n",
      "Requirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from transformers) (3.0.12)\n",
      "Requirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from transformers) (2.23.0)\n",
      "Requirement already satisfied: packaging in /usr/local/lib/python3.7/dist-packages (from transformers) (20.9)\n",
      "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata; python_version < \"3.8\"->transformers) (3.4.1)\n",
      "Requirement already satisfied: typing-extensions>=3.6.4; python_version < \"3.8\" in /usr/local/lib/python3.7/dist-packages (from importlib-metadata; python_version < \"3.8\"->transformers) (3.7.4.3)\n",
      "Requirement already satisfied: click in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (8.0.0)\n",
      "Requirement already satisfied: joblib in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (1.0.1)\n",
      "Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (1.15.0)\n",
      "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2.10)\n",
      "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (1.24.3)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2020.12.5)\n",
      "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (3.0.4)\n",
      "Requirement already satisfied: pyparsing>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging->transformers) (2.4.7)\n"
     ]
    }
   ],
   "source": [
    "! pip install transformers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "eLIQjXQ7vBNP",
    "outputId": "9b9011af-0c38-4763-e06f-e235f8ac8c19"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.8.1+cu101\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "device(type='cuda', index=0)"
      ]
     },
     "execution_count": 2,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import gc\n",
    "\n",
    "import random\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "import torch.nn.functional as F\n",
    "from torch.utils.data import Dataset, DataLoader\n",
    "\n",
    "from sklearn import preprocessing\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "# set a seed value\n",
    "torch.manual_seed(555)\n",
    "\n",
    "from sklearn.utils import shuffle\n",
    "from sklearn.metrics import roc_auc_score, accuracy_score\n",
    "\n",
    "import transformers\n",
    "from transformers import BertTokenizer, BertForSequenceClassification \n",
    "from transformers import XLMRobertaTokenizer, XLMRobertaForSequenceClassification\n",
    "from transformers import RobertaTokenizer, RobertaModel\n",
    "\n",
    "from transformers import AdamW, Adafactor\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "\n",
    "\n",
    "print(torch.__version__)\n",
    "\n",
    "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
    "device"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "U6V8Loa_vZlU"
   },
   "outputs": [],
   "source": [
    "class TextDataset(Dataset):\n",
    "\n",
    "  def __init__(self, sense_source,\tsense_target, targets, tokenizer, max_len):\n",
    "    self.sense_source = sense_source\n",
    "    self.sense_target = sense_target\n",
    "    self.targets = targets\n",
    "    self.tokenizer = tokenizer\n",
    "    self.max_len = max_len\n",
    "  \n",
    "  def __len__(self):\n",
    "    return len(self.sense_source)\n",
    "  \n",
    "  def __getitem__(self, item):\n",
    "    sense_source = str(self.sense_source[item])\n",
    "    sense_target = str(self.sense_target[item])\n",
    "\n",
    "    encoded_dict = self.tokenizer.encode_plus(\n",
    "      sense_source,\n",
    "      sense_target,\n",
    "      add_special_tokens=True,\n",
    "      max_length=self.max_len,\n",
    "      return_token_type_ids=True,\n",
    "      pad_to_max_length=True,\n",
    "      return_attention_mask=True,\n",
    "      return_tensors='pt',\n",
    "    )\n",
    "    # These are torch tensors already.\n",
    "    padded_token_list = encoded_dict['input_ids'][0]\n",
    "    att_mask = encoded_dict['attention_mask'][0]\n",
    "    #token_type_ids = encoded_dict['token_type_ids'][0]\n",
    "        \n",
    "    # Convert the target to a torch tensor\n",
    "    target = torch.tensor(self.targets[item])\n",
    "\n",
    "    sample = (padded_token_list, att_mask, target, item)\n",
    "\n",
    "    return sample"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "dTY7jIppvBRq"
   },
   "outputs": [],
   "source": [
    "DATASET_NAME = 'DE_all_MWSA.csv'\n",
    "BATCH_SIZE = 32\n",
    "MAX_LEN = 100\n",
    "MODEL_TYPE = 'xlm-roberta-base'\n",
    "NUM_EPOCHS = 50"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "sFK93X7PvKIK"
   },
   "source": [
    "Datasets: https://drive.google.com/drive/folders/1QOR_Vb-KKPwT0gdA7j1qvrrXUmJChFpM?usp=sharing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from google.colab import drive\n",
    "drive.mount('/content/drive')\n",
    "\n",
    "!mkdir ./data/\n",
    "!cp /content/drive/MyDrive/MWSA/HA_MWSA.csv ./"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv(DATASET_NAME)\n",
    "#df = df.dropna()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "iXi2j8m2CsAj",
    "outputId": "452ec999-4505-4845-c575-ab8c521b6cb4"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['broader', 'narrower', 'exact', 'related'], dtype=object)"
      ]
     },
     "execution_count": 37,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['semantic_relationship'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "2CBs-ZiqQl2_",
    "outputId": "6209e047-b7f1-436a-e88d-5a74363b057c"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4"
      ]
     },
     "execution_count": 38,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "NUM_CLASSES = len(df['semantic_relationship'].unique())\n",
    "NUM_CLASSES"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "TU8qWMNGCjaW"
   },
   "outputs": [],
   "source": [
    "df['targets'] = df['semantic_relationship'].map({'exact': 0, 'broader':1, 'narrower': 2, 'related':3})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer = XLMRobertaTokenizer.from_pretrained(MODEL_TYPE)\n",
    "\n",
    "model = XLMRobertaForSequenceClassification.from_pretrained(\n",
    "    MODEL_TYPE, \n",
    "    num_labels = NUM_CLASSES, # The number of output labels\n",
    ")\n",
    "\n",
    "for param in model.parameters():\n",
    "    param.requires_grad = True\n",
    "    \n",
    "model.to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "8xQvE_z9wEbL"
   },
   "outputs": [],
   "source": [
    "train, test = train_test_split(df[['sense_source', 'sense_target', 'targets']], test_size=0.2, random_state=42)\n",
    "\n",
    "train_data = TextDataset(\n",
    "  sense_source=train.sense_source.to_numpy(),\n",
    "  sense_target=train.sense_target.to_numpy(),\n",
    "  targets=train.targets.to_numpy(),\n",
    "  tokenizer=tokenizer,\n",
    "  max_len=MAX_LEN\n",
    "  )\n",
    "test_data = TextDataset(\n",
    "  sense_source=test.sense_source.to_numpy(),\n",
    "  sense_target=test.sense_target.to_numpy(),\n",
    "  targets=test.targets.to_numpy(),\n",
    "  tokenizer=tokenizer,\n",
    "  max_len=MAX_LEN\n",
    "  )\n",
    "\n",
    "train_dataloader = torch.utils.data.DataLoader(train_data,\n",
    "                                        batch_size=BATCH_SIZE,\n",
    "                                        shuffle=True)\n",
    "test_dataloader = torch.utils.data.DataLoader(test_data,\n",
    "                                        batch_size=BATCH_SIZE,\n",
    "                                        shuffle=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "JZfuJJri2A-Y"
   },
   "outputs": [],
   "source": [
    "train, test = train_test_split(df[['lemma', 'pos', 'sense_source', 'sense_target', \n",
    "                                   'semantic_relationship', 'targets']], test_size=0.2, random_state=42)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "eymtwButwSXT"
   },
   "outputs": [],
   "source": [
    "# Define the optimizer\n",
    "optimizer = AdamW(model.parameters(), \n",
    "              lr = 1e-5,\n",
    "            )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Set the seed.\n",
    "seed_val = 101\n",
    "\n",
    "random.seed(seed_val)\n",
    "np.random.seed(seed_val)\n",
    "torch.manual_seed(seed_val)\n",
    "torch.cuda.manual_seed_all(seed_val)\n",
    "\n",
    "# Store the average loss after each epoch so we can plot them.\n",
    "loss_values = []\n",
    "\n",
    "\n",
    "# For each epoch...\n",
    "for epoch in range(NUM_EPOCHS):\n",
    "    \n",
    "    print(\"\")\n",
    "    print('======== Epoch {:} / {:} ========'.format(epoch + 1, NUM_EPOCHS))\n",
    "    \n",
    "\n",
    "    stacked_val_labels = []\n",
    "    stacked_val_item_ids = []\n",
    "    targets_list = []\n",
    "\n",
    "    # ========================================\n",
    "    #               Training\n",
    "    # ========================================\n",
    "    \n",
    "    print('Training...')\n",
    "    \n",
    "    # put the model into train mode\n",
    "    model.train()\n",
    "    \n",
    "    # This turns gradient calculations on and off.\n",
    "    torch.set_grad_enabled(True)\n",
    "\n",
    "\n",
    "    # Reset the total loss for this epoch.\n",
    "    total_train_loss = 0\n",
    "\n",
    "    for i, batch in enumerate(train_dataloader):\n",
    "        \n",
    "        train_status = 'Batch ' + str(i) + ' of ' + str(len(train_dataloader))\n",
    "        \n",
    "        print(train_status, end='\\r')\n",
    "        # Zero the gradients\n",
    "        optimizer.zero_grad()\n",
    "\n",
    "        b_input_ids = batch[0].to(device)\n",
    "        b_input_mask = batch[1].to(device)\n",
    "        b_labels = batch[2].to(device)\n",
    "       \n",
    "\n",
    "        outputs = model(b_input_ids, \n",
    "                    attention_mask=b_input_mask,\n",
    "                    labels=b_labels)\n",
    "        \n",
    "        # Get the loss from the outputs tuple: (loss, logits)\n",
    "        loss = outputs[0]\n",
    "        # Convert the loss from a torch tensor to a number.\n",
    "        # Calculate the total loss.\n",
    "        total_train_loss += loss.item()\n",
    "        \n",
    "        \n",
    "        # Perform a backward pass to calculate the gradients.\n",
    "        loss.backward()\n",
    "\n",
    "        # Clip the norm of the gradients to 1.0.\n",
    "        # This is to help prevent the \"exploding gradients\" problem.\n",
    "        torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)\n",
    "        \n",
    "        \n",
    "        # Use the optimizer to update the weights.\n",
    "        \n",
    "        # Optimizer for GPU\n",
    "        optimizer.step() \n",
    "\n",
    "        \n",
    "    print('Train loss:' ,total_train_loss)\n",
    "\n",
    "\n",
    "    # ========================================\n",
    "    #               Validation\n",
    "    # ========================================\n",
    "    \n",
    "    print('\\nValidation...')\n",
    "\n",
    "    # Put the model in evaluation mode.\n",
    "    model.eval()\n",
    "\n",
    "    # Turn off the gradient calculations.\n",
    "    # This tells the model not to compute or store gradients.\n",
    "    # This step saves memory and speeds up validation.\n",
    "    torch.set_grad_enabled(False)\n",
    "    \n",
    "    \n",
    "    # Reset the total loss for this epoch.\n",
    "    total_val_loss = 0\n",
    "    \n",
    "\n",
    "    for j, batch in enumerate(test_dataloader):\n",
    "        \n",
    "        val_status = 'Batch ' + str(j) + ' of ' + str(len(test_dataloader))\n",
    "        \n",
    "        print(val_status, end='\\r')\n",
    "\n",
    "        b_input_ids = batch[0].to(device)\n",
    "        b_input_mask = batch[1].to(device)\n",
    "        b_labels = batch[2].to(device)  \n",
    "        b_item_ids = batch[3]\n",
    "\n",
    "\n",
    "        outputs = model(b_input_ids, \n",
    "                attention_mask=b_input_mask, \n",
    "                labels=b_labels)\n",
    "        # Get the loss from the outputs tuple: (loss, logits)\n",
    "        loss = outputs[0]\n",
    "        \n",
    "        # Convert the loss from a torch tensor to a number.\n",
    "        # Calculate the total loss.\n",
    "        total_val_loss += loss.item()\n",
    "        \n",
    "\n",
    "        # Get the preds\n",
    "        preds = outputs[1]\n",
    "\n",
    "\n",
    "        # Move preds to the CPU\n",
    "        val_preds = preds.detach().cpu().numpy()\n",
    "        \n",
    "        # Move the labels to the cpu\n",
    "        targets_np = b_labels.to('cpu').numpy()\n",
    "\n",
    "        # Append the labels to a numpy list\n",
    "        targets_list.extend(targets_np)\n",
    "\n",
    "        if j == 0:  # first batch\n",
    "            stacked_val_preds = val_preds\n",
    "\n",
    "        else:\n",
    "            stacked_val_preds = np.vstack((stacked_val_preds, val_preds))\n",
    "            \n",
    "        stacked_val_item_ids.extend(b_item_ids.numpy())\n",
    "\n",
    "    \n",
    "    # Calculate the validation accuracy\n",
    "    y_true = targets_list\n",
    "    y_pred = np.argmax(stacked_val_preds, axis=1)\n",
    "    \n",
    "    val_acc = accuracy_score(y_true, y_pred)\n",
    "    \n",
    "    \n",
    "    print('Val loss:' ,total_val_loss)\n",
    "    print('Val acc: ', val_acc)\n",
    "    \n",
    "\n",
    "    # Save the Model\n",
    "    #torch.save(model.state_dict(), 'RU_xlm_model.pth')\n",
    "    \n",
    "    # Use the garbage collector to save memory.\n",
    "    gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "DRZIGq-yPgII"
   },
   "outputs": [],
   "source": [
    "t = test.iloc[stacked_val_item_ids]\n",
    "t['pred'] = y_pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "spJnZOpr3DHe"
   },
   "outputs": [],
   "source": [
    "t.to_csv('DE_all_mistakes.csv')"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "name": "ИТОГОВАЯ_с_XNLI_курсовая_XLM-Roberta_MWSA",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}