Part-3-Test-The-Model.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import re\n",
    "import numpy as np\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>File</th>\n",
       "      <th>Task</th>\n",
       "      <th>Category</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>g0pA_taska.txt</td>\n",
       "      <td>a</td>\n",
       "      <td>non</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>g0pA_taskb.txt</td>\n",
       "      <td>b</td>\n",
       "      <td>cut</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>g0pA_taskc.txt</td>\n",
       "      <td>c</td>\n",
       "      <td>light</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>g0pA_taskd.txt</td>\n",
       "      <td>d</td>\n",
       "      <td>heavy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>g0pA_taske.txt</td>\n",
       "      <td>e</td>\n",
       "      <td>non</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             File Task Category\n",
       "0  g0pA_taska.txt    a      non\n",
       "1  g0pA_taskb.txt    b      cut\n",
       "2  g0pA_taskc.txt    c    light\n",
       "3  g0pA_taskd.txt    d    heavy\n",
       "4  g0pA_taske.txt    e      non"
      ]
     },
     "execution_count": 103,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_data_files = pd.read_csv (os.path.join (\"dataset_test\", \"file_information.csv\"))\n",
    "test_data_files.head ()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    g0pA_taska.txt\n",
       "1    g0pA_taskb.txt\n",
       "2    g0pA_taskc.txt\n",
       "3    g0pA_taskd.txt\n",
       "4    g0pA_taske.txt\n",
       "Name: File, dtype: object"
      ]
     },
     "execution_count": 104,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "files = test_data_files['File']\n",
    "files.head ()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [],
   "source": [
    "files_text = []\n",
    "for file in files :\n",
    "    files_text.append (open (os.path.join (\"dataset_test\", file), 'r', encoding = \"cp437\").read ())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "100\n"
     ]
    }
   ],
   "source": [
    "print (len (files_text))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Inheritance is a basic concept of Object-Oriented Programming where\\nthe basic idea is to create new classes that add extra detail to\\nexisting classes. This is done by allowing the new classes to reuse\\nthe methods and variables of the existing classes and new methods and\\nclasses are added to specialise the new class. Inheritance models the\\nΓÇ£is-kind-ofΓÇ¥ relationship between entities (or objects), for example,\\npostgraduates and undergraduates are both kinds of student. This kind\\nof relationship can be visualised as a tree structure, where ΓÇÿstudentΓÇÖ\\nwould be the more general root node and both ΓÇÿpostgraduateΓÇÖ and\\nΓÇÿundergraduateΓÇÖ would be more specialised extensions of the ΓÇÿstudentΓÇÖ\\nnode (or the child nodes). In this relationship ΓÇÿstudentΓÇÖ would be\\nknown as the superclass or parent class whereas, ΓÇÿpostgraduateΓÇÖ would\\nbe known as the subclass or child class because the ΓÇÿpostgraduateΓÇÖ\\nclass extends the ΓÇÿstudentΓÇÖ class.\\n\\nInheritance can occur on several layers, where if visualised would\\ndisplay a larger tree structure. For example, we could further extend\\nthe ΓÇÿpostgraduateΓÇÖ node by adding two extra extended classes to it\\ncalled, ΓÇÿMSc StudentΓÇÖ and ΓÇÿPhD StudentΓÇÖ as both these types of student\\nare kinds of postgraduate student. This would mean that both the ΓÇÿMSc\\nStudentΓÇÖ and ΓÇÿPhD StudentΓÇÖ classes would inherit methods and variables\\nfrom both the ΓÇÿpostgraduateΓÇÖ and ΓÇÿstudent classesΓÇÖ.\\n'"
      ]
     },
     "execution_count": 108,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "files_text[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package punkt to\n",
      "[nltk_data]     C:\\Users\\Shraeyas\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package punkt is already up-to-date!\n",
      "[nltk_data] Downloading package stopwords to\n",
      "[nltk_data]     C:\\Users\\Shraeyas\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package stopwords is already up-to-date!\n"
     ]
    }
   ],
   "source": [
    "import re\n",
    "import nltk\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.tokenize import word_tokenize\n",
    "from nltk.stem import LancasterStemmer\n",
    "\n",
    "nltk.download ('punkt')\n",
    "nltk.download ('stopwords')\n",
    "lancaster = LancasterStemmer()\n",
    "\n",
    "def pre_process (_sentences) :\n",
    "    en_stops = set (stopwords.words ('english')) # Remove Stopwords\n",
    "    processed_sentences = []\n",
    "\n",
    "    for sentence in _sentences :\n",
    "        sentence = re.sub (r'[^\\w\\s]','', sentence) # Remove special characters\n",
    "        lower_sen = sentence.lower () # Convert the sentence to Lowercase\n",
    "        token_sen = word_tokenize (lower_sen) # Tokenize the sentence\n",
    "\n",
    "        new_sentence = []\n",
    "        for word in token_sen:\n",
    "            if word not in en_stops and word.isnumeric() == False : # Remove words that contain only digits\n",
    "                word = lancaster.stem (word) # Perform Stemming\n",
    "                new_sentence.append (word)\n",
    "\n",
    "        if len (new_sentence) > 3 :\n",
    "            processed_sentences.append (new_sentence)\n",
    "    return processed_sentences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [],
   "source": [
    "documents = []\n",
    "for i, file_text in enumerate (files_text) :\n",
    "    sentences = file_text.split (\".\")\n",
    "    documents.append (pre_process (sentences))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {},
   "outputs": [],
   "source": [
    "flat_documents = []\n",
    "for document in documents :\n",
    "    flat_documents.append ([item for sublist in document for item in sublist])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "flat_documents[50]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [],
   "source": [
    "from gensim.models.doc2vec import Doc2Vec\n",
    "model = Doc2Vec.load (os.path.join (\"models\", \"doc2vec_20news.bin\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {},
   "outputs": [],
   "source": [
    "vectors = []\n",
    "for flat_document in flat_documents :\n",
    "    v = model.infer_vector (flat_document)\n",
    "    vectors.append (v)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "vectors[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "vectors[50]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "metadata": {},
   "outputs": [],
   "source": [
    "from scipy import spatial\n",
    "values = []\n",
    "for i in range (len (vectors)) :\n",
    "    for j in range (i + 1, len (vectors)) :\n",
    "        result = 1 - spatial.distance.cosine (vectors[i], vectors[j])\n",
    "        values.append ([result, i + 1, j + 1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 139,
   "metadata": {},
   "outputs": [],
   "source": [
    "values.sort ()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 140,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[[-0.0973544493317604, 44, 51],\n",
       " [-0.07829519361257553, 69, 71],\n",
       " [-0.07706417143344879, 21, 69],\n",
       " [-0.058547187596559525, 69, 96],\n",
       " [-0.05392792075872421, 44, 56]]"
      ]
     },
     "execution_count": 140,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "values[0 : 5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[[-0.0973544493317604, 44, 51],\n",
       " [-0.07829519361257553, 69, 71],\n",
       " [-0.07706417143344879, 21, 69],\n",
       " [-0.058547187596559525, 69, 96],\n",
       " [-0.05392792075872421, 44, 56]]"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "values[0 : 5]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Sentence wise similarity\n",
    "##### We'll calculate sentence wise similarity documents"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "```\n",
    "The following function takes 2 documents as input and returns cosine_distance and the start and end index in both the documents for the particular pair\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_similarity (doc_0, doc_1) :\n",
    "    sentences_dict = {}\n",
    "    \n",
    "    sentences_0 = []\n",
    "    sentences_0 = doc_0.split (\".\")\n",
    "    sentences_0_processed = []\n",
    "    sentences_0_processed.append (pre_process (sentences_0))\n",
    "    vectors_0 = []\n",
    "    for i, sentence in enumerate (sentences_0_processed[0]) :\n",
    "        v = model.infer_vector (sentence)\n",
    "        for match in re.finditer (sentences_0[i], doc_0) :\n",
    "            vectors_0.append ([v, match.start (), match.end ()]) \n",
    "    \n",
    "    sentences_1 = []\n",
    "    sentences_1 = doc_1.split (\".\")\n",
    "    sentences_1_processed = []\n",
    "    sentences_1_processed.append (pre_process (sentences_1))\n",
    "    vectors_1 = []\n",
    "    for i, sentence in enumerate (sentences_1_processed[0]) :\n",
    "        v = model.infer_vector (sentence)\n",
    "        for match in re.finditer (sentences_1[i], doc_1) :\n",
    "            vectors_1.append ([v, match.start (), match.end ()])\n",
    "    \n",
    "    values = []\n",
    "    for i in range (len (vectors_0)) :\n",
    "        for j in range (len (vectors_1)) :\n",
    "            result = 1 - spatial.distance.cosine (vectors_0[i][0], vectors_1[j][0])\n",
    "            values.append ([result, [vectors_0[i][1], vectors_0[i][2]], [vectors_1[j][1], vectors_1[j][2]]])\n",
    "    \n",
    "    return values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [],
   "source": [
    "vals = get_similarity (files_text[43], files_text[50])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [],
   "source": [
    "vals.sort (reverse = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[[0.7193825840950012, [713, 788], [963, 1049]],\n",
       " [0.6758325099945068, [713, 788], [1169, 1307]],\n",
       " [0.6283330917358398, [713, 788], [845, 962]],\n",
       " [0.5072965621948242, [232, 296], [963, 1049]],\n",
       " [0.5023082494735718, [713, 788], [381, 527]]]"
      ]
     },
     "execution_count": 101,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vals[0 : 5]"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}