Text-preprocessing-notebook.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 82,
   "id": "5b2d2c2a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['@UN', '@UN_Women']"
      ]
     },
     "execution_count": 82,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text7 = '@UN @UN_Women \"Ethics are built right into the ideals and objectives of the United Nations\" \\\n",
    "#UNSG @ NY Society for Ethical Culture bit.ly/2guVelr'\n",
    "import re\n",
    "mytext = text7.split(' ')\n",
    "[w for w in mytext if re.search ('@[A-Za-z0-9_]+', w)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "id": "2c1fca52",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['NLTK', 'is', 'a', 'leading', 'platform', 'for', 'building', 'Python', 'programs', 'to', 'work', 'with', 'human', 'language', 'data', '.', 'It', 'provides', 'easy-to-use', 'interfaces', 'to', 'over', '50', 'corpora', 'and', 'lexical', 'resources', '.']\n"
     ]
    }
   ],
   "source": [
    "import nltk\n",
    "str =\"NLTK is a leading platform for building Python programs to work with human language data. It provides easy-to-use interfaces to over 50 corpora and lexical resources.\"\n",
    "from nltk.tokenize import word_tokenize\n",
    "text_tokens = word_tokenize(str)\n",
    "print(text_tokens)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "id": "71e74e2e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# stop words removal\n",
    "import nltk\n",
    "from nltk.corpus import stopwords\n",
    "#print (stopwords.words('english'))\n",
    "stop_words = set(stopwords.words('english'))\n",
    "text_tokens = word_tokenize(str)\n",
    "filterd_text = [w for w in text_tokens if not  w.lower() in stop_words]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "id": "4907185b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['NLTK', 'leading', 'platform', 'building', 'Python', 'programs', 'work', 'human', 'language', 'data', '.', 'provides', 'easy-to-use', 'interfaces', '50', 'corpora', 'lexical', 'resources', '.']\n"
     ]
    }
   ],
   "source": [
    "print (filterd_text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "id": "15d853d9",
   "metadata": {},
   "outputs": [],
   "source": [
    "filterd_text = [w for w in filterd_text if w != '.']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "id": "b1b872f3",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['NLTK', 'leading', 'platform', 'building', 'Python', 'programs', 'work', 'human', 'language', 'data', 'provides', 'easy-to-use', 'interfaces', '50', 'corpora', 'lexical', 'resources']\n"
     ]
    }
   ],
   "source": [
    "print (filterd_text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "id": "aa32c5b2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['nltk', 'lead', 'platform', 'build', 'python', 'program', 'work', 'human', 'languag', 'data', 'provid', 'easy-to-us', 'interfac', '50', 'corpora', 'lexic', 'resourc']\n"
     ]
    }
   ],
   "source": [
    "# stemming\n",
    "from nltk.stem import PorterStemmer\n",
    "#str2 = \"I was sleeping. Did you read the book I was reading? I was speaking when you were slept.\"\n",
    "from nltk.tokenize import word_tokenize\n",
    "#text_tokens = word_tokenize(filterd_text)\n",
    "stemmer = PorterStemmer()\n",
    "stemmed_text = [stemmer.stem(w) for w in filterd_text]\n",
    "print (stemmed_text)\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "id": "e22dffdf",
   "metadata": {},
   "outputs": [],
   "source": [
    "#word2vec\n",
    "import gensim\n",
    "from gensim.models import Word2Vec\n",
    "from nltk.corpus import brown, movie_reviews, treebank\n",
    "\n",
    "b = Word2Vec(brown.sents())\n",
    "b.save ('brown.embedding')\n",
    "#len(movie_reviews.sents())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "id": "a43bef69",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<ipython-input-97-7009b7100184>:2: DeprecationWarning: Call to deprecated `__getitem__` (Method will be removed in 4.0.0, use self.wv.__getitem__() instead).\n",
      "  new_model['luxurious']\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([ 0.00280439, -0.04052062,  0.03445007,  0.05987329, -0.00445369,\n",
       "       -0.00788333,  0.01117999,  0.05222733, -0.04317435, -0.0156741 ,\n",
       "        0.02136648,  0.02013602,  0.05789823, -0.02516223, -0.01066493,\n",
       "       -0.00087927, -0.00814933, -0.01407449, -0.05362804, -0.04329726,\n",
       "       -0.00530956,  0.01113105, -0.05073878,  0.01740376,  0.0072285 ,\n",
       "        0.01963971, -0.05751493,  0.01775876, -0.0236617 ,  0.00796327,\n",
       "       -0.04030541, -0.02137259,  0.00979035,  0.03333796, -0.04430782,\n",
       "       -0.00947509, -0.00217238,  0.02206587,  0.01594679,  0.00407867,\n",
       "       -0.01919502, -0.032233  , -0.01931941, -0.05357175,  0.04875766,\n",
       "        0.03643737,  0.02712464, -0.01166884,  0.01511744, -0.02735651,\n",
       "       -0.02350183,  0.00552888, -0.00632858, -0.01752267, -0.00084344,\n",
       "        0.01122153, -0.02671883, -0.01490573, -0.01266249,  0.02621745,\n",
       "        0.04417945,  0.00179544, -0.03316855,  0.04582269, -0.03479574,\n",
       "        0.05253881, -0.01837257, -0.02049927, -0.01920311, -0.00161503,\n",
       "        0.04261051,  0.02318022,  0.02879585, -0.06502986,  0.01238524,\n",
       "        0.03043072, -0.0016209 ,  0.02280431, -0.0292137 , -0.0222376 ,\n",
       "       -0.00212527,  0.02152537,  0.00902196, -0.07777054,  0.04121765,\n",
       "        0.01339401,  0.00939879, -0.05969568, -0.01205128,  0.05157083,\n",
       "       -0.0350206 , -0.0064469 , -0.04360951,  0.00283879,  0.030083  ,\n",
       "        0.05046897, -0.02028229,  0.00391383,  0.02544644,  0.01251267],\n",
       "      dtype=float32)"
      ]
     },
     "execution_count": 97,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new_model = Word2Vec.load('brown.embedding')\n",
    "new_model['luxurious']\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "id": "abf3ff4b",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<ipython-input-98-646b69963cad>:1: DeprecationWarning: Call to deprecated `similarity` (Method will be removed in 4.0.0, use self.wv.similarity() instead).\n",
      "  new_model.similarity ('university', 'school')\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.8460974"
      ]
     },
     "execution_count": 98,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new_model.similarity ('university', 'school')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "id": "ee195612",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<ipython-input-99-c562daa13e5b>:1: DeprecationWarning: Call to deprecated `similarity` (Method will be removed in 4.0.0, use self.wv.similarity() instead).\n",
      "  new_model.similarity ('university', 'animal')\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.92897964"
      ]
     },
     "execution_count": 99,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new_model.similarity ('university', 'animal')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "id": "51215352",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<ipython-input-100-a25cbe5d6ed1>:1: DeprecationWarning: Call to deprecated `similarity` (Method will be removed in 4.0.0, use self.wv.similarity() instead).\n",
      "  new_model.similarity ('university', 'man')\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.4009484"
      ]
     },
     "execution_count": 100,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new_model.similarity ('university', 'man')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "id": "8da70960",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<ipython-input-101-55025506af69>:1: DeprecationWarning: Call to deprecated `most_similar` (Method will be removed in 4.0.0, use self.wv.most_similar() instead).\n",
      "  new_model.most_similar('money', topn = 5)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[('care', 0.9187424778938293),\n",
       " ('job', 0.9148074388504028),\n",
       " ('trouble', 0.9037779569625854),\n",
       " ('work', 0.8860379457473755),\n",
       " ('chance', 0.8836445808410645)]"
      ]
     },
     "execution_count": 101,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new_model.most_similar('money', topn = 5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f7650371",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}