Bag-of-Word Model.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "e537be6d",
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e8d432b1",
   "metadata": {},
   "source": [
    "### Import dependencies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "b29caa81",
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "from sklearn.svm import SVC\n",
    "from sklearn import model_selection\n",
    "from sklearn.model_selection import train_test_split, GridSearchCV\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.metrics import accuracy_score, f1_score\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.decomposition import PCA\n",
    "from sklearn.pipeline import Pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "760e3bb1",
   "metadata": {},
   "outputs": [],
   "source": [
    "import data_prep\n",
    "import ngram\n",
    "import linguistic_feature"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ca5bf9af",
   "metadata": {},
   "source": [
    "### Import Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "0e6fdb0c",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Label</th>\n",
       "      <th>Rating</th>\n",
       "      <th>Ori_Review</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>Affinia Chicago is one of the worst hotels I h...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>I recently stayed at the Affina Chicago hotel ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>I stayed at the Affina Chicago for my annivers...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>If you are looking for a high end hotel on the...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>I just returned from a long weekend in Chicago...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Label  Rating                                         Ori_Review\n",
       "0      1       1  Affinia Chicago is one of the worst hotels I h...\n",
       "1      1       1  I recently stayed at the Affina Chicago hotel ...\n",
       "2      1       1  I stayed at the Affina Chicago for my annivers...\n",
       "3      1       1  If you are looking for a high end hotel on the...\n",
       "4      1       1  I just returned from a long weekend in Chicago..."
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "df = data_prep.import_chi_review()\n",
    "display(df[:5])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "48138f8c",
   "metadata": {},
   "source": [
    "### Train-Test Split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "966b3df5",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train, df_test = train_test_split(df, test_size=0.2, random_state=0)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f1bd39be",
   "metadata": {},
   "source": [
    "### Preprocess & Feature Extraction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "b74c5369",
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "df_train = data_prep.preprocess_ngram(df_train)\n",
    "df_test = data_prep.preprocess_ngram(df_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "093bac77",
   "metadata": {},
   "outputs": [],
   "source": [
    "unigrams, bigrams, trigrams, unigrams_dict, bigrams_dict, trigrams_dict = ngram.find_universal_ngrams(df_train['PP_Review'], 16, 16, 10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "4001bdd6",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Label</th>\n",
       "      <th>Rating</th>\n",
       "      <th>Ori_Review</th>\n",
       "      <th>Clean_Review</th>\n",
       "      <th>PP_Review</th>\n",
       "      <th>Word_List</th>\n",
       "      <th>Ngram</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>-1</td>\n",
       "      <td>1</td>\n",
       "      <td>great bed, but when I first came in, I request...</td>\n",
       "      <td>great bed but when i first come in i request a...</td>\n",
       "      <td>great bed first come request lake view request...</td>\n",
       "      <td>[great, bed, first, come, request, lake, view,...</td>\n",
       "      <td>[great, bed, first, come, request, lake, view,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>Don't let the website fool you. I stayed at th...</td>\n",
       "      <td>do not let the website fool you i stay at the ...</td>\n",
       "      <td>let website fool amalfi last business trip pro...</td>\n",
       "      <td>[let, website, fool, amalfi, last, business, t...</td>\n",
       "      <td>[let, website, amalfi, last, business, trip, p...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>-1</td>\n",
       "      <td>1</td>\n",
       "      <td>Just returned from a one night stay at the Kni...</td>\n",
       "      <td>just return from a one night stay at the knick...</td>\n",
       "      <td>return one night knickerbocker return come now...</td>\n",
       "      <td>[return, one, night, knickerbocker, return, co...</td>\n",
       "      <td>[return, one, night, knickerbocker, return, co...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Label  Rating                                         Ori_Review  \\\n",
       "0     -1       1  great bed, but when I first came in, I request...   \n",
       "1      1       1  Don't let the website fool you. I stayed at th...   \n",
       "2     -1       1  Just returned from a one night stay at the Kni...   \n",
       "\n",
       "                                        Clean_Review  \\\n",
       "0  great bed but when i first come in i request a...   \n",
       "1  do not let the website fool you i stay at the ...   \n",
       "2  just return from a one night stay at the knick...   \n",
       "\n",
       "                                           PP_Review  \\\n",
       "0  great bed first come request lake view request...   \n",
       "1  let website fool amalfi last business trip pro...   \n",
       "2  return one night knickerbocker return come now...   \n",
       "\n",
       "                                           Word_List  \\\n",
       "0  [great, bed, first, come, request, lake, view,...   \n",
       "1  [let, website, fool, amalfi, last, business, t...   \n",
       "2  [return, one, night, knickerbocker, return, co...   \n",
       "\n",
       "                                               Ngram  \n",
       "0  [great, bed, first, come, request, lake, view,...  \n",
       "1  [let, website, amalfi, last, business, trip, p...  \n",
       "2  [return, one, night, knickerbocker, return, co...  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Label</th>\n",
       "      <th>Rating</th>\n",
       "      <th>Ori_Review</th>\n",
       "      <th>Clean_Review</th>\n",
       "      <th>PP_Review</th>\n",
       "      <th>Word_List</th>\n",
       "      <th>Ngram</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>My stay at the Hotel Monaco Chicago was amazin...</td>\n",
       "      <td>my stay at the hotel monaco chicago be amaze t...</td>\n",
       "      <td>monaco amaze staff polite well poise eager giv...</td>\n",
       "      <td>[monaco, amaze, staff, polite, well, poise, ea...</td>\n",
       "      <td>[monaco, amaze, staff, polite, well, give, hel...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>For the price, you would think this would be a...</td>\n",
       "      <td>for the price you would think this would be a ...</td>\n",
       "      <td>price think top quality nowhere close service ...</td>\n",
       "      <td>[price, think, top, quality, nowhere, close, s...</td>\n",
       "      <td>[price, think, top, quality, close, service, c...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>-1</td>\n",
       "      <td>5</td>\n",
       "      <td>We just got back from 3 nights at the Sofitel....</td>\n",
       "      <td>we just get back from night at the sofitel we ...</td>\n",
       "      <td>get back night sofitel really nothing bad nega...</td>\n",
       "      <td>[get, back, night, sofitel, really, nothing, b...</td>\n",
       "      <td>[get, back, night, sofitel, really, nothing, b...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Label  Rating                                         Ori_Review  \\\n",
       "0      1       5  My stay at the Hotel Monaco Chicago was amazin...   \n",
       "1      1       1  For the price, you would think this would be a...   \n",
       "2     -1       5  We just got back from 3 nights at the Sofitel....   \n",
       "\n",
       "                                        Clean_Review  \\\n",
       "0  my stay at the hotel monaco chicago be amaze t...   \n",
       "1  for the price you would think this would be a ...   \n",
       "2  we just get back from night at the sofitel we ...   \n",
       "\n",
       "                                           PP_Review  \\\n",
       "0  monaco amaze staff polite well poise eager giv...   \n",
       "1  price think top quality nowhere close service ...   \n",
       "2  get back night sofitel really nothing bad nega...   \n",
       "\n",
       "                                           Word_List  \\\n",
       "0  [monaco, amaze, staff, polite, well, poise, ea...   \n",
       "1  [price, think, top, quality, nowhere, close, s...   \n",
       "2  [get, back, night, sofitel, really, nothing, b...   \n",
       "\n",
       "                                               Ngram  \n",
       "0  [monaco, amaze, staff, polite, well, give, hel...  \n",
       "1  [price, think, top, quality, close, service, c...  \n",
       "2  [get, back, night, sofitel, really, nothing, b...  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "df_train['Ngram'] = df_train['Word_List'].apply(lambda x: ngram.find_ngrams(x, unigrams, bigrams, trigrams))\n",
    "df_test['Ngram'] = df_test['Word_List'].apply(lambda x: ngram.find_ngrams(x, unigrams, bigrams, trigrams))\n",
    "display(df_train.head(3))\n",
    "display(df_test.head(3))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "385ce483",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train = linguistic_feature.ling_feature(df_train)\n",
    "df_test = linguistic_feature.ling_feature(df_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "1d5ee3ec",
   "metadata": {},
   "outputs": [],
   "source": [
    "bigram_words = [' '.join(bigram) for bigram in bigrams]\n",
    "trigram_words = [' '.join(trigram) for trigram in trigrams]\n",
    "new_words = unigrams + bigram_words + trigram_words\n",
    "word2ind = dict(zip(new_words, range(len(new_words))))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "5aaa4629",
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, y_train = data_prep.df2matrix(df_train, word2ind)\n",
    "X_test, y_test = data_prep.df2matrix(df_test, word2ind)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "e48d53d2",
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train_pos = X_train[X_train['Rating'] == 5]\n",
    "y_train_pos = y_train[X_train['Rating'] == 5]\n",
    "X_train_neg = X_train[X_train['Rating'] == 1]\n",
    "y_train_neg = y_train[X_train['Rating'] == 1]\n",
    "\n",
    "X_test_pos = X_test[X_test['Rating'] == 5]\n",
    "y_test_pos = y_test[X_test['Rating'] == 5]\n",
    "X_test_neg = X_test[X_test['Rating'] == 1]\n",
    "y_test_neg = y_test[X_test['Rating'] == 1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "5ed412b2",
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train_std = StandardScaler().fit_transform(X_train)\n",
    "X_test_std = StandardScaler().fit_transform(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "0c7f2169",
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train_pos_std = X_train_std[X_train['Rating'] == 5]\n",
    "X_train_neg_std = X_train_std[X_train['Rating'] == 1]\n",
    "\n",
    "X_test_pos_std = X_test_std[X_test['Rating'] == 5]\n",
    "X_test_neg_std = X_test_std[X_test['Rating'] == 1]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "275f53d9",
   "metadata": {},
   "source": [
    "### Train Model"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9b7dfcfa",
   "metadata": {},
   "source": [
    "#### 1) Random Forest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "409eac2f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "RandomForestClassifier(random_state=0)"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rf_model = RandomForestClassifier(random_state=0)\n",
    "rf_model.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c45a69bd",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "rf_model = RandomForestClassifier(criterion='gini', random_state=0)\n",
    "parameters = {'n_estimators': [100, 500],\n",
    "              'max_features': [5, 10, 20],\n",
    "              'max_depth': [20, 50],\n",
    "              'max_samples': [100, 500, None]}\n",
    "grid_search = GridSearchCV(rf_model, param_grid=parameters, cv=5, scoring='accuracy', verbose=2)\n",
    "grid_search.fit(X_train, y_train)\n",
    "print('Best Parameters: ', grid_search.best_params_)\n",
    "print('Score', grid_search.best_score_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "19464cca",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Random Forest Performance\n",
      "- Train Overall Accuracy:  99.8%\n",
      "- Test Overall Accuracy:  90.3%\n",
      "- Test Positive Accuracy:  90.4%\n",
      "- Test Negative Accuracy:  90.2%\n"
     ]
    }
   ],
   "source": [
    "rf_model = RandomForestClassifier(random_state=0, \n",
    "                                  criterion='gini', \n",
    "                                  max_depth=20, \n",
    "                                  max_features=5, \n",
    "                                  max_samples=None, \n",
    "                                  n_estimators=500)\n",
    "rf_model.fit(X_train, y_train)\n",
    "print('Random Forest Performance')\n",
    "print('- Train Overall Accuracy: ', \"{:.1%}\".format(rf_model.score(X_train, y_train)))\n",
    "print('- Test Overall Accuracy: ', \"{:.1%}\".format(rf_model.score(X_test, y_test)))\n",
    "print('- Test Positive Accuracy: ', \"{:.1%}\".format(rf_model.score(X_test_pos, y_test_pos)))\n",
    "print('- Test Negative Accuracy: ', \"{:.1%}\".format(rf_model.score(X_test_neg, y_test_neg)))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8426435f",
   "metadata": {},
   "source": [
    "#### 2) PCA + SVM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "a86548dd",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best Parameters:  {'pca__n_components': 601, 'svm__C': 3}\n",
      "Score 0.8546875\n"
     ]
    }
   ],
   "source": [
    "pca = PCA()\n",
    "svm_model = SVC()\n",
    "pipeline = Pipeline(steps=[('pca', pca), ('svm', svm_model)])\n",
    "parameters = {'svm__C': [3, 4], 'pca__n_components': list(range(1, 1001, 50))}\n",
    "grid_search = GridSearchCV(pipeline, param_grid=parameters, cv=5, scoring='accuracy')\n",
    "grid_search.fit(X_train_std, y_train)\n",
    "print('Best Parameters: ', grid_search.best_params_)\n",
    "print('Score', grid_search.best_score_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "e4aed81c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SVM Performance\n",
      "- Train Overall Accuracy:  99.8%\n",
      "- Test Overall Accuracy:  86.6%\n",
      "- Test Positive Accuracy:  84.7%\n",
      "- Test Negative Accuracy:  88.3%\n"
     ]
    }
   ],
   "source": [
    "pca = PCA(n_components=601)\n",
    "X_train_pca = pca.fit_transform(X_train_std)\n",
    "X_test_pca = pca.transform(X_test_std)\n",
    "X_test_pos_pca = pca.transform(X_test_pos_std)\n",
    "X_test_neg_pca = pca.transform(X_test_neg_std)\n",
    "\n",
    "svm_model = SVC(C=3)\n",
    "svm_model.fit(X_train_pca, y_train)\n",
    "print('SVM Performance')\n",
    "print('- Train Overall Accuracy: ', \"{:.1%}\".format(svm_model.score(X_train_pca, y_train)))\n",
    "print('- Test Overall Accuracy: ', \"{:.1%}\".format(svm_model.score(X_test_pca, y_test)))\n",
    "print('- Test Positive Accuracy: ', \"{:.1%}\".format(svm_model.score(X_test_pos_pca, y_test_pos)))\n",
    "print('- Test Negative Accuracy: ', \"{:.1%}\".format(svm_model.score(X_test_neg_pca, y_test_neg)))"
   ]
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "770021e68aae5e5bc49f7a3b6edb919a4a251fef8e7696fd47196c25263b13c4"
  },
  "kernelspec": {
   "display_name": "Python 3.8.8 64-bit ('base': conda)",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}