k-KNNAD-implementation.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# implement MDS dimension reduction algorithm and then using scikit-learn algorithm KNN to fulfill anormaly detection\n",
    "# algorithm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from pymongo import MongoClient\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn import manifold"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def _connect_mongo():\n",
    "    \"\"\" A util for making a connection to mongo \"\"\"\n",
    "    host = \"localhost\"\n",
    "    port = 27017\n",
    "    username = \"\"\n",
    "    password = \"\"\n",
    "    db = \"5gopt\"\n",
    "\n",
    "    if username and password:\n",
    "        mongo_uri = 'mongodb://%s:%s@%s:%s/%s' % (username, password, host, port, db)\n",
    "        conn = MongoClient(mongo_uri)\n",
    "    else:\n",
    "        conn = MongoClient(host, port)\n",
    "\n",
    "\n",
    "    return conn[db]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def collection_read_mongo(collection, query={}, no_id = True):\n",
    "    db = _connect_mongo()\n",
    "    cursor = db[collection].find(query)\n",
    "    df = pd.DataFrame(list(cursor))\n",
    "\n",
    "    if no_id:\n",
    "        try:\n",
    "            del df[\"_id\"]\n",
    "        except:\n",
    "            pass\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# df = collection_read_mongo(collection=\"training\")\n",
    "\n",
    "df = pd.read_csv(\"training.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Time</th>\n",
       "      <th>UserID</th>\n",
       "      <th>LocationX</th>\n",
       "      <th>LocationY</th>\n",
       "      <th>RSRP1</th>\n",
       "      <th>RSRQ1</th>\n",
       "      <th>RSRP2</th>\n",
       "      <th>RSRQ2</th>\n",
       "      <th>RSRP3</th>\n",
       "      <th>RSRQ3</th>\n",
       "      <th>RSRP4</th>\n",
       "      <th>RSRQ4</th>\n",
       "      <th>Label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.2</td>\n",
       "      <td>1</td>\n",
       "      <td>1165.135603</td>\n",
       "      <td>157.155659</td>\n",
       "      <td>-71.781608</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-76.114469</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-76.421107</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-77.007771</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.2</td>\n",
       "      <td>2</td>\n",
       "      <td>18.101453</td>\n",
       "      <td>97.904367</td>\n",
       "      <td>-69.099606</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-71.002579</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-74.648307</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-76.255106</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.2</td>\n",
       "      <td>3</td>\n",
       "      <td>539.327246</td>\n",
       "      <td>151.255141</td>\n",
       "      <td>-67.527468</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-69.966291</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-70.339135</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-74.413793</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Time  UserID    LocationX   LocationY      RSRP1  RSRQ1      RSRP2  RSRQ2  \\\n",
       "0   0.2       1  1165.135603  157.155659 -71.781608    NaN -76.114469    NaN   \n",
       "1   0.2       2    18.101453   97.904367 -69.099606    NaN -71.002579    NaN   \n",
       "2   0.2       3   539.327246  151.255141 -67.527468    NaN -69.966291    NaN   \n",
       "\n",
       "       RSRP3  RSRQ3      RSRP4  RSRQ4  Label  \n",
       "0 -76.421107    NaN -77.007771    NaN      0  \n",
       "1 -74.648307    NaN -76.255106    NaN      0  \n",
       "2 -70.339135    NaN -74.413793    NaN      0  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[:3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "elapse time:  721.722332239151\n"
     ]
    }
   ],
   "source": [
    "df.dropna(axis=0, how=\"any\", inplace=True)\n",
    "df.drop(df.columns[:4], axis=1, inplace=True)\n",
    "y = df[\"Label\"]\n",
    "X = df.drop(df.columns[-1], axis=1)\n",
    "# apply MDS dimension reduction to signalDF, with only 3 dimensions left\n",
    "mds = manifold.MDS(3, max_iter=200, n_init=1)\n",
    "import time\n",
    "startTime = time.time()\n",
    "threeDimSig = mds.fit_transform(X)\n",
    "print(\"elapse time: \", (time.time()-startTime))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "data[:3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "type(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "type(data[\"Time\"][0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "testData = data[:10000]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# for each user A, we pick the top 4 highest RSRP, RSRQ value at a time point t.\n",
    "identiferList = list()\n",
    "pd.DataFrame(columns=[\"Time\", \"UserID\"])\n",
    "signalList = list()\n",
    "\n",
    "for ident, group in testData.groupby([\"Time\", \"UserID\"]):\n",
    "#     iterate the group object, and pick the top 4 highest rsrp value and then its corresponsing rsrq value in that row\n",
    "#     print(type(group))\n",
    "    top4Row = group.sort_values(by=[\"RSRP\"])[:4]\n",
    "#     print(top4Row)\n",
    "    try:\n",
    "        signalRow = [top4Row.iloc[0][\"RSRP\"], top4Row.iloc[0][\"RSRQ\"], top4Row.iloc[1][\"RSRP\"], top4Row.iloc[1][\"RSRQ\"],top4Row.iloc[2][\"RSRP\"], top4Row.iloc[2][\"RSRQ\"], top4Row.iloc[3][\"RSRP\"], top4Row.iloc[3][\"RSRQ\"]]\n",
    "        signalList.append(signalRow)\n",
    "        ident = (round(ident[0], 1), ident[1])\n",
    "        identiferList.append(ident)\n",
    "    except:\n",
    "        pass\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "signalDF = pd.DataFrame(signalList, columns=[\"RSRP_1\",\"RSRQ_1\",\"RSRP_2\",\"RSRQ_2\",\"RSRP_3\",\"RSRQ_3\",\"RSRP_4\",\"RSRQ_4\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "signalDF[:3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "identiferDF = pd.DataFrame(identiferList, columns=[\"Time\", \"UserID\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "identiferDF[:3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "identiferDF.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "signalDF.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# create a new Dataframe with merging two exist Dataframe, and length of these two dataframe is same\n",
    "referenceDF = identiferDF.merge(signalDF, left_index=True, right_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "referenceDF[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# apply MDS dimension reduction to signalDF, with only 3 dimensions left\n",
    "mds = manifold.MDS(3, max_iter=200, n_init=1)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# before applying MDS, need to remove rows which contain NaN value, so that MDS can run correctly.\n",
    "import time\n",
    "signalDF = signalDF.dropna(axis=0, how=\"any\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "signalDF[:3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "\n",
    "startTime = time.time()\n",
    "threeDimSig = mds.fit_transform(signalDF)\n",
    "print(\"elapse time: \", (time.time()-startTime))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "threeDimSigDF = pd.DataFrame(threeDimSig)\n",
    "threeDimSigDF[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# create a new Dataframe with merging two exist Dataframe, and length of these two dataframe is same\n",
    "referenceDF = identiferDF.merge(threeDimSigDF, left_index=True, right_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# implement k-KNNAD algorithm, the reason we don't implement LOFAD algorithm is k-KNNAD can achieve higher accuracy,\n",
    "#  auc score is about 0.91, while LOFAD is 0.85 according to the paper \n",
    "# \"a cell outage management framework for dense heterogeneous networks\"\n",
    "\n",
    "referenceDF[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "mds3D = MDS(n_components=3, max_iter=1000, eps=1e-7, random_state=seed, dissimilarity=\"euclidean\", n_jobs=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "out = mds3D.fit_transform(cleanedWantedDF)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "out[:3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from mpl_toolkits import mplot3d\n",
    "ax = plt.axes(projection=\"3d\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "ax.scatter3D(out[:,0], out[:,1], out[:,2])\n",
    "ax.view_init(azim=70, elev=50)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "\n",
    "plt.scatter(out[:, 0], out[:, 1], out[:,2])\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# after getting reduced-dimensional data, then calculate the nearest neighbor iterately\n",
    "# store the embedded measurement in csv file, reference database\n",
    "np.savetxt('embedding.csv', out, delimiter=\",\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from IPython.display import Latex\n",
    "Latex(r\"\"\"\\begin{eqnarray}\n",
    "\\nabla \\times \\vec{\\mathbf{B}} -\\, \\frac1c\\, \\frac{\\partial\\vec{\\mathbf{E}}}{\\partial t} & = \\frac{4\\pi}{c}\\vec{\\mathbf{j}} \\\\\n",
    "\\nabla \\cdot \\vec{\\mathbf{E}} & = 4 \\pi \\rho \\\\\n",
    "\\nabla \\times \\vec{\\mathbf{E}}\\, +\\, \\frac1c\\, \\frac{\\partial\\vec{\\mathbf{B}}}{\\partial t} & = \\vec{\\mathbf{0}} \\\\\n",
    "\\nabla \\cdot \\vec{\\mathbf{B}} & = 0 \n",
    "\\end{eqnarray}\"\"\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "Latex(r\"\"\"\\begin{eqnarray}\n",
    "D_{k-NNAD}(x_{i}, k, D_{M}) = \\frac{1}{N_{tr}} \\sum_{N_{tr}}^{i=1} I(d_{t} <= d_{i}) \\\\\n",
    "\\end{eqnarry}\"\"\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# use cross validation method, k-fold, k = 10\n",
    "from sklearn.model_selection import KFold\n",
    "k = 10\n",
    "kf = KFold(n_splits = k)\n",
    "indicesList = list()\n",
    "for train, test in kf.split(out):\n",
    "    indicesList.append((train, test))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "\n",
    "N_tr = len(indicesList[0][0])+len(indicesList[0][1])\n",
    "# k means the top k nearest neighbor\n",
    "k = 50 \n",
    "trainData = np.take(out, indicesList[0][0])\n",
    "testData = np.take(out, indicesList[0][1])\n",
    "\n",
    "# for each data in test dataset, calculate distance between it and each data point in train dataset\n",
    "# pick the top k(k=50)\n",
    "for i in testData:\n",
    "    distList = list()\n",
    "    for j in trainData:\n",
    "        distance = np.sqrt(np.sum((i-j)**2, axis=1))\n",
    "        distList.append(distance)\n",
    "#     sort distList and then pick the top k(k =50) elems\n",
    "    distList = sorted(distList, reverse=True)\n",
    "    topKDist = distList[:k]\n",
    "    \n",
    "        \n",
    "        \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "pos.get_params()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "mds.get_params()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def _connect_mongo():\n",
    "    \"\"\" A util for making a connection to mongo \"\"\"\n",
    "    host = \"localhost\"\n",
    "    port = 27017\n",
    "    username = \"\"\n",
    "    password = \"\"\n",
    "    db = \"5gopt\"\n",
    "\n",
    "    if username and password:\n",
    "        mongo_uri = 'mongodb://%s:%s@%s:%s/%s' % (username, password, host, port, db)\n",
    "        conn = MongoClient(mongo_uri)\n",
    "    else:\n",
    "        conn = MongoClient(host, port)\n",
    "\n",
    "\n",
    "    return conn[db]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def collection_read_mongo(collection, query={}, no_id = True):\n",
    "    db = _connect_mongo()\n",
    "    cursor = db[collection].find(query)\n",
    "    df = pd.DataFrame(list(cursor))\n",
    "\n",
    "    if no_id:\n",
    "        try:\n",
    "            del df[\"_id\"]\n",
    "        except:\n",
    "            pass\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "data = collection_read_mongo(collection=\"main_file_with_UserTHR\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "data[:3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "testData = data[:1000]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "groupedData = "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "max(data[\"Throughput\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "groupedData = data.groupby([\"Time\", \"CellID\"])[[\"Throughput\"]].sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "type(groupedData)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "groupedData[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df = groupedData.to_frame"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "groupedData[36.2, 2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "groupedData[\"Throughput\"]"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}