TxGNN_Demo.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using backend: pytorch\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found local copy...\n",
      "Downloading...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5fe5a61560b64695a9184deca8fc4c7f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0.00/8.89M [00:00<?, ?iB/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Done!\n",
      "Downloading...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6ab94b20e2164279a02e95b3154f3df1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0.00/387M [00:00<?, ?iB/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Done!\n",
      "First time usage... Mapping RxData raw KG to directed csv... it takes several minutes...\n",
      "Iterating over relations...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a63a4504709d4d36a11d9f7927c25ee5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/30 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Iterating over node types...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fe6b6a98b8b94ab295489b9ce3bba96d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/10 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Creating splits... it takes several minutes...\n",
      "Creating DGL graph....\n",
      "Done!\n"
     ]
    }
   ],
   "source": [
    "from txgnn import TxData, TxGNN, TxEval\n",
    "\n",
    "TxData = TxData(data_folder_path = '/n/scratch3/users/k/kh278/kg/')\n",
    "TxData.prepare_split(split = 'complex_disease', seed = 42, no_kg = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "TxGNN = TxGNN(data = TxData, \n",
    "              weight_bias_track = False,\n",
    "              proj_name = 'TxGNN',\n",
    "              exp_name = 'TxGNN'\n",
    "              )\n",
    "\n",
    "# to load a pretrained model: \n",
    "# TxGNN.load_pretrained('./model_ckpt')\n",
    "\n",
    "TxGNN.model_initialize(n_hid = 100, \n",
    "                      n_inp = 100, \n",
    "                      n_out = 100, \n",
    "                      proto = True,\n",
    "                      proto_num = 3,\n",
    "                      attention = False,\n",
    "                      sim_measure = 'all_nodes_profile',\n",
    "                      bert_measure = 'disease_name',\n",
    "                      agg_measure = 'rarity',\n",
    "                      num_walks = 200,\n",
    "                      walk_mode = 'bit',\n",
    "                      path_length = 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## here we did not run this, since the output is too long to fit into the notebook\n",
    "TxGNN.pretrain(n_epoch = 2, \n",
    "               learning_rate = 1e-3,\n",
    "               batch_size = 1024, \n",
    "               train_print_per_n = 20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 0 LR: 0.00050 Loss 0.6938, Train Micro AUROC 0.5109 Train Micro AUPRC 0.5127 Train Macro AUROC 0.5019 Train Macro AUPRC 0.5073\n",
      "----- AUROC Performance in Each Relation -----\n",
      "('drug', 'contraindication', 'disease'): 0.5349347764149285\n",
      "('drug', 'indication', 'disease'): 0.5475017777377161\n",
      "('drug', 'off-label use', 'disease'): 0.4750877230935641\n",
      "('disease', 'rev_contraindication', 'drug'): 0.5200441594466088\n",
      "('disease', 'rev_indication', 'drug'): 0.42414272145771154\n",
      "('disease', 'rev_off-label use', 'drug'): 0.5098947539210383\n",
      "----- AUPRC Performance in Each Relation -----\n",
      "('drug', 'contraindication', 'disease'): 0.5439511651288017\n",
      "('drug', 'indication', 'disease'): 0.551160435487897\n",
      "('drug', 'off-label use', 'disease'): 0.4725924166335391\n",
      "('disease', 'rev_contraindication', 'drug'): 0.5207788369454536\n",
      "('disease', 'rev_indication', 'drug'): 0.4539515055664601\n",
      "('disease', 'rev_off-label use', 'drug'): 0.5012481020151309\n",
      "----------------------------------------------\n",
      "Validation.....\n",
      "Epoch: 0 LR: 0.00050 Validation Loss 0.6929,  Validation Micro AUROC 0.5235 Validation Micro AUPRC 0.5232 Validation Macro AUROC 0.5299 Validation Macro AUPRC 0.5223 (Best Macro AUROC 0.5299)\n",
      "----- AUROC Performance in Each Relation -----\n",
      "('drug', 'contraindication', 'disease'): 0.5462669878118773\n",
      "('drug', 'indication', 'disease'): 0.579094545019629\n",
      "('drug', 'off-label use', 'disease'): 0.52843625279593\n",
      "('disease', 'rev_contraindication', 'drug'): 0.5162123733177675\n",
      "('disease', 'rev_indication', 'drug'): 0.4857158331258476\n",
      "('disease', 'rev_off-label use', 'drug'): 0.5233761677119425\n",
      "----- AUPRC Performance in Each Relation -----\n",
      "('drug', 'contraindication', 'disease'): 0.5284480700097016\n",
      "('drug', 'indication', 'disease'): 0.5599285087410649\n",
      "('drug', 'off-label use', 'disease'): 0.5109112572657095\n",
      "('disease', 'rev_contraindication', 'drug'): 0.5176517996384884\n",
      "('disease', 'rev_indication', 'drug'): 0.5032478920915975\n",
      "('disease', 'rev_off-label use', 'drug'): 0.5138745650969\n",
      "----------------------------------------------\n",
      "Epoch: 5 LR: 0.00050 Loss 0.6889, Train Micro AUROC 0.5626 Train Micro AUPRC 0.5641 Train Macro AUROC 0.5451 Train Macro AUPRC 0.5400\n",
      "----- AUROC Performance in Each Relation -----\n",
      "('drug', 'contraindication', 'disease'): 0.5852535608525192\n",
      "('drug', 'indication', 'disease'): 0.576565752926173\n",
      "('drug', 'off-label use', 'disease'): 0.4750885884261763\n",
      "('disease', 'rev_contraindication', 'drug'): 0.5540898457579201\n",
      "('disease', 'rev_indication', 'drug'): 0.5844817352055499\n",
      "('disease', 'rev_off-label use', 'drug'): 0.4953451595457004\n",
      "----- AUPRC Performance in Each Relation -----\n",
      "('drug', 'contraindication', 'disease'): 0.5853801035105486\n",
      "('drug', 'indication', 'disease'): 0.5694905248706956\n",
      "('drug', 'off-label use', 'disease'): 0.4763876836694141\n",
      "('disease', 'rev_contraindication', 'drug'): 0.5458827057813597\n",
      "('disease', 'rev_indication', 'drug'): 0.5727108380539363\n",
      "('disease', 'rev_off-label use', 'drug'): 0.49000961760804385\n",
      "----------------------------------------------\n",
      "Epoch: 10 LR: 0.00050 Loss 0.6851, Train Micro AUROC 0.5775 Train Micro AUPRC 0.5769 Train Macro AUROC 0.5673 Train Macro AUPRC 0.5589\n",
      "----- AUROC Performance in Each Relation -----\n",
      "('drug', 'contraindication', 'disease'): 0.5947743689635581\n",
      "('drug', 'indication', 'disease'): 0.6011430269281359\n",
      "('drug', 'off-label use', 'disease'): 0.4903127095727421\n",
      "('disease', 'rev_contraindication', 'drug'): 0.5679930937412638\n",
      "('disease', 'rev_indication', 'drug'): 0.633856013260421\n",
      "('disease', 'rev_off-label use', 'drug'): 0.5155235262303948\n",
      "----- AUPRC Performance in Each Relation -----\n",
      "('drug', 'contraindication', 'disease'): 0.5989037198168043\n",
      "('drug', 'indication', 'disease'): 0.5873969054236162\n",
      "('drug', 'off-label use', 'disease'): 0.4909531845932579\n",
      "('disease', 'rev_contraindication', 'drug'): 0.5535179369107939\n",
      "('disease', 'rev_indication', 'drug'): 0.615812028221359\n",
      "('disease', 'rev_off-label use', 'drug'): 0.5068521651632782\n",
      "----------------------------------------------\n",
      "Epoch: 15 LR: 0.00050 Loss 0.6813, Train Micro AUROC 0.5903 Train Micro AUPRC 0.5905 Train Macro AUROC 0.5784 Train Macro AUPRC 0.5690\n",
      "----- AUROC Performance in Each Relation -----\n",
      "('drug', 'contraindication', 'disease'): 0.6046423020712154\n",
      "('drug', 'indication', 'disease'): 0.6129430322696954\n",
      "('drug', 'off-label use', 'disease'): 0.510339534883721\n",
      "('disease', 'rev_contraindication', 'drug'): 0.5734694118386404\n",
      "('disease', 'rev_indication', 'drug'): 0.6437518528533943\n",
      "('disease', 'rev_off-label use', 'drug'): 0.525255922120065\n",
      "----- AUPRC Performance in Each Relation -----\n",
      "('drug', 'contraindication', 'disease'): 0.6111205319380861\n",
      "('drug', 'indication', 'disease'): 0.5991445557229728\n",
      "('drug', 'off-label use', 'disease'): 0.5043737838078705\n",
      "('disease', 'rev_contraindication', 'drug'): 0.5596717293236961\n",
      "('disease', 'rev_indication', 'drug'): 0.6286739920251833\n",
      "('disease', 'rev_off-label use', 'drug'): 0.5112296189418617\n",
      "----------------------------------------------\n",
      "Epoch: 20 LR: 0.00050 Loss 0.6771, Train Micro AUROC 0.6022 Train Micro AUPRC 0.6026 Train Macro AUROC 0.5877 Train Macro AUPRC 0.5775\n",
      "----- AUROC Performance in Each Relation -----\n",
      "('drug', 'contraindication', 'disease'): 0.620410356865875\n",
      "('drug', 'indication', 'disease'): 0.6157521583238187\n",
      "('drug', 'off-label use', 'disease'): 0.5313520822065981\n",
      "('disease', 'rev_contraindication', 'drug'): 0.5776603940652365\n",
      "('disease', 'rev_indication', 'drug'): 0.6434780728989311\n",
      "('disease', 'rev_off-label use', 'drug'): 0.5374131963223364\n",
      "----- AUPRC Performance in Each Relation -----\n",
      "('drug', 'contraindication', 'disease'): 0.6277603660899511\n",
      "('drug', 'indication', 'disease'): 0.6013951829845593\n",
      "('drug', 'off-label use', 'disease'): 0.5185372395906721\n",
      "('disease', 'rev_contraindication', 'drug'): 0.5631574167202016\n",
      "('disease', 'rev_indication', 'drug'): 0.6340424812424496\n",
      "('disease', 'rev_off-label use', 'drug'): 0.5202987044160109\n",
      "----------------------------------------------\n",
      "Validation.....\n",
      "Epoch: 20 LR: 0.00050 Validation Loss 0.6844,  Validation Micro AUROC 0.5791 Validation Micro AUPRC 0.5679 Validation Macro AUROC 0.5516 Validation Macro AUPRC 0.5408 (Best Macro AUROC 0.5516)\n",
      "----- AUROC Performance in Each Relation -----\n",
      "('drug', 'contraindication', 'disease'): 0.6153962023726334\n",
      "('drug', 'indication', 'disease'): 0.5832091553464736\n",
      "('drug', 'off-label use', 'disease'): 0.5016282180606113\n",
      "('disease', 'rev_contraindication', 'drug'): 0.5510605652586977\n",
      "('disease', 'rev_indication', 'drug'): 0.5668752054484362\n",
      "('disease', 'rev_off-label use', 'drug'): 0.4916560677163283\n",
      "----- AUPRC Performance in Each Relation -----\n",
      "('drug', 'contraindication', 'disease'): 0.5991434327210107\n",
      "('drug', 'indication', 'disease'): 0.5518445092218665\n",
      "('drug', 'off-label use', 'disease'): 0.49813432015807163\n",
      "('disease', 'rev_contraindication', 'drug'): 0.5413906938529918\n",
      "('disease', 'rev_indication', 'drug'): 0.5693761911164258\n",
      "('disease', 'rev_off-label use', 'drug'): 0.48515413698204796\n",
      "----------------------------------------------\n",
      "Epoch: 25 LR: 0.00050 Loss 0.6726, Train Micro AUROC 0.6156 Train Micro AUPRC 0.6128 Train Macro AUROC 0.5952 Train Macro AUPRC 0.5840\n",
      "----- AUROC Performance in Each Relation -----\n",
      "('drug', 'contraindication', 'disease'): 0.6394245568766965\n",
      "('drug', 'indication', 'disease'): 0.6199004383417129\n",
      "('drug', 'off-label use', 'disease'): 0.5366962682531098\n",
      "('disease', 'rev_contraindication', 'drug'): 0.5836349116571526\n",
      "('disease', 'rev_indication', 'drug'): 0.6439810892107178\n",
      "('disease', 'rev_off-label use', 'drug'): 0.5473875608436992\n",
      "----- AUPRC Performance in Each Relation -----\n",
      "('drug', 'contraindication', 'disease'): 0.6427936868163125\n",
      "('drug', 'indication', 'disease'): 0.6052729815037041\n",
      "('drug', 'off-label use', 'disease'): 0.52231299426086\n",
      "('disease', 'rev_contraindication', 'drug'): 0.5685857127274868\n",
      "('disease', 'rev_indication', 'drug'): 0.6345257470821545\n",
      "('disease', 'rev_off-label use', 'drug'): 0.530766184825923\n",
      "----------------------------------------------\n",
      "Testing...\n",
      "Testing Loss 0.6839 Testing Micro AUROC 0.5731 Testing Micro AUPRC 0.5624 Testing Macro AUROC 0.5595 Testing Macro AUPRC 0.5622\n",
      "----- AUROC Performance in Each Relation -----\n",
      "('disease', 'rev_contraindication', 'drug'): 0.5379341823437893\n",
      "('disease', 'rev_indication', 'drug'): 0.5546474849505152\n",
      "('disease', 'rev_off-label use', 'drug'): 0.5351516052318669\n",
      "('drug', 'contraindication', 'disease'): 0.6124091427126344\n",
      "('drug', 'indication', 'disease'): 0.6302499744923987\n",
      "('drug', 'off-label use', 'disease'): 0.4865487514863258\n",
      "----- AUPRC Performance in Each Relation -----\n",
      "('disease', 'rev_contraindication', 'drug'): 0.5293769910520773\n",
      "('disease', 'rev_indication', 'drug'): 0.5442612049842734\n",
      "('disease', 'rev_off-label use', 'drug'): 0.5330106792677878\n",
      "('drug', 'contraindication', 'disease'): 0.5986166731978727\n",
      "('drug', 'indication', 'disease'): 0.6382368517760889\n",
      "('drug', 'off-label use', 'disease'): 0.5295611297196345\n",
      "----------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "## here as a demo, the n_epoch is set to 30. Change it to n_epoch = 500 when you use it\n",
    "TxGNN.finetune(n_epoch = 30, \n",
    "               learning_rate = 5e-4,\n",
    "               train_print_per_n = 5,\n",
    "               valid_per_n = 20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "TxGNN.save_model('./model_ckpt')\n",
    "TxGNN.load_pretrained('./model_ckpt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "TxGNN.train_graphmask(relation = 'indication',\n",
    "                      learning_rate = 3e-4,\n",
    "                      allowance = 0.005,\n",
    "                      epochs_per_layer = 3,\n",
    "                      penalty_scaling = 1,\n",
    "                      epochs_per_layer = 5,\n",
    "                      valid_per_n = 20)\n",
    "\n",
    "output = TxGNN.retrieve_save_gates('./model_ckpt')\n",
    "TxGNN.save_graphmask_model('./graphmask_model_ckpt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "from txgnn import TxEval\n",
    "TxEval = TxEval(model = TxGNN)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluating relation: contraindication\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b80e7e5641254380a0b2c2965fcdec6d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=54.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "---------\n",
      "AUROC mean:  0.4827983690906781\n",
      "AUROC std:  0.1710535011763153\n",
      "---------\n",
      "---------\n",
      "AUPRC mean:  0.023629636234423025\n",
      "AUPRC std:  0.036442407633213866\n",
      "---------\n",
      "---------\n",
      "Accuracy mean:  0.974648837277499\n",
      "Accuracy std:  0.04358800960119679\n",
      "---------\n",
      "---------\n",
      "Sensitivity mean:  0.0\n",
      "Sensitivity std:  0.0\n",
      "---------\n",
      "---------\n",
      "Specificity mean:  1.0\n",
      "Specificity std:  0.0\n",
      "---------\n",
      "---------\n",
      "F1 mean:  0.0\n",
      "F1 std:  0.0\n",
      "---------\n",
      "---------\n",
      "PPV mean:  nan\n",
      "PPV std:  nan\n",
      "---------\n",
      "---------\n",
      "NPV mean:  0.974648837277499\n",
      "NPV std:  0.04358800960119679\n",
      "---------\n",
      "---------\n",
      "FPR mean:  0.0\n",
      "FPR std:  0.0\n",
      "---------\n",
      "---------\n",
      "FNR mean:  1.0\n",
      "FNR std:  0.0\n",
      "---------\n",
      "---------\n",
      "FDR mean:  nan\n",
      "FDR std:  nan\n",
      "---------\n",
      "---------\n",
      "# of Pos mean:  32.01851851851852\n",
      "# of Pos std:  55.05165612631155\n",
      "---------\n",
      "---------\n",
      "Recall@1% mean:  0.00129426532934482\n",
      "Recall@1% std:  0.004735312292905053\n",
      "---------\n",
      "---------\n",
      "Recall_Random@1% mean:  0.009496868416327983\n",
      "Recall_Random@1% std:  0.0031843050687695726\n",
      "---------\n",
      "---------\n",
      "Enrichment@1% mean:  0.13879378822533134\n",
      "Enrichment@1% std:  0.5155583324698909\n",
      "---------\n",
      "---------\n",
      "MRR@1% mean:  0.027623456790123457\n",
      "MRR@1% std:  0.094930991749981\n",
      "---------\n",
      "---------\n",
      "AP@1% mean:  0.02659465020576132\n",
      "AP@1% std:  0.09186380138670512\n",
      "---------\n",
      "---------\n",
      "Recall@5% mean:  0.014459524325020089\n",
      "Recall@5% std:  0.02962244293134301\n",
      "---------\n",
      "---------\n",
      "Recall_Random@5% mean:  0.05022658345318769\n",
      "Recall_Random@5% std:  0.007725713266865586\n",
      "---------\n",
      "---------\n",
      "Enrichment@5% mean:  0.28894271866027454\n",
      "Enrichment@5% std:  0.5868289806536506\n",
      "---------\n",
      "---------\n",
      "MRR@5% mean:  0.03327879563128982\n",
      "MRR@5% std:  0.094036116052397\n",
      "---------\n",
      "---------\n",
      "AP@5% mean:  0.0203188731797839\n",
      "AP@5% std:  0.04479226290630266\n",
      "---------\n",
      "---------\n",
      "Recall@10% mean:  0.07214201933124294\n",
      "Recall@10% std:  0.19202490477915607\n",
      "---------\n",
      "---------\n",
      "Recall_Random@10% mean:  0.10014631967458737\n",
      "Recall_Random@10% std:  0.009184361902598102\n",
      "---------\n",
      "---------\n",
      "Enrichment@10% mean:  0.7606310009665149\n",
      "Enrichment@10% std:  2.1358690120853674\n",
      "---------\n",
      "---------\n",
      "MRR@10% mean:  0.03447403039687176\n",
      "MRR@10% std:  0.09367551922639453\n",
      "---------\n",
      "---------\n",
      "AP@10% mean:  0.019252640214614487\n",
      "AP@10% std:  0.03613457694211353\n",
      "---------\n",
      "---------\n",
      "Recall@10 mean:  0.00129426532934482\n",
      "Recall@10 std:  0.004735312292905053\n",
      "---------\n",
      "---------\n",
      "Recall_Random@10 mean:  0.00793897095130406\n",
      "Recall_Random@10 std:  0.00248899732659056\n",
      "---------\n",
      "---------\n",
      "Enrichment@10 mean:  0.16849820395616966\n",
      "Enrichment@10 std:  0.6273341684069486\n",
      "---------\n",
      "---------\n",
      "MRR@10 mean:  0.027623456790123457\n",
      "MRR@10 std:  0.094930991749981\n",
      "---------\n",
      "---------\n",
      "AP@10 mean:  0.02659465020576132\n",
      "AP@10 std:  0.09186380138670512\n",
      "---------\n",
      "---------\n",
      "Recall@50 mean:  0.011867745121870797\n",
      "Recall@50 std:  0.027129619673790403\n",
      "---------\n",
      "---------\n",
      "Recall_Random@50 mean:  0.04008526319150218\n",
      "Recall_Random@50 std:  0.007612411863092311\n",
      "---------\n",
      "---------\n",
      "Enrichment@50 mean:  0.29263390570032755\n",
      "Enrichment@50 std:  0.6618921781175742\n",
      "---------\n",
      "---------\n",
      "MRR@50 mean:  0.03259778398536589\n",
      "MRR@50 std:  0.09420777842684455\n",
      "---------\n",
      "---------\n",
      "AP@50 mean:  0.02042982945782197\n",
      "AP@50 std:  0.04770458769819987\n",
      "---------\n",
      "---------\n",
      "Recall@100 mean:  0.04614684565945143\n",
      "Recall@100 std:  0.1428573231684311\n",
      "---------\n",
      "---------\n",
      "Recall_Random@100 mean:  0.07945713706374243\n",
      "Recall_Random@100 std:  0.008119063689330196\n",
      "---------\n",
      "---------\n",
      "Enrichment@100 mean:  0.6451371616663898\n",
      "Enrichment@100 std:  2.197048218887136\n",
      "---------\n",
      "---------\n",
      "MRR@100 mean:  0.03397931932031925\n",
      "MRR@100 std:  0.09383254985799716\n",
      "---------\n",
      "---------\n",
      "AP@100 mean:  0.019967913220195893\n",
      "AP@100 std:  0.04021788238214835\n",
      "---------\n",
      "Evaluating relation: indication\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b5c16261bf044ea88b2c56ae6ad97c5c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=66.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "---------\n",
      "AUROC mean:  0.43292636186911265\n",
      "AUROC std:  0.21921846534384767\n",
      "---------\n",
      "---------\n",
      "AUPRC mean:  0.006045431487876577\n",
      "AUPRC std:  0.01036997622355921\n",
      "---------\n",
      "---------\n",
      "Accuracy mean:  0.9958356468628541\n",
      "Accuracy std:  0.006893993307483263\n",
      "---------\n",
      "---------\n",
      "Sensitivity mean:  0.0\n",
      "Sensitivity std:  0.0\n",
      "---------\n",
      "---------\n",
      "Specificity mean:  1.0\n",
      "Specificity std:  0.0\n",
      "---------\n",
      "---------\n",
      "F1 mean:  0.0\n",
      "F1 std:  0.0\n",
      "---------\n",
      "---------\n",
      "PPV mean:  nan\n",
      "PPV std:  nan\n",
      "---------\n",
      "---------\n",
      "NPV mean:  0.9958356468628541\n",
      "NPV std:  0.006893993307483263\n",
      "---------\n",
      "---------\n",
      "FPR mean:  0.0\n",
      "FPR std:  0.0\n",
      "---------\n",
      "---------\n",
      "FNR mean:  1.0\n",
      "FNR std:  0.0\n",
      "---------\n",
      "---------\n",
      "FDR mean:  nan\n",
      "FDR std:  nan\n",
      "---------\n",
      "---------\n",
      "# of Pos mean:  7.5\n",
      "# of Pos std:  12.41608194677737\n",
      "---------\n",
      "---------\n",
      "Recall@1% mean:  0.0013331591779867642\n",
      "Recall@1% std:  0.006319964413607548\n",
      "---------\n",
      "---------\n",
      "Recall_Random@1% mean:  0.010273823015917559\n",
      "Recall_Random@1% std:  0.0032496026468089674\n",
      "---------\n",
      "---------\n",
      "Enrichment@1% mean:  nan\n",
      "Enrichment@1% std:  nan\n",
      "---------\n",
      "---------\n",
      "MRR@1% mean:  0.031144781144781142\n",
      "MRR@1% std:  0.17140535827717526\n",
      "---------\n",
      "---------\n",
      "AP@1% mean:  0.02446028916617152\n",
      "AP@1% std:  0.13903792689108074\n",
      "---------\n",
      "---------\n",
      "Recall@5% mean:  0.020825496342737723\n",
      "Recall@5% std:  0.123790328427321\n",
      "---------\n",
      "---------\n",
      "Recall_Random@5% mean:  0.049684295013387954\n",
      "Recall_Random@5% std:  0.005962749908639804\n",
      "---------\n",
      "---------\n",
      "Enrichment@5% mean:  0.39158214509466766\n",
      "Enrichment@5% std:  2.296330461295242\n",
      "---------\n",
      "---------\n",
      "MRR@5% mean:  0.0328741677425888\n",
      "MRR@5% std:  0.17131355525236838\n",
      "---------\n",
      "---------\n",
      "AP@5% mean:  0.016582891079795103\n",
      "AP@5% std:  0.07858720592360213\n",
      "---------\n",
      "---------\n",
      "Recall@10% mean:  0.04836476242398281\n",
      "Recall@10% std:  0.17669675806679766\n",
      "---------\n",
      "---------\n",
      "Recall_Random@10% mean:  0.10129925851075819\n",
      "Recall_Random@10% std:  0.006627600700416985\n",
      "---------\n",
      "---------\n",
      "Enrichment@10% mean:  0.4867953772925046\n",
      "Enrichment@10% std:  1.7979007225718213\n",
      "---------\n",
      "---------\n",
      "MRR@10% mean:  0.03343598286971356\n",
      "MRR@10% std:  0.17121755333331726\n",
      "---------\n",
      "---------\n",
      "AP@10% mean:  0.01109969909480007\n",
      "AP@10% std:  0.0459071455321114\n",
      "---------\n",
      "---------\n",
      "Recall@10 mean:  0.0005529432253570185\n",
      "Recall@10 std:  0.0033415170734230103\n",
      "---------\n",
      "---------\n",
      "Recall_Random@10 mean:  0.0055174552595105555\n",
      "Recall_Random@10 std:  0.0020375799191167453\n",
      "---------\n",
      "---------\n",
      "Enrichment@10 mean:  nan\n",
      "Enrichment@10 std:  nan\n",
      "---------\n",
      "---------\n",
      "MRR@10 mean:  0.030303030303030304\n",
      "MRR@10 std:  0.1714198257421934\n",
      "---------\n",
      "---------\n",
      "AP@10 mean:  0.030303030303030304\n",
      "AP@10 std:  0.1714198257421934\n",
      "---------\n",
      "---------\n",
      "Recall@50 mean:  0.005121037965865553\n",
      "Recall@50 std:  0.02327205426750195\n",
      "---------\n",
      "---------\n",
      "Recall_Random@50 mean:  0.0275210362917599\n",
      "Recall_Random@50 std:  0.004345581092724046\n",
      "---------\n",
      "---------\n",
      "Enrichment@50 mean:  0.17912891801427291\n",
      "Enrichment@50 std:  0.8157581978756483\n",
      "---------\n",
      "---------\n",
      "MRR@50 mean:  0.03266373003215109\n",
      "MRR@50 std:  0.1713452745551583\n",
      "---------\n",
      "---------\n",
      "AP@50 mean:  0.026338089728182607\n",
      "AP@50 std:  0.13913575026621883\n",
      "---------\n",
      "---------\n",
      "Recall@100 mean:  0.03696185997910136\n",
      "Recall@100 std:  0.17222995827103246\n",
      "---------\n",
      "---------\n",
      "Recall_Random@100 mean:  0.05565316385097494\n",
      "Recall_Random@100 std:  0.005952475125860074\n",
      "---------\n",
      "---------\n",
      "Enrichment@100 mean:  0.6776946220151775\n",
      "Enrichment@100 std:  3.2069023412868867\n",
      "---------\n",
      "---------\n",
      "MRR@100 mean:  0.033025682894103944\n",
      "MRR@100 std:  0.17128883366660938\n",
      "---------\n",
      "---------\n",
      "AP@100 mean:  0.014027896123484358\n",
      "AP@100 std:  0.06407317515540195\n",
      "---------\n",
      "Evaluating relation: off-label use\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "52388211d45c481fb0e9f8d97ed30fa9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=35.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "---------\n",
      "AUROC mean:  0.41947627382973357\n",
      "AUROC std:  0.2128356076699966\n",
      "---------\n",
      "---------\n",
      "AUPRC mean:  0.010609126220454445\n",
      "AUPRC std:  0.010198966578842047\n",
      "---------\n",
      "---------\n",
      "Accuracy mean:  0.9931381248151433\n",
      "Accuracy std:  0.005041650577382425\n",
      "---------\n",
      "---------\n",
      "Sensitivity mean:  0.0\n",
      "Sensitivity std:  0.0\n",
      "---------\n",
      "---------\n",
      "Specificity mean:  1.0\n",
      "Specificity std:  0.0\n",
      "---------\n",
      "---------\n",
      "F1 mean:  0.0\n",
      "F1 std:  0.0\n",
      "---------\n",
      "---------\n",
      "PPV mean:  nan\n",
      "PPV std:  nan\n",
      "---------\n",
      "---------\n",
      "NPV mean:  0.9931381248151433\n",
      "NPV std:  0.005041650577382425\n",
      "---------\n",
      "---------\n",
      "FPR mean:  0.0\n",
      "FPR std:  0.0\n",
      "---------\n",
      "---------\n",
      "FNR mean:  1.0\n",
      "FNR std:  0.0\n",
      "---------\n",
      "---------\n",
      "FDR mean:  nan\n",
      "FDR std:  nan\n",
      "---------\n",
      "---------\n",
      "# of Pos mean:  3.3142857142857145\n",
      "# of Pos std:  2.4351172288757157\n",
      "---------\n",
      "---------\n",
      "Recall@1% mean:  0.0\n",
      "Recall@1% std:  0.0\n",
      "---------\n",
      "---------\n",
      "Recall_Random@1% mean:  0.008490884353741499\n",
      "Recall_Random@1% std:  0.0020658633102465184\n",
      "---------\n",
      "---------\n",
      "Enrichment@1% mean:  0.0\n",
      "Enrichment@1% std:  0.0\n",
      "---------\n",
      "---------\n",
      "MRR@1% mean:  0.0\n",
      "MRR@1% std:  0.0\n",
      "---------\n",
      "---------\n",
      "AP@1% mean:  0.0\n",
      "AP@1% std:  0.0\n",
      "---------\n",
      "---------\n",
      "Recall@5% mean:  0.03214285714285714\n",
      "Recall@5% std:  0.16728626326614812\n",
      "---------\n",
      "---------\n",
      "Recall_Random@5% mean:  0.04842455782312925\n",
      "Recall_Random@5% std:  0.005168341111514603\n",
      "---------\n",
      "---------\n",
      "Enrichment@5% mean:  0.6979229279369532\n",
      "Enrichment@5% std:  3.6361977257507134\n",
      "---------\n",
      "---------\n",
      "MRR@5% mean:  0.0024327122153209105\n",
      "MRR@5% std:  0.009884095701194594\n",
      "---------\n",
      "---------\n",
      "AP@5% mean:  0.0024327122153209105\n",
      "AP@5% std:  0.009884095701194594\n",
      "---------\n",
      "---------\n",
      "Recall@10% mean:  0.08571428571428572\n",
      "Recall@10% std:  0.24233136362410534\n",
      "---------\n",
      "---------\n",
      "Recall_Random@10% mean:  0.09555843537414967\n",
      "Recall_Random@10% std:  0.007124273747626257\n",
      "---------\n",
      "---------\n",
      "Enrichment@10% mean:  0.9672630281600854\n",
      "Enrichment@10% std:  2.8081375771513057\n",
      "---------\n",
      "---------\n",
      "MRR@10% mean:  0.006192856765915733\n",
      "MRR@10% std:  0.013052222277169626\n",
      "---------\n",
      "---------\n",
      "AP@10% mean:  0.006751019474534368\n",
      "AP@10% std:  0.01419585485315457\n",
      "---------\n",
      "---------\n",
      "Recall@10 mean:  0.0\n",
      "Recall@10 std:  0.0\n",
      "---------\n",
      "---------\n",
      "Recall_Random@10 mean:  0.019949183673469387\n",
      "Recall_Random@10 std:  0.0035876093745798323\n",
      "---------\n",
      "---------\n",
      "Enrichment@10 mean:  0.0\n",
      "Enrichment@10 std:  0.0\n",
      "---------\n",
      "---------\n",
      "MRR@10 mean:  0.0\n",
      "MRR@10 std:  0.0\n",
      "---------\n",
      "---------\n",
      "AP@10 mean:  0.0\n",
      "AP@10 std:  0.0\n",
      "---------\n",
      "---------\n",
      "Recall@50 mean:  0.08928571428571429\n",
      "Recall@50 std:  0.24380068808394303\n",
      "---------\n",
      "---------\n",
      "Recall_Random@50 mean:  0.09923353741496599\n",
      "Recall_Random@50 std:  0.0077910102309228305\n",
      "---------\n",
      "---------\n",
      "Enrichment@50 mean:  0.9794676933853271\n",
      "Enrichment@50 std:  2.778680240874449\n",
      "---------\n",
      "---------\n",
      "MRR@50 mean:  0.006192856765915733\n",
      "MRR@50 std:  0.013052222277169626\n",
      "---------\n",
      "---------\n",
      "AP@50 mean:  0.0069372844567177185\n",
      "AP@50 std:  0.014509824616971445\n",
      "---------\n",
      "---------\n",
      "Recall@100 mean:  0.1\n",
      "Recall@100 std:  0.24964260167573274\n",
      "---------\n",
      "---------\n",
      "Recall_Random@100 mean:  0.20648462585034014\n",
      "Recall_Random@100 std:  0.010267204360089254\n",
      "---------\n",
      "---------\n",
      "Enrichment@100 mean:  0.5039102213728086\n",
      "Enrichment@100 std:  1.293153055208707\n",
      "---------\n",
      "---------\n",
      "MRR@100 mean:  0.006192856765915733\n",
      "MRR@100 std:  0.013052222277169626\n",
      "---------\n",
      "---------\n",
      "AP@100 mean:  0.007147293530178646\n",
      "AP@100 std:  0.01477194739973838\n",
      "---------\n"
     ]
    }
   ],
   "source": [
    "# evaluate individual diseases\n",
    "result = TxEval.eval_disease_centric(disease_idxs = [12661.0, 11318.0], \n",
    "                                     relation = 'indication', \n",
    "                                     save_result = False)\n",
    "\n",
    "# evaluate the entire test set\n",
    "result = RxEval.eval_disease_centric(disease_idxs = 'test_set', \n",
    "                                     show_plot = False, \n",
    "                                     verbose = True, \n",
    "                                     save_result = True,\n",
    "                                     return_raw = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 7701., 12661., 11195., 16569.,  6343.,  1479., 12621., 14416.,\n",
       "       13557., 16501., 13446., 13610.,  7105.,  9716.,  5822., 12320.,\n",
       "       10023., 14513., 12929.,  5683.,  9950., 14661.,  8395.,  7990.,\n",
       "        6031.,  6255., 14409.,  9589., 11885.,  4285.,  6377., 13130.,\n",
       "        7773.,  9741., 13381., 15355.,  4536., 12891.,  9087., 14873.,\n",
       "        6185.,  1812., 15301.,  6474.,  3381.,  6599., 16965.,  7880.,\n",
       "       16698.,  6252.,  5764.,  4192.,  4467., 13107., 14501., 11318.,\n",
       "        3452., 11349., 15911., 15100.,  1417.,  4152.,  1319.,  2220.,\n",
       "       15989.,  8973.])"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "TxEval.retrieve_disease_idxs_test_set('indication')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}