src/examples/training_demo.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Python 2.7.14\n"
     ]
    }
   ],
   "source": [
    "!python --version"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# ML Code Walkthrough"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from datetime import date\n",
    "from preprocess_raw import *\n",
    "from preprocess_transformed import *\n",
    "from model_setup_fit import *\n",
    "from model_optimization import *\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "pd.get_option(\"display.max_rows\",999)\n",
    "pd.get_option(\"display.max_columns\",999)\n",
    "pd.get_option(\"display.width\",None)\n",
    "\n",
    "GITHUB = os.environ.get(\"GITHUB\")\n",
    "sys.path.insert(0, GITHUB + 'Machine_Cleaning_JAM/modeling/src')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import time\n",
    "\n",
    "def print_time(start, end):\n",
    "    t_sec = round(end - start)\n",
    "    (t_min, t_sec) = divmod(t_sec,60)\n",
    "    (t_hour,t_min) = divmod(t_min,60) \n",
    "    print('Time passed: {}hour:{}min:{}sec'.format(t_hour,t_min,t_sec))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Pull in features from ESH Postgres databases"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/Users/sierra/Documents/ESH/ficher/Machine_Cleaning_JAM/src\n"
     ]
    }
   ],
   "source": [
    "!pwd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Querying data from DB connection\n",
      "Trying to establish initial connection to the server\n",
      "Success!\n",
      "Finished querying data\n",
      "raw_data data shape:  (47165, 118)\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "raw_data = get_table_from_db('get_data_2019_train.sql', 'file', \n",
    "                                     HOST=HOST_FORKED, \n",
    "                                     USER=USER_FORKED, \n",
    "                                     PASSWORD=PASSWORD_FORKED,\n",
    "                                     DB=DB_FORKED)\n",
    "print \"raw_data data shape: \", raw_data.shape\n",
    "print(\"\\n\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1. Preprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "*** STARTING PRE-PROCESSING FOR RAW DATA ***\n",
      "Dropped 0 duplicate rows\n",
      "Dropped null columns: \n",
      "['actual_start_date', 'fiber_type', 'fiber_sub_type', 'total_project_plant_route_feet', 'average_cost_per_foot_of_outside_plant', 'total_strands', 'number_of_erate_eligible_strands', 'match_amount', 'source_of_matching_funds', 'total_amount_financed', 'total_number_of_terms_in_months', 'annual_interest_rate', 'baloon_payment', 'special_construction_state_tribal_match_percentage', 'pending_reason', 'fcc_form486', 'fcc_form486_case_status', 'fcc_form486_invoicing_ready', 'last_date_to_invoice', 'wave_sequence_number', 'fcdl_letter_date', 'user_generated_fcdl_date', 'fcdl_comment_app', 'fcdl_comment_frn', 'appeal_wave_number', 'revised_fcdl_date', 'invoicing_mode', 'total_authorized_disbursement', 'connection_used_by', 'make', 'model', 'other_manufacture', 'unit']\n",
      "months_of_service duplicate with total_number_of_months_of_service, dropping months_of_service\n",
      "lease_or_non_purchase_agreement duplicate with is_installation_included_in_price, dropping lease_or_non_purchase_agreement\n",
      "Dropped duplicate columns: \n",
      "['months_of_service', 'lease_or_non_purchase_agreement']\n",
      "Dropped 0-variance columns: \n",
      "Index([u'funding_year', u'form_version', u'frn_status', u'service_type',\n",
      "       u'is_lease', u'is_installation_included_in_price'],\n",
      "      dtype='object')\n",
      "Dropped: id\n",
      "Dropped: frn\n",
      "Dropped: frn_number_from_the_previous_year\n",
      "Dropped: application_number\n",
      "Dropped: ben\n",
      "Dropped: account_number\n",
      "Dropped: service_provider_number\n",
      "Dropped: establishing_fcc_form470\n",
      "Dropped: user_entered_establishing_fcc_form470\n",
      "Dropped: line_item\n",
      "Dropped: award_date\n",
      "Dropped: expiration_date\n",
      "Dropped: contract_expiration_date\n",
      "Dropped: service_start_date\n",
      "Dropped: model\n",
      "Dropped: contract_number\n",
      "Dropped: restriction_citation\n",
      "Dropped: other_manufacture\n",
      "Dropped: download_speed\n",
      "Dropped: download_speed_units\n",
      "Dropped: upload_speed\n",
      "Dropped: upload_speed_units\n",
      "Dropped: burstable_speed\n",
      "Dropped: burstable_speed_units\n",
      "Dropped: purpose\n",
      "Dropped: billed_entity_name\n",
      "Dropped: type_of_product\n",
      "Dropped: updated_at\n",
      "Dropped: created_at\n",
      "Dropped: extended_contract_expiration_date\n",
      "Dropped: window_status\n",
      "Renamed purpose_adj to purpose\n",
      "contact_email float conversion failed\n",
      "funding_request_nickname float conversion failed\n",
      "old_form470_number converted to float\n",
      "was_fcc_form470_posted float conversion failed\n",
      "service_provider_name float conversion failed\n",
      "includes_voluntary_extensions float conversion failed\n",
      "remaining_voluntary_extensions converted to float\n",
      "total_remaining_contract_length converted to float\n",
      "pricing_confidentiality float conversion failed\n",
      "narrative float conversion failed\n",
      "total_monthly_recurring_charges converted to float\n",
      "total_monthly_ineligible_charges converted to float\n",
      "total_monthly_eligible_charges converted to float\n",
      "total_number_of_months_of_service converted to float\n",
      "total_eligible_pre_discount_recurring_charges converted to float\n",
      "total_one_time_charges converted to float\n",
      "total_ineligible_one_time_charges converted to float\n",
      "total_eligible_pre_discount_one_time_charges converted to float\n",
      "total_pre_discount_charges converted to float\n",
      "discount_rate converted to float\n",
      "funding_commitment_request converted to float\n",
      "num_bids_received converted to float\n",
      "based_on_state_master_contract float conversion failed\n",
      "based_on_multiple_award_schedule float conversion failed\n",
      "basic_firewall_protection float conversion failed\n",
      "connected_directly_to_school_library_or_nif float conversion failed\n",
      "connection_supports_school_library_or_nif float conversion failed\n",
      "monthly_quantity converted to float\n",
      "monthly_recurring_unit_costs converted to float\n",
      "monthly_recurring_unit_eligible_costs converted to float\n",
      "monthly_recurring_unit_ineligible_costs converted to float\n",
      "one_time_eligible_unit_costs converted to float\n",
      "one_time_ineligible_unit_costs converted to float\n",
      "one_time_quantity converted to float\n",
      "one_time_unit_costs converted to float\n",
      "pre_discount_extended_eligible_line_item_cost converted to float\n",
      "total_eligible_one_time_costs converted to float\n",
      "total_eligible_recurring_costs converted to float\n",
      "total_monthly_eligible_recurring_costs converted to float\n",
      "frn_adjusted float conversion failed\n",
      "upload_speed_mbps converted to float\n",
      "download_speed_mbps converted to float\n",
      "contract_end_time converted to float\n",
      "frn_previous_year_exists float conversion failed\n",
      "connection_supports_school_library_or_nif converted to float\n",
      "includes_voluntary_extensions converted to float\n",
      "basic_firewall_protection converted to float\n",
      "based_on_state_master_contract converted to float\n",
      "based_on_multiple_award_schedule converted to float\n",
      "pricing_confidentiality converted to float\n",
      "connected_directly_to_school_library_or_nif converted to float\n",
      "was_fcc_form470_posted converted to float\n",
      "frn_previous_year_exists converted to float\n",
      "Dummified columns: \n",
      "['pricing_confidentiality_type', 'purpose', 'function', 'postal_cd', 'billed_entity_type', 'contract_type']\n",
      "SHAPE (47165, 119)\n",
      "Dropped columns >= 74.0% NULL: \n",
      "['old_form470_number']\n",
      "Dropped total_monthly_recurring_charges due to 0.96 correlation with total_pre_discount_charges\n",
      "Dropped total_monthly_eligible_charges due to 0.959 correlation with total_pre_discount_charges\n",
      "Dropped total_eligible_recurring_costs due to 0.973 correlation with total_monthly_eligible_recurring_costs\n",
      "Dropped total_eligible_pre_discount_recurring_charges due to 0.999 correlation with total_pre_discount_charges\n",
      "Dropped total_eligible_pre_discount_one_time_charges due to 1.0 correlation with total_one_time_charges\n",
      "Dropped pre_discount_extended_eligible_line_item_cost due to 0.971 correlation with total_monthly_eligible_recurring_costs\n",
      "Dropped monthly_recurring_unit_costs due to 1.0 correlation with monthly_recurring_unit_eligible_costs\n",
      "Dropped funding_commitment_request due to 0.992 correlation with total_pre_discount_charges\n",
      "('Shape training_data_raw: ', (47165, 110))\n",
      "\n",
      "\n",
      "Time passed: 0.0hour:1.0min:25.0sec\n"
     ]
    }
   ],
   "source": [
    "start = time.time()\n",
    "print(\"*** STARTING PRE-PROCESSING FOR RAW DATA ***\")\n",
    "raw_preprocess = PreprocessRaw(raw_data, verbose=True, corr_threshold=0.93) #setting a higher threshold for dropping variables due to high correlation with others\n",
    "raw_preprocessed = raw_preprocess.applyall_raw()\n",
    "\n",
    "training_data_raw = raw_preprocessed.getdata()\n",
    "print(\"Shape training_data_raw: \", training_data_raw.shape)\n",
    "\n",
    "end = time.time()\n",
    "print(\"\\n\")\n",
    "print_time(start, end)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# you can save it as csv or pickle it if you don't want to keep reo-doing preprocessing\n",
    "training_data_raw.to_csv('../data/training_data_raw_qa.csv', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### variables I kept purposefully, could use more research/exploration for how or if to use!! but dropping for the demo"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('Final shape training_data_raw: ', (47165, 106))\n"
     ]
    }
   ],
   "source": [
    "string_vars = ['service_provider_name', 'narrative', 'funding_request_nickname', 'contact_email' ]\n",
    "training_data_raw.drop(string_vars, axis=1, inplace=True)\n",
    "print(\"Final shape training_data_raw: \", training_data_raw.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2. Modeling"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 'Model' class: under the hood\n",
    "<img src=\"http://scikit-learn.org/stable/_static/scikit-learn-logo-small.png\" alt=\"Logo\">\n",
    "\n",
    "Built on top of the `sklearn` package, a popular Python Machine Learning package (_not_ particularly geared toward Deep Learning, but a popular first start for regression, tree and ensemble methods).\n",
    "Initializes a model to fit to given training data and y variable, and fit a model using  GridSearch and/or RandomizedSearch cross validation. \n",
    "\n",
    "Input attributes: \n",
    "* **training_data**: the training dataset without the (clean) y variable\n",
    "* **yvar**: column name of y variable to predict\n",
    "* **model**: the type of model algorithm to fit. Options are one of: ['logisticregression', 'decisiontree', 'randomforest', 'gradientboosting'], but can add more under the hood if desired\n",
    "* **classification_type**: 'classification' or 'regression'\n",
    "* **imputer_strategy**: how to handle missing values – must be an array. \n",
    "* **p**: proportion of the training dataset to set for training; number between 0 and 1 (optional, default = 0.8)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Initialize Model object\n",
    "model_initial = Model(training_data = training_data_raw, \n",
    "                      yvar = 'purpose', \n",
    "                      model = 'randomforest', \n",
    "                      classification_type = 'classification',\n",
    "                      imputer_strategy = [\"mean\",\"median\",\"most_frequent\"])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### build_pipe()\n",
    "* Input: arrays of hyperparameters to test. For info on where to start, see our [wiki pages](https://educationsuperhighway.atlassian.net/wiki/spaces/SA/pages/541392967/Tuning+ML+Parameters). Must use syntax according to `sklearn` documentation. \n",
    "* Splits the data into training set and test (validation) set, and sets up the model pipeline in the format `sklearn` accepts\n",
    "\n",
    "_Further detail below:_\n",
    "* `training_setup()`: Obtain and merge the variable to predict, and encode the variable (necessary for classification). In Machine Cleaning case, our y variable came from a different database, hence the need for the two functions below. Could be simplified for other projects.\n",
    "    * `get_clean_y_query()`: query written to obtain the variable to predict (design varies by project)\n",
    "    * `merge_y_variable()`: execute the query and merge the y variable to the training data\n",
    "* `get_train_test_split()`:\n",
    "Splits the training data by a training proportion **p** into a training set and testing set at random (default 80% train, 20% test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of Estimators values to try: [200, 350, 500]\n",
      "Maximum Depth values to try: [10, 30, 50, None]\n"
     ]
    }
   ],
   "source": [
    "## SET ARRAYS OF INITIAL HYPERPARAMATER VALUES TO TRY\n",
    "## number of trees in the random forest\n",
    "n_estimators = [int(x) for x in np.linspace(start=200, stop=500, num=3)]\n",
    "print \"Number of Estimators values to try: \" + str(n_estimators)\n",
    "## number of features to consider at each split\n",
    "max_features = ['auto', 'sqrt']\n",
    "## maximum number of levels in tree\n",
    "max_depth = [int(x) for x in np.linspace(start=10, stop=50, num=3)]\n",
    "max_depth.append(None)\n",
    "print \"Maximum Depth values to try: \" + str(max_depth)\n",
    "## minimum number of samples required to split a node\n",
    "min_samples_split = [2,5,10]\n",
    "## minimum number of samples required at each leaf node\n",
    "min_samples_leaf = [1,2,4]\n",
    "## method of selecting samples for training each tree\n",
    "bootstrap = [True, False]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Querying data from DB connection\n",
      "Trying to establish initial connection to the server\n",
      "Success!\n",
      "Finished querying data\n",
      "Category counts for purpose(pre-encoding):\n",
      "[['backbone' 185]\n",
      " ['internet' 11215]\n",
      " ['isp' 1204]\n",
      " ['upstream' 3195]\n",
      " ['wan' 8459]]\n",
      "\n",
      "\n",
      "Time passed: 0.0hour:0.0min:3.0sec\n"
     ]
    }
   ],
   "source": [
    "start = time.time()\n",
    "\n",
    "## Pipeline setup\n",
    "model_initial = model_initial.build_pipe(n_estimators=n_estimators, \n",
    "                                         max_depth=max_depth, \n",
    "                                         class_weight=['balanced'], \n",
    "                                         max_features=max_features, \n",
    "                                         n_jobs=[-1], oob_score=[False], \n",
    "                                         criterion=['gini', 'entropy'], \n",
    "                                         min_samples_split=min_samples_split, \n",
    "                                         min_samples_leaf=min_samples_leaf, \n",
    "                                         bootstrap=bootstrap)\n",
    "\n",
    "end = time.time()\n",
    "print(\"\\n\")\n",
    "print_time(start, end)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### randomized_fit() and fit()\n",
    "Inputs below. Fit all the models to the full training set using cross-validation, and output the one with the best set of hyperparameters. \n",
    "* **n_jobs**: number of jobs (roughly: cores to allocate)\n",
    "* **verbose**: printing detail level\n",
    "* **scoring**: metric to optimize for (typically 'accuracy')\n",
    "* **n_iter**: number of iterations (`randomized_fit()` only)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Randomized Fit for testing out initial hyperparameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "*** STARTING RANDOMIZED SEARCH ***\n",
      "Fitting 3 folds for each of 50 candidates, totalling 150 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=4)]: Using backend LokyBackend with 4 concurrent workers.\n",
      "[Parallel(n_jobs=4)]: Done  24 tasks      | elapsed:  1.9min\n",
      "[Parallel(n_jobs=4)]: Done 120 tasks      | elapsed:  8.9min\n",
      "[Parallel(n_jobs=4)]: Done 150 out of 150 | elapsed: 10.8min finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished fit for all iterations.\n",
      "BEST PARAMETERS: {'estimator__min_samples_leaf': 1, 'estimator__oob_score': False, 'estimator__criterion': 'gini', 'imputer__strategy': 'most_frequent', 'estimator__min_samples_split': 5, 'estimator__max_features': 'auto', 'estimator__class_weight': 'balanced', 'estimator__bootstrap': False, 'estimator__max_depth': None, 'estimator__n_estimators': 350, 'estimator__n_jobs': -1}\n",
      "\n",
      "\n",
      "Time passed: 0.0hour:10.0min:54.0sec\n"
     ]
    }
   ],
   "source": [
    "start = time.time()\n",
    "print(\"*** STARTING RANDOMIZED SEARCH ***\")\n",
    "## Find the best model parameters\n",
    "## JUST FOR DEMO PURPOSES - USUALLY 50+ ITERATIONS ARE RECOMMENDED\n",
    "model_initial.randomized_fit(n_jobs=4, verbose=3, scoring= 'accuracy', n_iter=50)\n",
    "\n",
    "end = time.time()\n",
    "print(\"\\n\")\n",
    "print_time(start, end)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Narrowed down model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Querying data from DB connection\n",
      "Trying to establish initial connection to the server\n",
      "Success!\n",
      "Finished querying data\n",
      "Category counts for purpose(pre-encoding):\n",
      "[['backbone' 185]\n",
      " ['internet' 11215]\n",
      " ['isp' 1204]\n",
      " ['upstream' 3195]\n",
      " ['wan' 8459]]\n",
      "\n",
      "\n",
      "Time passed: 0.0hour:0.0min:3.0sec\n"
     ]
    }
   ],
   "source": [
    "start = time.time()\n",
    "model = Model(training_data_raw, 'purpose', 'randomforest', 'classification', imputer_strategy=['most_frequent'])\n",
    "model = model.build_pipe(n_estimators = [350], \n",
    "                         criterion = ['gini'], \n",
    "                         min_samples_leaf=[1], \n",
    "                         min_samples_split =[5], \n",
    "                         max_depth = [30], \n",
    "                         class_weight = ['balanced'], \n",
    "                         max_features = ['auto'],\n",
    "                         n_jobs = [-1], \n",
    "                         oob_score = [False],\n",
    "                         bootstrap = [False])\n",
    "\n",
    "end = time.time()\n",
    "print(\"\\n\")\n",
    "print_time(start, end)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "*** STARTING GRIDSEARCH ***\n",
      "Fitting 3 folds for each of 1 candidates, totalling 3 fits\n",
      "[CV] estimator__min_samples_leaf=1, estimator__oob_score=False, estimator__criterion=gini, estimator__class_weight=balanced, estimator__min_samples_split=5, estimator__max_features=auto, imputer__strategy=most_frequent, estimator__bootstrap=False, estimator__max_depth=30, estimator__n_estimators=350, estimator__n_jobs=-1 \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  estimator__min_samples_leaf=1, estimator__oob_score=False, estimator__criterion=gini, estimator__class_weight=balanced, estimator__min_samples_split=5, estimator__max_features=auto, imputer__strategy=most_frequent, estimator__bootstrap=False, estimator__max_depth=30, estimator__n_estimators=350, estimator__n_jobs=-1, score=0.921638330757, total=   6.0s\n",
      "[CV] estimator__min_samples_leaf=1, estimator__oob_score=False, estimator__criterion=gini, estimator__class_weight=balanced, estimator__min_samples_split=5, estimator__max_features=auto, imputer__strategy=most_frequent, estimator__bootstrap=False, estimator__max_depth=30, estimator__n_estimators=350, estimator__n_jobs=-1 \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:    6.6s remaining:    0.0s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  estimator__min_samples_leaf=1, estimator__oob_score=False, estimator__criterion=gini, estimator__class_weight=balanced, estimator__min_samples_split=5, estimator__max_features=auto, imputer__strategy=most_frequent, estimator__bootstrap=False, estimator__max_depth=30, estimator__n_estimators=350, estimator__n_jobs=-1, score=0.924884080371, total=   5.9s\n",
      "[CV] estimator__min_samples_leaf=1, estimator__oob_score=False, estimator__criterion=gini, estimator__class_weight=balanced, estimator__min_samples_split=5, estimator__max_features=auto, imputer__strategy=most_frequent, estimator__bootstrap=False, estimator__max_depth=30, estimator__n_estimators=350, estimator__n_jobs=-1 \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   2 out of   2 | elapsed:   13.0s remaining:    0.0s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  estimator__min_samples_leaf=1, estimator__oob_score=False, estimator__criterion=gini, estimator__class_weight=balanced, estimator__min_samples_split=5, estimator__max_features=auto, imputer__strategy=most_frequent, estimator__bootstrap=False, estimator__max_depth=30, estimator__n_estimators=350, estimator__n_jobs=-1, score=0.9189607176, total=   6.0s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   3 out of   3 | elapsed:   19.6s finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                                              Feature  Importance\n",
      "0                        purpose_data_connect_2ormore    0.090380\n",
      "1                     purpose_ias_includes_connection    0.087798\n",
      "2                            purpose_data_connect_hub    0.083043\n",
      "3                                    purpose_backbone    0.077329\n",
      "4                              purpose_ias_no_circuit    0.070189\n",
      "5                                   billed_consortium    0.048355\n",
      "6         connected_directly_to_school_library_or_nif    0.037990\n",
      "7                          total_pre_discount_charges    0.037272\n",
      "8               monthly_recurring_unit_eligible_costs    0.036854\n",
      "9                              billed_school_district    0.033941\n",
      "10             total_monthly_eligible_recurring_costs    0.032313\n",
      "11          connection_supports_school_library_or_nif    0.032140\n",
      "12                                download_speed_mbps    0.032065\n",
      "13                                  upload_speed_mbps    0.031498\n",
      "14                                  num_bids_received    0.024518\n",
      "15                                      discount_rate    0.017472\n",
      "16                                          postal_ia    0.013620\n",
      "17                     remaining_voluntary_extensions    0.013446\n",
      "18                                          postal_wi    0.013030\n",
      "19                                   monthly_quantity    0.012980\n",
      "20                    total_remaining_contract_length    0.012537\n",
      "21                                  contract_end_time    0.011371\n",
      "22                                          postal_ca    0.009274\n",
      "23                             total_one_time_charges    0.008087\n",
      "24                     based_on_state_master_contract    0.007040\n",
      "25                             was_fcc_form470_posted    0.006407\n",
      "26                                one_time_unit_costs    0.006306\n",
      "27                       one_time_eligible_unit_costs    0.006243\n",
      "28                                          postal_ut    0.005899\n",
      "29                      includes_voluntary_extensions    0.005765\n",
      "..                                                ...         ...\n",
      "75                                     function_other    0.000312\n",
      "76                                          postal_in    0.000301\n",
      "77                                          postal_nc    0.000296\n",
      "78                                          postal_vt    0.000272\n",
      "79                                          postal_tn    0.000264\n",
      "80                                          postal_mt    0.000257\n",
      "81                                          postal_ms    0.000236\n",
      "82                                    contract_tariff    0.000222\n",
      "83                                          postal_nm    0.000181\n",
      "84                                          postal_nv    0.000168\n",
      "85                                          postal_id    0.000158\n",
      "86                                          postal_md    0.000143\n",
      "87                                          postal_ky    0.000136\n",
      "88                                          postal_sc    0.000129\n",
      "89                                          postal_me    0.000125\n",
      "90                                          postal_nd    0.000119\n",
      "91                                          postal_ak    0.000106\n",
      "92                            pricing_confidentiality    0.000101\n",
      "93   pricing_contract_executed_with_restrictive_terms    0.000085\n",
      "94                                          postal_hi    0.000079\n",
      "95                                          postal_de    0.000053\n",
      "96                       pricing_state_law_or_statute    0.000012\n",
      "97                                          postal_dc    0.000007\n",
      "98                                          postal_as    0.000000\n",
      "99                                          postal_vi    0.000000\n",
      "100                                         postal_pr    0.000000\n",
      "101                             billed_library_system    0.000000\n",
      "102                                    billed_library    0.000000\n",
      "103                                         postal_mp    0.000000\n",
      "104                                         postal_gu    0.000000\n",
      "\n",
      "[105 rows x 2 columns]\n",
      "Result for: train\n",
      "0.9909306400082448\n",
      "Predicted  backbone  internet   isp  upstream   wan    All\n",
      "True                                                      \n",
      "backbone        155         0     0         0     0    155\n",
      "internet          0      8883    52        15    28   8978\n",
      "isp               0         0   959         0     0    959\n",
      "upstream          4         3     4      2547     3   2561\n",
      "wan               1        15     0        51  6686   6753\n",
      "All             160      8901  1015      2613  6717  19406\n",
      "Result for: test\n",
      "0.9245671887881286\n",
      "Predicted  backbone  internet  isp  upstream   wan   All\n",
      "True                                                    \n",
      "backbone         27         0    1         1     1    30\n",
      "internet          0      2132   45        24    36  2237\n",
      "isp               0        52  183         7     3   245\n",
      "upstream          3        29    9       555    38   634\n",
      "wan               0        65    0        52  1589  1706\n",
      "All              30      2278  238       639  1667  4852\n"
     ]
    }
   ],
   "source": [
    "print(\"*** STARTING GRIDSEARCH ***\")\n",
    "## Find the best model parameters\n",
    "model.fit()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3. Feature Elimination"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 'ModelOptimizer' class: under the hood\n",
    "* Also built on top of the `sklearn` package. \n",
    "* Serves as a starting point for eliminating features to reduce overfitting, since we started with ~300 features with limited intuition for which ones to select.\n",
    "* Helps with (small) incremental gains in _testing set score_. Good to choose a threshold right before the accuracy/score starts dropping more dramatically.\n",
    "* There are many complex methods and packages for feature elimination, but we wanted to start with the most rudimentary one possible.\n",
    "* Modeled after [this post.](https://machinelearningmastery.com/feature-importance-and-feature-selection-with-xgboost-in-python/) \n",
    "\n",
    "Input attributes: \n",
    "* **model_fit_obj**: An instance of the `Model` class _that has been fit_, mandatory.\n",
    "* **strategy**: Feature elimination testing strategy, mandatory. Options are one of: 'importance','manual','both'\n",
    "* **drop_features**: A list of column names of feature(s) to test elimination of, optional.\n",
    "* **threshold**: A decimal, optional.\n",
    "\n",
    "### optimize() pseudocode:\n",
    "<img src=\"strategy_desc.png\">\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "*** STARTING FEATURE ELIMINATION/OPTIMIZATION ***\n",
      "Thresh=0.00000730, n=98, Accuracy: 92.48%, Precision: 92.47%\n",
      "Thresh=0.00001196, n=97, Accuracy: 92.31%, Precision: 92.31%\n",
      "Thresh=0.00005288, n=96, Accuracy: 92.31%, Precision: 92.31%\n",
      "Thresh=0.00007868, n=95, Accuracy: 92.37%, Precision: 92.37%\n",
      "Thresh=0.00008460, n=94, Accuracy: 92.35%, Precision: 92.35%\n",
      "Thresh=0.00010081, n=93, Accuracy: 92.37%, Precision: 92.37%\n",
      "Thresh=0.00010612, n=92, Accuracy: 92.39%, Precision: 92.39%\n",
      "Thresh=0.00011919, n=91, Accuracy: 92.35%, Precision: 92.34%\n",
      "Thresh=0.00012469, n=90, Accuracy: 92.35%, Precision: 92.36%\n",
      "Thresh=0.00012920, n=89, Accuracy: 92.27%, Precision: 92.27%\n",
      "Thresh=0.00013587, n=88, Accuracy: 92.35%, Precision: 92.34%\n",
      "Thresh=0.00014311, n=87, Accuracy: 92.39%, Precision: 92.40%\n",
      "Thresh=0.00015842, n=86, Accuracy: 92.42%, Precision: 92.42%\n",
      "Thresh=0.00016798, n=85, Accuracy: 92.35%, Precision: 92.36%\n",
      "Thresh=0.00018131, n=84, Accuracy: 92.35%, Precision: 92.36%\n",
      "Thresh=0.00022192, n=83, Accuracy: 92.29%, Precision: 92.29%\n",
      "Thresh=0.00023630, n=82, Accuracy: 92.29%, Precision: 92.29%\n",
      "Thresh=0.00025710, n=81, Accuracy: 92.42%, Precision: 92.42%\n",
      "Thresh=0.00026381, n=80, Accuracy: 92.50%, Precision: 92.50%\n",
      "Thresh=0.00027223, n=79, Accuracy: 92.37%, Precision: 92.38%\n",
      "Thresh=0.00029613, n=78, Accuracy: 92.37%, Precision: 92.38%\n",
      "Thresh=0.00030096, n=77, Accuracy: 92.27%, Precision: 92.27%\n",
      "Thresh=0.00031237, n=76, Accuracy: 92.31%, Precision: 92.31%\n",
      "Thresh=0.00032637, n=75, Accuracy: 92.29%, Precision: 92.29%\n",
      "Thresh=0.00032707, n=74, Accuracy: 92.35%, Precision: 92.35%\n",
      "Thresh=0.00032981, n=73, Accuracy: 92.29%, Precision: 92.30%\n",
      "Thresh=0.00034599, n=72, Accuracy: 92.42%, Precision: 92.41%\n",
      "Thresh=0.00035600, n=71, Accuracy: 92.42%, Precision: 92.40%\n",
      "Thresh=0.00051179, n=70, Accuracy: 92.37%, Precision: 92.38%\n",
      "Thresh=0.00055503, n=69, Accuracy: 92.44%, Precision: 92.43%\n",
      "Thresh=0.00058143, n=68, Accuracy: 92.37%, Precision: 92.37%\n",
      "Thresh=0.00058503, n=67, Accuracy: 92.44%, Precision: 92.44%\n",
      "Thresh=0.00069921, n=66, Accuracy: 92.35%, Precision: 92.35%\n",
      "Thresh=0.00079881, n=65, Accuracy: 92.44%, Precision: 92.44%\n",
      "Thresh=0.00081724, n=64, Accuracy: 92.37%, Precision: 92.37%\n",
      "Thresh=0.00085268, n=63, Accuracy: 92.39%, Precision: 92.39%\n",
      "Thresh=0.00086043, n=62, Accuracy: 92.37%, Precision: 92.38%\n",
      "Thresh=0.00087952, n=61, Accuracy: 92.33%, Precision: 92.33%\n",
      "Thresh=0.00094059, n=60, Accuracy: 92.35%, Precision: 92.35%\n",
      "Thresh=0.00101821, n=59, Accuracy: 92.33%, Precision: 92.34%\n",
      "Thresh=0.00106341, n=58, Accuracy: 92.31%, Precision: 92.33%\n",
      "Thresh=0.00110160, n=57, Accuracy: 92.29%, Precision: 92.30%\n",
      "Thresh=0.00112660, n=56, Accuracy: 92.29%, Precision: 92.31%\n",
      "Thresh=0.00112758, n=55, Accuracy: 92.29%, Precision: 92.30%\n",
      "Thresh=0.00115039, n=54, Accuracy: 92.44%, Precision: 92.46%\n",
      "Thresh=0.00118651, n=53, Accuracy: 92.27%, Precision: 92.28%\n",
      "Thresh=0.00134353, n=52, Accuracy: 92.33%, Precision: 92.35%\n",
      "Thresh=0.00143651, n=51, Accuracy: 92.42%, Precision: 92.42%\n",
      "Thresh=0.00165665, n=50, Accuracy: 92.37%, Precision: 92.40%\n",
      "Thresh=0.00178886, n=49, Accuracy: 92.27%, Precision: 92.28%\n",
      "Thresh=0.00181099, n=48, Accuracy: 92.35%, Precision: 92.36%\n",
      "Thresh=0.00200111, n=47, Accuracy: 92.35%, Precision: 92.35%\n",
      "Thresh=0.00236776, n=46, Accuracy: 92.39%, Precision: 92.40%\n",
      "Thresh=0.00284027, n=45, Accuracy: 92.21%, Precision: 92.21%\n",
      "Thresh=0.00288183, n=44, Accuracy: 92.17%, Precision: 92.18%\n",
      "Thresh=0.00291456, n=43, Accuracy: 92.31%, Precision: 92.29%\n",
      "Thresh=0.00323041, n=42, Accuracy: 92.25%, Precision: 92.24%\n",
      "Thresh=0.00327806, n=41, Accuracy: 92.25%, Precision: 92.24%\n",
      "Thresh=0.00337079, n=40, Accuracy: 92.17%, Precision: 92.16%\n",
      "Thresh=0.00374699, n=39, Accuracy: 92.00%, Precision: 92.02%\n",
      "Thresh=0.00385787, n=38, Accuracy: 91.94%, Precision: 91.97%\n",
      "Thresh=0.00403966, n=37, Accuracy: 92.04%, Precision: 92.06%\n",
      "Thresh=0.00448844, n=36, Accuracy: 92.00%, Precision: 92.03%\n",
      "Thresh=0.00462600, n=35, Accuracy: 92.02%, Precision: 92.04%\n",
      "Thresh=0.00483726, n=34, Accuracy: 91.96%, Precision: 91.99%\n",
      "Thresh=0.00562230, n=33, Accuracy: 92.09%, Precision: 92.10%\n",
      "Thresh=0.00568767, n=32, Accuracy: 92.07%, Precision: 92.07%\n",
      "Thresh=0.00570458, n=31, Accuracy: 91.96%, Precision: 91.98%\n",
      "Thresh=0.00576492, n=30, Accuracy: 91.96%, Precision: 91.99%\n",
      "Thresh=0.00589880, n=29, Accuracy: 91.96%, Precision: 91.99%\n",
      "Thresh=0.00624310, n=28, Accuracy: 92.00%, Precision: 92.02%\n",
      "Thresh=0.00630612, n=27, Accuracy: 91.86%, Precision: 91.88%\n",
      "Thresh=0.00640733, n=26, Accuracy: 91.88%, Precision: 91.90%\n",
      "Thresh=0.00704009, n=25, Accuracy: 91.90%, Precision: 91.92%\n",
      "Thresh=0.00808705, n=24, Accuracy: 91.80%, Precision: 91.82%\n",
      "Thresh=0.00927369, n=23, Accuracy: 91.90%, Precision: 91.91%\n",
      "Thresh=0.01137129, n=22, Accuracy: 91.78%, Precision: 91.77%\n",
      "Thresh=0.01253732, n=21, Accuracy: 91.61%, Precision: 91.65%\n",
      "Thresh=0.01298003, n=20, Accuracy: 91.59%, Precision: 91.65%\n",
      "Thresh=0.01302993, n=19, Accuracy: 91.65%, Precision: 91.70%\n",
      "Thresh=0.01344641, n=18, Accuracy: 91.30%, Precision: 91.31%\n",
      "Thresh=0.01361969, n=17, Accuracy: 91.43%, Precision: 91.44%\n",
      "Thresh=0.01747234, n=16, Accuracy: 91.12%, Precision: 91.13%\n",
      "Thresh=0.02451797, n=15, Accuracy: 90.27%, Precision: 90.48%\n",
      "Thresh=0.03149808, n=14, Accuracy: 89.37%, Precision: 89.74%\n",
      "Thresh=0.03206498, n=13, Accuracy: 89.32%, Precision: 89.68%\n",
      "Thresh=0.03214003, n=12, Accuracy: 87.00%, Precision: 87.85%\n",
      "Thresh=0.03231277, n=11, Accuracy: 86.87%, Precision: 87.77%\n",
      "Thresh=0.03394067, n=10, Accuracy: 87.00%, Precision: 87.85%\n",
      "Thresh=0.03685433, n=9, Accuracy: 86.93%, Precision: 87.76%\n",
      "Thresh=0.03727171, n=8, Accuracy: 84.95%, Precision: 86.43%\n",
      "Thresh=0.03799040, n=7, Accuracy: 88.87%, Precision: 88.91%\n",
      "Thresh=0.04835489, n=6, Accuracy: 86.77%, Precision: 88.07%\n",
      "Thresh=0.07018919, n=5, Accuracy: 89.01%, Precision: 88.96%\n",
      "Thresh=0.07732893, n=4, Accuracy: 89.01%, Precision: 88.96%\n",
      "Thresh=0.08304322, n=3, Accuracy: 86.23%, Precision: 85.44%\n",
      "Thresh=0.08779807, n=2, Accuracy: 76.03%, Precision: 74.61%\n",
      "Thresh=0.09038033, n=1, Accuracy: 36.91%, Precision: 33.11%\n",
      "('Maximum Accuracy: ', '0.9249793899422918')\n",
      "\n",
      "\n",
      "Time passed: 0.0hour:22.0min:49.0sec\n"
     ]
    }
   ],
   "source": [
    "# By importance (all) for sake of demo; it's what we did the majority of the time\n",
    "start = time.time()\n",
    "model_optimizer = ModelOptimizer(model, 'importance')\n",
    "print(\"*** STARTING FEATURE ELIMINATION/OPTIMIZATION ***\")\n",
    "model_optimizer.optimize()\n",
    "\n",
    "end = time.time()\n",
    "print(\"\\n\")\n",
    "print_time(start, end)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Apply final importance threshold"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Thresh=0.00115000, n=54, Accuracy: 92.44%, Precision: 92.46%\n",
      "('Maximum Accuracy: ', '0.924361088211047')\n"
     ]
    }
   ],
   "source": [
    "mo_final = ModelOptimizer(model, 'importance', threshold=0.00115)\n",
    "mo_final.optimize()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 4. Finally, output results of final model\n",
    "**`output_results()`** function would be unique to your project: \n",
    "* extracts elements from a fit and/or optimized `Model` or `ModelOptimizer` object and outputs them to a database \n",
    "* stores final model object and features list (via `cPickle`)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Querying data from DB connection\n",
      "Trying to establish initial connection to the server\n",
      "Success!\n",
      "Finished querying data\n",
      "Results inserted\n",
      "All results inserted\n"
     ]
    }
   ],
   "source": [
    "output_results(mo_final, mo_final.getfeatures(),'Sierra',comment='rerun demo model')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# stopped here"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index([u'wide_area_network_text_narrative', u'was_fcc_form470_posted',\n",
       "       u'wan_text_narrative', u'wan_text_funding_request_nickname',\n",
       "       u'usac_school_district', u'usac_consortium',\n",
       "       u'transport_text_narrative', u'transport_text_funding_request_nickname',\n",
       "       u'transmission_text_narrative', u'total_remaining_contract_length',\n",
       "       ...\n",
       "       u'connected_directly_to_school_library_or_nif', u'connect_lit_fiber',\n",
       "       u'connect_isp_only', u'connect_ethernet', u'connect_dark_fiber',\n",
       "       u'basic_firewall_protection', u'based_on_state_master_contract',\n",
       "       u'bandwidth_in_mbps', u'backbone_text_narrative', u'app_new_sp'],\n",
       "      dtype='object', length=120)"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mo_final.getfeatures()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mo_final.bestmodel"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.14"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}