Sustainable_Industry.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Essential libraries and settings for the notebook\n",
    "%matplotlib inline\n",
    "import warnings\n",
    "from pathlib import Path\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "pd.set_option('display.max_columns', 40)\n",
    "data_path = Path('/home/devel/ML/selection_djakonov/')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# As we want to predict we should exclude data representing final rinse\n",
    "f = open('/home/devel/ML/selection_djakonov/train_values.csv', 'r')\n",
    "o = open('/home/devel/ML/selection_djakonov/train_values_nofinal.csv','w')\n",
    "for line in f:\n",
    "    if 'final_rinse' not in line:\n",
    "        s = line\n",
    "        o.write(s) \n",
    "f.close()\n",
    "o.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# As we want to use train data to tune weights of algorithm we should make it similar test one \n",
    "data = pd.read_csv('/home/devel/ML/selection_djakonov/train_values_nofinal.csv')\n",
    "data['process_phase'] = data.process_id.astype(str) + '_' + data.phase.astype(str)\n",
    "process_phases = data.process_phase.unique()\n",
    "rng = np.random.RandomState(100)\n",
    "to_keep = rng.choice(process_phases, size = np.int(len(process_phases) * 0.8), replace = False)\n",
    "data_limited = data[data.process_phase.isin(to_keep)]\n",
    "data_limited.to_csv('/home/devel/ML/selection_djakonov/train_values_nofinal_liketest.csv')                                                                                                                                                                                        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Loading training data for our model\n",
    "train_values = pd.read_csv(data_path / 'train_values_nofinal_liketest.csv',\n",
    "                           index_col = 0,\n",
    "                           parse_dates = ['timestamp'])\n",
    "\n",
    "train_labels = pd.read_csv(data_path / 'train_labels.csv',\n",
    "                           index_col = 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# The should be like train_values in id\n",
    "train_labels = train_labels.loc[train_values.process_id.unique()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>row_id</th>\n",
       "      <th>process_id</th>\n",
       "      <th>object_id</th>\n",
       "      <th>phase</th>\n",
       "      <th>timestamp</th>\n",
       "      <th>pipeline</th>\n",
       "      <th>supply_flow</th>\n",
       "      <th>supply_pressure</th>\n",
       "      <th>return_temperature</th>\n",
       "      <th>return_conductivity</th>\n",
       "      <th>return_turbidity</th>\n",
       "      <th>return_flow</th>\n",
       "      <th>supply_pump</th>\n",
       "      <th>supply_pre_rinse</th>\n",
       "      <th>supply_caustic</th>\n",
       "      <th>return_caustic</th>\n",
       "      <th>supply_acid</th>\n",
       "      <th>return_acid</th>\n",
       "      <th>supply_clean_water</th>\n",
       "      <th>return_recovery_water</th>\n",
       "      <th>return_drain</th>\n",
       "      <th>object_low_level</th>\n",
       "      <th>tank_level_pre_rinse</th>\n",
       "      <th>tank_level_caustic</th>\n",
       "      <th>tank_level_acid</th>\n",
       "      <th>tank_level_clean_water</th>\n",
       "      <th>tank_temperature_pre_rinse</th>\n",
       "      <th>tank_temperature_caustic</th>\n",
       "      <th>tank_temperature_acid</th>\n",
       "      <th>tank_concentration_caustic</th>\n",
       "      <th>tank_concentration_acid</th>\n",
       "      <th>tank_lsh_caustic</th>\n",
       "      <th>tank_lsh_acid</th>\n",
       "      <th>tank_lsh_clean_water</th>\n",
       "      <th>tank_lsh_pre_rinse</th>\n",
       "      <th>target_time_period</th>\n",
       "      <th>process_phase</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>20001</td>\n",
       "      <td>405</td>\n",
       "      <td>pre_rinse</td>\n",
       "      <td>2018-04-15 04:20:47</td>\n",
       "      <td>L4</td>\n",
       "      <td>8550.348</td>\n",
       "      <td>0.615451</td>\n",
       "      <td>18.044704</td>\n",
       "      <td>4.990765</td>\n",
       "      <td>0.177228</td>\n",
       "      <td>15776.9100</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>55.499672</td>\n",
       "      <td>41.555992</td>\n",
       "      <td>44.026875</td>\n",
       "      <td>49.474102</td>\n",
       "      <td>32.385708</td>\n",
       "      <td>83.036750</td>\n",
       "      <td>73.03241</td>\n",
       "      <td>45.394646</td>\n",
       "      <td>44.340126</td>\n",
       "      <td>False</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>20001_pre_rinse</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>20001</td>\n",
       "      <td>405</td>\n",
       "      <td>pre_rinse</td>\n",
       "      <td>2018-04-15 04:20:49</td>\n",
       "      <td>L4</td>\n",
       "      <td>11364.294</td>\n",
       "      <td>0.654297</td>\n",
       "      <td>18.229168</td>\n",
       "      <td>3.749680</td>\n",
       "      <td>0.122975</td>\n",
       "      <td>13241.4640</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>55.487920</td>\n",
       "      <td>41.624170</td>\n",
       "      <td>44.045685</td>\n",
       "      <td>49.457645</td>\n",
       "      <td>32.385708</td>\n",
       "      <td>83.015045</td>\n",
       "      <td>73.03241</td>\n",
       "      <td>45.394447</td>\n",
       "      <td>44.339380</td>\n",
       "      <td>False</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>20001_pre_rinse</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>20001</td>\n",
       "      <td>405</td>\n",
       "      <td>pre_rinse</td>\n",
       "      <td>2018-04-15 04:20:51</td>\n",
       "      <td>L4</td>\n",
       "      <td>12174.479</td>\n",
       "      <td>0.699870</td>\n",
       "      <td>18.395544</td>\n",
       "      <td>2.783954</td>\n",
       "      <td>0.387008</td>\n",
       "      <td>10698.7850</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>55.476166</td>\n",
       "      <td>41.638275</td>\n",
       "      <td>44.045685</td>\n",
       "      <td>49.462350</td>\n",
       "      <td>32.385708</td>\n",
       "      <td>83.015045</td>\n",
       "      <td>73.03241</td>\n",
       "      <td>45.396280</td>\n",
       "      <td>44.336735</td>\n",
       "      <td>False</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>20001_pre_rinse</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>20001</td>\n",
       "      <td>405</td>\n",
       "      <td>pre_rinse</td>\n",
       "      <td>2018-04-15 04:20:53</td>\n",
       "      <td>L4</td>\n",
       "      <td>13436.776</td>\n",
       "      <td>0.761502</td>\n",
       "      <td>18.583622</td>\n",
       "      <td>1.769353</td>\n",
       "      <td>0.213397</td>\n",
       "      <td>8007.8125</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>55.471466</td>\n",
       "      <td>41.647675</td>\n",
       "      <td>44.048030</td>\n",
       "      <td>49.462350</td>\n",
       "      <td>32.385708</td>\n",
       "      <td>83.036750</td>\n",
       "      <td>73.03241</td>\n",
       "      <td>45.401875</td>\n",
       "      <td>44.333110</td>\n",
       "      <td>False</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>20001_pre_rinse</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>20001</td>\n",
       "      <td>405</td>\n",
       "      <td>pre_rinse</td>\n",
       "      <td>2018-04-15 04:20:55</td>\n",
       "      <td>L4</td>\n",
       "      <td>13776.766</td>\n",
       "      <td>0.837240</td>\n",
       "      <td>18.627026</td>\n",
       "      <td>0.904020</td>\n",
       "      <td>0.148293</td>\n",
       "      <td>6004.0510</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>55.459705</td>\n",
       "      <td>41.654730</td>\n",
       "      <td>44.048030</td>\n",
       "      <td>49.462350</td>\n",
       "      <td>32.385708</td>\n",
       "      <td>83.015045</td>\n",
       "      <td>73.03241</td>\n",
       "      <td>45.398197</td>\n",
       "      <td>44.334373</td>\n",
       "      <td>False</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>20001_pre_rinse</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   row_id  process_id  object_id      phase           timestamp pipeline  \\\n",
       "0       0       20001        405  pre_rinse 2018-04-15 04:20:47       L4   \n",
       "1       1       20001        405  pre_rinse 2018-04-15 04:20:49       L4   \n",
       "2       2       20001        405  pre_rinse 2018-04-15 04:20:51       L4   \n",
       "3       3       20001        405  pre_rinse 2018-04-15 04:20:53       L4   \n",
       "4       4       20001        405  pre_rinse 2018-04-15 04:20:55       L4   \n",
       "\n",
       "   supply_flow  supply_pressure  return_temperature  return_conductivity  \\\n",
       "0     8550.348         0.615451           18.044704             4.990765   \n",
       "1    11364.294         0.654297           18.229168             3.749680   \n",
       "2    12174.479         0.699870           18.395544             2.783954   \n",
       "3    13436.776         0.761502           18.583622             1.769353   \n",
       "4    13776.766         0.837240           18.627026             0.904020   \n",
       "\n",
       "   return_turbidity  return_flow  supply_pump  supply_pre_rinse  \\\n",
       "0          0.177228   15776.9100         True              True   \n",
       "1          0.122975   13241.4640         True              True   \n",
       "2          0.387008   10698.7850         True              True   \n",
       "3          0.213397    8007.8125         True              True   \n",
       "4          0.148293    6004.0510         True              True   \n",
       "\n",
       "   supply_caustic  return_caustic  supply_acid  return_acid  \\\n",
       "0           False           False        False        False   \n",
       "1           False           False        False        False   \n",
       "2           False           False        False        False   \n",
       "3           False           False        False        False   \n",
       "4           False           False        False        False   \n",
       "\n",
       "   supply_clean_water  return_recovery_water  return_drain  object_low_level  \\\n",
       "0               False                  False          True              True   \n",
       "1               False                  False          True              True   \n",
       "2               False                  False          True              True   \n",
       "3               False                  False          True              True   \n",
       "4               False                  False          True              True   \n",
       "\n",
       "   tank_level_pre_rinse  tank_level_caustic  tank_level_acid  \\\n",
       "0             55.499672           41.555992        44.026875   \n",
       "1             55.487920           41.624170        44.045685   \n",
       "2             55.476166           41.638275        44.045685   \n",
       "3             55.471466           41.647675        44.048030   \n",
       "4             55.459705           41.654730        44.048030   \n",
       "\n",
       "   tank_level_clean_water  tank_temperature_pre_rinse  \\\n",
       "0               49.474102                   32.385708   \n",
       "1               49.457645                   32.385708   \n",
       "2               49.462350                   32.385708   \n",
       "3               49.462350                   32.385708   \n",
       "4               49.462350                   32.385708   \n",
       "\n",
       "   tank_temperature_caustic  tank_temperature_acid  \\\n",
       "0                 83.036750               73.03241   \n",
       "1                 83.015045               73.03241   \n",
       "2                 83.015045               73.03241   \n",
       "3                 83.036750               73.03241   \n",
       "4                 83.015045               73.03241   \n",
       "\n",
       "   tank_concentration_caustic  tank_concentration_acid  tank_lsh_caustic  \\\n",
       "0                   45.394646                44.340126             False   \n",
       "1                   45.394447                44.339380             False   \n",
       "2                   45.396280                44.336735             False   \n",
       "3                   45.401875                44.333110             False   \n",
       "4                   45.398197                44.334373             False   \n",
       "\n",
       "   tank_lsh_acid  tank_lsh_clean_water  tank_lsh_pre_rinse  \\\n",
       "0            0.0                 False                 0.0   \n",
       "1            0.0                 False                 0.0   \n",
       "2            0.0                 False                 0.0   \n",
       "3            0.0                 False                 0.0   \n",
       "4            0.0                 False                 0.0   \n",
       "\n",
       "   target_time_period    process_phase  \n",
       "0               False  20001_pre_rinse  \n",
       "1               False  20001_pre_rinse  \n",
       "2               False  20001_pre_rinse  \n",
       "3               False  20001_pre_rinse  \n",
       "4               False  20001_pre_rinse  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Let`s take a look at data\n",
    "train_values.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>row_id</th>\n",
       "      <th>process_id</th>\n",
       "      <th>object_id</th>\n",
       "      <th>supply_flow</th>\n",
       "      <th>supply_pressure</th>\n",
       "      <th>return_temperature</th>\n",
       "      <th>return_conductivity</th>\n",
       "      <th>return_turbidity</th>\n",
       "      <th>return_flow</th>\n",
       "      <th>tank_level_pre_rinse</th>\n",
       "      <th>tank_level_caustic</th>\n",
       "      <th>tank_level_acid</th>\n",
       "      <th>tank_level_clean_water</th>\n",
       "      <th>tank_temperature_pre_rinse</th>\n",
       "      <th>tank_temperature_caustic</th>\n",
       "      <th>tank_temperature_acid</th>\n",
       "      <th>tank_concentration_caustic</th>\n",
       "      <th>tank_concentration_acid</th>\n",
       "      <th>tank_lsh_acid</th>\n",
       "      <th>tank_lsh_pre_rinse</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>3.571941e+06</td>\n",
       "      <td>3.571941e+06</td>\n",
       "      <td>3.571941e+06</td>\n",
       "      <td>3.571941e+06</td>\n",
       "      <td>3.571941e+06</td>\n",
       "      <td>3.571941e+06</td>\n",
       "      <td>3.571941e+06</td>\n",
       "      <td>3.571941e+06</td>\n",
       "      <td>3.571941e+06</td>\n",
       "      <td>3.571941e+06</td>\n",
       "      <td>3.571941e+06</td>\n",
       "      <td>3.571941e+06</td>\n",
       "      <td>3.571941e+06</td>\n",
       "      <td>3.571941e+06</td>\n",
       "      <td>3.571941e+06</td>\n",
       "      <td>3.571941e+06</td>\n",
       "      <td>3.571941e+06</td>\n",
       "      <td>3.571941e+06</td>\n",
       "      <td>3571941.0</td>\n",
       "      <td>3571941.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>2.985055e+06</td>\n",
       "      <td>2.401463e+04</td>\n",
       "      <td>5.743699e+02</td>\n",
       "      <td>2.326928e+04</td>\n",
       "      <td>1.045687e+00</td>\n",
       "      <td>5.913386e+01</td>\n",
       "      <td>3.087263e+01</td>\n",
       "      <td>3.704040e+00</td>\n",
       "      <td>2.219825e+04</td>\n",
       "      <td>5.280756e+01</td>\n",
       "      <td>4.220547e+01</td>\n",
       "      <td>4.413573e+01</td>\n",
       "      <td>4.300557e+01</td>\n",
       "      <td>2.859020e+01</td>\n",
       "      <td>8.238858e+01</td>\n",
       "      <td>7.257215e+01</td>\n",
       "      <td>4.524451e+01</td>\n",
       "      <td>4.442097e+01</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>1.730014e+06</td>\n",
       "      <td>2.317472e+03</td>\n",
       "      <td>3.458863e+02</td>\n",
       "      <td>1.720663e+04</td>\n",
       "      <td>1.354687e+00</td>\n",
       "      <td>2.345633e+01</td>\n",
       "      <td>1.996129e+01</td>\n",
       "      <td>7.683539e+00</td>\n",
       "      <td>1.792751e+04</td>\n",
       "      <td>3.958271e+00</td>\n",
       "      <td>3.046017e+00</td>\n",
       "      <td>2.432799e+00</td>\n",
       "      <td>6.552683e+00</td>\n",
       "      <td>1.876694e+00</td>\n",
       "      <td>2.331478e+00</td>\n",
       "      <td>4.091700e-01</td>\n",
       "      <td>8.697421e-01</td>\n",
       "      <td>1.087087e+00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>2.000100e+04</td>\n",
       "      <td>1.020000e+02</td>\n",
       "      <td>-9.481482e+04</td>\n",
       "      <td>-1.145833e-01</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>-3.616898e-01</td>\n",
       "      <td>-1.251447e+03</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>2.778157e+01</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>6.729962e+01</td>\n",
       "      <td>2.569796e+01</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>1.482635e+06</td>\n",
       "      <td>2.201800e+04</td>\n",
       "      <td>3.000000e+02</td>\n",
       "      <td>6.622540e+03</td>\n",
       "      <td>8.854166e-02</td>\n",
       "      <td>3.250868e+01</td>\n",
       "      <td>2.453276e+00</td>\n",
       "      <td>4.195602e-01</td>\n",
       "      <td>4.108796e+03</td>\n",
       "      <td>5.032046e+01</td>\n",
       "      <td>4.070258e+01</td>\n",
       "      <td>4.273148e+01</td>\n",
       "      <td>4.089301e+01</td>\n",
       "      <td>2.717375e+01</td>\n",
       "      <td>8.225188e+01</td>\n",
       "      <td>7.239583e+01</td>\n",
       "      <td>4.512999e+01</td>\n",
       "      <td>4.430475e+01</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>2.994167e+06</td>\n",
       "      <td>2.406400e+04</td>\n",
       "      <td>4.200000e+02</td>\n",
       "      <td>2.324942e+04</td>\n",
       "      <td>3.760851e-01</td>\n",
       "      <td>7.000868e+01</td>\n",
       "      <td>4.281397e+01</td>\n",
       "      <td>1.124855e+00</td>\n",
       "      <td>2.193287e+04</td>\n",
       "      <td>5.370822e+01</td>\n",
       "      <td>4.247522e+01</td>\n",
       "      <td>4.420320e+01</td>\n",
       "      <td>4.414207e+01</td>\n",
       "      <td>2.849754e+01</td>\n",
       "      <td>8.253761e+01</td>\n",
       "      <td>7.260561e+01</td>\n",
       "      <td>4.529035e+01</td>\n",
       "      <td>4.451433e+01</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>4.477179e+06</td>\n",
       "      <td>2.599400e+04</td>\n",
       "      <td>9.340000e+02</td>\n",
       "      <td>3.308738e+04</td>\n",
       "      <td>1.841363e+00</td>\n",
       "      <td>7.888093e+01</td>\n",
       "      <td>4.460889e+01</td>\n",
       "      <td>2.466725e+00</td>\n",
       "      <td>3.484881e+04</td>\n",
       "      <td>5.595106e+01</td>\n",
       "      <td>4.401982e+01</td>\n",
       "      <td>4.567256e+01</td>\n",
       "      <td>4.744756e+01</td>\n",
       "      <td>2.988281e+01</td>\n",
       "      <td>8.280165e+01</td>\n",
       "      <td>7.282262e+01</td>\n",
       "      <td>4.545260e+01</td>\n",
       "      <td>4.474604e+01</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>5.987585e+06</td>\n",
       "      <td>2.798900e+04</td>\n",
       "      <td>9.770000e+02</td>\n",
       "      <td>1.031612e+05</td>\n",
       "      <td>6.194010e+00</td>\n",
       "      <td>9.697627e+01</td>\n",
       "      <td>7.356589e+01</td>\n",
       "      <td>1.009657e+02</td>\n",
       "      <td>1.031395e+05</td>\n",
       "      <td>5.834907e+01</td>\n",
       "      <td>5.135489e+01</td>\n",
       "      <td>5.244104e+01</td>\n",
       "      <td>5.063549e+01</td>\n",
       "      <td>3.778935e+01</td>\n",
       "      <td>8.352864e+01</td>\n",
       "      <td>7.399450e+01</td>\n",
       "      <td>6.146095e+01</td>\n",
       "      <td>6.524091e+01</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             row_id    process_id     object_id   supply_flow  \\\n",
       "count  3.571941e+06  3.571941e+06  3.571941e+06  3.571941e+06   \n",
       "mean   2.985055e+06  2.401463e+04  5.743699e+02  2.326928e+04   \n",
       "std    1.730014e+06  2.317472e+03  3.458863e+02  1.720663e+04   \n",
       "min    0.000000e+00  2.000100e+04  1.020000e+02 -9.481482e+04   \n",
       "25%    1.482635e+06  2.201800e+04  3.000000e+02  6.622540e+03   \n",
       "50%    2.994167e+06  2.406400e+04  4.200000e+02  2.324942e+04   \n",
       "75%    4.477179e+06  2.599400e+04  9.340000e+02  3.308738e+04   \n",
       "max    5.987585e+06  2.798900e+04  9.770000e+02  1.031612e+05   \n",
       "\n",
       "       supply_pressure  return_temperature  return_conductivity  \\\n",
       "count     3.571941e+06        3.571941e+06         3.571941e+06   \n",
       "mean      1.045687e+00        5.913386e+01         3.087263e+01   \n",
       "std       1.354687e+00        2.345633e+01         1.996129e+01   \n",
       "min      -1.145833e-01        0.000000e+00         0.000000e+00   \n",
       "25%       8.854166e-02        3.250868e+01         2.453276e+00   \n",
       "50%       3.760851e-01        7.000868e+01         4.281397e+01   \n",
       "75%       1.841363e+00        7.888093e+01         4.460889e+01   \n",
       "max       6.194010e+00        9.697627e+01         7.356589e+01   \n",
       "\n",
       "       return_turbidity   return_flow  tank_level_pre_rinse  \\\n",
       "count      3.571941e+06  3.571941e+06          3.571941e+06   \n",
       "mean       3.704040e+00  2.219825e+04          5.280756e+01   \n",
       "std        7.683539e+00  1.792751e+04          3.958271e+00   \n",
       "min       -3.616898e-01 -1.251447e+03          0.000000e+00   \n",
       "25%        4.195602e-01  4.108796e+03          5.032046e+01   \n",
       "50%        1.124855e+00  2.193287e+04          5.370822e+01   \n",
       "75%        2.466725e+00  3.484881e+04          5.595106e+01   \n",
       "max        1.009657e+02  1.031395e+05          5.834907e+01   \n",
       "\n",
       "       tank_level_caustic  tank_level_acid  tank_level_clean_water  \\\n",
       "count        3.571941e+06     3.571941e+06            3.571941e+06   \n",
       "mean         4.220547e+01     4.413573e+01            4.300557e+01   \n",
       "std          3.046017e+00     2.432799e+00            6.552683e+00   \n",
       "min          0.000000e+00     2.778157e+01            0.000000e+00   \n",
       "25%          4.070258e+01     4.273148e+01            4.089301e+01   \n",
       "50%          4.247522e+01     4.420320e+01            4.414207e+01   \n",
       "75%          4.401982e+01     4.567256e+01            4.744756e+01   \n",
       "max          5.135489e+01     5.244104e+01            5.063549e+01   \n",
       "\n",
       "       tank_temperature_pre_rinse  tank_temperature_caustic  \\\n",
       "count                3.571941e+06              3.571941e+06   \n",
       "mean                 2.859020e+01              8.238858e+01   \n",
       "std                  1.876694e+00              2.331478e+00   \n",
       "min                  0.000000e+00              0.000000e+00   \n",
       "25%                  2.717375e+01              8.225188e+01   \n",
       "50%                  2.849754e+01              8.253761e+01   \n",
       "75%                  2.988281e+01              8.280165e+01   \n",
       "max                  3.778935e+01              8.352864e+01   \n",
       "\n",
       "       tank_temperature_acid  tank_concentration_caustic  \\\n",
       "count           3.571941e+06                3.571941e+06   \n",
       "mean            7.257215e+01                4.524451e+01   \n",
       "std             4.091700e-01                8.697421e-01   \n",
       "min             6.729962e+01                2.569796e+01   \n",
       "25%             7.239583e+01                4.512999e+01   \n",
       "50%             7.260561e+01                4.529035e+01   \n",
       "75%             7.282262e+01                4.545260e+01   \n",
       "max             7.399450e+01                6.146095e+01   \n",
       "\n",
       "       tank_concentration_acid  tank_lsh_acid  tank_lsh_pre_rinse  \n",
       "count             3.571941e+06      3571941.0           3571941.0  \n",
       "mean              4.442097e+01            0.0                 0.0  \n",
       "std               1.087087e+00            0.0                 0.0  \n",
       "min               0.000000e+00            0.0                 0.0  \n",
       "25%               4.430475e+01            0.0                 0.0  \n",
       "50%               4.451433e+01            0.0                 0.0  \n",
       "75%               4.474604e+01            0.0                 0.0  \n",
       "max               6.524091e+01            0.0                 0.0  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Different statistics can help us realize data\n",
    "train_values.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# We want to use all the data that`s why id and pipeline at work\n",
    "def prep_metadata(df):\n",
    "    \n",
    "    # select process_id and pipeline\n",
    "    metadata = df[['process_id', 'pipeline']].drop_duplicates().set_index('process_id') \n",
    "    \n",
    "    # convert categorical pipeline data to dummy variables\n",
    "    metadata = pd.get_dummies(metadata)\n",
    "    \n",
    "    # pipeline L12 not in test data!!!\n",
    "    if 'L12' not in metadata.columns:\n",
    "        metadata['pipeline_L12'] = 0\n",
    "    \n",
    "    # calculate number of phases for each process_object\n",
    "    metadata['num_phases'] = df.groupby('process_id')['phase'].apply(lambda x: x.nunique())\n",
    "    \n",
    "    return metadata\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# As we saw later we want to solve regression problem hence our data should be in appropriate conditions\n",
    "def prep_time_series_features_for_regression(df, columns = None):\n",
    "   \n",
    "    need_cols = [\n",
    "    'process_id',\n",
    "    'supply_flow',\n",
    "    'supply_pressure',\n",
    "    'return_temperature',\n",
    "    'return_conductivity',\n",
    "    'return_turbidity',\n",
    "    'return_flow',\n",
    "    'tank_level_pre_rinse',\n",
    "    'tank_level_caustic',\n",
    "    'tank_level_acid',\n",
    "    'tank_level_clean_water',\n",
    "    'tank_temperature_pre_rinse',\n",
    "    'tank_temperature_caustic',\n",
    "    'tank_temperature_acid',\n",
    "    'tank_concentration_caustic',\n",
    "    'tank_concentration_acid',\n",
    "    ]\n",
    "    \n",
    "    if columns is None:\n",
    "        columns = df.columns\n",
    "    \n",
    "    need_data = df[need_cols].set_index('process_id')\n",
    "    \n",
    "    # create features: min, max, mean, standard deviation, and mean of the last five observations\n",
    "    need_features = need_data.groupby('process_id').agg(['min', 'max', 'mean', 'std', lambda x: x.tail(5).mean()])\n",
    "    \n",
    "    return need_features\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Generally speaking, I thought these data to be valuable but they are not\n",
    "def perform_bool_information(df):\n",
    "    \n",
    "    need_cols = [\n",
    "    'process_id',\n",
    "    'supply_pump',\n",
    "    'supply_pre_rinse',\n",
    "    'supply_caustic',\n",
    "    'return_caustic',\n",
    "    'supply_acid',\n",
    "    'return_acid',\n",
    "    'supply_clean_water',\n",
    "    'return_recovery_water',\n",
    "    'return_drain'\n",
    "    ]\n",
    "    need_features = df[need_cols].set_index('process_id')\n",
    "    \n",
    "    need_features = need_features.groupby('process_id').agg(lambda x: x[lambda y : y == True].sum())\n",
    "    \n",
    "    return need_features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# This function is a union of three previuos ones\n",
    "def create_feature_matrix(df):\n",
    "    \n",
    "    metadata = prep_metadata(df)\n",
    "    \n",
    "    time_series = prep_time_series_features_for_regression(df)\n",
    "    \n",
    "    bool_inf = perform_bool_information(df)\n",
    "    \n",
    "    # join metadata and time series features into a single dataframe\n",
    "    feature_matrix = pd.concat([metadata, time_series], axis=1)# bool_inf does not help\n",
    "    \n",
    "    return feature_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Transform raw data into features for our future model\n",
    "train_features = create_feature_matrix(train_values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Our metric here is not Least Squares but special  \n",
    "def metrics(X, Y):\n",
    "    Z = abs(X - Y)\n",
    "    K = Y.applymap(lambda x: max(abs(x), 290000))\n",
    "    Z = Z / K\n",
    "    return ((Z.sum()) / Z.size).values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Making train set and validation set to evaluate our results fairly\n",
    "X_train, X_valid, Y_train, Y_valid = train_test_split(train_features, train_labels, random_state = 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1.70690684]\n"
     ]
    }
   ],
   "source": [
    "# This model is one of the easiest that`s why we should try it, but our results are poor\n",
    "from sklearn.neighbors import KNeighborsRegressor\n",
    "\n",
    "KNR_model = KNeighborsRegressor(n_neighbors = 1)\n",
    "KNR_model.fit(X_train, Y_train)\n",
    "pred_KNR = KNR_model.predict(X_valid)\n",
    "cr = metrics(pred_KNR, Y_valid)\n",
    "print(cr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1.60811307]\n"
     ]
    }
   ],
   "source": [
    "# As we expected Tree methods usually work more regular and robust in different tasks, but it is not well then\n",
    "from sklearn.tree import DecisionTreeRegressor\n",
    "\n",
    "DT_model = DecisionTreeRegressor(max_leaf_nodes = 1500, max_depth = 20, random_state = 0)\n",
    "DT_model.fit(X_train, Y_train)\n",
    "pred_DT = DT_model.predict(X_valid)\n",
    "pred_DT.reshape(-1, 1)\n",
    "\n",
    "cr = metrics(pred_DT.reshape(-1, 1),Y_valid)\n",
    "print(cr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.40587567]\n"
     ]
    }
   ],
   "source": [
    "# Putting it mildly, it is one of the best models to tune and use, our results are increadible!\n",
    "from sklearn.ensemble import GradientBoostingRegressor\n",
    "\n",
    "GB_model = GradientBoostingRegressor(loss = 'lad', learning_rate = 0.1, n_estimators = 90, random_state = 0, max_depth = 6)\n",
    "GB_model.fit(X_train, Y_train)\n",
    "pred_GB = GB_model.predict(X_valid)\n",
    "cr = metrics(pred_GB.reshape(-1, 1), Y_valid)\n",
    "print(cr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.39328043]\n"
     ]
    }
   ],
   "source": [
    "# Adjusting parametres is always very necessary \n",
    "GB_model = GradientBoostingRegressor(loss = 'lad', learning_rate = 0.1, n_estimators = 50, random_state = 0, max_depth = 5)\n",
    "GB_model.fit(X_train, Y_train)\n",
    "pred_GB = GB_model.predict(X_valid)\n",
    "cr = metrics(pred_GB.reshape(-1, 1), Y_valid)\n",
    "print(cr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.71146876] ;i: 0.02 ;j: 20\n",
      "[0.59606305] ;i: 0.02 ;j: 40\n",
      "[0.52426157] ;i: 0.02 ;j: 60\n",
      "[0.47846577] ;i: 0.02 ;j: 80\n",
      "[0.44863653] ;i: 0.02 ;j: 100\n",
      "[0.42960834] ;i: 0.02 ;j: 120\n",
      "[0.42535864] ;i: 0.04 ;j: 60\n",
      "[0.40582901] ;i: 0.04 ;j: 80\n",
      "[0.40226227] ;i: 0.04 ;j: 100\n",
      "[0.39753303] ;i: 0.04 ;j: 120\n",
      "[0.39576022] ;i: 0.06 ;j: 120\n",
      "[0.39370191] ;i: 0.1 ;j: 60\n",
      "[0.39088247] ;i: 0.1 ;j: 80\n",
      "Stopped Process\n"
     ]
    }
   ],
   "source": [
    "# However we cannot change parameters by hand hence loops can help us\n",
    "minim = 1\n",
    "i = 0.02\n",
    "j = 20\n",
    "while i <= 0.12:\n",
    "    while j <= 120:\n",
    "        GB_model = GradientBoostingRegressor(loss = 'lad', learning_rate = i, n_estimators = j, random_state = 0, max_depth = 5)\n",
    "        GB_model.fit(X_train, Y_train)\n",
    "        pred_GB = GB_model.predict(X_valid)\n",
    "        cr = metrics(pred_GB.reshape(-1, 1), Y_valid)\n",
    "        if cr < minim:\n",
    "            minim = cr\n",
    "            print(cr, ';i:', i, ';j:', j)\n",
    "        j += 20\n",
    "    j = 20   \n",
    "    i += 0.02 \n",
    "    \n",
    "print('Stopped Process')     "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Let`s try a bit more models, however the need scaling to work well\n",
    "from sklearn.preprocessing import Imputer, PolynomialFeatures, MinMaxScaler, RobustScaler, StandardScaler, Normalizer\n",
    "\n",
    "scaler = Normalizer()\n",
    "scaler.fit(train_features)\n",
    "X = scaler.transform(train_features) \n",
    "X_train,X_valid,Y_train,Y_valid = train_test_split(X, train_labels, random_state = 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.85252112]\n"
     ]
    }
   ],
   "source": [
    "# This model is quite well, but not excellent in our notebook\n",
    "from sklearn.neural_network import MLPRegressor\n",
    "\n",
    "MLP_model = MLPRegressor(hidden_layer_sizes = (200, ), learning_rate_init = 0.001, alpha = 0.00003)\n",
    "MLP_model.fit(X_train, Y_train)\n",
    "pred_MLP = MLP_model.predict(X_valid)\n",
    "cr = metrics(pred_MLP.reshape(-1, 1), Y_valid)\n",
    "print(cr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.84815456] ; hl= 1\n",
      "[0.66224853] ; hl= 26\n",
      "[0.62961491] ; hl= 51\n",
      "Stopped Process\n"
     ]
    }
   ],
   "source": [
    "# May be the matter is our parameters, the answer is later no\n",
    "minim = 1\n",
    "m = 0.0001\n",
    "for i in range(1, 300, 25):\n",
    "    MLP_model = MLPRegressor(hidden_layer_sizes = (i, ), max_iter = 500, random_state = 0, alpha = 0.00003, learning_rate_init = 0.005)\n",
    "    MLP_model.fit(X_train, Y_train)\n",
    "    pred_MLP = MLP_model.predict(X_valid)\n",
    "    cr = metrics(pred_MLP.reshape(-1, 1), Y_valid)\n",
    "    if cr < minim:\n",
    "        minim = cr\n",
    "        print(cr, '; hl=', i)\n",
    "               \n",
    "print('Stopped Process')     "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.8728878]\n"
     ]
    }
   ],
   "source": [
    "# Sometimes this method works much better than Boosting, but it is hardly to be tuned properly \n",
    "from sklearn.svm import NuSVR\n",
    "\n",
    "SVR_model = NuSVR(nu = 0.999999, C = 0.5)\n",
    "SVR_model.fit(X_train, Y_train)\n",
    "pred_SVR = SVR_model.predict(X_valid)\n",
    "cr = metrics(pred_SVR.reshape(-1, 1), Y_valid)\n",
    "print(cr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load the test data\n",
    "test_values = pd.read_csv(data_path / 'test_values.csv',\n",
    "                         index_col = 0,\n",
    "                         parse_dates = ['timestamp'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create metadata and time series features\n",
    "test_features = create_feature_matrix(test_values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "# We use Boosting Trees to suggest our last solution\n",
    "preds = GB_model.predict(test_features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "# We load rules of making submissions\n",
    "submission_format = pd.read_csv(data_path / 'submission_format.csv', index_col = 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Confirm everything is in the right order\n",
    "assert np.all(test_features.index == submission_format.index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Our submission\n",
    "my_submission = pd.DataFrame(data = preds,\n",
    "                             columns = submission_format.columns,\n",
    "                             index = submission_format.index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Our task is done!\n",
    "my_submission.to_csv('submission20.csv')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}