99600.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from sklearn import ensemble, preprocessing\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.cluster import KMeans\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train = pd.read_csv('train.csv')\n",
    "test = pd.read_csv('test.csv')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "portfolio_id         0\n",
       "desk_id           3665\n",
       "office_id            0\n",
       "pf_category          0\n",
       "start_date           0\n",
       "sold                 2\n",
       "country_code         0\n",
       "euribor_rate         0\n",
       "currency             0\n",
       "libor_rate         474\n",
       "bought               2\n",
       "creation_date        0\n",
       "indicator_code    5699\n",
       "sell_date            0\n",
       "type                 0\n",
       "hedge_value       5701\n",
       "status            3084\n",
       "return               0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.isnull().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>portfolio_id</th>\n",
       "      <th>desk_id</th>\n",
       "      <th>office_id</th>\n",
       "      <th>pf_category</th>\n",
       "      <th>start_date</th>\n",
       "      <th>sold</th>\n",
       "      <th>country_code</th>\n",
       "      <th>euribor_rate</th>\n",
       "      <th>currency</th>\n",
       "      <th>libor_rate</th>\n",
       "      <th>bought</th>\n",
       "      <th>creation_date</th>\n",
       "      <th>indicator_code</th>\n",
       "      <th>sell_date</th>\n",
       "      <th>type</th>\n",
       "      <th>hedge_value</th>\n",
       "      <th>status</th>\n",
       "      <th>return</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>PF00001002</td>\n",
       "      <td>DSK00001001</td>\n",
       "      <td>OFF00001002</td>\n",
       "      <td>B</td>\n",
       "      <td>20040720</td>\n",
       "      <td>110000000.0</td>\n",
       "      <td>T</td>\n",
       "      <td>0.02074</td>\n",
       "      <td>USD</td>\n",
       "      <td>2.332216</td>\n",
       "      <td>1.098097e+08</td>\n",
       "      <td>20040720</td>\n",
       "      <td>NaN</td>\n",
       "      <td>20040812</td>\n",
       "      <td>B</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.02496</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>PF00001003</td>\n",
       "      <td>DSK00001002</td>\n",
       "      <td>OFF00001001</td>\n",
       "      <td>A</td>\n",
       "      <td>20040709</td>\n",
       "      <td>176671000.0</td>\n",
       "      <td>N</td>\n",
       "      <td>0.02074</td>\n",
       "      <td>GBP</td>\n",
       "      <td>5.269617</td>\n",
       "      <td>1.760084e+08</td>\n",
       "      <td>20040723</td>\n",
       "      <td>NaN</td>\n",
       "      <td>20040812</td>\n",
       "      <td>C</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.05496</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>PF00001005</td>\n",
       "      <td>DSK00001004</td>\n",
       "      <td>OFF00001001</td>\n",
       "      <td>A</td>\n",
       "      <td>20040723</td>\n",
       "      <td>56474000.0</td>\n",
       "      <td>T</td>\n",
       "      <td>0.02074</td>\n",
       "      <td>USD</td>\n",
       "      <td>2.332216</td>\n",
       "      <td>5.637953e+07</td>\n",
       "      <td>20040723</td>\n",
       "      <td>NaN</td>\n",
       "      <td>20040817</td>\n",
       "      <td>A</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.02496</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>PF00001006</td>\n",
       "      <td>DSK00001005</td>\n",
       "      <td>OFF00001001</td>\n",
       "      <td>A</td>\n",
       "      <td>20040609</td>\n",
       "      <td>164813000.0</td>\n",
       "      <td>T</td>\n",
       "      <td>0.02074</td>\n",
       "      <td>USD</td>\n",
       "      <td>2.332216</td>\n",
       "      <td>1.645088e+08</td>\n",
       "      <td>20040723</td>\n",
       "      <td>NaN</td>\n",
       "      <td>20040713</td>\n",
       "      <td>A</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.02496</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>PF00001007</td>\n",
       "      <td>DSK00001005</td>\n",
       "      <td>OFF00001002</td>\n",
       "      <td>B</td>\n",
       "      <td>20040609</td>\n",
       "      <td>140800000.0</td>\n",
       "      <td>T</td>\n",
       "      <td>0.02074</td>\n",
       "      <td>USD</td>\n",
       "      <td>2.332216</td>\n",
       "      <td>1.405402e+08</td>\n",
       "      <td>20040723</td>\n",
       "      <td>NaN</td>\n",
       "      <td>20040713</td>\n",
       "      <td>B</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.02496</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>PF00001008</td>\n",
       "      <td>DSK00001006</td>\n",
       "      <td>OFF00001001</td>\n",
       "      <td>A</td>\n",
       "      <td>20040707</td>\n",
       "      <td>48741000.0</td>\n",
       "      <td>T</td>\n",
       "      <td>0.02074</td>\n",
       "      <td>USD</td>\n",
       "      <td>2.332216</td>\n",
       "      <td>4.865127e+07</td>\n",
       "      <td>20040726</td>\n",
       "      <td>NaN</td>\n",
       "      <td>20040810</td>\n",
       "      <td>A</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.02490</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>PF00001010</td>\n",
       "      <td>DSK00001009</td>\n",
       "      <td>OFF00001001</td>\n",
       "      <td>A</td>\n",
       "      <td>20040706</td>\n",
       "      <td>60593500.0</td>\n",
       "      <td>T</td>\n",
       "      <td>0.02074</td>\n",
       "      <td>USD</td>\n",
       "      <td>2.332216</td>\n",
       "      <td>6.048181e+07</td>\n",
       "      <td>20040726</td>\n",
       "      <td>NaN</td>\n",
       "      <td>20040809</td>\n",
       "      <td>A</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.02493</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>PF00001011</td>\n",
       "      <td>DSK00001009</td>\n",
       "      <td>OFF00001002</td>\n",
       "      <td>B</td>\n",
       "      <td>20040706</td>\n",
       "      <td>134200000.0</td>\n",
       "      <td>T</td>\n",
       "      <td>0.02074</td>\n",
       "      <td>USD</td>\n",
       "      <td>2.332216</td>\n",
       "      <td>1.339526e+08</td>\n",
       "      <td>20040726</td>\n",
       "      <td>NaN</td>\n",
       "      <td>20040809</td>\n",
       "      <td>B</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.02493</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>PF00001012</td>\n",
       "      <td>DSK00001010</td>\n",
       "      <td>OFF00001001</td>\n",
       "      <td>A</td>\n",
       "      <td>20040419</td>\n",
       "      <td>82929000.0</td>\n",
       "      <td>T</td>\n",
       "      <td>0.02074</td>\n",
       "      <td>USD</td>\n",
       "      <td>2.332216</td>\n",
       "      <td>8.250616e+07</td>\n",
       "      <td>20040726</td>\n",
       "      <td>NaN</td>\n",
       "      <td>20040720</td>\n",
       "      <td>A</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.02460</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>PF00001016</td>\n",
       "      <td>DSK00001014</td>\n",
       "      <td>OFF00001001</td>\n",
       "      <td>A</td>\n",
       "      <td>20040414</td>\n",
       "      <td>212476000.0</td>\n",
       "      <td>T</td>\n",
       "      <td>0.02074</td>\n",
       "      <td>USD</td>\n",
       "      <td>2.332216</td>\n",
       "      <td>2.114140e+08</td>\n",
       "      <td>20040727</td>\n",
       "      <td>NaN</td>\n",
       "      <td>20040713</td>\n",
       "      <td>A</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.02466</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  portfolio_id      desk_id    office_id pf_category  start_date         sold  \\\n",
       "0   PF00001002  DSK00001001  OFF00001002           B    20040720  110000000.0   \n",
       "1   PF00001003  DSK00001002  OFF00001001           A    20040709  176671000.0   \n",
       "2   PF00001005  DSK00001004  OFF00001001           A    20040723   56474000.0   \n",
       "3   PF00001006  DSK00001005  OFF00001001           A    20040609  164813000.0   \n",
       "4   PF00001007  DSK00001005  OFF00001002           B    20040609  140800000.0   \n",
       "5   PF00001008  DSK00001006  OFF00001001           A    20040707   48741000.0   \n",
       "6   PF00001010  DSK00001009  OFF00001001           A    20040706   60593500.0   \n",
       "7   PF00001011  DSK00001009  OFF00001002           B    20040706  134200000.0   \n",
       "8   PF00001012  DSK00001010  OFF00001001           A    20040419   82929000.0   \n",
       "9   PF00001016  DSK00001014  OFF00001001           A    20040414  212476000.0   \n",
       "\n",
       "  country_code  euribor_rate currency  libor_rate        bought  \\\n",
       "0            T       0.02074      USD    2.332216  1.098097e+08   \n",
       "1            N       0.02074      GBP    5.269617  1.760084e+08   \n",
       "2            T       0.02074      USD    2.332216  5.637953e+07   \n",
       "3            T       0.02074      USD    2.332216  1.645088e+08   \n",
       "4            T       0.02074      USD    2.332216  1.405402e+08   \n",
       "5            T       0.02074      USD    2.332216  4.865127e+07   \n",
       "6            T       0.02074      USD    2.332216  6.048181e+07   \n",
       "7            T       0.02074      USD    2.332216  1.339526e+08   \n",
       "8            T       0.02074      USD    2.332216  8.250616e+07   \n",
       "9            T       0.02074      USD    2.332216  2.114140e+08   \n",
       "\n",
       "   creation_date indicator_code  sell_date type hedge_value status   return  \n",
       "0       20040720            NaN   20040812    B         NaN    NaN  0.02496  \n",
       "1       20040723            NaN   20040812    C         NaN    NaN  0.05496  \n",
       "2       20040723            NaN   20040817    A         NaN    NaN  0.02496  \n",
       "3       20040723            NaN   20040713    A         NaN    NaN  0.02496  \n",
       "4       20040723            NaN   20040713    B         NaN    NaN  0.02496  \n",
       "5       20040726            NaN   20040810    A         NaN    NaN  0.02490  \n",
       "6       20040726            NaN   20040809    A         NaN    NaN  0.02493  \n",
       "7       20040726            NaN   20040809    B         NaN    NaN  0.02493  \n",
       "8       20040726            NaN   20040720    A         NaN    NaN  0.02460  \n",
       "9       20040727            NaN   20040713    A         NaN    NaN  0.02466  "
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "col_more = []\n",
    "col_less = []\n",
    "train.replace(np.nan,-1,inplace= True)\n",
    "test.replace(np.nan,-1,inplace= True)\n",
    "for i in train.columns:\n",
    "    if train[i].dtype == 'object':\n",
    "        if len(np.unique(list(train[i])+list(test[i]))) >30:\n",
    "            col_more.append(i)\n",
    "        else:\n",
    "            col_less.append(i)\n",
    "        #print i, len(np.unique(list(train[i])+list(test[i]))),len(np.setdiff1d(np.unique(test[i]),np.unique(train[i])))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "E    2940\n",
       "A    1171\n",
       "B     351\n",
       "C     289\n",
       "F      26\n",
       "D      21\n",
       "H       3\n",
       "Name: type, dtype: int64"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test.type.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "E    5734\n",
       "A    2316\n",
       "B     638\n",
       "C     562\n",
       "F      61\n",
       "D      47\n",
       "H       7\n",
       "G       1\n",
       "Name: type, dtype: int64"
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.type.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "\n",
    "train['creation_date']  = pd.to_datetime(train['creation_date'],format='%Y%m%d', errors='ignore')\n",
    "train['start_date']  = pd.to_datetime(train['start_date'],format='%Y%m%d', errors='ignore')\n",
    "train['sell_date']  = pd.to_datetime(train['sell_date'],format='%Y%m%d', errors='ignore')\n",
    "\n",
    "test['creation_date']  = pd.to_datetime(test['creation_date'],format='%Y%m%d', errors='ignore')\n",
    "test['start_date']  = pd.to_datetime(test['start_date'],format='%Y%m%d', errors='ignore')\n",
    "test['sell_date']  = pd.to_datetime(test['sell_date'],format='%Y%m%d', errors='ignore')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train['sell_date_dow']  = train.sell_date.dt.dayofweek\n",
    "test['sell_date_dow']  = test.sell_date.dt.dayofweek\n",
    "\n",
    "train['sell_date_month']  = train.sell_date.dt.month\n",
    "test['sell_date_month']  = test.sell_date.dt.month\n",
    "\n",
    "train['sell_date_dayofyear']  = train.sell_date.dt.dayofyear\n",
    "test['sell_date_dayofyear']  = test.sell_date.dt.dayofyear\n",
    "\n",
    "train['start_date_dayofyear']  = train.start_date.dt.day\n",
    "test['start_date_dayofyear']  = test.start_date.dt.day\n",
    "\n",
    "\n",
    "train['sell_date_year']  = train.sell_date.dt.year\n",
    "test['sell_date_year']  = test.sell_date.dt.year\n",
    "\n",
    "train['sell_date_week_number']  = train.sell_date.dt.week\n",
    "test['sell_date_week_number']  = test.sell_date.dt.week\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "\n",
    "train['difference_s_s'] = (train.start_date - train.sell_date).apply(lambda x: int(str(x).split(' ')[0]) )\n",
    "test['difference_s_s'] = (test.start_date - test.sell_date).apply(lambda x: int(str(x).split(' ')[0]) )\n",
    "\n",
    "test['difference_s_s'] = test['difference_s_s'].astype(int)\n",
    "train['difference_s_s'] = train['difference_s_s'].astype(int)\n",
    "\n",
    "train['difference_c_s'] = (train.creation_date - train.sell_date).apply(lambda x: int(str(x).split(' ')[0]) )\n",
    "test['difference_c_s'] = (test.creation_date - test.sell_date).apply(lambda x: int(str(x).split(' ')[0]) )\n",
    "\n",
    "test['difference_c_s'] = test['difference_c_s'].astype(int)\n",
    "train['difference_c_s'] = train['difference_c_s'].astype(int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train['return1'] = (train['sold'] -train['bought'])*1.0/train['bought']\n",
    "test['return1'] = (test['sold'] -test['bought'])*1.0/test['bought']\n",
    "\n",
    "total = pd.concat((train,test))\n",
    "\n",
    "k = total[['desk_id','currency','portfolio_id']].groupby(['desk_id','currency']).agg('count').reset_index()\n",
    "k = k.groupby('desk_id').agg('count')['currency'].reset_index()\n",
    "k.columns = ['desk_id','desk_unique_count']\n",
    "train = train.merge(k,on='desk_id',how='left')\n",
    "test = test.merge(k,on='desk_id',how='left')\n",
    "\n",
    "\n",
    "k1 = total[['sell_date_month','sell_date_year','desk_id','return1']].groupby(['desk_id','sell_date_month','sell_date_year']).agg('mean')['return1'].reset_index()\n",
    "k1.columns = ['desk_id','sell_date_month','sell_date_year','mean_val_sell_date_month']\n",
    "train = train.merge(k1,on=['desk_id','sell_date_month','sell_date_year'],how='left')\n",
    "test = test.merge(k1,on=['desk_id','sell_date_month','sell_date_year'],how='left')\n",
    "\n",
    "k1 = total[['sell_date_month','return1']].groupby(['sell_date_month']).agg('mean')['return1'].reset_index()\n",
    "k1.columns = ['sell_date_month','mean_val_sell_date_month']\n",
    "train = train.merge(k1,on='sell_date_month',how='left')\n",
    "\n",
    "test = test.merge(k1,on='sell_date_month',how='left')\n",
    "k1 = total[['start_date_dayofyear','return1']].groupby(['start_date_dayofyear']).agg('mean')['return1'].reset_index()\n",
    "k1.columns = ['start_date_dayofyear','mean_val_start_date_dayofyear']\n",
    "train = train.merge(k1,on='start_date_dayofyear',how='left')\n",
    "test = test.merge(k1,on='start_date_dayofyear',how='left')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "return1\n"
     ]
    }
   ],
   "source": [
    "col_more = ['return1']\n",
    "for enu,i in enumerate(col_more):\n",
    "    print i\n",
    "    \n",
    "    total=pd.concat((train[[i,'sell_date_month','sell_date_year','currency']],test[[i,'sell_date_month','sell_date_year','currency']]))\n",
    "    k = total.groupby(['sell_date_month','sell_date_year','currency']).agg('min').reset_index()\n",
    "    k.columns = ['sell_date_month','sell_date_year','currency',i+'_min']\n",
    "    train = train.merge(k,on=['sell_date_month','sell_date_year','currency'],how='left')\n",
    "    test = test.merge(k,on=['sell_date_month','sell_date_year','currency'],how='left')\n",
    "    \n",
    "col_more = ['sold','bought']\n",
    "col_more.append('sell_date_week_number')\n",
    "train = train.drop(col_more,axis=1)\n",
    "test = test.drop(col_more,axis=1)\n",
    "\n",
    "y  = train['return'].values\n",
    "train = train.drop('return',axis =1)\n",
    "\n",
    "train['euribor_rate'] = np.log1p(train.euribor_rate)+1\n",
    "test['euribor_rate'] = np.log1p(test.euribor_rate)+1\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train.replace(np.nan,-1,inplace=True)\n",
    "test.replace(np.nan,-1,inplace=True)\n",
    "\n",
    "p_id = test.portfolio_id.values\n",
    "\n",
    "train = train.drop(['creation_date','start_date','sell_date','country_code'],axis =1)\n",
    "test = test.drop(['creation_date','start_date','sell_date','country_code'],axis =1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "text_columns = []\n",
    "for f in train.columns:\n",
    "    if train[f].dtype== 'object':    \n",
    "        text_columns.append(f)            \n",
    "        lbl = preprocessing.LabelEncoder()\n",
    "        lbl.fit(list(train[f].values) + list(test[f].values))\n",
    "        train[f] = lbl.transform(list(train[f].values))\n",
    "        test[f] = lbl.transform(list(test[f].values))    \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import xgboost as xgb\n",
    "from sklearn.metrics import r2_score\n",
    "from sklearn.metrics import  roc_auc_score\n",
    "def runXGB(train_X, train_y, test_X, test_y=None, feature_names=None, seed_val=0,depth =8):\n",
    "        params = {}\n",
    "        params[\"objective\"] = \"reg:linear\"\n",
    "        params['eval_metric'] = 'rmse'\n",
    "        params[\"eta\"] = 0.01 #0.00334\n",
    "        params[\"min_child_weight\"] = 1\n",
    "        params[\"subsample\"] = 0.8\n",
    "        params[\"colsample_bytree\"] = 0.3\n",
    "        params[\"silent\"] = 1\n",
    "        params[\"max_depth\"] = depth\n",
    "        params[\"seed\"] = seed_val\n",
    "        #params[\"max_delta_step\"] = 2\n",
    "        #params[\"gamma\"] = 0.5\n",
    "        num_rounds = 5000 #2500\n",
    "\n",
    "        plst = list(params.items())\n",
    "        xgtrain = xgb.DMatrix(train_X, label=train_y)\n",
    "\n",
    "        if test_y is not None:\n",
    "                xgtest = xgb.DMatrix(test_X, label=test_y)\n",
    "                watchlist = [ (xgtrain,'train'), (xgtest, 'test') ]\n",
    "                model = xgb.train(plst, xgtrain, num_rounds, watchlist, early_stopping_rounds= 500)\n",
    "        else:\n",
    "                xgtest = xgb.DMatrix(test_X)\n",
    "                #xgtest1 = xgb.DMatrix(test_X1)\n",
    "                model = xgb.train(plst, xgtrain, 4500)\n",
    "\n",
    "        if feature_names:\n",
    "                        create_feature_map(feature_names)\n",
    "                        model.dump_model('xgbmodel.txt', 'xgb.fmap', with_stats=True)\n",
    "                        importance = model.get_fscore(fmap='xgb.fmap')\n",
    "                        importance = sorted(importance.items(), key=operator.itemgetter(1), reverse=True)\n",
    "                        imp_df = pd.DataFrame(importance, columns=['feature','fscore'])\n",
    "                        imp_df['fscore'] = imp_df['fscore'] / imp_df['fscore'].sum()\n",
    "                        imp_df.to_csv(\"imp_feat.txt\", index=False)\n",
    "\n",
    "        pred_test_y = model.predict(xgtest)\n",
    "        \n",
    "        if test_y is not None:\n",
    "                loss = r2_score(test_y, pred_test_y)\n",
    "                print loss\n",
    "        \treturn pred_test_y, loss\n",
    "\telse:\n",
    "\t\treturn pred_test_y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"\\nfrom sklearn.linear_model import LinearRegression\\ng_train = np.zeros(len(y))\\ng1_train = np.zeros(len(y))\\n\\ng_test = np.zeros(test.shape[0])\\ng1_test = np.zeros(test.shape[0])\\n\\n\\nfor i in np.unique(train.currency):\\n    print i\\n    ind_train = train.loc[train.currency==i].index\\n    ind_test = test.loc[test.currency==i].index\\n    train_train = train.iloc[ind_train]\\n    y_train1 = y[ind_train]\\n    test_test = test.iloc[ind_test]\\n    \\n    \\n    model = LinearRegression()\\n    model.fit(train_train,y_train1)\\n    pred_train = model.predict(train_train)\\n    pred_test = model.predict(test_test)\\n    g1_train[ind_train] = pred_train\\n    g1_test[ind_test] = pred_test\\n    \\n    \\n    k = np.log1p(y_train1+1)    \\n    model.fit(train_train,k)\\n    pred_train = np.expm1(model.predict(train_train))-1\\n    pred_test = np.expm1(model.predict(test_test))-1\\n    g_train[ind_train] = pred_train\\n    g_test[ind_test] = pred_test\\n    \\ntrain['g1'] = g1_train\\ntest['g1'] = g1_test\\n\\ntrain['g'] = g_train\\ntest['g'] = g_test\\n\""
      ]
     },
     "execution_count": 76,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\"\"\"\n",
    "from sklearn.linear_model import LinearRegression\n",
    "g_train = np.zeros(len(y))\n",
    "g1_train = np.zeros(len(y))\n",
    "\n",
    "g_test = np.zeros(test.shape[0])\n",
    "g1_test = np.zeros(test.shape[0])\n",
    "\n",
    "\n",
    "for i in np.unique(train.currency):\n",
    "    print i\n",
    "    ind_train = train.loc[train.currency==i].index\n",
    "    ind_test = test.loc[test.currency==i].index\n",
    "    train_train = train.iloc[ind_train]\n",
    "    y_train1 = y[ind_train]\n",
    "    test_test = test.iloc[ind_test]\n",
    "    \n",
    "    \n",
    "    model = LinearRegression()\n",
    "    model.fit(train_train,y_train1)\n",
    "    pred_train = model.predict(train_train)\n",
    "    pred_test = model.predict(test_test)\n",
    "    g1_train[ind_train] = pred_train\n",
    "    g1_test[ind_test] = pred_test\n",
    "    \n",
    "    \n",
    "    k = np.log1p(y_train1+1)    \n",
    "    model.fit(train_train,k)\n",
    "    pred_train = np.expm1(model.predict(train_train))-1\n",
    "    pred_test = np.expm1(model.predict(test_test))-1\n",
    "    g_train[ind_train] = pred_train\n",
    "    g_test[ind_test] = pred_test\n",
    "    \n",
    "train['g1'] = g1_train\n",
    "test['g1'] = g1_test\n",
    "\n",
    "train['g'] = g_train\n",
    "test['g'] = g_test\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "X_train, X_test, y_train, y_test = train_test_split(train, y, test_size=0.3, random_state=42)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.582189107602\n",
      "0.991106572534\n",
      "0.996356250897\n",
      "0.997846136988\n",
      "0.993608128134\n",
      "0.912221239231\n"
     ]
    }
   ],
   "source": [
    "\n",
    "from sklearn.metrics import r2_score\n",
    "from sklearn.cross_validation import StratifiedKFold\n",
    "np.random.seed(0) # seed to shuffle the train set\n",
    "n_folds = 5\n",
    "#0.903907431605\n",
    "skf = list(StratifiedKFold(y, n_folds))\n",
    "train2 = train.copy()\n",
    "train2['pred'] = -1\n",
    "train2['fold'] =-1\n",
    "f = []\n",
    "for i, (train1, test1) in enumerate(skf): \n",
    "        #print \"Fold\", i\n",
    "        X_train = train.loc[train1]\n",
    "        y_train = y[train1]\n",
    "        X_test = train.loc[test1]\n",
    "        y_test = y[test1]\n",
    "        k = runXGB(X_train, y_train, X_test)\n",
    "        train2.loc[test1,'pred']=k\n",
    "        train2.loc[test1,'fold']=i\n",
    "        print (r2_score(y_test,k))\n",
    "        f.append(r2_score(y_test,k))\n",
    "print np.mean(f)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'\\n0.582189107602\\n0.991106572534\\n0.996356250897\\n0.997846136988\\n0.993608128134\\n0.912221239231\\n\\n'"
      ]
     },
     "execution_count": 79,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\"\"\"\n",
    "0.582189107602\n",
    "0.991106572534\n",
    "0.996356250897\n",
    "0.997846136988\n",
    "0.993608128134\n",
    "0.912221239231\n",
    "\n",
    "\"\"\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n",
      "5\n",
      "10\n",
      "15\n",
      "20\n",
      "25\n",
      "30\n",
      "35\n",
      "40\n",
      "45\n",
      "50\n",
      "55\n",
      "60\n",
      "65\n",
      "70\n",
      "75\n",
      "80\n",
      "85\n",
      "90\n",
      "95\n"
     ]
    }
   ],
   "source": [
    "final_prediction = []\n",
    "for i in range(0,100):\n",
    "    if i%5==0:\n",
    "        print i\n",
    "    k = (y-np.min(y)) *1.0/(np.max(y)-np.min(y))\n",
    "    pred6 = runXGB(train,k , test,depth =10,seed_val = i)\n",
    "    pred6 = pred6*(np.max(y)-np.min(y)) + np.min(y)\n",
    "    k = (y-np.min(y)) *1.0/(np.max(y)-np.min(y))\n",
    "    pred7 = runXGB(train,k , test,depth =12,seed_val =i)\n",
    "    pred7 = pred7*(np.max(y)-np.min(y)) + np.min(y)\n",
    "    k = (y-np.min(y)) *1.0/(np.max(y)-np.min(y))\n",
    "    pred8 = runXGB(train,k , test,depth =14,seed_val =i)\n",
    "    pred8 = pred8*(np.max(y)-np.min(y)) + np.min(y)\n",
    "    pred6_7 =(pred6+pred7+pred8)*1.0/3 \n",
    "    final_prediction.append(pred6)\n",
    "    final_prediction.append(pred7)\n",
    "    final_prediction.append(pred8)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.DataFrame({'portfolio_id':p_id,'return':np.median(final_prediction,axis = 0)}).to_csv('100_models.csv',index =False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 0.02519561,  0.0272034 ,  0.0245437 , ...,  0.0058769 ,\n",
       "        0.00534486,  0.00586923], dtype=float32)"
      ]
     },
     "execution_count": 82,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.mean(final_prediction,axis = 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 0.02514424,  0.02621793,  0.02434616, ...,  0.00591265,\n",
       "        0.00535696,  0.00586584], dtype=float32)"
      ]
     },
     "execution_count": 84,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pred6"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 0.02511298,  0.02619646,  0.02439666, ...,  0.0058863 ,\n",
       "        0.00533841,  0.00588477], dtype=float32)"
      ]
     },
     "execution_count": 85,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pred7"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 0.02509124,  0.02617314,  0.02439369, ...,  0.00587329,\n",
       "        0.00535029,  0.00589438], dtype=float32)"
      ]
     },
     "execution_count": 86,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pred8"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}