pytorch-resnet-training.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-02-01T03:36:22.212933Z",
     "iopub.status.busy": "2021-02-01T03:36:22.161139Z",
     "iopub.status.idle": "2021-02-01T03:38:56.870076Z",
     "shell.execute_reply": "2021-02-01T03:38:56.869284Z"
    },
    "papermill": {
     "duration": 154.732515,
     "end_time": "2021-02-01T03:38:56.870212",
     "exception": false,
     "start_time": "2021-02-01T03:36:22.137697",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import time\n",
    "import pickle\n",
    "import random\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from tqdm import tqdm\n",
    "from sklearn.metrics import log_loss, roc_auc_score\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "from torch.autograd import Variable\n",
    "from torch.utils.data import DataLoader\n",
    "from torch.nn import CrossEntropyLoss, MSELoss\n",
    "from torch.nn.modules.loss import _WeightedLoss\n",
    "import torch.nn.functional as F\n",
    "\n",
    "pd.set_option('display.max_columns', 100)\n",
    "pd.set_option('display.max_rows', 100)\n",
    "\n",
    "DATA_PATH = '../input/jane-street-market-prediction/'\n",
    "\n",
    "BATCH_SIZE =4096 # 8192\n",
    "EPOCHS = 200\n",
    "LEARNING_RATE = 1e-3\n",
    "WEIGHT_DECAY = 1e-5\n",
    "EARLYSTOP_NUM = 3\n",
    "FOLDS = 5\n",
    "\n",
    "TRAIN = True\n",
    "CACHE_PATH = './'\n",
    "\n",
    "train = pd.read_csv(f'{DATA_PATH}/train.csv')\n",
    "\n",
    "import numpy as np\n",
    "from sklearn.model_selection import KFold\n",
    "from sklearn.model_selection._split import _BaseKFold, indexable, _num_samples\n",
    "from sklearn.utils.validation import _deprecate_positional_args\n",
    "\n",
    "class PurgedGroupTimeSeriesSplit(_BaseKFold):\n",
    "    \"\"\"Time Series cross-validator variant with non-overlapping groups.\n",
    "    Allows for a gap in groups to avoid potentially leaking info from\n",
    "    train into test if the model has windowed or lag features.\n",
    "    Provides train/test indices to split time series data samples\n",
    "    that are observed at fixed time intervals according to a\n",
    "    third-party provided group.\n",
    "    In each split, test indices must be higher than before, and thus shuffling\n",
    "    in cross validator is inappropriate.\n",
    "    This cross-validation object is a variation of :class:`KFold`.\n",
    "    In the kth split, it returns first k folds as train set and the\n",
    "    (k+1)th fold as test set.\n",
    "    The same group will not appear in two different folds (the number of\n",
    "    distinct groups has to be at least equal to the number of folds).\n",
    "    Note that unlike standard cross-validation methods, successive\n",
    "    training sets are supersets of those that come before them.\n",
    "    Read more in the :ref:`User Guide <cross_validation>`.\n",
    "    Parameters\n",
    "    ----------\n",
    "    n_splits : int, default=5\n",
    "        Number of splits. Must be at least 2.\n",
    "    max_train_group_size : int, default=Inf\n",
    "        Maximum group size for a single training set.\n",
    "    group_gap : int, default=None\n",
    "        Gap between train and test\n",
    "    max_test_group_size : int, default=Inf\n",
    "        We discard this number of groups from the end of each train split\n",
    "    \"\"\"\n",
    "\n",
    "    @_deprecate_positional_args\n",
    "    def __init__(self,\n",
    "                 n_splits=5,\n",
    "                 *,\n",
    "                 max_train_group_size=np.inf,\n",
    "                 max_test_group_size=np.inf,\n",
    "                 group_gap=None,\n",
    "                 verbose=False\n",
    "                 ):\n",
    "        super().__init__(n_splits, shuffle=False, random_state=None)\n",
    "        self.max_train_group_size = max_train_group_size\n",
    "        self.group_gap = group_gap\n",
    "        self.max_test_group_size = max_test_group_size\n",
    "        self.verbose = verbose\n",
    "\n",
    "    def split(self, X, y=None, groups=None):\n",
    "        \"\"\"Generate indices to split data into training and test set.\n",
    "        Parameters\n",
    "        ----------\n",
    "        X : array-like of shape (n_samples, n_features)\n",
    "            Training data, where n_samples is the number of samples\n",
    "            and n_features is the number of features.\n",
    "        y : array-like of shape (n_samples,)\n",
    "            Always ignored, exists for compatibility.\n",
    "        groups : array-like of shape (n_samples,)\n",
    "            Group labels for the samples used while splitting the dataset into\n",
    "            train/test set.\n",
    "        Yields\n",
    "        ------\n",
    "        train : ndarray\n",
    "            The training set indices for that split.\n",
    "        test : ndarray\n",
    "            The testing set indices for that split.\n",
    "        \"\"\"\n",
    "        if groups is None:\n",
    "            raise ValueError(\n",
    "                \"The 'groups' parameter should not be None\")\n",
    "        X, y, groups = indexable(X, y, groups)\n",
    "        n_samples = _num_samples(X)\n",
    "        n_splits = self.n_splits\n",
    "        group_gap = self.group_gap\n",
    "        max_test_group_size = self.max_test_group_size\n",
    "        max_train_group_size = self.max_train_group_size\n",
    "        n_folds = n_splits + 1\n",
    "        group_dict = {}\n",
    "        u, ind = np.unique(groups, return_index=True)\n",
    "        unique_groups = u[np.argsort(ind)]\n",
    "        n_samples = _num_samples(X)\n",
    "        n_groups = _num_samples(unique_groups)\n",
    "        for idx in np.arange(n_samples):\n",
    "            if (groups[idx] in group_dict):\n",
    "                group_dict[groups[idx]].append(idx)\n",
    "            else:\n",
    "                group_dict[groups[idx]] = [idx]\n",
    "        if n_folds > n_groups:\n",
    "            raise ValueError(\n",
    "                (\"Cannot have number of folds={0} greater than\"\n",
    "                 \" the number of groups={1}\").format(n_folds,\n",
    "                                                     n_groups))\n",
    "\n",
    "        group_test_size = min(n_groups // n_folds, max_test_group_size)\n",
    "        group_test_starts = range(n_groups - n_splits * group_test_size,\n",
    "                                  n_groups, group_test_size)\n",
    "        for group_test_start in group_test_starts:\n",
    "            train_array = []\n",
    "            test_array = []\n",
    "\n",
    "            group_st = max(0, group_test_start - group_gap - max_train_group_size)\n",
    "            for train_group_idx in unique_groups[group_st:(group_test_start - group_gap)]:\n",
    "                train_array_tmp = group_dict[train_group_idx]\n",
    "                \n",
    "                train_array = np.sort(np.unique(\n",
    "                                      np.concatenate((train_array,\n",
    "                                                      train_array_tmp)),\n",
    "                                      axis=None), axis=None)\n",
    "\n",
    "            train_end = train_array.size\n",
    " \n",
    "            for test_group_idx in unique_groups[group_test_start:\n",
    "                                                group_test_start +\n",
    "                                                group_test_size]:\n",
    "                test_array_tmp = group_dict[test_group_idx]\n",
    "                test_array = np.sort(np.unique(\n",
    "                                              np.concatenate((test_array,\n",
    "                                                              test_array_tmp)),\n",
    "                                     axis=None), axis=None)\n",
    "\n",
    "            test_array  = test_array[group_gap:]\n",
    "            \n",
    "            \n",
    "            if self.verbose > 0:\n",
    "                    pass\n",
    "                    \n",
    "            yield [int(i) for i in train_array], [int(i) for i in test_array]\n",
    "\n",
    "def save_pickle(dic, save_path):\n",
    "    with open(save_path, 'wb') as f:\n",
    "    # with gzip.open(save_path, 'wb') as f:\n",
    "        pickle.dump(dic, f)\n",
    "\n",
    "def load_pickle(load_path):\n",
    "    with open(load_path, 'rb') as f:\n",
    "    # with gzip.open(load_path, 'rb') as f:\n",
    "        message_dict = pickle.load(f)\n",
    "    return message_dict\n",
    "\n",
    "class EarlyStopping:\n",
    "    def __init__(self, patience=7, mode=\"max\", delta=0.001):\n",
    "        self.patience = patience\n",
    "        self.counter = 0\n",
    "        self.mode = mode\n",
    "        self.best_score = None\n",
    "        self.early_stop = False\n",
    "        self.delta = delta\n",
    "        if self.mode == \"min\":\n",
    "            self.val_score = np.Inf\n",
    "        else:\n",
    "            self.val_score = -np.Inf\n",
    "\n",
    "    def __call__(self, epoch_score, model, model_path):\n",
    "\n",
    "        if self.mode == \"min\":\n",
    "            score = -1.0 * epoch_score\n",
    "        else:\n",
    "            score = np.copy(epoch_score)\n",
    "\n",
    "        if self.best_score is None:\n",
    "            self.best_score = score\n",
    "            self.save_checkpoint(epoch_score, model, model_path)\n",
    "        elif score < self.best_score: #  + self.delta\n",
    "            self.counter += 1\n",
    "            # print('EarlyStopping counter: {} out of {}'.format(self.counter, self.patience))\n",
    "            if self.counter >= self.patience:\n",
    "                self.early_stop = True\n",
    "        else:\n",
    "            self.best_score = score\n",
    "            # ema.apply_shadow()\n",
    "            self.save_checkpoint(epoch_score, model, model_path)\n",
    "            # ema.restore()\n",
    "            self.counter = 0\n",
    "\n",
    "    def save_checkpoint(self, epoch_score, model, model_path):\n",
    "        if epoch_score not in [-np.inf, np.inf, -np.nan, np.nan]:\n",
    "            # print('Validation score improved ({} --> {}). Saving model!'.format(self.val_score, epoch_score))\n",
    "            # if not DEBUG:\n",
    "            torch.save(model.state_dict(), model_path)\n",
    "        self.val_score = epoch_score\n",
    "\n",
    "def seed_everything(seed=xx):\n",
    "    random.seed(seed)\n",
    "    os.environ['PYTHONHASHSEED'] = str(seed)\n",
    "    np.random.seed(seed)\n",
    "    torch.manual_seed(seed)\n",
    "    torch.cuda.manual_seed(seed)\n",
    "    torch.backends.cudnn.deterministic = True\n",
    "seed_everything(seed=xx)\n",
    "\n",
    "feat_cols = [f'feature_{i}' for i in range(130)]\n",
    "\n",
    "if TRAIN:\n",
    "    train = train.loc[train.date > 85].reset_index(drop=True)\n",
    "    train = train[train['weight'] != 0]\n",
    "    train['action'] = (train['resp'] > 0).astype('int')\n",
    "    train['action_1'] = (train['resp_1'] > 0).astype('int')\n",
    "    train['action_2'] = (train['resp_2'] > 0).astype('int')\n",
    "    train['action_3'] = (train['resp_3'] > 0).astype('int')\n",
    "    train['action_4'] = (train['resp_4'] > 0).astype('int')\n",
    "target_cols = ['action', 'action_1', 'action_2', 'action_3', 'action_4']\n",
    "\n",
    "if TRAIN:\n",
    "    df = train[feat_cols]\n",
    "    f_mean = df.median() # df.mean()\n",
    "    f_mean = f_mean.values\n",
    "    np.save(f'{CACHE_PATH}/f_mean_online.npy', f_mean)\n",
    "\n",
    "    train.fillna(df.median(), inplace=True) # df.mean()\n",
    "\n",
    "else:\n",
    "    f_mean = np.load(f'{CACHE_PATH}/f_mean_online.npy')\n",
    "\n",
    "\n",
    "def fillna_npwhere_njit(array, values):\n",
    "    if np.isnan(array.sum()):\n",
    "        array = np.where(np.isnan(array), values, array)\n",
    "    return array\n",
    "\n",
    "class RunningEWMean:\n",
    "    def __init__(self, WIN_SIZE=20, n_size=1, lt_mean=None):\n",
    "        if lt_mean is not None:\n",
    "            self.s = lt_mean\n",
    "        else:\n",
    "            self.s = np.zeros(n_size)\n",
    "        self.past_value = np.zeros(n_size)\n",
    "        self.alpha = 2 / (WIN_SIZE + 1)\n",
    "\n",
    "    def clear(self):\n",
    "        self.s = 0\n",
    "\n",
    "    def push(self, x):\n",
    "\n",
    "        x = fillna_npwhere_njit(x, self.past_value)\n",
    "        self.past_value = x\n",
    "        self.s = self.alpha * x + (1 - self.alpha) * self.s\n",
    "\n",
    "    def get_mean(self):\n",
    "        return self.s\n",
    "\n",
    "if TRAIN:\n",
    "    all_feat_cols = [col for col in feat_cols]\n",
    "\n",
    "    train['cross_41_42_43'] = train['feature_41'] + train['feature_42'] + train['feature_43']\n",
    "    train['cross_1_2'] = train['feature_1'] / (train['feature_2'] + 1e-5)\n",
    "\n",
    "    all_feat_cols.extend(['cross_41_42_43', 'cross_1_2'])\n",
    "\n",
    "##### Model&Data fnc\n",
    "class SmoothBCEwLogits(_WeightedLoss):\n",
    "    def __init__(self, weight=None, reduction='mean', smoothing=0.0):\n",
    "        super().__init__(weight=weight, reduction=reduction)\n",
    "        self.smoothing = smoothing\n",
    "        self.weight = weight\n",
    "        self.reduction = reduction\n",
    "\n",
    "    @staticmethod\n",
    "    def _smooth(targets:torch.Tensor, n_labels:int, smoothing=0.0):\n",
    "        assert 0 <= smoothing < 1\n",
    "        with torch.no_grad():\n",
    "            targets = targets * (1.0 - smoothing) + 0.5 * smoothing\n",
    "        return targets\n",
    "\n",
    "    def forward(self, inputs, targets):\n",
    "        targets = SmoothBCEwLogits._smooth(targets, inputs.size(-1),\n",
    "            self.smoothing)\n",
    "        loss = F.binary_cross_entropy_with_logits(inputs, targets,self.weight)\n",
    "\n",
    "        if  self.reduction == 'sum':\n",
    "            loss = loss.sum()\n",
    "        elif  self.reduction == 'mean':\n",
    "            loss = loss.mean()\n",
    "\n",
    "        return loss\n",
    "\n",
    "class MarketDataset:\n",
    "    def __init__(self, df):\n",
    "        self.features = df[all_feat_cols].values\n",
    "\n",
    "        self.label = df[target_cols].values.reshape(-1, len(target_cols))\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.label)\n",
    "\n",
    "    def __getitem__(self, idx):\n",
    "        return {\n",
    "            'features': torch.tensor(self.features[idx], dtype=torch.float),\n",
    "            'label': torch.tensor(self.label[idx], dtype=torch.float)\n",
    "        }\n",
    "\n",
    "\n",
    "class Model(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(Model, self).__init__()\n",
    "        self.batch_norm0 = nn.BatchNorm1d(len(all_feat_cols))\n",
    "        self.dropout0 = nn.Dropout(0.2)\n",
    "\n",
    "        dropout_rate = 0.2\n",
    "        hidden_size = 256\n",
    "        self.dense1 = nn.Linear(len(all_feat_cols), hidden_size)\n",
    "        self.batch_norm1 = nn.BatchNorm1d(hidden_size)\n",
    "        self.dropout1 = nn.Dropout(dropout_rate)\n",
    "\n",
    "        self.dense2 = nn.Linear(hidden_size+len(all_feat_cols), hidden_size)\n",
    "        self.batch_norm2 = nn.BatchNorm1d(hidden_size)\n",
    "        self.dropout2 = nn.Dropout(dropout_rate)\n",
    "\n",
    "        self.dense3 = nn.Linear(hidden_size+hidden_size, hidden_size)\n",
    "        self.batch_norm3 = nn.BatchNorm1d(hidden_size)\n",
    "        self.dropout3 = nn.Dropout(dropout_rate)\n",
    "\n",
    "        self.dense4 = nn.Linear(hidden_size+hidden_size, hidden_size)\n",
    "        self.batch_norm4 = nn.BatchNorm1d(hidden_size)\n",
    "        self.dropout4 = nn.Dropout(dropout_rate)\n",
    "\n",
    "        self.dense5 = nn.Linear(hidden_size+hidden_size, len(target_cols))\n",
    "\n",
    "        self.Relu = nn.ReLU(inplace=True)\n",
    "        self.PReLU = nn.PReLU()\n",
    "        self.LeakyReLU = nn.LeakyReLU(negative_slope=0.01, inplace=True)\n",
    "        # self.GeLU = nn.GELU()\n",
    "        self.RReLU = nn.RReLU()\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = self.batch_norm0(x)\n",
    "        x = self.dropout0(x)\n",
    "\n",
    "        x1 = self.dense1(x)\n",
    "        x1 = self.batch_norm1(x1)\n",
    "        # x = F.relu(x)\n",
    "        # x = self.PReLU(x)\n",
    "        x1 = self.LeakyReLU(x1)\n",
    "        x1 = self.dropout1(x1)\n",
    "\n",
    "        x = torch.cat([x, x1], 1)\n",
    "\n",
    "        x2 = self.dense2(x)\n",
    "        x2 = self.batch_norm2(x2)\n",
    "        # x = F.relu(x)\n",
    "        # x = self.PReLU(x)\n",
    "        x2 = self.LeakyReLU(x2)\n",
    "        x2 = self.dropout2(x2)\n",
    "\n",
    "        x = torch.cat([x1, x2], 1)\n",
    "\n",
    "        x3 = self.dense3(x)\n",
    "        x3 = self.batch_norm3(x3)\n",
    "        # x = F.relu(x)\n",
    "        # x = self.PReLU(x)\n",
    "        x3 = self.LeakyReLU(x3)\n",
    "        x3 = self.dropout3(x3)\n",
    "\n",
    "        x = torch.cat([x2, x3], 1)\n",
    "\n",
    "        x4 = self.dense4(x)\n",
    "        x4 = self.batch_norm4(x4)\n",
    "        # x = F.relu(x)\n",
    "        # x = self.PReLU(x)\n",
    "        x4 = self.LeakyReLU(x4)\n",
    "        x4 = self.dropout4(x4)\n",
    "\n",
    "        x = torch.cat([x3, x4], 1)\n",
    "\n",
    "        x = self.dense5(x)\n",
    "\n",
    "        return x\n",
    "\n",
    "def train_fn(model, optimizer, scheduler, loss_fn, dataloader, device):\n",
    "    model.train()\n",
    "    final_loss = 0\n",
    "\n",
    "    for data in dataloader:\n",
    "        optimizer.zero_grad()\n",
    "        features = data['features'].to(device)\n",
    "        label = data['label'].to(device)\n",
    "        outputs = model(features)\n",
    "        loss = loss_fn(outputs, label)\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        if scheduler:\n",
    "            scheduler.step()\n",
    "\n",
    "        final_loss += loss.item()\n",
    "\n",
    "    final_loss /= len(dataloader)\n",
    "\n",
    "    return final_loss\n",
    "\n",
    "def inference_fn(model, dataloader, device):\n",
    "    model.eval()\n",
    "    preds = []\n",
    "\n",
    "    for data in dataloader:\n",
    "        features = data['features'].to(device)\n",
    "\n",
    "        with torch.no_grad():\n",
    "            outputs = model(features)\n",
    "\n",
    "        preds.append(outputs.sigmoid().detach().cpu().numpy())\n",
    "\n",
    "    preds = np.concatenate(preds).reshape(-1, len(target_cols))\n",
    "\n",
    "    return preds\n",
    "\n",
    "def utility_score_bincount(date, weight, resp, action):\n",
    "    count_i = len(np.unique(date))\n",
    "    Pi = np.bincount(date, weight * resp * action)\n",
    "    t = np.sum(Pi) / np.sqrt(np.sum(Pi ** 2)) * np.sqrt(250 / count_i)\n",
    "    u = np.clip(t, 0, 6) * np.sum(Pi)\n",
    "    return u"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-02-01T03:38:56.902540Z",
     "iopub.status.busy": "2021-02-01T03:38:56.901065Z",
     "iopub.status.idle": "2021-02-01T03:57:18.069267Z",
     "shell.execute_reply": "2021-02-01T03:57:18.069695Z"
    },
    "papermill": {
     "duration": 1101.194945,
     "end_time": "2021-02-01T03:57:18.069827",
     "exception": false,
     "start_time": "2021-02-01T03:38:56.874882",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fold0:\n",
      "FOLD0 EPOCH:  0 train_loss=0.69713 valid_u_score=648.76478 valid_auc=0.53225 time: 0.28min\n",
      "FOLD0 EPOCH:  1 train_loss=0.69025 valid_u_score=541.84478 valid_auc=0.53463 time: 0.46min\n",
      "FOLD0 EPOCH:  2 train_loss=0.68893 valid_u_score=386.15269 valid_auc=0.53200 time: 0.64min\n",
      "FOLD0 EPOCH:  3 train_loss=0.68705 valid_u_score=485.43847 valid_auc=0.53459 time: 0.81min\n",
      "FOLD0 EPOCH:  4 train_loss=0.68601 valid_u_score=851.83964 valid_auc=0.53431 time: 0.98min\n",
      "Early stopping\n",
      "Fold1:\n",
      "FOLD1 EPOCH:  0 train_loss=0.69417 valid_u_score=166.92038 valid_auc=0.53650 time: 1.27min\n",
      "FOLD1 EPOCH:  1 train_loss=0.69008 valid_u_score=408.34408 valid_auc=0.54000 time: 1.54min\n",
      "FOLD1 EPOCH:  2 train_loss=0.68911 valid_u_score=117.68339 valid_auc=0.53499 time: 1.82min\n",
      "FOLD1 EPOCH:  3 train_loss=0.68829 valid_u_score=261.08802 valid_auc=0.53999 time: 2.09min\n",
      "FOLD1 EPOCH:  4 train_loss=0.68788 valid_u_score=139.08922 valid_auc=0.53843 time: 2.35min\n",
      "Early stopping\n",
      "Fold2:\n",
      "FOLD2 EPOCH:  0 train_loss=0.69298 valid_u_score=727.87573 valid_auc=0.54048 time: 2.77min\n",
      "FOLD2 EPOCH:  1 train_loss=0.68963 valid_u_score=971.40254 valid_auc=0.54225 time: 3.15min\n",
      "FOLD2 EPOCH:  2 train_loss=0.68891 valid_u_score=1082.20919 valid_auc=0.54334 time: 3.54min\n",
      "FOLD2 EPOCH:  3 train_loss=0.68843 valid_u_score=1063.45954 valid_auc=0.54416 time: 3.93min\n",
      "FOLD2 EPOCH:  4 train_loss=0.68793 valid_u_score=1328.37022 valid_auc=0.54519 time: 4.31min\n",
      "FOLD2 EPOCH:  5 train_loss=0.68743 valid_u_score=910.50992 valid_auc=0.54392 time: 4.69min\n",
      "FOLD2 EPOCH:  6 train_loss=0.68718 valid_u_score=1231.28469 valid_auc=0.54447 time: 5.08min\n",
      "FOLD2 EPOCH:  7 train_loss=0.68683 valid_u_score=736.79148 valid_auc=0.54245 time: 5.45min\n",
      "Early stopping\n",
      "Fold3:\n",
      "FOLD3 EPOCH:  0 train_loss=0.69235 valid_u_score=546.70790 valid_auc=0.53720 time: 6.00min\n",
      "FOLD3 EPOCH:  1 train_loss=0.68964 valid_u_score=400.96937 valid_auc=0.53825 time: 6.51min\n",
      "FOLD3 EPOCH:  2 train_loss=0.68902 valid_u_score=309.17291 valid_auc=0.53643 time: 7.02min\n",
      "FOLD3 EPOCH:  3 train_loss=0.68851 valid_u_score=53.27706 valid_auc=0.53767 time: 7.53min\n",
      "FOLD3 EPOCH:  4 train_loss=0.68824 valid_u_score=225.07305 valid_auc=0.53916 time: 8.04min\n",
      "FOLD3 EPOCH:  5 train_loss=0.68781 valid_u_score=533.33657 valid_auc=0.54138 time: 8.55min\n",
      "FOLD3 EPOCH:  6 train_loss=0.68758 valid_u_score=598.96849 valid_auc=0.53998 time: 9.06min\n",
      "FOLD3 EPOCH:  7 train_loss=0.68721 valid_u_score=200.60890 valid_auc=0.53989 time: 9.57min\n",
      "FOLD3 EPOCH:  8 train_loss=0.68681 valid_u_score=370.20726 valid_auc=0.53908 time: 10.07min\n",
      "Early stopping\n",
      "Fold4:\n",
      "FOLD4 EPOCH:  0 train_loss=0.69211 valid_u_score=2278.79059 valid_auc=0.54639 time: 10.80min\n",
      "FOLD4 EPOCH:  1 train_loss=0.68972 valid_u_score=2040.21171 valid_auc=0.54673 time: 11.46min\n",
      "FOLD4 EPOCH:  2 train_loss=0.68916 valid_u_score=2102.59749 valid_auc=0.54971 time: 12.12min\n",
      "FOLD4 EPOCH:  3 train_loss=0.68882 valid_u_score=2207.75604 valid_auc=0.55032 time: 12.80min\n",
      "FOLD4 EPOCH:  4 train_loss=0.68849 valid_u_score=2542.51694 valid_auc=0.55090 time: 13.46min\n",
      "FOLD4 EPOCH:  5 train_loss=0.68821 valid_u_score=2035.29423 valid_auc=0.54976 time: 14.13min\n",
      "FOLD4 EPOCH:  6 train_loss=0.68792 valid_u_score=2209.18374 valid_auc=0.55078 time: 14.79min\n",
      "FOLD4 EPOCH:  7 train_loss=0.68770 valid_u_score=2197.27939 valid_auc=0.55258 time: 15.47min\n",
      "FOLD4 EPOCH:  8 train_loss=0.68742 valid_u_score=2461.51785 valid_auc=0.55068 time: 16.13min\n",
      "FOLD4 EPOCH:  9 train_loss=0.68708 valid_u_score=1833.53234 valid_auc=0.55098 time: 16.80min\n",
      "FOLD4 EPOCH: 10 train_loss=0.68680 valid_u_score=2188.06003 valid_auc=0.55055 time: 17.46min\n",
      "Early stopping\n",
      "5 models valid score: 2301.4194028695647\tauc_score: 0.5509\tlogloss_score:4.0475\n"
     ]
    }
   ],
   "source": [
    "if TRAIN:\n",
    "    gkf = PurgedGroupTimeSeriesSplit(n_splits = FOLDS, group_gap=20)\n",
    "    splits = list(gkf.split(train, groups=train['date'].values))\n",
    "    start_time = time.time()\n",
    "    for _fold, (train_indices, test_indices) in enumerate(splits):\n",
    "            print(f'Fold{_fold}:')\n",
    "            seed_everything(seed=xx)\n",
    "            train_df=train.iloc[train_indices,:]\n",
    "            valid=train.iloc[test_indices,:]\n",
    "            train_set = MarketDataset(train_df)\n",
    "            train_loader = DataLoader(train_set, batch_size=BATCH_SIZE, shuffle=True, num_workers=4)\n",
    "            valid_set = MarketDataset(valid)\n",
    "            valid_loader = DataLoader(valid_set, batch_size=BATCH_SIZE, shuffle=False, num_workers=4)\n",
    "            torch.cuda.empty_cache()\n",
    "            device = torch.device(\"cuda:0\")\n",
    "            model = Model()\n",
    "            model.to(device)\n",
    "            optimizer = torch.optim.Adam(model.parameters(), lr=LEARNING_RATE, weight_decay=WEIGHT_DECAY)\n",
    "            scheduler = None\n",
    "            loss_fn = SmoothBCEwLogits(smoothing=0.005)\n",
    "\n",
    "            model_weights = f\"{CACHE_PATH}/online_model{_fold}.pth\"\n",
    "            es = EarlyStopping(patience=EARLYSTOP_NUM, mode=\"max\")\n",
    "            for epoch in range(EPOCHS):\n",
    "                train_loss = train_fn(model, optimizer, scheduler, loss_fn, train_loader, device)\n",
    "\n",
    "                valid_pred = inference_fn(model, valid_loader, device)\n",
    "                valid_auc = roc_auc_score(valid[target_cols].values, valid_pred)\n",
    "                valid_logloss = log_loss(valid[target_cols].values, valid_pred)\n",
    "                valid_pred = np.median(valid_pred, axis=1)\n",
    "                valid_pred = np.where(valid_pred >= 0.5, 1, 0).astype(int)\n",
    "                valid_u_score = utility_score_bincount(date=valid.date.values, weight=valid.weight.values,\n",
    "                                                   resp=valid.resp.values, action=valid_pred)\n",
    "                print(f\"FOLD{_fold} EPOCH:{epoch:3} train_loss={train_loss:.5f} \"\n",
    "                      f\"valid_u_score={valid_u_score:.5f} valid_auc={valid_auc:.5f} \"\n",
    "                      f\"time: {(time.time() - start_time) / 60:.2f}min\")\n",
    "                es(valid_auc, model, model_path=model_weights)\n",
    "                if es.early_stop:\n",
    "                    print(\"Early stopping\")\n",
    "                    break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": 0.01398,
     "end_time": "2021-02-01T03:57:18.098486",
     "exception": false,
     "start_time": "2021-02-01T03:57:18.084506",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  },
  "papermill": {
   "duration": 1261.140455,
   "end_time": "2021-02-01T03:57:19.277627",
   "environment_variables": {},
   "exception": null,
   "input_path": "__notebook__.ipynb",
   "output_path": "__notebook__.ipynb",
   "parameters": {},
   "start_time": "2021-02-01T03:36:18.137172",
   "version": "2.1.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}