diff --git a/docs/CTR/leaderboard/avazu_x1.csv b/docs/CTR/leaderboard/avazu_x1.csv index 0b224db..8e4e8d1 100644 --- a/docs/CTR/leaderboard/avazu_x1.csv +++ b/docs/CTR/leaderboard/avazu_x1.csv @@ -7,7 +7,7 @@ Year,Publication,Model,Paper URL,AUC,Logloss,Running Steps,Contributor 2017,SIGIR'17,NFM,https://dl.acm.org/citation.cfm?id=3080777,0.7627,0.3677,https://github.com/reczoo/BARS/tree/main/ranking/ctr/NFM/NFM_avazu_x1,"Zhu et al." 2018,WWW'18,FwFM,https://arxiv.org/pdf/1806.03514.pdf,0.7602,0.3688,https://github.com/reczoo/BARS/tree/main/ranking/ctr/FwFM/FwFM_avazu_x1,"Zhu et al." 2021,WWW'21,FmFM,https://arxiv.org/abs/2102.12994,0.7595,0.3689,https://github.com/reczoo/BARS/tree/main/ranking/ctr/FmFM/FmFM_avazu_x1,"Zhu et al." -2016,RecSys'16,YoutubeDNN,https://research.google.com/pubs/archive/45530.pdf,0.763,0.3682,https://github.com/reczoo/BARS/tree/main/ranking/ctr/DNN/DNN_avazu_x1,"Zhu et al." +2016,RecSys'16,YoutubeDNN,https://research.google.com/pubs/archive/45530.pdf,0.763827,0.367394,https://github.com/reczoo/BARS/tree/main/ranking/ctr/DNN/DNN_avazu_x1,"Zhu et al." 2016,ICDM'16,IPNN,https://arxiv.org/pdf/1611.00144.pdf,0.763,0.3676,https://github.com/reczoo/BARS/tree/main/ranking/ctr/PNN/IPNN_avazu_x1,"Zhu et al." 2016,DLRS'16,Wide&Deep,https://arxiv.org/pdf/1606.07792.pdf,0.7649,0.3665,https://github.com/reczoo/BARS/tree/main/ranking/ctr/WideDeep/WideDeep_avazu_x1,"Zhu et al." 2017,IJCAI'17,DeepFM,https://arxiv.org/abs/1703.04247,0.7648,0.3667,https://github.com/reczoo/BARS/tree/main/ranking/ctr/DeepFM/DeepFM_avazu_x1,"Zhu et al." diff --git a/ranking/ctr/DNN/DNN_avazu_x1/DNN_avazu_x1_001_3da2d674.log b/ranking/ctr/DNN/DNN_avazu_x1/DNN_avazu_x1_001_3da2d674.log index 259ecd3..bd97b5c 100644 --- a/ranking/ctr/DNN/DNN_avazu_x1/DNN_avazu_x1_001_3da2d674.log +++ b/ranking/ctr/DNN/DNN_avazu_x1/DNN_avazu_x1_001_3da2d674.log @@ -1,4 +1,4 @@ -2022-02-08 09:57:44,716 P50417 INFO { +2024-06-11 15:24:47,922 P169311 INFO { "batch_norm": "False", "batch_size": "4096", "data_block_size": "-1", @@ -41,57 +41,101 @@ "verbose": "0", "version": "pytorch" } -2022-02-08 09:57:44,717 P50417 INFO Set up feature encoder... -2022-02-08 09:57:44,717 P50417 INFO Load feature_map from json: ../data/Avazu/avazu_x1_3fb65689/feature_map.json -2022-02-08 09:57:44,717 P50417 INFO Loading data... -2022-02-08 09:57:44,718 P50417 INFO Loading data from h5: ../data/Avazu/avazu_x1_3fb65689/train.h5 -2022-02-08 09:57:46,771 P50417 INFO Loading data from h5: ../data/Avazu/avazu_x1_3fb65689/valid.h5 -2022-02-08 09:57:47,098 P50417 INFO Train samples: total/28300276, pos/4953382, neg/23346894, ratio/17.50%, blocks/1 -2022-02-08 09:57:47,098 P50417 INFO Validation samples: total/4042897, pos/678699, neg/3364198, ratio/16.79%, blocks/1 -2022-02-08 09:57:47,098 P50417 INFO Loading train data done. -2022-02-08 09:57:51,209 P50417 INFO Total number of parameters: 13805192. -2022-02-08 09:57:51,209 P50417 INFO Start training: 6910 batches/epoch -2022-02-08 09:57:51,209 P50417 INFO ************ Epoch=1 start ************ -2022-02-08 10:02:20,462 P50417 INFO [Metrics] AUC: 0.742239 - logloss: 0.398721 -2022-02-08 10:02:20,465 P50417 INFO Save best model: monitor(max): 0.742239 -2022-02-08 10:02:20,538 P50417 INFO --- 6910/6910 batches finished --- -2022-02-08 10:02:20,587 P50417 INFO Train loss: 0.427262 -2022-02-08 10:02:20,587 P50417 INFO ************ Epoch=1 end ************ -2022-02-08 10:06:46,211 P50417 INFO [Metrics] AUC: 0.739514 - logloss: 0.401270 -2022-02-08 10:06:46,214 P50417 INFO Monitor(max) STOP: 0.739514 ! -2022-02-08 10:06:46,214 P50417 INFO Reduce learning rate on plateau: 0.000100 -2022-02-08 10:06:46,214 P50417 INFO --- 6910/6910 batches finished --- -2022-02-08 10:06:46,262 P50417 INFO Train loss: 0.427740 -2022-02-08 10:06:46,262 P50417 INFO ************ Epoch=2 end ************ -2022-02-08 10:11:10,596 P50417 INFO [Metrics] AUC: 0.743622 - logloss: 0.398500 -2022-02-08 10:11:10,599 P50417 INFO Save best model: monitor(max): 0.743622 -2022-02-08 10:11:10,667 P50417 INFO --- 6910/6910 batches finished --- -2022-02-08 10:11:10,713 P50417 INFO Train loss: 0.404701 -2022-02-08 10:11:10,713 P50417 INFO ************ Epoch=3 end ************ -2022-02-08 10:15:34,346 P50417 INFO [Metrics] AUC: 0.745833 - logloss: 0.396748 -2022-02-08 10:15:34,349 P50417 INFO Save best model: monitor(max): 0.745833 -2022-02-08 10:15:34,421 P50417 INFO --- 6910/6910 batches finished --- -2022-02-08 10:15:34,472 P50417 INFO Train loss: 0.405991 -2022-02-08 10:15:34,472 P50417 INFO ************ Epoch=4 end ************ -2022-02-08 10:19:57,705 P50417 INFO [Metrics] AUC: 0.745333 - logloss: 0.396666 -2022-02-08 10:19:57,708 P50417 INFO Monitor(max) STOP: 0.745333 ! -2022-02-08 10:19:57,708 P50417 INFO Reduce learning rate on plateau: 0.000010 -2022-02-08 10:19:57,708 P50417 INFO --- 6910/6910 batches finished --- -2022-02-08 10:19:57,759 P50417 INFO Train loss: 0.406534 -2022-02-08 10:19:57,759 P50417 INFO ************ Epoch=5 end ************ -2022-02-08 10:24:22,342 P50417 INFO [Metrics] AUC: 0.745134 - logloss: 0.397022 -2022-02-08 10:24:22,345 P50417 INFO Monitor(max) STOP: 0.745134 ! -2022-02-08 10:24:22,345 P50417 INFO Reduce learning rate on plateau: 0.000001 -2022-02-08 10:24:22,345 P50417 INFO Early stopping at epoch=6 -2022-02-08 10:24:22,345 P50417 INFO --- 6910/6910 batches finished --- -2022-02-08 10:24:22,395 P50417 INFO Train loss: 0.393572 -2022-02-08 10:24:22,395 P50417 INFO Training finished. -2022-02-08 10:24:22,395 P50417 INFO Load best model: /cache/FuxiCTR/benchmarks/Avazu/DNN_avazu_x1/avazu_x1_3fb65689/DNN_avazu_x1_001_3da2d674.model -2022-02-08 10:24:22,686 P50417 INFO ****** Validation evaluation ****** -2022-02-08 10:24:34,114 P50417 INFO [Metrics] AUC: 0.745833 - logloss: 0.396748 -2022-02-08 10:24:34,195 P50417 INFO ******** Test evaluation ******** -2022-02-08 10:24:34,195 P50417 INFO Loading data... -2022-02-08 10:24:34,195 P50417 INFO Loading data from h5: ../data/Avazu/avazu_x1_3fb65689/test.h5 -2022-02-08 10:24:34,880 P50417 INFO Test samples: total/8085794, pos/1232985, neg/6852809, ratio/15.25%, blocks/1 -2022-02-08 10:24:34,881 P50417 INFO Loading test data done. -2022-02-08 10:25:00,299 P50417 INFO [Metrics] AUC: 0.763019 - logloss: 0.368178 +2024-06-11 15:24:47,924 P169311 INFO Set up feature encoder... +2024-06-11 15:24:47,924 P169311 INFO Reading file: ../data/Avazu/Avazu_x1/train.csv +2024-06-11 15:25:48,605 P169311 INFO Reading file: ../data/Avazu/Avazu_x1/valid.csv +2024-06-11 15:25:56,754 P169311 INFO Reading file: ../data/Avazu/Avazu_x1/test.csv +2024-06-11 15:26:13,225 P169311 INFO Preprocess feature columns... +2024-06-11 15:26:15,317 P169311 INFO Fit feature encoder... +2024-06-11 15:26:15,317 P169311 INFO Processing column: {'active': True, 'dtype': 'float', 'name': 'feat_1', 'type': 'categorical'} +2024-06-11 15:26:25,500 P169311 INFO Processing column: {'active': True, 'dtype': 'float', 'name': 'feat_2', 'type': 'categorical'} +2024-06-11 15:26:35,766 P169311 INFO Processing column: {'active': True, 'dtype': 'float', 'name': 'feat_3', 'type': 'categorical'} +2024-06-11 15:26:46,412 P169311 INFO Processing column: {'active': True, 'dtype': 'float', 'name': 'feat_4', 'type': 'categorical'} +2024-06-11 15:26:56,793 P169311 INFO Processing column: {'active': True, 'dtype': 'float', 'name': 'feat_5', 'type': 'categorical'} +2024-06-11 15:27:06,938 P169311 INFO Processing column: {'active': True, 'dtype': 'float', 'name': 'feat_6', 'type': 'categorical'} +2024-06-11 15:27:17,414 P169311 INFO Processing column: {'active': True, 'dtype': 'float', 'name': 'feat_7', 'type': 'categorical'} +2024-06-11 15:27:27,648 P169311 INFO Processing column: {'active': True, 'dtype': 'float', 'name': 'feat_8', 'type': 'categorical'} +2024-06-11 15:27:37,620 P169311 INFO Processing column: {'active': True, 'dtype': 'float', 'name': 'feat_9', 'type': 'categorical'} +2024-06-11 15:27:50,724 P169311 INFO Processing column: {'active': True, 'dtype': 'float', 'name': 'feat_10', 'type': 'categorical'} +2024-06-11 15:28:14,989 P169311 INFO Processing column: {'active': True, 'dtype': 'float', 'name': 'feat_11', 'type': 'categorical'} +2024-06-11 15:28:25,269 P169311 INFO Processing column: {'active': True, 'dtype': 'float', 'name': 'feat_12', 'type': 'categorical'} +2024-06-11 15:28:35,297 P169311 INFO Processing column: {'active': True, 'dtype': 'float', 'name': 'feat_13', 'type': 'categorical'} +2024-06-11 15:28:45,089 P169311 INFO Processing column: {'active': True, 'dtype': 'float', 'name': 'feat_14', 'type': 'categorical'} +2024-06-11 15:28:55,233 P169311 INFO Processing column: {'active': True, 'dtype': 'float', 'name': 'feat_15', 'type': 'categorical'} +2024-06-11 15:29:05,185 P169311 INFO Processing column: {'active': True, 'dtype': 'float', 'name': 'feat_16', 'type': 'categorical'} +2024-06-11 15:29:15,109 P169311 INFO Processing column: {'active': True, 'dtype': 'float', 'name': 'feat_17', 'type': 'categorical'} +2024-06-11 15:29:25,233 P169311 INFO Processing column: {'active': True, 'dtype': 'float', 'name': 'feat_18', 'type': 'categorical'} +2024-06-11 15:29:34,955 P169311 INFO Processing column: {'active': True, 'dtype': 'float', 'name': 'feat_19', 'type': 'categorical'} +2024-06-11 15:29:44,869 P169311 INFO Processing column: {'active': True, 'dtype': 'float', 'name': 'feat_20', 'type': 'categorical'} +2024-06-11 15:29:54,723 P169311 INFO Processing column: {'active': True, 'dtype': 'float', 'name': 'feat_21', 'type': 'categorical'} +2024-06-11 15:30:04,695 P169311 INFO Processing column: {'active': True, 'dtype': 'float', 'name': 'feat_22', 'type': 'categorical'} +2024-06-11 15:30:14,458 P169311 INFO Set feature index... +2024-06-11 15:30:14,458 P169311 INFO Pickle feature_encoder: ../data/Avazu/avazu_x1_3fb65689/feature_encoder.pkl +2024-06-11 15:30:20,672 P169311 INFO Save feature_map to json: ../data/Avazu/avazu_x1_3fb65689/feature_map.json +2024-06-11 15:30:20,673 P169311 INFO Set feature encoder done. +2024-06-11 15:30:20,673 P169311 INFO Transform feature columns... +2024-06-11 15:36:56,128 P169311 INFO Saving data to h5: ../data/Avazu/avazu_x1_3fb65689/train.h5 +2024-06-11 15:36:59,477 P169311 INFO Preprocess feature columns... +2024-06-11 15:36:59,817 P169311 INFO Transform feature columns... +2024-06-11 15:37:57,112 P169311 INFO Saving data to h5: ../data/Avazu/avazu_x1_3fb65689/valid.h5 +2024-06-11 15:37:57,612 P169311 INFO Preprocess feature columns... +2024-06-11 15:37:58,172 P169311 INFO Transform feature columns... +2024-06-11 15:39:52,553 P169311 INFO Saving data to h5: ../data/Avazu/avazu_x1_3fb65689/test.h5 +2024-06-11 15:39:53,540 P169311 INFO Transform csv data to h5 done. +2024-06-11 15:39:53,540 P169311 INFO Loading data... +2024-06-11 15:39:53,546 P169311 INFO Loading data from h5: ../data/Avazu/avazu_x1_3fb65689/train.h5 +2024-06-11 15:39:55,407 P169311 INFO Loading data from h5: ../data/Avazu/avazu_x1_3fb65689/valid.h5 +2024-06-11 15:39:55,744 P169311 INFO Train samples: total/28300276, pos/4953382, neg/23346894, ratio/17.50%, blocks/1 +2024-06-11 15:39:55,744 P169311 INFO Validation samples: total/4042897, pos/678699, neg/3364198, ratio/16.79%, blocks/1 +2024-06-11 15:39:55,744 P169311 INFO Loading train data done. +2024-06-11 15:40:01,389 P169311 INFO Total number of parameters: 13395591. +2024-06-11 15:40:01,389 P169311 INFO Start training: 6910 batches/epoch +2024-06-11 15:40:01,389 P169311 INFO ************ Epoch=1 start ************ +2024-06-11 15:44:00,317 P169311 INFO [Metrics] AUC: 0.738388 - logloss: 0.400051 +2024-06-11 15:44:00,317 P169311 INFO Save best model: monitor(max): 0.738388 +2024-06-11 15:44:00,375 P169311 INFO --- 6910/6910 batches finished --- +2024-06-11 15:44:00,526 P169311 INFO Train loss: 0.428714 +2024-06-11 15:44:00,526 P169311 INFO ************ Epoch=1 end ************ +2024-06-11 15:47:58,946 P169311 INFO [Metrics] AUC: 0.740958 - logloss: 0.398840 +2024-06-11 15:47:58,948 P169311 INFO Save best model: monitor(max): 0.740958 +2024-06-11 15:47:59,078 P169311 INFO --- 6910/6910 batches finished --- +2024-06-11 15:47:59,240 P169311 INFO Train loss: 0.428697 +2024-06-11 15:47:59,240 P169311 INFO ************ Epoch=2 end ************ +2024-06-11 15:51:58,178 P169311 INFO [Metrics] AUC: 0.741541 - logloss: 0.399047 +2024-06-11 15:51:58,185 P169311 INFO Save best model: monitor(max): 0.741541 +2024-06-11 15:51:58,314 P169311 INFO --- 6910/6910 batches finished --- +2024-06-11 15:51:58,477 P169311 INFO Train loss: 0.428326 +2024-06-11 15:51:58,478 P169311 INFO ************ Epoch=3 end ************ +2024-06-11 15:56:00,843 P169311 INFO [Metrics] AUC: 0.739486 - logloss: 0.399609 +2024-06-11 15:56:00,845 P169311 INFO Monitor(max) STOP: 0.739486 ! +2024-06-11 15:56:00,845 P169311 INFO Reduce learning rate on plateau: 0.000100 +2024-06-11 15:56:00,846 P169311 INFO --- 6910/6910 batches finished --- +2024-06-11 15:56:01,004 P169311 INFO Train loss: 0.428737 +2024-06-11 15:56:01,004 P169311 INFO ************ Epoch=4 end ************ +2024-06-11 15:59:59,972 P169311 INFO [Metrics] AUC: 0.746336 - logloss: 0.396527 +2024-06-11 15:59:59,972 P169311 INFO Save best model: monitor(max): 0.746336 +2024-06-11 16:00:00,104 P169311 INFO --- 6910/6910 batches finished --- +2024-06-11 16:00:00,264 P169311 INFO Train loss: 0.403978 +2024-06-11 16:00:00,265 P169311 INFO ************ Epoch=5 end ************ +2024-06-11 16:03:58,553 P169311 INFO [Metrics] AUC: 0.745096 - logloss: 0.396898 +2024-06-11 16:03:58,554 P169311 INFO Monitor(max) STOP: 0.745096 ! +2024-06-11 16:03:58,554 P169311 INFO Reduce learning rate on plateau: 0.000010 +2024-06-11 16:03:58,554 P169311 INFO --- 6910/6910 batches finished --- +2024-06-11 16:03:58,720 P169311 INFO Train loss: 0.404432 +2024-06-11 16:03:58,720 P169311 INFO ************ Epoch=6 end ************ +2024-06-11 16:07:58,646 P169311 INFO [Metrics] AUC: 0.744883 - logloss: 0.397435 +2024-06-11 16:07:58,648 P169311 INFO Monitor(max) STOP: 0.744883 ! +2024-06-11 16:07:58,648 P169311 INFO Reduce learning rate on plateau: 0.000001 +2024-06-11 16:07:58,648 P169311 INFO Early stopping at epoch=7 +2024-06-11 16:07:58,648 P169311 INFO --- 6910/6910 batches finished --- +2024-06-11 16:07:58,839 P169311 INFO Train loss: 0.394071 +2024-06-11 16:07:58,839 P169311 INFO Training finished. +2024-06-11 16:07:58,839 P169311 INFO Load best model: /home/ma-user/work/DNN_avazu_x1/Avazu/DNN_avazu_x1/avazu_x1_3fb65689/DNN_avazu_x1_001_3da2d674.model +2024-06-11 16:07:58,987 P169311 INFO ****** Validation evaluation ****** +2024-06-11 16:08:13,177 P169311 INFO [Metrics] AUC: 0.746336 - logloss: 0.396527 +2024-06-11 16:08:13,250 P169311 INFO ******** Test evaluation ******** +2024-06-11 16:08:13,250 P169311 INFO Loading data... +2024-06-11 16:08:13,251 P169311 INFO Loading data from h5: ../data/Avazu/avazu_x1_3fb65689/test.h5 +2024-06-11 16:08:16,926 P169311 INFO Test samples: total/8085794, pos/1232985, neg/6852809, ratio/15.25%, blocks/1 +2024-06-11 16:08:16,927 P169311 INFO Loading test data done. +2024-06-11 16:08:45,943 P169311 INFO [Metrics] AUC: 0.763827 - logloss: 0.367394 diff --git a/ranking/ctr/DNN/DNN_avazu_x1/DNN_avazu_x1_tuner_config_01.csv b/ranking/ctr/DNN/DNN_avazu_x1/DNN_avazu_x1_tuner_config_01.csv new file mode 100644 index 0000000..c3894c2 --- /dev/null +++ b/ranking/ctr/DNN/DNN_avazu_x1/DNN_avazu_x1_tuner_config_01.csv @@ -0,0 +1 @@ + 20240611-160845,[command] python run_expid.py --config ./DNN_avazu_x1_tuner_config_01 --expid DNN_avazu_x1_001_3da2d674 --gpu 0,[exp_id] DNN_avazu_x1_001_3da2d674,[dataset_id] avazu_x1_3fb65689,[train] N.A.,[val] AUC: 0.746336 - logloss: 0.396527,[test] AUC: 0.763827 - logloss: 0.367394 diff --git a/ranking/ctr/DNN/DNN_avazu_x1/README.md b/ranking/ctr/DNN/DNN_avazu_x1/README.md index aae3c8a..ac351dd 100644 --- a/ranking/ctr/DNN/DNN_avazu_x1/README.md +++ b/ranking/ctr/DNN/DNN_avazu_x1/README.md @@ -13,18 +13,18 @@ Author: [BARS Benchmark](https://github.com/reczoo/BARS/blob/main/CITATION) + Hardware ```python - CPU: Intel(R) Xeon(R) Gold 6278C CPU @ 2.60GHz + CPU: Intel(R) Xeon(R) Gold 6151 CPU @ 3.00GHz GPU: Tesla V100 32G - RAM: 755G + RAM: 512G ``` + Software ```python CUDA: 10.2 - python: 3.6.4 + python: 3.7.10 pytorch: 1.0.0 - pandas: 0.22.0 + pandas: 1.1.5 numpy: 1.19.2 scipy: 1.5.4 sklearn: 0.22.1 @@ -44,7 +44,7 @@ We use [FuxiCTR-v1.1.0](https://github.com/reczoo/FuxiCTR/tree/v1.1.0) for this Running steps: -1. Download [FuxiCTR-v1.1.0](https://github.com/reczoo/FuxiCTR/archive/refs/tags/v1.1.0.zip) and install all the dependencies listed in the [environments](#environments). Then modify [run_expid.py](./run_expid.py#L5) to add the FuxiCTR library to system path +1. Download [FuxiCTR-v1.1.0](https://github.com/reczoo/FuxiCTR/archive/refs/tags/v1.1.0.zip) and install all the dependencies listed in the [environments](#environments). Then edit [run_expid.py](./run_expid.py#L5) to add the FuxiCTR library to system path ```python sys.path.append('YOUR_PATH_TO_FuxiCTR/') @@ -52,34 +52,26 @@ Running steps: 2. Create a data directory and put the downloaded csv files in `../data/Avazu/Avazu_x1`. -3. Both `dataset_config.yaml` and `model_config.yaml` files are available in [DNN_avazu_x1_tuner_config_seeds](./DNN_avazu_x1_tuner_config_seeds). Make sure the data paths in `dataset_config.yaml` are correctly set to what we create in the last step. +3. Both `dataset_config.yaml` and `model_config.yaml` files are available in [DNN_avazu_x1_tuner_config_01](./DNN_avazu_x1_tuner_config_01). Make sure that the data paths in `dataset_config.yaml` are correctly set to the directory we create in the last step. 4. Run the following script to start. ```bash - cd DNN_avazu_x1 + cd . nohup python run_expid.py --config ./DNN_avazu_x1_tuner_config_01 --expid DNN_avazu_x1_001_3da2d674 --gpu 0 > run.log & tail -f run.log ``` ### Results -Total 5 runs: - -| Runs | AUC | logloss | -|:----:|:----------------:|:----------------:| -| 1 | 0.763019 | 0.368178 | -| 2 | 0.763126 | 0.367452 | -| 3 | 0.763342 | 0.367366 | -| 4 | 0.763145 | 0.367455 | -| 5 | 0.763870 | 0.367038 | -| Avg | 0.763300 | 0.367498 | -| Std | ±0.00030329 | ±0.00037293 | +| AUC | logloss | +|:--------:|:--------:| +| 0.763827 | 0.367394 | ### Logs ```python -2022-02-08 09:57:44,716 P50417 INFO { +2024-06-11 15:24:47,922 P169311 INFO { "batch_norm": "False", "batch_size": "4096", "data_block_size": "-1", @@ -122,58 +114,102 @@ Total 5 runs: "verbose": "0", "version": "pytorch" } -2022-02-08 09:57:44,717 P50417 INFO Set up feature encoder... -2022-02-08 09:57:44,717 P50417 INFO Load feature_map from json: ../data/Avazu/avazu_x1_3fb65689/feature_map.json -2022-02-08 09:57:44,717 P50417 INFO Loading data... -2022-02-08 09:57:44,718 P50417 INFO Loading data from h5: ../data/Avazu/avazu_x1_3fb65689/train.h5 -2022-02-08 09:57:46,771 P50417 INFO Loading data from h5: ../data/Avazu/avazu_x1_3fb65689/valid.h5 -2022-02-08 09:57:47,098 P50417 INFO Train samples: total/28300276, pos/4953382, neg/23346894, ratio/17.50%, blocks/1 -2022-02-08 09:57:47,098 P50417 INFO Validation samples: total/4042897, pos/678699, neg/3364198, ratio/16.79%, blocks/1 -2022-02-08 09:57:47,098 P50417 INFO Loading train data done. -2022-02-08 09:57:51,209 P50417 INFO Total number of parameters: 13805192. -2022-02-08 09:57:51,209 P50417 INFO Start training: 6910 batches/epoch -2022-02-08 09:57:51,209 P50417 INFO ************ Epoch=1 start ************ -2022-02-08 10:02:20,462 P50417 INFO [Metrics] AUC: 0.742239 - logloss: 0.398721 -2022-02-08 10:02:20,465 P50417 INFO Save best model: monitor(max): 0.742239 -2022-02-08 10:02:20,538 P50417 INFO --- 6910/6910 batches finished --- -2022-02-08 10:02:20,587 P50417 INFO Train loss: 0.427262 -2022-02-08 10:02:20,587 P50417 INFO ************ Epoch=1 end ************ -2022-02-08 10:06:46,211 P50417 INFO [Metrics] AUC: 0.739514 - logloss: 0.401270 -2022-02-08 10:06:46,214 P50417 INFO Monitor(max) STOP: 0.739514 ! -2022-02-08 10:06:46,214 P50417 INFO Reduce learning rate on plateau: 0.000100 -2022-02-08 10:06:46,214 P50417 INFO --- 6910/6910 batches finished --- -2022-02-08 10:06:46,262 P50417 INFO Train loss: 0.427740 -2022-02-08 10:06:46,262 P50417 INFO ************ Epoch=2 end ************ -2022-02-08 10:11:10,596 P50417 INFO [Metrics] AUC: 0.743622 - logloss: 0.398500 -2022-02-08 10:11:10,599 P50417 INFO Save best model: monitor(max): 0.743622 -2022-02-08 10:11:10,667 P50417 INFO --- 6910/6910 batches finished --- -2022-02-08 10:11:10,713 P50417 INFO Train loss: 0.404701 -2022-02-08 10:11:10,713 P50417 INFO ************ Epoch=3 end ************ -2022-02-08 10:15:34,346 P50417 INFO [Metrics] AUC: 0.745833 - logloss: 0.396748 -2022-02-08 10:15:34,349 P50417 INFO Save best model: monitor(max): 0.745833 -2022-02-08 10:15:34,421 P50417 INFO --- 6910/6910 batches finished --- -2022-02-08 10:15:34,472 P50417 INFO Train loss: 0.405991 -2022-02-08 10:15:34,472 P50417 INFO ************ Epoch=4 end ************ -2022-02-08 10:19:57,705 P50417 INFO [Metrics] AUC: 0.745333 - logloss: 0.396666 -2022-02-08 10:19:57,708 P50417 INFO Monitor(max) STOP: 0.745333 ! -2022-02-08 10:19:57,708 P50417 INFO Reduce learning rate on plateau: 0.000010 -2022-02-08 10:19:57,708 P50417 INFO --- 6910/6910 batches finished --- -2022-02-08 10:19:57,759 P50417 INFO Train loss: 0.406534 -2022-02-08 10:19:57,759 P50417 INFO ************ Epoch=5 end ************ -2022-02-08 10:24:22,342 P50417 INFO [Metrics] AUC: 0.745134 - logloss: 0.397022 -2022-02-08 10:24:22,345 P50417 INFO Monitor(max) STOP: 0.745134 ! -2022-02-08 10:24:22,345 P50417 INFO Reduce learning rate on plateau: 0.000001 -2022-02-08 10:24:22,345 P50417 INFO Early stopping at epoch=6 -2022-02-08 10:24:22,345 P50417 INFO --- 6910/6910 batches finished --- -2022-02-08 10:24:22,395 P50417 INFO Train loss: 0.393572 -2022-02-08 10:24:22,395 P50417 INFO Training finished. -2022-02-08 10:24:22,395 P50417 INFO Load best model: /cache/FuxiCTR/benchmarks/Avazu/DNN_avazu_x1/avazu_x1_3fb65689/DNN_avazu_x1_001_3da2d674.model -2022-02-08 10:24:22,686 P50417 INFO ****** Validation evaluation ****** -2022-02-08 10:24:34,114 P50417 INFO [Metrics] AUC: 0.745833 - logloss: 0.396748 -2022-02-08 10:24:34,195 P50417 INFO ******** Test evaluation ******** -2022-02-08 10:24:34,195 P50417 INFO Loading data... -2022-02-08 10:24:34,195 P50417 INFO Loading data from h5: ../data/Avazu/avazu_x1_3fb65689/test.h5 -2022-02-08 10:24:34,880 P50417 INFO Test samples: total/8085794, pos/1232985, neg/6852809, ratio/15.25%, blocks/1 -2022-02-08 10:24:34,881 P50417 INFO Loading test data done. -2022-02-08 10:25:00,299 P50417 INFO [Metrics] AUC: 0.763019 - logloss: 0.368178 +2024-06-11 15:24:47,924 P169311 INFO Set up feature encoder... +2024-06-11 15:24:47,924 P169311 INFO Reading file: ../data/Avazu/Avazu_x1/train.csv +2024-06-11 15:25:48,605 P169311 INFO Reading file: ../data/Avazu/Avazu_x1/valid.csv +2024-06-11 15:25:56,754 P169311 INFO Reading file: ../data/Avazu/Avazu_x1/test.csv +2024-06-11 15:26:13,225 P169311 INFO Preprocess feature columns... +2024-06-11 15:26:15,317 P169311 INFO Fit feature encoder... +2024-06-11 15:26:15,317 P169311 INFO Processing column: {'active': True, 'dtype': 'float', 'name': 'feat_1', 'type': 'categorical'} +2024-06-11 15:26:25,500 P169311 INFO Processing column: {'active': True, 'dtype': 'float', 'name': 'feat_2', 'type': 'categorical'} +2024-06-11 15:26:35,766 P169311 INFO Processing column: {'active': True, 'dtype': 'float', 'name': 'feat_3', 'type': 'categorical'} +2024-06-11 15:26:46,412 P169311 INFO Processing column: {'active': True, 'dtype': 'float', 'name': 'feat_4', 'type': 'categorical'} +2024-06-11 15:26:56,793 P169311 INFO Processing column: {'active': True, 'dtype': 'float', 'name': 'feat_5', 'type': 'categorical'} +2024-06-11 15:27:06,938 P169311 INFO Processing column: {'active': True, 'dtype': 'float', 'name': 'feat_6', 'type': 'categorical'} +2024-06-11 15:27:17,414 P169311 INFO Processing column: {'active': True, 'dtype': 'float', 'name': 'feat_7', 'type': 'categorical'} +2024-06-11 15:27:27,648 P169311 INFO Processing column: {'active': True, 'dtype': 'float', 'name': 'feat_8', 'type': 'categorical'} +2024-06-11 15:27:37,620 P169311 INFO Processing column: {'active': True, 'dtype': 'float', 'name': 'feat_9', 'type': 'categorical'} +2024-06-11 15:27:50,724 P169311 INFO Processing column: {'active': True, 'dtype': 'float', 'name': 'feat_10', 'type': 'categorical'} +2024-06-11 15:28:14,989 P169311 INFO Processing column: {'active': True, 'dtype': 'float', 'name': 'feat_11', 'type': 'categorical'} +2024-06-11 15:28:25,269 P169311 INFO Processing column: {'active': True, 'dtype': 'float', 'name': 'feat_12', 'type': 'categorical'} +2024-06-11 15:28:35,297 P169311 INFO Processing column: {'active': True, 'dtype': 'float', 'name': 'feat_13', 'type': 'categorical'} +2024-06-11 15:28:45,089 P169311 INFO Processing column: {'active': True, 'dtype': 'float', 'name': 'feat_14', 'type': 'categorical'} +2024-06-11 15:28:55,233 P169311 INFO Processing column: {'active': True, 'dtype': 'float', 'name': 'feat_15', 'type': 'categorical'} +2024-06-11 15:29:05,185 P169311 INFO Processing column: {'active': True, 'dtype': 'float', 'name': 'feat_16', 'type': 'categorical'} +2024-06-11 15:29:15,109 P169311 INFO Processing column: {'active': True, 'dtype': 'float', 'name': 'feat_17', 'type': 'categorical'} +2024-06-11 15:29:25,233 P169311 INFO Processing column: {'active': True, 'dtype': 'float', 'name': 'feat_18', 'type': 'categorical'} +2024-06-11 15:29:34,955 P169311 INFO Processing column: {'active': True, 'dtype': 'float', 'name': 'feat_19', 'type': 'categorical'} +2024-06-11 15:29:44,869 P169311 INFO Processing column: {'active': True, 'dtype': 'float', 'name': 'feat_20', 'type': 'categorical'} +2024-06-11 15:29:54,723 P169311 INFO Processing column: {'active': True, 'dtype': 'float', 'name': 'feat_21', 'type': 'categorical'} +2024-06-11 15:30:04,695 P169311 INFO Processing column: {'active': True, 'dtype': 'float', 'name': 'feat_22', 'type': 'categorical'} +2024-06-11 15:30:14,458 P169311 INFO Set feature index... +2024-06-11 15:30:14,458 P169311 INFO Pickle feature_encoder: ../data/Avazu/avazu_x1_3fb65689/feature_encoder.pkl +2024-06-11 15:30:20,672 P169311 INFO Save feature_map to json: ../data/Avazu/avazu_x1_3fb65689/feature_map.json +2024-06-11 15:30:20,673 P169311 INFO Set feature encoder done. +2024-06-11 15:30:20,673 P169311 INFO Transform feature columns... +2024-06-11 15:36:56,128 P169311 INFO Saving data to h5: ../data/Avazu/avazu_x1_3fb65689/train.h5 +2024-06-11 15:36:59,477 P169311 INFO Preprocess feature columns... +2024-06-11 15:36:59,817 P169311 INFO Transform feature columns... +2024-06-11 15:37:57,112 P169311 INFO Saving data to h5: ../data/Avazu/avazu_x1_3fb65689/valid.h5 +2024-06-11 15:37:57,612 P169311 INFO Preprocess feature columns... +2024-06-11 15:37:58,172 P169311 INFO Transform feature columns... +2024-06-11 15:39:52,553 P169311 INFO Saving data to h5: ../data/Avazu/avazu_x1_3fb65689/test.h5 +2024-06-11 15:39:53,540 P169311 INFO Transform csv data to h5 done. +2024-06-11 15:39:53,540 P169311 INFO Loading data... +2024-06-11 15:39:53,546 P169311 INFO Loading data from h5: ../data/Avazu/avazu_x1_3fb65689/train.h5 +2024-06-11 15:39:55,407 P169311 INFO Loading data from h5: ../data/Avazu/avazu_x1_3fb65689/valid.h5 +2024-06-11 15:39:55,744 P169311 INFO Train samples: total/28300276, pos/4953382, neg/23346894, ratio/17.50%, blocks/1 +2024-06-11 15:39:55,744 P169311 INFO Validation samples: total/4042897, pos/678699, neg/3364198, ratio/16.79%, blocks/1 +2024-06-11 15:39:55,744 P169311 INFO Loading train data done. +2024-06-11 15:40:01,389 P169311 INFO Total number of parameters: 13395591. +2024-06-11 15:40:01,389 P169311 INFO Start training: 6910 batches/epoch +2024-06-11 15:40:01,389 P169311 INFO ************ Epoch=1 start ************ +2024-06-11 15:44:00,317 P169311 INFO [Metrics] AUC: 0.738388 - logloss: 0.400051 +2024-06-11 15:44:00,317 P169311 INFO Save best model: monitor(max): 0.738388 +2024-06-11 15:44:00,375 P169311 INFO --- 6910/6910 batches finished --- +2024-06-11 15:44:00,526 P169311 INFO Train loss: 0.428714 +2024-06-11 15:44:00,526 P169311 INFO ************ Epoch=1 end ************ +2024-06-11 15:47:58,946 P169311 INFO [Metrics] AUC: 0.740958 - logloss: 0.398840 +2024-06-11 15:47:58,948 P169311 INFO Save best model: monitor(max): 0.740958 +2024-06-11 15:47:59,078 P169311 INFO --- 6910/6910 batches finished --- +2024-06-11 15:47:59,240 P169311 INFO Train loss: 0.428697 +2024-06-11 15:47:59,240 P169311 INFO ************ Epoch=2 end ************ +2024-06-11 15:51:58,178 P169311 INFO [Metrics] AUC: 0.741541 - logloss: 0.399047 +2024-06-11 15:51:58,185 P169311 INFO Save best model: monitor(max): 0.741541 +2024-06-11 15:51:58,314 P169311 INFO --- 6910/6910 batches finished --- +2024-06-11 15:51:58,477 P169311 INFO Train loss: 0.428326 +2024-06-11 15:51:58,478 P169311 INFO ************ Epoch=3 end ************ +2024-06-11 15:56:00,843 P169311 INFO [Metrics] AUC: 0.739486 - logloss: 0.399609 +2024-06-11 15:56:00,845 P169311 INFO Monitor(max) STOP: 0.739486 ! +2024-06-11 15:56:00,845 P169311 INFO Reduce learning rate on plateau: 0.000100 +2024-06-11 15:56:00,846 P169311 INFO --- 6910/6910 batches finished --- +2024-06-11 15:56:01,004 P169311 INFO Train loss: 0.428737 +2024-06-11 15:56:01,004 P169311 INFO ************ Epoch=4 end ************ +2024-06-11 15:59:59,972 P169311 INFO [Metrics] AUC: 0.746336 - logloss: 0.396527 +2024-06-11 15:59:59,972 P169311 INFO Save best model: monitor(max): 0.746336 +2024-06-11 16:00:00,104 P169311 INFO --- 6910/6910 batches finished --- +2024-06-11 16:00:00,264 P169311 INFO Train loss: 0.403978 +2024-06-11 16:00:00,265 P169311 INFO ************ Epoch=5 end ************ +2024-06-11 16:03:58,553 P169311 INFO [Metrics] AUC: 0.745096 - logloss: 0.396898 +2024-06-11 16:03:58,554 P169311 INFO Monitor(max) STOP: 0.745096 ! +2024-06-11 16:03:58,554 P169311 INFO Reduce learning rate on plateau: 0.000010 +2024-06-11 16:03:58,554 P169311 INFO --- 6910/6910 batches finished --- +2024-06-11 16:03:58,720 P169311 INFO Train loss: 0.404432 +2024-06-11 16:03:58,720 P169311 INFO ************ Epoch=6 end ************ +2024-06-11 16:07:58,646 P169311 INFO [Metrics] AUC: 0.744883 - logloss: 0.397435 +2024-06-11 16:07:58,648 P169311 INFO Monitor(max) STOP: 0.744883 ! +2024-06-11 16:07:58,648 P169311 INFO Reduce learning rate on plateau: 0.000001 +2024-06-11 16:07:58,648 P169311 INFO Early stopping at epoch=7 +2024-06-11 16:07:58,648 P169311 INFO --- 6910/6910 batches finished --- +2024-06-11 16:07:58,839 P169311 INFO Train loss: 0.394071 +2024-06-11 16:07:58,839 P169311 INFO Training finished. +2024-06-11 16:07:58,839 P169311 INFO Load best model: /home/ma-user/work/DNN_avazu_x1/Avazu/DNN_avazu_x1/avazu_x1_3fb65689/DNN_avazu_x1_001_3da2d674.model +2024-06-11 16:07:58,987 P169311 INFO ****** Validation evaluation ****** +2024-06-11 16:08:13,177 P169311 INFO [Metrics] AUC: 0.746336 - logloss: 0.396527 +2024-06-11 16:08:13,250 P169311 INFO ******** Test evaluation ******** +2024-06-11 16:08:13,250 P169311 INFO Loading data... +2024-06-11 16:08:13,251 P169311 INFO Loading data from h5: ../data/Avazu/avazu_x1_3fb65689/test.h5 +2024-06-11 16:08:16,926 P169311 INFO Test samples: total/8085794, pos/1232985, neg/6852809, ratio/15.25%, blocks/1 +2024-06-11 16:08:16,927 P169311 INFO Loading test data done. +2024-06-11 16:08:45,943 P169311 INFO [Metrics] AUC: 0.763827 - logloss: 0.367394 ``` diff --git a/ranking/ctr/DNN/DNN_avazu_x1/environments.txt b/ranking/ctr/DNN/DNN_avazu_x1/environments.txt index 2d13d2a..0be063e 100644 --- a/ranking/ctr/DNN/DNN_avazu_x1/environments.txt +++ b/ranking/ctr/DNN/DNN_avazu_x1/environments.txt @@ -1,13 +1,13 @@ [Hardware] -CPU: Intel(R) Xeon(R) Gold 6278C CPU @ 2.60GHz +CPU: Intel(R) Xeon(R) Gold 6151 CPU @ 3.00GHz GPU: Tesla V100 32G -RAM: 755G +RAM: 512G [Software] CUDA: 10.2 -python: 3.6.4 +python: 3.7.10 pytorch: 1.0.0 -pandas: 0.22.0 +pandas: 1.1.5 numpy: 1.19.2 scipy: 1.5.4 sklearn: 0.22.1 @@ -15,4 +15,3 @@ pyyaml: 5.4.1 h5py: 2.8.0 tqdm: 4.60.0 fuxictr: 1.1.0 - diff --git a/ranking/ctr/DNN/DNN_avazu_x1/results.csv b/ranking/ctr/DNN/DNN_avazu_x1/results.csv index ded5cb0..c3894c2 100644 --- a/ranking/ctr/DNN/DNN_avazu_x1/results.csv +++ b/ranking/ctr/DNN/DNN_avazu_x1/results.csv @@ -1,5 +1 @@ -seed=2021, 20220208-102500,[command] python run_expid.py --version pytorch --config Avazu/DNN_avazu_x1/DNN_avazu_x1_tuner_config_randomseeds --expid DNN_avazu_x1_001_3da2d674 --gpu 0,[exp_id] DNN_avazu_x1_001_3da2d674,[dataset_id] avazu_x1_3fb65689,[train] N.A.,[val] AUC: 0.745833 - logloss: 0.396748,[test] AUC: 0.763019 - logloss: 0.368178 -seed=190034, 20220208-102927,[command] python run_expid.py --version pytorch --config Avazu/DNN_avazu_x1/DNN_avazu_x1_tuner_config_randomseeds --expid DNN_avazu_x1_002_874db960 --gpu 1,[exp_id] DNN_avazu_x1_002_874db960,[dataset_id] avazu_x1_3fb65689,[train] N.A.,[val] AUC: 0.746596 - logloss: 0.396269,[test] AUC: 0.763126 - logloss: 0.367452 -seed=992817, 20220208-102438,[command] python run_expid.py --version pytorch --config Avazu/DNN_avazu_x1/DNN_avazu_x1_tuner_config_randomseeds --expid DNN_avazu_x1_003_e0047608 --gpu 2,[exp_id] DNN_avazu_x1_003_e0047608,[dataset_id] avazu_x1_3fb65689,[train] N.A.,[val] AUC: 0.745725 - logloss: 0.396513,[test] AUC: 0.763342 - logloss: 0.367366 -seed=27011, 20220208-103816,[command] python run_expid.py --version pytorch --config Avazu/DNN_avazu_x1/DNN_avazu_x1_tuner_config_randomseeds --expid DNN_avazu_x1_004_e83b1aea --gpu 3,[exp_id] DNN_avazu_x1_004_e83b1aea,[dataset_id] avazu_x1_3fb65689,[train] N.A.,[val] AUC: 0.747169 - logloss: 0.395120,[test] AUC: 0.763145 - logloss: 0.367455 -seed=948432, 20220208-102518,[command] python run_expid.py --version pytorch --config Avazu/DNN_avazu_x1/DNN_avazu_x1_tuner_config_randomseeds --expid DNN_avazu_x1_005_64b90a5b --gpu 4,[exp_id] DNN_avazu_x1_005_64b90a5b,[dataset_id] avazu_x1_3fb65689,[train] N.A.,[val] AUC: 0.747358 - logloss: 0.395916,[test] AUC: 0.763870 - logloss: 0.367038 + 20240611-160845,[command] python run_expid.py --config ./DNN_avazu_x1_tuner_config_01 --expid DNN_avazu_x1_001_3da2d674 --gpu 0,[exp_id] DNN_avazu_x1_001_3da2d674,[dataset_id] avazu_x1_3fb65689,[train] N.A.,[val] AUC: 0.746336 - logloss: 0.396527,[test] AUC: 0.763827 - logloss: 0.367394