Nepal-Earthquake-predictor-NeuralNetwork.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Welcome to my Richter's Predictor Nepal Earthquake Damage Predictor Neural Network Model!"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Default code from Kaggle Notebook:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# This Python 3 environment comes with many helpful analytics libraries installed\n",
    "# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n",
    "# For example, here's several helpful packages to load\n",
    "\n",
    "import numpy as np # linear algebra\n",
    "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
    "\n",
    "# Input data files are available in the read-only \"../input/\" directory\n",
    "# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n",
    "\n",
    "import os\n",
    "for dirname, _, filenames in os.walk('/kaggle/input'):\n",
    "    for filename in filenames:\n",
    "        print(os.path.join(dirname, filename))\n",
    "\n",
    "# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n",
    "# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Making sure TensorFlow uses the first GPU"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024-01-13 00:20:24.905560: I tensorflow/core/util/port.cc:113] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`.\n",
      "2024-01-13 00:20:24.930467: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:9261] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n",
      "2024-01-13 00:20:24.930490: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:607] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n",
      "2024-01-13 00:20:24.931033: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1515] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n",
      "2024-01-13 00:20:24.936767: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n",
      "To enable the following instructions: AVX2 AVX512F AVX512_VNNI AVX512_BF16 FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
      "2024-01-13 00:20:25.490648: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"0\"\n",
    "\n",
    "import tensorflow as tf\n",
    "\n",
    "#tf.debugging.set_log_device_placement(True) # This will make TensorFlow print out which device (CPU OR GPU) is being used for each operation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Displaying some important libraries:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import tensorflow as tf\n",
    "# print(\"Tensorflow:\", tf.__version__)\n",
    "\n",
    "# import kerastuner as kt\n",
    "# print(\"kerastuner:\", kt.__version__)\n",
    "\n",
    "# import keras_tuner as kt2\n",
    "# print(\"keras_tuner:\", kt2.__version__)\n",
    "\n",
    "# import platform\n",
    "# print(\"Python:\", platform.python_version())\n",
    "\n",
    "# import numpy as np\n",
    "# print(\"numpy:\", np.__version__)\n",
    "\n",
    "# import pandas as pd\n",
    "# print(\"pandas:\", pd.__version__)\n",
    "\n",
    "# import sklearn\n",
    "# print(\"sklearn version:\", sklearn.__version__)\n",
    "\n",
    "# import sklearn\n",
    "# print(\"sklearn path:\", sklearn.__path__)\n",
    "\n",
    "# import matplotlib\n",
    "# print(\"matplotlib:\", matplotlib.__version__)\n",
    "\n",
    "# import seaborn as sns\n",
    "# print(\"seaborn:\", sns.__version__)\n",
    "\n",
    "# # WARNING:tensorflow:From c:\\Users\\Micha\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\keras\\src\\losses.py:2976: The name tf.losses.sparse_softmax_cross_entropy is deprecated. Please use tf.compat.v1.losses.sparse_softmax_cross_entropy instead.\n",
    "\n",
    "# # Tensorflow: 2.15.0\n",
    "# # C:\\Users\\Micha\\AppData\\Local\\Temp\\ipykernel_6936\\1753711907.py:4: DeprecationWarning: `import kerastuner` is deprecated, please use `import keras_tuner`.\n",
    "# #   import kerastuner as kt\n",
    "# # kerastuner: 1.0.5\n",
    "# # keras_tuner: 1.3.5\n",
    "# # Python: 3.10.11\n",
    "# # numpy: 1.24.3\n",
    "# # pandas: 2.1.4\n",
    "# # sklearn version: 1.2.2\n",
    "# # sklearn path: ['c:\\\\Users\\\\Micha\\\\AppData\\\\Local\\\\Programs\\\\Python\\\\Python310\\\\lib\\\\site-packages\\\\sklearn']\n",
    "# # matplotlib: 3.8.2\n",
    "# # seaborn: 0.13.0\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Set Global random seed to make sure we can replicate any model that we create (no randomness)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import random\n",
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "import os\n",
    "\n",
    "\n",
    "\n",
    "np.random.seed(42)\n",
    "random.seed(42)\n",
    "np.random.seed(42)\n",
    "tf.random.set_seed(42)\n",
    "\n",
    "os.environ['TF_DETERMINISTIC_OPS'] = '1'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Loading the training and testing data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "train_values are the features (X), and train_labels is the target/label (Y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_X = pd.read_csv(\"train_values.csv\")\n",
    "train_Y = pd.read_csv(\"train_labels.csv\")\n",
    "\n",
    "test_values = pd.read_csv(\"test_values.csv\")\n",
    "\n",
    "# print(\"train labels:\\n\", train_Y.head())\n",
    "\n",
    "# print(\"train values:\\n\", train_X.head())\n",
    "      \n",
    "# print(\"test_values:\\n\", test_values.head())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## I want to find out which features to use since there are so many. Here are some common data science techniques:\n",
    "\n",
    "1. **Correlation Matrix with Heatmap**: Correlation states how the features are related to each other or the target variable. You can use a heatmap to visualize the correlation matrix.\n",
    "\n",
    "2. **Univariate Selection**: Statistical tests can be used to select those features that have the strongest relationship with the output variable. The scikit-learn library provides the `SelectKBest` class that can be used with a suite of different statistical tests to select a specific number of features.\n",
    "\n",
    "3. **Recursive Feature Elimination (RFE)**: RFE is a popular feature selection method that fits a model and removes the weakest feature (or features) until the specified number of features is reached.\n",
    "\n",
    "4. **Feature Importance**: You can get the feature importance of each feature of your dataset by using the feature importance property of the model. For example, Decision Trees models in the scikit-learn library offer an importance property that can be accessed directly.\n",
    "\n",
    "For categorical features, you can convert them into numerical values using techniques like One-Hot Encoding or Label Encoding before applying these feature selection techniques. "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## First, I will try RFE (Recursive Feature Elimination)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# from sklearn.feature_selection import RFE\n",
    "# from lightgbm import LGBMRegressor #Lightgbm is a great gradient boosting model for large amount of data\n",
    "\n",
    "# # Assuming X is your feature set and y is the target value\n",
    "# X = train_X.drop('building_id', axis=1)\n",
    "# X = pd.get_dummies(X)\n",
    "\n",
    "# y = train_Y.drop('building_id', axis=1)\n",
    "# y = np.ravel(y) # converting dataframe to a one-dimensional array using the ravel function from numpy\n",
    "\n",
    "# estimator = LGBMRegressor(verbose = 0, random_state = 42)  # It's best to find the best model for you\n",
    "# selector = RFE(estimator, step=1)\n",
    "# selector = selector.fit(X, y)\n",
    "\n",
    "# # Assuming 'X' is your DataFrame with the feature data\n",
    "# feature_names = X.columns\n",
    "\n",
    "# # Map the feature names to the support array, which tells you which features were selected\n",
    "# support_dict = dict(zip(feature_names, selector.support_))\n",
    "\n",
    "# # Get the selected features\n",
    "# selected_features = [feature for feature, support in support_dict.items() if support]\n",
    "\n",
    "# # Print the selected features\n",
    "# print(\"Selected features:\\n\", selected_features)\n",
    "\n",
    "# # ['geo_level_1_id', 'geo_level_2_id', 'geo_level_3_id', 'count_floors_pre_eq', 'age', 'area_percentage', 'height_percentage',\n",
    "# # 'has_superstructure_adobe_mud', 'has_superstructure_mud_mortar_stone', 'has_superstructure_stone_flag', 'has_superstructure_cement_mortar_stone',\n",
    "# # 'has_superstructure_mud_mortar_brick', 'has_superstructure_cement_mortar_brick', 'has_superstructure_timber', 'has_superstructure_bamboo', \n",
    "# # 'has_superstructure_rc_non_engineered', 'has_superstructure_rc_engineered', 'has_superstructure_other', 'count_families', 'has_secondary_use', \n",
    "# # 'land_surface_condition_n', 'land_surface_condition_o', 'foundation_type_h', 'foundation_type_r', 'foundation_type_u', 'roof_type_n', \n",
    "# # 'roof_type_q', 'roof_type_x', 'ground_floor_type_f', 'ground_floor_type_v', 'ground_floor_type_x', 'other_floor_type_q', 'position_s',\n",
    "# # 'plan_configuration_u']\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "# # The feature ranking, such that ranking_[i] corresponds to the ranking position of the i-th feature. \n",
    "# # Selected features are assigned rank 1.\n",
    "# # Map the feature names to the ranking array\n",
    "# ranking_dict = dict(zip(feature_names, selector.ranking_))\n",
    "# print(ranking_dict)\n",
    "\n",
    "\n",
    "# # [ 1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  2 18 10 31\n",
    "# #  27 33 28 25 30  6  1  1  3  1  8  1  1  5  1  1  1  1 34  1  1 16  9  1\n",
    "# #  23  7  4 14  1 29 15 20 17 35 32 24 21 13 26  1 11 22 12 19]\n",
    "\n",
    "\n",
    "# # {'geo_level_1_id': 1, 'geo_level_2_id': 1, 'geo_level_3_id': 1, 'count_floors_pre_eq': 1, 'age': 1, 'area_percentage': 1, \n",
    "# #  'height_percentage': 1, 'has_superstructure_adobe_mud': 1, 'has_superstructure_mud_mortar_stone': 1, 'has_superstructure_stone_flag': 1, \n",
    "# #  'has_superstructure_cement_mortar_stone': 1, 'has_superstructure_mud_mortar_brick': 1, 'has_superstructure_cement_mortar_brick': 1, \n",
    "# #  'has_superstructure_timber': 1, 'has_superstructure_bamboo': 1, 'has_superstructure_rc_non_engineered': 1, 'has_superstructure_rc_engineered': 1, \n",
    "# #  'has_superstructure_other': 1, 'count_families': 1, 'has_secondary_use': 1, 'has_secondary_use_agriculture': 2, 'has_secondary_use_hotel': 18, \n",
    "# #  'has_secondary_use_rental': 10, 'has_secondary_use_institution': 31, 'has_secondary_use_school': 27, 'has_secondary_use_industry': 33, \n",
    "# #  'has_secondary_use_health_post': 28, 'has_secondary_use_gov_office': 25, 'has_secondary_use_use_police': 30, 'has_secondary_use_other': 6, \n",
    "# #  'land_surface_condition_n': 1, 'land_surface_condition_o': 1, 'land_surface_condition_t': 3, 'foundation_type_h': 1, 'foundation_type_i': 8, \n",
    "# #  'foundation_type_r': 1, 'foundation_type_u': 1, 'foundation_type_w': 5, 'roof_type_n': 1, 'roof_type_q': 1, 'roof_type_x': 1, \n",
    "# #  'ground_floor_type_f': 1, 'ground_floor_type_m': 34, 'ground_floor_type_v': 1, 'ground_floor_type_x': 1, 'ground_floor_type_z': 16, \n",
    "# #  'other_floor_type_j': 9, 'other_floor_type_q': 1, 'other_floor_type_s': 23, 'other_floor_type_x': 7, 'position_j': 4, 'position_o': 14, \n",
    "# #  'position_s': 1, 'position_t': 29, 'plan_configuration_a': 15, 'plan_configuration_c': 20, 'plan_configuration_d': 17, 'plan_configuration_f': 35, \n",
    "# #  'plan_configuration_m': 32, 'plan_configuration_n': 24, 'plan_configuration_o': 21, 'plan_configuration_q': 13, 'plan_configuration_s': 26, \n",
    "# #  'plan_configuration_u': 1, 'legal_ownership_status_a': 11, 'legal_ownership_status_r': 22, 'legal_ownership_status_v': 12, \n",
    "# #  'legal_ownership_status_w': 19}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "34\n",
      "New features:\n",
      " ['land_surface_condition_n', 'land_surface_condition_o', 'foundation_type_h', 'foundation_type_r', 'foundation_type_u', 'roof_type_n', 'roof_type_q', 'roof_type_x', 'ground_floor_type_f', 'ground_floor_type_v', 'ground_floor_type_x', 'other_floor_type_q', 'position_s', 'plan_configuration_u']\n",
      "27\n"
     ]
    }
   ],
   "source": [
    "features = ['geo_level_1_id', 'geo_level_2_id', 'geo_level_3_id', 'count_floors_pre_eq', 'age', 'area_percentage', 'height_percentage',\n",
    "'has_superstructure_adobe_mud', 'has_superstructure_mud_mortar_stone', 'has_superstructure_stone_flag', 'has_superstructure_cement_mortar_stone',\n",
    "'has_superstructure_mud_mortar_brick', 'has_superstructure_cement_mortar_brick', 'has_superstructure_timber', 'has_superstructure_bamboo', \n",
    "'has_superstructure_rc_non_engineered', 'has_superstructure_rc_engineered', 'has_superstructure_other', 'count_families', 'has_secondary_use', \n",
    "'land_surface_condition_n', 'land_surface_condition_o', 'foundation_type_h', 'foundation_type_r', 'foundation_type_u', 'roof_type_n', \n",
    "'roof_type_q', 'roof_type_x', 'ground_floor_type_f', 'ground_floor_type_v', 'ground_floor_type_x', 'other_floor_type_q', 'position_s',\n",
    "'plan_configuration_u']\n",
    "\n",
    "print(len(features))\n",
    "\n",
    "\n",
    "# Find out which features are created through one-hot-encoding\n",
    "import pandas as pd\n",
    "\n",
    "# Load the original data\n",
    "original_data = pd.read_csv('train_values.csv')\n",
    "\n",
    "# Get the original feature names\n",
    "original_features = original_data.columns\n",
    "\n",
    "# Check which features are not in the original data\n",
    "new_features = [feature for feature in features if feature not in original_features]\n",
    "\n",
    "# Print the new features\n",
    "print(\"New features:\\n\", new_features)\n",
    "\n",
    "\n",
    "#Manually remove the one-hot-encoding that pd.get_dummies() used on categorial \n",
    "features_before_dummies = ['geo_level_1_id', 'geo_level_2_id', 'geo_level_3_id', 'count_floors_pre_eq', 'age', 'area_percentage', 'height_percentage',\n",
    "'has_superstructure_adobe_mud', 'has_superstructure_mud_mortar_stone', 'has_superstructure_stone_flag', 'has_superstructure_cement_mortar_stone',\n",
    "'has_superstructure_mud_mortar_brick', 'has_superstructure_cement_mortar_brick', 'has_superstructure_timber', 'has_superstructure_bamboo', \n",
    "'has_superstructure_rc_non_engineered', 'has_superstructure_rc_engineered', 'has_superstructure_other', 'count_families', 'has_secondary_use', \n",
    "'land_surface_condition', 'foundation_type', 'roof_type', 'ground_floor_type','other_floor_type', 'position','plan_configuration']\n",
    "\n",
    "print(len(features_before_dummies))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data Analysis from ChatGPT-4 (second way to find best features):\n",
    "\n",
    "The categorical variables in the features dataset have been successfully encoded. Now, let's look at the correlation of these features with the `damage_grade`:\n",
    "\n",
    "#### Correlation with `damage_grade`\n",
    "The correlation values range between -1 and 1. A value closer to 1 indicates a strong positive correlation, meaning that as the feature increases, the `damage_grade` tends to increase. Conversely, a value closer to -1 indicates a strong negative correlation, where an increase in the feature leads to a decrease in `damage_grade`. Values around 0 imply weak or no linear correlation.\n",
    "\n",
    "#### Top Positively Correlated Features:\n",
    "- `has_superstructure_mud_mortar_stone`\n",
    "- `count_floors_pre_eq`\n",
    "- Other features like `legal_ownership_status`, `has_superstructure_stone_flag`, etc., also show positive correlation but to a lesser extent.\n",
    "\n",
    "#### Top Negatively Correlated Features:\n",
    "- `has_superstructure_cement_mortar_brick`\n",
    "- `ground_floor_type`\n",
    "- `has_superstructure_rc_engineered`\n",
    "- Other features like `roof_type`, `has_superstructure_rc_non_engineered`, etc., also show negative correlation.\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Here are the selected features based on the correlation threshold (of 0.05):\n",
    "\n",
    "1. `has_superstructure_mud_mortar_stone`\n",
    "2. `count_floors_pre_eq`\n",
    "3. `legal_ownership_status`\n",
    "4. `has_superstructure_stone_flag`\n",
    "5. `count_families`\n",
    "6. `has_superstructure_adobe_mud`\n",
    "7. `position`\n",
    "8. `has_superstructure_cement_mortar_stone`\n",
    "9. `has_superstructure_bamboo`\n",
    "10. `has_superstructure_timber`\n",
    "11. `geo_level_1_id`\n",
    "12. `has_secondary_use`\n",
    "13. `has_secondary_use_rental`\n",
    "14. `has_secondary_use_hotel`\n",
    "15. `foundation_type`\n",
    "16. `area_percentage`\n",
    "17. `has_superstructure_rc_non_engineered`\n",
    "18. `roof_type`\n",
    "19. `has_superstructure_rc_engineered`\n",
    "20. `ground_floor_type`\n",
    "21. `has_superstructure_cement_mortar_brick`\n",
    "\n",
    "These features were chosen because they have a correlation with the target variable `damage_grade` greater than the specified threshold of 0.05 (in absolute value). You can use these features for building your predictive model. \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "features2 = [\n",
    "    \"has_superstructure_mud_mortar_stone\",\n",
    "    \"count_floors_pre_eq\",\n",
    "    \"legal_ownership_status\",\n",
    "    \"has_superstructure_stone_flag\",\n",
    "    \"count_families\",\n",
    "    \"has_superstructure_adobe_mud\",\n",
    "    \"position\",\n",
    "    \"has_superstructure_cement_mortar_stone\",\n",
    "    \"has_superstructure_bamboo\",\n",
    "    \"has_superstructure_timber\",\n",
    "    \"geo_level_1_id\",\n",
    "    \"has_secondary_use\",\n",
    "    \"has_secondary_use_rental\",\n",
    "    \"has_secondary_use_hotel\",\n",
    "    \"foundation_type\",\n",
    "    \"area_percentage\",\n",
    "    \"has_superstructure_rc_non_engineered\",\n",
    "    \"roof_type\",\n",
    "    \"has_superstructure_rc_engineered\",\n",
    "    \"ground_floor_type\",\n",
    "    \"has_superstructure_cement_mortar_brick\"\n",
    "]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Third, I will try SelectKBest to find best features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# from sklearn.feature_selection import SelectKBest, f_classif\n",
    "\n",
    "\n",
    "# # Create the SelectKBest with the f_classif function. You can set the parameter \"k\" equal to a number if you want to limit the amount of features\n",
    "# selector = SelectKBest(f_classif, k = 20) # Getting the 20 best features\n",
    "\n",
    "# # Assuming X is your feature set and y is the target value\n",
    "# X = train_X.drop('building_id', axis=1)\n",
    "# X = pd.get_dummies(X)\n",
    "\n",
    "# y = train_Y.drop('building_id', axis=1)\n",
    "# y = np.ravel(y) # converting dataframe to a one-dimensional array using the ravel function from numpy\n",
    "\n",
    "\n",
    "# # Fit the selector to the data\n",
    "# selector.fit(X, y)\n",
    "\n",
    "# # Get the boolean mask of the selected features\n",
    "# mask = selector.get_support()\n",
    "\n",
    "# # Get the names of the selected features\n",
    "# selected_features = X.columns[mask]\n",
    "\n",
    "# # Convert the Index object to a list\n",
    "# features3 = selected_features.tolist()\n",
    "\n",
    "# print(features3)\n",
    "\n",
    "# # ['geo_level_1_id', 'count_floors_pre_eq', 'area_percentage', 'has_superstructure_mud_mortar_stone', 'has_superstructure_cement_mortar_brick', \n",
    "# #  'has_superstructure_rc_non_engineered', 'has_superstructure_rc_engineered', 'has_secondary_use_hotel', 'has_secondary_use_rental', \n",
    "# #  'foundation_type_i', 'foundation_type_r', 'foundation_type_u', 'foundation_type_w', 'roof_type_n', 'roof_type_x', 'ground_floor_type_f', \n",
    "# #  'ground_floor_type_v', 'other_floor_type_j', 'other_floor_type_q', 'other_floor_type_s']\n",
    "\n",
    "\n",
    "# # Get the scores\n",
    "# scores = selector.scores_\n",
    "\n",
    "# # Create a DataFrame with the scores\n",
    "# features_scores = pd.DataFrame({'Feature': X.columns, 'Score': scores})\n",
    "\n",
    "# # Sort the DataFrame by score in descending order\n",
    "# features_scores = features_scores.sort_values(by='Score', ascending=False)\n",
    "\n",
    "# # print the best 20 features\n",
    "# print(features_scores[0:19])\n",
    "\n",
    "# #                                    Feature         Score\n",
    "# # 35                       foundation_type_r  23787.275036\n",
    "# # 43                     ground_floor_type_v  20782.933584\n",
    "# # 40                             roof_type_x  16891.038184\n",
    "# # 8      has_superstructure_mud_mortar_stone  16490.386507\n",
    "# # 34                       foundation_type_i  16385.772905\n",
    "# # 12  has_superstructure_cement_mortar_brick  11120.193268\n",
    "# # 48                      other_floor_type_s  10507.484572\n",
    "# # 41                     ground_floor_type_f  10151.525359\n",
    "# # 16        has_superstructure_rc_engineered   7757.593854\n",
    "# # 47                      other_floor_type_q   7378.599061\n",
    "# # 15    has_superstructure_rc_non_engineered   4721.916051\n",
    "# # 37                       foundation_type_w   4568.674306\n",
    "# # 46                      other_floor_type_j   4533.708398\n",
    "# # 36                       foundation_type_u   2972.409108\n",
    "# # 0                           geo_level_1_id   2657.791274\n",
    "# # 3                      count_floors_pre_eq   2544.836052\n",
    "# # 5                          area_percentage   2529.046730\n",
    "# # 38                             roof_type_n   1776.396178\n",
    "# # 21                 has_secondary_use_hotel   1537.672773\n",
    "\n",
    "# # ...\n",
    "\n",
    "# # 22                has_secondary_use_rental   1342.099336\n",
    "# # 64                legal_ownership_status_a   1166.606551\n",
    "# # 19                       has_secondary_use    841.802928\n",
    "# # 39                             roof_type_q    761.885856\n",
    "# # 7             has_superstructure_adobe_mud    739.412821\n",
    "# # 13               has_superstructure_timber    659.199014\n",
    "# # 9            has_superstructure_stone_flag    576.438023\n",
    "# # 14               has_superstructure_bamboo    538.551492\n",
    "# # 66                legal_ownership_status_v    536.308634\n",
    "# # 11     has_superstructure_mud_mortar_brick    531.784659\n",
    "# # 63                    plan_configuration_u    515.087147\n",
    "# # 10  has_superstructure_cement_mortar_stone    478.844199\n",
    "# # 18                          count_families    476.562914\n",
    "# # 56                    plan_configuration_d    378.234531\n",
    "# # 53                              position_t    373.594539\n",
    "# # 6                        height_percentage    370.173817\n",
    "# # 20           has_secondary_use_agriculture    289.462856\n",
    "# # 1                           geo_level_2_id    264.447807\n",
    "# # 49                      other_floor_type_x    244.432657\n",
    "# # 4                                      age    219.626253\n",
    "# # 33                       foundation_type_h    209.425818\n",
    "# # 32                land_surface_condition_t    201.698101\n",
    "# # 30                land_surface_condition_n    182.152148\n",
    "# # 61                    plan_configuration_q    165.370472\n",
    "# # 23           has_secondary_use_institution    146.731486\n",
    "# # 17                has_superstructure_other    142.014204\n",
    "# # 50                              position_j    136.783490\n",
    "# # 67                legal_ownership_status_w    116.039402\n",
    "# # 52                              position_s    110.038831\n",
    "# # 55                    plan_configuration_c     75.423022\n",
    "# # 51                              position_o     75.104323\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "features3 = ['geo_level_1_id', 'count_floors_pre_eq', 'area_percentage', 'has_superstructure_mud_mortar_stone', 'has_superstructure_cement_mortar_brick', \n",
    " 'has_superstructure_rc_non_engineered', 'has_superstructure_rc_engineered', 'has_secondary_use_hotel', 'has_secondary_use_rental', \n",
    " 'foundation_type_i', 'foundation_type_r', 'foundation_type_u', 'foundation_type_w', 'roof_type_n', 'roof_type_x', 'ground_floor_type_f', \n",
    " 'ground_floor_type_v', 'other_floor_type_j', 'other_floor_type_q', 'other_floor_type_s']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Check to see if there are any missing values in the data. If so, we have to do imputation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of missing values in train_X: 0\n",
      "Number of missing values in train_Y: 0\n",
      "Number of missing values in test_values: 0\n"
     ]
    }
   ],
   "source": [
    "missing_train_X = train_X.isnull().sum().sum()\n",
    "print(\"Number of missing values in train_X:\", missing_train_X)\n",
    "\n",
    "missing_train_Y = train_Y.isnull().sum().sum()\n",
    "print(\"Number of missing values in train_Y:\", missing_train_Y)\n",
    "\n",
    "missing_test_values = test_values.isnull().sum().sum()\n",
    "print(\"Number of missing values in test_values:\", missing_test_values )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Since we have 0 missing values in each dataframe, we don't have to do imputation!"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Now, we have 3 different list of features (features, features2, and features3), which I found using RFE. Data Analysis ChatGPT-4, and SelectKBest respectively\n",
    "\n",
    "#### Now, I have to turn one-hot-encode the data using pd.get_dummies, and I'll be creating 3 seperate train_X, one for each list of possible best features. And also on the test_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 1. testX1 for the RFE features\n",
    "# Since the features from RFE are the one-hot-encoded features, we have to apply features after doing pd.get_dummies()\n",
    "\n",
    "trainX1 = pd.get_dummies(train_X)\n",
    "trainX1 = trainX1[features]\n",
    "\n",
    "\n",
    "# 2. testX2 for the Data Analysis ChatGPT-4\n",
    "# Since the features from Data Analysis are from the original feature set, we have to apply the features before doing pd.get_dummies()\n",
    "\n",
    "trainX2 = train_X[features2]\n",
    "trainX2 = pd.get_dummies(trainX2)\n",
    "\n",
    "\n",
    "\n",
    "# 3. testX3 for the SelectKBest\n",
    "# Since the features from RFE are the one-hot-encoded features, we have to apply features after doing pd.get_dummies()\n",
    "\n",
    "trainX3 = pd.get_dummies(train_X)\n",
    "trainX3 = trainX3[features3]\n",
    "\n",
    "\n",
    "\n",
    "# 4. Do pd.get_dummies() on test data. I will create a seperate test_data for each feature selection, since each test_data needs to have a certain set of features\n",
    "\n",
    "test_data1 = pd.get_dummies(test_values)\n",
    "test_data1 = test_data1[features + ['building_id'] ]\n",
    "\n",
    "test_data2 = test_values[features2 + ['building_id'] ]\n",
    "test_data2 = pd.get_dummies(test_data2)\n",
    "\n",
    "test_data3 = pd.get_dummies(test_values)\n",
    "test_data3 = test_data3[features3 + ['building_id'] ]\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### It's important to do pd.get_dummies() before doing the train_valid_test split. Now we can do the split\n",
    "I have to do train_valid_test split three times, one for each different train_X"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "# 1. train_valid_test split for train_X1\n",
    "train_X1, test_X1, train_Y1, test_Y1 = train_test_split(trainX1, train_Y, test_size=0.3, random_state = 42) # split into training (70%) and a test set (30%)\n",
    "\n",
    "valid_X1, test_X1, valid_Y1, test_Y1 = train_test_split(test_X1, test_Y1, test_size = 0.5, random_state = 42) # split test set into a validation (15%) and test set (15%)\n",
    "\n",
    "\n",
    "# 2. train_valid_test split for train_X2\n",
    "train_X2, test_X2, train_Y2, test_Y2 = train_test_split(trainX2, train_Y, test_size=0.3, random_state = 42) # split into training (70%) and a test set (30%)\n",
    "\n",
    "valid_X2, test_X2, valid_Y2, test_Y2 = train_test_split(test_X2, test_Y2, test_size = 0.5, random_state = 42) # split test set into a validation (15%) and test set (15%)\n",
    "\n",
    "\n",
    "# 3. train_valid_test split for train_X3\n",
    "train_X3, test_X3, train_Y3, test_Y3 = train_test_split(trainX3, train_Y, test_size=0.3, random_state = 42) # split into training (70%) and a test set (30%)\n",
    "\n",
    "valid_X3, test_X3, valid_Y3, test_Y3 = train_test_split(test_X3, test_Y3, test_size = 0.5, random_state = 42) # split test set into a validation (15%) and test set (15%)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Now, we can normalize the data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Creating the Normalizing Scaler**"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "There are many different Normalization Scalers:\n",
    "\n",
    "1. **MinMaxScaler**: This scaler scales and translates each feature individually such that it is in the given range on the training set, e.g., between zero and one.\n",
    "\n",
    "\n",
    "2. **StandardScaler**: This scaler standardizes features by removing the mean and scaling to unit variance.\n",
    "\n",
    "\n",
    "3. **RobustScaler**: This scaler scales features using statistics that are robust to outliers. It uses the Interquartile Range (IQR) to scale the data, making it a better choice for when the data has outliers.\n",
    "\n",
    "\n",
    "4. **Normalizer**: This scaler scales individual samples to have unit norm. This scaler works on the rows, not the columns!\n",
    "\n",
    "\n",
    "5. **MaxAbsScaler**: This scaler scales and translates each feature individually such that the maximal absolute value of each feature in the training set will be 1.0. It does not shift/center the data, and thus does not destroy any sparsity.\n",
    "\n",
    "\n",
    "Remember, the choice of scaler can depend on your specific dataset and the machine learning algorithm that you're using."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# I've actually decided to not do normalization since almost all the columns are categorial columns, and the non-categorial columns are mostly normalized already in the dataset, so there's no need    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "I've actually decided to not do normalization since almost all the columns are categorial columns, and the non-categorial columns are mostly normalized already in the dataset, so there's no need"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Change all the data to float32 so that it can be converted into tensor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_X1 = train_X1.astype('float32')\n",
    "valid_X1 = valid_X1.astype('float32')\n",
    "test_X1 = test_X1.astype('float32')\n",
    "\n",
    "train_X2 = train_X2.astype('float32')\n",
    "valid_X2 = valid_X2.astype('float32')\n",
    "test_X2 = test_X2.astype('float32')\n",
    "\n",
    "train_X3 = train_X3.astype('float32')\n",
    "valid_X3 = valid_X3.astype('float32')\n",
    "test_X3 = test_X3.astype('float32')\n",
    "\n",
    "test_data1 = test_data1.astype('float32')\n",
    "test_data2 = test_data2.astype('float32')\n",
    "test_data3 = test_data3.astype('float32')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model 1, TensorFlow Neural Network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import keras_tuner as kt\n",
    "# from tensorflow.keras.callbacks import EarlyStopping\n",
    "\n",
    "\n",
    "# %pip install tensorflow-addons\n",
    "\n",
    "# import tensorflow_addons as tfa\n",
    "\n",
    "\n",
    "# # TensorFlow expects the labels data to only include one column. Currently, they include two columns: building_id and damage_grade.\n",
    "# # So, I am making it so the labels data only includes one column (damage_grade) and changing the range from 1-3 to 0-2 since TensorFlow wants to start at 0\n",
    "# train_Y1_temp = train_Y1['damage_grade'] - 1  \n",
    "# valid_Y1_temp = valid_Y1['damage_grade'] - 1\n",
    "# test_Y1_temp = test_Y1['damage_grade'] - 1\n",
    "\n",
    "\n",
    "# def build_model(hp):\n",
    "#     model = tf.keras.Sequential()\n",
    "    \n",
    "#     # Input layer\n",
    "#     model.add(tf.keras.layers.Dense(\n",
    "#         units=hp.Int('input_units', min_value=32, max_value=256, step=20),\n",
    "#         activation=hp.Choice('input_activation', ['relu', 'tanh', 'sigmoid']),\n",
    "#         input_shape=(train_X1.shape[1],)\n",
    "#     ))\n",
    "    \n",
    "#     # Hidden layers\n",
    "#     for i in range(hp.Int('n_layers', 1, 5)):\n",
    "#         model.add(tf.keras.layers.Dense(\n",
    "#             units=hp.Int(f'hidden_units_{i}', min_value=32, max_value=256, step=20),\n",
    "#             activation=hp.Choice(f'hidden_activation_{i}', ['relu', 'tanh', 'sigmoid']),\n",
    "#             kernel_regularizer=tf.keras.regularizers.l2(hp.Float(f'regularization_{i}', min_value=0.0, max_value=0.1, step=0.01))\n",
    "#         ))\n",
    "#         # Dropout layer\n",
    "#         model.add(tf.keras.layers.Dropout(rate=hp.Float('dropout_rate', min_value=0.0, max_value=0.5, step=0.1)))\n",
    "    \n",
    "#     # Output layer\n",
    "#     model.add(tf.keras.layers.Dense(3, activation='softmax'))\n",
    "    \n",
    "#     # Compile the model\n",
    "#     model.compile(\n",
    "#         optimizer=hp.Choice('optimizer', ['adam', 'sgd', 'rmsprop']),\n",
    "#         loss='sparse_categorical_crossentropy',\n",
    "#         # metrics=['accuracy'])\n",
    "#         metrics=[tfa.metrics.F1Score(num_classes=3, average='micro')]) # for f1 micro score\n",
    "    \n",
    "#     return model\n",
    "\n",
    "\n",
    "# tuner = kt.BayesianOptimization(\n",
    "#     build_model,\n",
    "#     max_trials=20,\n",
    "#     objective=kt.Objective('f1_score', direction='max'), # for f1 micro score\n",
    "#     #objective='val_accuracy',\n",
    "#     #max_epochs=50,\n",
    "#     directory='directory_for_model_1',  #Make sure to change ethe directory  and project name if you want to start a fresh new tuning session\n",
    "#     project_name='keras_tuner_model1', # make sure to change the project and directory name if you want to start a fresh new tuning session\n",
    "#     #hyperband_iterations=1# Increase if you have more time\n",
    "#     )  \n",
    "\n",
    "\n",
    "# early_stopping = EarlyStopping(monitor='val_loss', patience=5) # If the validation loss does not decrease for 10 (`patience=10`) consecutive epochs, the training process will stop early. \n",
    "\n",
    "# # Search for the best hyperparameters\n",
    "# tuner.search(pd.concat([train_X1, valid_X1, test_X1], ignore_index=True), \n",
    "#              pd.concat([train_Y1_temp, valid_Y1_temp, test_Y1_temp], ignore_index=True),  # make sure to use the temp values, which is the label data without the building_id\n",
    "#              epochs=5, \n",
    "#              validation_split=0.2,  # 20% of the data will be used for validation\n",
    "#              batch_size=200, # make sure to change this and test out different values\n",
    "#              callbacks=[early_stopping])  # Add early stopping callback)  # Fixed batch size\n",
    "\n",
    "# # Get the best model\n",
    "# best_model = tuner.get_best_models()[0]\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Get information about the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# #1 Summary of the model: This will print a summary of the model's layers, output shapes, and the number of parameters.\n",
    "# print(\"Summary of Model: \")\n",
    "# best_model = tuner.get_best_models(num_models=1)[0]\n",
    "# best_model.summary()\n",
    "\n",
    "# #Writing it to a file\n",
    "# with open('TF_Model1_Summary.txt', 'w') as f:\n",
    "#     # Write the model summary\n",
    "#     f.write(\"Summary of Model: \\n\")\n",
    "#     f.write(str(best_model.summary()))\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "# #2 Inspect Hyperparameters: This will display the values of the hyperparameters like the number of units in each layer, learning rate, etc.\n",
    "# print(\"Hyperparameters: \")\n",
    "# best_hp = tuner.get_best_hyperparameters()[0]\n",
    "# for param in best_hp.values:\n",
    "#     print(param, best_hp.get(param))\n",
    "\n",
    "\n",
    "# #Writing it to a file\n",
    "# import json\n",
    "\n",
    "# # Get the best hyperparameters\n",
    "# best_hp = tuner.get_best_hyperparameters()[0]\n",
    "\n",
    "# # Convert the hyperparameters to a dictionary\n",
    "# hp_dict = {param: best_hp.get(param) for param in best_hp.values}\n",
    "\n",
    "# # Write the dictionary to a file in JSON format\n",
    "# with open('hyperparameters_TF_model1.json', 'w') as f:\n",
    "#     json.dump(hp_dict, f)\n",
    "\n",
    "\n",
    "\n",
    "    \n",
    "# #3 Model Configuration: This returns a Python dictionary containing the model configuration. It can be quite detailed and technical, showing layer types, activation functions, and other layer-specific settings.\n",
    "# print(\"Model Configuration: \")\n",
    "# config = best_model.get_config()\n",
    "\n",
    "# #Writing it to a file\n",
    "# with open('TF_Model1_Configuration.txt', 'w') as f:\n",
    "#     # Write the configuration\n",
    "#     f.write(\"Configuration: \\n\")\n",
    "#     f.write(str(best_model.get_config()))\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "# #4 Visualizing the Model: This creates a plot of the model architecture, showing the layers, their shapes, and how they're connected.\n",
    "# print(\"Visualizing the model: \")  \n",
    "# tf.keras.utils.plot_model(best_model, to_file='model1_TF.png', show_shapes=True, show_layer_names=True) # MAKE SURE TO CHANGE THE FILE NAME \n",
    "\n",
    "\n",
    "\n",
    "\n",
    "# #5 Weights and Biases: This will print out the weights and biases for each layer in the model. Be cautious with large models, as this can be a lot of data.\n",
    "# # print(\"Weights and Biases: \")\n",
    "# # for layer in best_model.layers:\n",
    "# #     weights, biases = layer.get_weights()\n",
    "# #     print(layer.name, weights, biases)\n",
    "\n",
    "# # # Open the file in write mode\n",
    "# # with open('TF_Model1_Weights_And_Biases.txt', 'w') as f:\n",
    "# #     # Write the weights and biases\n",
    "# #     f.write(\"Weights and Biases: \\n\")\n",
    "# #     for layer in best_model.layers:\n",
    "# #         weights, biases = layer.get_weights()\n",
    "# #         f.write(f\"Layer: {layer.name}\\n\")\n",
    "# #         f.write(f\"Weights: {weights}\\n\")\n",
    "# #         f.write(f\"Biases: {biases}\\n\\n\")\n",
    " \n",
    "\n",
    "\n",
    "# #6 see the best optimizer specifically \n",
    "# print(\"Optimizer: \")\n",
    "# # Get the optimizer's configuration\n",
    "# optimizer_config = best_model.optimizer.get_config()\n",
    "\n",
    "# # Print the optimizer configuration\n",
    "# print(optimizer_config)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Submitting Model 1. Score: 0.5670"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "# from tensorflow.keras.models import load_model\n",
    "\n",
    "# model1 = best_model\n",
    "\n",
    "# model1.save(\"model1_TF.h5\")\n",
    "\n",
    "# # Separate building_ids and features in the test data\n",
    "# competition_test_building_ids = test_data1['building_id']\n",
    "# competition_test_X = test_data1.drop('building_id', axis=1)\n",
    "\n",
    "# print(test_data1.shape)\n",
    "\n",
    "# # Generate predictions\n",
    "# predictions = model1.predict(competition_test_X)\n",
    "\n",
    "# # Get the class with the highest probability for each instance\n",
    "# predictions = predictions.argmax(axis=-1)\n",
    "\n",
    "# # Since the competition expects labels in the range 1-3, add 1 to the predictions\n",
    "# predictions = predictions + 1\n",
    "\n",
    "# # Create a DataFrame for submission\n",
    "# submission = pd.DataFrame({\n",
    "#     'building_id': competition_test_building_ids,\n",
    "#     'damage_grade': predictions\n",
    "# })\n",
    "    \n",
    " \n",
    "# # Save the DataFrame to a CSV file\n",
    "# submission.to_csv('submission_TF_1.csv', index=False)\n",
    "# print(\"Your submission was successfully saved!\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Set up GPU Acceleration for Tensorflow using CUDA and cuDNN"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For TensorFlow 2.15.0, you will need CUDA 12.2 and cuDNN 8.9\n",
    "\n",
    "Here are the steps to install CUDA and cuDNN:\n",
    "\n",
    "1. **Download CUDA 10.0**: Visit the [CUDA Toolkit Archive](https://developer.nvidia.com/cuda-toolkit-archive) and download CUDA Toolkit 10.0. Follow the on-screen instructions to install it.\n",
    "\n",
    "2. **Download cuDNN 7.4**: Visit the [NVIDIA cuDNN Archive](https://developer.nvidia.com/rdp/cudnn-archive). You'll need to create a free NVIDIA developer account if you don't have one. Download cuDNN v7.4.2 (Nov 8, 2018), for CUDA 10.0.\n",
    "\n",
    "3. **Install cuDNN**: Extract the cuDNN archive to a directory of your choice. Then, add the `bin` directory inside the extracted directory to your system's PATH environment variable. \n",
    "\n",
    "   On Windows, you can do this by going to System Properties -> Advanced -> Environment Variables, and then adding a new entry with the name `PATH` and the value being the path to the `bin` directory. \n",
    "\n",
    "   On Linux, you can add the line `export PATH=/path/to/cudnn/bin:$PATH` to your `~/.bashrc` or `~/.bash_profile` file.\n",
    "\n",
    "After you've installed CUDA and cuDNN, you should be able to use your GPU with TensorFlow. You can verify this by running your Python script again:\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Num GPUs Available:  1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024-01-13 00:20:29.641593: I external/local_xla/xla/stream_executor/cuda/cuda_executor.cc:887] could not open file to read NUMA node: /sys/bus/pci/devices/0000:01:00.0/numa_node\n",
      "Your kernel may have been built without NUMA support.\n",
      "2024-01-13 00:20:29.665024: I external/local_xla/xla/stream_executor/cuda/cuda_executor.cc:887] could not open file to read NUMA node: /sys/bus/pci/devices/0000:01:00.0/numa_node\n",
      "Your kernel may have been built without NUMA support.\n",
      "2024-01-13 00:20:29.665067: I external/local_xla/xla/stream_executor/cuda/cuda_executor.cc:887] could not open file to read NUMA node: /sys/bus/pci/devices/0000:01:00.0/numa_node\n",
      "Your kernel may have been built without NUMA support.\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "print(\"Num GPUs Available: \", len(tf.config.list_physical_devices('GPU')))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This will print the number of GPUs available. If no GPU is available, it will print `Num GPUs Available:  0`."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model 2, now I have GPU acceleration for my tensorflow"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import keras_tuner as kt\n",
    "# from tensorflow.keras.callbacks import EarlyStopping\n",
    "\n",
    "\n",
    "# %pip install tensorflow-addons\n",
    "\n",
    "# import tensorflow_addons as tfa\n",
    "\n",
    "\n",
    "# # TensorFlow expects the labels data to only include one column. Currently, they include two columns: building_id and damage_grade.\n",
    "# # So, I am making it so the labels data only includes one column (damage_grade) and changing the range from 1-3 to 0-2 since TensorFlow wants to start at 0\n",
    "# train_Y1_temp = train_Y1['damage_grade'] - 1  \n",
    "# valid_Y1_temp = valid_Y1['damage_grade'] - 1\n",
    "# test_Y1_temp = test_Y1['damage_grade'] - 1\n",
    "\n",
    "\n",
    "# def build_model(hp):\n",
    "#     model = tf.keras.Sequential()\n",
    "    \n",
    "#     # Input layer\n",
    "#     model.add(tf.keras.layers.Dense(\n",
    "#         units=hp.Int('input_units', min_value=80, max_value=256, step=20),\n",
    "#         #activation=hp.Choice('input_activation', ['relu', 'tanh', 'sigmoid']),\n",
    "#         activation = \"relu\",\n",
    "#         input_shape=(train_X1.shape[1],)\n",
    "#     ))\n",
    "    \n",
    "#     # Hidden layers\n",
    "#     for i in range(hp.Int('n_layers', 1, 5)):\n",
    "#         model.add(tf.keras.layers.Dense(\n",
    "#             units=hp.Int(f'hidden_units_{i}', min_value=80, max_value=256, step=20),\n",
    "#             #activation=hp.Choice(f'hidden_activation_{i}', ['relu', 'tanh', 'sigmoid']),\n",
    "#             activation = \"relu\",\n",
    "#             #kernel_regularizer=tf.keras.regularizers.l2(hp.Float(f'regularization_{i}', min_value=0.0, max_value=0.1, step=0.01))\n",
    "#         ))\n",
    "#         # Dropout layer\n",
    "#         #model.add(tf.keras.layers.Dropout(rate=hp.Float('dropout_rate', min_value=0.0, max_value=0.5, step=0.1)))\n",
    "    \n",
    "#     # Output layer\n",
    "#     model.add(tf.keras.layers.Dense(3, activation='softmax'))\n",
    "    \n",
    "#     # Compile the model\n",
    "#     model.compile(\n",
    "#         optimizer=hp.Choice('optimizer', ['adam', 'sgd', 'rmsprop']),\n",
    "#         loss='sparse_categorical_crossentropy', # sparse_categorical_crossentropy makes it so there's no need to do one-hot encoding on labels data, while categorical_crossentropy needs one-hot encodig of labels\n",
    "#         metrics=['accuracy'])\n",
    "#         #metrics=[tfa.metrics.F1Score(num_classes=3, average='micro', name='f1_score')]) # for f1 micro score\n",
    "    \n",
    "#     return model\n",
    "\n",
    "\n",
    "# tuner = kt.BayesianOptimization(\n",
    "#     build_model,\n",
    "#     max_trials=10,\n",
    "#     #objective=kt.Objective('f1_score', direction='max'), # for f1 micro score\n",
    "#     objective='val_accuracy',\n",
    "#     #max_epochs=50,\n",
    "#     directory='directory_for_model_2',  #Make sure to change ethe directory  and project name if you want to start a fresh new tuning session\n",
    "#     project_name='keras_tuner_model2', # make sure to change the project and directory name if you want to start a fresh new tuning session\n",
    "#     #hyperband_iterations=1# Increase if you have more time\n",
    "#     )  \n",
    "\n",
    "\n",
    "# #early_stopping = EarlyStopping(monitor='val_loss', patience=5) # If the validation loss does not decrease for 10 (`patience=10`) consecutive epochs, the training process will stop early. \n",
    "\n",
    "# # Search for the best hyperparameters\n",
    "# tuner.search(pd.concat([train_X1, valid_X1, test_X1], ignore_index=True), \n",
    "#              pd.concat([train_Y1_temp, valid_Y1_temp, test_Y1_temp], ignore_index=True),  # make sure to use the temp values, which is the label data without the building_id\n",
    "#              epochs=5, \n",
    "#              validation_split=0.2,  # 20% of the data will be used for validation\n",
    "#              batch_size=200, # make sure to change this and test out different values\n",
    "#              #callbacks=[early_stopping]  # Add early stopping callback)  # Fixed batch size\n",
    "#             )\n",
    "\n",
    "# # Get the best model\n",
    "# best_model = tuner.get_best_models()[0]\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Get information about the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "# #1 Summary of the model: This will print a summary of the model's layers, output shapes, and the number of parameters.\n",
    "# print(\"Summary of Model: \")\n",
    "# best_model = tuner.get_best_models(num_models=1)[0]\n",
    "# best_model.summary()\n",
    "\n",
    "# #Writing it to a file\n",
    "# with open('TF_Model2_Summary.txt', 'w') as f:\n",
    "#     # Write the model summary\n",
    "#     f.write(\"Summary of Model: \\n\")\n",
    "#     f.write(str(best_model.summary()))\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "# #2 Inspect Hyperparameters: This will display the values of the hyperparameters like the number of units in each layer, learning rate, etc.\n",
    "# print(\"Hyperparameters: \")\n",
    "# best_hp = tuner.get_best_hyperparameters()[0]\n",
    "# for param in best_hp.values:\n",
    "#     print(param, best_hp.get(param))\n",
    "\n",
    "\n",
    "# #Writing it to a file\n",
    "# import json\n",
    "\n",
    "# # Get the best hyperparameters\n",
    "# best_hp = tuner.get_best_hyperparameters()[0]\n",
    "\n",
    "# # Convert the hyperparameters to a dictionary\n",
    "# hp_dict = {param: best_hp.get(param) for param in best_hp.values}\n",
    "\n",
    "# # Write the dictionary to a file in JSON format\n",
    "# with open('hyperparameters_TF_model2.json', 'w') as f:\n",
    "#     json.dump(hp_dict, f)\n",
    "\n",
    "\n",
    "\n",
    "    \n",
    "# #3 Model Configuration: This returns a Python dictionary containing the model configuration. It can be quite detailed and technical, showing layer types, activation functions, and other layer-specific settings.\n",
    "# print(\"Model Configuration: \")\n",
    "# config = best_model.get_config()\n",
    "\n",
    "# #Writing it to a file\n",
    "# with open('TF_Model2_Configuration.txt', 'w') as f:\n",
    "#     # Write the configuration\n",
    "#     f.write(\"Configuration: \\n\")\n",
    "#     f.write(str(best_model.get_config()))\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "# #4 Visualizing the Model: This creates a plot of the model architecture, showing the layers, their shapes, and how they're connected.\n",
    "# print(\"Visualizing the model: \")  \n",
    "# tf.keras.utils.plot_model(best_model, to_file='model2_TF.png', show_shapes=True, show_layer_names=True) # MAKE SURE TO CHANGE THE FILE NAME \n",
    "\n",
    "\n",
    "\n",
    "\n",
    "# #5 Weights and Biases: This will print out the weights and biases for each layer in the model. Be cautious with large models, as this can be a lot of data.\n",
    "# # print(\"Weights and Biases: \")\n",
    "# # for layer in best_model.layers:\n",
    "# #     weights, biases = layer.get_weights()\n",
    "# #     print(layer.name, weights, biases)\n",
    "\n",
    "# # # Open the file in write mode\n",
    "# # with open('TF_Model1_Weights_And_Biases.txt', 'w') as f:\n",
    "# #     # Write the weights and biases\n",
    "# #     f.write(\"Weights and Biases: \\n\")\n",
    "# #     for layer in best_model.layers:\n",
    "# #         weights, biases = layer.get_weights()\n",
    "# #         f.write(f\"Layer: {layer.name}\\n\")\n",
    "# #         f.write(f\"Weights: {weights}\\n\")\n",
    "# #         f.write(f\"Biases: {biases}\\n\\n\")\n",
    "\n",
    "\n",
    "\n",
    "# #6 see the best optimizer specifically \n",
    "# print(\"Optimizer: \")\n",
    "# # Get the optimizer's configuration\n",
    "# optimizer_config = best_model.optimizer.get_config()\n",
    "\n",
    "# # Print the optimizer configuration\n",
    "# print(optimizer_config)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Submitting Model 2. Score: 0.5686"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "# from tensorflow.keras.models import load_model\n",
    "\n",
    "# model2 = best_model\n",
    "\n",
    "# model2.save(\"model2_TF.h5\")\n",
    "\n",
    "# # Separate building_ids and features in the test data\n",
    "# competition_test_building_ids = test_data1['building_id']\n",
    "# competition_test_X = test_data1.drop('building_id', axis=1)\n",
    "\n",
    "# print(test_data1.shape)\n",
    "\n",
    "# # Generate predictions\n",
    "# predictions = model2.predict(competition_test_X)\n",
    "\n",
    "# # Get the class with the highest probability for each instance\n",
    "# predictions = predictions.argmax(axis=-1)\n",
    "\n",
    "# # Since the competition expects labels in the range 1-3, add 1 to the predictions\n",
    "# predictions = predictions + 1\n",
    "\n",
    "# # Create a DataFrame for submission\n",
    "# submission = pd.DataFrame({\n",
    "#     'building_id': competition_test_building_ids,\n",
    "#     'damage_grade': predictions\n",
    "# })\n",
    "    \n",
    " \n",
    "# # Save the DataFrame to a CSV file\n",
    "# submission.to_csv('submission_TF_2.csv', index=False)\n",
    "# print(\"Your submission was successfully saved!\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model 3, now I have GPU acceleration for my tensorflow"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Trial 10 Complete [00h 00m 37s]\n",
      "val_accuracy: 0.5654342770576477\n",
      "\n",
      "Best val_accuracy So Far: 0.5667580962181091\n",
      "Total elapsed time: 00h 07m 08s\n",
      "INFO:tensorflow:Oracle triggered exit\n"
     ]
    }
   ],
   "source": [
    "import keras_tuner as kt\n",
    "from tensorflow.keras.callbacks import EarlyStopping\n",
    "\n",
    "\n",
    "%pip install tensorflow-addons\n",
    "\n",
    "import tensorflow_addons as tfa\n",
    "\n",
    "\n",
    "# TensorFlow expects the labels data to only include one column. Currently, they include two columns: building_id and damage_grade.\n",
    "# So, I am making it so the labels data only includes one column (damage_grade) and changing the range from 1-3 to 0-2 since TensorFlow wants to start at 0\n",
    "train_Y1_temp = train_Y1['damage_grade'] - 1  \n",
    "valid_Y1_temp = valid_Y1['damage_grade'] - 1\n",
    "test_Y1_temp = test_Y1['damage_grade'] - 1\n",
    "\n",
    "\n",
    "def build_model(hp):\n",
    "    model = tf.keras.Sequential()\n",
    "    \n",
    "    # Input layer\n",
    "    model.add(tf.keras.layers.Dense(\n",
    "        units=hp.Int('input_units', min_value=5, max_value=30, step=2),\n",
    "        #activation=hp.Choice('input_activation', ['relu', 'tanh', 'sigmoid']),\n",
    "        activation = \"relu\",\n",
    "        input_shape=(train_X1.shape[1],)\n",
    "    ))\n",
    "    \n",
    "    # Hidden layers\n",
    "    for i in range(hp.Int('n_layers', 1, 10)):\n",
    "        model.add(tf.keras.layers.Dense(\n",
    "            units=hp.Int(f'hidden_units_{i}', min_value=5, max_value=30, step=2),\n",
    "            #activation=hp.Choice(f'hidden_activation_{i}', ['relu', 'tanh', 'sigmoid']),\n",
    "            activation = \"relu\",\n",
    "            #kernel_regularizer=tf.keras.regularizers.l2(hp.Float(f'regularization_{i}', min_value=0.0, max_value=0.1, step=0.01))\n",
    "        ))\n",
    "        # Dropout layer\n",
    "        #model.add(tf.keras.layers.Dropout(rate=hp.Float('dropout_rate', min_value=0.0, max_value=0.5, step=0.1)))\n",
    "    \n",
    "    # Output layer\n",
    "    model.add(tf.keras.layers.Dense(3, activation='softmax'))\n",
    "    \n",
    "    # Compile the model\n",
    "    model.compile(\n",
    "        optimizer=hp.Choice('optimizer', ['adam', 'sgd', 'rmsprop']),\n",
    "        loss='sparse_categorical_crossentropy', # sparse_categorical_crossentropy makes it so there's no need to do one-hot encoding on labels data, while categorical_crossentropy needs one-hot encodig of labels\n",
    "        metrics=['accuracy'])\n",
    "        #metrics=[tfa.metrics.F1Score(num_classes=3, average='micro', name='f1_score')]) # for f1 micro score\n",
    "    \n",
    "    return model\n",
    "\n",
    "\n",
    "tuner = kt.BayesianOptimization(\n",
    "    build_model,\n",
    "    max_trials=10,\n",
    "    #objective=kt.Objective('f1_score', direction='max'), # for f1 micro score\n",
    "    objective='val_accuracy',\n",
    "    #max_epochs=50,\n",
    "    directory='directory_for_model_3',  #Make sure to change ethe directory and project name if you want to start a fresh new tuning session\n",
    "    project_name='keras_tuner_model3', # make sure to change the project and directory name if you want to start a fresh new tuning session\n",
    "    #hyperband_iterations=1# Increase if you have more time\n",
    "    )  \n",
    "\n",
    "\n",
    "#early_stopping = EarlyStopping(monitor='val_loss', patience=5) # If the validation loss does not decrease for 10 (`patience=10`) consecutive epochs, the training process will stop early. \n",
    "\n",
    "# Search for the best hyperparameters\n",
    "tuner.search(pd.concat([train_X1, valid_X1, test_X1], ignore_index=True), \n",
    "             pd.concat([train_Y1_temp, valid_Y1_temp, test_Y1_temp], ignore_index=True),  # make sure to use the temp values, which is the label data without the building_id\n",
    "             epochs=5, \n",
    "             validation_split=0.2,  # 20% of the data will be used for validation\n",
    "             batch_size=200, # make sure to change this and test out different values\n",
    "             #callbacks=[early_stopping]  # Add early stopping callback)  # Fixed batch size\n",
    "            )\n",
    "\n",
    "# Get the best model\n",
    "best_model = tuner.get_best_models()[0]\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Get information about the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Summary of Model: \n",
      "WARNING:tensorflow:Detecting that an object or model or tf.train.Checkpoint is being deleted with unrestored values. See the following logs for the specific values in question. To silence these warnings, use `status.expect_partial()`. See https://www.tensorflow.org/api_docs/python/tf/train/Checkpoint#restorefor details about the status object returned by the restore function.\n",
      "WARNING:tensorflow:Value in checkpoint could not be found in the restored object: (root).optimizer._variables.1\n",
      "WARNING:tensorflow:Value in checkpoint could not be found in the restored object: (root).optimizer._variables.2\n",
      "WARNING:tensorflow:Value in checkpoint could not be found in the restored object: (root).optimizer._variables.3\n",
      "WARNING:tensorflow:Value in checkpoint could not be found in the restored object: (root).optimizer._variables.4\n",
      "WARNING:tensorflow:Value in checkpoint could not be found in the restored object: (root).optimizer._variables.5\n",
      "WARNING:tensorflow:Value in checkpoint could not be found in the restored object: (root).optimizer._variables.6\n",
      "WARNING:tensorflow:Value in checkpoint could not be found in the restored object: (root).optimizer._variables.7\n",
      "WARNING:tensorflow:Value in checkpoint could not be found in the restored object: (root).optimizer._variables.8\n",
      "WARNING:tensorflow:Value in checkpoint could not be found in the restored object: (root).optimizer._variables.9\n",
      "WARNING:tensorflow:Value in checkpoint could not be found in the restored object: (root).optimizer._variables.10\n",
      "WARNING:tensorflow:Value in checkpoint could not be found in the restored object: (root).optimizer._variables.11\n",
      "WARNING:tensorflow:Value in checkpoint could not be found in the restored object: (root).optimizer._variables.12\n",
      "WARNING:tensorflow:Value in checkpoint could not be found in the restored object: (root).optimizer._variables.13\n",
      "WARNING:tensorflow:Value in checkpoint could not be found in the restored object: (root).optimizer._variables.14\n",
      "WARNING:tensorflow:Value in checkpoint could not be found in the restored object: (root).optimizer._variables.15\n",
      "WARNING:tensorflow:Value in checkpoint could not be found in the restored object: (root).optimizer._variables.16\n",
      "WARNING:tensorflow:Value in checkpoint could not be found in the restored object: (root).optimizer._variables.17\n",
      "WARNING:tensorflow:Value in checkpoint could not be found in the restored object: (root).optimizer._variables.18\n",
      "WARNING:tensorflow:Value in checkpoint could not be found in the restored object: (root).optimizer._variables.19\n",
      "WARNING:tensorflow:Value in checkpoint could not be found in the restored object: (root).optimizer._variables.20\n",
      "Model: \"sequential\"\n",
      "_________________________________________________________________\n",
      " Layer (type)                Output Shape              Param #   \n",
      "=================================================================\n",
      " dense (Dense)               (None, 9)                 315       \n",
      "                                                                 \n",
      " dense_1 (Dense)             (None, 9)                 90        \n",
      "                                                                 \n",
      " dense_2 (Dense)             (None, 5)                 50        \n",
      "                                                                 \n",
      " dense_3 (Dense)             (None, 13)                78        \n",
      "                                                                 \n",
      " dense_4 (Dense)             (None, 3)                 42        \n",
      "                                                                 \n",
      "=================================================================\n",
      "Total params: 575 (2.25 KB)\n",
      "Trainable params: 575 (2.25 KB)\n",
      "Non-trainable params: 0 (0.00 Byte)\n",
      "_________________________________________________________________\n",
      "Model: \"sequential\"\n",
      "_________________________________________________________________\n",
      " Layer (type)                Output Shape              Param #   \n",
      "=================================================================\n",
      " dense (Dense)               (None, 9)                 315       \n",
      "                                                                 \n",
      " dense_1 (Dense)             (None, 9)                 90        \n",
      "                                                                 \n",
      " dense_2 (Dense)             (None, 5)                 50        \n",
      "                                                                 \n",
      " dense_3 (Dense)             (None, 13)                78        \n",
      "                                                                 \n",
      " dense_4 (Dense)             (None, 3)                 42        \n",
      "                                                                 \n",
      "=================================================================\n",
      "Total params: 575 (2.25 KB)\n",
      "Trainable params: 575 (2.25 KB)\n",
      "Non-trainable params: 0 (0.00 Byte)\n",
      "_________________________________________________________________\n",
      "Hyperparameters: \n",
      "input_units 9\n",
      "n_layers 3\n",
      "hidden_units_0 9\n",
      "optimizer adam\n",
      "hidden_units_1 5\n",
      "hidden_units_2 13\n",
      "hidden_units_3 5\n",
      "hidden_units_4 17\n",
      "hidden_units_5 5\n",
      "hidden_units_6 17\n",
      "hidden_units_7 5\n",
      "Model Configuration: \n",
      "Visualizing the model: \n",
      "You must install pydot (`pip install pydot`) and install graphviz (see instructions at https://graphviz.gitlab.io/download/) for plot_model to work.\n",
      "Optimizer: \n",
      "{'name': 'Adam', 'weight_decay': None, 'clipnorm': None, 'global_clipnorm': None, 'clipvalue': None, 'use_ema': False, 'ema_momentum': 0.99, 'ema_overwrite_frequency': None, 'jit_compile': True, 'is_legacy_optimizer': False, 'learning_rate': 0.001, 'beta_1': 0.9, 'beta_2': 0.999, 'epsilon': 1e-07, 'amsgrad': False}\n"
     ]
    }
   ],
   "source": [
    "#1 Summary of the model: This will print a summary of the model's layers, output shapes, and the number of parameters.\n",
    "print(\"Summary of Model: \")\n",
    "best_model = tuner.get_best_models(num_models=1)[0]\n",
    "best_model.summary()\n",
    "\n",
    "#Writing it to a file\n",
    "with open('TF_Model3_Summary.txt', 'w') as f:\n",
    "    # Write the model summary\n",
    "    f.write(\"Summary of Model: \\n\")\n",
    "    f.write(str(best_model.summary()))\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "#2 Inspect Hyperparameters: This will display the values of the hyperparameters like the number of units in each layer, learning rate, etc.\n",
    "print(\"Hyperparameters: \")\n",
    "best_hp = tuner.get_best_hyperparameters()[0]\n",
    "for param in best_hp.values:\n",
    "    print(param, best_hp.get(param))\n",
    "\n",
    "\n",
    "#Writing it to a file\n",
    "import json\n",
    "\n",
    "# Get the best hyperparameters\n",
    "best_hp = tuner.get_best_hyperparameters()[0]\n",
    "\n",
    "# Convert the hyperparameters to a dictionary\n",
    "hp_dict = {param: best_hp.get(param) for param in best_hp.values}\n",
    "\n",
    "# Write the dictionary to a file in JSON format\n",
    "with open('hyperparameters_TF_model3.json', 'w') as f:\n",
    "    json.dump(hp_dict, f)\n",
    "\n",
    "\n",
    "\n",
    "    \n",
    "#3 Model Configuration: This returns a Python dictionary containing the model configuration. It can be quite detailed and technical, showing layer types, activation functions, and other layer-specific settings.\n",
    "print(\"Model Configuration: \")\n",
    "config = best_model.get_config()\n",
    "\n",
    "#Writing it to a file\n",
    "with open('TF_Model3_Configuration.txt', 'w') as f:\n",
    "    # Write the configuration\n",
    "    f.write(\"Configuration: \\n\")\n",
    "    f.write(str(best_model.get_config()))\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "#4 Visualizing the Model: This creates a plot of the model architecture, showing the layers, their shapes, and how they're connected.\n",
    "print(\"Visualizing the model: \")  \n",
    "tf.keras.utils.plot_model(best_model, to_file='model3_TF.png', show_shapes=True, show_layer_names=True) # MAKE SURE TO CHANGE THE FILE NAME \n",
    "\n",
    "\n",
    "\n",
    "\n",
    "#5 Weights and Biases: This will print out the weights and biases for each layer in the model. Be cautious with large models, as this can be a lot of data.\n",
    "# print(\"Weights and Biases: \")\n",
    "# for layer in best_model.layers:\n",
    "#     weights, biases = layer.get_weights()\n",
    "#     print(layer.name, weights, biases)\n",
    "\n",
    "# # Open the file in write mode\n",
    "# with open('TF_Model1_Weights_And_Biases.txt', 'w') as f:\n",
    "#     # Write the weights and biases\n",
    "#     f.write(\"Weights and Biases: \\n\")\n",
    "#     for layer in best_model.layers:\n",
    "#         weights, biases = layer.get_weights()\n",
    "#         f.write(f\"Layer: {layer.name}\\n\")\n",
    "#         f.write(f\"Weights: {weights}\\n\")\n",
    "#         f.write(f\"Biases: {biases}\\n\\n\")\n",
    "\n",
    "\n",
    "\n",
    "#6 see the best optimizer specifically \n",
    "print(\"Optimizer: \")\n",
    "# Get the optimizer's configuration\n",
    "optimizer_config = best_model.optimizer.get_config()\n",
    "\n",
    "# Print the optimizer configuration\n",
    "print(optimizer_config)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Submitting Model 3. Score: 0.5662"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(86868, 35)\n",
      "   1/2715 [..............................] - ETA: 3:47"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/michaelye22/.local/lib/python3.10/site-packages/keras/src/engine/training.py:3103: UserWarning: You are saving your model as an HDF5 file via `model.save()`. This file format is considered legacy. We recommend using instead the native Keras format, e.g. `model.save('my_model.keras')`.\n",
      "  saving_api.save_model(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2715/2715 [==============================] - 4s 1ms/step\n",
      "Your submission was successfully saved!\n"
     ]
    }
   ],
   "source": [
    "from tensorflow.keras.models import load_model\n",
    "\n",
    "model3 = best_model\n",
    "\n",
    "model3.save(\"model3_TF.h5\")\n",
    "\n",
    "# Separate building_ids and features in the test data\n",
    "competition_test_building_ids = test_data1['building_id']\n",
    "competition_test_X = test_data1.drop('building_id', axis=1)\n",
    "\n",
    "print(test_data1.shape)\n",
    "\n",
    "# Generate predictions\n",
    "predictions = model3.predict(competition_test_X)\n",
    "\n",
    "# Get the class with the highest probability for each instance\n",
    "predictions = predictions.argmax(axis=-1)\n",
    "\n",
    "# Since the competition expects labels in the range 1-3, add 1 to the predictions\n",
    "predictions = predictions + 1\n",
    "\n",
    "# Create a DataFrame for submission\n",
    "submission = pd.DataFrame({\n",
    "    'building_id': competition_test_building_ids,\n",
    "    'damage_grade': predictions\n",
    "})\n",
    "    \n",
    " \n",
    "# Save the DataFrame to a CSV file\n",
    "submission.to_csv('submission_TF_3.csv', index=False)\n",
    "print(\"Your submission was successfully saved!\")\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}