Feature_extraction_offine_evaluation.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import cufflinks as cf\n",
    "import numpy as np\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Object and Emotion detection - without bounding boxes"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This code snippet extract object and emotions from the image set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "import os\n",
    "from PIL import Image\n",
    "import glob\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from deepface import DeepFace\n",
    "import torch\n",
    "\n",
    "path_model = r\"path\"\n",
    "model = torch.hub.load('ultralytics/yolov5', 'custom', path=path_model)\n",
    "\n",
    "path_to_images = r\"Images\"\n",
    "\n",
    "# Get a list of image file paths\n",
    "image_paths = glob.glob(os.path.join(path_to_images, \"*.png\"))\n",
    "\n",
    "# Create an empty dictionary to store the results\n",
    "results = {}\n",
    "\n",
    "# Loop over each image in the folder\n",
    "for image_path in image_paths:\n",
    "    \n",
    "    # Load the image\n",
    "    try:\n",
    "        img = Image.open(image_path)\n",
    "    except:\n",
    "        results[os.path.basename(image_path)] = {'object': None, 'dominant_emotion': None}\n",
    "        continue\n",
    "    \n",
    "    # Convert image to numpy array\n",
    "    img = np.array(img)\n",
    "    \n",
    "    # Use YOLOv5 to detect objects in the image\n",
    "    results_yolo = model(img)\n",
    "    \n",
    "    # Get a list of detected objects\n",
    "    detections = results_yolo.pandas().xyxy[0]\n",
    "    \n",
    "    # Check if a person was detected\n",
    "    person_detected = False\n",
    "    for detection in detections.iterrows():\n",
    "        if detection[1]['name'] == 'person':\n",
    "            person_detected = True\n",
    "            break\n",
    "    \n",
    "    # If a person was detected, deepface library is used to extract dominant emotion from image\n",
    "    if person_detected:\n",
    "        emotions = DeepFace.analyze(img, actions = ['emotion'], enforce_detection=False)\n",
    "        dominant_emotion = emotions['dominant_emotion']\n",
    "        object_detected = 'person'\n",
    "    # If no person was detected, set object_detected to the name of the first object detected (if any)\n",
    "    else:\n",
    "        if len(detections) > 0:\n",
    "            object_detected = detections.iloc[0]['name']\n",
    "#         else:\n",
    "            object_detected = None\n",
    "        dominant_emotion = None\n",
    "    \n",
    "    # Store the results in the dictionary\n",
    "    results[os.path.basename(image_path)] = {'object': object_detected, 'dominant_emotion': dominant_emotion}\n",
    "\n",
    "# Convert the dictionary to a pandas DataFrame\n",
    "df_results = pd.DataFrame.from_dict(results, orient='index')\n",
    "\n",
    "#Removing unnecessary .png from the image folder\n",
    "df_results['CreativeId'] = df_results.index.str.replace('.png', '').astype(np.int64)\n",
    "\n",
    "#Loading the dataset created in the EDA-file\n",
    "emotion_results = pd.read_csv('emotion_results.csv')\n",
    "\n",
    "# Merge the two DataFrames based on the CreativeId column\n",
    "final_df = pd.merge(emotion_results, df_results[['CreativeId', 'object', 'dominant_emotion']], on='CreativeId', how='left')\n",
    "\n",
    "# Save the final DataFrame so it can be used for further analysis\n",
    "final_df.to_csv('final_results.csv', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Object detection with only objects and their bounding boxes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import torch\n",
    "from PIL import Image\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "model = torch.hub.load('ultralytics/yolov5', 'yolov5s')\n",
    "\n",
    "path_to_image_folder = r'Images'\n",
    "\n",
    "image_files = os.listdir(path_to_image_folder)\n",
    "\n",
    "obj_results = pd.DataFrame(columns=['id', 'label', 'confidence', 'x_min', 'y_min', 'x_max', 'y_max'])\n",
    "for image_file in image_files:\n",
    "    image_path = os.path.join(path_to_image_folder, image_file)\n",
    "    try:\n",
    "        im = Image.open(image_path)\n",
    "    except:\n",
    "        continue\n",
    "    results = model(np.array(im))\n",
    "    detections = results.xyxy[0]\n",
    "    for i in range(len(detections)):\n",
    "        label = results.names[int(detections[i][5])]\n",
    "        confidence = detections[i][4]\n",
    "        x_min, y_min, x_max, y_max = detections[i][:4]\n",
    "        obj_results = pd.concat([df_results, pd.DataFrame({'id': [image_file], 'label': [label], 'confidence': [confidence],\n",
    "                                        'x_min': [x_min], 'y_min': [y_min], 'x_max': [x_max], 'y_max': [y_max]})])\n",
    "        \n",
    "\n",
    "obj_results.to_csv('final_results_objects.csv', index=False)\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Preparing data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In this code snippet the results from the object detection and ground truth tables are prepared so they can be evaluated"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "\n",
    "# Load the ground truth table\n",
    "objects_detected = pd.read_csv('final_results_objects.csv')\n",
    "ground_truth= pd.read_csv('labels_gr-table.csv')\n",
    "\n",
    "import re\n",
    "\n",
    "#Helping method to only extract the numeric values(needed as there was string values in the columns)\n",
    "def extract_float_from_tensor(tensor_str):\n",
    "    match = re.search(r\"tensor\\((\\d+\\.\\d+)\\)\", tensor_str)\n",
    "    if match:\n",
    "        return float(match.group(1))\n",
    "    else:\n",
    "        return None\n",
    "\n",
    "objects_detected[['confidence', 'x_min', 'y_min', 'x_max', 'y_max']] = objects_detected[['confidence', 'x_min', 'y_min', 'x_max', 'y_max']].applymap(extract_float_from_tensor)\n",
    "\n",
    "# Load the final results of your model\n",
    "emotions = ['happy', 'surprise', 'fear', 'angry', 'sad']\n",
    "ground_truth = ground_truth[~ground_truth['label_name'].isin(emotions)]\n",
    "ground_truth = ground_truth.rename(columns={'image_name': 'id'})  # rename the column\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# IoU"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In this code snippet intersection over union is utilized to evaluate the ground truth table with the object detection model sing bounding boxes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Connecting the object detection results and ground truth results together\n",
    "evaluate = pd.merge(objects_detected, ground_truth, on='id')\n",
    "\n",
    "# Calculate the area of the intersection between the bounding boxes using the formula for intersection over union\n",
    "evaluate['intersection_area'] = (evaluate[['x_max', 'bbox_x']].min(axis=1) - evaluate[['x_min', 'bbox_x']].max(axis=1)) * \\\n",
    "                                  (evaluate[['y_max', 'bbox_y']].min(axis=1) - evaluate[['y_min', 'bbox_y']].max(axis=1))\n",
    "evaluate['intersection_area'] = evaluate['intersection_area'].clip(lower=0)\n",
    "\n",
    "# Calculate the area of the union between the bounding boxes\n",
    "evaluate['union_area'] = (evaluate['x_max'] - evaluate['x_min']) * (evaluate['y_max'] - evaluate['y_min']) + \\\n",
    "                          (evaluate['bbox_width'] * evaluate['bbox_height']) - evaluate['intersection_area']\n",
    "\n",
    "# Calculate the intersection over union (IoU) score\n",
    "evaluate['iou'] = evaluate['intersection_area'] / evaluate['union_area']\n",
    "\n",
    "# Group by label name and calculate the mean IoU score for each label\n",
    "mean_iou_scores = evaluate.groupby('label_name')['iou'].mean()\n",
    "mean_iou_scores\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Loading the dataset with object and emotion detection - withouth bounding boxes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "obj_emo_results = pd.read_csv('final_results.csv')\n",
    "\n",
    "obj_emo_results['object'].fillna('no object', inplace=True)\n",
    "obj_emo_results['dominant_emotion'].fillna('not_person', inplace=True)\n",
    "\n",
    "#CreativeId (advertisement's id) since it is no longer important, and will disturb further analysis with ML-models\n",
    "obj_emo_results.drop(\"CreativeId\", axis=1)\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Highest CTR-value for object and emotion"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sorted_obj_emo = obj_emo_results.sort_values(by='ctr', ascending=False)\n",
    "\n",
    "# Find the most popular objects based on CTR\n",
    "top_objects = sorted_obj_emo.groupby('object')['ctr'].mean().nlargest(5)\n",
    "\n",
    "\n",
    "# Find the most popular dominant emotions based on CTR\n",
    "top_emotions = sorted_obj_emo.groupby('dominant_emotion')['ctr'].mean().nlargest(5)\n",
    "merged_obj_emo = pd.concat([top_objects, top_emotions], axis=0)\n",
    "merged_obj_emo"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Distribution for emotions and objects"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "objects = obj_emo_results[\"object\"].value_counts()\n",
    "emotions = obj_emo_results[\"dominant_emotion\"].value_counts()\n",
    "# Bar chart of object counts\n",
    "plt.bar(objects.index, objects.values, color=['blue', 'red', 'green', 'orange'])\n",
    "plt.title(\"Object Counts\")\n",
    "plt.xlabel(\"Object\")\n",
    "plt.ylabel(\"Count\")\n",
    "plt.xticks(rotation=45)\n",
    "for i, v in enumerate(objects.values):\n",
    "    plt.text(i, v, str(v), color='black', fontweight='bold', ha='center')\n",
    "plt.savefig('objectsdistribution.png', dpi=300, bbox_inches='tight')\n",
    "\n",
    "plt.show()\n",
    "\n",
    "# Bar chart of emotion counts\n",
    "plt.bar(emotions.index, emotions.values, color=['red', 'blue', 'green', 'orange', 'purple', 'brown'])\n",
    "plt.title(\"Emotion Counts\")\n",
    "plt.xlabel(\"Emotion\")\n",
    "plt.ylabel(\"Count\")\n",
    "plt.xticks(rotation=45)\n",
    "for i, v in enumerate(emotions.values):\n",
    "    plt.text(i, v, str(v), color='black', fontweight='bold', ha='center')\n",
    "plt.savefig('emotionsdistribution.png', dpi=300, bbox_inches='tight')\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Making copy of data set to build further on"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "df_updated = obj_emo_results.copy()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Preprocessing of the data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In order to be able to perfom machine-learning models, some preprocessing of the data var neccesary to get appropriate data-values."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import LabelEncoder\n",
    "\n",
    "df_transform = df_updated.copy()\n",
    "\n",
    "# Encode the age_group variable\n",
    "age_encoder = LabelEncoder()\n",
    "df_transform['age_group'] = age_encoder.fit_transform(df_transform['age_group'])\n",
    "\n",
    "# Encode the gender variable\n",
    "gender_encoder = LabelEncoder()\n",
    "df_transform['gender'] = gender_encoder.fit_transform(df_transform['gender'])\n",
    "\n",
    "# Convert n_click and n_impressions_measurable to integers\n",
    "df_transform['n_click'] = df_transform['n_click'].astype(int)\n",
    "df_transform['n_impressions_measurable'] = df_transform['n_impressions_measurable'].astype(int)\n",
    "\n",
    "# Categorical variables into integers using LabelEncoder\n",
    "categorical = ['Adviser', 'industry', 'Category', 'page_type', 'format', 'object', 'dominant_emotion']\n",
    "for col in categorical:\n",
    "    label_encoder = LabelEncoder()\n",
    "    df_transform[col] = label_encoder.fit_transform(df_transform[col])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Predictive models"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Random forest model, using gridsearch to find best parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier, RandomForestRegressor\n",
    "from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score, roc_auc_score, mean_squared_error\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import mean_absolute_error, r2_score\n",
    "\n",
    "# Random Forest Classifier\n",
    "# Split the data into training and test sets, n_click as target variable\n",
    "X_train, X_test, y_train, y_test = train_test_split(df_transform.drop('n_click', axis=1), df_transform['n_click'], test_size=0.2, random_state=42)\n",
    "\n",
    "# The parameters under is obtained from the gridsearch\n",
    "rfc = RandomForestRegressor(n_estimators=50, max_features=0.8, max_samples=0.8, max_depth=9, min_samples_split=5, random_state=42)\n",
    "\n",
    "\n",
    "# Fit the model to the training set\n",
    "rfc.fit(X_train, y_train)\n",
    "\n",
    "# Predict the test set\n",
    "y_pred = rfc.predict(X_test)\n",
    "\n",
    "\n",
    "\n",
    "# # Remove comment to perform the grid search\n",
    "# parameters = {\n",
    "#     'n_estimators':np.arange(50,200,15),\n",
    "#               'max_features':np.arange(0.1, 1, 0.1),\n",
    "#               'max_depth': [3, 5, 7, 9],\n",
    "#               'max_samples': [0.3, 0.5, 0.8]\n",
    "# }\n",
    "\n",
    "# dtc = RandomForestClassifier(random_state=42)\n",
    "# clf = GridSearchCV(dtc, parameters, cv=3)\n",
    "# clf.fit(X_train, y_train)\n",
    "# paramstouse = clf.best_params_\n",
    "# print(f\"Most suitable parameters for the classifier: {paramstouse}\")\n",
    "\n",
    "\n",
    "# Compute evaluation metrics\n",
    "mse = mean_squared_error(y_test, y_pred)\n",
    "mae = mean_absolute_error(y_test, y_pred)\n",
    "rmse = np.sqrt(mse)\n",
    "r2 = r2_score(y_test, y_pred)\n",
    "\n",
    "print(\"Random Forest Regressor\")\n",
    "print(f\"Mean Squared Error (MSE): {mse:.2f}\")\n",
    "print(f\"Root Mean Squared Error (RMSE): {rmse:.2f}\")\n",
    "print(f\"Mean Absolute Error (MAE): {mae:.2f}\")\n",
    "print(f\"R2 Score: {r2:.2f}\")\n",
    "# Calculate feature importances\n",
    "importances = rfc.feature_importances_\n",
    "sorted_indices = np.argsort(importances)[::-1]\n",
    "features = X_train.columns\n",
    "\n",
    "# Print feature importances\n",
    "print('\\nFeature ranking:')\n",
    "for i, idx in enumerate(sorted_indices):\n",
    "    print(f\"{i+1}. {features[idx]} ({importances[idx]:.6f})\")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Baseline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Decision tree built as baseline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn.tree import DecisionTreeRegressor\n",
    "\n",
    "tree = DecisionTreeRegressor(random_state=42)\n",
    "\n",
    "tree.fit(X_train, y_train)\n",
    "\n",
    "y_pred = tree.predict(X_test)\n",
    "\n",
    "# Evaluation metrics\n",
    "mse = mean_squared_error(y_test, y_pred)\n",
    "mae = mean_absolute_error(y_test, y_pred)\n",
    "rmse = np.sqrt(mse)\n",
    "\n",
    "r2 = r2_score(y_test, y_pred)\n",
    "\n",
    "print(\"Performance of the Decision Tree Regressor\")\n",
    "\n",
    "print(f\"Mean Squared Error (MSE): {mse:.2f}\")\n",
    "print(f\"Mean Absolute Error (MAE): {mae:.2f}\")\n",
    "print(f\"Root Mean Squared Error (RMSE): {rmse:.2f}\")\n",
    "print(f\"R-squared (R^2): {r2:.2f}\")\n",
    "\n",
    "\n",
    "#feature importances\n",
    "importances = tree.feature_importances_\n",
    "sorted_indices = np.argsort(importances)[::-1]\n",
    "features = X_train.columns\n",
    "print('Feature ranking:')\n",
    "for i, idx in enumerate(sorted_indices):\n",
    "    print(f\"{i+1}. {features[idx]} ({importances[idx]:.6f})\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.8.10 64-bit (microsoft store)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  },
  "vscode": {
   "interpreter": {
    "hash": "f2004070539d7c66e4cf09a9c67e00e71495706ea1b7a871e73bb819f6cde484"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}