nb_arrow_custom_landmark.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import cv2\n",
    "import mediapipe as mp\n",
    "import numpy as np\n",
    "import mediapipe as mp\n",
    "\n",
    "from tensorflow.keras.models import Sequential\n",
    "from tensorflow.keras.layers import LSTM, Dense\n",
    "\n",
    "import feedforward"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Setup folder and register the training data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-12-09 13:38:58.311 Python[46867:5999257] WARNING: AVCaptureDeviceTypeExternal is deprecated for Continuity Cameras. Please use AVCaptureDeviceTypeContinuityCamera and add NSCameraUseContinuityCameraDeviceType to your Info.plist.\n",
      "OpenCV: FFMPEG: tag 0x58564944/'DIVX' is not supported with codec id 12 and format 'mp4 / MP4 (MPEG-4 Part 14)'\n",
      "OpenCV: FFMPEG: fallback to use tag 0x7634706d/'mp4v'\n",
      "OpenCV: FFMPEG: tag 0x58564944/'DIVX' is not supported with codec id 12 and format 'mp4 / MP4 (MPEG-4 Part 14)'\n",
      "OpenCV: FFMPEG: fallback to use tag 0x7634706d/'mp4v'\n",
      "OpenCV: FFMPEG: tag 0x58564944/'DIVX' is not supported with codec id 12 and format 'mp4 / MP4 (MPEG-4 Part 14)'\n",
      "OpenCV: FFMPEG: fallback to use tag 0x7634706d/'mp4v'\n",
      "OpenCV: FFMPEG: tag 0x58564944/'DIVX' is not supported with codec id 12 and format 'mp4 / MP4 (MPEG-4 Part 14)'\n",
      "OpenCV: FFMPEG: fallback to use tag 0x7634706d/'mp4v'\n",
      "OpenCV: FFMPEG: tag 0x58564944/'DIVX' is not supported with codec id 12 and format 'mp4 / MP4 (MPEG-4 Part 14)'\n",
      "OpenCV: FFMPEG: fallback to use tag 0x7634706d/'mp4v'\n",
      "OpenCV: FFMPEG: tag 0x58564944/'DIVX' is not supported with codec id 12 and format 'mp4 / MP4 (MPEG-4 Part 14)'\n",
      "OpenCV: FFMPEG: fallback to use tag 0x7634706d/'mp4v'\n",
      "OpenCV: FFMPEG: tag 0x58564944/'DIVX' is not supported with codec id 12 and format 'mp4 / MP4 (MPEG-4 Part 14)'\n",
      "OpenCV: FFMPEG: fallback to use tag 0x7634706d/'mp4v'\n",
      "OpenCV: FFMPEG: tag 0x58564944/'DIVX' is not supported with codec id 12 and format 'mp4 / MP4 (MPEG-4 Part 14)'\n",
      "OpenCV: FFMPEG: fallback to use tag 0x7634706d/'mp4v'\n",
      "OpenCV: FFMPEG: tag 0x58564944/'DIVX' is not supported with codec id 12 and format 'mp4 / MP4 (MPEG-4 Part 14)'\n",
      "OpenCV: FFMPEG: fallback to use tag 0x7634706d/'mp4v'\n",
      "OpenCV: FFMPEG: tag 0x58564944/'DIVX' is not supported with codec id 12 and format 'mp4 / MP4 (MPEG-4 Part 14)'\n",
      "OpenCV: FFMPEG: fallback to use tag 0x7634706d/'mp4v'\n",
      "OpenCV: FFMPEG: tag 0x58564944/'DIVX' is not supported with codec id 12 and format 'mp4 / MP4 (MPEG-4 Part 14)'\n",
      "OpenCV: FFMPEG: fallback to use tag 0x7634706d/'mp4v'\n",
      "OpenCV: FFMPEG: tag 0x58564944/'DIVX' is not supported with codec id 12 and format 'mp4 / MP4 (MPEG-4 Part 14)'\n",
      "OpenCV: FFMPEG: fallback to use tag 0x7634706d/'mp4v'\n",
      "OpenCV: FFMPEG: tag 0x58564944/'DIVX' is not supported with codec id 12 and format 'mp4 / MP4 (MPEG-4 Part 14)'\n",
      "OpenCV: FFMPEG: fallback to use tag 0x7634706d/'mp4v'\n",
      "OpenCV: FFMPEG: tag 0x58564944/'DIVX' is not supported with codec id 12 and format 'mp4 / MP4 (MPEG-4 Part 14)'\n",
      "OpenCV: FFMPEG: fallback to use tag 0x7634706d/'mp4v'\n",
      "OpenCV: FFMPEG: tag 0x58564944/'DIVX' is not supported with codec id 12 and format 'mp4 / MP4 (MPEG-4 Part 14)'\n",
      "OpenCV: FFMPEG: fallback to use tag 0x7634706d/'mp4v'\n",
      "OpenCV: FFMPEG: tag 0x58564944/'DIVX' is not supported with codec id 12 and format 'mp4 / MP4 (MPEG-4 Part 14)'\n",
      "OpenCV: FFMPEG: fallback to use tag 0x7634706d/'mp4v'\n",
      "OpenCV: FFMPEG: tag 0x58564944/'DIVX' is not supported with codec id 12 and format 'mp4 / MP4 (MPEG-4 Part 14)'\n",
      "OpenCV: FFMPEG: fallback to use tag 0x7634706d/'mp4v'\n",
      "OpenCV: FFMPEG: tag 0x58564944/'DIVX' is not supported with codec id 12 and format 'mp4 / MP4 (MPEG-4 Part 14)'\n",
      "OpenCV: FFMPEG: fallback to use tag 0x7634706d/'mp4v'\n",
      "OpenCV: FFMPEG: tag 0x58564944/'DIVX' is not supported with codec id 12 and format 'mp4 / MP4 (MPEG-4 Part 14)'\n",
      "OpenCV: FFMPEG: fallback to use tag 0x7634706d/'mp4v'\n",
      "OpenCV: FFMPEG: tag 0x58564944/'DIVX' is not supported with codec id 12 and format 'mp4 / MP4 (MPEG-4 Part 14)'\n",
      "OpenCV: FFMPEG: fallback to use tag 0x7634706d/'mp4v'\n",
      "OpenCV: FFMPEG: tag 0x58564944/'DIVX' is not supported with codec id 12 and format 'mp4 / MP4 (MPEG-4 Part 14)'\n",
      "OpenCV: FFMPEG: fallback to use tag 0x7634706d/'mp4v'\n",
      "OpenCV: FFMPEG: tag 0x58564944/'DIVX' is not supported with codec id 12 and format 'mp4 / MP4 (MPEG-4 Part 14)'\n",
      "OpenCV: FFMPEG: fallback to use tag 0x7634706d/'mp4v'\n",
      "OpenCV: FFMPEG: tag 0x58564944/'DIVX' is not supported with codec id 12 and format 'mp4 / MP4 (MPEG-4 Part 14)'\n",
      "OpenCV: FFMPEG: fallback to use tag 0x7634706d/'mp4v'\n",
      "OpenCV: FFMPEG: tag 0x58564944/'DIVX' is not supported with codec id 12 and format 'mp4 / MP4 (MPEG-4 Part 14)'\n",
      "OpenCV: FFMPEG: fallback to use tag 0x7634706d/'mp4v'\n",
      "OpenCV: FFMPEG: tag 0x58564944/'DIVX' is not supported with codec id 12 and format 'mp4 / MP4 (MPEG-4 Part 14)'\n",
      "OpenCV: FFMPEG: fallback to use tag 0x7634706d/'mp4v'\n",
      "OpenCV: FFMPEG: tag 0x58564944/'DIVX' is not supported with codec id 12 and format 'mp4 / MP4 (MPEG-4 Part 14)'\n",
      "OpenCV: FFMPEG: fallback to use tag 0x7634706d/'mp4v'\n",
      "OpenCV: FFMPEG: tag 0x58564944/'DIVX' is not supported with codec id 12 and format 'mp4 / MP4 (MPEG-4 Part 14)'\n",
      "OpenCV: FFMPEG: fallback to use tag 0x7634706d/'mp4v'\n",
      "OpenCV: FFMPEG: tag 0x58564944/'DIVX' is not supported with codec id 12 and format 'mp4 / MP4 (MPEG-4 Part 14)'\n",
      "OpenCV: FFMPEG: fallback to use tag 0x7634706d/'mp4v'\n",
      "OpenCV: FFMPEG: tag 0x58564944/'DIVX' is not supported with codec id 12 and format 'mp4 / MP4 (MPEG-4 Part 14)'\n",
      "OpenCV: FFMPEG: fallback to use tag 0x7634706d/'mp4v'\n",
      "OpenCV: FFMPEG: tag 0x58564944/'DIVX' is not supported with codec id 12 and format 'mp4 / MP4 (MPEG-4 Part 14)'\n",
      "OpenCV: FFMPEG: fallback to use tag 0x7634706d/'mp4v'\n"
     ]
    }
   ],
   "source": [
    "VIDEOS_PATH = os.path.join('dist/videos') \n",
    "NUM_VIDEOS = 30\n",
    "NUM_FRAMES_PER_VIDEO = 100\n",
    "feedforward.create_folder_if_not_exists(VIDEOS_PATH)\n",
    "feedforward.clear_folder(VIDEOS_PATH)\n",
    "video_prefix = 'arrow'\n",
    "feedforward.register_videos(VIDEOS_PATH, video_prefix, NUM_VIDEOS, NUM_FRAMES_PER_VIDEO, 'beautiful_gesture')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Classification"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Data Augmentation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "augmented_prefix = \"augmented_rotation_\"\n",
    "feedforward.augment_dataset_rotation(VIDEOS_PATH, augmented_prefix)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#feedforward.delete_files_with_prefix(VIDEOS_PATH, augmented_prefix)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Building the training set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "WINDOW_LEN = 20\n",
    "TARGET_LANDMARKS = [\n",
    "    mp.solutions.holistic.PoseLandmark.LEFT_SHOULDER,\n",
    "    mp.solutions.holistic.PoseLandmark.LEFT_HIP,\n",
    "]\n",
    "X_training, y_training = feedforward.build_training_arrow_data(VIDEOS_PATH, WINDOW_LEN, TARGET_LANDMARKS)\n",
    "X_training.shape, y_training.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Build and Train the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_training.shape[1], X_training.shape[2]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = Sequential()\n",
    "\n",
    "NUM_OUTPUT_UNITS = len(TARGET_LANDMARKS) * 2\n",
    "\n",
    "model.add(LSTM(64, input_shape=(X_training.shape[1], X_training.shape[2]), activation='relu', return_sequences=True)) # LSTM layer with input shape (X_training.shape[1], X_training.shape[2]))\n",
    "model.add(LSTM(64, activation='relu', return_sequences=False)) # LSTM layer with return_sequences=False for the final prediction# LSTM layer with return_sequences=False for the final prediction\n",
    "model.add(Dense(NUM_OUTPUT_UNITS))# Dense output layer with NUM_OUTPUT_UNITS units\n",
    "\n",
    "model.compile(optimizer='adam', loss='mean_squared_error', metrics=['accuracy'])\n",
    "\n",
    "history = model.fit(X_training, y_training, batch_size=32, epochs=100, validation_split=0.2)\n",
    "\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cap = cv2.VideoCapture(feedforward.CAMERA_INDEX)\n",
    "holistic = mp.solutions.holistic.Holistic()\n",
    "frame_buffer = []\n",
    "\n",
    "while cap.isOpened():\n",
    "    ret, frame = cap.read()\n",
    "    if not ret:\n",
    "        break\n",
    "\n",
    "    frame_rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)\n",
    "\n",
    "    results = holistic.process(frame_rgb)\n",
    "\n",
    "    if results.pose_landmarks:\n",
    "\n",
    "        pixel_coords = []\n",
    "        height, width, _ = frame.shape\n",
    "        data = []\n",
    "\n",
    "        for target_landmark in TARGET_LANDMARKS:\n",
    "            target_coords = results.pose_landmarks.landmark[target_landmark]\n",
    "            target_input = [target_coords.x,  target_coords.y]\n",
    "            data += target_input\n",
    "\n",
    "            pixel_coords_x, pixel_coords_y = int(target_coords.x * width), int(target_coords.y * height)\n",
    "            pixel_coords.append([pixel_coords_x, pixel_coords_y])\n",
    "            cv2.circle(frame, (pixel_coords_x, pixel_coords_y), 5, (255, 0, 0), -1)\n",
    "\n",
    "        nose_coords = results.pose_landmarks.landmark[mp.solutions.holistic.PoseLandmark.NOSE]\n",
    "        nose_input = [nose_coords.x, nose_coords.y]\n",
    "        data += nose_input\n",
    "        frame_buffer.append(data)\n",
    "\n",
    "        if len(frame_buffer) >= WINDOW_LEN/2:\n",
    "            prediction_input = np.array(frame_buffer)[np.newaxis, :, :]\n",
    "            prediction = model.predict(prediction_input)\n",
    "    \n",
    "            prediction = np.array(np.split(prediction[0], len(TARGET_LANDMARKS)))\n",
    "            print(prediction)\n",
    "\n",
    "            for i, target_landmark in enumerate(TARGET_LANDMARKS):\n",
    "                print(i)\n",
    "                prediction_x = int(prediction[i][0] * width)\n",
    "                prediction_y = int(prediction[i][1] * height)\n",
    "            \n",
    "                cv2.arrowedLine(frame, (pixel_coords[i][0], pixel_coords[i][1]), (prediction_x, prediction_y), (255, 0, 255), 2)\n",
    "\n",
    "            frame_buffer = frame_buffer[1:]\n",
    "\n",
    "    cv2.imshow('Video', frame)\n",
    "\n",
    "    if cv2.waitKey(1) & 0xFF == ord('q'):\n",
    "        break\n",
    "\n",
    "cap.release()\n",
    "cv2.destroyAllWindows()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "uit-proj",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.18"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}