THESIS-Indonesian-Sign-Language-Detection-w-MediaPipe-BiGRU_v1.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 1. Import and Install Dependencies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!pip install tensorflow==2.4.1 tensorflow-gpu==2.4.1 opencv-python mediapipe sklearn matplotlib"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Math and Data Processing\n",
    "import os\n",
    "import numpy as np\n",
    "from matplotlib import pyplot as plt\n",
    "import time\n",
    "\n",
    "# Sklearn\n",
    "from sklearn.model_selection import KFold\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import multilabel_confusion_matrix, accuracy_score, classification_report\n",
    "\n",
    "# Tensor & Keras\n",
    "from tensorflow.keras.layers import Dense, GRU, Bidirectional, Dropout\n",
    "from tensorflow.keras.utils import to_categorical\n",
    "from tensorflow.keras.callbacks import TensorBoard\n",
    "from tensorflow.keras.models import Sequential, load_model\n",
    "from keras.callbacks import ModelCheckpoint, EarlyStopping\n",
    "\n",
    "# Evaluation\n",
    "import seaborn as sns\n",
    "from sklearn.metrics import confusion_matrix, classification_report, accuracy_score, precision_score, recall_score, f1_score\n",
    "\n",
    "# Record and Testing Realtime\n",
    "from scipy import stats\n",
    "import mediapipe as mp\n",
    "import cv2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 2. Keypoints using MP Holistic"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mp_holistic = mp.solutions.holistic # Holistic model\n",
    "mp_drawing = mp.solutions.drawing_utils # Drawing utilities"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def mediapipe_detection(image, model):\n",
    "    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB) # COLOR CONVERSION BGR 2 RGB\n",
    "    image.flags.writeable = False                  # Image is no longer writeable\n",
    "    results = model.process(image)                 # Make prediction\n",
    "    image.flags.writeable = True                   # Image is now writeable \n",
    "    image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR) # COLOR COVERSION RGB 2 BGR\n",
    "    return image, results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def draw_styled_landmarks(image, results):\n",
    "    # Draw left hand connections\n",
    "    mp_drawing.draw_landmarks(image, results.left_hand_landmarks, mp_holistic.HAND_CONNECTIONS, \n",
    "                                mp_drawing.DrawingSpec(color=(121,22,76), thickness=2, circle_radius=4), \n",
    "                                mp_drawing.DrawingSpec(color=(121,44,250), thickness=2, circle_radius=2)\n",
    "                                ) \n",
    "    # Draw right hand connections  \n",
    "    mp_drawing.draw_landmarks(image, results.right_hand_landmarks, mp_holistic.HAND_CONNECTIONS, \n",
    "                                mp_drawing.DrawingSpec(color=(245,117,66), thickness=2, circle_radius=4), \n",
    "                                mp_drawing.DrawingSpec(color=(245,66,230), thickness=2, circle_radius=2)\n",
    "                                )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 3. Extract Keypoint Values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def extract_keypoints(results):\n",
    "    lh = np.array([[res.x, res.y, res.z] for res in results.left_hand_landmarks.landmark]).flatten() if results.left_hand_landmarks else np.zeros(21*3)\n",
    "    rh = np.array([[res.x, res.y, res.z] for res in results.right_hand_landmarks.landmark]).flatten() if results.right_hand_landmarks else np.zeros(21*3)\n",
    "    return np.concatenate([lh, rh])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 4. Setup Folders for Collection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Path for exported data, numpy arrays\n",
    "# Utility untuk Pembuatan Folder automatis untuk penyimpaman data koordinat lokasi pose, wajah, dan gerakan tangan -> .npy format\n",
    "# SIBI\n",
    "DATA_PATH = os.path.join('BIGRU_SIBI_DATASET_9Aug2024') \n",
    "\n",
    "# Actions that we try to detect\n",
    "actions = np.array(['Tolong', 'Terima Kasih', 'Maaf'])\n",
    "\n",
    "# Thirty videos worth of data\n",
    "no_sequences = 30\n",
    "\n",
    "# Videos are going to be 30 frames in length (30x30 = 900 frames)\n",
    "sequence_length = 30\n",
    "\n",
    "# augmented data total (30 frames x 12 augmentation + 30 frames original data)\n",
    "total_augmented_sequence = 390\n",
    "\n",
    "# Folder start\n",
    "start_folder = 30"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# RUN 1x UNTUK BUAT FOLDER SAJA (JANGAN DI RUN LAGI) UNTUK RECORDING DATA \n",
    "for action in actions: \n",
    "    for sequence in range(total_augmented_sequence):\n",
    "        try: \n",
    "            os.makedirs(os.path.join(DATA_PATH, action, str(sequence)))\n",
    "        except:\n",
    "            pass"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 5. Collect Keypoint Values for Training and Testing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# RUN 1x UNTUK BUAT FOLDER SAJA (JANGAN DI RUN LAGI) UNTUK RECORDING DATA \n",
    "cap = cv2.VideoCapture(0)\n",
    "# Set mediapipe model \n",
    "with mp_holistic.Holistic(min_detection_confidence=0.5, min_tracking_confidence=0.5) as holistic:\n",
    "    \n",
    "    # NEW LOOP\n",
    "    # Loop through actions\n",
    "    for action in actions:\n",
    "        # Loop through sequences aka videos\n",
    "        for sequence in range(0, no_sequences):\n",
    "            # Loop through video length aka sequence length\n",
    "            for frame_num in range(sequence_length):\n",
    "                \n",
    "                # Read feed\n",
    "                ret, frame = cap.read()\n",
    "\n",
    "                # Save Extracted Landmarks to .npy file\n",
    "                npy_path = os.path.join(DATA_PATH, action, str(sequence), str(frame_num))\n",
    "\n",
    "                # Save image\n",
    "                cv2.imwrite(npy_path + '.jpg', frame)\n",
    "\n",
    "                # Make detections\n",
    "                image, results = mediapipe_detection(frame, holistic)\n",
    "\n",
    "                # Draw landmarks\n",
    "                draw_styled_landmarks(image, results)\n",
    "                \n",
    "                # NEW Apply wait logic\n",
    "                if frame_num == 0: \n",
    "                    cv2.putText(image, 'STARTING COLLECTION', (120,200), cv2.FONT_HERSHEY_SIMPLEX, 2, (0,255, 0), 2, cv2.LINE_AA)\n",
    "                    cv2.putText(image, 'Collecting frames for {} Video Number {}'.format(action, sequence), (15,75), cv2.FONT_HERSHEY_SIMPLEX, 1.7, (0, 0, 255), 2, cv2.LINE_AA)\n",
    "                    # Show to screen\n",
    "                    cv2.imshow('Hazlan SIBI Record Webcam', image)\n",
    "                    cv2.waitKey(2000)\n",
    "                else: \n",
    "                    cv2.putText(image, 'Collecting frames for {} Video Number {}'.format(action, sequence), (15,75), cv2.FONT_HERSHEY_SIMPLEX, 1.7, (0, 0, 255), 2, cv2.LINE_AA)\n",
    "                    # Show to screen\n",
    "                    cv2.imshow('Hazlan SIBI Record Webcam', image)\n",
    "                \n",
    "                # NEW Export keypoints\n",
    "                keypoints = extract_keypoints(results)\n",
    "                # npy_path = os.path.join(DATA_PATH, action, str(sequence), str(frame_num))\n",
    "                np.save(npy_path, keypoints)\n",
    "\n",
    "                # Break gracefully\n",
    "                if cv2.waitKey(10) & 0xFF == ord('q'):\n",
    "                    break\n",
    "                    \n",
    "    cap.release()\n",
    "    cv2.destroyAllWindows()\n",
    "    cv2.waitKey(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cap.release()\n",
    "cv2.destroyAllWindows()\n",
    "cv2.waitKey(1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 6. Data Augmentation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Augmented Function Paramater = 12\n",
    "#### Deleted (rotate: 45, 315 | zoomin: 0.9 | zoomout: 0.9)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def augmentation_rotation(image, angle):\n",
    "    # Rotate the image\n",
    "    array = [15,30,345,330]\n",
    "    image_list = []\n",
    "    for element in array:\n",
    "        rows, cols, _ = image.shape\n",
    "        M = cv2.getRotationMatrix2D((cols / 2, rows / 2), element, 1)\n",
    "        augmented_image = cv2.warpAffine(image, M, (cols, rows))\n",
    "        image_list.append(augmented_image)\n",
    "    # Show to screen\n",
    "    return image_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def zoomin(image, zoom_factor):\n",
    "    height, width = image.shape[:2]\n",
    "    new_width = int(width * zoom_factor)\n",
    "    new_height = int(height * zoom_factor)\n",
    "    left = int((width - new_width))\n",
    "    top = int((height - new_height))\n",
    "    right = int((width + new_width))\n",
    "    bottom = int((height + new_height))\n",
    "    cropped_image = image[top:bottom, left:right]\n",
    "    zoom_in_width = int(image.shape[1])\n",
    "    zoom_in_height = int(image.shape[0])\n",
    "    zoom_in_image = cv2.resize(cropped_image, (zoom_in_width, zoom_in_height))\n",
    "    return zoom_in_image\n",
    "\n",
    "def augmentation_zoomin(image):\n",
    "    array = [0.8,0.7]\n",
    "    image_list = []\n",
    "    for element in array:\n",
    "        augmented_image = zoomin(image, element)\n",
    "        image_list.append(augmented_image)\n",
    "    return image_list\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def zoomout(image, zoom_factor):\n",
    "    height, width = image.shape[:2]\n",
    "    new_width = int(width * zoom_factor)\n",
    "    new_height = int(height * zoom_factor)\n",
    "\n",
    "    # Compute the aspect ratio difference\n",
    "    width_ratio = new_width / width\n",
    "    height_ratio = new_height / height\n",
    "    aspect_ratio = min(width_ratio, height_ratio)\n",
    "\n",
    "    # Compute the new dimensions\n",
    "    new_width = int(width * aspect_ratio)\n",
    "    new_height = int(height * aspect_ratio)\n",
    "\n",
    "    # Ensure that the new dimensions are not larger than the original image dimensions\n",
    "    new_width = min(new_width, width)\n",
    "    new_height = min(new_height, height)\n",
    "\n",
    "    # Compute the black border dimensions\n",
    "    border_width = width - new_width\n",
    "    border_height = height - new_height\n",
    "\n",
    "    # Create a black border around the zoomed-out image\n",
    "    border_color = (0, 0, 0)  # Black color\n",
    "    bordered_image = cv2.copyMakeBorder(image, border_height, border_height, border_width, border_width,\n",
    "                                        cv2.BORDER_CONSTANT, value=border_color)\n",
    "    zoomed_out_image = cv2.resize(bordered_image, (width, height))\n",
    "    return zoomed_out_image\n",
    "\n",
    "def augmentation_zoomout(image):\n",
    "    array = [0.8,0.7]\n",
    "    image_list = []\n",
    "    for element in array:\n",
    "        augmented_image = zoomout(image, element)\n",
    "        image_list.append(augmented_image)\n",
    "    return image_list\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def augmentation_meta(image):\n",
    "    # Adjust brightness and contrast of the image\n",
    "    array = np.arange(0.5, 2.5, 0.5)\n",
    "    image_list = []\n",
    "    for element in array:\n",
    "        brightness = element  # Brightness factor (0.0 to 1.0)\n",
    "        contrast = element + 1  # Contrast factor (>1.0 for higher contrast)\n",
    "        augmented_image = cv2.convertScaleAbs(image, alpha=contrast, beta=brightness)\n",
    "        image_list.append(augmented_image)\n",
    "    return image_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def augmented_background(frame_aug, file_num, frame_number):\n",
    "    frameaug = frame_aug\n",
    "    seq_num = file_num\n",
    "    augmented_rotation = augmentation_rotation(frameaug, 45)\n",
    "    augmented_zoomin = augmentation_zoomin(frameaug)\n",
    "    augmented_zoomout = augmentation_zoomout(frameaug)\n",
    "    augmented_meta = augmentation_meta(frameaug)\n",
    "    concated_image = augmented_rotation + augmented_zoomin + augmented_zoomout  + augmented_meta\n",
    "    for  index, aug_image in enumerate(concated_image):\n",
    "        image, results = mediapipe_detection(aug_image, holistic)\n",
    "        keypoints = extract_keypoints(results)\n",
    "        npy_path = os.path.join(DATA_PATH, action, str(seq_num), str(frame_number))\n",
    "        print(npy_path)\n",
    "        cv2.imwrite(f'{npy_path}.jpg', aug_image)\n",
    "        np.save(npy_path, keypoints)\n",
    "        seq_num+= 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Set mediapipe model\n",
    "with mp_holistic.Holistic(min_detection_confidence=0.5, min_tracking_confidence = 0.5) as holistic:\n",
    "    \n",
    "    # NEW LOOP\n",
    "    # Loop through actions\n",
    "    for action in actions:\n",
    "        seq_num = 30\n",
    "        # Loop through sequences aka videos\n",
    "        for sequence in range(no_sequences):\n",
    "            # Loop through sequence length aka video length\n",
    "            for frame_num in range(sequence_length):\n",
    "                # Read feed\n",
    "                npy_path = os.path.join(DATA_PATH, action, str(sequence), str(frame_num))\n",
    "                \n",
    "                frame = cv2.imread(f'{npy_path}.jpg')\n",
    "                augmented_background(frame,seq_num, frame_num)\n",
    "            # Break gracefully\n",
    "            seq_num += 12"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 7. Preprocess Data and Create Labels and Features"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Dataset untuk testing digunakan sejumlah 5% dari dataset total"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "from tensorflow.keras.utils import to_categorical"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "label_map = {label:num for num, label in enumerate(actions)}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "label_map"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sequences, labels = [], []\n",
    "for action in actions:\n",
    "    for sequence in np.array(os.listdir(os.path.join(DATA_PATH, action))).astype(int):\n",
    "        window = []\n",
    "        for frame_num in range(sequence_length):\n",
    "            res = np.load(os.path.join(DATA_PATH, action, str(sequence), \"{}.npy\".format(frame_num)))\n",
    "            window.append(res)\n",
    "        sequences.append(window)\n",
    "        labels.append(label_map[action])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sequences, labels = [], []\n",
    "for action in actions:\n",
    "    for sequence in np.array(os.listdir(os.path.join(DATA_PATH, action))).astype(int):\n",
    "        window = []\n",
    "        for frame_num in range(sequence_length):\n",
    "            res = np.load(os.path.join(DATA_PATH, action, str(sequence), \"{}.npy\".format(frame_num)))\n",
    "            window.append(res)\n",
    "        sequences.append(window)\n",
    "        labels.append(label_map[action])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = np.array(sequences)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "y = to_categorical(labels).astype(int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.05)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 8. Build and Train BiGRU Neural Network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "log_dir = os.path.join('Logs')\n",
    "tb_callback = TensorBoard(log_dir=log_dir)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = Sequential()\n",
    "model.add(Bidirectional(GRU(64, return_sequences=True, activation='relu', input_shape=(30,126))))\n",
    "model.add(Bidirectional(GRU(128, return_sequences=False, activation='relu')))\n",
    "model.add(Dense(64, activation='relu'))\n",
    "model.add(Dense(actions.shape[0], activation='softmax'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.compile(optimizer='Adam', loss='categorical_crossentropy', metrics=['categorical_accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "history = model.fit(X_train, y_train, epochs=120, callbacks=[tb_callback], validation_split = 0.1, batch_size=16)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Plot training & validation loss values\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "plt.plot(history.history['loss'])\n",
    "plt.plot(history.history['val_loss'])\n",
    "plt.title('Model loss')\n",
    "plt.ylabel('Loss')\n",
    "plt.xlabel('Epoch')\n",
    "plt.legend(['Train', 'Validation'], loc='upper left')\n",
    "plt.show()\n",
    "\n",
    "# Plot training & validation accuracy values\n",
    "plt.plot(history.history['categorical_accuracy'])\n",
    "plt.plot(history.history['val_categorical_accuracy'])\n",
    "plt.title('Model accuracy')\n",
    "plt.ylabel('Accuracy')\n",
    "plt.xlabel('Epoch')\n",
    "plt.legend(['Train', 'Validation'], loc='upper left')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "res = model.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "actions[np.argmax(y_test[0])]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 9. Save Weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.save('SIBI_24Agustus2024_V2.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.load_weights('SIBI_24Agustus2024_V2.h5')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 10. Evaluation using Confusion Matrix and Accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "yhat = model.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ytrue = np.argmax(y_test, axis=1).tolist()\n",
    "yhat = np.argmax(yhat, axis=1).tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "multilabel_confusion_matrix(ytrue, yhat)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "accuracy_score(ytrue, yhat)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# define class labels\n",
    "class_labels = ['Tolong', 'Terima Kasih', 'Maaf']\n",
    "\n",
    "# ytrue and yhat are the predicted and the actual labels\n",
    "conf_matrix = confusion_matrix(ytrue, yhat)\n",
    "\n",
    "accuracy = accuracy_score(ytrue, yhat)\n",
    "precision = precision_score(ytrue, yhat, average='weighted')\n",
    "recall = recall_score(ytrue, yhat, average='weighted')\n",
    "f1 = f1_score(ytrue, yhat, average='weighted')\n",
    "\n",
    "# Plot confusion matrix\n",
    "plt.figure(figsize=(8, 6))\n",
    "sns.set(font_scale=1.2)\n",
    "sns.heatmap(conf_matrix, annot=True, fmt=\"d\", cmap=\"Blues\", cbar=False,\n",
    "            xticklabels=class_labels, yticklabels=class_labels)\n",
    "plt.xlabel('Predicted')\n",
    "plt.ylabel('Actual')\n",
    "plt.title('Confusion Matrix')\n",
    "plt.show()\n",
    "\n",
    "print(\"\\nAccuracy:\", accuracy)\n",
    "print(\"Precision:\", precision)\n",
    "print(\"Recall:\", recall)\n",
    "print(\"F1-Score:\", f1)\n",
    "\n",
    "# Calculate classification report\n",
    "report = classification_report(ytrue, yhat, target_names=class_labels)\n",
    "print(\"\\nClassification Report:\\n\", report)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 11. Test in Real Time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "colors = [(245,117,16), (117,245,16), (16,117,245)]\n",
    "def prob_viz(res, actions, input_frame, colors):\n",
    "    output_frame = input_frame.copy()\n",
    "    for num, prob in enumerate(res):\n",
    "        # MAC OS FRAME RECTANGLE\n",
    "        \n",
    "        # cv2.rectangle(image, start_point, end_point, color, thickness)\n",
    "        # cv2.putText(image, text, org (coordinates), font, fontScale, color[, thickness[, lineType[, bottomLeftOrigin]]])\n",
    "        \n",
    "        # Pembesaran Frame Rectangle = WORKED but masih KECIL\n",
    "        cv2.rectangle(output_frame, (0, 200+num*200), (int(prob*480), 300+num*200), colors[num], -1)\n",
    "        cv2.putText(output_frame, actions[num], (0, 260+num*200), cv2.FONT_HERSHEY_SIMPLEX, 2, (0, 0, 0), 5, cv2.LINE_AA)\n",
    "\n",
    "        # Kasih Tambahan Angka Probabilitas \n",
    "        cv2.rectangle(output_frame, (500, 200+num*200), (650, 300+num*200), (0, 0, 0), 3)\n",
    "        cv2.putText(output_frame, ' ' + str(round(prob*100, 1)) + '%', (500, 260+num*200), cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 0, 0), 2, cv2.LINE_AA)\n",
    "        \n",
    "    return output_frame"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 1. New detection variables\n",
    "sequence = []\n",
    "sentence = []\n",
    "predictions = []\n",
    "threshold = 0.9\n",
    "\n",
    "cap = cv2.VideoCapture(0)\n",
    "# Set mediapipe model \n",
    "with mp_holistic.Holistic(min_detection_confidence=0.5, min_tracking_confidence=0.5) as holistic:\n",
    "    while cap.isOpened():\n",
    "\n",
    "        # Read feed\n",
    "        ret, frame = cap.read()\n",
    "\n",
    "        # Make detections\n",
    "        image, results = mediapipe_detection(frame, holistic)\n",
    "        # print(results)\n",
    "        \n",
    "        # Draw landmarks\n",
    "        draw_styled_landmarks(image, results)\n",
    "        \n",
    "        # 2. Prediction logic\n",
    "        keypoints = extract_keypoints(results)\n",
    "        sequence.append(keypoints)\n",
    "        sequence = sequence[-30:]\n",
    "        \n",
    "        if len(sequence) == 30:\n",
    "            res = model.predict(np.expand_dims(sequence, axis=0))[0]\n",
    "            # print(actions[np.argmax(res)])\n",
    "            predictions.append(np.argmax(res))\n",
    "            \n",
    "            \n",
    "        #3. Viz logic\n",
    "            # # Filter out the noise\n",
    "            if np.argmax(np.bincount(predictions[-10:]))==np.argmax(res):\n",
    "            # if np.unique(predictions[-10:])[0]==np.argmax(res): \n",
    "                if res[np.argmax(res)] > threshold: \n",
    "                    \n",
    "                    if len(sentence) > 0: \n",
    "                        if actions[np.argmax(res)] != sentence[-1]:\n",
    "                            sentence.append(actions[np.argmax(res)])\n",
    "                    else:\n",
    "                        sentence.append(actions[np.argmax(res)])\n",
    "\n",
    "            if len(sentence) > 5: \n",
    "                sentence = sentence[-5:]\n",
    "\n",
    "            # Viz probabilities\n",
    "            image = prob_viz(res, actions, image, colors)\n",
    "            \n",
    "        cv2.rectangle(image, (0,0), (2000, 150), (245, 117, 16), -1)\n",
    "        cv2.putText(image, ' '.join(sentence), (30,100), \n",
    "                       cv2.FONT_HERSHEY_SIMPLEX, 2, (0, 0, 0), 2, cv2.LINE_AA)\n",
    "        \n",
    "        # Show to screen\n",
    "        cv2.imshow('Hazlan SIBI REAL-TIME TEST | WEBCAM', image)\n",
    "\n",
    "        # Break gracefully\n",
    "        if cv2.waitKey(10) & 0xFF == ord('q'):\n",
    "            break\n",
    "    cap.release()\n",
    "    cv2.destroyAllWindows()\n",
    "    cv2.waitKey(1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 12. Bonus Step Perhitungan Arsitektur BiGRU"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn.functional as F\n",
    "\n",
    "# Input: Single keypoint value 0.1\n",
    "X = torch.tensor([[0.5]])\n",
    "\n",
    "# Define the GRU network parameters:\n",
    "hidden_size = 1  # GRU layer with only one unit\n",
    "\n",
    "### 2. First BiGRU Layer\n",
    "# Shared weights for update, reset, and new state gates (for simplicity)\n",
    "W_u = torch.tensor([[0.5]])\n",
    "W_r = torch.tensor([[0.5]])\n",
    "W = torch.tensor([[0.5]])  # W for candidate hidden state\n",
    "Q_u = torch.tensor([[0.2]])\n",
    "Q_r = torch.tensor([[0.2]])\n",
    "Q = torch.tensor([[0.2]])  # Q for candidate hidden state\n",
    "b_u = torch.tensor([0.1])\n",
    "b_r = torch.tensor([0.1])\n",
    "b = torch.tensor([0.1])    # b for candidate hidden state\n",
    "\n",
    "# Initialize the hidden state for the forward and backward cells of the first BiGRU layer\n",
    "h_t_f1 = torch.zeros(1)  # Forward cell\n",
    "h_t_b1 = torch.zeros(1)  # Backward cell\n",
    "\n",
    "# GRU cell calculation for the first BiGRU layer - forward cell\n",
    "u_f1 = torch.sigmoid(torch.matmul(X, W_u) + b_u + torch.matmul(h_t_f1, Q_u) + b_u)\n",
    "r_f1 = torch.sigmoid(torch.matmul(X, W_r) + b_r + torch.matmul(h_t_f1, Q_r) + b_r)\n",
    "h_tilde_f1 = torch.tanh(torch.matmul(X, W) + b + r_f1 * (torch.matmul(h_t_f1, Q) + b))\n",
    "h_t_f1_new = u_f1 * h_t_f1 + (1 - u_f1) * h_tilde_f1\n",
    "\n",
    "# GRU cell calculation for the first BiGRU layer - backward cell\n",
    "u_b1 = torch.sigmoid(torch.matmul(X, W_u) + b_u + torch.matmul(h_t_b1, Q_u) + b_u)\n",
    "r_b1 = torch.sigmoid(torch.matmul(X, W_r) + b_r + torch.matmul(h_t_b1, Q_r) + b_r)\n",
    "h_tilde_b1 = torch.tanh(torch.matmul(X, W) + b + r_b1 * (torch.matmul(h_t_b1, Q) + b))\n",
    "h_t_b1_new = u_b1 * h_t_b1 + (1 - u_b1) * h_tilde_b1\n",
    "\n",
    "# Combine the forward and backward hidden states using element-wise multiplication\n",
    "h_t_1 = h_t_f1_new * h_t_b1_new\n",
    "\n",
    "# Apply ReLU activation after combining\n",
    "h_t_1_relu = F.relu(h_t_1)\n",
    "\n",
    "### 3. Second BiGRU Layer\n",
    "# Initialize the hidden state for the forward and backward cells of the second BiGRU layer\n",
    "h_t_f2 = torch.zeros(1)  # Forward cell\n",
    "h_t_b2 = torch.zeros(1)  # Backward cell\n",
    "\n",
    "# GRU cell calculation for the second BiGRU layer - forward cell\n",
    "u_f2 = torch.sigmoid(torch.matmul(X, W_u) + b_u + torch.matmul(h_t_1_relu, Q_u) + b_u)\n",
    "r_f2 = torch.sigmoid(torch.matmul(X, W_r) + b_r + torch.matmul(h_t_1_relu, Q_r) + b_r)\n",
    "h_tilde_f2 = torch.tanh(torch.matmul(X, W) + b + r_f2 * (torch.matmul(h_t_1_relu, Q) + b))\n",
    "h_t_f2_new = u_f2 * h_t_1_relu + (1 - u_f2) * h_tilde_f2\n",
    "\n",
    "# GRU cell calculation for the second BiGRU layer - backward cell\n",
    "u_b2 = torch.sigmoid(torch.matmul(X, W_u) + b_u + torch.matmul(h_t_1_relu, Q_u) + b_u)\n",
    "r_b2 = torch.sigmoid(torch.matmul(X, W_r) + b_r + torch.matmul(h_t_1_relu, Q_r) + b_r)\n",
    "h_tilde_b2 = torch.tanh(torch.matmul(X, W) + b + r_b2 * (torch.matmul(h_t_1_relu, Q) + b))\n",
    "h_t_b2_new = u_b2 * h_t_1_relu + (1 - u_b2) * h_tilde_b2\n",
    "\n",
    "# Combine the forward and backward hidden states using element-wise multiplication\n",
    "h_t_2 = h_t_f2_new * h_t_b2_new\n",
    "\n",
    "# Apply ReLU activation after combining\n",
    "h_t_2_relu = F.relu(h_t_2)\n",
    "\n",
    "### 4. Dense Layer\n",
    "# Dense layer weights and bias\n",
    "W_dense = torch.tensor([[0.7]])\n",
    "b_dense = torch.tensor([0.2])\n",
    "\n",
    "# Dense layer output\n",
    "y_linear = torch.matmul(h_t_2_relu, W_dense.T) + b_dense\n",
    "\n",
    "# Apply ReLU activation after the dense layer\n",
    "h_dense_relu = F.relu(y_linear)\n",
    "\n",
    "### 5. Output Layer\n",
    "# Softmax activation for output layer\n",
    "y_linear_2 = torch.matmul(h_dense_relu, W_dense.T) + b_dense\n",
    "\n",
    "y_pred = F.softmax(y_linear_2, dim=0)\n",
    "\n",
    "# Print detailed calculations\n",
    "print(\"\\n=== First BiGRU Layer - Forward Cell ===\")\n",
    "print(f\"Update Gate (u_f1): {u_f1.item()}\")\n",
    "print(f\"Reset Gate (r_f1): {r_f1.item()}\")\n",
    "print(f\"Candidate Hidden State (h_tilde_f1): {h_tilde_f1.item()}\")\n",
    "print(f\"New Hidden State (h_t_f1_new): {h_t_f1_new.item()}\")\n",
    "\n",
    "print(\"\\n=== First BiGRU Layer - Backward Cell ===\")\n",
    "print(f\"Update Gate (u_b1): {u_b1.item()}\")\n",
    "print(f\"Reset Gate (r_b1): {r_b1.item()}\")\n",
    "print(f\"Candidate Hidden State (h_tilde_b1): {h_tilde_b1.item()}\")\n",
    "print(f\"New Hidden State (h_t_b1_new): {h_t_b1_new.item()}\")\n",
    "\n",
    "print(f\"\\n1st First Cell ReLU: {h_t_1_relu.item()}\")\n",
    "\n",
    "print(\"\\n=== Second BiGRU Layer - Forward Cell ===\")\n",
    "print(f\"Update Gate (u_f2): {u_f2.item()}\")\n",
    "print(f\"Reset Gate (r_f2): {r_f2.item()}\")\n",
    "print(f\"Candidate Hidden State (h_tilde_f2): {h_tilde_f2.item()}\")\n",
    "print(f\"New Hidden State (h_t_f2_new): {h_t_f2_new.item()}\")\n",
    "\n",
    "print(\"\\n=== Second BiGRU Layer - Backward Cell ===\")\n",
    "print(f\"Update Gate (u_b2): {u_b2.item()}\")\n",
    "print(f\"Reset Gate (r_b2): {r_b2.item()}\")\n",
    "print(f\"Candidate Hidden State (h_tilde_b2): {h_tilde_b2.item()}\")\n",
    "print(f\"New Hidden State (h_t_b2_new): {h_t_b2_new.item()}\")\n",
    "\n",
    "print(f\"\\n 2nd Second Cell ReLU: {h_t_2_relu.item()}\")\n",
    "\n",
    "print(\"\\n=== First Dense Layer ===\")\n",
    "print(f\"Linear Output (y_linear): {y_linear.item()}\")\n",
    "\n",
    "print(\"\\n=== Second Dense Layer (Output) ===\")\n",
    "print(f\"2nd Linear Output (y_linear_2): {y_linear_2.item()}\")\n",
    "print(f\"Softmax Output (y_pred): {y_pred.item()}\")"
   ]
  }
 ],
 "metadata": {
  "language_info": {
   "name": "python"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}