notebook534ce12ff7.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "ed98f066",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-09-30T14:50:48.334380Z",
     "iopub.status.busy": "2024-09-30T14:50:48.333571Z",
     "iopub.status.idle": "2024-09-30T14:50:48.343328Z",
     "shell.execute_reply": "2024-09-30T14:50:48.342522Z"
    },
    "id": "sb0zW1T7DIGh",
    "papermill": {
     "duration": 0.0203,
     "end_time": "2024-09-30T14:50:48.345243",
     "exception": false,
     "start_time": "2024-09-30T14:50:48.324943",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import random"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "2526282f",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-09-30T14:50:48.359409Z",
     "iopub.status.busy": "2024-09-30T14:50:48.359120Z",
     "iopub.status.idle": "2024-09-30T14:50:48.364305Z",
     "shell.execute_reply": "2024-09-30T14:50:48.363419Z"
    },
    "papermill": {
     "duration": 0.014379,
     "end_time": "2024-09-30T14:50:48.366319",
     "exception": false,
     "start_time": "2024-09-30T14:50:48.351940",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/kaggle\n"
     ]
    }
   ],
   "source": [
    "cd .."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "ee03d930",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-09-30T14:50:48.380259Z",
     "iopub.status.busy": "2024-09-30T14:50:48.380005Z",
     "iopub.status.idle": "2024-09-30T14:50:48.384855Z",
     "shell.execute_reply": "2024-09-30T14:50:48.383989Z"
    },
    "papermill": {
     "duration": 0.013974,
     "end_time": "2024-09-30T14:50:48.386817",
     "exception": false,
     "start_time": "2024-09-30T14:50:48.372843",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/\n"
     ]
    }
   ],
   "source": [
    "cd ..\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0d4b0aab",
   "metadata": {
    "papermill": {
     "duration": 0.006625,
     "end_time": "2024-09-30T14:50:48.401059",
     "exception": false,
     "start_time": "2024-09-30T14:50:48.394434",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e1cdc439",
   "metadata": {
    "papermill": {
     "duration": 0.006297,
     "end_time": "2024-09-30T14:50:48.413785",
     "exception": false,
     "start_time": "2024-09-30T14:50:48.407488",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "1cad4a90",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-09-30T14:50:48.427972Z",
     "iopub.status.busy": "2024-09-30T14:50:48.427654Z",
     "iopub.status.idle": "2024-09-30T14:50:49.444200Z",
     "shell.execute_reply": "2024-09-30T14:50:49.443262Z"
    },
    "papermill": {
     "duration": 1.025975,
     "end_time": "2024-09-30T14:50:49.446402",
     "exception": false,
     "start_time": "2024-09-30T14:50:48.420427",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "'=24.4'                                     \u001b[0m\u001b[01;36mlibx32\u001b[0m@\r\n",
      " NGC-DL-CONTAINER-LICENSE                   \u001b[01;34mmedia\u001b[0m/\r\n",
      " \u001b[01;36mbin\u001b[0m@                                       \u001b[01;34mmnt\u001b[0m/\r\n",
      " \u001b[01;34mboot\u001b[0m/                                      \u001b[01;34mopt\u001b[0m/\r\n",
      " cuda-keyring_1.0-1_all.deb                 \u001b[01;34mproc\u001b[0m/\r\n",
      " \u001b[01;34mdev\u001b[0m/                                       \u001b[01;34mroot\u001b[0m/\r\n",
      " \u001b[01;32mentrypoint.sh\u001b[0m*                             \u001b[01;34mrun\u001b[0m/\r\n",
      " \u001b[01;34metc\u001b[0m/                                       \u001b[01;32mrun_jupyter.sh\u001b[0m*\r\n",
      " \u001b[01;34mhome\u001b[0m/                                      \u001b[01;36msbin\u001b[0m@\r\n",
      " \u001b[01;32minstall_packages.sh\u001b[0m*                       \u001b[01;34msrv\u001b[0m/\r\n",
      " \u001b[01;34mkaggle\u001b[0m/                                    \u001b[01;34msys\u001b[0m/\r\n",
      " \u001b[01;36mlib\u001b[0m@                                       \u001b[30;42mtmp\u001b[0m/\r\n",
      " \u001b[01;36mlib32\u001b[0m@                                     \u001b[01;34musr\u001b[0m/\r\n",
      " \u001b[01;36mlib64\u001b[0m@                                     \u001b[01;34mvar\u001b[0m/\r\n",
      " libnvinfer8_8.6.1.6-1+cuda12.0_amd64.deb\r\n"
     ]
    }
   ],
   "source": [
    "ls"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "55c405f9",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-09-30T14:50:49.462283Z",
     "iopub.status.busy": "2024-09-30T14:50:49.461929Z",
     "iopub.status.idle": "2024-09-30T14:50:49.522072Z",
     "shell.execute_reply": "2024-09-30T14:50:49.521344Z"
    },
    "id": "U4mKEJSODK3p",
    "papermill": {
     "duration": 0.070492,
     "end_time": "2024-09-30T14:50:49.524121",
     "exception": false,
     "start_time": "2024-09-30T14:50:49.453629",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "### read words ###\n",
    "text_file = open('kaggle/input/dataset/words_250000_train.txt',\"r\")\n",
    "full_dictionary = text_file.read().splitlines()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "9033c365",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-09-30T14:50:49.538749Z",
     "iopub.status.busy": "2024-09-30T14:50:49.538433Z",
     "iopub.status.idle": "2024-09-30T14:50:49.546465Z",
     "shell.execute_reply": "2024-09-30T14:50:49.545573Z"
    },
    "papermill": {
     "duration": 0.017539,
     "end_time": "2024-09-30T14:50:49.548415",
     "exception": false,
     "start_time": "2024-09-30T14:50:49.530876",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/kaggle/working\n"
     ]
    }
   ],
   "source": [
    "cd /kaggle/working"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "bf0f3272",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-09-30T14:50:49.562783Z",
     "iopub.status.busy": "2024-09-30T14:50:49.562515Z",
     "iopub.status.idle": "2024-09-30T14:51:39.525807Z",
     "shell.execute_reply": "2024-09-30T14:51:39.524972Z"
    },
    "id": "uBBtPhXtrc6Q",
    "papermill": {
     "duration": 49.973223,
     "end_time": "2024-09-30T14:51:39.528191",
     "exception": false,
     "start_time": "2024-09-30T14:50:49.554968",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "random.seed(40)\n",
    "\n",
    "import numpy as np\n",
    "from tensorflow.keras.preprocessing.sequence import pad_sequences\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "random.seed(40)\n",
    "\n",
    "# Sample data for demonstration\n",
    "words = full_dictionary\n",
    "\n",
    "def generate_training_data_3(words):\n",
    "    X = []\n",
    "    y = []\n",
    "    for word in words:\n",
    "      # print(word)\n",
    "      for char in np.unique(list(word)):\n",
    "          X.append(word.replace(char,\"_\"))  # Input with missing character\n",
    "          y.append(char)\n",
    "          try:\n",
    "            X.append(word.replace(char,\"_\").replace(random.choice(list(word.replace(char,\"\"))),\"_\"))  # Input with missing character\n",
    "            y.append(char)                      # Target character (the missing one)\n",
    "          except:\n",
    "            pass\n",
    "          # print(word.replace(char,\"_\"))\n",
    "    return X, y\n",
    "\n",
    "\n",
    "# Toy dataset\n",
    "X_train_3, y_train_3 = generate_training_data_3(words)\n",
    "\n",
    "words_3 = X_train_3\n",
    "labels_3 = y_train_3\n",
    "\n",
    "max_length_3 = max(len(word) for word in words_3)\n",
    "\n",
    "# Create a dictionary for character to index mapping\n",
    "char_to_index = {char: idx for idx, char in enumerate(set(''.join(words_3)))}\n",
    "index_to_char = {idx: char for char, idx in char_to_index.items()}\n",
    "\n",
    "# Convert words and labels to numerical format\n",
    "X_padded_3 = pad_sequences([[char_to_index[char] for char in word] for word in words_3], maxlen=max_length_3, padding='post')\n",
    "y_padded_3 = np.array([char_to_index[label] for label in labels_3])\n",
    "\n",
    "X_train_3, X_temp_3, y_train_3, y_temp_3 = train_test_split(X_padded_3, y_padded_3, test_size=0.3, random_state=42)\n",
    "X_val_3, X_test_3, y_val_3, y_test_3 = train_test_split(X_temp_3, y_temp_3, test_size=0.5, random_state=42)\n",
    "\n",
    "# print(\"Padded data:\")\n",
    "# for i in range(len(words)):\n",
    "#     print(f\"{words[i]} -> {labels[i]}\")\n",
    "\n",
    "# print(\"\\nPadded and numerical data:\")\n",
    "# print(X_padded_2)\n",
    "# print(y_padded_2)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6f52ae8d",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-06-20T18:25:02.064844Z",
     "iopub.status.busy": "2024-06-20T18:25:02.063934Z",
     "iopub.status.idle": "2024-06-20T18:25:02.070731Z",
     "shell.execute_reply": "2024-06-20T18:25:02.069350Z",
     "shell.execute_reply.started": "2024-06-20T18:25:02.064810Z"
    },
    "papermill": {
     "duration": 0.006683,
     "end_time": "2024-09-30T14:51:39.542118",
     "exception": false,
     "start_time": "2024-09-30T14:51:39.535435",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# BiLSTM with attention\n",
    "\n",
    "## attention \n",
    "\n",
    "## LSTM 256\n",
    "\n",
    "## embedding - 50\n",
    "\n",
    "## no dropout"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "71b1957b",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-09-30T14:51:39.557606Z",
     "iopub.status.busy": "2024-09-30T14:51:39.556978Z",
     "iopub.status.idle": "2024-09-30T14:51:39.565972Z",
     "shell.execute_reply": "2024-09-30T14:51:39.565051Z"
    },
    "papermill": {
     "duration": 0.019077,
     "end_time": "2024-09-30T14:51:39.568001",
     "exception": false,
     "start_time": "2024-09-30T14:51:39.548924",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "from tensorflow.keras.callbacks import ModelCheckpoint\n",
    "from tensorflow.keras.callbacks import EarlyStopping\n",
    "\n",
    "# Define a checkpoint callback\n",
    "checkpoint_callback_6 = ModelCheckpoint('best_model_6.keras', \n",
    "                                      monitor='val_loss', \n",
    "                                      save_best_only=True, \n",
    "                                      mode='min', \n",
    "                                      verbose=1)\n",
    "\n",
    "early_stopping_callback = EarlyStopping(monitor='val_loss', \n",
    "                                       patience=3, \n",
    "                                       mode='min', \n",
    "                                       verbose=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "6a777437",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-09-30T14:51:39.583787Z",
     "iopub.status.busy": "2024-09-30T14:51:39.583175Z",
     "iopub.status.idle": "2024-09-30T14:51:39.587267Z",
     "shell.execute_reply": "2024-09-30T14:51:39.586414Z"
    },
    "papermill": {
     "duration": 0.013787,
     "end_time": "2024-09-30T14:51:39.589175",
     "exception": false,
     "start_time": "2024-09-30T14:51:39.575388",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# # detect and init the TPU\n",
    "# tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n",
    "\n",
    "# # instantiate a distribution strategy\n",
    "# tf.tpu.experimental.initialize_tpu_system(tpu)\n",
    "# tpu_strategy = tf.distribute.TPUStrategy(tpu)\n",
    "\n",
    "# # instantiating the model in the strategy scope creates the model on the TPU"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "2346b091",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-09-30T14:51:39.604687Z",
     "iopub.status.busy": "2024-09-30T14:51:39.604124Z",
     "iopub.status.idle": "2024-09-30T14:51:40.567236Z",
     "shell.execute_reply": "2024-09-30T14:51:40.566327Z"
    },
    "papermill": {
     "duration": 0.972988,
     "end_time": "2024-09-30T14:51:40.569240",
     "exception": false,
     "start_time": "2024-09-30T14:51:39.596252",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.10/site-packages/keras/src/layers/core/embedding.py:90: UserWarning: Argument `input_length` is deprecated. Just remove it.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\">Model: \"sequential\"</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1mModel: \"sequential\"\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┓\n",
       "┃<span style=\"font-weight: bold\"> Layer (type)                    </span>┃<span style=\"font-weight: bold\"> Output Shape           </span>┃<span style=\"font-weight: bold\">       Param # </span>┃\n",
       "┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━┩\n",
       "│ embedding (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Embedding</span>)           │ ?                      │   <span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> (unbuilt) │\n",
       "├─────────────────────────────────┼────────────────────────┼───────────────┤\n",
       "│ bidirectional (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Bidirectional</span>)   │ ?                      │   <span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> (unbuilt) │\n",
       "├─────────────────────────────────┼────────────────────────┼───────────────┤\n",
       "│ dropout (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Dropout</span>)               │ ?                      │   <span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> (unbuilt) │\n",
       "├─────────────────────────────────┼────────────────────────┼───────────────┤\n",
       "│ bidirectional_1 (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Bidirectional</span>) │ ?                      │   <span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> (unbuilt) │\n",
       "├─────────────────────────────────┼────────────────────────┼───────────────┤\n",
       "│ dropout_1 (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Dropout</span>)             │ ?                      │   <span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> (unbuilt) │\n",
       "├─────────────────────────────────┼────────────────────────┼───────────────┤\n",
       "│ bidirectional_2 (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Bidirectional</span>) │ ?                      │   <span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> (unbuilt) │\n",
       "├─────────────────────────────────┼────────────────────────┼───────────────┤\n",
       "│ dropout_2 (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Dropout</span>)             │ ?                      │   <span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> (unbuilt) │\n",
       "├─────────────────────────────────┼────────────────────────┼───────────────┤\n",
       "│ attention_layer                 │ ?                      │   <span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> (unbuilt) │\n",
       "│ (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">AttentionLayer</span>)                │                        │               │\n",
       "├─────────────────────────────────┼────────────────────────┼───────────────┤\n",
       "│ dense (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Dense</span>)                   │ ?                      │   <span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> (unbuilt) │\n",
       "└─────────────────────────────────┴────────────────────────┴───────────────┘\n",
       "</pre>\n"
      ],
      "text/plain": [
       "┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┓\n",
       "┃\u001b[1m \u001b[0m\u001b[1mLayer (type)                   \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mOutput Shape          \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1m      Param #\u001b[0m\u001b[1m \u001b[0m┃\n",
       "┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━┩\n",
       "│ embedding (\u001b[38;5;33mEmbedding\u001b[0m)           │ ?                      │   \u001b[38;5;34m0\u001b[0m (unbuilt) │\n",
       "├─────────────────────────────────┼────────────────────────┼───────────────┤\n",
       "│ bidirectional (\u001b[38;5;33mBidirectional\u001b[0m)   │ ?                      │   \u001b[38;5;34m0\u001b[0m (unbuilt) │\n",
       "├─────────────────────────────────┼────────────────────────┼───────────────┤\n",
       "│ dropout (\u001b[38;5;33mDropout\u001b[0m)               │ ?                      │   \u001b[38;5;34m0\u001b[0m (unbuilt) │\n",
       "├─────────────────────────────────┼────────────────────────┼───────────────┤\n",
       "│ bidirectional_1 (\u001b[38;5;33mBidirectional\u001b[0m) │ ?                      │   \u001b[38;5;34m0\u001b[0m (unbuilt) │\n",
       "├─────────────────────────────────┼────────────────────────┼───────────────┤\n",
       "│ dropout_1 (\u001b[38;5;33mDropout\u001b[0m)             │ ?                      │   \u001b[38;5;34m0\u001b[0m (unbuilt) │\n",
       "├─────────────────────────────────┼────────────────────────┼───────────────┤\n",
       "│ bidirectional_2 (\u001b[38;5;33mBidirectional\u001b[0m) │ ?                      │   \u001b[38;5;34m0\u001b[0m (unbuilt) │\n",
       "├─────────────────────────────────┼────────────────────────┼───────────────┤\n",
       "│ dropout_2 (\u001b[38;5;33mDropout\u001b[0m)             │ ?                      │   \u001b[38;5;34m0\u001b[0m (unbuilt) │\n",
       "├─────────────────────────────────┼────────────────────────┼───────────────┤\n",
       "│ attention_layer                 │ ?                      │   \u001b[38;5;34m0\u001b[0m (unbuilt) │\n",
       "│ (\u001b[38;5;33mAttentionLayer\u001b[0m)                │                        │               │\n",
       "├─────────────────────────────────┼────────────────────────┼───────────────┤\n",
       "│ dense (\u001b[38;5;33mDense\u001b[0m)                   │ ?                      │   \u001b[38;5;34m0\u001b[0m (unbuilt) │\n",
       "└─────────────────────────────────┴────────────────────────┴───────────────┘\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\"> Total params: </span><span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> (0.00 B)\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1m Total params: \u001b[0m\u001b[38;5;34m0\u001b[0m (0.00 B)\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\"> Trainable params: </span><span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> (0.00 B)\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1m Trainable params: \u001b[0m\u001b[38;5;34m0\u001b[0m (0.00 B)\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\"> Non-trainable params: </span><span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> (0.00 B)\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1m Non-trainable params: \u001b[0m\u001b[38;5;34m0\u001b[0m (0.00 B)\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "from tensorflow.keras.models import Sequential\n",
    "from tensorflow.keras.layers import Embedding, Bidirectional, LSTM, Dense, Dropout, Concatenate, Layer\n",
    "from tensorflow.keras import Input, Model\n",
    "import keras\n",
    "# Define parameters\n",
    "vocab_size = len(char_to_index)  # Size of vocabulary (unique characters)\n",
    "embedding_dim = 50  # Dimension of character embeddings\n",
    "lstm_units = 256  # Number of units in LSTM layers\n",
    "num_layers = 3  # Number of BiLSTM layers\n",
    "dropout_rate = 0.1  # Dropout rate\n",
    "\n",
    "# Custom Attention Layer\n",
    "@keras.saving.register_keras_serializable()\n",
    "class AttentionLayer(Layer):\n",
    "    def __init__(self, **kwargs):\n",
    "        super(AttentionLayer, self).__init__(**kwargs)\n",
    "\n",
    "    def build(self, input_shape):\n",
    "        self.W = self.add_weight(name=\"att_weight\", shape=(input_shape[-1], 1),\n",
    "                                 initializer=\"normal\")\n",
    "        self.b = self.add_weight(name=\"att_bias\", shape=(input_shape[1], 1),\n",
    "                                 initializer=\"zeros\")\n",
    "\n",
    "        super(AttentionLayer, self).build(input_shape)\n",
    "\n",
    "    def call(self, x):\n",
    "        et = tf.squeeze(tf.tanh(tf.matmul(x, self.W) + self.b), axis=-1)\n",
    "        at = tf.nn.softmax(et)\n",
    "        at = tf.expand_dims(at, axis=-1)\n",
    "        output = x * at\n",
    "        return tf.reduce_sum(output, axis=1)\n",
    "\n",
    "# Define Sequential model\n",
    "model_6 = Sequential()\n",
    "\n",
    "# Add layers to the model\n",
    "model_6.add(Embedding(input_dim=vocab_size, output_dim=embedding_dim, input_length=max_length_3))\n",
    "\n",
    "# Add BiLSTM layers with Dropout\n",
    "for _ in range(num_layers):\n",
    "    model_6.add(Bidirectional(LSTM(units=lstm_units, return_sequences=True)))\n",
    "    model_6.add(Dropout(rate=dropout_rate))\n",
    "\n",
    "# Add Attention Layer\n",
    "model_6.add(AttentionLayer())\n",
    "\n",
    "# Output layer\n",
    "model_6.add(Dense(units=vocab_size, activation='softmax'))\n",
    "\n",
    "# Compile the model\n",
    "model_6.compile(optimizer='adam',\n",
    "              loss='sparse_categorical_crossentropy',\n",
    "              metrics=['accuracy'])\n",
    "\n",
    "model_6.summary()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "c0a136f8",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-09-30T14:51:40.586961Z",
     "iopub.status.busy": "2024-09-30T14:51:40.586335Z",
     "iopub.status.idle": "2024-09-30T15:30:03.964284Z",
     "shell.execute_reply": "2024-09-30T15:30:03.963292Z"
    },
    "papermill": {
     "duration": 2305.468718,
     "end_time": "2024-09-30T15:30:06.046137",
     "exception": false,
     "start_time": "2024-09-30T14:51:40.577419",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/3\n",
      "\u001b[1m9194/9194\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 77ms/step - accuracy: 0.3581 - loss: 2.0191\n",
      "Epoch 1: val_loss improved from inf to 1.45258, saving model to best_model_6.keras\n",
      "\u001b[1m9194/9194\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m770s\u001b[0m 83ms/step - accuracy: 0.3581 - loss: 2.0190 - val_accuracy: 0.5045 - val_loss: 1.4526\n",
      "Epoch 2/3\n",
      "\u001b[1m9194/9194\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 78ms/step - accuracy: 0.5202 - loss: 1.3977\n",
      "Epoch 2: val_loss improved from 1.45258 to 1.33420, saving model to best_model_6.keras\n",
      "\u001b[1m9194/9194\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m766s\u001b[0m 83ms/step - accuracy: 0.5202 - loss: 1.3977 - val_accuracy: 0.5361 - val_loss: 1.3342\n",
      "Epoch 3/3\n",
      "\u001b[1m9194/9194\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 78ms/step - accuracy: 0.5572 - loss: 1.2683\n",
      "Epoch 3: val_loss improved from 1.33420 to 1.29104, saving model to best_model_6.keras\n",
      "\u001b[1m9194/9194\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m767s\u001b[0m 83ms/step - accuracy: 0.5572 - loss: 1.2683 - val_accuracy: 0.5480 - val_loss: 1.2910\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.src.callbacks.history.History at 0x7d1aecfe4880>"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Train the model\n",
    "model_6.fit( X_train_3, \n",
    "            y_train_3, \n",
    "            epochs=3, \n",
    "            batch_size=256, \n",
    "            validation_data=(X_val_3, y_val_3), \n",
    "            verbose=1,callbacks=[checkpoint_callback_6,early_stopping_callback])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "b14a505d",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-09-30T15:30:10.639162Z",
     "iopub.status.busy": "2024-09-30T15:30:10.638769Z",
     "iopub.status.idle": "2024-09-30T17:00:18.681108Z",
     "shell.execute_reply": "2024-09-30T17:00:18.680011Z"
    },
    "papermill": {
     "duration": 5418.10614,
     "end_time": "2024-09-30T17:00:26.451532",
     "exception": false,
     "start_time": "2024-09-30T15:30:08.345392",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/7\n",
      "\u001b[1m9194/9194\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 78ms/step - accuracy: 0.5783 - loss: 1.1963\n",
      "Epoch 1: val_loss improved from 1.29104 to 1.26879, saving model to best_model_6.keras\n",
      "\u001b[1m9194/9194\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m768s\u001b[0m 83ms/step - accuracy: 0.5783 - loss: 1.1963 - val_accuracy: 0.5529 - val_loss: 1.2688\n",
      "Epoch 2/7\n",
      "\u001b[1m9194/9194\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 78ms/step - accuracy: 0.5929 - loss: 1.1486\n",
      "Epoch 2: val_loss improved from 1.26879 to 1.25708, saving model to best_model_6.keras\n",
      "\u001b[1m9194/9194\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m766s\u001b[0m 83ms/step - accuracy: 0.5929 - loss: 1.1486 - val_accuracy: 0.5563 - val_loss: 1.2571\n",
      "Epoch 3/7\n",
      "\u001b[1m9194/9194\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 78ms/step - accuracy: 0.6031 - loss: 1.1154\n",
      "Epoch 3: val_loss improved from 1.25708 to 1.25475, saving model to best_model_6.keras\n",
      "\u001b[1m9194/9194\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m768s\u001b[0m 84ms/step - accuracy: 0.6031 - loss: 1.1154 - val_accuracy: 0.5583 - val_loss: 1.2548\n",
      "Epoch 4/7\n",
      "\u001b[1m9194/9194\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 78ms/step - accuracy: 0.6112 - loss: 1.0897\n",
      "Epoch 4: val_loss improved from 1.25475 to 1.25157, saving model to best_model_6.keras\n",
      "\u001b[1m9194/9194\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m768s\u001b[0m 84ms/step - accuracy: 0.6112 - loss: 1.0897 - val_accuracy: 0.5587 - val_loss: 1.2516\n",
      "Epoch 5/7\n",
      "\u001b[1m9194/9194\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 78ms/step - accuracy: 0.6174 - loss: 1.0696\n",
      "Epoch 5: val_loss did not improve from 1.25157\n",
      "\u001b[1m9194/9194\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m802s\u001b[0m 84ms/step - accuracy: 0.6174 - loss: 1.0696 - val_accuracy: 0.5598 - val_loss: 1.2517\n",
      "Epoch 6/7\n",
      "\u001b[1m9194/9194\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 78ms/step - accuracy: 0.6220 - loss: 1.0546\n",
      "Epoch 6: val_loss did not improve from 1.25157\n",
      "\u001b[1m9194/9194\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m768s\u001b[0m 84ms/step - accuracy: 0.6220 - loss: 1.0546 - val_accuracy: 0.5612 - val_loss: 1.2536\n",
      "Epoch 7/7\n",
      "\u001b[1m9194/9194\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 78ms/step - accuracy: 0.6254 - loss: 1.0433\n",
      "Epoch 7: val_loss did not improve from 1.25157\n",
      "\u001b[1m9194/9194\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m767s\u001b[0m 83ms/step - accuracy: 0.6254 - loss: 1.0433 - val_accuracy: 0.5602 - val_loss: 1.2537\n",
      "Epoch 7: early stopping\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.src.callbacks.history.History at 0x7d1a0aec6110>"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Train the model\n",
    "model_6.fit( X_train_3, \n",
    "            y_train_3, \n",
    "            epochs=7, \n",
    "            batch_size=256, \n",
    "            validation_data=(X_val_3, y_val_3), \n",
    "            verbose=1,callbacks=[checkpoint_callback_6,early_stopping_callback])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "23ca5f30",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-09-30T17:00:42.168121Z",
     "iopub.status.busy": "2024-09-30T17:00:42.167171Z",
     "iopub.status.idle": "2024-09-30T17:00:42.171713Z",
     "shell.execute_reply": "2024-09-30T17:00:42.170816Z"
    },
    "papermill": {
     "duration": 7.754129,
     "end_time": "2024-09-30T17:00:42.173573",
     "exception": false,
     "start_time": "2024-09-30T17:00:34.419444",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# checkpoint_callback_total6 = ModelCheckpoint('best_model_6.keras', \n",
    "#                                       monitor='val_loss', \n",
    "#                                       save_best_only=True, \n",
    "#                                       mode='min', \n",
    "#                                       verbose=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "28747ca1",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-09-30T17:00:57.694673Z",
     "iopub.status.busy": "2024-09-30T17:00:57.693785Z",
     "iopub.status.idle": "2024-09-30T17:39:22.451416Z",
     "shell.execute_reply": "2024-09-30T17:39:22.450320Z"
    },
    "papermill": {
     "duration": 2312.495013,
     "end_time": "2024-09-30T17:39:22.453553",
     "exception": false,
     "start_time": "2024-09-30T17:00:49.958540",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/3\n",
      "\u001b[1m9194/9194\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 78ms/step - accuracy: 0.6290 - loss: 1.0319\n",
      "Epoch 1: val_loss did not improve from 1.25157\n",
      "\u001b[1m9194/9194\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m768s\u001b[0m 84ms/step - accuracy: 0.6290 - loss: 1.0319 - val_accuracy: 0.5614 - val_loss: 1.2569\n",
      "Epoch 2/3\n",
      "\u001b[1m9194/9194\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 78ms/step - accuracy: 0.6325 - loss: 1.0216\n",
      "Epoch 2: val_loss did not improve from 1.25157\n",
      "\u001b[1m9194/9194\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m768s\u001b[0m 83ms/step - accuracy: 0.6325 - loss: 1.0216 - val_accuracy: 0.5606 - val_loss: 1.2518\n",
      "Epoch 3/3\n",
      "\u001b[1m9194/9194\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 78ms/step - accuracy: 0.6350 - loss: 1.0155\n",
      "Epoch 3: val_loss improved from 1.25157 to 1.25037, saving model to best_model_6.keras\n",
      "\u001b[1m9194/9194\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m768s\u001b[0m 84ms/step - accuracy: 0.6350 - loss: 1.0155 - val_accuracy: 0.5605 - val_loss: 1.2504\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.src.callbacks.history.History at 0x7d1a0aec6c50>"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Train the model\n",
    "model_6.fit(X_train_3, \n",
    "            y_train_3, \n",
    "            epochs=3,      \n",
    "            batch_size=256,                     \n",
    "            validation_data=(X_val_3, y_val_3), \n",
    "            verbose=1,\n",
    "            callbacks=[checkpoint_callback_6,early_stopping_callback])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b396f3b5",
   "metadata": {
    "papermill": {
     "duration": 10.318176,
     "end_time": "2024-09-30T17:39:43.103433",
     "exception": false,
     "start_time": "2024-09-30T17:39:32.785257",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Complete Training data\n",
    "# randome shuffling "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "46b17f52",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-09-30T17:40:03.412745Z",
     "iopub.status.busy": "2024-09-30T17:40:03.411850Z",
     "iopub.status.idle": "2024-09-30T17:40:03.501110Z",
     "shell.execute_reply": "2024-09-30T17:40:03.500199Z"
    },
    "papermill": {
     "duration": 10.178362,
     "end_time": "2024-09-30T17:40:03.503105",
     "exception": false,
     "start_time": "2024-09-30T17:39:53.324743",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\">Model: \"sequential_1\"</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1mModel: \"sequential_1\"\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┓\n",
       "┃<span style=\"font-weight: bold\"> Layer (type)                    </span>┃<span style=\"font-weight: bold\"> Output Shape           </span>┃<span style=\"font-weight: bold\">       Param # </span>┃\n",
       "┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━┩\n",
       "│ embedding_1 (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Embedding</span>)         │ ?                      │   <span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> (unbuilt) │\n",
       "├─────────────────────────────────┼────────────────────────┼───────────────┤\n",
       "│ bidirectional_3 (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Bidirectional</span>) │ ?                      │   <span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> (unbuilt) │\n",
       "├─────────────────────────────────┼────────────────────────┼───────────────┤\n",
       "│ dropout_3 (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Dropout</span>)             │ ?                      │   <span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> (unbuilt) │\n",
       "├─────────────────────────────────┼────────────────────────┼───────────────┤\n",
       "│ bidirectional_4 (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Bidirectional</span>) │ ?                      │   <span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> (unbuilt) │\n",
       "├─────────────────────────────────┼────────────────────────┼───────────────┤\n",
       "│ dropout_4 (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Dropout</span>)             │ ?                      │   <span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> (unbuilt) │\n",
       "├─────────────────────────────────┼────────────────────────┼───────────────┤\n",
       "│ bidirectional_5 (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Bidirectional</span>) │ ?                      │   <span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> (unbuilt) │\n",
       "├─────────────────────────────────┼────────────────────────┼───────────────┤\n",
       "│ dropout_5 (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Dropout</span>)             │ ?                      │   <span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> (unbuilt) │\n",
       "├─────────────────────────────────┼────────────────────────┼───────────────┤\n",
       "│ attention_layer_1               │ ?                      │   <span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> (unbuilt) │\n",
       "│ (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">AttentionLayer</span>)                │                        │               │\n",
       "├─────────────────────────────────┼────────────────────────┼───────────────┤\n",
       "│ dense_1 (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Dense</span>)                 │ ?                      │   <span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> (unbuilt) │\n",
       "└─────────────────────────────────┴────────────────────────┴───────────────┘\n",
       "</pre>\n"
      ],
      "text/plain": [
       "┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┓\n",
       "┃\u001b[1m \u001b[0m\u001b[1mLayer (type)                   \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mOutput Shape          \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1m      Param #\u001b[0m\u001b[1m \u001b[0m┃\n",
       "┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━┩\n",
       "│ embedding_1 (\u001b[38;5;33mEmbedding\u001b[0m)         │ ?                      │   \u001b[38;5;34m0\u001b[0m (unbuilt) │\n",
       "├─────────────────────────────────┼────────────────────────┼───────────────┤\n",
       "│ bidirectional_3 (\u001b[38;5;33mBidirectional\u001b[0m) │ ?                      │   \u001b[38;5;34m0\u001b[0m (unbuilt) │\n",
       "├─────────────────────────────────┼────────────────────────┼───────────────┤\n",
       "│ dropout_3 (\u001b[38;5;33mDropout\u001b[0m)             │ ?                      │   \u001b[38;5;34m0\u001b[0m (unbuilt) │\n",
       "├─────────────────────────────────┼────────────────────────┼───────────────┤\n",
       "│ bidirectional_4 (\u001b[38;5;33mBidirectional\u001b[0m) │ ?                      │   \u001b[38;5;34m0\u001b[0m (unbuilt) │\n",
       "├─────────────────────────────────┼────────────────────────┼───────────────┤\n",
       "│ dropout_4 (\u001b[38;5;33mDropout\u001b[0m)             │ ?                      │   \u001b[38;5;34m0\u001b[0m (unbuilt) │\n",
       "├─────────────────────────────────┼────────────────────────┼───────────────┤\n",
       "│ bidirectional_5 (\u001b[38;5;33mBidirectional\u001b[0m) │ ?                      │   \u001b[38;5;34m0\u001b[0m (unbuilt) │\n",
       "├─────────────────────────────────┼────────────────────────┼───────────────┤\n",
       "│ dropout_5 (\u001b[38;5;33mDropout\u001b[0m)             │ ?                      │   \u001b[38;5;34m0\u001b[0m (unbuilt) │\n",
       "├─────────────────────────────────┼────────────────────────┼───────────────┤\n",
       "│ attention_layer_1               │ ?                      │   \u001b[38;5;34m0\u001b[0m (unbuilt) │\n",
       "│ (\u001b[38;5;33mAttentionLayer\u001b[0m)                │                        │               │\n",
       "├─────────────────────────────────┼────────────────────────┼───────────────┤\n",
       "│ dense_1 (\u001b[38;5;33mDense\u001b[0m)                 │ ?                      │   \u001b[38;5;34m0\u001b[0m (unbuilt) │\n",
       "└─────────────────────────────────┴────────────────────────┴───────────────┘\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\"> Total params: </span><span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> (0.00 B)\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1m Total params: \u001b[0m\u001b[38;5;34m0\u001b[0m (0.00 B)\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\"> Trainable params: </span><span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> (0.00 B)\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1m Trainable params: \u001b[0m\u001b[38;5;34m0\u001b[0m (0.00 B)\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\"> Non-trainable params: </span><span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> (0.00 B)\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1m Non-trainable params: \u001b[0m\u001b[38;5;34m0\u001b[0m (0.00 B)\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "from tensorflow.keras.models import Sequential\n",
    "from tensorflow.keras.layers import Embedding, Bidirectional, LSTM, Dense, Dropout, Concatenate, Layer\n",
    "from tensorflow.keras import Input, Model\n",
    "import keras\n",
    "# Define parameters\n",
    "vocab_size = len(char_to_index)  # Size of vocabulary (unique characters)\n",
    "embedding_dim = 50  # Dimension of character embeddings\n",
    "lstm_units = 256  # Number of units in LSTM layers\n",
    "num_layers = 3  # Number of BiLSTM layers\n",
    "dropout_rate = 0.1  # Dropout rate\n",
    "\n",
    "# Custom Attention Layer\n",
    "@keras.saving.register_keras_serializable()\n",
    "class AttentionLayer(Layer):\n",
    "    def __init__(self, **kwargs):\n",
    "        super(AttentionLayer, self).__init__(**kwargs)\n",
    "\n",
    "    def build(self, input_shape):\n",
    "        self.W = self.add_weight(name=\"att_weight\", shape=(input_shape[-1], 1),\n",
    "                                 initializer=\"normal\")\n",
    "        self.b = self.add_weight(name=\"att_bias\", shape=(input_shape[1], 1),\n",
    "                                 initializer=\"zeros\")\n",
    "\n",
    "        super(AttentionLayer, self).build(input_shape)\n",
    "\n",
    "    def call(self, x):\n",
    "        et = tf.squeeze(tf.tanh(tf.matmul(x, self.W) + self.b), axis=-1)\n",
    "        at = tf.nn.softmax(et)\n",
    "        at = tf.expand_dims(at, axis=-1)\n",
    "        output = x * at\n",
    "        return tf.reduce_sum(output, axis=1)\n",
    "\n",
    "# Define Sequential model\n",
    "model_complete_6 = Sequential()\n",
    "\n",
    "# Add layers to the model\n",
    "model_complete_6.add(Embedding(input_dim=vocab_size, output_dim=embedding_dim, input_length=max_length_3))\n",
    "\n",
    "# Add BiLSTM layers with Dropout\n",
    "for _ in range(num_layers):\n",
    "    model_complete_6.add(Bidirectional(LSTM(units=lstm_units, return_sequences=True)))\n",
    "    model_complete_6.add(Dropout(rate=dropout_rate))\n",
    "\n",
    "# Add Attention Layer\n",
    "model_complete_6.add(AttentionLayer())\n",
    "\n",
    "# Output layer\n",
    "model_complete_6.add(Dense(units=vocab_size, activation='softmax'))\n",
    "\n",
    "# Compile the model\n",
    "model_complete_6.compile(optimizer='adam',\n",
    "              loss='sparse_categorical_crossentropy',\n",
    "              metrics=['accuracy'])\n",
    "\n",
    "model_complete_6.summary()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "e7abbae8",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-09-30T17:40:23.901247Z",
     "iopub.status.busy": "2024-09-30T17:40:23.900325Z",
     "iopub.status.idle": "2024-09-30T19:58:26.767523Z",
     "shell.execute_reply": "2024-09-30T19:58:26.766521Z"
    },
    "papermill": {
     "duration": 8293.076213,
     "end_time": "2024-09-30T19:58:26.769891",
     "exception": false,
     "start_time": "2024-09-30T17:40:13.693678",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/8\n",
      "\u001b[1m13135/13135\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1025s\u001b[0m 78ms/step - accuracy: 0.3869 - loss: 1.9100\n",
      "Epoch 2/8\n",
      "\u001b[1m13135/13135\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1024s\u001b[0m 78ms/step - accuracy: 0.5363 - loss: 1.3367\n",
      "Epoch 3/8\n",
      "\u001b[1m13135/13135\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1030s\u001b[0m 78ms/step - accuracy: 0.5664 - loss: 1.2294\n",
      "Epoch 4/8\n",
      "\u001b[1m13135/13135\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1035s\u001b[0m 79ms/step - accuracy: 0.5820 - loss: 1.1727\n",
      "Epoch 5/8\n",
      "\u001b[1m13135/13135\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1042s\u001b[0m 79ms/step - accuracy: 0.5911 - loss: 1.1403\n",
      "Epoch 6/8\n",
      "\u001b[1m13135/13135\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1042s\u001b[0m 79ms/step - accuracy: 0.5981 - loss: 1.1165\n",
      "Epoch 7/8\n",
      "\u001b[1m13135/13135\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1041s\u001b[0m 79ms/step - accuracy: 0.6033 - loss: 1.0981\n",
      "Epoch 8/8\n",
      "\u001b[1m13135/13135\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1036s\u001b[0m 79ms/step - accuracy: 0.6076 - loss: 1.0836\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.src.callbacks.history.History at 0x7d19e7416b60>"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.utils import shuffle\n",
    "\n",
    "X_train_complete_3, y_train_complete_3 = shuffle( X_padded_3, y_padded_3 )\n",
    "\n",
    "# Train the model\n",
    "model_complete_6.fit( X_train_complete_3, \n",
    "            y_train_complete_3,\n",
    "            epochs=8, \n",
    "            batch_size=256, \n",
    "#             validation_data=(X_val_3, y_val_3), \n",
    "            verbose=1,\n",
    "#             callbacks=[checkpoint_callback_6,early_stopping_callback]\n",
    "           )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "3c031679",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-09-30T19:59:09.559300Z",
     "iopub.status.busy": "2024-09-30T19:59:09.558189Z",
     "iopub.status.idle": "2024-09-30T19:59:09.739066Z",
     "shell.execute_reply": "2024-09-30T19:59:09.738117Z"
    },
    "papermill": {
     "duration": 21.749587,
     "end_time": "2024-09-30T19:59:09.741651",
     "exception": false,
     "start_time": "2024-09-30T19:58:47.992064",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "model_complete_6.save('best_model_6_complete.keras')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "b8b4f9cb",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-09-30T19:59:52.345017Z",
     "iopub.status.busy": "2024-09-30T19:59:52.344015Z",
     "iopub.status.idle": "2024-09-30T19:59:53.523670Z",
     "shell.execute_reply": "2024-09-30T19:59:53.522607Z"
    },
    "papermill": {
     "duration": 22.576708,
     "end_time": "2024-09-30T19:59:53.525919",
     "exception": false,
     "start_time": "2024-09-30T19:59:30.949211",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "here pliss work i am pissed \n"
     ]
    }
   ],
   "source": [
    "model_complete_6_recon = tf.keras.models.load_model('best_model_6_complete.keras')\n",
    "from IPython.display import FileLink\n",
    "FileLink(r'best_model_6_complete.keras')\n",
    "print(\"here pliss work i am pissed \")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "da2bfe3d",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-09-30T20:00:36.125699Z",
     "iopub.status.busy": "2024-09-30T20:00:36.125258Z",
     "iopub.status.idle": "2024-09-30T20:00:36.129870Z",
     "shell.execute_reply": "2024-09-30T20:00:36.128828Z"
    },
    "papermill": {
     "duration": 21.684258,
     "end_time": "2024-09-30T20:00:36.132096",
     "exception": false,
     "start_time": "2024-09-30T20:00:14.447838",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# model_complete_6_recon = tf.keras.models.load_model('best_model_6_complete.keras')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "b6d87c03",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-09-30T20:01:19.780561Z",
     "iopub.status.busy": "2024-09-30T20:01:19.780117Z",
     "iopub.status.idle": "2024-09-30T20:01:19.784706Z",
     "shell.execute_reply": "2024-09-30T20:01:19.783573Z"
    },
    "papermill": {
     "duration": 21.938923,
     "end_time": "2024-09-30T20:01:19.786992",
     "exception": false,
     "start_time": "2024-09-30T20:00:57.848069",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "## load model \n",
    "# model_6_recon = tf.keras.models.load_model('best_model_6.keras')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "34de4ce0",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-09-30T20:02:02.740160Z",
     "iopub.status.busy": "2024-09-30T20:02:02.739745Z",
     "iopub.status.idle": "2024-09-30T20:04:13.527576Z",
     "shell.execute_reply": "2024-09-30T20:04:13.526431Z"
    },
    "papermill": {
     "duration": 165.589274,
     "end_time": "2024-09-30T20:04:26.852971",
     "exception": false,
     "start_time": "2024-09-30T20:01:41.263697",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[1m15762/15762\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m131s\u001b[0m 8ms/step - accuracy: 0.6345 - loss: 1.0027\n",
      "validation set Accuracy: 63.64%\n"
     ]
    }
   ],
   "source": [
    "# Evaluate on test set\n",
    "loss, accuracy = model_complete_6_recon.evaluate(X_test_3, y_test_3, verbose=1)\n",
    "print(f\"validation set Accuracy: {accuracy*100:.2f}%\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "235d8b07",
   "metadata": {
    "papermill": {
     "duration": 21.724051,
     "end_time": "2024-09-30T20:05:10.168178",
     "exception": false,
     "start_time": "2024-09-30T20:04:48.444127",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "gpuType": "T4",
   "provenance": []
  },
  "kaggle": {
   "accelerator": "nvidiaTeslaT4",
   "dataSources": [
    {
     "datasetId": 5241531,
     "sourceId": 8732277,
     "sourceType": "datasetVersion"
    },
    {
     "datasetId": 5261218,
     "sourceId": 8757543,
     "sourceType": "datasetVersion"
    },
    {
     "datasetId": 5262822,
     "sourceId": 8759694,
     "sourceType": "datasetVersion"
    },
    {
     "datasetId": 5262852,
     "sourceId": 8759733,
     "sourceType": "datasetVersion"
    },
    {
     "datasetId": 5791774,
     "sourceId": 9514166,
     "sourceType": "datasetVersion"
    },
    {
     "sourceId": 184902460,
     "sourceType": "kernelVersion"
    }
   ],
   "dockerImageVersionId": 30776,
   "isGpuEnabled": true,
   "isInternetEnabled": true,
   "language": "python",
   "sourceType": "notebook"
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.14"
  },
  "papermill": {
   "default_parameters": {},
   "duration": 18889.447737,
   "end_time": "2024-09-30T20:05:34.717849",
   "environment_variables": {},
   "exception": null,
   "input_path": "__notebook__.ipynb",
   "output_path": "__notebook__.ipynb",
   "parameters": {},
   "start_time": "2024-09-30T14:50:45.270112",
   "version": "2.6.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}