t5_data_prep.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "4d35c170-601d-4e56-9f96-31601d36d737",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "hello, world\n"
     ]
    }
   ],
   "source": [
    "codequasprint('hello, world')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "b583c69f-368e-42f9-8d33-39972ab72164",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Sat Jan 27 23:32:31 2024       \n",
      "+---------------------------------------------------------------------------------------+\n",
      "| NVIDIA-SMI 535.129.03             Driver Version: 535.129.03   CUDA Version: 12.2     |\n",
      "|-----------------------------------------+----------------------+----------------------+\n",
      "| GPU  Name                 Persistence-M | Bus-Id        Disp.A | Volatile Uncorr. ECC |\n",
      "| Fan  Temp   Perf          Pwr:Usage/Cap |         Memory-Usage | GPU-Util  Compute M. |\n",
      "|                                         |                      |               MIG M. |\n",
      "|=========================================+======================+======================|\n",
      "|   0  NVIDIA A10                     On  | 00000000:07:00.0 Off |                    0 |\n",
      "|  0%   25C    P8               9W / 150W |      4MiB / 23028MiB |      0%      Default |\n",
      "|                                         |                      |                  N/A |\n",
      "+-----------------------------------------+----------------------+----------------------+\n",
      "                                                                                         \n",
      "+---------------------------------------------------------------------------------------+\n",
      "| Processes:                                                                            |\n",
      "|  GPU   GI   CI        PID   Type   Process name                            GPU Memory |\n",
      "|        ID   ID                                                             Usage      |\n",
      "|=======================================================================================|\n",
      "|  No running processes found                                                           |\n",
      "+---------------------------------------------------------------------------------------+\n"
     ]
    }
   ],
   "source": [
    "!nvidia-smi"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "289012e4-c77f-4a75-99ea-8001b0b73b42",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Defaulting to user installation because normal site-packages is not writeable\n",
      "Requirement already satisfied: transformers in /home/ubuntu/.local/lib/python3.10/site-packages (4.37.1)\n",
      "Requirement already satisfied: filelock in /usr/lib/python3/dist-packages (from transformers) (3.6.0)\n",
      "Requirement already satisfied: huggingface-hub<1.0,>=0.19.3 in /home/ubuntu/.local/lib/python3.10/site-packages (from transformers) (0.20.3)\n",
      "Requirement already satisfied: numpy>=1.17 in /home/ubuntu/.local/lib/python3.10/site-packages (from transformers) (1.25.2)\n",
      "Requirement already satisfied: packaging>=20.0 in /usr/lib/python3/dist-packages (from transformers) (21.3)\n",
      "Requirement already satisfied: pyyaml>=5.1 in /usr/lib/python3/dist-packages (from transformers) (5.4.1)\n",
      "Requirement already satisfied: regex!=2019.12.17 in /home/ubuntu/.local/lib/python3.10/site-packages (from transformers) (2023.12.25)\n",
      "Requirement already satisfied: requests in /home/ubuntu/.local/lib/python3.10/site-packages (from transformers) (2.31.0)\n",
      "Requirement already satisfied: tokenizers<0.19,>=0.14 in /home/ubuntu/.local/lib/python3.10/site-packages (from transformers) (0.15.1)\n",
      "Requirement already satisfied: safetensors>=0.3.1 in /home/ubuntu/.local/lib/python3.10/site-packages (from transformers) (0.4.2)\n",
      "Requirement already satisfied: tqdm>=4.27 in /home/ubuntu/.local/lib/python3.10/site-packages (from transformers) (4.66.1)\n",
      "Requirement already satisfied: fsspec>=2023.5.0 in /home/ubuntu/.local/lib/python3.10/site-packages (from huggingface-hub<1.0,>=0.19.3->transformers) (2023.10.0)\n",
      "Requirement already satisfied: typing-extensions>=3.7.4.3 in /home/ubuntu/.local/lib/python3.10/site-packages (from huggingface-hub<1.0,>=0.19.3->transformers) (4.8.0)\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in /home/ubuntu/.local/lib/python3.10/site-packages (from requests->transformers) (3.3.2)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /usr/lib/python3/dist-packages (from requests->transformers) (3.3)\n",
      "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/lib/python3/dist-packages (from requests->transformers) (1.26.5)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /usr/lib/python3/dist-packages (from requests->transformers) (2020.6.20)\n",
      "\u001b[33mDEPRECATION: flatbuffers 1.12.1-git20200711.33e2d80-dfsg1-0.6 has a non-standard version number. pip 24.0 will enforce this behaviour change. A possible replacement is to upgrade to a newer version of flatbuffers or contact the author to suggest that they release a version with a conforming version number. Discussion can be found at https://github.com/pypa/pip/issues/12063\u001b[0m\u001b[33m\n",
      "\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.3.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m23.3.2\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpython3 -m pip install --upgrade pip\u001b[0m\n",
      "Defaulting to user installation because normal site-packages is not writeable\n",
      "Requirement already satisfied: pytorch_lightning==0.7.5 in /home/ubuntu/.local/lib/python3.10/site-packages (0.7.5)\n",
      "Requirement already satisfied: tqdm>=4.41.0 in /home/ubuntu/.local/lib/python3.10/site-packages (from pytorch_lightning==0.7.5) (4.66.1)\n",
      "Requirement already satisfied: numpy>=1.16.4 in /home/ubuntu/.local/lib/python3.10/site-packages (from pytorch_lightning==0.7.5) (1.25.2)\n",
      "Requirement already satisfied: torch>=1.1 in /usr/lib/python3/dist-packages (from pytorch_lightning==0.7.5) (2.0.1)\n",
      "Requirement already satisfied: tensorboard>=1.14 in /usr/lib/python3/dist-packages (from pytorch_lightning==0.7.5) (2.13.0)\n",
      "Requirement already satisfied: future>=0.17.1 in /usr/lib/python3/dist-packages (from pytorch_lightning==0.7.5) (0.18.2)\n",
      "\u001b[33mDEPRECATION: flatbuffers 1.12.1-git20200711.33e2d80-dfsg1-0.6 has a non-standard version number. pip 24.0 will enforce this behaviour change. A possible replacement is to upgrade to a newer version of flatbuffers or contact the author to suggest that they release a version with a conforming version number. Discussion can be found at https://github.com/pypa/pip/issues/12063\u001b[0m\u001b[33m\n",
      "\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.3.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m23.3.2\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpython3 -m pip install --upgrade pip\u001b[0m\n",
      "Defaulting to user installation because normal site-packages is not writeable\n",
      "Requirement already satisfied: datasets in /home/ubuntu/.local/lib/python3.10/site-packages (2.16.1)\n",
      "Requirement already satisfied: filelock in /usr/lib/python3/dist-packages (from datasets) (3.6.0)\n",
      "Requirement already satisfied: numpy>=1.17 in /home/ubuntu/.local/lib/python3.10/site-packages (from datasets) (1.25.2)\n",
      "Requirement already satisfied: pyarrow>=8.0.0 in /home/ubuntu/.local/lib/python3.10/site-packages (from datasets) (15.0.0)\n",
      "Requirement already satisfied: pyarrow-hotfix in /home/ubuntu/.local/lib/python3.10/site-packages (from datasets) (0.6)\n",
      "Requirement already satisfied: dill<0.3.8,>=0.3.0 in /home/ubuntu/.local/lib/python3.10/site-packages (from datasets) (0.3.7)\n",
      "Requirement already satisfied: pandas in /usr/lib/python3/dist-packages (from datasets) (1.3.5)\n",
      "Requirement already satisfied: requests>=2.19.0 in /home/ubuntu/.local/lib/python3.10/site-packages (from datasets) (2.31.0)\n",
      "Requirement already satisfied: tqdm>=4.62.1 in /home/ubuntu/.local/lib/python3.10/site-packages (from datasets) (4.66.1)\n",
      "Requirement already satisfied: xxhash in /home/ubuntu/.local/lib/python3.10/site-packages (from datasets) (3.4.1)\n",
      "Requirement already satisfied: multiprocess in /home/ubuntu/.local/lib/python3.10/site-packages (from datasets) (0.70.15)\n",
      "Requirement already satisfied: fsspec<=2023.10.0,>=2023.1.0 in /home/ubuntu/.local/lib/python3.10/site-packages (from fsspec[http]<=2023.10.0,>=2023.1.0->datasets) (2023.10.0)\n",
      "Requirement already satisfied: aiohttp in /home/ubuntu/.local/lib/python3.10/site-packages (from datasets) (3.9.1)\n",
      "Requirement already satisfied: huggingface-hub>=0.19.4 in /home/ubuntu/.local/lib/python3.10/site-packages (from datasets) (0.20.3)\n",
      "Requirement already satisfied: packaging in /usr/lib/python3/dist-packages (from datasets) (21.3)\n",
      "Requirement already satisfied: pyyaml>=5.1 in /usr/lib/python3/dist-packages (from datasets) (5.4.1)\n",
      "Requirement already satisfied: attrs>=17.3.0 in /home/ubuntu/.local/lib/python3.10/site-packages (from aiohttp->datasets) (23.1.0)\n",
      "Requirement already satisfied: multidict<7.0,>=4.5 in /home/ubuntu/.local/lib/python3.10/site-packages (from aiohttp->datasets) (6.0.4)\n",
      "Requirement already satisfied: yarl<2.0,>=1.0 in /home/ubuntu/.local/lib/python3.10/site-packages (from aiohttp->datasets) (1.9.4)\n",
      "Requirement already satisfied: frozenlist>=1.1.1 in /home/ubuntu/.local/lib/python3.10/site-packages (from aiohttp->datasets) (1.4.1)\n",
      "Requirement already satisfied: aiosignal>=1.1.2 in /home/ubuntu/.local/lib/python3.10/site-packages (from aiohttp->datasets) (1.3.1)\n",
      "Requirement already satisfied: async-timeout<5.0,>=4.0 in /home/ubuntu/.local/lib/python3.10/site-packages (from aiohttp->datasets) (4.0.3)\n",
      "Requirement already satisfied: typing-extensions>=3.7.4.3 in /home/ubuntu/.local/lib/python3.10/site-packages (from huggingface-hub>=0.19.4->datasets) (4.8.0)\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in /home/ubuntu/.local/lib/python3.10/site-packages (from requests>=2.19.0->datasets) (3.3.2)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /usr/lib/python3/dist-packages (from requests>=2.19.0->datasets) (3.3)\n",
      "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/lib/python3/dist-packages (from requests>=2.19.0->datasets) (1.26.5)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /usr/lib/python3/dist-packages (from requests>=2.19.0->datasets) (2020.6.20)\n",
      "\u001b[33mDEPRECATION: flatbuffers 1.12.1-git20200711.33e2d80-dfsg1-0.6 has a non-standard version number. pip 24.0 will enforce this behaviour change. A possible replacement is to upgrade to a newer version of flatbuffers or contact the author to suggest that they release a version with a conforming version number. Discussion can be found at https://github.com/pypa/pip/issues/12063\u001b[0m\u001b[33m\n",
      "\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.3.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m23.3.2\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpython3 -m pip install --upgrade pip\u001b[0m\n",
      "Defaulting to user installation because normal site-packages is not writeable\n",
      "Requirement already satisfied: nltk in /home/ubuntu/.local/lib/python3.10/site-packages (3.8.1)\n",
      "Requirement already satisfied: click in /usr/lib/python3/dist-packages (from nltk) (8.0.3)\n",
      "Requirement already satisfied: joblib in /usr/lib/python3/dist-packages (from nltk) (0.17.0)\n",
      "Requirement already satisfied: regex>=2021.8.3 in /home/ubuntu/.local/lib/python3.10/site-packages (from nltk) (2023.12.25)\n",
      "Requirement already satisfied: tqdm in /home/ubuntu/.local/lib/python3.10/site-packages (from nltk) (4.66.1)\n",
      "\u001b[33mDEPRECATION: flatbuffers 1.12.1-git20200711.33e2d80-dfsg1-0.6 has a non-standard version number. pip 24.0 will enforce this behaviour change. A possible replacement is to upgrade to a newer version of flatbuffers or contact the author to suggest that they release a version with a conforming version number. Discussion can be found at https://github.com/pypa/pip/issues/12063\u001b[0m\u001b[33m\n",
      "\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.3.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m23.3.2\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpython3 -m pip install --upgrade pip\u001b[0m\n",
      "Defaulting to user installation because normal site-packages is not writeable\n",
      "Requirement already satisfied: SentencePiece in /home/ubuntu/.local/lib/python3.10/site-packages (0.1.99)\n"
     ]
    }
   ],
   "source": [
    "!pip install transformers\n",
    "!pip install pytorch_lightning==0.7.5\n",
    "!pip install datasets\n",
    "!pip install nltk\n",
    "!pip install SentencePiece"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "7b238b11",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "hello, world\n"
     ]
    }
   ],
   "source": [
    "print('hello, world')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "7d712a4d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Sat Jan 27 22:37:53 2024       \n",
      "+---------------------------------------------------------------------------------------+\n",
      "| NVIDIA-SMI 535.129.03             Driver Version: 535.129.03   CUDA Version: 12.2     |\n",
      "|-----------------------------------------+----------------------+----------------------+\n",
      "| GPU  Name                 Persistence-M | Bus-Id        Disp.A | Volatile Uncorr. ECC |\n",
      "| Fan  Temp   Perf          Pwr:Usage/Cap |         Memory-Usage | GPU-Util  Compute M. |\n",
      "|                                         |                      |               MIG M. |\n",
      "|=========================================+======================+======================|\n",
      "|   0  NVIDIA A10                     On  | 00000000:07:00.0 Off |                    0 |\n",
      "|  0%   25C    P8               9W / 150W |      4MiB / 23028MiB |      0%      Default |\n",
      "|                                         |                      |                  N/A |\n",
      "+-----------------------------------------+----------------------+----------------------+\n",
      "                                                                                         \n",
      "+---------------------------------------------------------------------------------------+\n",
      "| Processes:                                                                            |\n",
      "|  GPU   GI   CI        PID   Type   Process name                            GPU Memory |\n",
      "|        ID   ID                                                             Usage      |\n",
      "|=======================================================================================|\n",
      "|  No running processes found                                                           |\n",
      "+---------------------------------------------------------------------------------------+\n"
     ]
    }
   ],
   "source": [
    "!nvidia-smi"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3780a3df",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Defaulting to user installation because normal site-packages is not writeable\n",
      "Requirement already satisfied: transformers in /home/ubuntu/.local/lib/python3.10/site-packages (4.37.1)\n",
      "Requirement already satisfied: filelock in /usr/lib/python3/dist-packages (from transformers) (3.6.0)\n",
      "Requirement already satisfied: huggingface-hub<1.0,>=0.19.3 in /home/ubuntu/.local/lib/python3.10/site-packages (from transformers) (0.20.3)\n",
      "Requirement already satisfied: numpy>=1.17 in /home/ubuntu/.local/lib/python3.10/site-packages (from transformers) (1.25.2)\n",
      "Requirement already satisfied: packaging>=20.0 in /usr/lib/python3/dist-packages (from transformers) (21.3)\n",
      "Requirement already satisfied: pyyaml>=5.1 in /usr/lib/python3/dist-packages (from transformers) (5.4.1)\n",
      "Requirement already satisfied: regex!=2019.12.17 in /home/ubuntu/.local/lib/python3.10/site-packages (from transformers) (2023.12.25)\n",
      "Requirement already satisfied: requests in /home/ubuntu/.local/lib/python3.10/site-packages (from transformers) (2.31.0)\n",
      "Requirement already satisfied: tokenizers<0.19,>=0.14 in /home/ubuntu/.local/lib/python3.10/site-packages (from transformers) (0.15.1)\n",
      "Requirement already satisfied: safetensors>=0.3.1 in /home/ubuntu/.local/lib/python3.10/site-packages (from transformers) (0.4.2)\n",
      "Requirement already satisfied: tqdm>=4.27 in /home/ubuntu/.local/lib/python3.10/site-packages (from transformers) (4.66.1)\n",
      "Requirement already satisfied: fsspec>=2023.5.0 in /home/ubuntu/.local/lib/python3.10/site-packages (from huggingface-hub<1.0,>=0.19.3->transformers) (2023.10.0)\n",
      "Requirement already satisfied: typing-extensions>=3.7.4.3 in /home/ubuntu/.local/lib/python3.10/site-packages (from huggingface-hub<1.0,>=0.19.3->transformers) (4.8.0)\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in /home/ubuntu/.local/lib/python3.10/site-packages (from requests->transformers) (3.3.2)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /usr/lib/python3/dist-packages (from requests->transformers) (3.3)\n",
      "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/lib/python3/dist-packages (from requests->transformers) (1.26.5)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /usr/lib/python3/dist-packages (from requests->transformers) (2020.6.20)\n",
      "\u001b[33mDEPRECATION: flatbuffers 1.12.1-git20200711.33e2d80-dfsg1-0.6 has a non-standard version number. pip 24.0 will enforce this behaviour change. A possible replacement is to upgrade to a newer version of flatbuffers or contact the author to suggest that they release a version with a conforming version number. Discussion can be found at https://github.com/pypa/pip/issues/12063\u001b[0m\u001b[33m\n",
      "\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.3.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m23.3.2\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpython3 -m pip install --upgrade pip\u001b[0m\n",
      "Defaulting to user installation because normal site-packages is not writeable\n",
      "Requirement already satisfied: pytorch_lightning==0.7.5 in /home/ubuntu/.local/lib/python3.10/site-packages (0.7.5)\n",
      "Requirement already satisfied: tqdm>=4.41.0 in /home/ubuntu/.local/lib/python3.10/site-packages (from pytorch_lightning==0.7.5) (4.66.1)\n",
      "Requirement already satisfied: numpy>=1.16.4 in /home/ubuntu/.local/lib/python3.10/site-packages (from pytorch_lightning==0.7.5) (1.25.2)\n",
      "Requirement already satisfied: torch>=1.1 in /usr/lib/python3/dist-packages (from pytorch_lightning==0.7.5) (2.0.1)\n",
      "Requirement already satisfied: tensorboard>=1.14 in /usr/lib/python3/dist-packages (from pytorch_lightning==0.7.5) (2.13.0)\n",
      "Requirement already satisfied: future>=0.17.1 in /usr/lib/python3/dist-packages (from pytorch_lightning==0.7.5) (0.18.2)\n",
      "\u001b[33mDEPRECATION: flatbuffers 1.12.1-git20200711.33e2d80-dfsg1-0.6 has a non-standard version number. pip 24.0 will enforce this behaviour change. A possible replacement is to upgrade to a newer version of flatbuffers or contact the author to suggest that they release a version with a conforming version number. Discussion can be found at https://github.com/pypa/pip/issues/12063\u001b[0m\u001b[33m\n",
      "\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.3.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m23.3.2\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpython3 -m pip install --upgrade pip\u001b[0m\n",
      "Defaulting to user installation because normal site-packages is not writeable\n",
      "Requirement already satisfied: datasets in /home/ubuntu/.local/lib/python3.10/site-packages (2.16.1)\n",
      "Requirement already satisfied: filelock in /usr/lib/python3/dist-packages (from datasets) (3.6.0)\n",
      "Requirement already satisfied: numpy>=1.17 in /home/ubuntu/.local/lib/python3.10/site-packages (from datasets) (1.25.2)\n",
      "Requirement already satisfied: pyarrow>=8.0.0 in /home/ubuntu/.local/lib/python3.10/site-packages (from datasets) (15.0.0)\n",
      "Requirement already satisfied: pyarrow-hotfix in /home/ubuntu/.local/lib/python3.10/site-packages (from datasets) (0.6)\n",
      "Requirement already satisfied: dill<0.3.8,>=0.3.0 in /home/ubuntu/.local/lib/python3.10/site-packages (from datasets) (0.3.7)\n",
      "Requirement already satisfied: pandas in /usr/lib/python3/dist-packages (from datasets) (1.3.5)\n",
      "Requirement already satisfied: requests>=2.19.0 in /home/ubuntu/.local/lib/python3.10/site-packages (from datasets) (2.31.0)\n",
      "Requirement already satisfied: tqdm>=4.62.1 in /home/ubuntu/.local/lib/python3.10/site-packages (from datasets) (4.66.1)\n",
      "Requirement already satisfied: xxhash in /home/ubuntu/.local/lib/python3.10/site-packages (from datasets) (3.4.1)\n",
      "Requirement already satisfied: multiprocess in /home/ubuntu/.local/lib/python3.10/site-packages (from datasets) (0.70.15)\n",
      "Requirement already satisfied: fsspec<=2023.10.0,>=2023.1.0 in /home/ubuntu/.local/lib/python3.10/site-packages (from fsspec[http]<=2023.10.0,>=2023.1.0->datasets) (2023.10.0)\n",
      "Requirement already satisfied: aiohttp in /home/ubuntu/.local/lib/python3.10/site-packages (from datasets) (3.9.1)\n",
      "Requirement already satisfied: huggingface-hub>=0.19.4 in /home/ubuntu/.local/lib/python3.10/site-packages (from datasets) (0.20.3)\n",
      "Requirement already satisfied: packaging in /usr/lib/python3/dist-packages (from datasets) (21.3)\n",
      "Requirement already satisfied: pyyaml>=5.1 in /usr/lib/python3/dist-packages (from datasets) (5.4.1)\n",
      "Requirement already satisfied: attrs>=17.3.0 in /home/ubuntu/.local/lib/python3.10/site-packages (from aiohttp->datasets) (23.1.0)\n",
      "Requirement already satisfied: multidict<7.0,>=4.5 in /home/ubuntu/.local/lib/python3.10/site-packages (from aiohttp->datasets) (6.0.4)\n",
      "Requirement already satisfied: yarl<2.0,>=1.0 in /home/ubuntu/.local/lib/python3.10/site-packages (from aiohttp->datasets) (1.9.4)\n",
      "Requirement already satisfied: frozenlist>=1.1.1 in /home/ubuntu/.local/lib/python3.10/site-packages (from aiohttp->datasets) (1.4.1)\n",
      "Requirement already satisfied: aiosignal>=1.1.2 in /home/ubuntu/.local/lib/python3.10/site-packages (from aiohttp->datasets) (1.3.1)\n",
      "Requirement already satisfied: async-timeout<5.0,>=4.0 in /home/ubuntu/.local/lib/python3.10/site-packages (from aiohttp->datasets) (4.0.3)\n",
      "Requirement already satisfied: typing-extensions>=3.7.4.3 in /home/ubuntu/.local/lib/python3.10/site-packages (from huggingface-hub>=0.19.4->datasets) (4.8.0)\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in /home/ubuntu/.local/lib/python3.10/site-packages (from requests>=2.19.0->datasets) (3.3.2)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /usr/lib/python3/dist-packages (from requests>=2.19.0->datasets) (3.3)\n",
      "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/lib/python3/dist-packages (from requests>=2.19.0->datasets) (1.26.5)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /usr/lib/python3/dist-packages (from requests>=2.19.0->datasets) (2020.6.20)\n",
      "\u001b[33mDEPRECATION: flatbuffers 1.12.1-git20200711.33e2d80-dfsg1-0.6 has a non-standard version number. pip 24.0 will enforce this behaviour change. A possible replacement is to upgrade to a newer version of flatbuffers or contact the author to suggest that they release a version with a conforming version number. Discussion can be found at https://github.com/pypa/pip/issues/12063\u001b[0m\u001b[33m\n",
      "\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.3.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m23.3.2\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpython3 -m pip install --upgrade pip\u001b[0m\n",
      "Defaulting to user installation because normal site-packages is not writeable\n",
      "Requirement already satisfied: nltk in /home/ubuntu/.local/lib/python3.10/site-packages (3.8.1)\n",
      "Requirement already satisfied: click in /usr/lib/python3/dist-packages (from nltk) (8.0.3)\n",
      "Requirement already satisfied: joblib in /usr/lib/python3/dist-packages (from nltk) (0.17.0)\n",
      "Requirement already satisfied: regex>=2021.8.3 in /home/ubuntu/.local/lib/python3.10/site-packages (from nltk) (2023.12.25)\n",
      "Requirement already satisfied: tqdm in /home/ubuntu/.local/lib/python3.10/site-packages (from nltk) (4.66.1)\n",
      "\u001b[33mDEPRECATION: flatbuffers 1.12.1-git20200711.33e2d80-dfsg1-0.6 has a non-standard version number. pip 24.0 will enforce this behaviour change. A possible replacement is to upgrade to a newer version of flatbuffers or contact the author to suggest that they release a version with a conforming version number. Discussion can be found at https://github.com/pypa/pip/issues/12063\u001b[0m\u001b[33m\n",
      "\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.3.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m23.3.2\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpython3 -m pip install --upgrade pip\u001b[0m\n",
      "Defaulting to user installation because normal site-packages is not writeable\n",
      "Requirement already satisfied: SentencePiece in /home/ubuntu/.local/lib/python3.10/site-packages (0.1.99)\n"
     ]
    }
   ],
   "source": [
    "!pip install transformers\n",
    "!pip install pytorch_lightning==0.7.5\n",
    "!pip install datasets\n",
    "!pip install nltk\n",
    "!pip install SentencePiece"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "0054749a",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package punkt to /home/ubuntu/nltk_data...\n",
      "[nltk_data]   Package punkt is already up-to-date!\n"
     ]
    }
   ],
   "source": [
    "import argparse\n",
    "import glob\n",
    "import os\n",
    "import json\n",
    "import time\n",
    "import logging\n",
    "import random\n",
    "import re\n",
    "from itertools import chain\n",
    "from string import punctuation\n",
    "\n",
    "import nltk\n",
    "nltk.download('punkt')\n",
    "from nltk.tokenize import sent_tokenize\n",
    "\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import torch\n",
    "from torch.utils.data import Dataset, DataLoader\n",
    "import pytorch_lightning as pl\n",
    "import datasets\n",
    "\n",
    "from transformers import (\n",
    "    AdamW,\n",
    "    T5ForConditionalGeneration,\n",
    "    T5Tokenizer,\n",
    "    get_linear_schedule_with_warmup\n",
    ")\n",
    "\n",
    "hf_spider_ds = 'semiotic/spider_dataset_tuning'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "b9723e88-79f9-4d10-a837-e41c78c16c67",
   "metadata": {},
   "outputs": [],
   "source": [
    "def set_seed(seed):\n",
    "  random.seed(seed)\n",
    "  np.random.seed(seed)\n",
    "  torch.manual_seed(seed)\n",
    "  if torch.cuda.is_available():\n",
    "    torch.cuda.manual_seed_all(seed)\n",
    "\n",
    "set_seed(42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "c7c42db7-5779-46fb-a738-0b494e00e587",
   "metadata": {},
   "outputs": [],
   "source": [
    "class T5FineTuner(pl.LightningModule):\n",
    "  def __init__(self, hparams):\n",
    "    super(T5FineTuner, self).__init__()\n",
    "    self.hparams = hparams\n",
    "    \n",
    "    self.model = T5ForConditionalGeneration.from_pretrained(hparams.model_name_or_path)\n",
    "    self.tokenizer = T5Tokenizer.from_pretrained(hparams.tokenizer_name_or_path)\n",
    "  \n",
    "  def is_logger(self):\n",
    "    return self.trainer.proc_rank <= 0\n",
    "  \n",
    "  def forward(\n",
    "      self, input_ids, attention_mask=None, decoder_input_ids=None, decoder_attention_mask=None, labels=None, # lm_labels=None\n",
    "  ):\n",
    "    return self.model(\n",
    "        input_ids,\n",
    "        attention_mask=attention_mask,\n",
    "        decoder_input_ids=decoder_input_ids,\n",
    "        decoder_attention_mask=decoder_attention_mask,\n",
    "        labels=labels,\n",
    "        #lm_labels=lm_labels,\n",
    "    )\n",
    "\n",
    "  def _step(self, batch):\n",
    "    lm_labels = batch[\"target_ids\"]\n",
    "    lm_labels[lm_labels[:, :] == self.tokenizer.pad_token_id] = -100\n",
    "\n",
    "    outputs = self(\n",
    "        input_ids=batch[\"source_ids\"],\n",
    "        attention_mask=batch[\"source_mask\"],\n",
    "        #lm_labels=lm_labels,\n",
    "        labels=lm_labels,\n",
    "        decoder_attention_mask=batch['target_mask']\n",
    "    )\n",
    "\n",
    "    loss = outputs[0]\n",
    "\n",
    "    return loss\n",
    "\n",
    "  def training_step(self, batch, batch_idx):\n",
    "    loss = self._step(batch)\n",
    "\n",
    "    tensorboard_logs = {\"train_loss\": loss}\n",
    "    return {\"loss\": loss, \"log\": tensorboard_logs}\n",
    "  \n",
    "  def training_epoch_end(self, outputs):\n",
    "    avg_train_loss = torch.stack([x[\"loss\"] for x in outputs]).mean()\n",
    "    tensorboard_logs = {\"avg_train_loss\": avg_train_loss}\n",
    "    return {\"avg_train_loss\": avg_train_loss, \"log\": tensorboard_logs, 'progress_bar': tensorboard_logs}\n",
    "\n",
    "  def validation_step(self, batch, batch_idx):\n",
    "    loss = self._step(batch)\n",
    "    return {\"val_loss\": loss}\n",
    "  \n",
    "  def validation_epoch_end(self, outputs):\n",
    "    avg_loss = torch.stack([x[\"val_loss\"] for x in outputs]).mean()\n",
    "    tensorboard_logs = {\"val_loss\": avg_loss}\n",
    "    return {\"avg_val_loss\": avg_loss, \"log\": tensorboard_logs, 'progress_bar': tensorboard_logs}\n",
    "\n",
    "  def configure_optimizers(self):\n",
    "    \"Prepare optimizer and schedule (linear warmup and decay)\"\n",
    "\n",
    "    model = self.model\n",
    "    no_decay = [\"bias\", \"LayerNorm.weight\"]\n",
    "    optimizer_grouped_parameters = [\n",
    "        {\n",
    "            \"params\": [p for n, p in model.named_parameters() if not any(nd in n for nd in no_decay)],\n",
    "            \"weight_decay\": self.hparams.weight_decay,\n",
    "        },\n",
    "        {\n",
    "            \"params\": [p for n, p in model.named_parameters() if any(nd in n for nd in no_decay)],\n",
    "            \"weight_decay\": 0.0,\n",
    "        },\n",
    "    ]\n",
    "    optimizer = AdamW(optimizer_grouped_parameters, lr=self.hparams.learning_rate, eps=self.hparams.adam_epsilon)\n",
    "    self.opt = optimizer\n",
    "    return [optimizer]\n",
    "  \n",
    "  def optimizer_step(self, epoch, batch_idx, optimizer, optimizer_idx, second_order_closure=None):\n",
    "    if self.trainer.use_tpu:\n",
    "      xm.optimizer_step(optimizer)\n",
    "    else:\n",
    "      optimizer.step()\n",
    "    optimizer.zero_grad()\n",
    "    self.lr_scheduler.step()\n",
    "  \n",
    "  def get_tqdm_dict(self):\n",
    "    tqdm_dict = {\"loss\": \"{:.3f}\".format(self.trainer.avg_loss), \"lr\": self.lr_scheduler.get_last_lr()[-1]}\n",
    "\n",
    "    return tqdm_dict\n",
    "\n",
    "  def train_dataloader(self):\n",
    "    train_dataset = get_dataset(tokenizer=self.tokenizer, type_path=\"train\", args=self.hparams)\n",
    "    dataloader = DataLoader(train_dataset, batch_size=self.hparams.train_batch_size, drop_last=True, shuffle=True, num_workers=4)\n",
    "    t_total = (\n",
    "        (len(dataloader.dataset) // (self.hparams.train_batch_size * max(1, self.hparams.n_gpu)))\n",
    "        // self.hparams.gradient_accumulation_steps\n",
    "        * float(self.hparams.num_train_epochs)\n",
    "    )\n",
    "    scheduler = get_linear_schedule_with_warmup(\n",
    "        self.opt, num_warmup_steps=self.hparams.warmup_steps, num_training_steps=t_total\n",
    "    )\n",
    "    self.lr_scheduler = scheduler\n",
    "    return dataloader\n",
    "\n",
    "  def val_dataloader(self):\n",
    "    val_dataset = get_dataset(tokenizer=self.tokenizer, type_path=\"val\", args=self.hparams)\n",
    "    return DataLoader(val_dataset, batch_size=self.hparams.eval_batch_size, num_workers=4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "4b1ababe-116d-4c9f-8868-abadec491ad9",
   "metadata": {},
   "outputs": [],
   "source": [
    "logger = logging.getLogger(__name__)\n",
    "\n",
    "class LoggingCallback(pl.Callback):\n",
    "  def on_validation_end(self, trainer, pl_module):\n",
    "    logger.info(\"***** Validation results *****\")\n",
    "    if pl_module.is_logger():\n",
    "      metrics = trainer.callback_metrics\n",
    "      # Log results\n",
    "      for key in sorted(metrics):\n",
    "        if key not in [\"log\", \"progress_bar\"]:\n",
    "          logger.info(\"{} = {}\\n\".format(key, str(metrics[key])))\n",
    "\n",
    "  def on_test_end(self, trainer, pl_module):\n",
    "    logger.info(\"***** Test results *****\")\n",
    "\n",
    "    if pl_module.is_logger():\n",
    "      metrics = trainer.callback_metrics\n",
    "\n",
    "      # Log and save results to file\n",
    "      output_test_results_file = os.path.join(pl_module.hparams.output_dir, \"test_results.txt\")\n",
    "      with open(output_test_results_file, \"w\") as writer:\n",
    "        for key in sorted(metrics):\n",
    "          if key not in [\"log\", \"progress_bar\"]:\n",
    "            logger.info(\"{} = {}\\n\".format(key, str(metrics[key])))\n",
    "            writer.write(\"{} = {}\\n\".format(key, str(metrics[key])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "90c7bec3-b5e3-4533-8b7c-549dbc890f73",
   "metadata": {},
   "outputs": [],
   "source": [
    "args_dict = dict(\n",
    "    data_dir=\"\", # path for data files\n",
    "    output_dir=\"\", # path to save the checkpoints\n",
    "    model_name_or_path='t5-base',\n",
    "    tokenizer_name_or_path='t5-base',\n",
    "    max_seq_length=512,\n",
    "    learning_rate=3e-4,\n",
    "    weight_decay=0.0,\n",
    "    adam_epsilon=1e-8,\n",
    "    warmup_steps=0,\n",
    "    train_batch_size=8,\n",
    "    eval_batch_size=8,\n",
    "    num_train_epochs=2,\n",
    "    gradient_accumulation_steps=16,\n",
    "    n_gpu=1,\n",
    "    early_stop_callback=False,\n",
    "    fp_16=False, # if you want to enable 16-bit training then install apex and set this to true\n",
    "    opt_level='O1', # you can find out more on optimisation levels here https://nvidia.github.io/apex/amp.html#opt-levels-and-properties\n",
    "    max_grad_norm=1.0, # if you enable 16-bit training then set this to a sensible value, 0.5 is a good default\n",
    "    seed=42,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "ee1773c9-48bf-4c96-b973-2c15c7369e04",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c98ae8afddc74a539c43e4e53b3ac3fa",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading readme:   0%|          | 0.00/631 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ab0abc250ae8429a9f47e62ec43e9879",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading data:   0%|          | 0.00/226k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "09f64d6dfcf3486b9c0fe56d7af18079",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading data:   0%|          | 0.00/61.9k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c66d806d6db041f882c7c0dbec77e17e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading data:   0%|          | 0.00/62.6k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "130a5a8ab09c445682f2ef441275e9ce",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Generating train split:   0%|          | 0/2860 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a19df82937d34d7ea9e95ef48068ab58",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Generating validation split:   0%|          | 0/716 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f9deffcd5c764b12a60035683dd13b7e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Generating test split:   0%|          | 0/1034 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "hf_ds = datasets.load_dataset(hf_spider_ds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "dd2f6449-74fd-4a82-a904-9d3df1418b2e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "468ef7acbe804cbab5b8b1c6bdd215ca",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "tokenizer_config.json:   0%|          | 0.00/25.0 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5e5d01acf5e444419a752d98eec1c7f9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "spiece.model:   0%|          | 0.00/792k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8cdb0d44ad2440f890d095ec377bb521",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "special_tokens_map.json:   0%|          | 0.00/1.79k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "96ac6b0c301b41fda85472ebcb1a7c9c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "config.json:   0%|          | 0.00/1.23k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "The `xla_device` argument has been deprecated in v4.4.0 of Transformers. It is ignored and you can safely remove it from your `config.json` file.\n"
     ]
    }
   ],
   "source": [
    "#tokenizer = T5Tokenizer.from_pretrained('t5-base')\n",
    "tokenizer = T5Tokenizer.from_pretrained('mrm8488/t5-base-finetuned-wikiSQL')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "78bb8dc5-95b9-4560-b32e-d12cc31809eb",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "hello\n"
     ]
    }
   ],
   "source": [
    "print('hello')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "18c25792-d54f-48e2-ab80-e724ff56e3c2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DatasetDict({\n",
       "    train: Dataset({\n",
       "        features: ['type', 'question', 'query', 'db_id', 'schema'],\n",
       "        num_rows: 2860\n",
       "    })\n",
       "    validation: Dataset({\n",
       "        features: ['type', 'question', 'query', 'db_id', 'schema'],\n",
       "        num_rows: 716\n",
       "    })\n",
       "    test: Dataset({\n",
       "        features: ['type', 'question', 'query', 'db_id', 'schema'],\n",
       "        num_rows: 1034\n",
       "    })\n",
       "})"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hf_ds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "27fe7032-c556-4324-bc73-47debcd3f32f",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_df = hf_ds['train'].to_pandas()\n",
    "eval_df = hf_ds['validation'].to_pandas()\n",
    "test_df = hf_ds['test'].to_pandas()\n",
    "\n",
    "aggregate = pd.concat([train_df, eval_df, test_df])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "451c25b0-61f4-4de2-9144-586c09c936fb",
   "metadata": {},
   "outputs": [],
   "source": [
    "input_template = \"Schema: {schema} Question: {question} </s>\"\n",
    "target_template = \"{query} </s>\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "d5b5a636-3956-4d22-9a34-77562c9b71d1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4610\n",
      "4610\n"
     ]
    }
   ],
   "source": [
    "inputs = []\n",
    "targets = []\n",
    "\n",
    "for i in range(len(aggregate)):\n",
    "    question = aggregate['question'].iloc[i]\n",
    "    query = aggregate['query'].iloc[i]\n",
    "    schema = aggregate['schema'].iloc[i]\n",
    "    input = input_template.format(\n",
    "        schema=schema,\n",
    "        question=question\n",
    "    )\n",
    "    target = target_template.format(\n",
    "        query=query,\n",
    "    )\n",
    "    inputs.append(input)\n",
    "    targets.append(target)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "47ab6434-ec73-498e-afb2-ce6cabc68bd4",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Token indices sequence length is longer than the specified maximum sequence length for this model (994 > 512). Running this sequence through the model will result in indexing errors\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4610\n",
      "4610\n"
     ]
    }
   ],
   "source": [
    "inputs_encoded = []\n",
    "targets_encoded = []\n",
    "\n",
    "for i in range(len(inputs)):\n",
    "    input_encoded = tokenizer.encode(inputs[i])\n",
    "    target_encoded = tokenizer.encode(targets[i])\n",
    "    inputs_encoded.append(input_encoded)\n",
    "    targets_encoded.append(target_encoded)\n",
    "\n",
    "print(len(inputs_encoded))\n",
    "print(len(targets_encoded))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "172fdb97-878d-4386-9df7-a962b58f327c",
   "metadata": {},
   "outputs": [],
   "source": [
    "input_count_over = []\n",
    "target_count_over = []\n",
    "\n",
    "for i in range(len(inputs_encoded)):\n",
    "    if len(inputs_encoded[i]) > 512: \n",
    "        input_count_over.append(1)\n",
    "    else: \n",
    "        input_count_over.append(0)\n",
    "    if len(targets_encoded[i]) > 512: \n",
    "        target_count_over.append(1)\n",
    "    else:\n",
    "        target_count_over.append(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "cb48a497-1646-461d-a4e5-f4b7017905a5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7470715835140997"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sum(input_count_over) / len(input_count_over)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "1354506d-1ffa-49b8-8945-6dc471b4e6c8",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3444\n",
      "4610\n"
     ]
    }
   ],
   "source": [
    "print(sum(input_count_over))\n",
    "print(len(input_count_over))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "54872788-5865-43ff-8143-ac469e115a82",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}