gec-flan-t5-small.ipynb

{"cells":[{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","trusted":true},"outputs":[],"source":["# This Python 3 environment comes with many helpful analytics libraries installed\n","# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n","# For example, here's several helpful packages to load\n","\n","import numpy as np # linear algebra\n","import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n","\n","# Input data files are available in the read-only \"../input/\" directory\n","# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n","\n","import os\n","for dirname, _, filenames in os.walk('/kaggle/input'):\n","    for filename in filenames:\n","        print(os.path.join(dirname, filename))\n","\n","# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n","# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session"]},{"cell_type":"code","execution_count":1,"metadata":{"execution":{"iopub.execute_input":"2023-12-08T09:22:56.192103Z","iopub.status.busy":"2023-12-08T09:22:56.191839Z","iopub.status.idle":"2023-12-08T09:23:14.639913Z","shell.execute_reply":"2023-12-08T09:23:14.639166Z","shell.execute_reply.started":"2023-12-08T09:22:56.192077Z"},"trusted":true},"outputs":[{"name":"stderr","output_type":"stream","text":["/opt/conda/lib/python3.10/site-packages/scipy/__init__.py:146: UserWarning: A NumPy version >=1.16.5 and <1.23.0 is required for this version of SciPy (detected version 1.24.3\n","  warnings.warn(f\"A NumPy version >={np_minversion} and <{np_maxversion}\"\n"]}],"source":["from transformers import T5Tokenizer, T5ForConditionalGeneration\n","from transformers import DataCollatorForSeq2Seq\n","import datasets\n","import pandas as pd\n"]},{"cell_type":"code","execution_count":2,"metadata":{"execution":{"iopub.execute_input":"2023-12-08T09:24:31.983215Z","iopub.status.busy":"2023-12-08T09:24:31.982100Z","iopub.status.idle":"2023-12-08T09:24:41.013986Z","shell.execute_reply":"2023-12-08T09:24:41.013017Z","shell.execute_reply.started":"2023-12-08T09:24:31.983177Z"},"trusted":true},"outputs":[{"data":{"application/vnd.jupyter.widget-view+json":{"model_id":"8a69c2358ad9460f9f43c15a5fc119b5","version_major":2,"version_minor":0},"text/plain":["tokenizer_config.json:   0%|          | 0.00/2.54k [00:00<?, ?B/s]"]},"metadata":{},"output_type":"display_data"},{"data":{"application/vnd.jupyter.widget-view+json":{"model_id":"7280e7d8c80741b6ac6eebb088fea150","version_major":2,"version_minor":0},"text/plain":["spiece.model:   0%|          | 0.00/792k [00:00<?, ?B/s]"]},"metadata":{},"output_type":"display_data"},{"data":{"application/vnd.jupyter.widget-view+json":{"model_id":"4faabb5aca21402ead0868a4a31b2f6f","version_major":2,"version_minor":0},"text/plain":["special_tokens_map.json:   0%|          | 0.00/2.20k [00:00<?, ?B/s]"]},"metadata":{},"output_type":"display_data"},{"data":{"application/vnd.jupyter.widget-view+json":{"model_id":"2cf58bd99f484d3f85e326f40a5cac75","version_major":2,"version_minor":0},"text/plain":["tokenizer.json:   0%|          | 0.00/2.42M [00:00<?, ?B/s]"]},"metadata":{},"output_type":"display_data"},{"name":"stderr","output_type":"stream","text":["You are using the default legacy behaviour of the <class 'transformers.models.t5.tokenization_t5.T5Tokenizer'>. This is expected, and simply means that the `legacy` (previous) behavior will be used so nothing changes for you. If you want to use the new behaviour, set `legacy=False`. This should only be set if you understand what it means, and thouroughly read the reason why this was added as explained in https://github.com/huggingface/transformers/pull/24565\n","Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"]},{"data":{"application/vnd.jupyter.widget-view+json":{"model_id":"8034be317ea84565b6b99de1dc969064","version_major":2,"version_minor":0},"text/plain":["config.json:   0%|          | 0.00/1.40k [00:00<?, ?B/s]"]},"metadata":{},"output_type":"display_data"},{"data":{"application/vnd.jupyter.widget-view+json":{"model_id":"9496c6142f9144d6a7edb5ad4e858706","version_major":2,"version_minor":0},"text/plain":["model.safetensors:   0%|          | 0.00/990M [00:00<?, ?B/s]"]},"metadata":{},"output_type":"display_data"},{"data":{"application/vnd.jupyter.widget-view+json":{"model_id":"c395903ffe7048bc849ab8d7049f9873","version_major":2,"version_minor":0},"text/plain":["generation_config.json:   0%|          | 0.00/147 [00:00<?, ?B/s]"]},"metadata":{},"output_type":"display_data"}],"source":["tokenizer = T5Tokenizer.from_pretrained(\"google/flan-t5-small\")\n","model = T5ForConditionalGeneration.from_pretrained(\"google/flan-t5-small\")\n"]},{"cell_type":"code","execution_count":3,"metadata":{"execution":{"iopub.execute_input":"2023-12-08T09:26:16.043549Z","iopub.status.busy":"2023-12-08T09:26:16.042593Z","iopub.status.idle":"2023-12-08T09:26:16.167169Z","shell.execute_reply":"2023-12-08T09:26:16.166138Z","shell.execute_reply.started":"2023-12-08T09:26:16.043511Z"},"trusted":true},"outputs":[{"data":{"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>input</th>\n","      <th>target</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>So I think we can not live if old people could...</td>\n","      <td>So I think we would not be alive if our ancest...</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>So I think we can not live if old people could...</td>\n","      <td>So I think we could not live if older people d...</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>So I think we can not live if old people could...</td>\n","      <td>So I think we can not live if old people could...</td>\n","    </tr>\n","    <tr>\n","      <th>3</th>\n","      <td>So I think we can not live if old people could...</td>\n","      <td>So I think we can not live if old people can n...</td>\n","    </tr>\n","    <tr>\n","      <th>4</th>\n","      <td>For not use car.</td>\n","      <td>Not for use with a car.</td>\n","    </tr>\n","  </tbody>\n","</table>\n","</div>"],"text/plain":["                                               input  \\\n","0  So I think we can not live if old people could...   \n","1  So I think we can not live if old people could...   \n","2  So I think we can not live if old people could...   \n","3  So I think we can not live if old people could...   \n","4                                  For not use car.    \n","\n","                                              target  \n","0  So I think we would not be alive if our ancest...  \n","1  So I think we could not live if older people d...  \n","2  So I think we can not live if old people could...  \n","3  So I think we can not live if old people can n...  \n","4                           Not for use with a car.   "]},"execution_count":3,"metadata":{},"output_type":"execute_result"}],"source":["import pandas as pd\n","\n","train_df = pd.read_csv(\"/kaggle/input/jfleg-dataset/train.csv\")\n","test_def = pd.read_csv(\"/kaggle/input/jfleg-dataset/eval.csv\")\n","\n","combined_df = pd.concat([train_df, test_def], ignore_index=True)\n","combined_df.to_csv(\"/kaggle/working/combined.csv\", index=False)\n","combined_df.head()"]},{"cell_type":"code","execution_count":4,"metadata":{"execution":{"iopub.execute_input":"2023-12-08T09:26:19.121560Z","iopub.status.busy":"2023-12-08T09:26:19.120821Z","iopub.status.idle":"2023-12-08T09:26:19.302482Z","shell.execute_reply":"2023-12-08T09:26:19.301439Z","shell.execute_reply.started":"2023-12-08T09:26:19.121520Z"},"trusted":true},"outputs":[],"source":["import json\n","# Convert combined_df to json format and save it.\n","data = {'data': []}\n","\n","for i in range(len(combined_df)):\n","    json_data = {}\n","    json_data = {\n","        \"input\": \"Correct Grammer Errors: \" + combined_df['input'][i],\n","        \"target\": combined_df['target'][i]\n","    }   \n","    data['data'].append(json_data)\n","with open('/kaggle/working/combined.json', 'w') as f:\n","    json.dump(data, f, indent=4, separators=(',', ': ') )"]},{"cell_type":"code","execution_count":5,"metadata":{"execution":{"iopub.execute_input":"2023-12-08T09:26:22.509534Z","iopub.status.busy":"2023-12-08T09:26:22.508789Z","iopub.status.idle":"2023-12-08T09:26:22.767817Z","shell.execute_reply":"2023-12-08T09:26:22.766940Z","shell.execute_reply.started":"2023-12-08T09:26:22.509493Z"},"trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":["Downloading and preparing dataset json/default to /root/.cache/huggingface/datasets/json/default-b00bfe3bde116fcc/0.0.0/ac0ca5f5289a6cf108e706efcf040422dbbfa8e658dee6a819f20d76bb84d26b...\n"]},{"data":{"application/vnd.jupyter.widget-view+json":{"model_id":"9c1c756de6b94ef4b24065f5bb6ded50","version_major":2,"version_minor":0},"text/plain":["Downloading data files:   0%|          | 0/1 [00:00<?, ?it/s]"]},"metadata":{},"output_type":"display_data"},{"data":{"application/vnd.jupyter.widget-view+json":{"model_id":"bbbd869ccfdb4a9795c26c693f31543d","version_major":2,"version_minor":0},"text/plain":["Extracting data files:   0%|          | 0/1 [00:00<?, ?it/s]"]},"metadata":{},"output_type":"display_data"},{"name":"stdout","output_type":"stream","text":["Dataset json downloaded and prepared to /root/.cache/huggingface/datasets/json/default-b00bfe3bde116fcc/0.0.0/ac0ca5f5289a6cf108e706efcf040422dbbfa8e658dee6a819f20d76bb84d26b. Subsequent calls will reuse this data.\n"]},{"data":{"application/vnd.jupyter.widget-view+json":{"model_id":"a117f03d2a654cadbf716de3cb725522","version_major":2,"version_minor":0},"text/plain":["  0%|          | 0/1 [00:00<?, ?it/s]"]},"metadata":{},"output_type":"display_data"}],"source":["from datasets import load_dataset\n","dataset = load_dataset('json', data_files='/kaggle/working/combined.json',field=\"data\")"]},{"cell_type":"code","execution_count":6,"metadata":{"execution":{"iopub.execute_input":"2023-12-08T09:26:26.124807Z","iopub.status.busy":"2023-12-08T09:26:26.124465Z","iopub.status.idle":"2023-12-08T09:26:26.144141Z","shell.execute_reply":"2023-12-08T09:26:26.142971Z","shell.execute_reply.started":"2023-12-08T09:26:26.124780Z"},"trusted":true},"outputs":[{"data":{"text/plain":["DatasetDict({\n","    train: Dataset({\n","        features: ['input', 'target'],\n","        num_rows: 4803\n","    })\n","    test: Dataset({\n","        features: ['input', 'target'],\n","        num_rows: 1201\n","    })\n","})"]},"execution_count":6,"metadata":{},"output_type":"execute_result"}],"source":["dataset = dataset['train'].train_test_split(test_size=0.2)\n","dataset"]},{"cell_type":"code","execution_count":7,"metadata":{"execution":{"iopub.execute_input":"2023-12-08T09:26:28.847640Z","iopub.status.busy":"2023-12-08T09:26:28.847277Z","iopub.status.idle":"2023-12-08T09:26:28.853224Z","shell.execute_reply":"2023-12-08T09:26:28.852162Z","shell.execute_reply.started":"2023-12-08T09:26:28.847608Z"},"trusted":true},"outputs":[],"source":["def preprocess_func(examples):\n","    inputs = [source for source in examples['input']]\n","    targets = [target for target in examples['target']]\n","    model_inputs = tokenizer(inputs, text_target=targets, max_length=1024, truncation=True)\n","    return model_inputs"]},{"cell_type":"code","execution_count":8,"metadata":{"execution":{"iopub.execute_input":"2023-12-08T09:26:32.920955Z","iopub.status.busy":"2023-12-08T09:26:32.920594Z","iopub.status.idle":"2023-12-08T09:26:35.364585Z","shell.execute_reply":"2023-12-08T09:26:35.363654Z","shell.execute_reply.started":"2023-12-08T09:26:32.920923Z"},"trusted":true},"outputs":[{"data":{"application/vnd.jupyter.widget-view+json":{"model_id":"f8755385ea17480eaf83342d2e60cd0b","version_major":2,"version_minor":0},"text/plain":["  0%|          | 0/5 [00:00<?, ?ba/s]"]},"metadata":{},"output_type":"display_data"},{"data":{"application/vnd.jupyter.widget-view+json":{"model_id":"4b045599205c46b28139ab06a219da2c","version_major":2,"version_minor":0},"text/plain":["  0%|          | 0/2 [00:00<?, ?ba/s]"]},"metadata":{},"output_type":"display_data"}],"source":["tokenized_data = dataset.map(preprocess_func, batched=True)"]},{"cell_type":"code","execution_count":13,"metadata":{"execution":{"iopub.execute_input":"2023-12-08T08:34:25.724566Z","iopub.status.busy":"2023-12-08T08:34:25.724165Z","iopub.status.idle":"2023-12-08T08:34:25.732704Z","shell.execute_reply":"2023-12-08T08:34:25.731825Z","shell.execute_reply.started":"2023-12-08T08:34:25.724534Z"},"trusted":true},"outputs":[{"data":{"text/plain":["DatasetDict({\n","    train: Dataset({\n","        features: ['input', 'target', 'input_ids', 'attention_mask', 'labels'],\n","        num_rows: 4803\n","    })\n","    test: Dataset({\n","        features: ['input', 'target', 'input_ids', 'attention_mask', 'labels'],\n","        num_rows: 1201\n","    })\n","})"]},"execution_count":13,"metadata":{},"output_type":"execute_result"}],"source":["tokenized_data"]},{"cell_type":"code","execution_count":9,"metadata":{"execution":{"iopub.execute_input":"2023-12-08T09:26:38.068420Z","iopub.status.busy":"2023-12-08T09:26:38.067768Z","iopub.status.idle":"2023-12-08T09:26:52.333062Z","shell.execute_reply":"2023-12-08T09:26:52.331895Z","shell.execute_reply.started":"2023-12-08T09:26:38.068384Z"},"trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":["Collecting evaluate\n","  Obtaining dependency information for evaluate from https://files.pythonhosted.org/packages/70/63/7644a1eb7b0297e585a6adec98ed9e575309bb973c33b394dae66bc35c69/evaluate-0.4.1-py3-none-any.whl.metadata\n","  Downloading evaluate-0.4.1-py3-none-any.whl.metadata (9.4 kB)\n","Collecting sacrebleu\n","  Obtaining dependency information for sacrebleu from https://files.pythonhosted.org/packages/0a/a6/2ac47e71e526bbcd97ea08f20d9ef7d3852e2594ec7b2d55f5d2bbfd7aae/sacrebleu-2.3.3-py3-none-any.whl.metadata\n","  Downloading sacrebleu-2.3.3-py3-none-any.whl.metadata (57 kB)\n","\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m57.7/57.7 kB\u001b[0m \u001b[31m2.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25hRequirement already satisfied: datasets>=2.0.0 in /opt/conda/lib/python3.10/site-packages (from evaluate) (2.1.0)\n","Requirement already satisfied: numpy>=1.17 in /opt/conda/lib/python3.10/site-packages (from evaluate) (1.24.3)\n","Requirement already satisfied: dill in /opt/conda/lib/python3.10/site-packages (from evaluate) (0.3.7)\n","Requirement already satisfied: pandas in /opt/conda/lib/python3.10/site-packages (from evaluate) (2.0.3)\n","Requirement already satisfied: requests>=2.19.0 in /opt/conda/lib/python3.10/site-packages (from evaluate) (2.31.0)\n","Requirement already satisfied: tqdm>=4.62.1 in /opt/conda/lib/python3.10/site-packages (from evaluate) (4.66.1)\n","Requirement already satisfied: xxhash in /opt/conda/lib/python3.10/site-packages (from evaluate) (3.4.1)\n","Requirement already satisfied: multiprocess in /opt/conda/lib/python3.10/site-packages (from evaluate) (0.70.15)\n","Requirement already satisfied: fsspec[http]>=2021.05.0 in /opt/conda/lib/python3.10/site-packages (from evaluate) (2023.12.1)\n","Requirement already satisfied: huggingface-hub>=0.7.0 in /opt/conda/lib/python3.10/site-packages (from evaluate) (0.19.4)\n","Requirement already satisfied: packaging in /opt/conda/lib/python3.10/site-packages (from evaluate) (21.3)\n","Requirement already satisfied: responses<0.19 in /opt/conda/lib/python3.10/site-packages (from evaluate) (0.18.0)\n","Collecting portalocker (from sacrebleu)\n","  Obtaining dependency information for portalocker from https://files.pythonhosted.org/packages/17/9e/87671efcca80ba6203811540ed1f9c0462c1609d2281d7b7f53cef05da3d/portalocker-2.8.2-py3-none-any.whl.metadata\n","  Downloading portalocker-2.8.2-py3-none-any.whl.metadata (8.5 kB)\n","Requirement already satisfied: regex in /opt/conda/lib/python3.10/site-packages (from sacrebleu) (2023.8.8)\n","Requirement already satisfied: tabulate>=0.8.9 in /opt/conda/lib/python3.10/site-packages (from sacrebleu) (0.9.0)\n","Requirement already satisfied: colorama in /opt/conda/lib/python3.10/site-packages (from sacrebleu) (0.4.6)\n","Requirement already satisfied: lxml in /opt/conda/lib/python3.10/site-packages (from sacrebleu) (4.9.3)\n","Requirement already satisfied: pyarrow>=5.0.0 in /opt/conda/lib/python3.10/site-packages (from datasets>=2.0.0->evaluate) (11.0.0)\n","Requirement already satisfied: aiohttp in /opt/conda/lib/python3.10/site-packages (from datasets>=2.0.0->evaluate) (3.8.5)\n","Requirement already satisfied: filelock in /opt/conda/lib/python3.10/site-packages (from huggingface-hub>=0.7.0->evaluate) (3.12.2)\n","Requirement already satisfied: pyyaml>=5.1 in /opt/conda/lib/python3.10/site-packages (from huggingface-hub>=0.7.0->evaluate) (6.0.1)\n","Requirement already satisfied: typing-extensions>=3.7.4.3 in /opt/conda/lib/python3.10/site-packages (from huggingface-hub>=0.7.0->evaluate) (4.5.0)\n","Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /opt/conda/lib/python3.10/site-packages (from packaging->evaluate) (3.0.9)\n","Requirement already satisfied: charset-normalizer<4,>=2 in /opt/conda/lib/python3.10/site-packages (from requests>=2.19.0->evaluate) (3.2.0)\n","Requirement already satisfied: idna<4,>=2.5 in /opt/conda/lib/python3.10/site-packages (from requests>=2.19.0->evaluate) (3.4)\n","Requirement already satisfied: urllib3<3,>=1.21.1 in /opt/conda/lib/python3.10/site-packages (from requests>=2.19.0->evaluate) (1.26.15)\n","Requirement already satisfied: certifi>=2017.4.17 in /opt/conda/lib/python3.10/site-packages (from requests>=2.19.0->evaluate) (2023.11.17)\n","Requirement already satisfied: python-dateutil>=2.8.2 in /opt/conda/lib/python3.10/site-packages (from pandas->evaluate) (2.8.2)\n","Requirement already satisfied: pytz>=2020.1 in /opt/conda/lib/python3.10/site-packages (from pandas->evaluate) (2023.3)\n","Requirement already satisfied: tzdata>=2022.1 in /opt/conda/lib/python3.10/site-packages (from pandas->evaluate) (2023.3)\n","Requirement already satisfied: attrs>=17.3.0 in /opt/conda/lib/python3.10/site-packages (from aiohttp->datasets>=2.0.0->evaluate) (23.1.0)\n","Requirement already satisfied: multidict<7.0,>=4.5 in /opt/conda/lib/python3.10/site-packages (from aiohttp->datasets>=2.0.0->evaluate) (6.0.4)\n","Requirement already satisfied: async-timeout<5.0,>=4.0.0a3 in /opt/conda/lib/python3.10/site-packages (from aiohttp->datasets>=2.0.0->evaluate) (4.0.3)\n","Requirement already satisfied: yarl<2.0,>=1.0 in /opt/conda/lib/python3.10/site-packages (from aiohttp->datasets>=2.0.0->evaluate) (1.9.2)\n","Requirement already satisfied: frozenlist>=1.1.1 in /opt/conda/lib/python3.10/site-packages (from aiohttp->datasets>=2.0.0->evaluate) (1.4.0)\n","Requirement already satisfied: aiosignal>=1.1.2 in /opt/conda/lib/python3.10/site-packages (from aiohttp->datasets>=2.0.0->evaluate) (1.3.1)\n","Requirement already satisfied: six>=1.5 in /opt/conda/lib/python3.10/site-packages (from python-dateutil>=2.8.2->pandas->evaluate) (1.16.0)\n","Downloading evaluate-0.4.1-py3-none-any.whl (84 kB)\n","\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m84.1/84.1 kB\u001b[0m \u001b[31m4.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25hDownloading sacrebleu-2.3.3-py3-none-any.whl (106 kB)\n","\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m106.4/106.4 kB\u001b[0m \u001b[31m10.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25hDownloading portalocker-2.8.2-py3-none-any.whl (17 kB)\n","Installing collected packages: portalocker, sacrebleu, evaluate\n","Successfully installed evaluate-0.4.1 portalocker-2.8.2 sacrebleu-2.3.3\n"]}],"source":["!pip install evaluate sacrebleu"]},{"cell_type":"code","execution_count":11,"metadata":{"execution":{"iopub.execute_input":"2023-12-08T09:26:55.108634Z","iopub.status.busy":"2023-12-08T09:26:55.107751Z","iopub.status.idle":"2023-12-08T09:26:55.436548Z","shell.execute_reply":"2023-12-08T09:26:55.435616Z","shell.execute_reply.started":"2023-12-08T09:26:55.108598Z"},"trusted":true},"outputs":[],"source":["import evaluate\n","metric = evaluate.load(\"sacrebleu\")\n","\n","# Then create a function that passes your predictions and labels to compute to calculate the SacreBLEU score:\n","import numpy as np\n","\n","def postprocess_text(preds, labels):\n","    preds = [pred.strip() for pred in preds]\n","    labels = [[label.strip()] for label in labels]\n","    return preds, labels\n","\n","def compute_metrics(eval_preds):\n","    preds, labels = eval_preds\n","    if isinstance(preds, tuple):\n","        preds = preds[0]\n","    decoded_preds = tokenizer.batch_decode(preds, skip_special_tokens=True, max_new_tokens=1024)\n","#     print(f\"Decoded preds: {decoded_preds} \\n Len = {len(decoded_preds)}\")\n","    labels = np.where(labels != -100, labels, tokenizer.pad_token_id)\n","    decoded_labels = tokenizer.batch_decode(labels, skip_special_tokens=True)\n","\n","    decoded_preds, decoded_labels = postprocess_text(decoded_preds, decoded_labels)\n","\n","    result = metric.compute(predictions=decoded_preds, references=decoded_labels)\n","    result = {\"bleu\": result[\"score\"]}\n","\n","    prediction_lens = [np.count_nonzero(pred != tokenizer.pad_token_id) for pred in preds]\n","    result[\"gen_len\"] = np.mean(prediction_lens)\n","    result = {k: round(v, 4) for k, v in result.items()}\n","    return result"]},{"cell_type":"code","execution_count":13,"metadata":{"execution":{"iopub.execute_input":"2023-12-08T09:26:55.478279Z","iopub.status.busy":"2023-12-08T09:26:55.477974Z","iopub.status.idle":"2023-12-08T09:26:55.482884Z","shell.execute_reply":"2023-12-08T09:26:55.482001Z","shell.execute_reply.started":"2023-12-08T09:26:55.478252Z"},"trusted":true},"outputs":[],"source":["from transformers import Seq2SeqTrainingArguments, Seq2SeqTrainer, DataCollatorForSeq2Seq\n","\n","data_collator = DataCollatorForSeq2Seq(tokenizer=tokenizer, model=\"google/flan-t5-base\")"]},{"cell_type":"code","execution_count":14,"metadata":{"execution":{"iopub.execute_input":"2023-12-08T09:27:01.718396Z","iopub.status.busy":"2023-12-08T09:27:01.718017Z","iopub.status.idle":"2023-12-08T09:27:08.413425Z","shell.execute_reply":"2023-12-08T09:27:08.412593Z","shell.execute_reply.started":"2023-12-08T09:27:01.718367Z"},"trusted":true},"outputs":[],"source":["\n","training_args = Seq2SeqTrainingArguments(\n","    output_dir=\"trainer-state\",\n","    evaluation_strategy=\"epoch\",\n","    learning_rate=2e-5,\n","    per_device_train_batch_size=32,\n","    per_device_eval_batch_size=32,\n","    weight_decay=0.01,\n","    save_total_limit=2,\n","    num_train_epochs=50,\n","    predict_with_generate=True,\n","    fp16=False,\n","    push_to_hub=False,\n","    logging_dir=\"logging\",\n","    logging_strategy=\"epoch\",\n",")\n","\n","trainer = Seq2SeqTrainer(\n","    model=model,\n","    args=training_args,\n","    train_dataset=tokenized_data[\"train\"],\n","    eval_dataset=tokenized_data[\"test\"],\n","    tokenizer=tokenizer,\n","    data_collator=data_collator,\n","    compute_metrics=compute_metrics,\n",")"]},{"cell_type":"code","execution_count":null,"metadata":{"execution":{"iopub.execute_input":"2023-12-08T09:27:13.686315Z","iopub.status.busy":"2023-12-08T09:27:13.685925Z","iopub.status.idle":"2023-12-08T09:28:28.195167Z","shell.execute_reply":"2023-12-08T09:28:28.193372Z","shell.execute_reply.started":"2023-12-08T09:27:13.686280Z"},"trusted":true},"outputs":[],"source":["trainer.train()"]},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":["trainer.save_model(\"gec-flan-t5-small\")"]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.10.12"}},"nbformat":4,"nbformat_minor":4}