05_disaster_tweet_cleanlab.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Text Classification - Detect & Clean incorrect labels using cleanlab"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mounted at /content/gdrive\n"
     ]
    }
   ],
   "source": [
    "# make your Google Drive accessible\n",
    "from google.colab import drive\n",
    "drive.mount('/content/gdrive', force_remount=True)\n",
    "\n",
    "root_dir = \"/content/gdrive/My Drive\"\n",
    "drive_data = f\"{root_dir}/Colab Notebooks/data/disaster_tweet\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# !pip install light-the-torch \n",
    "# !ltt install torch torchvision"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# !pip install -Uqq pandas --upgrade\n",
    "# !pip install -Uqq fastcore --upgrade\n",
    "# !pip install -Uqq fastai --upgrade \n",
    "# !pip install -q nbdev\n",
    "# !pip install sentencepiece \n",
    "\n",
    "from pathlib import Path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%reload_ext autoreload\n",
    "%autoreload 2\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from fastai.text.all import *\n",
    "from fastcore.test import * \n",
    "#import fastai; fastai.utils.show_install()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def replace_space(text, sep='_'):\n",
    "  return sep.join(text.split(' '))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def modify_metadata(column_value, tag_name):\n",
    "  data = f\"{tag_name}_{replace_space(column_value)}\"\n",
    "  data.replace(',', '')\n",
    "  return data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'aut_name_Jeremy_Howard'"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "modify_metadata('Jeremy Howard', 'aut_name')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def clean_text(word):\n",
    "\n",
    "  #Remove urls\n",
    "  urls = re.findall(r'https?://\\S+', word)\n",
    "  if len(urls) > 0:\n",
    "    word = \"\"\n",
    "    return word\n",
    "  \n",
    "  return word.strip()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "Config.DEFAULT_CONFIG = {\n",
    "        'data_path': f'{drive_data}',\n",
    "        'data_archive_path': f'{drive_data}',\n",
    "        'model_path': f'{drive_data}' + '/models'\n",
    "    }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'data_archive_path': '/content/gdrive/My Drive/Colab Notebooks/data/disaster_tweet',\n",
       " 'data_path': '/content/gdrive/My Drive/Colab Notebooks/data/disaster_tweet',\n",
       " 'model_path': '/content/gdrive/My Drive/Colab Notebooks/data/disaster_tweet/models'}"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Config.DEFAULT_CONFIG"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "bs=64"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "disaster_tweet = f\"{drive_data}\"; disaster_tweet\n",
    "disaster = Path(disaster_tweet)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_df = pd.read_csv(disaster/'train.csv');train_df.head()\n",
    "test_df = pd.read_csv(disaster/'test.csv');"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_df['text'] = train_df['text'].apply(lambda x: ' '.join([clean_text(each) for each in x.split(' ')]))\n",
    "test_df['text'] = test_df['text'].apply(lambda x: ' '.join([clean_text(each) for each in x.split(' ')]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "train_df['keyword'] = train_df['keyword'].replace(np.nan, 'NA', regex=True)\n",
    "test_df['keyword'] = test_df['keyword'].replace(np.nan, 'NA', regex=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       ""
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.7/dist-packages/numpy/core/_asarray.py:83: VisibleDeprecationWarning: Creating an ndarray from ragged nested sequences (which is a list-or-tuple of lists-or-tuples-or ndarrays with different lengths or shapes) is deprecated. If you meant to do this, you must specify 'dtype=object' when creating the ndarray\n",
      "  return array(a, dtype, copy=False, order=order)\n"
     ]
    }
   ],
   "source": [
    "dls_lm = TextDataLoaders.from_df(pd.concat([train_df, test_df]), seed=42, text_col='text', is_lm=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "lang_learn = language_model_learner(dls_lm, AWD_LSTM, metrics=Perplexity())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       ""
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "SuggestedLRs(lr_min=0.05248074531555176, lr_steep=0.05754399299621582)"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAELCAYAAADURYGZAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3dd3wc5b3v8c9P3eqyJDeMe8EGY2NEtWnHEAIJgZBKCSUQQkIIpB2Se5Kbc27OuclJuwlJwHFoKYQUh5pCSU7o2MYVsI2JLTfZyJabVpLVVvrdP3blCFmW5TK7s9rv+/Xal3ZnZme+Wsz+9Mwz8zzm7oiISPrKSHYAERFJLhUCEZE0p0IgIpLmVAhERNKcCoGISJpTIRARSXOBFQIzm2xmy7s9ImZ2e49tzMzuNLO1Zvaamc0MKo+IiPQuK6gdu/saYAaAmWUCW4BHemx2ETAx/jgNuDv+84AqKip8zJgxRzuuiMiAtmTJkh3uXtnbusAKQQ9zgHXuvrHH8kuBX3jsrrYFZlZqZsPd/e0D7WjMmDEsXrw4yKwiIgOOmfX8/t0nUX0EHwUe6mX5McDmbq9r4svewcxuMrPFZra4rq4uoIgiIukp8EJgZjnA+4DfH+4+3H2eu1e5e1VlZa8tGxEROUyJaBFcBCx19229rNsCHNvt9cj4MhERSZBEFIIr6P20EMDjwDXxq4dOB+r76h8QEZGjL9DOYjMrAC4APtlt2c0A7j4X+DNwMbAW2AtcH2QeERHZX6CFwN2bgPIey+Z2e+7ALUFmEBGRvunOYhGRFPDMqm2s3d4QyL5VCEREUsAtDy7lD0uDuZZGhUBEJORaox20dXRSmBvM2XwVAhGRkGtsiQJQlKdCICKSlhpbY4VALQIRkTTVEG8RFKgQiIikp64WQZEKgYhIemrqOjWkPgIRkfSkPgIRkTTX1UegFoGISJr6Zx9BdiD7VyEQEQm5xpYomRlGXnYwX9kqBCIiIdfYGqUgJxMzC2T/KgQiIiHX0BKlKC+Y00KgQiAiEnpNrdHArhgCFQIRkdBrbI0GdsUQqBCIiIReg1oEIiLprbGlXS0CEZF01tgapTAnRQuBmZWa2Xwze9PMVpvZGT3Wl5jZE2a2wsxWmpkmrxcR6aGxJdg+gkAnrwd+CDzp7h80sxwgv8f6W4BV7n6JmVUCa8zsQXdvCziXiEhK6Ox0mto6Au0jCGzPZlYCnA1cBxD/cu/5Be9AkcXukigEdgHRoDKJiKSaprZgZyeDYE8NjQXqgPvNbJmZ3WNmBT22+TEwBdgKvA7c5u6dPXdkZjeZ2WIzW1xXVxdgZBGRcAl65FEIthBkATOBu939JKAJ+HKPbS4ElgMjgBnAj82suOeO3H2eu1e5e1VlZWWAkUVEwqUx4JFHIdhCUAPUuPvC+Ov5xApDd9cDD3vMWmA9cFyAmUREUkpDKrcI3L0W2Gxmk+OL5gCremy2Kb4cMxsKTAaqg8okIpJq9rUIUrGzOO5W4MH4FUPVwPVmdjOAu88FvgE8YGavAwbc4e47As4kIpIyGgOephICLgTuvhyo6rF4brf1W4F3BZlBRCSVpXpnsYiIHKGuU0NBzU4GKgQiIqHW1SIoyM0M7BgqBCIiIdbYGmVQdiZZmcF9XasQiIiEWEPA4wyBCoGISKg1BjwXAagQiIiEWmNLuwqBiEg6a2oNduRRUCEQEQm1hoDnKwYVAhGRUGtsbadILQIRkfQV9OxkoEIgIhJa7k5ja5QCtQhERNJTa7ST9g5XZ7GISLrqGl4iyGkqQYVARCS0mhIw8iioEIiIhFZDAialARUCEZHQSsSkNKBCICISWomYiwBUCEREQisRcxFAwIXAzErNbL6ZvWlmq83sjF62OdfMlpvZSjN7Lsg8IiKppCFBp4aCnrz+h8CT7v7B+AT2+d1XmlkpcBfwbnffZGZDAs4jIpIyuq4aCvrUUGCFwMxKgLOB6wDcvQ1o67HZlcDD7r4pvs32oPKIiKSaxpYomRlGXnawZ/GD3PtYoA6438yWmdk9ZlbQY5tJQJmZPWtmS8zsmt52ZGY3mdliM1tcV1cXYGQRkfDompTGzAI9TpCFIAuYCdzt7icBTcCXe9nmZOA9wIXA18xsUs8dufs8d69y96rKysoAI4uIhEdDS/Czk0GwhaAGqHH3hfHX84kVhp7bPOXuTe6+A3gemB5gJhGRlNHY2h748BIQYCFw91pgs5lNji+aA6zqsdljwGwzyzKzfOA0YHVQmUREUkkiRh6F4K8auhV4MH7FUDVwvZndDODuc919tZk9CbwGdAL3uPsbAWcSEUkJjS1RSvNzAj9OoIXA3ZcDVT0Wz+2xzXeA7wSZQ0QkFTW2Rhk5OP/gGx4h3VksIhJSja3RwKepBBUCEZHQahwAVw2JiMhh6uh0mto6Ah9eAlQIRERCqaktMXMRgAqBiEgoNSZoUhpQIRARCaVETUoDKgQiIqHU0NIOqEUgIpK2Is2xFkHJoGCHoAYVAhGRUKpvjrUIVAhERNKUCoGISJrrKgTFKgQiIumpvrmd/JxMsjOD/5pWIRARCaH65vaEnBYCFQIRkVCKqBCIiKS3+ub2hPQPgAqBiEgo6dSQiEiaizS3U5ynQiAikrYGTIvAzErNbL6ZvWlmq83sjANsd4qZRc3sg0HmERFJBe0dnTS1dSSsEAQ9mtEPgSfd/YPxCez3m3zTzDKB/waeDjiLiEhKaGjpGmco+AHnIMAWgZmVAGcD9wK4e5u77+ll01uBPwDbg8oiIpJK9g0vkZ/6p4bGAnXA/Wa2zMzuMbOC7huY2THA+4G7+9qRmd1kZovNbHFdXV1wiUVEQiCR4wxBsIUgC5gJ3O3uJwFNwJd7bPMD4A537+xrR+4+z92r3L2qsrIymLQiIiGxb5yhBF01FOQJqBqgxt0Xxl/PZ/9CUAX8xswAKoCLzSzq7o8GmEtEJNQS3SIIrBC4e62ZbTazye6+BpgDrOqxzdiu52b2APBHFQERSXcDphDE3Qo8GL9iqBq43sxuBnD3uQEfW0QkJUUSOAQ1BFwI3H05sdM/3fVaANz9uiCziIikikhzO7lZGeRlZybkeLqzWEQkZBI54ByoEIiIhE4ih5cAFQIRkdBRIRARSXMqBCIiaS7SokIgIpLW6veqEIiIpK3OTqehNUpxXmJGHgUVAhGRUGloieKeuJvJQIVARCRUEj28BKgQiIiESmgLgZkVmFlG/PkkM3ufmSUupYhImoi0hLQQAM8DefGJZJ4GPgY8EFQoEZF0VZ/gAeeg/4XA3H0vcDlwl7t/CDg+uFgiIukptKeGADOzM4CrgD/FlyVmWDwRkTQS5kJwO/AV4BF3X2lm44C/BxdLRCQ91Te3k5Vh5Ock7m/tft2x4O7PAc8BxDuNd7j7Z4MMJiKSjrrGGYpP4ZsQ/b1q6NdmVmxmBcAbwCoz+1Kw0URE0k8kwQPOQf9PDU119whwGfAXYCyxK4dEROQoqm9upyikhSA7ft/AZcDj7t4O+MHeZGalZjbfzN40s9XxDufu668ys9fM7HUze9nMph/6ryAiMnCEuUXwU2ADUAA8b2ajgUg/3vdD4El3Pw6YDqzusX49cI67TwO+AczrZx4RkQEp0XMRQP87i+8E7uy2aKOZndfXe8ysBDgbuC6+jzagrcd+X+72cgEwsj95REQGqlghSNzIo9D/zuISM/u+mS2OP75HrHXQl7FAHXC/mS0zs3vinc0HcgOx/gcRkbTk7kRaoqE9NXQf0AB8OP6IAPcf5D1ZwEzgbnc/CWgCvtzbhvHWxQ3AHQdYf1NXEaqrq+tnZBGR1NLU1kFHp1OcF85CMN7dv+7u1fHHfwDjDvKeGqDG3RfGX88nVhjewcxOBO4BLnX3nb3tyN3nuXuVu1dVVlb2M7KISGpJxl3F0P9C0Gxms7temNksoLmvN7h7LbDZzCbHF80BVnXfxsxGAQ8DH3P3t/qdWkRkAKrfm5xC0N8eiZuBX8Q7gAF2A9f24323Ag+aWQ5QDVxvZjcDuPtc4H8D5cBd8bvoou5edQj5D0lrtIPcLA2RJCLhlKwWQX+vGloBTDez4vjriJndDrx2kPctB3p+sc/ttv5G4MZDSnyYnl5Zy/965HWeuHU2w0sGJeKQIiKHJBlDUMMhzlDm7pH4HcYAnw8gT2CmDC8m0hLl20+uSXYUEZFeJWNSGjiyqSoTNyLSUXDs4HxunD2WR5ZtYdmm3cmOIyKynz17Y7daleSnTiE46BATYfPp8yZQUZjLN/64CveUiy8iA9z2SCt52RkU5YbohjIzazCzSC+PBmBEgjIeNYW5WXzpwkks3bSHJ157O9lxRETeoTbSwrDivIQOQQ0HKQTuXuTuxb08itw9sSXrKPngyccydXgx3/rzava2RZMdR0Rkn22RFoYW5yX8uEdyaiglZWYYX3vvVLbWt3Da//0bt/1mGX967W1a2juSHU1E0lxtpIVhJSoECXHG+HIevPE03n38MJ5/q45bfr2UD859eV9HjYhIork72yKtDEtCiyAlT+8cDbMmVDBrQgXRjk7+8kYtX/jdCj527yJ+dcNpCe+xFxHZvbedtmgnQ3RqKPGyMjO4ZPoI5n5sJmtqG/jYfQv33dQhIpIotfUtAElpEaR9IejyL8cN5e6rZ7L67Qg3PPAqHZ26vFREEmdbJF4ISnITfmwVgm7mTBnKty4/kcUbd/PQok3JjiMiaaQ2Xgh01VAIXD7zGM4YV863n3yTHY2tyY4jImmi69TQkCIVgqQzM75x2fE0t3fwrb+8mew4IpImtje0UFGYQ05W4r+WVQh6MWFIEZ84axzzl9SwaP2uQI7R0NLOG1vqWbppNwurd7Ji8x5ao7qXQSRd1dYn52YySOPLRw/m1n+ZyGPLt/LVR1/n4U/PovAojf1R39zOvS+u5/4X19PQ+s47m3OzMpg+spTpx5aQl51JR6fjwMQhhZwxvlzDZ4sMYLWRVkYk4WYyUCE4oEE5mfzn+0/gxp8v5kNzX+Hea6sYUXr4X8Q7G1v5xSsbue+l9TS0RLnohGFcOmMEudmZZGdkEGlpZ+nG3by6cTcPvLyBjk4nIz7eSDR+BdPYigJmHFvK2IoCxlYUMHFoIROHFJGZkVIDwYpIL7ZFWphxbGlSjq1C0IfzJg/hvutO4TMPLuXSn7zEz66pYnhJHguqd7J4w27GVhTwkVOOpaCP1sK6ukbueWE9Dy+toTXayYXHD+W2OZOYOqJ4v20vnjZ8v2Wdnc6btQ28vG4HC6p3smj9Lh5ZtmXf+qLcLGaMKuXk0WVMHlrEhCGFjC4vSMp5RhE5PK3RDnY1tSXlHgIAS7XhmKuqqnzx4sUJPeZb2xr4+AOvsnVPM123FwzKzqS5vYOSQdlcc8ZoPjBzJMNL88jNyqSlvYOnVtbym0WbeaV6JzlZGXxg5jHcMHscE4YUHnGe5rYONu5qYtXWCEs27mbJxt2s2dZA13/KzAyjLD+b4rxsigdlM6I0jynDijlueDHTjy1JylUJInJgm3ft5axv/53//sA0PnLKqECOYWZLDjQVsApBP+1sbGXuc+sYUpTH6ePKmTqimBU1e/jpc+t4etW2fV/CZfnZRDuchtYoowbn8+GqkXz01FFUFAZ7k8jetijrtjextq6Bddub2LW3jUhzO/XN7WzatZeNO/cCYAZnjCvnspOO4d0nDKM4r/fhNFraO4g0t7OnuZ38nExGluUHml8knb26YRcfmvsKD1x/CudOHhLIMZJWCMysFLgHOIHYRDYfd/dXuq034IfAxcBe4Dp3X9rXPpNVCPpSXdfIqxt2sT3SyraGFjo64ZITh3P6uHIyQnL+vrE1ypraCC/8YwePLtvChnhhKMzNojgvi4LcLNo6OmlsidLYGqU12vmO908YUsic44Zw5oQKCnMzyczIIMNi46PUNbSyo7GVorwsxlUUMr6ygMqi3ISPqS6Sqp5YsZVbH1rGk7efxXHD9j9tfDT0VQiC7iP4IfCku3/QzHKAnn9WXgRMjD9OA+6O/0wp4yoLGVd55Kd8glSYm8XJowdz8ujB3DZnIitq6nn+rTr27G0n0tJOY0uU3OwMCnOzYsVhUDal+dmUDsphW6SF/3lzO/e9tJ6fPl/dr+MV5GQyqryAUYMHMbxkEHnZmeRkZZCXncHIsnzGVRQwpqLgqF2NJZLK9g0vMdAuHzWzEuBs4DoAd28Deo7zfCnwC481SxaYWamZDXd3TR8WIDNjxrGlh3SFwsdnj43f+xChvaOTaGcnHZ2xU2GVRbmUF+ayZ28b1XVNVNc1smHnXjbv2su6uiZeWruTtmgnbR2d++13wpBCZk+o4KyJFRTkZrF88x6Wb9rDurpGWqOdtEU76XRnTHkBxw0vYvKwIsZWFDC6vIBhxXlkWKy1U9fQys6mNur3xk6HtXV0MnlYEVOHF5OXnXk0Pz6Ro25bpIXcrIyET1rfJcg/x8YCdcD9ZjYdWALc5u5N3bY5Btjc7XVNfNk7CoGZ3QTcBDBqVDAdKXJwRXnZnDG+/IDrC3OzGFmWz9mTKntd7+60tHeyadde1u9oYl1dIwvX7+KhRZt44OUN+7YbXZ7P5KFF5OfEWhHuUL2jiYeXbqGx270XOZkZZGRAS/v+BaZLdqYxZXgxw4rzKMrLpnhQFmMrCjhjXDkThhTq9JWEQm2klWEliZ+iskuQhSALmAnc6u4LzeyHwJeBrx3qjtx9HjAPYn0ERzWlJIyZMSgnk8nDYn/ZA9xyXqxjeunG3bRGOzlxZAnlB+hYd3e27Glm485Y5/fGXU10dDhDinOpLMplcEEupYOyKRmUTWaGsXJrhOWb9/D6lj1s2rWXhpYo9c3t+4pJRWEOU4YXk52ZQYYZmRmxK67MjKwMY3R5AVOHF3P8iFirYldTG7ua2sjPyeSEY0p0/4YcNdvqWxiaxKv5giwENUCNuy+Mv55PrBB0twU4ttvrkfFlkkbysjM5c0LFQbczM0aW5TOyLJ9ZEw6+32MH5/PuE4a9Y5m7s3lXM69U7+CVdTtZv6OJDnc6OmP3bHR67NHW0ckTK7ZyoNHIy/KzOXtSJaeOHUyGGW3RTqKdTkVhDiPLBnFMaT75uZn7riZrjXbEr+KK0hrtoHRQDoMLcigryCY3S6eu0l1tpIXpSbqZDAIsBO5ea2abzWyyu68B5gCremz2OPAZM/sNsU7ievUPSJDMjFHl+YwqH3XQ67Wb2zp4szbCqrcjdHY6ZQWxL+8djW08u2Y7z79Vx2PLtx5xplGD85k8rIjjhhW94xxxaX4OU4YXMXFIkW4QHMDcndpICxcWJ34egi5BX7JxK/Bg/IqhauB6M7sZwN3nAn8mdunoWmKXj14fcB6RfhuUk8lJo8o4aVTZfuveN30EnZ3O1vpmsjIyyM40MjOM7Q2tbNndTM2eZlrbY4MImhk5mUZx/LRVTlYGkeZ2djW1s72hhX9sb2RNbQN/W72t1xZIVoYxYUghJ44sYdrIUqYdU8Jxw4rUCT5A7IlPUZmsAecg4ELg7suBntetzu223oFbgswgEpSMDNvvRrvS/BwmDS06rP21RTv3jUDrwPZIK6vfjrD67QhvbI3wzKpt/G5xTezYBuMrC5kyvJgzx5dz/tShgd+0KMGo3Tcz2QAtBCLSfzlZGe84BVScl82EIYVcMn0E8M/O8tdr6ln9doRVbzewaP0uHl+xFXvkdapGl/G+6SO4fObIPse/knCpTfI9BKBCIJIyuneWXxQfoNDdWfV2hKdXbuOplbV87bGVfPupNXyk6liuOWMMo8o1NEjYbatP3hSVXVQIRFKYmXH8iBKOH1HC7edPZOmmPdz30nruf3kD97y4npmjSrlk+gjeM204Q5L4RSMHti0SmxJXhUBEjpiZcfLoMk4eXcbWPc08smwLT6zYyn88sYr/88dVzBpfwWUnHcOFxw+l6ACDDUri1UZaKC9IzhSVXVQIRAagEaWDuOW8Cdxy3gTWbm/k8RVbeWz5Fr74+xV89dEM3n/SSG4+ZxyjywuSHTXt1ezey/DS5LbWVAhEBrgJQwr5/AWT+Nz5E1m2eQ+/X7yZPyyp4bevbuK9J47glvMm7LvTWxLL3Vm5NcKc44IZerq/VAhE0oSZMXNUGTNHlfG58ydx74vr+dWCjTzx2lbeN30Et58/ibEVaiEkUm2khV1NbRzfy4yFiaTbFUXS0JDiPL5y8RRevONfuPmc8Ty9chvnf/85vvC7FazaGkl2vLSxckvssz7+mJKk5lCLQCSNlRXkcMe7j+Pjs8Zy97PreGjRJv6wtIbTxg7mhtljuWDqUI3QGqCVWyOYwZThahGISJJVFuXyvy+ZyoKvzOErFx1Hze5mbvrlEj7xiyXUNbQmO96AtXJrPWPKkz9BkwqBiOxTkp/NJ88Zz3NfOpevvmcKz/+jjgt/8DxPvlGb7GgD0sqtEaYmuX8AVAhEpBdZmRnceNY4/nTrbEaU5nHzr5bw1Udfpy164EmA5NDs2dvGlj3NSe8oBhUCEenDxKFFPPLpWXzy7HH8asEmPjrvlX3z68qR6eqUP35EcjuKQYVARA4iOzODr1w8hZ9cOZM3axt4z50vsrB6Z7JjpbyV+wqBWgQikiLec+JwHr1lFkV5WVx5z0J+8ve1dB5oCjc5qJVb6xlanBuK4cNVCESk3yYNLeLxz8zi4mnD+c5Ta7j2/kW6qugwrdwa4YQQnBYCFQIROURFednc+dEZfPPyaSxav4v3/ugFVmzek+xYKaW5rYN1dY2hOC0EKgQichjMjCtOHcUjn55FVkYGH/7pKzy+4sjnb04Xb9ZG6HSYmg4tAjPbYGavm9lyM1vcy/oSM3vCzFaY2Uoz05zFIilk6ohiHv/MLKaPLOWzDy3je0+vITYDrfQlTB3FkJgWwXnuPsPde85dDLH5ile5+3TgXOB78YnuRSRFlBfm8qsbT+MjVcfyo/9Zyxd+v4L2Dt1v0JeVWyOUDMpmZNmgZEcBkj/WkANFFhvMpBDYBUSTG0lEDlVOVgbf+sA0RpYN4nvPvMXupjZ+ctVM8nOS/RUTTqu21jN1eHFoxnEKukXgwNNmtsTMbupl/Y+BKcBW4HXgNnff708JM7vJzBab2eK6urpgE4vIYTEzbp0zkW9ePo3n3qrjyp8tZFdTW7JjhU5Lewer325g2shw9A9A8IVgtrvPBC4CbjGzs3usvxBYDowAZgA/NrP9Tpq5+zx3r3L3qsrKyoAji8iRuOLUUdx99cmsejvC5Xe9RHVdY7IjhcrSTbtp6+jk9HGDkx1ln0ALgbtvif/cDjwCnNpjk+uBhz1mLbAeOC7ITCISvAuPH8ZDnzidSEuUy+9+WXcid7Ng3U4yDKrGpEEhMLMCMyvqeg68C3ijx2abgDnxbYYCk4HqoDKJSOKcPLqMRz89i8EFOVx970IeW74l2ZFCYUH1LqYdU0JxXnayo+wTZItgKPCima0AFgF/cvcnzexmM7s5vs03gDPN7HXgb8Ad7r4jwEwikkCjyvN55FOzmDmqjNt+s5z7X1qf7EhJ1dzWwbLNuzl9XHmyo7xDYF367l4NTO9l+dxuz7cSaymIyABVkp/Nzz9+Kp99aBn/8cQqdjW18fkLJoXmiplEWrppN+0dHrpCoDuLRSRwedmZ3HXVTD56Suxeg3979A060nDAugXVO8nMMKrGlCU7yjvoIl8RSYiszAy+efk0ygpyuPvZdTS1Rvnuh6aTnZk+f4++sm4nJxxTQlGI+gdAhUBEEsjMuOPdx1Gcl81/P/kmTa1RfnzlTPKyM5MdLXB726KsqNnDDbPHJTvKftKnFItIaHzq3PF849Lj+evq7Xz8gVdpah34Awos2djVPxCey0a7qBCISFJ87IwxfP/D01lQvZNr71tEQ0t7siMFqqt/4JQQ3T/QRYVARJLm8pkj+fGVM1m+eQ9X37uI+r0Dtxi8sm4nJ44soSA3fGfkVQhEJKkunjacuVefzOqtEa742YIBOT5RU2uU12rqQ3fZaBcVAhFJuvOnDuVn11axrq6Rq+9ZyJ69A6sYvLR2B9FOZ/aEimRH6ZUKgYiEwjmTKpl3TRVrtzdyzX2LiAygPoO/rt5GUV4Wp44NX/8AqBCISIicM6mSu6+eyeq3IwOmA7mj0/nb6u2cN3lIaO+ZCGcqEUlbc6YM5UdXzOS1mnquumchOxpbkx3piCzbtJudTW2cP3VosqMckAqBiITOu08Yxk+vPpm3tjXwgbtfZsOOpmRHOmzPrN5GdqZx7uTwzqWiQiAioXT+1KH8+hOnE2lu5wN3v8yKzXuSHemwPLNqG6ePKw/VsNM9qRCISGjNHFXG/E+dyaCcTK742QJeXpdao9Svq2ukuq6JC0J8WghUCEQk5MZXFvKHT53JyLJBXHf/q/x11bZkR+q3rqxzpqgQiIgckaHFefz2pjOYMqyIT/5qScrMdvbMqm0cP6KYY0oHJTtKn1QIRCQllBXk8OAnTueUMWXc/tvlPLy0JtmR+rSjsZUlm3aH/rQQqBCISAopzM3igetP5Yxx5Xzx9ytC3TJ4amUt7qgQmNkGM3vdzJab2eIDbHNufP1KM3suyDwikvrysjO599pTOHXsYD732+U8sWJrsiPtx925/6UNHD+imKnDi5Md56AS0SI4z91nuHtVzxVmVgrcBbzP3Y8HPpSAPCKS4gblxIpB1ejB3P7b5aFrGTz3Vh1rtzdy41ljU2Ju5mSfGroSeNjdNwG4+/Yk5xGRFFGQm8V915/CyaNjfQa/e3VzsiPtc++L6xlanMt7po1IdpR+CboQOPC0mS0xs5t6WT8JKDOzZ+PbXBNwHhEZQApzs/j59acye0IF//qH1/j5yxuSHYk3ayO88I8dXHvmGHKykv23dv8EnXK2u88ELgJuMbOze6zPAk4G3gNcCHzNzCb13ImZ3WRmi81scV1dXcCRRSSVDMrJ5J5rq7hg6lC+/vhKfvL3tbh70vLc+8J6BmVncuWpo5KW4VAFWgjcfUv853bgEeDUHpvUAE+5e5O77wCeB6b3sp957l7l7lWVleEdr0NEkiM3K5O7rprJZTNG8J2n1vCff1pNZ2fii0FdQyuPLd/Kh6pGUhh7BP8AAAnWSURBVJqfk/DjH67ACoGZFZhZUddz4F3AGz02ewyYbWZZZpYPnAasDiqTiAxc2ZkZfP/DM7juzDHc++J6vjh/Be0dnQnN8ItXNtDe2cn1s8Ym9LhHKsjJM4cCj8R7zLOAX7v7k2Z2M4C7z3X31Wb2JPAa0Anc4+49i4WISL9kZBhfv2Qqgwty+P4zb7GrqY0fXXESRQkY8G3P3jYeeGkDF04dxtiKgsCPdzQFVgjcvZreT/PM7fH6O8B3gsohIunFzPjsnIlUFuXytUff4P13vcw911QxJuAv53nPV9PYFuVzF+zXzRl6qdGlLSJyiK44dRS/vOE0djS2culPXuKltcGNXLqjsZX7X9rAJSeOYPKwosCOExQVAhEZsM4YX87jt8xmaHEu19y3iF8t2BjIceY+u47WaAe3nT8xkP0HTYVARAa0UeX5/OFTZ3LOpEq++ugb/PvjK4kexU7kbZEWfrlgI5fPHMn4ysKjtt9EUiEQkQGvKC+bn11TxQ2zx/LAyxv4+M8Xs3VP8xHv1935wV/foqPTuW1OarYGQIVARNJEZobxtfdO5ZuXT2PBup2c+91n+eafV1O/t/2w9tfUGuX23y7noUWbufr00Rw7OP8oJ06cIC8fFREJnStOHcVZEyv4/jNvMe+Fah5atIlPnjOe62eNIT+nf1+J/9jWwKceXEp1XSNfuGASt5w3IeDUwbJk3op9OKqqqnzx4l5HtBYROSSr347wnafW8D9vbqeiMIdPnzuBC08YRlaGYQYFOVkU5Ga9Y/sHXtrAI8u3UJyXxZ0fPYkzJ1Qk8TfoPzNb0tso0KBCICLCko27+O5Tb/FK9c791lUU5jC6vAB3Z+mmPeRlZ3D5zJHcPmciQ4rzkpD28KgQiIj0w6sbdlFd10hHJ3S6E2lpZ9POvWzY2USkOcqlM0bwkVOOTalxhLr0VQjURyAiEnfKmMGcMmZwsmMknK4aEhFJcyoEIiJpToVARCTNqRCIiKQ5FQIRkTSnQiAikuZUCERE0pwKgYhImku5O4vNrA7oml2iBKjv43nPZdnAoU5T1H0f/VnXc1l/M3b9rDjEjInK17VMn2G48qVCxrDnO5KMfS0L22c42t0re927u6fsA5jX1/Oey4DFR3KM/qzruay/Gbv9PKSMicqnzzCc+VIhY9jzHUnGg2QN1WfY1yPVTw09cZDnB1p/uMfoz7qey/qbMez5DnasvugzPPhx+nKw94U9Y9jzHWh9fzIebNmhCPozPKCUOzV0JMxssR9g0KWwCHvGsOeD8GcMez4If8aw54PUyNgl1VsEh2pesgP0Q9gzhj0fhD9j2PNB+DOGPR+kRkYgzVoEIiKyv3RrEYiISA8qBCIiaU6FQEQkzakQiIikORWCODM7y8zmmtk9ZvZysvP0xswyzOy/zOxHZnZtsvP0ZGbnmtkL8c/x3GTn6Y2ZFZjZYjN7b7Kz9MbMpsQ/v/lm9qlk5+mNmV1mZj8zs9+a2buSnacnMxtnZvea2fxkZ+kS/3f38/jndlWy8/Q0IAqBmd1nZtvN7I0ey99tZmvMbK2Zfbmvfbj7C+5+M/BH4OdhzAhcCowE2oGaEOZzoBHIC2k+gDuA3x3NbEczo7uvjv87/DAwK6QZH3X3TwA3Ax8JYb5qd7/haObqzSFmvRyYH//c3hd0tkN2KLdAh/UBnA3MBN7otiwTWAeMA3KAFcBUYBqxL/vujyHd3vc7oCiMGYEvA5+Mv3d+CPNlxN83FHgwhPkuAD4KXAe8N4z/jePveR/wF+DKsGaMv+97wMwQ5zuq/48cYdavADPi2/w6yFyH88hiAHD3581sTI/FpwJr3b0awMx+A1zq7t8Eej0tYGajgHp3bwhjRjOrAdriLzvClq+b3UBu2PLFT1cVEPsfs9nM/uzunWHKGN/P48DjZvYn4NdHK9/RymhmBnwL+Iu7Lw1bvkQ5lKzEWsgjgeWE8EzMgCgEB3AMsLnb6xrgtIO85wbg/sAS7e9QMz4M/MjMzgKeDzJY3CHlM7PLgQuBUuDHwUYDDjGfu/8bgJldB+w4mkWgD4f6GZ5L7DRCLvDnQJP906H+O7wVOB8oMbMJ7j43yHAc+mdYDvwXcJKZfSVeMBLlQFnvBH5sZu/h8MciCsxALgSHzN2/nuwMfXH3vcSKVSi5+8PEilWoufsDyc5wIO7+LPBskmP0yd3vJPbFFkruvpNY/0VouHsTcH2ycxxI6JooR9EW4Nhur0fGl4VJ2DMq35FTxiMX9nzdpVLWfQZyIXgVmGhmY80sh1gn4eNJztRT2DMq35FTxiMX9nzdpVLWf0p2b/VR6r1/CHibf15WeUN8+cXAW8R68f9NGZVPGcOdMez5UjXrwR4afVREJM0N5FNDIiLSDyoEIiJpToVARCTNqRCIiKQ5FQIRkTSnQiAikuZUCGRAMLPGBB/vqMxZYbE5HOrNbLmZvWlm3+3Hey4zs6lH4/gioEIg0isz63McLnc/8yge7gV3nwGcBLzXzA42D8FlxEZQFTkqVAhkwDKz8Wb2pJktsdjMacfFl19iZgvNbJmZ/dXMhsaX/7uZ/dLMXgJ+GX99n5k9a2bVZvbZbvtujP88N75+fvwv+gfjwzRjZhfHly0xszvN7I995XX3ZmLDFB8Tf/8nzOxVM1thZn8ws3wzO5PYfAXfibcixh/o9xTpLxUCGcjmAbe6+8nAF4G74stfBE5395OA3wD/2u09U4Hz3f2K+OvjiA2tfSrwdTPL7uU4JwG3x987DphlZnnAT4GL4sevPFhYMysDJvLPIcYfdvdT3H06sJrYEAYvExu75kvuPsPd1/Xxe4r0i4ahlgHJzAqBM4Hfx/9Ah39OljMS+K2ZDSc2i9T6bm99PP6XeZc/uXsr0Gpm24nNvtZzGs5F7l4TP+5yYAyxKTur3b1r3w8BNx0g7llmtoJYEfiBu9fGl59gZv9JbH6HQuCpQ/w9RfpFhUAGqgxgT/zce08/Ar7v7o/HJ4L5927rmnps29rteQe9/z/Tn2368oK7v9fMxgILzOx37r4ceAC4zN1XxCfTObeX9/b1e4r0i04NyYDk7hFgvZl9CGLTK5rZ9PjqEv45Rvy1AUVYA4zrNpXhQSd5j7cevgXcEV9UBLwdPx11VbdNG+LrDvZ7ivSLCoEMFPlmVtPt8XliX543xE+7rCQ2dyzEWgC/N7MlwI4gwsRPL30aeDJ+nAagvh9vnQucHS8gXwMWAi8Bb3bb5jfAl+Kd3eM58O8p0i8ahlokIGZW6O6N8auIfgL8w93/X7JzifSkFoFIcD4R7zxeSex01E+TnEekV2oRiIikObUIRETSnAqBiEiaUyEQEUlzKgQiImlOhUBEJM39f0t/xlMH2EV0AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "lang_learn.lr_find()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>perplexity</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>4.869072</td>\n",
       "      <td>4.222111</td>\n",
       "      <td>68.177238</td>\n",
       "      <td>00:15</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>perplexity</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>3.882690</td>\n",
       "      <td>3.805628</td>\n",
       "      <td>44.953487</td>\n",
       "      <td>00:17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>3.605679</td>\n",
       "      <td>3.606257</td>\n",
       "      <td>36.827946</td>\n",
       "      <td>00:18</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>3.268116</td>\n",
       "      <td>3.602493</td>\n",
       "      <td>36.689598</td>\n",
       "      <td>00:18</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "lang_learn.fine_tune(3, 5e-2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "lang_learn.save_encoder('fine_tuned_enc')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       ""
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.7/dist-packages/numpy/core/_asarray.py:83: VisibleDeprecationWarning: Creating an ndarray from ragged nested sequences (which is a list-or-tuple of lists-or-tuples-or ndarrays with different lengths or shapes) is deprecated. If you meant to do this, you must specify 'dtype=object' when creating the ndarray\n",
      "  return array(a, dtype, copy=False, order=order)\n"
     ]
    }
   ],
   "source": [
    "dls = TextDataLoaders.from_df(train_df, seed=42, text_col='text', label_col='target', y_block=CategoryBlock(), is_lm=False, text_vocab=dls_lm.vocab)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn = text_classifier_learner(dls, arch=AWD_LSTM, metrics=accuracy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<fastai.text.learner.TextLearner at 0x7f621643a7d0>"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learn.load_encoder('fine_tuned_enc')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       ""
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "SuggestedLRs(lr_min=0.017378008365631102, lr_steep=0.007585775572806597)"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEKCAYAAAAIO8L1AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deXxU9dX48c+ZrCQkbEkAE3bCEkCCREAQxR1XcEPQWvel1tal6qOtj1prn1rrVkWtWMVdVIqU/kSRuoEISpBF1hDClrAlAbJC1vP7YyY4hoEkJDczk5z36zUvM99779yTOMyZ7y6qijHGGFOby98BGGOMCUyWIIwxxvhkCcIYY4xPliCMMcb4ZAnCGGOMT5YgjDHG+BTq7wCaSlxcnPbs2dPfYRhjTFBZtmxZnqrG+zrWYhJEz549SU9P93cYxhgTVERk65GOWROTMcYYnyxBGGOM8ckShDHGGJ8sQRhjjPHJEoQxxhifLEEYY4zxyRKEn23KLeZAeZW/wzDGmMNYgvCjeWt2cfYzC7hjxnJ/h2KMMYexBOGA7XtL+fPHaykpqzziOQs35vKbd5cTFR7CZ2t3811WfjNGaIwxdbME0cRUlf/51ypeWbiZd77zPUExfctebn5zGb3jo5l/16l0bRfJ/81dR3W17e5njAkcjiYIERkvIhtEJFNE7vdx/BkRWeF5ZIjIfq9j3UXkMxFZJyJrRaSnk7E2lTkrd/DtpnxiI0N57ZstlFdW/+z4hl1FXDd9KV3aRfLWDSPp0i6Se87uz8rsAv6zaoefojbGmMM5liBEJAR4ATgXSAGmiEiK9zmqepeqpqpqKvA8MMvr8JvA31R1IDAC2ONUrE2l8GAFj328jqFJ7Xh2ciq7Cg/yn5U/fehXVSv3zlxJRJiLt28cSXxMBAAXD0skpWssT3y6gYMV1mFtjAkMTtYgRgCZqpqlquXADGDCUc6fArwH4Ekkoao6H0BVi1W11MFYm8TTn2WQV1zGYxOHcFr/BPp3juGVhVmoupuO3ly8hVXZBTx04SAS27c5dJ3LJfzh/IHk7D/A0/Mz+OfCLK557XuGPDyP619fyuqcAj/9RsaY1szJBJEIbPd6nu0pO4yI9AB6AV94ivoB+0VklogsF5G/eWokta+7WUTSRSQ9Nze3icNvmNU5Bby5eAtXj+rBkKR2iAg3ndKb9buK+Dojlx37D/DkvA2c2i+eC4/vetj1Y/rGcVr/eKYtyOKxj9eRva+Uswd1YdnWfVzw/Dfc9s4yNuUWN/8vZoxptQJlue/JwExVrWlfCQXGAsOAbcD7wLXAq94Xqeo0YBpAWlqaX3t4n5i3gY7R4fzu7P6Hyi4aehxPztvAy19nERMZSpUqj00cjIj4fo3LhrIoM48Te3U8VMMoOFDBq99s5tWFWXy7KZ+F951GTGRYs/xOxpjWzckaRA7Qzet5kqfMl8l4mpc8soEVnuapSmA2cIIjUTaRNTkFnDmwM+3a/PThHR7q4roxPVmclc9na3dz55n96NYx6oivER8TwcRhiT9rfmrXJoy7z+rHuzeNYn9pBe99v83R38MYY2o4mSCWAski0ktEwnEngTm1TxKRAUAHYHGta9uLSM0uR6cDax2MtVEKD1aQX1JOj07Rhx2bMrI7MRGhDOgSww0n9zrmewzt1p4xfTvxz4WbKasMjo7stTsK+XjVzkN9MMaY4OJYgvB8878dmAesAz5Q1TUi8qiIXOR16mRghnp9iniamu4BPheRHwEBXnEq1sbamufuP+8Vd3jtIDYyjDm/OZkZN48iLKRxf+5fndqXPUVlfPTDkSpisK+knKlfbKTgQEWj7tVYG3YVccW0xfz63R94YNaPhw33BSxxGBPgHO2DUNW5wNxaZQ/Vev7IEa6dDxzvWHBNaEt+CYDPGgRArzjf5Q01pm8nhiS24+UFWVye1o0Q18/7MnYVHOTqV79j455iyiurudurP6Q57dh/gGunf0+bsBAuG57E9EVbyMot4aVfnECVKjOXZfP+0u3ERIby1vUj6RAd7pc4jTFHZzOpm8CWPHeC6HmEBNFURIRbT+3D5rwSPluz67AYLvvHt+wsOMjArrG8+/02vzRFFZRWcO307yk+WMnr143g4QsH8ffJqazM3s+ZT3/N6L98wROfbiAhJoKM3cVc9c/v2F9a3uxxGmPqFiijmILa5vwSusRG0ib8sJG4TW784C707BTFS19v4qyUzmzfd4B1Owt56N9rqFblvZtGsbe0nGte+55PV+9iQqrPkcVN5s3FW/gwPZs24SHERISyfV8pm/NKeOO6EaQcFwvAhNREenaK5snPNjCwayxXnNiNPvFt+Tojl5veTOeqf37HOzeOJDIshP+u281HP+TQvVMUD56fclgtyRjTfKSltAOnpaVpenq6X+596UvfEuoS3r/lpGa533vfb+OBWT8SFiJUVLn//x3XLpI3bxhJ34S2VFcrZzz9NR2iwph12xhHYqiuVh77eB2vLdrM4MRYosNDKSmvpLyymrvO7Me5Qw6f6+HLVxv2cPOby+jaPpJ9JeUUHqykU3Q4+SXlnDekC89ckUpEqPOJ15jWSkSWqWqar2NWg2gCW/JKOCulc7Pd75ITElm/s5DIsBD6JLQlOaEtA7rEHqrBuFzC1aN68Oj/W8uP2QUMSWrXpPc/WFHF7z5Yycc/7uTa0T353wuO/Zv+uP4JvHz1cH7/0Y+cPiCBy4Z346Q+nZi+aDOPfbyOooPpvHz1cKLC7a1qTHOzf3WNVDPEtWcTdUTXR0RoCH+cMPio51yWlsSTn23gzcVb+NvlQwHYWXCA2ct30DE6jO4do+neKYrj2kUeceKeLxVV1Vw3fSmLs/L5w3kDuXFsrwZd78tpAxJY/MAZPyu7cWxvYiPDuH/WKq7653c8fOEgUru1b9R9jDENYwmikWqGuPbsdOQJcP4QGxnGJSck8kF6NveO78+/l+/gmf9mUFpr97qJqcfx7ORh9X7dVxZmsTgrnycuO55Jad3qvqARJp3YjZjIUO6duYqJLywitVt7rh3dk4SYCFbvKGDNjkJKyiq566x+DDquaWtJxhhLEI222TPEtTlrEPX1y5N68vaSbZzx1NcUHazk9AEJPHj+QEJdLrbtLeWT1Tt557ttnDekK2cP6lLn62XlFvPsfzcyflAXx5NDjXOHdOXk5Dhm/ZDDG4u3cOf7Kw4d69oukvLKaiZMXcRvz0jmV+P6NHquiTHmJ5YgGmmrZ4hrj46BlyD6dY7hjAEJrNtZyJOXD+XslM6HmoO6d4piZO+OpG/ZxyNz1jCmbxzREUd+O1RXK/fP+pHIUBePThjUXL8CADGRYVwzuidXj+rBd5v3UlFVzeDEdnSMDmdfSTkPz1nD0/MzmL92N89PGRaQydqYYGRftxqpOYe4Hotpv0xj0f2nc86gLof1FYSFuPi/Swazo+Agz8zPOOrrvPv9Nr7fvJcHz08hITbSyZCPyOUSTurTiVP6xdPRM7muQ3Q4z00ZxotXncD2faVc/dp37C2xeRXGNAVLEI20Nb+Unj6W2AgUIS45aify8B4dmTKiO9O/3eJz3wlVZeX2/Tz+yXrG9O3E5WlJToZ7zM4b0pXp157I7sIybn17mc+lPYwxDWNNTI3U3ENcnXD/+AHMX7uLez5cyeQTu9EuKoy2EWH8sG0fn/y4ky35pcREhPKXi49v9IglJw3r3oG/XXY8d8xYwUP/Xs1fLhlyKF5VDejYjQlEliAawR9DXJ3QLiqMxyYO5rczVvDIf35aNDfEJYzu04mbT+nDOYM606lthB+jrJ8JqYls3F3M1C8ziQwLoVqVFdv3s35XEecN7sJjFw+h7VH6WowxP7F/KY0QqENcj8X4wV1Z88fOFB6ooOBABYUHK+neMepQW38wufusfmTuKeb1b7cQHR7C8UntufD44/hoeTarsguYeuUJh5YBMcYcmSWIRgjkIa7HIizERae2EUFRUzgal0t4/sphbM0vpVdc9KFZ3penJfHb95Yz8cVFPHRBCleN7G7NTsYchXVSN0IgD3Ft7cJCXPRNaPuzJUBG9e7E3DvGMrJXRx6cvZrrXl/KroKDfozSmMBmCaIRAn2IqzlcXNsI3rhuBH+8aBBLsvI5+5mvmb08xzYvMsYHRxOEiIwXkQ0ikiki9/s4/oyIrPA8MkRkf63jsSKSLSJTnYzzWG3JKwnoIa7GN5dLuGZ0Tz654xSSO8dw5/sruPSlb/li/W5LFMZ4cSxBiEgI8AJwLpACTBGRFO9zVPUuVU1V1VTgeWBWrZf5E7DAqRgbq6aN2wSnXnHRfHDLSTw2cTC7C8u4/vV0Lnj+Gz5ft9vfoRkTEJysQYwAMlU1S1XLgRnAhKOcPwV4r+aJiAwHOgOfORjjMasZ4nqkbUZNcAhxCb8Y1YOv7h3HE5cdT2l5FTe8kc7vPlhJ4UH/7uttjL85mSASge1ez7M9ZYcRkR5AL+ALz3MX8BRwz9FuICI3i0i6iKTn5uY2SdB1Kaus4qsNe/jjHPd8Aae3GTXNIyzExaS0bnx21yn89vS+fLQ8m3OfXciSrHx/h2aM3wTKMNfJwExVrVmL+jZgrqpmH20YoqpOA6aBe0c5p4OcvmgzT87bQEl5FW3CQjh3cBdO6t3J6duaZhQW4uLus/szbkACd7+/gsnTltArLpph3dqT2r09Z6d0oUs7/6xFZUxzczJB5ADea0Inecp8mQz82uv5ScBYEbkNaAuEi0ixqh7W0d2c5v64k7iYCKZeOIiT+nQiMsxGL7VUJ3TvwNw7xvL2kq0s3bKPBRvzmLU8hxe/3MSnd46lfVTwTSA0pqGcTBBLgWQR6YU7MUwGrqx9kogMADoAi2vKVPUqr+PXAmn+Tg4AuUVlHJ/UntMGJPg7FNMMosJDufmUPtx8instp/St+5gybQl/mL2aqVOG2SQ70+I51gehqpXA7cA8YB3wgaquEZFHReQir1MnAzM0CMYX5haVEdfWvjm2RiLCiT07ctdZ/fh41U5mrzhSZdiYlsPRPghVnQvMrVX2UK3nj9TxGq8DrzdxaA1WWl5JSXkV8THBvQyFaZxbT+3DVxv28NDsNaT16Ei3jjYPxrRcNpO6nvKK3JvQxAf5OkWmcUJcwtOTUlHgrvdXsDqngIMVVXVeZ0wwCpRRTAEvt9i9Zo/VIEy3jlE8NnEwd76/ggue/4YQl9ArLprrx/TiypHd/R2eMU3GEkQ95RaVAe61fIyZOCyRE7p3YPWOAtbtLGRBRi4Pzv6RnnFRjO4T5+/wjGkS1sRUT7nF7iamBKtBGI/unaI4b0hXfnd2f969aRS94qK5Y8aKQ18mjAl2liDqKbeoDBGCcgMd47zoiFBevGo4RQcruGPGcqqqA35QnjF1sgRRT7lFZXSKDic0xP5kxrf+XWJ49KLBfLspn+e/2OjvcIxpNPu0qyf3HAhrXjJHd3laEpeckMjfP9/I6pwCf4djTKNYgqinvOIyG8Fk6iQiPHLRINq1CeOvn673dzjGNIoliHrKLSqzORCmXmIjw7j9tL4s3JjHosw8f4djzDGzBFEPqkqu1SBMA/xiVA8S27fh8U/WU20d1iZIWYKoh8KDlZRXVlsfhKm3yLAQ7jqrHz/mFDB39U5/h2PMMbEEUQ95xe5x7VaDMA1x8bBE+neO4cl5G6ioqvZ3OMY0mCWIeqiZ+GQJwjREiEu4b3x/tuSX8vaSrf4Ox5gGswRRD7bMhjlWpw9IYGxyHI9/sp71uwr9HY4xDWIJoh6sickcKxH36q+xbcL49Ts/UFJW6e+QjKk3RxOEiIwXkQ0ikikih+0IJyLPiMgKzyNDRPZ7ylNFZLGIrBGRVSJyhZNx1iW3qIxQl9C+TZg/wzBBKj4mgr9fkUpWXgkPzl5NEOyNZQzgYIIQkRDgBeBcIAWYIiIp3ueo6l2qmqqqqcDzwCzPoVLgl6o6CBgPPCsi7Z2KtS41s6hdLtti0hyb0X3juOOMZD5ansOH6dn+DseYenGyBjECyFTVLFUtB2YAE45y/hTgPQBVzVDVjZ6fdwB7gHgHYz2q3OIy4mJskT7TOL85PZnRfTrxh9k/8vgn6ym25iYT4JxMEInAdq/n2Z6yw4hID6AX8IWPYyOAcGCTj2M3i0i6iKTn5uY2SdC+5BXbLGrTeCEu4YUrT+CioYn84+tNnPbkV3yQvt0m0pmAFSid1JOBmar6s70bRaQr8BZwnaoeNpBcVaepapqqpsXHO1fByC2yWdSmaXSIDuepSUOZ/esxJHVow30zVzH1y0x/h2WMT04miBygm9fzJE+ZL5PxNC/VEJFY4GPgD6q6xJEI66G6WskrLrcEYZpUarf2zPrVaE7tF8+bi7faRDoTkJxMEEuBZBHpJSLhuJPAnNonicgAoAOw2KssHPgIeFNVZzoYY532H6igqlptDoRpciLCNaN7kFdcxvy1u/0djjGHcSxBqGolcDswD1gHfKCqa0TkURG5yOvUycAM/fnYv0nAKcC1XsNgU52K9WhsFrVx0qn9Ekhs34Z3vrOZ1ibwhDr54qo6F5hbq+yhWs8f8XHd28DbTsZWX4cShNUgjANCXMKVI7vzt3kbyMotpnd8W3+HZMwhgdJJHbByiw8CVoMwzrk8LYlQl/De99v8HYoxP2MJog55ReUAxFmCMA5JiInknEFd+HBZNgcrquq+wJhmYgmilu17S0nfsvfQ89ziMiJCXcREONoaZ1q5q0Z2Z39pBXN/tL0jTOCwBFHLc59v5Mp/fse2/FLgpzkQIrbMhnHOSX060TsumreWbLW1mkzAsARRS8GBCsorq/nz3LWATZIzzUNEuP7kXizftp9Xv9ns73CMASxBHKak3L0+zrw1u1mUmUdecZnNgTDN4qqR3Tk7pTOPf7KepV7NnMb4iyWIWorLqhjVuyPdO0bxx/+sYXfhQatBmGYhIjw5aShJHdrw63d+ODTE2hh/sQRRS0lZJR2jw/nD+QPJ2F3MvtIKmwNhmk1sZBgv/WI4hQcr+M17P1BpS3AYP7IEUUtpWSXR4aGcndKZk/vGATYHwjSvgV1j+fPEISzJ2sv1b6Szp/Cgv0MyrZQliFqKyyqJjghFRHj4whTiYyIYdFysv8Myrcylw5N4bOJgvsvK55xnF/Dp6l3+Dsm0QpYgvKgqJeVVREeEAJDcOYbvf38Gw7p38HNkpjX6xagefPzbsSR2aMOtby/j9x/9SJXtHWGakSUIL2WV1VRVK9Fek+Js/oPxp74JbZn1qzHcckpv3v1uG/d+uNKShGk2Nj3YS4lnC8i2NmvaBJDwUBcPnDeQthGhPDU/AxHhicuOJ8T2SDcOs09CLyVl7nVwosLtz2ICz2/OSKZa4Zn/ZuAS+Oulx+OyJGEcZJ+EXooP1SBC/ByJMb7dcWYy1ar8/fONdI6N5J5z+vs7JNOCWYLwUjOLOtqamEwAu/PMZHYVHGTql5mkdmvPmSmd/R2SaaEc7aQWkfEiskFEMkXkfh/Hn/HaMS5DRPZ7HbtGRDZ6Htc4GWeNmj4ISxAmkIkIf5wwiMGJsdz1wYpDC0sa09QcSxAiEgK8AJwLpABTRCTF+xxVvUtVU1U1FXgemOW5tiPwMDASGAE8LCKOjzWt6YOItj4IE+Aiw0J46arhuES49e1lto+EcYSTNYgRQKaqZqlqOTADmHCU86cA73l+PgeYr6p7VXUfMB8Y72CsgHcNwvogTODr1jGKZ69IZd2uQh7+9xp/h2NaICcTRCKw3et5tqfsMCLSA+gFfNGQa0XkZhFJF5H03NzcRgdcbMNcTZA5bUACt57ah/fTt7Mgo/H/BozxFigT5SYDM1W1QfVkVZ2mqmmqmhYfH9/oIEo9ndQ2zNUEkzvOSKZ3fDS//+jHQ+9hY5qCkwkiB+jm9TzJU+bLZH5qXmrotU2muKyK8BAX4aGBkjeNqVtkWAiPX3I82fsO8NRnGf4Ox7QgTn4SLgWSRaSXiITjTgJzap8kIgOADsBir+J5wNki0sHTOX22p8xRJWWV1v9ggtKIXh35xajuTF+0mRXb99d9gTH14FiCUNVK4HbcH+zrgA9UdY2IPCoiF3mdOhmYoV4b8arqXuBPuJPMUuBRT5mjSjwruRoTjO4bP4CEmEju/9cqyittHwnTeI62pajqXFXtp6p9VPXPnrKHVHWO1zmPqOphcyRU9TVV7et5THcyzhol5ZXWQW2CVmxkGI9NHMz6XUVcO/17CkorfnY8v7iMf6/IYX9puZ8iNMHGGtu9lJRVERVuTUwmeJ2Z0pmnLh/K0i17ueSlRWzLL6Wyqpo3vt3CaU9+xR0zVjDm8S/4y9x17CmyjYjM0dnXZS/FZZXERNqfxAS3S4cnkdihDbe8tYyLX1xEfEwE63cVMaZvJ248uTcfLc/hlYVZvP7tFh66MIWrRvbwd8gmQFkNwktJmTUxmZZhVO9OzLptNDGRoRQeqOClq07g7RtGctqABJ6bMowvfjeO4T068Oh/1tpSHeaILEF4KS2vsk5q02L0iW/LZ3edyoL7TuPcIV1/tvlVz7honp6USqhL+ON/bBa28c0ShJfiskqirQ/CtCDhoS5CQ3z/M+/SLpI7z+zH5+v3MH/t7maOzASDeiUIEYkWEZfn534icpGIhDkbWvNSVRvmalqda8f0pF/ntjwyZw0Hym3BP/Nz9a1BLAAiRSQR+Ay4GnjdqaD8oayymspa+1Eb09KFhbh4dMJgcvYf4MWvMv0djgkw9U0QoqqlwCXAi6p6OTDIubCaX6nn25N1UpvWZlTvTkxMPY6Xv85ix/4D/g7HBJB6JwgROQm4CvjYU9aiGutrlvq2eRCmNbrnnP5UVrvnSxhTo74J4k7gAeAjz3IZvYEvnQur+dlS36Y1S+oQxblDuvLu99sO/Vswpl4JQlW/VtWLVPWvns7qPFX9rcOxNSvbbtS0djeN7U3RwUreX7q97pNNq1DfUUzvikisiEQDq4G1InKvs6E1rxJPH4QlCNNapXZrz4k9O/DaN5uprLLF/kz9m5hSVLUQmAh8gnv3t6sdi8oPbLtRY+DGsb3J2X+AT9fs8ncoJgDUN0GEeeY9TATmqGoFoHVcE1Rq2l2jbTc504qdObAzPTpF8crCzXitwG9aqfomiJeBLUA0sMCzh3ShU0H5Q4l1UhtDiEu44eRerNy+n6Vb9vk7HONn9e2kfk5VE1X1PHXbCpxW13UiMl5ENohIpogctueD55xJIrJWRNaIyLte5U94ytaJyHPivZCMA0qtD8IYAC4bnkRc23BueGMpH6Zvt5pEK1bfTup2IvK0iKR7Hk/hrk0c7ZoQ4AXgXCAFmCIiKbXOScY9fHaMqg7CPZwWERkNjAGOBwYDJwKnNug3a6DiskrCQsT2ozatXlR4KDNvHc3ALrHcO3MVN7yRzq4C2zuiNarvp+FrQBEwyfMoBOra5W0EkKmqWapaDswAJtQ65ybgBVXdB6CqezzlCkQC4UAEEAY4upqYrcNkzE96xkUz4+ZRPHRBCt9uyuO85xays8BmWbc29U0QfVT1Yc+HfZaq/hHoXcc1iYD3gOpsT5m3fkA/EVkkIktEZDyAqi7GPRFvp+cxT1XX1TPWY+JeydUShDE1XC7h+pN7Mef2kzlYUcU9H66kutqam1qT+iaIAyJycs0TERkDNMXXiVAgGRgHTAFeEZH2ItIXGAgk4U4qp4vI2NoXi8jNNc1eubm5jQqktKzKOqiN8aFf5xj+94IUFmXm89qizf4OxzSj+iaIW4EXRGSLiGwBpgK31HFNDtDN63mSp8xbNp5hs6q6GcjAnTAuBpaoarGqFuOee3FS7Ruo6jRVTVPVtPj4+Hr+Kr6VlFcSZXMgjPFp8ondOHNgZ56Yt4H1u1rUAEZzFPUdxbRSVYfi7jQ+XlWHAafXcdlSIFlEeolIODAZmFPrnNm4aw+ISBzuJqcsYBtwqoiEeuZfnAo43sRkNQhjfBMRHr90CLGRodw5YwVllbZ3RGvQoCE7qlromVENcHcd51YCtwPzcH+4f+BZ6O9REbnIc9o8IF9E1uLuc7hXVfOBmcAm4EdgJbBSVf/TkFgbqsT6IIw5qri2ETxx2fGs31XE45+s93c4phk05hOxznkJqjoXmFur7CGvnxV3orm71jlV1N2E1aRKymw/amPqcvqAzlw7uifTF21hZK+OjB/c1d8hGQc1ZtB/ixrOUFJeaeswGVMPvz9vIEOT2nHvzFVsyy/1dzjGQUdNECJSJCKFPh5FwHHNFGOzsHkQxtRPeKiLqVeegAC3vbuMgxXWH9FSHTVBqGqMqsb6eMSoaov5NC2rrKKiSq2T2ph66tYxiqcmpbI6p5DHPl7r73CMQ2xdCdxzIACibbtRY+rtrJTO3HJqb95eso23lmz1dzjGAZYg+Gmp7yirQRjTIPedM4AzBiTwyJw1fJ3RuMmqJvBYgsDdQQ221LcxDRXiEp6bMox+nWP49Ts/sGFXkb9DMk3IEgS2H7UxjREdEcqr16QRFR7C9a8vJa+4zN8hmSZiCQL3HAiAtjbM1Zhjclz7Nrx6zYnsKjzIKwuz/B2OaSKWIPipBhFlM6mNOWZDktpxWv8EZv2QQ2VVtb/DMU3AEgQ/dVJbH4QxjTMpLYncojLrsG4hLEFgfRDGNJXTBiQQ1zacD9K3132yCXiWIICSQ/tRWx+EMY0RFuLi4mGJfL5uj3VWtwCWIHDXIEJdQniI/TmMaazL07pRWa3MXl57+xcTbOwTkZ/WYRKpc4FaY0wd+nWOIbVbez5I3457wWYTrCxBAMW23agxTWpSWjcydhezKrvA36GYRrAEAZTaUt/GNKkLhnYlMszFjKXWWR3MHE0QIjJeRDaISKaI3H+EcyaJyFoRWSMi73qVdxeRz0Rkned4T6fiLC6rtDkQxjSh2MgwJqYmMmPpNqYv2uzvcMwxcuxTUURCgBeAs4BsYKmIzFHVtV7nJAMPAGNUdZ+IJHi9xJvAn1V1voi0BRybeVNi+1Eb0+QeuWgQ+0rL+eN/1rKnqIz7zulv/XxBxskaxAggU1WzVLUcmAFMqHXOTcALqroPQFX3AIhIChCqqvM95cWq6tjWVe7tRq2JyZimFHAnNj8AABYrSURBVBkWwotXDefKkd156atN3PPhKpthHWScTBCJgHcDZLanzFs/oJ+ILBKRJSIy3qt8v4jMEpHlIvI3T43kZ0TkZhFJF5H03Nxjn7np3m7UahDGNLUQl/DniYO568x+/OuHbKbZOk1Bxd+d1KFAMjAOmAK8IiLtPeVjgXuAE4HewLW1L1bVaaqapqpp8fHxxxyENTEZ4xwR4Y4zkzkrpTNTv8hkd+FBf4dk6snJBJEDdPN6nuQp85YNzFHVClXdDGTgThjZwApP81QlMBs4walAS8qqrJPaGIc9eP5AKquUv36y3t+hmHpyMkEsBZJFpJeIhAOTgTm1zpmNu/aAiMThblrK8lzbXkRqqgWnA45sfFteWU15VbUt9W2Mw3p0iuamU3oxa3kOy7bu83c4ph4cSxCeb/63A/OAdcAHqrpGRB4VkYs8p80D8kVkLfAlcK+q5qtqFe7mpc9F5EdAgFeciLO03BbqM6a53DauL11iI3lkzhqqq22WdaCTljIVPi0tTdPT0xt8XXFZJdMWZHH6gARSu7V3IDJjjLd/r8jhjhkr+OulQ7jixO7+DqfVE5Flqprm65i/O6n9rm1EKHef1c+SgzHN5KKhx5HWowN/m5dxaKl9E5hafYIwxjQvEeH35w8kr7iMV7+xWdaNNeP7bbzx7RZHXtsShDGm2Z3QvQPjB3Xh5a83kW/7RjTK++nb+XT1Lkde2xKEMcYv7h3fn4OV1Tz/Raa/Qwlaqkrm7mKSO7d15PUtQRhj/KJPfFuuOLEb73y3la35Jf4OJyjtLiyjqKyS5ARLEMaYFubOM5IJdbl46rMMf4cSlDJ2FwHQNyHGkde3BGGM8ZuE2EhuHNuLOSt3sG5nob/DCTob9xQDWBOTMaZlunFsb6LCQ3jNRjQ1WOaeIjpGhxPXNsKR17cEYYzxq3Ztwrj0hCT+vXKHjWhqoI27i+nrUP8DWIIwxgSAa0b3pLyymve+3+bvUIKGqrJxT7FjHdRgCcIYEwD6JrTllH7xvLVkKxW2qVC95BaXUXCgwhKEMablu250T3YXlvGJQ5O+WprM3TUd1M6MYAJLEMaYAHFqv3h6xUUzfZF1VtdHzRBXq0EYY1o8l0u45qQeLN+2nxXb9/s7nIC3cU8xsZGhxMc4M4IJLEEYYwLIZWndiIkIZeoXmbSUrQicsnFPMf06xyAijt3D0QQhIuNFZIOIZIrI/Uc4Z5KIrBWRNSLybq1jsSKSLSJTnYzTGBMY2kaEcvvpffnvut18uCzb3+EEtMw9zq3BVMOxBCEiIcALwLlACjBFRFJqnZMMPACMUdVBwJ21XuZPwAKnYjTGBJ6bxvbmpN6deGTOGrbk2RpNvuQXl7G3pNyxJTZqOFmDGAFkqmqWqpYDM4AJtc65CXhBVfcBqOqemgMiMhzoDHzmYIzGmADjcglPTRpKWIiLO95fYcNefTi0xIaDHdTgbIJIBLZ7Pc/2lHnrB/QTkUUiskRExgOIiAt4Cve+1MaYVua49m34v4uHsHL7fp77fKO/wwk4Tq/BVCPU0Vev3/2TgXFAErBARIYAvwDmqmr20TpgRORm4GaA7t1tb1tjWpLzj+/KVxuSmPplJlXVyp1n9iM81MbVAGTuLiImIpQusZGO3sfJBJEDdPN6nuQp85YNfKeqFcBmEcnAnTBOAsaKyG1AWyBcRIpV9Wcd3ao6DZgGkJaWZkMejGlhHp0wGBF48atNfLkhl2evSKV/F2fb3YNBxu5i+nZu6+gIJnC2iWkpkCwivUQkHJgMzKl1zmzctQdEJA53k1OWql6lqt1VtSfuZqY3aycHY0zL1yY8hCcuG8orv0wjt+ggFz7/DTNtdJPjazDVcCxBqGolcDswD1gHfKCqa0TkURG5yHPaPCBfRNYCXwL3qmq+UzEZY4LTWSmdmXfnKZzYqwP/869VLNyY6++Q/GZfSTl5xWUkOzyCCUBaymSUtLQ0TU9P93cYxhgHFZdVctlL35Kz/wAf3Tba8WGegSh9y14u+8dipl97IqcNSGj064nIMlVN83XMenyMMUGjbUQor157IhGhIVz3+tJWuX9EVq57bkjv+GjH72UJwhgTVBLbt+GVXw5nT2EZt7y1jOKySn+H1Kyy8koICxGSOkQ5fi9LEMaYoDOseweeuSKV5dv3c+UrS8hrRTWJrNxienSKJsTl7AgmsARhjAlS5w3pyiu/HE7G7iIue+lbtuWX+jukZpGVV0LvOOebl8AShDEmiJ0+oDPv3DiK/QcquOSlb9mwq8jfITmqsqqarfkl9I53fogrWIIwxgS54T06MPPWkwDlf2evbtHLhOfsP0BFlVoNwhhj6qtvQgx3nJHM91v28k1mnr/DcUxzjmACSxDGmBZi0ondSGzfhic/y2ixtYhNue5F+qyJyRhjGiAiNITfnN6Xldv388X6PXVfEIQ255XQPiqMjtHhzXI/SxDGmBbj0uFJdO8YxdPzW2YtIiu3hF7N1P8AliCMMS1IWIiLO85IZs2OQuat2eXvcJpcVl4xveOap3kJLEEYY1qYicMS6R0fzTPzN1Jd3XJqEcVllewuLGu2DmqwBGGMaWFCXMIdZySzYXcR89ft9nc4TaZmf+7mGuIKliCMMS3Q+UO60qNTFC9+mdli+iKaewQTWIIwxrRAoSEubjmlDyuzC1iU2TK2mMnKLUEEenRyfpG+GpYgjDEt0qXDE0mIieCFLzP9HUqT2JxXQmL7NkSGhTTbPR1NECIyXkQ2iEimiPjcMlREJonIWhFZIyLvespSRWSxp2yViFzhZJzGmJYnIjSEm0/pzeKsfH7Yts/f4TRaVl5xszYvgYMJQkRCgBeAc4EUYIqIpNQ6Jxl4ABijqoOAOz2HSoFfesrGA8+KSHunYjXGtExTRnSnfVQYL365yd+hNIqqsjm3+VZxreFkDWIEkKmqWapaDswAJtQ65ybgBVXdB6Cqezz/zVDVjZ6fdwB7gHgHYzXGtEDREaFcN7oX/123mzU7CvwdzjHbXVhGSXkVfZpxiCs4myASge1ez7M9Zd76Af1EZJGILBGR8bVfRERGAOHAYV8BRORmEUkXkfTc3Na7ibkx5siuGd2DjtHh3PRGetDuGZGV5x7B1KsZJ8mB/zupQ4FkYBwwBXjFuylJRLoCbwHXqWp17YtVdZqqpqlqWny8VTCMMYdrHxXOWzeMoLSiisnTFgdlkmjuVVxrOJkgcoBuXs+TPGXesoE5qlqhqpuBDNwJAxGJBT4G/qCqSxyM0xjTwg06rh3v3DgyKJOEqvLD1n20CQuhS2xks97byQSxFEgWkV4iEg5MBubUOmc27toDIhKHu8kpy3P+R8CbqjrTwRiNMa2Ed5K4dvr3VFYd1igRcHYVHOSa6UuZtTyHc4d0wdUM+1B7cyxBqGolcDswD1gHfKCqa0TkURG5yHPaPCBfRNYCXwL3qmo+MAk4BbhWRFZ4HqlOxWqMaR0GHdeOxy85nqy8Ej5ZHbiL+akqHy3P5uxnvmbp5r08OmEQT142tNnjkJYyDT0tLU3T09P9HYYxJsBVVytnPv010RGhzLl9DCLN96286GAFy7ft5+S+cUesDZSWV/Lg7NXM+iGHtB4dePLyofR0cHiriCxT1TRfx/zdSW2MMc3K5RJuHNubH3MKWJzVvMtwTP0yk1++9j0Xv7iIldv3H3Z84+4iJkxdxEfLc7jjjGTev+UkR5NDXUL9dmdjjPGTS05I5On5G5i2IIvRfeKa7b5fb8ilZ6codhQcZOKLi7girRsDu8ayu/AguwoP8smPu4gKD+Gt60dycnLzxXUkliCMMa1OZFgI15zUk6fmZ7BhVxH9u8Q4fs9dBQdZv6uIB84dwJUju/P3/25k+rdbqKpWQl1CQkwEY5Pj+NPEwXRu5tFKR2IJwhjTKv1iVA9e/GoT0xZk8dQk5zuAF2S4J/Oe2j+emMgwHrwghV+N60O1Qqfo8GYfoVQf1gdhjGmVOkSHc8WJ3ZizMoedBQccv9/XG3PpHBtB/84/1VY6tY0gPiYiIJMDWIIwxrRiN5zcCxHht+8tp6yyyrH7VFZV883GPE5Jjm/WUVONZQnCGNNqdesYxdOThrJ0yz7um7nKsd3nVmYXUHCgglP7B9eSQNYHYYxp1S44/ji25pfyt3kb6NEpmrvP6tfk91iQkYtL4OS+/h+Z1BCWIIwxrd5t4/qwNb+E5z7fSGxkKFeN7EGb8Kbbue3rjFyGdmtP+6jwJnvN5mAJwhjT6okIf754CDsLDvLYx+t4Zn4G5wzuwoVDj2Ngl1gSGtGRvK+knFXZ+/nN6clNHLXzLEEYYwwQFuLi9etG8F1WPrNX5PDJ6l3M+iHHc0zo0i6S8wZ35b7xAwiplSy+3ZRHbGQYgxPbHfa632TmUa0EXf8DWIIwxphDQlzC6L5xjO4bx6MTBrN0y1625peSs/8AGbuKeHlBFtv2lvLs5FQiQkOorlaenp/B1C8zEXFvcXrfOf1/1pT0dUYu7dqEMTQp+HZNtgRhjDE+RIaFMDY5nrFeLUP/XJjFYx+vo2D6Up6aNJQHP1rN5+v3MCktibYRYbyxeAufrt7F9WN6sruwjFU5BazJKeCcwV0Oq3UEA1vN1RhjGmDWD9ncO3MVAAI8dGEKV4/qgYiwdkch//vv1Szbuo+2EaEMToxlSGI7fjGqBz06+W/RvaM52mquVoMwxpgGuOSEJDpEhfP8Fxu5b/wARvXudOhYynGxfHjLSewpKmtUx3agsARhjDENdNqABE4bkODzmMvl7tBuCRydSS0i40Vkg4hkisj9RzhnkoisFZE1IvKuV/k1IrLR87jGyTiNMcYczrEahIiEAC8AZwHZwFIRmaOqa73OSQYeAMao6j4RSfCUdwQeBtIABZZ5rt3nVLzGGGN+zskaxAggU1WzVLUcmAFMqHXOTcALNR/8qrrHU34OMF9V93qOzQfGOxirMcaYWpxMEInAdq/n2Z4yb/2AfiKySESWiMj4BlyLiNwsIukikp6bm9uEoRtjjPH3aq6hQDIwDpgCvCIi9Z5NoqrTVDVNVdPi44NvlqIxxgQyJxNEDtDN63mSp8xbNjBHVStUdTOQgTth1OdaY4wxDnIyQSwFkkWkl4iEA5OBObXOmY279oCIxOFucsoC5gFni0gHEekAnO0pM8YY00wcG8WkqpUicjvuD/YQ4DVVXSMijwLpqjqHnxLBWqAKuFdV8wFE5E+4kwzAo6q616lYjTHGHK7FLLUhIrnAVs/TdkDBUX72VRYH5DXwtt6vU99jtcvritVX3E0d65GOHy1WX3EFQ6y+YvR3rEeKz1es3mWt+T0bTLEe6XigvGd7qKrvTlxVbXEPYNrRfj5CWXpj7lPfY7XL64rVV9xNHeuRjh8t1nr8PQMy1iPE6NdYG/L/3sl4g+k9G0yx1vd9ECjvWe+Hv0cxOeU/dfx8pOONuU99j9UurytW75+divVIx48Wq/fzYIrV++dAibV2mb1n6742mGI90vFAfc8e0mKamBpLRNL1CCsaBhqL1RnBFCsEV7wWqzOcjrWl1iCOxTR/B9AAFqszgilWCK54LVZnOBqr1SCMMcb4ZDUIY4wxPlmCMMYY45MlCGOMMT5ZgqiDiIwVkX+IyD9F5Ft/x1MXEXGJyJ9F5PlA32hJRMaJyELP33ecv+Opi4hEe1YPvsDfsRyNiAz0/E1nisiv/B1PXURkooi8IiLvi8jZ/o7naESkt4i8KiIz/R2LL5736Buev+dVjX29Fp0gROQ1EdkjIqtrlde5010NVV2oqrcC/w94I9Djxb3nRhJQgXsxxECOVYFiIDIIYgX4H+ADZ6I8FFNTvGfXed6zk4AxQRDvbFW9CbgVuCLAY81S1RucitGXBsZ9CTDT8/e8qNE3b+gsvGB6AKcAJwCrvcpCgE1AbyAcWAmkAENwJwHvR4LXdR8AMYEeL3A/cIvn2pkBHqvLc11n4J0Aj/Us3AtOXgtcEMixeq65CPgEuDLQ37Ne1z0FnBAksTr2b6uRcT8ApHrOebex93Zssb5AoKoLRKRnreJDO90BiMgMYIKq/gXw2XQgIt2BAlUtcjDcJolXRLKBcs/TqkCO1cs+IMKJOKHJ/q7jgGjc/wgPiMhcVa0OxFg9rzMHmCMiHwPv+jonUOIVEQEeBz5R1R8COVZ/aEjcuGviScAKmqCFqEUniCPwtVvdyDquuQGY7lhER9fQeGcBz4vIWGCBk4H50KBYReQS3NvLtgemOhvaYRoUq6r+AUBErgXynEgOR9HQv+s43E0NEcBcRyPzraHv2d8AZwLtRKSvqv7DyeBqaejfthPwZ2CYiDzgSST+cKS4nwOmisj5NG45DqB1JogGU9WH/R1DfalqKe6EFvBUdRbuhBY0VPV1f8dQF1X9CvjKz2HUm6o+h/uDLeCpezuCW/0dx5GoaglwXVO9XovupD6CYNutLpjitVidEUyxQnDFG0yxemuWuFtjgqjPTneBJJjitVidEUyxQnDFG0yxemueuJurJ94fD+A9YCc/Dfm8wVN+Hu79rzcBf/B3nMEYr8VqsQZbvMEUa6DEbYv1GWOM8ak1NjEZY4ypB0sQxhhjfLIEYYwxxidLEMYYY3yyBGGMMcYnSxDGGGN8sgRhWjQRKW7m+zXJniHi3iujQERWiMh6EXmyHtdMFJGUpri/MWAJwpgGEZGjrl+mqqOb8HYLVTUVGAZcICJ17e0wEfdqs8Y0CUsQptURkT4i8qmILBP3jnYDPOUXish3IrJcRP4rIp095Y+IyFsisgh4y/P8NRH5SkSyROS3Xq9d7PnvOM/xmZ4awDueZa0RkfM8ZctE5DkR+X9Hi1dVD+BevjnRc/1NIrJURFaKyL9EJEpERuPeA+JvnlpHnyP9nsbUlyUI0xpNA36jqsOBe4AXPeXfAKNUdRgwA7jP65oU4ExVneJ5PgD3UuUjgIdFJMzHfYYBd3qu7Q2MEZFI4GXgXM/94+sKVkQ6AMn8tHz7LFU9UVWHAutwL73wLe61eO5V1VRV3XSU39OYerHlvk2rIiJtgdHAh54v9PDTZkVJwPsi0hX3Ll2bvS6d4/kmX+NjVS0DykRkD+5d8Wpvm/q9qmZ77rsC6Il7i9UsVa157feAm48Q7lgRWYk7OTyrqrs85YNF5DHc+2i0BeY18Pc0pl4sQZjWxgXs97Tt1/Y88LSqzvFsuvOI17GSWueWef1che9/S/U552gWquoFItILWCIiH6jqCuB1YKKqrvRsYDTOx7VH+z2NqRdrYjKtiqoWAptF5HJwb3cpIkM9h9vx05r61zgUwgagt9cWklfUdYGntvE48D+eohhgp6dZ6yqvU4s8x+r6PY2pF0sQpqWLEpFsr8fduD9Ub/A036zBvZcvuGsMH4rIMiDPiWA8zVS3AZ967lMEFNTj0n8Ap3gSy/8C3wGLgPVe58wA7vV0svfhyL+nMfViy30b08xEpK2qFntGNb0AbFTVZ/wdlzG1WQ3CmOZ3k6fTeg3uZq2X/RyPMT5ZDcIYY4xPVoMwxhjjkyUIY4wxPlmCMMYY45MlCGOMMT5ZgjDGGOOTJQhjjDE+/X//bTxXHuQApQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn.lr_find()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0.517189</td>\n",
       "      <td>0.488224</td>\n",
       "      <td>0.777267</td>\n",
       "      <td>00:09</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0.468717</td>\n",
       "      <td>0.454412</td>\n",
       "      <td>0.796321</td>\n",
       "      <td>00:10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>0.433061</td>\n",
       "      <td>0.447050</td>\n",
       "      <td>0.798949</td>\n",
       "      <td>00:10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>0.360648</td>\n",
       "      <td>0.467646</td>\n",
       "      <td>0.795664</td>\n",
       "      <td>00:10</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn.fine_tune(3, 7e-3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting cleanlab\n",
      "  Downloading cleanlab-0.1.1-py2.py3-none-any.whl (58 kB)\n",
      "\u001b[?25l\r\u001b[K     |█████▋                          | 10 kB 36.9 MB/s eta 0:00:01\r\u001b[K     |███████████▏                    | 20 kB 40.5 MB/s eta 0:00:01\r\u001b[K     |████████████████▊               | 30 kB 25.4 MB/s eta 0:00:01\r\u001b[K     |██████████████████████▎         | 40 kB 29.0 MB/s eta 0:00:01\r\u001b[K     |███████████████████████████▉    | 51 kB 27.6 MB/s eta 0:00:01\r\u001b[K     |████████████████████████████████| 58 kB 6.7 MB/s \n",
      "\u001b[?25hRequirement already satisfied: scikit-learn>=0.18 in /usr/local/lib/python3.7/dist-packages (from cleanlab) (0.22.2.post1)\n",
      "Requirement already satisfied: numpy>=1.11.3 in /usr/local/lib/python3.7/dist-packages (from cleanlab) (1.19.5)\n",
      "Requirement already satisfied: scipy>=1.1.0 in /usr/local/lib/python3.7/dist-packages (from cleanlab) (1.4.1)\n",
      "Requirement already satisfied: joblib>=0.11 in /usr/local/lib/python3.7/dist-packages (from scikit-learn>=0.18->cleanlab) (1.0.1)\n",
      "Installing collected packages: cleanlab\n",
      "Successfully installed cleanlab-0.1.1\n"
     ]
    }
   ],
   "source": [
    "!pip install cleanlab"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<fastai.text.data.SortedDL at 0x7f621766dfd0>"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learn.dls.train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       ""
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.7/dist-packages/numpy/core/_asarray.py:83: VisibleDeprecationWarning: Creating an ndarray from ragged nested sequences (which is a list-or-tuple of lists-or-tuples-or ndarrays with different lengths or shapes) is deprecated. If you meant to do this, you must specify 'dtype=object' when creating the ndarray\n",
      "  return array(a, dtype, copy=False, order=order)\n"
     ]
    }
   ],
   "source": [
    "train_dl = learn.dls.test_dl(tokenize_df(train_df, text_cols='text')[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       ""
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "train_preds = learn.get_preds(dl=train_dl, with_decoded=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([7613, 2])"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_preds[0].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([1, 1, 1, ..., 1, 1, 1])"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df.target.values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(7613,)"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df.target.values.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.4174578 , 0.58254224],\n",
       "       [0.00218836, 0.9978116 ],\n",
       "       [0.18324117, 0.8167588 ],\n",
       "       ...,\n",
       "       [0.02872721, 0.97127277],\n",
       "       [0.01679658, 0.9832034 ],\n",
       "       [0.01257689, 0.9874232 ]], dtype=float32)"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_preds[0].numpy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from cleanlab.pruning import get_noise_indices"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ordered_label_errors = get_noise_indices(s=train_df.target.values, psx=train_preds[0].numpy(), sorted_index_method='normalized_margin')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "383"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(ordered_label_errors)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>3991</th>\n",
       "      <td>@madonnamking RSPCA site multiple 7 story high rise buildings next to low density character residential in an area that floods</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3806</th>\n",
       "      <td>Former Township fire truck being used in Philippines - Langley Times  #filipino</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2269</th>\n",
       "      <td>Rand Paul's Debate Strategy 'demolish Some other bad ideas out there or point out maybe that there are some em...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1315</th>\n",
       "      <td>My hand is burning</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5463</th>\n",
       "      <td>.@slosheriff: 2 South Gate police officers and 2 Huntington Park officers arrested after child abuse investigation at boot camp</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2877</th>\n",
       "      <td>Large rain drops falling in Rock Hill off Anderson Road. #rain #scwx #drought</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2236</th>\n",
       "      <td>If you're in search of powerful content to improve your business or have been frustrated with the deluge of</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>547</th>\n",
       "      <td>STAR WARS POWER OF THE JEDI COLLECTION 1 BATTLE DROID HASBRO - Full read by eBay</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3913</th>\n",
       "      <td>Spot Flood Combo 53inch 300W Curved Cree LED Work Light Bar 4X4 Offroad Fog Lamp - Full reÛ_</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3278</th>\n",
       "      <td>#Tanzania elephant population declined by 60% in five years census reveals</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3057</th>\n",
       "      <td>#Sismo M 1.3 - 1km NNE of The Geysers California: Time2015-08-05 23:40:21 UTC2015-08-05 16:40:21 -07:00 a...  #CS</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6131</th>\n",
       "      <td>'Amateur Night' Actress Reprises Role for 'Siren' - HorrorMovies.ca #horror</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5563</th>\n",
       "      <td>RAIN RAIN GO AWAY... A soaker is on the way ------&amp;gt;  &amp;lt;-----</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6913</th>\n",
       "      <td>@lucysforsale funny cause my dumb ass was the young one to get n trouble the most lol</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1358</th>\n",
       "      <td>if firefighters acted like cops they'd drive around shooting a flamethrower at burning buildings</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2544</th>\n",
       "      <td>destroy the free fandom honestly</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>358</th>\n",
       "      <td>One Direction Is my pick for  Fan Army #Directioners  x1392</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>556</th>\n",
       "      <td>Black Eye 9: A space battle occurred at Star M27329 involving 1 fleets totaling 1236 ships with 7 destroyed</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3056</th>\n",
       "      <td>#USGS M 1.4 - 4km E of Interlaken California: Time2015-08-06 00:52:25 UTC2015-08-05 17:52:25 -07:00 at ep...  #SM</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1847</th>\n",
       "      <td>'@jorrynja: 6. @ your bf/gf/crush ??' @Ter_ell ??</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5507</th>\n",
       "      <td>Reddit's new content policy goes into effect many horrible subreddits banned or quarantined</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4497</th>\n",
       "      <td>@Hurricane_Dame ???????? I don't have them they out here</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5464</th>\n",
       "      <td>Reddit Will Now QuarantineÛ_  #onlinecommunities #reddit #amageddon #freespeech #Business</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2832</th>\n",
       "      <td>.POTUS #StrategicPatience is a strategy for #Genocide; refugees; IDP Internally displaced people; horror; etc.</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3058</th>\n",
       "      <td>Earthquake drill ??????</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6855</th>\n",
       "      <td>Trauma injuries involving kids and sport usually cycling related - CBC.ca</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4732</th>\n",
       "      <td>Check out my Lava lamp dude ????</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4444</th>\n",
       "      <td>I went to pick up my lunch today and the bartender was holding my change hostage because he wanted my number. ??</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2905</th>\n",
       "      <td>I can't drown my demons they know how to swim</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6277</th>\n",
       "      <td>this storm????</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4713</th>\n",
       "      <td>Imagine a room with walls that are lava lamps.</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>430</th>\n",
       "      <td>Arsonists being blamed for a blaze at a plastics recycling business in Adelaide | @pcaldicott7 reports. #7NewsAdl</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4217</th>\n",
       "      <td>JAX issues Hazardous Weather Outlook (HWO)</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6707</th>\n",
       "      <td>@HaydnExists so glad i saved them all at once then didnÛªt want you stealing my thunder :P</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4509</th>\n",
       "      <td>My back is so sunburned :(</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4305</th>\n",
       "      <td>Hellfire is surrounded by desires so be careful and donÛªt let your desires control you! #Afterlife</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4364</th>\n",
       "      <td>Remove the  and Linkury Browser Hijacker</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4357</th>\n",
       "      <td>Remove the  and Linkury Browser Hijacker</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1565</th>\n",
       "      <td>2Leezy its like you're about to fall down a cliff but you're grabbing my hand but my palms are sweaty ... Why bruh?</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4096</th>\n",
       "      <td>@Flow397 Coming atcha from Boston. Had golfball sized hail yesterday sunny amazing skies today! #ParkChat</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7203</th>\n",
       "      <td>Even if u have your weapon and your badge we gonna put them choppas on your ass????</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4704</th>\n",
       "      <td>#landslide while on a trip in #skardu</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5370</th>\n",
       "      <td>@elielcruz just watching the streams was bad - I don't think I could actually walk in there I'd panic and get sick</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>857</th>\n",
       "      <td>people with a #tattoo out there.. Are u allowed to donate blood and receive blood as well or not?</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4311</th>\n",
       "      <td>@gg_keeponrockin @StrawberrySoryu Oh okay I just got the message twice and got suspicious. Sorry.  I'll check it out!</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1232</th>\n",
       "      <td>Fire hazard associated with installation of non-compliant external cladding on  - By @www.cbplawyers</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7127</th>\n",
       "      <td>nside a Dragon's belly. Or an ice cave under\\na volcano in Kamchatka | Photography by\\nå©Daniel</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>842</th>\n",
       "      <td>I call it a little bit of your blizzard?</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1005</th>\n",
       "      <td>'I did another one I did another one. You still ain't done shit about the other one.' Nigga body bagging Meek.</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6011</th>\n",
       "      <td>When you go to a concert and someone screams in your ear... Does it look like I wanna loose my hearing anytime soon???</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                                                                                 text  target\n",
       "3991   @madonnamking RSPCA site multiple 7 story high rise buildings next to low density character residential in an area that floods       0\n",
       "3806                                                  Former Township fire truck being used in Philippines - Langley Times  #filipino       0\n",
       "2269               Rand Paul's Debate Strategy 'demolish Some other bad ideas out there or point out maybe that there are some em...        1\n",
       "1315                                                                                                               My hand is burning       1\n",
       "5463  .@slosheriff: 2 South Gate police officers and 2 Huntington Park officers arrested after child abuse investigation at boot camp       0\n",
       "2877                                                    Large rain drops falling in Rock Hill off Anderson Road. #rain #scwx #drought       0\n",
       "2236                     If you're in search of powerful content to improve your business or have been frustrated with the deluge of        1\n",
       "547                                                STAR WARS POWER OF THE JEDI COLLECTION 1 BATTLE DROID HASBRO - Full read by eBay         1\n",
       "3913                                  Spot Flood Combo 53inch 300W Curved Cree LED Work Light Bar 4X4 Offroad Fog Lamp - Full reÛ_         1\n",
       "3278                                                     #Tanzania elephant population declined by 60% in five years census reveals         0\n",
       "3057                #Sismo M 1.3 - 1km NNE of The Geysers California: Time2015-08-05 23:40:21 UTC2015-08-05 16:40:21 -07:00 a...  #CS       0\n",
       "6131                                                     'Amateur Night' Actress Reprises Role for 'Siren' - HorrorMovies.ca #horror        1\n",
       "5563                                                               RAIN RAIN GO AWAY... A soaker is on the way ------&gt;  &lt;-----        0\n",
       "6913                                            @lucysforsale funny cause my dumb ass was the young one to get n trouble the most lol       1\n",
       "1358                                 if firefighters acted like cops they'd drive around shooting a flamethrower at burning buildings       0\n",
       "2544                                                                                                 destroy the free fandom honestly       1\n",
       "358                                                                       One Direction Is my pick for  Fan Army #Directioners  x1392       1\n",
       "556                       Black Eye 9: A space battle occurred at Star M27329 involving 1 fleets totaling 1236 ships with 7 destroyed       1\n",
       "3056                #USGS M 1.4 - 4km E of Interlaken California: Time2015-08-06 00:52:25 UTC2015-08-05 17:52:25 -07:00 at ep...  #SM       0\n",
       "1847                                                                                '@jorrynja: 6. @ your bf/gf/crush ??' @Ter_ell ??       1\n",
       "5507                                    Reddit's new content policy goes into effect many horrible subreddits banned or quarantined         1\n",
       "4497                                                                         @Hurricane_Dame ???????? I don't have them they out here       1\n",
       "5464                                      Reddit Will Now QuarantineÛ_  #onlinecommunities #reddit #amageddon #freespeech #Business        1\n",
       "2832                  .POTUS #StrategicPatience is a strategy for #Genocide; refugees; IDP Internally displaced people; horror; etc.        0\n",
       "3058                                                                                                          Earthquake drill ??????       0\n",
       "6855                                                       Trauma injuries involving kids and sport usually cycling related - CBC.ca        1\n",
       "4732                                                                                                Check out my Lava lamp dude ????        1\n",
       "4444                 I went to pick up my lunch today and the bartender was holding my change hostage because he wanted my number. ??       1\n",
       "2905                                                                                    I can't drown my demons they know how to swim       1\n",
       "6277                                                                                                                   this storm????       0\n",
       "4713                                                                                   Imagine a room with walls that are lava lamps.       1\n",
       "430                Arsonists being blamed for a blaze at a plastics recycling business in Adelaide | @pcaldicott7 reports. #7NewsAdl        0\n",
       "4217                                                                                      JAX issues Hazardous Weather Outlook (HWO)        0\n",
       "6707                                      @HaydnExists so glad i saved them all at once then didnÛªt want you stealing my thunder :P       1\n",
       "4509                                                                                                       My back is so sunburned :(       1\n",
       "4305                             Hellfire is surrounded by desires so be careful and donÛªt let your desires control you! #Afterlife       1\n",
       "4364                                                                                       Remove the  and Linkury Browser Hijacker         1\n",
       "4357                                                                                       Remove the  and Linkury Browser Hijacker         1\n",
       "1565             2Leezy its like you're about to fall down a cliff but you're grabbing my hand but my palms are sweaty ... Why bruh?        1\n",
       "4096                        @Flow397 Coming atcha from Boston. Had golfball sized hail yesterday sunny amazing skies today! #ParkChat       0\n",
       "7203                                              Even if u have your weapon and your badge we gonna put them choppas on your ass????       1\n",
       "4704                                                                                           #landslide while on a trip in #skardu        0\n",
       "5370               @elielcruz just watching the streams was bad - I don't think I could actually walk in there I'd panic and get sick       1\n",
       "857                                 people with a #tattoo out there.. Are u allowed to donate blood and receive blood as well or not?       1\n",
       "4311            @gg_keeponrockin @StrawberrySoryu Oh okay I just got the message twice and got suspicious. Sorry.  I'll check it out!       1\n",
       "1232                             Fire hazard associated with installation of non-compliant external cladding on  - By @www.cbplawyers       0\n",
       "7127                                 nside a Dragon's belly. Or an ice cave under\\na volcano in Kamchatka | Photography by\\nå©Daniel        1\n",
       "842                                                                                          I call it a little bit of your blizzard?       1\n",
       "1005                   'I did another one I did another one. You still ain't done shit about the other one.' Nigga body bagging Meek.       1\n",
       "6011           When you go to a concert and someone screams in your ear... Does it look like I wanna loose my hearing anytime soon???       1"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df.loc[ordered_label_errors][['text', 'target']][:50]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "error_df = train_df.loc[ordered_label_errors]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "clean_idx = [idx for idx, _ in enumerate(train_df.values) if not idx in ordered_label_errors]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>keyword</th>\n",
       "      <th>location</th>\n",
       "      <th>text</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>NA</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Our Deeds are the Reason of this #earthquake May ALLAH Forgive us all</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>4</td>\n",
       "      <td>NA</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Forest fire near La Ronge Sask. Canada</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>5</td>\n",
       "      <td>NA</td>\n",
       "      <td>NaN</td>\n",
       "      <td>All residents asked to 'shelter in place' are being notified by officers. No other evacuation or shelter in place orders are expected</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>6</td>\n",
       "      <td>NA</td>\n",
       "      <td>NaN</td>\n",
       "      <td>13,000 people receive #wildfires evacuation orders in California</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>7</td>\n",
       "      <td>NA</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Just got sent this photo from Ruby #Alaska as smoke from #wildfires pours into a school</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7608</th>\n",
       "      <td>10869</td>\n",
       "      <td>NA</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Two giant cranes holding a bridge collapse into nearby homes</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7609</th>\n",
       "      <td>10870</td>\n",
       "      <td>NA</td>\n",
       "      <td>NaN</td>\n",
       "      <td>@aria_ahrary @TheTawniest The out of control wild fires in California even in the Northern part of the state. Very troubling.</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7610</th>\n",
       "      <td>10871</td>\n",
       "      <td>NA</td>\n",
       "      <td>NaN</td>\n",
       "      <td>M1.94 [01:04 UTC]?5km S of Volcano Hawaii.</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7611</th>\n",
       "      <td>10872</td>\n",
       "      <td>NA</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Police investigating after an e-bike collided with a car in Little Portugal. E-bike rider suffered serious non-life threatening injuries.</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7612</th>\n",
       "      <td>10873</td>\n",
       "      <td>NA</td>\n",
       "      <td>NaN</td>\n",
       "      <td>The Latest: More Homes Razed by Northern California Wildfire - ABC News</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>7230 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         id  ... target\n",
       "0         1  ...      1\n",
       "1         4  ...      1\n",
       "2         5  ...      1\n",
       "3         6  ...      1\n",
       "4         7  ...      1\n",
       "...     ...  ...    ...\n",
       "7608  10869  ...      1\n",
       "7609  10870  ...      1\n",
       "7610  10871  ...      1\n",
       "7611  10872  ...      1\n",
       "7612  10873  ...      1\n",
       "\n",
       "[7230 rows x 5 columns]"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df.loc[clean_idx]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "clean_train_df = train_df.loc[clean_idx]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       ""
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.7/dist-packages/numpy/core/_asarray.py:83: VisibleDeprecationWarning: Creating an ndarray from ragged nested sequences (which is a list-or-tuple of lists-or-tuples-or ndarrays with different lengths or shapes) is deprecated. If you meant to do this, you must specify 'dtype=object' when creating the ndarray\n",
      "  return array(a, dtype, copy=False, order=order)\n"
     ]
    }
   ],
   "source": [
    "dls = TextDataLoaders.from_df(clean_train_df, seed=42, text_col=['keyword', 'text'], label_col='target', y_block=CategoryBlock(), is_lm=False, text_vocab=dls_lm.vocab)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<fastai.text.learner.TextLearner at 0x7f61cb998e90>"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learn = text_classifier_learner(dls, arch=AWD_LSTM, metrics=accuracy)\n",
    "learn.load_encoder('fine_tuned_enc')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       ""
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "SuggestedLRs(lr_min=0.02089296132326126, lr_steep=6.309573450380412e-07)"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEKCAYAAAAIO8L1AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deXxU5dn/8c+VyUYIhISENQm7QGQJEBBRFDdEq4I7KG5V0Nali9Vq25/2UfvUVq0rqFgRqxW0aC2PG2oRQRBIkEV2QlgStgRCAlnIev3+yIBjGEICOTmT5Hq/XvNi5ixzvgnJXLnPfe77iKpijDHGVBfkdgBjjDGByQqEMcYYv6xAGGOM8csKhDHGGL+sQBhjjPHLCoQxxhi/gt0OUF9iY2O1a9eubscwxphGZfny5ftUNc7fuiZTILp27UpaWprbMYwxplERke3HW2enmIwxxvhlBcIYY4xfViCMMcb4ZQXCGGOMX1YgjDHG+GUFwhhjjF/NvkAcLqtg3oa97Nhf5HYUY4wJKM2+QBSWlPPTGWl8uX6v21GMMSagNPsCEdMylIhQD5kHrAVhjDG+mn2BEBESYyLIzLUCYYwxvpp9gQCIj44gM7fY7RjGGBNQrEAACTEtyDxQhN2f2xhjfmAFAkiMiaCotIL9haVuRzHGmIBhBQJIiI4AsH4IY4zxYQUCSGxbVSB2WIEwxpijrEAA8dEtAMg6YB3VxhhzhBUIICI0mNjIUBtNbYwxPqxAeCXERNhgOWOM8WEFwish2gqEMcb4crRAiMgYEdkoIuki8pCf9V1E5L8islpE5otIvM+6W0Rks/dxi5M5oepS1115hymvqHT6UMYY0yg4ViBExANMAS4BkoAJIpJUbbOngX+o6gDgMeDP3n1jgEeBM4BhwKMiEu1UVqgaLFdRqezOP3zK71VSXkH2oVN/H2OMcZOTLYhhQLqqZqhqKTALGFttmyRgnvf5Vz7rLwa+UNVcVT0AfAGMcTArCTF1v9S1stL/yOs//HsNI/48j798toHDZRX1ks8YYxqakwWiM5Dp8zrLu8zXKuAq7/MrgVYi0raW+9arug6W+2zNHgY9/sUx22cdKOKDFTvpHN2Cl+dv4eLnFvDN5n31ntcYY5zmdif1b4BzRWQFcC6wE6j1n9wiMllE0kQkLScn55SCdIwKxxMktW5BfLFuL/nFZTz52YYfLX9tQQZBAjMnDeedO85AgImvL+Wed75jV56NszDGNB5OFoidQILP63jvsqNUdZeqXqWqg4Dfe5fl1WZf77bTVDVFVVPi4uJOKWywJ4jObVqQWcvBcqnbcgn1BPHx6t0s334AgH0FJcxKzWRccmc6tWnBiJ6xfPbLc/jFBb34Yt1ezn9mPi/+dzO5haXkFZWyv6CEgpLyU8ptjDFOcbJApAK9RKSbiIQC44E5vhuISKyIHMnwMDDd+3wuMFpEor2d06O9yxyVENOiVqeYducXsyO3iPsu6Em7VmE8/tE6VJUZi7ZRWlHJXaN6HN02PMTDry46jS9/fS7n92nHM19sYvDjX5D82BcMeeJLkv/nc976dptzX5QxxpykYKfeWFXLReQeqj7YPcB0VV0rIo8Baao6BxgF/FlEFFgA3O3dN1dEHqeqyAA8pqq5TmU9IjEmgs/XnvjWo8u2VkUZ1bsd7VqH8+Ds1cxclsk/vt3GmNM70CMu8ph9EmIimHrjEFK35bI6Kx+PQFCQ8NWGbP7ff9aSlVfMby/uQ1CQ1PeXZYwxJ8WxAgGgqp8An1Rb9ojP89nA7OPsO50fWhQNIj46gv2FpRSWlNMy7PjfmmVbc4kMC6Zvx9b07diaGYu28YcPv6dS4Wc+rQd/hnaNYWjXmKOvbzyjC3+cs5ZXv85g54Finr52IOEhnnr7mowx5mS53UkdUBK9l7qeaET1sq25pHSNxhMkeIKEP/ykL5UKZ/eMZUB8mzod0xMkPDb2dB6+pA8frd7NEx+vO+n8xhhTn6xA+DgyFqKm24/uLyhhc3YBw7r90AoY0TOWp64ZwOPj+p3UcUWEO8/twdWD4/lwxS6KS23shDHGfVYgfCR4p/2uqaM6dVvVFUtn+BQIgGtTEugW2/KUjn9tSjwFJeXMXbvnlN7HGGPqgxUIHzEtQ2kZ6qlxLMSyrbmEBQfRv3PdTiXVxrCuMSTEtGD28qx6f29jjKkrKxA+RISEmAiyauiDWLZtP4MTowkNrv9vXVCQcPXgeBZt2cdOG1RnjHGZFYhqEmMi2JJT6HfdwcNlrNt18Ef9D/Xt6sHxqMK/v6vfVsTXm3K45PmF3PT6UuZvzEbV/zxSxhhzhBWIaoZ1i2HrvkK//RDLtx+gUo/tf6hPCTERDO8ew+zlWfXyIZ5bWMqv3l3JLdOXUVxazsY9h7j1jVRGP7uAOat21UNiY0xTZQWimvP6tANg/qZj53ZatjWX4CBhUKKjM49zzZAEtu0vOjqFx8koLq1g+jdbufBvX/PR6l3cd35PPvvlOXzz2/P523UD8QQJ981cwf9ZkTDGHIejA+Uao+6xLUmMiWD+hmxuGt7lR+sWb9lP//goWoQ6O5Dtkn4deOQ/a5i9PIuUrnVrrRSWlPP2ku28tjCDfQWlDO8ew/9c0Y/eHVod3eaqwfFcNqATN7y2hAdnr6ZHXCRJnVofXV9WUcm6XQdZlZXHqsx8tu8vZEjXaEYndWBQQhuCgoTi0gq25BRQVFpBSpdoGwFuTBNkBaIaEeH8Pu2YlbqDw2UVR0c1p2cfYlVmHg9d0sfxDC3Dgrm0f0c+Wr2bP1yWRGQNo7p9qSo3vb6U73bkMbJXLPee3+u4/SWhwUFMnTiYy1/8hjvfTmPO3WfTJiKET77fw5OfrT86FiQ2MpTO0RG8vnArr36dQWxkGBGhHjIPFHHkDFjfjq35xQW9GJ3U3gqFMU2IFQg/RvWOY8bibSzJ2M+o3lWnnGYuyyTEI1wzJP4Ee9ePG89IZPbyLN5LzeSnZ3er1T5fb8rhux15PDb2dG4+s+sJt2/XKpxXJg7h+leXcOdby6lUJW37Afp0aMXz45MZ0iWazm1aICLkF5cxf2M28zZkU16hXDW4M6e1b0VhSTlT52/hrreX07dja4Z1jaZFaDARoR5Oax/Jxad3QMSKhjGNkRUIP4Z3b0t4SBDzN+Ywqnc7DpdV8MF3WYxO6kBsZFiDZBiUGE1Kl2imL9rKzWd2Idhz4u6iKV+l0ykqnPFDE+t0nCfG9ePB91cTGxnKn6/qz3UpCXiqtQSiWoQwNrkzY5OPvW/TlYM6M2fVLqYtyODDlVUjwUu99/a+tH8H/nzlAKIiQmqdyRgTGKxA+BEe4uGsHrHM25DNo5cnMXftHg4UlTFhWO0/eOvDHSO7c9fby5m7di8/GdCxxm2XZuwnddsB/nh5Up3HaFw3NIGe7SPp1S6SVuF1/yAP9gRx1eB4rhr8Q+uqrKKS17/ZytNzN7IqcyHPj0+uc3+KMcZddhXTcYzq044duUVs3VfIzGU7SIyJYESPtg2a4aKk9nRpG8FrCzNOeMnrS1+lExsZyviTLGKDE6NPqjgcT4gniLvO7cHsn40g2CNc9+q3vDRv83Hv422MCTxWII5j1GlVd6h7/ZutLMnI5fqhCQ3eAesJEu44uxsrM/NqvOR1VWYeCzfv4/azuwfcVOHJCW346N6zuXxgJ57+fBO3vLGMnEMlbscyxtSCnWI6joSYCHq1i+SfS3cQHCRcm9IwndPVXTMkgWe+2MS0BRnHPUUz5at0WocHM3F4w54Cq61W4SE8d30yZ3Zvy6Nz1nLpCwu585zuhIV4CJKq9aOT2gdccTOmubMCUYPz+7Rjc3YBF/ZtT7tW4a5kaBHqYeIZXZgyP52t+wqPmTF2/sZsPl+3l/su6FWvp4jqm4gwflgiAxPacO/MFTzx8fofrY+NDOOuc7tz4xldHB9nYoypHWkqc/KkpKRoWlpavb7ndzsOcM3Li3n79jMY0TO2Xt+7LrIPHebcv86nXeswpt869OgtTRel7+OnM1LpERfJrDuH0zqAC4Svikolv7iMikqlUpUtOQVM+SqdRen7iY0M5bdj+nBtSoLbMY1pFkRkuaqm+F1nBaJmeUWltIkIrff3ravl23OZ/I/llFVU8vLEIYR4grhl+jISYyKYOXk4MS3dz3iq0rbl8tfPNrJsWy7XDonn8XH97LSTMQ6zAtFEZOYW8dMZqWzdV0hocBAdo8KZNflM4lo1zNiMhlBRqTz35SZenJdO346tefnGwXQ9xRsxGWOOr6YCYVcxNSIJMRG8//MRnHtaHPHRLXhn0vAmVRyg6sqt+0f35o3bhrI7v5ixUxbVeH8OY4xzrAXRSKlqk5/CIiOngCteWkTvDq14d/LwWo0mN8bUjbUgmqCmXhwAusdF8qcr+7F8+wGe+3Kz23GMaXasQJiANja5M9cOiWfK/HQWp+9zO44xzYqNgzAB73/Gns7yHQf45bsr+fmoHhSVVVBYUk5sZBhXDOxE2waaQNGY5sbRPggRGQM8D3iAv6vqk9XWJwJvAm282zykqp+ISFdgPbDRu+kSVb2rpmM1tz6I5mbdroNc9+q3FJSUAxAkUKkQ4hFGJ3Vg/LAEzu4Z2yxOvRlTn1y5zFVEPMAm4CIgC0gFJqjqOp9tpgErVPVlEUkCPlHVrt4C8ZGq9qvt8axANH2FJeWUlFcSEeohLDiITXsLeDc1kw9WZJFXVMblAzvx5FX9aVnLGywZY9zrpB4GpKtqhqqWArOAsdW2UeDIvS6jALtBsjmulmHBxLQMJTzEg4jQu0MrHrk8iaW/u4AHLu7Nx6t3ccVL37Bp7yG3oxrTJDhZIDoDmT6vs7zLfP0RmCgiWcAnwL0+67qJyAoR+VpERvo7gIhMFpE0EUnLycmpx+imMQkL9nD3eT355x3DyS8uZ+xLi/hotf2tYcypcvsqpgnADFWNBy4F3hKRIGA3kKiqg4BfA++ISOvqO6vqNFVNUdWUuLi4Bg1uAs+ZPdryyX1nc3qn1tw7cwXvpWaeeCdjzHE5WSB2Ar4zrsV7l/m6HXgPQFW/BcKBWFUtUdX93uXLgS3AaQ5mNU1Eu9bhvH3HGYzsFceD76/m7SXb3Y5kTKPlZIFIBXqJSDcRCQXGA3OqbbMDuABARPpSVSByRCTO28mNiHQHegEZDmY1TUh4iIdpNw3hwr7t+MOHa3htQQYVdic7Y+rMsQKhquXAPcBcqi5ZfU9V14rIYyJyhXez+4FJIrIKmAncqlWXVZ0DrBaRlcBs4C5VzXUqq2l6wkM8TL1xCJf068CfPlnPWU/O48lPN5CebR3YxtSWzcVkmrSKSmXu2j28vzyL+ZtyqKhUBie2YeLwLlzav6NNJ26aPZvu2xgg51AJH67YycxlO8jYV0h0RAg3n9mV+y7ohaeB7zduTKCwAmGMD1Vl8Zb9vLl4G5+v28ul/Tvw7PXJhAVba8I0PzUVCBtyapodEeGsnrGc1TOWvy/M4ImP13OwOI1Xbxpio7CN8eH2OAhjXHXHyO48c+1Avs3Yzw2vLSG/qMztSMYEDCsQptm7ekg8r04cwvc783n56y1uxzEmYFiBMAa4MKk9Y/p14J2l2yn0zhhrTHNnBcIYr9vP7sbBw+W8/12W21GMCQhWIIzxGpwYTXJCG6Z/s5VKG3ltjBUIY44QEW4/uxvb9hfx3w3ZbscxxnVWIIzxcUm/DnSKCuf1b2zqL2OsQBjjI9gTxK1ndWVJRi5rdua7HccYV1mBMKaa64cmEhHq4cV5m60vwjRrViCMqSaqRQg/H9WDuWv38vsPv7ciYZotm1fAGD/uPq8nxWUVTPlqC2UVyl+uHmAT+plmxwqEMX6ICL8Z3ZsQTxDPfbmZikrlqWsGEOyxRrdpPqxAGHMcIsIvLzyNEE8QT83dyMHiMl68YRARofZrY5oH+3PImBO4+7yePDGuH19tzGbCa0vZX1DidiRjGoQVCGNqYeLwLrwycQgbdh/k6pcXs31/oduRjHGcFQhjamn06R14Z9Jw8orLmPj6UvZZS8I0cVYgjKmDIV2imXHbMLIPljD5H2kcLqtwO5IxjrECYUwdJSe04bnrk1mRmcf9/1pl4yRMk2UFwpiTcEn/jjw0pg8fr97NU59vdDuOMY6w6/WMOUmTz+nOtv1FvDx/C93atuS6oQluRzKmXlmBMOYkiQiPjT2drANF/O7f3xMf3YIRPWPdjmVMvXH0FJOIjBGRjSKSLiIP+VmfKCJficgKEVktIpf6rHvYu99GEbnYyZzGnKwQTxBTbhxMt9iW3PX2ctKzC9yOZEy9caxAiIgHmAJcAiQBE0QkqdpmfwDeU9VBwHhgqnffJO/r04ExwFTv+xkTcFqHhzD91qGEBgfx0xmpNpDONBlOtiCGAemqmqGqpcAsYGy1bRRo7X0eBezyPh8LzFLVElXdCqR738+YgJQQE8FrN6ew9+Bhfvb2d5SWV7odyZhT5mSB6Axk+rzO8i7z9UdgoohkAZ8A99ZhX0RksoikiUhaTk5OfeU25qQMSozmr9cMYNm2XB75zxpU7fJX07i5fZnrBGCGqsYDlwJviUitM6nqNFVNUdWUuLg4x0IaU1tjkztz93k9mJWayYzF29yOY8wpcfIqpp2A73V/8d5lvm6nqo8BVf1WRMKB2Frua0xAuv+i3mzaW8DjH62jZ7tIRvayP15M4+RkCyIV6CUi3UQklKpO5znVttkBXAAgIn2BcCDHu914EQkTkW5AL2CZg1mNqTdBQcJz1ydzWvtW/GLWSvKLy9yOZMxJcaxAqGo5cA8wF1hP1dVKa0XkMRG5wrvZ/cAkEVkFzARu1SprgfeAdcBnwN2qapPemEajZVgwT187kANFpUz5Kt3tOMacFGkqHWkpKSmalpbmdgxjfuSBf63iPyt38cWvz6FL25ZuxzHmGCKyXFVT/K1zu5PamCbtNxf3JtgjPPnpBrejGFNnViCMcVD71uHcdW4PPl2zh6UZ+wHIzC3ifz9Zz98XZtilsCag2VxMxjhs0sjuzFy2g0fnrKVnu0g++X43R2YI35JTwONj+xHssb/VTOCp1U+liLQ8Mj5BRE4TkStEJMTZaMY0DS1CPTw4pjcb9hzi6405TBrZncUPnc895/Vk5rJM7nxrOUWl5W7HNOYYteqkFpHlwEggGlhE1SWspap6o7Pxas86qU0gU1WWZOTSr3NrWoX/8LfV20u288h/1tCvcxS/uug0zukVhydIXExqmpuaOqlre4pJVLVIRG4HpqrqX0VkZf1FNKZpExHO7NH2mOUTh3ehXaswHv7ge257I5VOUeFcm5LAT8/uRlQLa6Qbd9W6QIjImcCNVI1+BrDZVY2pB6NP78Co3u34Yt1eZqXu4Pn/biY9u4ApNw52O5pp5mrbM/ZL4GHg397Bbt2Br5yLZUzzEhocxE8GdOSt28/gznO78+ma3WTmFrkdyzRztSoQqvq1ql6hqn/xdlbvU9X7HM5mTLN0y5ldERHetMn+jMtqexXTOyLSWkRaAmuAdSLygLPRjGmeOrVpwU/6d+Td1EwOHbZ5nIx7anuKKUlVDwLjgE+BbsBNjqUyppm7/exuHCop5720LLejmGastgUixDvuYRwwR1XLqLobnDHGAQMT2pDSJZoZi7dSUWm/asYdtS0QrwLbgJbAAhHpAhx0KpQxBu4Y2Y3M3GK+WLfH7SimmaptJ/ULqtpZVS/1Tse9HTjP4WzGNGsXJXUgIaYFL3+dYSOtjStq20kdJSJ/O3L/ZxF5hqrWhDHGIZ4g4RcXnMaqzDzGPLeQxen73I5kmpnanmKaDhwCrvM+DgJvOBXKGFPlmiHxzJo8nCCBG/6+lIc/WE1hibUmTMOo7UjqHqp6tc/r/7GpNoxpGMO7t+XTX5zDc19u4rWFGeQVlTH1xsGI2JxNxlm1bUEUi8jZR16IyFlAsTORjDHVtQj18PClffntmD58umYPby3Z7nYk0wzUtgVxF/APEYnyvj4A3OJMJGPM8Uwa2Z0lGft54qP1DE6Mpl/nqBPvZMxJqu1VTKtUdSAwABigqoOA8x1NZow5RlCQ8Mx1ycS0DOXud76zkdbGUXW6jZWqHvSOqAb4tQN5jDEnENMylBdvGETWgWJ+9e4qSsor3I5kmqhTuc+h9ZAZ45KhXWN45LIkvly/l4l/X0puYanbkUwTdCoFwsb/G+OiW0Z05aUbBrEqK5+rpi4iI6fA7UimiamxQIjIIRE56OdxCOjUQBmNMcdx2YBOzJw0nIOHy7nq5cVs3VfodiTThNRYIFS1laq29vNopaonvAJKRMaIyEYRSReRh/ysf1ZEVnofm0Qkz2ddhc+6OSf35RnT9A3pEs2/fz6C8grliY/WuR3HNLB/r8jivdRMR977VE4x1UhEPMAU4BIgCZggIkm+26jqr1Q1WVWTgReBD3xWFx9Zp6pXOJXTmKagS9uW3HN+T/67IZsFm3LcjmMa0NtLdjD7O2emhXesQADDgHRVzVDVUmAWMLaG7ScAMx3MY0yTdttZXenSNoLHP1pHeUWl23FMA1BVNu89RK92kY68v5MFojPg2+7J8i47hnf68G7APJ/F4d6JAZeIyDjnYhrTNIQFe/jdpX3ZnF3AP5fucDuOaQA5BSUcPFzeKAtEXYwHZquq7wXdXVQ1BbgBeE5EelTfSUQmH5lhNifHmtXGjE5qz4gebXn2y03kFdmlr01d+t6qK9d6tW/lyPs7WSB2Agk+r+O9y/wZT7XTS6q60/tvBjAfGFR9J1WdpqopqpoSFxdXH5mNadREhEcuT+JgcRlPf77R7TjGYZuzvQWiEbYgUoFeItJNREKpKgLHXI0kIn2AaOBbn2XRIhLmfR4LnAXY5RnG1EKfDq25ZURX3l6yg8Vb7B4STdnm7EO0Cg8mrlWYI+/vWIFQ1XLgHmAusB54T1XXishjIuJ7VdJ4YJaq+g686wukicgq4CvgSVW1AmFMLT14cR+6to3gwdmrKbD7RzRZm/cW0KtdpGNTv9d2NteToqqfAJ9UW/ZItdd/9LPfYqC/k9mMacpahHp4+tqBXPvqt/z5k/X86Ur7dWqKtuQUcEGf9o69f6B0Uhtj6llK1xgmjezOP5fuYOFmu4ijqcktLGVfQSm92jvT/wBWIIxp0n590Wn0iGvJb2ev5oBN6NekpHs7qHs41EENViCMadLCQzw8e30y+wpKuXfmChtA14Rszj4EOHcFE1iBMKbJGxDfhieu7Mc36fv4y2cb3I5j6kl6dgERoR46RbVw7BiOdlIbYwLDdSkJrN2Zz2sLt3J6pyjGDfI7qYFpRNKzC+jZLpKgIOduzWMtCGOaiT9clsSwbjH89v3VrNmZ73Ycc4o2760qEE6yAmFMMxHiCWLqjYNp2zKUu95ebp3WjdjBw2XsOXjYCoQxpv7ERoYxdeIQsg+W8Mt3V1JRaTeGbIzSj06x4cwcTEdYgTCmmUlOaMOjVyTx9aYcXvjvZrfjmJOQ7vAcTEdYgTCmGbphWCJXD47nhXmb+WpDtttxTB2lZxcQGhxEQkyEo8exAmFMMyQiPDGuH306tOaX765kT/5htyOZOti89xDdY1vicfAKJrACYUyz1SLUw5QbBlFaXskDs1dRaf0Rjcbm7ALH7gHhywqEMc1Y97hI/nBZXxZu3seb325zO46phaLScrIOFDve/wBWIIxp9m4YlsgFfdrx5Kcb2Lz3kNtxzAl8sW4vAAPioxw/lo2kNqaZExGevHoAY55bwH2zVnLZgI6s2ZnPml359IyL5JnrkolpGep2TAOoKi/P30KPuJac08v5u2haC8IYQ1yrMJ68egDrdx/kqbkbWbf7IH07tGbRlv1cOXURW3IK3I5ogPmbctiw5xB3ndvD0Sk2jrAWhDEGgIuS2jPv/nNp2zKMqIgQAJZvP8Dkf6Rx5ZRFvDJxCCN6xrqcsnl7ef4WOkaFMza5YebSshaEMeao7nGRR4sDwJAu0Xx491l0iArn5unLWJKx38V0zdvy7bks25rLHSO7ExrcMB/dViCMMTVKiIngX3eNICEmgl/MWsH+ghK3IzVLL8/PoE1ECBOGJTTYMa1AGGNOKKpFCC/dMIgDRWX86j0bM9HQNu09xJfr93LriK5EhDZcz4AVCGNMrZzeKYpHLktiwaYcXlmwxe04zco7S3cQHhLELWd2bdDjWoEwxtTajWck8pMBHXnm800stf6IBrMkYz9Du8YQ3cCXG1uBMMbUmojw5FX9SYyJ4I4300jblut2pCYvv7iMjXsPMbRrTIMf2wqEMaZOWoWH8M6kM4hrFcbN05fx7RZrSThp+fZcVLECYYxpHDpGtWDWncPp3KYFt76xjK835bgdqclatvUAIR5hUGKbBj+2owVCRMaIyEYRSReRh/ysf1ZEVnofm0Qkz2fdLSKy2fu4xcmcxpi6a9cqnFmTh9M9LpJJb6bx2Zo9bkdqklK35dK/cxThIZ4GP7ZjBUJEPMAU4BIgCZggIkm+26jqr1Q1WVWTgReBD7z7xgCPAmcAw4BHRSTaqazGmJPTNjKMWZOGc3rn1tz9zne8vzzL7UhNyuGyClZn5TG0W8OfXgJnWxDDgHRVzVDVUmAWMLaG7ScAM73PLwa+UNVcVT0AfAGMcTCrMeYkRUWE8PbtZzC8ewz3/2sVby7e5nakJmPFjjzKKpRhLvQ/gLMFojOQ6fM6y7vsGCLSBegGzKvLviIyWUTSRCQtJ8fOgRrjlpZhwbx+y1BGJ7Xn0TlreWneZlRtMN2pSt2WiwikdGl6BaIuxgOzVbWiLjup6jRVTVHVlLg456e+NcYcX3iIh6k3DuaqQZ15+vNN/OWzjVYkTlHqtlx6t2/1o/mxGpKTBWIn4DtpSLx3mT/j+eH0Ul33NcYEiGBPEE9fO5CJwxN55estPPKftTYtx0kqr6jku+0HXLm89QgnJ/VIBXqJSDeqPtzHAzdU30hE+gDRwLc+i+cC/+vTMT0aeNjBrMaYehIUJDw+th8tw4J59esMducXc8fI7gzrGtMg9zBoKtbtPkhhaQXDXOqgBgcLhKqWi8g9VH3Ye4DpqrpWRB4D0lR1jnfT8cAs9WmLqmquiDxOVZEBeCjSN98AAA/ySURBVExVbcimMY2EiPDQmD7ERITy4rx0vlyfTWJMBNcOiWfSOd1duWSzsVm2teojz80CIU3lHGFKSoqmpaW5HcMYU01RaTlz1+7hX2lZLN6yn5uGd+Hxcf3cjhXw7nwrjfW7D7HgwfMcPY6ILFfVFH/rAqWT2hjTREWEBnPloHjemTScSSO78daS7Xy5bq/bsQJaaXklqdvc7X8AKxDGmAb0m4t7k9SxNQ/MXsXeg4fdjhOwZi7bQW5hKWOTO7mawwqEMabBhAV7eGHCIIrLKrjfbjzkV2FJOS/O28yZ3dsyspe79wC3AmGMaVA920Xy6OWn8036Pv46dyMVViR+ZPo3W9lXUMqDY3oj4u5VX1YgjDENbvzQBK4ZEs8rX2/h6pcXs2nvIbcjBYTcwlJeXZDBxae3Z1Ci+9PPWYEwxjQ4EeGpawbw/PhkduQWcdkL3/DSvM3N/pTT1K/SKSot5zeje7sdBbACYYxxiYgwNrkzX/zqHEaf3p6nP9/E7z9c02yLxK68Yv6xZDtXD46nV/tWbscBnB1JbYwxJ9Q2MowXJwwiMSaCqfO3IAJPjO3X7EZdz127h9LySu45v6fbUY6yAmGMcZ2I8MDFvVHg5flbCBJ4fGw/1ztpG9KuvGLCgoNIjIlwO8pRViCMMQFBRHjw4t5UqvLq1xmUllfypyv7E+JpHmfCd+UfpmNUeEAVRSsQxpiAcWQOp1BPEC/OSyf7UAlTbhhMy7Cm/1G1O6+YjlEt3I7xI82jNBtjGg0R4f7RvfnfK/uzYFMO10/7luxDTX/U9e78w3RsE+52jB+xAmGMCUg3nJHIazensCW7kKumLmbrvkK3IzmmvKKSvQcP08laEMYYUzsX9G3PzMnDKSqt4NpXFrNmZ77bkRyRfaiESsVaEMYYUxfJCW34111nEhbsYfy0JSzess/tSPVud34xgLUgjDGmrnrERfL+z0bQqU04t05PZf7GbLcj1atdeVV9LNaCMMaYk9AhKpz37jyTnu0iufuf3zWp001HWxBtrAVhjDEnpU1EKG/cNpSoFiH8dEYqO/OK3Y5UL3blHSYyLJjW4SFuR/kRKxDGmEalfetw3rhtGMWlFdz2xjLyi8vcjnTKduUV0zEqsE4vgRUIY0wj1LtDK165aQgZOYXc9sayRn93uqoxEIF1egmsQBhjGqmzesbywoRBrN99iEufX8g3mxvv1U2784vpZC0IY4ypP5f278ice84ipmUoN01fyrNfbGp004WXlFewr6A04KbZACsQxphGrlf7VvznnrO4clBnnv/vZn7x7kpKyyvdjlVre/ID8xJXsMn6jDFNQERoMH+7Lpne7Vvx5083kFdUyisThzSKSf6OjIEItEFy4HALQkTGiMhGEUkXkYeOs811IrJORNaKyDs+yytEZKX3McfJnMaYpuHOc3vw12sGsCh9Hzf8fSm5haVuRzqhI2MgmlULQkQ8wBTgIiALSBWROaq6zmebXsDDwFmqekBE2vm8RbGqJjuVzxjTNF2XkkB0RCj3vPMdl7/4Dc9en8ywbjFuxzqu3fnNswUxDEhX1QxVLQVmAWOrbTMJmKKqBwBUtWmNnzfGuOKipPa8d+eZBHuE8dO+5W9fbKK8IjD7JXblFdMmIoQWoR63oxzDyQLRGcj0eZ3lXebrNOA0EVkkIktEZIzPunARSfMuH+fvACIy2btNWk5OTv2mN8Y0agMT2vDxfSO5clA8L/x3M+OnLSGvKPBOOe3OPxyQVzCB+1cxBQO9gFHABOA1EWnjXddFVVOAG4DnRKRH9Z1VdZqqpqhqSlxcXENlNsY0EpFhwTxz3UCeH5/M6qx8bnp9GflFgTXyeldeYI6BAGcLxE4gwed1vHeZryxgjqqWqepWYBNVBQNV3en9NwOYDwxyMKsxpgkbm9yZV28awsY9h7hp+tKAmp5jd/7hgJuk7wgnC0Qq0EtEuolIKDAeqH410odUtR4QkViqTjlliEi0iIT5LD8LWIcxxpyk8/q04+WJg1m/+yA3v76Ug4fdLxJFpeXkF5cF5BVM4GCBUNVy4B5gLrAeeE9V14rIYyJyhXezucB+EVkHfAU8oKr7gb5Amois8i5/0vfqJ2OMORkX9G3PyzcOYd3ug/z87e9c77gO5DEQ4PBAOVX9BPik2rJHfJ4r8Gvvw3ebxUB/J7MZY5qnC5Pa86dx/Xnw/dU88fF6/njF6a5lOToGIkD7IAJ/mKExxtSz64YmsGHPIaYv2kqfDq0YPyzRlRy7j7QgmmEfhDHGBKzfXdqHc06L4//9Zw3Ltua6kmFnXjEiVfe4CERWIIwxzVKwJ4gXJwwiITqC22ekMnftngbPsDu/mNjIMEKDA/OjODBTGWNMA4hqEcJbd5xB97iW3PnWcv708TrKGqjjWlXZkVsUsGMgwAqEMaaZ69ymBe/ddSY3n9mF1xZuZcK0JazOynPseIUl5bz17TYuenYBSzJyGRDf5oT7uEWqLiRq/FJSUjQtLc3tGMaYRuz/Vu3idx98z6GScoZ0iebWEV0Z068DIZ76+Vs6dVsut89I5eDhcvp1bs0tZ3bliuROhAW7Nw+TiCz3zlpx7DorEMYY84NDh8uYvTyLNxdvY9v+Ivp0aMXrtw6l8yleabQlp4Crpi6mbctQnrp2IIMT2yAi9ZT65NVUIOwUkzHG+GgVHsJtZ3Vj3v2jmHrjYHYeKGbclEWsyqzdaafyikpW7Djwo4kBcw6VcOsbywjxCDNuG8aQLtEBURxOxMZBGGOMH0FBwqX9O9KzXSQ/nZHK9dO+5alrBvo95VRRqXy/M58PV+zko9W72VdQQqgniAuT2jE2uTNTvkpn36FSZk0eTmLbCJe+orqzU0zGGHMC+wpKmPSPNFbsyCM4SEiMiaBbbEtKKyrJzC1iZ14xZRVKqCeIC/q2Y/Tp7VmVmc+cVbvILSwlSGDaTSlcmNTe7S/lGNYHYYwxp+hwWQWfrtnN5r0FbN1XyNZ9hYSFeIiPbkFCdAS92kVyYVJ7olqEHN2nrKKSrzfmEBocxDmnBeYtCWoqEHaKyRhjaiE8xMOVg+LrtE+IJyggWw21ZZ3Uxhhj/LICYYwxxi8rEMYYY/yyAmGMMcYvKxDGGGP8sgJhjDHGLysQxhhj/LICYYwxxq8mM5JaRHKA7d6XUUB+LZ8f+TcW2FfHw/q+X23WnWhZTfmcylnbrDW9DuSsgfT/35iy2s9qw2d16/+/i6r6H+atqk3uAUyr7XOff9NO5Ti1WXeiZSfI50jO2mat6XUgZw2k///GlNV+VpvPz2pNj6Z6iun/6vDcd9mpHKc26060rKZ8TuU83vrqy2p6HchZA+n/39/yQM1qP6sn1lR+Vo+ryZxiOlUikqbHmbAqkDSWnGBZndJYsjaWnGBZj6eptiBOxjS3A9RSY8kJltUpjSVrY8kJltUva0EYY4zxy1oQxhhj/LICYYwxxi8rEMYYY/yyAnECIjJSRF4Rkb+LyGK389RERIJE5E8i8qKI3OJ2npqIyCgRWej93o5yO8+JiEhLEUkTkcvcznI8ItLX+/2cLSI/cztPTURknIi8JiLvishot/PURES6i8jrIjLb7SzVeX8u3/R+L2+s7/dv0gVCRKaLSLaIrKm2fIyIbBSRdBF5qKb3UNWFqnoX8BHwZiBnBcYC8UAZkBXgWRUoAMIbQVaA3wLvOZOy3n5W13t/Vq8DzgrwrB+q6iTgLuD6AM+aoaq3O5WxujpmvgqY7f1eXlHvYU5mdF1jeQDnAIOBNT7LPMAWoDsQCqwCkoD+VBUB30c7n/3eA1oFclbgIeBO776zAzxrkHe/9sA/AzzrRcB44FbgskDN6d3nCuBT4IZA/p767PcMMLiRZHXsd+oUMj8MJHu3eae+swTThKnqAhHpWm3xMCBdVTMARGQWMFZV/wz4PX0gIolAvqoeCuSsIpIFlHpfVgRyVh8HgDAnckK9fV9HAS2p+oUsFpFPVLUy0HJ632cOMEdEPgbeqc+M9ZlVRAR4EvhUVb9zImd9ZW1odclMVes7HliJA2eEmnSBOI7OQKbP6yzgjBPsczvwhmOJjq+uWT8AXhSRkcACJ4P5UaesInIVcDHQBnjJ2WjHqFNWVf09gIjcCuyr7+JQg7p+T0dRdcohDPjE0WTHquvP6r3AhUCUiPRU1VecDFdNXb+vbYE/AYNE5GFvIWlox8v8AvCSiPyEU5uKw6/mWCDqTFUfdTtDbahqEVXFLOCp6gdUFbRGQ1VnuJ2hJqo6H5jvcoxaUdUXqPpwC3iqup+qvpKAo6qFwG1OvX+T7qQ+jp1Ags/reO+yQGRZndFYsjaWnGBZneZK5uZYIFKBXiLSTURCqep8nONypuOxrM5oLFkbS06wrE5zJ3ND9Mq79QBmArv54bLP273LLwU2UXVVwO/dzmlZLWtjyWlZm1dmm6zPGGOMX83xFJMxxphasAJhjDHGLysQxhhj/LICYYwxxi8rEMYYY/yyAmGMMcYvKxCmSRORggY+Xr3cM0Sq7peRLyIrRWSDiDxdi33GiUhSfRzfGLACYUydiEiN85ep6oh6PNxCVU0GBgGXiciJ7vEwjqoZZ42pF1YgTLMjIj1E5DMRWS5Vd7Xr411+uYgsFZEVIvKliLT3Lv+jiLwlIouAt7yvp4vIfBHJEJH7fN67wPvvKO/62d4WwD+9U1wjIpd6ly0XkRdE5KOa8qpqMVXTOXf27j9JRFJFZJWIvC8iESIygqp7QTzlbXX0ON7XaUxtWYEwzdE04F5VHQL8BpjqXf4NMFxVBwGzgAd99kkCLlTVCd7XfaiarnwY8KiIhPg5ziDgl959uwNniUg48Cpwiff4cScKKyLRQC9+mML9A1UdqqoDgfVUTcWwmKq5eR5Q1WRV3VLD12lMrdh036ZZEZFIYATwL+8f9PDDDYvigXdFpCNVd+3a6rPrHO9f8kd8rKolQImIZFN1Z7zqt05dpqpZ3uOuBLpSdZvVDFU98t4zgcnHiTtSRFZRVRyeU9U93uX9ROQJqu6lEQnMrePXaUytWIEwzU0QkOc9t1/di8DfVHWO9+Y7f/RZV1ht2xKf5xX4/12qzTY1Waiql4lIN2CJiLynqiuBGcA4VV3lvYnRKD/71vR1GlMrdorJNCuqehDYKiLXQtWtL0VkoHd1FD/MsX+LQxE2At19bil5/Yl28LY2ngR+613UCtjtPa11o8+mh7zrTvR1GlMrViBMUxchIlk+j19T9aF6u/f0zVqq7u0LVS2Gf4nIcmCfE2G8p6l+DnzmPc4hIL8Wu74CnOMtLP8PWAosAjb4bDMLeMDbyd6D43+dxtSKTfdtTAMTkUhVLfBe1TQF2Kyqz7qdy5jqrAVhTMOb5O20XkvVaa1XXc5jjF/WgjDGGOOXtSCMMcb4ZQXCGGOMX1YgjDHG+GUFwhhjjF9WIIwxxvhlBcIYY4xf/x+Mju4YbLMfigAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn.lr_find()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0.485073</td>\n",
       "      <td>0.425624</td>\n",
       "      <td>0.811203</td>\n",
       "      <td>00:08</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0.407019</td>\n",
       "      <td>0.346976</td>\n",
       "      <td>0.856155</td>\n",
       "      <td>00:10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>0.372527</td>\n",
       "      <td>0.343793</td>\n",
       "      <td>0.858921</td>\n",
       "      <td>00:10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>0.323204</td>\n",
       "      <td>0.633997</td>\n",
       "      <td>0.769710</td>\n",
       "      <td>00:10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>0.262023</td>\n",
       "      <td>0.370186</td>\n",
       "      <td>0.858921</td>\n",
       "      <td>00:10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>0.215317</td>\n",
       "      <td>0.383070</td>\n",
       "      <td>0.843015</td>\n",
       "      <td>00:10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>0.142278</td>\n",
       "      <td>0.513578</td>\n",
       "      <td>0.857538</td>\n",
       "      <td>00:10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6</td>\n",
       "      <td>0.095367</td>\n",
       "      <td>0.589843</td>\n",
       "      <td>0.847165</td>\n",
       "      <td>00:10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7</td>\n",
       "      <td>0.065511</td>\n",
       "      <td>0.615701</td>\n",
       "      <td>0.857538</td>\n",
       "      <td>00:10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8</td>\n",
       "      <td>0.041297</td>\n",
       "      <td>0.651564</td>\n",
       "      <td>0.854772</td>\n",
       "      <td>00:10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9</td>\n",
       "      <td>0.040339</td>\n",
       "      <td>0.665134</td>\n",
       "      <td>0.852006</td>\n",
       "      <td>00:10</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn.fine_tune(10, 2e-2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       ""
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "(#2) [0.665134072303772,0.8520055413246155]"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learn.validate()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "name": "python3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}