ToyTask.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:faiss.loader:Loading faiss with AVX2 support.\n",
      "INFO:faiss.loader:Loading faiss.\n"
     ]
    }
   ],
   "source": [
    "import argparse\n",
    "import os\n",
    "import torch\n",
    "import pytorch_lightning as pl\n",
    "import matplotlib.pyplot as plt\n",
    "from tqdm.notebook import tqdm, trange\n",
    "\n",
    "from diffmask.utils.callbacks import CallbackToyTask, CallbackToyTaskDiffMask\n",
    "from diffmask.models.toy_task import ToyTaskModel\n",
    "from diffmask.models.toy_task_diffmask import ToyTaskModelDiffMask\n",
    "\n",
    "from diffmask.attributions.integrated_gradient import integrated_gradient\n",
    "from diffmask.attributions.schulz import schulz_explainer, toy_hidden_states_statistics\n",
    "from diffmask.attributions.guan import guan_explainer\n",
    "from diffmask.utils.getter_setter import toy_getter, toy_setter"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Train the Toy Task model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "parser = argparse.ArgumentParser()\n",
    "parser.add_argument(\"--gpu\", type=str, default=\"0\")\n",
    "parser.add_argument(\"--epochs\", type=int, default=50)\n",
    "parser.add_argument(\"--num_embeddings\", type=int, default=10)\n",
    "parser.add_argument(\"--embedding_dim\", type=int, default=64)\n",
    "parser.add_argument(\"--hidden_size\", type=int, default=64)\n",
    "parser.add_argument(\"--data_size\", type=int, default=10000)\n",
    "parser.add_argument(\"--data_length\", type=int, default=10)\n",
    "parser.add_argument(\"--batch_size\", type=int, default=128)\n",
    "parser.add_argument(\"--learning_rate\", type=float, default=1e-3)\n",
    "parser.add_argument(\"--learning_rate_alpha\", type=float, default=3e-1)    \n",
    "parser.add_argument(\"--eps\", type=float, default=5e-2)\n",
    "parser.add_argument(\"--seed\", type=float, default=0)\n",
    "parser.add_argument(\n",
    "    \"--model_path\",\n",
    "    type=str,\n",
    "    default=\"./outputs/toy/version_2/checkpoints\",\n",
    ")\n",
    "hparams, _ = parser.parse_known_args()\n",
    "\n",
    "torch.manual_seed(hparams.seed)\n",
    "\n",
    "os.environ[\"CUDA_VISIBLE_DEVICES\"] = hparams.gpu"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_orig = ToyTaskModel(hparams)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:lightning:GPU available: True, used: True\n",
      "INFO:lightning:VISIBLE GPUS: 0\n"
     ]
    }
   ],
   "source": [
    "trainer = pl.Trainer(\n",
    "    gpus=int(hparams.gpu != \"\"),\n",
    "    progress_bar_refresh_rate=10,\n",
    "    max_epochs=hparams.epochs,\n",
    "    logger=pl.loggers.TensorBoardLogger(\"outputs\", name=\"toy\"),\n",
    "    callbacks=[CallbackToyTask()]\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:lightning:\n",
      "  | Name            | Type       | Params\n",
      "-------------------------------------------\n",
      "0 | embedding_query | Embedding  | 640   \n",
      "1 | embedding_input | Embedding  | 640   \n",
      "2 | encoder         | Sequential | 12 K  \n",
      "3 | encoder.0       | Linear     | 12 K  \n",
      "4 | encoder.1       | Tanh       | 0     \n",
      "5 | encoder.2       | Linear     | 130   \n",
      "6 | gru             | GRU        | 13 K  \n",
      "7 | head            | Linear     | 65    \n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validation sanity check'), FloatProgress(value=0.0, layout=Layout(flex='2'), max=5.…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "939caf5d72c144aa99bc2926c8f5ec66",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), max=1.0), H…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/ndecao/.anaconda3/envs/kilt37/lib/python3.7/site-packages/pytorch_lightning/utilities/warnings.py:18: RuntimeWarning: Displayed epoch numbers in the progress bar start from \"1\" until v0.6.x, but will start from \"0\" in v0.8.0.\n",
      "  warnings.warn(*args, **kwargs)\n",
      "/home/ndecao/.anaconda3/envs/kilt37/lib/python3.7/site-packages/pytorch_lightning/utilities/warnings.py:18: UserWarning: The dataloader, train dataloader, does not have many workers which may be a bottleneck. Consider increasing the value of the `num_workers` argument` in the `DataLoader` init to improve performance.\n",
      "  warnings.warn(*args, **kwargs)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=0.0, layout=Layout(flex='2'), max=8.0), HTML(valu…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1: Validation accuracy = 63.77\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=0.0, layout=Layout(flex='2'), max=8.0), HTML(valu…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 2: Validation accuracy = 63.77\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=0.0, layout=Layout(flex='2'), max=8.0), HTML(valu…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 3: Validation accuracy = 66.86\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=0.0, layout=Layout(flex='2'), max=8.0), HTML(valu…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 4: Validation accuracy = 78.04\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=0.0, layout=Layout(flex='2'), max=8.0), HTML(valu…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 5: Validation accuracy = 86.24\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=0.0, layout=Layout(flex='2'), max=8.0), HTML(valu…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 6: Validation accuracy = 94.11\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=0.0, layout=Layout(flex='2'), max=8.0), HTML(valu…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 7: Validation accuracy = 94.25\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=0.0, layout=Layout(flex='2'), max=8.0), HTML(valu…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 8: Validation accuracy = 95.44\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=0.0, layout=Layout(flex='2'), max=8.0), HTML(valu…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 9: Validation accuracy = 95.47\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=0.0, layout=Layout(flex='2'), max=8.0), HTML(valu…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 10: Validation accuracy = 98.29\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=0.0, layout=Layout(flex='2'), max=8.0), HTML(valu…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 11: Validation accuracy = 97.42\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=0.0, layout=Layout(flex='2'), max=8.0), HTML(valu…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 12: Validation accuracy = 98.69\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=0.0, layout=Layout(flex='2'), max=8.0), HTML(valu…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 13: Validation accuracy = 95.96\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=0.0, layout=Layout(flex='2'), max=8.0), HTML(valu…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 14: Validation accuracy = 97.86\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=0.0, layout=Layout(flex='2'), max=8.0), HTML(valu…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 15: Validation accuracy = 99.02\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=0.0, layout=Layout(flex='2'), max=8.0), HTML(valu…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 16: Validation accuracy = 97.78\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=0.0, layout=Layout(flex='2'), max=8.0), HTML(valu…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 17: Validation accuracy = 98.39\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=0.0, layout=Layout(flex='2'), max=8.0), HTML(valu…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 18: Validation accuracy = 99.02\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=0.0, layout=Layout(flex='2'), max=8.0), HTML(valu…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 19: Validation accuracy = 97.83\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=0.0, layout=Layout(flex='2'), max=8.0), HTML(valu…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 20: Validation accuracy = 98.22\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=0.0, layout=Layout(flex='2'), max=8.0), HTML(valu…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 21: Validation accuracy = 99.32\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=0.0, layout=Layout(flex='2'), max=8.0), HTML(valu…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 22: Validation accuracy = 97.71\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=0.0, layout=Layout(flex='2'), max=8.0), HTML(valu…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 23: Validation accuracy = 99.22\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=0.0, layout=Layout(flex='2'), max=8.0), HTML(valu…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 24: Validation accuracy = 99.32\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=0.0, layout=Layout(flex='2'), max=8.0), HTML(valu…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 25: Validation accuracy = 99.20\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=0.0, layout=Layout(flex='2'), max=8.0), HTML(valu…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 26: Validation accuracy = 99.80\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=0.0, layout=Layout(flex='2'), max=8.0), HTML(valu…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 27: Validation accuracy = 99.32\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=0.0, layout=Layout(flex='2'), max=8.0), HTML(valu…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 28: Validation accuracy = 99.71\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=0.0, layout=Layout(flex='2'), max=8.0), HTML(valu…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 29: Validation accuracy = 99.71\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=0.0, layout=Layout(flex='2'), max=8.0), HTML(valu…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 30: Validation accuracy = 99.61\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=0.0, layout=Layout(flex='2'), max=8.0), HTML(valu…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 31: Validation accuracy = 99.80\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=0.0, layout=Layout(flex='2'), max=8.0), HTML(valu…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 32: Validation accuracy = 99.80\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=0.0, layout=Layout(flex='2'), max=8.0), HTML(valu…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 33: Validation accuracy = 99.90\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=0.0, layout=Layout(flex='2'), max=8.0), HTML(valu…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 34: Validation accuracy = 99.90\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=0.0, layout=Layout(flex='2'), max=8.0), HTML(valu…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 35: Validation accuracy = 99.90\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=0.0, layout=Layout(flex='2'), max=8.0), HTML(valu…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 36: Validation accuracy = 99.90\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=0.0, layout=Layout(flex='2'), max=8.0), HTML(valu…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 37: Validation accuracy = 96.66\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=0.0, layout=Layout(flex='2'), max=8.0), HTML(valu…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 38: Validation accuracy = 99.32\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=0.0, layout=Layout(flex='2'), max=8.0), HTML(valu…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 39: Validation accuracy = 99.80\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=0.0, layout=Layout(flex='2'), max=8.0), HTML(valu…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 40: Validation accuracy = 99.61\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=0.0, layout=Layout(flex='2'), max=8.0), HTML(valu…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 41: Validation accuracy = 99.71\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=0.0, layout=Layout(flex='2'), max=8.0), HTML(valu…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 42: Validation accuracy = 99.71\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=0.0, layout=Layout(flex='2'), max=8.0), HTML(valu…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 43: Validation accuracy = 99.80\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=0.0, layout=Layout(flex='2'), max=8.0), HTML(valu…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 44: Validation accuracy = 99.80\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=0.0, layout=Layout(flex='2'), max=8.0), HTML(valu…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 45: Validation accuracy = 99.80\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=0.0, layout=Layout(flex='2'), max=8.0), HTML(valu…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 46: Validation accuracy = 99.80\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=0.0, layout=Layout(flex='2'), max=8.0), HTML(valu…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 47: Validation accuracy = 99.71\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=0.0, layout=Layout(flex='2'), max=8.0), HTML(valu…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 48: Validation accuracy = 99.80\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=0.0, layout=Layout(flex='2'), max=8.0), HTML(valu…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 49: Validation accuracy = 99.80\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=0.0, layout=Layout(flex='2'), max=8.0), HTML(valu…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 50: Validation accuracy = 99.80\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "1"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trainer.fit(model_orig)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "hparams.model_path_ = os.path.join(hparams.model_path, os.listdir(hparams.model_path)[0])\n",
    "model_orig = ToyTaskModel.load_from_checkpoint(hparams.model_path_)\n",
    "model_orig.prepare_data()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Create an example"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor([[8, 1]]),\n",
       " tensor([[8, 8, 3, 1, 3, 9, 0, 0, 0, 0]]),\n",
       " tensor([[1, 1, 1, 1, 1, 1, 0, 0, 0, 0]]),\n",
       " tensor([1]),\n",
       " tensor([[1., 1., 0., 1., 0., 0., 0., 0., 0., 0.]]))"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "device = \"cpu:0\"\n",
    "model_orig.to(device)\n",
    "\n",
    "i = 0\n",
    "query_ids, input_ids, mask, labels = model_orig.val_dataset[:]\n",
    "filtered = (\n",
    "    (labels == 1) \n",
    "    & (mask.sum(-1) == 6) \n",
    "    & (input_ids == query_ids[:,0].unsqueeze(-1)).any(-1) \n",
    "    & (input_ids == query_ids[:,1].unsqueeze(-1)).any(-1)\n",
    ")\n",
    "query_ids, input_ids, mask, labels = tuple(e[filtered][i].unsqueeze(0).to(device)\n",
    "                                           for e in model_orig.val_dataset[:])\n",
    "attributions_gt = ((input_ids == query_ids[0, 0]) | (input_ids == query_ids[0, 1])).float().cpu()\n",
    "query_ids, input_ids, mask, labels, attributions_gt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_attribution(\n",
    "    query_ids,\n",
    "    input_ids,\n",
    "    attributions,\n",
    "    attributions_gt,\n",
    "    name=None,\n",
    "    legend=False,\n",
    "    save=False,\n",
    "    ymax=1\n",
    "):\n",
    "    \n",
    "    fig, axs = plt.subplots(1, 1,figsize=(6, 4), sharex=True)\n",
    "    fig.patch.set_facecolor('white')\n",
    "    plt.xticks(\n",
    "        torch.arange(input_ids.shape[-1]),\n",
    "        [r\"$\\bf{{{}}}$\".format(e) if e in query_ids else \"${}$\".format(e) for e in input_ids],\n",
    "        size=32\n",
    "    )\n",
    "    \n",
    "    axs.bar(\n",
    "        torch.arange(input_ids.shape[-1]),\n",
    "        attributions / attributions.sum(),\n",
    "        label=\"attribution\"\n",
    "    )\n",
    "\n",
    "    axs.locator_params(axis='y', nbins=5)\n",
    "    axs.axis(ymin=0, ymax=ymax)\n",
    "    axs.tick_params(axis='y', labelsize=24)\n",
    "    \n",
    "    if save:\n",
    "        plt.savefig(name, bbox_inches=\"tight\", pad_inches=0)\n",
    "    else:\n",
    "        plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "hidden_state_idx = 1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Sundararajan et al. (2017) - Integrated Gradient"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "bd1bc2511b614565bc617214be7334ea",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=0.0, max=500.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "attributions_ig = integrated_gradient(\n",
    "        model_orig,\n",
    "        {\"query_ids\": query_ids, \"input_ids\": input_ids, \"mask\": mask, \"labels\": labels},\n",
    "        getter=toy_getter,\n",
    "        setter=toy_setter,\n",
    "        label_getter=lambda outputs, inputs_dict: outputs,\n",
    "        hidden_state_idx=hidden_state_idx,\n",
    "        steps=500,\n",
    "    ).sum(-1).abs().cpu()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZoAAAENCAYAAAA2ZaOYAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAAmm0lEQVR4nO3de1TUdf4/8OcMILcBbbkoYgvoKVE0QVg13ewgJpVmtikl7tH6Q+ziZh4kr5u2fcUyrOx0MvGS7a520aVVwbxf1lBZR0UDxWpFVAQFVBhgQHHevz/4zSdxLnxmmA8D0/NxzpwzM+/3+zOv9/kUTz93lRBCgIiISCFqZxdARESujUFDRESKYtAQEZGiGDRERKQoBg0RESnK3dkFdDSBgYEIDw93dhlERJ3KxYsXUVlZabaNQXOf8PBwaLVaZ5dBRNSpxMXFWWzjrjMiIlIUg4aIiBTFoCEiIkUxaIiISFEMGiIiUhSDhoiIFMWgISIiRTFoiIhIUQwaIiJSFIOGiIgUJTtodDodtm3bhr/+9a946qmnEBgYCJVKBZVKhaKiojYXUlNTg0WLFqFfv37w8fFBQEAAEhISsGXLFlnjN2/ejFGjRiEgIAA+Pj7o168fFi1aBJ1O1+baiIioDYRM3333nQBg9nXu3Dm5izHr8uXLIiIiQlqeRqMR7u7u0udXX33V6vjp06dLfd3d3YVGo5E+9+7dW5SWlsquJTY2tk1zISL6LbL2t9OmXWfBwcF4+umnsXjxYmRmZjoq6DBx4kQUFxcjPDwcubm50Ol00Ol0WL58OdRqNVatWoU1a9aYHW9sU6vV+OCDD1BbWwudTofc3FyEhYXhwoULSEpKckitRERkB7lp1dTU1OJzcXGxQ7ZojFtKarVanDp1yqT9zTffFABEjx49RGNjY4u2hoYGERwcLACI2bNnm4w9efKkUKlUAoDYtm2brHq4RUNEZDuHbNG4ubkpEnQbN24EAIwePRrR0dEm7XPmzIFKpUJ5eTn279/fom3v3r24fv06VCoVUlNTTcbGxMRg9OjRLX6HiIjal9PPOjtw4AAAIDEx0Wx7aGgooqKiAMAkaIxjBwwYgNDQULPjjcu9fywREbUPpwbN9evXUVVVBQBSmJjTv39/AMDZs2dbfG/8LGdsRUWFxae/ERGRcpwaNGVlZdL7nj17WuxnbLu3/72f5Yw1N56IiJTn1Ec519XVSe+9vb0t9vPx8QEA1NbWmh0vZ6y58UaZmZnSWXQVFRWtVE1ERLZw+jGajiAlJQVarRZarRZBQUHOLoeIyKU4NWh8fX2l93q93mK/+vp6AIBGozE7Xs5Yc+OJiEh5Tg2ae4+fXL161WI/Y1tISIjZ8XLGmhtPRETKc2rQBAUFITAwEABQWFhosZ/x7DLjGWRGxs9yxt77W0RE1H6cfowmPj4eALBnzx6z7aWlpVKQJCQkmB1bWFho8Yyy3bt3mx1LRETtw+lBk5ycDKA5EE6fPm3S/uGHH0IIgZCQEClYjBISEhAcHAyDwYAVK1aYjD19+jT27t0LAJgyZYoC1RMRUWtsCprKykrpdfPmTen7W7dutWgzGAwtxhkfJ7BkyRKTZT777LMYOnQoDAYDnnvuORw7dgwA0NjYiBUrVuDjjz8GALzzzjvo0qVLi7Genp7SMj/66COsWLECjY2NAICjR4/iueeeg8FgwIgRIzBu3DhbpkpERI5iy03TYOExAfe/iouLzY5bvHix2eW29piAV155xWpd9z4mwMPDg48JICJqZw57TIBSevXqhfz8fCxYsACRkZFoamqCn58f4uPj8e2332LVqlVWx2dmZuKbb75BfHw8NBoNmpqaEBkZiYULFyI/P9/qnQOIiEhZKiGEcHYRHUlcXBy0Wq2zyyAi6lSs/e3sEFs0RETkuhg0RESkKAYNEREpikFDRESKYtAQEZGiGDRERKQoBg0RESmKQUNERIpi0BARkaIYNEREpCgGDRERKYpBQ0REimLQEBGRohg0RESkKAYNEREpikFDRESKYtAQEZGiGDRERKQoBg0RESmKQUNERIpi0BARkaIYNEREpCgGDRERKYpBQ0REimLQEBGRohg0RESkKAYNEREpikFDRESKYtAQEZGiGDRERKQoBg0RESnK5qApLy/HrFmz0KdPH3h5eaF79+545plnsG/fPrsKCA8Ph0qlkvX68ssvTcbLGbdlyxa7aiMiorZzt6XzmTNnMGrUKFRVVQEA/P39UVlZiezsbOTk5CA9PR3z5s2zqYCgoCA0NDRYbK+rq0NtbS0AYPDgwRb7BQYGws3NzWybl5eXTTUREZEDCZnq6+tFWFiYACBiYmJEQUGBEEKI6upqkZqaKgAIlUoldu3aJXeRsowfP14AEIMHDzbbDkAAEMXFxQ75vdjYWIcsh4jot8Ta307Zu85Wr16NkpISaDQabN++HVFRUQCat2oyMjIwYcIECCEwf/58h4VgRUUFvv/+ewDAtGnTHLZcIiJqP7KDZuPGjQCA5ORkhIaGmrSnpaUBAE6ePInz5887pLhNmzbhzp078PDwQHJyskOWSURE7UtW0Oh0Opw4cQIAkJiYaLbPsGHD0LVrVwCw+8SA+xkP/o8dOxaBgYEOWSYREbUvWUFz7tw5CCEAQNplZrIgtRp9+/YFAJw9e7bNhf344484deoUAHm7zZKSkvDAAw/A09MTvXr1wvPPP4+cnJw210FERG0jK2jKysqk9z179rTYz9h2b397bdiwAUDz2WRjx45ttf/x48dx9+5deHh4oLS0FFlZWRg3bhySkpJw+/btNtdDRET2kRU0dXV10ntvb2+L/Xx8fABAOh3ZXk1NTS2OCXl4eFjsO23aNOzcuRM3b95ETU0Namtrce7cObz88ssAgM2bN2PmzJlWfy8zMxNxcXGIi4tDRUVFm2onIqKWOuSdAXbt2oVr164BaH232YYNG5CYmIhu3bpJ30VGRmL9+vXSCQpr1661eoJCSkoKtFottFotgoKC2j4BIiKSyAoaX19f6b1er7fYr76+HgCg0WjaVJTxJICBAwdavUizNYsXL4a3tzeEEMjOzm5TTUREZB9ZQXPvcZmrV69a7GdsCwkJsbugmzdvYtu2bQDafu2Mr68vBgwYAAC4cOFCm5ZFRET2kRU0kZGRUKlUAIDCwkKzfQwGg7R7qn///nYX9PXXX6OxsRHu7u6YMmWK3cshIqKOQVbQ+Pn5IS4uDgCwZ88es33y8vJQXV0NAEhISLC7IONus8TERPTo0cPu5QDNJzEUFBQAACIiItq0LCIiso/skwGMV+Zv3LjR7OnLGRkZAIDY2FjpehpbnT9/Hnl5eQDk7TYzXttjybvvvgu9Xg+VSoWnn37arpqIiKhtZAfNjBkzEBYWBp1Oh3HjxkkXZep0Orz11lvIysoCAKSnp5uMNd6uf8mSJVZ/w7g188ADD2D8+PGt1pSUlISFCxdCq9W2uFbm/PnzmD59Ot5//30AzaHVlt15RERkP9mPCfD29sbWrVuRkJCAkydPIioqCv7+/qitrYXBYIBKpUJ6ejrGjBljVyEGgwH/+Mc/AAAvvvgiPD09Wx1TUVGBLVu2ID09HW5ubujatSsaGxtbXPczceJEfP7553bVREREbWfT82gGDRqEgoICLFu2DNnZ2SgtLUVAQACGDBmC2bNnt+nYzP79+3HlyhUA8s82W7BgAR555BEcO3YMV65cwY0bN6BWqxEREYFhw4bhpZdesjv4iIjIMVSitQMdvzFxcXHQarXOLoOIqFOx9rezQ94ZgIiIXAeDhoiIFMWgISIiRTFoiIhIUQwaIiJSFIOGiIgUxaAhIiJFMWiIiEhRDBoiIlIUg4aIiBTFoCEiIkUxaIiISFEMGiIiUhSDhoiIFMWgISIiRTFoiIhIUQwaIiJSFIOGiIgUxaAhIiJFMWiIiEhRDBoiIlIUg4aIiBTFoCEiIkUxaIiISFEMGiIiUhSDhoiIFMWgISIiRTFoiIhIUQwaIiJSFIOGiIgUZXPQlJeXY9asWejTpw+8vLzQvXt3PPPMM9i3b59dBRw8eBAqlarVV2VlpdXlbN68GaNGjUJAQAB8fHzQr18/LFq0CDqdzq66iIjIMdxt6XzmzBmMGjUKVVVVAAB/f39UVlYiOzsbOTk5SE9Px7x58+wqRK1WIygoyGq7JSkpKVizZg0AwN3dHV5eXigqKsLSpUvx1Vdf4fDhw+jZs6dddRERUdvI3qLR6/UYP348qqqqEBMTg4KCAlRXV+PmzZtITU2FEAILFizA7t277SrkwQcfRHl5ucXX7373O7PjVq1ahTVr1kCtVuODDz5AbW0tdDodcnNzERYWhgsXLiApKcmumoiIqO1kB83q1atRUlICjUaD7du3IyoqCkDzVk1GRgYmTJgAIQTmz5+vWLH3a2xsxJIlSwAAs2bNwpw5c+Dp6QkAGD58OL777juoVCrk5uZi+/bt7VYXERH9SnbQbNy4EQCQnJyM0NBQk/a0tDQAwMmTJ3H+/HkHlWfd3r17cf36dahUKqSmppq0x8TEYPTo0QB+rZ+IiNqXrKDR6XQ4ceIEACAxMdFsn2HDhqFr164AYPeJAbY6cOAAAGDAgAFmww/4td79+/e3S01ERNSSrKA5d+4chBAAIO0yM1mQWo2+ffsCAM6ePWtzIRUVFRg8eDB8fX3h6+uLhx9+GCkpKfjxxx8tjjH+jqWaAKB///7S8ls7c42IiBxPVtCUlZVJ762dvWVsu7e/XPX19Th16hQ8PT3R1NSEn3/+GWvWrEFMTAwyMjKs1iWnJnvrIiKitpEVNHV1ddJ7b29vi/18fHwAALW1tbIL6NatG9LS0qDVaqHX63Hjxg3U19fj0KFDGD58OO7evYu0tDRs2rTJYl1yarJWV2ZmJuLi4hAXF4eKigrZtRMRUeucfmeA6OhoLF++HLGxsfDy8gIAuLm5YeTIkThw4ABGjBgBAJg7dy4MBoMiNaSkpECr1UKr1Vq9loeIiGwnK2h8fX2l93q93mK/+vp6AIBGo2ljWc26dOmCd999FwBw5coVnDp1ymxdcmpyZF1ERCSfrKC59zjH1atXLfYztoWEhLSxrF8NHTpUen/hwgWzdcmpydF1ERGRPLKCJjIyEiqVCgBQWFhoto/BYJCunzGe6aU04+9Yqgn49cy0oKAgBAYGtktdRET0K1lB4+fnh7i4OADAnj17zPbJy8tDdXU1ACAhIcFB5TUv1ygiIqJFW3x8PIDmoLF0RpnxljiOrImIiOSTfTJAcnIygOYr7M39UTeeghwbGytdTyOH8focc+7cuYO3334bQPNur8GDB7doT0hIQHBwMAwGA1asWGEy/vTp09i7dy8AYMqUKbJrIiIiBxIy1dfXi7CwMAFADB48WBQWFgohhKipqRFpaWkCgAAgdu3aZTLW2LZ48WKTtv79+4tPPvlE/PTTT8JgMAghhGhqahKHDx8Wjz32mDR2w4YNZuv67LPPBAChVqtFRkaGaGhoEEIIceTIERERESEAiBEjRsidpoiNjZXdl4iImln72yk7aIQQIj8/XwQEBEh//P39/YVarRYAhEqlEsuWLTP/I1aCxtgGQHh6eorAwEDRpUsX6Tt3d3fx3nvvWa1r+vTpUn8PDw+h0Wikz7179xalpaWy58igISKynbW/nTZdRzNo0CAUFBTgjTfeQO/evdHY2IiAgACMHTsWe/bssetZNKtXr8bUqVMRFRUFf39/3Lp1C56enhg4cCBmzpyJ06dPY+7cuVaXkZmZiW+++Qbx8fHQaDRoampCZGQkFi5ciPz8fD6LhojIiVRCWDlI8hsUFxcHrVbr7DKIiDoVa387nX5nACIicm0MGiIiUhSDhoiIFMWgISIiRTFoiIhIUQwaIiJSFIOGiIgUxaAhIiJFMWiIiEhRDBoiIlIUg4aIiBTFoCEiIkW5O7sAVxI+L8fZJchy8b2xzi6BiH5DuEVDRESKYtAQEZGiGDRERKQoBg0RESmKQUNERIpi0BARkaIYNEREpCgGDRERKYpBQ0REimLQEBGRohg0RESkKAYNEREpikFDRESKYtAQEZGiGDRERKQoBg0RESmKQUNERIpi0BARkaJsDpry8nLMmjULffr0gZeXF7p3745nnnkG+/bts6uAiooKrF69GpMmTZKW6evri379+mHmzJn45ZdfrI5XqVStvrZs2WJXbURE1HbutnQ+c+YMRo0ahaqqKgCAv78/KisrkZ2djZycHKSnp2PevHk2FdCzZ080NTVJnzUaDW7fvo2ioiIUFRVh3bp1WL9+PSZPnmx1OYGBgXBzczPb5uXlZVNNRETkOLK3aPR6PcaPH4+qqirExMSgoKAA1dXVuHnzJlJTUyGEwIIFC7B7926bCmhqasLIkSPx5ZdfoqysDDqdDvX19fjhhx8QHR2NhoYGTJ06FWfOnLG6nOPHj6O8vNzsa9y4cTbVREREjiM7aFavXo2SkhJoNBps374dUVFRAJq3ajIyMjBhwgQIITB//nybCjh06BAOHTqEqVOnokePHgAANzc3jBgxArt370ZwcDCamprw0Ucf2bRcIiLqGGQHzcaNGwEAycnJCA0NNWlPS0sDAJw8eRLnz5+XXcDIkSMttgUFBeHpp58GAJw4cUL2MomIqOOQFTQ6nU76Q5+YmGi2z7Bhw9C1a1cAsPvEAHMCAgIAAHfv3nXYMomIqP3ICppz585BCAEA0i4zkwWp1ejbty8A4OzZsw4qr3nXGgAMGDDAar+kpCQ88MAD8PT0RK9evfD8888jJyfHYXUQEZF9ZJ11VlZWJr3v2bOnxX7Gtnv7t8XWrVuh1WoBAC+//LLVvsePH4efnx88PDxQWlqKrKwsZGVlYdKkSfjnP/+JLl26WBybmZmJzMxMAM2nWxN1JuHzOsc/qC6+N9bZJZCTyNqiqaurk957e3tb7Ofj4wMAqK2tbWNZQGlpKVJSUgAA48ePx5NPPmm237Rp07Bz507cvHkTNTU1qK2txblz56Rg2rx5M2bOnGn1t1JSUqDVaqHVahEUFNTm2omI6Fcd8s4AtbW1mDBhAq5fv46wsDCsW7fOYt8NGzYgMTER3bp1k76LjIzE+vXrpRMU1q5da9MJCkRE5DiygsbX11d6r9frLfarr68H0HzRpb0aGhrw7LPPSlsXu3btQmBgoF3LWrx4Mby9vSGEQHZ2tt01ERGR/WQFzb3HZa5evWqxn7EtJCTErmJu376NiRMnYv/+/ejWrRt2794tnWBgD19fX+kkggsXLti9HCIisp+soImMjIRKpQIAFBYWmu1jMBik3VP9+/e3uZCmpiZMnjwZOTk50Gg02LFjB6Kjo21eDhERdSyygsbPzw9xcXEAgD179pjtk5eXh+rqagBAQkKCTUUYDAZMmzYNWVlZ8Pb2xrZt2/Doo4/atAxz6urqUFBQAACIiIho8/KIiMh2sk8GSE5OBtB8hwBzpy9nZGQAAGJjY23a3SWEQEpKCjZt2oQuXbogKysL8fHxssda8+6770Kv10OlUkl3GCAiovYlO2hmzJiBsLAw6HQ6jBs3TrooU6fT4a233kJWVhYAID093WSs8Xb9S5YsMWmbPXs21q1bB3d3d3z77bcWT2M2JykpCQsXLoRWq8Xt27el78+fP4/p06fj/fffB9B8CrQ9u/OIiKjtZD8mwNvbG1u3bkVCQgJOnjyJqKgo+Pv7o7a2FgaDASqVCunp6RgzZozsH7906RJWrlwJoDmMZsyYgRkzZljsX15e3uJzRUUFtmzZgvT0dLi5uaFr165obGxscd3PxIkT8fnnn8uuiYiIHMum59EMGjQIBQUFWLZsGbKzs1FaWoqAgAAMGTIEs2fPtuvYjNGdO3dw7do1m8YvWLAAjzzyCI4dO4YrV67gxo0bUKvViIiIwLBhw/DSSy/ZFHxEROR4NgUNAPTo0QMrV66UtkTksHQsJTw8vNXjLNaMGTOGQUJE1MF1yDsDEBGR62DQEBGRohg0RESkKJuP0dBvC29BT0RtxS0aIiJSFIOGiIgUxaAhIiJFMWiIiEhRDBoiIlIUg4aIiBTFoCEiIkUxaIiISFEMGiIiUhSDhoiIFMWgISIiRTFoiIhIUQwaIiJSFIOGiIgUxccE0G8KH3tA1P64RUNERIpi0BARkaIYNEREpCgGDRERKYpBQ0REimLQEBGRohg0RESkKAYNEREpikFDRESKYtAQEZGieAsaIupQeJsg12Nz0JSXl2PZsmXIzs5GaWkpunbtiiFDhuDNN99EQkKC3YXU1NRg+fLl+Ne//oWSkhJ4e3sjOjoar776KiZOnNjq+M2bN2PVqlU4ffo09Ho9wsLC8Pzzz2Pu3Lnw8/Ozuy4iorb6rYenTbvOzpw5gwEDBuCTTz7BhQsX4OnpicrKSmRnZ+OJJ57Ae++9Z1cRV65cQXR0NJYuXYqioiK4ubmhpqYG+/fvx6RJk/Daa69ZHZ+SkoKkpCQcOHAANTU1cHNzQ1FREZYuXYro6GhcvXrVrrqIiKjtZAeNXq/H+PHjUVVVhZiYGBQUFKC6uho3b95EamoqhBBYsGABdu/ebVMBQghMnDgRxcXFCA8PR25uLnQ6HXQ6HZYvXw61Wo1Vq1ZhzZo1Zscb29RqNT744APU1tZCp9MhNzcXYWFhuHDhApKSkmyqiYiIHEd20KxevRolJSXQaDTYvn07oqKiAAD+/v7IyMjAhAkTIITA/PnzbSpg69atyMvLg1qtxnfffYfhw4cDALy8vJCWloY33ngDAPD222/j9u3bLcY2NjZiyZIlAIBZs2Zhzpw58PT0BAAMHz4c3333HVQqFXJzc7F9+3ab6iIiIseQHTQbN24EACQnJyM0NNSkPS0tDQBw8uRJnD9/XnYBxuWOHj0a0dHRJu1z5syBSqVCeXk59u/f36Jt7969uH79OlQqFVJTU03GxsTEYPTo0S1+h4iI2pesoNHpdDhx4gQAIDEx0WyfYcOGoWvXrgCAffv2yS7gwIEDVpcbGhoqbT3dHzTGsQMGDDAbfvcu9/6xRETUPmQFzblz5yCEAADpj77JgtRq9O3bFwBw9uxZWT9+/fp1VFVVWV0uAPTv39/sco2f5YytqKhAZWWlrLqIiMhxZAVNWVmZ9L5nz54W+xnb7u2v5HKNn+WMtaUuIiJyHFnX0dTV1Unvvb29Lfbz8fEBANTW1sr68bYu1zhezlhrdWVmZiIzMxMAUFRUhLi4uFYqNy/QrlHWVVRUICgoyKHLjItbLLuvq83J1eYDuN6cXG0+gGvO6X4XL1602MY7A6D5OpyUlBRnl2FWXFwctFqts8twKFebk6vNB3C9ObnafIDONSdZu858fX2l93q93mK/+vp6AIBGo5H1421drnG8nLG21EVERI4jK2juPc5h7Sp7Y1tISIisH2/rco3j5Yy1pS4iInIcWUETGRkJlUoFACgsLDTbx2AwSNfPGM/0ak1QUBACAwOtLhf49eyy+5dr/Cxn7L2/1Zl01F16beFqc3K1+QCuNydXmw/QyeYkZPrDH/4gAIhXXnnFbPuRI0cEAAFAFBUVyV2smDRpkgAgnnzySbPtV65cESqVSgAQO3fubNGWk5MjAAi1Wi2uXr1qdvyYMWMEAPHiiy/KromIiBxH9p0BkpOTATRfYW/uNOGMjAwAQGxsrHQ9jS3L3b17N06fPm3S/uGHH0IIgZCQEMTHx7doS0hIQHBwMAwGA1asWGEy9vTp09i7dy8AYMqUKbJrIiIiB5KbSPX19SIsLEwAEIMHDxaFhYVCCCFqampEWlqatDWza9cuk7HGtsWLF5u0GQwGMXToUAFAREREiKNHjwohhGhoaBAZGRlCrVYLACIzM9NsXZ999pm0VZORkSEaGhqEEM1bWBEREQKAGDFihNxpEhGRg8kOGiGEyM/PFwEBAVJw+Pv7S0GgUqnEsmXLzP+IlaARQojLly9LoQBAaDQa4e7uLn22tLvOaPr06VJfDw8PodFopM+9e/cWpaWltkyTiIgcyKbraAYNGoSCgoIWDz4LCAjAkCFDMHv2bLsffNarVy/k5+fj/fffR1ZWFi5evAg/Pz/pwWeTJk2yOj4zMxOjR4/G559/jvz8fOj1ekRGRna6B5/V19dj+/bt+P7771FQUIBLly5Bp9Phzp078PX1RUBAAB566CE8+uijmDBhgtmbkHY0rjSnX375BXl5edBqtcjPz0dZWRkqKytRXV0NX19f9OzZE7GxsZg4cSLGjRsHNzc3Z5fcqqqqKmi1Whw/flx6mds1/vjjj+PgwYPtX6CNXHEdGdXW1mL9+vXYsWMHzpw5g6qqKmg0GoSGhmLMmDF46aWXMGDAAGeXaZ6zk46affXVV6JHjx7Slpic11NPPSXKysqcXbpFrjanPn36yJ7HoEGDxJkzZ5xdslWbNm2SPZ/HH3/c2eXK4mrryGjbtm0t9iaZe7m7u4sFCxYIg8Hg7HJN8M4AHcDatWsxffp0m8d9//33GDFiBE6cOIFu3bo5vrA2cMU5GYWGhmLw4MHo3bs3AgIC4OHhgaqqKpw8eRKHDh3C3bt3cfr0aYwcORJHjhxBv379nF2yWY2Njc4uQTGuso4AYMuWLUhKSpJubBwUFITx48cjIiICer0eeXl52LdvH5qampCeno66ujp8/PHHzi36fs5Out+6W7duCV9fX5v+1X//a9GiRc6eRguuOCchhMjKyhIlJSVW+5w/f15ERUVJ84iPj2+n6mz3xRdfuNwWjauto/Ly8hbHnJOTk0Vtba1Jv9zcXBEUFCT1y87OdkK1ljFonMzS7ovg4GDx9ddfi/LycqHT6cSJEyfE2LFjzfYdOnSos6fRgivOyRYFBQXSPFQqlSgvL3d2SWZ98cUXwsvLSwwdOlT85S9/EX//+99FUVFRpw4auTrLOlq4cKFUZ0xMjLhz547Fvrt27ZL6RkZGdqhdaAwaJ1u2bJnZ/7E3bNhg0lev1wsvLy+Tvn379nVC5Za54pxsFRwcLM3l2LFjzi7HLJ1OJ27fvm3y/W8haIToHOto8ODBUo3r169vtf/AgQOl/keOHGmHCuWRfcEmKcPSIw6Mt/xp7TsAePjhhx1aU1u54pxscefOnRaPpLj3URUdiUajgYeHh7PLcIrOso7+97//Se9jY2Nb7X9vny1btihSkz0YNE72xz/+0ez38+bNw7///W/cuHEDDQ0NOHPmDF544QU0NDS06KdSqTBr1qz2KFU2V5yTLT799FPpruHBwcGy7/1H7aezrKN770xv7blb5vqcOHFCkZrs4uxNKhJi3Lhxdh0w79Kli/jss8+cXb5ZrjgnS+7evSuqqqrE/v37xbRp01rMZ82aNc4uz2bm1ktn33XWWddRSEiIVOfevXtb7X/vMc+AgIB2qFAeBk0HUFNTY/GguLlXt27dxOuvvy5+/vlnZ5dukSvO6V5z5861Oh9vb2/x5ZdfOrtMu7hK0LjCOnrmmWekemfOnGm1740bN1qcoQbA6skD7Ym7zjoAPz8/ZGdnY8eOHbL2w1ZXV+Ps2bMoKSlph+rs44pzkuuFF15ASUkJpk6d6uxSyILOso5eeOEF6X1mZiZyc3PN9jMYDHj99ddNHldfU1OjaH1yqYT4/1cBkVPt2bMHixcvxtGjR20al5aWhuXLlytUVdu44pyMDh8+jLy8PADNB5avX7+Oo0ePSt+Fh4djxYoV+NOf/uTMMu1i7gSNznILmnu5wjq6e/cuhg4dKh1v8fb2xty5czF58mSEh4dDr9fjv//9L9LT03Hw4EF4eXm1OOZ5+fJl9OrVy1nl/8rZm1QkxP/93/+ZbNY/8MADYtWqVaK4uFjcuHFDHDp0SAwfPtzsLoBPP/3U2VMw4YpzkiMvL6/FbVA64zzMrY/OuOvMks62ji5fviwefvjhVnc/+/j4iLVr17b4rrq62tnlCyF4jMbp9uzZY/Y/mh07dpj0raurE+Hh4SZ9g4KCRGNjoxOqN88V52SLixcvSndG8PDw6DTHnYxcPWiE6HzrqKamRqSmpgo/Pz+L6+fcuXPi2LFj0ndubm4d5qJNHqNxsg8//NDku5CQEDz11FMm3/v4+GDy5Mkm31dUVECr1SpSnz1ccU62CAsLw5///GcAzbtsNmzY4NyCyERnW0d+fn7IyMhARUUFDh06hHXr1mHlypXYtGkTfv75Zxw8eBCRkZEtrrvp06ePxevU2htvqulkx44dM/nuwQcftNjfUtulS5cwfPhwh9XVFq44J1s98sgj0ntzT44l5+uM68jT0xMjR47EyJEjzbbn5+dL7x999NF2qqp13KJxsrq6OpPvLl++bLG/pTZPT0+H1dRWrjgnWzU1NUnv796968RKyBJXXEdbt26V3j/55JNOrKQlBo2TBQcHm3xXVlaGXbt2mXyv1+vx9ddfm12OtS2G9uaKc7LV8ePHpfcd4qwfMuFq62jHjh346aefAADdu3fvUGfTMWiczNLtWqZMmYLMzEyUlJTg1q1b+OGHH5CYmIji4mKTvgEBAYiJiVG6VNlccU62KCwsxLfffit9HjNmjBOrsUwIgaamJpNXW/t2Bp1lHcl17do1zJw5U/q8cOFCdOnSxYkV3cfZZyP81v3nP/+RffW8pdfSpUudPY0WXHFOf/vb38Q777wjLl++bLVfdnZ2i6eK9uvXr8OePXfgwIE2r6fi4mJnT0PiiutIiOZ6Dx48aPEMsmPHjonIyEhpPvHx8R3mbDMjngzgZI899hgWLFiA9PR0u8Y/8cQTSE1NdXBVbeOKc6qqqsLKlSuxZMkSDBo0CNHR0ejVqxc0Gg30ej0uXryIw4cP48KFC9KYwMBAfPXVVx3rX5YuzFXX0dGjR7F06VL07NkTI0eOxEMPPQRvb29cv34dubm50Gq10tM3hw4diqysrA5ztpnE2UlHzdasWdPiCXmtvdzd3UVqaqrQ6/XOLt0iV5rT7NmzbfqX/ujRozv8tRmutkXjiutIiJYPP7P0cnNzE6+99prQ6XTOLtcs3oKmA6mrq0N2djZ27tyJH3/8EZcuXYJOp0NTUxM0Gg0CAwMxcOBAjBw5Ei+88AJCQkKcXXKrXGVOTU1NOHLkCA4ePIjjx4/jp59+QllZGerq6uDl5YVu3bqhb9++GDJkCCZNmiTr/m7OdvDgQcTHx7dpGcXFxQgPD3dMQW3kiusIAEpLS5GTk4MDBw6goKAA165dw61btxAQEIDf//73SExMxOTJk9GvXz9nl2oRg4aIiBTFs86IiEhRDBoiIlIUg4aIiBTFoCEiIkUxaIiISFEMGiIiUhSDhoiIFMWgISIiRTFoiIhIUQwaIiJSFIOGiIgUxaAhIiJF/T8lThapOiwWYQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_attribution(\n",
    "    query_ids[0],\n",
    "    input_ids[0,:mask.sum()],\n",
    "    attributions_ig[0,:mask.sum()],\n",
    "    attributions_gt[0,:mask.sum()],\n",
    "    ymax=1.1\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Schulz et al. (2020)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "70f2cf2991cf43219c71c6666f365c41",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=0.0, max=71.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "all_q_z_loc, all_q_z_scale = toy_hidden_states_statistics(model_orig)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1db3b3712cb742a69aa32de4c32987cd",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=0.0, max=500.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "attributions_schulz = schulz_explainer(\n",
    "    model_orig,\n",
    "    {\"query_ids\": query_ids, \"input_ids\": input_ids, \"labels\": labels, \"mask\": mask},\n",
    "    getter=toy_getter,\n",
    "    setter=toy_setter,\n",
    "    q_z_loc=all_q_z_loc[hidden_state_idx].to(device).unsqueeze(0).detach(),\n",
    "    q_z_scale=all_q_z_scale[hidden_state_idx].to(device).unsqueeze(0).detach(),\n",
    "    loss_fn=lambda outputs, hidden_states, inputs_dict: torch.nn.functional.binary_cross_entropy_with_logits(\n",
    "        outputs, inputs_dict[\"labels\"].float(), reduction=\"none\"\n",
    "    ).mean(-1),\n",
    "    loss_kl_fn=lambda kl, inputs_dict: ((\n",
    "        kl * inputs_dict[\"mask\"]).sum(-1) / inputs_dict[\"mask\"].sum(-1)).mean(-1),\n",
    "    hidden_state_idx=hidden_state_idx,\n",
    "    steps=500,\n",
    "    lr=1e-1,\n",
    "    la=10,\n",
    ").cpu()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZoAAAENCAYAAAA2ZaOYAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAAmp0lEQVR4nO3de1TUdf4/8OcMILcBbQdQxBbQU4JYgrBpurkHMam8ZJtS4p6sP6Sbm+sh8rpp21csw26nk4lltnu0ix5aFSzvuoXKOhIaKFYrmiLERYUBBhTm/fuD33wS58JnLh8GpufjnDlnZt7v92de7/Mpnn7uKiGEABERkULU7i6AiIg8G4OGiIgUxaAhIiJFMWiIiEhRDBoiIlKUt7sL6G1CQkIQFRXl7jKIiPqU8+fPo66uzmIbg+YWUVFR0Ol07i6DiKhPSUpKstrGXWdERKQoBg0RESmKQUNERIpi0BARkaIYNEREpCgGDRERKYpBQ0REimLQEBGRohg0RESkKAYNEREpSnbQ6PV67NixA3//+9/x4IMPIiQkBCqVCiqVCuXl5U4X0tjYiOXLlyM2NhYBAQHQarVISUnBtm3bZI3funUrJk6cCK1Wi4CAAMTGxmL58uXQ6/VO10ZERE4QMn355ZcCgMXXmTNn5C7GoosXL4ro6GhpeRqNRnh7e0ufn332WZvj582bJ/X19vYWGo1G+jx06FBRWVkpu5bExESn5kJE9Ftk62+nXbvOwsLC8NBDD2HFihXIzc11VdBh5syZqKioQFRUFAoLC6HX66HX67FmzRqo1WqsW7cOGzZssDje1KZWq/HGG2+gqakJer0ehYWFiIyMxLlz55CWluaSWomIyAFy06q9vb3L54qKCpds0Zi2lNRqtfjuu+/M2v/2t78JAGLQoEGira2tS1tra6sICwsTAMTChQvNxhYXFwuVSiUAiB07dsiqh1s0RET2c8kWjZeXlyJBt3nzZgDApEmTEB8fb9b+4osvQqVSobq6GgcOHOjStm/fPtTU1EClUiEzM9NsbEJCAiZNmtTld4iIqGe5/ayzgwcPAgBSU1MttkdERCAuLg4AzILGNHbkyJGIiIiwON603FvHEhFRz3Br0NTU1KC+vh4ApDCxZMSIEQCA06dPd/ne9FnO2NraWqtPfyMiIuW4NWiqqqqk94MHD7baz9R2c/+bP8sZa2k8EREpz62Pcm5ubpbe+/v7W+0XEBAAAGhqarI4Xs5YS+NNcnNzpbPoamtru6maiIjs4fZjNL1BRkYGdDoddDodQkND3V0OEZFHcWvQBAYGSu8NBoPVfi0tLQAAjUZjcbycsZbGExGR8twaNDcfP7l8+bLVfqa28PBwi+PljLU0noiIlOfWoAkNDUVISAgAoKyszGo/09llpjPITEyf5Yy9+beIiKjnuP0YTXJyMgBg7969FtsrKyulIElJSbE4tqyszOoZZXv27LE4loiIeobbgyY9PR1AZyCcPHnSrP3NN9+EEALh4eFSsJikpKQgLCwMRqMRa9euNRt78uRJ7Nu3DwAwZ84cBaonIqLu2BU0dXV10uvq1avS99euXevSZjQau4wzPU5g5cqVZst8+OGHMWbMGBiNRjzyyCM4duwYAKCtrQ1r167F22+/DQB45ZVX0K9fvy5jfX19pWW+9dZbWLt2Ldra2gAAR48exSOPPAKj0Yjx48dj6tSp9kyViIhcxZ6bpsHKYwJufVVUVFgct2LFCovL7e4xAc8884zNum5+TICPjw8fE0BE1MNc9pgApQwZMgQlJSVYunQpYmJi0N7ejqCgICQnJ+OLL77AunXrbI7Pzc3F559/juTkZGg0GrS3tyMmJgbLli1DSUmJzTsHEBGRslRCCOHuInqTpKQk6HQ6d5dBRNSn2Prb2Su2aIiIyHMxaIiISFEMGiIiUhSDhoiIFMWgISIiRTFoiIhIUQwaIiJSFIOGiIgUxaAhIiJFMWiIiEhRDBoiIlIUg4aIiBTFoCEiIkUxaIiISFEMGiIiUhSDhoiIFMWgISIiRTFoiIhIUQwaIiJSFIOGiIgUxaAhIiJFMWiIiEhRDBoiIlIUg4aIiBTFoCEiIkUxaIiISFEMGiIiUhSDhoiIFMWgISIiRTFoiIhIUQwaIiJSlN1BU11djQULFmDYsGHw8/PDwIEDMW3aNOzfv9+hAqKioqBSqWS9PvnkE7PxcsZt27bNodqIiMh53vZ0PnXqFCZOnIj6+noAQHBwMOrq6pCfn4+CggJkZ2dj8eLFdhUQGhqK1tZWq+3Nzc1oamoCAIwePdpqv5CQEHh5eVls8/Pzs6smIiJyISFTS0uLiIyMFABEQkKCKC0tFUII0dDQIDIzMwUAoVKpxO7du+UuUpbp06cLAGL06NEW2wEIAKKiosIlv5eYmOiS5RAR/ZbY+tspe9fZ+vXrceHCBWg0GuzcuRNxcXEAOrdqcnJyMGPGDAghsGTJEpeFYG1tLb766isAwNy5c122XCIi6jmyg2bz5s0AgPT0dERERJi1Z2VlAQCKi4tx9uxZlxS3ZcsW3LhxAz4+PkhPT3fJMomIqGfJChq9Xo8TJ04AAFJTUy32GTt2LPr37w8ADp8YcCvTwf8pU6YgJCTEJcskIqKeJStozpw5AyEEAEi7zMwWpFZj+PDhAIDTp087Xdj333+P7777DoC83WZpaWm47bbb4OvriyFDhuDRRx9FQUGB03UQEZFzZAVNVVWV9H7w4MFW+5nabu7vqE2bNgHoPJtsypQp3fY/fvw4Ojo64OPjg8rKSuTl5WHq1KlIS0vD9evXna6HiIgcIytompubpff+/v5W+wUEBACAdDqyo9rb27scE/Lx8bHad+7cufj6669x9epVNDY2oqmpCWfOnMFTTz0FANi6dSvmz59v8/dyc3ORlJSEpKQk1NbWOlU7ERF11SvvDLB792788ssvALrfbbZp0yakpqZiwIAB0ncxMTHYuHGjdILChx9+aPMEhYyMDOh0Ouh0OoSGhjo/ASIiksgKmsDAQOm9wWCw2q+lpQUAoNFonCrKdBLAXXfdZfMize6sWLEC/v7+EEIgPz/fqZqIiMgxsoLm5uMyly9fttrP1BYeHu5wQVevXsWOHTsAOH/tTGBgIEaOHAkAOHfunFPLIiIix8gKmpiYGKhUKgBAWVmZxT5Go1HaPTVixAiHC/rss8/Q1tYGb29vzJkzx+HlEBFR7yAraIKCgpCUlAQA2Lt3r8U+RUVFaGhoAACkpKQ4XJBpt1lqaioGDRrk8HKAzpMYSktLAQDR0dFOLYuIiBwj+2QA05X5mzdvtnj6ck5ODgAgMTFRup7GXmfPnkVRUREAebvNTNf2WPPqq6/CYDBApVLhoYcecqgmIiJyjuygefrppxEZGQm9Xo+pU6dKF2Xq9Xq89NJLyMvLAwBkZ2ebjTXdrn/lypU2f8O0NXPbbbdh+vTp3daUlpaGZcuWQafTdblW5uzZs5g3bx5ef/11AJ2h5czuPCIicpzsxwT4+/tj+/btSElJQXFxMeLi4hAcHIympiYYjUaoVCpkZ2dj8uTJDhViNBrxr3/9CwDw+OOPw9fXt9sxtbW12LZtG7Kzs+Hl5YX+/fujra2ty3U/M2fOxAcffOBQTURE5Dy7nkczatQolJaWYvXq1cjPz0dlZSW0Wi3uueceLFy40KljMwcOHMClS5cAyD/bbOnSpbj77rtx7NgxXLp0CVeuXIFarUZ0dDTGjh2LJ5980uHgIyIi11CJ7g50/MYkJSVBp9O5uwwioj7F1t/OXnlnACIi8hwMGiIiUhSDhoiIFMWgISIiRTFoiIhIUQwaIiJSFIOGiIgUxaAhIiJFMWiIiEhRDBoiIlIUg4aIiBTFoCEiIkUxaIiISFEMGiIiUhSDhoiIFMWgISIiRTFoiIhIUQwaIiJSFIOGiIgUxaAhIiJFMWiIiEhRDBoiIlIUg4aIiBTFoCEiIkUxaIiISFEMGiIiUhSDhoiIFMWgISIiRTFoiIhIUQwaIiJSlN1BU11djQULFmDYsGHw8/PDwIEDMW3aNOzfv9+hAg4dOgSVStXtq66uzuZytm7diokTJ0Kr1SIgIACxsbFYvnw59Hq9Q3UREZFreNvT+dSpU5g4cSLq6+sBAMHBwairq0N+fj4KCgqQnZ2NxYsXO1SIWq1GaGiozXZrMjIysGHDBgCAt7c3/Pz8UF5ejlWrVuHTTz/FN998g8GDBztUFxEROUf2Fo3BYMD06dNRX1+PhIQElJaWoqGhAVevXkVmZiaEEFi6dCn27NnjUCG33347qqurrb5+97vfWRy3bt06bNiwAWq1Gm+88Qaampqg1+tRWFiIyMhInDt3DmlpaQ7VREREzpMdNOvXr8eFCxeg0Wiwc+dOxMXFAejcqsnJycGMGTMghMCSJUsUK/ZWbW1tWLlyJQBgwYIFePHFF+Hr6wsAGDduHL788kuoVCoUFhZi586dPVYXERH9SnbQbN68GQCQnp6OiIgIs/asrCwAQHFxMc6ePeui8mzbt28fampqoFKpkJmZadaekJCASZMmAfi1fiIi6lmygkav1+PEiRMAgNTUVIt9xo4di/79+wOAwycG2OvgwYMAgJEjR1oMP+DXeg8cONAjNRERUVeygubMmTMQQgCAtMvMbEFqNYYPHw4AOH36tN2F1NbWYvTo0QgMDERgYCDuvPNOZGRk4Pvvv7c6xvQ71moCgBEjRkjL7+7MNSIicj1ZQVNVVSW9t3X2lqnt5v5ytbS04LvvvoOvry/a29vx448/YsOGDUhISEBOTo7NuuTU5GhdRETkHFlB09zcLL339/e32i8gIAAA0NTUJLuAAQMGICsrCzqdDgaDAVeuXEFLSwsOHz6McePGoaOjA1lZWdiyZYvVuuTUZKuu3NxcJCUlISkpCbW1tbJrJyKi7rn9zgDx8fFYs2YNEhMT4efnBwDw8vLChAkTcPDgQYwfPx4AsGjRIhiNRkVqyMjIgE6ng06ns3ktDxER2U9W0AQGBkrvDQaD1X4tLS0AAI1G42RZnfr164dXX30VAHDp0iV89913FuuSU5Mr6yIiIvlkBc3NxzkuX75stZ+pLTw83MmyfjVmzBjp/blz5yzWJacmV9dFRETyyAqamJgYqFQqAEBZWZnFPkajUbp+xnSml9JMv2OtJuDXM9NCQ0MREhLSI3UREdGvZAVNUFAQkpKSAAB79+612KeoqAgNDQ0AgJSUFBeV17lck+jo6C5tycnJADqDxtoZZaZb4riyJiIikk/2yQDp6ekAOq+wt/RH3XQKcmJionQ9jRym63MsuXHjBl5++WUAnbu9Ro8e3aU9JSUFYWFhMBqNWLt2rdn4kydPYt++fQCAOXPmyK6JiIhcSMjU0tIiIiMjBQAxevRoUVZWJoQQorGxUWRlZQkAAoDYvXu32VhT24oVK8zaRowYId59913xww8/CKPRKIQQor29XXzzzTfivvvuk8Zu2rTJYl3vv/++ACDUarXIyckRra2tQgghjhw5IqKjowUAMX78eLnTFImJibL7EhFRJ1t/O2UHjRBClJSUCK1WK/3xDw4OFmq1WgAQKpVKrF692vKP2AgaUxsA4evrK0JCQkS/fv2k77y9vcVrr71ms6558+ZJ/X18fIRGo5E+Dx06VFRWVsqeI4OGiMh+tv522nUdzahRo1BaWooXXngBQ4cORVtbG7RaLaZMmYK9e/c69Cya9evX44knnkBcXByCg4Nx7do1+Pr64q677sL8+fNx8uRJLFq0yOYycnNz8fnnnyM5ORkajQbt7e2IiYnBsmXLUFJSwmfREBG5kUoIGwdJfoOSkpKg0+ncXQYRUZ9i62+n2+8MQEREno1BQ0REimLQEBGRohg0RESkKAYNEREpikFDRESKYtAQEZGiGDRERKQoBg0RESmKQUNERIpi0BARkaIYNEREpChvdxfgSaIWF7i7BFnOvzbF3SUQ0W8Ig4ZsYngSkbO464yIiBTFoCEiIkUxaIiISFEMGiIiUhSDhoiIFMWgISIiRTFoiIhIUQwaIiJSFIOGiIgUxaAhIiJFMWiIiEhRDBoiIlIUg4aIiBTFoCEiIkUxaIiISFEMGiIiUhSDhoiIFGV30FRXV2PBggUYNmwY/Pz8MHDgQEybNg379+93qIDa2lqsX78es2bNkpYZGBiI2NhYzJ8/Hz/99JPN8SqVqtvXtm3bHKqNiIicZ9ejnE+dOoWJEyeivr4eABAcHIy6ujrk5+ejoKAA2dnZWLx4sV0FDB48GO3t7dJnjUaD69evo7y8HOXl5fjoo4+wceNGzJ492+ZyQkJC4OXlZbHNz8/PrpqIiMh1ZG/RGAwGTJ8+HfX19UhISEBpaSkaGhpw9epVZGZmQgiBpUuXYs+ePXYV0N7ejgkTJuCTTz5BVVUV9Ho9Wlpa8O233yI+Ph6tra144okncOrUKZvLOX78OKqrqy2+pk6daldNRETkOrKDZv369bhw4QI0Gg127tyJuLg4AJ1bNTk5OZgxYwaEEFiyZIldBRw+fBiHDx/GE088gUGDBgEAvLy8MH78eOzZswdhYWFob2/HW2+9ZddyiYiod5AdNJs3bwYApKenIyIiwqw9KysLAFBcXIyzZ8/KLmDChAlW20JDQ/HQQw8BAE6cOCF7mURE1HvIChq9Xi/9oU9NTbXYZ+zYsejfvz8AOHxigCVarRYA0NHR4bJlEhFRz5EVNGfOnIEQAgCkXWZmC1KrMXz4cADA6dOnXVRe5641ABg5cqTNfmlpabjtttvg6+uLIUOG4NFHH0VBQYHL6iAiIsfICpqqqirp/eDBg632M7Xd3N8Z27dvh06nAwA89dRTNvseP34cHR0d8PHxQWVlJfLy8jB16lSkpaXh+vXrNsfm5uYiKSkJSUlJqK2tdUntRETUSVbQNDc3S+/9/f2t9gsICAAANDU1OVkWUFlZiYyMDADA9OnT8cADD1jsN3fuXHz99de4evUqGhsb0dTUhDNnzkjBtHXrVsyfP9/mb2VkZECn00Gn0yE0NNTp2omI6Fe98s4ATU1NmDFjBmpqahAZGYmPPvrIat9NmzYhNTUVAwYMkL6LiYnBxo0bpRMUPvzwQ7tOUCAiIteRFTSBgYHSe4PBYLVfS0sLgM6LLh3V2tqKhx9+WNq62L17N0JCQhxa1ooVK+Dv7w8hBPLz8x2uiYiIHCcraG4+LnP58mWr/Uxt4eHhDhVz/fp1zJw5EwcOHMCAAQOwZ88e6QQDRwQGBkonEZw7d87h5RARkeNk3YImJiYGKpUKQgiUlZVZ/ONvNBql3VMjRoywu5D29nbMnj0bBQUF0Gg02LVrF+Lj4+1eDhFRbxO1uG+cAXv+tSmKLFfWFk1QUBCSkpIAAHv37rXYp6ioCA0NDQCAlJQUu4owGo2YO3cu8vLy4O/vjx07duDee++1axmWNDc3o7S0FAAQHR3t9PKIiMh+sk8GSE9PB9B5hwBLpy/n5OQAABITE+3a3SWEQEZGBrZs2YJ+/fohLy8PycnJssfa8uqrr8JgMEClUkl3GCAiop4lO2iefvppREZGQq/XY+rUqdJFmXq9Hi+99BLy8vIAANnZ2WZjTbfrX7lypVnbwoUL8dFHH8Hb2xtffPGF1dOYLUlLS8OyZcug0+m6XCtz9uxZzJs3D6+//jqAzlOgHdmdR0REzpP9mAB/f39s374dKSkpKC4uRlxcHIKDg9HU1ASj0QiVSoXs7GxMnjxZ9o///PPPeOeddwB0htHTTz+Np59+2mr/6urqLp9ra2uxbds2ZGdnw8vLC/3790dbW1uX635mzpyJDz74QHZNRETkWnY9j2bUqFEoLS3F6tWrkZ+fj8rKSmi1Wtxzzz1YuHChQ8dmTG7cuIFffvnFrvFLly7F3XffjWPHjuHSpUu4cuUK1Go1oqOjMXbsWDz55JN2BR8REbmeXUEDAIMGDcI777wjbYnIYe1YSlRUVLfHWWyZPHkyg4SIqJfrlXcGICIiz8GgISIiRTFoiIhIUQwaIiJSFIOGiIgUxaAhIiJFMWiIiEhRDBoiIlIUg4aIiBTFoCEiIkUxaIiISFEMGiIiUhSDhoiIFMWgISIiRTFoiIhIUQwaIiJSFIOGiIgUxaAhIiJFMWiIiEhRDBoiIlIUg4aIiBTFoCEiIkUxaIiISFEMGiIiUhSDhoiIFMWgISIiRTFoiIhIUQwaIiJSFIOGiIgUxaAhIiJF2R001dXVWLBgAYYNGwY/Pz8MHDgQ06ZNw/79+50qpLGxEcuXL0dsbCwCAgKg1WqRkpKCbdu2yRq/detWTJw4EVqtFgEBAYiNjcXy5cuh1+udqouIiJxjV9CcOnUKI0eOxLvvvotz587B19cXdXV1yM/Px/3334/XXnvNoSIuXbqE+Ph4rFq1CuXl5fDy8kJjYyMOHDiAWbNm4bnnnrM5PiMjA2lpaTh48CAaGxvh5eWF8vJyrFq1CvHx8bh8+bJDdRERkfNkB43BYMD06dNRX1+PhIQElJaWoqGhAVevXkVmZiaEEFi6dCn27NljVwFCCMycORMVFRWIiopCYWEh9Ho99Ho91qxZA7VajXXr1mHDhg0Wx5va1Go13njjDTQ1NUGv16OwsBCRkZE4d+4c0tLS7KqJiIhcR3bQrF+/HhcuXIBGo8HOnTsRFxcHAAgODkZOTg5mzJgBIQSWLFliVwHbt29HUVER1Go1vvzyS4wbNw4A4Ofnh6ysLLzwwgsAgJdffhnXr1/vMratrQ0rV64EACxYsAAvvvgifH19AQDjxo3Dl19+CZVKhcLCQuzcudOuuoiIyDVkB83mzZsBAOnp6YiIiDBrz8rKAgAUFxfj7NmzsgswLXfSpEmIj483a3/xxRehUqlQXV2NAwcOdGnbt28fampqoFKpkJmZaTY2ISEBkyZN6vI7RETUs2QFjV6vx4kTJwAAqampFvuMHTsW/fv3BwC7Tgw4ePCgzeVGRERIW0+3Bo1p7MiRIy2G383LvXUsERH1DFlBc+bMGQghAED6o2+2ILUaw4cPBwCcPn1a1o/X1NSgvr7e5nIBYMSIERaXa/osZ2xtbS3q6upk1UVERK7jLadTVVWV9H7w4MFW+5nabu6v5HJNn+WMNfUPCQmRVRt5pqjFBe4uQZbzr01xdwluw3XkeWQFTXNzs/Te39/far+AgAAAQFNTk6wfd3a5pvFyxtqqKzc3F7m5uQCA8vJyJCUldVO5ZUpEWG1tLUJDQ126zKSkFbL7etqcPG0+SlFiTnJ54jryxDnd6vz581bbZAWNp8vIyEBGRoa7y7AoKSkJOp3O3WW4lKfNydPmA3jenDxtPkDfmpOsYzSBgYHSe4PBYLVfS0sLAECj0cj6cWeXaxovZ6w9dRERkevICpqbj3PYusre1BYeHi7rx51drmm8nLH21EVERK4jK2hiYmKgUqkAAGVlZRb7GI1G6foZ05le3QkNDZUOzltbLvDr2WW3Ltf0Wc7Ym3+rL+mtu/Sc4Wlz8rT5AJ43J0+bD9DH5iRk+sMf/iAAiGeeecZi+5EjRwQAAUCUl5fLXayYNWuWACAeeOABi+2XLl0SKpVKABBff/11l7aCggIBQKjVanH58mWL4ydPniwAiMcff1x2TURE5Dqy7wyQnp4OoPMKe0unL+fk5AAAEhMTpetp7Fnunj17cPLkSbP2N998E0IIhIeHIzk5uUtbSkoKwsLCYDQasXbtWrOxJ0+exL59+wAAc+bMkV0TERG5kNxEamlpEZGRkQKAGD16tCgrKxNCCNHY2CiysrKkrZndu3ebjTW1rVixwqzNaDSKMWPGCAAiOjpaHD16VAghRGtrq8jJyRFqtVoAELm5uRbrev/996WtmpycHNHa2iqE6NzCio6OFgDE+PHj5U6TiIhcTHbQCCFESUmJ0Gq1UnAEBwdLQaBSqcTq1ast/4iNoBFCiIsXL0qhAEBoNBrh7e0tfba2u85k3rx5Ul8fHx+h0Wikz0OHDhWVlZX2TJOIiFzIrutoRo0ahdLSUqxevRr5+fmorKyEVqvFPffcg4ULFyIlJcWhraohQ4agpKQEr7/+OvLy8nD+/HkEBQUhPj4ezz77LGbNmmVzfG5uLiZNmoQPPvgAJSUlMBgMiImJwaOPPopFixYhKCjIobp6WktLC3bu3ImvvvoKpaWl+Pnnn6HX63Hjxg0EBgZCq9XijjvuwL333osZM2ZYvAlpb+NJc/rpp59QVFQEnU6HkpISVFVVoa6uDg0NDQgMDMTgwYORmJiImTNnYurUqfDy8nJ3yd2qr6+HTqfD8ePHpZelXeN/+tOfcOjQoZ4v0E6euI5MmpqasHHjRuzatQunTp1CfX09NBoNIiIiMHnyZDz55JMYOXKku8u0zN1JR50+/fRTMWjQIGlLTM7rwQcfFFVVVe4u3SpPm9OwYcNkz2PUqFHi1KlT7i7Zpi1btsiez5/+9Cd3lyuLp60jkx07dnTZm2Tp5e3tLZYuXSqMRqO7yzXDOwP0Ah9++CHmzZtn97ivvvoK48ePx4kTJzBgwADXF+YET5yTSUREBEaPHo2hQ4dCq9XCx8cH9fX1KC4uxuHDh9HR0YGTJ09iwoQJOHLkCGJjY91dskVtbW3uLkExnrKOAGDbtm1IS0uTbmwcGhqK6dOnIzo6GgaDAUVFRdi/fz/a29uRnZ2N5uZmvP322+4t+lbuTrrfumvXronAwEC7/tV/62v58uXunkYXnjgnIYTIy8sTFy5csNnn7NmzIi4uTppHcnJyD1Vnv48//tjjtmg8bR1VV1d3Oeacnp4umpqazPoVFhaK0NBQqV9+fr4bqrWOQeNm1nZfhIWFic8++0xUV1cLvV4vTpw4IaZMmWKx75gxY9w9jS48cU72KC0tleahUqlEdXW1u0uy6OOPPxZ+fn5izJgx4q9//av45z//KcrLy/t00MjVV9bRsmXLpDoTEhLEjRs3rPbdvXu31DcmJqZX7UJj0LjZ6tWrLf6PvWnTJrO+BoNB+Pn5mfUdPny4Gyq3zhPnZK+wsDBpLseOHXN3ORbp9Xpx/fp1s+9/C0EjRN9YR6NHj5Zq3LhxY7f977rrLqn/kSNHeqBCeWRfsEnKsPaIA9Mtf7r7DgDuvPNOl9bkLE+ckz1u3LjR5ZEUNz+qojfRaDTw8fFxdxlu0VfW0f/+9z/pfWJiYrf9b+6zbds2RWpyBIPGzf74xz9a/H7x4sX497//jStXrqC1tRWnTp3CY489htbW1i79VCoVFixY0BOlyuaJc7LHe++9J901PCwsTPa9/6jn9JV1dPOd6W09d8tSnxMnTihSk0PcvUlFQkydOtWhA+b9+vUT77//vrvLt8gT52RNR0eHqK+vFwcOHBBz587tMp8NGza4uzy7WVovfX3XWV9dR+Hh4VKd+/bt67b/zcc8tVptD1QoD4OmF2hsbLR6UNzSa8CAAeL5558XP/74o7tLt8oT53SzRYsW2ZyPv7+/+OSTT9xdpkM8JWg8YR1NmzZNqnf+/Pk2+165cqXLGWoAbJ480JO466wXCAoKQn5+Pnbt2iVrP2xDQwNOnz6NCxcu9EB1jvHEOcn12GOP4cKFC3jiiSfcXQpZ0VfW0WOPPSa9z83NRWFhocV+RqMRzz//vNnj6hsbGxWtTy6VEP//KiByq71792LFihU4evSoXeOysrKwZs0ahapyjifOyeSbb75BUVERgM4DyzU1NTh69Kj0XVRUFNauXYs///nP7izTIZZO0Ogrt6C5mSeso46ODowZM0Y63uLv749FixZh9uzZiIqKgsFgwH//+19kZ2fj0KFD8PPz63LM8+LFixgyZIi7yv+VuzepSIj/+7//M9usv+2228S6detERUWFuHLlijh8+LAYN26cxV0A7733nrunYMYT5yRHUVFRl9ug9MV5WFoffXHXmTV9bR1dvHhR3Hnnnd3ufg4ICBAffvhhl+8aGhrcXb4Qgsdo3G7v3r0W/6PZtWuXWd/m5mYRFRVl1jc0NFS0tbW5oXrLPHFO9jh//rx0ZwQfH58+c9zJxNODRoi+t44aGxtFZmamCAoKsrp+zpw5I44dOyZ95+Xl1Wsu2uQxGjd78803zb4LDw/Hgw8+aPZ9QEAAZs+ebfZ9bW0tdDqdIvU5whPnZI/IyEj85S9/AdC5y2bTpk3uLYjM9LV1FBQUhJycHNTW1uLw4cP46KOP8M4772DLli348ccfcejQIcTExHS57mbYsGFWr1PrabypppsdO3bM7Lvbb7/dan9rbT///DPGjRvnsrqc4Ylzstfdd98tvbf05Fhyv764jnx9fTFhwgRMmDDBYntJSYn0/t577+2hqrrHLRo3a25uNvvu4sWLVvtba/P19XVZTc7yxDnZq729XXrf0dHhxkrIGk9cR9u3b5feP/DAA26spCsGjZuFhYWZfVdVVYXdu3ebfW8wGPDZZ59ZXI6tLYae5olzstfx48el973irB8y42nraNeuXfjhhx8AAAMHDuxVZ9MxaNzM2u1a5syZg9zcXFy4cAHXrl3Dt99+i9TUVFRUVJj11Wq1SEhIULpU2TxxTvYoKyvDF198IX2ePHmyG6uxTgiB9vZ2s5ezffuCvrKO5Prll18wf/586fOyZcvQr18/N1Z0C3efjfBb95///Ef21fPWXqtWrXL3NLrwxDn94x//EK+88oq4ePGizX75+fldnioaGxvba8+eO3jwoNPrqaKiwt3TkHjiOhKis95Dhw5ZPYPs2LFjIiYmRppPcnJyrznbzIQnA7jZfffdh6VLlyI7O9uh8ffffz8yMzNdXJVzPHFO9fX1eOedd7By5UqMGjUK8fHxGDJkCDQaDQwGA86fP49vvvkG586dk8aEhITg008/7V3/svRgnrqOjh49ilWrVmHw4MGYMGEC7rjjDvj7+6OmpgaFhYXQ6XTS0zfHjBmDvLy8XnO2mcTdSUedNmzY0OUJed29vL29RWZmpjAYDO4u3SpPmtPChQvt+pf+pEmTev21GZ62ReOJ60iIrg8/s/by8vISzz33nNDr9e4u1yLegqYXaW5uRn5+Pr7++mt8//33+Pnnn6HX69He3g6NRoOQkBDcddddmDBhAh577DGEh4e7u+Ruecqc2tvbceTIERw6dAjHjx/HDz/8gKqqKjQ3N8PPzw8DBgzA8OHDcc8992DWrFmy7u/mbocOHUJycrJTy6ioqEBUVJRrCnKSJ64jAKisrERBQQEOHjyI0tJS/PLLL7h27Rq0Wi1+//vfIzU1FbNnz0ZsbKy7S7WKQUNERIriWWdERKQoBg0RESmKQUNERIpi0BARkaIYNEREpCgGDRERKYpBQ0REimLQEBGRohg0RESkKAYNEREpikFDRESKYtAQEZGi/h/WpwwXMxYq4AAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_attribution(\n",
    "    query_ids[0],\n",
    "    input_ids[0,:mask.sum()],\n",
    "    attributions_schulz[0,:mask.sum()],\n",
    "    attributions_gt[0,:mask.sum()],\n",
    "    ymax=1.1\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Guan et al. (2019)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "369e6bdd18fb461490ea5ec67c38c83f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=0.0, max=500.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "attributions_guan = 1 / guan_explainer(\n",
    "    model_orig,\n",
    "    {\"query_ids\": query_ids, \"input_ids\": input_ids, \"labels\": labels, \"mask\": mask},\n",
    "    getter=toy_getter,\n",
    "    setter=toy_setter,\n",
    "    s_fn=lambda outputs, hidden_states: outputs,\n",
    "    loss_l2_fn=lambda s, inputs_dict: sum(s_i.sum(-1).mean(-1) for s_i in s),\n",
    "    loss_h_fn=lambda h, inputs_dict: (h * inputs_dict[\"mask\"]).sum(-1).mean(-1),\n",
    "    hidden_state_idx=hidden_state_idx,\n",
    "    steps=500,\n",
    "    lr=1e-1,\n",
    "    la=1,\n",
    ").cpu()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZoAAAENCAYAAAA2ZaOYAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAAmw0lEQVR4nO3deVTU9f4/8OewyDa4XBZF7AvoKUE02W6a3uwgJpVLdlNMvEfrD7HF8nqQNPWGLWIWVnY7mbhkt6MtevCqYO7LNReu4xooVlc0RVBAhQEGFOf9+4PffBJn4TPLh4Hp+ThnzpmZ9/v9mdf7fIqnn10lhBAgIiJSiJuzCyAiItfGoCEiIkUxaIiISFEMGiIiUhSDhoiIFOXh7ALam8DAQISHhzu7DCKiDuXixYuorKw02caguU94eDg0Go2zyyAi6lASEhLMtnHXGRERKYpBQ0REimLQEBGRohg0RESkKAYNEREpikFDRESKYtAQEZGiGDRERKQoBg0RESmKQUNERIqSHTRarRZbtmzBP/7xDzz11FMIDAyESqWCSqVCcXGx3YXU1NRgwYIFiIqKgq+vLwICApCUlISNGzfKGr9hwwYMHz4cAQEB8PX1RVRUFBYsWACtVmt3bUREZAch06ZNmwQAk69z587JXYxJly9fFhEREdLy1Gq18PDwkD6//PLLFsdPmzZN6uvh4SHUarX0uXfv3qK0tFR2LfHx8XbNhYjoj8jS306rdp0FBwfj6aefRmZmJnJychwVdBg/fjxKSkoQHh6OQ4cOQavVQqvV4oMPPoCbmxuWL1+OlStXmhxvaHNzc8OHH36I2tpaaLVaHDp0CGFhYbhw4QJSUlIcUisREdlAblo1NTW1+FxSUuKQLRrDlpKbm5s4efKkUfvf//53AUD06NFDNDY2tmhraGgQwcHBAoCYNWuW0dgTJ04IlUolAIgtW7bIqodbNERE1nPIFo27u7siQbdu3ToAwIgRIxATE2PUPnv2bKhUKpSXl2Pv3r0t2nbv3o3r169DpVIhPT3daGxsbCxGjBjR4neIiKhtOf2ss3379gEAkpOTTbaHhoYiOjoaAIyCxjC2f//+CA0NNTnesNz7xxIRUdtwatBcv34dVVVVACCFiSn9+vUDAJw9e7bF94bPcsZWVFSYffobEREpx6lBU1ZWJr3v2bOn2X6Gtnv73/tZzlhT44mISHlOfZRzXV2d9N7Hx8dsP19fXwBAbW2tyfFyxpoab5CTkyOdRVdRUdFK1UREZA2nH6NpD9LS0qDRaKDRaBAUFOTscoiIXIpTg8bPz096r9PpzParr68HAKjVapPj5Yw1NZ6IiJTn1KC59/jJ1atXzfYztIWEhJgcL2esqfFERKQ8pwZNUFAQAgMDAQBFRUVm+xnOLjOcQWZg+Cxn7L2/RUREbcfpx2gSExMBALt27TLZXlpaKgVJUlKSybFFRUVmzyjbuXOnybFERNQ2nB40qampAJoD4fTp00btH330EYQQCAkJkYLFICkpCcHBwdDr9Vi6dKnR2NOnT2P37t0AgMmTJytQPRERtcaqoKmsrJReN2/elL6/detWiza9Xt9inOFxAgsXLjRa5jPPPINBgwZBr9fj2WefxdGjRwEAjY2NWLp0KT755BMAwNtvv41OnTq1GOvl5SUt8+OPP8bSpUvR2NgIADhy5AieffZZ6PV6DB06FKNHj7ZmqkRE5CjW3DQNZh4TcP+rpKTE5LjMzEyTy23tMQEvvfSSxbrufUyAp6cnHxNARNTGHPaYAKX06tULp06dwrx58xAZGYmmpib4+/sjMTER33//PZYvX25xfE5ODr777jskJiZCrVajqakJkZGRmD9/Pk6dOmXxzgFERKQslRBCOLuI9iQhIQEajcbZZRARdSiW/na2iy0aIiJyXQwaIiJSFIOGiIgUxaAhIiJFMWiIiEhRDBoiIlIUg4aIiBTFoCEiIkUxaIiISFEMGiIiUhSDhoiIFMWgISIiRTFoiIhIUQwaIiJSFIOGiIgUxaAhIiJFMWiIiEhRDBoiIlIUg4aIiBTFoCEiIkUxaIiISFEMGiIiUhSDhoiIFMWgISIiRTFoiIhIUQwaIiJSFIOGiIgUxaAhIiJFMWiIiEhRDBoiIlIUg4aIiBRlddCUl5dj5syZ6NOnD7y9vdG9e3eMGTMGe/bssamA8PBwqFQqWa+vvvrKaLyccRs3brSpNiIisp+HNZ3PnDmD4cOHo6qqCgDQuXNnVFZWIi8vD/n5+cjKysLcuXOtKiAoKAgNDQ1m2+vq6lBbWwsAiIuLM9svMDAQ7u7uJtu8vb2tqomIiBxIyFRfXy/CwsIEABEbGysKCwuFEEJUV1eL9PR0AUCoVCqxY8cOuYuUZezYsQKAiIuLM9kOQAAQJSUlDvm9+Ph4hyyHiOiPxNLfTtm7zlasWIFLly5BrVZj69atiI6OBtC8VZOdnY1x48ZBCIE333zTYSFYUVGBH374AQAwdepUhy2XiIjajuygWbduHQAgNTUVoaGhRu0ZGRkAgBMnTuD8+fMOKW79+vW4c+cOPD09kZqa6pBlEhFR25IVNFqtFsePHwcAJCcnm+wzePBgdOnSBQBsPjHgfoaD/6NGjUJgYKBDlklERG1LVtCcO3cOQggAkHaZGS3IzQ19+/YFAJw9e9buwn766SecPHkSgLzdZikpKejWrRu8vLzQq1cvPPfcc8jPz7e7DiIiso+soCkrK5Pe9+zZ02w/Q9u9/W21du1aAM1nk40aNarV/seOHcPdu3fh6emJ0tJS5ObmYvTo0UhJScHt27ftroeIiGwjK2jq6uqk9z4+Pmb7+fr6AoB0OrKtmpqaWhwT8vT0NNt36tSp2L59O27evImamhrU1tbi3LlzePHFFwEAGzZswIwZMyz+Xk5ODhISEpCQkICKigq7aiciopba5Z0BduzYgWvXrgFofbfZ2rVrkZycjK5du0rfRUZGYs2aNdIJCqtWrbJ4gkJaWho0Gg00Gg2CgoLsnwAREUlkBY2fn5/0XqfTme1XX18PAFCr1XYVZTgJYMCAARYv0mxNZmYmfHx8IIRAXl6eXTUREZFtZAXNvcdlrl69arafoS0kJMTmgm7evIktW7YAsP/aGT8/P/Tv3x8AcOHCBbuWRUREtpEVNJGRkVCpVACAoqIik330er20e6pfv342F/Ttt9+isbERHh4emDx5ss3LISKi9kFW0Pj7+yMhIQEAsGvXLpN9CgoKUF1dDQBISkqyuSDDbrPk5GT06NHD5uUAzScxFBYWAgAiIiLsWhYREdlG9skAhivz161bZ/L05ezsbABAfHy8dD2Ntc6fP4+CggIA8nabGa7tMefdd9+FTqeDSqXC008/bVNNRERkH9lBM336dISFhUGr1WL06NHSRZlarRZvvPEGcnNzAQBZWVlGYw2361+4cKHF3zBszXTr1g1jx45ttaaUlBTMnz8fGo2mxbUy58+fx7Rp07BkyRIAzaFlz+48IiKynezHBPj4+GDz5s1ISkrCiRMnEB0djc6dO6O2thZ6vR4qlQpZWVkYOXKkTYXo9Xp8/fXXAIDnn38eXl5erY6pqKjAxo0bkZWVBXd3d3Tp0gWNjY0trvsZP348vvjiC5tqIiIi+1n1PJqBAweisLAQixcvRl5eHkpLSxEQEIBHHnkEs2bNsuvYzN69e3HlyhUA8s82mzdvHh5++GEcPXoUV65cwY0bN+Dm5oaIiAgMHjwYL7zwgs3BR0REjqESrR3o+INJSEiARqNxdhlERB2Kpb+d7fLOAERE5DoYNEREpCgGDRERKYpBQ0REimLQEBGRohg0RESkKAYNEREpikFDRESKYtAQEZGiGDRERKQoBg0RESmKQUNERIpi0BARkaIYNEREpCgGDRERKYpBQ0REimLQEBGRohg0RESkKAYNEREpikFDRESKYtAQEZGiGDRERKQoBg0RESmKQUNERIpi0BARkaIYNEREpCgGDRERKYpBQ0REimLQEBGRohg0RESkKKuDpry8HDNnzkSfPn3g7e2N7t27Y8yYMdizZ49NBezfvx8qlarVV2VlpcXlbNiwAcOHD0dAQAB8fX0RFRWFBQsWQKvV2lQXERE5hoc1nc+cOYPhw4ejqqoKANC5c2dUVlYiLy8P+fn5yMrKwty5c20qxM3NDUFBQRbbzUlLS8PKlSsBAB4eHvD29kZxcTEWLVqEb775BgcPHkTPnj1tqouIiOwje4tGp9Nh7NixqKqqQmxsLAoLC1FdXY2bN28iPT0dQgjMmzcPO3futKmQBx54AOXl5WZff/rTn0yOW758OVauXAk3Nzd8+OGHqK2thVarxaFDhxAWFoYLFy4gJSXFppqIiMh+soNmxYoVuHTpEtRqNbZu3Yro6GgAzVs12dnZGDduHIQQePPNNxUr9n6NjY1YuHAhAGDmzJmYPXs2vLy8AABDhgzBpk2boFKpcOjQIWzdurXN6iIiot/JDpp169YBAFJTUxEaGmrUnpGRAQA4ceIEzp8/76DyLNu9ezeuX78OlUqF9PR0o/bY2FiMGDECwO/1ExFR25IVNFqtFsePHwcAJCcnm+wzePBgdOnSBQBsPjHAWvv27QMA9O/f32T4Ab/Xu3fv3japiYiIWpIVNOfOnYMQAgCkXWZGC3JzQ9++fQEAZ8+etbqQiooKxMXFwc/PD35+fnjooYeQlpaGn376yewYw++YqwkA+vXrJy2/tTPXiIjI8WQFTVlZmfTe0tlbhrZ7+8tVX1+PkydPwsvLC01NTfjll1+wcuVKxMbGIjs722JdcmqytS4iIrKPrKCpq6uT3vv4+Jjt5+vrCwCora2VXUDXrl2RkZEBjUYDnU6HGzduoL6+HgcOHMCQIUNw9+5dZGRkYP369WbrklOTpbpycnKQkJCAhIQEVFRUyK6diIha5/Q7A8TExOCDDz5AfHw8vL29AQDu7u4YNmwY9u3bh6FDhwIA5syZA71er0gNaWlp0Gg00Gg0Fq/lISIi68kKGj8/P+m9Tqcz26++vh4AoFar7SyrWadOnfDuu+8CAK5cuYKTJ0+arEtOTY6si4iI5JMVNPce57h69arZfoa2kJAQO8v63aBBg6T3Fy5cMFmXnJocXRcREckjK2giIyOhUqkAAEVFRSb76PV66foZw5leSjP8jrmagN/PTAsKCkJgYGCb1EVERL+TFTT+/v5ISEgAAOzatctkn4KCAlRXVwMAkpKSHFRe83INIiIiWrQlJiYCaA4ac2eUGW6J48iaiIhIPtknA6SmpgJovsLe1B91wynI8fHx0vU0chiuzzHlzp07eOuttwA07/aKi4tr0Z6UlITg4GDo9XosXbrUaPzp06exe/duAMDkyZNl10RERA4kZKqvrxdhYWECgIiLixNFRUVCCCFqampERkaGACAAiB07dhiNNbRlZmYatfXr1098+umn4ueffxZ6vV4IIURTU5M4ePCgeOyxx6Sxa9euNVnX559/LgAINzc3kZ2dLRoaGoQQQhw+fFhEREQIAGLo0KFypyni4+Nl9yUiomaW/naqhLCwSXGf06dPIykpqcVjAmpra6HX66FSqcw+JsBwfCczM1O6Ceb9bQDg5eUFf39/1NTU4Pbt2wCab/v/3nvvYc6cOWbruvcxAZ6envDy8pKumendu7dVjwlISEiARqOR1fePIHxuvrNLkOXi+6OcXQLRH5qlv51WXUczcOBAFBYW4vXXX0fv3r3R2NiIgIAAjBo1Crt27bLpWTQrVqzAlClTEB0djc6dO+PWrVvw8vLCgAEDMGPGDJw+fdpiyADNF1x+9913SExMhFqtRlNTEyIjIzF//nycOnWKz6IhInIiq7Zo/gi4RdMSt2iISA6HbdEQERFZi0FDRESKYtAQEZGiGDRERKQoBg0RESmKQUNERIpi0BARkaIYNEREpCgGDRERKYpBQ0REimLQEBGRohg0RESkKAYNEREpikFDRESKYtAQEZGiGDRERKQoBg0RESmKQUNERIpi0BARkaIYNEREpCgGDRERKYpBQ0REimLQEBGRohg0RESkKAYNEREpikFDRESKYtAQEZGiGDRERKQoD2cX4ErC5+Y7uwRZLr4/ytklENEfCLdoiIhIUQwaIiJSlNVBU15ejpkzZ6JPnz7w9vZG9+7dMWbMGOzZs8emAioqKrBixQpMmDBBWqafnx+ioqIwY8YM/PrrrxbHq1SqVl8bN260qTYiIrKfVcdozpw5g+HDh6OqqgoA0LlzZ1RWViIvLw/5+fnIysrC3LlzrSqgZ8+eaGpqkj6r1Wrcvn0bxcXFKC4uxurVq7FmzRpMmjTJ4nICAwPh7u5uss3b29uqmoiIyHFkb9HodDqMHTsWVVVViI2NRWFhIaqrq3Hz5k2kp6dDCIF58+Zh586dVhXQ1NSEYcOG4auvvkJZWRm0Wi3q6+vx448/IiYmBg0NDZgyZQrOnDljcTnHjh1DeXm5ydfo0aOtqomIiBxHdtCsWLECly5dglqtxtatWxEdHQ2geasmOzsb48aNgxACb775plUFHDhwAAcOHMCUKVPQo0cPAIC7uzuGDh2KnTt3Ijg4GE1NTfj444+tWi4REbUPsoNm3bp1AIDU1FSEhoYatWdkZAAATpw4gfPnz8suYNiwYWbbgoKC8PTTTwMAjh8/LnuZRETUfsgKGq1WK/2hT05ONtln8ODB6NKlCwDYfGKAKQEBAQCAu3fvOmyZRETUdmQFzblz5yCEAABpl5nRgtzc0LdvXwDA2bNnHVRe8641AOjfv7/FfikpKejWrRu8vLzQq1cvPPfcc8jP7xgXUBIRuTJZQVNWVia979mzp9l+hrZ7+9tj8+bN0Gg0AIAXX3zRYt9jx47h7t278PT0RGlpKXJzczF69GikpKTg9u3bFsfm5OQgISEBCQkJqKiocEjtRETUTFbQ1NXVSe99fHzM9vP19QUA1NbW2lkWUFpairS0NADA2LFj8eSTT5rsN3XqVGzfvh03b95ETU0Namtrce7cOSmYNmzYgBkzZlj8rbS0NGg0Gmg0GgQFBdldOxER/a5d3hmgtrYW48aNw/Xr1xEWFobVq1eb7bt27VokJyeja9eu0neRkZFYs2aNdILCqlWrrDpBgYiIHEdW0Pj5+UnvdTqd2X719fUAmi+6tFVDQwOeeeYZaetix44dCAwMtGlZmZmZ8PHxgRACeXl5NtdERES2kxU09x6XuXr1qtl+hraQkBCbirl9+zbGjx+PvXv3omvXrti5c6d0goEt/Pz8pJMILly4YPNyiIjIdrKCJjIyEiqVCgBQVFRkso9er5d2T/Xr18/qQpqamjBp0iTk5+dDrVZj27ZtiImJsXo5RETUvsgKGn9/fyQkJAAAdu3aZbJPQUEBqqurAQBJSUlWFaHX6zF16lTk5ubCx8cHW7ZswaOPPmrVMkypq6tDYWEhACAiIsLu5RERkfVknwyQmpoKoPkOAaZOX87OzgYAxMfHW7W7SwiBtLQ0rF+/Hp06dUJubi4SExNlj7Xk3XffhU6ng0qlku4wQEREbUt20EyfPh1hYWHQarUYPXq0dFGmVqvFG2+8gdzcXABAVlaW0VjD7foXLlxo1DZr1iysXr0aHh4e+P77782exmxKSkoK5s+fD41G0+JamfPnz2PatGlYsmQJgOZToG3ZnUdERPaT/ZgAHx8fbN68GUlJSThx4gSio6PRuXNn1NbWQq/XQ6VSISsrCyNHjpT947/99huWLVsGoDmMpk+fjunTp5vtX15e3uJzRUUFNm7ciKysLLi7u6NLly5obGxscd3P+PHj8cUXX8iuiYiIHMuq59EMHDgQhYWFWLx4MfLy8lBaWoqAgAA88sgjmDVrlk3HZgzu3LmDa9euWTV+3rx5ePjhh3H06FFcuXIFN27cgJubGyIiIjB48GC88MILVgUfERE5nlVBAwA9evTAsmXLpC0ROcwdSwkPD2/1OIslI0eOZJAQEbVz7fLOAERE5DoYNEREpCgGDRERKYpBQ0REimLQEBGRohg0RESkKAYNEREpikFDRESKYtAQEZGiGDRERKQoBg0RESmKQUNERIpi0BARkaIYNEREpCgGDRERKYpBQ0REimLQEBGRohg0RESkKAYNEREpysPZBRARubrwufnOLkGWi++PUmS5DBoialf+6H+UXRF3nRERkaIYNEREpCgGDRERKYpBQ0REimLQEBGRohg0RESkKAYNEREpikFDRESKYtAQEZGirA6a8vJyzJw5E3369IG3tze6d++OMWPGYM+ePXYVUlNTgwULFiAqKgq+vr4ICAhAUlISNm7cKGv8hg0bMHz4cAQEBMDX1xdRUVFYsGABtFqtXXUREZF9rAqaM2fOoH///vj0009x4cIFeHl5obKyEnl5eXjiiSfw/vvv21TElStXEBMTg0WLFqG4uBju7u6oqanB3r17MWHCBLzyyisWx6elpSElJQX79u1DTU0N3N3dUVxcjEWLFiEmJgZXr161qS4iIrKf7Hud6XQ6jB07FlVVVYiNjcXXX3+N6Oho1NTU4J133sHSpUsxb948xMXFYeTIkbILEEJg/PjxKCkpQXh4ONatW4chQ4agoaEB//znPzF37lwsX74csbGxmDZtmtH45cuXY+XKlXBzc8OSJUvw2muvwcvLC4cPH0ZqaiouXLiAlJQU/Pjjj7JrItflivfRcsU5kWuRvUWzYsUKXLp0CWq1Glu3bkV0dDQAoHPnzsjOzsa4ceMghMCbb75pVQGbN29GQUEB3NzcsGnTJgwZMgQA4O3tjYyMDLz++usAgLfeegu3b99uMbaxsRELFy4EAMycOROzZ8+Gl5cXAGDIkCHYtGkTVCoVDh06hK1bt1pVFxEROYbsoFm3bh0AIDU1FaGhoUbtGRkZAIATJ07g/PnzsgswLHfEiBGIiYkxap89ezZUKhXKy8uxd+/eFm27d+/G9evXoVKpkJ6ebjQ2NjYWI0aMaPE7RETUtmQFjVarxfHjxwEAycnJJvsMHjwYXbp0AQCrTgzYt2+fxeWGhoZKW0/3B41hbP/+/U2G373LvX8sERG1DVlBc+7cOQghAED6o2+0IDc39O3bFwBw9uxZWT9+/fp1VFVVWVwuAPTr18/kcg2f5YytqKhAZWWlrLqIiMhxZAVNWVmZ9L5nz55m+xna7u2v5HINn+WMtaYuIiJyHFlnndXV1UnvfXx8zPbz9fUFANTW1sr6cXuXaxgvZ6ylunJycpCTkwMAKC4uRkJCQiuVmxZo0yjLKioqEBQU5NBlJiRkyu7ranNytfkArjcnV5sP4Jpzut/FixfNtvFRzmi+DictLc3ZZZiUkJAAjUbj7DIcytXm5GrzAVxvTq42H6BjzUnWrjM/Pz/pvU6nM9uvvr4eAKBWq2X9uL3LNYyXM9aauoiIyHFkBc29xzksXWVvaAsJCZH14/Yu1zBezlhr6iIiIseRFTSRkZFQqVQAgKKiIpN99Hq9dP2M4Uyv1gQFBSEwMNDicoHfzy67f7mGz3LG3vtbHUl73aVnD1ebk6vNB3C9ObnafIAONich05///GcBQLz00ksm2w8fPiwACACiuLhY7mLFhAkTBADx5JNPmmy/cuWKUKlUAoDYvn17i7b8/HwBQLi5uYmrV6+aHD9y5EgBQDz//POyayIiIseRfWeA1NRUAM1X2Js6TTg7OxsAEB8fL11PY81yd+7cidOnTxu1f/TRRxBCICQkBImJiS3akpKSEBwcDL1ej6VLlxqNPX36NHbv3g0AmDx5suyaiIjIgeQmUn19vQgLCxMARFxcnCgqKhJCCFFTUyMyMjKkrZkdO3YYjTW0ZWZmGrXp9XoxaNAgAUBERESII0eOCCGEaGhoENnZ2cLNzU0AEDk5OSbr+vzzz6WtmuzsbNHQ0CCEaN7CioiIEADE0KFD5U6TiIgcTHbQCCHEqVOnREBAgBQcnTt3loJApVKJxYsXm/4RC0EjhBCXL1+WQgGAUKvVwsPDQ/psbnedwbRp06S+np6eQq1WS5979+4tSktLrZkmERE5kFXX0QwcOBCFhYVYvHgx8vLyUFpaioCAADzyyCOYNWsWkpKSbNqq6tWrF06dOoUlS5YgNzcXFy9ehL+/P2JiYvDyyy9jwoQJFsfn5ORgxIgR+OKLL3Dq1CnodDpERkbiueeew5w5c+Dv729TXW2tvr4eW7duxQ8//IDCwkL89ttv0Gq1uHPnDvz8/BAQEIAHH3wQjz76KMaNG2fyJqTtjSvN6ddff0VBQQE0Gg1OnTqFsrIyVFZWorq6Gn5+fujZsyfi4+Mxfvx4jB49Gu7u7s4uuVVVVVXQaDQ4duyY9DK1a/zxxx/H/v37275AK7niOjKora3FmjVrsG3bNpw5cwZVVVVQq9UIDQ3FyJEj8cILL6B///7OLtM0ZycdNfvmm29Ejx49pC0xOa+nnnpKlJWVObt0s1xtTn369JE9j4EDB4ozZ844u2SL1q9fL3s+jz/+uLPLlcXV1pHBli1bWuxNMvXy8PAQ8+bNE3q93tnlGuGdAdqBVatWmXyoW2t++OEHDB06FMePH0fXrl0dX5gdXHFOBqGhoYiLi0Pv3r0REBAAT09PVFVV4cSJEzhw4ADu3r2L06dPY9iwYTh8+DCioqKcXbJJjY2Nzi5BMa6yjgBg48aNSElJkW5sHBQUhLFjxyIiIgI6nQ4FBQXYs2cPmpqakJWVhbq6OnzyySfOLfp+zk66P7pbt24JPz8/q/7Vf/9rwYIFzp5GC644JyGEyM3NFZcuXbLY5/z58yI6OlqaR2JiYhtVZ70vv/zS5bZoXG0dlZeXtzjmnJqaKmpra436HTp0SAQFBUn98vLynFCteQwaJzO3+yI4OFh8++23ory8XGi1WnH8+HExatQok30HDRrk7Gm04IpzskZhYaE0D5VKJcrLy51dkklffvml8Pb2FoMGDRKvvfaa+Ne//iWKi4s7dNDI1VHW0fz586U6Y2NjxZ07d8z23bFjh9Q3MjKyXe1CY9A42eLFi03+j7127VqjvjqdTnh7exv17du3rxMqN88V52St4OBgaS5Hjx51djkmabVacfv2baPv/whBI0THWEdxcXFSjWvWrGm1/4ABA6T+hw8fboMK5ZF9wSYpw9wjDgy3/GntOwB46KGHHFqTvVxxTta4c+dOi0dS3PuoivZErVbD09PT2WU4RUdZR//73/+k9/Hx8a32v7fPxo0bFanJFgwaJ/vLX/5i8vu5c+fi3//+N27cuIGGhgacOXMGEydORENDQ4t+KpUKM2fObItSZXPFOVnjs88+k+4aHhwcLPvef9R2Oso6uvfO9Jaeu2Wqz/HjxxWpySbO3qQiIUaPHm3TAfNOnTqJzz//3Nnlm+SKczLn7t27oqqqSuzdu1dMnTq1xXxWrlzp7PKsZmq9dPRdZx11HYWEhEh17t69u9X+9x7zDAgIaIMK5WHQtAM1NTVmD4qbenXt2lW8+uqr4pdffnF26Wa54pzuNWfOHIvz8fHxEV999ZWzy7SJqwSNK6yjMWPGSPXOmDHDYt8bN260OEMNgMWTB9oSd521A/7+/sjLy8O2bdtk7Yetrq7G2bNncenSpTaozjauOCe5Jk6ciEuXLmHKlCnOLoXM6CjraOLEidL7nJwcHDp0yGQ/vV6PV1991ehx9TU1NYrWJ5dKiP9/FRA51a5du5CZmYkjR45YNS4jIwMffPCBQlXZxxXnZHDw4EEUFBQAaD6wfP36dRw5ckT6Ljw8HEuXLsVf//pXZ5ZpE1MnaHSUW9DcyxXW0d27dzFo0CDpeIuPjw/mzJmDSZMmITw8HDqdDv/973+RlZWF/fv3w9vbu8Uxz8uXL6NXr17OKv93zt6kIiHee+89o836bt26ieXLl4uSkhJx48YNceDAATFkyBCTuwA+++wzZ0/BiCvOSY6CgoIWt0HpiPMwtT464q4zczraOrp8+bJ46KGHWt397OvrK1atWtXiu+rqameXL4TgMRqn27Vrl8n/aLZt22bUt66uToSHhxv1DQoKEo2NjU6o3jRXnJM1Ll68KN0ZwdPTs8McdzJw9aARouOto5qaGpGeni78/f3Nrp9z586Jo0ePSt+5u7u3m4s2eYzGyT766COj70JCQvDUU08Zfe/r64tJkyYZfV9RUQGNRqNIfbZwxTlZIywsDH/7298ANO+yWbt2rXMLIiMdbR35+/sjOzsbFRUVOHDgAFavXo1ly5Zh/fr1+OWXX7B//35ERka2uO6mT58+Zq9Ta2u8qaaTHT161Oi7Bx54wGx/c22//fYbhgwZ4rC67OGKc7LWww8/LL039eRYcr6OuI68vLwwbNgwDBs2zGT7qVOnpPePPvpoG1XVOm7ROFldXZ3Rd5cvXzbb31ybl5eXw2qylyvOyVpNTU3S+7t37zqxEjLHFdfR5s2bpfdPPvmkEytpiUHjZMHBwUbflZWVYceOHUbf63Q6fPvttyaXY2mLoa254pysdezYMel9uzjrh4y42jratm0bfv75ZwBA9+7d29XZdAwaJzN3u5bJkycjJycHly5dwq1bt/Djjz8iOTkZJSUlRn0DAgIQGxurdKmyueKcrFFUVITvv/9e+jxy5EgnVmOeEAJNTU1GL3v7dgQdZR3Jde3aNcyYMUP6PH/+fHTq1MmJFd3H2Wcj/NH95z//kX31vLnXokWLnD2NFlxxTu+88454++23xeXLly32y8vLa/FU0aioqHZ79ty+ffvsXk8lJSXOnobEFdeREM317t+/3+wZZEePHhWRkZHSfBITE9vN2WYGPBnAyR577DHMmzcPWVlZNo1/4oknkJ6e7uCq7OOKc6qqqsKyZcuwcOFCDBw4EDExMejVqxfUajV0Oh0uXryIgwcP4sKFC9KYwMBAfPPNN+3rX5YuzFXX0ZEjR7Bo0SL07NkTw4YNw4MPPggfHx9cv34dhw4dgkajkZ6+OWjQIOTm5rabs80kzk46arZy5coWT8hr7eXh4SHS09OFTqdzdulmudKcZs2aZdW/9EeMGNHur81wtS0aV1xHQrR8+Jm5l7u7u3jllVeEVqt1drkm8RY07UhdXR3y8vKwfft2/PTTT/jtt9+g1WrR1NQEtVqNwMBADBgwAMOGDcPEiRMREhLi7JJb5SpzampqwuHDh7F//34cO3YMP//8M8rKylBXVwdvb2907doVffv2xSOPPIIJEybIur+bs+3fvx+JiYl2LaOkpATh4eGOKchOrriOAKC0tBT5+fnYt28fCgsLce3aNdy6dQsBAQH4v//7PyQnJ2PSpEmIiopydqlmMWiIiEhRPOuMiIgUxaAhIiJFMWiIiEhRDBoiIlIUg4aIiBTFoCEiIkUxaIiISFEMGiIiUhSDhoiIFMWgISIiRTFoiIhIUQwaIiJS1P8DVP4pNrAKEswAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_attribution(\n",
    "    query_ids[0],\n",
    "    input_ids[0,:mask.sum()],\n",
    "    attributions_guan[0,:mask.sum()],\n",
    "    attributions_gt[0,:mask.sum()],\n",
    "    ymax=1.1,\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Plotting latent space"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXsAAAFqCAYAAAD7tPo1AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAAwu0lEQVR4nO3de3hU1b038O+PyW2SEEJIIBpAvFVuCQSiNRgawVLxCiIcsK1obYXyqD31bdG+WpHS2tZy2lov9YDPeXuopRZBRNQWSw+YGg1HgYQESI7CkYvBSEIukGRCksl6/0hmyMzsyUyS2XvPzP5+nidPMmvPzP4ReL6zWHvttUQpBSIiim5DzC6AiIj0x7AnIrIAhj0RkQUw7ImILIBhT0RkAQx7IiILiDG7AH/S09PVuHHjzC6DiCii7Nu3r04pleHdHrZhP27cOOzdu9fsMoiIIoqIHNdq5zAOEZEFMOyJiCyAYU9EZAEMeyIiC2DYExFZAMOeiMgCdJt6KSLXA9itcahJKZUainOcPXsWp0+fRkdHRyjejgwQGxuLkSNHIiUlxexSiCzFiHn23wPwUa/HnaF407Nnz+KLL75AVlYW7HY7RCQUb0s6UkrB4XCguroaABj4pL/Vw3q+N5lbRxgwIuwrlVJ7Qv2mp0+fRlZWFhITE0P91qQTEUFiYiKysrJw6tQphj2RgSJ2zL6jowN2u93sMmgA7HY7h96IDGZE2G8UEaeInBGRP4vI2FC9MYduIhP/3khXq4ddGL4hNz2HcZoA/BpAEYCzAHIBPAagRERylVKnvV8gIssALAOAsWND9plARFbRO+T9/QxYcgxft7BXSpUCKO3VVCQi/wTwIbov2v5Y4zXrAawHgLy8PO6ETkQUIoaueqmU2i8iHwO42sjzEpFFrG7ynIHD2ThuZi1xHDa9dqVUn2PIgY4Tkcn8jc97D+MEG/hR+gFh6GwcEckDcBW6h3JM99udH2PNW4ehlPZnj1IKa946jN/u/NjgyoiIQku3sBeRjSLyMxFZICKzReQHAHYAqAbwrF7nDZZSCmfbOvCH949pBr4r6P/w/jGcbevw+4EQzqZOnYp7770XL730EiZOnAi73Y4ZM2bg6NGjaGpqwkMPPYRRo0Zh+PDhePDBByPyz0jk0QPv62eLz9LRcxjnIIC7ADwEIBFADYCtAJ5UStXpeN6giAhW3ToRAPCH948BAFbdOhEi4hH037punLvdSEopOJ3OgM8TEdhsNp/29vZ2HD58GPX19WhsbMTTTz+NhoYGrFixAitWrEBDQwNuuukmbNy4Edu3b8dzzz2HOXPmYN68eXr8cYjMEWVDMYOh52ycXwD4hV7vHwr+At/soAeAoqIizJo1K+DzCgsL8e677/q0Hzx4EB0dHcjPz8emTZvc7W+++Sa2bNmCzZs3Y+HChQCAWbNm4cUXX8Thw4cZ9mQ9wYz5AxH/wRG2e9AaxTvwXaFvZtADwPTp0/HRRx8FfN7QoUM120tLu2e9rlmzxqO9paUFOTk57qAHAIfDgc7OTowYMWIQFRMZIBwunoZDDQNg+bAHLgS+K+gBmBr0AJCcnIypU6cGfJ6/GktLSzF27FhcddVVPu133323R9uBAwcAAFOmTBlYsURGC3YGTjCv9Q7tKOvRu0Ts2jih5Bqj762vWTpGKCoqQmxsbMCvG264QfP1paWlmDZtmkdbTU0NampqfNpLS0ths9mQk5Oj25+HKGJFyYVdy/fstS7Guh4D5vXwBzOM09XVhQMHDuDRRx/1aHcN7eTm5vq0jx8/ngvLUeToz81TUdpT7y9Lh72/WTf+ZukYaejQocjLyxvQaz/55BO0tLRo9uCTk5Nx5ZVX+rR7P5coLAxkuGagY+qBeu/+zuN+HN4fIpYN+76mV4ZL4A+UqwevFfZTpkzBkCEXRu86Ojpw6NAhLF261NAaichYlg17EUFKQqzfWTe9Az8lITZigh4AlixZgiVLlvi0b9682actNjYW58+fN6Isov7zd/HU3zBOKM4V7HuFeU/em2XDHgAenvOlPte+cQV+JAU9UdQKZvniUMzGiVKWn40TKMgZ9ERRznu2zeqmqPwAsHzYE1GE8LfuzUDep69ZO1Ey1dKbpYdxiCjCaIZ0gJuiCADDnoispq8Pg/58UETYsgkcxiGiyBNOvfe+LgSHUZ3s2RNR5ArVtMtQCePePnv2REShEE4fOhrYsyeiyGbWBdow7L33hWFPROEvlDdI9fdO2UA1BVp8LUxu4mLYE1H0CCbAQ7W0Qu95+RGAYU9E4a+vNXK82wypp5/nCoMhH4Z9b3+4pfv7t942tw4i6r/+7Dhl6AeD+UEPcDZOVJs8eTLuv/9+vPzyy8jNzYXdbsfEiROxe/dus0sjim7eHyxhMNTDnn2YUkrB6XQGfJ6IwGaz+bS3tbWhqqoKLS0tqK6uxhNPPIHY2FisXLkSS5cuxcmTJ/Uom8h8wV7MNdrqYab28hn2YaqoqAizZs0K+LzCwkK8++67Pu0VFRVwOp2YPn06tmzZ4m4/efIkHnjgATgcDm5DSJErTIZG3LRu7tKa9WNi4Fs77F1j9C7Hi7XbTRjDH8wetMCF3arWrFnj0V5XV4eUlBQGPUWvUE2tHFQN5vbitVg77MNYcnIypk6dGvB5/tbbLysrw+jRozFx4kSP9tLSUuTk5ISiRKLwYla4uz9cvD5kAg0nGfxhYO2w9+6xh9FsnMEO45SWlmL69Oma7fPmzQtFiUQ0GAb3/q0d9mFsMMM4XV1dqKiowE033eTR3tDQgOPHjyM3NzdkdRKFjWCmXurR+x9MT93AwGfYh6mhQ4ciLy9vQK/9+OOP0dLS4tOzd43jT5s2bdD1EUUcvYd5zJ7tEwDDPgqVlZUB8A310tJSxMfH+4zjE5EJDB6zF6WUoScMVl5entq7d6/f45WVlZgwYYKBFVEo8e+PdGf2kgqB6BT2IrJPKeUzLMCePRFFp3ANeReDV8PkcglERAMRZvPoA2HPnoiinx43WvVrc/I+7rA1CHv2RETeVjeFLoz78z46LprGsCci6/AX4t7tAwldj9eH3xCPYWEvIjtERInIz4w6JxGRYfxtRRgmF4cNGbMXkbsATDHiXEREAZmxWFqg3r7Otege9iIyHMBvATwM4M96n4+IaMD8BXIwc/Y99qUNchgnlBupB2BEz/5pAAeVUq+ICMOeiMJHf0O0r953mAzX+KNr2ItIAYCl4BAOEVlJsD1zf/9T0OECr25hLyJxANYB+Del1P8E+ZplAJYBwNixY/UqjYhoYEK5cmaw/0sIUfDr2bN/BIAdwFPBvkAptR7AeqB7bRyd6iIi6p8BLV8cXtMvdQl7ERkL4HEA3wEQLyLxvQ7Hi0gqgHNKqcA7apOmHTt2YNKkSRgzZozZpRBZi7+ZPP29QNvX+vs6fFDoNc/+MgAJAP4EoKHXFwD8sOfnbJ3OHfXKyspw55134tVXX+3zeVOnTsW9996Ll156CRMnToTdbseMGTNw9OhRNDU14aGHHsKoUaMwfPhwPPjgg+jvCqiTJ0/G/fffj5dffhm5ubmw2+2YOHEidu/ePZg/HpG16fQ/Ar2GccoAaO2ptxvdHwD/AeCITuceEGeXE8XVxaisr8SEtAkoyCqAbYjNtHpaW1tx4sQJn/a2tjbMnz8fU6dOxS233IITJ05oXt9ob2/H4cOHUV9fj8bGRjz99NNoaGjAihUrsGLFCjQ0NOCmm27Cxo0bsX37djz33HOYM2dO0FsWtrW1oaqqCi0tLaiursYTTzyB2NhYrFy5EkuXLsXJkycH/TsgCmve4/d93UgVBkM6uoS9UqoRwLve7T2bYx9XSvkcM5Ozy4nlO5ejoq4Cjk4H7DF2ZKdnY92cdaYF/gcffIA5c+b4PX78+HFMmDDB7x60Bw8eREdHB/Lz87Fp0yZ3+5tvvoktW7Zg8+bNWLhwIQBg1qxZePHFF3H48OGgw76iogJOpxPTp0/Hli1b3O0nT57EAw88AIfDAbvdHuSflsgiTNpsHOCqlwCA4upiVNRVoLWzFQDQ2tmK8rpyFFcXo3BMoSk1ffWrX/UYVlFK4Y477kB5eTn279+P1NTUPl/v2oJwzZo1Hu0tLS3IyclxBz0AOBwOdHZ2YsSIEUHX5+/96+rqkJKSwqAna/A37h6GF3QNXQhNKSVKqR8bec5gVNZXwtHp8Ghr62xDVX2VSRX5Wrt2Lf72t7/h1VdfDRj0QHcYjx07FldddZVP+4033ujRduDAAQDAlCnB3w5RVlaG0aNH+2xxWFpaipycnKDfh4iMwVUvAUxImwB7jGdPNCEmAePTxptUEfCPf/wDIuL+evTRR9He3o6rr77ao/3666/XfH1paanPHrQ1NTWoqanR3JvWZrP1K6RLS0t9NjT3d14iMh+HcQAUZBUgOz0b5XXlaOtsQ0JMAnLSc1CQVWBaTTNmzEBlZSXef/99fOc738HatWtx6623+jwvMTHRp62rqwsHDhzAo48+6tHuGnrJzc31aR8/fnzQQy9dXV2oqKjATTfd5NHe0NCA48eP+7w/kWUZuPZNIAx7ALYhNqybsw7F1cWoqq/C+LTxps/GSUxMRHp6Oh577DF897vfxQ9/+MOgX/vJJ5+gpaVFswefnJyMK6+80qfd+7nHjh3DpZdeiieffBKrV6/2OPbxxx+jpaXFp2fv+jBhz54sKwxm3fjDsO9hG2JD4ZhC0y7IaklPT8ezzz6LO++8s1+v8xe6paWlmDJlCoYMuTB619HRgUOHDmHp0qUez21paQEAZGZm+rx/WVmZ3/ePj4/3GccnsqxQXsAdJOnvjTRGycvLU3v37vV7vLKyEhMmTDCwImtZv349Hn/8cRw/flxzqGiw+PdHlmRA2IvIPqVUnnc7L9CSpqKiIjz88MO6BD0RGY/DOKRp48aNZpdARCHEsCciMoqJF3A5jENEZAEMeyIiC4josA/XmUTUN/69ERkvYsM+NjYWDocj8BMp7DgcDsTGxppdBpGlRGzYjxw5EtXV1WhtbWVPMUIopdDa2orq6mqMHDnS7HKILCViZ+OkpKQAAE6dOoWOjg6Tq6FgxcbGYtSoUe6/PyIyRsSGPdAd+AwNIqLAInYYh4iIgsewJyKyAIY9EZEFMOyJiCyAYU9EZAEMeyIiC2DYExFZAMOeiMgCGPZERBbAsCcisgCGPRGRBTDsiYgsgGFPRGQBDHsiIgtg2BMRWQDDnojIAhj2REQWwLAnIrIAhj0RkQUw7ImILEC3sBeRG0Vkl4jUiMh5EflMRF4VkYl6nZOIiLTF6PjeaQD2Afg9gFoAYwH8CMAeEclWSh3X8dxERNSLbmGvlHoFwCu920TkQwBVABYC+LVe5yYiIk9Gj9mf6fneafB5iYgsTfewFxGbiMSJyJUA1gGogVePn4iI9GVEz/6/AZwH8DGAHACzlVKntZ4oIstEZK+I7K2trTWgNCIiazAi7O8GcC2ArwM4C2CniIzTeqJSar1SKk8plZeRkWFAaURE1qB72CulKpVS/91zwfYGAMnonpVDREQGMfQCrVKqEcARAFcYeV4iIqszNOxFZBSA8QCOGnleCj+OdgdWFq3E3NfmYmXRSjjaHWaXRBTVdJtnLyKvA9gPoBzdY/VfAvAwuqddco69hTnaHcj/Sz6cygkAqG6uxs7jO1GypAT2OLvJ1RFFJz179nsAzAewAcDbAP4PgCIAU5VSH+t4Xgpzq0pWuYPexamcWFWyyqSKujW3NeOev92DGa/MwD1/uwfNbc2m1kMUSnreQfs0gKf1en+KXBV1FZrtB+sOGlzJBc1tzcjflO9+vP/0fuRvykfJ4hIkJySbVhdRqHDVSzJcdnq2Zvvk9MkGV3LBA7sf6Fc7UaRh2JPh1uSvgU1sHm02sWFN/hrN57d3tuP50ufx7Xe+jedLn0d7Z3vIa/qk8RPN9iONR0J+LiIzMOzJcPY4O0qWlGDuuLkYnTwac8fN9Xtxtr2zHbM2z8K68nX4sOZDrCtfh1mbZ4U88K9MvVKz/YpUzhKm6KDnEsdEftnj7FhbuDbg89ZXrMfZ9rMebWfbz2J9xXo8mPtgyOp5YdYLHmP2vduJogF79tQvtc21mPmXmcjZkIOZf5mJ2mZ91zAqPV2q2V52uiyk50lOSEbJ4hLkZuTCHmPHRUkXYe3MtZwKSlGDYU9Bq22uxezXZqPxfCMUFBrPN2L2a7P7DHxnlxNFJ4vwQukLWFm0EvftuK9f4+65I3M126eOnDqQP0Kf7HF2xNniIBDUtNTgyZInsXzncji7nIFfTBTmOIxDQVvw1gK/7e8tec+n3dnlxPKdy1FeVw5H54U7ZD/64iO8UvUKdi/ajbiYONS31GPxXxej1lGLDHsGNt28CWlJaahvqce2T7b5vK8NNnz9S18PWK+j3YFVJatQUVeB7PRsrMlf02dPvbi6GBV1FWjtbAUAtHa2oryuHMXVxSgcUxjwfEThjGFPQWs639Svdld49g56F9e4+9e/9HUUbrkQpDWtNSjcUojXb38dd2y/Q/N9nXCicEshihcVo+xMGSrrKzEhbQIKsgpgG2KDo92Bx4sfx86TO92vqW6uxs5jOzF5xGQcPXsUoxJHYfaY2Zgycor7dZX1le6gd3F0OlBVX8Wwp4jHsKegDYsfhsbzjZrtWirrKzWD3mV9+Xq8VP6S5rEF27X/F9Fb4eZCxNpicd55HvYYO7LTs/FM4TMo2FQAJ3yHXpxw4sCZAwCA5qZmHG06CnuMHTnpOVg3Zx0uS7lM8zyXplwasBaicMcxewra1lu39qt9QtoEJNgS/L6fgkIXuvweC8QJJ9qcbVBQaO1sxYc1H+Krr31VM+j9cXQ63EM17xx/R/M5/tqJIgnDnoKWkZyBXXfuQmp8KgSC1PhU7LpzFzKStTeaKcgqMLQ+BYWWzpZ+v66tsw1V9VU4dOaQ5vGDteYt40AUKhzGoX7JSM7QvBirxTbEhk4V/nvLKyg8X/Y84hGvefxU6yk42h2chkkRjT170tVF9osG9Dp7jB2zL54d4mr6dh7n/R4ze0VOosFiz55CprmtGQ/sfgCfNH6CK1OvxM+v+zlOtJzo9/sk2BIwRA3B7lO7dahyYMxckZMoFBj2FJSaczWYv30+WjpbkBSThG23b0Pm0Ez3ca0lgue+Prff5xEI2pxtIak5lMxckZMoFDiMQwHVnKvBnK1z3Bc/WzpbMGfrHNScq3E/57v/+G5IzhXMLByj2cSGJ7/8JIpOFuHfD/w7ik4W8a5aijjs2VNA87fP99u+5xt7AAAVZ7Q3JIl0c8fNxZNffhLfL/q++wYx15z+dXPWwTbEFvhNiMIAe/YUkL/pjL3b/c2Xj2SZiZlYW7gW+2r3uZdRcM3pd83NJ4oUDHsKKCkmKWD7JUmXGFVOSKTHpWNIgH/+m27eBED7TmDX3HyiSMGwp4C23b4tYPvGmzcaU0yI1LXX9fm/ke3ztgMA5myegxfLXvS5liAQHGk8osuuWUR6YNhTQJlDM7FzwU53Tz4pJgk7F+z0mI0zLHEYihcV4/JhlyPeFo/Lh12Op/KfMqvkQbv9jdtRuKUQNa01mh8KXejCjmM7dNk1i0gPvEBLQckcmum+GOvPsMRh2DZ/m/vxs/ufHdC5ZmTOwJHGIzjddnpArzfS2faz+P2B3+P707/v0e7scqK4uthnRU4iszDsSTcHag8M6HUf1HwQ4kr09R8H/wPLs5e7l1NwrePP2TsUTjiMQ7rxt8tUNHrigyfcP/feBIWzdyhcMOwpJFzbD/a+6WhZ9jIkDkk0uzRD7P1iL+pb6jFn8xw8tOshn01QOHuHzMZhHBq0voYt3rvrPTyz7xls+ngTOro6MNI+Eq/e8ipmbZkVVXPzh8YM9dhxy1tCTAIuTroYM/8yE03nmzAsfhi23rrV7/LQRKEmSoXf7ekAkJeXp/bu3Wt2GRSEopNFeOSfj3j0Zu0xdqz9ylq/2/nd8JcbcPp8+F+ADdZQ21Ccc57zezx7WDYqmnzvMt515y7EDYnDPX+/ByebT2JM8hhs+NoGDEvU3v2LKBAR2aeUyvNu5zAODVp/bzqqb6lHU4f2vrWRqq+gB4DKs5Wa7fO2zUPB5gIcbTqKdmc7jjYdRcHmAjS1Rtfvh8zHsKdBm5A2AfYYz409EmIScEXqFXi+9Hl8+51v4/nS59He2Y76lnoUbinE+S7/a8dHI3+buPj7kLjn7/foWQ5ZEMfsadAKsgqQnZ6N8rpytHW2ISEmAZNHTMYT7z+Bcx3dYfZhzYdYX74+LFe1DIUkWxJanP3fEtGfz5o/C9l7EQEMewoB2xAb1s1Zh+LqYlTVV2F82niU15bjoy8+8nhetAY9gAEHfSxi0YEOn/bRyaMHWxKRB4Y9hYRtiA2FYwrdF2T/ePiPJlcUfi5NuhSftnzq0ebvA3DD1zYYURJZCMfsSRdWuqEqWCOSR0AgHm1OOHHfpPs81hQqXlTM2TgUcuzZky6WZS/DK1Wv4Gz7WbNLCQvxtnh8c8I3cfjMYY8pqgkxCZg2ahoeznvYxOrICtizJ13ExcRh96LdWJ6zHNPSp2k+54YxNxhclTlihsTg6ZlP4/ox1yM7PRv2GDsEAnuMHTnpOSjIKjC7RLIA3Xr2IrIQwF0A8gCMBHACwFYAP1dK9T0pmaJCXEwcHsx9EMgFmlqbcM/f78FnzZ9hdPJobPjaBiz+22KzSzREZ1cn9n2xDzOzZvpcyOZqmGQU3e6gFZE96A74NwB8BiAXwGoAVQBmKKX6vFeed9BGv5VFK7Hj2A6zyzBMckwyihYXIS4mzuxSKIqZcQftbUqpf1FKbVRKFSmlngHwPQBfBnC9juelCLEmfw1sYp1ebXNnM9ZXrDe7DLIo3cJeKVWr0eyaeJ2l13kpctjj7ChZUmJ2GYYq/aLU7BLIooy+QOtaFUt7oRCyHHucHTFinUlhZbVlaO9sh6PdgZVFKzH3tblYWbQSjnZH4BcTDYJhq16KSBaAUgAHlFJz/DxnGYBlADB27Njpx48fN6Q20k97ZzvWV6xH6elS5I7MxbLsZT5j1ieaTuCWbbeYVKHx4obEob3Lc99am9hQsqTEvdsV0UD5G7M3JOxFJBnAuwAuBnCNUirgwh+8QBv52jvbMWvzLI+59vYYO+6ZeA8mp0/2mIlyoukE5r0xz++CYVYwd9xcrMlfg1Ulq1BRV4Hs9GysyV/DDwDqF9PCXkTsAP4KYAqAQqWU76LeGhj2ke/50uexrnyd5rHEmETNfVm11sa3iosTL8YXji/gVE53G3v81F+mrGcvIrEAtqB7rv3NwQY9RYfS0/4vRvrbl9W1gqYN1pml49KFLo+gBwCncmJVySqTKqJoolvYi8gQABsBzAYwXym1R69zUXgKtD6O1gYnrhU0L0+9XM/SwpLq0v5f9s7jO3kRlwZNz579CwAWAfg1gBYRubbXF9dvtYBl2cuQEpfi93hCTALGp433abcNsWHW2Fl6lhaWGtsbNdudyokdx3Yg/y/5DHwaMD3D/qae748DKPH6+o6O56Uw0Xt9nGtGXYOspKyg14VZeMVCg6s1X6DduzikQ4Oh2wRnpdQ4vd6bIod7fRwAzi5n0OvC3L3jbiPLjBgH6w6aXQJFKOvczUKm897gpC+1Dq0bsGly+mSzS6AIxSWOKSxl2DPMLsE0qfGpuDjxYp92m9iwJn+NCRVRNGDYU1jadPMmzfadC3b2edE3EgRaHqJLdeGdRe/gw7s+xNxxczE6eTTmjpvL+fY0KBzGobBR31KPxX9djFpHLTLsGXj99tex4h8r3I833bwJaUlp2L1oNxZsW4DjLZG5nEagu4SvSL0CQPe6QWsL1xpRElkAw5505Wh3BHX7f31LPQq3XBjLr2mtwR3b70DRwiKkJaV5PDcuJg4nW07qXrtZXpj1gtklUBTiMA7pxtHuQP5f8rHj2A5UN1f3OVd88V+1d63y167Q9zIfSTFJ/S84DNw78V4kJySbXQZFIYY96WZVyaqgb//3N/vGX3ugC7jbbt8WkRujVNZz9W/SB8OedFNRp70UktZccX/h7a998y2bNdvHJo9F8aJiZA7NxPZ524OsNHxMHTm1z+NNrU2Yv20+pv9pOuZvm4+m1iZjCqOIx7An3WSnZ2u2a80V9zf7xl97WlIaihYWITMxEzaxITMxE0ULi/D2nW9jWOIwAMA3/vaNAVZunmXZy/wea2ptQsHmAhxtOop2ZzuONh1FweYCBj4FxbDNS/qLSxxHPteYfbBL9nrPxnHNvhmonA05Acf2w82+b+zz2Nyl9wXuxrZGtHS2+Lzm8mGXY9v8bQZWSeHM3xLHnI1DunHtMbuqZBUO1h3E5PTJfW7GkZaUhp2Ldobs/MPih6HxfGO/XxcrsehQHSGroz9u23Ybbrv8NizLXgZnl9Pnw1LLZ80B9wIiYs+eoldtcy1mvza7369LjU8d0IdEKKXEpeDazGvx9xN/D/hc9uypN1M2LyEyU0ZyBnbduQup8akQCFLjUpE7ou819gHoGvTZw7WvY3g7234WJZ+XBPXcDV/bMJiSyCI4jENRLSM5A+8teQ9A95aHP9j9A1Prae1qRZwtDu3O9oDP9Td1NCkmCZ2qE6OTR2PD1za4L0gT9YVhT5ZRWV+J80p7zfihtqE45zynew2fNX+GMcljcLTpaMDnjksZh4ozFT4XuHcv2s01cqjfOIxDljEhbQIEonms2dlsSA2u3ngw4mxxKFlSwsXQKCTYsyfLKMgqwPD44ag/X+9zbKAzd7x9OfPL+En+T3D7ttvRrnyHavozvp47KpeLoVHIsGdPlmEbYsPb89/GEK9/9jaxYeutWzVfEzckDpcPuxx2W3C96akjpyIzORO/mfUb3DfpPmQmZrrfo3hRMYYlDsPSd5YGfJ+UuJQ+b7Ai6i/27MlSkhOSseeuPZpz/0sWl+CB3Q/gSOMRXJF6BV6Y9YJ7UbKaczWYs3VOn++dEpeCb0/6NpbvXI6Kugo4Oh2wx9iROzIX6+asc2/BePyc9tLMAsE1mddg6sipWJa9zOPmKqLB4jx7oiDVnKvB/O3z0dLZgqSYJLx666vY/r/bUXa6zB3QJZ+X4JF/PoLWzlb36+wxdqz9ylr3doxXv3w12rrafN4/YUgCPrr7I8P+PBSdeAct0SBlDs3Enm/s8WhzbabuUllfCUen5xLObZ1tqKqvcof94qsWY0Ol79j94qu0l3MmCgWO2ROF0IS0CbDHeI7vJ8QkYHzaePfj7037HhKHJHo8J3FIIr437XuG1EjWxJ49UQgVZBUgOz0b5XXlaOtsQ0JMAnLSc1CQVeB+TlxMHN676z2sr1jvMQTEMXrSE8fsiULM2eVEcXUxquqrMD5tPAqyCtwXZ4n0xjF7oj7UNtdiwVsL0HS+CcPih2HrrVuRkdz3blj+2IbYUDim0D1GTxQOOGZPludaHbPxfCMUFBrPN2L2a7NR26y9JSJRJGLYk+UteGtBv9qJIhHDniyv6bz2tn7+2okiEcOeLG9YvPYSwf7aiSIRw54sz9+6OP7aiSIRw54sz2dHq/hU7Lpz14Bn4xCFI069JILnjlZE0Yg9eyIiC2DYExFZAMOeiMgCdAt7ERktIs+JSImItIqIEpFxep2PiIj807NnfwWAfwHQAIBXvoiITKRn2P9TKTVKKXUzgM06noeIiALQLeyVUl16vTcREfUPL9ASEVlAWIW9iCwTkb0isre2lsvLEhGFSliFvVJqvVIqTymVl5HBW9WJiEIlrMKeiIj0wbAnIrIAhj0RkQXouuqliCzs+XF6z/ebRKQWQK1SqkjPcxMR0QV6L3HsfTPV73u+FwG4XudzExFRD13DXikler4/EREFh2P2REQWwLAnIrIAhj0RkQUw7ImILIBhT0RkAQx7IiILYNgTEVkAw56IyAIY9kREFsCwJyKyAIY9EZEFMOyJiCyAYU9EZAEMeyIiC2DYExFZAMOeiMgCGPZERBbAsCcisgCGPRGRBTDsiYgsgGFPRGQBDHsiIgtg2BMRWQDDnojIAhj2REQWwLAnIrIAhj0RkQUw7ImILIBhT0RkAQx7IiILYNgTEVkAw56IyAIY9kREFsCwJyKyAIY9EZEF6Br2IjJGRLaISJOInBWRrSIyVs9zEhGRL93CXkQSAewCMB7APQDuBnAlgN0ikqTXeYmIyFeMju99P4DLAFyllDoCACJSDuATAMsB/EbHcxMRUS96DuPcDmCPK+gBQCn1KYD3AczT8bxERORFz7CfBOCgRvshABN1PC8REXnRM+zTADRotNcDGK71AhFZJiJ7RWRvbW2tjqUREVlLWE29VEqtV0rlKaXyMjIyzC6HiChq6Bn2DdDuwfvr8RMRkU70DPtD6B639zYRwGEdz0tERF70DPvtAK4VkctcDSIyDsB1PceIiMggeob9SwCOAXhDROaJyO0A3gBwEsA6Hc9LRERedAt7pVQLgNkAPgbwMoCNAD4FMFsp1azXeYmIyJeed9BCKXUCwJ16noOIiAILq6mXRESkD4Y9EZEFMOyJiCyAYU9EZAEMeyIiC2DYExFZAMM+CJ9//vmgjhMRmY1hH8C4H72N/N/t9xvon3/+OfJ/tx/jfvS2wZUREQUvqsK+pqZmUMe99Q54rcB3Bb3W84mIwknUhP24H72Na5/Z5zfQP/30U1z7zD6PHvjZs2fdPzscDp/XJCUloeRfp7kfuwK/ra3NJ+hL/nUaLrroIgBAW1vboP88REShpOtyCUbpHfDXPrMPe74/HZmZme427yGWmpoaXPvMPgBA+WMzERsbiwk/2QUAOPbLWwB0fxDk/Pw9AN1B7gr23gHv4h3041f/l8d7ERGZLSp69pmZmdjz/enux64efkNDAz799FOP576++GIkJia6H+f8/D130APdHww1NTXuoAd8e/i9+Qt612MionAgSimza9CUl5en9u7dG9Rz6+vrkZaW5tFj70vJv05De3s7Cl/U2g/dU/ljM5GSkgLAd4weAKpW34CEhASfoK9afQM6OzuRnJwc1J+BiCgURGSfUirPpz3Sw941RLP/kfx+BX4wXMNBTU1NaG1t1RzCAbqDXSvoJ/+sCACHc4jIOP7CPqKHcerr690/T/tVCerr632GdAbj2mf24fPPP8eUXxT7DXoAfQY9ADQ3c/l+IjJXRId9Wloa9j+S73487VclOHz4sN+e/e7lE/t9jr5C3ptW0B/8cSGHcojIdBE/jAN09/Cn/apE54oG7tgvb8G5c+cwdOhQs0shoigXlcM4Lmlpafjr0ks1j/lrN9LBgweR/dQ/eZctEZkmKsK+pqYGN//xU5/2/Y/ku6dFmuWtb16CW/903P3YX+DX1tYaVRIRWVBU3FTV+waq3swe2tmycJRH0Lu4poq61NbW4upffwiAM3eISB9R0bPvPSsHgMdFWzMt3PKFT9s737rCb9C7HhMRhVrEh733xdn9j+RDRPp8zUBm5YTKjX844g5076D/6AfXICMjw6zSiCiKRXTYawV9Wloahg8f3ufr0tLSUProDL3L8/DHmy/MxLn61x/6BP1b37wEra2thtZERNYR8VMvve+g7a2hoQFKKc2x+96Lm5nN+yIux+2JaKCidrkEwPeCp7/jTU1NmPKL4lCVqJuiFZNxySWXmF0GEUWgqJ9nH8zxYcOG+V29Mlww6IlID1ER9sHqazGzcPDmN8b2GfS9N1shIuoPS4V973Xsw9FtG0/g1KlTmsdcm6nwLlwiGghLhf3hw4fNLiGgGc+W+gR6712zXI+JiPrDMmE/7kdveyyp8Prii02sJjBXD9876HtvpkJEFCxLhL13j/6vSy9Fbm4u9v7wyyZVFNiMZ0sZ9EQUMpYI+4kTPe+YvfmPn6K6uhrhOO30lXkXZhYx6IkoVCwR9kD3jUq9lzu+7rkyjztYge6g/egH1xhdmoe73qjHm98Y69H2/kNTGfRENCiWCXugu4f//kNT/R7PyclBRkYG3nsgx7iiNNy28YTH4+ueK+MsHCIaFEuFPQBkZWX5DfxJP30XZ86cwcwXyg2tyWX97FiPx951VldXG1gNEUUT3cJeRP6PiLwpIp+LiBKR1Xqdqz+qq6tx3XNlfo9PX7vHuGK8LNvV4f75/Yem+tR53XNlnHZJRAOiZ8/+fgAjAWzT8Rz94h30fQ3p9PbqggzsWjZBl5r+39fsPm3vfOsKv3Xm/Pw9Bj4R9ZueYT9JKfVlAA/peI6gaQV9X0M6Lq8uyMA111yDyy67DNu/Pibkdd33dwfe+dYVHm03/uGI++eKx7+CrKwslD82093Gi7VE1F+6hb1Sqkuv9+4vf0EP9D2GDwCTJk0CADQ3N+P2P5/UfM4/vnPVoOq78Q9H8MZdo33aKx7/CoYO7V4HPyUlBeWPzeTyx0Q0IJa4QOsKdsAz6F1SU1M9Hh964nr3z5N++i4+//xzJCcn+33/urq6Qdc475XPfNqyn/onzp07537MHj0RDVRYhb2ILBORvSKyN9R7sR775S2aQd/S0oJJP33X/fjQE9cjKSnJI/Dzf7cf+/bt8/veS7adCWmtFY9/xf2zd+ATEQ1EUGEvIl/tmVET6OvdwRSjlFqvlMpTSuXpsRdrsEEPAElJSR5r39+5ucb98wffy0XpozOw497Lfc7x6oLB1e0aumHgE1EoBduz/wDAhCC+lupQo25cwQ54Br3LRRddhNcWZXq07bj3clx88cWw2WyY+59Hfd5z9OjR2LfyWr/n3P9Ivt9jvcfoGfhEFEpBhb1SqlUpVRXE14nA7xZejv3yFs2gd5k+fbrH47n/edRngbLePfz4+Pg+5+pP+1WJZuD3DnoX78D3Pk5EFCzd96AVkRgAHQB+opRaHezr+rMHrRGqqqo0e/KuBcpOnToVMOj7fP/VN6C1tVVzi8Vz584x6IkoKIbvQSsieSKyEMCCnqaJIrKw5yu8t4zSMH78eI+57oDnSpRaQX941aygh3TGr/4vTPtVieYaOAx6IhosPWfjPAhgM4BNPY8X9TzejO47ayOK99ANcOFu1jNnzmgGfWJiIkaMGIGd3/6S5numpaWhavUNPu319fWhK5yICPreVHWvUkr8fB3T67x60NpExCXn5+8hNvbCAmb7Vl7rDnqXMWN877x19epbW1t9joX7XrlEFHnCap59OPK3W5R34Jc/NhP7Vl6LESNG+IR1QkKCz/tO+1UJ6uvrMe1XJT7HtJ5PRDQYul+gHahwuEAbaFvA/m4b2NbWhtbWVs2A3/9IPhITExn0RDQohl+gjQa9g1sryL17+IGWM0hISEBaWprP1Mv9j+QjLS2NQU9EumHPPghnz57tM8gDHe/N39CNK/CJiAaDPftBCBTkAw363j181xg+EZEeGPYG0Qp67yEdBj4R6YVhbwB/QQ+AgU9EhmDYG6D3WLzW2Lx34HPsnohCjRdoDVRfX99nkAc6TkQUCC/QhoFAQc6gJyK9MOyJiCyAYU9EZAEMeyIiC2DYExFZAMOeiMgCGPZERBbAsCcisoCwvalKRGoBHDe7Dj/SAdSZXYSF8PdtPP7OjRXK3/clSqkM78awDftwJiJ7te5QI33w9208/s6NZcTvm8M4REQWwLAnIrIAhv3ArDe7AIvh79t4/J0bS/ffN8fsiYgsgD17IiILYNgHSUTGiMgWEWkSkbMislVExppdVzQSkYUi8pqIHBcRh4j8j4j8QkSGml2blYjIDhFRIvIzs2uJZiJys4j8U0Sae7Jlr4jMDvV5GPZBEJFEALsAjAdwD4C7AVwJYLeIJJlZW5T6IQAngMcAzAXwIoAVAHaKCP/NGkBE7gIwxew6op2ILAfwBoB9AO4AsAjAZgCJoT5XTKjfMErdD+AyAFcppY4AgIiUA/gEwHIAvzGxtmh0m1KqttfjIhGpB7ABwPXo/uAlnYjIcAC/BfAwgD+bXE7UEpFxAJ4BsFIp9UyvQ+/ocT72koJzO4A9rqAHAKXUpwDeBzDPtKqilFfQu3zU8z3LyFos6mkAB5VSr5hdSJS7D0AXgH834mQM++BMAnBQo/0QgIkG12JVhT3fK02tIsqJSAGApQAeMLsWCygAUAVgiYgcFZFOETkiIrr87jmME5w0AA0a7fUAhhtci+WISBaANQD+oZSKrl3ow4iIxAFYB+DflFL/Y3Y9FnBxz9dadF+fOoruMfvnRSRGKfW7UJ6MYU9hTUSS0X0BqxPAt0wuJ9o9AsAO4CmzC7GIIQCGArhXKbW1p21Xz1j+/xWRZ1UIb4TiME5wGqDdg/fX46cQEBE7gDfRfXH8RqXUZyaXFLV6phE/DuAJAPEikioiqT2HXY9tphUYnc70fN/p1f53AKMAXBTKkzHsg3MI3eP23iYCOGxwLZYgIrEAtgDIA3CzUqrC5JKi3WUAEgD8Cd0dGNcX0D0VtgFAtjmlRa1DAY53hfJkDPvgbAdwrYhc5mro+a/WdT3HKIR65tJvBDAbwHyl1B6TS7KCMgCzNL6A7g+AWQCOaL6SBur1nu83erXPBfCZUqomlCfj2jhB6Llx6gAAB4AfA1AAforu8bYcpVSzieVFHRF5EcB30T12/JbX4c84nGMcEVEAnlJK/djsWqKNiAiA/0L3zWuPA/hfdF+g/Q6Abyml/jOk52PYB6dnTPO3AOYAcP0lfV8pdczMuqKRiBwDcImfwz9RSq02rhprY9jrS0RSAPwCwEJ0XxesAvBLpVTIb2Zj2BMRWQDH7ImILIBhT0RkAQx7IiILYNgTEVkAw56IyAIY9kREFsCwJyKyAIY9EZEFMOyJiCzg/wPsBLZyoeuh9wAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x432 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "_, hidden_states = toy_getter(model_orig,{\n",
    "    \"query_ids\": model_orig.val_dataset[:][0].to(device),\n",
    "    \"input_ids\": model_orig.val_dataset[:][1].to(device),\n",
    "    \"mask\": model_orig.val_dataset[:][2].to(device),\n",
    "    \"labels\": model_orig.val_dataset[:][3].to(device),\n",
    "},)\n",
    "hidden_state = hidden_states[2].flatten(0, 1).detach().cpu()[\n",
    "    model_orig.val_dataset[:][2].flatten(0, 1).bool()\n",
    "]\n",
    "\n",
    "plt.figure(figsize=(6, 6))\n",
    "\n",
    "mask0 = (model_orig.val_dataset[:][1] == model_orig.val_dataset[:][0][:,:1]).flatten(0, 1)[\n",
    "    model_orig.val_dataset[:][2].flatten(0, 1).bool()\n",
    "]\n",
    "plt.scatter(*hidden_state[mask0].T, marker=\"x\", s=100, label=\"$=m$\")\n",
    "\n",
    "mask1 = (model_orig.val_dataset[:][1] == model_orig.val_dataset[:][0][:,1:]).flatten(0, 1)[\n",
    "    model_orig.val_dataset[:][2].flatten(0, 1).bool()\n",
    "]\n",
    "plt.scatter(*hidden_state[mask1].T, marker=\"+\", s=100, label=\"$=n$\")\n",
    "\n",
    "plt.scatter(*hidden_state[~mask0 & ~mask1].T, marker=\".\", s=100, label=r\"$\\neq m,n$\")\n",
    "plt.legend(fontsize=16, )\n",
    "\n",
    "plt.tick_params(axis='both', labelsize=16)\n",
    "# plt.savefig(\"plots/toy-hidden-projection.pdf\", bbox_inches=\"tight\", pad_inches=0)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Training DiffMask"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = ToyTaskModelDiffMask(hparams)\n",
    "model.load_state_dict(model_orig.state_dict(), strict=False)\n",
    "model.train_dataset = model_orig.train_dataset\n",
    "model.val_dataset = model_orig.val_dataset\n",
    "model.hparams.learning_rate = 3e-4\n",
    "model.hparams.learning_rate_alpha = 3e-1\n",
    "model.hparams.epochs = 100\n",
    "model.hparams.eps = 0.05"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "_, hidden_states = toy_getter(model_orig,{\n",
    "    \"query_ids\": model_orig.val_dataset[:][0].to(device),\n",
    "    \"input_ids\": model_orig.val_dataset[:][1].to(device),\n",
    "    \"mask\": model_orig.val_dataset[:][2].to(device),\n",
    "    \"labels\": model_orig.val_dataset[:][3].to(device),\n",
    "},)\n",
    "model.placeholder[0,0,0].data[:] = hidden_states[1].flatten(0, 1).detach().cpu()[\n",
    "    model_orig.val_dataset[:][2].flatten(0, 1).bool()\n",
    "].mean(0)\n",
    "\n",
    "model.placeholder[0,1,0].data[:2] = hidden_states[2].flatten(0, 1).detach().cpu()[\n",
    "    model_orig.val_dataset[:][2].flatten(0, 1).bool()\n",
    "].mean(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Uncomment to load a trained model\n",
    "# hparams.model_path = \"./outputs/toy-diffmask/version_0/checkpoints\"\n",
    "# hparams.model_path_ = os.path.join(hparams.model_path, os.listdir(hparams.model_path)[0])\n",
    "# model = ToyTaskModelDiffMask.load_from_checkpoint(hparams.model_path_).to(device)\n",
    "# model.to(device)\n",
    "# model.train_dataset = model_orig.train_dataset\n",
    "# model.val_dataset = model_orig.val_dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:lightning:GPU available: True, used: True\n",
      "INFO:lightning:VISIBLE GPUS: 0\n"
     ]
    }
   ],
   "source": [
    "trainer = pl.Trainer(\n",
    "    gpus=int(hparams.gpu != \"\"),\n",
    "    progress_bar_refresh_rate=10,\n",
    "    max_epochs=hparams.epochs,\n",
    "    logger=pl.loggers.TensorBoardLogger(\"outputs\", name=\"toy-diffmask\"),\n",
    "    callbacks=[CallbackToyTaskDiffMask()],\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:lightning:\n",
      "   | Name            | Type       | Params\n",
      "-------------------------------------------\n",
      "0  | embedding_query | Embedding  | 640   \n",
      "1  | embedding_input | Embedding  | 640   \n",
      "2  | encoder         | Sequential | 12 K  \n",
      "3  | encoder.0       | Linear     | 12 K  \n",
      "4  | encoder.1       | Tanh       | 0     \n",
      "5  | encoder.2       | Linear     | 130   \n",
      "6  | gru             | GRU        | 13 K  \n",
      "7  | head            | Linear     | 65    \n",
      "8  | gate            | ModuleList | 4 K   \n",
      "9  | gate.0          | MLPMaxGate | 4 K   \n",
      "10 | gate.0.f        | Sequential | 4 K   \n",
      "11 | gate.0.f.0      | Linear     | 4 K   \n",
      "12 | gate.0.f.1      | Tanh       | 0     \n",
      "13 | gate.0.f.2      | Linear     | 34    \n",
      "14 | gate.0.f.3      | Tanh       | 0     \n",
      "15 | gate.1          | MLPMaxGate | 227   \n",
      "16 | gate.1.f        | Sequential | 226   \n",
      "17 | gate.1.f.0      | Linear     | 192   \n",
      "18 | gate.1.f.1      | Tanh       | 0     \n",
      "19 | gate.1.f.2      | Linear     | 34    \n",
      "20 | gate.1.f.3      | Tanh       | 0     \n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validation sanity check'), FloatProgress(value=0.0, layout=Layout(flex='2'), max=5.…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "60b7cd59d4304e1baac5642f5424d5c8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), max=1.0), H…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/ndecao/diffmask/diffmask/optim/lookahead.py:39: UserWarning: This overload of add_ is deprecated:\n",
      "\tadd_(Number alpha, Tensor other)\n",
      "Consider using one of the following signatures instead:\n",
      "\tadd_(Tensor other, *, Number alpha) (Triggered internally at  /pytorch/torch/csrc/utils/python_arg_parser.cpp:882.)\n",
      "  slow.add_(group[\"lookahead_alpha\"], fast_p.data - slow)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=0.0, layout=Layout(flex='2'), max=8.0), HTML(valu…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1: Validation accuracy = 98.59, gates at zero = 1.53%, constraint = 0.07787\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/ndecao/.anaconda3/envs/kilt37/lib/python3.7/site-packages/torch/optim/lr_scheduler.py:216: UserWarning: Please also save or load the state of the optimizer when saving or loading the scheduler.\n",
      "  warnings.warn(SAVE_STATE_WARNING, UserWarning)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=0.0, layout=Layout(flex='2'), max=8.0), HTML(valu…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 2: Validation accuracy = 98.51, gates at zero = 1.80%, constraint = 0.08481\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=0.0, layout=Layout(flex='2'), max=8.0), HTML(valu…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 3: Validation accuracy = 99.20, gates at zero = 1.36%, constraint = 0.04800\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=0.0, layout=Layout(flex='2'), max=8.0), HTML(valu…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 4: Validation accuracy = 98.83, gates at zero = 0.84%, constraint = 0.07254\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=0.0, layout=Layout(flex='2'), max=8.0), HTML(valu…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 5: Validation accuracy = 99.51, gates at zero = 0.72%, constraint = 0.02206\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=0.0, layout=Layout(flex='2'), max=8.0), HTML(valu…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 6: Validation accuracy = 99.02, gates at zero = 0.71%, constraint = 0.05949\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=0.0, layout=Layout(flex='2'), max=8.0), HTML(valu…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 7: Validation accuracy = 99.39, gates at zero = 0.54%, constraint = 0.02836\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=0.0, layout=Layout(flex='2'), max=8.0), HTML(valu…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 8: Validation accuracy = 99.51, gates at zero = 0.71%, constraint = 0.02910\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=0.0, layout=Layout(flex='2'), max=8.0), HTML(valu…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 9: Validation accuracy = 98.81, gates at zero = 0.51%, constraint = 0.07339\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=0.0, layout=Layout(flex='2'), max=8.0), HTML(valu…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 10: Validation accuracy = 99.27, gates at zero = 0.39%, constraint = 0.04633\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=0.0, layout=Layout(flex='2'), max=8.0), HTML(valu…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 11: Validation accuracy = 99.59, gates at zero = 0.39%, constraint = 0.02400\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=0.0, layout=Layout(flex='2'), max=8.0), HTML(valu…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 12: Validation accuracy = 99.41, gates at zero = 0.40%, constraint = 0.03648\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=0.0, layout=Layout(flex='2'), max=8.0), HTML(valu…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 13: Validation accuracy = 99.17, gates at zero = 0.41%, constraint = 0.04531\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=0.0, layout=Layout(flex='2'), max=8.0), HTML(valu…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 14: Validation accuracy = 99.71, gates at zero = 0.43%, constraint = 0.01006\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=0.0, layout=Layout(flex='2'), max=8.0), HTML(valu…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 15: Validation accuracy = 99.61, gates at zero = 0.48%, constraint = 0.02015\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=0.0, layout=Layout(flex='2'), max=8.0), HTML(valu…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 16: Validation accuracy = 99.32, gates at zero = 0.43%, constraint = 0.04247\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=0.0, layout=Layout(flex='2'), max=8.0), HTML(valu…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 17: Validation accuracy = 99.02, gates at zero = 0.61%, constraint = 0.05807\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=0.0, layout=Layout(flex='2'), max=8.0), HTML(valu…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 18: Validation accuracy = 99.59, gates at zero = 0.39%, constraint = 0.02748\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=0.0, layout=Layout(flex='2'), max=8.0), HTML(valu…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 19: Validation accuracy = 99.49, gates at zero = 0.39%, constraint = 0.02847\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=0.0, layout=Layout(flex='2'), max=8.0), HTML(valu…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 20: Validation accuracy = 99.22, gates at zero = 0.74%, constraint = 0.04348\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=0.0, layout=Layout(flex='2'), max=8.0), HTML(valu…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 21: Validation accuracy = 99.61, gates at zero = 0.30%, constraint = 0.02158\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=0.0, layout=Layout(flex='2'), max=8.0), HTML(valu…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 22: Validation accuracy = 99.27, gates at zero = 0.36%, constraint = 0.03807\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=0.0, layout=Layout(flex='2'), max=8.0), HTML(valu…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 23: Validation accuracy = 99.90, gates at zero = 0.44%, constraint = 0.00191\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=0.0, layout=Layout(flex='2'), max=8.0), HTML(valu…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 24: Validation accuracy = 98.93, gates at zero = 0.83%, constraint = 0.06029\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=0.0, layout=Layout(flex='2'), max=8.0), HTML(valu…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 25: Validation accuracy = 99.56, gates at zero = 0.59%, constraint = 0.02752\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=0.0, layout=Layout(flex='2'), max=8.0), HTML(valu…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 26: Validation accuracy = 99.78, gates at zero = 0.36%, constraint = 0.00901\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=0.0, layout=Layout(flex='2'), max=8.0), HTML(valu…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 27: Validation accuracy = 99.39, gates at zero = 0.60%, constraint = 0.03774\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=0.0, layout=Layout(flex='2'), max=8.0), HTML(valu…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 28: Validation accuracy = 98.69, gates at zero = 0.76%, constraint = 0.06794\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=0.0, layout=Layout(flex='2'), max=8.0), HTML(valu…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 29: Validation accuracy = 99.61, gates at zero = 0.73%, constraint = 0.01290\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=0.0, layout=Layout(flex='2'), max=8.0), HTML(valu…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 30: Validation accuracy = 99.61, gates at zero = 0.60%, constraint = 0.01910\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=0.0, layout=Layout(flex='2'), max=8.0), HTML(valu…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 31: Validation accuracy = 98.96, gates at zero = 0.79%, constraint = 0.06909\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=0.0, layout=Layout(flex='2'), max=8.0), HTML(valu…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 32: Validation accuracy = 99.49, gates at zero = 0.55%, constraint = 0.02804\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=0.0, layout=Layout(flex='2'), max=8.0), HTML(valu…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 33: Validation accuracy = 99.49, gates at zero = 0.50%, constraint = 0.02995\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=0.0, layout=Layout(flex='2'), max=8.0), HTML(valu…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 34: Validation accuracy = 99.41, gates at zero = 0.55%, constraint = 0.03051\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=0.0, layout=Layout(flex='2'), max=8.0), HTML(valu…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 35: Validation accuracy = 99.20, gates at zero = 0.84%, constraint = 0.03998\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=0.0, layout=Layout(flex='2'), max=8.0), HTML(valu…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 36: Validation accuracy = 99.51, gates at zero = 0.62%, constraint = 0.02752\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=0.0, layout=Layout(flex='2'), max=8.0), HTML(valu…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 37: Validation accuracy = 98.98, gates at zero = 0.65%, constraint = 0.04705\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=0.0, layout=Layout(flex='2'), max=8.0), HTML(valu…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 38: Validation accuracy = 99.39, gates at zero = 0.70%, constraint = 0.02968\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=0.0, layout=Layout(flex='2'), max=8.0), HTML(valu…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 39: Validation accuracy = 99.10, gates at zero = 1.14%, constraint = 0.04397\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=0.0, layout=Layout(flex='2'), max=8.0), HTML(valu…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 40: Validation accuracy = 99.68, gates at zero = 0.97%, constraint = 0.01625\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=0.0, layout=Layout(flex='2'), max=8.0), HTML(valu…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 41: Validation accuracy = 99.49, gates at zero = 1.51%, constraint = 0.03025\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=0.0, layout=Layout(flex='2'), max=8.0), HTML(valu…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 42: Validation accuracy = 96.95, gates at zero = 3.69%, constraint = 0.20641\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=0.0, layout=Layout(flex='2'), max=8.0), HTML(valu…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 43: Validation accuracy = 99.61, gates at zero = 1.96%, constraint = 0.02186\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=0.0, layout=Layout(flex='2'), max=8.0), HTML(valu…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 44: Validation accuracy = 99.51, gates at zero = 3.07%, constraint = 0.02898\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=0.0, layout=Layout(flex='2'), max=8.0), HTML(valu…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 45: Validation accuracy = 99.39, gates at zero = 1.82%, constraint = 0.02902\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=0.0, layout=Layout(flex='2'), max=8.0), HTML(valu…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 46: Validation accuracy = 98.90, gates at zero = 5.85%, constraint = 0.05318\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=0.0, layout=Layout(flex='2'), max=8.0), HTML(valu…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 47: Validation accuracy = 99.71, gates at zero = 2.26%, constraint = 0.02058\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=0.0, layout=Layout(flex='2'), max=8.0), HTML(valu…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 48: Validation accuracy = 99.61, gates at zero = 2.34%, constraint = 0.01446\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=0.0, layout=Layout(flex='2'), max=8.0), HTML(valu…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 49: Validation accuracy = 98.98, gates at zero = 9.72%, constraint = 0.04581\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=0.0, layout=Layout(flex='2'), max=8.0), HTML(valu…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 50: Validation accuracy = 99.29, gates at zero = 8.76%, constraint = 0.03741\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=0.0, layout=Layout(flex='2'), max=8.0), HTML(valu…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 51: Validation accuracy = 99.27, gates at zero = 6.46%, constraint = 0.03356\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=0.0, layout=Layout(flex='2'), max=8.0), HTML(valu…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 52: Validation accuracy = 99.29, gates at zero = 6.76%, constraint = 0.03598\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=0.0, layout=Layout(flex='2'), max=8.0), HTML(valu…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 53: Validation accuracy = 99.88, gates at zero = 5.17%, constraint = 0.00601\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=0.0, layout=Layout(flex='2'), max=8.0), HTML(valu…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 54: Validation accuracy = 98.39, gates at zero = 5.40%, constraint = 0.08033\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=0.0, layout=Layout(flex='2'), max=8.0), HTML(valu…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 55: Validation accuracy = 98.81, gates at zero = 3.72%, constraint = 0.07464\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=0.0, layout=Layout(flex='2'), max=8.0), HTML(valu…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 56: Validation accuracy = 99.05, gates at zero = 13.82%, constraint = 0.05209\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=0.0, layout=Layout(flex='2'), max=8.0), HTML(valu…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 57: Validation accuracy = 99.61, gates at zero = 14.98%, constraint = 0.01539\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=0.0, layout=Layout(flex='2'), max=8.0), HTML(valu…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 58: Validation accuracy = 98.59, gates at zero = 14.54%, constraint = 0.07912\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=0.0, layout=Layout(flex='2'), max=8.0), HTML(valu…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 59: Validation accuracy = 99.03, gates at zero = 8.39%, constraint = 0.04504\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=0.0, layout=Layout(flex='2'), max=8.0), HTML(valu…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 60: Validation accuracy = 99.51, gates at zero = 13.67%, constraint = 0.02102\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=0.0, layout=Layout(flex='2'), max=8.0), HTML(valu…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 61: Validation accuracy = 99.51, gates at zero = 10.45%, constraint = 0.02545\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=0.0, layout=Layout(flex='2'), max=8.0), HTML(valu…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 62: Validation accuracy = 98.98, gates at zero = 10.88%, constraint = 0.05327\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=0.0, layout=Layout(flex='2'), max=8.0), HTML(valu…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 63: Validation accuracy = 98.90, gates at zero = 24.30%, constraint = 0.05439\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=0.0, layout=Layout(flex='2'), max=8.0), HTML(valu…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 64: Validation accuracy = 99.20, gates at zero = 19.93%, constraint = 0.03971\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=0.0, layout=Layout(flex='2'), max=8.0), HTML(valu…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 65: Validation accuracy = 99.20, gates at zero = 17.23%, constraint = 0.03596\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=0.0, layout=Layout(flex='2'), max=8.0), HTML(valu…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 66: Validation accuracy = 99.20, gates at zero = 14.11%, constraint = 0.04198\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=0.0, layout=Layout(flex='2'), max=8.0), HTML(valu…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 67: Validation accuracy = 99.15, gates at zero = 26.99%, constraint = 0.04434\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=0.0, layout=Layout(flex='2'), max=8.0), HTML(valu…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 68: Validation accuracy = 99.51, gates at zero = 4.93%, constraint = 0.02789\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=0.0, layout=Layout(flex='2'), max=8.0), HTML(valu…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 69: Validation accuracy = 99.10, gates at zero = 18.73%, constraint = 0.04044\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=0.0, layout=Layout(flex='2'), max=8.0), HTML(valu…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 70: Validation accuracy = 98.88, gates at zero = 29.04%, constraint = 0.05300\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=0.0, layout=Layout(flex='2'), max=8.0), HTML(valu…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 71: Validation accuracy = 98.78, gates at zero = 28.85%, constraint = 0.05822\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=0.0, layout=Layout(flex='2'), max=8.0), HTML(valu…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 72: Validation accuracy = 99.02, gates at zero = 24.57%, constraint = 0.04906\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=0.0, layout=Layout(flex='2'), max=8.0), HTML(valu…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 73: Validation accuracy = 99.05, gates at zero = 13.98%, constraint = 0.03420\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=0.0, layout=Layout(flex='2'), max=8.0), HTML(valu…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 74: Validation accuracy = 99.17, gates at zero = 20.59%, constraint = 0.03864\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=0.0, layout=Layout(flex='2'), max=8.0), HTML(valu…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 75: Validation accuracy = 98.57, gates at zero = 16.74%, constraint = 0.05346\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=0.0, layout=Layout(flex='2'), max=8.0), HTML(valu…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 76: Validation accuracy = 98.90, gates at zero = 26.50%, constraint = 0.03743\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=0.0, layout=Layout(flex='2'), max=8.0), HTML(valu…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 77: Validation accuracy = 99.44, gates at zero = 21.27%, constraint = 0.03108\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=0.0, layout=Layout(flex='2'), max=8.0), HTML(valu…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 78: Validation accuracy = 99.22, gates at zero = 25.54%, constraint = 0.03544\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=0.0, layout=Layout(flex='2'), max=8.0), HTML(valu…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 79: Validation accuracy = 99.59, gates at zero = 10.98%, constraint = 0.02244\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=0.0, layout=Layout(flex='2'), max=8.0), HTML(valu…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 80: Validation accuracy = 99.10, gates at zero = 32.40%, constraint = 0.04777\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=0.0, layout=Layout(flex='2'), max=8.0), HTML(valu…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 81: Validation accuracy = 99.32, gates at zero = 22.17%, constraint = 0.04142\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=0.0, layout=Layout(flex='2'), max=8.0), HTML(valu…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 82: Validation accuracy = 98.49, gates at zero = 27.80%, constraint = 0.08546\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=0.0, layout=Layout(flex='2'), max=8.0), HTML(valu…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 83: Validation accuracy = 99.08, gates at zero = 21.91%, constraint = 0.04913\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=0.0, layout=Layout(flex='2'), max=8.0), HTML(valu…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 84: Validation accuracy = 99.12, gates at zero = 22.39%, constraint = 0.03874\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=0.0, layout=Layout(flex='2'), max=8.0), HTML(valu…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 85: Validation accuracy = 99.32, gates at zero = 28.52%, constraint = 0.03182\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=0.0, layout=Layout(flex='2'), max=8.0), HTML(valu…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 86: Validation accuracy = 99.56, gates at zero = 11.58%, constraint = 0.02303\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=0.0, layout=Layout(flex='2'), max=8.0), HTML(valu…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 87: Validation accuracy = 98.93, gates at zero = 34.26%, constraint = 0.04856\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=0.0, layout=Layout(flex='2'), max=8.0), HTML(valu…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 88: Validation accuracy = 99.00, gates at zero = 34.14%, constraint = 0.04128\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=0.0, layout=Layout(flex='2'), max=8.0), HTML(valu…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 89: Validation accuracy = 98.69, gates at zero = 28.58%, constraint = 0.06077\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=0.0, layout=Layout(flex='2'), max=8.0), HTML(valu…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 90: Validation accuracy = 99.12, gates at zero = 17.29%, constraint = 0.03956\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=0.0, layout=Layout(flex='2'), max=8.0), HTML(valu…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 91: Validation accuracy = 99.71, gates at zero = 28.08%, constraint = 0.01036\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=0.0, layout=Layout(flex='2'), max=8.0), HTML(valu…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 92: Validation accuracy = 99.32, gates at zero = 12.03%, constraint = 0.02913\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=0.0, layout=Layout(flex='2'), max=8.0), HTML(valu…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 93: Validation accuracy = 99.51, gates at zero = 29.54%, constraint = 0.01754\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=0.0, layout=Layout(flex='2'), max=8.0), HTML(valu…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 94: Validation accuracy = 99.22, gates at zero = 28.74%, constraint = 0.03640\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=0.0, layout=Layout(flex='2'), max=8.0), HTML(valu…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 95: Validation accuracy = 99.49, gates at zero = 34.72%, constraint = 0.03013\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=0.0, layout=Layout(flex='2'), max=8.0), HTML(valu…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 96: Validation accuracy = 98.88, gates at zero = 35.35%, constraint = 0.05273\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=0.0, layout=Layout(flex='2'), max=8.0), HTML(valu…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 97: Validation accuracy = 99.61, gates at zero = 17.38%, constraint = 0.01645\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=0.0, layout=Layout(flex='2'), max=8.0), HTML(valu…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 98: Validation accuracy = 98.81, gates at zero = 35.79%, constraint = 0.05967\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=0.0, layout=Layout(flex='2'), max=8.0), HTML(valu…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 99: Validation accuracy = 99.41, gates at zero = 11.89%, constraint = 0.02681\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=0.0, layout=Layout(flex='2'), max=8.0), HTML(valu…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 100: Validation accuracy = 99.25, gates at zero = 23.74%, constraint = 0.04601\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "1"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trainer.fit(model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.to(device)\n",
    "\n",
    "attributions_ours = [\n",
    "    model.forward_explainer(\n",
    "        query_ids=query_ids,\n",
    "        input_ids=input_ids,\n",
    "        mask=mask,\n",
    "        layer_drop=0,\n",
    "        layer_pred=0,\n",
    "    )[3].exp().cpu().detach(),\n",
    "    model.forward_explainer(\n",
    "        query_ids=query_ids,\n",
    "        input_ids=input_ids,\n",
    "        mask=mask,\n",
    "        layer_drop=1,\n",
    "        layer_pred=1,\n",
    "    )[3].exp().cpu().detach(),\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZoAAAENCAYAAAA2ZaOYAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAAmgElEQVR4nO3de1TUdf4/8OcMILcBbQdQxBbQU4JYgsym6eYexKTykm1KiXuy/pBubq6HyOumbV+xDLudTiaa2e7RLnpoVbS86xYq62hooFiteEOQiwoDDCjO+/cHv/kkzYXPXD4MTM/HOXPOzLwv83qfT/H085nP5zMqIYQAERGRQtSeLoCIiLwbg4aIiBTFoCEiIkUxaIiISFEMGiIiUpSvpwvobsLCwhATE+PpMoiIepRz586htrbWahuD5ldiYmKg1+s9XQYRUY+i0+lstvHQGRERKYpBQ0REimLQEBGRohg0RESkKAYNEREpikFDRESKYtAQEZGiGDRERKQoBg0RESmKQUNERIqSHTQGgwFbt27F3//+dzz88MMICwuDSqWCSqVCWVmZy4U0NDRg8eLFiI+PR1BQELRaLVJTU7F582ZZ4zdt2oSxY8dCq9UiKCgI8fHxWLx4MQwGg8u1ERGRC4RMX331lQBg9XH69Gm501h18eJFERsbK82n0WiEr6+v9Pr555+3O37WrFlSX19fX6HRaKTXAwcOFBUVFbJrSU5OdmktRES/Rfb+djp06CwiIgKPPPIIlixZgry8PHcFHaZOnYry8nLExMSgsLAQBoMBBoMBK1asgFqtxqpVq7BmzRqr481tarUab731FhobG2EwGFBYWIjo6GicPXsW6enpbqmViIicIDet2traOrwuLy93yx6NeU9JrVaL77//3qL9b3/7mwAg+vXrJ1pbWzu0tbS0iIiICAFAzJ0712Ls8ePHhUqlEgDE1q1bZdXDPRoiIse5ZY/Gx8dHkaDbsGEDAGDcuHFITEy0aH/55ZehUqlQVVWFffv2dWjbs2cPqquroVKpkJWVZTE2KSkJ48aN6/A5RETUtTx+1tn+/fsBAGlpaVbbo6KikJCQAAAWQWMeO3ToUERFRVkdb57312OJiKhreDRoqqurUVdXBwBSmFgzZMgQAMCpU6c6vG9+LWdsTU2NzV9/IyIi5Xg0aCorK6Xn/fv3t9nP3HZ7/9tfyxlrbTwRESnPoz/l3NTUJD0PDAy02S8oKAgA0NjYaHW8nLHWxpvl5eVJZ9HV1NR0UjURETnC49/RdAeZmZnQ6/XQ6/UIDw/3dDlERF7Fo0ETHBwsPTcajTb7NTc3AwA0Go3V8XLGWhtPRETK82jQ3P79yeXLl232M7dFRkZaHS9nrLXxRESkPI8GTXh4OMLCwgAApaWlNvuZzy4zn0FmZn4tZ+ztn0VERF3H49/RpKSkAAB2795ttb2iokIKktTUVKtjS0tLbZ5RtmvXLqtjiYioa3g8aDIyMgC0B8KJEycs2t9++20IIRAZGSkFi1lqaioiIiJgMpmwcuVKi7EnTpzAnj17AAAzZsxQoHoiIuqMQ0FTW1srPa5duya9f/369Q5tJpOpwzjzzwksXbrUYs5HH30UI0aMgMlkwmOPPYYjR44AAFpbW7Fy5Uq8++67AIDXXnsNvXr16jDW399fmvOdd97BypUr0draCgA4fPgwHnvsMZhMJowePRoTJ050ZKlEROQujtw0DTZ+JuDXj/LycqvjlixZYnXezn4m4LnnnrNb1+0/E+Dn58efCSAi6mJu+5kApQwYMADFxcVYuHAh4uLi0NbWhpCQEKSkpODLL7/EqlWr7I7Py8vDF198gZSUFGg0GrS1tSEuLg6LFi1CcXGx3TsHEBGRslRCCOHpIroTnU4HvV7v6TKIiHoUe387u8UeDREReS8GDRERKYpBQ0REimLQEBGRohg0RESkKAYNEREpikFDRESKYtAQEZGiGDRERKQoBg0RESmKQUNERIpi0BARkaIYNEREpCgGDRERKYpBQ0REimLQEBGRohg0RESkKAYNEREpikFDRESKYtAQEZGiGDRERKQoBg0RESmKQUNERIpi0BARkaIYNEREpCgGDRERKYpBQ0REimLQEBGRohg0RESkKAYNEREpikFDRESKcjhoqqqqMGfOHAwaNAgBAQHo27cvJk2ahL179zpVQExMDFQqlazHp59+ajFezrjNmzc7VRsREbnO15HOJ0+exNixY1FXVwcACA0NRW1tLQoKCrB9+3bk5ORg/vz5DhUQHh6OlpYWm+1NTU1obGwEAAwfPtxmv7CwMPj4+FhtCwgIcKgmIiJyIyFTc3OziI6OFgBEUlKSKCkpEUIIUV9fL7KysgQAoVKpxM6dO+VOKcvkyZMFADF8+HCr7QAEAFFeXu6Wz0tOTnbLPEREvyX2/nbKPnS2evVqnD9/HhqNBtu2bUNCQgKA9r2a3NxcTJkyBUIILFiwwG0hWFNTg6+//hoAMHPmTLfNS0REXUd20GzYsAEAkJGRgaioKIv27OxsAMDx48dx5swZtxS3ceNG3Lx5E35+fsjIyHDLnERE1LVkBY3BYMCxY8cAAGlpaVb7jBw5Er179wYAp08M+DXzl/8TJkxAWFiYW+YkIqKuJStoTp8+DSEEAEiHzCwmUqsxePBgAMCpU6dcLuyHH37A999/D0DeYbP09HTccccd8Pf3x4ABA/D4449j+/btLtdBRESukRU0lZWV0vP+/fvb7Gduu72/s9avXw+g/WyyCRMmdNr/6NGjuHXrFvz8/FBRUYH8/HxMnDgR6enpuHHjhsv1EBGRc2QFTVNTk/Q8MDDQZr+goCAAkE5HdlZbW1uH74T8/Pxs9p05cya++eYbXLt2DQ0NDWhsbMTp06fxzDPPAAA2bdqE2bNn2/28vLw86HQ66HQ61NTUuFQ7ERF11C3vDLBz505cuXIFQOeHzdavX4+0tDT06dNHei8uLg7r1q2TTlBYu3at3RMUMjMzodfrodfrER4e7voCiIhIIitogoODpedGo9Fmv+bmZgCARqNxqSjzSQD33HOP3Ys0O7NkyRIEBgZCCIGCggKXaiIiIufICprbv5e5fPmyzX7mtsjISKcLunbtGrZu3QrA9WtngoODMXToUADA2bNnXZqLiIicIyto4uLioFKpAAClpaVW+5hMJunw1JAhQ5wu6PPPP0drayt8fX0xY8YMp+chIqLuQVbQhISEQKfTAQB2795ttU9RURHq6+sBAKmpqU4XZD5slpaWhn79+jk9D9B+EkNJSQkAIDY21qW5iIjIObJPBjBfmb9hwwarpy/n5uYCAJKTk6XraRx15swZFBUVAZB32Mx8bY8tr7/+OoxGI1QqFR555BGnaiIiItfIDppnn30W0dHRMBgMmDhxonRRpsFgwCuvvIL8/HwAQE5OjsVY8+36ly5davczzHszd9xxByZPntxpTenp6Vi0aBH0en2Ha2XOnDmDWbNm4c033wTQHlquHM4jIiLnyf6ZgMDAQGzZsgWpqak4fvw4EhISEBoaisbGRphMJqhUKuTk5GD8+PFOFWIymfCvf/0LAPDkk0/C39+/0zE1NTXYvHkzcnJy4OPjg969e6O1tbXDdT9Tp07FRx995FRNRETkOod+j2bYsGEoKSnB8uXLUVBQgIqKCmi1Wtx3332YO3euS9/N7Nu3D5cuXQIg/2yzhQsX4t5778WRI0dw6dIlXL16FWq1GrGxsRg5ciSefvppp4OPiIjcQyU6+6LjN0an00Gv13u6DCKiHsXe385ueWcAIiLyHgwaIiJSFIOGiIgUxaAhIiJFMWiIiEhRDBoiIlIUg4aIiBTFoCEiIkUxaIiISFEMGiIiUhSDhoiIFMWgISIiRTFoiIhIUQwaIiJSFIOGiIgUxaAhIiJFMWiIiEhRDBoiIlIUg4aIiBTFoCEiIkUxaIiISFEMGiIiUhSDhoiIFMWgISIiRTFoiIhIUQwaIiJSFIOGiIgUxaAhIiJFMWiIiEhRDBoiIlKUw0FTVVWFOXPmYNCgQQgICEDfvn0xadIk7N2716kCDhw4AJVK1emjtrbW7jybNm3C2LFjodVqERQUhPj4eCxevBgGg8GpuoiIyD18Hel88uRJjB07FnV1dQCA0NBQ1NbWoqCgANu3b0dOTg7mz5/vVCFqtRrh4eF2223JzMzEmjVrAAC+vr4ICAhAWVkZli1bhs8++wzffvst+vfv71RdRETkGtl7NEajEZMnT0ZdXR2SkpJQUlKC+vp6XLt2DVlZWRBCYOHChdi1a5dThdx5552oqqqy+fjd735nddyqVauwZs0aqNVqvPXWW2hsbITBYEBhYSGio6Nx9uxZpKenO1UTERG5TnbQrF69GufPn4dGo8G2bduQkJAAoH2vJjc3F1OmTIEQAgsWLFCs2F9rbW3F0qVLAQBz5szByy+/DH9/fwDAqFGj8NVXX0GlUqGwsBDbtm3rsrqIiOgXsoNmw4YNAICMjAxERUVZtGdnZwMAjh8/jjNnzripPPv27NmD6upqqFQqZGVlWbQnJSVh3LhxAH6pn4iIupasoDEYDDh27BgAIC0tzWqfkSNHonfv3gDg9IkBjtq/fz8AYOjQoVbDD/il3n379nVJTURE1JGsoDl9+jSEEAAgHTKzmEitxuDBgwEAp06dcriQmpoaDB8+HMHBwQgODsbdd9+NzMxM/PDDDzbHmD/HVk0AMGTIEGn+zs5cIyIi95MVNJWVldJze2dvmdtu7y9Xc3Mzvv/+e/j7+6OtrQ0//fQT1qxZg6SkJOTm5tqtS05NztZFRESukRU0TU1N0vPAwECb/YKCggAAjY2Nsgvo06cPsrOzodfrYTQacfXqVTQ3N+PgwYMYNWoUbt26hezsbGzcuNFmXXJqsldXXl4edDoddDodampqZNdORESd8/idARITE7FixQokJycjICAAAODj44MxY8Zg//79GD16NABg3rx5MJlMitSQmZkJvV4PvV5v91oeIiJynKygCQ4Olp4bjUab/ZqbmwEAGo3GxbLa9erVC6+//joA4NKlS/j++++t1iWnJnfWRURE8skKmtu/57h8+bLNfua2yMhIF8v6xYgRI6TnZ8+etVqXnJrcXRcREckjK2ji4uKgUqkAAKWlpVb7mEwm6foZ85leSjN/jq2agF/OTAsPD0dYWFiX1EVERL+QFTQhISHQ6XQAgN27d1vtU1RUhPr6egBAamqqm8prn9csNja2Q1tKSgqA9qCxdUaZ+ZY47qyJiIjkk30yQEZGBoD2K+yt/VE3n4KcnJwsXU8jh/n6HGtu3ryJV199FUD7Ya/hw4d3aE9NTUVERARMJhNWrlxpMf7EiRPYs2cPAGDGjBmyayIiIjcSMjU3N4vo6GgBQAwfPlyUlpYKIYRoaGgQ2dnZAoAAIHbu3Gkx1ty2ZMkSi7YhQ4aI999/X/z444/CZDIJIYRoa2sT3377rXjggQeksevXr7da14cffigACLVaLXJzc0VLS4sQQohDhw6J2NhYAUCMHj1a7jJFcnKy7L5ERNTO3t9O2UEjhBDFxcVCq9VKf/xDQ0OFWq0WAIRKpRLLly+3/iF2gsbcBkD4+/uLsLAw0atXL+k9X19f8cYbb9ita9asWVJ/Pz8/odFopNcDBw4UFRUVstfIoCEicpy9v50OXUczbNgwlJSU4KWXXsLAgQPR2toKrVaLCRMmYPfu3U79Fs3q1avx1FNPISEhAaGhobh+/Tr8/f1xzz33YPbs2Thx4gTmzZtnd468vDx88cUXSElJgUajQVtbG+Li4rBo0SIUFxfzt2iIiDxIJYSdL0l+g3Q6HfR6vafLICLqUez97fT4nQGIiMi7MWiIiEhRDBoiIlIUg4aIiBTFoCEiIkUxaIiISFEMGiIiUhSDhoiIFMWgISIiRTFoiIhIUQwaIiJSFIOGiIgUxaAhIiJFMWiIiEhRDBoiIlIUg4aIiBTFoCEiIkUxaIiISFEMGiIiUhSDhoiIFMWgISIiRTFoiIhIUQwaIiJSFIOGiIgUxaAhIiJFMWiIiEhRDBoiIlIUg4aIiBTFoCEiIkUxaIiISFEMGiIiUpTDQVNVVYU5c+Zg0KBBCAgIQN++fTFp0iTs3bvXqQJqamqwevVqTJs2TZozODgY8fHxmD17Nn7++We741UqVaePzZs3O1UbERG5zteRzidPnsTYsWNRV1cHAAgNDUVtbS0KCgqwfft25OTkYP78+Q4V0L9/f7S1tUmvNRoNbty4gbKyMpSVleHjjz/GunXrMH36dLvzhIWFwcfHx2pbQECAQzUREZH7yN6jMRqNmDx5Murq6pCUlISSkhLU19fj2rVryMrKghACCxcuxK5duxwqoK2tDWPGjMGnn36KyspKGAwGNDc347vvvkNiYiJaWlrw1FNP4eTJk3bnOXr0KKqqqqw+Jk6c6FBNRETkPrKDZvXq1Th//jw0Gg22bduGhIQEAO17Nbm5uZgyZQqEEFiwYIFDBRw8eBAHDx7EU089hX79+gEAfHx8MHr0aOzatQsRERFoa2vDO++849C8RETUPcgOmg0bNgAAMjIyEBUVZdGenZ0NADh+/DjOnDkju4AxY8bYbAsPD8cjjzwCADh27JjsOYmIqPuQFTQGg0H6Q5+Wlma1z8iRI9G7d28AcPrEAGu0Wi0A4NatW26bk4iIuo6soDl9+jSEEAAgHTKzmEitxuDBgwEAp06dclN57YfWAGDo0KF2+6Wnp+OOO+6Av78/BgwYgMcffxzbt293Wx1EROQcWUFTWVkpPe/fv7/Nfua22/u7YsuWLdDr9QCAZ555xm7fo0eP4tatW/Dz80NFRQXy8/MxceJEpKen48aNG3bH5uXlQafTQafToaamxi21ExFRO1lB09TUJD0PDAy02S8oKAgA0NjY6GJZQEVFBTIzMwEAkydPxkMPPWS138yZM/HNN9/g2rVraGhoQGNjI06fPi0F06ZNmzB79my7n5WZmQm9Xg+9Xo/w8HCXayciol90yzsDNDY2YsqUKaiurkZ0dDQ+/vhjm33Xr1+PtLQ09OnTR3ovLi4O69atk05QWLt2rUMnKBARkfvICprg4GDpudFotNmvubkZQPtFl85qaWnBo48+Ku1d7Ny5E2FhYU7NtWTJEgQGBkIIgYKCAqdrIiIi58kKmtu/l7l8+bLNfua2yMhIp4q5ceMGpk6din379qFPnz7YtWuXdIKBM4KDg6WTCM6ePev0PERE5DxZQRMXFweVSgUAKC0ttdrHZDJJh6eGDBnicCFtbW2YPn06tm/fDo1Ggx07diAxMdHheYiIqHuRFTQhISHQ6XQAgN27d1vtU1RUhPr6egBAamqqQ0WYTCbMnDkT+fn5CAwMxNatW3H//fc7NIc1TU1NKCkpAQDExsa6PB8RETlO9skAGRkZANrvEGDt9OXc3FwAQHJyskOHu4QQyMzMxMaNG9GrVy/k5+cjJSVF9lh7Xn/9dRiNRqhUKukOA0RE1LVkB82zzz6L6OhoGAwGTJw4Uboo02Aw4JVXXkF+fj4AICcnx2Ks+Xb9S5cutWibO3cuPv74Y/j6+uLLL7+0eRqzNenp6Vi0aBH0en2Ha2XOnDmDWbNm4c033wTQfgq0M4fziIjIdbJ/JiAwMBBbtmxBamoqjh8/joSEBISGhqKxsREmkwkqlQo5OTkYP3687A+/cOEC3nvvPQDtYfTss8/i2Weftdm/qqqqw+uamhps3rwZOTk58PHxQe/evdHa2trhup+pU6fio48+kl0TERG5l0O/RzNs2DCUlJRg+fLlKCgoQEVFBbRaLe677z7MnTvXqe9mzG7evIkrV644NH7hwoW49957ceTIEVy6dAlXr16FWq1GbGwsRo4ciaefftqh4CMiIvdTic6+6PiN0el00m1viIhIHnt/O7vlnQGIiMh7MGiIiEhRDBoiIlIUg4aIiBTFoCEiIkU5dHoz2Rczv2f8oue5NybI7utta/K29QDetyZvWw/gnWtyBPdoiIhIUQwaIiJSFIOGiIgUxaAhIiJFMWiIiEhRDBoiIlIUg4aIiBTFoCEiIkUxaIiISFEMGiIiUhSDhoiIFMWgISIiRTFoiIhIUQwaIiJSFIOGiIgUxaAhIiJFMWiIiEhRDBoiIlIUg4aIiBTFoCEiIkUxaIiISFEMGiIiUhSDhoiIFMWgISIiRTkcNFVVVZgzZw4GDRqEgIAA9O3bF5MmTcLevXtdKqShoQGLFy9GfHw8goKCoNVqkZqais2bN8sav2nTJowdOxZarRZBQUGIj4/H4sWLYTAYXKqLiIhc41DQnDx5EkOHDsX777+Ps2fPwt/fH7W1tSgoKMCDDz6IN954w6kiLl26hMTERCxbtgxlZWXw8fFBQ0MD9u3bh2nTpuGFF16wOz4zMxPp6enYv38/Ghoa4OPjg7KyMixbtgyJiYm4fPmyU3UREZHrZAeN0WjE5MmTUVdXh6SkJJSUlKC+vh7Xrl1DVlYWhBBYuHAhdu3a5VABQghMnToV5eXliImJQWFhIQwGAwwGA1asWAG1Wo1Vq1ZhzZo1Vseb29RqNd566y00NjbCYDCgsLAQ0dHROHv2LNLT0x2qiYiI3Ed20KxevRrnz5+HRqPBtm3bkJCQAAAIDQ1Fbm4upkyZAiEEFixY4FABW7ZsQVFREdRqNb766iuMGjUKABAQEIDs7Gy89NJLAIBXX30VN27c6DC2tbUVS5cuBQDMmTMHL7/8Mvz9/QEAo0aNwldffQWVSoXCwkJs27bNobqIiMg9ZAfNhg0bAAAZGRmIioqyaM/OzgYAHD9+HGfOnJFdgHnecePGITEx0aL95ZdfhkqlQlVVFfbt29ehbc+ePaiuroZKpUJWVpbF2KSkJIwbN67D5xARUdeSFTQGgwHHjh0DAKSlpVntM3LkSPTu3RsAHDoxYP/+/XbnjYqKkvaefh005rFDhw61Gn63z/vrsURE1DVkBc3p06chhAAA6Y++xURqNQYPHgwAOHXqlKwPr66uRl1dnd15AWDIkCFW5zW/ljO2pqYGtbW1suoiIiL3kRU0lZWV0vP+/fvb7Gduu72/kvOaX8sZ60hdRETkPr5yOjU1NUnPAwMDbfYLCgoCADQ2Nsr6cFfnNY+XM9ZeXXl5ecjLywMAlJWVQafTdVK5dWFOjbKvpqYG4eHhbp1Tp1siu6+3rcnb1gN435q8bT2Ad67p186dO2ezTVbQeLvMzExkZmZ6ugyrdDod9Hq9p8twK29bk7etB/C+NXnbeoCetSZZh86Cg4Ol50aj0Wa/5uZmAIBGo5H14a7Oax4vZ6wjdRERkfvICprbv+ewd5W9uS0yMlLWh7s6r3m8nLGO1EVERO4jK2ji4uKgUqkAAKWlpVb7mEwm6foZ85lenQkPD0dYWJjdeYFfzi779bzm13LG3v5ZPUl3PaTnCm9bk7etB/C+NXnbeoAetiYh0x/+8AcBQDz33HNW2w8dOiQACACirKxM7rRi2rRpAoB46KGHrLZfunRJqFQqAUB88803Hdq2b98uAAi1Wi0uX75sdfz48eMFAPHkk0/KromIiNxH9p0BMjIyALRfYW/tNOHc3FwAQHJysnQ9jSPz7tq1CydOnLBof/vttyGEQGRkJFJSUjq0paamIiIiAiaTCStXrrQYe+LECezZswcAMGPGDNk1ERGRG8lNpObmZhEdHS0AiOHDh4vS0lIhhBANDQ0iOztb2pvZuXOnxVhz25IlSyzaTCaTGDFihAAgYmNjxeHDh4UQQrS0tIjc3FyhVqsFAJGXl2e1rg8//FDaq8nNzRUtLS1CiPY9rNjYWAFAjB49Wu4yiYjIzWQHjRBCFBcXC61WKwVHaGioFAQqlUosX77c+ofYCRohhLh48aIUCgCERqMRvr6+0mtbh+vMZs2aJfX18/MTGo1Gej1w4EBRUVHhyDKJiMiNHLqOZtiwYSgpKcHy5ctRUFCAiooKaLVa3HfffZg7dy5SU1Od2qsaMGAAiouL8eabbyI/Px/nzp1DSEgIEhMT8fzzz2PatGl2x+fl5WHcuHH46KOPUFxcDKPRiLi4ODz++OOYN28eQkJCnKqrqzU3N2Pbtm34+uuvUVJSggsXLsBgMODmzZsIDg6GVqvFXXfdhfvvvx9TpkyxehPS7sab1vTzzz+jqKgIer0excXFqKysRG1tLerr6xEcHIz+/fsjOTkZU6dOxcSJE+Hj4+PpkjtVV1cHvV6Po0ePSg9rh8b/9Kc/4cCBA11foIO8cRuZNTY2Yt26ddixYwdOnjyJuro6aDQaREVFYfz48Xj66acxdOhQT5dpnaeTjtp99tlnol+/ftKemJzHww8/LCorKz1duk3etqZBgwbJXsewYcPEyZMnPV2yXRs3bpS9nj/96U+eLlcWb9tGZlu3bu1wNMnaw9fXVyxcuFCYTCZPl2uBdwboBtauXYtZs2Y5PO7rr7/G6NGjcezYMfTp08f9hbnAG9dkFhUVheHDh2PgwIHQarXw8/NDXV0djh8/joMHD+LWrVs4ceIExowZg0OHDiE+Pt7TJVvV2trq6RIU4y3bCAA2b96M9PR06cbG4eHhmDx5MmJjY2E0GlFUVIS9e/eira0NOTk5aGpqwrvvvuvZon/N00n3W3f9+nURHBzs0L/6f/1YvHixp5fRgTeuSQgh8vPzxfnz5+32OXPmjEhISJDWkZKS0kXVOe6TTz7xuj0ab9tGVVVVHb5zzsjIEI2NjRb9CgsLRXh4uNSvoKDAA9XaxqDxMFuHLyIiIsTnn38uqqqqhMFgEMeOHRMTJkyw2nfEiBGeXkYH3rgmR5SUlEjrUKlUoqqqytMlWfXJJ5+IgIAAMWLECPHXv/5V/POf/xRlZWU9Omjk6inbaNGiRVKdSUlJ4ubNmzb77ty5U+obFxfXrQ6hMWg8bPny5Vb/x16/fr1FX6PRKAICAiz6Dh482AOV2+aNa3JURESEtJYjR454uhyrDAaDuHHjhsX7v4WgEaJnbKPhw4dLNa5bt67T/vfcc4/U/9ChQ11QoTyyL9gkZdj6iQPzLX86ew8A7r77brfW5CpvXJMjbt682eEnKW7/qYruRKPRwM/Pz9NleERP2Ub/+9//pOfJycmd9r+9z+bNmxWpyRkMGg/74x//aPX9+fPn49///jeuXr2KlpYWnDx5Ek888QRaWlo69FOpVJgzZ05XlCqbN67JER988IF01/CIiAjZ9/6jrtNTttHtd6a397tb1vocO3ZMkZqc4uldKhJi4sSJTn1h3qtXL/Hhhx96unyrvHFNtty6dUvU1dWJffv2iZkzZ3ZYz5o1azxdnsOsbZeefuisp26jyMhIqc49e/Z02v/27zy1Wm0XVCgPg6YbaGhosPmluLVHnz59xIsvvih++uknT5dukzeu6Xbz5s2zu57AwEDx6aeferpMp3hL0HjDNpo0aZJU7+zZs+32vXr1aocz1ADYPXmgK/HQWTcQEhKCgoIC7NixQ9Zx2Pr6epw6dQrnz5/vguqc441rkuuJJ57A+fPn8dRTT3m6FLKhp2yjJ554Qnqel5eHwsJCq/1MJhNefPFFi5+rb2hoULQ+uVRC/P+rgMijdu/ejSVLluDw4cMOjcvOzsaKFSsUqso13rgms2+//RZFRUUA2r9Yrq6uxuHDh6X3YmJisHLlSvz5z3/2ZJlOsXaCRk+5Bc3tvGEb3bp1CyNGjJC+bwkMDMS8efMwffp0xMTEwGg04r///S9ycnJw4MABBAQEdPjO8+LFixgwYICnyv+Fp3epSIj/+7//s9itv+OOO8SqVatEeXm5uHr1qjh48KAYNWqU1UMAH3zwgaeXYMEb1yRHUVFRh9ug9MR1WNsePfHQmS09bRtdvHhR3H333Z0efg4KChJr167t8F59fb2nyxdC8Dsaj9u9e7fV/2h27Nhh0bepqUnExMRY9A0PDxetra0eqN46b1yTI86dOyfdGcHPz6/HfO9k5u1BI0TP20YNDQ0iKytLhISE2Nw+p0+fFkeOHJHe8/Hx6TYXbfI7Gg97++23Ld6LjIzEww8/bPF+UFAQpk+fbvF+TU0N9Hq9IvU5wxvX5Ijo6Gj85S9/AdB+yGb9+vWeLYgs9LRtFBISgtzcXNTU1ODgwYP4+OOP8d5772Hjxo346aefcODAAcTFxXW47mbQoEE2r1PraryppocdOXLE4r0777zTZn9bbRcuXMCoUaPcVpcrvHFNjrr33nul59Z+OZY8ryduI39/f4wZMwZjxoyx2l5cXCw9v//++7uoqs5xj8bDmpqaLN67ePGizf622vz9/d1Wk6u8cU2Oamtrk57funXLg5WQLd64jbZs2SI9f+ihhzxYSUcMGg+LiIiweK+yshI7d+60eN9oNOLzzz+3Oo+9PYau5o1rctTRo0el593irB+y4G3baMeOHfjxxx8BAH379u1WZ9MxaDzM1u1aZsyYgby8PJw/fx7Xr1/Hd999h7S0NJSXl1v01Wq1SEpKUrpU2bxxTY4oLS3Fl19+Kb0eP368B6uxTQiBtrY2i4erfXuCnrKN5Lpy5Qpmz54tvV60aBF69erlwYp+xdNnI/zW/ec//5F99bytx7Jlyzy9jA68cU3/+Mc/xGuvvSYuXrxot19BQUGHXxWNj4/vtmfP7d+/3+XtVF5e7ullSLxxGwnRXu+BAwdsnkF25MgRERcXJ60nJSWl25xtZsaTATzsgQcewMKFC5GTk+PU+AcffBBZWVlurso13rimuro6vPfee1i6dCmGDRuGxMREDBgwABqNBkajEefOncO3336Ls2fPSmPCwsLw2Wefda9/WXoxb91Ghw8fxrJly9C/f3+MGTMGd911FwIDA1FdXY3CwkLo9Xrp1zdHjBiB/Pz8bnO2mcTTSUft1qxZ0+EX8jp7+Pr6iqysLGE0Gj1duk3etKa5c+c69C/9cePGdftrM7xtj8Ybt5EQHX/8zNbDx8dHvPDCC8JgMHi6XKt4C5pupKmpCQUFBfjmm2/www8/4MKFCzAYDGhra4NGo0FYWBjuuecejBkzBk888QQiIyM9XXKnvGVNbW1tOHToEA4cOICjR4/ixx9/RGVlJZqamhAQEIA+ffpg8ODBuO+++zBt2jRZ93fztAMHDiAlJcWlOcrLyxETE+OeglzkjdsIACoqKrB9+3bs378fJSUluHLlCq5fvw6tVovf//73SEtLw/Tp0xEfH+/pUm1i0BARkaJ41hkRESmKQUNERIpi0BARkaIYNEREpCgGDRERKYpBQ0REimLQEBGRohg0RESkKAYNEREpikFDRESKYtAQEZGiGDRERKSo/we7VQqrryHdWgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZoAAAENCAYAAAA2ZaOYAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAAmrklEQVR4nO3df1RUdf4/8OcMIL8G1B1AAVtATwliCcKq6WYHMan8kW1KiXuy/hD74eZ6iDR107aPWIaVnU4m/sh2j/ZDD60Klr91C5UcFQ0UqxV/ISSgwgADivP+/sF3buL84M4Ml4Hp+ThnzpmZ9/t95/U+t3h677zvHZUQQoCIiEghalcXQERE7o1BQ0REimLQEBGRohg0RESkKAYNEREpytPVBXQ1QUFBiIyMdHUZRETdyvnz51FdXW2xjUFzl8jISOh0OleXQUTUrSQmJlpt46kzIiJSFIOGiIgUxaAhIiJFMWiIiEhRDBoiIlIUg4aIiBTFoCEiIkUxaIiISFEMGiIiUhSDhoiIFCU7aPR6PbZt24Z//OMfeOyxxxAUFASVSgWVSoXS0lKnC6mrq8OiRYsQExMDPz8/aLVaJCcnY8uWLbLGb968GWPGjIFWq4Wfnx9iYmKwaNEi6PV6p2sjIiInCJm+/vprAcDi48yZM3I3Y9GlS5dEVFSUtD2NRiM8PT2l1y+++KLN8TNnzpT6enp6Co1GI73u37+/KC8vl11LQkKCU3MhIvo9svW3065TZyEhIXj88cexePFi5OTkdFTQYcqUKSgrK0NkZCQKCgqg1+uh1+uxfPlyqNVqrFq1CmvWrLE43tSmVqvx7rvvor6+Hnq9HgUFBYiIiMC5c+eQmpraIbUSEZED5KZVS0tLm9dlZWUdckRjOlJSq9XixIkTZu1///vfBQDRt29f0dzc3KatqalJhISECABi7ty5ZmOPHz8uVCqVACC2bdsmqx4e0RAR2a9Djmg8PDwUCbqNGzcCAMaOHYu4uDiz9ldffRUqlQqVlZXYt29fm7Y9e/bg6tWrUKlUyMjIMBsbHx+PsWPHtvkcIiLqXC5fdbZ//34AQEpKisX28PBwxMbGAoBZ0JjGDh48GOHh4RbHm7Z791giIuocLg2aq1evoqamBgCkMLFk0KBBAIDTp0+3ed/0Ws7Yqqoqq7/+RkREynFp0FRUVEjPw8LCrPYztd3Z/87XcsZaGk9ERMpz6U85NzQ0SM99fX2t9vPz8wMA1NfXWxwvZ6yl8SY5OTnSKrqqqqp2qiYiInu4/DuariA9PR06nQ46nQ7BwcGuLoeIyK24NGj8/f2l5waDwWq/xsZGAIBGo7E4Xs5YS+OJiEh5Lg2aO78/uXLlitV+prbQ0FCL4+WMtTSeiIiU59KgCQ4ORlBQEACgpKTEaj/T6jLTCjIT02s5Y+/8LCIi6jwu/44mKSkJALB7926L7eXl5VKQJCcnWxxbUlJidUXZrl27LI4lIqLO4fKgSUtLA9AaCCdPnjRrf++99yCEQGhoqBQsJsnJyQgJCYHRaMSKFSvMxp48eRJ79uwBAEyfPl2B6omIqD12BU11dbX0uH79uvT+jRs32rQZjcY240w/J7BkyRKzbT7xxBMYPnw4jEYjnnzySRw5cgQA0NzcjBUrVuCDDz4AALz55pvo0aNHm7He3t7SNt9//32sWLECzc3NAIDDhw/jySefhNFoxKhRozBhwgR7pkpERB3FnpumwcrPBNz9KCsrszhu8eLFFrfb3s8EvPDCCzbruvNnAry8vPgzAUREnazDfiZAKf369UNRUREWLFiA6OhotLS0ICAgAElJSfjqq6+watUqm+NzcnLw5ZdfIikpCRqNBi0tLYiOjsbChQtRVFRk884BRESkLJUQQri6iK4kMTEROp3O1WUQEXUrtv52dokjGiIicl8MGiIiUhSDhoiIFMWgISIiRTFoiIhIUQwaIiJSFIOGiIgUxaAhIiJFMWiIiEhRDBoiIlIUg4aIiBTFoCEiIkUxaIiISFEMGiIiUhSDhoiIFMWgISIiRTFoiIhIUQwaIiJSFIOGiIgUxaAhIiJFMWiIiEhRDBoiIlIUg4aIiBTFoCEiIkUxaIiISFEMGiIiUhSDhoiIFMWgISIiRTFoiIhIUQwaIiJSFIOGiIgUZXfQVFZWYs6cORgwYAB8fHzQp08fTJw4EXv37nWogMjISKhUKlmPzz77zGy8nHFbtmxxqDYiInKepz2dT506hTFjxqCmpgYAEBgYiOrqauTl5SE/Px9ZWVmYP3++XQUEBwejqanJantDQwPq6+sBAEOHDrXaLygoCB4eHhbbfHx87KqJiIg6kJCpsbFRRERECAAiPj5eFBcXCyGEqK2tFRkZGQKAUKlUYufOnXI3KcukSZMEADF06FCL7QAEAFFWVtYhn5eQkNAh2yEi+j2x9bdT9qmz1atX48KFC9BoNNi+fTtiY2MBtB7VZGdnY/LkyRBC4PXXX++wEKyqqsI333wDAJgxY0aHbZeIiDqP7KDZuHEjACAtLQ3h4eFm7ZmZmQCA48eP4+zZsx1S3KZNm3Dr1i14eXkhLS2tQ7ZJRESdS1bQ6PV6HDt2DACQkpJisc+IESPQs2dPAHB4YcDdTF/+jx8/HkFBQR2yTSIi6lyygubMmTMQQgCAdMrMbENqNQYOHAgAOH36tNOF/fjjjzhx4gQAeafNUlNT0bt3b3h7e6Nfv3546qmnkJ+f73QdRETkHFlBU1FRIT0PCwuz2s/Udmd/R23YsAFA62qy8ePHt9v/6NGjuH37Nry8vFBeXo7c3FxMmDABqampuHnzptP1EBGRY2QFTUNDg/Tc19fXaj8/Pz8AkJYjO6qlpaXNd0JeXl5W+86YMQPffvstrl+/jrq6OtTX1+PMmTN4/vnnAQCbN2/G7NmzbX5eTk4OEhMTkZiYiKqqKqdqJyKitrrknQF27tyJX3/9FUD7p802bNiAlJQU9OrVS3ovOjoa69evlxYorF271uYChfT0dOh0Ouh0OgQHBzs/ASIiksgKGn9/f+m5wWCw2q+xsREAoNFonCrKtAjg/vvvt3mRZnsWL14MX19fCCGQl5fnVE1EROQYWUFz5/cyV65csdrP1BYaGupwQdevX8e2bdsAOH/tjL+/PwYPHgwAOHfunFPbIiIix8gKmujoaKhUKgBASUmJxT5Go1E6PTVo0CCHC/riiy/Q3NwMT09PTJ8+3eHtEBFR1yAraAICApCYmAgA2L17t8U+hYWFqK2tBQAkJyc7XJDptFlKSgr69u3r8HaA1kUMxcXFAICoqCintkVERI6RvRjAdGX+xo0bLS5fzs7OBgAkJCRI19PY6+zZsygsLAQg77SZ6doea9566y0YDAaoVCo8/vjjDtVERETOkR00s2bNQkREBPR6PSZMmCBdlKnX6/Haa68hNzcXAJCVlWU21nS7/iVLltj8DNPRTO/evTFp0qR2a0pNTcXChQuh0+naXCtz9uxZzJw5E++88w6A1tBy5nQeERE5TvbPBPj6+mLr1q1ITk7G8ePHERsbi8DAQNTX18NoNEKlUiErKwvjxo1zqBCj0Yh///vfAIBnnnkG3t7e7Y6pqqrCli1bkJWVBQ8PD/Ts2RPNzc1trvuZMmUKPvnkE4dqIiIi59n1ezRDhgxBcXExli1bhry8PJSXl0Or1WLYsGGYO3euU9/N7Nu3D5cvXwYgf7XZggUL8MADD+DIkSO4fPkyrl27BrVajaioKIwYMQLPPfecw8FHREQdQyXa+6LjdyYxMRE6nc7VZRARdSu2/nZ2yTsDEBGR+2DQEBGRohg0RESkKAYNEREpikFDRESKYtAQEZGiGDRERKQoBg0RESmKQUNERIpi0BARkaIYNEREpCgGDRERKYpBQ0REimLQEBGRohg0RESkKAYNEREpikFDRESKYtAQEZGiGDRERKQoBg0RESmKQUNERIpi0BARkaIYNEREpCgGDRERKYpBQ0REimLQEBGRohg0RESkKAYNEREpikFDRESKYtAQEZGi7A6ayspKzJkzBwMGDICPjw/69OmDiRMnYu/evQ4VcODAAahUqnYf1dXVNrezefNmjBkzBlqtFn5+foiJicGiRYug1+sdqouIiDqGpz2dT506hTFjxqCmpgYAEBgYiOrqauTl5SE/Px9ZWVmYP3++Q4Wo1WoEBwfbbLcmPT0da9asAQB4enrCx8cHpaWlWLp0KT7//HN89913CAsLc6guIiJyjuwjGoPBgEmTJqGmpgbx8fEoLi5GbW0trl+/joyMDAghsGDBAuzatcuhQu655x5UVlZaffzhD3+wOG7VqlVYs2YN1Go13n33XdTX10Ov16OgoAARERE4d+4cUlNTHaqJiIicJztoVq9ejQsXLkCj0WD79u2IjY0F0HpUk52djcmTJ0MIgddff12xYu/W3NyMJUuWAADmzJmDV199Fd7e3gCAkSNH4uuvv4ZKpUJBQQG2b9/eaXUREdFvZAfNxo0bAQBpaWkIDw83a8/MzAQAHD9+HGfPnu2g8mzbs2cPrl69CpVKhYyMDLP2+Ph4jB07FsBv9RMRUeeSFTR6vR7Hjh0DAKSkpFjsM2LECPTs2RMAHF4YYK/9+/cDAAYPHmwx/IDf6t23b1+n1ERERG3JCpozZ85ACAEA0ikzsw2p1Rg4cCAA4PTp03YXUlVVhaFDh8Lf3x/+/v647777kJ6ejh9//NHqGNPnWKsJAAYNGiRtv72Va0RE1PFkBU1FRYX03NbqLVPbnf3lamxsxIkTJ+Dt7Y2Wlhb8/PPPWLNmDeLj45GdnW2zLjk1OVoXERE5R1bQNDQ0SM99fX2t9vPz8wMA1NfXyy6gV69eyMzMhE6ng8FgwLVr19DY2IiDBw9i5MiRuH37NjIzM7Fp0yardcmpyVZdOTk5SExMRGJiIqqqqmTXTkRE7XP5nQHi4uKwfPlyJCQkwMfHBwDg4eGB0aNHY//+/Rg1ahQAYN68eTAajYrUkJ6eDp1OB51OZ/NaHiIisp+soPH395eeGwwGq/0aGxsBABqNxsmyWvXo0QNvvfUWAODy5cs4ceKExbrk1NSRdRERkXyygubO7zmuXLlitZ+pLTQ01MmyfjN8+HDp+blz5yzWJaemjq6LiIjkkRU00dHRUKlUAICSkhKLfYxGo3T9jGmll9JMn2OtJuC3lWnBwcEICgrqlLqIiOg3soImICAAiYmJAIDdu3db7FNYWIja2loAQHJycgeV17pdk6ioqDZtSUlJAFqDxtqKMtMtcTqyJiIikk/2YoC0tDQArVfYW/qjblqCnJCQIF1PI4fp+hxLbt26hTfeeANA62mvoUOHtmlPTk5GSEgIjEYjVqxYYTb+5MmT2LNnDwBg+vTpsmsiIqIOJGRqbGwUERERAoAYOnSoKCkpEUIIUVdXJzIzMwUAAUDs3LnTbKypbfHixWZtgwYNEh9++KH46aefhNFoFEII0dLSIr777jvx0EMPSWM3bNhgsa6PP/5YABBqtVpkZ2eLpqYmIYQQhw4dElFRUQKAGDVqlNxpioSEBNl9iYiola2/nbKDRgghioqKhFarlf74BwYGCrVaLQAIlUolli1bZvlDbASNqQ2A8Pb2FkFBQaJHjx7Se56enuLtt9+2WdfMmTOl/l5eXkKj0Uiv+/fvL8rLy2XPkUFDRGQ/W3877bqOZsiQISguLsYrr7yC/v37o7m5GVqtFuPHj8fu3bsd+i2a1atX49lnn0VsbCwCAwNx48YNeHt74/7778fs2bNx8uRJzJs3z+Y2cnJy8OWXXyIpKQkajQYtLS2Ijo7GwoULUVRUxN+iISJyIZUQNr4k+R1KTEyETqdzdRlERN2Krb+dLr8zABERuTcGDRERKYpBQ0REimLQEBGRohg0RESkKAYNEREpikFDRESKYtAQEZGiGDRERKQoBg0RESmKQUNERIpi0BARkaIYNEREpCgGDRERKYpBQ0REimLQEBGRohg0RESkKAYNEREpikFDRESKYtAQEZGiGDRERKQoBg0RESmKQUNERIrydHUB7iRyfr6rS5Dl/NvjZfd1xzkRUefiEQ0RESmKRzRE3RyPOqmr4xENEREpikFDRESKYtAQEZGiGDRERKQoBg0RESnK7qCprKzEnDlzMGDAAPj4+KBPnz6YOHEi9u7d61ABVVVVWL16NaZOnSpt09/fHzExMZg9ezZ++eUXm+NVKlW7jy1btjhUGxEROc+u5c2nTp3CmDFjUFNTAwAIDAxEdXU18vLykJ+fj6ysLMyfP9+uAsLCwtDS0iK91mg0uHnzJkpLS1FaWop169Zh/fr1mDZtms3tBAUFwcPDw2Kbj4+PXTUREVHHkX1EYzAYMGnSJNTU1CA+Ph7FxcWora3F9evXkZGRASEEFixYgF27dtlVQEtLC0aPHo3PPvsMFRUV0Ov1aGxsxPfff4+4uDg0NTXh2WefxalTp2xu5+jRo6isrLT4mDBhgl01ERFRx5EdNKtXr8aFCxeg0Wiwfft2xMbGAmg9qsnOzsbkyZMhhMDrr79uVwEHDx7EwYMH8eyzz6Jv374AAA8PD4waNQq7du1CSEgIWlpa8P7779u1XSIi6hpkB83GjRsBAGlpaQgPDzdrz8zMBAAcP34cZ8+elV3A6NGjrbYFBwfj8ccfBwAcO3ZM9jaJiKjrkBU0er1e+kOfkpJisc+IESPQs2dPAHB4YYAlWq0WAHD79u0O2yYREXUeWUFz5swZCCEAQDplZrYhtRoDBw4EAJw+fbqDyms9tQYAgwcPttkvNTUVvXv3hre3N/r164ennnoK+fnd4x5QRETuTFbQVFRUSM/DwsKs9jO13dnfGVu3boVOpwMAPP/88zb7Hj16FLdv34aXlxfKy8uRm5uLCRMmIDU1FTdv3rQ5NicnB4mJiUhMTERVVVWH1E5ERK1kBU1DQ4P03NfX12o/Pz8/AEB9fb2TZQHl5eVIT08HAEyaNAmPPvqoxX4zZszAt99+i+vXr6Ourg719fU4c+aMFEybN2/G7NmzbX5Weno6dDoddDodgoODna6diIh+0yXvDFBfX4/Jkyfj6tWriIiIwLp166z23bBhA1JSUtCrVy/pvejoaKxfv15aoLB27Vq7FigQEVHHkRU0/v7+0nODwWC1X2NjI4DWiy4d1dTUhCeeeEI6uti5cyeCgoIc2tbixYvh6+sLIQTy8vIcromIiBwnK2ju/F7mypUrVvuZ2kJDQx0q5ubNm5gyZQr27duHXr16YdeuXdICA0f4+/tLiwjOnTvn8HaIiMhxsoImOjoaKpUKAFBSUmKxj9FolE5PDRo0yO5CWlpaMG3aNOTn50Oj0WDHjh2Ii4uzeztERNS1yAqagIAAJCYmAgB2795tsU9hYSFqa2sBAMnJyXYVYTQaMWPGDOTm5sLX1xfbtm3Dgw8+aNc2LGloaEBxcTEAICoqyuntERGR/WQvBkhLSwPQeocAS8uXs7OzAQAJCQl2ne4SQiA9PR2bNm1Cjx49kJubi6SkJNljbXnrrbdgMBigUqmkOwwQEVHnkh00s2bNQkREBPR6PSZMmCBdlKnX6/Haa68hNzcXAJCVlWU21nS7/iVLlpi1zZ07F+vWrYOnpye++uorq8uYLUlNTcXChQuh0+naXCtz9uxZzJw5E++88w6A1iXQjpzOIyIi58n+mQBfX19s3boVycnJOH78OGJjYxEYGIj6+noYjUaoVCpkZWVh3Lhxsj/84sWLWLlyJYDWMJo1axZmzZpltX9lZWWb11VVVdiyZQuysrLg4eGBnj17orm5uc11P1OmTMEnn3wiuyYiIupYdv0ezZAhQ1BcXIxly5YhLy8P5eXl0Gq1GDZsGObOnevQdzMmt27dwq+//mrX+AULFuCBBx7AkSNHcPnyZVy7dg1qtRpRUVEYMWIEnnvuObuCj4iIOp5dQQMAffv2xcqVK6UjETmsfZcSGRnZ7vcstowbN45BQkTUxXXJOwMQEZH7YNAQEZGiGDRERKQoBg0RESmKQUNERIpi0BARkaIYNEREpCgGDRERKYpBQ0REimLQEBGRohg0RESkKAYNEREpikFDRESKYtAQEZGiGDRERKQoBg0RESmKQUNERIpi0BARkaIYNEREpCgGDRERKYpBQ0REimLQEBGRohg0RESkKAYNEREpikFDRESKYtAQEZGiGDRERKQoBg0RESmKQUNERIpi0BARkaLsDprKykrMmTMHAwYMgI+PD/r06YOJEydi7969ThVSV1eHRYsWISYmBn5+ftBqtUhOTsaWLVtkjd+8eTPGjBkDrVYLPz8/xMTEYNGiRdDr9U7VRUREzrEraE6dOoXBgwfjww8/xLlz5+Dt7Y3q6mrk5eXhkUcewdtvv+1QEZcvX0ZcXByWLl2K0tJSeHh4oK6uDvv27cPUqVPx0ksv2Ryfnp6O1NRU7N+/H3V1dfDw8EBpaSmWLl2KuLg4XLlyxaG6iIjIebKDxmAwYNKkSaipqUF8fDyKi4tRW1uL69evIyMjA0IILFiwALt27bKrACEEpkyZgrKyMkRGRqKgoAB6vR56vR7Lly+HWq3GqlWrsGbNGovjTW1qtRrvvvsu6uvrodfrUVBQgIiICJw7dw6pqal21URERB1HdtCsXr0aFy5cgEajwfbt2xEbGwsACAwMRHZ2NiZPngwhBF5//XW7Cti6dSsKCwuhVqvx9ddfY+TIkQAAHx8fZGZm4pVXXgEAvPHGG7h582absc3NzViyZAkAYM6cOXj11Vfh7e0NABg5ciS+/vprqFQqFBQUYPv27XbVRUREHUN20GzcuBEAkJaWhvDwcLP2zMxMAMDx48dx9uxZ2QWYtjt27FjExcWZtb/66qtQqVSorKzEvn372rTt2bMHV69ehUqlQkZGhtnY+Ph4jB07ts3nEBFR55IVNHq9HseOHQMApKSkWOwzYsQI9OzZEwDsWhiwf/9+m9sNDw+Xjp7uDhrT2MGDB1sMvzu3e/dYIiLqHJ5yOp05cwZCCACQ/ujfTa1WY+DAgfjhhx9w+vRpWR9+9epV1NTU2NwuAAwaNAjFxcVm2zW9bm8sAFRVVaG6uhpBQUGyaiP3FDk/39UlyHL+7fGuLsFl3HEfueOc7CHriKaiokJ6HhYWZrWfqe3O/kpu1/Razlh76iIioo4j64imoaFBeu7r62u1n5+fHwCgvr5e1oc7u13TeDljbdWVk5ODnJwcAEBpaSkSExPbqdwyJY6VqqqqEBwc3KHbTExcLLuvu83J3eYDuN+c3G0+gHvO6W7nz5+32iYraNxdeno60tPTXV2GRYmJidDpdK4uo0O525zcbT6A+83J3eYDdK85yTp15u/vLz03GAxW+zU2NgIANBqNrA93drum8XLG2lMXERF1HFlBc+f3HLausje1hYaGyvpwZ7drGi9nrD11ERFRx5EVNNHR0VCpVACAkpISi32MRqN0/YxppVd7goODpVVg1rYL/La67O7tml7LGXvnZ3UnXfWUnjPcbU7uNh/A/ebkbvMButmchEx/+tOfBADxwgsvWGw/dOiQACAAiNLSUrmbFVOnThUAxKOPPmqx/fLly0KlUgkA4ttvv23Tlp+fLwAItVotrly5YnH8uHHjBADxzDPPyK6JiIg6juw7A6SlpQFovcLe0jLh7OxsAEBCQgIGDhwoO+hM2921axdOnjxp1v7ee+9BCIHQ0FAkJSW1aUtOTkZISAiMRiNWrFhhNvbkyZPYs2cPAGD69OmyayIiog4kN5EaGxtFRESEACCGDh0qSkpKhBBC1NXViczMTOloZufOnWZjTW2LFy82azMajWL48OECgIiKihKHDx8WQgjR1NQksrOzhVqtFgBETk6Oxbo+/vhj6agmOztbNDU1CSFaj7CioqIEADFq1Ci50yQiog4mO2iEEKKoqEhotVopOAIDA6UgUKlUYtmyZZY/xEbQCCHEpUuXpFAAIDQajfD09JReWztdZzJz5kypr5eXl9BoNNLr/v37i/LycnumSUREHciu62iGDBmC4uJiLFu2DHl5eSgvL4dWq8WwYcMwd+5cJCcnO3RU1a9fPxQVFeGdd95Bbm4uzp8/j4CAAMTFxeHFF1/E1KlTbY7PycnB2LFj8cknn6CoqAgGgwHR0dF46qmnMG/ePAQEBDhUV2drbGzE9u3b8c0336C4uBgXL16EXq/HrVu34O/vD61Wi3vvvRcPPvggJk+ebPEmpF2NO83pl19+QWFhIXQ6HYqKilBRUYHq6mrU1tbC398fYWFhSEhIwJQpUzBhwgR4eHi4uuR21dTUQKfT4ejRo9LD0qnxhx9+GAcOHOj8Au3kjvvIpL6+HuvXr8eOHTtw6tQp1NTUQKPRIDw8HOPGjcNzzz2HwYMHu7pMy1yddNTq888/F3379pWOxOQ8HnvsMVFRUeHq0q1ytzkNGDBA9jyGDBkiTp065eqSbdq0aZPs+Tz88MOuLlcWd9tHJtu2bWtzNsnSw9PTUyxYsEAYjUZXl2uGdwboAtauXYuZM2faPe6bb77BqFGjcOzYMfTq1avjC3OCO87JJDw8HEOHDkX//v2h1Wrh5eWFmpoaHD9+HAcPHsTt27dx8uRJjB49GocOHUJMTIyrS7aoubnZ1SUoxl32EQBs2bIFqamp0o2Ng4ODMWnSJERFRcFgMKCwsBB79+5FS0sLsrKy0NDQgA8++MC1Rd/N1Un3e3fjxg3h7+9v17/6734sWrTI1dNowx3nJIQQubm54sKFCzb7nD17VsTGxkrzSEpK6qTq7Pfpp5+63RGNu+2jysrKNt85p6Wlifr6erN+BQUFIjg4WOqXl5fngmqtY9C4mLXTFyEhIeKLL74QlZWVQq/Xi2PHjonx48db7Dt8+HBXT6MNd5yTPYqLi6V5qFQqUVlZ6eqSLPr000+Fj4+PGD58uPjb3/4m/vWvf4nS0tJuHTRydZd9tHDhQqnO+Ph4cevWLat9d+7cKfWNjo7uUqfQGDQutmzZMov/Y2/YsMGsr8FgED4+PmZ9Bw4c6ILKrXPHOdkrJCREmsuRI0dcXY5Fer1e3Lx50+z930PQCNE99tHQoUOlGtevX99u//vvv1/qf+jQoU6oUB7ZF2ySMqz9xIHplj/tvQcA9913X4fW5Cx3nJM9bt261eYnKe78qYquRKPRwMvLy9VluER32Uf/+9//pOcJCQnt9r+zz5YtWxSpyREMGhf785//bPH9+fPn4z//+Q+uXbuGpqYmnDp1Ck8//TSampra9FOpVJgzZ05nlCqbO87JHh999JF01/CQkBDZ9/6jztNd9tGdd6a39btblvocO3ZMkZoc4upDKhJiwoQJDn1h3qNHD/Hxxx+7unyL3HFO1ty+fVvU1NSIffv2iRkzZrSZz5o1a1xdnt0s7Zfufuqsu+6j0NBQqc49e/a02//O7zy1Wm0nVCgPg6YLqKurs/qluKVHr169xMsvvyx+/vlnV5dulTvO6U7z5s2zOR9fX1/x2WefubpMh7hL0LjDPpo4caJU7+zZs232vXbtWpsVagBsLh7oTDx11gUEBAQgLy8PO3bskHUetra2FqdPn8aFCxc6oTrHuOOc5Hr66adx4cIFPPvss64uhazoLvvo6aeflp7n5OSgoKDAYj+j0YiXX37Z7Ofq6+rqFK1PLpUQ//8qIHKp3bt3Y/HixTh8+LBd4zIzM7F8+XKFqnKOO87J5LvvvkNhYSGA1i+Wr169isOHD0vvRUZGYsWKFfjLX/7iyjIdYmmBRne5Bc2d3GEf3b59G8OHD5e+b/H19cW8efMwbdo0REZGwmAw4IcffkBWVhYOHDgAHx+fNt95Xrp0Cf369XNV+b9x9SEVCfF///d/Zof1vXv3FqtWrRJlZWXi2rVr4uDBg2LkyJEWTwF89NFHrp6CGXeckxyFhYVtboPSHedhaX90x1Nn1nS3fXTp0iVx3333tXv62c/PT6xdu7bNe7W1ta4uXwjB72hcbvfu3Rb/o9mxY4dZ34aGBhEZGWnWNzg4WDQ3N7ugesvccU72OH/+vHRnBC8vr27zvZOJuweNEN1vH9XV1YmMjAwREBBgdf+cOXNGHDlyRHrPw8Ojy1y0ye9oXOy9994zey80NBSPPfaY2ft+fn6YNm2a2ftVVVXQ6XSK1OcId5yTPSIiIvDXv/4VQOspmw0bNri2IDLT3fZRQEAAsrOzUVVVhYMHD2LdunVYuXIlNm3ahJ9//hkHDhxAdHR0m+tuBgwYYPU6tc7Gm2q62JEjR8zeu+eee6z2t9Z28eJFjBw5ssPqcoY7zsleDzzwgPTc0i/Hkut1x33k7e2N0aNHY/To0Rbbi4qKpOcPPvhgJ1XVPh7RuFhDQ4PZe5cuXbLa31qbt7d3h9XkLHeck71aWlqk57dv33ZhJWSNO+6jrVu3Ss8fffRRF1bSFoPGxUJCQszeq6iowM6dO83eNxgM+OKLLyxux9YRQ2dzxznZ6+jRo9LzLrHqh8y42z7asWMHfvrpJwBAnz59utRqOgaNi1m7Xcv06dORk5ODCxcu4MaNG/j++++RkpKCsrIys75arRbx8fFKlyqbO87JHiUlJfjqq6+k1+PGjXNhNdYJIdDS0mL2cLZvd9Bd9pFcv/76K2bPni29XrhwIXr06OHCiu7i6tUIv3f//e9/ZV89b+2xdOlSV0+jDXec0z//+U/x5ptvikuXLtnsl5eX1+ZXRWNiYrrs6rn9+/c7vZ/KyspcPQ2JO+4jIVrrPXDggNUVZEeOHBHR0dHSfJKSkrrMajMTLgZwsYceeggLFixAVlaWQ+MfeeQRZGRkdHBVznHHOdXU1GDlypVYsmQJhgwZgri4OPTr1w8ajQYGgwHnz5/Hd999h3PnzkljgoKC8Pnnn3etf1m6MXfdR4cPH8bSpUsRFhaG0aNH495774Wvry+uXr2KgoIC6HQ66dc3hw8fjtzc3C6z2kzi6qSjVmvWrGnzC3ntPTw9PUVGRoYwGAyuLt0qd5rT3Llz7fqX/tixY7v8tRnudkTjjvtIiLY/fmbt4eHhIV566SWh1+tdXa5FvAVNF9LQ0IC8vDx8++23+PHHH3Hx4kXo9Xq0tLRAo9EgKCgI999/P0aPHo2nn34aoaGhri65Xe4yp5aWFhw6dAgHDhzA0aNH8dNPP6GiogINDQ3w8fFBr169MHDgQAwbNgxTp06VdX83Vztw4ACSkpKc2kZZWRkiIyM7piAnueM+AoDy8nLk5+dj//79KC4uxq+//oobN25Aq9Xij3/8I1JSUjBt2jTExMS4ulSrGDRERKQorjojIiJFMWiIiEhRDBoiIlIUg4aIiBTFoCEiIkUxaIiISFEMGiIiUhSDhoiIFMWgISIiRTFoiIhIUQwaIiJSFIOGiIgU9f8Az+gRePMgOmIAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_attribution(\n",
    "    query_ids[0],\n",
    "    input_ids[0,:mask.sum()],\n",
    "    attributions_ours[0][0,:mask.sum()],\n",
    "    attributions_gt[0,:mask.sum()],\n",
    "    name=\"plots/toy-ours-input.pdf\",\n",
    "    ymax=1.1,\n",
    ")\n",
    "\n",
    "plot_attribution(\n",
    "    query_ids[0],\n",
    "    input_ids[0,:mask.sum()],\n",
    "    attributions_ours[1][0,:mask.sum()],\n",
    "    attributions_gt[0,:mask.sum()],\n",
    "    name=\"plots/toy-ours-hidden.pdf\",\n",
    "    ymax=1.1,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZoAAAENCAYAAAA2ZaOYAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAAmm0lEQVR4nO3de1TUdf4/8OcMILcBbbkoYgvoKVE0QVg13ewgJpVmtikl7tH6Q+ziZh4kr5u2fcUyrOx0MvGS7a520aVVwbxf1lBZR0UDxWpFVAQFVBhgQHHevz/4zSdxLnxmmA8D0/NxzpwzM+/3+zOv9/kUTz93lRBCgIiISCFqZxdARESujUFDRESKYtAQEZGiGDRERKQoBg0RESnK3dkFdDSBgYEIDw93dhlERJ3KxYsXUVlZabaNQXOf8PBwaLVaZ5dBRNSpxMXFWWzjrjMiIlIUg4aIiBTFoCEiIkUxaIiISFEMGiIiUhSDhoiIFMWgISIiRTFoiIhIUQwaIiJSFIOGiIgUJTtodDodtm3bhr/+9a946qmnEBgYCJVKBZVKhaKiojYXUlNTg0WLFqFfv37w8fFBQEAAEhISsGXLFlnjN2/ejFGjRiEgIAA+Pj7o168fFi1aBJ1O1+baiIioDYRM3333nQBg9nXu3Dm5izHr8uXLIiIiQlqeRqMR7u7u0udXX33V6vjp06dLfd3d3YVGo5E+9+7dW5SWlsquJTY2tk1zISL6LbL2t9OmXWfBwcF4+umnsXjxYmRmZjoq6DBx4kQUFxcjPDwcubm50Ol00Ol0WL58OdRqNVatWoU1a9aYHW9sU6vV+OCDD1BbWwudTofc3FyEhYXhwoULSEpKckitRERkB7lp1dTU1OJzcXGxQ7ZojFtKarVanDp1yqT9zTffFABEjx49RGNjY4u2hoYGERwcLACI2bNnm4w9efKkUKlUAoDYtm2brHq4RUNEZDuHbNG4ubkpEnQbN24EAIwePRrR0dEm7XPmzIFKpUJ5eTn279/fom3v3r24fv06VCoVUlNTTcbGxMRg9OjRLX6HiIjal9PPOjtw4AAAIDEx0Wx7aGgooqKiAMAkaIxjBwwYgNDQULPjjcu9fywREbUPpwbN9evXUVVVBQBSmJjTv39/AMDZs2dbfG/8LGdsRUWFxae/ERGRcpwaNGVlZdL7nj17WuxnbLu3/72f5Yw1N56IiJTn1Ec519XVSe+9vb0t9vPx8QEA1NbWmh0vZ6y58UaZmZnSWXQVFRWtVE1ERLZw+jGajiAlJQVarRZarRZBQUHOLoeIyKU4NWh8fX2l93q93mK/+vp6AIBGozE7Xs5Yc+OJiEh5Tg2ae4+fXL161WI/Y1tISIjZ8XLGmhtPRETKc2rQBAUFITAwEABQWFhosZ/x7DLjGWRGxs9yxt77W0RE1H6cfowmPj4eALBnzx6z7aWlpVKQJCQkmB1bWFho8Yyy3bt3mx1LRETtw+lBk5ycDKA5EE6fPm3S/uGHH0IIgZCQEClYjBISEhAcHAyDwYAVK1aYjD19+jT27t0LAJgyZYoC1RMRUWtsCprKykrpdfPmTen7W7dutWgzGAwtxhkfJ7BkyRKTZT777LMYOnQoDAYDnnvuORw7dgwA0NjYiBUrVuDjjz8GALzzzjvo0qVLi7Genp7SMj/66COsWLECjY2NAICjR4/iueeeg8FgwIgRIzBu3DhbpkpERI5iy03TYOExAfe/iouLzY5bvHix2eW29piAV155xWpd9z4mwMPDg48JICJqZw57TIBSevXqhfz8fCxYsACRkZFoamqCn58f4uPj8e2332LVqlVWx2dmZuKbb75BfHw8NBoNmpqaEBkZiYULFyI/P9/qnQOIiEhZKiGEcHYRHUlcXBy0Wq2zyyAi6lSs/e3sEFs0RETkuhg0RESkKAYNEREpikFDRESKYtAQEZGiGDRERKQoBg0RESmKQUNERIpi0BARkaIYNEREpCgGDRERKYpBQ0REimLQEBGRohg0RESkKAYNEREpikFDRESKYtAQEZGiGDRERKQoBg0RESmKQUNERIpi0BARkaIYNEREpCgGDRERKYpBQ0REimLQEBGRohg0RESkKAYNEREpikFDRESKYtAQEZGiGDRERKQoBg0RESnK5qApLy/HrFmz0KdPH3h5eaF79+545plnsG/fPrsKCA8Ph0qlkvX68ssvTcbLGbdlyxa7aiMiorZzt6XzmTNnMGrUKFRVVQEA/P39UVlZiezsbOTk5CA9PR3z5s2zqYCgoCA0NDRYbK+rq0NtbS0AYPDgwRb7BQYGws3NzWybl5eXTTUREZEDCZnq6+tFWFiYACBiYmJEQUGBEEKI6upqkZqaKgAIlUoldu3aJXeRsowfP14AEIMHDzbbDkAAEMXFxQ75vdjYWIcsh4jot8Ta307Zu85Wr16NkpISaDQabN++HVFRUQCat2oyMjIwYcIECCEwf/58h4VgRUUFvv/+ewDAtGnTHLZcIiJqP7KDZuPGjQCA5ORkhIaGmrSnpaUBAE6ePInz5887pLhNmzbhzp078PDwQHJyskOWSURE7UtW0Oh0Opw4cQIAkJiYaLbPsGHD0LVrVwCw+8SA+xkP/o8dOxaBgYEOWSYREbUvWUFz7tw5CCEAQNplZrIgtRp9+/YFAJw9e7bNhf344484deoUAHm7zZKSkvDAAw/A09MTvXr1wvPPP4+cnJw210FERG0jK2jKysqk9z179rTYz9h2b397bdiwAUDz2WRjx45ttf/x48dx9+5deHh4oLS0FFlZWRg3bhySkpJw+/btNtdDRET2kRU0dXV10ntvb2+L/Xx8fABAOh3ZXk1NTS2OCXl4eFjsO23aNOzcuRM3b95ETU0Namtrce7cObz88ssAgM2bN2PmzJlWfy8zMxNxcXGIi4tDRUVFm2onIqKWOuSdAXbt2oVr164BaH232YYNG5CYmIhu3bpJ30VGRmL9+vXSCQpr1661eoJCSkoKtFottFotgoKC2j4BIiKSyAoaX19f6b1er7fYr76+HgCg0WjaVJTxJICBAwdavUizNYsXL4a3tzeEEMjOzm5TTUREZB9ZQXPvcZmrV69a7GdsCwkJsbugmzdvYtu2bQDafu2Mr68vBgwYAAC4cOFCm5ZFRET2kRU0kZGRUKlUAIDCwkKzfQwGg7R7qn///nYX9PXXX6OxsRHu7u6YMmWK3cshIqKOQVbQ+Pn5IS4uDgCwZ88es33y8vJQXV0NAEhISLC7IONus8TERPTo0cPu5QDNJzEUFBQAACIiItq0LCIiso/skwGMV+Zv3LjR7OnLGRkZAIDY2FjpehpbnT9/Hnl5eQDk7TYzXttjybvvvgu9Xg+VSoWnn37arpqIiKhtZAfNjBkzEBYWBp1Oh3HjxkkXZep0Orz11lvIysoCAKSnp5uMNd6uf8mSJVZ/w7g188ADD2D8+PGt1pSUlISFCxdCq9W2uFbm/PnzmD59Ot5//30AzaHVlt15RERkP9mPCfD29sbWrVuRkJCAkydPIioqCv7+/qitrYXBYIBKpUJ6ejrGjBljVyEGgwH/+Mc/AAAvvvgiPD09Wx1TUVGBLVu2ID09HW5ubujatSsaGxtbXPczceJEfP7553bVREREbWfT82gGDRqEgoICLFu2DNnZ2SgtLUVAQACGDBmC2bNnt+nYzP79+3HlyhUA8s82W7BgAR555BEcO3YMV65cwY0bN6BWqxEREYFhw4bhpZdesjv4iIjIMVSitQMdvzFxcXHQarXOLoOIqFOx9rezQ94ZgIiIXAeDhoiIFMWgISIiRTFoiIhIUQwaIiJSFIOGiIgUxaAhIiJFMWiIiEhRDBoiIlIUg4aIiBTFoCEiIkUxaIiISFEMGiIiUhSDhoiIFMWgISIiRTFoiIhIUQwaIiJSFIOGiIgUxaAhIiJFMWiIiEhRDBoiIlIUg4aIiBTFoCEiIkUxaIiISFEMGiIiUhSDhoiIFMWgISIiRTFoiIhIUQwaIiJSFIOGiIgUZXPQlJeXY9asWejTpw+8vLzQvXt3PPPMM9i3b59dBRw8eBAqlarVV2VlpdXlbN68GaNGjUJAQAB8fHzQr18/LFq0CDqdzq66iIjIMdxt6XzmzBmMGjUKVVVVAAB/f39UVlYiOzsbOTk5SE9Px7x58+wqRK1WIygoyGq7JSkpKVizZg0AwN3dHV5eXigqKsLSpUvx1Vdf4fDhw+jZs6dddRERUdvI3qLR6/UYP348qqqqEBMTg4KCAlRXV+PmzZtITU2FEAILFizA7t277SrkwQcfRHl5ucXX7373O7PjVq1ahTVr1kCtVuODDz5AbW0tdDodcnNzERYWhgsXLiApKcmumoiIqO1kB83q1atRUlICjUaD7du3IyoqCkDzVk1GRgYmTJgAIQTmz5+vWLH3a2xsxJIlSwAAs2bNwpw5c+Dp6QkAGD58OL777juoVCrk5uZi+/bt7VYXERH9SnbQbNy4EQCQnJyM0NBQk/a0tDQAwMmTJ3H+/HkHlWfd3r17cf36dahUKqSmppq0x8TEYPTo0QB+rZ+IiNqXrKDR6XQ4ceIEACAxMdFsn2HDhqFr164AYPeJAbY6cOAAAGDAgAFmww/4td79+/e3S01ERNSSrKA5d+4chBAAIO0yM1mQWo2+ffsCAM6ePWtzIRUVFRg8eDB8fX3h6+uLhx9+GCkpKfjxxx8tjjH+jqWaAKB///7S8ls7c42IiBxPVtCUlZVJ762dvWVsu7e/XPX19Th16hQ8PT3R1NSEn3/+GWvWrEFMTAwyMjKs1iWnJnvrIiKitpEVNHV1ddJ7b29vi/18fHwAALW1tbIL6NatG9LS0qDVaqHX63Hjxg3U19fj0KFDGD58OO7evYu0tDRs2rTJYl1yarJWV2ZmJuLi4hAXF4eKigrZtRMRUeucfmeA6OhoLF++HLGxsfDy8gIAuLm5YeTIkThw4ABGjBgBAJg7dy4MBoMiNaSkpECr1UKr1Vq9loeIiGwnK2h8fX2l93q93mK/+vp6AIBGo2ljWc26dOmCd999FwBw5coVnDp1ymxdcmpyZF1ERCSfrKC59zjH1atXLfYztoWEhLSxrF8NHTpUen/hwgWzdcmpydF1ERGRPLKCJjIyEiqVCgBQWFhoto/BYJCunzGe6aU04+9Yqgn49cy0oKAgBAYGtktdRET0K1lB4+fnh7i4OADAnj17zPbJy8tDdXU1ACAhIcFB5TUv1ygiIqJFW3x8PIDmoLF0RpnxljiOrImIiOSTfTJAcnIygOYr7M39UTeeghwbGytdTyOH8focc+7cuYO3334bQPNur8GDB7doT0hIQHBwMAwGA1asWGEy/vTp09i7dy8AYMqUKbJrIiIiBxIy1dfXi7CwMAFADB48WBQWFgohhKipqRFpaWkCgAAgdu3aZTLW2LZ48WKTtv79+4tPPvlE/PTTT8JgMAghhGhqahKHDx8Wjz32mDR2w4YNZuv67LPPBAChVqtFRkaGaGhoEEIIceTIERERESEAiBEjRsidpoiNjZXdl4iImln72yk7aIQQIj8/XwQEBEh//P39/YVarRYAhEqlEsuWLTP/I1aCxtgGQHh6eorAwEDRpUsX6Tt3d3fx3nvvWa1r+vTpUn8PDw+h0Wikz7179xalpaWy58igISKynbW/nTZdRzNo0CAUFBTgjTfeQO/evdHY2IiAgACMHTsWe/bssetZNKtXr8bUqVMRFRUFf39/3Lp1C56enhg4cCBmzpyJ06dPY+7cuVaXkZmZiW+++Qbx8fHQaDRoampCZGQkFi5ciPz8fD6LhojIiVRCWDlI8hsUFxcHrVbr7DKIiDoVa387nX5nACIicm0MGiIiUhSDhoiIFMWgISIiRTFoiIhIUQwaIiJSFIOGiIgUxaAhIiJFMWiIiEhRDBoiIlIUg4aIiBTFoCEiIkW5O7sAVxI+L8fZJchy8b2xzi6BiH5DuEVDRESKYtAQEZGiGDRERKQoBg0RESmKQUNERIpi0BARkaIYNEREpCgGDRERKYpBQ0REimLQEBGRohg0RESkKAYNEREpikFDRESKYtAQEZGiGDRERKQoBg0RESmKQUNERIpi0BARkaJsDpry8nLMmjULffr0gZeXF7p3745nnnkG+/bts6uAiooKrF69GpMmTZKW6evri379+mHmzJn45ZdfrI5XqVStvrZs2WJXbURE1HbutnQ+c+YMRo0ahaqqKgCAv78/KisrkZ2djZycHKSnp2PevHk2FdCzZ080NTVJnzUaDW7fvo2ioiIUFRVh3bp1WL9+PSZPnmx1OYGBgXBzczPb5uXlZVNNRETkOLK3aPR6PcaPH4+qqirExMSgoKAA1dXVuHnzJlJTUyGEwIIFC7B7926bCmhqasLIkSPx5ZdfoqysDDqdDvX19fjhhx8QHR2NhoYGTJ06FWfOnLG6nOPHj6O8vNzsa9y4cTbVREREjiM7aFavXo2SkhJoNBps374dUVFRAJq3ajIyMjBhwgQIITB//nybCjh06BAOHTqEqVOnokePHgAANzc3jBgxArt370ZwcDCamprw0Ucf2bRcIiLqGGQHzcaNGwEAycnJCA0NNWlPS0sDAJw8eRLnz5+XXcDIkSMttgUFBeHpp58GAJw4cUL2MomIqOOQFTQ6nU76Q5+YmGi2z7Bhw9C1a1cAsPvEAHMCAgIAAHfv3nXYMomIqP3ICppz585BCAEA0i4zkwWp1ejbty8A4OzZsw4qr3nXGgAMGDDAar+kpCQ88MAD8PT0RK9evfD8888jJyfHYXUQEZF9ZJ11VlZWJr3v2bOnxX7Gtnv7t8XWrVuh1WoBAC+//LLVvsePH4efnx88PDxQWlqKrKwsZGVlYdKkSfjnP/+JLl26WBybmZmJzMxMAM2nWxN1JuHzOsc/qC6+N9bZJZCTyNqiqaurk957e3tb7Ofj4wMAqK2tbWNZQGlpKVJSUgAA48ePx5NPPmm237Rp07Bz507cvHkTNTU1qK2txblz56Rg2rx5M2bOnGn1t1JSUqDVaqHVahEUFNTm2omI6Fcd8s4AtbW1mDBhAq5fv46wsDCsW7fOYt8NGzYgMTER3bp1k76LjIzE+vXrpRMU1q5da9MJCkRE5DiygsbX11d6r9frLfarr68H0HzRpb0aGhrw7LPPSlsXu3btQmBgoF3LWrx4Mby9vSGEQHZ2tt01ERGR/WQFzb3HZa5evWqxn7EtJCTErmJu376NiRMnYv/+/ejWrRt2794tnWBgD19fX+kkggsXLti9HCIisp+soImMjIRKpQIAFBYWmu1jMBik3VP9+/e3uZCmpiZMnjwZOTk50Gg02LFjB6Kjo21eDhERdSyygsbPzw9xcXEAgD179pjtk5eXh+rqagBAQkKCTUUYDAZMmzYNWVlZ8Pb2xrZt2/Doo4/atAxz6urqUFBQAACIiIho8/KIiMh2sk8GSE5OBtB8hwBzpy9nZGQAAGJjY23a3SWEQEpKCjZt2oQuXbogKysL8fHxssda8+6770Kv10OlUkl3GCAiovYlO2hmzJiBsLAw6HQ6jBs3TrooU6fT4a233kJWVhYAID093WSs8Xb9S5YsMWmbPXs21q1bB3d3d3z77bcWT2M2JykpCQsXLoRWq8Xt27el78+fP4/p06fj/fffB9B8CrQ9u/OIiKjtZD8mwNvbG1u3bkVCQgJOnjyJqKgo+Pv7o7a2FgaDASqVCunp6RgzZozsH7906RJWrlwJoDmMZsyYgRkzZljsX15e3uJzRUUFtmzZgvT0dLi5uaFr165obGxscd3PxIkT8fnnn8uuiYiIHMum59EMGjQIBQUFWLZsGbKzs1FaWoqAgAAMGTIEs2fPtuvYjNGdO3dw7do1m8YvWLAAjzzyCI4dO4YrV67gxo0bUKvViIiIwLBhw/DSSy/ZFHxEROR4NgUNAPTo0QMrV66UtkTksHQsJTw8vNXjLNaMGTOGQUJE1MF1yDsDEBGR62DQEBGRohg0RESkKJuP0dBvC29BT0RtxS0aIiJSFIOGiIgUxaAhIiJFMWiIiEhRDBoiIlIUg4aIiBTFoCEiIkUxaIiISFEMGiIiUhSDhoiIFMWgISIiRTFoiIhIUQwaIiJSFIOGiIgUxccE0G8KH3tA1P64RUNERIpi0BARkaIYNEREpCgGDRERKYpBQ0REimLQEBGRohg0RESkKAYNEREpikFDRESKYtAQEZGieAsaIupQeJsg12Nz0JSXl2PZsmXIzs5GaWkpunbtiiFDhuDNN99EQkKC3YXU1NRg+fLl+Ne//oWSkhJ4e3sjOjoar776KiZOnNjq+M2bN2PVqlU4ffo09Ho9wsLC8Pzzz2Pu3Lnw8/Ozuy4iorb6rYenTbvOzpw5gwEDBuCTTz7BhQsX4OnpicrKSmRnZ+OJJ57Ae++9Z1cRV65cQXR0NJYuXYqioiK4ubmhpqYG+/fvx6RJk/Daa69ZHZ+SkoKkpCQcOHAANTU1cHNzQ1FREZYuXYro6GhcvXrVrrqIiKjtZAeNXq/H+PHjUVVVhZiYGBQUFKC6uho3b95EamoqhBBYsGABdu/ebVMBQghMnDgRxcXFCA8PR25uLnQ6HXQ6HZYvXw61Wo1Vq1ZhzZo1Zscb29RqNT744APU1tZCp9MhNzcXYWFhuHDhApKSkmyqiYiIHEd20KxevRolJSXQaDTYvn07oqKiAAD+/v7IyMjAhAkTIITA/PnzbSpg69atyMvLg1qtxnfffYfhw4cDALy8vJCWloY33ngDAPD222/j9u3bLcY2NjZiyZIlAIBZs2Zhzpw58PT0BAAMHz4c3333HVQqFXJzc7F9+3ab6iIiIseQHTQbN24EACQnJyM0NNSkPS0tDQBw8uRJnD9/XnYBxuWOHj0a0dHRJu1z5syBSqVCeXk59u/f36Jt7969uH79OlQqFVJTU03GxsTEYPTo0S1+h4iI2pesoNHpdDhx4gQAIDEx0WyfYcOGoWvXrgCAffv2yS7gwIEDVpcbGhoqbT3dHzTGsQMGDDAbfvcu9/6xRETUPmQFzblz5yCEAADpj77JgtRq9O3bFwBw9uxZWT9+/fp1VFVVWV0uAPTv39/sco2f5YytqKhAZWWlrLqIiMhxZAVNWVmZ9L5nz54W+xnb7u2v5HKNn+WMtaUuIiJyHFnX0dTV1Unvvb29Lfbz8fEBANTW1sr68bYu1zhezlhrdWVmZiIzMxMAUFRUhLi4uFYqNy/QrlHWVVRUICgoyKHLjItbLLuvq83J1eYDuN6cXG0+gGvO6X4XL1602MY7A6D5OpyUlBRnl2FWXFwctFqts8twKFebk6vNB3C9ObnafIDONSdZu858fX2l93q93mK/+vp6AIBGo5H1421drnG8nLG21EVERI4jK2juPc5h7Sp7Y1tISIisH2/rco3j5Yy1pS4iInIcWUETGRkJlUoFACgsLDTbx2AwSNfPGM/0ak1QUBACAwOtLhf49eyy+5dr/Cxn7L2/1Zl01F16beFqc3K1+QCuNydXmw/QyeYkZPrDH/4gAIhXXnnFbPuRI0cEAAFAFBUVyV2smDRpkgAgnnzySbPtV65cESqVSgAQO3fubNGWk5MjAAi1Wi2uXr1qdvyYMWMEAPHiiy/KromIiBxH9p0BkpOTATRfYW/uNOGMjAwAQGxsrHQ9jS3L3b17N06fPm3S/uGHH0IIgZCQEMTHx7doS0hIQHBwMAwGA1asWGEy9vTp09i7dy8AYMqUKbJrIiIiB5KbSPX19SIsLEwAEIMHDxaFhYVCCCFqampEWlqatDWza9cuk7HGtsWLF5u0GQwGMXToUAFAREREiKNHjwohhGhoaBAZGRlCrVYLACIzM9NsXZ999pm0VZORkSEaGhqEEM1bWBEREQKAGDFihNxpEhGRg8kOGiGEyM/PFwEBAVJw+Pv7S0GgUqnEsmXLzP+IlaARQojLly9LoQBAaDQa4e7uLn22tLvOaPr06VJfDw8PodFopM+9e/cWpaWltkyTiIgcyKbraAYNGoSCgoIWDz4LCAjAkCFDMHv2bLsffNarVy/k5+fj/fffR1ZWFi5evAg/Pz/pwWeTJk2yOj4zMxOjR4/G559/jvz8fOj1ekRGRna6B5/V19dj+/bt+P7771FQUIBLly5Bp9Phzp078PX1RUBAAB566CE8+uijmDBhgtmbkHY0rjSnX375BXl5edBqtcjPz0dZWRkqKytRXV0NX19f9OzZE7GxsZg4cSLGjRsHNzc3Z5fcqqqqKmi1Whw/flx6mds1/vjjj+PgwYPtX6CNXHEdGdXW1mL9+vXYsWMHzpw5g6qqKmg0GoSGhmLMmDF46aWXMGDAAGeXaZ6zk46affXVV6JHjx7Slpic11NPPSXKysqcXbpFrjanPn36yJ7HoEGDxJkzZ5xdslWbNm2SPZ/HH3/c2eXK4mrryGjbtm0t9iaZe7m7u4sFCxYIg8Hg7HJN8M4AHcDatWsxffp0m8d9//33GDFiBE6cOIFu3bo5vrA2cMU5GYWGhmLw4MHo3bs3AgIC4OHhgaqqKpw8eRKHDh3C3bt3cfr0aYwcORJHjhxBv379nF2yWY2Njc4uQTGuso4AYMuWLUhKSpJubBwUFITx48cjIiICer0eeXl52LdvH5qampCeno66ujp8/PHHzi36fs5Out+6W7duCV9fX5v+1X//a9GiRc6eRguuOCchhMjKyhIlJSVW+5w/f15ERUVJ84iPj2+n6mz3xRdfuNwWjauto/Ly8hbHnJOTk0Vtba1Jv9zcXBEUFCT1y87OdkK1ljFonMzS7ovg4GDx9ddfi/LycqHT6cSJEyfE2LFjzfYdOnSos6fRgivOyRYFBQXSPFQqlSgvL3d2SWZ98cUXwsvLSwwdOlT85S9/EX//+99FUVFRpw4auTrLOlq4cKFUZ0xMjLhz547Fvrt27ZL6RkZGdqhdaAwaJ1u2bJnZ/7E3bNhg0lev1wsvLy+Tvn379nVC5Za54pxsFRwcLM3l2LFjzi7HLJ1OJ27fvm3y/W8haIToHOto8ODBUo3r169vtf/AgQOl/keOHGmHCuWRfcEmKcPSIw6Mt/xp7TsAePjhhx1aU1u54pxscefOnRaPpLj3URUdiUajgYeHh7PLcIrOso7+97//Se9jY2Nb7X9vny1btihSkz0YNE72xz/+0ez38+bNw7///W/cuHEDDQ0NOHPmDF544QU0NDS06KdSqTBr1qz2KFU2V5yTLT799FPpruHBwcGy7/1H7aezrKN770xv7blb5vqcOHFCkZrs4uxNKhJi3Lhxdh0w79Kli/jss8+cXb5ZrjgnS+7evSuqqqrE/v37xbRp01rMZ82aNc4uz2bm1ktn33XWWddRSEiIVOfevXtb7X/vMc+AgIB2qFAeBk0HUFNTY/GguLlXt27dxOuvvy5+/vlnZ5dukSvO6V5z5861Oh9vb2/x5ZdfOrtMu7hK0LjCOnrmmWekemfOnGm1740bN1qcoQbA6skD7Ym7zjoAPz8/ZGdnY8eOHbL2w1ZXV+Ps2bMoKSlph+rs44pzkuuFF15ASUkJpk6d6uxSyILOso5eeOEF6X1mZiZyc3PN9jMYDHj99ddNHldfU1OjaH1yqYT4/1cBkVPt2bMHixcvxtGjR20al5aWhuXLlytUVdu44pyMDh8+jLy8PADNB5avX7+Oo0ePSt+Fh4djxYoV+NOf/uTMMu1i7gSNznILmnu5wjq6e/cuhg4dKh1v8fb2xty5czF58mSEh4dDr9fjv//9L9LT03Hw4EF4eXm1OOZ5+fJl9OrVy1nl/8rZm1QkxP/93/+ZbNY/8MADYtWqVaK4uFjcuHFDHDp0SAwfPtzsLoBPP/3U2VMw4YpzkiMvL6/FbVA64zzMrY/OuOvMks62ji5fviwefvjhVnc/+/j4iLVr17b4rrq62tnlCyF4jMbp9uzZY/Y/mh07dpj0raurE+Hh4SZ9g4KCRGNjoxOqN88V52SLixcvSndG8PDw6DTHnYxcPWiE6HzrqKamRqSmpgo/Pz+L6+fcuXPi2LFj0ndubm4d5qJNHqNxsg8//NDku5CQEDz11FMm3/v4+GDy5Mkm31dUVECr1SpSnz1ccU62CAsLw5///GcAzbtsNmzY4NyCyERnW0d+fn7IyMhARUUFDh06hHXr1mHlypXYtGkTfv75Zxw8eBCRkZEtrrvp06ePxevU2htvqulkx44dM/nuwQcftNjfUtulS5cwfPhwh9XVFq44J1s98sgj0ntzT44l5+uM68jT0xMjR47EyJEjzbbn5+dL7x999NF2qqp13KJxsrq6OpPvLl++bLG/pTZPT0+H1dRWrjgnWzU1NUnv796968RKyBJXXEdbt26V3j/55JNOrKQlBo2TBQcHm3xXVlaGXbt2mXyv1+vx9ddfm12OtS2G9uaKc7LV8ePHpfcd4qwfMuFq62jHjh346aefAADdu3fvUGfTMWiczNLtWqZMmYLMzEyUlJTg1q1b+OGHH5CYmIji4mKTvgEBAYiJiVG6VNlccU62KCwsxLfffit9HjNmjBOrsUwIgaamJpNXW/t2Bp1lHcl17do1zJw5U/q8cOFCdOnSxYkV3cfZZyP81v3nP/+RffW8pdfSpUudPY0WXHFOf/vb38Q777wjLl++bLVfdnZ2i6eK9uvXr8OePXfgwIE2r6fi4mJnT0PiiutIiOZ6Dx48aPEMsmPHjonIyEhpPvHx8R3mbDMjngzgZI899hgWLFiA9PR0u8Y/8cQTSE1NdXBVbeOKc6qqqsLKlSuxZMkSDBo0CNHR0ejVqxc0Gg30ej0uXryIw4cP48KFC9KYwMBAfPXVVx3rX5YuzFXX0dGjR7F06VL07NkTI0eOxEMPPQRvb29cv34dubm50Gq10tM3hw4diqysrA5ztpnE2UlHzdasWdPiCXmtvdzd3UVqaqrQ6/XOLt0iV5rT7NmzbfqX/ujRozv8tRmutkXjiutIiJYPP7P0cnNzE6+99prQ6XTOLtcs3oKmA6mrq0N2djZ27tyJH3/8EZcuXYJOp0NTUxM0Gg0CAwMxcOBAjBw5Ei+88AJCQkKcXXKrXGVOTU1NOHLkCA4ePIjjx4/jp59+QllZGerq6uDl5YVu3bqhb9++GDJkCCZNmiTr/m7OdvDgQcTHx7dpGcXFxQgPD3dMQW3kiusIAEpLS5GTk4MDBw6goKAA165dw61btxAQEIDf//73SExMxOTJk9GvXz9nl2oRg4aIiBTFs86IiEhRDBoiIlIUg4aIiBTFoCEiIkUxaIiISFEMGiIiUhSDhoiIFMWgISIiRTFoiIhIUQwaIiJSFIOGiIgUxaAhIiJF/T8lThapOiwWYQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZoAAAENCAYAAAA2ZaOYAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAAmp0lEQVR4nO3de1TUdf4/8OcMILcBbQdQxBbQU4JYgrBpurkHMam8ZJtS4p6sP6Sbm+sh8rpp21csw26nk4lltnu0ix5aFSzvuoXKOhIaKFYrmiLERYUBBhTm/fuD33wS58JnLh8GpufjnDlnZt7v92de7/Mpnn7uKiGEABERkULU7i6AiIg8G4OGiIgUxaAhIiJFMWiIiEhRDBoiIlKUt7sL6G1CQkIQFRXl7jKIiPqU8+fPo66uzmIbg+YWUVFR0Ol07i6DiKhPSUpKstrGXWdERKQoBg0RESmKQUNERIpi0BARkaIYNEREpCgGDRERKYpBQ0REimLQEBGRohg0RESkKAYNEREpSnbQ6PV67NixA3//+9/x4IMPIiQkBCqVCiqVCuXl5U4X0tjYiOXLlyM2NhYBAQHQarVISUnBtm3bZI3funUrJk6cCK1Wi4CAAMTGxmL58uXQ6/VO10ZERE4QMn355ZcCgMXXmTNn5C7GoosXL4ro6GhpeRqNRnh7e0ufn332WZvj582bJ/X19vYWGo1G+jx06FBRWVkpu5bExESn5kJE9Ftk62+nXbvOwsLC8NBDD2HFihXIzc11VdBh5syZqKioQFRUFAoLC6HX66HX67FmzRqo1WqsW7cOGzZssDje1KZWq/HGG2+gqakJer0ehYWFiIyMxLlz55CWluaSWomIyAFy06q9vb3L54qKCpds0Zi2lNRqtfjuu+/M2v/2t78JAGLQoEGira2tS1tra6sICwsTAMTChQvNxhYXFwuVSiUAiB07dsiqh1s0RET2c8kWjZeXlyJBt3nzZgDApEmTEB8fb9b+4osvQqVSobq6GgcOHOjStm/fPtTU1EClUiEzM9NsbEJCAiZNmtTld4iIqGe5/ayzgwcPAgBSU1MttkdERCAuLg4AzILGNHbkyJGIiIiwON603FvHEhFRz3Br0NTU1KC+vh4ApDCxZMSIEQCA06dPd/ne9FnO2NraWqtPfyMiIuW4NWiqqqqk94MHD7baz9R2c/+bP8sZa2k8EREpz62Pcm5ubpbe+/v7W+0XEBAAAGhqarI4Xs5YS+NNcnNzpbPoamtru6maiIjs4fZjNL1BRkYGdDoddDodQkND3V0OEZFHcWvQBAYGSu8NBoPVfi0tLQAAjUZjcbycsZbGExGR8twaNDcfP7l8+bLVfqa28PBwi+PljLU0noiIlOfWoAkNDUVISAgAoKyszGo/09llpjPITEyf5Yy9+beIiKjnuP0YTXJyMgBg7969FtsrKyulIElJSbE4tqyszOoZZXv27LE4loiIeobbgyY9PR1AZyCcPHnSrP3NN9+EEALh4eFSsJikpKQgLCwMRqMRa9euNRt78uRJ7Nu3DwAwZ84cBaonIqLu2BU0dXV10uvq1avS99euXevSZjQau4wzPU5g5cqVZst8+OGHMWbMGBiNRjzyyCM4duwYAKCtrQ1r167F22+/DQB45ZVX0K9fvy5jfX19pWW+9dZbWLt2Ldra2gAAR48exSOPPAKj0Yjx48dj6tSp9kyViIhcxZ6bpsHKYwJufVVUVFgct2LFCovL7e4xAc8884zNum5+TICPjw8fE0BE1MNc9pgApQwZMgQlJSVYunQpYmJi0N7ejqCgICQnJ+OLL77AunXrbI7Pzc3F559/juTkZGg0GrS3tyMmJgbLli1DSUmJzTsHEBGRslRCCOHuInqTpKQk6HQ6d5dBRNSn2Prb2Su2aIiIyHMxaIiISFEMGiIiUhSDhoiIFMWgISIiRTFoiIhIUQwaIiJSFIOGiIgUxaAhIiJFMWiIiEhRDBoiIlIUg4aIiBTFoCEiIkUxaIiISFEMGiIiUhSDhoiIFMWgISIiRTFoiIhIUQwaIiJSFIOGiIgUxaAhIiJFMWiIiEhRDBoiIlIUg4aIiBTFoCEiIkUxaIiISFEMGiIiUhSDhoiIFMWgISIiRTFoiIhIUQwaIiJSlN1BU11djQULFmDYsGHw8/PDwIEDMW3aNOzfv9+hAqKioqBSqWS9PvnkE7PxcsZt27bNodqIiMh53vZ0PnXqFCZOnIj6+noAQHBwMOrq6pCfn4+CggJkZ2dj8eLFdhUQGhqK1tZWq+3Nzc1oamoCAIwePdpqv5CQEHh5eVls8/Pzs6smIiJyISFTS0uLiIyMFABEQkKCKC0tFUII0dDQIDIzMwUAoVKpxO7du+UuUpbp06cLAGL06NEW2wEIAKKiosIlv5eYmOiS5RAR/ZbY+tspe9fZ+vXrceHCBWg0GuzcuRNxcXEAOrdqcnJyMGPGDAghsGTJEpeFYG1tLb766isAwNy5c122XCIi6jmyg2bz5s0AgPT0dERERJi1Z2VlAQCKi4tx9uxZlxS3ZcsW3LhxAz4+PkhPT3fJMomIqGfJChq9Xo8TJ04AAFJTUy32GTt2LPr37w8ADp8YcCvTwf8pU6YgJCTEJcskIqKeJStozpw5AyEEAEi7zMwWpFZj+PDhAIDTp087Xdj333+P7777DoC83WZpaWm47bbb4OvriyFDhuDRRx9FQUGB03UQEZFzZAVNVVWV9H7w4MFW+5nabu7vqE2bNgHoPJtsypQp3fY/fvw4Ojo64OPjg8rKSuTl5WHq1KlIS0vD9evXna6HiIgcIytompubpff+/v5W+wUEBACAdDqyo9rb27scE/Lx8bHad+7cufj6669x9epVNDY2oqmpCWfOnMFTTz0FANi6dSvmz59v8/dyc3ORlJSEpKQk1NbWOlU7ERF11SvvDLB792788ssvALrfbbZp0yakpqZiwIAB0ncxMTHYuHGjdILChx9+aPMEhYyMDOh0Ouh0OoSGhjo/ASIiksgKmsDAQOm9wWCw2q+lpQUAoNFonCrKdBLAXXfdZfMize6sWLEC/v7+EEIgPz/fqZqIiMgxsoLm5uMyly9fttrP1BYeHu5wQVevXsWOHTsAOH/tTGBgIEaOHAkAOHfunFPLIiIix8gKmpiYGKhUKgBAWVmZxT5Go1HaPTVixAiHC/rss8/Q1tYGb29vzJkzx+HlEBFR7yAraIKCgpCUlAQA2Lt3r8U+RUVFaGhoAACkpKQ4XJBpt1lqaioGDRrk8HKAzpMYSktLAQDR0dFOLYuIiBwj+2QA05X5mzdvtnj6ck5ODgAgMTFRup7GXmfPnkVRUREAebvNTNf2WPPqq6/CYDBApVLhoYcecqgmIiJyjuygefrppxEZGQm9Xo+pU6dKF2Xq9Xq89NJLyMvLAwBkZ2ebjTXdrn/lypU2f8O0NXPbbbdh+vTp3daUlpaGZcuWQafTdblW5uzZs5g3bx5ef/11AJ2h5czuPCIicpzsxwT4+/tj+/btSElJQXFxMeLi4hAcHIympiYYjUaoVCpkZ2dj8uTJDhViNBrxr3/9CwDw+OOPw9fXt9sxtbW12LZtG7Kzs+Hl5YX+/fujra2ty3U/M2fOxAcffOBQTURE5Dy7nkczatQolJaWYvXq1cjPz0dlZSW0Wi3uueceLFy40KljMwcOHMClS5cAyD/bbOnSpbj77rtx7NgxXLp0CVeuXIFarUZ0dDTGjh2LJ5980uHgIyIi11CJ7g50/MYkJSVBp9O5uwwioj7F1t/OXnlnACIi8hwMGiIiUhSDhoiIFMWgISIiRTFoiIhIUQwaIiJSFIOGiIgUxaAhIiJFMWiIiEhRDBoiIlIUg4aIiBTFoCEiIkUxaIiISFEMGiIiUhSDhoiIFMWgISIiRTFoiIhIUQwaIiJSFIOGiIgUxaAhIiJFMWiIiEhRDBoiIlIUg4aIiBTFoCEiIkUxaIiISFEMGiIiUhSDhoiIFMWgISIiRTFoiIhIUQwaIiJSlN1BU11djQULFmDYsGHw8/PDwIEDMW3aNOzfv9+hAg4dOgSVStXtq66uzuZytm7diokTJ0Kr1SIgIACxsbFYvnw59Hq9Q3UREZFreNvT+dSpU5g4cSLq6+sBAMHBwairq0N+fj4KCgqQnZ2NxYsXO1SIWq1GaGiozXZrMjIysGHDBgCAt7c3/Pz8UF5ejlWrVuHTTz/FN998g8GDBztUFxEROUf2Fo3BYMD06dNRX1+PhIQElJaWoqGhAVevXkVmZiaEEFi6dCn27NnjUCG33347qqurrb5+97vfWRy3bt06bNiwAWq1Gm+88Qaampqg1+tRWFiIyMhInDt3DmlpaQ7VREREzpMdNOvXr8eFCxeg0Wiwc+dOxMXFAejcqsnJycGMGTMghMCSJUsUK/ZWbW1tWLlyJQBgwYIFePHFF+Hr6wsAGDduHL788kuoVCoUFhZi586dPVYXERH9SnbQbN68GQCQnp6OiIgIs/asrCwAQHFxMc6ePeui8mzbt28fampqoFKpkJmZadaekJCASZMmAfi1fiIi6lmygkav1+PEiRMAgNTUVIt9xo4di/79+wOAwycG2OvgwYMAgJEjR1oMP+DXeg8cONAjNRERUVeygubMmTMQQgCAtMvMbEFqNYYPHw4AOH36tN2F1NbWYvTo0QgMDERgYCDuvPNOZGRk4Pvvv7c6xvQ71moCgBEjRkjL7+7MNSIicj1ZQVNVVSW9t3X2lqnt5v5ytbS04LvvvoOvry/a29vx448/YsOGDUhISEBOTo7NuuTU5GhdRETkHFlB09zcLL339/e32i8gIAAA0NTUJLuAAQMGICsrCzqdDgaDAVeuXEFLSwsOHz6McePGoaOjA1lZWdiyZYvVuuTUZKuu3NxcJCUlISkpCbW1tbJrJyKi7rn9zgDx8fFYs2YNEhMT4efnBwDw8vLChAkTcPDgQYwfPx4AsGjRIhiNRkVqyMjIgE6ng06ns3ktDxER2U9W0AQGBkrvDQaD1X4tLS0AAI1G42RZnfr164dXX30VAHDp0iV89913FuuSU5Mr6yIiIvlkBc3NxzkuX75stZ+pLTw83MmyfjVmzBjp/blz5yzWJacmV9dFRETyyAqamJgYqFQqAEBZWZnFPkajUbp+xnSml9JMv2OtJuDXM9NCQ0MREhLSI3UREdGvZAVNUFAQkpKSAAB79+612KeoqAgNDQ0AgJSUFBeV17lck+jo6C5tycnJADqDxtoZZaZb4riyJiIikk/2yQDp6ekAOq+wt/RH3XQKcmJionQ9jRym63MsuXHjBl5++WUAnbu9Ro8e3aU9JSUFYWFhMBqNWLt2rdn4kydPYt++fQCAOXPmyK6JiIhcSMjU0tIiIiMjBQAxevRoUVZWJoQQorGxUWRlZQkAAoDYvXu32VhT24oVK8zaRowYId59913xww8/CKPRKIQQor29XXzzzTfivvvuk8Zu2rTJYl3vv/++ACDUarXIyckRra2tQgghjhw5IqKjowUAMX78eLnTFImJibL7EhFRJ1t/O2UHjRBClJSUCK1WK/3xDw4OFmq1WgAQKpVKrF692vKP2AgaUxsA4evrK0JCQkS/fv2k77y9vcVrr71ms6558+ZJ/X18fIRGo5E+Dx06VFRWVsqeI4OGiMh+tv522nUdzahRo1BaWooXXngBQ4cORVtbG7RaLaZMmYK9e/c69Cya9evX44knnkBcXByCg4Nx7do1+Pr64q677sL8+fNx8uRJLFq0yOYycnNz8fnnnyM5ORkajQbt7e2IiYnBsmXLUFJSwmfREBG5kUoIGwdJfoOSkpKg0+ncXQYRUZ9i62+n2+8MQEREno1BQ0REimLQEBGRohg0RESkKAYNEREpikFDRESKYtAQEZGiGDRERKQoBg0RESmKQUNERIpi0BARkaIYNEREpChvdxfgSaIWF7i7BFnOvzbF3SUQ0W8Ig4ZsYngSkbO464yIiBTFoCEiIkUxaIiISFEMGiIiUhSDhoiIFMWgISIiRTFoiIhIUQwaIiJSFIOGiIgUxaAhIiJFMWiIiEhRDBoiIlIUg4aIiBTFoCEiIkUxaIiISFEMGiIiUhSDhoiIFGV30FRXV2PBggUYNmwY/Pz8MHDgQEybNg379+93qIDa2lqsX78es2bNkpYZGBiI2NhYzJ8/Hz/99JPN8SqVqtvXtm3bHKqNiIicZ9ejnE+dOoWJEyeivr4eABAcHIy6ujrk5+ejoKAA2dnZWLx4sV0FDB48GO3t7dJnjUaD69evo7y8HOXl5fjoo4+wceNGzJ492+ZyQkJC4OXlZbHNz8/PrpqIiMh1ZG/RGAwGTJ8+HfX19UhISEBpaSkaGhpw9epVZGZmQgiBpUuXYs+ePXYV0N7ejgkTJuCTTz5BVVUV9Ho9Wlpa8O233yI+Ph6tra144okncOrUKZvLOX78OKqrqy2+pk6daldNRETkOrKDZv369bhw4QI0Gg127tyJuLg4AJ1bNTk5OZgxYwaEEFiyZIldBRw+fBiHDx/GE088gUGDBgEAvLy8MH78eOzZswdhYWFob2/HW2+9ZddyiYiod5AdNJs3bwYApKenIyIiwqw9KysLAFBcXIyzZ8/KLmDChAlW20JDQ/HQQw8BAE6cOCF7mURE1HvIChq9Xi/9oU9NTbXYZ+zYsejfvz8AOHxigCVarRYA0NHR4bJlEhFRz5EVNGfOnIEQAgCkXWZmC1KrMXz4cADA6dOnXVRe5641ABg5cqTNfmlpabjtttvg6+uLIUOG4NFHH0VBQYHL6iAiIsfICpqqqirp/eDBg632M7Xd3N8Z27dvh06nAwA89dRTNvseP34cHR0d8PHxQWVlJfLy8jB16lSkpaXh+vXrNsfm5uYiKSkJSUlJqK2tdUntRETUSVbQNDc3S+/9/f2t9gsICAAANDU1OVkWUFlZiYyMDADA9OnT8cADD1jsN3fuXHz99de4evUqGhsb0dTUhDNnzkjBtHXrVsyfP9/mb2VkZECn00Gn0yE0NNTp2omI6Fe98s4ATU1NmDFjBmpqahAZGYmPPvrIat9NmzYhNTUVAwYMkL6LiYnBxo0bpRMUPvzwQ7tOUCAiIteRFTSBgYHSe4PBYLVfS0sLgM6LLh3V2tqKhx9+WNq62L17N0JCQhxa1ooVK+Dv7w8hBPLz8x2uiYiIHCcraG4+LnP58mWr/Uxt4eHhDhVz/fp1zJw5EwcOHMCAAQOwZ88e6QQDRwQGBkonEZw7d87h5RARkeNk3YImJiYGKpUKQgiUlZVZ/ONvNBql3VMjRoywu5D29nbMnj0bBQUF0Gg02LVrF+Lj4+1eDhFRbxO1uG+cAXv+tSmKLFfWFk1QUBCSkpIAAHv37rXYp6ioCA0NDQCAlJQUu4owGo2YO3cu8vLy4O/vjx07duDee++1axmWNDc3o7S0FAAQHR3t9PKIiMh+sk8GSE9PB9B5hwBLpy/n5OQAABITE+3a3SWEQEZGBrZs2YJ+/fohLy8PycnJssfa8uqrr8JgMEClUkl3GCAiop4lO2iefvppREZGQq/XY+rUqdJFmXq9Hi+99BLy8vIAANnZ2WZjTbfrX7lypVnbwoUL8dFHH8Hb2xtffPGF1dOYLUlLS8OyZcug0+m6XCtz9uxZzJs3D6+//jqAzlOgHdmdR0REzpP9mAB/f39s374dKSkpKC4uRlxcHIKDg9HU1ASj0QiVSoXs7GxMnjxZ9o///PPPeOeddwB0htHTTz+Np59+2mr/6urqLp9ra2uxbds2ZGdnw8vLC/3790dbW1uX635mzpyJDz74QHZNRETkWnY9j2bUqFEoLS3F6tWrkZ+fj8rKSmi1Wtxzzz1YuHChQ8dmTG7cuIFffvnFrvFLly7F3XffjWPHjuHSpUu4cuUK1Go1oqOjMXbsWDz55JN2BR8REbmeXUEDAIMGDcI777wjbYnIYe1YSlRUVLfHWWyZPHkyg4SIqJfrlXcGICIiz8GgISIiRTFoiIhIUQwaIiJSFIOGiIgUxaAhIiJFMWiIiEhRDBoiIlIUg4aIiBTFoCEiIkUxaIiISFEMGiIiUhSDhoiIFMWgISIiRTFoiIhIUQwaIiJSFIOGiIgUxaAhIiJFMWiIiEhRDBoiIlIUg4aIiBTFoCEiIkUxaIiISFEMGiIiUhSDhoiIFMWgISIiRTFoiIhIUQwaIiJSFIOGiIgUxaAhIiJF2R001dXVWLBgAYYNGwY/Pz8MHDgQ06ZNw/79+50qpLGxEcuXL0dsbCwCAgKg1WqRkpKCbdu2yRq/detWTJw4EVqtFgEBAYiNjcXy5cuh1+udqouIiJxjV9CcOnUKI0eOxLvvvotz587B19cXdXV1yM/Px/3334/XXnvNoSIuXbqE+Ph4rFq1CuXl5fDy8kJjYyMOHDiAWbNm4bnnnrM5PiMjA2lpaTh48CAaGxvh5eWF8vJyrFq1CvHx8bh8+bJDdRERkfNkB43BYMD06dNRX1+PhIQElJaWoqGhAVevXkVmZiaEEFi6dCn27NljVwFCCMycORMVFRWIiopCYWEh9Ho99Ho91qxZA7VajXXr1mHDhg0Wx5va1Go13njjDTQ1NUGv16OwsBCRkZE4d+4c0tLS7KqJiIhcR3bQrF+/HhcuXIBGo8HOnTsRFxcHAAgODkZOTg5mzJgBIQSWLFliVwHbt29HUVER1Go1vvzyS4wbNw4A4Ofnh6ysLLzwwgsAgJdffhnXr1/vMratrQ0rV64EACxYsAAvvvgifH19AQDjxo3Dl19+CZVKhcLCQuzcudOuuoiIyDVkB83mzZsBAOnp6YiIiDBrz8rKAgAUFxfj7NmzsgswLXfSpEmIj483a3/xxRehUqlQXV2NAwcOdGnbt28fampqoFKpkJmZaTY2ISEBkyZN6vI7RETUs2QFjV6vx4kTJwAAqampFvuMHTsW/fv3BwC7Tgw4ePCgzeVGRERIW0+3Bo1p7MiRIy2G383LvXUsERH1DFlBc+bMGQghAED6o2+2ILUaw4cPBwCcPn1a1o/X1NSgvr7e5nIBYMSIERaXa/osZ2xtbS3q6upk1UVERK7jLadTVVWV9H7w4MFW+5nabu6v5HJNn+WMNfUPCQmRVRt5pqjFBe4uQZbzr01xdwluw3XkeWQFTXNzs/Te39/far+AgAAAQFNTk6wfd3a5pvFyxtqqKzc3F7m5uQCA8vJyJCUldVO5ZUpEWG1tLUJDQ126zKSkFbL7etqcPG0+SlFiTnJ54jryxDnd6vz581bbZAWNp8vIyEBGRoa7y7AoKSkJOp3O3WW4lKfNydPmA3jenDxtPkDfmpOsYzSBgYHSe4PBYLVfS0sLAECj0cj6cWeXaxovZ6w9dRERkevICpqbj3PYusre1BYeHi7rx51drmm8nLH21EVERK4jK2hiYmKgUqkAAGVlZRb7GI1G6foZ05le3QkNDZUOzltbLvDr2WW3Ltf0Wc7Ym3+rL+mtu/Sc4Wlz8rT5AJ43J0+bD9DH5iRk+sMf/iAAiGeeecZi+5EjRwQAAUCUl5fLXayYNWuWACAeeOABi+2XLl0SKpVKABBff/11l7aCggIBQKjVanH58mWL4ydPniwAiMcff1x2TURE5Dqy7wyQnp4OoPMKe0unL+fk5AAAEhMTpetp7Fnunj17cPLkSbP2N998E0IIhIeHIzk5uUtbSkoKwsLCYDQasXbtWrOxJ0+exL59+wAAc+bMkV0TERG5kNxEamlpEZGRkQKAGD16tCgrKxNCCNHY2CiysrKkrZndu3ebjTW1rVixwqzNaDSKMWPGCAAiOjpaHD16VAghRGtrq8jJyRFqtVoAELm5uRbrev/996WtmpycHNHa2iqE6NzCio6OFgDE+PHj5U6TiIhcTHbQCCFESUmJ0Gq1UnAEBwdLQaBSqcTq1ast/4iNoBFCiIsXL0qhAEBoNBrh7e0tfba2u85k3rx5Ul8fHx+h0Wikz0OHDhWVlZX2TJOIiFzIrutoRo0ahdLSUqxevRr5+fmorKyEVqvFPffcg4ULFyIlJcWhraohQ4agpKQEr7/+OvLy8nD+/HkEBQUhPj4ezz77LGbNmmVzfG5uLiZNmoQPPvgAJSUlMBgMiImJwaOPPopFixYhKCjIobp6WktLC3bu3ImvvvoKpaWl+Pnnn6HX63Hjxg0EBgZCq9XijjvuwL333osZM2ZYvAlpb+NJc/rpp59QVFQEnU6HkpISVFVVoa6uDg0NDQgMDMTgwYORmJiImTNnYurUqfDy8nJ3yd2qr6+HTqfD8ePHpZelXeN/+tOfcOjQoZ4v0E6euI5MmpqasHHjRuzatQunTp1CfX09NBoNIiIiMHnyZDz55JMYOXKku8u0zN1JR50+/fRTMWjQIGlLTM7rwQcfFFVVVe4u3SpPm9OwYcNkz2PUqFHi1KlT7i7Zpi1btsiez5/+9Cd3lyuLp60jkx07dnTZm2Tp5e3tLZYuXSqMRqO7yzXDOwP0Ah9++CHmzZtn97ivvvoK48ePx4kTJzBgwADXF+YET5yTSUREBEaPHo2hQ4dCq9XCx8cH9fX1KC4uxuHDh9HR0YGTJ09iwoQJOHLkCGJjY91dskVtbW3uLkExnrKOAGDbtm1IS0uTbmwcGhqK6dOnIzo6GgaDAUVFRdi/fz/a29uRnZ2N5uZmvP322+4t+lbuTrrfumvXronAwEC7/tV/62v58uXunkYXnjgnIYTIy8sTFy5csNnn7NmzIi4uTppHcnJyD1Vnv48//tjjtmg8bR1VV1d3Oeacnp4umpqazPoVFhaK0NBQqV9+fr4bqrWOQeNm1nZfhIWFic8++0xUV1cLvV4vTpw4IaZMmWKx75gxY9w9jS48cU72KC0tleahUqlEdXW1u0uy6OOPPxZ+fn5izJgx4q9//av45z//KcrLy/t00MjVV9bRsmXLpDoTEhLEjRs3rPbdvXu31DcmJqZX7UJj0LjZ6tWrLf6PvWnTJrO+BoNB+Pn5mfUdPny4Gyq3zhPnZK+wsDBpLseOHXN3ORbp9Xpx/fp1s+9/C0EjRN9YR6NHj5Zq3LhxY7f977rrLqn/kSNHeqBCeWRfsEnKsPaIA9Mtf7r7DgDuvPNOl9bkLE+ckz1u3LjR5ZEUNz+qojfRaDTw8fFxdxlu0VfW0f/+9z/pfWJiYrf9b+6zbds2RWpyBIPGzf74xz9a/H7x4sX497//jStXrqC1tRWnTp3CY489htbW1i79VCoVFixY0BOlyuaJc7LHe++9J901PCwsTPa9/6jn9JV1dPOd6W09d8tSnxMnTihSk0PcvUlFQkydOtWhA+b9+vUT77//vrvLt8gT52RNR0eHqK+vFwcOHBBz587tMp8NGza4uzy7WVovfX3XWV9dR+Hh4VKd+/bt67b/zcc8tVptD1QoD4OmF2hsbLR6UNzSa8CAAeL5558XP/74o7tLt8oT53SzRYsW2ZyPv7+/+OSTT9xdpkM8JWg8YR1NmzZNqnf+/Pk2+165cqXLGWoAbJ480JO466wXCAoKQn5+Pnbt2iVrP2xDQwNOnz6NCxcu9EB1jvHEOcn12GOP4cKFC3jiiSfcXQpZ0VfW0WOPPSa9z83NRWFhocV+RqMRzz//vNnj6hsbGxWtTy6VEP//KiByq71792LFihU4evSoXeOysrKwZs0ahapyjifOyeSbb75BUVERgM4DyzU1NTh69Kj0XVRUFNauXYs///nP7izTIZZO0Ogrt6C5mSeso46ODowZM0Y63uLv749FixZh9uzZiIqKgsFgwH//+19kZ2fj0KFD8PPz63LM8+LFixgyZIi7yv+VuzepSIj/+7//M9usv+2228S6detERUWFuHLlijh8+LAYN26cxV0A7733nrunYMYT5yRHUVFRl9ug9MV5WFoffXHXmTV9bR1dvHhR3Hnnnd3ufg4ICBAffvhhl+8aGhrcXb4Qgsdo3G7v3r0W/6PZtWuXWd/m5mYRFRVl1jc0NFS0tbW5oXrLPHFO9jh//rx0ZwQfH58+c9zJxNODRoi+t44aGxtFZmamCAoKsrp+zpw5I44dOyZ95+Xl1Wsu2uQxGjd78803zb4LDw/Hgw8+aPZ9QEAAZs+ebfZ9bW0tdDqdIvU5whPnZI/IyEj85S9/AdC5y2bTpk3uLYjM9LV1FBQUhJycHNTW1uLw4cP46KOP8M4772DLli348ccfcejQIcTExHS57mbYsGFWr1PrabypppsdO3bM7Lvbb7/dan9rbT///DPGjRvnsrqc4Ylzstfdd98tvbf05Fhyv764jnx9fTFhwgRMmDDBYntJSYn0/t577+2hqrrHLRo3a25uNvvu4sWLVvtba/P19XVZTc7yxDnZq729XXrf0dHhxkrIGk9cR9u3b5feP/DAA26spCsGjZuFhYWZfVdVVYXdu3ebfW8wGPDZZ59ZXI6tLYae5olzstfx48el973irB8y42nraNeuXfjhhx8AAAMHDuxVZ9MxaNzM2u1a5syZg9zcXFy4cAHXrl3Dt99+i9TUVFRUVJj11Wq1SEhIULpU2TxxTvYoKyvDF198IX2ePHmyG6uxTgiB9vZ2s5ezffuCvrKO5Prll18wf/586fOyZcvQr18/N1Z0C3efjfBb95///Ef21fPWXqtWrXL3NLrwxDn94x//EK+88oq4ePGizX75+fldnioaGxvba8+eO3jwoNPrqaKiwt3TkHjiOhKis95Dhw5ZPYPs2LFjIiYmRppPcnJyrznbzIQnA7jZfffdh6VLlyI7O9uh8ffffz8yMzNdXJVzPHFO9fX1eOedd7By5UqMGjUK8fHxGDJkCDQaDQwGA86fP49vvvkG586dk8aEhITg008/7V3/svRgnrqOjh49ilWrVmHw4MGYMGEC7rjjDvj7+6OmpgaFhYXQ6XTS0zfHjBmDvLy8XnO2mcTdSUedNmzY0OUJed29vL29RWZmpjAYDO4u3SpPmtPChQvt+pf+pEmTev21GZ62ReOJ60iIrg8/s/by8vISzz33nNDr9e4u1yLegqYXaW5uRn5+Pr7++mt8//33+Pnnn6HX69He3g6NRoOQkBDcddddmDBhAh577DGEh4e7u+Ruecqc2tvbceTIERw6dAjHjx/HDz/8gKqqKjQ3N8PPzw8DBgzA8OHDcc8992DWrFmy7u/mbocOHUJycrJTy6ioqEBUVJRrCnKSJ64jAKisrERBQQEOHjyI0tJS/PLLL7h27Rq0Wi1+//vfIzU1FbNnz0ZsbKy7S7WKQUNERIriWWdERKQoBg0RESmKQUNERIpi0BARkaIYNEREpCgGDRERKYpBQ0REimLQEBGRohg0RESkKAYNEREpikFDRESKYtAQEZGi/h/WpwwXMxYq4AAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZoAAAENCAYAAAA2ZaOYAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAAmw0lEQVR4nO3deVTU9f4/8OewyDa4XBZF7AvoKUE02W6a3uwgJpVLdlNMvEfrD7HF8nqQNPWGLWIWVnY7mbhkt6MtevCqYO7LNReu4xooVlc0RVBAhQEGFOf9+4PffBJn4TPLh4Hp+ThnzpmZ9/v9mdf7fIqnn10lhBAgIiJSiJuzCyAiItfGoCEiIkUxaIiISFEMGiIiUhSDhoiIFOXh7ALam8DAQISHhzu7DCKiDuXixYuorKw02caguU94eDg0Go2zyyAi6lASEhLMtnHXGRERKYpBQ0REimLQEBGRohg0RESkKAYNEREpikFDRESKYtAQEZGiGDRERKQoBg0RESmKQUNERIqSHTRarRZbtmzBP/7xDzz11FMIDAyESqWCSqVCcXGx3YXU1NRgwYIFiIqKgq+vLwICApCUlISNGzfKGr9hwwYMHz4cAQEB8PX1RVRUFBYsWACtVmt3bUREZAch06ZNmwQAk69z587JXYxJly9fFhEREdLy1Gq18PDwkD6//PLLFsdPmzZN6uvh4SHUarX0uXfv3qK0tFR2LfHx8XbNhYjoj8jS306rdp0FBwfj6aefRmZmJnJychwVdBg/fjxKSkoQHh6OQ4cOQavVQqvV4oMPPoCbmxuWL1+OlStXmhxvaHNzc8OHH36I2tpaaLVaHDp0CGFhYbhw4QJSUlIcUisREdlAblo1NTW1+FxSUuKQLRrDlpKbm5s4efKkUfvf//53AUD06NFDNDY2tmhraGgQwcHBAoCYNWuW0dgTJ04IlUolAIgtW7bIqodbNERE1nPIFo27u7siQbdu3ToAwIgRIxATE2PUPnv2bKhUKpSXl2Pv3r0t2nbv3o3r169DpVIhPT3daGxsbCxGjBjR4neIiKhtOf2ss3379gEAkpOTTbaHhoYiOjoaAIyCxjC2f//+CA0NNTnesNz7xxIRUdtwatBcv34dVVVVACCFiSn9+vUDAJw9e7bF94bPcsZWVFSYffobEREpx6lBU1ZWJr3v2bOn2X6Gtnv73/tZzlhT44mISHlOfZRzXV2d9N7Hx8dsP19fXwBAbW2tyfFyxpoab5CTkyOdRVdRUdFK1UREZA2nH6NpD9LS0qDRaKDRaBAUFOTscoiIXIpTg8bPz096r9PpzParr68HAKjVapPj5Yw1NZ6IiJTn1KC59/jJ1atXzfYztIWEhJgcL2esqfFERKQ8pwZNUFAQAgMDAQBFRUVm+xnOLjOcQWZg+Cxn7L2/RUREbcfpx2gSExMBALt27TLZXlpaKgVJUlKSybFFRUVmzyjbuXOnybFERNQ2nB40qampAJoD4fTp00btH330EYQQCAkJkYLFICkpCcHBwdDr9Vi6dKnR2NOnT2P37t0AgMmTJytQPRERtcaqoKmsrJReN2/elL6/detWiza9Xt9inOFxAgsXLjRa5jPPPINBgwZBr9fj2WefxdGjRwEAjY2NWLp0KT755BMAwNtvv41OnTq1GOvl5SUt8+OPP8bSpUvR2NgIADhy5AieffZZ6PV6DB06FKNHj7ZmqkRE5CjW3DQNZh4TcP+rpKTE5LjMzEyTy23tMQEvvfSSxbrufUyAp6cnHxNARNTGHPaYAKX06tULp06dwrx58xAZGYmmpib4+/sjMTER33//PZYvX25xfE5ODr777jskJiZCrVajqakJkZGRmD9/Pk6dOmXxzgFERKQslRBCOLuI9iQhIQEajcbZZRARdSiW/na2iy0aIiJyXQwaIiJSFIOGiIgUxaAhIiJFMWiIiEhRDBoiIlIUg4aIiBTFoCEiIkUxaIiISFEMGiIiUhSDhoiIFMWgISIiRTFoiIhIUQwaIiJSFIOGiIgUxaAhIiJFMWiIiEhRDBoiIlIUg4aIiBTFoCEiIkUxaIiISFEMGiIiUhSDhoiIFMWgISIiRTFoiIhIUQwaIiJSFIOGiIgUxaAhIiJFMWiIiEhRDBoiIlIUg4aIiBRlddCUl5dj5syZ6NOnD7y9vdG9e3eMGTMGe/bssamA8PBwqFQqWa+vvvrKaLyccRs3brSpNiIisp+HNZ3PnDmD4cOHo6qqCgDQuXNnVFZWIi8vD/n5+cjKysLcuXOtKiAoKAgNDQ1m2+vq6lBbWwsAiIuLM9svMDAQ7u7uJtu8vb2tqomIiBxIyFRfXy/CwsIEABEbGysKCwuFEEJUV1eL9PR0AUCoVCqxY8cOuYuUZezYsQKAiIuLM9kOQAAQJSUlDvm9+Ph4hyyHiOiPxNLfTtm7zlasWIFLly5BrVZj69atiI6OBtC8VZOdnY1x48ZBCIE333zTYSFYUVGBH374AQAwdepUhy2XiIjajuygWbduHQAgNTUVoaGhRu0ZGRkAgBMnTuD8+fMOKW79+vW4c+cOPD09kZqa6pBlEhFR25IVNFqtFsePHwcAJCcnm+wzePBgdOnSBQBsPjHgfoaD/6NGjUJgYKBDlklERG1LVtCcO3cOQggAkHaZGS3IzQ19+/YFAJw9e9buwn766SecPHkSgLzdZikpKejWrRu8vLzQq1cvPPfcc8jPz7e7DiIiso+soCkrK5Pe9+zZ02w/Q9u9/W21du1aAM1nk40aNarV/seOHcPdu3fh6emJ0tJS5ObmYvTo0UhJScHt27ftroeIiGwjK2jq6uqk9z4+Pmb7+fr6AoB0OrKtmpqaWhwT8vT0NNt36tSp2L59O27evImamhrU1tbi3LlzePHFFwEAGzZswIwZMyz+Xk5ODhISEpCQkICKigq7aiciopba5Z0BduzYgWvXrgFofbfZ2rVrkZycjK5du0rfRUZGYs2aNdIJCqtWrbJ4gkJaWho0Gg00Gg2CgoLsnwAREUlkBY2fn5/0XqfTme1XX18PAFCr1XYVZTgJYMCAARYv0mxNZmYmfHx8IIRAXl6eXTUREZFtZAXNvcdlrl69arafoS0kJMTmgm7evIktW7YAsP/aGT8/P/Tv3x8AcOHCBbuWRUREtpEVNJGRkVCpVACAoqIik330er20e6pfv342F/Ttt9+isbERHh4emDx5ss3LISKi9kFW0Pj7+yMhIQEAsGvXLpN9CgoKUF1dDQBISkqyuSDDbrPk5GT06NHD5uUAzScxFBYWAgAiIiLsWhYREdlG9skAhivz161bZ/L05ezsbABAfHy8dD2Ntc6fP4+CggIA8nabGa7tMefdd9+FTqeDSqXC008/bVNNRERkH9lBM336dISFhUGr1WL06NHSRZlarRZvvPEGcnNzAQBZWVlGYw2361+4cKHF3zBszXTr1g1jx45ttaaUlBTMnz8fGo2mxbUy58+fx7Rp07BkyRIAzaFlz+48IiKynezHBPj4+GDz5s1ISkrCiRMnEB0djc6dO6O2thZ6vR4qlQpZWVkYOXKkTYXo9Xp8/fXXAIDnn38eXl5erY6pqKjAxo0bkZWVBXd3d3Tp0gWNjY0trvsZP348vvjiC5tqIiIi+1n1PJqBAweisLAQixcvRl5eHkpLSxEQEIBHHnkEs2bNsuvYzN69e3HlyhUA8s82mzdvHh5++GEcPXoUV65cwY0bN+Dm5oaIiAgMHjwYL7zwgs3BR0REjqESrR3o+INJSEiARqNxdhlERB2Kpb+d7fLOAERE5DoYNEREpCgGDRERKYpBQ0REimLQEBGRohg0RESkKAYNEREpikFDRESKYtAQEZGiGDRERKQoBg0RESmKQUNERIpi0BARkaIYNEREpCgGDRERKYpBQ0REimLQEBGRohg0RESkKAYNEREpikFDRESKYtAQEZGiGDRERKQoBg0RESmKQUNERIpi0BARkaIYNEREpCgGDRERKYpBQ0REimLQEBGRohg0RESkKKuDpry8HDNnzkSfPn3g7e2N7t27Y8yYMdizZ49NBezfvx8qlarVV2VlpcXlbNiwAcOHD0dAQAB8fX0RFRWFBQsWQKvV2lQXERE5hoc1nc+cOYPhw4ejqqoKANC5c2dUVlYiLy8P+fn5yMrKwty5c20qxM3NDUFBQRbbzUlLS8PKlSsBAB4eHvD29kZxcTEWLVqEb775BgcPHkTPnj1tqouIiOwje4tGp9Nh7NixqKqqQmxsLAoLC1FdXY2bN28iPT0dQgjMmzcPO3futKmQBx54AOXl5WZff/rTn0yOW758OVauXAk3Nzd8+OGHqK2thVarxaFDhxAWFoYLFy4gJSXFppqIiMh+soNmxYoVuHTpEtRqNbZu3Yro6GgAzVs12dnZGDduHIQQePPNNxUr9n6NjY1YuHAhAGDmzJmYPXs2vLy8AABDhgzBpk2boFKpcOjQIWzdurXN6iIiot/JDpp169YBAFJTUxEaGmrUnpGRAQA4ceIEzp8/76DyLNu9ezeuX78OlUqF9PR0o/bY2FiMGDECwO/1ExFR25IVNFqtFsePHwcAJCcnm+wzePBgdOnSBQBsPjHAWvv27QMA9O/f32T4Ab/Xu3fv3japiYiIWpIVNOfOnYMQAgCkXWZGC3JzQ9++fQEAZ8+etbqQiooKxMXFwc/PD35+fnjooYeQlpaGn376yewYw++YqwkA+vXrJy2/tTPXiIjI8WQFTVlZmfTe0tlbhrZ7+8tVX1+PkydPwsvLC01NTfjll1+wcuVKxMbGIjs722JdcmqytS4iIrKPrKCpq6uT3vv4+Jjt5+vrCwCora2VXUDXrl2RkZEBjUYDnU6HGzduoL6+HgcOHMCQIUNw9+5dZGRkYP369WbrklOTpbpycnKQkJCAhIQEVFRUyK6diIha5/Q7A8TExOCDDz5AfHw8vL29AQDu7u4YNmwY9u3bh6FDhwIA5syZA71er0gNaWlp0Gg00Gg0Fq/lISIi68kKGj8/P+m9Tqcz26++vh4AoFar7SyrWadOnfDuu+8CAK5cuYKTJ0+arEtOTY6si4iI5JMVNPce57h69arZfoa2kJAQO8v63aBBg6T3Fy5cMFmXnJocXRcREckjK2giIyOhUqkAAEVFRSb76PV66foZw5leSjP8jrmagN/PTAsKCkJgYGCb1EVERL+TFTT+/v5ISEgAAOzatctkn4KCAlRXVwMAkpKSHFRe83INIiIiWrQlJiYCaA4ac2eUGW6J48iaiIhIPtknA6SmpgJovsLe1B91wynI8fHx0vU0chiuzzHlzp07eOuttwA07/aKi4tr0Z6UlITg4GDo9XosXbrUaPzp06exe/duAMDkyZNl10RERA4kZKqvrxdhYWECgIiLixNFRUVCCCFqampERkaGACAAiB07dhiNNbRlZmYatfXr1098+umn4ueffxZ6vV4IIURTU5M4ePCgeOyxx6Sxa9euNVnX559/LgAINzc3kZ2dLRoaGoQQQhw+fFhEREQIAGLo0KFypyni4+Nl9yUiomaW/naqhLCwSXGf06dPIykpqcVjAmpra6HX66FSqcw+JsBwfCczM1O6Ceb9bQDg5eUFf39/1NTU4Pbt2wCab/v/3nvvYc6cOWbruvcxAZ6envDy8pKumendu7dVjwlISEiARqOR1fePIHxuvrNLkOXi+6OcXQLRH5qlv51WXUczcOBAFBYW4vXXX0fv3r3R2NiIgIAAjBo1Crt27bLpWTQrVqzAlClTEB0djc6dO+PWrVvw8vLCgAEDMGPGDJw+fdpiyADNF1x+9913SExMhFqtRlNTEyIjIzF//nycOnWKz6IhInIiq7Zo/gi4RdMSt2iISA6HbdEQERFZi0FDRESKYtAQEZGiGDRERKQoBg0RESmKQUNERIpi0BARkaIYNEREpCgGDRERKYpBQ0REimLQEBGRohg0RESkKAYNEREpikFDRESKYtAQEZGiGDRERKQoBg0RESmKQUNERIpi0BARkaIYNEREpCgGDRERKYpBQ0REimLQEBGRohg0RESkKAYNEREpikFDRESKYtAQEZGiGDRERKQoD2cX4ErC5+Y7uwRZLr4/ytklENEfCLdoiIhIUQwaIiJSlNVBU15ejpkzZ6JPnz7w9vZG9+7dMWbMGOzZs8emAioqKrBixQpMmDBBWqafnx+ioqIwY8YM/PrrrxbHq1SqVl8bN260qTYiIrKfVcdozpw5g+HDh6OqqgoA0LlzZ1RWViIvLw/5+fnIysrC3LlzrSqgZ8+eaGpqkj6r1Wrcvn0bxcXFKC4uxurVq7FmzRpMmjTJ4nICAwPh7u5uss3b29uqmoiIyHFkb9HodDqMHTsWVVVViI2NRWFhIaqrq3Hz5k2kp6dDCIF58+Zh586dVhXQ1NSEYcOG4auvvkJZWRm0Wi3q6+vx448/IiYmBg0NDZgyZQrOnDljcTnHjh1DeXm5ydfo0aOtqomIiBxHdtCsWLECly5dglqtxtatWxEdHQ2geasmOzsb48aNgxACb775plUFHDhwAAcOHMCUKVPQo0cPAIC7uzuGDh2KnTt3Ijg4GE1NTfj444+tWi4REbUPsoNm3bp1AIDU1FSEhoYatWdkZAAATpw4gfPnz8suYNiwYWbbgoKC8PTTTwMAjh8/LnuZRETUfsgKGq1WK/2hT05ONtln8ODB6NKlCwDYfGKAKQEBAQCAu3fvOmyZRETUdmQFzblz5yCEAABpl5nRgtzc0LdvXwDA2bNnHVRe8641AOjfv7/FfikpKejWrRu8vLzQq1cvPPfcc8jP7xgXUBIRuTJZQVNWVia979mzp9l+hrZ7+9tj8+bN0Gg0AIAXX3zRYt9jx47h7t278PT0RGlpKXJzczF69GikpKTg9u3bFsfm5OQgISEBCQkJqKiocEjtRETUTFbQ1NXVSe99fHzM9vP19QUA1NbW2lkWUFpairS0NADA2LFj8eSTT5rsN3XqVGzfvh03b95ETU0Namtrce7cOSmYNmzYgBkzZlj8rbS0NGg0Gmg0GgQFBdldOxER/a5d3hmgtrYW48aNw/Xr1xEWFobVq1eb7bt27VokJyeja9eu0neRkZFYs2aNdILCqlWrrDpBgYiIHEdW0Pj5+UnvdTqd2X719fUAmi+6tFVDQwOeeeYZaetix44dCAwMtGlZmZmZ8PHxgRACeXl5NtdERES2kxU09x6XuXr1qtl+hraQkBCbirl9+zbGjx+PvXv3omvXrti5c6d0goEt/Pz8pJMILly4YPNyiIjIdrKCJjIyEiqVCgBQVFRkso9er5d2T/Xr18/qQpqamjBp0iTk5+dDrVZj27ZtiImJsXo5RETUvsgKGn9/fyQkJAAAdu3aZbJPQUEBqqurAQBJSUlWFaHX6zF16lTk5ubCx8cHW7ZswaOPPmrVMkypq6tDYWEhACAiIsLu5RERkfVknwyQmpoKoPkOAaZOX87OzgYAxMfHW7W7SwiBtLQ0rF+/Hp06dUJubi4SExNlj7Xk3XffhU6ng0qlku4wQEREbUt20EyfPh1hYWHQarUYPXq0dFGmVqvFG2+8gdzcXABAVlaW0VjD7foXLlxo1DZr1iysXr0aHh4e+P77782exmxKSkoK5s+fD41G0+JamfPnz2PatGlYsmQJgOZToG3ZnUdERPaT/ZgAHx8fbN68GUlJSThx4gSio6PRuXNn1NbWQq/XQ6VSISsrCyNHjpT947/99huWLVsGoDmMpk+fjunTp5vtX15e3uJzRUUFNm7ciKysLLi7u6NLly5obGxscd3P+PHj8cUXX8iuiYiIHMuq59EMHDgQhYWFWLx4MfLy8lBaWoqAgAA88sgjmDVrlk3HZgzu3LmDa9euWTV+3rx5ePjhh3H06FFcuXIFN27cgJubGyIiIjB48GC88MILVgUfERE5nlVBAwA9evTAsmXLpC0ROcwdSwkPD2/1OIslI0eOZJAQEbVz7fLOAERE5DoYNEREpCgGDRERKYpBQ0REimLQEBGRohg0RESkKAYNEREpikFDRESKYtAQEZGiGDRERKQoBg0RESmKQUNERIpi0BARkaIYNEREpCgGDRERKYpBQ0REimLQEBGRohg0RESkKAYNEREpysPZBRARubrwufnOLkGWi++PUmS5DBoialf+6H+UXRF3nRERkaIYNEREpCgGDRERKYpBQ0REimLQEBGRohg0RESkKAYNEREpikFDRESKYtAQEZGirA6a8vJyzJw5E3369IG3tze6d++OMWPGYM+ePXYVUlNTgwULFiAqKgq+vr4ICAhAUlISNm7cKGv8hg0bMHz4cAQEBMDX1xdRUVFYsGABtFqtXXUREZF9rAqaM2fOoH///vj0009x4cIFeHl5obKyEnl5eXjiiSfw/vvv21TElStXEBMTg0WLFqG4uBju7u6oqanB3r17MWHCBLzyyisWx6elpSElJQX79u1DTU0N3N3dUVxcjEWLFiEmJgZXr161qS4iIrKf7Hud6XQ6jB07FlVVVYiNjcXXX3+N6Oho1NTU4J133sHSpUsxb948xMXFYeTIkbILEEJg/PjxKCkpQXh4ONatW4chQ4agoaEB//znPzF37lwsX74csbGxmDZtmtH45cuXY+XKlXBzc8OSJUvw2muvwcvLC4cPH0ZqaiouXLiAlJQU/Pjjj7JrItflivfRcsU5kWuRvUWzYsUKXLp0CWq1Glu3bkV0dDQAoHPnzsjOzsa4ceMghMCbb75pVQGbN29GQUEB3NzcsGnTJgwZMgQA4O3tjYyMDLz++usAgLfeegu3b99uMbaxsRELFy4EAMycOROzZ8+Gl5cXAGDIkCHYtGkTVCoVDh06hK1bt1pVFxEROYbsoFm3bh0AIDU1FaGhoUbtGRkZAIATJ07g/PnzsgswLHfEiBGIiYkxap89ezZUKhXKy8uxd+/eFm27d+/G9evXoVKpkJ6ebjQ2NjYWI0aMaPE7RETUtmQFjVarxfHjxwEAycnJJvsMHjwYXbp0AQCrTgzYt2+fxeWGhoZKW0/3B41hbP/+/U2G373LvX8sERG1DVlBc+7cOQghAED6o2+0IDc39O3bFwBw9uxZWT9+/fp1VFVVWVwuAPTr18/kcg2f5YytqKhAZWWlrLqIiMhxZAVNWVmZ9L5nz55m+xna7u2v5HINn+WMtaYuIiJyHFlnndXV1UnvfXx8zPbz9fUFANTW1sr6cXuXaxgvZ6ylunJycpCTkwMAKC4uRkJCQiuVmxZo0yjLKioqEBQU5NBlJiRkyu7ranNytfkArjcnV5sP4Jpzut/FixfNtvFRzmi+DictLc3ZZZiUkJAAjUbj7DIcytXm5GrzAVxvTq42H6BjzUnWrjM/Pz/pvU6nM9uvvr4eAKBWq2X9uL3LNYyXM9aauoiIyHFkBc29xzksXWVvaAsJCZH14/Yu1zBezlhr6iIiIseRFTSRkZFQqVQAgKKiIpN99Hq9dP2M4Uyv1gQFBSEwMNDicoHfzy67f7mGz3LG3vtbHUl73aVnD1ebk6vNB3C9ObnafIAONich05///GcBQLz00ksm2w8fPiwACACiuLhY7mLFhAkTBADx5JNPmmy/cuWKUKlUAoDYvn17i7b8/HwBQLi5uYmrV6+aHD9y5EgBQDz//POyayIiIseRfWeA1NRUAM1X2Js6TTg7OxsAEB8fL11PY81yd+7cidOnTxu1f/TRRxBCICQkBImJiS3akpKSEBwcDL1ej6VLlxqNPX36NHbv3g0AmDx5suyaiIjIgeQmUn19vQgLCxMARFxcnCgqKhJCCFFTUyMyMjKkrZkdO3YYjTW0ZWZmGrXp9XoxaNAgAUBERESII0eOCCGEaGhoENnZ2cLNzU0AEDk5OSbr+vzzz6WtmuzsbNHQ0CCEaN7CioiIEADE0KFD5U6TiIgcTHbQCCHEqVOnREBAgBQcnTt3loJApVKJxYsXm/4RC0EjhBCXL1+WQgGAUKvVwsPDQ/psbnedwbRp06S+np6eQq1WS5979+4tSktLrZkmERE5kFXX0QwcOBCFhYVYvHgx8vLyUFpaioCAADzyyCOYNWsWkpKSbNqq6tWrF06dOoUlS5YgNzcXFy9ehL+/P2JiYvDyyy9jwoQJFsfn5ORgxIgR+OKLL3Dq1CnodDpERkbiueeew5w5c+Dv729TXW2tvr4eW7duxQ8//IDCwkL89ttv0Gq1uHPnDvz8/BAQEIAHH3wQjz76KMaNG2fyJqTtjSvN6ddff0VBQQE0Gg1OnTqFsrIyVFZWorq6Gn5+fujZsyfi4+Mxfvx4jB49Gu7u7s4uuVVVVVXQaDQ4duyY9DK1a/zxxx/H/v37275AK7niOjKora3FmjVrsG3bNpw5cwZVVVVQq9UIDQ3FyJEj8cILL6B///7OLtM0ZycdNfvmm29Ejx49pC0xOa+nnnpKlJWVObt0s1xtTn369JE9j4EDB4ozZ844u2SL1q9fL3s+jz/+uLPLlcXV1pHBli1bWuxNMvXy8PAQ8+bNE3q93tnlGuGdAdqBVatWmXyoW2t++OEHDB06FMePH0fXrl0dX5gdXHFOBqGhoYiLi0Pv3r0REBAAT09PVFVV4cSJEzhw4ADu3r2L06dPY9iwYTh8+DCioqKcXbJJjY2Nzi5BMa6yjgBg48aNSElJkW5sHBQUhLFjxyIiIgI6nQ4FBQXYs2cPmpqakJWVhbq6OnzyySfOLfp+zk66P7pbt24JPz8/q/7Vf/9rwYIFzp5GC644JyGEyM3NFZcuXbLY5/z58yI6OlqaR2JiYhtVZ70vv/zS5bZoXG0dlZeXtzjmnJqaKmpra436HTp0SAQFBUn98vLynFCteQwaJzO3+yI4OFh8++23ory8XGi1WnH8+HExatQok30HDRrk7Gm04IpzskZhYaE0D5VKJcrLy51dkklffvml8Pb2FoMGDRKvvfaa+Ne//iWKi4s7dNDI1VHW0fz586U6Y2NjxZ07d8z23bFjh9Q3MjKyXe1CY9A42eLFi03+j7127VqjvjqdTnh7exv17du3rxMqN88V52St4OBgaS5Hjx51djkmabVacfv2baPv/whBI0THWEdxcXFSjWvWrGm1/4ABA6T+hw8fboMK5ZF9wSYpw9wjDgy3/GntOwB46KGHHFqTvVxxTta4c+dOi0dS3PuoivZErVbD09PT2WU4RUdZR//73/+k9/Hx8a32v7fPxo0bFanJFgwaJ/vLX/5i8vu5c+fi3//+N27cuIGGhgacOXMGEydORENDQ4t+KpUKM2fObItSZXPFOVnjs88+k+4aHhwcLPvef9R2Oso6uvfO9Jaeu2Wqz/HjxxWpySbO3qQiIUaPHm3TAfNOnTqJzz//3Nnlm+SKczLn7t27oqqqSuzdu1dMnTq1xXxWrlzp7PKsZmq9dPRdZx11HYWEhEh17t69u9X+9x7zDAgIaIMK5WHQtAM1NTVmD4qbenXt2lW8+uqr4pdffnF26Wa54pzuNWfOHIvz8fHxEV999ZWzy7SJqwSNK6yjMWPGSPXOmDHDYt8bN260OEMNgMWTB9oSd521A/7+/sjLy8O2bdtk7Yetrq7G2bNncenSpTaozjauOCe5Jk6ciEuXLmHKlCnOLoXM6CjraOLEidL7nJwcHDp0yGQ/vV6PV1991ehx9TU1NYrWJ5dKiP9/FRA51a5du5CZmYkjR45YNS4jIwMffPCBQlXZxxXnZHDw4EEUFBQAaD6wfP36dRw5ckT6Ljw8HEuXLsVf//pXZ5ZpE1MnaHSUW9DcyxXW0d27dzFo0CDpeIuPjw/mzJmDSZMmITw8HDqdDv/973+RlZWF/fv3w9vbu8Uxz8uXL6NXr17OKv93zt6kIiHee+89o836bt26ieXLl4uSkhJx48YNceDAATFkyBCTuwA+++wzZ0/BiCvOSY6CgoIWt0HpiPMwtT464q4zczraOrp8+bJ46KGHWt397OvrK1atWtXiu+rqameXL4TgMRqn27Vrl8n/aLZt22bUt66uToSHhxv1DQoKEo2NjU6o3jRXnJM1Ll68KN0ZwdPTs8McdzJw9aARouOto5qaGpGeni78/f3Nrp9z586Jo0ePSt+5u7u3m4s2eYzGyT766COj70JCQvDUU08Zfe/r64tJkyYZfV9RUQGNRqNIfbZwxTlZIywsDH/7298ANO+yWbt2rXMLIiMdbR35+/sjOzsbFRUVOHDgAFavXo1ly5Zh/fr1+OWXX7B//35ERka2uO6mT58+Zq9Ta2u8qaaTHT161Oi7Bx54wGx/c22//fYbhgwZ4rC67OGKc7LWww8/LL039eRYcr6OuI68vLwwbNgwDBs2zGT7qVOnpPePPvpoG1XVOm7ROFldXZ3Rd5cvXzbb31ybl5eXw2qylyvOyVpNTU3S+7t37zqxEjLHFdfR5s2bpfdPPvmkEytpiUHjZMHBwUbflZWVYceOHUbf63Q6fPvttyaXY2mLoa254pysdezYMel9uzjrh4y42jratm0bfv75ZwBA9+7d29XZdAwaJzN3u5bJkycjJycHly5dwq1bt/Djjz8iOTkZJSUlRn0DAgIQGxurdKmyueKcrFFUVITvv/9e+jxy5EgnVmOeEAJNTU1GL3v7dgQdZR3Jde3aNcyYMUP6PH/+fHTq1MmJFd3H2Wcj/NH95z//kX31vLnXokWLnD2NFlxxTu+88454++23xeXLly32y8vLa/FU0aioqHZ79ty+ffvsXk8lJSXOnobEFdeREM317t+/3+wZZEePHhWRkZHSfBITE9vN2WYGPBnAyR577DHMmzcPWVlZNo1/4oknkJ6e7uCq7OOKc6qqqsKyZcuwcOFCDBw4EDExMejVqxfUajV0Oh0uXryIgwcP4sKFC9KYwMBAfPPNN+3rX5YuzFXX0ZEjR7Bo0SL07NkTw4YNw4MPPggfHx9cv34dhw4dgkajkZ6+OWjQIOTm5rabs80kzk46arZy5coWT8hr7eXh4SHS09OFTqdzdulmudKcZs2aZdW/9EeMGNHur81wtS0aV1xHQrR8+Jm5l7u7u3jllVeEVqt1drkm8RY07UhdXR3y8vKwfft2/PTTT/jtt9+g1WrR1NQEtVqNwMBADBgwAMOGDcPEiRMREhLi7JJb5SpzampqwuHDh7F//34cO3YMP//8M8rKylBXVwdvb2907doVffv2xSOPPIIJEybIur+bs+3fvx+JiYl2LaOkpATh4eGOKchOrriOAKC0tBT5+fnYt28fCgsLce3aNdy6dQsBAQH4v//7PyQnJ2PSpEmIiopydqlmMWiIiEhRPOuMiIgUxaAhIiJFMWiIiEhRDBoiIlIUg4aIiBTFoCEiIkUxaIiISFEMGiIiUhSDhoiIFMWgISIiRTFoiIhIUQwaIiJS1P8DVP4pNrAKEswAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZoAAAENCAYAAAA2ZaOYAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAAmgElEQVR4nO3de1TUdf4/8OcMILcBbQdQxBbQU4JYgsym6eYexKTykm1KiXuy/pBubq6HyOumbV+xDLudTiaa2e7RLnpoVbS86xYq62hooFiteEOQiwoDDCjO+/cHv/kkzYXPXD4MTM/HOXPOzLwv83qfT/H085nP5zMqIYQAERGRQtSeLoCIiLwbg4aIiBTFoCEiIkUxaIiISFEMGiIiUpSvpwvobsLCwhATE+PpMoiIepRz586htrbWahuD5ldiYmKg1+s9XQYRUY+i0+lstvHQGRERKYpBQ0REimLQEBGRohg0RESkKAYNEREpikFDRESKYtAQEZGiGDRERKQoBg0RESmKQUNERIqSHTQGgwFbt27F3//+dzz88MMICwuDSqWCSqVCWVmZy4U0NDRg8eLFiI+PR1BQELRaLVJTU7F582ZZ4zdt2oSxY8dCq9UiKCgI8fHxWLx4MQwGg8u1ERGRC4RMX331lQBg9XH69Gm501h18eJFERsbK82n0WiEr6+v9Pr555+3O37WrFlSX19fX6HRaKTXAwcOFBUVFbJrSU5OdmktRES/Rfb+djp06CwiIgKPPPIIlixZgry8PHcFHaZOnYry8nLExMSgsLAQBoMBBoMBK1asgFqtxqpVq7BmzRqr481tarUab731FhobG2EwGFBYWIjo6GicPXsW6enpbqmViIicIDet2traOrwuLy93yx6NeU9JrVaL77//3qL9b3/7mwAg+vXrJ1pbWzu0tbS0iIiICAFAzJ0712Ls8ePHhUqlEgDE1q1bZdXDPRoiIse5ZY/Gx8dHkaDbsGEDAGDcuHFITEy0aH/55ZehUqlQVVWFffv2dWjbs2cPqquroVKpkJWVZTE2KSkJ48aN6/A5RETUtTx+1tn+/fsBAGlpaVbbo6KikJCQAAAWQWMeO3ToUERFRVkdb57312OJiKhreDRoqqurUVdXBwBSmFgzZMgQAMCpU6c6vG9+LWdsTU2NzV9/IyIi5Xg0aCorK6Xn/fv3t9nP3HZ7/9tfyxlrbTwRESnPoz/l3NTUJD0PDAy02S8oKAgA0NjYaHW8nLHWxpvl5eVJZ9HV1NR0UjURETnC49/RdAeZmZnQ6/XQ6/UIDw/3dDlERF7Fo0ETHBwsPTcajTb7NTc3AwA0Go3V8XLGWhtPRETK82jQ3P79yeXLl232M7dFRkZaHS9nrLXxRESkPI8GTXh4OMLCwgAApaWlNvuZzy4zn0FmZn4tZ+ztn0VERF3H49/RpKSkAAB2795ttb2iokIKktTUVKtjS0tLbZ5RtmvXLqtjiYioa3g8aDIyMgC0B8KJEycs2t9++20IIRAZGSkFi1lqaioiIiJgMpmwcuVKi7EnTpzAnj17AAAzZsxQoHoiIuqMQ0FTW1srPa5duya9f/369Q5tJpOpwzjzzwksXbrUYs5HH30UI0aMgMlkwmOPPYYjR44AAFpbW7Fy5Uq8++67AIDXXnsNvXr16jDW399fmvOdd97BypUr0draCgA4fPgwHnvsMZhMJowePRoTJ050ZKlEROQujtw0DTZ+JuDXj/LycqvjlixZYnXezn4m4LnnnrNb1+0/E+Dn58efCSAi6mJu+5kApQwYMADFxcVYuHAh4uLi0NbWhpCQEKSkpODLL7/EqlWr7I7Py8vDF198gZSUFGg0GrS1tSEuLg6LFi1CcXGx3TsHEBGRslRCCOHpIroTnU4HvV7v6TKIiHoUe387u8UeDREReS8GDRERKYpBQ0REimLQEBGRohg0RESkKAYNEREpikFDRESKYtAQEZGiGDRERKQoBg0RESmKQUNERIpi0BARkaIYNEREpCgGDRERKYpBQ0REimLQEBGRohg0RESkKAYNEREpikFDRESKYtAQEZGiGDRERKQoBg0RESmKQUNERIpi0BARkaIYNEREpCgGDRERKYpBQ0REimLQEBGRohg0RESkKAYNEREpikFDRESKcjhoqqqqMGfOHAwaNAgBAQHo27cvJk2ahL179zpVQExMDFQqlazHp59+ajFezrjNmzc7VRsREbnO15HOJ0+exNixY1FXVwcACA0NRW1tLQoKCrB9+3bk5ORg/vz5DhUQHh6OlpYWm+1NTU1obGwEAAwfPtxmv7CwMPj4+FhtCwgIcKgmIiJyIyFTc3OziI6OFgBEUlKSKCkpEUIIUV9fL7KysgQAoVKpxM6dO+VOKcvkyZMFADF8+HCr7QAEAFFeXu6Wz0tOTnbLPEREvyX2/nbKPnS2evVqnD9/HhqNBtu2bUNCQgKA9r2a3NxcTJkyBUIILFiwwG0hWFNTg6+//hoAMHPmTLfNS0REXUd20GzYsAEAkJGRgaioKIv27OxsAMDx48dx5swZtxS3ceNG3Lx5E35+fsjIyHDLnERE1LVkBY3BYMCxY8cAAGlpaVb7jBw5Er179wYAp08M+DXzl/8TJkxAWFiYW+YkIqKuJStoTp8+DSEEAEiHzCwmUqsxePBgAMCpU6dcLuyHH37A999/D0DeYbP09HTccccd8Pf3x4ABA/D4449j+/btLtdBRESukRU0lZWV0vP+/fvb7Gduu72/s9avXw+g/WyyCRMmdNr/6NGjuHXrFvz8/FBRUYH8/HxMnDgR6enpuHHjhsv1EBGRc2QFTVNTk/Q8MDDQZr+goCAAkE5HdlZbW1uH74T8/Pxs9p05cya++eYbXLt2DQ0NDWhsbMTp06fxzDPPAAA2bdqE2bNn2/28vLw86HQ66HQ61NTUuFQ7ERF11C3vDLBz505cuXIFQOeHzdavX4+0tDT06dNHei8uLg7r1q2TTlBYu3at3RMUMjMzodfrodfrER4e7voCiIhIIitogoODpedGo9Fmv+bmZgCARqNxqSjzSQD33HOP3Ys0O7NkyRIEBgZCCIGCggKXaiIiIufICprbv5e5fPmyzX7mtsjISKcLunbtGrZu3QrA9WtngoODMXToUADA2bNnXZqLiIicIyto4uLioFKpAAClpaVW+5hMJunw1JAhQ5wu6PPPP0drayt8fX0xY8YMp+chIqLuQVbQhISEQKfTAQB2795ttU9RURHq6+sBAKmpqU4XZD5slpaWhn79+jk9D9B+EkNJSQkAIDY21qW5iIjIObJPBjBfmb9hwwarpy/n5uYCAJKTk6XraRx15swZFBUVAZB32Mx8bY8tr7/+OoxGI1QqFR555BGnaiIiItfIDppnn30W0dHRMBgMmDhxonRRpsFgwCuvvIL8/HwAQE5OjsVY8+36ly5davczzHszd9xxByZPntxpTenp6Vi0aBH0en2Ha2XOnDmDWbNm4c033wTQHlquHM4jIiLnyf6ZgMDAQGzZsgWpqak4fvw4EhISEBoaisbGRphMJqhUKuTk5GD8+PFOFWIymfCvf/0LAPDkk0/C39+/0zE1NTXYvHkzcnJy4OPjg969e6O1tbXDdT9Tp07FRx995FRNRETkOod+j2bYsGEoKSnB8uXLUVBQgIqKCmi1Wtx3332YO3euS9/N7Nu3D5cuXQIg/2yzhQsX4t5778WRI0dw6dIlXL16FWq1GrGxsRg5ciSefvppp4OPiIjcQyU6+6LjN0an00Gv13u6DCKiHsXe385ueWcAIiLyHgwaIiJSFIOGiIgUxaAhIiJFMWiIiEhRDBoiIlIUg4aIiBTFoCEiIkUxaIiISFEMGiIiUhSDhoiIFMWgISIiRTFoiIhIUQwaIiJSFIOGiIgUxaAhIiJFMWiIiEhRDBoiIlIUg4aIiBTFoCEiIkUxaIiISFEMGiIiUhSDhoiIFMWgISIiRTFoiIhIUQwaIiJSFIOGiIgUxaAhIiJFMWiIiEhRDBoiIlKUw0FTVVWFOXPmYNCgQQgICEDfvn0xadIk7N2716kCDhw4AJVK1emjtrbW7jybNm3C2LFjodVqERQUhPj4eCxevBgGg8GpuoiIyD18Hel88uRJjB07FnV1dQCA0NBQ1NbWoqCgANu3b0dOTg7mz5/vVCFqtRrh4eF2223JzMzEmjVrAAC+vr4ICAhAWVkZli1bhs8++wzffvst+vfv71RdRETkGtl7NEajEZMnT0ZdXR2SkpJQUlKC+vp6XLt2DVlZWRBCYOHChdi1a5dThdx5552oqqqy+fjd735nddyqVauwZs0aqNVqvPXWW2hsbITBYEBhYSGio6Nx9uxZpKenO1UTERG5TnbQrF69GufPn4dGo8G2bduQkJAAoH2vJjc3F1OmTIEQAgsWLFCs2F9rbW3F0qVLAQBz5szByy+/DH9/fwDAqFGj8NVXX0GlUqGwsBDbtm3rsrqIiOgXsoNmw4YNAICMjAxERUVZtGdnZwMAjh8/jjNnzripPPv27NmD6upqqFQqZGVlWbQnJSVh3LhxAH6pn4iIupasoDEYDDh27BgAIC0tzWqfkSNHonfv3gDg9IkBjtq/fz8AYOjQoVbDD/il3n379nVJTURE1JGsoDl9+jSEEAAgHTKzmEitxuDBgwEAp06dcriQmpoaDB8+HMHBwQgODsbdd9+NzMxM/PDDDzbHmD/HVk0AMGTIEGn+zs5cIyIi95MVNJWVldJze2dvmdtu7y9Xc3Mzvv/+e/j7+6OtrQ0//fQT1qxZg6SkJOTm5tqtS05NztZFRESukRU0TU1N0vPAwECb/YKCggAAjY2Nsgvo06cPsrOzodfrYTQacfXqVTQ3N+PgwYMYNWoUbt26hezsbGzcuNFmXXJqsldXXl4edDoddDodampqZNdORESd8/idARITE7FixQokJycjICAAAODj44MxY8Zg//79GD16NABg3rx5MJlMitSQmZkJvV4PvV5v91oeIiJynKygCQ4Olp4bjUab/ZqbmwEAGo3GxbLa9erVC6+//joA4NKlS/j++++t1iWnJnfWRURE8skKmtu/57h8+bLNfua2yMhIF8v6xYgRI6TnZ8+etVqXnJrcXRcREckjK2ji4uKgUqkAAKWlpVb7mEwm6foZ85leSjN/jq2agF/OTAsPD0dYWFiX1EVERL+QFTQhISHQ6XQAgN27d1vtU1RUhPr6egBAamqqm8prn9csNja2Q1tKSgqA9qCxdUaZ+ZY47qyJiIjkk30yQEZGBoD2K+yt/VE3n4KcnJwsXU8jh/n6HGtu3ryJV199FUD7Ya/hw4d3aE9NTUVERARMJhNWrlxpMf7EiRPYs2cPAGDGjBmyayIiIjcSMjU3N4vo6GgBQAwfPlyUlpYKIYRoaGgQ2dnZAoAAIHbu3Gkx1ty2ZMkSi7YhQ4aI999/X/z444/CZDIJIYRoa2sT3377rXjggQeksevXr7da14cffigACLVaLXJzc0VLS4sQQohDhw6J2NhYAUCMHj1a7jJFcnKy7L5ERNTO3t9O2UEjhBDFxcVCq9VKf/xDQ0OFWq0WAIRKpRLLly+3/iF2gsbcBkD4+/uLsLAw0atXL+k9X19f8cYbb9ita9asWVJ/Pz8/odFopNcDBw4UFRUVstfIoCEicpy9v50OXUczbNgwlJSU4KWXXsLAgQPR2toKrVaLCRMmYPfu3U79Fs3q1avx1FNPISEhAaGhobh+/Tr8/f1xzz33YPbs2Thx4gTmzZtnd468vDx88cUXSElJgUajQVtbG+Li4rBo0SIUFxfzt2iIiDxIJYSdL0l+g3Q6HfR6vafLICLqUez97fT4nQGIiMi7MWiIiEhRDBoiIlIUg4aIiBTFoCEiIkUxaIiISFEMGiIiUhSDhoiIFMWgISIiRTFoiIhIUQwaIiJSFIOGiIgUxaAhIiJFMWiIiEhRDBoiIlIUg4aIiBTFoCEiIkUxaIiISFEMGiIiUhSDhoiIFMWgISIiRTFoiIhIUQwaIiJSFIOGiIgUxaAhIiJFMWiIiEhRDBoiIlIUg4aIiBTFoCEiIkUxaIiISFEMGiIiUpTDQVNVVYU5c+Zg0KBBCAgIQN++fTFp0iTs3bvXqQJqamqwevVqTJs2TZozODgY8fHxmD17Nn7++We741UqVaePzZs3O1UbERG5zteRzidPnsTYsWNRV1cHAAgNDUVtbS0KCgqwfft25OTkYP78+Q4V0L9/f7S1tUmvNRoNbty4gbKyMpSVleHjjz/GunXrMH36dLvzhIWFwcfHx2pbQECAQzUREZH7yN6jMRqNmDx5Murq6pCUlISSkhLU19fj2rVryMrKghACCxcuxK5duxwqoK2tDWPGjMGnn36KyspKGAwGNDc347vvvkNiYiJaWlrw1FNP4eTJk3bnOXr0KKqqqqw+Jk6c6FBNRETkPrKDZvXq1Th//jw0Gg22bduGhIQEAO17Nbm5uZgyZQqEEFiwYIFDBRw8eBAHDx7EU089hX79+gEAfHx8MHr0aOzatQsRERFoa2vDO++849C8RETUPcgOmg0bNgAAMjIyEBUVZdGenZ0NADh+/DjOnDkju4AxY8bYbAsPD8cjjzwCADh27JjsOYmIqPuQFTQGg0H6Q5+Wlma1z8iRI9G7d28AcPrEAGu0Wi0A4NatW26bk4iIuo6soDl9+jSEEAAgHTKzmEitxuDBgwEAp06dclN57YfWAGDo0KF2+6Wnp+OOO+6Av78/BgwYgMcffxzbt293Wx1EROQcWUFTWVkpPe/fv7/Nfua22/u7YsuWLdDr9QCAZ555xm7fo0eP4tatW/Dz80NFRQXy8/MxceJEpKen48aNG3bH5uXlQafTQafToaamxi21ExFRO1lB09TUJD0PDAy02S8oKAgA0NjY6GJZQEVFBTIzMwEAkydPxkMPPWS138yZM/HNN9/g2rVraGhoQGNjI06fPi0F06ZNmzB79my7n5WZmQm9Xg+9Xo/w8HCXayciol90yzsDNDY2YsqUKaiurkZ0dDQ+/vhjm33Xr1+PtLQ09OnTR3ovLi4O69atk05QWLt2rUMnKBARkfvICprg4GDpudFotNmvubkZQPtFl85qaWnBo48+Ku1d7Ny5E2FhYU7NtWTJEgQGBkIIgYKCAqdrIiIi58kKmtu/l7l8+bLNfua2yMhIp4q5ceMGpk6din379qFPnz7YtWuXdIKBM4KDg6WTCM6ePev0PERE5DxZQRMXFweVSgUAKC0ttdrHZDJJh6eGDBnicCFtbW2YPn06tm/fDo1Ggx07diAxMdHheYiIqHuRFTQhISHQ6XQAgN27d1vtU1RUhPr6egBAamqqQ0WYTCbMnDkT+fn5CAwMxNatW3H//fc7NIc1TU1NKCkpAQDExsa6PB8RETlO9skAGRkZANrvEGDt9OXc3FwAQHJyskOHu4QQyMzMxMaNG9GrVy/k5+cjJSVF9lh7Xn/9dRiNRqhUKukOA0RE1LVkB82zzz6L6OhoGAwGTJw4Uboo02Aw4JVXXkF+fj4AICcnx2Ks+Xb9S5cutWibO3cuPv74Y/j6+uLLL7+0eRqzNenp6Vi0aBH0en2Ha2XOnDmDWbNm4c033wTQfgq0M4fziIjIdbJ/JiAwMBBbtmxBamoqjh8/joSEBISGhqKxsREmkwkqlQo5OTkYP3687A+/cOEC3nvvPQDtYfTss8/i2Weftdm/qqqqw+uamhps3rwZOTk58PHxQe/evdHa2trhup+pU6fio48+kl0TERG5l0O/RzNs2DCUlJRg+fLlKCgoQEVFBbRaLe677z7MnTvXqe9mzG7evIkrV644NH7hwoW49957ceTIEVy6dAlXr16FWq1GbGwsRo4ciaefftqh4CMiIvdTic6+6PiN0el00m1viIhIHnt/O7vlnQGIiMh7MGiIiEhRDBoiIlIUg4aIiBTFoCEiIkU5dHoz2Rczv2f8oue5NybI7utta/K29QDetyZvWw/gnWtyBPdoiIhIUQwaIiJSFIOGiIgUxaAhIiJFMWiIiEhRDBoiIlIUg4aIiBTFoCEiIkUxaIiISFEMGiIiUhSDhoiIFMWgISIiRTFoiIhIUQwaIiJSFIOGiIgUxaAhIiJFMWiIiEhRDBoiIlIUg4aIiBTFoCEiIkUxaIiISFEMGiIiUhSDhoiIFMWgISIiRTkcNFVVVZgzZw4GDRqEgIAA9O3bF5MmTcLevXtdKqShoQGLFy9GfHw8goKCoNVqkZqais2bN8sav2nTJowdOxZarRZBQUGIj4/H4sWLYTAYXKqLiIhc41DQnDx5EkOHDsX777+Ps2fPwt/fH7W1tSgoKMCDDz6IN954w6kiLl26hMTERCxbtgxlZWXw8fFBQ0MD9u3bh2nTpuGFF16wOz4zMxPp6enYv38/Ghoa4OPjg7KyMixbtgyJiYm4fPmyU3UREZHrZAeN0WjE5MmTUVdXh6SkJJSUlKC+vh7Xrl1DVlYWhBBYuHAhdu3a5VABQghMnToV5eXliImJQWFhIQwGAwwGA1asWAG1Wo1Vq1ZhzZo1Vseb29RqNd566y00NjbCYDCgsLAQ0dHROHv2LNLT0x2qiYiI3Ed20KxevRrnz5+HRqPBtm3bkJCQAAAIDQ1Fbm4upkyZAiEEFixY4FABW7ZsQVFREdRqNb766iuMGjUKABAQEIDs7Gy89NJLAIBXX30VN27c6DC2tbUVS5cuBQDMmTMHL7/8Mvz9/QEAo0aNwldffQWVSoXCwkJs27bNobqIiMg9ZAfNhg0bAAAZGRmIioqyaM/OzgYAHD9+HGfOnJFdgHnecePGITEx0aL95ZdfhkqlQlVVFfbt29ehbc+ePaiuroZKpUJWVpbF2KSkJIwbN67D5xARUdeSFTQGgwHHjh0DAKSlpVntM3LkSPTu3RsAHDoxYP/+/XbnjYqKkvaefh005rFDhw61Gn63z/vrsURE1DVkBc3p06chhAAA6Y++xURqNQYPHgwAOHXqlKwPr66uRl1dnd15AWDIkCFW5zW/ljO2pqYGtbW1suoiIiL3kRU0lZWV0vP+/fvb7Gduu72/kvOaX8sZ60hdRETkPr5yOjU1NUnPAwMDbfYLCgoCADQ2Nsr6cFfnNY+XM9ZeXXl5ecjLywMAlJWVQafTdVK5dWFOjbKvpqYG4eHhbp1Tp1siu6+3rcnb1gN435q8bT2Ad67p186dO2ezTVbQeLvMzExkZmZ6ugyrdDod9Hq9p8twK29bk7etB/C+NXnbeoCetSZZh86Cg4Ol50aj0Wa/5uZmAIBGo5H14a7Oax4vZ6wjdRERkfvICprbv+ewd5W9uS0yMlLWh7s6r3m8nLGO1EVERO4jK2ji4uKgUqkAAKWlpVb7mEwm6foZ85lenQkPD0dYWJjdeYFfzi779bzm13LG3v5ZPUl3PaTnCm9bk7etB/C+NXnbeoAetiYh0x/+8AcBQDz33HNW2w8dOiQACACirKxM7rRi2rRpAoB46KGHrLZfunRJqFQqAUB88803Hdq2b98uAAi1Wi0uX75sdfz48eMFAPHkk0/KromIiNxH9p0BMjIyALRfYW/tNOHc3FwAQHJysnQ9jSPz7tq1CydOnLBof/vttyGEQGRkJFJSUjq0paamIiIiAiaTCStXrrQYe+LECezZswcAMGPGDNk1ERGRG8lNpObmZhEdHS0AiOHDh4vS0lIhhBANDQ0iOztb2pvZuXOnxVhz25IlSyzaTCaTGDFihAAgYmNjxeHDh4UQQrS0tIjc3FyhVqsFAJGXl2e1rg8//FDaq8nNzRUtLS1CiPY9rNjYWAFAjB49Wu4yiYjIzWQHjRBCFBcXC61WKwVHaGioFAQqlUosX77c+ofYCRohhLh48aIUCgCERqMRvr6+0mtbh+vMZs2aJfX18/MTGo1Gej1w4EBRUVHhyDKJiMiNHLqOZtiwYSgpKcHy5ctRUFCAiooKaLVa3HfffZg7dy5SU1Od2qsaMGAAiouL8eabbyI/Px/nzp1DSEgIEhMT8fzzz2PatGl2x+fl5WHcuHH46KOPUFxcDKPRiLi4ODz++OOYN28eQkJCnKqrqzU3N2Pbtm34+uuvUVJSggsXLsBgMODmzZsIDg6GVqvFXXfdhfvvvx9TpkyxehPS7sab1vTzzz+jqKgIer0excXFqKysRG1tLerr6xEcHIz+/fsjOTkZU6dOxcSJE+Hj4+PpkjtVV1cHvV6Po0ePSg9rh8b/9Kc/4cCBA11foIO8cRuZNTY2Yt26ddixYwdOnjyJuro6aDQaREVFYfz48Xj66acxdOhQT5dpnaeTjtp99tlnol+/ftKemJzHww8/LCorKz1duk3etqZBgwbJXsewYcPEyZMnPV2yXRs3bpS9nj/96U+eLlcWb9tGZlu3bu1wNMnaw9fXVyxcuFCYTCZPl2uBdwboBtauXYtZs2Y5PO7rr7/G6NGjcezYMfTp08f9hbnAG9dkFhUVheHDh2PgwIHQarXw8/NDXV0djh8/joMHD+LWrVs4ceIExowZg0OHDiE+Pt7TJVvV2trq6RIU4y3bCAA2b96M9PR06cbG4eHhmDx5MmJjY2E0GlFUVIS9e/eira0NOTk5aGpqwrvvvuvZon/N00n3W3f9+nURHBzs0L/6f/1YvHixp5fRgTeuSQgh8vPzxfnz5+32OXPmjEhISJDWkZKS0kXVOe6TTz7xuj0ab9tGVVVVHb5zzsjIEI2NjRb9CgsLRXh4uNSvoKDAA9XaxqDxMFuHLyIiIsTnn38uqqqqhMFgEMeOHRMTJkyw2nfEiBGeXkYH3rgmR5SUlEjrUKlUoqqqytMlWfXJJ5+IgIAAMWLECPHXv/5V/POf/xRlZWU9Omjk6inbaNGiRVKdSUlJ4ubNmzb77ty5U+obFxfXrQ6hMWg8bPny5Vb/x16/fr1FX6PRKAICAiz6Dh482AOV2+aNa3JURESEtJYjR454uhyrDAaDuHHjhsX7v4WgEaJnbKPhw4dLNa5bt67T/vfcc4/U/9ChQ11QoTyyL9gkZdj6iQPzLX86ew8A7r77brfW5CpvXJMjbt682eEnKW7/qYruRKPRwM/Pz9NleERP2Ub/+9//pOfJycmd9r+9z+bNmxWpyRkMGg/74x//aPX9+fPn49///jeuXr2KlpYWnDx5Ek888QRaWlo69FOpVJgzZ05XlCqbN67JER988IF01/CIiAjZ9/6jrtNTttHtd6a397tb1vocO3ZMkZqc4uldKhJi4sSJTn1h3qtXL/Hhhx96unyrvHFNtty6dUvU1dWJffv2iZkzZ3ZYz5o1azxdnsOsbZeefuisp26jyMhIqc49e/Z02v/27zy1Wm0XVCgPg6YbaGhosPmluLVHnz59xIsvvih++uknT5dukzeu6Xbz5s2zu57AwEDx6aeferpMp3hL0HjDNpo0aZJU7+zZs+32vXr1aocz1ADYPXmgK/HQWTcQEhKCgoIC7NixQ9Zx2Pr6epw6dQrnz5/vguqc441rkuuJJ57A+fPn8dRTT3m6FLKhp2yjJ554Qnqel5eHwsJCq/1MJhNefPFFi5+rb2hoULQ+uVRC/P+rgMijdu/ejSVLluDw4cMOjcvOzsaKFSsUqso13rgms2+//RZFRUUA2r9Yrq6uxuHDh6X3YmJisHLlSvz5z3/2ZJlOsXaCRk+5Bc3tvGEb3bp1CyNGjJC+bwkMDMS8efMwffp0xMTEwGg04r///S9ycnJw4MABBAQEdPjO8+LFixgwYICnyv+Fp3epSIj/+7//s9itv+OOO8SqVatEeXm5uHr1qjh48KAYNWqU1UMAH3zwgaeXYMEb1yRHUVFRh9ug9MR1WNsePfHQmS09bRtdvHhR3H333Z0efg4KChJr167t8F59fb2nyxdC8Dsaj9u9e7fV/2h27Nhh0bepqUnExMRY9A0PDxetra0eqN46b1yTI86dOyfdGcHPz6/HfO9k5u1BI0TP20YNDQ0iKytLhISE2Nw+p0+fFkeOHJHe8/Hx6TYXbfI7Gg97++23Ld6LjIzEww8/bPF+UFAQpk+fbvF+TU0N9Hq9IvU5wxvX5Ijo6Gj85S9/AdB+yGb9+vWeLYgs9LRtFBISgtzcXNTU1ODgwYP4+OOP8d5772Hjxo346aefcODAAcTFxXW47mbQoEE2r1PraryppocdOXLE4r0777zTZn9bbRcuXMCoUaPcVpcrvHFNjrr33nul59Z+OZY8ryduI39/f4wZMwZjxoyx2l5cXCw9v//++7uoqs5xj8bDmpqaLN67ePGizf622vz9/d1Wk6u8cU2Oamtrk57funXLg5WQLd64jbZs2SI9f+ihhzxYSUcMGg+LiIiweK+yshI7d+60eN9oNOLzzz+3Oo+9PYau5o1rctTRo0el593irB+y4G3baMeOHfjxxx8BAH379u1WZ9MxaDzM1u1aZsyYgby8PJw/fx7Xr1/Hd999h7S0NJSXl1v01Wq1SEpKUrpU2bxxTY4oLS3Fl19+Kb0eP368B6uxTQiBtrY2i4erfXuCnrKN5Lpy5Qpmz54tvV60aBF69erlwYp+xdNnI/zW/ec//5F99bytx7Jlyzy9jA68cU3/+Mc/xGuvvSYuXrxot19BQUGHXxWNj4/vtmfP7d+/3+XtVF5e7ullSLxxGwnRXu+BAwdsnkF25MgRERcXJ60nJSWl25xtZsaTATzsgQcewMKFC5GTk+PU+AcffBBZWVlurso13rimuro6vPfee1i6dCmGDRuGxMREDBgwABqNBkajEefOncO3336Ls2fPSmPCwsLw2Wefda9/WXoxb91Ghw8fxrJly9C/f3+MGTMGd911FwIDA1FdXY3CwkLo9Xrp1zdHjBiB/Pz8bnO2mcTTSUft1qxZ0+EX8jp7+Pr6iqysLGE0Gj1duk3etKa5c+c69C/9cePGdftrM7xtj8Ybt5EQHX/8zNbDx8dHvPDCC8JgMHi6XKt4C5pupKmpCQUFBfjmm2/www8/4MKFCzAYDGhra4NGo0FYWBjuuecejBkzBk888QQiIyM9XXKnvGVNbW1tOHToEA4cOICjR4/ixx9/RGVlJZqamhAQEIA+ffpg8ODBuO+++zBt2jRZ93fztAMHDiAlJcWlOcrLyxETE+OeglzkjdsIACoqKrB9+3bs378fJSUluHLlCq5fvw6tVovf//73SEtLw/Tp0xEfH+/pUm1i0BARkaJ41hkRESmKQUNERIpi0BARkaIYNEREpCgGDRERKYpBQ0REimLQEBGRohg0RESkKAYNEREpikFDRESKYtAQEZGiGDRERKSo/we7VQqrryHdWgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZoAAAENCAYAAAA2ZaOYAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAAmrklEQVR4nO3df1RUdf4/8OcMIL8G1B1AAVtATwliCcKq6WYHMan8kW1KiXuy/hD74eZ6iDR107aPWIaVnU4m/sh2j/ZDD60Klr91C5UcFQ0UqxV/ISSgwgADivP+/sF3buL84M4Ml4Hp+ThnzpmZ9/t95/U+t3h677zvHZUQQoCIiEghalcXQERE7o1BQ0REimLQEBGRohg0RESkKAYNEREpytPVBXQ1QUFBiIyMdHUZRETdyvnz51FdXW2xjUFzl8jISOh0OleXQUTUrSQmJlpt46kzIiJSFIOGiIgUxaAhIiJFMWiIiEhRDBoiIlIUg4aIiBTFoCEiIkUxaIiISFEMGiIiUhSDhoiIFCU7aPR6PbZt24Z//OMfeOyxxxAUFASVSgWVSoXS0lKnC6mrq8OiRYsQExMDPz8/aLVaJCcnY8uWLbLGb968GWPGjIFWq4Wfnx9iYmKwaNEi6PV6p2sjIiInCJm+/vprAcDi48yZM3I3Y9GlS5dEVFSUtD2NRiM8PT2l1y+++KLN8TNnzpT6enp6Co1GI73u37+/KC8vl11LQkKCU3MhIvo9svW3065TZyEhIXj88cexePFi5OTkdFTQYcqUKSgrK0NkZCQKCgqg1+uh1+uxfPlyqNVqrFq1CmvWrLE43tSmVqvx7rvvor6+Hnq9HgUFBYiIiMC5c+eQmpraIbUSEZED5KZVS0tLm9dlZWUdckRjOlJSq9XixIkTZu1///vfBQDRt29f0dzc3KatqalJhISECABi7ty5ZmOPHz8uVCqVACC2bdsmqx4e0RAR2a9Djmg8PDwUCbqNGzcCAMaOHYu4uDiz9ldffRUqlQqVlZXYt29fm7Y9e/bg6tWrUKlUyMjIMBsbHx+PsWPHtvkcIiLqXC5fdbZ//34AQEpKisX28PBwxMbGAoBZ0JjGDh48GOHh4RbHm7Z791giIuocLg2aq1evoqamBgCkMLFk0KBBAIDTp0+3ed/0Ws7Yqqoqq7/+RkREynFp0FRUVEjPw8LCrPYztd3Z/87XcsZaGk9ERMpz6U85NzQ0SM99fX2t9vPz8wMA1NfXWxwvZ6yl8SY5OTnSKrqqqqp2qiYiInu4/DuariA9PR06nQ46nQ7BwcGuLoeIyK24NGj8/f2l5waDwWq/xsZGAIBGo7E4Xs5YS+OJiEh5Lg2aO78/uXLlitV+prbQ0FCL4+WMtTSeiIiU59KgCQ4ORlBQEACgpKTEaj/T6jLTCjIT02s5Y+/8LCIi6jwu/44mKSkJALB7926L7eXl5VKQJCcnWxxbUlJidUXZrl27LI4lIqLO4fKgSUtLA9AaCCdPnjRrf++99yCEQGhoqBQsJsnJyQgJCYHRaMSKFSvMxp48eRJ79uwBAEyfPl2B6omIqD12BU11dbX0uH79uvT+jRs32rQZjcY240w/J7BkyRKzbT7xxBMYPnw4jEYjnnzySRw5cgQA0NzcjBUrVuCDDz4AALz55pvo0aNHm7He3t7SNt9//32sWLECzc3NAIDDhw/jySefhNFoxKhRozBhwgR7pkpERB3FnpumwcrPBNz9KCsrszhu8eLFFrfb3s8EvPDCCzbruvNnAry8vPgzAUREnazDfiZAKf369UNRUREWLFiA6OhotLS0ICAgAElJSfjqq6+watUqm+NzcnLw5ZdfIikpCRqNBi0tLYiOjsbChQtRVFRk884BRESkLJUQQri6iK4kMTEROp3O1WUQEXUrtv52dokjGiIicl8MGiIiUhSDhoiIFMWgISIiRTFoiIhIUQwaIiJSFIOGiIgUxaAhIiJFMWiIiEhRDBoiIlIUg4aIiBTFoCEiIkUxaIiISFEMGiIiUhSDhoiIFMWgISIiRTFoiIhIUQwaIiJSFIOGiIgUxaAhIiJFMWiIiEhRDBoiIlIUg4aIiBTFoCEiIkUxaIiISFEMGiIiUhSDhoiIFMWgISIiRTFoiIhIUQwaIiJSFIOGiIgUZXfQVFZWYs6cORgwYAB8fHzQp08fTJw4EXv37nWogMjISKhUKlmPzz77zGy8nHFbtmxxqDYiInKepz2dT506hTFjxqCmpgYAEBgYiOrqauTl5SE/Px9ZWVmYP3++XQUEBwejqanJantDQwPq6+sBAEOHDrXaLygoCB4eHhbbfHx87KqJiIg6kJCpsbFRRERECAAiPj5eFBcXCyGEqK2tFRkZGQKAUKlUYufOnXI3KcukSZMEADF06FCL7QAEAFFWVtYhn5eQkNAh2yEi+j2x9bdT9qmz1atX48KFC9BoNNi+fTtiY2MBtB7VZGdnY/LkyRBC4PXXX++wEKyqqsI333wDAJgxY0aHbZeIiDqP7KDZuHEjACAtLQ3h4eFm7ZmZmQCA48eP4+zZsx1S3KZNm3Dr1i14eXkhLS2tQ7ZJRESdS1bQ6PV6HDt2DACQkpJisc+IESPQs2dPAHB4YcDdTF/+jx8/HkFBQR2yTSIi6lyygubMmTMQQgCAdMrMbENqNQYOHAgAOH36tNOF/fjjjzhx4gQAeafNUlNT0bt3b3h7e6Nfv3546qmnkJ+f73QdRETkHFlBU1FRIT0PCwuz2s/Udmd/R23YsAFA62qy8ePHt9v/6NGjuH37Nry8vFBeXo7c3FxMmDABqampuHnzptP1EBGRY2QFTUNDg/Tc19fXaj8/Pz8AkJYjO6qlpaXNd0JeXl5W+86YMQPffvstrl+/jrq6OtTX1+PMmTN4/vnnAQCbN2/G7NmzbX5eTk4OEhMTkZiYiKqqKqdqJyKitrrknQF27tyJX3/9FUD7p802bNiAlJQU9OrVS3ovOjoa69evlxYorF271uYChfT0dOh0Ouh0OgQHBzs/ASIiksgKGn9/f+m5wWCw2q+xsREAoNFonCrKtAjg/vvvt3mRZnsWL14MX19fCCGQl5fnVE1EROQYWUFz5/cyV65csdrP1BYaGupwQdevX8e2bdsAOH/tjL+/PwYPHgwAOHfunFPbIiIix8gKmujoaKhUKgBASUmJxT5Go1E6PTVo0CCHC/riiy/Q3NwMT09PTJ8+3eHtEBFR1yAraAICApCYmAgA2L17t8U+hYWFqK2tBQAkJyc7XJDptFlKSgr69u3r8HaA1kUMxcXFAICoqCintkVERI6RvRjAdGX+xo0bLS5fzs7OBgAkJCRI19PY6+zZsygsLAQg77SZ6doea9566y0YDAaoVCo8/vjjDtVERETOkR00s2bNQkREBPR6PSZMmCBdlKnX6/Haa68hNzcXAJCVlWU21nS7/iVLltj8DNPRTO/evTFp0qR2a0pNTcXChQuh0+naXCtz9uxZzJw5E++88w6A1tBy5nQeERE5TvbPBPj6+mLr1q1ITk7G8ePHERsbi8DAQNTX18NoNEKlUiErKwvjxo1zqBCj0Yh///vfAIBnnnkG3t7e7Y6pqqrCli1bkJWVBQ8PD/Ts2RPNzc1trvuZMmUKPvnkE4dqIiIi59n1ezRDhgxBcXExli1bhry8PJSXl0Or1WLYsGGYO3euU9/N7Nu3D5cvXwYgf7XZggUL8MADD+DIkSO4fPkyrl27BrVajaioKIwYMQLPPfecw8FHREQdQyXa+6LjdyYxMRE6nc7VZRARdSu2/nZ2yTsDEBGR+2DQEBGRohg0RESkKAYNEREpikFDRESKYtAQEZGiGDRERKQoBg0RESmKQUNERIpi0BARkaIYNEREpCgGDRERKYpBQ0REimLQEBGRohg0RESkKAYNEREpikFDRESKYtAQEZGiGDRERKQoBg0RESmKQUNERIpi0BARkaIYNEREpCgGDRERKYpBQ0REimLQEBGRohg0RESkKAYNEREpikFDRESKYtAQEZGi7A6ayspKzJkzBwMGDICPjw/69OmDiRMnYu/evQ4VcODAAahUqnYf1dXVNrezefNmjBkzBlqtFn5+foiJicGiRYug1+sdqouIiDqGpz2dT506hTFjxqCmpgYAEBgYiOrqauTl5SE/Px9ZWVmYP3++Q4Wo1WoEBwfbbLcmPT0da9asAQB4enrCx8cHpaWlWLp0KT7//HN89913CAsLc6guIiJyjuwjGoPBgEmTJqGmpgbx8fEoLi5GbW0trl+/joyMDAghsGDBAuzatcuhQu655x5UVlZaffzhD3+wOG7VqlVYs2YN1Go13n33XdTX10Ov16OgoAARERE4d+4cUlNTHaqJiIicJztoVq9ejQsXLkCj0WD79u2IjY0F0HpUk52djcmTJ0MIgddff12xYu/W3NyMJUuWAADmzJmDV199Fd7e3gCAkSNH4uuvv4ZKpUJBQQG2b9/eaXUREdFvZAfNxo0bAQBpaWkIDw83a8/MzAQAHD9+HGfPnu2g8mzbs2cPrl69CpVKhYyMDLP2+Ph4jB07FsBv9RMRUeeSFTR6vR7Hjh0DAKSkpFjsM2LECPTs2RMAHF4YYK/9+/cDAAYPHmwx/IDf6t23b1+n1ERERG3JCpozZ85ACAEA0ikzsw2p1Rg4cCAA4PTp03YXUlVVhaFDh8Lf3x/+/v647777kJ6ejh9//NHqGNPnWKsJAAYNGiRtv72Va0RE1PFkBU1FRYX03NbqLVPbnf3lamxsxIkTJ+Dt7Y2Wlhb8/PPPWLNmDeLj45GdnW2zLjk1OVoXERE5R1bQNDQ0SM99fX2t9vPz8wMA1NfXyy6gV69eyMzMhE6ng8FgwLVr19DY2IiDBw9i5MiRuH37NjIzM7Fp0yardcmpyVZdOTk5SExMRGJiIqqqqmTXTkRE7XP5nQHi4uKwfPlyJCQkwMfHBwDg4eGB0aNHY//+/Rg1ahQAYN68eTAajYrUkJ6eDp1OB51OZ/NaHiIisp+soPH395eeGwwGq/0aGxsBABqNxsmyWvXo0QNvvfUWAODy5cs4ceKExbrk1NSRdRERkXyygubO7zmuXLlitZ+pLTQ01MmyfjN8+HDp+blz5yzWJaemjq6LiIjkkRU00dHRUKlUAICSkhKLfYxGo3T9jGmll9JMn2OtJuC3lWnBwcEICgrqlLqIiOg3soImICAAiYmJAIDdu3db7FNYWIja2loAQHJycgeV17pdk6ioqDZtSUlJAFqDxtqKMtMtcTqyJiIikk/2YoC0tDQArVfYW/qjblqCnJCQIF1PI4fp+hxLbt26hTfeeANA62mvoUOHtmlPTk5GSEgIjEYjVqxYYTb+5MmT2LNnDwBg+vTpsmsiIqIOJGRqbGwUERERAoAYOnSoKCkpEUIIUVdXJzIzMwUAAUDs3LnTbKypbfHixWZtgwYNEh9++KH46aefhNFoFEII0dLSIr777jvx0EMPSWM3bNhgsa6PP/5YABBqtVpkZ2eLpqYmIYQQhw4dElFRUQKAGDVqlNxpioSEBNl9iYiola2/nbKDRgghioqKhFarlf74BwYGCrVaLQAIlUolli1bZvlDbASNqQ2A8Pb2FkFBQaJHjx7Se56enuLtt9+2WdfMmTOl/l5eXkKj0Uiv+/fvL8rLy2XPkUFDRGQ/W3877bqOZsiQISguLsYrr7yC/v37o7m5GVqtFuPHj8fu3bsd+i2a1atX49lnn0VsbCwCAwNx48YNeHt74/7778fs2bNx8uRJzJs3z+Y2cnJy8OWXXyIpKQkajQYtLS2Ijo7GwoULUVRUxN+iISJyIZUQNr4k+R1KTEyETqdzdRlERN2Krb+dLr8zABERuTcGDRERKYpBQ0REimLQEBGRohg0RESkKAYNEREpikFDRESKYtAQEZGiGDRERKQoBg0RESmKQUNERIpi0BARkaIYNEREpCgGDRERKYpBQ0REimLQEBGRohg0RESkKAYNEREpikFDRESKYtAQEZGiGDRERKQoBg0RESmKQUNERIrydHUB7iRyfr6rS5Dl/NvjZfd1xzkRUefiEQ0RESmKRzRE3RyPOqmr4xENEREpikFDRESKYtAQEZGiGDRERKQoBg0RESnK7qCprKzEnDlzMGDAAPj4+KBPnz6YOHEi9u7d61ABVVVVWL16NaZOnSpt09/fHzExMZg9ezZ++eUXm+NVKlW7jy1btjhUGxEROc+u5c2nTp3CmDFjUFNTAwAIDAxEdXU18vLykJ+fj6ysLMyfP9+uAsLCwtDS0iK91mg0uHnzJkpLS1FaWop169Zh/fr1mDZtms3tBAUFwcPDw2Kbj4+PXTUREVHHkX1EYzAYMGnSJNTU1CA+Ph7FxcWora3F9evXkZGRASEEFixYgF27dtlVQEtLC0aPHo3PPvsMFRUV0Ov1aGxsxPfff4+4uDg0NTXh2WefxalTp2xu5+jRo6isrLT4mDBhgl01ERFRx5EdNKtXr8aFCxeg0Wiwfft2xMbGAmg9qsnOzsbkyZMhhMDrr79uVwEHDx7EwYMH8eyzz6Jv374AAA8PD4waNQq7du1CSEgIWlpa8P7779u1XSIi6hpkB83GjRsBAGlpaQgPDzdrz8zMBAAcP34cZ8+elV3A6NGjrbYFBwfj8ccfBwAcO3ZM9jaJiKjrkBU0er1e+kOfkpJisc+IESPQs2dPAHB4YYAlWq0WAHD79u0O2yYREXUeWUFz5swZCCEAQDplZrYhtRoDBw4EAJw+fbqDyms9tQYAgwcPttkvNTUVvXv3hre3N/r164ennnoK+fnd4x5QRETuTFbQVFRUSM/DwsKs9jO13dnfGVu3boVOpwMAPP/88zb7Hj16FLdv34aXlxfKy8uRm5uLCRMmIDU1FTdv3rQ5NicnB4mJiUhMTERVVVWH1E5ERK1kBU1DQ4P03NfX12o/Pz8/AEB9fb2TZQHl5eVIT08HAEyaNAmPPvqoxX4zZszAt99+i+vXr6Ourg719fU4c+aMFEybN2/G7NmzbX5Weno6dDoddDodgoODna6diIh+0yXvDFBfX4/Jkyfj6tWriIiIwLp166z23bBhA1JSUtCrVy/pvejoaKxfv15aoLB27Vq7FigQEVHHkRU0/v7+0nODwWC1X2NjI4DWiy4d1dTUhCeeeEI6uti5cyeCgoIc2tbixYvh6+sLIQTy8vIcromIiBwnK2ju/F7mypUrVvuZ2kJDQx0q5ubNm5gyZQr27duHXr16YdeuXdICA0f4+/tLiwjOnTvn8HaIiMhxsoImOjoaKpUKAFBSUmKxj9FolE5PDRo0yO5CWlpaMG3aNOTn50Oj0WDHjh2Ii4uzeztERNS1yAqagIAAJCYmAgB2795tsU9hYSFqa2sBAMnJyXYVYTQaMWPGDOTm5sLX1xfbtm3Dgw8+aNc2LGloaEBxcTEAICoqyuntERGR/WQvBkhLSwPQeocAS8uXs7OzAQAJCQl2ne4SQiA9PR2bNm1Cjx49kJubi6SkJNljbXnrrbdgMBigUqmkOwwQEVHnkh00s2bNQkREBPR6PSZMmCBdlKnX6/Haa68hNzcXAJCVlWU21nS7/iVLlpi1zZ07F+vWrYOnpye++uorq8uYLUlNTcXChQuh0+naXCtz9uxZzJw5E++88w6A1iXQjpzOIyIi58n+mQBfX19s3boVycnJOH78OGJjYxEYGIj6+noYjUaoVCpkZWVh3Lhxsj/84sWLWLlyJYDWMJo1axZmzZpltX9lZWWb11VVVdiyZQuysrLg4eGBnj17orm5uc11P1OmTMEnn3wiuyYiIupYdv0ezZAhQ1BcXIxly5YhLy8P5eXl0Gq1GDZsGObOnevQdzMmt27dwq+//mrX+AULFuCBBx7AkSNHcPnyZVy7dg1qtRpRUVEYMWIEnnvuObuCj4iIOp5dQQMAffv2xcqVK6UjETmsfZcSGRnZ7vcstowbN45BQkTUxXXJOwMQEZH7YNAQEZGiGDRERKQoBg0RESmKQUNERIpi0BARkaIYNEREpCgGDRERKYpBQ0REimLQEBGRohg0RESkKAYNEREpikFDRESKYtAQEZGiGDRERKQoBg0RESmKQUNERIpi0BARkaIYNEREpCgGDRERKYpBQ0REimLQEBGRohg0RESkKAYNEREpikFDRESKYtAQEZGiGDRERKQoBg0RESmKQUNERIpi0BARkaLsDprKykrMmTMHAwYMgI+PD/r06YOJEydi7969ThVSV1eHRYsWISYmBn5+ftBqtUhOTsaWLVtkjd+8eTPGjBkDrVYLPz8/xMTEYNGiRdDr9U7VRUREzrEraE6dOoXBgwfjww8/xLlz5+Dt7Y3q6mrk5eXhkUcewdtvv+1QEZcvX0ZcXByWLl2K0tJSeHh4oK6uDvv27cPUqVPx0ksv2Ryfnp6O1NRU7N+/H3V1dfDw8EBpaSmWLl2KuLg4XLlyxaG6iIjIebKDxmAwYNKkSaipqUF8fDyKi4tRW1uL69evIyMjA0IILFiwALt27bKrACEEpkyZgrKyMkRGRqKgoAB6vR56vR7Lly+HWq3GqlWrsGbNGovjTW1qtRrvvvsu6uvrodfrUVBQgIiICJw7dw6pqal21URERB1HdtCsXr0aFy5cgEajwfbt2xEbGwsACAwMRHZ2NiZPngwhBF5//XW7Cti6dSsKCwuhVqvx9ddfY+TIkQAAHx8fZGZm4pVXXgEAvPHGG7h582absc3NzViyZAkAYM6cOXj11Vfh7e0NABg5ciS+/vprqFQqFBQUYPv27XbVRUREHUN20GzcuBEAkJaWhvDwcLP2zMxMAMDx48dx9uxZ2QWYtjt27FjExcWZtb/66qtQqVSorKzEvn372rTt2bMHV69ehUqlQkZGhtnY+Ph4jB07ts3nEBFR55IVNHq9HseOHQMApKSkWOwzYsQI9OzZEwDsWhiwf/9+m9sNDw+Xjp7uDhrT2MGDB1sMvzu3e/dYIiLqHJ5yOp05cwZCCACQ/ujfTa1WY+DAgfjhhx9w+vRpWR9+9epV1NTU2NwuAAwaNAjFxcVm2zW9bm8sAFRVVaG6uhpBQUGyaiP3FDk/39UlyHL+7fGuLsFl3HEfueOc7CHriKaiokJ6HhYWZrWfqe3O/kpu1/Razlh76iIioo4j64imoaFBeu7r62u1n5+fHwCgvr5e1oc7u13TeDljbdWVk5ODnJwcAEBpaSkSExPbqdwyJY6VqqqqEBwc3KHbTExcLLuvu83J3eYDuN+c3G0+gHvO6W7nz5+32iYraNxdeno60tPTXV2GRYmJidDpdK4uo0O525zcbT6A+83J3eYDdK85yTp15u/vLz03GAxW+zU2NgIANBqNrA93drum8XLG2lMXERF1HFlBc+f3HLausje1hYaGyvpwZ7drGi9nrD11ERFRx5EVNNHR0VCpVACAkpISi32MRqN0/YxppVd7goODpVVg1rYL/La67O7tml7LGXvnZ3UnXfWUnjPcbU7uNh/A/ebkbvMButmchEx/+tOfBADxwgsvWGw/dOiQACAAiNLSUrmbFVOnThUAxKOPPmqx/fLly0KlUgkA4ttvv23Tlp+fLwAItVotrly5YnH8uHHjBADxzDPPyK6JiIg6juw7A6SlpQFovcLe0jLh7OxsAEBCQgIGDhwoO+hM2921axdOnjxp1v7ee+9BCIHQ0FAkJSW1aUtOTkZISAiMRiNWrFhhNvbkyZPYs2cPAGD69OmyayIiog4kN5EaGxtFRESEACCGDh0qSkpKhBBC1NXViczMTOloZufOnWZjTW2LFy82azMajWL48OECgIiKihKHDx8WQgjR1NQksrOzhVqtFgBETk6Oxbo+/vhj6agmOztbNDU1CSFaj7CioqIEADFq1Ci50yQiog4mO2iEEKKoqEhotVopOAIDA6UgUKlUYtmyZZY/xEbQCCHEpUuXpFAAIDQajfD09JReWztdZzJz5kypr5eXl9BoNNLr/v37i/LycnumSUREHciu62iGDBmC4uJiLFu2DHl5eSgvL4dWq8WwYcMwd+5cJCcnO3RU1a9fPxQVFeGdd95Bbm4uzp8/j4CAAMTFxeHFF1/E1KlTbY7PycnB2LFj8cknn6CoqAgGgwHR0dF46qmnMG/ePAQEBDhUV2drbGzE9u3b8c0336C4uBgXL16EXq/HrVu34O/vD61Wi3vvvRcPPvggJk+ebPEmpF2NO83pl19+QWFhIXQ6HYqKilBRUYHq6mrU1tbC398fYWFhSEhIwJQpUzBhwgR4eHi4uuR21dTUQKfT4ejRo9LD0qnxhx9+GAcOHOj8Au3kjvvIpL6+HuvXr8eOHTtw6tQp1NTUQKPRIDw8HOPGjcNzzz2HwYMHu7pMy1yddNTq888/F3379pWOxOQ8HnvsMVFRUeHq0q1ytzkNGDBA9jyGDBkiTp065eqSbdq0aZPs+Tz88MOuLlcWd9tHJtu2bWtzNsnSw9PTUyxYsEAYjUZXl2uGdwboAtauXYuZM2faPe6bb77BqFGjcOzYMfTq1avjC3OCO87JJDw8HEOHDkX//v2h1Wrh5eWFmpoaHD9+HAcPHsTt27dx8uRJjB49GocOHUJMTIyrS7aoubnZ1SUoxl32EQBs2bIFqamp0o2Ng4ODMWnSJERFRcFgMKCwsBB79+5FS0sLsrKy0NDQgA8++MC1Rd/N1Un3e3fjxg3h7+9v17/6734sWrTI1dNowx3nJIQQubm54sKFCzb7nD17VsTGxkrzSEpK6qTq7Pfpp5+63RGNu+2jysrKNt85p6Wlifr6erN+BQUFIjg4WOqXl5fngmqtY9C4mLXTFyEhIeKLL74QlZWVQq/Xi2PHjonx48db7Dt8+HBXT6MNd5yTPYqLi6V5qFQqUVlZ6eqSLPr000+Fj4+PGD58uPjb3/4m/vWvf4nS0tJuHTRydZd9tHDhQqnO+Ph4cevWLat9d+7cKfWNjo7uUqfQGDQutmzZMov/Y2/YsMGsr8FgED4+PmZ9Bw4c6ILKrXPHOdkrJCREmsuRI0dcXY5Fer1e3Lx50+z930PQCNE99tHQoUOlGtevX99u//vvv1/qf+jQoU6oUB7ZF2ySMqz9xIHplj/tvQcA9913X4fW5Cx3nJM9bt261eYnKe78qYquRKPRwMvLy9VluER32Uf/+9//pOcJCQnt9r+zz5YtWxSpyREMGhf785//bPH9+fPn4z//+Q+uXbuGpqYmnDp1Ck8//TSampra9FOpVJgzZ05nlCqbO87JHh999JF01/CQkBDZ9/6jztNd9tGdd6a39btblvocO3ZMkZoc4upDKhJiwoQJDn1h3qNHD/Hxxx+7unyL3HFO1ty+fVvU1NSIffv2iRkzZrSZz5o1a1xdnt0s7Zfufuqsu+6j0NBQqc49e/a02//O7zy1Wm0nVCgPg6YLqKurs/qluKVHr169xMsvvyx+/vlnV5dulTvO6U7z5s2zOR9fX1/x2WefubpMh7hL0LjDPpo4caJU7+zZs232vXbtWpsVagBsLh7oTDx11gUEBAQgLy8PO3bskHUetra2FqdPn8aFCxc6oTrHuOOc5Hr66adx4cIFPPvss64uhazoLvvo6aeflp7n5OSgoKDAYj+j0YiXX37Z7Ofq6+rqFK1PLpUQ//8qIHKp3bt3Y/HixTh8+LBd4zIzM7F8+XKFqnKOO87J5LvvvkNhYSGA1i+Wr169isOHD0vvRUZGYsWKFfjLX/7iyjIdYmmBRne5Bc2d3GEf3b59G8OHD5e+b/H19cW8efMwbdo0REZGwmAw4IcffkBWVhYOHDgAHx+fNt95Xrp0Cf369XNV+b9x9SEVCfF///d/Zof1vXv3FqtWrRJlZWXi2rVr4uDBg2LkyJEWTwF89NFHrp6CGXeckxyFhYVtboPSHedhaX90x1Nn1nS3fXTp0iVx3333tXv62c/PT6xdu7bNe7W1ta4uXwjB72hcbvfu3Rb/o9mxY4dZ34aGBhEZGWnWNzg4WDQ3N7ugesvccU72OH/+vHRnBC8vr27zvZOJuweNEN1vH9XV1YmMjAwREBBgdf+cOXNGHDlyRHrPw8Ojy1y0ye9oXOy9994zey80NBSPPfaY2ft+fn6YNm2a2ftVVVXQ6XSK1OcId5yTPSIiIvDXv/4VQOspmw0bNri2IDLT3fZRQEAAsrOzUVVVhYMHD2LdunVYuXIlNm3ahJ9//hkHDhxAdHR0m+tuBgwYYPU6tc7Gm2q62JEjR8zeu+eee6z2t9Z28eJFjBw5ssPqcoY7zsleDzzwgPTc0i/Hkut1x33k7e2N0aNHY/To0Rbbi4qKpOcPPvhgJ1XVPh7RuFhDQ4PZe5cuXbLa31qbt7d3h9XkLHeck71aWlqk57dv33ZhJWSNO+6jrVu3Ss8fffRRF1bSFoPGxUJCQszeq6iowM6dO83eNxgM+OKLLyxux9YRQ2dzxznZ6+jRo9LzLrHqh8y42z7asWMHfvrpJwBAnz59utRqOgaNi1m7Xcv06dORk5ODCxcu4MaNG/j++++RkpKCsrIys75arRbx8fFKlyqbO87JHiUlJfjqq6+k1+PGjXNhNdYJIdDS0mL2cLZvd9Bd9pFcv/76K2bPni29XrhwIXr06OHCiu7i6tUIv3f//e9/ZV89b+2xdOlSV0+jDXec0z//+U/x5ptvikuXLtnsl5eX1+ZXRWNiYrrs6rn9+/c7vZ/KyspcPQ2JO+4jIVrrPXDggNUVZEeOHBHR0dHSfJKSkrrMajMTLgZwsYceeggLFixAVlaWQ+MfeeQRZGRkdHBVznHHOdXU1GDlypVYsmQJhgwZgri4OPTr1w8ajQYGgwHnz5/Hd999h3PnzkljgoKC8Pnnn3etf1m6MXfdR4cPH8bSpUsRFhaG0aNH495774Wvry+uXr2KgoIC6HQ66dc3hw8fjtzc3C6z2kzi6qSjVmvWrGnzC3ntPTw9PUVGRoYwGAyuLt0qd5rT3Llz7fqX/tixY7v8tRnudkTjjvtIiLY/fmbt4eHhIV566SWh1+tdXa5FvAVNF9LQ0IC8vDx8++23+PHHH3Hx4kXo9Xq0tLRAo9EgKCgI999/P0aPHo2nn34aoaGhri65Xe4yp5aWFhw6dAgHDhzA0aNH8dNPP6GiogINDQ3w8fFBr169MHDgQAwbNgxTp06VdX83Vztw4ACSkpKc2kZZWRkiIyM7piAnueM+AoDy8nLk5+dj//79KC4uxq+//oobN25Aq9Xij3/8I1JSUjBt2jTExMS4ulSrGDRERKQorjojIiJFMWiIiEhRDBoiIlIUg4aIiBTFoCEiIkUxaIiISFEMGiIiUhSDhoiIFMWgISIiRTFoiIhIUQwaIiJSFIOGiIgU9f8Az+gRePMgOmIAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZoAAAENCAYAAAA2ZaOYAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAAm6ElEQVR4nO3de1BTZ/4/8HcA5Ra8LBdF7BfQaQXRKpitt112EFfaitZulVbcqe0fYt26dR2kKrrV3nBrsZfZTq2g1m5H2yqDq4Ktd92WKmtEpXirXdEqQkVUSCCgIc/vD345lSaBk5BDIH2/ZjKT5DzPk88zp+XtOXnOiUoIIUBERKQQD1cXQERE7o1BQ0REimLQEBGRohg0RESkKAYNEREpysvVBXQ1QUFBiIiIcHUZRETdyuXLl3Hz5k2r2xg0vxAREQGtVuvqMoiIuhWNRmNzG0+dERGRohg0RESkKAYNEREpikFDRESKYtAQEZGiGDRERKQoBg0RESmKQUNERIpi0BARkaIYNEREpCjZQaPT6bBz5078/e9/x2OPPYagoCCoVCqoVCqcP3++w4XU1dVh+fLliI6Ohp+fHwIDA5GYmIi8vDxZ/bdt24YJEyYgMDAQfn5+iI6OxvLly6HT6TpcGxERdYCQafv27QKA1ce5c+fkDmPV1atXRWRkpDSeWq0WXl5e0ut58+a12X/OnDlSWy8vL6FWq6XXgwYNEhUVFbJrGTVqVIfmQkT0a9TW3067Tp2FhITg8ccfx4oVK5CTk+OsoMP06dNRXl6OiIgIFBUVQafTQafTYfXq1fDw8MDatWuRm5trtb95m4eHB95++23o9XrodDoUFRUhPDwcly5dQkpKilNqJSIiB8hNK6PR2Op1eXm5U45ozEdKHh4e4uTJkxbb//a3vwkAon///qKpqanVtsbGRhESEiIAiIULF1r0LSkpESqVSgAQO3fulFUPj2iIiOznlCMaT09PRYJu8+bNAICJEydi5MiRFtsXLVoElUqFqqoqHDx4sNW2/fv348aNG1CpVEhPT7foGxsbi4kTJ7b6HCIi6lwuX3V26NAhAEBSUpLV7WFhYYiJiQEAi6Ax9x02bBjCwsKs9jeP+8u+RETUOVwaNDdu3EBNTQ0ASGFizdChQwEAZ8+ebfW++bWcvtXV1TZ//Y2IiJTj0qCprKyUng8YMMBmO/O2+9vf/1pOX2v9iYhIeS79Kef6+nrpua+vr812fn5+AAC9Xm+1v5y+1vqb5eTkSKvoqqur26natoglhQ737UyX/zHZ1SUQ0a+Iy7+j6QrS0tKg1Wqh1WoRHBzs6nKIiNyKS4PG399fem4wGGy2a2hoAACo1Wqr/eX0tdafiIiU59Kguf/7k+vXr9tsZ94WGhpqtb+cvtb6ExGR8lwaNMHBwQgKCgIAnDlzxmY78+oy8woyM/NrOX3v/ywiIuo8Lv+OJiEhAQCwb98+q9srKiqkIElMTLTa98yZMzZXlO3du9dqXyIi6hwuD5rU1FQALYFw+vRpi+3vvPMOhBAIDQ2VgsUsMTERISEhMJlMWLNmjUXf06dPY//+/QCAWbNmKVA9ERG1x66guXnzpvS4ffu29P6dO3dabTOZTK36mX9OYOXKlRZjPvHEExg9ejRMJhOefPJJHDt2DADQ1NSENWvW4L333gMAvPrqq+jZs2ervt7e3tKY7777LtasWYOmpiYAwNGjR/Hkk0/CZDJh/PjxSE5OtmeqRETkJHZdR2Nr6e/YsWNbvTbfiVkOlUqFvLw8xMfHo7y8HGPHjoVarUZjYyOMRiMA4IUXXsCcOXOs9p83bx5OnjyJ3NxcLFq0CEuXLoW3t7d0zcygQYOwdetWmTMkIiJnc/mpMwAYOHAgTp06hczMTERFRcFoNCIgIAAJCQnYunUr1q5d22b/nJwcfPHFF0hISIBarYbRaERUVBSWLVuGU6dOtXnnACIiUpZKCCFcXURXotFooNVqHerLOwMQ0a9VW387u8QRDRERuS8GDRERKYpBQ0REimLQEBGRohg0RESkKAYNEREpikFDRESKYtAQEZGiGDRERKQoBg0RESmKQUNERIpi0BARkaIYNEREpCgGDRERKYpBQ0REimLQEBGRohg0RESkKAYNEREpikFDRESKYtAQEZGiGDRERKQoBg0RESmKQUNERIpi0BARkaIYNEREpCgGDRERKYpBQ0REimLQEBGRohg0RESkKAYNEREpikFDRESKsjtoqqqqsGDBAgwePBg+Pj7o168fpkyZggMHDjhUQEREBFQqlazHJ598YtFfTr+8vDyHaiMioo7zsqdxaWkpJkyYgJqaGgBAr169cPPmTRQUFKCwsBBZWVlYsmSJXQUEBwejsbHR5vb6+nro9XoAQFxcnM12QUFB8PT0tLrNx8fHrpqIiMh5ZAeNwWDA1KlTUVNTg9jYWHz66aeIiYlBXV0dXnvtNaxZswaZmZmIi4vDpEmTZBdw/PjxNrc/8cQT2LlzJ+Li4jB8+PA2x4mIiJD9uURE1Dlknzpbt24drly5ArVajV27diEmJgZAy1FNdnY2pk2bBiEEli5d6rTiqqur8eWXXwIAZs+e7bRxiYio88gOms2bNwMAUlNTERYWZrE9IyMDAFBSUoILFy44pbgtW7bg3r176NGjB1JTU50yJhERdS5ZQaPT6XDixAkAQFJSktU2Y8aMQe/evQHA4YUBv2T+8n/y5MkICgpyyphERNS5ZAXNuXPnIIQAAOmUmcVAHh4YMmQIAODs2bMdLuy7777DyZMnAcg7bZaSkoK+ffvC29sbAwcOxFNPPYXCwsIO10FERB0jK2gqKyul5wMGDLDZzrzt/vaO2rRpE4CW1WSTJ09ut/3x48fR3NyMHj16oKKiAvn5+UhOTkZKSgru3r3b4XqIiMgxsoKmvr5eeu7r62uznZ+fHwBIy5EdZTQaW30n1KNHD5ttZ8+eja+++gq3b99GXV0d9Ho9zp07h+effx4AsG3bNsyfP7/Nz8vJyYFGo4FGo0F1dXWHaiciota65J0B9uzZg59++glA+6fNNm3ahKSkJPTp00d6LyoqChs3bpQWKKxfv77NBQppaWnQarXQarUIDg7u+ASIiEgiK2j8/f2l5waDwWa7hoYGAIBare5QUeZFAMOHD2/zIs32rFixAr6+vhBCoKCgoEM1ERGRY2QFzf3fy1y/ft1mO/O20NBQhwu6ffs2du7cCaDj1874+/tj2LBhAIBLly51aCwiInKMrKCJioqCSqUCAJw5c8ZqG5PJJJ2eGjp0qMMFff7552hqaoKXlxdmzZrl8DhERNQ1yAqagIAAaDQaAMC+ffustikuLkZtbS0AIDEx0eGCzKfNkpKS0L9/f4fHAVoWMZSVlQEAIiMjOzQWERE5RvZiAPOV+Zs3b7a6fDk7OxsAMGrUKOl6GntduHABxcXFAOSdNjNf22PL66+/DoPBAJVKhccff9yhmoiIqGNkB83cuXMRHh4OnU6H5ORk6aJMnU6Hl19+Gfn5+QCArKwsi77m2/WvXLmyzc8wH8307dsXU6dObbemlJQULFu2DFqtttW1MhcuXMCcOXPw1ltvAWgJrY6cziMiIsfJvnuzr68vduzYgcTERJSUlCAmJga9evWCXq+HyWSCSqVCVlaWXXduvp/JZMKnn34KAHjmmWfg7e3dbp/q6mrk5eUhKysLnp6e6N27N5qamlpd9zN9+nR89NFHDtVEREQdZ9fv0YwYMQJlZWVYtWoVCgoKUFFRgcDAQDzyyCNYuHBhh76bOXjwIK5duwZA/mqzzMxMPPzwwzh27BiuXbuGW7duwcPDA5GRkRgzZgyee+45h4OPiIicQyXa+6LjV0aj0UCr1TrUN2JJ97i32uV/tH9LHyIie7T1t7NL3hmAiIjcB4OGiIgUxaAhIiJFMWiIiEhRDBoiIlIUg4aIiBTFoCEiIkUxaIiISFEMGiIiUhSDhoiIFMWgISIiRTFoiIhIUQwaIiJSFIOGiIgUxaAhIiJFMWiIiEhRDBoiIlIUg4aIiBTFoCEiIkUxaIiISFEMGiIiUhSDhoiIFMWgISIiRTFoiIhIUQwaIiJSFIOGiIgUxaAhIiJFMWiIiEhRDBoiIlIUg4aIiBRld9BUVVVhwYIFGDx4MHx8fNCvXz9MmTIFBw4ccKiAw4cPQ6VStfu4efNmm+Ns27YNEyZMQGBgIPz8/BAdHY3ly5dDp9M5VBcRETmHlz2NS0tLMWHCBNTU1AAAevXqhZs3b6KgoACFhYXIysrCkiVLHCrEw8MDwcHBbW63JS0tDbm5uQAALy8v+Pj44Pz583jzzTfx2Wef4euvv8aAAQMcqouIiDpG9hGNwWDA1KlTUVNTg9jYWJSVlaG2tha3b99Geno6hBDIzMzE3r17HSrkgQceQFVVlc3Hb37zG6v91q5di9zcXHh4eODtt9+GXq+HTqdDUVERwsPDcenSJaSkpDhUExERdZzsoFm3bh2uXLkCtVqNXbt2ISYmBkDLUU12djamTZsGIQSWLl2qWLG/1NTUhJUrVwIAFixYgEWLFsHb2xsAMG7cOGzfvh0qlQpFRUXYtWtXp9VFREQ/kx00mzdvBgCkpqYiLCzMYntGRgYAoKSkBBcuXHBSeW3bv38/bty4AZVKhfT0dIvtsbGxmDhxIoCf6ycios4lK2h0Oh1OnDgBAEhKSrLaZsyYMejduzcAOLwwwF6HDh0CAAwbNsxq+AE/13vw4MFOqYmIiFqTFTTnzp2DEAIApFNmFgN5eGDIkCEAgLNnz9pdSHV1NeLi4uDv7w9/f3889NBDSEtLw3fffWezj/lzbNUEAEOHDpXGb2/lGhEROZ+soKmsrJSet7V6y7zt/vZyNTQ04OTJk/D29obRaMTFixeRm5uL2NhYZGdnt1mXnJocrYuIiDpGVtDU19dLz319fW228/PzAwDo9XrZBfTp0wcZGRnQarUwGAy4desWGhoacOTIEYwbNw7Nzc3IyMjAli1bbNYlp6a26srJyYFGo4FGo0F1dbXs2omIqH0uvzPAyJEjsXr1aowaNQo+Pj4AAE9PT8THx+PQoUMYP348AGDx4sUwmUyK1JCWlgatVgutVtvmtTxERGQ/WUHj7+8vPTcYDDbbNTQ0AADUanUHy2rRs2dPvP766wCAa9eu4eTJk1brklOTM+siIiL5ZAXN/d9zXL9+3WY787bQ0NAOlvWz0aNHS88vXbpktS45NTm7LiIikkdW0ERFRUGlUgEAzpw5Y7WNyWSSrp8xr/RSmvlzbNUE/LwyLTg4GEFBQZ1SFxER/UxW0AQEBECj0QAA9u3bZ7VNcXExamtrAQCJiYlOKq9lXLPIyMhW2xISEgC0BI2tFWXmW+I4syYiIpJP9mKA1NRUAC1X2Fv7o25egjxq1Cjpeho5zNfnWHPv3j288sorAFpOe8XFxbXanpiYiJCQEJhMJqxZs8ai/+nTp7F//34AwKxZs2TXREREziM7aObOnYvw8HDodDokJydLp6R0Oh1efvll5OfnAwCysrIs+ppv9W++L9n9hg0bhn/+85+4ePGiFDrNzc345ptvkJiYiG+++QYAsGrVKos7OHt7e0tjvvvuu1izZg2ampoAAEePHsWTTz4Jk8mE8ePHIzk5We5UiYjIiWT/TICvry927NiBxMRElJSUICYmBr169YJer4fJZIJKpUJWVhYmTZpkVwFnz57FSy+9BKAlOAICAlBXV4e7d++2FOjlhTfeeAOzZ8+22n/evHk4efIkcnNzsWjRIixduhTe3t7SNTODBg3C1q1b7aqJiIicx67raEaMGIGysjK89NJLGDRoEJqamhAYGIjJkydj3759Dv0Wzbp16/Dss89KwXXnzh14e3tj+PDhmD9/Pk6fPo3Fixe3OUZOTg6++OILJCQkQK1Ww2g0IioqCsuWLcOpU6f4WzRERC6kEm19SfIrpNFooNVqHeobsaTQydUo4/I/Jru6BCJyM2397XT5nQGIiMi9MWiIiEhRDBoiIlIUg4aIiBTFoCEiIkUxaIiISFEMGiIiUhSDhoiIFMWgISIiRTFoiIhIUQwaIiJSFIOGiIgUxaAhIiJFMWiIiEhRDBoiIlIUg4aIiBTFoCEiIkUxaIiISFEMGiIiUhSDhoiIFMWgISIiRTFoiIhIUQwaIiJSFIOGiIgUxaAhIiJFMWiIiEhRDBoiIlIUg4aIiBTFoCEiIkUxaIiISFEMGiIiUpTdQVNVVYUFCxZg8ODB8PHxQb9+/TBlyhQcOHDAoQKqq6uxbt06zJgxQxrT398f0dHRmD9/Pn744Yc2+6tUqnYfeXl5DtVGREQd52VP49LSUkyYMAE1NTUAgF69euHmzZsoKChAYWEhsrKysGTJErsKGDBgAIxGo/RarVbj7t27OH/+PM6fP48NGzZg48aNmDlzZpvjBAUFwdPT0+o2Hx8fu2oiIiLnkX1EYzAYMHXqVNTU1CA2NhZlZWWora3F7du3kZ6eDiEEMjMzsXfvXrsKMBqNiI+PxyeffILKykrodDo0NDTgm2++wciRI9HY2Ihnn30WpaWlbY5z/PhxVFVVWX0kJyfbVRMRETmP7KBZt24drly5ArVajV27diEmJgZAy1FNdnY2pk2bBiEEli5dalcBR44cwZEjR/Dss8+if//+AABPT0+MHz8ee/fuRUhICIxGI9599127xiUioq5BdtBs3rwZAJCamoqwsDCL7RkZGQCAkpISXLhwQXYB8fHxNrcFBwfj8ccfBwCcOHFC9phERNR1yAoanU4n/aFPSkqy2mbMmDHo3bs3ADi8MMCawMBAAEBzc7PTxiQios4jK2jOnTsHIQQASKfMLAby8MCQIUMAAGfPnnVSeS2n1gBg2LBhbbZLSUlB37594e3tjYEDB+Kpp55CYWGh0+ogIiLHyAqayspK6fmAAQNstjNvu799R+zYsQNarRYA8Pzzz7fZ9vjx42hubkaPHj1QUVGB/Px8JCcnIyUlBXfv3m2zb05ODjQaDTQaDaqrq51SOxERtZAVNPX19dJzX19fm+38/PwAAHq9voNlARUVFUhLSwMATJ06FY8++qjVdrNnz8ZXX32F27dvo66uDnq9HufOnZOCadu2bZg/f36bn5WWlgatVgutVovg4OAO105ERD/rkncG0Ov1mDZtGm7cuIHw8HBs2LDBZttNmzYhKSkJffr0kd6LiorCxo0bpQUK69evt2uBAhEROY+soPH395eeGwwGm+0aGhoAtFx06ajGxkY88cQT0tHFnj17EBQU5NBYK1asgK+vL4QQKCgocLgmIiJynKyguf97mevXr9tsZ94WGhrqUDF3797F9OnTcfDgQfTp0wd79+6VFhg4wt/fX1pEcOnSJYfHISIix8kKmqioKKhUKgDAmTNnrLYxmUzS6amhQ4faXYjRaMTMmTNRWFgItVqN3bt3Y+TIkXaPQ0REXYusoAkICIBGowEA7Nu3z2qb4uJi1NbWAgASExPtKsJkMmH27NnIz8+Hr68vdu7cibFjx9o1hjX19fUoKysDAERGRnZ4PCIisp/sxQCpqakAWu4QYG35cnZ2NgBg1KhRdp3uEkIgLS0NW7ZsQc+ePZGfn4+EhATZfdvy+uuvw2AwQKVSSXcYICKiziU7aObOnYvw8HDodDokJydLF2XqdDq8/PLLyM/PBwBkZWVZ9DXfrn/lypUW2xYuXIgNGzbAy8sLW7dutbmM2ZqUlBQsW7YMWq221bUyFy5cwJw5c/DWW28BaFkC7cjpPCIi6jjZPxPg6+uLHTt2IDExESUlJYiJiUGvXr2g1+thMpmgUqmQlZWFSZMmyf7wH3/8Ee+//z6AljCaO3cu5s6da7N9VVVVq9fV1dXIy8tDVlYWPD090bt3bzQ1NbW67mf69On46KOPZNdERETOZdfv0YwYMQJlZWVYtWoVCgoKUFFRgcDAQDzyyCNYuHChQ9/NmN27dw8//fSTXf0zMzPx8MMP49ixY7h27Rpu3boFDw8PREZGYsyYMXjuuefsCj4iInI+lWjvi45fGY1GI932xl4RS7rHvdUu/2Oyq0sgIjfT1t/OLnlnACIich8MGiIiUhSDhoiIFMWgISIiRTFoiIhIUQwaIiJSFIOGiIgUxaAhIiJFMWiIiEhRDBoiIlIUg4aIiBTFoCEiIkUxaIiISFEMGiIiUhSDhoiIFMWgISIiRTFoiIhIUQwaIiJSFIOGiIgUxaAhIiJFMWiIiEhRDBoiIlIUg4aIiBTFoCEiIkUxaIiISFEMGiIiUhSDhoiIFMWgISIiRTFoiIhIUQwaIiJSlN1BU1VVhQULFmDw4MHw8fFBv379MGXKFBw4cKBDhdTV1WH58uWIjo6Gn58fAgMDkZiYiLy8PFn9t23bhgkTJiAwMBB+fn6Ijo7G8uXLodPpOlQXERF1jEoIIeQ2Li0txYQJE1BTUwMA6NWrF/R6PUwmE1QqFbKysrBkyRK7i7h27Rri4+NRXl4OAFCr1WhsbITRaAQAzJs3Dx9++KHN/mlpacjNzQUAeHl5wcfHB3q9HgAwaNAgfP311xgwYICsWjQaDbRard1zAICIJYUO9etsl/8x2dUlEJGbaetvp+wjGoPBgKlTp6KmpgaxsbEoKytDbW0tbt++jfT0dAghkJmZib1799pVnBAC06dPR3l5OSIiIlBUVASdTgedTofVq1fDw8MDa9eulYLkl8zbPDw88Pbbb0Ov10On06GoqAjh4eG4dOkSUlJS7KqJiIicR3bQrFu3DleuXIFarcauXbsQExMDoOWoJjs7G9OmTYMQAkuXLrWrgB07dqC4uBgeHh7Yvn07xo0bBwDw8fFBRkYGXnrpJQDAK6+8grt377bq29TUhJUrVwIAFixYgEWLFsHb2xsAMG7cOGzfvh0qlQpFRUXYtWuXXXUREZFzyA6azZs3AwBSU1MRFhZmsT0jIwMAUFJSggsXLsguwDzuxIkTMXLkSIvtixYtgkqlQlVVFQ4ePNhq2/79+3Hjxg2oVCqkp6db9I2NjcXEiRNbfQ4REXUuWUGj0+lw4sQJAEBSUpLVNmPGjEHv3r0BwK6FAYcOHWpz3LCwMOno6ZdBY+47bNgwq+F3/7i/7EtERJ1DVtCcO3cO5jUD5j/6FgN5eGDIkCEAgLNnz8r68Bs3bkgLC2yNCwBDhw61Oq75tZy+1dXVuHnzpqy6iIjIeWQFTWVlpfS8rdVb5m33t1dyXPNrOX3tqYuIiJzHS06j+vp66bmvr6/Ndn5+fgAgLS1Welxzfzl926orJycHOTk5AIDz589Do9G0U7l1QQ71alt1dTWCg4OdOqZGs8Kp49lLiTm5krvNB3C/ObnbfICuN6fLly/b3CYraNxdWloa0tLSXF2GVR25rqercrc5udt8APebk7vNB+hec5J16szf3196bjAYbLZraGgA0HLBZWeMa+4vp689dRERkfPICpr7v+e4fv26zXbmbaGhobI+vKPjmvvL6WtPXURE5DyygiYqKgoqlQoAcObMGattTCaTdP2MeaVXe4KDgxEUFNTmuMDPq8t+Oa75tZy+939Wd9JVT+l1hLvNyd3mA7jfnNxtPkA3m5OQ6be//a0AIF544QWr27/99lsBQAAQ58+flzusmDFjhgAgHn30Uavbr127JlQqlQAgvvrqq1bbCgsLBQDh4eEhrl+/brX/pEmTBADxzDPPyK6JiIicR/adAVJTUwG0XGFvbZlwdnY2AGDUqFHS9TT2jLt3716cPn3aYvs777wDIQRCQ0ORkJDQaltiYiJCQkJgMpmwZs0ai76nT5/G/v37AQCzZs2SXRMRETmR3ERqaGgQ4eHhAoCIi4sTZ86cEUIIUVdXJzIyMqSjmT179lj0NW9bsWKFxTaTySRGjx4tAIjIyEhx9OhRIYQQjY2NIjs7W3h4eAgAIicnx2pdH374oXRUk52dLRobG4UQLUdYkZGRAoAYP3683GkSEZGTyQ4aIYQ4deqUCAwMlIKjV69eUhCoVCqxatUq6x/SRtAIIcTVq1elUAAg1Gq18PLykl7bOl1nNmfOHKltjx49hFqtll4PGjRIVFRU2DNNIiJyIruuoxkxYgTKysqwatUqFBQUoKKiAoGBgXjkkUewcOFCJCYmOnRUNXDgQJw6dQpvvfUW8vPzcfnyZQQEBGDkyJGYN28eZsyY0Wb/nJwcTJw4ER999BFOnToFg8GAqKgoPPXUU1i8eDECAgIcqquzNTQ0YNeuXfjyyy9RVlaGH3/8ETqdDvfu3YO/vz8CAwPx4IMPYuzYsZg2bZrVm5B2Ne40px9++AHFxcXQarU4deoUKisrcfPmTdTW1sLf3x8DBgzAqFGjMH36dCQnJ8PT09PVJberpqYGWq0Wx48flx7WTo3/4Q9/wOHDhzu/QDu54z4y0+v12LhxI3bv3o3S0lLU1NRArVYjLCwMkyZNwnPPPYdhw4a5ukzrXJ101OKzzz4T/fv3l47E5Dwee+wxUVlZ6erSbXK3OQ0ePFj2PEaMGCFKS0tdXXKbtmzZIns+f/jDH1xdrizuto/Mdu7c2epskrWHl5eXyMzMFCaTydXlWuCdAbqA9evXY86cOXb3+/LLLzF+/HicOHECffr0cX5hHeCOczILCwtDXFwcBg0ahMDAQPTo0QM1NTUoKSnBkSNH0NzcjNOnTyM+Ph7ffvstoqOjXV2yVU1NTa4uQTHuso8AIC8vDykpKdKNjYODgzF16lRERkbCYDCguLgYBw4cgNFoRFZWFurr6/Hee++5tuhfcnXS/drduXNH+Pv72/Wv/l8+li9f7upptOKOcxJCiPz8fHHlypU221y4cEHExMRI80hISOik6uz38ccfu90Rjbvto6qqqlbfOaempgq9Xm/RrqioSAQHB0vtCgoKXFCtbQwaF7N1+iIkJER8/vnnoqqqSuh0OnHixAkxefJkq21Hjx7t6mm04o5zskdZWZk0D5VKJaqqqlxdklUff/yx8PHxEaNHjxZ//etfxb/+9S9x/vz5bh00cnWXfbRs2TKpztjYWHHv3j2bbffs2SO1jYqK6lKn0Bg0LrZq1Sqr/2Nv2rTJoq3BYBA+Pj4WbYcMGeKCym1zxznZKyQkRJrLsWPHXF2OVTqdTty9e9fi/V9D0AjRPfZRXFycVOPGjRvbbT98+HCp/bffftsJFcoj+4JNUoatnzgw3/KnvfcA4KGHHnJqTR3ljnOyx71791r9JMX9P1XRlajVavTo0cPVZbhEd9lH//vf/6Tno0aNarf9/W3y8vIUqckRDBoX+93vfmf1/SVLluDf//43bt26hcbGRpSWluLpp59GY2Njq3YqlQoLFizojFJlc8c52eODDz6Q7hoeEhIi+95/1Hm6yz66/870bf3ulrU2J06cUKQmh7j6kIqESE5OdugL8549e4oPP/zQ1eVb5Y5zsqW5uVnU1NSIgwcPitmzZ7eaT25urqvLs5u1/dLdT511130UGhoq1bl///5229//nWdgYGAnVCgPg6YLqKurs/mluLVHnz59xIsvviguXrzo6tJtcsc53W/x4sVtzsfX11d88sknri7TIe4SNO6wj6ZMmSLVO3/+/Dbb3rp1q9UKNQBtLh7oTDx11gUEBASgoKAAu3fvlnUetra2FmfPnsWVK1c6oTrHuOOc5Hr66adx5coVPPvss64uhWzoLvvo6aeflp7n5OSgqKjIajuTyYQXX3zR4ufq6+rqFK1PLpUQ//8qIHKpffv2YcWKFTh69Khd/TIyMrB69WqFquoYd5yT2ddff43i4mIALV8s37hxA0ePHpXei4iIwJo1a/CnP/3JlWU6xNoCje5yC5r7ucM+am5uxujRo6XvW3x9fbF48WLMnDkTERERMBgM+O9//4usrCwcPnwYPj4+rb7zvHr1KgYOHOiq8n/m6kMqEuKNN96wOKzv27evWLt2rSgvLxe3bt0SR44cEePGjbN6CuCDDz5w9RQsuOOc5CguLm51G5TuOA9r+6M7njqzpbvto6tXr4qHHnqo3dPPfn5+Yv369a3eq62tdXX5Qgh+R+Ny+/bts/ofze7duy3a1tfXi4iICIu2wcHBoqmpyQXVW+eOc7LH5cuXpTsj9OjRo9t872Tm7kEjRPfbR3V1dSI9PV0EBATY3D/nzp0Tx44dk97z9PTsMhdt8jsaF3vnnXcs3gsNDcVjjz1m8b6fnx9mzpxp8X51dTW0Wq0i9TnCHedkj/DwcPz5z38G0HLKZtOmTa4tiCx0t30UEBCA7OxsVFdX48iRI9iwYQPef/99bNmyBRcvXsThw4cRFRXV6rqbwYMH27xOrbPxppouduzYMYv3HnjgAZvtbW378ccfMW7cOKfV1RHuOCd7Pfzww9Jza78cS67XHfeRt7c34uPjER8fb3X7qVOnpOdjx47tpKraxyMaF6uvr7d47+rVqzbb29rm7e3ttJo6yh3nZC+j0Sg9b25udmElZIs77qMdO3ZIzx999FEXVtIag8bFQkJCLN6rrKzEnj17LN43GAz4/PPPrY7T1hFDZ3PHOdnr+PHj0vMuseqHLLjbPtq9eze+//57AEC/fv261Go6Bo2L2bpdy6xZs5CTk4MrV67gzp07+Oabb5CUlITy8nKLtoGBgYiNjVW6VNnccU72OHPmDLZu3Sq9njRpkgursU0IAaPRaPHoaNvuoLvsI7l++uknzJ8/X3q9bNky9OzZ04UV/YKrVyP82v3nP/+RffW8rcebb77p6mm04o5zeu2118Srr74qrl692ma7goKCVr8qGh0d3WVXzx06dKjD+6m8vNzV05C44z4SoqXew4cP21xBduzYMREVFSXNJyEhocusNjPjYgAX+/3vf4/MzExkZWU51P+Pf/wj0tPTnVxVx7jjnGpqavD+++9j5cqVGDFiBEaOHImBAwdCrVbDYDDg8uXL+Prrr3Hp0iWpT1BQED777LOu9S9LN+au++jo0aN48803MWDAAMTHx+PBBx+Er68vbty4gaKiImi1WunXN0ePHo38/Pwus9pM4uqkoxa5ubmtfiGvvYeXl5dIT08XBoPB1aXb5E5zWrhwoV3/0p84cWKXvzbD3Y5o3HEfCdH6x89sPTw9PcVf/vIXodPpXF2uVbwFTRdSX1+PgoICfPXVV/juu+/w448/QqfTwWg0Qq1WIygoCMOHD0d8fDyefvpphIaGurrkdrnLnIxGI7799lscPnwYx48fx/fff4/KykrU19fDx8cHffr0wZAhQ/DII49gxowZsu7v5mqHDx9GQkJCh8YoLy9HRESEcwrqIHfcRwBQUVGBwsJCHDp0CGVlZfjpp59w584dBAYG4v/+7/+QlJSEmTNnIjo62tWl2sSgISIiRXHVGRERKYpBQ0REimLQEBGRohg0RESkKAYNEREpikFDRESKYtAQEZGiGDRERKQoBg0RESmKQUNERIpi0BARkaIYNEREpKj/B0ZSJr3HbnTDAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_attribution(\n",
    "    query_ids[0],\n",
    "    input_ids[0,:mask.sum()],\n",
    "    attributions_ig[0,:mask.sum()],\n",
    "    attributions_gt[0,:mask.sum()],\n",
    "    name=\"plots/toy-ig.pdf\",\n",
    "    ymax=1.1,\n",
    "    save=True\n",
    ")\n",
    "plot_attribution(\n",
    "    query_ids[0],\n",
    "    input_ids[0,:mask.sum()],\n",
    "    attributions_schulz[0,:mask.sum()],\n",
    "    attributions_gt[0,:mask.sum()],\n",
    "    \"plots/toy-schulz.pdf\",\n",
    "    ymax=1.1,\n",
    "    save=True\n",
    ")\n",
    "plot_attribution(\n",
    "    query_ids[0],\n",
    "    input_ids[0,:mask.sum()],\n",
    "    attributions_guan[0,:mask.sum()],\n",
    "    attributions_gt[0,:mask.sum()],\n",
    "    \"plots/toy-guan.pdf\",\n",
    "    ymax=1.1,\n",
    "    save=True\n",
    ")\n",
    "plot_attribution(\n",
    "    query_ids[0],\n",
    "    input_ids[0,:mask.sum()],\n",
    "    attributions_ours[0][0,:mask.sum()],\n",
    "    attributions_gt[0,:mask.sum()],\n",
    "    \"plots/toy-ours-input.pdf\",\n",
    "    ymax=1.1,\n",
    "    save=True,\n",
    "    legend=True\n",
    ")\n",
    "plot_attribution(\n",
    "    query_ids[0],\n",
    "    input_ids[0,:mask.sum()],\n",
    "    attributions_ours[1][0,:mask.sum()],\n",
    "    attributions_gt[0,:mask.sum()],\n",
    "    \"plots/toy-ours-hidden.pdf\",\n",
    "    ymax=1.1,\n",
    "    save=True,\n",
    "    legend=True\n",
    ")\n",
    "\n",
    "plot_attribution(\n",
    "    query_ids[0],\n",
    "    input_ids[0,:mask.sum()],\n",
    "    torch.tensor([1.] + [0] * (mask.sum().item() - 1)),\n",
    "    attributions_gt[0,:mask.sum()],\n",
    "    \"plots/toy-exact.pdf\",\n",
    "    ymax=1.1,\n",
    "    save=True,\n",
    "    legend=True\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Evaluate on all validation set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "hidden_state_idx = 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c92b698145a84ecabb98e3e7b5777e2d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=0.0, max=500.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "attributions_ig_all = integrated_gradient(\n",
    "    model_orig,\n",
    "    {\n",
    "        \"query_ids\": model_orig.val_dataset[:][0].to(device),\n",
    "        \"input_ids\": model_orig.val_dataset[:][1].to(device),\n",
    "        \"mask\": model_orig.val_dataset[:][2].to(device),\n",
    "        \"labels\": model_orig.val_dataset[:][3].to(device),\n",
    "    },\n",
    "    getter=toy_getter,\n",
    "    setter=toy_setter,\n",
    "    label_getter=lambda outputs, inputs_dict: outputs,\n",
    "    hidden_state_idx=hidden_state_idx,\n",
    "    steps=500,\n",
    ").sum(-1).abs().cpu() * model_orig.val_dataset[:][2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6dae82a1d77c4d22b1b14ccc0ae2d283",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=0.0, max=500.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "attributions_schulz_all = schulz_explainer(\n",
    "    model_orig,\n",
    "    {\n",
    "        \"query_ids\": model.val_dataset[:][0].to(device),\n",
    "        \"input_ids\": model.val_dataset[:][1].to(device),\n",
    "        \"mask\": model.val_dataset[:][2].to(device),\n",
    "        \"labels\": model.val_dataset[:][3].to(device),\n",
    "    },\n",
    "    getter=toy_getter,\n",
    "    setter=toy_setter,\n",
    "    q_z_loc=all_q_z_loc[hidden_state_idx].to(device).unsqueeze(0).detach(),\n",
    "    q_z_scale=all_q_z_scale[hidden_state_idx].to(device).unsqueeze(0).detach(),\n",
    "    loss_fn=lambda outputs, hidden_states, inputs_dict: torch.nn.functional.binary_cross_entropy_with_logits(\n",
    "        outputs, inputs_dict[\"labels\"].float(), reduction=\"none\"\n",
    "    ).mean(-1),\n",
    "    loss_kl_fn=lambda kl, inputs_dict: kl.mean(-1).mean(-1),\n",
    "    hidden_state_idx=hidden_state_idx,\n",
    "    steps=500,\n",
    "    lr=1e-1,\n",
    "    la=10,\n",
    ").cpu() * model.val_dataset[:][2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fe95e2dfa2f247aba661764a20624953",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=0.0, max=500.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "attributions_guan_all = 1 / guan_explainer(\n",
    "    model_orig,\n",
    "    {\n",
    "        \"query_ids\": model.val_dataset[:][0].to(device),\n",
    "        \"input_ids\": model.val_dataset[:][1].to(device),\n",
    "        \"mask\": model.val_dataset[:][2].to(device),\n",
    "        \"labels\": model.val_dataset[:][3].to(device),\n",
    "    },\n",
    "    getter=toy_getter,\n",
    "    setter=toy_setter,\n",
    "    s_fn=lambda outputs, hidden_states: outputs,\n",
    "    loss_l2_fn=lambda s, inputs_dict: sum(s_i.sum(-1).mean(-1) for s_i in s),\n",
    "    loss_h_fn=lambda h, inputs_dict: h.sum(-1).mean(-1),\n",
    "    hidden_state_idx=hidden_state_idx,\n",
    "    steps=500,\n",
    "    lr=1e-1,\n",
    "    la=1,\n",
    ").cpu() * model.val_dataset[:][2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "attributions_ours_all = model.forward_explainer(\n",
    "    query_ids=model.val_dataset[:][0].to(device),\n",
    "    input_ids=model.val_dataset[:][1].to(device),\n",
    "    mask=model.val_dataset[:][2].to(device),\n",
    "    layer_drop=1,\n",
    "    layer_pred=1,\n",
    ")[3].exp().cpu() * model.val_dataset[:][2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "attributions_gt_all = ((model.val_dataset[:][1] == model.val_dataset[:][0][:,:1]) | (\n",
    "    model.val_dataset[:][1] == model.val_dataset[:][0][:,1:])).float().cpu() * model.val_dataset[:][2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "attributions_exact_tmp = (torch.where(\n",
    "    model.val_dataset[:][3].unsqueeze(-1).bool(),\n",
    "    (model.val_dataset[:][1] == model.val_dataset[:][0][:,:1]).float(),\n",
    "    (model.val_dataset[:][1] == model.val_dataset[:][0][:,1:]).float(),\n",
    ") * model.val_dataset[:][2])\n",
    "attributions_exact = torch.zeros_like(attributions_exact_tmp)\n",
    "attributions_exact[attributions_exact_tmp.sum(-1) > 0,\n",
    "                   attributions_exact_tmp[attributions_exact_tmp.sum(-1) > 0].argmax(-1),] = 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from collections import defaultdict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "metrics = defaultdict(dict)\n",
    "\n",
    "for method_name, attr in zip(\n",
    "    (\"exact\", \"ig\", \"cv\", \"nlp\", \"ours\"),\n",
    "    (attributions_exact, attributions_ig_all, attributions_schulz_all, attributions_guan_all, attributions_ours_all)\n",
    "):\n",
    "\n",
    "    attr = attr[attributions_gt_all.sum(-1) > 0]\n",
    "    \n",
    "    p = torch.distributions.Categorical(probs=attr / attr.sum(-1, keepdim=True))\n",
    "    q = torch.distributions.Categorical(\n",
    "        probs=(attributions_gt_all / attributions_gt_all.sum(-1, keepdim=True))[attributions_gt_all.sum(-1) > 0]\n",
    "    )\n",
    "    m = torch.distributions.Categorical(probs=0.5 * (p.probs + q.probs))\n",
    "\n",
    "    metrics[method_name][\"$D_\\mathrm{KL}$\"] = torch.distributions.kl_divergence(q, p)\n",
    "\n",
    "    metrics[method_name][\"$D_\\mathrm{JS}$\"] = 0.5 * (\n",
    "        torch.distributions.kl_divergence(p, m) + torch.distributions.kl_divergence(q, m)\n",
    "    )\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>$D_\\mathrm{KL}$</th>\n",
       "      <th>$D_\\mathrm{JS}$</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>exact</th>\n",
       "      <td>inf +/- nan</td>\n",
       "      <td>0.2664 +/- 0.156</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ig</th>\n",
       "      <td>1.3551 +/- 1.016</td>\n",
       "      <td>0.2703 +/- 0.155</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cv</th>\n",
       "      <td>1.1589 +/- 0.718</td>\n",
       "      <td>0.1892 +/- 0.090</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>nlp</th>\n",
       "      <td>0.8610 +/- 0.546</td>\n",
       "      <td>0.2320 +/- 0.133</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ours</th>\n",
       "      <td>0.0372 +/- 0.036</td>\n",
       "      <td>0.0128 +/- 0.012</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        $D_\\mathrm{KL}$   $D_\\mathrm{JS}$\n",
       "exact       inf +/- nan  0.2664 +/- 0.156\n",
       "ig     1.3551 +/- 1.016  0.2703 +/- 0.155\n",
       "cv     1.1589 +/- 0.718  0.1892 +/- 0.090\n",
       "nlp    0.8610 +/- 0.546  0.2320 +/- 0.133\n",
       "ours   0.0372 +/- 0.036  0.0128 +/- 0.012"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.DataFrame(\n",
    "    {k: {kk: \"{:.4f} +/- {:.3f}\".format(vv.mean().item(), vv.std().item()) for kk, vv in v.items()} \n",
    "     for k, v in metrics.items()}\n",
    ").T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\\begin{tabular}{lll}\n",
      "\\toprule\n",
      "{} & $D_\\mathrm{KL}$ & $D_\\mathrm{JS}$ \\\\\n",
      "\\midrule\n",
      "Exact                             &           $inf$ &          $0.27$ \\\\\n",
      "\\citet{sundararajan2017axiomatic} &          $1.36$ &          $0.27$ \\\\\n",
      "\\citet{schulz2020restricting}     &          $1.16$ &          $0.19$ \\\\\n",
      "\\citet{guan2019towards}           &          $0.86$ &          $0.23$ \\\\\n",
      "\\textsc{DiffMask}                 &          $0.04$ &          $0.01$ \\\\\n",
      "\\bottomrule\n",
      "\\end{tabular}\n",
      "\n"
     ]
    }
   ],
   "source": [
    "name_map = {\n",
    "    \"exact\": \"Exact\",\n",
    "    \"ig\": \"\\citet{sundararajan2017axiomatic}\",\n",
    "    \"cv\": \"\\citet{schulz2020restricting}\",\n",
    "    \"nlp\": \"\\citet{guan2019towards}\",\n",
    "    \"ours\": \"\\textsc{DiffMask}\",\n",
    "}\n",
    "\n",
    "print(\n",
    "    pd.DataFrame(\n",
    "        {name_map[k]: \n",
    "         {kk:\n",
    "          \"${:.2f}$\".format(vv.mean().item(), vv.std().item())\n",
    "          for kk, vv in v.items()\n",
    "         } \n",
    "         for k, v in metrics.items()\n",
    "        }\n",
    "    ).T.to_latex()\n",
    "    .replace(\"\\$\", \"$\")\n",
    "    .replace(r\"\\textbackslash \", \"\\\\\")\n",
    "    .replace(\"\\{\", \"{\")\n",
    "    .replace(\"\\}\", \"}\")\n",
    "    .replace(\"\\_\", \"_\")\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}