mlp_lm_gpu_data.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "import time\n",
    "import wandb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "device = \"cuda\" if torch.cuda.is_available() else \"cpu\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# load les données\n",
    "\n",
    "fichier = open('villes.txt')\n",
    "donnees = fichier.read()\n",
    "villes = donnees.replace('\\n', ',').split(',')\n",
    "villes = [ville for ville in villes if len(ville) > 2]\n",
    "villes = sorted(villes, key=len)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# création du vocabulaire\n",
    "\n",
    "vocabulaire = sorted(list(set(''.join(villes))))\n",
    "vocabulaire = [\"<pad>\", \"<SOS>\", \"<EOS>\"] + vocabulaire\n",
    "\n",
    "# pour convertir char <-> int\n",
    "char_to_int = {}\n",
    "int_to_char = {}\n",
    "\n",
    "for (c, i) in zip(vocabulaire, range(len(vocabulaire))):\n",
    "    char_to_int[c] = i\n",
    "    int_to_char[i] = c"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_sequences = len(villes)\n",
    "max_len = max([len(ville) for ville in villes]) + 2 # account for <SOS> and <EOS>, which are appened at the beg. and end of each seq\n",
    "\n",
    "X = torch.zeros((num_sequences, max_len), dtype=torch.int32)\n",
    "\n",
    "for i in range(num_sequences):\n",
    "    X[i] = torch.tensor([char_to_int['<SOS>']] + [char_to_int[c] for c in villes[i]] + [char_to_int['<EOS>']] + [char_to_int['<pad>']] * (max_len - len(villes[i]) - 2))\n",
    "\n",
    "n_split = int(0.9*X.shape[0])\n",
    "\n",
    "idx_permut = torch.randperm(X.shape[0])\n",
    "idx_train, _ = torch.sort(idx_permut[:n_split])\n",
    "idx_val, _ = torch.sort(idx_permut[n_split:])\n",
    "\n",
    "X_train = X[idx_train]\n",
    "X_val = X[idx_val]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_batch(split, batch_size):\n",
    "    # returns a batch, according to the data pipeline written in the W&B report\n",
    "    data = X_train if split == 'train' else X_val\n",
    "\n",
    "    idx_seed = torch.randint(high=data.shape[0], size=(1,), dtype=torch.int32).item() #sample la ligne seed autour de laquelle on va piocher les exemples\n",
    "\n",
    "    if split == 'train':\n",
    "        idx = torch.randint(low = max(0, idx_seed - 4 * batch_size), high = min(data.shape[0], idx_seed + 4 * batch_size), size=(batch_size,), dtype=torch.int32) #samples les indices du batch à produire\n",
    "        \n",
    "    else:\n",
    "        start = max(0, idx_seed-batch_size/2)\n",
    "        end =  min(data.shape[0]-1, idx_seed+batch_size/2)\n",
    "\n",
    "        if end-start != batch_size:\n",
    "            if start == 0:\n",
    "                end = end - (idx_seed-batch_size/2 - start)\n",
    "            else:\n",
    "                start = start - (idx_seed+batch_size/2 - end)\n",
    "\n",
    "        start, end = int(start), int(end)\n",
    "\n",
    "        idx = torch.arange(start=start, end=end, dtype=torch.int64)\n",
    "        \n",
    "    #pq 4 ? bon compromis entre assez large pour pas bcp de répétitions, assez petit pour pas bcp de padding (cf data.ipynb)\n",
    "    #en moyenne sur un batch, 6.7 d'écart en max_len et min_len (donc en moyenne pour une séq., 3,3 de padding) (ça fait bcp finalement? a comparer devant la longueur d'un mot) et max_len 17.8\n",
    "    #longueur moyenne d'une séq. : 11.6. donc en moyenne pour une séq., on rajoute 28% de compute inutile...\n",
    "    #on aura forcement pas mal de padding sur le val car dataset comparable devant 4*batch_size\n",
    "\n",
    "    #pour 2, on aura 3.5 d'écart, donc 1.7 de padding\n",
    "    # donc 14% de compute inutile par mot\n",
    "\n",
    "    #pour 2, on a 12% des exemples qui sont des répétitions (122 pour un batch de 1024, en moyenne)\n",
    "    #pour 4, seulement 6% (67 exemples)\n",
    "\n",
    "    #pour batch_size=512\n",
    "    #pour 2, 5% de répétitions, 1.8 de disp. donc 0.9 de padding (soit 7% de compute inutile par séq.)\n",
    "    #pour 4, 3% de répétitions, 3.5 de disp. donc 1.7 de padding (soit 14% de compute inutile par séq.)\n",
    "\n",
    "    #pour privilégier des données iid, je choisi de partir sur 4 (pour avoir des batch diversifiés, au détriment d'un peu de compute inutile)\n",
    "    #voir considérer plus ?\n",
    "    \n",
    "    #pour le val. set, 4 semble bcp trop\n",
    "    #pour 4, 12% de répétitions, 15 de disp. donc 7 de padding (soit plus de 50% de compute lost)\n",
    "    #c'est dommage, vu qu'avoir des batchs diversifiés n'a aucun interet pour le calcul du loss, donc on ne fait que perdre du compute\n",
    "    #mais le pb avec 1 (par exemple) c'est qu'on a enormément de répétitions (25%) au sein d'un batch... donc estimation du cout totalement erronée\n",
    "    #je choisis d'adopter une stratégie un peu différente: sample idx_seed, et on prends tous les index entre idx_seed-batch_size/2 et idx_seed+batch_size/2\n",
    "\n",
    "    idx_sorted, _ = torch.sort(idx) #on les ordonne pour recuperer facilement la longueur de la plus grande seq. du batch\n",
    "\n",
    "    X_batch = data[idx_sorted] #on extrait la matrice qui va produire Xb et Yb\n",
    "\n",
    "    max_len_batch = torch.sum(torch.ne(X_batch[-1], char_to_int['<pad>'])) #longueur de la plus grande seq. du batch : torch.ne(X_batch[-1], 0) crée une matrice masque, avec True si diff de 0, False si egal a 0\n",
    "\n",
    "    Xb = X_batch[:, :max_len_batch-1] #on selectionne que jusqu'a la len max - 1 (<EOS> du plus long inutile) (le reste n'est que padding)\n",
    "    Yb = X_batch[:, 1:max_len_batch] #meme que Xb, mais décalé de 1 (avec le <EOS> mais sans le <SOS>)\n",
    "\n",
    "    #Xb[Xb == 1] = -1 #on remplace le <EOS> par du padding (totalement optionnel)\n",
    "\n",
    "    Xb = Xb.pin_memory().to('cuda', non_blocking=True)\n",
    "    Yb = Yb.type(torch.LongTensor).pin_memory().to('cuda', non_blocking=True)\n",
    "\n",
    "    return Xb, Yb\n",
    "\n",
    "#todo: mettre les shapes a cote"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {},
   "outputs": [],
   "source": [
    "lr = 0.03\n",
    "batch_size = 1024\n",
    "embed_dim = 16\n",
    "hidden_dim = 100\n",
    "context_len = 3\n",
    "data = \"new\"\n",
    "\n",
    "eval_interval = 500\n",
    "eval_iter = 50"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {},
   "outputs": [],
   "source": [
    "class BengioLM(nn.Module):\n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "\n",
    "        self.embed = nn.Embedding(len(vocabulaire), embed_dim, padding_idx=0)\n",
    "        self.fc1 = nn.Linear(3 * embed_dim, hidden_dim)\n",
    "\n",
    "        self.lm_head = nn.Linear(hidden_dim, len(vocabulaire))\n",
    "\n",
    "    \"\"\"\n",
    "    def forward(self, x):\n",
    "        x = self.embed(x).view(-1, context_len*embed_dim)\n",
    "\n",
    "        z1 = self.fc1(x)\n",
    "        a1 = F.tanh(z1)\n",
    "\n",
    "        logits = self.lm_head(a1)\n",
    "\n",
    "        return z1, a1, logits\n",
    "    \"\"\"\n",
    "    \n",
    "    def forward(self, x, y=None):\n",
    "        # x : (B, T)\n",
    "        \n",
    "        embeddings = [] #liste de tensors (B, T, embed_dim)\n",
    "          \n",
    "        embed_char_1 = self.embed(x) #matrice qui stocke les embeddings du premire char du contexte (B, T, embed_dim)\n",
    "        #embed_char_1 = x.view(64, -1, 1)\n",
    "\n",
    "        embeddings.append(embed_char_1)\n",
    "\n",
    "        embed_char_2 = torch.roll(embed_char_1, shifts=1, dims=1) #shift le time de 1 pour obtenir la matrice qui stocke les embeddings du second char du contexte (B, T, embed_dim)\n",
    "        embed_char_2[:, 0] = self.embed(torch.tensor(1, device=device)) #on met l'embedding de <SOS>\n",
    "\n",
    "        #embed_char_2[:, 0] = self.embed(torch.zeros(embed_char_2.size(0), dtype=torch.long, device=device))\n",
    "\n",
    "        embeddings.append(embed_char_2)\n",
    "\n",
    "        embed_char_3 = torch.roll(embed_char_2, shifts=1, dims=1) # (B, T, embed_dim)\n",
    "        embed_char_3[:, 0] = self.embed(torch.tensor(1, device=device)) #on met l'embedding de <SOS>\n",
    "        #embed_char_3[:, 0] = self.embed(torch.zeros(embed_char_2.size(0), dtype=torch.long, device=device))\n",
    "\n",
    "        embeddings.append(embed_char_3)\n",
    "\n",
    "        embeddings.reverse()\n",
    "\n",
    "        x = torch.cat(embeddings, dim=2) # (B, T, 3*embed_dim)\n",
    "        x = x.view(-1, 3*embed_dim) # (B*T, 3*embed_dim), l'input du MLP\n",
    "\n",
    "        x = self.fc1(x)\n",
    "        x = F.tanh(x)\n",
    "\n",
    "        logits = self.lm_head(x) # (B, T, voc_size)\n",
    "\n",
    "        loss = None\n",
    "        if y is not None:\n",
    "            y = y.view(-1) # (B, T)\n",
    "\n",
    "            loss = F.cross_entropy(logits, y, ignore_index=0)\n",
    "\n",
    "        return logits, loss\n",
    "    \n",
    "    def sample(self, prompt=\"\", max_new_tokens=None):\n",
    "        return\n",
    "    \n",
    "#todo : remplacer les 0 et -1 par les char_to_int[...]\n",
    "#todo : voc_size - 1 ou pas pour la lm head ?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "BengioLM(\n",
       "  (embed): Embedding(46, 16, padding_idx=0)\n",
       "  (fc1): Linear(in_features=48, out_features=100, bias=True)\n",
       "  (lm_head): Linear(in_features=100, out_features=46, bias=True)\n",
       ")"
      ]
     },
     "execution_count": 130,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#writer = SummaryWriter(log_dir=\"runs/mlp/batch_size=\" + str(batch_size) + \"_lr=\" + str(lr))\n",
    "\"\"\"\n",
    "wandb.init(project=\"bengio_lm\",\n",
    "           config={\n",
    "               \"learning_rate\": lr,\n",
    "               \"batch_size\": batch_size,\n",
    "               \"embed_dim\": embed_dim,\n",
    "               \"hidden_dim\": hidden_dim,\n",
    "               \"context_len\": context_len,\n",
    "               \"data_loading\": data\n",
    "           })\n",
    "\"\"\"\n",
    "\n",
    "model = BengioLM()\n",
    "model.to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {},
   "outputs": [],
   "source": [
    "losses = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "training throughput = 325718.1811448856 examples/s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'\\nwandb.log({\"training_throughput\": num_examples_processed/(end_time-start_time)})\\nwandb.log({\"params_num\": sum([p.numel() for p in model.parameters()])})\\n'"
      ]
     },
     "execution_count": 132,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "N = 10000\n",
    "start_time = time.time()\n",
    "\n",
    "#wandb.watch(model, log=\"all\")\n",
    "\n",
    "for update_num in range(N):\n",
    "    Xb, Yb = get_batch('train', batch_size)\n",
    "\n",
    "    logits, loss = model(Xb, Yb)\n",
    "\n",
    "    losses.append(loss.item())\n",
    "\n",
    "    for p in model.parameters():\n",
    "        p.grad = None\n",
    "\n",
    "    loss.backward()\n",
    "\n",
    "    for p in model.parameters():\n",
    "        p.data += -lr * p.grad\n",
    "\n",
    "    \"\"\"\n",
    "    # eval : track loss (train & val), update_to_data\n",
    "    if update_num % eval_interval == 0:\n",
    "        to_log = {}\n",
    "\n",
    "        with torch.no_grad():\n",
    "            model.eval()\n",
    "            for split in ['train', 'val']:\n",
    "                loss_mean = 0\n",
    "                for i in range(eval_iter):\n",
    "                    Xb, Yb = get_batch(split, batch_size)\n",
    "                    logits, loss = model(Xb, Yb)\n",
    "\n",
    "                    loss_mean += loss.item()\n",
    "                loss_mean /= eval_iter\n",
    "                to_log[\"loss_\" + split] = loss_mean\n",
    "            model.train()\n",
    "\n",
    "            scalars_dict = {}\n",
    "\n",
    "            for name, p in model.named_parameters():\n",
    "                scalars_dict[name] = (lr*p.grad.std() / p.data.std()).log10().item()\n",
    "        \n",
    "        wandb.log(to_log | {\"update_to_data\": scalars_dict}, step=update_num)\n",
    "    \"\"\"\n",
    "\n",
    "end_time = time.time()\n",
    "num_examples_processed = N * batch_size\n",
    "\n",
    "print(\"training throughput = {} examples/s\".format(str(num_examples_processed/(end_time-start_time))))\n",
    "\"\"\"\n",
    "wandb.log({\"training_throughput\": num_examples_processed/(end_time-start_time)})\n",
    "wandb.log({\"params_num\": sum([p.numel() for p in model.parameters()])})\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "Waiting for W&B process to finish... <strong style=\"color:green\">(success).</strong>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       " View run <strong style=\"color:#cdcd00\">generous-glitter-8</strong> at: <a href='https://wandb.ai/alexandretl/bengio_lm/runs/79y7hnnx' target=\"_blank\">https://wandb.ai/alexandretl/bengio_lm/runs/79y7hnnx</a><br/>Synced 5 W&B file(s), 0 media file(s), 0 artifact file(s) and 0 other file(s)"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Find logs at: <code>./wandb/run-20230710_221917-79y7hnnx/logs</code>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "#wandb.finish()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x7fc5aaf7f2d0>]"
      ]
     },
     "execution_count": 133,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAiMAAAGdCAYAAADAAnMpAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAABIOElEQVR4nO3dd1xT5+IG8CessIPsraAWByIIDlTce1S7ta7u2mqttdY6uq0/vbfeVu3Qaltba716W7W11VqxihMXDhAVtTIc4GQpENb5/QGEhCSQkJAD5Pl+Pnxucs6bkzenXvLwTokgCAKIiIiIRGIhdgWIiIjIvDGMEBERkagYRoiIiEhUDCNEREQkKoYRIiIiEhXDCBEREYmKYYSIiIhExTBCREREorISuwK6KC8vx40bN+Dk5ASJRCJ2dYiIiEgHgiAgPz8fvr6+sLDQ3v7RJMLIjRs3EBAQIHY1iIiIqB6uXr0Kf39/reebRBhxcnICUPFhnJ2dRa4NERER6SIvLw8BAQGK73FtmkQYqeqacXZ2ZhghIiJqYuoaYsEBrERERCQqhhEiIiISFcMIERERiYphhIiIiETFMEJERESiYhghIiIiUTGMEBERkagYRoiIiEhUDCNEREQkKoYRIiIiEhXDCBEREYmKYYSIiIhE1SQ2ymsomxOuIel6LoaHeqN7sJvY1SEiIjJLZt0yEnfxNr4/nIbkG3liV4WIiMhsmXUYkVpVfHx5abnINSEiIjJfDCMA5KVlIteEiIjIfJl5GLEEABSzZYSIiEg05h1GrNlNQ0REJDazDiM2luymISIiEptZhxFFy0gJW0aIiIjEYt5hpHLMCLtpiIiIxGPmYaTi43MAKxERkXjMOozYcGovERGR6Mw6jHDRMyIiIvGZeRjhmBEiIiKxmXcYsWY3DRERkdjMO4xwACsREZHoGEYAXLx5X+SaEBERmS+zDiO5hSViV4GIiMjsmXUYuS+vHitSVi6IWBMiIiLzZdZhpH+Ih+IxB7ESERGJw6zDiL2NleIx96chIiISh1mHEUsLieLx1ewCEWtCRERkvsw6jCgTOGSEiIhIFGYfRgJd7QEApeXspiEiIhKD2YcRa8uKrpqSMjaNEBERiYFhxLLiFpQyjBAREYnC7MOIlaJlhN00REREYjD7MFLVMsIwQkREJA6GEYvKbhquwEpERCQKsw8jx9LuAQCSb+SKXBMiIiLzZPZhpMqXe/8RuwpERERmiWGkUkxbd7GrQEREZJYYRioduHRH7CoQERGZJYYRIiIiEpXZh5GWbvZiV4GIiMismX0Ymdi9JQDgkQg/kWtCRERknsw+jEgqFmDFngu3xK0IERGRmTL7MFJQXAYAaOvpKHJNiIiIzJPZh5EOPs4AgBKuwEpERCQKsw8jUuuKWyAvKRO5JkREROaJYcTKEgBQXMqN8oiIiMTAMGJV2TLCMEJERCQKhpGqbppSdtMQERGJgWGksptGXsKWESIiIjEwjLCbhoiISFRmH0ZsrSsHsJaVo7CYXTVERESmZvZhpKplBAB+SbgqYk2IiIjME8OIUhjJLSwRsSZERETmyezDiJVl9S3gWiNERESmZ/ZhRJm8jGGEiIjI1BhGlES1dBW7CkRERGaHYURJfhHHjBAREZkaw4iSJX9eELsKREREZodhRIlfCzuxq0BERGR2GEYAdA5wAQAM6+gtbkWIiIjMEMMIgA4+TgC4JDwREZEYGEZQvVleUQmXgyciIjI1hhEA13MKAQA/HkkXuSZERETmh2EEQOy5mwCA/KJSkWtCRERkfhhGAMwe8hAAwNvZVuSaEBERmR+GEQAt3RwAAEHuDiLXhIiIyPwwjACwrtwsr4R70xAREZmcXmFk5cqVCAsLg7OzM5ydnREdHY0///xTa/m4uDhIJBK1nwsXGtdKpzZWEgAMI0RERGKw0qewv78/lixZgjZt2gAAfvjhB4wZMwanTp1Cx44dtb4uJSUFzs7OiuceHh71rG7DqGoZKS4TRK4JERGR+dErjIwePVrl+aJFi7By5UocOXKk1jDi6ekJFxeXelXQFKwsKsLI+cw8kWtCRERkfuo9ZqSsrAwbN27EgwcPEB0dXWvZiIgI+Pj4YODAgdi7d2+d15bL5cjLy1P5aUgMIUREROLRO4wkJSXB0dERUqkUU6dOxdatW9GhQweNZX18fLB69Wps3rwZW7ZsQUhICAYOHIj9+/fX+h6LFy+GTCZT/AQEBOhbTb10C3Jt0OsTERGRdhJBEPQaKFFcXIyMjAzk5ORg8+bN+Oabb7Bv3z6tgaSm0aNHQyKRYNu2bVrLyOVyyOVyxfO8vDwEBAQgNzdXZeyJsWQ/KEbEwlgAwOVFw2FlyUlGREREhsrLy4NMJqvz+1uvMSMAYGNjoxjAGhUVhePHj2P58uX4+uuvdXp9jx49sH79+lrLSKVSSKVSfatWbzZW1eGjpExA5VY1REREZAIGNwEIgqDSilGXU6dOwcfHx9C3NSprpZaQYu7cS0REZFJ6tYzMnz8fw4cPR0BAAPLz87Fx40bExcVh586dAIB58+bh+vXrWLduHQBg2bJlaNWqFTp27Iji4mKsX78emzdvxubNm43/SQxgbSlRPP4hPg0zBrYVsTZERETmRa8wcvPmTUyaNAmZmZmQyWQICwvDzp07MXjwYABAZmYmMjIyFOWLi4sxe/ZsXL9+HXZ2dujYsSO2b9+OESNGGPdTGEgiqQ4j3x1KZRghIiIyIb0HsIpB1wEwhmg1d7vicdqSkQ3yHkREROZE1+9vThshIiIiUTGMEBERkagYRoiIiEhUDCM1KI1lJSIiIhNgGKmh8Q/nJSIial4YRio521bMcu4c4CJuRYiIiMwMw0ilKT1bAQDC/WXiVoSIiMjMMIxUklbuT1NUwuXgiYiITIlhpJKtdcXuePLSMpFrQkREZF4YRipJK8MIW0aIiIhMi2GkkqKbhi0jREREJsUwUqmqmyYu5TbKyzm/l4iIyFQYRipVtYwAwOXb90WsCRERkXlhGKl0936x2FUgIiIySwwjlVrYW4tdBSIiIrPEMFIp1K96sbOcghIRa0JERGReGEYqWVpU75AXl3JLxJoQERGZF4aRShZK2/WWlHGtESIiIlNhGKlkbakcRji1l4iIyFQYRiq5OUoVj+/LS0WsCRERkXlhGNHgl4RrYleBiIjIbDCMEBERkagYRoiIiEhUDCNEREQkKoYRIiIiEhXDCBEREYmKYYSIiIhExTBCREREomIY0aK4lEvCExERmQLDiBa7z98UuwpERERmgWFEi6KSMrGrQEREZBYYRpQEuTsoHgvcK4+IiMgkGEaUfP9sV7GrQEREZHYYRpQ4Sq0Uj9kwQkREZBoMI0osLSSKx+XspyEiIjIJhhElEkl1GEm/+0DEmhAREZkPhhElzrbV3TRf7v1HxJoQERGZD4YRJcotI0RERGQaDCNEREQkKoYRIiIiEhXDCBEREYmKYYSIiIhExTBCREREomIYISIiIlExjBAREZGoGEZqGBnmo3gscEl4IiKiBscwUkPvNu6KxyfSs0WsCRERkXlgGKlhRKfqlpEnVsWLWBMiIiLzwDBSg5UFl4QnIiIyJYaRGiwZRoiIiEyKYaQGtowQERGZFsNIDVaWvCVERESmxG9eIiIiEhXDCBEREYmKYaQO5eVc+IyIiKghMYzU4ePt58WuAhERUbPGMFKH7w6lil0FIiKiZo1hhIiIiETFMEJERESiYhghIiIiUTGMEBERkagYRjQY1N5L7CoQERGZDYYRDV6MCVJ5Lghca4SIiKihMIxo0DnAReX57Xy5OBUhIiIyAwwjGljW2Ll3yZ8XRKoJERFR88cwooF1jZ17t5y6LlJNiIiImj+GESIiIhIVwwgRERGJimFER0ev3BW7CkRERM0Sw4iOlv99SewqEBERNUsMI1rMGvyQyvPD/7BlhIiIqCHoFUZWrlyJsLAwODs7w9nZGdHR0fjzzz9rfc2+ffsQGRkJW1tbBAcHY9WqVQZV2FR8XezErgIREZFZ0CuM+Pv7Y8mSJThx4gROnDiBAQMGYMyYMUhOTtZYPjU1FSNGjEBMTAxOnTqF+fPnY8aMGdi8ebNRKt+QrGqsNUJEREQNw0qfwqNHj1Z5vmjRIqxcuRJHjhxBx44d1cqvWrUKgYGBWLZsGQCgffv2OHHiBJYuXYrHHnus/rU2AQsNYaSopAy21pYi1IaIiKj5qveYkbKyMmzcuBEPHjxAdHS0xjLx8fEYMmSIyrGhQ4fixIkTKCkp0XptuVyOvLw8lR9Ta+ftpHbsx/h0k9eDiIioudM7jCQlJcHR0RFSqRRTp07F1q1b0aFDB41ls7Ky4OWlugOul5cXSktLcefOHa3vsXjxYshkMsVPQECAvtU02ENe6mEkp7DY5PUgIiJq7vQOIyEhITh9+jSOHDmCV155BVOmTMG5c+e0lpdIVLs7qnbArXlc2bx585Cbm6v4uXr1qr7VbBAXMvPFrgIREVGzo9eYEQCwsbFBmzZtAABRUVE4fvw4li9fjq+//lqtrLe3N7KyslSO3bp1C1ZWVnBzc9P6HlKpFFKpVN+qNbjcQu1dS0RERFQ/Bq8zIggC5HK5xnPR0dGIjY1VObZr1y5ERUXB2tra0Lc2uRPp2bh0k60jRERExqRXGJk/fz4OHDiAtLQ0JCUlYcGCBYiLi8OECRMAVHSvTJ48WVF+6tSpSE9Px6xZs3D+/Hl89913+PbbbzF79mzjfgoTGvzZfuQUcOwIERGRsegVRm7evIlJkyYhJCQEAwcOxNGjR7Fz504MHjwYAJCZmYmMjAxF+aCgIOzYsQNxcXEIDw/HwoULsWLFikY/rbcuL61LELsKREREzYZEqBpR2ojl5eVBJpMhNzcXzs7OJnvfVnO3az2XtmSkyepBRETUFOn6/c29aWoxspOP2FUgIiJq9hhGarFgZHuxq0BERNTsMYzUwsZK++3p98leXL7FmTVERESGYhiphbWF9tuTdrcAgz7dj6KSMhPWiIiIqPlhGKmFzN4abT0day2TU8CF0IiIiAzBMFKH2Fl9az1fXFpuopoQERE1TwwjOpjev43Wc8Vl7KYhIiIyBMOIDmYPDdF6rowNI0RERAZhGDGQgEa/ZhwREVGjxjBioMSruWJXgYiIqEljGDHQJ7tSxK4CERFRk8YwoqN3tKzGejtfjlMZ2Sjh4BEiIqJ6YRjRUbCHg9Zzj3x1GPO3JJmwNkRERM0Hw4iR/JxwDWXlHMxKRESkL4YRHQk65Iwz13IavB5ERETNDcOIEekSWIiIiEgVw4gRXbzJXXyJiIj0xTBiRPM4iJWIiEhvDCM60rUL5lp2AQDgek4h7t6XN2CNiIiImgeGESMbsHQfvjlwBb2W7EHkx7vFrg4REVGjxzBiZMVl5fh4+3mxq0FERNRkMIzoqGuQq9hVICIiapYYRnQks7Ou92tzC0qQfveBEWtDRETUfDCM6EFqVb/b1fmjXej7SRwy7hYYuUZERERNH8OIHk6/N0Tv1whK03BOpN8zZnWIiIiaBYYRPdjZWGJ8t0C9XsNVWYmIiGrHMNLAmEWIiIhqxzDSwJ7/4bjYVSAiImrUGEYaWFzKbcXjWf87g5+OpkMQBHz8xzl8dzBVxJoRERE1DlZiV6DpMazjZcHWs+js74JvKoPIc72DjFEpIiKiJostIyK4fOu+2FUgIiJqNBhG9CSzszH4GjM3nVY87r80Djfzigy+JhERUVPFMKKnV/u3Rr8QDwS62hvleql3HuDTXRcBAFtPXUPitRyjXJeIiKip4JgRPTnbWuP7Z7sBAFrN3W6Ua5aUl+PIlbt4Y9MZAMBfM/sgxNvJKNcmIiJq7NgyYoBB7T2Ndq1LSuNIRq44YLTrEhERNXYMIwbo1cbdKNfZcvI6kq/nKp6XlnOpNCIiMh8MIwbQd2n42mw8ftVo1yIiImpKGEYMYGttiQsLhzXItd/77SzK2EJCRERmgGHEQLbWlg1y3XXx6dh25nqDXJuIiKgxYRhpxG7lySEvLcP2xEzkFBSLXR0iIqIGwam9RhAe4ILTV3OMft0T6dnYkZSJM9cqBrd2C3LFxhd7wMJCYvT3IiIiEgtbRoxg9aTIBrlu7LmbiiACAMdS7+Fff11okPciIiISC8OIEXg625rsvb7ed8Vk70VERGQKDCNNUHFpOQSBM22IiKh5kAhN4FstLy8PMpkMubm5cHZ2Frs6GiVey8HRK/fQv50HBn26v8Hfr1+IB17p2xq+LnYIMNI+OURERMak6/c3B7AaSZi/C8L8XUz2fnEptxGXclvlmJ+LHT4a0xED23uZrB5ERESGYjdNM3I9pxDP/3BC7GoQERHphWGkGRIEAel3H3BcCRERNQkMI83QZ7svoe8ncVi6K0XsqhAREdWJYaQZWvH3JQDAl3v/EbkmREREdWMYaeZyC0uw98ItbrpHRESNFmfTNHN9P9mLnIISPNOzFS7dykdMWw/cLyrF/ku3semlaNjZNMxGf0RERLpiGGnmcgpKAADfH04DABy6fFdx7peEq5gU3UqEWhEREVVjN00DGNbRW+wq6OTd35JxKiNb8TztzgPIS8vUypWUleOv5CxkP+DOwUREZHwMIw3gkyfC8ELvILGroZNHvjqMB/JS7L94G/2WxuHxlfFqZT7fcxkv/5iAp1arnyMiIjIUu2kagJOtNd4Z1QEd/ZzxxqYzYlenTh3f/0vxOOl6rsq5bw5cUczOuXjzvknrRURE5oEtIw1oTGc/PNbFX+xq6E0QBJSXCxi54gA+3n5ep9fczCvCj/FpeCAvbeDaERFRc8Mw0oAsLCSY2jdY5djUvq1Fqo3u5m1Jwtf7ryD5Rp7auaIS9TElfyTeQPf/+xvv/paMD39PNkUViYioGWE3TQNzsrVWee7pJBWpJrrbePyq1nOa1iuZvuGU4vHeGpv3aSMIAsoFIL+oBKev5iCmrQcsLST6V5aIiJo8hpEG5i2zVXne1JceKxMEnMrIRnsfZ9haW+JsjTEmuvhgWzK+P5wG/xZ2EISKDf7eHdUBz9cx6Le8XIC8tJxroxARNTPspjGxJ6Oa3hgSZWv2X8EjXx1Gu3d34sCl2xj1+UGV87fz5SgpK9f6+vyiEsWaJ9eyC3E9pxBARVdPXcavOYL27+3Erfyi+n8AIiJqdBhGTKxmt01T8/mey4rHk749prFM2wV/4v3fzmL/xYouG0EQkJCejdzCEp2XpddU7mjqPQDAzrNZ+labiIgaMYYRE4hp6y52FUzuh/h0TP7uGOL/uYudZ7Pw2MrDGLZsP3TJIpm5hQj/cBdeWncChy7fUTsvNPW+LiIiUsEwIoK1z3YVuwomk5B+DzsqWzIyc4uw4Wh6na9Zvf8K8uWl2HXuJiZ8cxRHrtxVOf/+tmQ8uSoe5dz8j4ioWWAYMYERnXwAAL6Vg1n7tPUQszomtXTXRfx+5obKc01OZeQg+UbFYFgLieqsmhNp99TKH0u7h7M39B88S0REjQ9n05jAU1EBCGhhj1A/ZwAAJ7BqNnLFQZz9cCh0neHLhhEiouaBLSMmYGEhQe+27nCxtwEASJhGtAp9/y9sPKa6zsnSXReRdueBWtnxq49AEHEAyeId5/HBNi7yRkRkKIYREUjqSCN+LnYmqknjlK9hSfl+S+PUjhWWlOGHw2k4mZGtNn4kM7cQ/ZfG4duDqQCAn09cxaxNp2uddqyPopIyfL3/Cr4/nIasXE41JiIyBLtpqEn74Pdziscjw3zwcp9ghPm74N87U5B65wEW/nEOz/cOwlu/JAIAvGS2GBPui+Op9/B095Zqq74WlZTB1rr2RdVqtsYcuHQbT0QFGOkTERGZH7aMULOxPTETD39xCFfvFWDrqeuK45uOZyger4z7B8OWHcC7vyWj9fwdkJdW77Wz5M8LaPfuTiSkqw6YlZeWYcPRDFzIysPxtHvotWQPVu37R3G+KugQEVH96BVGFi9ejK5du8LJyQmenp4YO3YsUlJSan1NXFwcJBKJ2s+FCxcMqjiRNjH/3qvy/O3NSVrLTv72GH44nIalf6UoAsZHf5xHflGJoswXey5j/tYkDFt2AE+siseN3CIs233J4Hqeu5GncSwMEZG50SuM7Nu3D9OmTcORI0cQGxuL0tJSDBkyBA8e1P0LNSUlBZmZmYqftm3b1rvSRPqwttQ+Rudo6j28vy0ZX+ytXln2zNUcdPpgF+7el6OguBQHNSy8VtP3h1KRducB3th0GilZ+XWWv3tfjhErDqDf0jicvZ6L5Bu5mLclEbfyOP6EiMyPRDBgOsLt27fh6emJffv2oU+fPhrLxMXFoX///sjOzoaLi0u93icvLw8ymQy5ublwdnaub3UblZSsfDwoLkWXwBZoNXe7yjk/FzvFni1kOGtLCUrK6j/r5iEvR1y8eV/n8vY2ljj30bBay6w/ko53fj2rdtzLWYqj8wfpXUcA+DE+DV/vv4KfXuiOlm4O9boGEZEx6fr9bdCYkdzcikWnXF1d6ywbEREBHx8fDBw4EHv37q21rFwuR15enspPcxPi7YQugS00nuPUX+MyJIgA0CuIAEBBcVmdZfZV7ttT0808OX44nIbrOYV46+czOJ+Zh9KyciSk30NxaXmtq86++1syrmUX4kOlQb3GtCs5C72W7NG4CB0RkSHqPZtGEATMmjULvXv3RmhoqNZyPj4+WL16NSIjIyGXy/Hjjz9i4MCBiIuL09qasnjxYnz44Yf1rRpRk/b+tmRsOXkNZ67l4ueEa3gxJghrDqRieKg3km/koZO/DF8+3UXr6w2Zvnz6ag42Hc/AW0Pb4cf4dFhbSfBqvzYAgJd+TABQsUHi+YW1t/wQEemj3mFk+vTpSExMxMGDB2stFxISgpCQEMXz6OhoXL16FUuXLtUaRubNm4dZs2Ypnufl5SEgoPlOnQx2d8AVDmRslvam3MLCP87h4zGh6NnGHT8eSUeQmwNu58trfd2Za9VL3a85ULFWyp+Ve/xk3CvA4PbX0SWwBQLd7NVeq9zxWlRShsLiMrRwsIEgCPh8z2V08HHGoA5eAIDluy8hwNUOj3bxBwCM/fIQACDtTgHiK/cEeqZnK9jbVP+qUJ6B1FyUlpXjZEYOwvxldU7tJiLjq1cYee2117Bt2zbs378f/v7+er++R48eWL9+vdbzUqkUUqm0PlVrkna90Qf/2nlB8aVDTV9cyi2czMjBir8rZt08/c1RONhY4oEOXTi6mLnpNADg8qLhsLJU7W09ePkOBEHA74mZmPHfUwCAU+8OxulrOfg0tmJvoLQlI5F4LQef7a54nltYgkHtvRTXuJBV3TVaUioANtXXb46r8C/ddRGr9v2DYR29sWpSpNjVITI7eo0ZEQQB06dPx5YtW7Bnzx4EBQXV601PnToFHx+fer22ObKytMDc4e0Vz90czSeINVfPrD2uCCJVjBVElLVZ8CcmfKO+LP7MTacVQQQATl/LQUJatkqZ7ILq6csf/n5OZUq08mJwZTWuXXPIe5mOmwQJgoCr9wpEXcK/ptQ7D5BbUILvKlfq3ZmcJXKNiMyTXmFk2rRpWL9+PTZs2AAnJydkZWUhKysLhYXVMz/mzZuHyZMnK54vW7YMv/76Ky5duoTk5GTMmzcPmzdvxvTp0433KZoB5V/+UkuuRUe6O3T5LrYp7YwMAL+dVn2ekJatMn35xyPp2HLymtZrKu+c/PjKw1rL/WvnBbSevwN7L9zSeP7LvZexZv8VAMC3B1MR8++9WPjHee0fpgZBEDDxm6N4du0xo4eY1DsP0H9pHDp/tAuCUntPbkEJpv6YgL/qGUwaU9giair06qZZuXIlAKBfv34qx9euXYtnnnkGAJCZmYmMjOoVL4uLizF79mxcv34ddnZ26NixI7Zv344RI0YYVvNmzKqWdTGINHl94+lazysHEQB4V8O0YmXKYaS28Uwr4yoWinv2++NIWzJScXzn2Sy8+b/Titagcd0C8PH2ihDy3aFUWFtJMLF7SwS4qo95UZaVV6RY5yUrrwgpWfno3cZdrWuqPo6l3tV4/NPYFOxMzsLO5CyVz6SLPRdu4o1NZ/CfJzorxuUQUd307qbR9FMVRADg+++/R1xcnOL5nDlzcPnyZRQWFuLevXs4cOAAg0gdWjjY4IXe9esCIzKG+kwvv3NfrpjJM3V9gkq3VKcPdqmU/XrfFYxbfaTOayo3MoxbfQTPrD2OrytbWpTtTbmFD39PRnGp4Rsh/hCfrvH4pZv5uFnHonTPfX8CuYUleGHdCQAVY3FqKi4txzcHrui0OB6RuWB/QCP19vB2YleBzFhWjS/dUZ8fqPM1UR/vxqgVtc+uU6bLwn45SuNa0u8WAAA++SsF/ztxFQ9/cRCnr+YAAJ5dexxrD6Vhw1HNQcIQl2/dR+j7f2HwZ/vR/f/+1vl1v52+js4f7sJnlYOGq6w5cAUfbz+Pocv2G7uqRE0Ww0gjMrRjRbPu872DYG1pgfdHd9BY7pV+rU1ZLTJDNYc9nL2uvvDgv3eq7y+VclP/v/bLywX8c/s+Dl++gy/2XMLd+xXTnpNv5GLECs0haM4viUi8louxXx7CqYzqgbk3cuu3nH5tC+PN/vkM7stL9b7m/C0VeyIt//sSikqqW4kSr+Xofa2aEtLvKYJYU1NcWo6dZ7OQU1AsdlWoEWEYaURWTYzE6fcGK1ZmfbZXECICXdTKvT2MrSYkrqv3CvBV3D8azw3T8y/+D35PxsD/7MPT3xzF0l0XEfnxbtzOl2NuLRscKnvkq+oBtpp6l3IKitVm/GTlFmlcjl/Ta2t+6ZdWdkWlZOVj4jdHkZCereGVqpTH6Eg01lJ3+UUleGxlPMZ+ecigBe4MdSOnEElK6+GUlJXjeNo9la4yTTOtPtt9EVPXJ+DpNUdNUk9qGhhGGhGJRAIXexuVY1WLUVWRWvE/GYmv5s7Iyi7oMRYi/e4DrNMwRiMhPdso2yJcupmP8I9i0Xr+DnwaexE7z2YCAHos/lunbQLCP4pVO1a1IN2U747h4OU7eKxyttHPJ65qvc7PCdeQeC0H41bHI/1egeJ4bcv7A0BhcZnKAnm38+X453b1gGJdw0h5uYCfjqbjvd/OGq1FoueSPRj9xUGkVg5wfu+3ZDyxKh7vb6sIXteyCxD+4S4s3qE6e2pb5Uyvc5nG3+ZjV3IWXv0pQeNYHWrc+M3WyE3oFqjyvHuwm8rz1h4O2Du7Hwa19zRltYiMou8ncRqPT12fgESlv7p19fX+K3j1pwR8sC0Z0zacxODPqltpVvx9CVPXnzR4BdnHVh7GyYxslXE1giDgrV8SVcrVXFfm4S8O4ciVeziv9CUc/tEuZNwtQE1V04OjPo5F10W7cSu/CMWl5ei6aLdilVygIqyUlQtY8fclHEut3jNozi9nMO2nk4rnvyfewIKtZ7EuPh3hH8Vi/OojOg+g/d+Jq3hx3QkUalkn59yNPBSXluO/xypmUf73WEUo+2LPZeTLS9UGHCuPFSo1csvOSz8mYEdSFl6qHEBMTQfDSCNnYVH3n4dB7g74ZkpXfDSmowlqRNS47UjKwveH07A9MVPj+V8StK+voqtHv1Jde2XK2uP1uk5eUSn+E5uiciz5Ri66LIzFuvg0RaBJSMvW+Nd+5Me7sS4+DZ/GXsSTX8cj/p+7yCsqwf9OXMP2pEycvZ6L8nIBu8+rrgMTf+Uuhi7bj7JyAe/9dha/nb6ucv73MzcwbcNJFBSXYs4viYg9dxPr4tM0fob/7ErBQ+/8qXKsoFh1jE15uYBzN/LUum0k9Wz+KisXkFekvfXjaCo3c2xq6r03DZnOf1/sgfFrKqZB1ragElcnIarbgq11jxXR134tuzDro7xcwN0HxZjzSyKyC0rw3m/JKue1rQ6782z18fFrjuDMe0MUzzefvIY5V+5p7RL5au9lrItPx7r4dIwJ98Om4xnwltnhtcrVe52k1V8ROVq6PjStQ9Phvb9Unn+2+yI+33MZ47vptsfYwj/OwdXBBtP6t9F4/smv45GQno39b/VX7M+UmVv37KzG7vKtfOxIysLzvYPgIDWvr2fz+rRNVHTr6q4ZZ1trEWtCRMa25/wtzNuSqOje0EbbQnUXa8xg+vZQ9R5Xaw+l1XrN/yhNO76QlYe3awwa3ni8uk43dJiKrc3neyoW3avtMwqCAIlEgsu37uPbyuX5x0b4wc/FTq1s1aDhPp/sxewhD8HOxgo/amm50aSsXMD9olLI7I33+/TqvQKsPZSGV/q1hodT/bb0GPRpRbfinftyfDQm1Gh1awrYTdNELB8Xji6BLnhnVMUeNiM7VeztM7Uvp/kSNWX58tI6g8grSuM/alLeYwiA2p5IusqsY1r0b6dvYMPRDPzv+FXM26LbTCddPZCXov/SOLz321mVLp7e/9pT52uX7rqIhX+cQ1qNsTe17Zn0+KrD6PzRLsXgW6AiDM3fmoQXfjih9yDfO/fliPn3Xnx3KLVijE9eEd76+YzaNO57D4px+J87dW4ZYOxp24Ig4Oz1XFy+dd+o1zUmtow0EWPC/TAm3E/xfMX4CMwZFoKWbg7VhYwx/YCISIv5W40bQtYeSsUTUQHYduYG0u4WIC0+HY9EVP+eM2Sbn9D3/8LOmTEqvyPlpWX4+cQ1nMrIAVCxMN3MQQ8BAJ5ecxTxVyq2CHjpxwR8PDYUZ67m4PFI/1rHtpy+mqMyqBgAulUujvdzwjVEtWyBFg42+PTJzuiysGJ21vJx4Sq/z2sy9vZGW05ex5s/nwEAfPJ4GA5cuoOZg9oi2MPRuG9kALaMNFGWFhLVIAKo/J+YiEgfz9ZzEK4hPt5+Hp0/3KXy7VtupG/iwpIy/GvnBQiCAHlpGU5mZCPknZ1a15epCiIAcCz1HoZ8th9v/ZKI7UmZyH5QjLv35cgtLMHczYmI/6e67Pojta/6eyI9G7Hnbqqsy1NX91ltXlmfgOe+P67Thoz35aXIKypRdHsBwFu/JGLbmRt4SoftGEyJLSPNiKOZDXgioubhXaXBus//oH1a7tnruRj1ue5bDuxIysKEb47i8D+aN0VctvsSbufLseiRTlqvceZqDqZvqBjQ+0SkP35OuIaNx6/qvYniSqUwcue+XOVcXlEJ/m+75t2s41Ju4fKt+3ghJhgP5KX4s3LAclZeEXxkFeNptp66hjX7U/H1pEgEuNoruqhC368YSNzO20nturfz5Ui8loMwfxfkFpYgK7cIIRrKmQq/vZqZmLbuOHDpjtjVICKql5waY2AiPtqFwpIyhHg5KRac04e2IFLlp6MZeL6WjUlLlcae/FxjWri8tKxeU8WvZasOBv7PXykqg4WVPVPZYhXqJ0Oon0xxXLlh5I1NFV0w7/52Fp8+GY7+S+PQq031xAdtCxE+/MUhpC0ZiV5L9uC+vBRbX+2JiMoVwE2N3TTNzBfju4hdBSIio8kuKEFRSXm9goiulu5K0XqutuX7ey6ue4CtLlI1LHxX07jVR/BwHa1CcSm38czaY8gtLMGOJM1TwWsqKC5V7L20N8XwKer1xTDSzMjsrfFEpH/dBYmICABq/eL+TmmqtLKdZ7Nw94FhS+tXbQpZc52apOu5uJatHlA0relSk74rF0cbKVAZimGkGfrkic7Y+mpPsatBRNRsTV2fYNDrV8b9g8iPd+ObA1c0nu/9r70m2dm4sezjwzDSTIUHuODlvsFiV4OIiDT4184LACpmFGmzPUnzlgbNEcNIMyWRSDBveHuxq0FERPW09C/tY1kagqGbSBqCYaSZs7Gq+E/8aIQf10QjImpCaq6uW1P63Qcax5bUV2mZkVdb0wOn9jZz8XMH4NKt++ge5IqtSjtzdg5wwRmlJYcHd/BC7LmbItSQiIj0lVdUgmHLDhj1msZe+VUfbBlp5twcpegR7AaJRIKlj3cGAMwZFqI2r/71gW3FqB4REdXDtXtNf5diZQwjZuSxSH8kfjAEr/Zro7aUsCAAe97sK1LNiIhIHy+s075SbX2J2ZXPMGJmnG01b5ndyt1eZdOkEC/xlgUmIiLTYzcNmZzyP7qtr/aEU2VImTmoLWwsLfDTi9017mdARERkbAwjZsrCoro9zq+FneLxzEEP4eKi4XB3lKKNZ+PZXpqIiJovhhEz1UapS8bTyVZjGXNacIeIiMTDqb1mqoOvM5aPC4efi53WMmL2HxIRkWkJ4DojJIIx4X61ng8PcMFppbVIiIiIGgK7aUirkrJysatARERmgGGEtFowomJvG2tLCVZPihS5NkRE1JAkEG+hEXbTkFY927gj+cOhcJDynwkRETUctoxQrRhEiIiooTGMEBERkaizaRhGSGfRwW5iV4GIiBoIl4OnJmHNlCh8/2xXvV/XI9i1AWpDRETNBcMI6cxRaoV+IZ44/d5g9GxdeyvJvrf6KR6vf757A9eMiIiaMoYR0puLvQ02vNgDm17qAQB4unug4twjEX5Y91w3BLraY0Qnb4zrGgArS/4zIyJq7AQR+2k4VYLqrXuwGy4tGg5rSwtM7dMaAgS0dHNQnP9qAtcmISKiuvFPVjKIdWWrR6CbvUoQqWlsuK+pqkRERE0MwwiZhIu9jdhVICKiWoi5NyrDCJlcex9nsatAREQ1cGovNXvKA6P+fD0Gf74eo7Xs19wHh4jIrDCMkEk8FukPAOjkJwNQ0TpyfMEgHJk3EP1CPFTKDunghSB37eNPiIjI+LgCKzV7Yf4uODJvILa82lNxzMNJCm+ZLb5/thuSPxyqUl65JeXQ3AHwdrY1WV2JiMi0GEbIZLxltorZNzU5SK0QP28Aji8YBIlEdRtrPxc7zB/Z3hRVJCIiETCMUKPhI7ODh5MUABBducKrg40lAGB0mA9XciUiaqa46Bk1SgtGdkCQuwOGdfQBAEgkEvRu6460JSNRXi7gk10pWBn3j8i1JCIiY2DLCDVKjlIrvNSnNQLd7NXOWVhIMGdoiE7XsbSQYGPlsvWaxLR1r3cdiYiak8LictHem2GEmqSa40q06fuQB9wdpVrPvxATbKwqERE1aXsu3BTtvRlGqFmriCyap6v5ymwR2bKFKatDRNRo6fpHXkNgGKFmLbKV5rDRs7Ub9s/pD0epFZI+GIJebdxMXDMiosZFzF17GUaoyYoIdFE8buPpqHY+smULvNBbczdMoKs9rCqnGTvZWqONh/rriYjINDibhpqslRMisfzvS3B1sMazvYJQVi4gr7AEgz/bDwD412OdYGOlOW+PClPdRXhwB2/8EJ9u9Dq+PawdYtq6Y9TnB41+bSIiYyoXcW8ahhFqsrxltlj8aCeVY17Otjg0dwBu5RWhjacTAEBmp7pj8OAOXuhdYxZN77bu2Da9Fx7+4pDR6xnqJ8PrA9ti+d+XjH5tIiJjEbObhmGEmh0/Fzv4udgpnns4SfH5+AjY21iiz0MeWleBDfN3MVENiYhIGcMImYXRnX3rLtSARGz9JCLSiZi/pziAlUiDhWNDjXKdquXtiYgaPRHTCMMIkZIPRnfA6M6+GN81wCjXeyTCDwDwlJGuR0TUUNgyQtRIPNMrCJ+Pj4CVpQU2vdQDHXycsXxcOAJd1Zelr0viB0NgaVGxiJCfix0OzR1g7OoSERkN1xkhaoS6B7thx+sxGBPuh7/f7It23k46v/bFmCA421qrHPNzscPwUG+96jCxR6Be5T8Y3UGv8kREjQEHsBLpwNrSAh5OUlzIyq+13JdPd4GHkxTdglw1nl85MRKt5m5viCoCAByk/L80EdWPmN00/M1FpCNd9m0YGeZj8PvI7KyRW1hS8Z7Qb68IT2dbg9+fiMyTzM667kINhN00RDpSXrukIbw1NAR/vNYb++f0Vxxzsdfvl0Oftu6YMaCNsatGRGZgeKjhf0zVF8MIkY7eHhaCMeG+WPdcN6Nfe8+bfTGtfxuE+skgs7OGk21Fo+UzPVvpdR2JRIJZQ0KMXj8iav4sxNu0l2GESFcu9jZYPi4CfR7yUBybEt0SwzpWDEp9tlcrna4zb3g7tWM199A5vmAQTr07GG6O6uuUzBjYVo9aExHpRoee6AbDMSNE9bBwbCiOpd7Du6M6oLRcwMn0bHTVMmi1ppf7tsZLfYJxPacQI1ccRFFJGbxqjPWwtbaErbUlAGDVxEhMXZ+gOGct5p8vRNRsiTizl2GEqD4m9WiJST1aAgCsLIGebdzreIUqiUQC/xb2OL5gEMoFQet+OQAwrJbpwL9O64WxX6pv7ufhJMXtfLledSIi88ZFz4jMlI2VhaIFpDYvxgQpHndp2ULxODzARWN5B5u6r0lE1FgwjBA1AW8NrR5n8pCXE/a82Ren3h2s8+trCycbXuxuUN2qHHy7f92FiKjRErMDmGGEqAmwsbLAwrGheGdke3g4SRHs4YgWDjYAgFUTu9T5+q8mRmo917O1O6b1b21wHf1b6Ldk/jsj2xv8nkRkPOymIaI6TerREi/EBKsdHxbqg38/HobvnolSHFOehfPTC93RV2kGkCaD2ntpPN5dx0G5+rKykGBi5ZgbIiK9wsjixYvRtWtXODk5wdPTE2PHjkVKSkqdr9u3bx8iIyNha2uL4OBgrFq1qt4VJiJ1T0YFYEC76kDx6ZOd0T3IFWuf7YpeNQbX9mztpvb6ci1/Ei0cG4pebdTLG8raUrexMkRkHvQKI/v27cO0adNw5MgRxMbGorS0FEOGDMGDBw+0viY1NRUjRoxATEwMTp06hfnz52PGjBnYvHmzwZUnIs1aujlg08vR6B/iqXYuOlg9XGjarfPcR0PxkJcTlj0Voff7h3g5obO/TOv5dc+rLxz3YkyQ2norjVF9dnAmagqazNTenTt3qjxfu3YtPD09kZCQgD59+mh8zapVqxAYGIhly5YBANq3b48TJ05g6dKleOyxx+pXayKqtxCl3Yf3vNkXANDOx1mtnL1Nxa8HD6fqLp+2no7wdJbi0OW7tb7HuG4BeLZXkMZNAf1c7NC1lXr3z4KRHbDhaAaKdfsYRNSMGLTOSG5uLgDA1VV7v3J8fDyGDBmicmzo0KH49ttvUVJSAmtr9b035HI55PLqNRLy8vIMqSYRAfj7zb7ILypFZ38ZxncLRICrHYI9HAEAjlIrJH4wBPO2JGF7YqbWa7TzcUZxaZnW8/99sQf2X7qtcTzIqDAf/JGYiZ+nRhv+YYioWal3m6ggCJg1axZ69+6N0NBQreWysrLg5aU6OM7LywulpaW4c+eOxtcsXrwYMplM8RMQEFDfahJRpdYejggPcIFEIsHiRzvh1X6qG+o521pr7MIBgN+m9cLEHoH48OGOKjt7Vg2andA9EAAQ3doNbw9rp3ERt4/GhCJtyUj4Km04uGpiF7T3ccaOGTEAgJmDHlJ5TUs3dokQmYN6t4xMnz4diYmJOHjwYJ1la269XtU/rW1L9nnz5mHWrFmK53l5eQwkRCYwvlsgrCwkakvbdw5wQefKBdbmDGuH9LsFGN8tEAPaeeH0e4O1bj2+cExHvPtbMv71WCe4Vk5FVjYs1AfDlHYKfSEmCP3beWDQp/sBAAEt7JF+twAA8Eq/1lgZ948xPqZBBFEnQBI1HDH/bdcrjLz22mvYtm0b9u/fD39//1rLent7IysrS+XYrVu3YGVlBTc3zX+FSaVSSKXqG4QRUcOytJBgXLfAWsu4O0qx6eXqrhYXe/WQUWVSdCtMim6l8/tLJBK08XSCrbUFikrK0butOw5ermhBtddh9s3T3QOx4WiG4vnk6JZYF5+u8/vrQsxBfkTNlV5hRBAEvPbaa9i6dSvi4uIQFBRU52uio6Px+++/qxzbtWsXoqKiNI4XISLa82Y/HLlyF6M7+2LJnxcAaF+QqZ23E7bPiIEgCLCytFAJI9P7tzF6GCEi49NrzMi0adOwfv16bNiwAU5OTsjKykJWVhYKCwsVZebNm4fJkycrnk+dOhXp6emYNWsWzp8/j++++w7ffvstZs+ebbxPQUTNiq+LHR7t4g9rSwuMCfdFSzd7DGinPk0ZABykVrC0kMCqls0Giahx0+v/vStXrkRubi769esHHx8fxc+mTZsUZTIzM5GRUf2XSVBQEHbs2IG4uDiEh4dj4cKFWLFiBaf1EpFOlo+LwN43+6GVu4NO5VdOqF4e31nLWBZDDK9lF2Uiqh+9u2nq8v3336sd69u3L06ePKnPWxERKVhYSOAotcL+t/rjvrwUI1YcUJyLaau6wuxDSuuo2Fpb4sCc/oj5917Fsf880Rlv/nymzvds5WaPtMrBs8rGhPthzYHU+nwMokZNzPFQbNckoiYj0M0eHXydYWVRPROv5hTl1h6O2PBid+yeVbEQY4CrveIxAIwM84EuHu2ieXC+pmnLRGQY/r+KiJqcDS/2QICrHdY+01XjEvI9W7ujjWd1C0kbTyec/XAoEj8YAltrS5WulhdjghAe4IJlT4UjvHL6MgBM7dsaA7WMUzE3XVu1ELsK1MwxjBBRk9MtyBUH5gxAfz3CgqPUCs62FWNIVk6MVBz3c7HDr9N6YWyEH76qHG/i6mADGysLrJ4cpfFadVn8aCfFY/8WdrWUrDZjYFuV58vHhdda/pPHwxSPnW0NWky7Vq/2a43Ilg2zezNRlYb7F0xE1MT4utjh+IJBcKr8cre0UF+YUctajQoJ7wyCm6MUvVq7w9fFFlaWFpi/NUllyrGy53oFwc3RBtP6t8GKvy8BAF4b0AZjwv3w+sbTOtV7dGdfvD+6Ix5650+dyuujc4ALTmXkGP26RMrYMkJEZq2lm+osHQ8nKWyVFlh7Z2R7ra/V1CLh5lixYGOgm71iuvH8Ee3xXK8gtenJMwa0wXujO2Baf9VxL57OtgAAF3vV2UCvDVAt98vUaEzsEYi3h7eDjZUFOmjY8NDTybAFJCXgqrPmQpdJKg2FYYSIzNKml3rgvVEd0C/Eo9ZyL8QEKwJJZMsWUG4YqRkitHGUWuG90R3Qq031zJ+/ZvbBG4NV9+L592NhGBnmgyejKgbPxs3upzhnbSnBrBrlo1q54uOxnRTdT3Y26qvU7pndD/Y1jr/cN1inegPat+34t1I3EZGh2E1DRGape7AbumvZGLCm53oFIczfBaF+zriRU73I4/O9gxTjVp5dexzTB9QeTp7uFog9F25iQDsvhChNQa7yZNcAPNm1eh8uF3sbJH84FJuOX8WQjl4qwUCXWT1PRvnDUWqF0+8NUenCmTe8PfqHeGLc6iMaX+fuKMWd+9U7pzvaVH9VtPZwwHO9g/B4F390CXTB02uO4la+XNNlqInRFjxNgS0jRER1sLCQoFuQK+xtrGBR4xf2Q15OeMjLCYfmDsD4Ovb1sbOxxE8v9MDzveveSqOKg9QKz/UOgn8L1R2MQ/3Uu2Ra1ehy+nhsxUBaTTOOlMNM9xobIyrPNvJvYYdnler7/bPdMKF7S1hYVOwj9FTX5r+J6YB2nnjIy1HsajS4rq3EG6jMMEJEpIcgdwcM6eCFJyL9RVmCfu/sfvjfy9EqU5ervDuqPYZ08EKYvwx/vh6jMYRUUR6bu2ZKFHq1qWgleq5XEN4ZVT1Opp23ExylVlj/fHd88XQEAlxVQ5GtDhsYapO2ZGS9X2tKYyP81IJec+TlLN4GteymISLSg0QiqfeUX2MIcndAkJal8V3sbbTWbeWELvghPg3Lx0UAAML8XdA9yBX+LezhbGuNn17ooVL+3EdDYSGRKJrue9dY6bbKlJ6t8MlfKfX9OE2GdS3BjgzHu0tEZAaGd/LBxpei4VU5U8fSQoJNL0fjP0921lje3sZKp1YPR6lhf9NqWk/l6e6Btc5iMsTSJzR/3rrY1NEKVte6MFQ7hhEiIjLIKA1L7O+YEYM9b/bF8nHh+GZyFB7t4ocV4yMU54M9Klp3xoT7qbxuWEdvLBobihdiglUWdtNm9pCHMLqzr9pxW2sL9NcwU0pTV8QPz3Wrs4sizF9W6/man6NKZ6VVfUk7dtMQEZFBPh8fgZKycvyVfFNxLMjdAXY2lgj2qBj4OaiDFwCgWytXrItPw4QeLTVf6+kIRdfQE1EB6BHsprLR4ZxhIfj3zopuoW5Brpg+oGLl2t/P3FC5zi9TeyLUT4aE9Ht4bGV8rfUPqRyEfDNP86wgCYBJPVriw9/P1XodTWz17N4Z3MELsedu1l2wmWHLCBERGUQikeDrSdVjVeysLTWueQIA3jJbzBnWDn4u6svkP9crSG3KcoCrPV6MqZ7No7wxYic/za0VnzwehtDKc5EtXXF50XDFOT8XO4wN94W7o43Ka2pbM6Z7sKvKYOXHI/1xaO4AreUNMUlLSDMFEWf2MowQEZFx1Vw5ti5rn+2KRyL8MHNwW43n29dYWXbHjBjMGNBGbRG4KjXXEbWytMCWV3vi60mRCPZwxLJxEdg9q69KmR7Bbho3Rlz2VDg8nSrG2ax9piuGdfTG/BHt4edih0HtvXT8hNW6BLrUel45ELg62Cj+952R7fHP/43A45H+eKlPMH6d1qvW6xyZN1CvenVt1QLhAeJtiMhuGiIiElX/EE/0D9G+6eHYcD/kF5UismXFl2UHX2d08FUNKBte7I6n1xwFALjYqYehLoGqX7QSqDcDfP50BP5IzESorwwjVhwAAMQozSLq386z1s0ZR4X54I/ETK3nIwJdsOXVXnggL0XH9//SUqa6niM7+eDlvsHwkdkp9knSZQBuTFt3eMts8d8Xe2D8Gs0L2ynzdrbFz1N71lmuIbFlhIiIjGJq39YAgHdHdTDqdS0sJJjSs5Wi60WTnq3d8cnjYZjQPbBeLRZAxQyiJ6MCVKZO29to/5t9Ss+KLpWqNVq+eLqLWhnllo45Q9sBqFjITllMW3d8MzkK5z8apjY7yb+FvcYNGwHghToWz9P2uprGdRN/4Tq2jBARkVHMHd4Or/RrDZmGlglTeCIqAE9E6fbFai+tHtNSs1vJzsYS/3s5WvFYm5i2Hjg8d4DKZoRezlKVgbAxbT1w5Mo9AEB06+rtB5QH4v74fHeV6z7c2RfbztzAM71a1foZXhvQFt8cTFU7XhVougS6IDzABaev5tR6nYkijlOpwjBCRERGI1YQ0Ze1pQUOzR2A8nJB43oq3YJ0Wxrdt8ZA3G+ndMWozw8qnr/UJxjezrYqQQSoCACbE65haEdv1LR8XDj+9VhYrUEIAOSlZYrH/UM8cCI9G8EejnhvdEXLlJWlBX6d1gut5m6v9ToijltVYBghIiKzpGlGj6GUu5LsbSxhbWmBxyL91co521rj7zf7abyGRCKpM4gAFQNbXR1sIEFFCLLQ0i0TN7sfEtKzMbC9J57/4QTGRvjh3V/PKs63sLfR+DpTYhghIiJqAHOGhjTo9a0sLRA/bwAsJBKtQQQAWrk7oFXlOJjNr1QMVA10tceU747hkQi/Wl9rKhJBEGrOgmp08vLyIJPJkJubC2dn9Z0qiYiIGosrt+/jyJV7eDJKnM0UdZVXVAInqZVikbkGeQ8dv7/ZMkJERGREwR6OipVnGzNn28YzvqfxRjYiIiIyCwwjREREJCqGESIiIhIVwwgRERGJimGEiIiIRMUwQkRERKJiGCEiIiJRMYwQERGRqBhGiIiISFQMI0RERCQqhhEiIiISFcMIERERiYphhIiIiETVJHbtFQQBQMVWxERERNQ0VH1vV32Pa9Mkwkh+fj4AICAgQOSaEBERkb7y8/Mhk8m0npcIdcWVRqC8vBw3btyAk5MTJBKJ0a6bl5eHgIAAXL16Fc7Ozka7LqnjvTYN3mfT4H02Dd5n02jI+ywIAvLz8+Hr6wsLC+0jQ5pEy4iFhQX8/f0b7PrOzs78h24ivNemwftsGrzPpsH7bBoNdZ9raxGpwgGsREREJCqGESIiIhKVWYcRqVSK999/H1KpVOyqNHu816bB+2wavM+mwftsGo3hPjeJAaxERETUfJl1ywgRERGJj2GEiIiIRMUwQkRERKJiGCEiIiJRmXUY+eqrrxAUFARbW1tERkbiwIEDYlep0Vq8eDG6du0KJycneHp6YuzYsUhJSVEpIwgCPvjgA/j6+sLOzg79+vVDcnKyShm5XI7XXnsN7u7ucHBwwMMPP4xr166plMnOzsakSZMgk8kgk8kwadIk5OTkNPRHbJQWL14MiUSCmTNnKo7xPhvH9evXMXHiRLi5ucHe3h7h4eFISEhQnOd9NlxpaSneeecdBAUFwc7ODsHBwfjoo49QXl6uKMP7XD/79+/H6NGj4evrC4lEgl9//VXlvCnva0ZGBkaPHg0HBwe4u7tjxowZKC4u1u8DCWZq48aNgrW1tbBmzRrh3Llzwuuvvy44ODgI6enpYletURo6dKiwdu1a4ezZs8Lp06eFkSNHCoGBgcL9+/cVZZYsWSI4OTkJmzdvFpKSkoSnnnpK8PHxEfLy8hRlpk6dKvj5+QmxsbHCyZMnhf79+wudO3cWSktLFWWGDRsmhIaGCocPHxYOHz4shIaGCqNGjTLp520Mjh07JrRq1UoICwsTXn/9dcVx3mfD3bt3T2jZsqXwzDPPCEePHhVSU1OF3bt3C5cvX1aU4X023Mcffyy4ubkJf/zxh5Camir8/PPPgqOjo7Bs2TJFGd7n+tmxY4ewYMECYfPmzQIAYevWrSrnTXVfS0tLhdDQUKF///7CyZMnhdjYWMHX11eYPn26Xp/HbMNIt27dhKlTp6oca9eunTB37lyRatS03Lp1SwAg7Nu3TxAEQSgvLxe8vb2FJUuWKMoUFRUJMplMWLVqlSAIgpCTkyNYW1sLGzduVJS5fv26YGFhIezcuVMQBEE4d+6cAEA4cuSIokx8fLwAQLhw4YIpPlqjkJ+fL7Rt21aIjY0V+vbtqwgjvM/G8fbbbwu9e/fWep732ThGjhwpPPfccyrHHn30UWHixImCIPA+G0vNMGLK+7pjxw7BwsJCuH79uqLMf//7X0EqlQq5ubk6fwaz7KYpLi5GQkIChgwZonJ8yJAhOHz4sEi1alpyc3MBAK6urgCA1NRUZGVlqdxTqVSKvn37Ku5pQkICSkpKVMr4+voiNDRUUSY+Ph4ymQzdu3dXlOnRowdkMplZ/beZNm0aRo4ciUGDBqkc5302jm3btiEqKgpPPPEEPD09ERERgTVr1ijO8z4bR+/evfH333/j4sWLAIAzZ87g4MGDGDFiBADe54ZiyvsaHx+P0NBQ+Pr6KsoMHToUcrlcpduzLk1iozxju3PnDsrKyuDl5aVy3MvLC1lZWSLVqukQBAGzZs1C7969ERoaCgCK+6bpnqanpyvK2NjYoEWLFmplql6flZUFT09Ptff09PQ0m/82GzduxMmTJ3H8+HG1c7zPxnHlyhWsXLkSs2bNwvz583Hs2DHMmDEDUqkUkydP5n02krfffhu5ublo164dLC0tUVZWhkWLFmH8+PEA+O+5oZjyvmZlZam9T4sWLWBjY6PXvTfLMFJFIpGoPBcEQe0YqZs+fToSExNx8OBBtXP1uac1y2gqby7/ba5evYrXX38du3btgq2trdZyvM+GKS8vR1RUFP7v//4PABAREYHk5GSsXLkSkydPVpTjfTbMpk2bsH79emzYsAEdO3bE6dOnMXPmTPj6+mLKlCmKcrzPDcNU99UY994su2nc3d1haWmpltpu3bqllvBI1WuvvYZt27Zh79698Pf3Vxz39vYGgFrvqbe3N4qLi5GdnV1rmZs3b6q97+3bt83iv01CQgJu3bqFyMhIWFlZwcrKCvv27cOKFStgZWWluAe8z4bx8fFBhw4dVI61b98eGRkZAPjv2VjeeustzJ07F+PGjUOnTp0wadIkvPHGG1i8eDEA3ueGYsr76u3trfY+2dnZKCkp0evem2UYsbGxQWRkJGJjY1WOx8bGomfPniLVqnETBAHTp0/Hli1bsGfPHgQFBamcDwoKgre3t8o9LS4uxr59+xT3NDIyEtbW1iplMjMzcfbsWUWZ6Oho5Obm4tixY4oyR48eRW5urln8txk4cCCSkpJw+vRpxU9UVBQmTJiA06dPIzg4mPfZCHr16qU2Nf3ixYto2bIlAP57NpaCggJYWKh+zVhaWiqm9vI+NwxT3tfo6GicPXsWmZmZijK7du2CVCpFZGSk7pXWeahrM1M1tffbb78Vzp07J8ycOVNwcHAQ0tLSxK5ao/TKK68IMplMiIuLEzIzMxU/BQUFijJLliwRZDKZsGXLFiEpKUkYP368xqlk/v7+wu7du4WTJ08KAwYM0DiVLCwsTIiPjxfi4+OFTp06NespenVRnk0jCLzPxnDs2DHByspKWLRokXDp0iXhp59+Euzt7YX169cryvA+G27KlCmCn5+fYmrvli1bBHd3d2HOnDmKMrzP9ZOfny+cOnVKOHXqlABA+PTTT4VTp04plqcw1X2tmto7cOBA4eTJk8Lu3bsFf39/Tu3Vx5dffim0bNlSsLGxEbp06aKYpkrqAGj8Wbt2raJMeXm58P777wve3t6CVCoV+vTpIyQlJalcp7CwUJg+fbrg6uoq2NnZCaNGjRIyMjJUyty9e1eYMGGC4OTkJDg5OQkTJkwQsrOzTfApG6eaYYT32Th+//13ITQ0VJBKpUK7du2E1atXq5znfTZcXl6e8PrrrwuBgYGCra2tEBwcLCxYsECQy+WKMrzP9bN3716Nv5OnTJkiCIJp72t6erowcuRIwc7OTnB1dRWmT58uFBUV6fV5JIIgCLq3oxAREREZl1mOGSEiIqLGg2GEiIiIRMUwQkRERKJiGCEiIiJRMYwQERGRqBhGiIiISFQMI0RERCQqhhEiIiISFcMIERERiYphhIiIiETFMEJERESiYhghIiIiUf0/0sPGx81GAQ4AAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(losses)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "...montol.\n",
      "...naiselphin-di-rouer-shén-le-blans.\n",
      "...houx.\n",
      "...troffes-d'ole-fe-des.\n",
      "...luphe-bon-pergdebotches.\n",
      "...ossies.\n",
      "...le vaivones.\n",
      "...saincourt-sézan.\n",
      "...bois.\n",
      "...soppesse-ssis.\n"
     ]
    }
   ],
   "source": [
    "# sample\n",
    "\n",
    "g = torch.Generator().manual_seed(40 + 7)\n",
    "\n",
    "for _ in range(10):\n",
    "    nom = \"...\"\n",
    "    while nom[-1] != \".\" or len(nom) == 3:\n",
    "        char_moins_3 = nom[-3]\n",
    "        char_moins_2 = nom[-2]\n",
    "        char_moins_1 = nom[-1]\n",
    "\n",
    "        id_moins_3 = char_to_int[char_moins_3]\n",
    "        id_moins_2 = char_to_int[char_moins_2]\n",
    "        id_moins_1 = char_to_int[char_moins_1]\n",
    "\n",
    "        x = torch.asarray([id_moins_3, id_moins_2, id_moins_1]).view(-1, context_len)\n",
    "\n",
    "        Z1 = C[x].view(-1, context_len*16) @ W1 + b1\n",
    "        A1 = torch.tanh(Z1)\n",
    "\n",
    "        Z2 = A1 @ W2 + b2\n",
    "        A2 = F.softmax(Z2, dim=1)\n",
    "\n",
    "        next_id = torch.multinomial(A2, num_samples=1, replacement=True, generator=g).item()\n",
    "        next_char = int_to_char[next_id]\n",
    "\n",
    "        nom = nom + next_char\n",
    "    print(nom)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [],
   "source": [
    "# batch_size    # TT    # GPU util\n",
    "#  8            11,5k       32%\n",
    "#  16           23k         32%\n",
    "#  32           45k         32%\n",
    "#  64           86k         36%\n",
    "#  128          168k        40%\n",
    "#  256          330k        36%\n",
    "#  512          640k        38%\n",
    "#  1024         1.14M       46% (optimal)\n",
    "#  2048         1.5M        48%"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "torch23",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.3"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}