mlp_lm_gpu.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "import time\n",
    "import wandb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "device = \"cuda\" if torch.cuda.is_available() else \"cpu\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# load les données\n",
    "\n",
    "fichier = open('villes.txt')\n",
    "donnees = fichier.read()\n",
    "villes = donnees.replace('\\n', ',').split(',')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# preparation des données\n",
    "\n",
    "# on rajoute le token . au début et en fin\n",
    "for ville, i in zip(villes, range(len(villes))):\n",
    "    villes[i] = ville + '.'\n",
    "\n",
    "# création du vocabulaire\n",
    "vocabulaire = []\n",
    "\n",
    "for ville in villes:\n",
    "    for c in ville:\n",
    "        if c not in vocabulaire:\n",
    "            vocabulaire.append(c)\n",
    "\n",
    "vocabulaire = sorted(vocabulaire)\n",
    "vocabulaire[0] = '.'\n",
    "vocabulaire[3] = \" \"\n",
    "\n",
    "# pour convertir char <-> int\n",
    "char_to_int = {}\n",
    "int_to_char = {}\n",
    "\n",
    "for (c, i) in zip(vocabulaire, range(len(vocabulaire))):\n",
    "    char_to_int[c] = i\n",
    "    int_to_char[i] = c"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# création du dataset\n",
    "\n",
    "context_len = 8\n",
    "\n",
    "X = []\n",
    "Y = []\n",
    "\n",
    "for ville in villes:\n",
    "    context = [0] * context_len\n",
    "\n",
    "    for ch in ville:\n",
    "        X.append(context)\n",
    "        Y.append(char_to_int[ch])\n",
    "\n",
    "        context = context[1:] + [char_to_int[ch]]\n",
    "\n",
    "X = torch.tensor(X) # (M, context_len), int64\n",
    "Y = torch.tensor(Y) # (M), int64\n",
    "\n",
    "n1 = int(0.8*X.shape[0])\n",
    "\n",
    "X_train = X[:n1]\n",
    "X_val = X[n1:]\n",
    "\n",
    "Y_train = Y[:n1]\n",
    "Y_val = Y[n1:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_batch(batch_size, split):\n",
    "    if split == 'train':\n",
    "        ix = torch.randint(X_train.shape[0], (batch_size,))\n",
    "\n",
    "        if device == 'cuda':\n",
    "            Xb = X_train[ix].pin_memory().to(device, non_blocking=True)\n",
    "            Yb = Y_train[ix].pin_memory().to(device, non_blocking=True)\n",
    "        else:\n",
    "            Xb = X_train[ix].to(device)\n",
    "            Yb = Y_train[ix].to(device)\n",
    "    else:\n",
    "        ix = torch.randint(X_val.shape[0], (batch_size,))\n",
    "\n",
    "        if device == 'cuda':\n",
    "            Xb = X_val[ix].pin_memory().to(device, non_blocking=True)\n",
    "            Yb = Y_val[ix].pin_memory().to(device, non_blocking=True)\n",
    "        else:\n",
    "            Xb = X_val[ix].to(device)\n",
    "            Yb = Y_val[ix].to(device)\n",
    "    \n",
    "    return Xb, Yb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "lr = 10**(-3.901)\n",
    "batch_size = 1024\n",
    "embed_dim = 32\n",
    "hidden_dim = 500\n",
    "\n",
    "eval_interval = 500\n",
    "eval_iter = 50"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "class BengioLM(nn.Module):\n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "\n",
    "        self.embed = nn.Embedding(len(vocabulaire), embed_dim)\n",
    "        self.fc1 = nn.Linear(context_len * embed_dim, hidden_dim)\n",
    "\n",
    "        self.lm_head = nn.Linear(hidden_dim, len(vocabulaire))\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = self.embed(x).view(-1, context_len*embed_dim)\n",
    "\n",
    "        z1 = self.fc1(x)\n",
    "        a1 = F.tanh(z1)\n",
    "\n",
    "        logits = self.lm_head(a1)\n",
    "\n",
    "        return logits\n",
    "    \n",
    "    def sample(self, prompt='', max_new_tokens=None):\n",
    "        prompt = context_len * '.' + prompt\n",
    "        init_len = len(prompt)\n",
    "\n",
    "        while True:\n",
    "            context = prompt[-context_len:]\n",
    "            context_tokenized = [char_to_int[c] for c in context]\n",
    "\n",
    "            logits = self.forward(torch.tensor(context_tokenized, device=device))\n",
    "            probs = F.softmax(logits, dim=1)\n",
    "\n",
    "            next_token = torch.multinomial(probs, num_samples=1, replacement=True).item()\n",
    "            next_char = int_to_char[next_token]\n",
    "\n",
    "            if next_char == '.':\n",
    "                break\n",
    "\n",
    "            prompt += next_char\n",
    "\n",
    "            if len(prompt)-init_len == max_new_tokens:\n",
    "                break\n",
    "        \n",
    "        return prompt[context_len:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "151952"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model = BengioLM()\n",
    "\n",
    "sum([p.numel() for p in model.parameters()])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "Finishing last run (ID:1q20snyt) before initializing another..."
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Waiting for W&B process to finish... <strong style=\"color:green\">(success).</strong>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<style>\n",
       "    table.wandb td:nth-child(1) { padding: 0 10px; text-align: left ; width: auto;} td:nth-child(2) {text-align: left ; width: 100%}\n",
       "    .wandb-row { display: flex; flex-direction: row; flex-wrap: wrap; justify-content: flex-start; width: 100% }\n",
       "    .wandb-col { display: flex; flex-direction: column; flex-basis: 100%; flex: 1; padding: 10px; }\n",
       "    </style>\n",
       "<div class=\"wandb-row\"><div class=\"wandb-col\"><h3>Run history:</h3><br/><table class=\"wandb\"><tr><td>loss_train</td><td>█▃▃▂▂▂▂▂▂▂▂▂▂▂▂▂▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁</td></tr><tr><td>loss_val</td><td>█▃▂▂▂▂▂▂▂▂▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁</td></tr><tr><td>params_num</td><td>▁</td></tr><tr><td>training_throughput</td><td>▁</td></tr></table><br/></div><div class=\"wandb-col\"><h3>Run summary:</h3><br/><table class=\"wandb\"><tr><td>loss_train</td><td>1.5279</td></tr><tr><td>loss_val</td><td>1.62666</td></tr><tr><td>params_num</td><td>151952</td></tr><tr><td>training_throughput</td><td>704585.40058</td></tr></table><br/></div></div>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       " View run <strong style=\"color:#cdcd00\">soft-sea-15</strong> at: <a href='https://wandb.ai/alexandretl/bengio_lm/runs/1q20snyt' target=\"_blank\">https://wandb.ai/alexandretl/bengio_lm/runs/1q20snyt</a><br/>Synced 6 W&B file(s), 0 media file(s), 0 artifact file(s) and 0 other file(s)"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Find logs at: <code>./wandb/run-20230714_095622-1q20snyt/logs</code>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Successfully finished last run (ID:1q20snyt). Initializing new run:<br/>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Tracking run with wandb version 0.15.5"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Run data is saved locally in <code>/home/alex/Bureau/llm/blablateurbinaire/wandb/run-20230714_101705-lrizt1ou</code>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Syncing run <strong><a href='https://wandb.ai/alexandretl/bengio_lm/runs/lrizt1ou' target=\"_blank\">sandy-elevator-16</a></strong> to <a href='https://wandb.ai/alexandretl/bengio_lm' target=\"_blank\">Weights & Biases</a> (<a href='https://wandb.me/run' target=\"_blank\">docs</a>)<br/>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       " View project at <a href='https://wandb.ai/alexandretl/bengio_lm' target=\"_blank\">https://wandb.ai/alexandretl/bengio_lm</a>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       " View run at <a href='https://wandb.ai/alexandretl/bengio_lm/runs/lrizt1ou' target=\"_blank\">https://wandb.ai/alexandretl/bengio_lm/runs/lrizt1ou</a>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<button onClick=\"this.nextSibling.style.display='block';this.style.display='none';\">Display W&B run</button><iframe src='https://wandb.ai/alexandretl/bengio_lm/runs/lrizt1ou?jupyter=true' style='border:none;width:100%;height:420px;display:none;'></iframe>"
      ],
      "text/plain": [
       "<wandb.sdk.wandb_run.Run at 0x7f81191f1690>"
      ]
     },
     "execution_count": 84,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "wandb.init(project=\"bengio_lm\",\n",
    "           config={\n",
    "               \"log_learning_rate\": np.log10(lr),\n",
    "               \"batch_size\": batch_size,\n",
    "               \"embed_dim\": embed_dim,\n",
    "               \"hidden_dim\": hidden_dim,\n",
    "               \"context_len\": context_len\n",
    "           })"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = BengioLM()\n",
    "model.to(device)\n",
    "\n",
    "optimizer = torch.optim.Adam(model.parameters(), lr=lr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "training throughput = 664015.5310009143 examples/s\n"
     ]
    }
   ],
   "source": [
    "N = 20000\n",
    "start_time = time.time()\n",
    "\n",
    "#wandb.watch(model, log=\"all\")\n",
    "\n",
    "for update_num in range(N):\n",
    "    Xb, Yb = get_batch(batch_size, 'train')\n",
    "\n",
    "    logits = model(Xb)\n",
    "\n",
    "    loss = F.cross_entropy(logits, Yb)\n",
    "\n",
    "    optimizer.zero_grad()\n",
    "    loss.backward()\n",
    "    optimizer.step()\n",
    "\n",
    "    # eval : track loss (train & val), update_to_data\n",
    "    if update_num % eval_interval == 0:\n",
    "        to_log = {}\n",
    "\n",
    "        with torch.no_grad():\n",
    "            model.eval()\n",
    "            for split in ['train', 'val']:\n",
    "                loss_mean = 0\n",
    "                for i in range(eval_iter):\n",
    "                    Xb, Yb = get_batch(batch_size, split)\n",
    "                    logits = model(Xb)\n",
    "\n",
    "                    loss_mean += F.cross_entropy(logits, Yb).item()\n",
    "                loss_mean /= eval_iter\n",
    "                to_log[\"loss_\" + split] = loss_mean\n",
    "            model.train()\n",
    "\n",
    "            scalars_dict = {}\n",
    "\n",
    "            for name, p in model.named_parameters():\n",
    "                scalars_dict[name] = (lr*p.grad.std() / p.data.std()).log10().item()\n",
    "        \n",
    "        #wandb.log(to_log | {\"update_to_data\": scalars_dict}, step=update_num)\n",
    "\n",
    "end_time = time.time()\n",
    "num_examples_processed = N * batch_size\n",
    "\n",
    "print(\"training throughput = {} examples/s\".format(str(num_examples_processed/(end_time-start_time))))\n",
    "#wandb.log({\"training_throughput\": num_examples_processed/(end_time-start_time)})\n",
    "#wandb.log({\"params_num\": sum([p.numel() for p in model.parameters()])})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'loss_train': 1.4038723802566528, 'loss_val': 1.6244510459899901}"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "to_log"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "wandb.finish()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mézérian\n",
      "braucourt\n",
      "le trize\n",
      "nans\n",
      "frayssinet\n",
      "mascarquissey\n",
      "frassonne\n",
      "suvillace\n",
      "saint-julien-d'ariagne\n",
      "saint-médère\n"
     ]
    }
   ],
   "source": [
    "for _ in range(10):\n",
    "    print(model.sample())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "villes_gen = []\n",
    "\n",
    "for _ in range(1000):\n",
    "    villes_gen.append(model.sample(\"\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "lens_gen = [len(ville) for ville in villes_gen]\n",
    "occurences_lens = [0] * (max(lens_gen)+1)\n",
    "\n",
    "for ville in villes_gen:\n",
    "    occurences_lens[len(ville)] += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x7ff49d5f4590>]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAigAAAGdCAYAAAA44ojeAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAABGfUlEQVR4nO3deXhU5d0+8PvMTDLZJpN9JoEAAYIgYZEtElZFaFFxoa5QxWpbLW641EppNdqaWGyRCi5VEVFEePtTrLVWiUoDGNn3qBAgQCCZ7MxknUlmzu+PyZlkyEImOZPZ7s91zXW9zJyZeXLKS26f5/t8H0EURRFEREREXkTh6QEQERERXYwBhYiIiLwOAwoRERF5HQYUIiIi8joMKEREROR1GFCIiIjI6zCgEBERkddhQCEiIiKvo/L0AHrCZrOhuLgYGo0GgiB4ejhERETUDaIooqamBklJSVAoup4j8cmAUlxcjOTkZE8Pg4iIiHqgqKgI/fv37/IanwwoGo0GgP0HjIyM9PBoiIiIqDtMJhOSk5Mdv8e74pMBRVrWiYyMZEAhIiLyMd0pz2CRLBEREXkdBhQiIiLyOgwoRERE5HUYUIiIiMjrMKAQERGR12FAISIiIq/DgEJERERehwGFiIiIvA4DChEREXkdBhQiIiLyOgwoRERE5HUYUIiIiMjrMKCQyypqzXjtfydQVtPo6aEQEZGfYkAhl7377Wks/+IY1n572tNDISIiP8WAQi47W1UPADAYOYNCRETuwYBCLjOY7MGkotbs4ZEQEZG/YkAhl5W1BJSqOouHR0JERP6KAYVcIoqiYwaFAYWIiNyFAYVcYmpsRmOTDQBQWWeBKIoeHhEREfkjBhRySamptTDW0mxDncXqwdEQEZG/YkAhl1y8c6eShbJEROQGDCjkkrYzKIB9mYeIiEhuDCjkkosDSlUtAwoREcmPAYVcYrg4oHAGhYiI3IABhVxSarLXnAiC/c9c4iEiIndgQCGXSEs8KbHhAFgkS0RE7sGAQi6RdvGMSIoEwCUeIiJyDwYU6rZmq81x/s7lifaAwiUeIiJyBwYU6raKWgtsIqBUCBim0wDgDAoREbkHAwp1m1R/kqBRIy4iGAADChERuQcDCnWbtMU4ITIEcRFqAEBFrZnn8RARkewYUKjbpBkUfaQaMeH2GRRzsw31PI+HiIhkxoBC3dYaUEIQFqyEWmX/68NlHiIikhsDCnWbwWjfwZMQGQJBEBDbMovCnTxERCQ3BhTqtrYzKAAQ4yiUZbM2IiKSFwMKdZsjoGjtASU23F4oW8kDA4mISGYMKNRt0i4eXaQ9mHCJh4iI3IUBhbql3tKMmsZmAIBOWuIJZy8UIiJyDwYU6hbpFOOwYCUi1CoArTUoXOIhIiK5MaBQt0iHBOpbdvAArUs8LJIlIiK5MaBQt5Q66k9CHM9JRbJc4iEiIrkxoFC3lF5UIAu0LvFUcImHiIhk5nJA2bZtG+bNm4ekpCQIgoBPPvnE8VpTUxN+97vfYdSoUQgPD0dSUhLuvvtuFBcXO32G2WzGww8/jLi4OISHh+OGG27AuXPnev3DkPs4dvBo286gsEiWiIjcw+WAUldXhzFjxmD16tXtXquvr8f+/fvxxz/+Efv378fHH3+M48eP44YbbnC6bsmSJdi8eTM2btyIHTt2oLa2Ftdffz2sVp7p4q3KWopk9W2WeKRdPA1NVjTwPB4iIpKRytU3zJ07F3Pnzu3wNa1Wi5ycHKfnVq1ahUmTJuHs2bMYMGAAjEYj1qxZg/fffx/XXHMNAGD9+vVITk7GV199hZ/85Cc9+DHI3Qwd1KBEqFUIVipgsdpQWWdG/+AwTw2PiIj8jNtrUIxGIwRBQFRUFABg3759aGpqwpw5cxzXJCUlIS0tDXl5eR1+htlshslkcnpQ35J28bQNKIIgIDaCyzxERCQ/twaUxsZGPP3001iwYAEiIyMBAAaDAcHBwYiOjna6VqfTwWAwdPg52dnZ0Gq1jkdycrI7h00XEUURZTXObe4l0jIPe6EQEZGc3BZQmpqacMcdd8Bms+G111675PWiKDr6a1xs6dKlMBqNjkdRUZHcw6UuVNVZ0GQVAQDxEWqn12LY7p6IiNzALQGlqakJt912GwoLC5GTk+OYPQEAvV4Pi8WC6upqp/eUlZVBp9N1+HlqtRqRkZFOD+o7Uv1JXEQwglXOf2XYrI2IiNxB9oAihZOCggJ89dVXiI2NdXp9/PjxCAoKciqmLSkpwdGjR5GRkSH3cEgG0g6eBE1Iu9dipBONOYNCREQycnkXT21tLU6cOOH4c2FhIQ4ePIiYmBgkJSXhlltuwf79+/HZZ5/BarU66kpiYmIQHBwMrVaL++67D0888QRiY2MRExODJ598EqNGjXLs6iHvIs2gXFx/AqC1SJY1KEREJCOXA8revXtx1VVXOf78+OOPAwAWLVqEzMxMfPrppwCAsWPHOr1v69atmDlzJgDg5Zdfhkqlwm233YaGhgbMmjUL7777LpRKZQ9/DHKnjnbwSGJZg0JERG7gckCZOXMmRFHs9PWuXpOEhIRg1apVWLVqlatfTx4g7eBp2+ZewiJZIiJyB57FQ5fU9iTji7X2QWGRLBERyYcBhS7J0FIkq+ugBkUqkmUNChERyYkBhS6pTGpz38EuHmkGpc5iRWMTz+MhIiJ5MKBQl8zNVkd9SUe7eDRqFYKU9gZ7rEMhIiK5MKBQl8pr7Ms7wUoFosOC2r0uCIKjUJbLPEREJBcGFOpSacvyTkKkutOjCFqbtbFQloiI5MGAQl0yGO2ho6MdPJLWdvecQSEiInkwoFCXpBmUjpq0SVq3GjOgEBGRPBhQqEvdCShs1kZERHJjQKEutZ7D076LrMTR7r6WNShERCQPBhTqUvdmUFqatXEGhYiIZMKAQl0qlbrIcomHiIj6EAMKdUoUxS7P4ZHEsUiWiIhkxoBCnaoxN6OhpX19d2ZQ2KiNiIjkwoBCnSptmT2JDFEhNFjZ6XWxLTUoNeZmmJt5Hg8REfUeAwp1Sqo/6egMnrYiQ1VQKexdZrnMQ0REcmBAoU4ZurGDB7CfxxPt2GrMgEJERL3HgEKd6s4WYwnb3RMRkZwYUKhTUkDpagePhO3uiYhITgwo1Clpi7EusvMushKpWVsFu8kSEZEMGFCoU1ziISIiT2FAoU51p4usJIYBhYiIZMSAQh2y2kSU13ZvmzHQWoPCdvdERCQHBhTqUEWtGVabCIUAxEVcugaFSzxERCQnBhTqkFR/Eq9RQ9nShK0rUpFsJYtkiYhIBgwo1KHuHBLYFk80JiIiOTGgUIdc2cEDtC7x1DQ2w9Jsc9u4iIgoMDCgUIdc2cEDANrQIMdSUHU9Z1GIiKh3GFCoQ9I5PN3ZwQMACoWA6DCex0NERPJgQKEOubrEA7Qu81TWsVCWiIh6hwGFOtQaUC69xVjCZm1ERCQXBhTqkKu7eAAgJoJLPEREJA8GFGqnwWKFqbEZAJDgQkCJ4wwKERHJhAGF2pGWd0KDlIgMUXX7fY5mbQwoRETUSwwo1E7bHTyCcOkuspLWJR4WyRIRUe8woFA70gxKgqb7BbIAz+MhIiL5MKBQO6Uu9kCRcBcPERHJhQGF2jEY7Us0ruzgAYC4CJ7HQ0RE8mBAoXZKa1qWeFwMKFKRrLGhCU1WnsdDREQ9x4BC7ZT2oAcKAESFBqHlOB6ex0NERL3ickDZtm0b5s2bh6SkJAiCgE8++cTpdVEUkZmZiaSkJISGhmLmzJnIz893usZsNuPhhx9GXFwcwsPDccMNN+DcuXO9+kFIPtIMil7rWpEsz+MhIiK5uBxQ6urqMGbMGKxevbrD15cvX44VK1Zg9erV2LNnD/R6PWbPno2amhrHNUuWLMHmzZuxceNG7NixA7W1tbj++uthtVp7/pOQLERRdJxknKBxbQYFYKEsERHJo/tduFrMnTsXc+fO7fA1URSxcuVKLFu2DPPnzwcArFu3DjqdDhs2bMD9998Po9GINWvW4P3338c111wDAFi/fj2Sk5Px1Vdf4Sc/+Ukvfhzqrer6Jlia7fUjCS6cwyOJjQhGQRkLZYmIqHdkrUEpLCyEwWDAnDlzHM+p1WrMmDEDeXl5AIB9+/ahqanJ6ZqkpCSkpaU5rrmY2WyGyWRyepB7SFuMY8KDoVYpXX5/bEuhbBWbtRERUS/IGlAMBgMAQKfTOT2v0+kcrxkMBgQHByM6OrrTay6WnZ0NrVbreCQnJ8s5bGrD4DjF2PXlHYBLPEREJA+37OK5uD26KIqXbJne1TVLly6F0Wh0PIqKimQbKzlr3cHj+vIO0BpQKhhQiIioF2QNKHq9HgDazYSUlZU5ZlX0ej0sFguqq6s7veZiarUakZGRTg9yD6lAtqczKFKztiru4iEiol6QNaCkpKRAr9cjJyfH8ZzFYkFubi4yMjIAAOPHj0dQUJDTNSUlJTh69KjjGvKc3i/xtNSgcAaFiIh6weVdPLW1tThx4oTjz4WFhTh48CBiYmIwYMAALFmyBFlZWUhNTUVqaiqysrIQFhaGBQsWAAC0Wi3uu+8+PPHEE4iNjUVMTAyefPJJjBo1yrGrhzznXHU9ANfP4ZFISzyVdSySJSKinnM5oOzduxdXXXWV48+PP/44AGDRokV499138dRTT6GhoQGLFy9GdXU10tPTsWXLFmg0Gsd7Xn75ZahUKtx2221oaGjArFmz8O6770KpdH3XCMmnqKoe356oAABcMSCqR58RG8EiWSIi6j1BFEXR04NwlclkglarhdFoZD2KjDI/zce7eacxLTUO79+X3qPPqKg1Y8KfvwIAnHhhLlRKnqZARER2rvz+5m8PAgBcqLdg0x777qhfTx/c48+JDguG4DiPp0mOoRERUQBiQCEAwPqdZ9DQZMWIxEhMHRrX489RtjmPh8s8RETUUwwohMYmK97NOwMA+PX0lEv2rLkUFsoSEVFvMaAQPjlwHhW1ZiRqQ3D96KRefx67yRIRUW8xoAQ4m03Em9tPAQDunZKCIBmKWmOlGRQ2ayMioh5iQAlw3/xYhlPlddCoVbhjkjxnHElbjXmiMRER9RQDSoB7c5t99mRB+gBoQoJk+czWbrKsQSEiop5hQAlgB85WY/fpKqgUAn4xJUW2z41lDQoREfUSA0oAe6ul9uSGsUk9bm3fkRjWoBARUS8xoASoM5V1+OKo/dTp3jRm64ijSJYzKERE1EMMKAFqzY5C2ERgxrB4DNfLe1xAbARPNCYiot5hQAlAVXUW/N/e3re174y0xFNdb4HV5nNHPRERkRdgQAlA63eeQWOTDSOTIpExJFb2z48Os+8GEkX7GT9ERESuYkAJMI1NVqzLOw3APnvS27b2HVEpFYhqCSlc5iEiop5gQAkwH+0/h8o6C/pFheLaUYlu+54YFsoSEVEvMKAEEJtNxNvbCwEAv5gySJa29p2Ja2nWxq3GRETUEwwoASTnh1IUVtRBE6LCHZMGuPW7Wg8MZDdZIiJyHQNKAFnTMnuyMH0gItQqt35XDM/jISKiXmBACRBWm4j9Z6sBAHdMlOdQwK6w3T0REfUGA0qAqKw1o9kmQiEA/aND3f59LJIlIqLeYEAJECXGRgBAgiYEKjcWx0qkbrLlNaxBISIi1zGgBIgSYwMAyHooYFcGxoQBAAor6vrk+4iIyL8woAQIaQYlsY8CypCECAD2GRRjQ1OffCcREfkPBpQAYXAEFPfXnwBAhFoFfaQ9DJ0sr+2T7yQiIv/BgBIg+noGBQCGtsyinChjQCEiItcwoASIvq5BAYAh8eEAOINCRESuY0AJENIMSlJU38+gnOQMChERuYgBJQDYbCJKTfaAou+jGhQAGBLfElDKuZOHiIhcw4ASACrrLGiyihAEIEGj7rPvlWZQzlTWwdxs7bPvJSIi38eAEgCk+pP4CLVbTzC+WLxGDY1aBZsInKms77PvJSIi38eAEgAcO3ii+m55BwAEQXD0Q+FOHiIicgUDSgBw9ECJ7LsCWYmjDoUBhYiIXMCAEgCkGZS+3GIscfRC4VZjIiJyAQNKAJBqUPqySZuEvVCIiKgnGFACgKdqUIC2vVDqYLOJff79RETkmxhQAoDBA23uJQNiwhCkFNDQZEVxy0wOERHRpTCg+DmbTXQEFL0HimRVSgUGxUrLPGzYRkRE3cOA4ueq6i2wWG0QBEDngYAC8NBAIiJyHQOKn5NmT+Ii1AhWeeZ/7taW9wwoRETUPbL/xmpubsYf/vAHpKSkIDQ0FIMHD8bzzz8Pm83muEYURWRmZiIpKQmhoaGYOXMm8vPz5R4KoU2BrAfqTyScQSEiIlfJHlD+8pe/4I033sDq1avxww8/YPny5XjppZewatUqxzXLly/HihUrsHr1auzZswd6vR6zZ89GTU2N3MMJeNIWY0/Un0ikGZRTnEEhIqJukj2gfPfdd7jxxhtx3XXXYdCgQbjlllswZ84c7N27F4B99mTlypVYtmwZ5s+fj7S0NKxbtw719fXYsGGD3MMJeN4wgzK4pRdKRa0FF+otHhsHERH5DtkDytSpU/H111/j+PHjAIBDhw5hx44duPbaawEAhYWFMBgMmDNnjuM9arUaM2bMQF5eXoefaTabYTKZnB7UPQYP9kCRhKtVSGoJSKxDISKi7lDJ/YG/+93vYDQaMXz4cCiVSlitVrzwwgu48847AQAGgwEAoNPpnN6n0+lw5syZDj8zOzsbzz33nNxDDQie7CLb1pCECBQbG3GirBbjB8Z4dCxEROT9ZJ9B2bRpE9avX48NGzZg//79WLduHf76179i3bp1TtcJguD0Z1EU2z0nWbp0KYxGo+NRVFQk97D9VokHe6C0JdWhsFCWiIi6Q/YZlN/+9rd4+umncccddwAARo0ahTNnziA7OxuLFi2CXq8HYJ9JSUxMdLyvrKys3ayKRK1WQ61Wyz1UvyeKYpsaFM8t8QBtWt6zWRsREXWD7DMo9fX1UCicP1apVDq2GaekpECv1yMnJ8fxusViQW5uLjIyMuQeTkCrrm+Cpdl+33VazwY8zqAQEZErZJ9BmTdvHl544QUMGDAAI0eOxIEDB7BixQrce++9AOxLO0uWLEFWVhZSU1ORmpqKrKwshIWFYcGCBXIPJ6BJ9SdxEcFQq5QeHYs0g1JUXY/GJitCgjw7HiIi8m6yB5RVq1bhj3/8IxYvXoyysjIkJSXh/vvvxzPPPOO45qmnnkJDQwMWL16M6upqpKenY8uWLdBoNHIPJ6CVXGipP/FwgSxgD0mRISqYGptRWFGHEYmRnh4SERF5MdkDikajwcqVK7Fy5cpOrxEEAZmZmcjMzJT766mNEpNUIOvZ+hPA/r/50IQI7D97ASfLaxlQiIioSzyLx48ZWpZ4kqI8P4MCsA6FiIi6jwHFjzm2GHvBEg/AnTxERNR9DCh+TKpB8XSTNgkPDSQiou5iQPFjBpN39ECRtD000GoTPTwaIiLyZgwofsrepM072txLkmPCEKxUwNxsQ/GFBk8Ph4iIvBgDip8yNjShsamlSZuH29xLlAoBKXH2k425zENERF1hQPFTxS31JzHhwV7VFK21UJYBhYiIOseA4qcMJu9a3pEMiecMChERXRoDip9qPSTQywIKZ1CIiKgbGFD8lMHLeqBI2KyNiIi6gwHFTxVf8K4txhIpoFTXN6GqzuLh0RARkbdiQPFT3lqDEhqsRL8oe2jiLAoREXWGAcVPeVub+7a4k4eIiC6FAcUPiaLYps29dy3xAKxDISKiS2NA8UOmhmY0NFkBAHovadLWFs/kISKiS2FA8UMlLfUn0WFBCA32niZtEqkXCpd4iIioMwwofqi1/sT7lneA1hmU8xca0GCxeng0RETkjRhQ/FBr/Yn3Le8A9vb7UWFBEEXgVAVnUYiIqD0GFD9kaDnF2Bt38ACAIAgYykJZIiLqAgOKH5KWeJK8NKAArTt5TpbXeXgkRETkjRhQ/JDB5N01KECbXiicQSEiog4woPih4gve2UW2LTZrIyKirjCg+BlRFL26i6xEWuI5VVEHq0308GiIiMjbMKD4mRpzM+pbtu568wxKv+hQqFUKWJptKKqq9/RwiIjIyzCg+BlDy+yJNjQIYcEqD4+mc0qFgJQ4NmwjIqKOMaD4GV+oP5Gw5T0REXWGAcXPGHyg/kTSutWYAYWIiJwxoPgZqUDWG08xvhhnUIiIqDMMKH7GYPTuNvdtsVkbERF1hgHFzxR7eZv7tgbFhQEAjA1NuFBv8fBoiIjImzCg+BmDo8299y/xhAWroItUAwBOV3KrMRERtWJA8TO+VCQLAANj7VuNz1RymYeIiFoxoPiRmsYm1JibAfhOQBkUa1/mOV3BGRQiImrFgOJHpNkTTYgKEWrvbdLWFmdQiIioIwwofqTEh+pPJINaAsppBhQiImqDAcWP+Fr9CQAMbFniOcMiWSIiaoMBxY+U+FAPFMmAloBSWWdBTWOTh0dDRETeggHFj5T4UA8USWRIEGLDgwFwFoWIiFoxoPgRX6xBAbjMQ0RE7TGg+BFfrEEBWChLRETtuSWgnD9/Hj//+c8RGxuLsLAwjB07Fvv27XO8LooiMjMzkZSUhNDQUMycORP5+fnuGEpAkdrc+1INCsCtxkRE1J7sAaW6uhpTpkxBUFAQ/vvf/+L777/H3/72N0RFRTmuWb58OVasWIHVq1djz5490Ov1mD17NmpqauQeTsCoNTejptG3mrRJpDN52O6eiIgksnfz+stf/oLk5GSsXbvW8dygQYMc/7coili5ciWWLVuG+fPnAwDWrVsHnU6HDRs24P7775d7SAHB0aRNrYImJMjDo3ENZ1CIiOhiss+gfPrpp5gwYQJuvfVWJCQk4IorrsBbb73leL2wsBAGgwFz5sxxPKdWqzFjxgzk5eV1+Jlmsxkmk8npQc58tf4EAAbG2GdQSk1mNFisHh4NERF5A9kDyqlTp/D6668jNTUVX375JR544AE88sgjeO+99wAABoMBAKDT6Zzep9PpHK9dLDs7G1qt1vFITk6We9g+r9gHtxhLosKCEBlin8w7W8VlHiIickNAsdlsGDduHLKysnDFFVfg/vvvx69+9Su8/vrrTtcJguD0Z1EU2z0nWbp0KYxGo+NRVFQk97B9nsEHm7RJBEHAoDju5CEiolayB5TExERcfvnlTs+NGDECZ8+eBQDo9XoAaDdbUlZW1m5WRaJWqxEZGen0IGetXWR9qweKhHUoRETUluwBZcqUKTh27JjTc8ePH8fAgQMBACkpKdDr9cjJyXG8brFYkJubi4yMDLmHEzAMPrrFWDIoljt5iIioley7eB577DFkZGQgKysLt912G3bv3o0333wTb775JgD7dP6SJUuQlZWF1NRUpKamIisrC2FhYViwYIHcwwkYJT5cJAtwBoWIiJzJHlAmTpyIzZs3Y+nSpXj++eeRkpKClStXYuHChY5rnnrqKTQ0NGDx4sWorq5Geno6tmzZAo1GI/dwAoavL/E4ZlAqOINCRESAIIqi6OlBuMpkMkGr1cJoNLIeBUC9pRmXP/MlAOBw5hxE+lgfFAAoq2nEpBe+hiAAP/7pp1CrlJ4eEhERycyV3988i8cPSDt4woKV0KhlnxTrE/ERaoQFKyGKQFFVg6eHQ0REHsaA4gdKTWYAgD4ypNOt2t5OEARHHcrZKtahEBEFOgYUP1BWY59BSYhUe3gkvcM6FCIikjCg+IFSkz2g6CJ9cwePhDt5iIhIwoDiB6QlHl8PKOyFQkREEgYUP1BWYw8oCRrfXuLhDAoREUkYUPyAvyzxDIqzz6Ccq25Ak9Xm4dEQEZEnMaD4gTI/CSg6TQiCVQo020QUX+BWYyKiQMaA4uNEUXTUoPj6Eo9CIWBgDOtQiIiIAcXn1Zib0dBkBeD724yB1jqUs6xDISIKaAwoPk5a3tGEqBAW7JtdZNviTh4iIgIYUHyev2wxlgyM404eIiJiQPF5rTt4fH95B+AMChER2TGg+DjHDIrGP2ZQBjlqUOphtfncQdtERCQTBhQfJ82gJPjJEk+iNgRBSgEWqw2Glp+NiIgCDwOKjyuvkWpQ/GOJR6VUoH+0fZnnTAXrUIiIAhUDio/zly6ybQ1sqUM5U8U6FCKiQMWA4uNKa/yrSBZorUM5zZ08REQBiwHFhzl3kfXDGZQKzqAQEQUqBhQfZmxogqXZfqhevI+3uW+LMyhERMSA4sOk2ZOosCCEBCk9PBr5OGZQKushitxqTEQUiBhQfJijQNaPlncAoH90GBQC0NBkdexSIiKiwMKA4sNae6D4z/IOAASrFOgXHQqAHWWJiAIVA4oPK6vxr3N42hoYwzoUIqJAxoDiw/ztHJ62pDqUs5xBISIKSAwoPqzMz04ybos7eYiIAhsDig+TmrT5Uw8USdudPEREFHgYUHxY6wyK/y3xDIprnUHhVmMiosDDgOKjbDYRZTX+dw6PZECMfQalprEZ1fVNHh4NERH1NQYUH1Vdb0GT1T6zEBfhfzMoIUFKJGrtwYt1KEREgYcBxUdJXWRjw4MRrPLP/xlb61AYUIiIAo1//mYLAI4CWT9c3pE4eqHw0EAiooDDgOKjyvy4B4pkYFxLL5QqBhQiokDDgOKjpCUefzuHpy32QiEiClwMKD7Kn7vIStgLhYgocDGg+CjpHB6/rkFpmUGpqrPA2MCtxkREgYQBxUe11qD4b0CJUKscW6h5Jg8RUWBhQPFRpX7cRbatQS3LPKxDISIKLAwoPshqE1Fe678HBbYlLfOwFwoRUWBxe0DJzs6GIAhYsmSJ4zlRFJGZmYmkpCSEhoZi5syZyM/Pd/dQ/EZlnRlWmwiFYG/U5s8GOmZQuMRDRBRI3BpQ9uzZgzfffBOjR492en758uVYsWIFVq9ejT179kCv12P27Nmoqalx53D8hnRIYGyEGiqlf0+CSQGFNShERIHFbb/damtrsXDhQrz11luIjo52PC+KIlauXIlly5Zh/vz5SEtLw7p161BfX48NGza4azh+JRC2GEvYC4WIKDC5LaA8+OCDuO6663DNNdc4PV9YWAiDwYA5c+Y4nlOr1ZgxYwby8vI6/Cyz2QyTyeT0CGSB0KRNIgWUshqz4/RmIiLyf24JKBs3bsT+/fuRnZ3d7jWDwQAA0Ol0Ts/rdDrHaxfLzs6GVqt1PJKTk+UftA+RZlD8uQeKRBsWhH5RoQCAWX/LxT9yT6KxyerhURERkbvJHlCKiorw6KOPYv369QgJ6fwXqCAITn8WRbHdc5KlS5fCaDQ6HkVFRbKO2ddIMwmBsMQDAK8uHIfheg1qGpuR/d8fMetvudh84BxsNtHTQyMiIjeRPaDs27cPZWVlGD9+PFQqFVQqFXJzc/HKK69ApVI5Zk4uni0pKytrN6siUavViIyMdHoEsjJTYGwxloxNjsJ/HpmGl24ZDX1kCM5faMBjmw5h3uod+PZEhaeHR0REbiB7QJk1axaOHDmCgwcPOh4TJkzAwoULcfDgQQwePBh6vR45OTmO91gsFuTm5iIjI0Pu4fil0gCbQQEApULArROSsfXJmfjtTy5DhFqF/GITFr69C/es3Y1jBu4AIyLyJyq5P1Cj0SAtLc3pufDwcMTGxjqeX7JkCbKyspCamorU1FRkZWUhLCwMCxYskHs4fkkqkk0IgCLZi4UGK/HgVUNxx8RkrPrmBNbvPIP/HSvHtuPluHV8Mp6ZdznC1bL/te6R3OPlKL7QgDsmJne6fElERB3zyL/kTz31FBoaGrB48WJUV1cjPT0dW7ZsgUaj8cRwfEqz1YaKAOki25XYCDUybxiJRRmD8NKXP+LzIwZs2luEYmMD1iyaiGCVZ/vDfHHUgMUf7INNBAbEhGHK0DiPjoeIyNcIoij6XKWhyWSCVquF0WgMuHoUg7ERV2Z/DaVCQMGf50Kh4H+ZA0DeyQr8ct1e1FusmDcmCX+/fazH7s3OU5W4+53dsDTbAADXjUrEqwvHeWQsRETexJXf3/7dhtQPObYYa9QMJ21kDInDGz8fD5VCwL8PFeP5z76HJ7J3frERv1q3F5ZmG8YPtDco/DLfwB4uREQuYkDxMW0DCjmbPiwef7ttDADg3bzTeHXriT79/rOV9Vj0zh7UmJsxKSUGH/wyHeMGRKHZJuKfe8/16ViIiHwdA4qPKa1pKZAN4PqTrtw4th+euf5yAMBftxzHh7vP9sn3lteYcdc7u1BRa8ZwvQZv3T0BIUFKLEwfCAD4cPdZWNm3hYio2xhQfExZAJ3D01P3Tk3B4plDAADLNh/BF0c77lAsl5rGJtyzdjfOVNYjOSYU7907CdrQIADAdaMToQ0NwrnqBmwrKHfrOIiI/AkDio8pC6BzeHrjtz+5DLdPSIZNBB7ZeAA7T1W65XvMzVbc//4+5BebEBsejPfuTXea3QoJUuJn4/oDADbs6pvZHCIif8CA4mNam7QxoHRFEAS8cHMaZl+ug6XZhl+t24vvi+U9ZNJqE/HYpoPIO1mJCLUK6+6dhJS48HbXLUi3nx319Q+lKDE2yDoGIiJ/xYDiYxxN2rjEc0kqpQKr7rwCkwbFoMbcjEVrd+NsZb0sny2KIp799Cg+P2JAsFKBN+8aj7R+2g6vHZqgQXpKDGwisGlPYJ8jRUTUXQwoPqa1BoUzKN0REqTEW4smYLheg/IaM+5+Z1evt/yKooiVXxVg/c6zEARg5R1jkXGJRmwL0gcAADbuLkKz1dar7yciCgQMKD7E0mxDZZ0FAAOKK7ShQXjv3knoHx2K05X1uPnVvB4v91iabVj2yVH8/esCAMDzN6bh2lGJl3zfT9P0iAkPhsHUiK3HWCxLRHQpDCg+pLylxX2QUkB0WJCHR+NbEiJD8MEv05ESF47zFxrws9fz8MXREpc+o6rOgrvW7MKGXfaZkz9cNwJ3XTmwW+9Vq5S4dYK9WPaDXWdcHj8RUaBhQPEhrU3aQnj4XA8MjA3HJ4unYFpqHBqarHhg/X688nVBtzrO/mgw4YbVO7CrsAoRahXWLJqAX04b7NL33znRvsyTe7wcRVXy1MIQEfkrBhQfItWfsEC257RhQVh7z0T8YsogAMCKnON4aMMBNFisnb5nS74BP3stD+eqGzAwNgybF2fg6uE6l797UFw4pqXGQRSBjXu45ZiIqCsMKD6klD1QZKFSKvDsvJH4y89GIUgp4D9HSnDLG3kovuC8BVgURaz+pgC/fn8f6ixWZAyJxSeLpyBV1/NTtxdMss+ibNpzDk0sliUi6hQDig8pZRdZWd0+cQA2/OpKxIYHI7/YhBtWf4t9Z6oBAA0WKx7ZeBB/3XIcALBo8kCsu3cSosODe/Wd11yuQ7xGjYpaM3K+L+31z0BE5K8YUHxIGc/hkd3EQTH410NTMCIxEhW1Ztz55k6s2VGI2/7xHf59qBgqhYCsm0fhuRvTEKTs/f+7BCkVuH2CvXEbi2WJiDrHgOJDStkDxS36R4fh/z0wGT8dqYfFasOfPvseR84bER0WhPW/THf0MJHLHZOSIQjAtycqUVhRJ+tnExH5CwYUH+I4h4dLPLILV6vw2sJxeHRWKgDgMp0Gnz40FVcOjpX9u/pHh+GqyxIAoM9OWyYi8jUMKD6E5/C4l0Ih4LHZw7Dr97Pwn0emIjkmzG3fJRXL/nNvERqbOt9BJLcfDSac5qwNEfkABhQf0dhkxYX6JgDcxeNuusgQqGSoN+nKVcMTkKQNQXV9E77MN7j1uyQnymoxb9UOXPfKdoYUIvJ6DCg+orylQFatUiAyVOXh0VBvKRUCbm9p3PbBzr5Z5nk55ziarCLqLFY8svEALM3c5kxE3osBxUe0LZBlF1n/cPvEZCgVAnafrsKfPvse2wvK3bbcc/S8Ef85UgJBADQhKhw+Z8Rftxxzy3cREcmBAcVHlLJA1u/otSG4aWw/AMCaHYW4a81ujHluC+5aswtvbz+FY4aabrXh746/tYSReaOT8LdbxwAA3tx2Cv87VibL5xMRyY1rBT6i7Tk85D9e/NkozLwsHtsLyrHteAUMpkZsL6jA9oIKAD8gQaPGtNR4zLwsHnPT9D2qjdl3pgpbj5VD2VIEnBIXjrsnD8R7353Bk/88hM8fnca/V0TkdRhQfIS0g4fn8PiXIKUC88YkYd6YJIiiiBNltdhWUIHtBeXYeaoSZTVmfLT/HD7afw43jU3Cy7ePdWmJTxRFvPSlffbklnH9kRIXDgD4/bUjsLuwCj8aavDE/x3Cul9MgkLBpUMi8h5c4vER5Y4lHv6Xrr8SBAGpOg3um5qCd38xCQefmYMPfpmOX08fDKVCwCcHi/HR/vMufea3Jyqx81QVgpUKPHJNquP5kCAlVi+4AiFBCmwvqMBb20/J/eMQEfUKA4qPaO2BwhmUQBESpMSUoXH4/bUj8PjsYQCAZ/51FKfKa7v1flEU8VJL7cmC9AHoFxXq9PrQBA0y540EALz05TEcLLog3+CJiHqJAcVH8CTjwPbAjCHIGBKLeosVD394AObmS+/2yfm+FIeKLiA0SIkHrxra4TW3T0zGdaMS0WwT8ciHB1DT2CT30ImIeoQBxUc4imS5xBOQlAoBL98+FjEtJy//5b9dbxG22USsyLGfxHzPlEGI13Q88yYIArLmj0K/qFCcrarHHz45KtvOISKi3mBA8QH1lmbUNDYD4BJPINNFhuCvt44GALzzbSG++bG002v/fbgYPxpqoAlR4f7pg7v8XG1oEF658wooFQL+1YM6FyIid2BA8QHSIYFhwUpEqLnxKpBdPVyHX0wZBAB48p+HHTNrbTVZbXi5Zfbk19MGIyos+JKfO35gdI/qXIiI3IUBxQewiyy19fTc4bg8MRJVdRY8tukgrDbnJZmP9p3D6cp6xIQH4xdTU7r9uRfXuTRZ2QqfiDyHAcUHlLacw5PQSR0BBRa1SolVC65AWLASeScr8UbuScdr5mYrXvm6AACweOYQl2bcpDqXqLAg5Beb+uwQQyKijjCg+IAyFsjSRYbER+C5G+xbhFfkHMe+M9UAgA27zqLY2AhdpBo/v3Kgy5+riwzB3S3v27Crbw4xJCLqCAOKD3As8XAGhdq4ZXx/3DAmCdaWLcKlpka8uvUEAOCRWakICVL26HNvnzQACgHIO1mJk6xFISIPYUDxAWU17CJL7QmCgBduTkNyTCjOX2jA9at2oKLWggExYbhtQnKPP7dfVCiuuiwBAPAhZ1GIyEMYUHxAaw8UzqCQM01IEFbdOQ4qhYDyliC75JpUBPXgUMG2Fl45AADw//afQ2PTpZvCERHJjQHFB5TxHB7qwtjkKDz5k8sAAMN0EbhxbL9ef+aMYQnoFxWKC/VN+O/Rkl5/HhGRqxhQfEDbbcZEHbl/+mC8c88ErLt3EpQynEqsVAi4Y6J9mYjFskTkCQwoXq7W3Iw6i32KnduMqTOCIODq4TokakMvfXE33TYxGUqFgD2nq3HMUCPb5xIRdQcDipeTZk80ahXC2UWW+pAuMgSzR+gAABt2nfHwaIgo0MgeULKzszFx4kRoNBokJCTgpptuwrFjzgebiaKIzMxMJCUlITQ0FDNnzkR+fr7cQ/ELLJAlT1qQbi+W/fjAedRbmj08GiIKJLIHlNzcXDz44IPYuXMncnJy0NzcjDlz5qCurs5xzfLly7FixQqsXr0ae/bsgV6vx+zZs1FTw2nkixVV1QOArFP3RN01dWgcBsSEoaaxGZ8dYrEsEfUd2QPKF198gXvuuQcjR47EmDFjsHbtWpw9exb79u0DYJ89WblyJZYtW4b58+cjLS0N69atQ319PTZs2CD3cHzewSIjAGBkv0gPj4QCkUIhOGZRPtjNYlki6jtur0ExGu2/YGNiYgAAhYWFMBgMmDNnjuMatVqNGTNmIC8vr8PPMJvNMJlMTo9AcbDoAgDgiuQoj46DAtct4/sjSCngUNEFHD1v9PRwiChAuDWgiKKIxx9/HFOnTkVaWhoAwGCwH0Cm0+mcrtXpdI7XLpadnQ2tVut4JCf3vEumL6m3NOOYwR7GxiZHe3g0FKjiItT4aVoiAOADbjkmoj7i1oDy0EMP4fDhw/jwww/bvSYIzr0aRFFs95xk6dKlMBqNjkdRUZFbxuttjp43wSYC+sgQ6LXsgUKes2CSfZnn04PnUWtmsSwRuZ/bAsrDDz+MTz/9FFu3bkX//v0dz+v1egBoN1tSVlbWblZFolarERkZ6fQIBAeL7CfUjknWengkFOiuHByDwfHhqLNY8cmB854eDhEFANkDiiiKeOihh/Dxxx/jm2++QUpKitPrKSkp0Ov1yMnJcTxnsViQm5uLjIwMuYfj06T6Ey7vkKcJgoCF6QMB2Jd5RFH08IiIyN/JHlAefPBBrF+/Hhs2bIBGo4HBYIDBYEBDQwMA+z90S5YsQVZWFjZv3oyjR4/innvuQVhYGBYsWCD3cHzawbMXANjPWiHytJ+N64dglQI/lJgc4ZmIyF1kb036+uuvAwBmzpzp9PzatWtxzz33AACeeuopNDQ0YPHixaiurkZ6ejq2bNkCjUYj93B8VpmpEcXGRigEYHR/LvGQ50WFBeP60Yn4eP95fLDrLK4YwJk9InIftyzxdPSQwglgn0XJzMxESUkJGhsbkZub69jlQ3YHWv4LdZhOwxb35DUWtvRE+exwMYz1TR4eDRH5M57F46Va60+iPDoOorbGDYjGcL0GjU02fHzgnKeHQ0R+jAHFSx1qCShjGFDIiwhCa2fZDSyWJSI3YkDxQlabiMPn7B07OYNC3uamK/ohNEiJgrJa3P3ObryzoxAnymoYVohIVixu8EIny2tRa25GWLASw3QsHCbvEhkShF9PH4y/f12A7QUV2F5QAQBI0oZgWmo8pg2Lw9ShcYgKC/bwSInIlzGgeCFpe/GoflooFR131yXypCXXpOKnaXpsLyjHtuMV2H26CsXGRmzaW4RNe4sgCMDo/lGYkRqH+6YOhjYsyNNDJiIfw4DihaQdPGMHRHl0HESdEQQBIxIjMSIxEr+ePgQNFit2n67CtuPl2F5QjuOltThUdAGHii7gu1OV+PBXV0Kl5IoyEXUfA4oX4gnG5GtCg5WYMSweM4bFAwAMxkZsO16O5z/7HntOV2PVNyfw2OxhHh4lEfkS/ieNl2l7gjF38JCv0mtDcNvEZLxws72/0apvCrDzVKWHR0VEvoQBxctIJxjrItVI1IZ6ejhEvXLj2H64ZXx/2ETgsU0HUV1n8fSQiMhHMKB4GekEY24vJn/x3A0jMTguHCXGRjz10WFuRyaibmFA8TI8wZj8TbhahVfuvALBSgVyvi/F+p1nPD0kIvIBDChehicYkz9K66fF7+YOBwD86T8/4IcSk4dHRETejgHFi/AEY/Jn904ZhKsui4el2YaHPzyABovV00MiIi/GgOJFpP4nqQk8wZj8jyAI+OutYxCvUeNEWS2e/yzf00MiIi/GgOJFeIIx+bvYCDVW3j4WggB8uLsI/zlc4ukhEZGXYkDxIo76E3aQJT82ZWgcfjNjCADg6Y8Po6iq3sMjIiJvxIDiJaw2EUfO8wRjCgyPzR6GKwZEoaaxGY9uPIBmq83TQyIiL8OA4iV4gjEFkiClAq/ccQU0ahX2n72Al7Yc8/SQiMjLMKB4CZ5gTIEmOSYMWfNHAQD+kXsK73932rMDIiKvwoDiJQ6wQJYC0LwxSXjsGvshgs98mo/PDhd7eETdd7K8Fo1N3CpN5C4MKF6CO3goUD0yayjunjwQYst5PTsKKjw9pEt6Z0chZv0tF7e+8R3rZ4jchAHFC7Q9wZg7eCjQCIKAZ+eNxHWjEtFkFXH/+3tx+NwFTw+rU58cOI/nP/seAHDkvBHrvmPrfiJ3YEDxAkfOGXmCMQU0pULAitvHYMrQWNRZrPjF2j0orKjz9LDa+d+xMjz5z0MAWrs9v5xzHKWmRk8Oi8gvMaB4gUMt/7XI5R0KZGqVEv+4awLS+kWiss6Cu9bs8qpf/AfOVuM36/ej2SbihjFJ+Pg3GRiTHIVaczP+/J8fPD08Ir/DgOIFeIIxkV2EWoV3fzEJg2LDcK66AYve2Q1jQ5Onh4UTZbW49909aGiyYvqwePz11jFQKRV44aY0KATg34eK8e0J76+dIfIlDCheQNpiPCaZBwQSxUWo8f596YjXqPGjoQa/WrfXo7tlSowNuHvNLlTXN2FMchReXzgOwSr7P51p/bS468qBAIA//usozM3c1UMkFwYUD5NOMBYEYHT/KE8Ph8grJMeE4b17J0ETosLu01V4aINnus1eqLfg7jW7UWxsxOD4cKy9Z2K7gzwfn3MZ4iLUOFVeh7e3F/b5GIn8FY/M9TCp/8mwBA0ieIIxkcOIxEi8ffcE3PXObnz1QylufPVbxIQHd/meqLBgTBkSi2nD4tEvqncF5w0WK+59dw8KymqhjwzBe/dO6vD7taFBWHbdcDy26RBWfVOAG8YkITkmrFffTUQMKB7H/idEnUsfHIvVd16BB9bvQ36xqVvv+fche7O3IfHhmJYaj+nD4pCeEttu5qMrTVYbFn+wD/vPXoA2NAjv3TcJ/aM7Dx03je2HjbuLsKuwCs9/9j3euntCt7+LiDrGgOJhPMGYqGtzRurxxZLpONpymGZXzlTWY8eJChw4W42T5XU4WV6Hd/NOI0gpYPzAaEwfFo9Jg2IQEqTs8nPe2VGIrcfKERKkwDv3TLjk+ViCIOBPN6Xh2r9vR873pfj6h1LMGqFz6efsSrPVhopaC/TaENk+k8jbMaB4EE8wJuqeYTpNtw/RfGz2MBgbmvDdyQpsK6jAtuPlOFfdgJ2nqrDzVFW3v1OpEPDawnEYPzCm22O8b2oK/rHtFDL/nY8pQ+MuGYQuRRRFfJlvwF++OIbCijr8/trh+PX0Ib36TCJfwYDiQdIJxqFBSqQmRHh6OER+QxsahJ+mJeKnaYkQRRGnK+uxvaAc245X4IcSE2yi2OX7Q4OVeHz2MFw93LVZkEdmpeLTQ8UoqmrAa1tP4PE5l/X4Z9h3pgpZn/+IfWeqHc9lff4jYsLVuGV8/x5/LpGvYEDxoANn7f/wjOqvhUrJDVVE7iAIAlLiwpESF467Jw9y63eFq1V45vrL8ZsP9uON3FO4eVx/pMSFu/QZp8prsfyLY/gi3wAACA1S4lfTB6O2sRnvfFuI3310GNFhQbIuIRF5IwYUDzlbWY+/bjkOAJg0qHtTyETk/X6apseMYfHIPV6OZ/51FO/dOwmCIFzyfRW1ZrzydQE27DqLZpsIhQDcNiEZj80eBl1kCGw2ERcaLPh4/3k8uGE/PvhlereXn4h8kSCKl5jr9EImkwlarRZGoxGRkZGeHo7LKmrNuOX1PJyurMdwvQab7p8MbWiQp4dFRDI5XVGHOSu3wdJsw4vzRyF9cGyn14qiiM+PlOCN3FOoNTcDAK4enoCn5w5vV3fTZLXh/vf34ZsfyxAZosI/H8jAZfru1eYQeQNXfn8zoPSxmsYm3PnWThw9b0L/6FB89JsM6CJZmU/kb1bkHMcrXxe49J60fpH4/bUjkDEkrtNrGixW/HzNLuw7Uw1dpBof/Sajyy3QRN6EAcVLmZvtp7TmnaxEbHgw/vnAZAyOZ3EskT9qbLLinrW7kX/+0v1bdNoQPHz1UMwbnQSF4tLLQRfqLbjtH9/heGktBseH4/89kHHJJnZE3oABxQtZbSIe+fAA/nOkBOHBSmz89WSM6s+zd4ioZ0qMDfjZa3koNjZiTH8tNvzqSpea0XXE3GzFvjPV2HmyEjHhwbh94gCEBvduqzRRWwwoXkYURTzzr3y8v/MMgpQC1t4zCVNTO5/CJSLqjhNltbj1jTxU1zdhWmoc1iya6DjIsDtEUcTJ8jpsLyjH9oIK7DxViXpL64GH+sgQPDFnGOaP6w9lN2Z2iC7FZwLKa6+9hpdeegklJSUYOXIkVq5ciWnTpl3yfb4WUP7+VQFe/uo4BAFYdecVuH50kqeHRER+4sDZaix4axcamqy4YUwS/nD9CAjoPEw0WW04cPaCI5Scv9Dg9HpchBoZQ2Kx70y147Xheg2WXjsCM4bFu+VnqLfY+0F1Z7eTN5Fr3JZm+0GYroTLjoiiiIYmK8KCe79B19jQhIpaM4bIXIbgEwFl06ZNuOuuu/Daa69hypQp+Mc//oG3334b33//PQYMGNDle30poKzfeQZ/+OQoAOD5G0e6vQ8DEQWe3OPluO/dPWi2uf7PebBKgUmDYjAtNQ7TUuMxXK+BQiGgscmK9747jdXfnICp0b67aFpqHJ6eOxwjk3q3PF1vacauU1XY1hKSTpTVol9UKKYPs49hypA4aMO8b2djvaUZuwqrsP14BbYVlDvGLd27KUNjERV26VogURRxvLQW246XY1tBOXYX2jscpw+OxfTUOEwfFo/UhIhuBZ/qOgu+PWnvmLy9oAIlxkYM00VgWmo8pqXaz6HqzjJds9WGQ+cuYNvxCmwvKMfBoguYMDAG//fA5EvfGBf4REBJT0/HuHHj8PrrrzueGzFiBG666SZkZ2d3+V5fCSifHynBgxv2QxSBR64e2quukkREXfn3oWI886+juNDQdMlrh8ZHYPqw7v0Cq66zYPXWE3jvu9NosooQBODmK/rhiTmXdfvEaJtNxA8Gk+OX397T1bBYbZ1erxCA0f2jMH1YPKanxmFscpRHmlmKoogfSmpaglQ59hR2c9ypcZg2LB5jk6MQ1DLuylozdpyocNyDshpzl9+ti1S3HHYZj6lD4xxF0G1nwLYdL8fh80Z09VtcCqBS+Buu1ziCT1FVvf1nO16Bb09WoKYliEqG6zX47OGpst57rw8oFosFYWFh+Oc//4mbb77Z8fyjjz6KgwcPIjc31+l6s9kMs7n1f0yTyYTk5GTZA0pFrRmvbj0hy2dZbSI27i6CxWrDnZMGIOvmNJ+bviQikhRV1eOlL4/h05bTooNVCtw4JgkRIV0vJ1TVWfDtiQpU1FqcnpdmTKanxmPcwGj8UNIaYArKap2u1ahVyBgai6RuBiI52MddiYpa5yDRdqZnfMu4txfYx328tP240wfHwGBqxNGLdnOFBCmQnhLrCIoAWmZUKrDrVCXMza1BSBCAUf20iI9QY1dhlaNfjuQyncY+izMsHpfpNNh3ptoRYIqNjU7XxmvUGD8gGsdKa1BYUef0mjY0CFNT4zA9NQ5TU+O7HUBd4fUBpbi4GP369cO3336LjIwMx/NZWVlYt24djh075nR9ZmYmnnvuuXafI3dAOVVei6v/lnvpC13w05F6vLpwHAvMiMgvHCq6gKzPf8Cuwu4fvAgAYcFKTB7c+gs5JS680/9oKzE2OJZRdpyowIX6S88KuYs0bikADL7UuAsqsL2gAjsKylF90bhHJEY6Qtn4gdGdHibZ2GTF3tPV2NYSMn401Di9HhMejKlD4xxLS52dcn1xEfR3JyvR0NRaBK1SCBg3INrxs43qp3X77yqfCSh5eXmYPLl1feuFF17A+++/jx9//NHp+r6aQamqs2DNjlOyfZ5eG4rbJvSHWsVtekTkP0RRRO5x+1KNiK5/hYSolJiYEoNxA6J7VARqtYnILzYi72Qlahr7LqioVUpMHBSD8QN7Nm6bTUR+sQm7Cu1btqemxiFB07OmnGWmRmwvqEB1vQVXDo7F5YmR3eqXczFpG/mhIiOGxIdj8pBYaEL6ttbH6wOKq0s8F/OVGhQiIiJq5crvb48coRscHIzx48cjJyfH6fmcnBynJR8iIiIKTB47zfjxxx/HXXfdhQkTJmDy5Ml48803cfbsWTzwwAOeGhIRERF5CY8FlNtvvx2VlZV4/vnnUVJSgrS0NHz++ecYOHCgp4ZEREREXoKt7omIiKhPeH0NChEREVFXGFCIiIjI6zCgEBERkddhQCEiIiKvw4BCREREXocBhYiIiLwOAwoRERF5HQYUIiIi8joMKEREROR1PNbqvjek5rcmk8nDIyEiIqLukn5vd6eJvU8GlJqaGgBAcnKyh0dCRERErqqpqYFWq+3yGp88i8dms6G4uBgajQaCIMj62SaTCcnJySgqKuI5P32A97tv8X73Ld7vvsX73bd6cr9FUURNTQ2SkpKgUHRdZeKTMygKhQL9+/d363dERkbyL3gf4v3uW7zffYv3u2/xfvctV+/3pWZOJCySJSIiIq/DgEJERERehwHlImq1Gs8++yzUarWnhxIQeL/7Fu933+L97lu8333L3ffbJ4tkiYiIyL9xBoWIiIi8DgMKEREReR0GFCIiIvI6DChERETkdRhQ2njttdeQkpKCkJAQjB8/Htu3b/f0kPzGtm3bMG/ePCQlJUEQBHzyySdOr4uiiMzMTCQlJSE0NBQzZ85Efn6+Zwbr47KzszFx4kRoNBokJCTgpptuwrFjx5yu4f2Wz+uvv47Ro0c7mlVNnjwZ//3vfx2v8167V3Z2NgRBwJIlSxzP8Z7LJzMzE4IgOD30er3jdXfeawaUFps2bcKSJUuwbNkyHDhwANOmTcPcuXNx9uxZTw/NL9TV1WHMmDFYvXp1h68vX74cK1aswOrVq7Fnzx7o9XrMnj3bce4SdV9ubi4efPBB7Ny5Ezk5OWhubsacOXNQV1fnuIb3Wz79+/fHiy++iL1792Lv3r24+uqrceONNzr+kea9dp89e/bgzTffxOjRo52e5z2X18iRI1FSUuJ4HDlyxPGaW++1SKIoiuKkSZPEBx54wOm54cOHi08//bSHRuS/AIibN292/Nlms4l6vV588cUXHc81NjaKWq1WfOONNzwwQv9SVlYmAhBzc3NFUeT97gvR0dHi22+/zXvtRjU1NWJqaqqYk5MjzpgxQ3z00UdFUeTfb7k9++yz4pgxYzp8zd33mjMoACwWC/bt24c5c+Y4PT9nzhzk5eV5aFSBo7CwEAaDwen+q9VqzJgxg/dfBkajEQAQExMDgPfbnaxWKzZu3Ii6ujpMnjyZ99qNHnzwQVx33XW45pprnJ7nPZdfQUEBkpKSkJKSgjvuuAOnTp0C4P577ZOHBcqtoqICVqsVOp3O6XmdTgeDweChUQUO6R53dP/PnDnjiSH5DVEU8fjjj2Pq1KlIS0sDwPvtDkeOHMHkyZPR2NiIiIgIbN68GZdffrnjH2nea3lt3LgR+/fvx549e9q9xr/f8kpPT8d7772HYcOGobS0FH/+85+RkZGB/Px8t99rBpQ2BEFw+rMoiu2eI/fh/ZffQw89hMOHD2PHjh3tXuP9ls9ll12GgwcP4sKFC/joo4+waNEi5ObmOl7nvZZPUVERHn30UWzZsgUhISGdXsd7Lo+5c+c6/u9Ro0Zh8uTJGDJkCNatW4crr7wSgPvuNZd4AMTFxUGpVLabLSkrK2uXDEl+UkU477+8Hn74YXz66afYunUr+vfv73ie91t+wcHBGDp0KCZMmIDs7GyMGTMGf//733mv3WDfvn0oKyvD+PHjoVKpoFKpkJubi1deeQUqlcpxX3nP3SM8PByjRo1CQUGB2/9+M6DA/o/L+PHjkZOT4/R8Tk4OMjIyPDSqwJGSkgK9Xu90/y0WC3Jzc3n/e0AURTz00EP4+OOP8c033yAlJcXpdd5v9xNFEWazmffaDWbNmoUjR47g4MGDjseECROwcOFCHDx4EIMHD+Y9dyOz2YwffvgBiYmJ7v/73esyWz+xceNGMSgoSFyzZo34/fffi0uWLBHDw8PF06dPe3pofqGmpkY8cOCAeODAARGAuGLFCvHAgQPimTNnRFEUxRdffFHUarXixx9/LB45ckS88847xcTERNFkMnl45L7nN7/5jajVasX//e9/YklJieNRX1/vuIb3Wz5Lly4Vt23bJhYWFoqHDx8Wf//734sKhULcsmWLKIq8132h7S4eUeQ9l9MTTzwh/u9//xNPnTol7ty5U7z++utFjUbj+N3oznvNgNLGq6++Kg4cOFAMDg4Wx40b59iWSb23detWEUC7x6JFi0RRtG9Xe/bZZ0W9Xi+q1Wpx+vTp4pEjRzw7aB/V0X0GIK5du9ZxDe+3fO69917Hvxvx8fHirFmzHOFEFHmv+8LFAYX3XD633367mJiYKAYFBYlJSUni/Pnzxfz8fMfr7rzXgiiKYu/nYYiIiIjkwxoUIiIi8joMKEREROR1GFCIiIjI6zCgEBERkddhQCEiIiKvw4BCREREXocBhYiIiLwOAwoRERF5HQYUIiIi8joMKEREROR1GFCIiIjI6zCgEBERkdf5/1ahBqbcCE+nAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(occurences_lens)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [],
   "source": [
    "# batch_size    # TT    # GPU util\n",
    "#  8            11,5k       32%\n",
    "#  16           23k         32%\n",
    "#  32           45k         32%\n",
    "#  64           86k         36%\n",
    "#  128          168k        40%\n",
    "#  256          330k        36%\n",
    "#  512          640k        38%\n",
    "#  1024         1.14M       46% (optimal)\n",
    "#  2048         1.5M        48%"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "torch23",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.3"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}