part1exercises.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "########################\n",
    "#\n",
    "# Excersises\n",
    "#\n",
    "########################"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\n",
    "### E01: train a trigram language model, i.e. take two characters as an input to predict the 3rd one. Feel free to use either counting or a neural net. Evaluate the loss; Did it improve over a bigram model?\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import matplotlib.pyplot as plt \n",
    "import torch.nn.functional as F\n",
    "\n",
    "import requests \n",
    "from tqdm import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "device = \"mps\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "32033 15 2\n"
     ]
    }
   ],
   "source": [
    "url = \"https://raw.githubusercontent.com/karpathy/makemore/master/names.txt\"\n",
    "# url = \"https://raw.githubusercontent.com/dwyl/english-words/master/words_alpha.txt\" #English dictionary\n",
    "\n",
    "res = requests.get(url)\n",
    "words = (res.text).splitlines()\n",
    "print(len(words), max(len(w) for w in words), min(len(w) for w in words))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "chars = sorted(list(set((''.join(words)))))  #get unique characters \n",
    "stoi = {s:i+1 for i, s in enumerate(chars)}  # map char to int\n",
    "stoi[\".\"] = 0 \n",
    "itos = {i:s for s,i in stoi.items()}         # map int to char"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "# calculate trigram frequencies in Matrix Form \n",
    "N = torch.zeros( 27, 27, 27, dtype=int, device=device)\n",
    "b = {}\n",
    "for w in  words:\n",
    "    chs =  ['.']  + ['.']  +  list(w) + ['.'] \n",
    "    \n",
    "    for ch1,ch2,ch3 in zip(chs, chs[1:], chs[2:]):\n",
    "        \n",
    "        ix1 = stoi[ch1]\n",
    "        ix2 = stoi[ch2]\n",
    "        ix3 = stoi[ch3]\n",
    "        N[ix1][ix2][ix3] += 1\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([27, 27, 27])\n"
     ]
    }
   ],
   "source": [
    "# Convert trigram frequency matrix to probability matrix\n",
    "P=(N+ 0.0001).float()# adding 1 to prevent probability from being 0 giving infinite loss\n",
    "P=P/P.sum(1, keepdim=True)\n",
    "print(P.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "sanvitxaightobryaze.\n",
      "bohoshuzz.\n",
      "jayuxxjqpvfpkfqpjfjp.\n",
      "sevohn.\n",
      "chapp.\n",
      "shuxxtjubygraitzchuxxwfxkzqmtormimmin.\n",
      "xon.\n",
      "m.\n",
      "gwywayqushwewisenzonyktqxgxjkehrichugackwuwaffkpqqkthaevrishadjulquayachiffe.\n",
      "jujhonzuhabby.\n",
      "m.\n",
      "fayvayvervigbqftobbod.\n",
      "glynciji.\n",
      "h.\n",
      "kiyaquippqjvijaimbuckwu.\n",
      "p.\n",
      "jovamakutchimbelorykiljolbudwehuxxwxwxx.\n",
      "hriggiusfiffiqualyn.\n",
      "wyelwojowymewecildoukwebubremulquaqus.\n",
      "b.\n"
     ]
    }
   ],
   "source": [
    "# sampling dumb lang  model basesd on probability matrix \n",
    "for i in range(20):\n",
    "\n",
    "  ix1 = 0\n",
    "  ix2 = 0 \n",
    "  out = []\n",
    "  while True:\n",
    "    p = P[ix1][ix2] \n",
    "    \n",
    "    # print(ix1,ix2)\n",
    "    ix1=ix2\n",
    "    ix2 = torch.multinomial(p,num_samples=1,replacement=True).item()\n",
    "    \n",
    "    out.append(itos[ix2])\n",
    "    \n",
    "    if ix2 == 0:\n",
    "      break\n",
    "  print(''.join(out))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "log -416811.03125\n",
      "nll 416811.03125\n",
      "LOSS nll/n 1.826948642730713\n"
     ]
    }
   ],
   "source": [
    "# loss calculation for model based on probability materix \n",
    "logLikelihood = 0.0\n",
    "nll = 0\n",
    "n = 0\n",
    "for w in  words:\n",
    "    chs = ['.'] + ['.'] + list(w) + ['.']\n",
    "    for ch1,ch2,ch3 in zip(chs, chs[1:], chs[2:]):\n",
    "        ix1 = stoi[ch1]\n",
    "        ix2 = stoi[ch2]\n",
    "        ix3 = stoi[ch3]\n",
    "        prob = P[ix1][ix2][ix3]\n",
    "        logprob = torch.log(prob) #multiplication of probabilites same as addition of negative log probabilites\n",
    "        logLikelihood += logprob\n",
    "        n += 1  \n",
    "    nll = -logLikelihood\n",
    "      \n",
    "print(f\"log {logLikelihood}\")\n",
    "print(f\"nll {nll}\")\n",
    "print(f\"LOSS nll/n {nll/n}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "#This model has lower loss compaerd to the bigram probability matrix model (2.45)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Now neural net work approach"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "#create training set \n",
    "xs, ys = [], []\n",
    "n = 0\n",
    "for w in words:\n",
    "    chs = ['.'] + ['.'] + list(w) + ['.']\n",
    "    for ch1, ch2, ch3 in zip(chs, chs[1:], chs[2:]):\n",
    "        ix1 = stoi[ch1]\n",
    "        ix2 = stoi[ch2]\n",
    "        ix3 = stoi[ch3]\n",
    "        # print(ch1,ch2) be carful with prints and for loops long for loops are very hard to stop ,kernel become un alive \n",
    "        xs.append([ix1, ix2])\n",
    "        ys.append(ix3)\n",
    "\n",
    "xs = torch.tensor(xs)\n",
    "ys = torch.tensor(ys)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "W = torch.randn((54,27),requires_grad=True).float() "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 200/200 [00:12<00:00, 16.11it/s, loss=tensor(2.4114)]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x17dbec5e0>]"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAiMAAAGdCAYAAADAAnMpAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/P9b71AAAACXBIWXMAAA9hAAAPYQGoP6dpAABCLElEQVR4nO3deXyU5b3///dkmck6k4SsJGGHoGxVRAyotYoLehDaHk+L/BpsObVisGpbv554tBb9aWhpbe2GS93O19K0ekROFeTgEpCyL6kBFNlCIiSELTNZJ8nM/f0jyUAk22S7s7yej8c8MnPPNTOf25tk3l7XdV+3xTAMQwAAACYJMLsAAAAwuBFGAACAqQgjAADAVIQRAABgKsIIAAAwFWEEAACYijACAABMRRgBAACmCjK7gI7wer06ceKEIiMjZbFYzC4HAAB0gGEYKi8v19ChQxUQ0Hr/R78IIydOnFBqaqrZZQAAgE4oKipSSkpKq8/3izASGRkpqWFn7Ha7ydUAAICOcLlcSk1N9X2Pt6ZfhJGmoRm73U4YAQCgn2lvigUTWAEAgKkIIwAAwFSEEQAAYCrCCAAAMBVhBAAAmIowAgAATEUYAQAApiKMAAAAU3UpjCxbtkwWi0UPPPBAm+3eeOMNjR8/XiEhIZo0aZLWrFnTlY8FAAADSKfDyI4dO/T8889r8uTJbbbbvHmz5s+fr0WLFmnPnj2aN2+e5s2bp71793b2owEAwADSqTBSUVGhBQsW6MUXX1R0dHSbbZ999lndcssteuihh3TJJZfoySef1OWXX67f//73nSoYAAAMLJ0KI5mZmbrttts0a9asdttu2bLlonY333yztmzZ0upr3G63XC5XsxsAABiY/L5QXk5Ojnbv3q0dO3Z0qH1JSYkSEhKabUtISFBJSUmrr8nOztbSpUv9Lc1vL286qiOnK7QwfYTGJrR9RUEAANAz/OoZKSoq0v33368///nPCgkJ6amalJWVJafT6bsVFRX1yOf8/ZMTen1roY6cruyR9wcAAO3zq2dk165dKi0t1eWXX+7b5vF4tHHjRv3+97+X2+1WYGBgs9ckJibq5MmTzbadPHlSiYmJrX6OzWaTzWbzp7ROsYcES5Jc1XU9/lkAAKBlfvWM3HDDDcrPz1deXp7vdsUVV2jBggXKy8u7KIhIUnp6uj744INm29avX6/09PSuVd4N7KENYcRJGAEAwDR+9YxERkZq4sSJzbaFh4dryJAhvu0ZGRlKTk5Wdna2JOn+++/XV7/6Vf3qV7/SbbfdppycHO3cuVMvvPBCN+1C5zlCG3bfVVNvciUAAAxe3b4Ca2FhoYqLi32PZ8yYoZUrV+qFF17QlClT9Oabb+rtt9++KNSYgWEaAADM5/fZNF+Wm5vb5mNJuuOOO3THHXd09aO6XdMwDWEEAADzDOpr0ziawkgNYQQAALMM6jDSNEzDBFYAAMwzqMOIr2ekmgmsAACYZVCHEbvvbBp6RgAAMMvgDiMM0wAAYLpBHUaahmmqaj2q83hNrgYAgMFpUIeRyJDzZzaXs/AZAACmGNRhJCgwQOHWhiXsGaoBAMAcgzqMSBeeUUMYAQDADIM+jNhZ+AwAAFMRRjijBgAAUxFGWPgMAABTEUZY+AwAAFMRRhimAQDAVIM+jHA2DQAA5hr0YeT82TTMGQEAwAyDPow09YwwTAMAgDkGfRixNy4JzzANAADmIIyw6BkAAKYa9GGECawAAJhr0IeRCxc9MwzD5GoAABh8CCONc0ZqPV65670mVwMAwOAz6MNIhC1IAZaG+5xRAwBA7xv0YcRisVwwVEMYAQCgtw36MCKdXxKeM2oAAOh9hBGx8BkAAGYijOiCK/dWsyQ8AAC9jTCiC9YaYZgGAIBeRxjR+TkjzirCCAAAvY0wovMLnzFnBACA3kcY0flhmjLCCAAAvY4wImlIuFWSdLay1uRKAAAYfPwKIytWrNDkyZNlt9tlt9uVnp6utWvXtvma3/zmN0pLS1NoaKhSU1P14IMPqqampktFd7eYxjByhjACAECvC/KncUpKipYtW6axY8fKMAy99tprmjt3rvbs2aMJEyZc1H7lypX6j//4D7388suaMWOGPv/8c911112yWCx65plnum0numpIRFPPiNvkSgAAGHz8CiNz5sxp9vipp57SihUrtHXr1hbDyObNmzVz5kzdeeedkqQRI0Zo/vz52rZtWxdK7n4x4TZJ0tkKekYAAOhtnZ4z4vF4lJOTo8rKSqWnp7fYZsaMGdq1a5e2b98uSTpy5IjWrFmjW2+9tc33drvdcrlczW49qWmYprLWo5o6T49+FgAAaM6vnhFJys/PV3p6umpqahQREaFVq1bp0ksvbbHtnXfeqdOnT+vqq6+WYRiqr6/XPffco0ceeaTNz8jOztbSpUv9La3T7CFBCgqwqN5r6FxVrZIcob322QAADHZ+94ykpaUpLy9P27Zt0+LFi7Vw4ULt37+/xba5ubl6+umn9cc//lG7d+/WW2+9pXfffVdPPvlkm5+RlZUlp9PpuxUVFflbpl8sFouimyaxMlQDAECvshiGYXTlDWbNmqXRo0fr+eefv+i5a665RldddZWWL1/u2/b666/r7rvvVkVFhQICOpaFXC6XHA6HnE6n7HZ7V8pt1S2/2ajPSsr1X9+7UteOi+uRzwAAYDDp6Pd3l9cZ8Xq9crtbPgulqqrqosARGBgoSepiBup2Maw1AgCAKfyaM5KVlaXZs2dr2LBhKi8v18qVK5Wbm6t169ZJkjIyMpScnKzs7GxJDWffPPPMM7rssss0ffp0HTp0SI899pjmzJnjCyV9BWuNAABgDr/CSGlpqTIyMlRcXCyHw6HJkydr3bp1uvHGGyVJhYWFzXpCHn30UVksFj366KM6fvy44uLiNGfOHD311FPduxfdoGkV1nOEEQAAelWX54z0ht6YM/Kb9z/Xb94/qPlXDlP2Nyb1yGcAADCY9NqckYHi/PVpWIUVAIDeRBhp5FuFlWEaAAB6FWGkERNYAQAwB2Gk0fmL5RFGAADoTYSRRk09I87qOtV7vCZXAwDA4EEYaRQVGixJMgyprLrO5GoAABg8CCONggIDFBXWEEgYqgEAoPcQRi4Qw8XyAADodYSRCwzh+jQAAPQ6wsgFfBfLqyKMAADQWwgjF/AtfMYwDQAAvYYwcoGY8KYJrCwJDwBAbyGMXKCpZ4RVWAEA6D2EkQswgRUAgN5HGLlADGEEAIBeRxi5QGxEwzDNqXLmjAAA0FsIIxeIt5+fM1LH9WkAAOgVhJELxIRZFRRgkSSdrqB3BACA3kAYuUBAgEVxkQ29I6UuwggAAL2BMPIl8U1hhHkjAAD0CsLIl8RFhkiSTrpqTK4EAIDBgTDyJU2TWOkZAQCgdxBGvqRpmOZUOT0jAAD0BsLIl8Q3DtMwgRUAgN5BGPmSBIZpAADoVYSRL/H1jDBMAwBAryCMfEnTBNbTFbXyeA2TqwEAYOAjjHzJkHCrLBbJ4zW4YB4AAL2AMPIlQYEBGhLe0DvCWiMAAPQ8wkgLzp/eyyRWAAB6GmGkBecXPqNnBACAnuZXGFmxYoUmT54su90uu92u9PR0rV27ts3XlJWVKTMzU0lJSbLZbBo3bpzWrFnTpaJ7WgJrjQAA0GuC/GmckpKiZcuWaezYsTIMQ6+99prmzp2rPXv2aMKECRe1r62t1Y033qj4+Hi9+eabSk5O1rFjxxQVFdVd9fcIloQHAKD3+BVG5syZ0+zxU089pRUrVmjr1q0thpGXX35ZZ8+e1ebNmxUcHCxJGjFiROer7SXnr9zLMA0AAD2t03NGPB6PcnJyVFlZqfT09Bbb/M///I/S09OVmZmphIQETZw4UU8//bQ8Hk+b7+12u+VyuZrdelOcb+EzekYAAOhpfvWMSFJ+fr7S09NVU1OjiIgIrVq1SpdeemmLbY8cOaIPP/xQCxYs0Jo1a3To0CHde++9qqur0+OPP97qZ2RnZ2vp0qX+ltZtfMM0zBkBAKDHWQzD8GuZ0draWhUWFsrpdOrNN9/Un/70J23YsKHFQDJu3DjV1NTo6NGjCgwMlCQ988wzWr58uYqLi1v9DLfbLbf7fBBwuVxKTU2V0+mU3W73p9xO+eJcla7++UeyBgbowP9/iywWS49/JgAAA43L5ZLD4Wj3+9vvnhGr1aoxY8ZIkqZOnaodO3bo2Wef1fPPP39R26SkJAUHB/uCiCRdcsklKikpUW1traxWa4ufYbPZZLPZ/C2t28Q1zhmp9XhVVlWn6PCW6wQAAF3X5XVGvF5vs16MC82cOVOHDh2S1+v1bfv888+VlJTUahDpC2xBgRrSGECKnUxiBQCgJ/kVRrKysrRx40YVFBQoPz9fWVlZys3N1YIFCyRJGRkZysrK8rVfvHixzp49q/vvv1+ff/653n33XT399NPKzMzs3r3oAUlRDZNYi53VJlcCAMDA5tcwTWlpqTIyMlRcXCyHw6HJkydr3bp1uvHGGyVJhYWFCgg4n29SU1O1bt06Pfjgg5o8ebKSk5N1//336+GHH+7evegBSY5Q7T3u0gl6RgAA6FF+hZGXXnqpzedzc3Mv2paenq6tW7f6VVRfMNTR0DNyooyeEQAAehLXpmnF0KhQSVIxYQQAgB5FGGlFUmMYYZgGAICeRRhpRdMwDRNYAQDoWYSRVjQN05Q4a+T1+rUuHAAA8ANhpBXxkTYFWKQ6j6HTFSwLDwBATyGMtCIoMEAJ9sYzapg3AgBAjyGMtCGpad4IZ9QAANBjCCNtaJo3cpwwAgBAjyGMtMG31gjDNAAA9BjCSBuSOL0XAIAeRxhpQ5KjaZiGnhEAAHoKYaQNySwJDwBAjyOMtCEpqmGY5lSFW7X1XpOrAQBgYCKMtGFIuFXWoAAZhnTSxVANAAA9gTDSBovF4pvEeoKhGgAAegRhpB1DHU1X7yWMAADQEwgj7UiJbggjX5wljAAA0BMII+0YFhMmSSo8W2VyJQAADEyEkXYMG0IYAQCgJxFG2pHa2DNSRBgBAKBHEEba0TRMU+yqYa0RAAB6AGGkHUPCrQoNDpRhcPVeAAB6AmGkHRaLhUmsAAD0IMJIB6QSRgAA6DGEkQ4YxiRWAAB6DGGkA4bFNCx8VniGMAIAQHcjjHRA01ojRecIIwAAdDfCSAekRjfOGTlTJcMwTK4GAICBhTDSASmNYaTcXS9ndZ3J1QAAMLAQRjog1Bqo+EibJM6oAQCguxFGOoi1RgAA6BmEkQ4ijAAA0DP8CiMrVqzQ5MmTZbfbZbfblZ6errVr13botTk5ObJYLJo3b15n6jRdCmuNAADQI/wKIykpKVq2bJl27dqlnTt36vrrr9fcuXO1b9++Nl9XUFCgn/zkJ7rmmmu6VKyZRjSe3nv0dKXJlQAAMLD4FUbmzJmjW2+9VWPHjtW4ceP01FNPKSIiQlu3bm31NR6PRwsWLNDSpUs1atSoLhdsllFxEZKkI6cIIwAAdKdOzxnxeDzKyclRZWWl0tPTW233xBNPKD4+XosWLerwe7vdbrlcrmY3s42MDZcklZa7VV7D6b0AAHSXIH9fkJ+fr/T0dNXU1CgiIkKrVq3SpZde2mLbTZs26aWXXlJeXp5fn5Gdna2lS5f6W1qPcoQGKzbCqtMVtSo4XaVJKQ6zSwIAYEDwu2ckLS1NeXl52rZtmxYvXqyFCxdq//79F7UrLy/Xd77zHb344ouKjY316zOysrLkdDp9t6KiIn/L7BGjYhuHak5XmFwJAAADh989I1arVWPGjJEkTZ06VTt27NCzzz6r559/vlm7w4cPq6CgQHPmzPFt83q9DR8aFKQDBw5o9OjRLX6GzWaTzWbzt7QeNyouXNsLzuow80YAAOg2foeRL/N6vXK73RdtHz9+vPLz85tte/TRR1VeXq5nn31WqampXf3oXjcqrmHeCGfUAADQffwKI1lZWZo9e7aGDRum8vJyrVy5Urm5uVq3bp0kKSMjQ8nJycrOzlZISIgmTpzY7PVRUVGSdNH2/mJk0zDNKYZpAADoLn6FkdLSUmVkZKi4uFgOh0OTJ0/WunXrdOONN0qSCgsLFRAwcBd1vbBnxDAMWSwWkysCAKD/sxiGYZhdRHtcLpccDoecTqfsdrtpddR5vBr/2HvyeA1tzbpBiY4Q02oBAKCv6+j398DtxugBwYEBvmvUMFQDAED3IIz4aVTj4meHmcQKAEC3IIz4qWneCD0jAAB0D8KIn5quUcPpvQAAdA/CiJ+arlFzmJ4RAAC6BWHET2PjG3pGvjhXraraepOrAQCg/yOM+GlIhE2xEVYZhnSolN4RAAC6ijDSCeMSIiVJB0rKTa4EAID+jzDSCU1h5POThBEAALqKMNIJaYmNPSMnGaYBAKCrCCOd4OsZYZgGAIAuI4x0wriEhjNqSlw1clbVmVwNAAD9G2GkEyJDgpUcFSpJ+ryU3hEAALqCMNJJvnkjDNUAANAlhJFO4owaAAC6B2Gkk9ISG+aN0DMCAEDXEEY66cKeEcMwTK4GAID+izDSSaPjIhRgkc5V1elUudvscgAA6LcII50UEhyoUXENQzX7i10mVwMAQP9FGOmCCUPtkqR9JwgjAAB0FmGkCyYOdUiS9h53mlwJAAD9F2GkCyYkN/SM7D1BGAEAoLMII10wobFnpOhsNcvCAwDQSYSRLnCEBmtYTJgkaR+9IwAAdAphpIuYxAoAQNcQRrpoYnLjJFZ6RgAA6BTCSBc19YxwRg0AAJ1DGOmipkmsR05XqtJdb3I1AAD0P4SRLoqLtCnRHiLDkD5lJVYAAPxGGOkGExvXG8lnqAYAAL8RRrrB5JQoSVJeUZmpdQAA0B/5FUZWrFihyZMny263y263Kz09XWvXrm21/YsvvqhrrrlG0dHRio6O1qxZs7R9+/YuF93XXDYsSpK0p7DM1DoAAOiP/AojKSkpWrZsmXbt2qWdO3fq+uuv19y5c7Vv374W2+fm5mr+/Pn66KOPtGXLFqWmpuqmm27S8ePHu6X4vmJKapQsFqnwbJVOV7jNLgcAgH7FYhiG0ZU3iImJ0fLly7Vo0aJ223o8HkVHR+v3v/+9MjIyOvwZLpdLDodDTqdTdru9K+X2mBuf2aCDpRX6U8YVmnVpgtnlAABguo5+f3d6zojH41FOTo4qKyuVnp7eoddUVVWprq5OMTExbbZzu91yuVzNbn2db6im6Jy5hQAA0M/4HUby8/MVEREhm82me+65R6tWrdKll17aodc+/PDDGjp0qGbNmtVmu+zsbDkcDt8tNTXV3zJ73WXDoiUxbwQAAH/5HUbS0tKUl5enbdu2afHixVq4cKH279/f7uuWLVumnJwcrVq1SiEhIW22zcrKktPp9N2Kior8LbPXNfWM/LOoTB5vl0a+AAAYVIL8fYHVatWYMWMkSVOnTtWOHTv07LPP6vnnn2/1Nb/85S+1bNkyvf/++5o8eXK7n2Gz2WSz2fwtzVRj4yMVbg1UZa1Hn58s1yVJfXNuCwAAfU2X1xnxer1yu1s/g+QXv/iFnnzySb333nu64ooruvpxfVZggEVTUqMkMVQDAIA//AojWVlZ2rhxowoKCpSfn6+srCzl5uZqwYIFkqSMjAxlZWX52v/85z/XY489ppdfflkjRoxQSUmJSkpKVFFR0b170Uc0DdXsLmQSKwAAHeXXME1paakyMjJUXFwsh8OhyZMna926dbrxxhslSYWFhQoIOJ9vVqxYodraWv3rv/5rs/d5/PHH9bOf/azr1fcxU4c3TGLdWXDW5EoAAOg//AojL730UpvP5+bmNntcUFDgbz392tThMbJYpIIzVTrpqlGCve2JugAAgGvTdCtHaLAubZy4uu0ovSMAAHQEYaSbTR85RJK0/egZkysBAKB/IIx0sytHNqwuu+0IPSMAAHQEYaSbNYWRg6UVOsNF8wAAaBdhpJvFhFuVlhApSdrBWTUAALSLMNIDmnpHtjJUAwBAuwgjPWD6qIYwsp0zagAAaBdhpAc0nVHzaYlLZytrTa4GAIC+jTDSA+IibRqfGCnDkP5x6LTZ5QAA0KcRRnrINWNjJUkfHzxlciUAAPRthJEecvXYOEnSpoOnZRiGydUAANB3EUZ6yJUjYmQNCtAJZ40On6o0uxwAAPoswkgPCbUG6soRDWfVbGKoBgCAVhFGetDVvnkjTGIFAKA1hJEe1DSJdcuRM6qt95pcDQAAfRNhpAddkmhXbIRVVbUe7TzGAmgAALSEMNKDAgIsunZcw1k1H35aanI1AAD0TYSRHjbrkgRJ0gefEUYAAGgJYaSHXTM2VsGBFh09XanDpyrMLgcAgD6HMNLDIkOCddWohmvVfPDpSZOrAQCg7yGM9IIbxsdLkt5n3ggAABchjPSCGxrnjew6dk5lVVzFFwCACxFGekFqTJjGJUTI4zWUe4DVWAEAuBBhpJc0nVWzbl+JyZUAANC3EEZ6ya2TkiRJHx0oVVVtvcnVAADQdxBGesmEoXYNiwlTTZ1XH33GUA0AAE0II73EYrFo9qRESdKavcUmVwMAQN9BGOlFtzUO1Xz4aamqaz0mVwMAQN9AGOlFk5IdSokOVXWdRxs+Z80RAAAkwkivslgsvoms7+ZzVg0AABJhpNc1DdWs31+iCjdn1QAA4FcYWbFihSZPniy73S673a709HStXbu2zde88cYbGj9+vEJCQjRp0iStWbOmSwX3d5NTHBoVG66aOq/W7aV3BAAAv8JISkqKli1bpl27dmnnzp26/vrrNXfuXO3bt6/F9ps3b9b8+fO1aNEi7dmzR/PmzdO8efO0d+/ebim+P7JYLJp3WbIkadWe4yZXAwCA+SyGYRhdeYOYmBgtX75cixYtuui5b33rW6qsrNQ777zj23bVVVfpK1/5ip577rkOf4bL5ZLD4ZDT6ZTdbu9KuX1C4ZkqXbv8I1ks0tasG5RgDzG7JAAAul1Hv787PWfE4/EoJydHlZWVSk9Pb7HNli1bNGvWrGbbbr75Zm3ZsqWzHzsgDBsSpiuGR8swpNV59I4AAAY3v8NIfn6+IiIiZLPZdM8992jVqlW69NJLW2xbUlKihISEZtsSEhJUUtL2XAm32y2Xy9XsNtB8/fKGoZq3dhNGAACDm99hJC0tTXl5edq2bZsWL16shQsXav/+/d1aVHZ2thwOh++Wmprare/fF/zLpKGyBgbos5Jy7T3uNLscAABM43cYsVqtGjNmjKZOnars7GxNmTJFzz77bIttExMTdfLkyWbbTp48qcTExDY/IysrS06n03crKiryt8w+zxEWrJsnNvx3+Mv2QpOrAQDAPF1eZ8Tr9crtdrf4XHp6uj744INm29avX9/qHJMmNpvNd/pw020gmj+tocfnf/JOcCVfAMCgFeRP46ysLM2ePVvDhg1TeXm5Vq5cqdzcXK1bt06SlJGRoeTkZGVnZ0uS7r//fn31q1/Vr371K912223KycnRzp079cILL3T/nvRDV40aomExYSo8W6V3PynWHVcMvOEoAADa41fPSGlpqTIyMpSWlqYbbrhBO3bs0Lp163TjjTdKkgoLC1VcfP6KtDNmzNDKlSv1wgsvaMqUKXrzzTf19ttva+LEid27F/1UQIBF32rsHcnZMfCGogAA6IgurzPSGwbaOiMXKnXVKH3Zh/J4Df3vg9dqXEKk2SUBANAtenydEXSPeHuIZl0SL0n6v1uOmVwNAAC9jzDSByycMUKS9N+7v5Crps7cYgAA6GWEkT4gfdQQpSVEqqrWozd2fmF2OQAA9CrCSB9gsViUMWO4JOm/thTI6+3z03gAAOg2hJE+4uuXJcseEqRjZ6qU+3mp2eUAANBrCCN9RJg1yHea758+PmpyNQAA9B7CSB9y18yRCgqwaPPhM/rkizKzywEAoFcQRvqQ5KhQ3T5lqCTp+Q1HTK4GAIDeQRjpY+7+6ihJ0tq9xSo4XWlyNQAA9DzCSB8zPtGu69Li5DWkFz+mdwQAMPARRvqgH1w7WpL0xs4vVOKsMbkaAAB6FmGkD7pqVIyuHBGjWo9Xz204bHY5AAD0KMJIH2SxWHT/rLGSpJXbC3XSRe8IAGDgIoz0UTNGD9G0EdGqrfdqRS69IwCAgYsw0kdZLBbdf8M4SQ29I8XOapMrAgCgZxBG+rCZY4Y0zB2p9+rZ9w+aXQ4AAD2CMNKHWSwWPTw7TZL0t51FOlRaYXJFAAB0P8JIHzd1eIxmXZIgryH9ct0Bs8sBAKDbEUb6gf9zS5oCLNJ7+0q0u/Cc2eUAANCtCCP9wLiESH3z8hRJ0pPv7JdhGCZXBABA9yGM9BM/uTlNYdZA7Sks0+q8E2aXAwBAtyGM9BMJ9hBlfm2MJGnZ2s9UVVtvckUAAHQPwkg/sujqkUqJDlWJq0bPsRAaAGCAIIz0IyHBgfrPWy+RJD234YiOnOJUXwBA/0cY6WdumZior46LU63Hq5+u3sdkVgBAv0cY6WcsFouemDtBtqAAbTp0Wv/zTyazAgD6N8JIPzR8SLhvMuuT73yqsqpakysCAKDzCCP91A++Okpj4yN0usKtJ97Zb3Y5AAB0GmGkn7IFBern/zpZFov01u7j+uizUrNLAgCgUwgj/djlw6K1aOZISVLWW/lyVteZXBEAAP4jjPRzP74pTSOGhKnEVaPHV+81uxwAAPxGGOnnQq2B+tW/fUUBFuntvBP6O2fXAAD6Gb/CSHZ2tqZNm6bIyEjFx8dr3rx5OnCg/cva/+Y3v1FaWppCQ0OVmpqqBx98UDU1NZ0uGs1NHR6tJY1n1/znqnwVO6tNrggAgI7zK4xs2LBBmZmZ2rp1q9avX6+6ujrddNNNqqysbPU1K1eu1H/8x3/o8ccf16effqqXXnpJf/3rX/XII490uXicd98NYzUlxSFXTb2WrNyjOo/X7JIAAOgQi9GFJTxPnTql+Ph4bdiwQddee22LbZYsWaJPP/1UH3zwgW/bj3/8Y23btk2bNm3q0Oe4XC45HA45nU7Z7fbOljvgHTtTqX/53SaV19Rr0dUj9di/XGp2SQCAQayj399dmjPidDolSTExMa22mTFjhnbt2qXt27dLko4cOaI1a9bo1ltvbfU1brdbLper2Q3tGz4kXL+6Y4ok6aVNR7U2v9jkigAAaF+nw4jX69UDDzygmTNnauLEia22u/POO/XEE0/o6quvVnBwsEaPHq3rrruuzWGa7OxsORwO3y01NbWzZQ46N01I1A+uHSVJeujNT3T0dOtDaAAA9AWdDiOZmZnau3evcnJy2myXm5urp59+Wn/84x+1e/duvfXWW3r33Xf15JNPtvqarKwsOZ1O362oqKizZQ5KD92cpitHxKjCXa/Fr+9Sda3H7JIAAGhVp+aMLFmyRKtXr9bGjRs1cuTINttec801uuqqq7R8+XLfttdff1133323KioqFBDQfh5izoj/Sl01uvW3m3S6wq1vXJ6sX90xRRaLxeyyAACDSI/MGTEMQ0uWLNGqVav04YcfthtEJKmqquqiwBEYGOh7P/SMeHuIfjf/MgU0Lhf/x9zDZpcEAECL/AojmZmZev3117Vy5UpFRkaqpKREJSUlqq4+v65FRkaGsrKyfI/nzJmjFStWKCcnR0ePHtX69ev12GOPac6cOb5Qgp6RPnqIfnb7BEnS8nUHtDrvuMkVAQBwsSB/Gq9YsUKSdN111zXb/sorr+iuu+6SJBUWFjbrCXn00UdlsVj06KOP6vjx44qLi9OcOXP01FNPda1ydEhG+ggVnqnSnzYd1UNvfKKhUaGaNqL1s58AAOhtXVpnpLcwZ6RrvF5Di/+8S+v2nVRUWLDeWjxDo+IizC4LADDA9co6I+gfAgIs+s23LtOU1CiVVdXpu6/u0JkKt9llAQAgiTAyaIRaA/WnjCuUEh2qY2eq9J2XtstZVWd2WQAAEEYGk7hIm1773pWKjbBqf7FLGS9vU3kNgQQAYC7CyCAzOi5Cf/73qxQdFqx/fuHUd1/ZoUp3vdllAQAGMcLIIJSWGKn/u2i67CFB2nnsnBa9toNVWgEApiGMDFITkx36r0XTFWEL0tYjZ/X9/9qpqlp6SAAAvY8wMoh9JTVKr353msKsgdp06LT+vz9tU1lVrdllAQAGGcLIIHfFiBjfkM3uwjJ96/mtKnXVmF0WAGAQIYxAU4dH62/3pCs+0qYDJ8v1zec269iZSrPLAgAMEoQRSJLGJ9r15j0zNCwmTEVnq/XNFVu074TT7LIAAIMAYQQ+w4aE6c170jU+MVKnK9z61xVbtG5fidllAQAGOMIImom3h+ivP0jX1WNiVV3n0Q/+7y794aND6geXMAIA9FOEEVzEERqsV787TQvTh0uSlq87oB/97Z+qqWMtEgBA9yOMoEVBgQFaOneinpw3UYEBFq3ac1zffmGrTpRVm10aAGCAIYygTd+5arj+63tXyh4SpLyiMt3224+Ve6DU7LIAAAMIYQTtmjkmVu/cd40mJtt1rqpOd72yQ79cd0D1Hq/ZpQEABgDCCDqk4UybGfrOVQ3zSH7/0SEt+NM2nWSBNABAFxFG0GEhwYF6ct5E/W7+ZQq3Bmrb0bO6+Tcb9c4nJ8wuDQDQjxFG4Lc5U4bq7/ddrUnJDpVV1WnJyj26P2ePnFV1ZpcGAOiHCCPolFFxEXrr3hn64fVjFBhg0eq8E7r5Nxu18fNTZpcGAOhnCCPotODAAP3opjS9eU+6RsWGq8RVo4yXt+vHf/unzlZy9V8AQMcQRtBllw2L1rs/vEZ3zRghi0X6791faNYzG7Rqzxes3AoAaBdhBN0i1Bqon90+Qf+9eIbSEiJ1trJWD/71n8p4ebsKTnMFYABA6wgj6FaXD4vWOz+8Wg/dnCZbUIA+PnhaN/56g7LXfqoKd73Z5QEA+iDCCLpdcGCAMr82RuseuFbXjotTncfQ8xuO6Gu/zNWbu76Q18vQDQDgPIvRDwb1XS6XHA6HnE6n7Ha72eXAD4Zh6MPPSvXkO/tVcKZKkjQlNUqPzB6v6aOGmFwdAKAndfT7mzCCXuGu9+iVfxTodx8cVGVtw9V/v5YWp/9zy3hdksQxBYCBiDCCPqm0vEa//eCgcrYXqd5ryGKRvv6VZD144zilxoSZXR4AoBsRRtCnHT1dqV/+7wG9+0mxJCkwwKKvX5aszK+N0cjYcJOrAwB0B8II+oVPvijTL//3c9/KrQGWhuXml3xtjMYmRJpcHQCgKwgj6Ffyisr0uw8O6oPPSiVJFot068QkLbl+DHNKAKCf6uj3t1+n9mZnZ2vatGmKjIxUfHy85s2bpwMHDrT7urKyMmVmZiopKUk2m03jxo3TmjVr/PloDHBfSY3SS3dN0zv3Xa2bJyTIMKR384s1+9mPtfDl7drw+SlWcwWAAcqvnpFbbrlF3/72tzVt2jTV19frkUce0d69e7V//36Fh7c8zl9bW6uZM2cqPj5ejzzyiJKTk3Xs2DFFRUVpypQpHfpcekYGn89KXPr9h4f0bn6xmv6Fjo2P0HdnjtQ3Lk9WSHCguQUCANrVK8M0p06dUnx8vDZs2KBrr722xTbPPfecli9frs8++0zBwcGd+hzCyOB17EylXt1coL/tKPKdEhwdFqwF04frO+nDlWAPMblCAEBreiWMHDp0SGPHjlV+fr4mTpzYYptbb71VMTExCgsL0+rVqxUXF6c777xTDz/8sAIDW/6/W7fbLbfb3WxnUlNTCSODmKumTn/bUaRXNxfoi3PVkqSgAItuvDRB868cpqvHxCogwGJylQCAC/V4GPF6vbr99ttVVlamTZs2tdpu/PjxKigo0IIFC3Tvvffq0KFDuvfee/XDH/5Qjz/+eIuv+dnPfqalS5detJ0wAo/X0Pr9JXpp01HtKDjn254aE6pvTxumO6amKJ7eEgDoE3o8jCxevFhr167Vpk2blJKS0mq7cePGqaamRkePHvX1hDzzzDNavny5iouLW3wNPSPoiM9KXMrZXqT/3v2FymsaLsIXGGDRrEvi9e1pw3TN2FgFBXL5JQAwS0fDSFBn3nzJkiV65513tHHjxjaDiCQlJSUpODi42ZDMJZdcopKSEtXW1spqtV70GpvNJpvN1pnSMIiMT7TrZ7dP0MO3jNea/GL9ZXuhdh47p3X7TmrdvpOKjbBqzpSh+vplyZqU7JDFwjAOAPRFfoURwzB03333adWqVcrNzdXIkSPbfc3MmTO1cuVKeb1eBQQ0/F/q559/rqSkpBaDCOCvUGugvjk1Rd+cmqKDJ8v1l+1FWp13XKcravXKPwr0yj8KNDouXF+/LFlzv5LMsvMA0Mf4NUxz7733auXKlVq9erXS0tJ82x0Oh0JDQyVJGRkZSk5OVnZ2tiSpqKhIEyZM0MKFC3Xffffp4MGD+t73vqcf/vCH+s///M8OfS5n08BfdR6vPj54Sqv2nND/7iuRu97re+6K4dG6dVKSZk9KVJIj1MQqAWBg65E5I611c7/yyiu66667JEnXXXedRowYoVdffdX3/JYtW/Tggw8qLy9PycnJWrRoUZtn03R2Z4CWlNfU6b29JVq157i2HDmjC//FXz4sqjGYJCk5imACAN2J5eCBFpQ4a7R2b7HW5Bdr57FzzYLJlNQo3TwhQTeMT9C4hAjmmABAFxFGgHacdNXovb0lWpNfrO0FZ5sFk5ToUN0wPl7XX5Kgq0bFyBbEiq8A4C/CCOCH0vIa/e++k/rg05P6x+Ezqr1gjkmYNVDXjI3VDeMT9LXx8YqL5EwvAOgIwgjQSVW19dp86Iw++OykPvi0VKXl7mbPT0y26+oxcbp6TKyuGBHNdXIAoBWEEaAbGIahfSdcev/Tk/rws1J98oWz2fO2oABNGxGjq8fG6uoxsbo0yc6y9ADQiDAC9IDS8hr949BpbTp4RpsOndJJV/Nek+iwYM0YE6sZo4do+sgYjY5jIiyAwYswAvQwwzB0+FSFPj54Wv84dFpbDp/xXVm4SUy4VdNGROvKkUN05YgYXZIUyRL1AAYNwgjQy+o8Xv2zqEwfHzyt7UfPanfhuWaLrUlShC1IU4dH68qRMbpieLQmpTgUZu3UVRkAoM8jjAAmq633Kv+4U9uPntX2o2e0s+Ccyt31zdoEBlg0PjFSlw+L1mXDonTZsGiNGBLG0A6AAYEwAvQxHq+hz0pcjeGkoefky3NOJCkqLFiXpTYEkympUZo41K4hEZxODKD/IYwAfZxhGCp21mhPYZn2FJ7TnqIy5R93NlvjpMlQR4gmJDs0cahDk1LsmjjUoXh7iAlVA0DHEUaAfqi23qtPi13nw8kXTh05Xdli27hImyYlOzRxqF0Tkx2akOzQUEcIQzwA+gzCCDBAlNfU6dPicuUfd2rfcafyjzt1+FSFvC385tpDgpSWGKm0xEiNT7RrfGKkxiVGyh4S3PuFAxj0CCPAAFZVW69Pi8u197iz4XbCpYMny1XfUkKRlBwVekFIaQgqI2PDZQ3iNGMAPYcwAgwy7nqPjpyq1IGScn1a4tKBknIdKClXsbOmxfaBARYNiwnT6LhwjY6L0Oj4CI2Oi9CYuAg5wuhJAdB1hBEAkiRnVZ0OnCzXZyUufdYYUD4vKb/oNOMLxUZYmwWUpsCSHBXKcvcAOowwAqBVhmGotNytw6UVOnyqQodPVepQ4/3WelKkhmvxDB8SpmEx4RoxJEzDYxt+jhgSriRHCKvLAmiGMAKgUyrc9TpyqjGklFbq8KkKHSqtUMGZStV5Wv9zERRgUWpMmIY3hpNhMWEaERum4UPClRIdKlsQVzcGBhvCCIBuVe/x6otz1Tp2tkrHzlTq2JmGnwVnqlR4pkq1novXR7lQfKRNKdGhSokOU3J06Pn7UQ33Q4IJK8BA09Hvby6KAaBDggIDNCI2XCNiwyXFNXvO4zVU4qrxhZSCM5U6drrKF1yqaj0qLXertNyt3YVlLb5/bITtgpASqpSohrCSEh2qpKhQRdj4cwUMVPSMAOhRhmHoXFWdvjhXpS/OVeuLc1U6fq668X7D4y9f7bglkbYgJTpClOgIUZIjRImOUCU13k9yhCrRESJ7SBCLvgF9CD0jAPoEi8WimHCrYsKtmpwSddHzhmHIWV3XLJx8ca5ax8vOPy6vqVe5u17lpRU6WFrR6meFWQPPhxV7aGNoCVF8pE3x9hDFRdoUF2FjfRWgjyGMADCVxWJRVJhVUWFWTUx2tNimwl2vEmeNSpw1KnZWN/x0NT2uUYmzWueq6lRV27DWypFTLS+h3yQ6LFjxkQ3hJD7Spji7rdnj+Eib4iJtirDR0wL0BsIIgD4vwhakMfERGhMf0WqbmjrP+XDiqm4MKQ2PS8vdOuWq0akKt+o8DcNG5xrXX2lLaHCg4u3nw0lshE1Dwm2KibAqNtyqIRE2xYRbFRthlT0kmDVYgE4ijAAYEEKCAy+YYNsywzBUVlXXOJm2RqcaJ9WWuhoel5a7dbpxW4W7XtV1nsazhqra/fygAIuiw60aEm5VbGNIGRJhbQww1sbHNsVGNNyn1wU4jzACYNCwWBoCQ3S4VWmJkW22raqtV6nLrVMV58PK2cpana6o1ZkKt85U1jY+dqu8pl71XkOnyt06Ve6W1HaPiyRZgwIUE2ZVVFiwosOsig4PVlSYVdFhwYoKbWm7VY7QYAXS+4IBiDACAC0IswZpRGxQmz0tTdz1Hp2rrNNpX0hx60zF+eBytrJWpyvP36+q9ai23qsSV41KXK2veNsSe0iQosOtvuASfUGgiQo7H2gcoQ03e0iwIkOCWB0XfRphBAC6yBYUqERHw5k8HVFVW68zFbU6V1Wrc1V1Kquq1bnKhvvO6rrm26tqVVZZ57uWkKumXq6a+g4NHV0o3BrYEE4aA4o9NOiC+8GyhwRd9HzT4whrEPNh0KMIIwDQy8KsQQqLCVJqTFiHX1Pn8cpZ3RRQ6nSuslZlVXUqq74w0DQEmbKqOrlq6uSqrvOt4VJZ61FlrUcn2rj2UGssloZ1XhxhjUGlMaxEhgQrwhakyJAgRdiCFNH4s+Hxxc8F0zuDVhBGAKAfCA4MUGxEwxk9/qjzeFVeUy9XdUNAcVbXyVVdf8H9puBS3/C4McQ4qxteU+vxyjDO98hI1Z3eB1tQwEXBJcIW3EqYOb8t0hascFugwm1BCrMGKswaxNyZAYYwAgADWHBggG/Ruc6oqfM0DyiN913VDUNHFTX1qmj82eyxu17lNfWqcNeppq7hukXueq/cjXNpuiokOEDh1iBfQPH9tAYpzPaln608H25rCDZN2+i5MQ9hBADQqpDgQIUEByo+smPzYVpS5/Gq0hdOWgovdW2EmXqV1zQsaFfprpe38QImNXVe1dTV6kxl14NNE2tQgMIbe17CbYEKtQYpLDhQodZAhTb+dwizNjwOCW7YFtb0XOPP0Avaf/lncKCF07lbQRgBAPSo4MAA3yq7XWEYhtz1DcGmqtajytp6Vbo9qrrwZ61HVe4v/Wzh+abXV7k9vitO19Z7VVvv1bmquu7Y7YsEBliah5TGENNe4AkNDvCFwpDgANmCAmVr2hbUsO3C50OCAvvdhGO/wkh2drbeeustffbZZwoNDdWMGTP085//XGlpaR16fU5OjubPn6+5c+fq7bff7ky9AIBBymKx+L50h3Tj+9bWe1XdFE4ag0tTUKmua7jV1HlUVetRdW3D/eqmx3Ue1dSef3zhczW1HlXVeeRp7M7xeA1fr09PswYGyBYUIFvwhWEloDG8NIaaxjBja9x+14wRGjak45Oqu5NfYWTDhg3KzMzUtGnTVF9fr0ceeUQ33XST9u/fr/Dwts/FLygo0E9+8hNdc801XSoYAIDuZA0KkDUoQI6w4B55/zqP93xQ+XJwaXzs+9nKc+56T+PQlEc1F96v88rduK3OY/g+s9bjVa3H6zslvCP+ZUpS/wgj7733XrPHr776quLj47Vr1y5de+21rb7O4/FowYIFWrp0qT7++GOVlZV1qlgAAPqb4MAAOUID5AjtmbDTxOM1moeWxrDSEF48cjdud9df8Hz9+VAz1BHao/W1pUtzRpxOpyQpJiamzXZPPPGE4uPjtWjRIn388cftvq/b7Zbb7fY9drlcXSkTAIABLzDA0rCGTdem5pii0+cxeb1ePfDAA5o5c6YmTpzYartNmzbppZde0osvvtjh987OzpbD4fDdUlNTO1smAADo4zodRjIzM7V3717l5OS02qa8vFzf+c539OKLLyo2NrbD752VlSWn0+m7FRUVdbZMAADQx3VqmGbJkiV65513tHHjRqWkpLTa7vDhwyooKNCcOXN827zehlOogoKCdODAAY0ePfqi19lsNtls/q0yCAAA+ie/wohhGLrvvvu0atUq5ebmauTIkW22Hz9+vPLz85tte/TRR1VeXq5nn32W4RcAAOBfGMnMzNTKlSu1evVqRUZGqqSkRJLkcDgUGtowCzcjI0PJycnKzs5WSEjIRfNJoqKiJKnNeSYAAGDw8CuMrFixQpJ03XXXNdv+yiuv6K677pIkFRYWKiCA9f0BAEDHWAzDMNpvZi6XyyWHwyGn0ym73W52OQAAoAM6+v1NFwYAADAVYQQAAJiKMAIAAExFGAEAAKYijAAAAFMRRgAAgKm6dNXe3tJ09jFX7wUAoP9o+t5ubxWRfhFGysvLJYnl4wEA6IfKy8vlcDhafb5fLHrm9Xp14sQJRUZGymKxdNv7ulwupaamqqioaMAupsY+9n8Dff8k9nEgGOj7Jw38feyJ/TMMQ+Xl5Ro6dGibq7P3i56RgICANq8O3FV2u31A/sO6EPvY/w30/ZPYx4FgoO+fNPD3sbv3r60ekSZMYAUAAKYijAAAAFMN6jBis9n0+OOPy2azmV1Kj2Ef+7+Bvn8S+zgQDPT9kwb+Ppq5f/1iAisAABi4BnXPCAAAMB9hBAAAmIowAgAATEUYAQAAphrUYeQPf/iDRowYoZCQEE2fPl3bt283u6ROyc7O1rRp0xQZGan4+HjNmzdPBw4caNbmuuuuk8ViaXa75557TKrYfz/72c8uqn/8+PG+52tqapSZmakhQ4YoIiJC3/zmN3Xy5EkTK/bfiBEjLtpHi8WizMxMSf3vGG7cuFFz5szR0KFDZbFY9Pbbbzd73jAM/fSnP1VSUpJCQ0M1a9YsHTx4sFmbs2fPasGCBbLb7YqKitKiRYtUUVHRi3vRtrb2sa6uTg8//LAmTZqk8PBwDR06VBkZGTpx4kSz92jpuC9btqyX96R17R3Hu+6666L6b7nllmZt+vJxbG//WvqdtFgsWr58ua9NXz6GHfl+6Mjfz8LCQt12220KCwtTfHy8HnroIdXX13dbnYM2jPz1r3/Vj370Iz3++OPavXu3pkyZoptvvlmlpaVml+a3DRs2KDMzU1u3btX69etVV1enm266SZWVlc3aff/731dxcbHv9otf/MKkijtnwoQJzerftGmT77kHH3xQf//73/XGG29ow4YNOnHihL7xjW+YWK3/duzY0Wz/1q9fL0m64447fG360zGsrKzUlClT9Ic//KHF53/xi1/ot7/9rZ577jlt27ZN4eHhuvnmm1VTU+Nrs2DBAu3bt0/r16/XO++8o40bN+ruu+/urV1oV1v7WFVVpd27d+uxxx7T7t279dZbb+nAgQO6/fbbL2r7xBNPNDuu9913X2+U3yHtHUdJuuWWW5rV/5e//KXZ8335OLa3fxfuV3FxsV5++WVZLBZ985vfbNaurx7Djnw/tPf30+Px6LbbblNtba02b96s1157Ta+++qp++tOfdl+hxiB15ZVXGpmZmb7HHo/HGDp0qJGdnW1iVd2jtLTUkGRs2LDBt+2rX/2qcf/995tXVBc9/vjjxpQpU1p8rqyszAgODjbeeOMN37ZPP/3UkGRs2bKllyrsfvfff78xevRow+v1GobRv4+hJGPVqlW+x16v10hMTDSWL1/u21ZWVmbYbDbjL3/5i2EYhrF//35DkrFjxw5fm7Vr1xoWi8U4fvx4r9XeUV/ex5Zs377dkGQcO3bMt2348OHGr3/9654trpu0tI8LFy405s6d2+pr+tNx7MgxnDt3rnH99dc329afjuGXvx868vdzzZo1RkBAgFFSUuJrs2LFCsNutxtut7tb6hqUPSO1tbXatWuXZs2a5dsWEBCgWbNmacuWLSZW1j2cTqckKSYmptn2P//5z4qNjdXEiROVlZWlqqoqM8rrtIMHD2ro0KEaNWqUFixYoMLCQknSrl27VFdX1+x4jh8/XsOGDeu3x7O2tlavv/66vve97zW7OGR/P4ZNjh49qpKSkmbHzOFwaPr06b5jtmXLFkVFRemKK67wtZk1a5YCAgK0bdu2Xq+5OzidTlksFkVFRTXbvmzZMg0ZMkSXXXaZli9f3q3d370hNzdX8fHxSktL0+LFi3XmzBnfcwPpOJ48eVLvvvuuFi1adNFz/eUYfvn7oSN/P7ds2aJJkyYpISHB1+bmm2+Wy+XSvn37uqWufnGhvO52+vRpeTyeZv9hJSkhIUGfffaZSVV1D6/XqwceeEAzZ87UxIkTfdvvvPNODR8+XEOHDtUnn3yihx9+WAcOHNBbb71lYrUdN336dL366qtKS0tTcXGxli5dqmuuuUZ79+5VSUmJrFbrRX/gExISVFJSYk7BXfT222+rrKxMd911l29bfz+GF2o6Li39DjY9V1JSovj4+GbPBwUFKSYmpl8e15qaGj388MOaP39+s4uQ/fCHP9Tll1+umJgYbd68WVlZWSouLtYzzzxjYrUdd8stt+gb3/iGRo4cqcOHD+uRRx7R7NmztWXLFgUGBg6o4/jaa68pMjLyoiHg/nIMW/p+6Mjfz5KSkhZ/V5ue6w6DMowMZJmZmdq7d2+z+RSSmo3PTpo0SUlJSbrhhht0+PBhjR49urfL9Nvs2bN99ydPnqzp06dr+PDh+tvf/qbQ0FATK+sZL730kmbPnq2hQ4f6tvX3YziY1dXV6d/+7d9kGIZWrFjR7Lkf/ehHvvuTJ0+W1WrVD37wA2VnZ/eLZce//e1v++5PmjRJkydP1ujRo5Wbm6sbbrjBxMq638svv6wFCxYoJCSk2fb+cgxb+37oCwblME1sbKwCAwMvmi188uRJJSYmmlRV1y1ZskTvvPOOPvroI6WkpLTZdvr06ZKkQ4cO9UZp3S4qKkrjxo3ToUOHlJiYqNraWpWVlTVr01+P57Fjx/T+++/r3//939ts15+PYdNxaet3MDEx8aIJ5fX19Tp79my/Oq5NQeTYsWNav359u5dmnz59uurr61VQUNA7BXazUaNGKTY21vfvcqAcx48//lgHDhxo9/dS6pvHsLXvh478/UxMTGzxd7Xpue4wKMOI1WrV1KlT9cEHH/i2eb1effDBB0pPTzexss4xDENLlizRqlWr9OGHH2rkyJHtviYvL0+SlJSU1MPV9YyKigodPnxYSUlJmjp1qoKDg5sdzwMHDqiwsLBfHs9XXnlF8fHxuu2229ps15+P4ciRI5WYmNjsmLlcLm3bts13zNLT01VWVqZdu3b52nz44Yfyer2+INbXNQWRgwcP6v3339eQIUPafU1eXp4CAgIuGtroL7744gudOXPG9+9yIBxHqaG3curUqZoyZUq7bfvSMWzv+6Ejfz/T09OVn5/fLFQ2BetLL7202wodlHJycgybzWa8+uqrxv79+427777biIqKajZbuL9YvHix4XA4jNzcXKO4uNh3q6qqMgzDMA4dOmQ88cQTxs6dO42jR48aq1evNkaNGmVce+21JlfecT/+8Y+N3Nxc4+jRo8Y//vEPY9asWUZsbKxRWlpqGIZh3HPPPcawYcOMDz/80Ni5c6eRnp5upKenm1y1/zwejzFs2DDj4Ycfbra9Px7D8vJyY8+ePcaePXsMScYzzzxj7Nmzx3cmybJly4yoqChj9erVxieffGLMnTvXGDlypFFdXe17j1tuucW47LLLjG3bthmbNm0yxo4da8yfP9+sXbpIW/tYW1tr3H777UZKSoqRl5fX7Hez6QyEzZs3G7/+9a+NvLw84/Dhw8brr79uxMXFGRkZGSbv2Xlt7WN5ebnxk5/8xNiyZYtx9OhR4/333zcuv/xyY+zYsUZNTY3vPfrycWzv36lhGIbT6TTCwsKMFStWXPT6vn4M2/t+MIz2/37W19cbEydONG666SYjLy/PeO+994y4uDgjKyur2+octGHEMAzjd7/7nTFs2DDDarUaV155pbF161azS+oUSS3eXnnlFcMwDKOwsNC49tprjZiYGMNmsxljxowxHnroIcPpdJpbuB++9a1vGUlJSYbVajWSk5ONb33rW8ahQ4d8z1dXVxv33nuvER0dbYSFhRlf//rXjeLiYhMr7px169YZkowDBw40294fj+FHH33U4r/LhQsXGobRcHrvY489ZiQkJBg2m8244YYbLtrvM2fOGPPnzzciIiIMu91ufPe73zXKy8tN2JuWtbWPR48ebfV386OPPjIMwzB27dplTJ8+3XA4HEZISIhxySWXGE8//XSzL3KztbWPVVVVxk033WTExcUZwcHBxvDhw43vf//7F/1PXV8+ju39OzUMw3j++eeN0NBQo6ys7KLX9/Vj2N73g2F07O9nQUGBMXv2bCM0NNSIjY01fvzjHxt1dXXdVqelsVgAAABTDMo5IwAAoO8gjAAAAFMRRgAAgKkIIwAAwFSEEQAAYCrCCAAAMBVhBAAAmIowAgAATEUYAQAApiKMAAAAUxFGAACAqQgjAADAVP8P2fLxyTHIhLYAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "lossi= []\n",
    "progress_bar = tqdm(range(200))\n",
    "xenc=F.one_hot(xs,num_classes=27).float()\n",
    "for k in progress_bar:\n",
    "    # forward pass\n",
    "    num=xs.shape[0]\n",
    "    logits= xenc.view(-1,54)@W\n",
    "    counts = logits.exp()\n",
    "     \n",
    "    probs = counts/counts.sum(1,keepdims=True)\n",
    "    # loss calculation\n",
    "    loss = -probs[torch.arange(num),ys].log().mean() +0.01*(W**2).mean() #regularization ie smoothing the weights\n",
    "    \n",
    "    # loss=F.cross_entropy(probs,ys)\n",
    "    W.grad=None \n",
    "    loss.backward()\n",
    "    progress_bar.set_postfix(loss = loss.data)\n",
    "    lossi.append(loss.data)\n",
    "    # update\n",
    "    W.data += -10 * W.grad\n",
    "     \n",
    "plt.plot(lossi)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([228146, 729])"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xenc.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mul.\n",
      "trreelahrasanardennitzfd.\n",
      "tven.\n",
      "ai.\n",
      "naakirandigszmbgcald.\n",
      "kadyt.\n",
      "sne.\n",
      "kari.\n",
      "hy.\n",
      "zemdeahockana.\n"
     ]
    }
   ],
   "source": [
    "for i in range(10):\n",
    "\n",
    "  ix = torch.tensor([0,0])\n",
    "  out = []\n",
    "  \n",
    "  while True:\n",
    "    # p=P[ix]   \n",
    "    xenc = F.one_hot(ix,num_classes=27).view(-1,54).float()\n",
    "    logits = xenc @ W\n",
    "    counts = logits.exp()\n",
    "    p = counts/counts.sum(1,keepdims=True)\n",
    "    isample = torch.multinomial(p,num_samples=1,replacement=True, ).item()\n",
    "    \n",
    "    out.append(itos[isample])\n",
    "    ix[0] = ix[1]\n",
    "    ix[1] = isample\n",
    "    if isample == 0:\n",
    "      break\n",
    "    \n",
    "     \n",
    "  print(''.join(out))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 200/200 [00:30<00:00,  6.57it/s, loss=tensor(2.3377)]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x16be26460>]"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAiMAAAGdCAYAAADAAnMpAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/P9b71AAAACXBIWXMAAA9hAAAPYQGoP6dpAAA8iElEQVR4nO3deXiU9b3//9dMlsk62feEsG8CCoiIKFIFhHoUa1f0FG05tdLQql2+FH/u7TG0tFq7HGw9budQpLWKtApyQAVENlmirGELJJCVhOx75v79MclAJPt2ZzLPx3XNleS+75l5394k8/Kz3RbDMAwBAACYxGp2AQAAwLMRRgAAgKkIIwAAwFSEEQAAYCrCCAAAMBVhBAAAmIowAgAATEUYAQAApvI2u4COcDgcys7OVnBwsCwWi9nlAACADjAMQ2VlZYqPj5fV2nr7h1uEkezsbCUlJZldBgAA6IKsrCwlJia2ut8twkhwcLAk58nY7XaTqwEAAB1RWlqqpKQk1+d4a9wijDR1zdjtdsIIAABupr0hFgxgBQAApiKMAAAAUxFGAACAqQgjAADAVIQRAABgKsIIAAAwFWEEAACYijACAABMRRgBAACmIowAAABTEUYAAICpCCMAAMBUHh1GXvskQ4+uPaiT+eVmlwIAgMfy6DCy7rNsrd6dSRgBAMBEHh1GooJskqSC8hqTKwEAwHN5dhgJbgwjpdUmVwIAgOfy6DASHewniZYRAADM5NFhxNUyUkYYAQDALB4dRqIbw0g+YQQAANN4dBihZQQAAPMRRuQMIw6HYXI1AAB4Jo8OI5GNU3vrHYaKq+pMrgYAAM/k0WHE19uqsAAfSXTVAABgFo8OI9Klrpr8MtYaAQDADB4fRlxrjdAyAgCAKTw+jEQxvRcAAFMRRpjeCwCAqTw+jEQTRgAAMJXHhxEGsAIAYC7CSBAtIwAAmMnjw0i0nTACAICZPD6MRAU5p/aWVteruq7B5GoAAPA8Hh9G7P7e8vV2/megdQQAgL7n8WHEYrG4xo2w1ggAAH3P48OIxFojAACYiTCiy9YaKSeMAADQ1wgjuqxlpJS1RgAA6GuEEUkxdueMmlzCCAAAfY4wIik+1F+SlF1MGAEAoK8RRiTFhzpbRs4XV5lcCQAAnocwIikxNECSM4wYhmFyNQAAeBbCiKTYED9ZLFJtvUMXymvNLgcAAI9CGJHk6211Te+lqwYAgL7VqTCycuVKTZgwQXa7XXa7XdOmTdOGDRtaPf6ll17STTfdpLCwMIWFhWnWrFnas2dPt4vuDQmuQayEEQAA+lKnwkhiYqKWL1+uffv2ae/evbrllls0f/58HT58uMXjt2zZogULFuijjz7Szp07lZSUpDlz5uj8+fM9UnxPSghrHDdykTACAEBfshjdHLEZHh6uFStWaNGiRe0e29DQoLCwMP3xj3/UwoULO/wepaWlCgkJUUlJiex2e3fKbVXqhqP689bTuv+GwXrqzqt65T0AAPAkHf389u7qGzQ0NOjNN99URUWFpk2b1qHnVFZWqq6uTuHh4W0eV1NTo5qaS0uzl5aWdrXMDkts7KZhzAgAAH2r0wNYDx48qKCgINlsNj344INau3atxo4d26HnLl26VPHx8Zo1a1abx6WmpiokJMT1SEpK6myZnZYQ1hhG6KYBAKBPdTqMjBo1Smlpadq9e7cWL16s++67T0eOHGn3ecuXL9eaNWu0du1a+fn5tXnssmXLVFJS4npkZWV1tsxOS7hsrREAANB3Ot1N4+vrq+HDh0uSJk+erE8//VQvvPCC/vznP7f6nN/85jdavny5Nm/erAkTJrT7HjabTTabrbOldUvTKqwlVXUqr6lXkK3LPVgAAKATur3OiMPhaDa+44t+/etf6xe/+IXef/99XXvttd19u14T7Ocju58zgDC9FwCAvtOp//1ftmyZ5s2bp0GDBqmsrEyrV6/Wli1btHHjRknSwoULlZCQoNTUVEnSr371Kz3xxBNavXq1Bg8erNzcXElSUFCQgoKCevhUui8hLEClOaU6f7FKI2OCzS4HAACP0Kkwkp+fr4ULFyonJ0chISGaMGGCNm7cqNmzZ0uSMjMzZbVeamxZuXKlamtr9bWvfa3Z6zz55JN66qmnul99D0sI9dPRnFLGjQAA0Ic6FUZefvnlNvdv2bKl2c9nzpzpbD2mSmB6LwAAfY5701yG6b0AAPQ9wshl4rk/DQAAfY4wcpnExvvTZBZVmlwJAACegzBymcERzjCSX1ajytp6k6sBAMAzEEYuExrgqxB/H0m0jgAA0FcII1/Q1Dpy5gJhBACAvkAY+YLBkYGSpLOFFSZXAgCAZyCMfEFyhDOMnCmkZQQAgL5AGPmCpm4aWkYAAOgbhJEvaGoZOUvLCAAAfYIw8gVNLSPZJVWqrmswuRoAAAY+wsgXhAf6KtjmLcOQzl2kdQQAgN5GGPkCi8Wi5Eim9wIA0FcIIy24NKOGQawAAPQ2wkgLLs2ooWUEAIDeRhhpAS0jAAD0HcJICwYzvRcAgD5DGGlBUzfNuYuVqq13mFwNAAADG2GkBVHBNgX4eslhSFlM7wUAoFcRRlpgsVg0NMrZVXMqv9zkagAAGNgII60YFhUkSTpVwCBWAAB6E2GkFZfCCC0jAAD0JsJIKwgjAAD0DcJIK4ZFXxozYhiGydUAADBwEUZaMTgiUFaLVFpdrwvltWaXAwDAgEUYaYWfj5eSwp3rjdBVAwBA7yGMtIFxIwAA9D7CSBuGudYaYXovAAC9hTDSBlpGAADofYSRNgyLJowAANDbCCNtaGoZOV9cparaBpOrAQBgYCKMtCE80FdhAT4yDCnjAuNGAADoDYSRdjS1jpykqwYAgF5BGGnHiBhnGDmRV2ZyJQAADEyEkXaMiA6WJB0njAAA0CsII+0YFdsURuimAQCgNxBG2jEyxhlGzhRWqLqOGTUAAPQ0wkg7IoMuzag5mU/rCAAAPY0w0g6LxeJqHWHcCAAAPY8w0gFN40bSCSMAAPQ4wkgHuFpGcgkjAAD0NMJIB1zqpmHMCAAAPY0w0gEjYy7do6asus7kagAAGFgIIx0QGuCrGLtNknSCGTUAAPQowkgHMW4EAIDeQRjpoKYwwowaAAB6FmGkg5qm9x7LIYwAANCTCCMdNDbOLkk6mlsqwzBMrgYAgIGDMNJBw6OD5G21qLiyTjkl1WaXAwDAgEEY6SA/Hy8Nj3ZO8T2SXWpyNQAADByEkU5o6qo5kkMYAQCgpxBGOmFM07gRwggAAD2GMNIJY+NpGQEAoKcRRjqhqWXkbGEly8IDANBDCCOdEB7oq7gQP0nSMVZiBQCgRxBGOolxIwAA9CzCSCe5ZtQwvRcAgB5BGOmkpkGshwkjAAD0CMJIJzW1jKTnlam23mFyNQAAuL9OhZGVK1dqwoQJstvtstvtmjZtmjZs2NDmc958802NHj1afn5+Gj9+vNavX9+tgs2WHBGgYD9v1dY7dJw7+AIA0G2dCiOJiYlavny59u3bp7179+qWW27R/Pnzdfjw4RaP37FjhxYsWKBFixbpwIEDuuuuu3TXXXfp0KFDPVK8GSwWiyYkhkiSDp4vMbkaAADcn8Xo5i1ow8PDtWLFCi1atOiKfd/85jdVUVGhd99917Xt+uuv1zXXXKMXX3yxw+9RWlqqkJAQlZSUyG63d6fcHrF8wzG9uPWUFlw3SKl3jze7HAAA+qWOfn53ecxIQ0OD1qxZo4qKCk2bNq3FY3bu3KlZs2Y123bbbbdp586dbb52TU2NSktLmz36k0stI8XmFgIAwADQ6TBy8OBBBQUFyWaz6cEHH9TatWs1duzYFo/Nzc1VTExMs20xMTHKzc1t8z1SU1MVEhLieiQlJXW2zF41PsEZRtJzy1Rd12ByNQAAuLdOh5FRo0YpLS1Nu3fv1uLFi3XffffpyJEjPVrUsmXLVFJS4npkZWX16Ot3V2KYv8IDfVXXYCidlVgBAOiWTocRX19fDR8+XJMnT1ZqaqquvvpqvfDCCy0eGxsbq7y8vGbb8vLyFBsb2+Z72Gw214ydpkd/YrFYXK0jnzOIFQCAbun2OiMOh0M1NTUt7ps2bZo++OCDZts2bdrU6hgTd+IaN3Ku2NxCAABwc96dOXjZsmWaN2+eBg0apLKyMq1evVpbtmzRxo0bJUkLFy5UQkKCUlNTJUkPPfSQbr75Zv32t7/V7bffrjVr1mjv3r36y1/+0vNn0sdcLSPnaBkBAKA7OhVG8vPztXDhQuXk5CgkJEQTJkzQxo0bNXv2bElSZmamrNZLjS033HCDVq9erccee0yPPvqoRowYoXfeeUfjxo3r2bMwwYTEUEnS8bwyVdU2yN/Xy9yCAABwU91eZ6Qv9Ld1RiTJMAxd9+wHKiir0ZsPTtOUweFmlwQAQL/S6+uMeDqLxaKJSaGSpP1nL5pbDAAAboww0g2TksMkSfszCSMAAHQVYaQbJg1qCiPFcoPeLgAA+iXCSDdMSAyRt9WigrIanbtYZXY5AAC4JcJIN/j5eGlsvHNADl01AAB0DWGkm5q6ag5kFptbCAAAboow0k0TB4VKomUEAICuIox0U1PLyJHsUu7gCwBAFxBGuikxzF9RwTbVOwyWhgcAoAsII91ksVg0qbGrZu/ZInOLAQDADRFGekDTUvCfZhBGAADoLMJID7h+aIQkae+Zi2pwsPgZAACdQRjpAWPi7Aqyeauspl5Hc0rNLgcAALdCGOkBXlaLrh3snFWzm64aAAA6hTDSQ6YOcXbV7MkoNLkSAADcC2Gkh1w3xDmIdU9GkRyMGwEAoMMIIz1kfEKI/H28dLGyTicLys0uBwAAt0EY6SG+3lZNSg6VxLgRAAA6gzDSg64b7Bw3sus040YAAOgowkgPmjasMYycKmTcCAAAHUQY6UHXJIUqwNdLhRW1OpZbZnY5AAC4BcJID/L1tmpq46yaT05eMLkaAADcA2Gkh00fHilJ2k4YAQCgQwgjPezGEc4wsiejSDX1DSZXAwBA/0cY6WGjYoIVGeSrqroGHcgsNrscAAD6PcJID7NYLK6uGsaNAADQPsJIL2DcCAAAHUcY6QU3NoaRz7KKVVxZa3I1AAD0b4SRXhAf6q9RMcFyGNK2E7SOAADQFsJIL5k5OkqStOVYvsmVAADQvxFGesmXRkVLkrYcL1ADS8MDANAqwkgvmZwcpmA/bxVV1Orzc8VmlwMAQL9FGOklPl5WzRjh7Kr5KL3A5GoAAOi/CCO9aOaoxnEj6YwbAQCgNYSRXnRzYxj5/FyJ8suqTa4GAID+iTDSi6KD/TQhMUSS9OFRWkcAAGgJYaSXzRkbI0naeDjX5EoAAOifCCO97LarYiVJn5wsVHlNvcnVAADQ/xBGetnw6CANjQxUbYODgawAALSAMNLLLBaLZl/V1FWTZ3I1AAD0P4SRPtDUVfPRsXzV1DeYXA0AAP0LYaQPXJMYquhgm8pr6rXzVKHZ5QAA0K8QRvqA1WrR7MZZNRsOMqsGAIDLEUb6yO0T4iRJ7x/OVW29w+RqAADoPwgjfWTqkAhFBdtUUlWn7Se5Vw0AAE0II33Ey2rR7eOdrSPvfpZjcjUAAPQfhJE+9G+NXTX/dyRP1XXMqgEAQCKM9KlJg8IUF+Kn8pp6bUmnqwYAAIkw0qesl3XV/OvzbJOrAQCgfyCM9LE7r4mXJG0+kqfS6jqTqwEAwHyEkT42PiFEw6ODVFPv0IaDDGQFAIAw0scsFovunpQgSXpr/3mTqwEAwHyEERN8ZWKCLBZpT0aRsooqzS4HAABTEUZMEBfir+nDIiVJb9M6AgDwcIQRkzR11bx94JwMwzC5GgAAzEMYMcltV8Uq0NdLZwsrtTujyOxyAAAwDWHEJIE2b9c03zV7Mk2uBgAA8xBGTPStKYMkSesP5aq4stbkagAAMAdhxEQTEkM0Js6u2noHA1kBAB6rU2EkNTVVU6ZMUXBwsKKjo3XXXXcpPT293ef97ne/06hRo+Tv76+kpCQ98sgjqq6u7nLRA4XFYtE91yVJktZ8mslAVgCAR+pUGNm6datSUlK0a9cubdq0SXV1dZozZ44qKipafc7q1av185//XE8++aSOHj2ql19+WX/729/06KOPdrv4gWD+xAT5+Vh1PK9c+zMvml0OAAB9zrszB7///vvNfn7ttdcUHR2tffv2acaMGS0+Z8eOHZo+fbruueceSdLgwYO1YMEC7d69u4slDyx2Px/dMSFeb+47p9d3nNXk5HCzSwIAoE91a8xISUmJJCk8vPUP0BtuuEH79u3Tnj17JEmnT5/W+vXr9eUvf7nV59TU1Ki0tLTZYyC774bBkqT1B3OUX0r3FQDAs3Q5jDgcDj388MOaPn26xo0b1+px99xzj5555hndeOON8vHx0bBhwzRz5sw2u2lSU1MVEhLieiQlJXW1TLcwLiFEk5PDVO8wtJppvgAAD9PlMJKSkqJDhw5pzZo1bR63ZcsWPfvss/qv//ov7d+/X2+//bbee+89/eIXv2j1OcuWLVNJSYnrkZWV1dUy3cbCacmSpL/uzlRtvcPkagAA6DsWowtTOJYsWaJ169Zp27ZtGjJkSJvH3nTTTbr++uu1YsUK17ZVq1bpgQceUHl5uazW9vNQaWmpQkJCVFJSIrvd3tly3UJtvUPTf/WhCspq9PsFE3Xn1fFmlwQAQLd09PO7Uy0jhmFoyZIlWrt2rT788MN2g4gkVVZWXhE4vLy8XK8HJ19vq+6d6lwE7eWPT/PfBgDgMToVRlJSUrRq1SqtXr1awcHBys3NVW5urqqqqlzHLFy4UMuWLXP9fMcdd2jlypVas2aNMjIytGnTJj3++OO64447XKEETv9+fbJ8va367FyJPj3DNF8AgGfo1NTelStXSpJmzpzZbPurr76q+++/X5KUmZnZrCXksccek8Vi0WOPPabz588rKipKd9xxh/7zP/+ze5UPQJFBNn11UqLe2JOpv2w7reuGMM0XADDwdWnMSF/zhDEjTU4VlOvW326VxSJt/vHNGhYVZHZJAAB0Sa+MGUHvGxYVpFljYmQY0n9/nGF2OQAA9DrCSD/0wIyhkqS39p1THougAQAGOMJIPzRlcJimDA5TbYNDL207bXY5AAD0KsJIP2SxWJTypeGSnIugFVXUmlwRAAC9hzDST908MkrjE0JUVdegV7YzdgQAMHARRvqpy1tHXt9xRiWVdSZXBABA7yCM9GNzxsZoVEywymrq9d/bGTsCABiYCCP9mNVq0SOzR0qSXtmewdgRAMCARBjp5267KkbjEuyqqG3Qn7edMrscAAB6HGGkn7NYLPpxY+vI6zvOKL+MdUcAAAMLYcQNfGlUtCYOClV1nUMvbD5hdjkAAPQowogbsFgsWjZvjCTpjT2ZOpFXZnJFAAD0HMKIm7huSLhuuypGDkN6dv1Rs8sBAKDHEEbcyM/njZG31aKP0gu0/cQFs8sBAKBHEEbcyJDIQH17WrIk6ZfvHVGDwzC5IgAAuo8w4mZ+dMsI2f28dSy3TG/tO2d2OQAAdBthxM2EBfrqh7eMkCT95v/SVVFTb3JFAAB0D2HEDS28IVmDwgOUX1ajv2xjmXgAgHsjjLghm7eXls4dLUl6cespZRZWmlwRAABdRxhxU18eH6sbhkWopt6hx9cdkmEwmBUA4J4II27KYrHoF3eNk6+XVVuPF+i9gzlmlwQAQJcQRtzYsKggLZ45TJL09L+OqLS6zuSKAADoPMKIm1s8c5iGRgaqoKxGv9mYbnY5AAB0GmHEzfn5eOmXXxknSfrfXWeVllVsbkEAAHQSYWQAuGFYpO6emCDDkB59+6DqGxxmlwQAQIcRRgaIR28foxB/Hx3JKdVLH2eYXQ4AAB1GGBkgIoNseuz2MZKk5zcd19GcUpMrAgCgYwgjA8jXJidq1pgY1TY49Mjf0lRT32B2SQAAtIswMoBYLBal3j1e4YG+OpZbphc2nzC7JAAA2kUYGWCigm169ivjJTmXit93tsjkigAAaBthZACaOy5Wd09KkMOQfvz3z7izLwCgXyOMDFBP3nGV4kL8dLawUs+uP2p2OQAAtIowMkCF+PvoN1+/WpL0192ZWs+9awAA/RRhZACbPjxS3795qCRp6T8+19nCCpMrAgDgSoSRAe6nc0bp2uQwldXU6wd/3a/qOqb7AgD6F8LIAOfjZdUf7pmo8EBfHc4u1S/fO2J2SQAANEMY8QBxIf567hvO8SOrdmXqX59lm1wRAACXEEY8xMxR0Ur50jBJ0s/f+lynC8pNrggAACfCiAd5ZNZITR0SroraBn3vf/aqtLrO7JIAACCMeBLvxvEjcSF+OlVQoR+9cUANDsPssgAAHo4w4mGig/300sJr5edj1Zb0Av3q/WNmlwQA8HCEEQ80LiFEK77mHND6l22n9da+cyZXBADwZIQRD3XH1fH64S3DJUnL3j6o/ZkXTa4IAOCpCCMe7JFZIzVnbIxqGxx64H/2Kauo0uySAAAeiDDiwaxWi57/5jUaE2fXhfIa3ffKHhVV1JpdFgDAwxBGPFygzVuvfWeKEkL9dfpChRa9/qmqalkyHgDQdwgjUIzdT69/d4pC/H10ILNYP3xjv+obHGaXBQDwEIQRSJKGRwfr5fuulc3bqs1H8/X4ukMyDNYgAQD0PsIIXK4dHK7fL5goq0V6Y0+Wfvt/x80uCQDgAQgjaOa2q2L1zPxxkqQ/fnRSf/ropMkVAQAGOsIIrvDv1ydr2bzRkqQVG9P13x+fNrkiAMBARhhBi75/8zA9MmukJOmX7x3V/+46a3JFAICBijCCVv3o1uFaPHOYJOnxdw7p73uzTK4IADAQEUbQKovFov932yh9Z/pgSdLStz7X3z7NNLcoAMCAQxhBmywWi574t7H69vXJMgxp6VsH9fqOM2aXBQAYQAgjaJfFYtEz86/SohuHSJKe/Odh/XnrKZOrAgAMFIQRdIjFYtFjt4/Rki857/SbuuGYXth8goXRAADdRhhBh1ksFv30tlH66RznLJvnNx9X6oZjcjgIJACAriOMoNOW3DJCj90+RpL0l22n9ZM3P1NtPfeyAQB0TafCSGpqqqZMmaLg4GBFR0frrrvuUnp6ervPKy4uVkpKiuLi4mSz2TRy5EitX7++y0XDfP9x01Ct+NoEeVktWnvgvBa9/qkqaurNLgsA4IY6FUa2bt2qlJQU7dq1S5s2bVJdXZ3mzJmjioqKVp9TW1ur2bNn68yZM/rHP/6h9PR0vfTSS0pISOh28TDX169N0n/fd638fbz08YkLWvDSLl0orzG7LACAm7EY3RiBWFBQoOjoaG3dulUzZsxo8ZgXX3xRK1as0LFjx+Tj49Ol9yktLVVISIhKSkpkt9u7Wi56SVpWsb772qcqqqhVckSAXrl/ioZFBZldFgDAZB39/O7WmJGSkhJJUnh4eKvH/POf/9S0adOUkpKimJgYjRs3Ts8++6waGhq689boR65JCtU/HpympHB/nS2s1Ff+9Ik+OXnB7LIAAG6iy2HE4XDo4Ycf1vTp0zVu3LhWjzt9+rT+8Y9/qKGhQevXr9fjjz+u3/72t/rlL3/Z6nNqampUWlra7IH+bWhUkNb+YLomDQpVaXW9Fr6yR3/dzf1sAADt63I3zeLFi7VhwwZt375diYmJrR43cuRIVVdXKyMjQ15eXpKk5557TitWrFBOTk6Lz3nqqaf09NNPX7Gdbpr+r7quQUvf+lzr0rIlSd+dPkT/3+1j5GW1mFwZAKCv9Wo3zZIlS/Tuu+/qo48+ajOISFJcXJxGjhzpCiKSNGbMGOXm5qq2trbF5yxbtkwlJSWuR1YWN2hzF34+XvrdN6/RT2Y71yJ55ZMMfee1T1Vc2fK1BgCgU2HEMAwtWbJEa9eu1YcffqghQ4a0+5zp06fr5MmTcjgurUNx/PhxxcXFydfXt8Xn2Gw22e32Zg+4D4vFoh/eOkJ/vGei/Hys2na8QHf8cbuOZNPdBgC4UqfCSEpKilatWqXVq1crODhYubm5ys3NVVVVleuYhQsXatmyZa6fFy9erKKiIj300EM6fvy43nvvPT377LNKSUnpubNAv/RvE+L11uIblBTur6yiKt298hOtSztvdlkAgH6mU2NGLJaW+/1fffVV3X///ZKkmTNnavDgwXrttddc+3fu3KlHHnlEaWlpSkhI0KJFi7R06dJmXTdtYWqveyuurNWP1qRp2/ECSdJ3pg/Wsnlj5OvNAsAAMJB19PO7W+uM9BXCiPtrcBh6ftNx/fGjk5Kc04H/sGCiksIDTK4MANBb+mSdEaCjvKzOm+y9tPBa2f28lZZVrNt//7HeP5RrdmkAAJMRRtCnZo+N0fqHbtLExvVIHly1T0/987Bq6lkEDwA8FWEEfS4xLEB///40ff/moZKk13ac0fw/fqJjucy2AQBPRBiBKXy8rFo2b4xevX+KwgN9dSy3THf+4RP9ZdspNTj6/TAmAEAPIozAVF8aHa2ND8/QrDHRqm1w6Nn1x3TPS7t07mKl2aUBAPoIYQSmiwq26aWF12r53eMV4Oul3RlFmvu7j/Xm3iy5wWQvAEA3EUbQL1gsFn3rukHa8NBNujY5TOU19frZPz7Xg6v26UJ5jdnlAQB6EWEE/UpyRKD+9v1p+n9zR8nHy6KNh/M067mttJIAwABGGEG/42W16Aczh+udlOkaG2dXcWWdfvaPz/XvL+/W2cIKs8sDAPQwwgj6raviQ7RuyXT9fN5o2byt+uRkoW773Ta9uPWU6hsc7b8AAMAtEEbQr/l4WfXgzcP0f4/M0PThEaquc2j5hmOa/6dPdOh8idnlAQB6AGEEbiE5IlCrFk3Viq9NUIi/jw5nl+rOP27XE+sOqbiy1uzyAADdQBiB27BYLPr6tUn64Cc3686r4+UwpP/ZeVZf+s0Wrd6dyWJpAOCmuGsv3NaOUxf09D+PKD2vTJI0LsGup++8SpOTw02uDAAgdfzzmzACt1bf4ND/7jqr5zYdV1l1vSTp7okJ+vm80Yq2+5lcHQB4NsIIPMqF8hqteD9df9+XJcOQAn299MCMYfrejCEK8PU2uzwA8EiEEXikz7KK9cQ/D+uzrGJJzqXmfzx7pL4+OVHeXgyRAoC+RBiBx3I4DL13MEe/3nhMWUVVkqTh0UH6+dzRunVMtCwWi8kVAoBnIIzA49XUN2jVrkz94cMTKq6skyRdNyRcy+aN1sRBYSZXBwADH2EEaFRSVaeVW07plU8yVFvvXLn11tHRemT2SI1LCDG5OgAYuAgjwBecL67S85uO6+3959S0JMltV8Xo4VkjNSaOf1cA0NMII0ArTheU64UPTuifn2Wr6V//7ePj9PCsERoRE2xucQAwgBBGgHacyCvT7zaf0HsHcyRJFoszlKR8aTgtJQDQAwgjQAcdzSnV7zYf18bDea5tt4yOVsqXhrGaKwB0A2EE6KTD2SVaueWU3juY4+q+mTokXClfGq6bRkQyJRgAOokwAnRRxoUK/XnrKb21/5zqGpy/HuMTQvTAjKGaNy6WxdMAoIMII0A35ZRU6aVtGXpjT6aq6hokSQmh/vrO9MH6xpQk2f18TK4QAPo3wgjQQ4oqavX6jjNateusCitqJUlBNm9949okfWf6YCWFB5hcIQD0T4QRoIdV1zVoXdp5/ffHGTqRXy5JslqkueNitXDaYE0dEs64EgC4DGEE6CWGYWjr8QK9vD1DH5+44No+IjpI356WrK9MTFAwXTgAQBgB+sLRnFL9z86zeufAede4kkBfL31lUoL+/fpkjY7l3ysAz0UYAfpQaXWd3t53Tv+766xOFVS4tl83OFz/Pi1Zc6+Kla83s3AAeBbCCGACwzC083ShVu06q42H89TQeBOcyCBf3T0pUd+4NlHDo1lyHoBnIIwAJsstqdaaTzP1xp5M5ZXWuLZPHBSqb1ybpH+bEMfYEgADGmEE6CfqGhzakl6gv+/N0ofH8l2tJX4+Vn15fJy+cW0SM3EADEiEEaAfyi+r1jsHzuvve8/pZOP0YEkaFB6gr09O1F0TE1i3BMCAQRgB+jHDMJSWVay/7z2nf32WrfKaete+yclhuuuaeH15fJwigmwmVgkA3UMYAdxEVW2DNhzK0Vv7z2nHqULXTfq8rRbdNCJS869J0OyxMQq0eZtbKAB0EmEEcEN5pdX612fZWpeWrYPnS1zb/X28NHtsjO6aGK8bh0cxTRiAWyCMAG7uVEG51qVla13aeZ0trHRtt/t5a/bYWH15fKxuHBEpm7eXiVUCQOsII8AA0TS+ZF1att79PEcXyi9NEw62eWvW2BjNGxerGSOj5OdDMAHQfxBGgAGowWFo39mLWn8wRxsO5TRbvyTQ10u3jonRl8c7g0mAL2NMAJiLMAIMcA6HoQNZF7X+YK42HMxRdkm1a5/N26obh0dq1tgY3TomWtHBfiZWCsBTEUYAD+JwGPrsXLE2HMrVhkM5yiqqarb/mqRQzR4bo9ljYzQiOogF1gD0CcII4KEMw9DxvHJtOpKrTUfz9VlWcbP9g8IDNGtMjG4ZHa0pQ8IYAAug1xBGAEhyThf+4Gi+Nh3J1SenClVb73DtC/D10g3DInTzqGjNHBnF6q8AehRhBMAVKmrq9fGJAm0+mq+txwtUUFbTbP/QqEDNHBmtmaOidN2QcGbnAOgWwgiANjkcho7klGrr8QJtTS/QvsyLrpv4Sc4b+U0bGqGZo6I1Y2SUBkcEMNYEQKcQRgB0SklVnT45eUFb0p2tJpdPG5ak+BA/3TA8UjcOj9QNwyIUbWeGDoC2EUYAdJlhGDqWW6Yt6QXakp6v/ZkXVdfQ/E/FiOggTW8MJtcPi5Ddz8ekagH0V4QRAD2msrZen565qB0nL+iTUxd0OLtUl//lsFqkCYmhmj48QtOHRWrioDD5+zLeBPB0hBEAveZiRa12nS7U9pMXtONUoTIuVDTb7+Nl0YTEUF03JFzXDQnXtclhCqblBPA4hBEAfSa7uEqfNAaTnacKlVta3Wy/1SJdFR/iCidTBocrPNDXpGoB9BXCCABTGIahrKIq7c4o1J6MIu05U9TsrsNNRsYENbaahGvSoDAlhfszWwcYYAgjAPqN3JJq7TlTpD0Zhdp9ukgn8suvOCYyyKZJg0I1OTlMk5LDND4hhHVOADdHGAHQbxWW1+jTMxe1J6NI+zMv6nB2yRWzdXy8LBobH6JJg0I1aVCYJieHKT7U36SKAXQFYQSA26iua9Ch8yXan3lR+85e1P7M4itWh5WkWLufJiU7w8mExFCNS7ArwNfbhIoBdARhBIDbMgxD5y5WaX/mRe0/e1H7Mi/qaE5ZsxViJefA2BHRwRqfGKKrE0M0ITFUo+OCufkf0E8QRgAMKJW19fosy9l6kpZVrM/PFV+xSqzk7N4ZE2fXhMQQTUgI1YSkEI2IDpaXlcGxQF/rlTCSmpqqt99+W8eOHZO/v79uuOEG/epXv9KoUaM69Pw1a9ZowYIFmj9/vt55552Ovi1hBECL8kqr9fm5En1+rlifNX4trqy74jh/Hy+NS7DrqvgQXRXv/DoiJkg+XlYTqgY8R6+Ekblz5+pb3/qWpkyZovr6ej366KM6dOiQjhw5osDAwDafe+bMGd14440aOnSowsPDCSMAelxT985n54r1+bkSfZZVrEPnS1RR23DFsb5eVo2ICXKFk7Hxdo2JsyvIxhgUoKf0STdNQUGBoqOjtXXrVs2YMaPV4xoaGjRjxgx997vf1ccff6zi4mLCCIA+4XAYOn2hXJ9llehwdqkOZ5foSE6pyqrrrzjWYpEGRwRqbJxdY+Odj6vi7YoKsrEGCtAFHf387tb/ApSUlEiSwsPD2zzumWeeUXR0tBYtWqSPP/64O28JAJ1itVo0PDpYw6OD9dXJzm1NLSiHs0t0JLu0MaSUKre0WhkXKpRxoULvHcxxvUZ4oK9GxQRrVGywRsc6v46MCVYgrShAj+jyb5LD4dDDDz+s6dOna9y4ca0et337dr388stKS0vr8GvX1NSopubSwLTS0tKulgkAV7BYLEoKD1BSeIDmjotzbS8sr9GRnEvh5Eh2iU5fqFBRRa12ni7UztOFzV4nKdxfo2LsroAyOjZYQyID5c1YFKBTuhxGUlJSdOjQIW3fvr3VY8rKyvTtb39bL730kiIjIzv82qmpqXr66ae7WhoAdElEkE03jYjSTSOiXNuqaht0Ir9Mx3LLlJ5bpuN5zu8LymqUVVSlrKIqbT6a5zre18uqYdFBGhUTpFGxzqAyPDpICaH+sjKjB2hRl8aMLFmyROvWrdO2bds0ZMiQVo9LS0vTxIkT5eV1ac6/w+GQJFmtVqWnp2vYsGFXPK+llpGkpCTGjADoN4oqanUst1TpjSHlWGNQqWxhsKwk+flYNSwqSMOjgzQ8KkgjYpzfJ0cEMqsHA1avDGA1DEM//OEPtXbtWm3ZskUjRoxo8/jq6mqdPHmy2bbHHntMZWVleuGFFzRy5Ej5+rZ/504GsAJwBw6HofPFVY2tKKU6llumE3nlyrhQodoGR4vP8bZalBwRoBHRzhaUpsewqCD5+7J4G9xbrwxgTUlJ0erVq7Vu3ToFBwcrNzdXkhQSEiJ/f+c9IxYuXKiEhASlpqbKz8/vivEkoaGhktTmOBMAcEdW66WxKLPHxri21zc4lFlUqZP55TpZUO78ml+uU/nlqqht0KmCCp0qqJAOX3oti0VKCPXXsKggDYkM1NCoQA2JdD7iQ+jywcDSqTCycuVKSdLMmTObbX/11Vd1//33S5IyMzNltdLkCABNvL2sGhoVpKFRQZpz2XbDMJRTUq2T+eU6cVlAOZFfpouVdTp3sUrnLlZp6/GCZq9n87ZqcESgBkcGaEhkkIZGBmpIY1iJCPRlGjLcDsvBA0A/VFheo5P55TrdONX4dEGFzhRW6GxhxRV3OL5csM3bFUwufwyODJTdz6cPzwDg3jQAMCDVNziUXVyt0xecY1HOXKhwBZbzxVVq6y96WICPBkUEKjk8QIPCAzQoIsD5fUSAYoL96PpBjyOMAICHqa5rUGZRpasVJaOgsVXlQoUulF95U8HL2bytSmoKKeEBSo649DUxLEB+PgymRef1yQqsAID+w8/HSyNjnKvDflFZdZ2yiqqUWVShs4WVyixyPs4WVup8cZVq6h2ugbUtibX7uVpSEsMClBDmr8TGR6zdj4Xe0C20jACAh2vq+jlbVOEMKYWVzQJLec2V9/G5nJfVoli7X2M4CVBimL8rrCSFBSg2xI+1VDwULSMAgA7x9rJqUIRz7MgXGYahoopaVzDJbGxJcc70cX5f1+BcX+V8cZV2ZxRd8RpWixrDSoCrNSXhsuASG+InmzfdQJ6MlhEAQJc5HIYKymt07mKlayqyK6hcrNK54irV1re84NvlIoN8FRfir7gQP8WHOr/GhforvvFrTLCNriA3RMsIAKDXWa0Wxdj9FGP30+TkK/c7HIYulNfo3GWtKecuVjmDSuP3NfUOXSiv1YXyWh08X9Ly+1ik6GA/xYX6OYPKF4JLfKi/ooJszAhyU4QRAECvsVotirb7Kdrup0mDwq7YbxiGLlbWKbu4Sjkl1copqVJ2sfNrTnG1skuqlFdarboGQ7ml1cotrdaBVt7LuzEYxYc2hpVQP8UE+yk2xE8xdpuig/0UbbfRJdQPEUYAAKaxWCwKD/RVeKCvxiWEtHiMw2HoQkWNcoqbh5XskmrlNIaYvNJq1TsujV2RLrb6nuGBvooOtinG7qdYe2NQcX3v/DkiyCYvWln6DGEEANCvWa0WZ6tGsJ+uTgpt8Zj6BocKymuuaFXJL6tRXkm18sqqlVdao9p6h4oqahvvulzW6nt6WS2KCrIpxm5zdUNd/n10Y0tLqL8PXUM9gDACAHB73l7WxnEk/pKu7A6SnF1CxZV1yiurVm5JtfJLa5TX2PWTV1qj/MbtF8pr1OC41C0ktTyORXJ2DUUE+Soq2KaoIJvzq+t7v0s/B9sU6OvFfYNaQRgBAHgEi8WisEBfhQX6anRs6zM76hscKqyoVW5j909eU+vKZd8XlNeoqKJW9Q5DeaU1yitte4VbSfL38fpCWPlieHE+IoNs8vX2rJlDhBEAAC7j7WV1dce0pa7BocLyWhWU1aigvNr5telRXtPs54raBlU1LtefWVTZbg2hAT6KDLIpPNBXkUG+igi0KSLIVxGBvooIsjX7GjIAuooIIwAAdIGPl1WxIc7ZOlLLg2+bVNTU60L5lWHlwhdCS0F5jeoanN1JxZV1HarDy+ocBOwMKJeCS1OYaQoukUHOgcJBNu9+111EGAEAoJcF2rwVaPNWckRgm8cZhqGSqjrlNwaVwnLnYNvC8hpdaPzatO1CeY1Kq+vV4DBcYaYjfL2timwMKOGNASYyyKZ7rhukwZFt19dbCCMAAPQTFotFoQG+Cg3wbfGGh1/UNDuosMIZUpq+XiivVVHT95eFmKq6BtXWO5RdUq3skupmrzV3XKwGizACAAA6wdf78q6i9lXW1jeGlsaAUlHr/Lm8Rolh/r1cbesIIwAAeIgAX28FhHsrKfzKmyKaybPmDgEAgH6HMAIAAExFGAEAAKYijAAAAFMRRgAAgKkIIwAAwFSEEQAAYCrCCAAAMBVhBAAAmIowAgAATEUYAQAApiKMAAAAUxFGAACAqdzirr2GYUiSSktLTa4EAAB0VNPndtPneGvcIoyUlZVJkpKSkkyuBAAAdFZZWZlCQkJa3W8x2osr/YDD4VB2draCg4NlsVh67HVLS0uVlJSkrKws2e32Hnvd/oRzdH8D/fwkznEgGOjnJw38c+yN8zMMQ2VlZYqPj5fV2vrIELdoGbFarUpMTOy117fb7QPyH9blOEf3N9DPT+IcB4KBfn7SwD/Hnj6/tlpEmjCAFQAAmIowAgAATOXRYcRms+nJJ5+UzWYzu5Rewzm6v4F+fhLnOBAM9POTBv45mnl+bjGAFQAADFwe3TICAADMRxgBAACmIowAAABTEUYAAICpPDqM/OlPf9LgwYPl5+enqVOnas+ePWaX1CWpqamaMmWKgoODFR0drbvuukvp6enNjpk5c6YsFkuzx4MPPmhSxZ331FNPXVH/6NGjXfurq6uVkpKiiIgIBQUF6atf/ary8vJMrLjzBg8efMU5WiwWpaSkSHK/a7ht2zbdcccdio+Pl8Vi0TvvvNNsv2EYeuKJJxQXFyd/f3/NmjVLJ06caHZMUVGR7r33XtntdoWGhmrRokUqLy/vw7NoW1vnWFdXp6VLl2r8+PEKDAxUfHy8Fi5cqOzs7Gav0dJ1X758eR+fSevau47333//FfXPnTu32TH9+Tq2d34t/U5aLBatWLHCdUx/voYd+XzoyN/PzMxM3X777QoICFB0dLR+9rOfqb6+vsfq9Ngw8re//U0//vGP9eSTT2r//v26+uqrddtttyk/P9/s0jpt69atSklJ0a5du7Rp0ybV1dVpzpw5qqioaHbc9773PeXk5Lgev/71r02quGuuuuqqZvVv377dte+RRx7Rv/71L7355pvaunWrsrOzdffdd5tYbed9+umnzc5v06ZNkqSvf/3rrmPc6RpWVFTo6quv1p/+9KcW9//617/W73//e7344ovavXu3AgMDddttt6m6utp1zL333qvDhw9r06ZNevfdd7Vt2zY98MADfXUK7WrrHCsrK7V//349/vjj2r9/v95++22lp6frzjvvvOLYZ555ptl1/eEPf9gX5XdIe9dRkubOndus/jfeeKPZ/v58Hds7v8vPKycnR6+88oosFou++tWvNjuuv17Djnw+tPf3s6GhQbfffrtqa2u1Y8cOvf7663rttdf0xBNP9Fyhhoe67rrrjJSUFNfPDQ0NRnx8vJGammpiVT0jPz/fkGRs3brVte3mm282HnroIfOK6qYnn3zSuPrqq1vcV1xcbPj4+Bhvvvmma9vRo0cNScbOnTv7qMKe99BDDxnDhg0zHA6HYRjufQ0lGWvXrnX97HA4jNjYWGPFihWubcXFxYbNZjPeeOMNwzAM48iRI4Yk49NPP3Uds2HDBsNisRjnz5/vs9o76ovn2JI9e/YYkoyzZ8+6tiUnJxvPP/987xbXQ1o6x/vuu8+YP39+q89xp+vYkWs4f/5845Zbbmm2zZ2u4Rc/Hzry93P9+vWG1Wo1cnNzXcesXLnSsNvtRk1NTY/U5ZEtI7W1tdq3b59mzZrl2ma1WjVr1izt3LnTxMp6RklJiSQpPDy82fa//vWvioyM1Lhx47Rs2TJVVlaaUV6XnThxQvHx8Ro6dKjuvfdeZWZmSpL27dunurq6Ztdz9OjRGjRokNtez9raWq1atUrf/e53m90c0t2vYZOMjAzl5uY2u2YhISGaOnWq65rt3LlToaGhuvbaa13HzJo1S1arVbt37+7zmntCSUmJLBaLQkNDm21fvny5IiIiNHHiRK1YsaJHm7/7wpYtWxQdHa1Ro0Zp8eLFKiwsdO0bSNcxLy9P7733nhYtWnTFPne5hl/8fOjI38+dO3dq/PjxiomJcR1z2223qbS0VIcPH+6RutziRnk97cKFC2poaGj2H1aSYmJidOzYMZOq6hkOh0MPP/ywpk+frnHjxrm233PPPUpOTlZ8fLw+//xzLV26VOnp6Xr77bdNrLbjpk6dqtdee02jRo1STk6Onn76ad100006dOiQcnNz5evre8Uf+JiYGOXm5ppTcDe98847Ki4u1v333+/a5u7X8HJN16Wl38Gmfbm5uYqOjm6239vbW+Hh4W55Xaurq7V06VItWLCg2U3IfvSjH2nSpEkKDw/Xjh07tGzZMuXk5Oi5554zsdqOmzt3ru6++24NGTJEp06d0qOPPqp58+Zp586d8vLyGlDX8fXXX1dwcPAVXcDucg1b+nzoyN/P3NzcFn9Xm/b1BI8MIwNZSkqKDh061Gw8haRm/bPjx49XXFycbr31Vp06dUrDhg3r6zI7bd68ea7vJ0yYoKlTpyo5OVl///vf5e/vb2JlvePll1/WvHnzFB8f79rm7tfQk9XV1ekb3/iGDMPQypUrm+378Y9/7Pp+woQJ8vX11fe//32lpqa6xbLj3/rWt1zfjx8/XhMmTNCwYcO0ZcsW3XrrrSZW1vNeeeUV3XvvvfLz82u23V2uYWufD/2BR3bTREZGysvL64rRwnl5eYqNjTWpqu5bsmSJ3n33XX300UdKTExs89ipU6dKkk6ePNkXpfW40NBQjRw5UidPnlRsbKxqa2tVXFzc7Bh3vZ5nz57V5s2b9R//8R9tHufO17DpurT1OxgbG3vFgPL6+noVFRW51XVtCiJnz57Vpk2b2r01+9SpU1VfX68zZ870TYE9bOjQoYqMjHT9uxwo1/Hjjz9Wenp6u7+XUv+8hq19PnTk72dsbGyLv6tN+3qCR4YRX19fTZ48WR988IFrm8Ph0AcffKBp06aZWFnXGIahJUuWaO3atfrwww81ZMiQdp+TlpYmSYqLi+vl6npHeXm5Tp06pbi4OE2ePFk+Pj7Nrmd6eroyMzPd8nq++uqrio6O1u23397mce58DYcMGaLY2Nhm16y0tFS7d+92XbNp06apuLhY+/btcx3z4YcfyuFwuIJYf9cURE6cOKHNmzcrIiKi3eekpaXJarVe0bXhLs6dO6fCwkLXv8uBcB0lZ2vl5MmTdfXVV7d7bH+6hu19PnTk7+e0adN08ODBZqGyKViPHTu2xwr1SGvWrDFsNpvx2muvGUeOHDEeeOABIzQ0tNloYXexePFiIyQkxNiyZYuRk5PjelRWVhqGYRgnT540nnnmGWPv3r1GRkaGsW7dOmPo0KHGjBkzTK68437yk58YW7ZsMTIyMoxPPvnEmDVrlhEZGWnk5+cbhmEYDz74oDFo0CDjww8/NPbu3WtMmzbNmDZtmslVd15DQ4MxaNAgY+nSpc22u+M1LCsrMw4cOGAcOHDAkGQ899xzxoEDB1wzSZYvX26EhoYa69atMz7//HNj/vz5xpAhQ4yqqirXa8ydO9eYOHGisXv3bmP79u3GiBEjjAULFph1Sldo6xxra2uNO++800hMTDTS0tKa/W42zUDYsWOH8fzzzxtpaWnGqVOnjFWrVhlRUVHGwoULTT6zS9o6x7KyMuOnP/2psXPnTiMjI8PYvHmzMWnSJGPEiBFGdXW16zX683Vs79+pYRhGSUmJERAQYKxcufKK5/f3a9je54NhtP/3s76+3hg3bpwxZ84cIy0tzXj//feNqKgoY9myZT1Wp8eGEcMwjD/84Q/GoEGDDF9fX+O6664zdu3aZXZJXSKpxcerr75qGIZhZGZmGjNmzDDCw8MNm81mDB8+3PjZz35mlJSUmFt4J3zzm9804uLiDF9fXyMhIcH45je/aZw8edK1v6qqyvjBD35ghIWFGQEBAcZXvvIVIycnx8SKu2bjxo2GJCM9Pb3Zdne8hh999FGL/y7vu+8+wzCc03sff/xxIyYmxrDZbMatt956xXkXFhYaCxYsMIKCggy73W585zvfMcrKykw4m5a1dY4ZGRmt/m5+9NFHhmEYxr59+4ypU6caISEhhp+fnzFmzBjj2WefbfZBbra2zrGystKYM2eOERUVZfj4+BjJycnG9773vSv+p64/X8f2/p0ahmH8+c9/Nvz9/Y3i4uIrnt/fr2F7nw+G0bG/n2fOnDHmzZtn+Pv7G5GRkcZPfvITo66ursfqtDQWCwAAYAqPHDMCAAD6D8IIAAAwFWEEAACYijACAABMRRgBAACmIowAAABTEUYAAICpCCMAAMBUhBEAAGAqwggAADAVYQQAAJiKMAIAAEz1/wP8aaZ2HBERMwAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "W = torch.randn((729, 27))*0.01;W.requires_grad = True\n",
    "lossi= []\n",
    "\n",
    "xenc=F.one_hot(xs[:,0] * 27 + xs[:,1],num_classes=729).float()\n",
    "progress_bar = tqdm(range(200))\n",
    "for k in progress_bar:\n",
    "    # forward pass\n",
    "    num=xs.shape[0]\n",
    "    logits= xenc.view(-1,729)@W\n",
    "    counts = logits.exp()\n",
    "     \n",
    "    probs = counts/counts.sum(1,keepdims=True)\n",
    "    # loss calculation\n",
    "    loss = -probs[torch.arange(num),ys].log().mean() +0.01*(W**2).mean() #regularization ie smoothing the weights\n",
    "    # loss=F.cross_entropy(probs,ys) \n",
    "    W.grad=None \n",
    "    loss.backward()\n",
    "    lossi.append(loss.data)\n",
    "    # update\n",
    "    progress_bar.set_postfix(loss = loss.data)\n",
    "    W.data += -50 * W.grad\n",
    "    \n",
    "plt.plot(lossi)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 208,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "nokszxlharleen.\n",
      "adi.\n",
      "betane.\n",
      "zrhen.\n",
      "kie.\n",
      "emxfccorl.\n",
      "sapbwjvdrci.\n",
      "fiene.\n",
      "nmwvqfxwh.\n",
      "zskcsqamaartertkoblqczjjbfigh.\n",
      "boswsvon.\n",
      "anaochadenton.\n",
      "da.\n",
      "jaqkosf.\n",
      "sa.\n",
      "ay.\n",
      "da.\n",
      "saoorqehlorrkxkoehdvfyizki.\n",
      "jduqesty.\n",
      "emeri.\n"
     ]
    }
   ],
   "source": [
    "#Yay almost the same loss as the model with direct Prob matrix\n",
    "\n",
    "# Now we will sample the MLP model\n",
    "for i in range(20):\n",
    "\n",
    "  ix1 = 0\n",
    "  ix2 = 0\n",
    "  out = []\n",
    "  \n",
    "  while True:\n",
    "    # p=P[ix]   \n",
    "    xenc = F.one_hot(torch.tensor([ix1 * 27 + ix2]),num_classes=729).float()\n",
    "    logits = xenc @ W\n",
    "    counts = logits.exp()\n",
    "    p = counts/counts.sum(1,keepdims=True)\n",
    "    isample = torch.multinomial(p,num_samples=1,replacement=True, ).item()\n",
    "    \n",
    "    out.append(itos[isample])\n",
    "    ix1 = ix2\n",
    "    ix2 = isample\n",
    "    if isample == 0:\n",
    "      break\n",
    "  print(''.join(out))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.6"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "31f2aee4e71d21fbe5cf8b01ff0e069b9275f58929596ceb00d14d90e3e16cd6"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}