Exercise 12 - Building a Neural Network Using Matrices (Algorithm).ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%config InlineBackend.figure_format = 'retina'\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "np.set_printoptions(precision=3)\n",
    "np.set_printoptions(suppress=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = np.array([[ 1.2, 0.7],\n",
    "                 [-0.3,-0.5],\n",
    "                 [ 3.0, 0.1],\n",
    "                 [-0.1,-1.0],\n",
    "                 [-0.0, 1.1],\n",
    "                 [ 2.1,-1.3],\n",
    "                 [ 4.0, 1.0],\n",
    "                 [ 3.1,-1.8],\n",
    "                 [ 1.1,-0.1],\n",
    "                 [ 1.5,-2.2],\n",
    "                 [ 4.0,-1.0],\n",
    "                 [-1.0, 2.0],\n",
    "                 [ 2.0,1.0]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "labels = np.array([  1,\n",
    "                     2,\n",
    "                     1,\n",
    "                     2,\n",
    "                     2,\n",
    "                     1,\n",
    "                     3,\n",
    "                     2,\n",
    "                     1,\n",
    "                     2,\n",
    "                     2,\n",
    "                     3,\n",
    "                     3])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_data(data, labels):\n",
    "    fig = plt.figure(figsize=(5,5))\n",
    "    ax = fig.add_subplot(111)\n",
    "    ax.scatter(data[:,0], data[:,1], c=labels, cmap=plt.cm.Set3, s=50,zorder=50)\n",
    "    nudge = 0.08\n",
    "    for i, d in enumerate(data):\n",
    "        ax.annotate(f'{i}',(d[0]+nudge,d[1]+nudge))\n",
    "    ax.set_aspect('equal', 'datalim')\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAn0AAAJgCAYAAAD/O9WIAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAAWJQAAFiUBSVIk8AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvIxREBQAAIABJREFUeJzs3Xl4lOWh///PPZN9hSSEEINhERC0iolFxDXuol08ovYcwaJitT2K9Nf2SH/9VmmrHnt+tC79Wm31VC20ly0U0NalVkVQFpUEqFi2sgskQFgSsmfm/v1BoAIJZJmZZ2bu9+u6cg2Z57knnxjDfLif57kfY60VAAAA4pvP6wAAAAAIP0ofAACAAyh9AAAADqD0AQAAOIDSBwAA4ABKHwAAgAMofQAAAA6g9AEAADiA0gcAAOAASh8AAIADKH0AAAAOoPQBAAA4gNIHAADgAEofAACAAyh9AAAADkjwOkAkGGM2ScqStNnjKAAAACczQFKNtXZgKF/UidInKSs1NTVn+PDhOV4HAQAAOJHVq1eroaEh5K/rSunbPHz48Jzy8nKvcwAAAJxQaWmpKioqNof6dTmnDwAAwAGUPgAAAAdQ+gAAABxA6QMAAHAApQ8AAMABlD4AAAAHUPoAAAAcQOkDAABwAKUPAADAAZQ+AAAAB1D6AAAAHEDpAwAAcAClDwAAwAGUPgAAAAdQ+gAAABxA6QMAAHAApQ8AAMABlD4AAAAHUPoAAAAcQOkDAABwAKUPAADAAZQ+AAAAB1D6AAAAHEDpAwAAcAClDwAAwAGUPgAAAAdQ+gAAABxA6QMAAHAApQ8AAMABlD4AAAAHUPoAAAAcQOkDAABwAKUPAADAAZQ+AAAAB1D6AAAAHEDpAwAAcAClDwAAwAGUPg/Nnj1b9913ny666CJlZWXJGKPx48e3u29LS4uefPJJ3X777Ro5cqSSkpJkjNHzzz8f4dQAACAWJXgdwGUPP/ywVq5cqYyMDBUVFWnNmjUd7ltXV6cpU6ZIkvr27auCggJt27YtUlEBAECMY6bPQ48//rjWrVunmpoaPfPMMyfcNy0tTa+//rp27NihyspK3XHHHRFKCQAA4gEzfR4qKyvr9L5JSUm69tprw5gGAADEM2b6AAAAHEDpAwAAcAClDwAAwAGUPgAAAAdQ+gAAABxA6QMAAHAApQ8AAMABlD4AAAAHsDhzqNhmKbBLsk2SSZL8fSSTcsIh8+bN07x58yRJlZWVkqQlS5Zo4sSJkqS8vDxNnz79yP6PPfbYkVu1rVixQpL0wgsv6IMPPpAkXXjhhZo0aVJIvy0AABAfKH09FaiSGhdJzR9JtvFzG5Kk5FIp+UIp4ZR2h65YsUIvvfTSUc9t3LhRGzdulCQVFxcfVfrefPNNLViw4Kj9Fy9erMWLFx/5nNIHAADaY6y1XmcIO2NMeUlJSUl5eXnoXtRaqfFtqeF1SSf5b5hSJqV+STIcTQcAACdWWlqqioqKCmttaShfNypaiDEm1xgzyRgz1xjzT2NMgzHmgDHmA2PMncZEYVtqeENqeE0nLXyS1Dhfqp99qCgCAAB4IFoO794k6RlJOyXNl7RVUl9J/ybpeUnXGmNustEyLdm8Wmp8q2tjmhZLCQOl5C+GJxMAAMAJREvpWyfpy5Jes9YGDz9pjPl/JX0k6UYdKoB/8ibeMRrf6/64pHMlY0KZBgAA4KSi4rCptfZda+2fP1/42p6vlPRs26eXRjxYewK7pda13Ry7XQpsCW0eAACAToiK0ncSLW2PrZ6mOKzl056Nb+7heAAAgG6IlsO77TLGJEi6re3TNzuxf0eX554eslDB+p6Nt3WhyQEAANAF0T7T95ikMyW9bq39q9dhJEnG38MX6Ol4AACArovamT5jzGRJ35G0RtKEzozpaD2bthnAkpAE8+V4Ox4AAKAbonKmzxjzn5KelPQPSWXW2r0eR/qXpLMkJXdzsO/QXToAAAAiLOpKnzFmiqT/K2mVDhW+So8jHc0kS8mjujc26WzJlxXaPAAAAJ0QVaXPGPOApMclrdChwrfL40jtSymTTFoXByVKKVeGJQ4AAMDJRE3pM8b8UIcu3CiXdLm1do/HkTrmz5Ey7pKU0skBCVLG7VJCYThTAQAAdCgqLuQwxnxd0o8lBSS9L2myOf6uFZuttS9GOFrHEgdKWZOluj+ceMFlf6GUdtOh/QEAADwSFaVP0uFG5Jc0pYN9Fkh6MSJpOiuhUMr+ttS6TWpaJLVukWzjofP+/KdIyRccut8ut10DAAAei4rSZ62dJmmaxzG6L6G/lPA1r1MAAAB0KGrO6QMAAED4UPoAAAAcQOkDAABwAKUPAADAAZQ+AAAAB1D6AAAAHEDpAwAAcAClDwAAwAGUPgAAAAdQ+gAAABxA6QMAAHAApQ8AAMABlD4AAAAHUPoAAAAcQOkDAABwAKUPAADAAZQ+AAAAB1D6AAAAHEDpAwAAcAClDwAAwAGUPgAAAAdQ+gAAABxA6QMAAHAApQ8AAMABlD4AAAAHUPoAAAAcQOkDAABwAKUPAADAAZQ+AAAAB1D6AAAAHEDpAwAAcAClDwAAwAGUPgAAAAdQ+gAAABxA6QMAAHAApQ8AAMABlD4AAAAHUPoAAAAcQOkDAABwAKUPAADAAZQ+AAAAB1D6AAAAHEDpAwAAcAClDwAAwAGUPgAAAAdQ+gAAABxA6QMAAHAApQ8AAMABlD4AAAAHUPoAAAAcQOkDAABwAKUPAADAAZQ+AAAAB1D6AAAAHEDpAwAAcAClDwAAwAGUPgAAAAdQ+gAAABxA6QMAAHAApQ8AAMABlD7gBGbMmCFjjIwxev75572OAwBAt1H6gA5s27ZN9913nzIyMryOAgBAj1H6gHZYa3X77bcrNzdX99xzj9dxAADoMUof0I6nnnpK7777rl544QWlp6d7HQcAgB6j9AHHWL16taZOnar7779fF198sddxAAAICUof8Dmtra2aMGGCTj31VD366KNexwEAIGQSvA4ARJMf//jHWr58uT744AOlpqZ6HQcAgJBhpg9o89FHH+nRRx/Vd77zHZ1//vlexwEAIKQofYD+dVh36NCh+slPfuJ1HAAAQo7SB0g6ePCg1q1bp9WrVyslJeXIgszGGP3oRz+SJN11110yxmjKlCkepwUAoOs4pw+QlJycrDvvvLPdbRUVFVq+fLkuvPBCDRs2jEO/AICYROkDJKWmpnZ4m7Vp06Zp+fLl+vrXv65JkyZFOBkQfWbPnq0FCxZoxYoVWrlypWpra3Xrrbdq5syZx+27fv16zZkzR3/961+1fv16VVVVqXfv3ho9erSmTJmisrIyD74DIHLef/99PfHEE1q8eLH27t2rnJwcfeELX9CUKVM0duzYiGah9AEAuuThhx/WypUrlZGRoaKiIq1Zs6bDfX/4wx/qD3/4g0aMGKGxY8cqJydHa9eu1auvvqpXX31VTz75pCZPnhzB9EDkPPzww/rhD3+ovLw8XX/99erXr5/27Nmj5cuX67333qP0AaFysKVKB5q2KWCblGCSlZ1crPTEPl7HAmLe448/rqKiIp122mlasGDBCWfrrrnmGj3wwAM655xzjnp+wYIFuvLKK/W9731PN910k/r16xfu2EBEzZo1Sz/84Q91xRVXaM6cOcrMzDxqe0tLS8QzUfoQV6wNqrJ+pTbXLFR149rjtvdJHa7izEvUN+0LMsZ06jWnTZumadOmhTgpELu6ckh24sSJ7T5/ySWX6NJLL9Xf/vY3LV68WDfeeGOI0gHeCwaDeuCBB5SWlqbf//73xxU+SUpMTIx4Lkof4kYg2Kzlu19SZf3yDvfZ3bBauxtWqzD9XJ3dZ4L8JvK/dAAOOfyml5DAWxHiy+LFi7Vp0yaNGzdOvXv31muvvaZVq1YpJSVFo0aN8uyCQH7TEBesDapi929UVf/3Tu2/o26ZrIIq6XOHjGHlIiDStmzZonfeeUdpaWnc4xpx5+OPP5Yk9e3bVyUlJfrkk0+O2n7xxRdr9uzZ6tMnsqcc8W6HuLCl9v1OF77DdtZVaNvBJWFKBKAjTU1NuvXWW9XU1KRp06apd+/eXkcCQmrXrl2SpGeffVYNDQ16++23VVtbq1WrVunqq6/WwoULddNNN0U8F6UPMc9aq00173Vr7OaaBbLWhjYQgA4FAgFNmDBBixYt0i233KLvfve7XkcCQi4QCEg69P40e/ZsXX755crIyNAZZ5yhuXPnqqioSAsWLNCSJZGdeKD0IeZVN65TXUtVt8bWNH+mfU0bQ5wIQHsCgYDGjx+vWbNm6eabb9bMmTM7fUEVEEsOz14PGjRIZ5999lHbUlNTdfXVV0s6dM/3SKL0IeZVN67r4fj1IUoCoCOtra3693//d7388sv6j//4D/3+97/nAg7ErWHDhkmSevXq1e72w6WwoaEhYpkkSh/iQEuwZ780LcH6ECUB0J7m5maNGzdOs2bN0m233aYZM2bI7/d7HQsIm4svvlgJCQlav369mpubj9u+atUqSdKAAQMimovSh5jX02VX/CYpREkAHKupqUk33HCDXnnlFd1555164YUX5PPx1oP4lpeXp1tuuUUHDhzQj3/846O2/e1vf9Nf//pXZWdn65prroloLubWEfMyEgt6OL5viJIAMSh4QArskmyzZFKlhELJpJxwyLx58zRv3jxJUmVlpSRpyZIlRxZizsvL0/Tp0yVJ99xzj15//XXl5eXplFNOOe4NUJIuvfRSXXrppaH7noBQC1RLwWrJtrb9nhRJJ5lw+PnPf64PP/xQjzzyiBYuXKhRo0Zpy5Ytmjt3rvx+v5577rkOD/+GC6UPMa9f+jn6tHqWWm1jl8cm+tJVkDYyDKmAKGat1LpWavxAavlU0uevYE+Wkr8oJV8gJbR/a7QVK1bopZdeOuq5jRs3auPGQxdFFRcXHyl9mzZtkiTt2bOn3cJ3GKUPUccGpJZVh35PWo8599ukS8nnHfo98ee2Ozw/P18ffvihHn74Yc2dO1dLly5VZmamrrvuOn3/+9/X6NGjI/BNHM24sFyFMaa8pKSkpLy83OsoCJNV1X/U5m4s2zIo+wqNyPm30AcCopVtlg7+TmpZeZIdjZT6JSmlTOIKW7gmWCcd/I3UuuEkO/ql9H+Xks8N6ZcvLS1VRUVFhbW2NJSvGxUnVhhjxhljfmGMed8YU2OMscaYmV7nQuw4Lftqpfi7Nk2empCrwdlXhikREIVs4NAb2UkLnyRZqeFVqfHtsMcCooptkmqf6UThk6SAVDdTavo47LFCISpKn6T/I+leSSMlbfc4C2JQSkK2ziu4V8n+rM7t7++t8/req2T/8TfBBuJW4ztSy5qujWl4TWphLUs4pH6eFPisa2PqXpYCe8KTJ4SipfR9W9JQSVmSvulxFsSozKRCXVj4XypML5Xp4H9tI78K07+oCwu/p4wkLuCAQ2zg0LlJ3dH0fmizANEqWNfNWbuA1LQo5HFCLSou5LDWzj/8Z1ZnR0+kJuSoJP9ONbYe0LaDS3Sgaatag41K8KWoV3Kx+meOYXYPbmr5RLI13RvbvPLQVb6+7NBmAqJN00eSWrs59kMp9VopipcBi4rSB4RaSkK2hvSK7PpHQFRrXt2DwUGpZX3IT1YHok5XT3/4PFsvtW6TEgeHLk+IxVXpM8Z0dHnu6RENAgDRxtb1cDx3roED4vz3JFrO6QMAhFMP71wj9XQ8EAN6/HsS3XNp0Z2uizpaz6ZtBrAkwnEAIHr48no2voMFaIG44suVtKn746P894SZPgBwQfKo7o/15UoJp4UuCxCtks/r/tiEQZI/P3RZwoDSBwAu8PeREoZ1b2zyGMnwdgEHJJwm+bq5nFfyBaHNEgb8FgOAK9KuU5fP6vHlScnnhyUOEHWMkdK+LKmLy8f5i6Wk6L+PO6UPAFyRcKqU8XVJ/s7tb3pJmXdLvrSwxgKiStIZUto4dbr4+ftJmZMk08nfKw9FxYUcxpivSvpq26cFbY/nG2NebPvzHmvtdyMeDADiTdIXpMz/lOrnnOBWU0ZKPFNKH8eCzHBTygWH/t+vf1UK7upgJ7+UVCKl/ZvkS41ovO6KitKnQ/fc/foxzw1q+5CkLZIofQAQComDpKzvSIGtUuNiKVApqVkyqVLC4EOHc/05XqcEvJV0ppR4htS6Xmpa2nZv3VbJpEmJw6Tk0ZIvtu7wFBWlz1o7TdI0j2MAgDuMkRKKpYxir5MA0csYKXHooY84wDl9AJzz2Wef6Y477lBhYaGSk5M1YMAATZkyRfv27fM6GgCETVTM9AFApGzYsEFjxozRrl279JWvfEWnn366PvroIz355JN68803tWjRIuXmRvcCqwDQHcz0AXDKt771Le3atUtPPfWU5s2bp8cee0zvvvuuvv3tb2vt2rX6wQ9+4HVEAAgLY631OkPYGWPKS0pKSsrLy72OAsBDGzdu1ODBgzVgwABt2LBBPt+//t1bW1urfv36yVqrXbt2KT093cOkAFxWWlqqioqKio5uL9tdzPQBcMa7774rSbrqqquOKnySlJmZqQsuuED19fVaunSpF/EAIKwofQCcsXbtWknS0KHtX4k3ZMgQSdK6desilgkAIoXSB8AZBw4ckCRlZ7e/4PDh5/fv3x+xTAAQKZQ+AGhz+BxnY7p4300AiAGUPgDOODyTd3jG71g1NTVH7QcA8YTSB8AZw4YNk9TxOXvr16+X1PE5fwAQyyh9AJxRVlYmSXrrrbcUDAaP2lZbW6tFixYpNTVVo0eP9iIeAIQVpQ+AMwYPHqyrrrpKmzdv1tNPP33Utoceekh1dXW67bbbWKMPQFziNmwAYl7QWhl17gKMX/7ylxozZowmT56sd955R8OHD9eHH36o+fPna+jQoXrkkUfCHxgAPEDpAxBzgtZqU+0+rdhbpR31tWoJBuU3RrnJaTorJ1+nZ+cpye9vd+zgwYO1bNkyPfjgg3rzzTf1+uuvq1+/fpo8ebIeeugh5eTkRPi7AYDIoPQBiCk76mv1+rZ/qqal6ajnA9ZqV2Od3t6xSQsrt+rSfsU6s3d+u6/Rv39/vfDCC5GICwBRg9IHIGZsPXhAc7esUeAk9wxvDgb01vaNamht1Rf7FEYoHQBENy7kABATDjQ36tWt605a+D7v/aqt2lCzN4ypACB2UPoAxITl1ZVqDga6PG7p7u1hSAMAsYfSByDqtQQD+nTf7m6NrWqoU2XDwRAnAoDYQ+kDEPU21x5QUzdm+Q5bs39PCNMAQGyi9AGIegdbm3s0vq61JURJACB2UfoARD2rzl+80e74Llz8AQDxitIHIOqlJyT2aHxaD8cDQDyg9AGIegMyeinR1/2/roZm54YwDQDEJkofgKiX7E/Q8Oy8bo3NTU7VKWmZIU4EALGH0gcgJpyT209+Y7o87ot9CmW6MQ4A4g2lD0BMyE1J1bVFp6kr9a00r59G9OoTtkwAEEu49y6AmDE0O1cJPp/e+Oyfagp0vG6fT0Zj+hbpi3ncdxcADqP0AYgpgzJ76xvDSrR6/x6t3Ful3Y31R7ZlJCTprJx8ndk7XxmJSR6mBIDoQ+kDEHMSfX6dldNXZ+X0VUswoKZAQIk+n5J8fs7fA4AOUPoAxLREn1+JPr/XMQAg6nEhBwAAgAMofQAAAA6g9AEAADiA0gcAAOAASh8AAIADKH0AAAAOoPQBAAA4gNIHAADgAEofAACAAyh9AAAADqD0AQAAOIDSBwAA4ABKHwAAgAMofQAAAA6g9AEAADiA0gcAAOAASh8AAIADKH0AAAAOoPQBAAA4gNIHAADgAEofAACAAyh9AAAADqD0AQAAOIDSBwAA4ABKHwAAgAMofQAAAA6g9AEAADiA0gcAAOAASh8AAIADKH0AAMSw6upqPf/887rhhht02mmnKTU1VdnZ2brwwgv1v//7vwoGg15HRJRI8DoAAADovlmzZumb3/ym+vXrp7KyMp166qmqqqrSnDlzNGnSJL3xxhuaNWuWjDFeR4XHKH0AAMSwoUOH6tVXX9V1110nn+9fB/AeffRRjRo1Sn/60580Z84c3XjjjR6mRDTg8C4AADHssssu05e+9KWjCp8kFRQU6J577pEkvffeex4kQ7Sh9AEAEKcSExMlSQkJHNgDpQ8AgLjU2tqq3/72t5Kka665xuM0iAaUPgAA4tDUqVO1atUqjR07VldffbXXcRAFKH0AAMSZp556Sj/72c90+umna8aMGV7HQZSg9AEAEEeefvpp3X///RoxYoTmz5+vnJwcryMhSlD6AACIE0888YTuvfdenXnmmZo/f74KCgq8joQoQukDACAO/PSnP9W3v/1tjRw5UvPnz1d+fr7XkRBlKH0AAMS4n/zkJ5o6dapKS0v1zjvvKC8vz+tIiEIs3AMAQAx76aWX9OCDD8rv9+uiiy7SU089ddw+AwYM0MSJEyMfDlGF0gcAQJSw1mpbXY32NTWo1Vol+/3qn56l7KSUDsds2rRJkhQIBPTEE0+0u88ll1xC6QOlDwAArzUHAvr7viqt3FulA81Nx20fkNFL5+QWaGBmr+O2TZs2TdOmTYtASsQ6Sh8AAB6qbW7SnC1rVN3U0OE+mw/u1+aD+1WSW6BLCopljIlgQsQLLuQAAMAjDa0tmr159QkL3+dVVFdqYdXWMKdCvKL0AQDgkYWVW7WvubFLY8r37NS2upowJUI8o/QBAOCBhtYWrTmwp1tjV1ZXhjgNXEDpAwDAA5/u362Atd0a+8+afTrY0hziRIh3lD4AADywva6222ODsqpsOBjCNHABpQ9A3Hjttdd01VVXqaioSKmpqRo0aJBuuukmLVmyxOtowHGag4EejW8K9Gw83EPpAxAXHnjgAV1//fWqqKjQNddco/vvv18lJSV65ZVXdMEFF2jmzJleRwSOkuDr2VtwYg/Hwz2s0wcg5lVWVmr69Onq27ev/v73vx91o/n58+frsssu04MPPqjx48d7mBI4Wl5ymjbV7u/2+NzktBCmgQv4ZwKAmLdlyxYFg0Gdd955RxU+SSorK1NmZqZ2797tUTqgfV/onX/ynTpwSlqmclNSQ5gGLqD0AYh5Q4YMUVJSkj766CPt2XP0EhgLFy5UbW2trrjiCo/SAe3rlZyiARnZ3Rp7dk7fEKeBC6Km9BljiowxvzHG7DDGNBljNhtjnjDG9PY6G4DolpOTo5/+9KeqqqrSiBEj9I1vfEPf//73dfPNN+uqq67SlVdeqV/96ldexwSOc3FBsZJ8/i6NKc7I1tDs3DAlQjyLinP6jDGDJS2WlC/pFUlrJI2SdL+ka4wxF1hrqz2MCCDKTZkyRQMGDNAdd9yh55577sjzp512miZOnHjcYV8gGuSlpOmrxcM0b8vaTl3NW5SWqev7D5GPe++iG6Jlpu+XOlT4Jltrv2qtnWqtvUzS45KGSXrE03QAot7//M//aNy4cZo4caI2bNiguro6lZeXa9CgQbr11lv1X//1X15HBNpVlJ6l/xh8poZk5aijKpfmT9ToPqfo3wYMV7I/KuZrEIOM7eZq4CELYMwgSRskbZY02Fob/Ny2TEk7JRlJ+dbaum5+jfKSkpKS8vLyECQGEG3ee+89lZWV6YYbbtCcOXOO2lZfX6+hQ4dq586dWr9+vQYNGuRRSuDkalua9Y99u7WvuUGtwaCS/Qnqn56lIVk58rNEizNKS0tVUVFRYa0tDeXrRsM/Fy5re3zr84VPkqy1tcaYRZKukjRa0juRDgcg+v3lL3+RdOhK3WOlpaVp1KhRmjt3rpYvX07pQ1TLTEzSefmneB0DcSoaSt+wtsd1HWxfr0Olb6hOUvqMMR1N5Z3evWgAYkFTU5Mkdbgsy+Hnk5KSIpYJAKJNNMwVH75e/UAH2w8/3ysCWQDEoIsuukiS9Otf/1rbt28/atsbb7yhRYsWKSUlRWPGjPEiHgBEhWiY6TuZw+e1nvTkw46OfbfNAJaEMhSA8GkOBLS3qUHNwYASfX7lJKec8OT1cePG6YorrtDbb7+t4cOH64YbblBBQYFWr16tv/zlL7LW6rHHHlNuLstcAHBXNJS+wzN5Ha1QmXXMfgDi1O7Geq3cW6nV+/eoJfivU3wTjE+n98rV2TkF6puaftw4n8+n119/XU8//bRefvllzZ07V/X19crJydHYsWM1efJkXXXVVZH8VgAg6kRD6Vvb9ji0g+1D2h47OucPQIyz1mrxrs/04e7t7W5vtUGt2rdbq/btVmluP11ccKrMMeuUJSYmasqUKZoyZUokIgNAzImGc/rmtz1eZYw5Kk/bki0XSGqQtDTSwQBExsLKrR0WvmOVV+/UOzs2yevlpgAg1nhe+qy1GyS9JWmApP88ZvOPJKVL+m131+gDEN3+WbNX5dU7uzTm7/t2ac0BbtIDAF0RDYd3JelbOnQbtqeMMZdLWi3pPEllOnRY9wceZgMQRhV7ulb4joyr3qnhvfJCnAYA4pfnM33Skdm+cyW9qENl7zuSBkt6StL53HcXiE97Guv1WX1tt8ZWNdSpsv5giBMBQPyKlpk+WWu3Sbrd6xwAImdT7f4ejd9Yu08FaRkhSgMA8S0qZvoAuKkx0OrpeABwCaUPgGd8xyy70lV+w19hANBZ/I0JwDPZSck9Gp/Vw/EA4BJKHwDPDMnKUaKve38N+Y3R6dncVg0AOovSB8Azyf4EjejVp1tjh2XnKjUhMcSJACB+UfoAeOrcvH5K8XdtIYEkn1+j+pwSpkQAEJ8ofQA8lZ2Uoq8WD1OSz9+p/RONT18+dahyklPDnAwA4gulD4DnCtMy9bVBZ6jwJGvu9U1N182DztCpGdkRSgYA8SNqFmcG4La8lDR9bdCZ2tVQp5V7q1TVcFDNwaASfT7lp6Tr7Jy+LMQMAD1A6QMQVfJT03XlKYO8jgEAcYfDuwAAAA6g9AEAADiA0gcAAOAASh8AAIADKH0AAAAOoPQBAAA4gNIHAADgAEofAACAAyh9AAAADqD0AQAAOIDSh6gze/Zs3XfffbrooouUlZUlY4zGjx/vdSwAAGIa995F1Hn44Ye1cuVKZWRkqKioSGvWrPE6EgAAMY+ZPkSdxx9/XOvWrVNNTY2eeeZ+gWFjAAAfEklEQVQZr+MAABAXmOlD1CkrK/M6AgAAcYeZPgAAAAdQ+gAAABxA6QMAAHAApQ8AAMABlD4AAAAHUPoAAAAcQOkDAABwAKUPAADAASzOjKgzb948zZs3T5JUWVkpSVqyZIkmTpwoScrLy9P06dO9igcAQEyi9CFsgrZVO+uW67ODS1XfUq2gWpXkS1d+2pk6NfMCpSbktDtuxYoVeumll456buPGjdq4caMkqbi4mNIHAEAXGWut1xnCzhhTXlJSUlJeXu51FGdsrV2kNXtfVXOwtoM9jArTS/SF3H9Xoj8totkAAIhmpaWlqqioqLDWlobydZnpQ8it3fdnrd//xkn2stpRV67a5h0a3W+Kkv2ZEckGAICruJADIbW1dlEnCt+/1Lbs1MdVzypoA2FMBQAAKH0ImaANaO2+v3R53P6mTaqq/yQMiQAAwGGUPoRMZf1KNQUOdGvslpqFIU4DAAA+j9KHkPmsdmm3x+5pXKOG1r0hTAMAAD6P0oeQaWit7uH4fSFKAgAAjkXpQ8j09GKMoG0NURIAAHAsSh9CJsmf7ul4AADQMUofQiY/7QvdHpuakKvMxMIQpgEAAJ9H6UPInJo5Rkb+bo0tzrxIxvC/IwAA4cK7LEIm2Z+looxRXR6XYFJ0auaYMCQCAACHUfoQUmfk3qTspP6d3t/Ip9L8SUryZ4QxFQAAoPQhpBJ8KRpdcL9yU4adfF+TqlEF96pP2ogIJAMAwG0JXgdA/En0p2l0wX3a3bBaW2oXqqp+lSR7ZHtaQh8VZ12k/hmjmeEDACBCKH0IC2N8yk87Q/lpZ6gpUKuG1r0K2lYl+dKVnpjPRRsAAEQYpQ9hl+zPVLI/0+sYAAA4jekWAAAAB1D6AAAAHEDpAwAAcAClDwAAwAGUPgAAAAdQ+gAAABxA6QMAAHAApQ8AAMABlD4AAAAHUPoAAAAcQOkDAABwAKUPAADAAZQ+AAAAB1D6AAAAHEDpAwAAcAClDwAAwAGUPgAAAAdQ+gAAABxA6QMAAHAApQ9OeOCBB3T55Zerf//+Sk1NVU5Ojs455xz96Ec/UnV1tdfxAABxZvbs2brvvvt00UUXKSsrS8YYjR8//oRjFi9erLFjx2rlypWSdI4x5u/GmCnGGH8oMhlrbSheJ6oZY8pLSkpKysvLvY4CjyQlJamkpEQjRoxQfn6+6urqtHTpUi1btkyFhYVaunSp+vfv73VMAECcGDlypFauXKmMjAwVFRVpzZo1uvXWWzVz5sx293/llVd04403KiUlRSkpKaqurq6StF/SMEmzrbU39TRTQk9fAIgFNTU1SklJOe75H/zgB3r00Uf13//93/rlL3/pQTIAQDx6/PHHVVRUpNNOO00LFixQWVlZh/vW1NTorrvukt/v13vvvae7775b1dXV2yVdIOldSeOMMV+z1r7ck0wc3oUT2it8knTzzTdLktavXx/JOACAOFdWVqYhQ4bIGHPSfWfPnq3du3fra1/7ms4999wjz1trGyX9n7ZPv9nTTJQ+OO3Pf/6zJOmss87yOAkAwFXvvvuuJOmaa65pb/NCSfWSxhhjknvydTi8C6dMnz5dBw8e1IEDB7Rs2TJ98MEHOuusszR16lSvowEAHLV27VpJ0tChQ4/bZq1tNcZsknSGpEGSVnf361D64JTp06erqqrqyOfXXHONXnzxRfXp08fDVAAAlx04cECSlJ2d3eEubY+9evJ1OLwLp1RWVspaq8rKSs2ZM0cbN27UOeeco4qKCq+jAQDQkcMnBvZoyRVKH5zUt29f3XDDDXrrrbdUXV2t2267zetIAABHHZ7hOzzj146stscOd+gMSh+cVlxcrBEjRujTTz/Vnj17vI4DAHDQsGHDJEnr1q07bpsxJkHSQEmtkjb25OtQ+uC8HTt2SJL8/pAseA4AQJdcdtllkqQ333yzvc0XS0qTtNha29STr0PpQ9xbs2aNKisrj3s+GAzqBz/4gXbt2qUxY8aod+/eHqQDALhu3LhxysvL08svv6xly5Yded4YkyLp4bZPn+np1+HqXcS9N998U9/73vd08cUXa/DgwcrNzVVVVZUWLFigjRs3qqCgQM8995zXMQEAcWTevHmaN2+eJB2ZeFiyZIkmTpwoScrLy9P06dMlSVlZWXruuec0btw4XXrppYdvKHCKpBVquw2bpD/0NBP33kXMOdC0VTXNOxSwzUr0pap38iClJeZ2uP+qVav0zDPPaNGiRfrss8+0f/9+paena+jQobruuus0efJk5eTkRPA7AADECmuD2tu0UfUtuxW0LUr0pSk3daiS/VknHDdt2jT96Ec/6nB7cXGxNm/efNRzixYt0iOPPKK33npLgUDASvpU0m8kPWWtDfT0e6H0ISYEbau2H/xYm2sW6EDz1mO2GuWnnqEBWZeoT+qITt3yBgCAE2kNNmpr7WJtqV2oupZdR20z8qtf+jkamFWm3ikDQ/61S0tLVVFRUWGtLQ3l63J4F1GvOVCvZbue1d7Gf3awh9WuhlXa1bBKxZkX6Yzcm+UzXJQBAOie+pZqfVT1tA62HH8+uCRZBbSjbpl21C3T6b2/osHZV8XEhIPnF3IYYxKNMfcbY14wxqwwxjQbY6wxZpLX2eC9QLBZH1U9fYLCd7Qtte/r0+o/yoUZbABA6DUFarW08skOC9+x1ux7RRsPvB3mVKHheemTlC7pCUkTJRVI6tx/ZThh/f43tL9pU5fGbKl9X1X1n4QpEQAgnn1a/UfVt3Zt3dbV++aqpnl7mBKFTjSUvnpJYyUVWmsLdOiERUCBYIu21i7q1tjNNe+FNgwAIO41tu7Xzrrl3Rq7pWZhiNOEnuelz1rbbK19w1q70+ssiC476yvUHDzYrbF7Gtfo4DEn3gIAcCJbaxfJKtitsZ8d/FAtwYYQJwotz0sf0JHqhs6dx9eRvY3rQ5QEAOCC6h68bwRssw40Hbu6RHSJq6t3jTEdrclyekSDICRabc/+xRTt/+ICAESX1h6+b/R0fLgx04eo5TeJPRqfYJJClAQA4AJ/D983ejo+3EIy02eM2SypuAtDfmetHR+Kr/15HS1i2DYDWBLqr4fwykgs8HQ8AMAtGUkF2tu0odvj0xP7hjBN6IXq8O4GSY1d2H9HiL4u4lhRxmit3fcXWXX9zjPpiX2VkzIkDKkAAPHq1MwLu71qRJ/UESe8JWg0CEnps9ZeHorXAT4vJSFb/dLP0Y66ZV0eW5x5UUysjg4AiB69kouVnVSsA81bujx2QNbFYUgUWpzTh6g2rPf1SvSld2lMVlJ/nZp5QZgSAQDi2Rm542TUtVt55qeeofzUM8OUKHQofYhq6Yn5GtX3W0r0pXVq/8zEQo3q+y0l+JLDnAwAEI9yUgarNH+SfJ08GJqbMlQl+XfKmOivVFGxZIsxZqr+tazKyLbH240xF7b9+QNr7fORT4Zo0DtloC4o/C+t2TtXVfWftLtwpt8kqyjjPJ2e8xUl+lI9SAkAiBcF6Wfr/ML/R2v3vqo9jWva3SfJl6HirIs0pNe18pmoqFMnFS0pr5F0yTHPjWn7OIzS57CMxHyd2/duNbTu07baJapt2aFAsEkJvlTlpAzWKRmjKHsAgJDpnTxAo/tN1sHmKm07uFh1LXsUtC1K9KepT+pw9Usv6fHSYpEWFaXPWnup1xkQG1ITemto77FexwAAOCIjqa+G59zgdYyQiP4D0ACAmDRgwAAZY9r9KChgHU0g0qJipg8AEJ+ys7M1ZcqU457PyMjwIA3gNkofACBsevXqpWnTpnkdA4A4vAsAAOAEZvoAAGHT1NSkmTNnauvWrUpPT9dZZ52liy++WH5/1xa/BdBzlD4AQNhUVlZqwoQJRz03cOBAvfDCC7rkkmNX6gIQThzeBQCExe2336533nlHlZWVqqur0yeffKK7775bmzdv1rXXXquVK1d6HRFwCjN9AICweOihh476/Mwzz9Szzz6rjIwM/exnP9O0adM0d+5cj9IB7mGmDwAQUffcc48kaeHChR4nAdxC6QMARFR+fr4kqa6uzuMkgFsofQCAiFqyZIkkadCgQR4nAdxC6QMAhNynn36qvXv3Hvf8li1bdO+990qSxo8fH+lYgNO4kAMAcFLWWjUGWtUcDCjJ51eKP0HGmA73nzVrlh577DGVlZVp4MCByszM1IYNG/Taa6+psbFRY8eO1Xe/+90IfgcAKH0AgA7VtTZr1d7d+vu+KtW2NB95PjspWWfn9NUZvfooNSHxuHFlZWVau3atli9friVLlqiurk69evXShRdeqAkTJmjChAknLI0AQo/SBwBo1/LqSi3YuUVB2eO2HWhu0sLKrVpc9ZkuLxyoM3r3OWr7JZdcwuLLQJSh9AEAjvPhru1atGvbSfdrtUH9dfsGtQQDGplbEIFkALqLCzkAAEfZWLuvU4Xv8+bv3KztdbXhCQQgJCh9AICjfLx7R5fHWEnL9nR9HIDIofQBAI7Y01iv7fXdm7HbWLtPtc1NIU4EIFQofQCAI9bXHL+2XmdZSf+s3Re6MABCitIHADiivrXF0/EAwofSBwA4wqhna+ex8h4QvSh9AIAjMhKPX2i5a+OTQpQEQKhR+gAARwzLzu32WL8xGpKVE8I0AEKJ0gcAOCI7KUWDMnt1a+yw7Nx2b8kGIDpQ+gAARxnV5xT5unh2XoLx6dy8wjAlAhAKlD4AwFEK0zJ1VdGgTtc+n4yu63+a8lLSwpoLQM9w710AwHFG9OqjZF+C/rZj4wmXYclMTNLVpwzWqRnZEUwHoDsofQCAdg3O6q0BGefon7V7tXLvLu1qqFNLMKAkn18FaRk6O6evBmX2ls+wUAsQCyh9AIAO+X0+DcvO07DsPK+jAOghzukDAABwAKUPAADAAZQ+AAAAB1D6AAAAHEDpAwAAcAClDwAAwAGUPgAAAAdQ+gAAABxA6QMAAHAApQ8AAMABlD4AAAAHUPoAAAAcQOkDAABwAKUPAADAAZQ+AAAAB1D6AAAAHEDpAwAAcAClDwAAwAGUPgAAAAdQ+gAAABxA6QMAwAEvvviijDEn/PD7/V7HRBgleB0AAACE38iRI/XQQw+1u+3999/Xu+++q2uvvTbCqRBJlD4AABwwcuRIjRw5st1t559/viTpG9/4RiQjIcI4vAsAgMNWrVqlpUuX6pRTTtF1113ndRyEEaUPAACH/epXv5Ik3XnnnZzTF+cofQAAOKqhoUEzZ86Uz+fTpEmTvI6DMKP0AQDgqD/+8Y/av3+/rr32WvXv39/rOAgzSh8AAI769a9/LUm6++67PU6CSKD0AQDgoH/84x9avHixioqKNHbsWK/jIAIofQAAOIgLONxD6QMAwDGNjY2aMWOGfD6f7rzzTq/jIEIofQAAOGbWrFnat2+fxo4dywUcDqH0AQDgmMMXcHAHDrdwGzYAAGJMc6BOnx38ULXN2xWwzUrwpah38mAVppfI70s64djVq1frgw8+4AIOB1H6AACIEQ2t+7R235+1o65cQdty1LattYv0j72zdWrmBRrS61ol+FLafY3hw4fLWhuJuIgylD4AAGJATfN2fVj5f9UUONDhPi3Bem048DftaVijUQX3KtmfGcGEiHac0wcAQJRraN130sL3eQeat+njqmcUCDaHORliCaUPAIAot27fXzpd+A7b37RZW2sXhSkRYhGlDwCAKNYcqNf2umXdGruldiHn7+EISh8AAFFs+8EPj7too7MOtlRpb+P6ECdCrKL0AQAQxWqat/dsfMuOECVBrKP0AQAQxQK2ZxdjcDEHDqP0AQAQxTpab6+zEns4HvGD0gcAQBTLSRnco/G9ezge8YPSBwBAFOuXVqJEX3q3xuYkD1ZW0ikhToRYRekDACCK+X2JOjXzgm6NLc66JMRpEMsofQAARLkhva5VdtKpXRpTmP5FFaaXhCkRYhGlDwCAKJfgS9Z5BfeqV/LATu1fmP5Fnd1nvIzhbR7/kuB1AAAAcHJJ/gydXzBFW2s/0Jba93WwpfK4fXKSB6s46xIVppdQ+HAcSh8AADHC70vUwOwyDci6VHubNqi2ebtag01K9KWqd8ogLtrACVH6AACIMcYY5aacptyU07yOghjC3C8AAIADKH0AAAAOoPQBAAA4gNIHAADgAEofAACAAzwvfcaYIcaYB4wx7xpjthljmo0xVcaYV4wxZV7nAwAAiAfRsGTLTyTdIukfkl6XtFfSMElflvRlY8z91tqnPMwHAAAQ86Kh9L0p6afW2uWff9IYc4mkv0n6/4wxs6y1Oz1JBwAAEAc8P7xrrX3x2MLX9vwCSe9JSpI0JtK5AAAA4onnpe8kWtoeWz1NAQAAEOOitvQZY4olXS6pXtLCTo4pb+9D0unhzAoAJ2Ot1W9+8xuNHj1amZmZSktL0znnnKOnnnpKgUDA63gAHBCVpc8Ykyzpd5KSJU2z1u7zOBIA9MjXv/513Xnnndq0aZNuueUW3XXXXWpubtb999+vW265RdZaryMCiHMhuZDDGLNZUnEXhvzOWju+g9fyS5oh6QJJf5A0vbMvaq0t7eA1yyWVdCEfAITMvHnzNGPGDA0cOFAfffSR8vLyJEktLS26+eab9ac//UkvvfSSJk6c6G1QAHEtVDN9GySt7cLHjvZepK3wzZR0k6Q/Shpv+ecvgBg3Z84cSdJ3vvOdI4VPkhITE/WTn/xEkvSLX/zCk2wA3BGSmT5r7eU9fQ1jTIKk3+tQ4fu9pNustZzoAiDmVVZWSpIGDRp03LbDz1VUVGj//v3q1atXRLMBcEdUnNNnjEmSNFuHCt9vJU2g8AGIF4dn9zZt2nTcto0bNx7585o1ayKWCYB7PC99bRdtzJX0FUn/K+l2a23Q21QAEDrXX3+9JOnnP/+59u7de+T51tZWPfTQQ0c+37ePa9YAhE803JHjWUljJe2RtF3Sg8aYY/d5z1r7XoRzAUBIfO1rX9PMmTP1xhtvaMSIEfryl7+stLQ0vf3229qwYYOGDBmi9evXy+/3ex0VQByLhtI3sO0xT9KDJ9jvvfBHAYDQ8/l8evXVV/Xkk09qxowZmjFjhhITEzVmzBi99NJLuvfee7V+/Xrl5+d7HRVAHDMuXBxrjCkvKSkpKS8v9zoKAByloaFBOTk5MsbowIEDSkxM9DoSAI+VlpaqoqKioqOl6LrL83P6AMBlM2bMUGNjo26++WYKH4CwovQBQATU1NQc99zHH3+sqVOnKiMjQw8+eKKzWwCg56LhnD4AiCnWBlXbvENNwVoZ+ZSS0FsZiSc+H+/KK69UamqqzjzzTGVmZurTTz/V66+/ruTkZM2ZM6fdNfwAIJQofQDQSc2Bg9p2cIm21Hyg+tbdR23rlTxAxZkXqzC9VH7f8Ydpx40bp5dfflkzZ85UQ0ODCgsLNWnSJE2dOlUDBgyI0HcAwGVcyAEAnbC38Z/6uOpXagnWnXC/jMR+GtX3W0pLzI1QMgDxhgs5AMAj+xo3amnlUyctfJJ0sGWnFu/8uRpb90cgGQB0HqUPAE4gEGzWsqpfKWhbOz2mMbBPy3e/GL5QANANlD4AOIHtdcvUFKzt8rjqxnWqafosDIkAoHsofQBwAltqFnZ77Oba7o8FgFCj9AFAB5oCtTrQvLXb43fX/yOEaQCgZyh9ANCBlkB9j8Y3d+LCDwCIFEofAHTAZ/w9HM9SqACiB6UPADqQ7M+Sz3T/frhpCazVByB6UPoAoAN+X5JOST+32+P7Z44JYRoA6BlKHwCcQHHWJd0a5zfJKsoYFeI0ANB9lD4AOIFeyaeqML3rd0Ia2musEnwpYUgEAN1D6QOAkzg7b4LyUk7v9P4Ds8o0KPuKMCYCgK6j9AHASfh9SRpV8C0NyrpcfpPU4X7J/iydkXuzRuSMkzEmggkB4ORYTwAAOsFnEjQi90YN6T1Wn9UuVWX9SjUHDsoYn1L8vVSUMUoF6SNZpgVA1OJvJwDogkRfqgZml2lgdpnXUQCgSzi8CwAA4ABKHwAAgAMofQAAAA6g9AEAADiA0gcAAOAASh8AAIADKH0AAAAOoPQBAAA4gNIHAADgAEofAACAAyh9AAAADqD0AQAAOIDSBwAA4ABKHwAAgAMofQAAAA6g9AEAADiA0gcAAOAASh8AAIADKH0AAAAOoPQBAAA4gNIHAADgAEofAACAAyh9AAAADqD0AQAAOIDSBwAA4ABKHwAAgAMofQAAAA6g9AEAADiA0gcAAOAASh8AAIADKH0AAAAOoPQBAAA4gNIHAADgAEofAACAAyh9AAAADqD0AQAAOIDSBwAA4ABKHwAAgAMofQAAAA6g9AEAADiA0gcAAOAASh8AAIADKH0AAAAOoPQBAAA4gNIHAADgAEofAACAAyh9AAAADqD0AQAAOIDSBwAA4ABKHwAAgAOMtdbrDGFnjKlOTU3NGT58uNdRAAAATmj16tVqaGjYa63NDeXrulL6NknKkrQ5Al/u9LbHNRH4Wjg5fh7Rh59J9OFnEn34mUSfSP5MBkiqsdYODOWLOlH6IskYUy5J1tpSr7OAn0c04mcSffiZRB9+JtEnHn4mnNMHAADgAEofAACAAyh9AAAADqD0AQAAOIDSBwAA4ACu3gUAAHAAM30AAAAOoPQBAAA4gNIHAADgAEofAACAAyh9AAAADqD0AQAAOIDSBwAA4ABKXxgZYxKNMfcbY14wxqwwxjQbY6wxZpLX2eKdMabIGPMbY8wOY0yTMWazMeYJY0xvr7O5yBgzzhjzC2PM+8aYmrbfg5le53KVMSbXGDPJGDPXGPNPY0yDMeaAMeYDY8ydxhjeGzxijPmpMeYdY8y2tp/LXmPMcmPMQ8aYXK/zQTLGTGj7Oyzm3s9ZnDmMjDG9JO1r+7RKUrOk/pLustY+71mwOGeMGSxpsaR8Sa9IWiNplKQySWslXWCtrfYuoXuMMSsknS3poKTPJJ0u6XfW2vGeBnOUMeYeSc9I2ilpvqStkvpK+jdJ2ZL+JOkmyxtExBljmiVVSPqHpF2S0iWNlnSupB2SRltrt3mX0G3GmP6SPpHkl5ShGHs/519z4VUvaaykQmttgaTfeJzHFb/UocI32Vr7VWvtVGvtZZIelzRM0iOepnPTtyUNlZQl6ZseZ4G0TtKXJRVZa2+11n7fWnuHDpXxbZJu1KECiMjLstaOttbe0fZ3133W2i9KelRSoaTve5zPWcYYI+kFSdWSnvU4TrdQ+sLIWttsrX3DWrvT6yyuMMYMknSVpM2Snj5m80OS6iRNMMakRzia06y1862165k5ig7W2nettX+21gaPeb5S/3ozuzTiwSBrbWMHm/7Y9jgkUllwnMmSLpN0uw69l8QcSh/izWVtj2+184ZWK2mRpDQdOlwC4HgtbY+tnqbAsb7U9vh3T1M4yhgzXNJjkp601i70Ok93JXgdAAixYW2P6zrYvl6HZgKHSvr/27t7FjmrMA7j1x1TqmuRYrEzhVgIrkGU6ILFYtgyXyBGC5sEQsAFRYkYbQJi4VrFIKzmA6QLiSiiBk3jSxEiBgKm8TW+FRItwm1xzuAQZ9jE3Z2zs+f6wcOZmWcX7mJn5z/nOed+PphIRdKUiIjtwJP16ZmWtfQuIpYoa8ZmKOv55imB71jLunpU3xcnKWtfX2hczpoY+rTVzNTxjzHnB6/fNYFapGlzDLgfOJ2ZZ1sX07klyuaagTPAU5n5c6N6evYS8CAwn5nXWhezFl7eXUVt9ZG3cNiGYnOLOrq2TBoSEYeAZym73fc1Lqd7mTmbmQHMUjbV7AS+jIhdbSvrS0Q8TJndez0zP2tdz1o507e6y8C4hbWjfLdRheimDGbyZsacv/OGn5O6FxEHgTcobUIWMvPXxiWpyswfgVMR8QVl2cq7lNlYbbChy7qXgCONy1kXhr5VZOZC6xp0S76p471jzg92vo1b8yd1JSIOU9oZXaAEvp8al6QRMvNKRFwE5iJiR2ZebV1TB27n38+Sv0rHlv84EREnKBs8Dk+ssv/J0Ket5sM67omIbcM7eCPiDuAx4BpwvkVx0mYSEc9R1vF9BTxhkNj07q7j9aZV9ONv4O0x53ZR1vmdo0w2TMWlX0OftpTMvBwR71F26B4E3hw6fZTS3f54Zk5ljyVpvUTEEeAV4HNgj5d024uI+4Dfa7/E4de3Aa9Sms5/mpm/jfp9ra+6aWPkbdYi4mVK6Htnmu7IYejbYBHxPKXLPcBcHZ+OiPn6+Nw0/cFMiQOU27AtR8QC8DXwCOU2bJeAFxvW1qWI2AvsrU9n67g7Ilbq46uZuTTxwjoVEfspge868AlwaMSlq28zc2XCpfVuEXgtIj6mrCf/hbKD93HKRo4fgGfaladpZ+jbeIuUN+ywR+sxYOhbR3W27yHKh9oi5VZ43wPLwFFnNJqYA/bf8NrOegBcobSo0GTcU8fbgHHrkD4CViZSjQbeB96iLEN5gNJa6k/Kl9WTwLL/v7QW4V2RJEmStj779EmSJHXA0CdJktQBQ58kSVIHDH2SJEkdMPRJkiR1wNAnSZLUAUOfJElSBwx9kiRJHTD0SZIkdcDQJ0mS1AFDnyRJUgcMfZIkSR0w9EmSJHXA0CdJktQBQ58kSVIHDH2SJEkdMPRJkiR14B9vl5G0NqM7VQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 360x360 with 1 Axes>"
      ]
     },
     "metadata": {
      "image/png": {
       "height": 304,
       "width": 318
      },
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_data(data, labels)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Task 1: Data normalisation and label conversion"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Compute the mean (average) value of all the values in the data set:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Compute the standard deviation (average distance from the mean) of all the values in the data set:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Compute the mean (average) value of X coordinates, and of Y coordinates:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Compute the standard deviation of X coordinates, and of Y coordinates:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Normalise the data by subtracting the respective mean from each data point's components and dividing each data point's coordinate by its respective standard deviation value:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Plot the normalised data set and compare it with the original one. What has changed?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Implement a function `convert_label(x)`, which converts a given label `x` into a vector. The vector has as many components as there are labels. Each of vector's components maps to one of the existing labels and encodes the probability that the label `x` is of its kind. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Test the previously defined function on the number of vectors:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Interpret the result in terms of the mapping of the input to the vectorial output:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Implement the sigmoid function that takes vector input:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Task 2: More useful numpy functions for manipulating data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "`xvec` is a given 5-dimensional vector:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "xvec = sigmoid(np.random.randn(5,1))\n",
    "xvec"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Transform this vector so that each of its components gets rounded to eighter 0 or 1, depending to which one it is closer:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "`xvec2` is a given 10-dimensional row vector:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "xvec2 = np.random.randint(10, size=10)\n",
    "print (xvec2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Return the index of the maximum value within this vector:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Task 3:"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Here we have our neural network equations in vector form:"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Forward pass:"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$\n",
    "\\mathbf{z}^{(l)}=\\mathbf{w}^{(l)}\\cdot \\mathbf{a}^{(l-1)}+\\mathbf{b}^{(l)}\\\\\n",
    "\\mathbf{a}^{(l)}\n",
    "=\n",
    "\\sigma(\\mathbf{z}^{(l)})\n",
    "\\\n",
    "$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Cost:"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$\n",
    "\\mathbf{TC} =\\|\\mathbf{a}^{(l_{final})}-\\mathbf{y} \\|^2\n",
    "$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Backward pass:"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$\n",
    "\\begin{align*}\n",
    "\\delta^{(l_{final})}&=2\\left(\\mathbf{a}^{(l_{final})}-\\mathbf{y}\\right)\\odot \\mathbf{a}^{(l_{final})} \\odot \\left(\\mathbf{1}-\\mathbf{a}^{(l_{final})}\\right) \\\\\n",
    "\\delta^{(l)}&=\\mathbf{a}^{(l)}\\left(1-\\mathbf{a}^{(l)}\\right)\\odot \\mathbf{w^T}^{(l+1)} \\cdot \\mathbf{\\delta}^{(l+1)} \\\\ \\partial \\mathbf{b}^{(l)}&=\\delta^{(l)} \\\\\n",
    "\\partial \\mathbf{W}^{(l)}&=\n",
    "\\delta^{(l)}\\cdot\\left(a^{(l-1)}\\right)^T\n",
    "\\end{align*}\n",
    "$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Implement a 2x5x3 neural network using vectors and train it to classify between 3 groups of points associated with different labels:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Task 4:"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Apply the functions `create_meshgrid` and `plot_neural_simple` to visualise the results of the classification. Add the necessary code to the function `plot_neural_simple` where marked:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_meshgrid(data):\n",
    "    h = 0.02\n",
    "    x_min, x_max = data[:, 0].min() - 1, data[:, 0].max() + 1\n",
    "    y_min, y_max = data[:, 1].min() - 1, data[:, 1].max() + 1\n",
    "    xx, yy = np.meshgrid(np.arange(x_min, x_max, h), np.arange(y_min, y_max, h))\n",
    "    return (xx,yy,np.ones(xx.shape))\n",
    "\n",
    "def plot_neural_simple(params, grid, data, labels, epoch, accuracy):\n",
    "    nudge = 0.06\n",
    "    W_3, W_2, b_3, b_2 = params\n",
    "    xx,yy,Z = grid\n",
    "    \n",
    "    for i in range(xx.shape[0]): # row\n",
    "        for j in range(yy.shape[1]): #column\n",
    "            dpoint = np.array([xx[i][j], yy[i][j]]).reshape(2,1)\n",
    "            \n",
    "            #compute the score \n",
    "            \n",
    "            Z[i][j] = score\n",
    "            \n",
    "    fig = plt.figure(figsize=(5,5))\n",
    "    ax = fig.add_subplot(111)\n",
    "    plt.title(f'accuracy at the epoch {epoch}: {accuracy}')\n",
    "    ax.contourf(xx, yy, Z, cmap=plt.cm.binary, alpha=0.1, zorder=15)\n",
    "    ax.scatter(data[:, 0], data[:, 1], c=labels, s=50,  cmap=plt.cm.Set3,zorder=50)\n",
    "    ax.set_aspect('equal')\n",
    "    for i in range(data.shape[0]):\n",
    "        d = data[i]\n",
    "        ax.annotate(f'{i}',(d[0]+nudge,d[1]+nudge))\n",
    "    plt.xlim(xx.min(), xx.max())\n",
    "    plt.ylim(yy.min(), yy.max())\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Task 5:"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Modify the algorithm to implement batch training:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}