07. NLP/4.3 Word Embeddings.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# <center> Word Embeddings\n",
    "![](img/embeddings.png)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import spacy\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.svm import LinearSVC\n",
    "from tqdm import tqdm_notebook as tqdm\n",
    "from sklearn.decomposition import PCA\n",
    "from sklearn.manifold import TSNE\n",
    "\n",
    "# Need to load the large model to get the vectors\n",
    "# python -m spacy download en_core_web_lg\n",
    "nlp   = spacy.load('en_core_web_sm')\n",
    "#nlp_lg = spacy.load('en_core_web_lg')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Toy Example"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(16, 96)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Token</th>\n",
       "      <th>Has vector?</th>\n",
       "      <th>Vector</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>These</td>\n",
       "      <td>True</td>\n",
       "      <td>[-5.0837116, 2.8361223, 3.4696412, -1.6743262,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>vectors</td>\n",
       "      <td>True</td>\n",
       "      <td>[-1.3429472, -2.240938, -3.213147, -2.2139177,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>can</td>\n",
       "      <td>True</td>\n",
       "      <td>[0.13003206, -0.124765396, -5.763321, -2.27762...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>be</td>\n",
       "      <td>True</td>\n",
       "      <td>[-2.5554748, 4.446042, -0.04874468, -2.2856915...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>used</td>\n",
       "      <td>True</td>\n",
       "      <td>[4.079047, 1.0927283, -0.76209986, -2.8814154,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>as</td>\n",
       "      <td>True</td>\n",
       "      <td>[3.1916761, 2.1806138, 5.0337315, 0.60693055, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>features</td>\n",
       "      <td>True</td>\n",
       "      <td>[-1.1637183, -0.8492511, -1.9874053, 0.2503561...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>for</td>\n",
       "      <td>True</td>\n",
       "      <td>[0.92133707, -0.29976475, 5.2616405, -0.381117...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>machine</td>\n",
       "      <td>True</td>\n",
       "      <td>[-1.217228, 2.2428694, -0.35657167, 2.7428248,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>learning</td>\n",
       "      <td>True</td>\n",
       "      <td>[4.2102685, -1.5389627, -2.9267843, 2.2330406,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>models</td>\n",
       "      <td>True</td>\n",
       "      <td>[1.1083796, 0.2575071, -2.1047792, 1.7764406, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>.</td>\n",
       "      <td>True</td>\n",
       "      <td>[-1.8922297, 0.10376406, -2.6942015, -1.113549...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>January</td>\n",
       "      <td>True</td>\n",
       "      <td>[-1.9741617, -2.5218587, -1.5795851, 1.5570469...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>February</td>\n",
       "      <td>True</td>\n",
       "      <td>[1.9770939, -1.9603626, -2.6785517, 2.4473314,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>dog</td>\n",
       "      <td>True</td>\n",
       "      <td>[-0.22021368, 1.9151164, -3.3270593, 2.9729428...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>cat</td>\n",
       "      <td>True</td>\n",
       "      <td>[1.8899665, 0.8104883, -0.7978307, 1.7478522, ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       Token  Has vector?                                             Vector\n",
       "0      These         True  [-5.0837116, 2.8361223, 3.4696412, -1.6743262,...\n",
       "1    vectors         True  [-1.3429472, -2.240938, -3.213147, -2.2139177,...\n",
       "2        can         True  [0.13003206, -0.124765396, -5.763321, -2.27762...\n",
       "3         be         True  [-2.5554748, 4.446042, -0.04874468, -2.2856915...\n",
       "4       used         True  [4.079047, 1.0927283, -0.76209986, -2.8814154,...\n",
       "5         as         True  [3.1916761, 2.1806138, 5.0337315, 0.60693055, ...\n",
       "6   features         True  [-1.1637183, -0.8492511, -1.9874053, 0.2503561...\n",
       "7        for         True  [0.92133707, -0.29976475, 5.2616405, -0.381117...\n",
       "8    machine         True  [-1.217228, 2.2428694, -0.35657167, 2.7428248,...\n",
       "9   learning         True  [4.2102685, -1.5389627, -2.9267843, 2.2330406,...\n",
       "10    models         True  [1.1083796, 0.2575071, -2.1047792, 1.7764406, ...\n",
       "11         .         True  [-1.8922297, 0.10376406, -2.6942015, -1.113549...\n",
       "12   January         True  [-1.9741617, -2.5218587, -1.5795851, 1.5570469...\n",
       "13  February         True  [1.9770939, -1.9603626, -2.6785517, 2.4473314,...\n",
       "14       dog         True  [-0.22021368, 1.9151164, -3.3270593, 2.9729428...\n",
       "15       cat         True  [1.8899665, 0.8104883, -0.7978307, 1.7478522, ..."
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text = \"These vectors can be used as features for machine learning models. January February dog cat\"\n",
    "\n",
    "tokens      = []\n",
    "vectors     = []\n",
    "has_vector  = []\n",
    "for token in nlp(text):\n",
    "    tokens.append(token)\n",
    "    has_vector.append(token.has_vector)\n",
    "    vectors.append(token.vector)\n",
    "\n",
    "result = pd.DataFrame({'Token': tokens, 'Has vector?': has_vector, 'Vector': vectors })\n",
    "\n",
    "vectors = np.array(vectors)\n",
    "print(vectors.shape)\n",
    "result"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Embedding Dimensonality Reduction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "vectors_pca  = PCA(n_components=2).fit_transform(vectors)\n",
    "vectors_tsne = TSNE(n_components=2).fit_transform(vectors)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAewAAAHVCAYAAAAzX8gFAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3XtcVVX+//H38kCKmqLZRWkSmim8cJe85r3E71fH0DQrazTHNMuymV9MUg/Lmm/lL+3XfHUqs3K0q+XdrEbHJsc0yyAQ0STKKEPHa5AYqMD6/YGcQcVLyuGw4PV8PHhwzj777PXZ26Nv19rr7G2stQIAADVbPX8XAAAAzozABgDAAQQ2AAAOILABAHAAgQ0AgAMIbAAAHEBgAwDgAAIbAAAHENgAADggwN8FVNSiRQsbGhrq7zIAAKg2qamp+6y1F59pvRoV2KGhoUpJSfF3GQAAVBtjzHdnsx5D4gAAOIDABgDAAQQ2AAAOILABAHAAgQ0AgAMIbAAAHEBgAwDgAAIbAAAHENgAADiAwAYAwAEENgAADiCwAQBwAIENAIADCGwAqEFeffVVRUVFKTo6WrfffrveffddderUSbGxsbruuuu0e/duSdKUKVM0evRo9erVS1deeaVmzJjh58rhazXq9poAUJdt2bJFTzzxhNavX68WLVrowIEDMsbo008/lTFGL7/8sp5++mk988wzkqRt27bpo48+0sGDBxUeHq7x48crMDDQz3sBXyGwAcCPlqblatrKLO3MK5TZ+nfFXZugFi1aSJKaN2+uzZs3a/jw4dq1a5eOHDmisLAw73sHDBig+vXrq379+rrkkku0e/duXX755f7aFfgYQ+IA4CdL03KVvHizcvMKZSXl/XxEa7L2amlarnede++9VxMmTNDmzZv14osvqqioyPta/fr1vY89Ho+Ki4urs3xUMwIbAPxk2sosFR4t8T5v0Dpa+VvX6snFGyVJBw4cUH5+vkJCQiRJ8+bN80udqBkIbADwk515hcc9v+Di1mraZbjSZ92v6Oho/fGPf9SUKVM0bNgwde/e3TtUjrrJWGv9XYNXfHy8TUlJ8XcZAFAtuk39p3JPCG1JCgkO0vpJffxQEfzBGJNqrY0/03r0sAHAT5ISwhUU6DluWVCgR0kJ4X6qCDUZs8QBwE8SY8vOTZfPEm8VHKSkhHDvcqAiAhsA/CgxNoSAxllhSBwAAAcQ2AAAOIDABgDAAQQ2AAAOILABAHAAgQ0AgAMIbAAAHEBgAwDgAAIbAAAHENgAADiAwAYAwAEENgAADiCwAQBwAIENAIADCGwAABxAYAMA4AACGwAABxDYAAA4gMAGAMABBDYAAA4gsAEAcACBDQCAAwhsAAAcQGADAOAAAhsAAAcQ2AAAOIDABgDAAQQ2AAAOILABAHBAlQS2MWaOMWaPMSazwrIpxphcY0z6sZ//roq2AACoi6qqhz1XUv9Klj9rrY059vN+FbUFAECdUyWBba1dK+lAVWwLAACczNfnsCcYYzKODZk3q2wFY8xYY0yKMSZl7969Pi4HAAA3+TKwX5D0a0kxknZJeqaylay1s6218dba+IsvvtiH5QAA4C6fBba1dre1tsRaWyrpJUkdfdUWAAC1nc8C2xjTssLTwZIyT7UuAAA4vYCq2Igx5i1JvSS1MMb8IOlRSb2MMTGSrKQcSeOqoi0AAOqiKglsa+0tlSx+pSq2DQAAuNIZAABOILABAHAAgQ0AgAMIbAAAHEBgAwDgAAIbAAAHENgAADiAwAYAwAEENgAADiCwAQBwAIENAIADCGwAABxAYAMA4AACGwAABxDYAAA4gMAGAMABBDYAAA4gsAEAcACBDQCAAwhsAAAcQGADAOAAAhsAAAcQ2AAAOIDABgDAAQQ2AAAOILABAHAAgQ0AgAMIbAAAHEBgAwDgAAIbAAAHENgAADiAwAYAwAEENgAADiCwAQBwAIENAIADCGwAABxAYAMA4AACGwAABxDYAAA4gMAGAMABBDYAAA4gsAEAcACBDQCAAwhsAAAcQGADAOAAAhsAAAcQ2AAAOIDABgDAAQQ2AAAOILABAHAAgQ0AgAMIbAAAHEBgAwDgAAIbAAAHENgAADiAwAYAwAEENgAADiCwAQBwAIENAIADCGwAABxAYAMA4IAqCWxjzBxjzB5jTGaFZc2NMf8wxmQf+92sKtoCAKAuqqoe9lxJ/U9YNknSh9baqyR9eOw5AAA4B1US2NbatZIOnLD4Bknzjj2eJymxKtoCAKAu8uU57Euttbsk6djvS3zYFgAAtZrfJ50ZY8YaY1KMMSl79+71dzkAANRIvgzs3caYlpJ07Peeylay1s621sZba+MvvvhiH5YDAIC7fBnYyyWNPPZ4pKRlPmwLAIBaraq+1vWWpA2Swo0xPxhjfi9pqqTrjTHZkq4/9hwAAJyDgKrYiLX2llO81Lcqtg8AQF3n90lnAADgzAhsAHBMTk6OIiIi/F0GqhmBDQCAAwhsAHBQcXGxRo4cqaioKA0dOlQ///yzUlNT1bNnT3Xo0EEJCQnatWuXv8tEFSKwAcBBWVlZGjt2rDIyMtSkSRM999xzuvfee7Vw4UKlpqZq9OjRevjhh/1dJqpQlcwSBwD41tK0XE1bmaWdeYVqbvPV4rJW6tatmyTptttu05NPPqnMzExdf/31kqSSkhK1bNnSnyWjihHYAFDDLU3LVfLizSo8WiJJ2v1TkfJ+LtbStFwlxoZIki688EK1b99eGzZs8Gep8CGGxAGghpu2Mssb1uWKf9qjR2YvliS99dZb6ty5s/bu3esN7KNHj2rLli3VXit8h8AGgBpuZ17hScsCL/qVvv30fUVFRenAgQPe89cPPvigoqOjFRMTo08++cQP1cJXGBIHgBquVXCQciuEdkDTS9VqzAsKCQ7S+kl9vMtjYmK0du1af5SIakAPGwBquKSEcAUFeo5bFhToUVJCuJ8qgj/QwwaAGq58Yln5LPFWwUFKSgj3LkfdQGADgAMSY0MI6DqOIXEAABxAYAMA4AACGwAABxDYAAA4gMAGAMABBDYAAA4gsAEAcACBDQCAAwhsAAAcQGCfhaVLl2rr1q3+LgMAUIcR2GfhXAK7uLjYR9UAAOoiY631dw1e8fHxNiUlpUq3+eCDD6p169a6++67JUlTpkzRhRdeqNLSUr3zzjs6fPiwBg8erMcee0yS9Oqrr2r69OkyxigqKkrjx4/XwIED1bRpUzVt2lSLFi3SwYMHddddd+nnn3/Wr3/9a82ZM0fNmjVTr1691LVrV61fv16DBg3SFVdcoccee0wej0dNmzbltncAgJMYY1KttfFnXNFaW2N+OnToYKvaF198YXv06OF93rZtWztv3jx755132tLSUltSUmIHDBhg//Wvf9nMzEx79dVX271791prrd2/f7+11tqRI0faBQsWeLcRGRlp16xZY621dvLkyXbixInWWmt79uxpx48f710vIiLC/vDDD9Zaa3/88ccq3zcAgPskpdizyMhae7eupWm53lvR7f4yR3NWparDpQFq1qyZMjIytGrVKsXGxkqSCgoKlJ2drU2bNmno0KFq0aKFJKl58+YnbTc/P195eXnq2bOnJGnkyJEaNmyY9/Xhw4d7H3fr1k2jRo3STTfdpCFDhvhydwEAtVytDOylablKXrxZhUdLJEkX/KaLkp95Wd1DAnTzzTcrJydHycnJGjdu3HHvmzFjhowx59V2o0aNvI9nzZqlzz77TO+9955iYmKUnp6uiy666Ly2DwCom2rlpLNpK7O8YS1JDdv2UH7mGr23fImGDh2qhIQEzZkzRwUFBZKk3Nxc7dmzR3379tU777yj/fv3S5IOHDggSbrwwgt18OBBSVLTpk3VrFkzffzxx5Kk1157zdvbPtE333yjTp066fHHH1eLFi20Y8cOn+0zAKB2q5U97J15hcc9v+Di1io9UihPo+Zq2bKlWrZsqS+//FJdunSRJDVu3Fivv/662rdvr4cfflg9e/aUx+NRbGys5s6dq5tvvll33nmnZsyYoYULF2revHneSWdXXnml/va3v1VaR1JSkrKzs2WtVd++fRUdHe3zfQcA1E61cpZ4t6n/VO4JoS1JIcFBWj+pz3lvHwCAqnK2s8Rr5ZB4UkK4ggI9xy0LCvQoKSHcTxUBAHB+amVgJ8aG6KkhkQoJDpJRWc/6qSGRSowN8XdpVWbGjBlq27atRowY8Yvel5OTozfffNNHVQEAfKVWnsOWykK7NgX0iZ5//nl98MEHCgsL+0XvKw/sW2+99Re9r6SkRB6P58wrAgB8olb2sGu7u+66S9u3b9egQYP0xBNPaPTo0brmmmsUGxurZcuWSSoL5u7duysuLk5xcXH65JNPJEmTJk3Sxx9/rJiYGD377LOaO3euJkyY4N32wIEDtWbNGkllk/EeeeQRderUSRs2bFBqaqp69uypDh06KCEhQbt27ZJU1ttv166doqKidPPNN1fvwQCAuuJsrq5SXT++uNJZbdW6dWu7d+9em5ycbF977TVrbdnV1K666ipbUFBgDx06ZAsLC6211n711Ve2/Nh+9NFHdsCAAd7t/O1vf7P33HOP9/mAAQPsRx99ZK21VpJ9++23rbXWHjlyxHbp0sXu2bPHWmvt/Pnz7R133GGttbZly5a2qKjIWwMA4Oyprl/prLapeOW2VsFB+vlI2ffMV61apeXLl2v69OmSpKKiIn3//fdq1aqVJkyYoPT0dHk8Hn311Ve/uE2Px6Mbb7xRkpSVlaXMzExdf/31ksqGyFu2bClJioqK0ogRI5SYmKjExMSq2F0AwAkIbAeceOW23LxC/fjzEb2fsUvWWi1atEjh4cfPgJ8yZYouvfRSbdq0SaWlpWrQoEGl2w4ICFBpaan3eVFRkfdxgwYNvOetrbVq3769NmzYcNI23nvvPa1du1bLly/Xn//8Z23ZskUBAXy0AKAqcQ7bASdeuU2SrJX++tHXSkhI0MyZM2WPfZ8+LS1NUtk1z1u2bKl69erptddeU0lJ2fsrXrVNkkJDQ5Wenq7S0lLt2LFDGzdurLSG8PBw7d271xvYR48e1ZYtW7zv6927t55++mnl5eV5ryAHAKg6dIMccOKV28r9O79Qk//vZN1///2KioqStVahoaFasWKF7r77bt14441asGCBevfu7b3GeVRUlAICAhQdHa1Ro0bp/vvvV1hYmCIjIxUREaG4uLhK27rgggu0cOFC3XfffcrPz1dxcbHuv/9+XX311brtttuUn58va63+8Ic/KDg42GfHAgDqqlp5pbPahiu3AUDtVaevdFbbcOU2AABD4g4ovwBMxVniSQnhtfrCMACA4xHYjqjtV24DAJweQ+IAADiAwAYAwAEENgAADiCwAQBwAIENAIADCGwAABxAYAMA4AACGwAABxDYAAA4gMAGAMABBDYAAA4gsAEAcACBDQCAAwhsAAAcQGADAOAAAhsAAAcQ2ACAGqdXr15KSUnxdxk1CoENAIADAnzdgDEmR9JBSSWSiq218b5uEwBQfXJycjRw4EBlZmZKkqZPn66CggI1b95cs2bNUkBAgNq1a6f58+fr0KFDuvfee7V582YVFxdrypQpuuGGG1RYWKg77rhDW7duVdu2bVVYWOjnvap5fB7Yx/S21u6rprYAADXA1KlT9e2336p+/frKy8uTJD3xxBPq06eP5syZo7y8PHXs2FHXXXedXnzxRTVs2FAZGRnKyMhQXFycn6uveaorsAEAtcjStFxNW5mlnXmFam7z9VNR8UnrREVFacSIEUpMTFRiYqIkadWqVVq+fLmmT58uSSoqKtL333+vtWvX6r777vO+Lyoqqvp2xhHVcQ7bSlpljEk1xow98UVjzFhjTIoxJmXv3r3VUA4A4HwsTctV8uLNys0rlJW0++BR7c7/WUvTciWVhbAkvffee7rnnnuUmpqqDh06qLi4WNZaLVq0SOnp6UpPT9f333+vtm3bSpKMMf7aJSdUR2B3s9bGSfovSfcYY3pUfNFaO9taG2+tjb/44ouroZzaKTQ0VPv2nf6sw9msAwBnMm1llgqPlnifexoFq/hQvp5cvFGHDx/WihUrVFpaqh07dqh37956+umnlZeXp4KCAiUkJGjmzJmy1kqS0tLSJEk9evTQG2+8IUnKzMxURkZG9e9YDefzIXFr7c5jv/cYY5ZI6ihpra/bBQD4xs684yeEGU+Amna9WWl/vUcDP22vNm3aqKSkRLfddpvy8/NlrdUf/vAHBQcHa/Lkybr//vsVFRUla61CQ0O1YsUKjR8/XnfccYeioqIUExOjjh07+mnvai6fBrYxppGketbag8ce95P0uC/bdElOTo769++va6+9Vp9++qmio6N1xx136NFHH9WePXv0xhtv6De/+Y1Gjx6t7du3q2HDhpo9e7aioqK0f/9+3XLLLdq7d686duzo/d+qJL3++uuaMWOGjhw5ok6dOun555+Xx+Pxvn7o0CHddNNN+uGHH1RSUqLJkydr+PDh/jgEABzUKjhIuSeEdpP4QWp73XD9Y1Kf0743KChIL774YqXL58+fX6V11ja+HhK/VNI6Y8wmSRslvWet/buP23TK119/rYkTJyojI0Pbtm3Tm2++qXXr1mn69Ol68skn9eijjyo2NlYZGRl68skn9bvf/U6S9Nhjj+naa69VWlqaBg0apO+//16S9OWXX+rtt9/W+vXrlZ6eLo/H4x1mKvf3v/9drVq10qZNm5SZman+/ftX+34DcFdSQriCAj3HLQsK9CgpIdxPFdUNPu1hW2u3S4r2ZRsuKp9d+d13OQoMvkzfFDdXZL16at++vfr27StjjCIjI5WTk6PvvvtOixYtkiT16dNH+/fvV35+vtauXavFixdLkgYMGKBmzZpJkj788EOlpqbqmmuukSQVFhbqkksuOa79yMhIPfDAA3rwwQc1cOBAde/evRr3HoDrEmNDJMk7S7xVcJCSEsK9y+EbfK2rmpXPriyfsFFiPEpevFmSVK9ePdWvX9/7uLi4WAEBJ/8Rlc+krGxGpbVWI0eO1FNPPXXKGq6++mqlpqbq/fffV3Jysvr166dHHnnkvPcNQN2RGBtCQFczLk1azU6cXSlJhUdLNG1lVqXrV5w5uWbNGrVo0UJNmjQ5bvkHH3ygH3/8UZLUt29fLVy4UHv27JEkHThwQN99991x29y5c6caNmyo2267TQ888IC++OKLKt1HAEDVo4ddzU6cXVlx+VWVLJ8yZYp35mTDhg01b948SdKjjz6qW265RXFxcerZs6euuOIKSVK7du30P//zP+rXr59KS0sVGBio5557Tq1bt/Zuc/PmzUpKSlK9evUUGBioF154ocr3EwBQtUzF2cX+Fh8fb2v73Vm6Tf3nSbMrJSkkOEjrzzC7EgBQ+xhjUs/mPhsMiVczZlcCAM4FQ+LVjNmVAIBzQWD7AbMrAQC/FEPiAAA4gMAGAMABBDYAAA4gsAEAcACBDQCAAwhsAAAcQGADAOAAAhsAAAcQ2AAAOIDABgDAAQQ2AAAOILABAHAAgQ0AgAMIbAAAHEBgAwDgAAIbAAAHENgAADiAwAYAwAEENgAADiCwAQBwAIENAIADCGwAABxAYAMA4AACGwAABxDYAAA4gMAGAMABBDYAAA4gsAEAcACBDQCAAwhsAAAcQGADAOAAAhsAAAcQ2AAAOIDABgDAAQQ2AAAOILABAHAAgQ0AgAMIbAAAHEBgAwDgAAIbAAAHENgAADiAwAYAwAEENgAADiCwAQBwAIENAIADCGwAABxAYAMA4AACGwAABxDYAAA4gMAGAMABBDYAAA4gsAEAcACBDQCAAwhs4DyEhoZq3759Jy1fvny5pk6d6oeKANRWPg9sY0x/Y0yWMeZrY8wkX7cH1ASDBg3SpEl83AFUHZ8GtjHGI+k5Sf8lqZ2kW4wx7XzZJnAqOTk5atOmjcaMGaOIiAiNGDFCq1evVrdu3XTVVVdp48aN2rhxo7p27arY2Fh17dpVWVlZkqSSkhI98MADioyMVFRUlGbOnOnd7syZMxUXF6fIyEht27ZNkjR37lxNmDBBkjRq1Cjdd9996tq1q6688kotXLjQ+95p06bpmmuuUVRUlB599NFqPBoAXOPrHnZHSV9ba7dba49Imi/pBh+3CZzS119/rYkTJyojI0Pbtm3Tm2++qXXr1mn69Ol68skn1aZNG61du1ZpaWl6/PHH9dBDD0mSZs+erW+//VZpaWnKyMjQiBEjvNts0aKFvvjiC40fP17Tp0+vtN1du3Zp3bp1WrFihbfnvWrVKmVnZ2vjxo1KT09Xamqq1q5d6/uDAMBJAT7efoikHRWe/yCpU8UVjDFjJY2VpCuuuMLH5aCuWZqWq2krs7Qzr1DNbb4uafUrRUZGSpLat2+vvn37yhijyMhI5eTkKD8/XyNHjlR2draMMTp69KgkafXq1brrrrsUEFD2V6Z58+beNoYMGSJJ6tChgxYvXlxpHYmJiapXr57atWun3bt3SyoL7FWrVik2NlaSVFBQoOzsbPXo0cM3BwOA03wd2KaSZfa4J9bOljRbkuLj420l6wPnZGlarpIXb1bh0RJJ0u6firS/yGppWq4SY0NUr1491a9fX5JUr149FRcXa/Lkyerdu7eWLFminJwc9erVS5JkrZUxlX2c5d2Gx+NRcXHxadcp31b57+TkZI0bN65K9hdA7ebrIfEfJP2qwvPLJe30cZuAJGnayixvWJez1mrayqxTvic/P18hISGSys5Dl+vXr59mzZrlDeQDBw6cd30JCQmaM2eOCgoKJEm5ubnas2fPeW8XQO3k68D+XNJVxpgwY8wFkm6WtNzHbQKSpJ15hb9ouST96U9/UnJysrp166aSkv+E/ZgxY3TFFVcoKipK0dHRevPNN8+7vn79+unWW29Vly5dFBkZqaFDh+rgwYPnvV0AtZMpH57zWQPG/Lekv0jySJpjrX3iVOvGx8fblJQUn9aDuqPb1H8qt5JwDgkO0vpJffxQEQCczBiTaq2NP9N6Pv8etrX2fWvt1dbaX58urIGqlpQQrqBAz3HLggI9SkoI91NFAHDufD3pDPCbxNiyc9Hls8RbBQcpKSHcuxwAXEJgo1ZLjA0hoAHUClxLHAAABxDYAAA4gMAGAMABBDYAAA4gsAEAcACBDQCAAwhsAAAcQGADAOAAAhsAAAcQ2AAAOIDABmqhNWvW6JNPPvF3GQCqEIEN1EIENlD7ENiAQ1599VVFRUUpOjpat99+u95991116tRJsbGxuu6667R7927l5ORo1qxZevbZZxUTE6OPP/74uG3s379fMTExiomJ0WWXXaaQkBDFxMQoODhY7dq189OeATgTY631dw1e8fHxNiUlxd9lADXSli1bNGTIEK1fv14tWrTQgQMHZIxRcHCwjDF6+eWX9eWXX+qZZ57RlClT1LhxYz3wwAOn3WbF9XJycjRw4EBlZmZW0x4BkCRjTKq1Nv5M69HDBmq4pWm56jb1n+px/0wdahWvdTsOS5KaN2+uH374QQkJCYqMjNS0adO0ZcuW82qrpKREd955p9q3b69+/fqpsLBQkvTNN9+of//+6tChg7p3765t27ZJkhYsWKCIiAhFR0erR48e3m0kJSXpmmuuUVRUlF588cXzqglAGQIbqMGWpuUqefFm5eYVylqrg4dLlLx4s5am5UqS7r33Xk2YMEGbN2/Wiy++qKKiovNqLzs7W/fcc4+2bNmi4OBgLVq0SJI0duxYzZw5U6mpqZo+fbruvvtuSdLjjz+ulStXatOmTVq+fLkk6ZVXXlHTpk31+eef6/PPP9dLL72kb7/99rzqAiAF+LsAAKc2bWWWCo+WSJIatI7W3iVPqOCaGzRtZZZ6tA5Sfn6+QkJCJEnz5s3zvu/CCy/UTz/9dNy2lqblatrKLO3MK1Sr4CAlJYSf1F5YWJhiYmIkSR06dFBOTo4KCgr0ySefaNiwYd71Dh8u6+V369ZNo0aN0k033aQhQ4ZIklatWqWMjAwtXLhQkpSfn6/s7GyFhYVV1WEB6iQCG6gmZ3teuaKdeYXexxdc3FpNuwzX7jcnabeppz9u66kpU6Zo2LBhCgkJUefOnb092d/+9rcaOnSoli1bppkzZ2p/4yuVvHizN/xz8wqVvHizovf8pPirGnvbqF+/vvexx+NRYWGhSktLFRwcrPT09JPqmzVrlj777DO99957iomJUXp6uqy1mjlzphISEn7xMQJwagQ2JEmNGzdWQUGBT9tYvny5tm7dqkmTJvm0ndqkVXCQciuEduPIvmoc2VchwUGaO6mPJOmGG2446X1XX321MjIyvM+7Tf2nN6zLFR4t0fpv9iv+qlanraFJkyYKCwvTggULNGzYMFlrlZGRoejoaH3zzTfq1KmTOnXqpHfffVc7duxQQkKCXnjhBfXp00eBgYH66quvFBISokaNGp3PoQDqPM5ho0qVlJSc8rVBgwbVubB+4oknFB4eruuuu05ZWVmSpPT0dHXu3FlRUVEaPHiwfvzxR0nS559/rqioKHXp0kVJSUmKiIhQUkK4ggI9x20zKNBT6XD26VTsqVd0sOjoWb3/jTfe0CuvvKLo6Gi1b99ey5YtkyQlJSUpMjJSERER6tGjh6KjozVmzBi1a9dOcXFxioiI0Lhx41RcXPyL6gVwMr7WBUnH97CnTZumd955R4cPH9bgwYP12GOPSZISExO1Y8cOFRUVaeLEiRo7dqz3vX/84x+1cuVKPfPMM7rttts0cuRIvfvuuzp69KgWLFigNm3aaO7cuUpJSdFf//pXjRo1Sk2aNFFKSor+/e9/6+mnn9bQoUNVWlqqCRMm6F//+pfCwsJUWlqq0aNHa+jQoX47NucqNTVVo0aN0meffabi4mLFxcXprrvu0quvvqqZM2eqZ8+eeuSRR/TTTz/pL3/5iyIiIjR79mx17dpVkyZN0ooVK5SZmVnpuefE2JBfVEu3qf88rqdeLiQ4SOuP9dQB+Adf68I5WbVqlbKzs7Vx40alp6crNTVVa9eulSTNmTNHqampSklJ0YwZM7R//35J0qFDhxQREaF6jS+HAAAT10lEQVTPPvtM1157rSSpRYsW+uKLLzR+/HhNnz690rZ27dqldevWacWKFd6e9+LFi5WTk6PNmzfr5Zdf1oYNG6phr33j448/1uDBg9WwYUM1adJEgwYN0qFDh5SXl6eePXtKkkaOHKm1a9cqLy9PBw8eVNeuXSVJt956q3c7ibEhWj+pj76dOkDrJ/X5xWEtqcp66gD8h3PYdVjFnlvh0RItTcvVulWrtGrVKsXGxkqSCgoKlJ2drR49emjGjBlasmSJJGnHjh3Kzs7WRRddJI/HoxtvvPG4bZfPGO7QoYMWL15cafuJiYmqV6+e2rVrp927d0uS1q1bp2HDhqlevXq67LLL1Lt3b1/tvs+UH9cv/7FVjVSkuLTcM4asr0e6yts/3546AP8hsOuo8u/3lk9EslZKXrxZV+8+qOTkZI0bN+649desWaPVq1drw4YNatiwoXr16uX9zm+DBg3k8RzfeyufbezxeE55/rLijOTywKpJp2jORcXjWv9X7bX7/b/owbdTVHioQO+++67GjRunZs2a6eOPP1b37t312muvqWfPnmrWrJkuvPBCffrpp+rcubPmz59f5bUlxoYQ0IDDGBKvoyp+v7dc4dESfR34a82ZM8d7Pjs3N1d79uxRfn6+mjVrpoYNG2rbtm369NNPfVLXtddeq0WLFqm0tFS7d+/WmjVrfNKOr1Q8rvUv+40atemu7S/do/F3jFD37t0llX1fOikpSVFRUUpPT9cjjzwiqeyCI2PHjlWXLl1krVXTpk39th8Aah562HXUqWYN/3xJe42Nrq8uXbpIKptQ9vrrr6t///6aNWuWoqKiFB4ers6dO/ukrhtvvFEffvihIiIidPXVV6tTp05OBdeJx7Vp1+Fq2nW4jKQ5Uwd4l1f2H5727dt7v4o1depUxcefcQ4KgDqEWeJ1VE2eNVxQUKDGjRtr//796tixo9avX6/LLrvMrzWdrfM5rm+//baeeuopFRcXq3Xr1po7d64uvvhiX5UKoIY421ni9LDrqKSE8OPOYUs1Z9bwwIEDlZeXpyNHjmjy5MnOhLV0fsd1+PDhGj58uC/LA+AwAruOqsmzhl07b11RTT6uANzGkDgAAH7EhVMAAKhFCGwAABxAYAMA4AACGwAABxDYAAA4gMAGAMABBDYAAA4gsAEAcACBDQCAAwhsAAAcQGADAOAAAhsAAAcQ2AAAOIDABgDAAQQ2AAAOILABAHAAgQ0AgAMIbAAAHEBgAwDgAAIbAAAHENgAADiAwAYAwAEENgAADiCwAQBwAIENAIADCGwAABxAYAMA4AACGwAABxDYAAA4wGeBbYyZYozJNcakH/v5b1+1BQBAbRfg4+0/a62d7uM2AACo9RgSBwDAAb4O7AnGmAxjzBxjTLPKVjDGjDXGpBhjUvbu3evjcgAAcJOx1p77m41ZLemySl56WNKnkvZJspL+LKmltXb06bYXHx9vU1JSzrkeAABcY4xJtdbGn2m98zqHba297iyLeUnSivNpCwCAusyXs8RbVng6WFKmr9oCAKC28+Us8aeNMTEqGxLPkTTOh20BAFCr+SywrbW3+2rbAADUNXytCwAABxDYAAA4gMAGAMABBDYAAA4gsAEAcACBDQCAAwhsAAAc4OvbawIAjlmalqtpK7O0M69QrYKDlJQQrsTYEH+XBUcQ2ABQDZam5Sp58WYVHi2RJOXmFSp58WZJIrRxVhgSB4BqMG1lljesyxUeLdG0lVl+qgiuIbABoBrszCs87vnuBY+q+OD+k5YDp0JgA0A1aBUcdNzzS4c9poALLzppOXAqBDYAVIOkhHAFBXqOWxYU6FFSQrifKoJrmHQGANWgfGIZs8RxrghsAKgmibEhBDTOGUPiAAA4gMAGAMABBDYAAA4gsAEAcACBDQCAAwhsAAAcQGADAOAAAhsAAAcQ2AAAOIDABgDAAQQ2AAAOILABAHAAgQ0AgAMIbB9q3Lixv0sAANQSBHYtU1JS4u8SAAA+QGD7WEFBgfr27au4uDhFRkZq2bJlkqScnBy1bdtWd955p9q3b69+/fqpsLBQktSrVy+lpKRIkvbt26fQ0FDve7p37664uDjFxcXpk08+kSStWbNGvXv31q233qrIyEhNnjxZ//u//+ut4eGHH9aMGTOqca8BAFUtwN8F1HYNGjTQkiVL1KRJE+3bt0+dO3fWoEGDJEnZ2dl666239NJLL+mmm27SokWLdNttt51yW5dccon+8Y9/qEGDBsrOztYtt9ziDfaNGzcqMzNTYWFhysnJ0ZAhQzRx4kSVlpZq/vz52rhxY7XsLwDANwjsKrQ0LVfTVmZpZ16hWgUHqaTUylqrhx56SGvXrlW9evWUm5ur3bt3S5LCwsIUExMjSerQoYNycnJOu/2jR49qwoQJSk9Pl8fj0VdffeV9rWPHjgoLC5MkhYaG6qKLLlJaWpp2796t2NhYXXTRRb7ZaQCogxITE7Vjxw4VFRVp4sSJ+v3vf6/f//73SklJkTFGo0eP1h/+8IcqbZPAriJL03KVvHizCo+WnUPOzSvU4eJS/Z8n/6q9e/cqNTVVgYGBCg0NVVFRkSSpfv363vd7PB7vkHhAQIBKS0slybuuJD377LO69NJLtWnTJpWWlqpBgwbe1xo1anRcPWPGjNHcuXP173//W6NHj/bNTgNAHTVnzhw1b95chYWFuuaaa9ShQwfl5uYqMzNTkpSXl1flbRLYVWTayixvWFf0bsrXGhh6iQIDA/XRRx/pu+++O+O2QkNDlZqaqo4dO2rhwoXe5fn5+br88stVr149zZs377QTzAYPHqxHHnlER48e1ZtvvnluOwUAkHTyCOqvvl2hLz/9UJK0Y8cOHTlyRNu3b9e9996rAQMGqF+/flVeA5POqsjOvMLjntvSEhlPoErCuiklJUXx8fF644031KZNmzNu64EHHtALL7ygrl27at++fd7ld999t+bNm6fOnTvrq6++OqlXXdEFF1yg3r1766abbpLH4zn3HQOAOq58BDU3r1BW0jcZn2n5+yv10AuLtGnTJsXGxurw4cPatGmTevXqpeeee05jxoyp8jqMtbbKN3qu4uPjbfkkKtd0m/pP5VYI7SN7tmv/32cq/r5ZWj+pT7XXU1paqri4OC1YsEBXXXVVtbcPALXFif++/5z9qQo2rVTsmKl6JbGVYmJi9Prrr6tfv35q0qSJ0tPTNWrUKKWnp5/V9o0xqdba+DOtx5B4FUlKCPeewz6Y9r4Opr6ry/qNU1JCeLXXsnXrVg0cOFCDBw8mrAHgPJ04ghoU1kEH0z7Q5//v95qcGq/OnTsrNzdXvXr18s4/euqpp6q8DnrYVejEcxxJCeFKjA3xd1kAgPNwYg+7XEhwUJWMoNLD9oPE2BACGgBqmYojqOWCAj3VPoJKYAMAcBrlHTF/j6AS2AAAnEFNGEHla10AADiAwAYAwAEENgAADiCwAQBwAIENAKhWM2bMUNu2bTVixAh/l+IUZokDAKrV888/rw8++MB7S+DTKS4uVkAAUSUR2ACAanTXXXdp+/btGjRokEaNGqWPP/5Y27dvV8OGDTV79mxFRUVpypQp2rlzp3JyctSiRQvuOHgMQ+IAgGoza9YstWrVSh999JFycnIUGxurjIwMPfnkk/rd737nXS81NVXLli0jrCughw0A8It169Zp0aJFkqQ+ffpo//79ys/PlyQNGjRIQUFB/iyvxiGwAQA+V/HmSP/OL9L7GbtU2c2njDGSpEaNGlV3iTUeQ+IAAJ9ampar5MWblZtXKCupuNTqz+9t1eVt4/TGG29IktasWaMWLVqoSZMm/i22BqOHDQDwqWkrs46705UkFR0t0Z6rBikl5W+KiopSw4YNNW/ePD9V6Abuhw0A8KmwSe+psqQxkr6dOqC6y6lxzvZ+2AyJAwB8qlVw5ZPHTrUclSOwAQA+lZQQrqBAz3HLggI9SkoI91NFbuIcNgDAp8rvI10+S7xVcJCSEsL9fn9p1xDYAACfS4wNIaDPE0PiAAA4gMAGAMAB5xXYxphhxpgtxphSY0z8Ca8lG2O+NsZkGWMSzq9MAADqtvM9h50paYikFysuNMa0k3SzpPaSWklabYy52lpbcvImAADAmZxXD9ta+6W1NquSl26QNN9ae9ha+62kryV1PJ+2AACoy3x1DjtE0o4Kz384tuwkxpixxpgUY0zK3r17fVSOGzwej2JiYrw/OTk5p1x3zZo1GjhwYPUVBwDwqzMOiRtjVku6rJKXHrbWLjvV2ypZVuk1UK21syXNlsouTXqmemqzoKAgpaenV+k2S0pK5PF4zrziKRQXFysggG//AYC/nbGHba29zlobUcnPqcJaKutR/6rC88sl7TzfYuuikpISJSUl6ZprrlFUVJRefPE/0wV++uknDR48WO3atdNdd92l0tJSSVLjxo31yCOPqFOnTtqwYYNCQ0O1b98+SVJKSop69eolSdq4caO6du2q2NhYde3aVVlZZWc35s6dq2HDhum3v/2t+vXrp9tvv13Llv3nj3vEiBFavnx5NR0BAIDkuwunLJf0pjHm/6ls0tlVkjb6qK1ao7CwUDExMZKksLAwLVmyRK+88oqaNm2qzz//XIcPH1a3bt3Ur18/SWWBu3XrVrVu3Vr9+/fX4sWLNXToUB06dEgRERF6/PHHT9temzZttHbtWgUEBGj16tV66KGHvDeT37BhgzIyMtS8eXP961//0rPPPqsbbrhB+fn5+uSTT7irDgBUs/MKbGPMYEkzJV0s6T1jTLq1NsFau8UY846krZKKJd3DDPHKVbypuwIu0JS/vXfc1YBWrVqljIwMLVy4UJKUn5+v7OxsXXDBBerYsaOuvPJKSdItt9yidevWaejQofJ4PLrxxhvP2HZ+fr5Gjhyp7OxsGWN09OhR72vXX3+9mjdvLknq2bOn7rnnHu3Zs0eLFy/WjTfeyDA5AFSz8/pX11q7RNKSU7z2hKQnzmf7tV35Td3L7xNrrZS8eLOk/1x711qrmTNnKiHh+K+yr1mzRsYcP1Wg/HmDBg2OO28dEBDgHS4vKiryLp88ebJ69+6tJUuWKCcnxztULkmNGjU6btu333673njjDc2fP19z5sw5n90GAJwDrnTmR5Xd1L3waImmrfzPN+USEhL0wgsveHu/X331lQ4dOiSpbEj822+/VWlpqd5++21de+21lbYTGhqq1NRUSfIOeUtlPeyQkLL/GMydO/e0tY4aNUp/+ctfJEnt27f/BXsJAKgKBLYf7cwrPOPyMWPGqF27doqLi1NERITGjRun4uJiSVKXLl00adIkRUREKCwsTIMHD650e48++qgmTpyo7t27H9fz/tOf/qTk5GR169ZNJSWnP2Nx6aWXqm3btrrjjjt+6W4CAKqAsbbmfJMqPj7epqSk+LuMatNt6j+VW0lohwQHaf2kPn6o6NR+/vlnRUZG6osvvlDTpk39XQ4A1BrGmFRrbfyZ1qOH7Ueu3NR99erVatOmje69917CGgD8hKm+fuTKTd2vu+46ff/99/4uAwDqNALbz7ipOwDgbDAkDgCAAwhsAAAcQGADAOAAAhsAAAcQ2AAAOIDABgDAAQQ2AAAOILABAHAAgQ0AgAMIbAAAHEBgAwDgAAIbAAAHENgAADiAwAYAwAEENgAADjDWWn/X4GWM2SvpO3/XcYIWkvb5u4hagmNZdTiWVYdjWTU4jueutbX24jOtVKMCuyYyxqRYa+P9XUdtwLGsOhzLqsOxrBocR99jSBwAAAcQ2AAAOIDAPrPZ/i6gFuFYVh2OZdXhWFYNjqOPcQ4bAAAH0MMGAMABBDYAAA4gsE/BGDPMGLPFGFNqjIk/4bVkY8zXxpgsY0yCv2p0kTFmijEm1xiTfuznv/1dk0uMMf2Pfe6+NsZM8nc9LjPG5BhjNh/7HKb4ux6XGGPmGGP2GGMyKyxrboz5hzEm+9jvZv6ssTYisE8tU9IQSWsrLjTGtJN0s6T2kvpLet4Y46n+8pz2rLU25tjP+/4uxhXHPmfPSfovSe0k3XLs84hz1/vY55DvD/8yc1X2719FkyR9aK29StKHx56jChHYp2Ct/dJam1XJSzdImm+tPWyt/VbS15I6Vm91qKM6SvraWrvdWntE0nyVfR6BamWtXSvpwAmLb5A079jjeZISq7WoOoDA/uVCJO2o8PyHY8tw9iYYYzKODasxbHb2+OxVLStplTEm1Rgz1t/F1AKXWmt3SdKx35f4uZ5aJ8DfBfiTMWa1pMsqeelha+2yU72tkmV8N66C0x1XSS9I+rPKjtmfJT0jaXT1Vec0PntVq5u1dqcx5hJJ/zDGbDvWcwRqpDod2Nba687hbT9I+lWF55dL2lk1FdUOZ3tcjTEvSVrh43JqEz57Vchau/PY7z3GmCUqO+VAYJ+73caYltbaXcaYlpL2+Lug2oYh8V9uuaSbjTH1jTFhkq6StNHPNTnj2F/kcoNVNrkPZ+dzSVcZY8KMMReobPLjcj/X5CRjTCNjzIXljyX1E5/F87Vc0shjj0dKOtUoJc5Rne5hn44xZrCkmZIulvSeMSbdWptgrd1ijHlH0lZJxZLusdaW+LNWxzxtjIlR2VBujqRx/i3HHdbaYmPMBEkrJXkkzbHWbvFzWa66VNISY4xU9u/gm9bav/u3JHcYY96S1EtSC2PMD5IelTRV0jvGmN9L+l7SMP9VWDtxaVIAABzAkDgAAA4gsAEAcACBDQCAAwhsAAAcQGADAOAAAhsAAAcQ2AAAOOD/A4g7yNkO+h/YAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 576x576 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "vectors_dimRed = vectors_pca\n",
    "\n",
    "plt.figure(figsize=(8,8))\n",
    "x = vectors_dimRed[:,0]\n",
    "y = vectors_dimRed[:,1]\n",
    "plt.scatter(x, y);\n",
    "\n",
    "for i, txt in enumerate(tokens):\n",
    "    plt.annotate(str(txt), (x[i], y[i]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Read data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "##### DOWNLOAD DATASET. ONLY IN COLAB !!!\n",
    "#!wget -P ./data \"https://raw.githubusercontent.com/SaturdaysAI-Murcia/machine-learning/master/4. NLP/data/movieReviews_trainLabel.tsv\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Loading the spam data\n",
    "# ham is the label for non-spam messages\n",
    "df = pd.read_csv(\"data/movieReviews_trainLabel.tsv\", sep=\"\\t\", index_col=\"id\")\n",
    "\n",
    "# Only top 1000\n",
    "df = df.sample(1000, random_state=0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Reviews to embeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "db8cd6edf5ff4f84b27f313dacb79543",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1000), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(1000, 96)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "embeddings = []\n",
    "\n",
    "with nlp.disable_pipes():\n",
    "    for text in tqdm(df.review):\n",
    "        embeddings.append(nlp(text).vector)\n",
    "    \n",
    "embeddings = np.array(embeddings)\n",
    "embeddings.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Split data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_train, x_test, y_train, y_test = train_test_split(embeddings, df[\"sentiment\"],\n",
    "                                                    test_size=0.2, shuffle=True, random_state=0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Model: Linear Support Vector Machine"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 70.000%\n"
     ]
    }
   ],
   "source": [
    "# Set dual=False to speed up training, and it's not needed\n",
    "model = LinearSVC(random_state=1, dual=False, max_iter=10000)\n",
    "model.fit(x_train, y_train)\n",
    "print(f\"Accuracy: {model.score(x_test, y_test) * 100:.3f}%\", )"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}