mf_test.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "from scipy.sparse.linalg import spsolve\n",
    "from scipy import sparse\n",
    "from data import *\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "class MFModel(object):\n",
    "\n",
    "    def __init__(self, rates_dict, test_rates_dict, num_users, num_items, \n",
    "                 hidden_dim, lr, gamma, epochs, optimizer, reg_u, reg_i, confidence=1):\n",
    "        \"\"\" Params:\n",
    "        1) ratings - rating matrix (as dict)\n",
    "        2) hidden_dim  - number of latent dimensions (as integer)\n",
    "        3) lr - learning rate (as float)\n",
    "        4) gamma - regulaizer (as float)\n",
    "        5) opechs - as int\n",
    "        \"\"\"\n",
    "\n",
    "        self.ratings_dict = rates_dict\n",
    "        self.test_ratings_dict = test_rates_dict\n",
    "        self.num_users = num_users\n",
    "        self.num_items = num_items\n",
    "        self.K = hidden_dim\n",
    "        self.optimizer = optimizer\n",
    "        self.lr = lr\n",
    "        self.gamma = gamma\n",
    "        self.regulizer_items = reg_i\n",
    "        self.regulizer_users = reg_u\n",
    "        self.confidence = confidence\n",
    "        self.epochs = epochs\n",
    "        self.scbd = []\n",
    "        \n",
    "    def print_hyper_params(self):\n",
    "        print(\"latent dimention %d\" % (self.K))\n",
    "        print(\"learning rate %r\" % (self.lr))\n",
    "        print(\"lambda SGD %r\" % (self.gamma))\n",
    "        print(\"lambda ALS users %r\" % (self.regulizer_users))\n",
    "        print(\"lambda ALS items %r\" % (self.regulizer_items))\n",
    "        print(\"Confidence ALS %d\" % (self.confidence))\n",
    "        print(\"Epochs %d\" % (self.epochs))\n",
    "\n",
    "    def init(self, isize, ik, kind='normal'):\n",
    "        if kind == 'normal':\n",
    "            return np.random.normal(scale=1.0/ik, size=(isize, ik))\n",
    "        elif kind == 'uniform':\n",
    "            return np.random.uniform(-1.0/ik, 1.0/ik)\n",
    "\n",
    "\n",
    "    def train_model(self):\n",
    "\n",
    "        def reset(samples):\n",
    "            np.random.shuffle(samples)\n",
    "\n",
    "        self.samples = []\n",
    "        # prepare the samples samples\n",
    "        for u, items in self.ratings_dict.items():\n",
    "            for i, r, _ in items:\n",
    "                self.samples += [(u, i, r)]\n",
    "\n",
    "        self.U = self.init(self.num_users, self.K, kind='normal')\n",
    "        self.V = self.init(self.num_items, self.K, kind='normal')\n",
    "        \n",
    "        #sparse intermidiate representation\n",
    "        self.Us = sparse.csr_matrix(self.U)\n",
    "        self.Vs = sparse.csr_matrix(self.V)\n",
    "\n",
    "        # Initialize the biases\n",
    "        self.bu = np.zeros(self.num_users)\n",
    "        self.bv = np.zeros(self.num_items)\n",
    "\n",
    "\n",
    "        for epoch in range(self.epochs):\n",
    "            reset(self.samples)\n",
    "            if self.optimizer == 'sgd':\n",
    "                self.LearnModelFromDataUsingSGD()\n",
    "            elif self.optimizer == 'als':\n",
    "                self.LearnModelFromDataUsingALS()\n",
    "            train_rmse = self.RMSE(is_train=True)\n",
    "            val_rmse = self.RMSE()\n",
    "            print(\"[Epoch %d] : train RMSE = %.4f\" % (epoch, train_rmse))\n",
    "            if epoch % 5 == 0:\n",
    "                print(\"[Epoch %d] : validation RMSE = %.4f\" % (epoch, val_rmse))\n",
    "            self.scbd.append((epoch, train_rmse, val_rmse))\n",
    "\n",
    "\n",
    "    def LearnModelFromDataUsingSGD (self):\n",
    "        \"\"\"\n",
    "        Performs one iteration on one of the User/Items matrices\n",
    "        \"\"\"    \n",
    "        for u, v, rate in self.samples:\n",
    "            prediction = self.bu[u] + self.bv[v] + self.U[u, :].dot(self.V[v, :].T)\n",
    "            err = (rate - prediction)\n",
    "\n",
    "            # update biases\n",
    "            self.bu[u] += self.lr * (err - self.gamma * self.bu[u])\n",
    "            self.bv[v] += self.lr * (err - self.gamma * self.bv[v])\n",
    "\n",
    "            # update user item metrices\n",
    "            self.U[u, :] += self.lr * (err * self.V[v, :] - self.gamma * self.U[u, :])\n",
    "            self.V[v, :] += self.lr * (err * self.U[u, :] - self.gamma * self.V[v, :])\n",
    "            \n",
    "    \n",
    "    def LearnModelFromDataUsingALS(self):\n",
    "        \"\"\"\n",
    "        Performs one iteration on one of the User/Items matrices\n",
    "        \"\"\"\n",
    "        rate_mat = np.zeros((self.num_users, self.num_items))\n",
    "        for user, item in self.ratings_dict.items():\n",
    "            for movie, rate, _ in item:\n",
    "                rate_mat[user][movie] = rate\n",
    "        rate_mat = sparse.csr_matrix(rate_mat) * self.confidence\n",
    "\n",
    "        U_i = sparse.eye(self.num_users)\n",
    "        V_i = sparse.eye(self.num_items)\n",
    "\n",
    "        # UtU = self.Us.T.dot(self.Us)\n",
    "        # VtV = self.Vs.T.dot(self.Vs)\n",
    "\n",
    "        # Users update\n",
    "        lambda_u = sparse.eye(self.K) * self.regulizer_users\n",
    "        for u_idx, u_rate in enumerate(rate_mat):\n",
    "            u_dense = u_rate.toarray()\n",
    "            u_bin = u_dense.copy()\n",
    "            u_bin[u_bin != 0] = 1.0\n",
    "            u_diag = sparse.diags(u_dense, [0])\n",
    "            VtV = self.Vs.T.dot(u_diag).dot(self.Vs)\n",
    "            VtV_u = self.Vs.T.dot(u_diag).dot(u_dense.T)\n",
    "            self.Us[u_idx, :] = spsolve(VtV + lambda_u, VtV_u)\n",
    "\n",
    "\n",
    "\n",
    "        # Items update\n",
    "        lambda_i = sparse.eye(self.K) * self.regulizer_items\n",
    "        for i_idx, i_rate in enumerate(rate_mat.T):\n",
    "            i_dense = i_rate.toarray()\n",
    "            i_bin = i_dense.copy()\n",
    "            i_bin[i_bin != 0] = 1.0\n",
    "            i_diag = sparse.diags(i_dense, [0])\n",
    "            UtU = self.Us.T.dot(i_diag).dot(self.Us)\n",
    "            UtU_i = self.Us.T.dot(i_diag).dot(i_dense.T)\n",
    "            self.Vs[i_idx, :] = spsolve(UtU + lambda_i, UtU_i)\n",
    "\n",
    "        self.U = self.Us.toarray()\n",
    "        self.V = self.Vs.toarray()\n",
    "\n",
    "\n",
    "\n",
    "            \n",
    "    def RMSE(self, is_train=False):\n",
    "        mod_pred = self.bu[:,np.newaxis] + self.bv[np.newaxis:,] + self.U.dot(self.V.T)\n",
    "        total_err = 0\n",
    "        r_cnt = 0\n",
    "        ratings_dict = self.ratings_dict if is_train else self.test_ratings_dict\n",
    "        for u, items in ratings_dict.items():\n",
    "            for i, r, _ in items:\n",
    "                total_err += pow(r - mod_pred[u, i], 2)\n",
    "                r_cnt += 1\n",
    "        return np.sqrt(total_err / r_cnt)\n",
    "\n",
    "# Load the data\n",
    "X_train, X_test, users, items = create_user_item_map('../ml-1m/ratings.dat', '', percent=0.8, sort_ratings=True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Starting training SGD...\n",
      "latent dimention 40\n",
      "learning rate 0.01\n",
      "lambda SGD 0.1\n",
      "lambda ALS users 0.1\n",
      "lambda ALS items 0.1\n",
      "Epochs 22\n",
      "[Epoch 0] : train RMSE = 1.0481\n",
      "[Epoch 0] : validation RMSE = 1.0967\n",
      "[Epoch 1] : train RMSE = 0.9747\n",
      "[Epoch 2] : train RMSE = 0.9505\n",
      "[Epoch 3] : train RMSE = 0.9372\n",
      "[Epoch 4] : train RMSE = 0.9301\n",
      "[Epoch 5] : train RMSE = 0.9250\n",
      "[Epoch 5] : validation RMSE = 0.9443\n",
      "[Epoch 6] : train RMSE = 0.9184\n",
      "[Epoch 7] : train RMSE = 0.9107\n",
      "[Epoch 8] : train RMSE = 0.9026\n",
      "[Epoch 9] : train RMSE = 0.8951\n",
      "[Epoch 10] : train RMSE = 0.8885\n",
      "[Epoch 10] : validation RMSE = 0.9132\n",
      "[Epoch 11] : train RMSE = 0.8831\n",
      "[Epoch 12] : train RMSE = 0.8783\n",
      "[Epoch 13] : train RMSE = 0.8730\n",
      "[Epoch 14] : train RMSE = 0.8686\n",
      "[Epoch 15] : train RMSE = 0.8660\n",
      "[Epoch 15] : validation RMSE = 0.8979\n",
      "[Epoch 16] : train RMSE = 0.8630\n",
      "[Epoch 17] : train RMSE = 0.8607\n",
      "[Epoch 18] : train RMSE = 0.8581\n",
      "[Epoch 19] : train RMSE = 0.8560\n",
      "[Epoch 20] : train RMSE = 0.8543\n",
      "[Epoch 20] : validation RMSE = 0.8906\n",
      "[Epoch 21] : train RMSE = 0.8527\n"
     ]
    }
   ],
   "source": [
    "# 1a - SGD\n",
    "model = MFModel(X_train, X_test, users, items, hidden_dim=40, lr=0.01, gamma=0.1, reg_u=0.1, reg_i=0.1, epochs=22, optimizer='sgd')\n",
    "print(\"Starting training SGD...\")\n",
    "model.print_hyper_params()\n",
    "model.train_model()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA7oAAAEJCAYAAAC6xVufAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzs3XmcnFWd7/HPqb27el/S2feEEAiE0EQUhACJo86ICvpS\ndMYZXCKDXgd1ruIdvSzqFR3vzIAyMtHBZVzQK+IyqGjQAAoCAQMEQvaVLL2k9+qq6qo694/nqaW3\nLN1VXdWd7/v1qtezneepX7lAvjnnOcdYaxERERERERGZKjzFLkBEREREREQknxR0RUREREREZEpR\n0BUREREREZEpRUFXREREREREphQFXREREREREZlSFHRFRERERERkSlHQFRERERERkSlFQVdERERE\nRESmFAVdERERERERmVJ8xS4gnxoaGuz8+fOLXYaIiIiIiIjk2TPPPNNmrW08lbZTKujOnz+fzZs3\nF7sMERERERERyTNjzP5TbauhyyIiIiIiIjKlKOiKiIiIiIjIlFKwoGuMudcY02KM2TrK9WXGmCeM\nMTFjzD8OufZ6Y8x2Y8wuY8zNhapRREREREREpp5CvqP7LeCrwHdGuX4c+AjwltyTxhgvcDewDjgE\nPG2M+bm19qXClSoiIiIiIlKaBgYGOHToENFotNilTIhQKMTs2bPx+/1jfkbBgq619lFjzPwTXG8B\nWowxfznk0mpgl7V2D4Ax5j7gzYCCroiIiIiInHEOHTpEZWUl8+fPxxhT7HIKylpLe3s7hw4dYsGC\nBWN+Tim+ozsLOJhzfMg9JyIiIiIicsaJRqPU19dP+ZALYIyhvr5+3L3Xk355IWPMemA9QFNTE5s2\nbSpuQSIiIiIiInlUXV1Nb29vscuYUNFodFzZrhSD7ivAnJzj2e65EVlrNwAbAJqbm+2aNWsKWtyY\n7X0UXnkWLr2p2JWIiIiIiMgksm3bNiorK4tdxoQKhUJccMEFY76/FIcuPw0sMcYsMMYEgHcCPy9y\nTeO3/Vfw8G3QuqPYlYiIiIiIiJyy9vZ2Vq5cycqVK5k+fTqzZs3KHMfj8VN6xvXXX8/27dsLXGlW\nwXp0jTE/ANYADcaYQ8AtgB/AWnuPMWY6sBmoAlLGmJuA5dbabmPMh4GHAC9wr7X2xULVOWFe+3F4\n9r/gd7fDO75b7GpEREREREROSX19PVu2bAHg1ltvpaKign/8x0ErxGKtxVqLxzNyX+o3v/nNgteZ\nq2A9utba66y1M6y1fmvtbGvtf1pr77HW3uNeP+qer7LW1rj73e61X1prl1prF1lrP1+oGidUuAFe\n8z9g2y/g0OZiVyMiIiIiIjIuu3btYvny5bz73e/mnHPO4ciRI6xfv57m5mbOOeccbr/99kzbSy+9\nlC1btpBIJKipqeHmm2/m/PPP59WvfjUtLS15r60U39Gdul79IXhqA2y8Ff72F3AGzJomIiIiIiL5\nc9svXuSlw915febymVXc8qZzxnTvyy+/zHe+8x2am5sBuOOOO6irqyORSHDFFVfwtre9jeXLlw+6\np6uri8svv5w77riDj33sY9x7773cfPPN4/4duUrxHd2pK1gBl38C9j0Gux8udjUiIiIiIiLjsmjR\nokzIBfjBD37AqlWrWLVqFdu2beOll14adk9ZWRlveMMbALjwwgvZt29f3utSj+5Eu/B6eOJup1d3\n4ZUwyhh2ERERERGRocba81oo4XA4s79z507uvPNOnnrqKWpqavjrv/7rEdfDDQQCmX2v10sikch7\nXUpZE80XgCs/DUdfgBd/UuxqRERERERE8qK7u5vKykqqqqo4cuQIDz30UNFqUdAthnPfBk3nwu8+\nC4lTm45bRERERESklK1atYrly5ezbNky3vOe93DJJZcUrRZjrS3al+dbc3Oz3bx5ksxovOM38P23\nwxu/DKs/UOxqRERERESkRG3bto2zzz672GVMqJF+szHmGWtt8yi3DKIe3WJZsg7mXQKPfAlivcWu\nRkREREREZMpQ0C0WY2DtbdDXAn/6WrGrERERERERmTIUdItpzkWw7K/g8bugr73Y1YiIiIiIiEwJ\nCrrFduVnIN4Lf/iXYlciIiIiIiIyJSjoFtu0ZXD+u+CpDdB5sNjViIiIiIiITHoKuqVgzc2AgU13\nFLsSERERERGRSU9BtxTUzHGWGHru+9DycrGrERERERERyWhvb2flypWsXLmS6dOnM2vWrMxxPB4/\n5efce++9HD16tICVZinolorXfhwCFfDw7cWuREREREREJKO+vp4tW7awZcsWbrjhBj760Y9mjgOB\nwCk/R0H3TFReB5d8BLY/CAeeLHY1IiIiIiIiJ/Xtb3+b1atXs3LlSm688UZSqRSJRIK/+Zu/YcWK\nFZx77rncdddd/PCHP2TLli284x3vOO2e4LHwFfTpcnouvhGe3AAbb4Xrf+mstSsiIiIiIpL2q5vh\n6Av5feb0FfCG058vaOvWrTzwwAM8/vjj+Hw+1q9fz3333ceiRYtoa2vjhRecOjs7O6mpqeErX/kK\nX/3qV1m5cmV+6x+BenRLSSAMl38CDjwOO39b7GpERERERERGtXHjRp5++mmam5tZuXIljzzyCLt3\n72bx4sVs376dj3zkIzz00ENUV1dPeG3q0S01F/4dPHE3PHwbLF4LHv1dhIiIiIiIuMbQ81oo1lre\n+9738tnPfnbYteeff55f/epX3H333dx///1s2LBhQmtTiio1Xj9c+Wk4thW2/rjY1YiIiIiIiIxo\n7dq1/OhHP6KtrQ1wZmc+cOAAra2tWGt5+9vfzu23386zzz4LQGVlJT09PRNSm3p0S9E518Af74Tf\nfQ6WvwV8pz6TmYiIiIiIyERYsWIFt9xyC2vXriWVSuH3+7nnnnvwer28733vw1qLMYYvfvGLAFx/\n/fW8//3vp6ysjKeeeuq0Zmw+XcZaW7CHT7Tm5ma7efPmYpeRH7s2wnevhTd8CV71wWJXIyIiIiIi\nRbJt2zbOPvvsYpcxoUb6zcaYZ6y1zadyv4Yul6pFV8H818IjX4LYxHTvi4iIiIiITAUKuqXKGFh7\nG0Ta4Il/L3Y1IiIiIiIik4aCbimbfSGc/SZ4/C7oayt2NSIiIiIiUiRT6ZXTk8nHb1XQLXVX/m8Y\niMCjXy52JSIiIiIiUgShUIj29vYzIuxaa2lvbycUCo3rOZp1udQ1LoUL/ho2/ydc/PdQO6/YFYmI\niIiIyASaPXs2hw4dorW1tdilTIhQKMTs2bPH9QwF3cng8pvh+R/Bpi/AW+8pdjUiIiIiIjKB/H4/\nCxYsKHYZk4qGLk8G1bNg9Xp47j449mKxqxERERERESlpCrqTxaUfhVAVPHx7sSsREREREREpaQq6\nk0V5HVxyE+z4Nex/otjViIiIiIiIlCwF3cnkVTdAxXTYeCucATOuiYiIiIiIjEXBgq4x5l5jTIsx\nZuso140x5i5jzC5jzPPGmFU51/YZY14wxmwxxmwuVI2TTqAc1nwSDv7J6dkVERERERGRYQrZo/st\n4PUnuP4GYIn7WQ98bcj1K6y1K621zYUpb5K64G+gbpHzrm4qWexqRERERERESk7Bgq619lHg+Ama\nvBn4jnX8CagxxswoVD1ThtcPV30GWl5ylhwSERERERGRQYr5ju4s4GDO8SH3HIAFNhpjnjHGrJ/w\nykrd2W+GGSvh9/8HErFiVyMiIiIiIlJSfMUuYBSXWmtfMcZMA35rjHnZ7SEexg3C6wGamprYtGnT\nBJZ56pIpS2u/ZXo4P3+3UNv4Vs5//hZ2/eBTHJpzdV6eKSIiIiIiMhUUM+i+AszJOZ7tnsNam962\nGGMeAFYDIwZda+0GYANAc3OzXbNmTQFLHruP/WgLj+9q5zcfu4SqkD8PT1wDvb9n8ZGfsvjttzpr\n7IqIiIiIiEhRhy7/HHiPO/vyxUCXtfaIMSZsjKkEMMaEgdcBI87cPJm859XzaemJ8oVfvpy/h151\nC0Ta4Ymv5u+ZIiIiIiIik1whlxf6AfAEcJYx5pAx5n3GmBuMMTe4TX4J7AF2AV8HbnTPNwF/MMY8\nBzwFPGitnfRr6aycU8P7Ll3AD546wOO72/Lz0FmrYPlb4PGvQm9Lfp4pIiIiIiIyyRlrbbFryJvm\n5ma7eXPpLrvbH0/y+judEdi//ofLKAt4x//Qtl1w92q46P3wxi+N/3kiIiIiIiIlyBjzzKkuP1vM\noctnnLKAly9cs4L97RH+5bfb8/PQhsWw6j2w+V44vjc/zxQREREREZnEFHQn2GsWNXDd6rn85x/2\nsuVgZ34eevknweNzlhsSERERERE5wynoFsGn3riMaZUhPvnj54knUuN/YNUMuPgGeOH/wdEXxv88\nERERERGRSUxBtwiqQn4+95Zz2X6sh3/ftCs/D73kJghVw8O35+d5IiIiIiIik5SCbpGsXd7E1efP\n5O7f72L70Z7xP7CsBl77Mdj5G9j3x/E/T0REREREZJJS0C2iW960nMqQn0/c/zzJVB5mv169Hipn\nwsZbYArNpi0iIiIiInI6FHSLqL4iyC1vWs5zBzv55h/zMGOyvwzW3AyHnobtvxz/80RERERERCYh\nBd0iu/r8mVy1bBpf/s129rf3jf+BK98N9Uucd3VTyfE/T0REREREZJJR0C0yYwyfe+u5+D0ebr7/\nBex4hxx7fXDVZ6D1ZXjuB/kpUkREREREZBJR0C0BM6rL+NQbz+aJPe3c9/TB8T/w7Kth1oXw+y/A\nQHT8zxMREREREZlEFHRLxDsvmsPFC+v4Pw9u42jXOMOpMbD2Vug+BE9/Ix/liYiIiIiITBoKuiXC\n4zHccc15DKRSfPqneRjCvOAyWHQVPPZliHblp0gREREREZFJQEG3hMxvCPPxdWexcVsLv3j+yPgf\nuPYW6O+AP941/meJiIiIiIhMEgq6Jea9ly7g/Dk13PrzFzneFx/fw2acD+deC3/6d+g5lp8CRURE\nRERESpyCbonxegxfuvY8eqID3PaLF8f/wCv+CZJxePRL43+WiIiIiIjIJKCgW4LOml7JjWsW87Mt\nh3l42zh7YusXwaq/hWe+Be2781KfiIiIiIhIKVPQLVEfumIxZzVV8k8PbKUnOjC+h13+SfAG4Pef\nz09xIiIiIiIiJUxBt0QFfB6++LbzaOmJ8oVfvTy+h1U2wcU3wtb74chz+SlQRERERESkRCnolrCV\nc2p47yUL+P6TB3hid/v4HnbJR6CsFjbelp/iRERERERESpSCbon7+OvOYm5dOTf/5Hn648mxPyhU\nDa/9OOx+GPY+mr8CRURERERESoyCbokrC3i549oV7G+P8K8bd4zvYRd9AKpmw8Zbwdq81CciIiIi\nIlJqFHQngdcsauC61XP4xmN7eO5g59gf5A/BFZ+CV56Bbb/IX4EiIiIiIiIlREF3kvjUG8+msTLI\nJ+9/nngiNfYHnX8dNC6Dh2+HZCJ/BYqIiIiIiJQIBd1Joirk5/NvWcHLR3v42qZxrIfr8cKVn4H2\nnbDle/krUEREREREpEQo6E4ia5c3cfX5M/nq73ey41jP2B+07C9h9mrYdAcM9OevQBERERERkRKg\noDvJ3PKm5VQEfXzix8+TTI1xQiljYO2t0HMYntqQz/JERERERESKTkF3kqmvCHLr1eew5WAn3/zj\n3rE/aP4lsHgdPPYv0D+OCa5ERERERERKjILuJHT1+TO5atk0vvyb7Rxoj4z9QWtvgWgX/PHf8lec\niIiIiIhIkSnoTkLGGD731nPxeTzc/JPnsWNdE3f6CljxdvjTPdB9JL9FioiIiIiIFImC7iQ1o7qM\nT71xGY/vbueHTx8c+4Ou/CdIJeB7b4NXns1fgSIiIiIiIkWioDuJXXfRXC5eWMfnH9zG0a7o2B5S\nOx/e+X2ItMM3roLffBri4xgOLSIiIiIiUmQKupOYx2O445rziCdTfPqnL4x9CPPS18GHnoRV74HH\nvwJfezXseSS/xYqIiIiIiEyQggVdY8y9xpgWY8zWUa4bY8xdxphdxpjnjTGrcq693hiz3b12c6Fq\nnArmN4T5+OuWsnFbC//9/Djesw1Vw5vuhL97EIwHvnM1/OzD0N+Rv2JFREREREQmQCF7dL8FvP4E\n198ALHE/64GvARhjvMDd7vXlwHXGmOUFrHPSe+8lCzh/djW3/vxFjvfFx/ew+ZfC3z8Ol9wEW74P\nd78KXvpZfgoVERERERGZAAULutbaR4HjJ2jyZuA71vEnoMYYMwNYDeyy1u6x1saB+9y2Mgqf18MX\n33Ye3dEBbv/Fi+N/oL8M1t0G638PFU3wo/fAfe+GnqPjf7aIiIiIiEiBFfMd3VlA7nTBh9xzo52X\nE1g2vYob1yzmp1sO87uXj+XnoTPOhw/8HtbeCrs2wldXwzPfhrG+CywiIiIiIjIBfMUuYLyMMetx\nhj7T1NTEpk2biltQEZ3rtcyqMPzjfc/w+UvLKPOZPD35AspW/StLd9xN7S8+QsdjX2fH0g/RXz4j\nT88XERERERHJn2IG3VeAOTnHs91z/lHOj8hauwHYANDc3GzXrFmT90Ink4bFHVz7tcf5Q08Dn3/r\nivw+/PXvhD9/h9rffIZXPXsTXPG/4OIPgXfS/32JiIiIiIhMIcUcuvxz4D3u7MsXA13W2iPA08AS\nY8wCY0wAeKfbVk7BBXNree8lC/jekwf40572/D7c44EL/w4+9BQsXgu//d/wjSvhyPP5/R4RERER\nEZFxKOTyQj8AngDOMsYcMsa8zxhzgzHmBrfJL4E9wC7g68CNANbaBPBh4CFgG/Aja20eZlg6c3zs\ndUuZW1fOzfc/T388mf8vqJoB7/guvP3b0H0ENqyBjbfBQDT/3yUiIiIiInKajJ1CEws1NzfbzZs3\nF7uMkvD4rjbe9Y0n+eBlC/nUG88u3BdFjsNvPgNbvgv1i+FNd8H8Swr3fSIiIiIickYyxjxjrW0+\nlbbFHLosBfSaxQ1ct3oOX39sD88d7CzcF5XXwVvuhr/5KSQH4FtvhF/cBNGuwn2niIiIiIjICSjo\nTmE3v+FsGiuDfPL+54knUoX9skVXwI1PwKs/DM9+G+6+GF7+ZWG/U0REREREZAQKulNYdZmfz71l\nBS8f7eGeR3YX/gsDYfiLz8P7N0JZLdx3Hfy/v4PelsJ/t4iIiIiIiOuEQdcYc2XO/oIh164pVFGS\nP+uWN/Gm82fyld/tZMexnon50lkXwvpNcOWn4eUH4asXwZbvwxR6H1xERERERErXyXp0v5yzf/+Q\na5/Ocy1SILe+aTkVQR+f+PHzJFMTFDZ9Abjsf8INf4TGZfDTv4fvXgMd+ybm+0VERERE5Ix1sqBr\nRtkf6VhKVH1FkFuvPoctBzv51uP7JvbLG5fC9b+CN34ZDj4F//5qeOJuSBVg2SMRERERERFOHnTt\nKPsjHUsJu/r8mVy5bBpffmg7B9ojE/vlHg+s/gB86EmY/1p46H/Bf66DY1oeWURERERE8u9kQXeh\nMebnxphf5Oynjxec5F4pIcYYPveWc/F6DDf/5HmKsn5y9Wx41w/h2v90hjD/x2Xwu89DIjbxtYiI\niIiIyJRlThR4jDGXn+hma+0jea9oHJqbm+3mzZuLXUZJ+96T+/mnB7byxWtX8I6L5havkL52eOhT\n8PwPoeEsuPorMPdVxatHRERERERKmjHmGWtt86m0PWGPrrX2kdwP8DjQDWwrtZArp+a6i+byqgV1\nfO7BbRzrjhavkHA9XLMB3v1jGIjAvX8Bv/yfEJugmaFFRERERGTKOtnyQvcYY85x96uB54DvAH82\nxlw3AfVJnnk8hjuuPY94IsU/PbC1OEOYcy1ZBzc+AavXw1Nfh7svhp2/LW5NIiIiIiIyqZ3sHd3X\nWmvTMwZdD+yw1q4ALgQ+UdDKpGAWNIT5+OuWsnHbMR584Uixy4FgJbzxS/C+30AgDN97G9z/Aehr\nK3ZlIiIiIiIyCZ0s6MZz9tcBPwWw1h4tWEUyId57yQLOm13NLT97keN98ZPfMBHmrIYbHoPLb4YX\nH4C7V8Nz90EyUezKRERERERkEjlZ0O00xvyVMeYC4BLg1wDGGB9QVujipHB8Xg9fvPY8uvoH+Ox/\nv1TscrJ8QbjiU/DBR6F2ATzwQfi/Z8GDH4f9j0MqVewKRURERESkxPlOcv2DwF3AdOCmnJ7cq4AH\nC1mYFN7ZM6q48YrF3PXwTlbNq+Vdq+fi9Zhil+VoWu4MZX75Qdh6P/z5e/D0N6ByJpx7jfOZuQpM\nidQrIiIiIiIl44TLC002Wl7o9MUSSf76G0/y9L4OFjWG+Ye1S/mrFTPwlErgTYv1wo5fO6F3528h\nNQC18+Gca+Dca6HpHIVeEREREZEp7HSWFzrZOrp3nehma+1HTrO2glLQHZtUyvLQi0f514072HGs\nl6VNFdy0dimvP2d66QVegP6ObE/vnkfAJp21eM+91unpbVhS7ApFRERERCTP8hl048BW4EfAYWBQ\n6rHWfnscdeadgu74pFKWX249wr9t3Mmull6WTa/ko+uW8rrlTZhS7S3tbYVtP4OtD8D+PwIWpp/n\nBN5zroHaecWuUERERERE8iCfQbceeDvwDiAB/BD4sbW2Mx+F5puCbn4kU5b/fv4wd27cyZ62Ps6Z\nWcXH1i3lymXTSjfwAnQfhhd/6vT0vuL+72D2RU5P7/K3QNWM4tYnIiIiIiJjlregO+Shs4F3Ah8D\nPmmt/a+xl1gYCrr5lUim+NmWw9z58E4OHI9w/uxqblq3lDVLG0s78AJ07IOtP3E+x14ADMy/FM55\nKyx/M4Qbil2hiIiIiIichrwHXWPMKuA6nLV0nwH+r7W2hNakcSjoFsZAMsUDz77CXb/byaGOfi6Y\nW8PH1i3l0sUNpR94AVp3wIs/gRd+DO07wXhh4Rqnp3fZX0JZTbErFBERERGRk8jn0OXbgb8EtgH3\nAb+21ibyUmUBKOgWVjyR4sfPHOKrv9vJ4a4oF82v5aPrlvKaRZOkd9RaOLbVGdq89X7oPADeACxe\n57zTe9YbIBAudpUiIiIiIjKCfAbdFLAXiLin0o0NYK21542n0HxT0J0YsUSSHz19kLt/v5uj3VEu\nXljHR9cu5VUL64td2qmzFl551gm8L/4Eeo6ArwzOer3T07t4HfhDxa5SRERERERc+Qy6J5yy1lq7\n/zRrKygF3YkVHUhy31MHuHvTblp7Yly6uIGPrlvChfPqil3a6Uml4MATTuh96WcQaYNAJZz9V07o\nXbgGvP5iVykiIiIickYryGRUQ77AA1xnrf3ead9cQAq6xREdSPLdP+3nnkd209Yb57KljXx07RIu\nmFtb7NJOXzIB+x51Qu+2X0C0C8pq4eyrndA7/1LweItdpYiIiIjIGSefPbpVwIeAWcDPgd8CHwY+\nDjxnrX3z+MvNHwXd4orEE/zXE/v5j0f3cLwvzpXLpvHRtUtZMbu62KWNTSIGu3/nhN6XfwkDfVDR\n5CxVdO61ztJFHk+xqxQREREROSPkM+j+DOgAngCuAqbhvJ/7D9baLXmoNa8UdEtDXyzBt5/Yx4ZH\n99AZGWDd8iZuWruEc2ZO0sALEI/Azt84oXfHQ5CMQXkDzFoFM1dltxWNxa5URERERGRKymfQfcFa\nu8Ld9wJHgLnW2mheKs0zBd3S0hMd4Ft/3MfXH9tDdzTBG86dzk1rl3LW9MpilzY+0W7Y/ivYswkO\nPwut28nM01Y9B2ZekA2+M1dCaBIHfBERERGREpHPoPustXbVaMelRkG3NHX1D3DvH/Zy7x/20htP\n8JcrZnDT2iUsnjbJA29arAeOPOfM4nz4WWfbmTNPW/2SwT2/01eAv6x49YqIiIiITEL5DLpJoC99\nCJThLDWUXl6oapy15pWCbmnrjMT5xmN7+eYf9xIZSHL1+TP5yFVLWNRYUezS8q+vHQ7/ORt8Dz8L\nvcecax4fTDt78JDnaWdrZmcRERERkRMo+KzLpUpBd3I43hdnw6N7+Pbj+4glkrzlgll85MolzG8I\nF7u0wrHWWas3t9f38LPOrM4AvhBMP29wz2/dIk12JSIiIiLiKpmga4x5PXAn4AW+Ya29Y8j1WuBe\nYBEQBd5rrd3qXtsH9ABJIHEqP0hBd3Jp643xH4/s5r/+tJ+BpOXaVbP4H1cuYU5debFLmxjWwvE9\nTs9vOvgeeQ4GIs71YBXMOH9w+K2eA8YUt24RERERkSIoiaDrTl61A1gHHAKexll796WcNv8M9Fpr\nbzPGLAPuttZe5V7bBzRba9tO9TsVdCenlp4o92zaw3ef3E8qZXl78xw+fOViZtWcge+xJhPQtn1w\nz++xFyE14FwPNzqTXWmmZxERERE5w5RK0H01cKu19i/c408BWGu/kNPmQeAOa+1j7vFu4DXW2mMK\numeeo11RvrZpFz946iAWyzsumsOHrljMjOozMPDmSsTg6NbBQ54107OIiIiInGFOJ+j6CljHLOBg\nzvEh4FVD2jwHXAM8ZoxZDcwDZgPHcP4Uv9GdEOs/rLUbClirlIDp1SFue/O5fPDyRdz9+1388OmD\n3PfUQS6cV8tlSxu5fGkjy2dU4fGcYUN3fUGYfaHzSYv1wJHnB4ffbT/PXq+cATXzoHYe1MzN2Z8H\nVbPAW8j/64uIiIiIFFex/7R7B3CnMWYL8ALwZ5x3cgEutda+YoyZBvzWGPOytfbRoQ8wxqwH1gM0\nNTWxadOmialcCmpdLVxwaYjfHUjwQlsn/7z3OP/80HaqAnBOg5cVDT7OrfdSFTzDQu8wK6BxBTT+\nLb6Bbip7dlHZs4uy/iOUdbcQavkdwVg7hlTmDouHaKiBaKiJaGiau3X2+8uaiAdqwWgSLBERERGZ\nvIo6dHlIewPsBc6z1nYPuXYrzru8Xz7Rd2ro8tTV0hPlsR1tPLqzlcd2tnG8Lw7AOTOruGxpI5ct\naeTCebUEfApowyQHoOuQs7Zv5wHo2D94v/fo4PbegDMcOt0DXDPX3Z/v7IcbNCGWiIiIiEy4UnlH\n14czGdVVwCs4k1G9y1r7Yk6bGiBirY0bYz4AvNZa+x5jTBjwWGt73P3fArdba399ou9U0D0zpFKW\nFw9388jRiz85AAAgAElEQVSOFh7d0cazBzpIpCzhgJdXL2rg8qUNXLa0kXn1U3i5onwa6IfOg07w\n7dznBuEDThju2A/9xwe394fd4dBzRwjD86Cspig/Q0RERESmtpIIum4hbwT+DWd5oXuttZ83xtwA\nYK29x+31/TbO+7gvAu+z1nYYYxYCD7iP8QHft9Z+/mTfp6B7ZuqJDvD47nYe3dHKoztbOXi8H4B5\n9eVctqSRy5Y28upF9VQEiz1Sf5KK9eT0BOcE4PR+rHtw+2A11LrvBee+G5wOwwH9BYSIiIiInL6S\nCboTTUFXrLXsa484oXdHK4/vbqd/IInfazKTWl225Ayd1KoQrIX+jtGHRXcegET/4HsqZ0DdIqhf\nCPWL3f3FUDsf/KGi/AwRERERKX0KuiKuWCLJM/s6eGRnK4/uaGPbEaf3saEiwGuXNHLZ0gZeu6SR\nhopgkSudoqyFvtZsAO7YC+174PhuaN8NkdzVw4zzbnD9IveTDsGLnN5gr79oP0NEREREik9BV2QU\nLd1RHts5fFKrc2dVZYY5r5qrSa0mTH9nNvS273b3dzlhONaVbefxOcOf6xdng3A6BFfNBo/++xIR\nERGZ6hR0RU5BKmXZerjLHeasSa1KirXQ15YTgne5+25v8EAk29YXgtoFQwKwG4grmjRDtIiIiMgU\noaArMgYnm9TqcndSq7AmtSoua6HniNvzu3twj3DHXkjGs20DFVC3cMhQaDcEl9cV7zeIiIiIyGlT\n0BUZp/SkVo9sb+HRnW08McKkVq9aUMeixgpqygPFLlfSUknoOjjCUOjdzsRYNpltG6rJGQq92AnE\n6Vmiw43qCRYREREpMQq6Ink22qRWALXlfhY0hFnQUMHCxjALG8IsaAwzvz5MyO8tYtUySCLuTIg1\naCi0++k+NLitv9xdK3ieMxt07ZClkkJVRfkJIiIiImcyBV2RAmvpjvLCK13sbetjd2sfe9t62dvW\nx7HuWKaNMTCzuoyFjWE3CDufRY0VzKwpw6vljUrHQH/OzND7cvb3j7xWcFmtE4Bzw2/tPKiZDzVz\nwKdZvEVERETyTUFXpEh6Ywn2tfWxp62PvTkBeE9rHz2xRKZdwOthXn05CxrCLGysyPQCL2gIUx8O\nYDRstnSk1wru2Dc4/KYDcdfBwe8FY6Bq5pAQPD+7XzlDs0SLiIiIjIGCrkiJsdbS1htnb5sTfve4\n4XdvWx/72/sYSGb/f1gV8rEgHX4bwoN6hMsDmgir5KRSzuRYo/UGdx8Gcv456w046wXXzhuhV3i+\n01usv+gQERERGUZBV2QSSSRTHO6Msrut1+0Fzn5e6ewf1HZ6VWhQ+HW2FcyuLcPvVS9hSUrEoPMg\ndO4beXh0f8fg9oHK4e8F1y2C+oVQPRe8+ssOEREROTMp6IpMEf3xJPvas8F3d2t2KHRX/0Cmnc9j\nmFtfnukFnlVTxvTqEE1VIWZUl9FQEcCnIFyaot2j9wZ37IdEzl92ePxQt8ANvosGzxpdOUM9wSIi\nIjKlKeiKnAE6+uLuEOjeQb3Ae9v6iCVSg9p6DDRWBpleFWJ6dYjpVSGa3O30nK2GRpcYa6H3GBzf\nk50tun1X9jiZnfwMf3jwmsGZ7WKtGSwiIiJTgoKuyBkslbIcj8Q52hXlWHeUI+72aFeUo93Zcz3R\nxLB7K0M+Zrg9wZkQnA7G7nFdeQCPZowuvlTKWRYps1zSnmwQ7tg/yprB6QCcXjt4EQQrivcbRERE\nRE6Dgq6InFQknsiE30wIHnLc2hMjNeQfEQGvh2lVwUyv8Aw3ADdVDQ7FAZ+GShdNcsAJu5n1gt0A\n3L5n+JrBFdPdALwwG4brFjlDpLVMkoiIiJQQBV0RyYtEMkVbb5wjXf05vcIxjnb1u73DMY52Rekf\nSA67tz4cyIbfnOHR8+rKmVcfZlplUD3DxRCPOL2/mQC8O9srHGnLtjMeZ3bo3CHQ6XeDa+aCx1u8\n3yAiIiJnpNMJunohT0RG5fN6MkF1NNZauvsTTk+w2yt8JGeY9NGuKFsOdnK8Lz7ovpDfw9y6cubW\nhZlfX868eicAz6svZ1ZNmSbPKpRAOUw/1/kM1d/pBuDdg98JPvgDiPdk23kDzszQ6Z7f2gXZbc1c\n8AUm7OeIiIiIjEQ9uiIyIWKJJEe7ouxvj7D/eIT9bX3Otr2PA8cjRAeyE2h5PYbZtWXMrStnvht+\n59aVM78hzNy6ckJ+9SZOKGuhrzWnBzhnUqyOfTAQyWlsoHq2G4Td8Ju7X1ZTnN8gIiIik56GLovI\npGKtpaUnxr6c8Lu/PcKB4xH2tfXRPWTirOlVIebWl7s9weFMIJ5bX051mb9Iv+IMZS30tkDHXji+\n19l27Mvu97UObl9W664TvGBIb/B8qJwJHvXki4iIyMgUdEVkSumMxNnXng3ATgjuY197hNae2KC2\nteV+5taHmVfnBOG59WF3W05jRRCjtWYnVqzHXSN4Xzb8predBwfPDu0NQu284b3AdQugZh74Rx9C\nLyIiIlOfgq6InDH6YgkOHB8cfg+0R9jX3sfhzv5Bs0aXB7zMrXPeB073AM+rc4ZGz6wpw6vJsSZW\nMgFdB3PC7z53393Gewe3r5w58nDougVOT7H+EkNERGRKU9AVEQHiiRSHOpx3gtPhN709eLyfeDL7\nXnDA52FhQ5jF0yoGfRY0hAn69E7whLMW+tpywu/ewYG49+jg9sFqpzc4HX6rZ0PldKhogoppztZf\nVoxfIiIiInmioCsichKplOVod5R97nDovW197GrpZVdLLwc7IqT/0egxMK8+zKLGbPhdMq2CRdMq\nqAhq4vqiiUeyIXjosOjOA5AaGH5PsDobeiubckLw9Oz5iiYor9e7wiIiIiVIQVdEZByiA0l2tzqh\nd3dLL7tae9l5rJd97X0MJLP/zJxRHWLxtAoWNVawpKmCxW4Yrq8IFrF6IZV0JsHqPeZMlNVzNLvf\neyzn0zJ8eDSA8brBd4QQPCggN0EgPPG/T0RE5AyldXRFRMYh5PdyzsxqzplZPej8QDLFgeORTM9v\n+vOjzQeJxLOTKtWW+1kyrZJF0wb3As+oDmkyrIng8TrDliunn7xtrHdICG5xhkVnQvIROPIc9LWA\nTQ2/P1A5Si9x0+CQHG5w6hIREZEJoR5dEZFxSqUsR7qj7DzW4/QCt2ZDcEckO4Q2HPA64bfRGfq8\nxA3Bc+vK8Xk1VLakpZIQOT44BPceg55jw3uLY93D7zceCDc6wTc8zdkPN7hb91PhbssbNMO0iIjI\nCDR0WUSkRLT3xtiZ0/u72x0GfbQ7mmkT8HqY31A+uBe4sYKFjWFCfvUCTjrxiNMDPNqw6b5Wd2h1\nKyT6R35GsGpIEG4YPSCX1eqdYhEROSMo6IqIlLie6AC7W/tyhkA7vcEHjkcySyJ5DMypcwLw0qYK\nzppeydKmShY2aiboKSPelw29fbmfNnfbkt2PtI88fNp4c8Lv0HCc3p+WPQ6UT/zvFBERyQO9oysi\nUuIqQ35Wzqlh5ZyaQeejA0n2tWcD8M6WXnYe62HT9hYSbgL2egzz68s5a3olS6ZVugG4gvn1YQ2B\nnmwCYedTO//kbVNJ6O/I6RHOCcG54bhjs7MdaaItAH94eCCumOYMmS6rgVCN00tc5m5DNRpKLSIi\nk46CrohICQn5vSybXsWy6VWDzscTKfa197H9aA87jjmfbUd6+PXWo5ke4IDXw8LGMEubKjO9v0ub\nKphTW47Ho0mwJj1Puue2ATj75O3jEYi0Dekhdvd7W5z9rkNw+M/Ovk2O/ixfKBt6cwPwoP3a4UE5\nVANe/VFDREQmnoYui4hMYtGBJLtaet3w62y3H+3hlc7su59lfi+Lp1W4AbiCJU2VnNVUqVmgJSuV\ngmin8+lPbzuG7/d3QLQrZ79z9J7jtEBlTgA+UVAesh+s1rvHIiIyiN7RFRE5w/XGEuw81jMsALf0\nxDJtKoM+lrjv/maHQFfSUBFQAJZTlxzICcQ5AXikoDw0NCdjJ3iwgVC12zvsfsrroKxuhP1aZ7+8\nzpnIS//7FRGZkhR0RURkRJ2RODuO9bL9WA873fC741jPoGWQasv9meHP6d7fpU0V1JQHili5TEkD\n/aP0Go8QmiPHof94tld5NMabE4RzAvCJwnJ5HfjLJu53i4jImJRM0DXGvB64E/AC37DW3jHkei1w\nL7AIiALvtdZuPZV7R6KgKyJy+qy1tPXGM+/+pnt/dx7rpSeWyLSbVhnM6f11hkAvnlZBVchfxOrl\njJRMOAE44gbf/uODg/Cg/Zzroy3nBO57yCOF4tywPLQ3uQa8+t+/iMhEKYmga4zxAjuAdcAh4Gng\nOmvtSzlt/hnotdbeZoxZBtxtrb3qVO4diYKuiEj+WGs50hXN6f11hkDvbOkhOpBd5qaxMsjixgoW\nTQu72woWNVboHWApPQP9IwTh4zlBuTNnPycgn2iirkCF815xqNodap2zf7LzwUoNsxYROQ2lsrzQ\namCXtXaPW9R9wJuB3LC6HLgDwFr7sjFmvjGmCVh4CveKiEgBGWOYWVPGzJoyrjhrWuZ8KmU52BFh\nx7Fedrf2srull12tvfxsy2F6otke4PKAl0WNFSyeVsGixnBmf159mIBPkwxJEfjLnE/VzFO/x1qI\ndQ8JxR3ZIBztyk7QFe2CzoMQfcHZj3Wf+NnGkxN+RwvGNaMHZg23FhEZVSGD7izgYM7xIeBVQ9o8\nB1wDPGaMWQ3MA2af4r0iIlIEHo9hXn2YefVh1tGUOW+tpbU3xu6WPna3OusA727t5am9x3ngz69k\n2nk9hrl15SwaoRe4ukzDQKXEGJMNlqey3nGuVDIbhDOfzuHhOPda27HstRMNtQbwBkfvMQ5Wgi/o\nfLzB7L4vBN6As/W520HXh7T3BjX7tYhMSsVe3O4O4E5jzBbgBeDPwAnGBw1njFkPrAdoampi06ZN\n+a5RRERO02xgdhWsqQIWeYgmyjnal+Jwn+VIX4ojvVG2HYyw6WVLIucNmuqgYUbYMDPsYUbYw4wK\nw4ywh7qQ0TBomSKq3Q/OLCRh9zMCkxrAl+jDl+h1t87HPzD42JfoxdfXh69rf865CB47MPKDT1PK\n+Eh5fKQ8Aazxk/KkP4HM+fQ553ogp83g9tb4SXoDpDxBkt7gKNsQSW8Qa3wa2i0iY1bIoPsKMCfn\neLZ7LsNa2w1cD2CcP8HsBfYAZSe7N+cZG4AN4Lyju2bNmvxULyIiBZdIpjjU0Z/p/U1vN7f00n0w\nnmmXHga9qDHsDoV2eoHnaxi0yOishWQcEjHnk4xl9xNR91oUEvEhx4Pbe9yPczxS+/Rxn7ONp49z\nvo8xzAljPOAvzw4594fdbc65wAjn/OU5n7LB14e294UUpkWmqEIG3aeBJcaYBTgh9Z3Au3IbGGNq\ngIi1Ng68H3jUWtttjDnpvSIiMvn5vB7mN4SZ3xBm7ZBh0G29cecd4EwA7uPpfR38dMvhTLuhw6DT\n7wEvbAhrOSQRY7JDkIvJWme95WQMBqJOCB7oh4GIu+1zt7nnIhCPDD+X3o+0Db4Wd6+ddqA2g4Ov\n1+8M7fb4wesbx777GbYfcNqNtj/q/YFsfSJySgoWdK21CWPMh4GHcAbn3GutfdEYc4N7/R7gbODb\nxhgLvAi870T3FqpWEREpLcYYGiuDNFYGuXhh/aBrkXiCPa19gybC2t3Sx6M7Wokns7NBV4V8zKsP\nM7e+nHl15cyrL2duXZh59eVMrwrh8agXR2RCGOO+Dxxw3h0uFGud3uNMMM4N0ZHBYToeGRKg3WvJ\nOKQG3GA+kN2PRyDZCamEey0++v6JZukeL+N1Q3kIfOne6tx9t5c60yY0pL177AvltB/tvHq7ZXIr\n6Dq6E03LC4mInLmSKcvB4xF2tfSyr72PA8cj7G+PcOB4hIPHIyRS2X/fBXwe5tSWOUHYDcHpIDyn\nroygz1vEXyIik1oq5QbfUULziPtxZ33o0fYzveH92VCeiLohPff8KG3GKh18faMF6jJ30rLAkN7w\ndC91INs7ndsmve8Z5fxI7T2+nH29snKmKpXlhURERCaM12Myw6CHSiRTHOmKsr89wv7jfRxoj7j7\nEZ7c005fPNsDYwzMqAq5PcFuj3DOvmaGFpET8njA4/Zgl4J0T/ewAOyG4MxQ8n63zYnO5+z3tWbf\n6U4H+2Q827OdjJ+8trEy3pxg7Bs9FKeDuC/du+2GdV8w51pZtvd7WLvQKNdC4NFfiJY6BV0REZny\nfF4Pc+rKmVNXzqU0DLpmraW9L+72/vY5WzcEP/xyC229sUHta8r9zKsrZ2592N2mh0aHmVYZ1JBo\nESktxrhDk0NQVjtx32uts8TW0ACc6c0eGoxHOJ8aGB6eB7XLOT80bKcnQoscdwN5NNvjnYg5oX0s\nk6SlefwjhOXgkCAdGh6W/WXZMJ75eN0e7iHnBh0P/eRc9/pHae916swcn1k94Qq6IiJyRjPG0FAR\npKEiyIXzhv8hsC+WyBkG3ZcZDv3cwU5++cIRkjlDooM+T2YodPp94HQQnl1brhmiReTMYYzb2+oD\nyotdzXDpSdLSPdSJ6JBAnDNxWrpHPB2QT6Vdb8vI7RLRIv5oMzgIe0cIxhf+HVz60SLWmD8KuiIi\nIicQDvo4e0YVZ8+oGnZtIJnicGd/Zhj0gfZsEP7jrnb6B7JDoj0GZlSXMaumjBk1IWZUlzHT3c6o\nDjGzpozacr/WCxYRmQi5k6SFqifue9PLfqWSTi/0iJ+ca8mBEdqmjweGt8/cd4Jnnug7q2ZP3H8W\nBaagKyIiMkZ+r4d59WHm1Q9/L9haS2tvbND7wAfa+zjcFeXZAx0c7TrCQHLwsLmQ35MJvoOCcE0o\nc64q5FMYFhGZrNLLfknBKeiKiIgUgDGGaZUhplWGaJ5fN+x6KmVp64txpDPKka5+Dqe3XVGOdPbz\n+O42jnVHSQ15hSwc8DKjxu0FdkPwzOrBvcTlAf3rXUREzmz6N6GIiEgReDzZIHz+nJoR2ySSKVp6\nYoODsLs90hVl25GeYZNlAVSX+TPDoXO36SA8vTqkJZRERGRKU9AVEREpUT6vh5k1ZcysKePCeSO3\niSWSHOuKcbirf3AQ7oxmhkl3RgaG3ddQERj0fvD06hDTq0JMqwrSVBWiqSpERVB/TBARkclJ/wYT\nERGZxII+L3Pd2Z1H0x9PZnqBD3c623Qo3tfexxO72+mJJYbdFw54aRoSfqdVZvebqoJMqwxRFlDv\nsIiIlBYFXRERkSmuLOBlYWMFCxsrRm3TEx2gpSfGse4oLd3O9lh3jGM9UVq6o/z5QCfHuqPEEqlh\n91aFfNkgnA7FbiCe5gbixsqghkuLiMiEUdAVERERKkN+KkN+Fp0gDFtr6e5PcKwnmg3C3U4QTofi\nJ/f00dITHTajNEBdOJDTI5wThHN6iRsqAvi8Wm9YRETGR0FXRERETokxhupyP9XlfpY2VY7aLpWy\ndETig3qEj+X0Erf0RHn5aDetPbFhs0obAw0VQScIVzpBeHqVu7xSzrrDYb0/LCIiJ6B/S4iIiEhe\neTyG+oog9RVBllM1artkytLeG8uG4B43CHc7PcZHuqJsOdhJe1982L1VIV9mEq0Z1WXMrHZmk86d\nYVrvDouInLkUdEVERKQovB7DNHf48gqqR22Xnlk6M6FWVz9Hu6KZGaZfONQ1YhiuKfdneoAzs0tX\nhTJrD0+vDhHyKwyLiExFCroiIiJS0k5lZunoQJKjXdHMjNKZrbvM0p8PdNAxwjJLdeFAJgjPqC5z\nh0ene4nLaKrWJFoiIpORgq6IiIhMeiG/l/kNYeY3hEdtk15m6WiXE36PdPZzpNvZHuro5+l9HXT1\nj77m8PTqEDOrQ8zIGR49rTJIQ2WQcMCLMaaQP1FERE6Dgq6IiIicEU5lmaW+WIIjXVE3DDs9wke7\nnTWH97f38ac97fREh685XOb30lgZpKEi4G6dJZUG7btbDZcWESk8BV0RERERVzjoY/G0ChZPGz0M\n98YSTm9wV5TWnhhtvTFae2K09jr7e9v6eHpfB8dHeG8YoDLooyEn+I4WjhsqggR8WmpJRGQsFHRF\nRERETkNF0MeSpkqWnGCJJYCBZIr23ng2CLthODccbzvaTVtPjO4ReokBqsv8md7g0cLxtMogdWGt\nPywikktBV0RERKQA/F4P091lj04mOpCkvS+eCcRtQwJxa0+MFw510toToy+eHHa/MVBXng2/deEA\nlSEfFSEflUEfFUEfFSE/FUGfcz6Ycy3ko8yvd4xFZGpR0BUREREpspDfy6yaMmbVlJ20bSSeoK0n\nTmtvlNae+LBe4rbeGAc7IvRGE/TEEsQTqZM+02NwQ7AThsNBLxUhf05I9g0Lydljf+a4IujD61Fg\nFpHiU9AVERERmUTKAz7m1vtOuNxSrlgiSV8s6QbfAXqjCXpjzqcnmqAvZ783lshc7+of4JWOSObc\nSD3JI9fnHdRjHA4O7kGuDPmpKvNRFfJTVeZ3t9njypAPv4Zhi8g4KeiKiIiITGFBn5egz0tdODCu\n5yRTlr54NggPDsYDw4JyT85+e68TmLujA/TFEqTsib+rzO8dEoZ9g0JxZWh4QE63qQz5tPaxiCjo\nioiIiMjJeT3GCZUh/7iek3IDc3c0QXe/E5C7+wfojg6429xjJxy39cbZ09aXuZ48SVIO+jwjBOTB\nx5WZ/ey5iqCP8qCXcEBDsEUmOwVdEREREZkwHo+hMuSnMuQ/pXeSh7LWEoknBwXhTGAeJSx3ROIc\nOB6hu3+Arv4BEifrUgZCfo8TfAPp4ddeygM+95yXsPsuczjoI+y2CWfOu9cC2f2A16MJv0QmkIKu\niIiIiEwaxphMmJxRffr3W2uJDqRyQnE2MPfFkvTFEvTFE+7WPY4l6Isl6YjEOdQRcdq5bU4hMwPg\n85iThuH0fvo953I3YKfPpycDCwd9BH0KziInoqArIiIiImcMYwxlAS9lAS9NVSdf+ulErLXEEil6\nc8JwJiTn7EfiSXpjCSKxBL1DwnR7b9y55rY5lVmyAfxek53oK5gNwMMmAXPPhYODz1eGsvsBnyb/\nkqlHQVdEREREZAyMMYT8XkJ+Lw0Vwbw8cyCZIpITknvd0JwO0725H3fW7B73WkckzsHjkcz1yCnO\nlB3wetww7KUi6HcDsddde9mbDdFDlptKD+0uD3gpd4d2l/m9er9ZSoKCroiIiIhIifB7PVSXe6gu\nH9+kX5CdKbsvZwbs3iH7uUHZOZ+kNzZAa2+Mfe2RzBJU/QOnFprBmQwsHHRCb3kg/XECcdmQY+ec\nM6S7zD2fu597f8iv4dpy6hR0RURERESmoEEzZY/hfeZciWSKvni2Zzm9nFTE7TmODCTpjzu9z/0D\nSSJx53x/PElf3Ll2tHvAPc5eO5WJwdKMwQ3PuSHZeYc5G6Cz19O97SG/hzJ3v8zvJej3ZPazW+ec\n3n2eOhR0RURERETkhHxeD9VlHqrLxt/TnCueSGVCcTr85h5H4k5vcl/MCcvZUJ3M9DRH4kna++KD\nwnUkfuoTheUyxumRHhyMTxaWs9dGCtehnCCdPhf0Occa5l04BQ26xpjXA3cCXuAb1to7hlyvBr4L\nzHVr+bK19pvutX1AD5AEEtba5kLWKiIiIiIiEyvg8xDwBagpz+9zrbXEkymiAyliA04vc3Qg5W6d\n49zz0SH7UTdMRxMp+uNJYgnnXHd0wHmOey7d5mRrO4/G5zGZ0Bv0eQgO3eZe8zmBOehzg7a7HXYt\nfY9/+P25x54pHrILFnSNMV7gbmAdcAh42hjzc2vtSznNPgS8ZK19kzGmEdhujPmetTbuXr/CWttW\nqBpFRERERGTqMca4wc4Lee6FHspay0DSEnXDcDSeIpoOwSME6OhAklgiRSyRytl32sQSTjCPutue\naIK23jixYe2SDCTHFq7T/F6TE5ad7TsvmsP6yxbl6T+Z4ipkj+5qYJe1dg+AMeY+4M1AbtC1QKVx\nBsJXAMeBRAFrEhERERERyRtjDAGfIeDzOO9DT5BkyhJLJImlA3JOCM6E6BNdS6SIDTihPOZey9fs\n4aWgkEF3FnAw5/gQ8Kohbb4K/Bw4DFQC77DWphcPs8BGY0wS+A9r7YYC1ioiIiIiIjJpeD3GnXir\n2JWUpmJPRvUXwBbgSmAR8FtjzGPW2m7gUmvtK8aYae75l621jw59gDFmPbAeoKmpiU2bNk1c9SIi\nIiIiIlJyChl0XwHm5BzPds/luh64w1prgV3GmL3AMuApa+0rANbaFmPMAzhDoYcFXbendwNAc3Oz\nXbNmTb5/h4iIiIiIiEwingI++2lgiTFmgTEmALwTZ5hyrgPAVQDGmCbgLGCPMSZsjKl0z4eB1wFb\nC1iriIiIiIiITBEF69G11iaMMR8GHsJZXuhea+2Lxpgb3Ov3AJ8FvmWMeQEwwCettW3GmIXAA+5i\nzT7g+9baXxeqVhEREREREZk6jDNqeGpobm62mzdvLnYZIiIiIiIikmfGmGestc2n0raQQ5dFRERE\nREREJpyCroiIiIiIiEwpCroiIiIiIiIypUypd3SNMa3A/mLXcQINQFuxixhFqdZWqnWBahuLUq0L\nVNtYlGpdoNrGolTrAtU2FqVaF6i2sSjVuqB0ayvVukC1jcc8a23jqTScUkG31BljNp/qy9MTrVRr\nK9W6QLWNRanWBaptLEq1LlBtY1GqdYFqG4tSrQtU21iUal1QurWVal2g2iaKhi6LiIiIiIjIlKKg\nKyIiIiIiIlOKgu7E2lDsAk6gVGsr1bpAtY1FqdYFqm0sSrUuUG1jUap1gWobi1KtC1TbWJRqXVC6\ntZVqXaDaJoTe0RUREfn/7d1/zJVlHcfx90eERppomfgjG5JWDFQ0zCAVeBBF80faD2tWzlwz/rB0\n0+Ef/dGcfzT+sURasy1HAXOtJWouVH7DICEdoticlfaDAePn4yhAxG9/3NcZN+e5z6b58FwXx89r\ne2uT6wwAAAijSURBVMZ93/AcPuc851zf67qv674fMzMz6yqe0TUzMzMzM7Ou4oHuAJH0gKSVkn6W\nO0udpNMlvSBpn6Rjc+epk3SxpNWSVkl6IHeeFkljUq6Vkh6RpNyZ2km6S9Kq3DlaJI2QtFXSMknP\n5M7TTtJ3JC1O+c7InadF0rSUaZmkzZK+nDsTgKQPS3oq5Xpc0odyZ2qRdKykRyUtlTSzgDx92lhJ\n96R2bZ6kwaVkkzRY0hpJeySdnStXh2xnpTZ3haT5kgYVkuvkVA+WS3pC0tAcuZqy1Y7fKOlfuXJ1\nyiapt9a+fbSgXFMlLUm5PpcjV1M2SWNrr9frku4sKNsxkuamz+ciSSeXkCsdm5Ves1/lajdSjj59\n2hJqQXuukupAf/BAdwBIuhA4PiIuBYZIuih3ppqdwBTgT7mDNPgH0BMRlwCnSDo3d6Dk1YiYkH6e\nAEXdgj0NOsbmztHg2YiYFBFX5A5Slwa2EyNiSsq3KXemlohYmDJNAv4JLMocqWUa8FzKtTbtl+IG\n4MWImAwMlXR+5jyHtbGSTgEmp3ZtA5Dz5EV7+/82VZ7fZUt0SHu23cA1EXEZ8DpwdSG5dgGXRMRE\n4Hngmky5oHM9/yqQdaBLc7aXWu1bROwsIVc6UXE7MDXlej5Trj7ZImJ9rR5sAP5QSjaqPsdb6fP5\nCHBzCblSf3tIes02kvfz2d6nnUgZteCwXMBnKacOvG8e6A6MLwDPpu1FwPiMWQ4TEfsiYlfuHE0i\nYktE7Eu7B4CDOfO0RMSB2u5+8ncg2t0GzMkdosHkNCNzV+4gba4EBqma0Z2V84xvJ5JGAlsjYk/u\nLMnfgOPS9onAjoxZ2o2k6jQArAcmZMzS1MaOA5al7az1oD1bVLbmylPXkG1XRPSm3Wz1oCHXwYh4\nJ+0OAl7LkStl6VPPJV1N9T57p/m7BkaHvsaoVBN+IuVZGdWQazzVa/VHSb+RdFyHbz3iOvXPUqZT\nI+KvGWIBjdk2Ub3/IWNNaMhVTD1o6NOOpoBa0NTXLqUO9AcPdAfGicCbabs37du7JOk84OMR8Uru\nLC2SrpP0MjCcgjr5aenLpIhYkjtLm83Ap4HJwOXpZ1qK4VRnfKcA/wWuz5ynyY3AY7lD1LwGjJe0\nkWrgtjpznrpXgYlpezLltbeuB++DpNOBqUAxl0BI+rykPwM9VLPNJbkFmJs7RAfnAJcBJwHXZs7S\nMhw4DbiKql27PW+cRlcBC3OHaLOdagXNX4DpwO8z52mp14MeCmhvW31aqlUqxdSCEvva/cED3YHR\nC5yQtk+genPbu5Cu23mIapayGBHxRESMAf5N3qUw7b4NzM8dol1E7I+I/0TE21TLrcbkzlTTCyxP\n20uAURmzdHIt8ETuEDW3AE9GxGjgKeBbmfPUPUnV4VpMteKitDPTrgf/p3RZxhzge6ktKUJErI2I\ncVQno76bO0+LpB5gTUS8lTtLk4jYGdWv/lhAOTWhF1gVEQcptx7cQDkDyZYrgG0RMQr4MXB33jiV\niFgPvCxpKVV7m7UetPVpi6kFpfa1+4MHugNjDdU1AwCXU+b1sMVJNxKYC9wdEVty52nR4TfeeRPY\nmytLg88A0yUtBEZLuiN3IABJH6ntfpFq6WspVgOtGeaxFDYjI+lUqmufilk5AIjqWiiozuQPy5jl\nMGkp6R1phv4g8HTuTG3WcWiGwfXgvXkYmF3SjIOkIbXd0urBGOC6Wj24P3egFknH1S4TKakmrOPQ\n4LbEejAYGBURL+bO0qbkmnBfumfDDqoTs1k09GmLqAWl9rX7iwe6AyAiXgD2SVpJtfZ9be5MLenu\naouA84GnJV2cO1PN14CLgJnpjnmlXNs8TdUdNpdTLXMqZgldRMyIiCsjYhqwMSJm5c6UXCrpeUmr\ngU0R8VzuQC3pjO9eScuo3m+l3YDheuDx3CHazAe+nl6zm4F5eeMcIumM1F4sAVbnvrlYexsLnAWs\nUHVX9LFUs1lFZEt33/wt1ezMHEnZlvE3ZJtItYT/zvTzvaGEXMAFqR4spbop269z5OqQ7bmI6KnV\ngx8VlG0MsE7SCuBMMrW7DblGAstTrluBX+TI1ZQt9c96qGaas2p43XZQXXO9DLgP+HkJuVKbtiyt\n8Hkrc9/jsD4t8CnKqAV9+tql1IH+oGrViJmZmZmZmVl38IyumZmZmZmZdRUPdM3MzMzMzKyreKBr\nZmZmZmZmXcUDXTMzMzMzM+sqHuiamZmZmZlZV/FA18zMLANJByWtr33d24+PPULSy/31eGZmZkeb\nY3MHMDMz+4DaGxFjc4cwMzPrRp7RNTMzK4ikNyTNlPSSpLWSzk7HR0haImmDpMWSPpmOD5f0mKQX\n09eE9FCDJP1S0kZJz0gamv79DyS9kh7n0UxP08zM7IjyQNfMzCyPoW1Ll2+q/V1vRJwLPAT8NB2b\nBcyJiPOAecCD6fiDwPKIOB+4ENiYjp8DzI6I0cBu4Cvp+L3ABelxvn+knpyZmVlOiojcGczMzD5w\nJO2JiOMbjr8B9ETE3yUNBrZExMckbQdOi4gD6fjmiDhZ0jbgExGxv/YYI4BnI+KctD8DGBwR90ta\nCOwBFgALImLPEX6qZmZmA84zumZmZuWJDtvvxf7a9kEO3ZfjS8BsqtnfdZJ8vw4zM+s6HuiamZmV\n56ban2vS9mrgG2n7ZmBl2l4MTAeQNEjSsE4PKukY4MyIWArMAIYBfWaVzczMjnY+i2tmZpbHUEnr\na/sLI6L1K4ZOkrSBalb2m+nYHcAjku4BtgG3puM/BB6WdBvVzO10YHOH/3MQMDcNhgU8GBG7++0Z\nmZmZFcLX6JqZmRUkXaM7LiK2585iZmZ2tPLSZTMzMzMzM+sqntE1MzMzMzOzruIZXTMzMzMzM+sq\nHuiamZmZmZlZV/FA18zMzMzMzLqKB7pmZmZmZmbWVTzQNTMzMzMzs67iga6ZmZmZmZl1lf8BOSFU\nUuDAxRIAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1b83471e160>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "train_rmse = [t for e, t, v in model.scbd]\n",
    "test_rmse = [v for e, t, v in model.scbd]\n",
    "epochs = [e for e, t, v in model.scbd]\n",
    "plt.figure(figsize=((16,4)))\n",
    "plt.plot(epochs, train_rmse, label=\"Train\")\n",
    "plt.plot(epochs, test_rmse, label=\"Test\")\n",
    "plt.legend()\n",
    "plt.xticks(epochs, fontsize=8)\n",
    "plt.xlabel(\"Epochs\")\n",
    "plt.ylabel(\"RMSE\")\n",
    "plt.grid(axis=\"y\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Starting training ALS...\n",
      "latent dimention 15\n",
      "learning rate 0.001\n",
      "lambda SGD 0.1\n",
      "lambda ALS users 0.1\n",
      "lambda ALS items 0.1\n",
      "Confidence ALS 1\n",
      "Epochs 21\n",
      "[Epoch 0] : train RMSE = 1.9278\n",
      "[Epoch 0] : validation RMSE = 2.3866\n",
      "[Epoch 1] : train RMSE = 0.8301\n",
      "[Epoch 2] : train RMSE = 0.7815\n",
      "[Epoch 3] : train RMSE = 0.7645\n",
      "[Epoch 4] : train RMSE = 0.7558\n",
      "[Epoch 5] : train RMSE = 0.7504\n",
      "[Epoch 5] : validation RMSE = 1.0312\n",
      "[Epoch 6] : train RMSE = 0.7467\n",
      "[Epoch 7] : train RMSE = 0.7440\n",
      "[Epoch 8] : train RMSE = 0.7420\n",
      "[Epoch 9] : train RMSE = 0.7403\n",
      "[Epoch 10] : train RMSE = 0.7390\n",
      "[Epoch 10] : validation RMSE = 1.0462\n",
      "[Epoch 11] : train RMSE = 0.7379\n",
      "[Epoch 12] : train RMSE = 0.7370\n",
      "[Epoch 13] : train RMSE = 0.7362\n",
      "[Epoch 14] : train RMSE = 0.7355\n",
      "[Epoch 15] : train RMSE = 0.7349\n",
      "[Epoch 15] : validation RMSE = 1.0577\n",
      "[Epoch 16] : train RMSE = 0.7343\n",
      "[Epoch 17] : train RMSE = 0.7338\n",
      "[Epoch 18] : train RMSE = 0.7334\n",
      "[Epoch 19] : train RMSE = 0.7330\n",
      "[Epoch 20] : train RMSE = 0.7326\n",
      "[Epoch 20] : validation RMSE = 1.0672\n"
     ]
    }
   ],
   "source": [
    "# 1b - ALS\n",
    "model = MFModel(X_train, X_test, users, items, hidden_dim=15, lr=0.001, gamma=0.1, reg_u=0.1, reg_i=0.1, epochs=21, optimizer='als')\n",
    "print(\"Starting training ALS...\")\n",
    "model.print_hyper_params()\n",
    "model.train_model()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA7oAAAEJCAYAAAC6xVufAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xuc3HWd5/vXp6u7cyeBhHQgFzogAo1AwJaLJhIuzYA7\nrjuzOiurzo6Xw+gZHZ1ZZ4edM2f0OHv2MHN2Zo4Oug66OLrj5cyugzqjQoISLqLcnAgkAYwkSEJu\nBEhCIJfu/u4f9etOdXVVdSfp6l919ev5eLT1q9/v+/3+3lXBrvr07/f9/SKlhCRJkiRJzaIl7wCS\nJEmSJI0lC11JkiRJUlOx0JUkSZIkNRULXUmSJElSU7HQlSRJkiQ1FQtdSZIkSVJTsdCVJEmSJDUV\nC11JkiRJUlOx0JUkSZIkNZXWvAOMpXnz5qXOzs68Y0iSJEmSxtgjjzzyfErp5NG0bapCt7Ozk4cf\nfjjvGJIkSZKkMRYRz4y2racuS5IkSZKaioWuJEmSJKmpWOhKkiRJkppKU83RlSRJkqRmc/jwYbZs\n2cKBAwfyjjIupk6dyqJFi2hrazvmMSx0JUmSJKmBbdmyhVmzZtHZ2UlE5B2nrlJK7N69my1btrB0\n6dJjHsdTlyVJkiSpgR04cIC5c+c2fZELEBHMnTv3uI9eW+hKkiRJUoObDEXugLF4rRa64+WpVXDf\nX+WdQpIkSZKanoXueHl6Ddz1/8DhyTGBXJIkSVJz2L17N8uWLWPZsmUsWLCAhQsXDj4/dOjQqMZ4\n73vfy5NPPlnnpEd4Marx0rkcfvJZ2PpwcVmSJEmSJoC5c+eydu1aAD75yU8yc+ZMPv7xjw9pk1Ii\npURLS+VjqV/60pfqnrOUR3THy2mXAQGb78s7iSRJkiQdt40bN9LV1cW73vUuzj33XLZt28YNN9xA\nd3c35557Lp/61KcG2y5fvpy1a9fS29vLnDlzuPHGG7ngggu47LLL2Llz55hn84jueJl2Iiw4z0JX\nkiRJ0jH7v/5xHeuf2zumY3adegKfeOu5x9T3iSee4Ctf+Qrd3d0A3HTTTZx00kn09vZyxRVX8Pa3\nv52urq4hffbs2cPll1/OTTfdxO///u9z6623cuONNx736yjlEd3x1LkCnn3QebqSJEmSmsIZZ5wx\nWOQCfP3rX+eiiy7ioosuYsOGDaxfv35Yn2nTpnHdddcB8PrXv57NmzePeS6P6I6npSucpytJkiTp\nmB3rkdd6mTFjxuDyz3/+cz796U/z4IMPMmfOHN797ndXvB9ue3v74HKhUKC3t3fMc3lEdzwtyebp\nbro37ySSJEmSNKb27t3LrFmzOOGEE9i2bRt33HFHblk8ojueps2BU853nq4kSZKkpnPRRRfR1dXF\n2WefzWmnncab3vSm3LJESim3nY+17u7u9PDDD+cdo7Y7/g948Atw4y+hbWreaSRJkiQ1uA0bNnDO\nOefkHWNcVXrNEfFISqm7SpchPHV5vHUuh76DsOWhvJNIkiRJUlOy0B1vSy6DaPH0ZUmSJEmqEwvd\n8TZtDixwnq4kSZIk1YuFbh46l8OWB+Hwq3knkSRJkqSmU7dCNyIWR8RdEbE+ItZFxEcrtHlXRDwa\nEY9FxP0RcUHJts3Z+rUR0eBXmDpKnSug75DzdCVJkiSpDup5e6Fe4N+nlH4aEbOARyJidUppfUmb\nTcDlKaUXI+I64BbgkpLtV6SUnq9jxnwsufTIPN2lb847jSRJkiQ1lboVuimlbcC2bHlfRGwAFgLr\nS9rcX9LlJ8CieuVpKM7TlSRJkjRB7N69m6uuugqA7du3UygUOPnkkwF48MEHaW9vH9U4t956K295\ny1tYsGBB3bIOqOcR3UER0QlcCDxQo9n7ge+XPE/AnRHRB/xNSumWugXMw9IV8MDfFOfptk3LO40k\nSZIkVTR37lzWrl0LwCc/+UlmzpzJxz/+8aMe59Zbb+Wiiy5qjkI3ImYC3wQ+llLaW6XNFRQL3eUl\nq5enlLZGxHxgdUQ8kVK6p0LfG4AbADo6OlizZs1Yv4S6OGnfHM7vO8Taf/oCL514ft5xJEmSJDWo\n2bNns2/fvrxjAHDw4EHa2toG83z1q1/lC1/4AocPH+biiy/mL/7iL+jv7+dDH/oQjz32GCklfuu3\nfov58+ezdu1a3vGOdzBt2jTuuuuumkeCDxw4cFy1XV0L3Yhoo1jkfjWl9A9V2pwPfBG4LqW0e2B9\nSmlr9rgzIm4DLgaGFbrZkd5bALq7u9PKlSvH+mXUx4EL4fH/m2Wz98FEySxJkiRp3G3YsIFZs2YV\nn3z/Rtj+2NjuYMF5cN1No2o6ZcoUpkyZwqxZs3j88ce5/fbbeeCBB2htbeWGG27gu9/9LmeccQZ7\n9uxh3bp1ALz00kvMmTOHL37xi9x8880sW7ZsxP1MnTqVCy+88JhfUj2vuhzAfwM2pJT+skqbJcA/\nAO9JKT1Vsn5GdgErImIGcA3weL2y5mLqbDjlAufpSpIkSZqQ7rzzTh566CG6u7tZtmwZd999N7/4\nxS94zWtew5NPPsnv/u7vcscddzB79uxxz1bPI7pvAt4DPBYRa7N1fwQsAUgpfR74E2Au8LliXUxv\nSqkb6ABuy9a1Al9LKd1ex6z56FxenKd76BVon553GkmSJEmNbpRHXsdDSon3ve99/Omf/umwbY8+\n+ijf//73+exnP8s3v/lNbrllfC+5VM+rLt8HxAhtPgB8oML6p4ELhvdoMp0r4P6/Lt5P9/TL804j\nSZIkSaN29dVX8/a3v52PfvSjzJs3j927d7N//36mTZvG1KlTecc73sGZZ57JBz5QLPlmzZo1bnON\nx+Wqy6piyWVH7qdroStJkiRpAjnvvPP4xCc+wdVXX01/fz9tbW18/vOfp1Ao8P73v5+UEhHBn/3Z\nnwHw3ve+lw984ANMmzbtqG5LdCwipVS3wcdbd3d3evjhh/OOcXRuuQJap8L7vj9yW0mSJEmTzoYN\nGzjnnHPyjjGuKr3miHgkm+o6orpdjEqj1Lm8eOryoVfyTiJJkiRJTcFCN2+dK6D/MGx5MO8kkiRJ\nktQULHTztuTSI/N0JUmSJKmCZppyOpKxeK0WunmbegKcssxCV5IkSVJFU6dOZffu3ZOi2E0psXv3\nbqZOnXpc43jV5UawdAX8+HPeT1eSJEnSMIsWLWLLli3s2rUr7yjjYurUqSxatOi4xrDQbQSdK+BH\nny7O0z19Zd5pJEmSJDWQtrY2li5dmneMCcVTlxvB4ksgCrDp3ryTSJIkSdKEZ6HbCKaeAKc6T1eS\nJEmSxoKFbqPoXA5bH4FD+/NOIkmSJEkTmoVuoxi4n+6z3k9XkiRJko6HhW6jWHJpcZ6upy9LkiRJ\n0nGx0G0UU2bBqRda6EqSJEnScbLQbSTO05UkSZKk42ah20gG5+k+kHcSSZIkSZqwLHQbyZJLnKcr\nSZIkScfJQreROE9XkiRJko6bhW6jWbrCebqSJEmSdBwsdBtN53Lo73WeriRJkiQdIwvdRrM4u5/u\npnvzTiJJkiRJE1LdCt2IWBwRd0XE+ohYFxEfrdAmIuIzEbExIh6NiItKtl0bEU9m226sV86GM2Um\nLLzIebqSJEmSdIzqeUS3F/j3KaUu4FLgdyKiq6zNdcCZ2c8NwH8FiIgC8NlsexdwfYW+zatzOTz3\nUzj4ct5JJEmSJGnCqVuhm1LallL6aba8D9gALCxr9jbgK6noJ8CciDgFuBjYmFJ6OqV0CPhG1nZy\ncJ6uJEmSJB2z1vHYSUR0AhcC5ZXbQuDZkudbsnWV1l9SZewbKB4NpqOjgzVr1oxF5FwVeg/zpijw\n7D1fY9OWQt5xJEmSJGlCqXuhGxEzgW8CH0sp7R3r8VNKtwC3AHR3d6eVK1eO9S7ysfn1nNb/S05r\nltcjSZIkSeOkrlddjog2ikXuV1NK/1ChyVZgccnzRdm6ausnD+fpSpIkSdIxqedVlwP4b8CGlNJf\nVmn2HeA3s6svXwrsSSltAx4CzoyIpRHRDrwzazt5DM7T/UneSSRJkiRpQqnnqctvAt4DPBYRa7N1\nfwQsAUgpfR74HvAWYCPwCvDebFtvRHwYuAMoALemlNbVMWvjWXwJtLQWbzP0mqvzTiNJkiRJE0bd\nCt2U0n1AjNAmAb9TZdv3KBbCk9OUmbDw9d5PV5IkSZKOUl3n6Oo4dS6Hrc7TlSRJkqSjYaHbyDqX\nQ+pznq4kSZIkHQUL3XHyjQd/yY3ffPToOg3M0910b31CSZIkSVITstAdJ8/tOcDfP/wsu18+OPpO\n7TOcpytJkiRJR8lCd5xc09VBf4IfPLHz6Dp2Lofn/hkO7qtPMEmSJElqMha64+TcU0/g1NlTWb1+\nx9F17FxRnKf7ywfqE0ySJEmSmoyF7jiJCHq6Orj357t49VDf6Dsuvhha2mCz83QlSZIkaTQsdMdR\nT9cCDhzu596f7xp9J+fpSpIkSdJRsdAdR5ecfhKzprYew+nLztOVJEmSpNGy0B1HbYUWrjhrPj98\nYid9/Wn0HQfup/tL76crSZIkSSOx0B1n15zbwe79h/jpL18cfafFlzhPV5IkSZJGyUJ3nF3+2pNp\nKwSr1m0ffaf26bCo23m6kiRJkjQKFrrjbNbUNi47Yx6r1+8gpaM8ffm5tXBgb/3CSZIkSVITsNDN\nQU9XB5t3v8LGnS+PvtPAPN1nvZ+uJEmSJNVioZuDnnM6AFh1NFdfXpTdT3fTPXVKJUmSJEnNwUI3\nBwtmT+WCRbOP7jZDztOVJEmSpFGx0M1JT1cHa599iR17D4y+U+dy2OY8XUmSJEmqxUI3Jz1dCwC4\nc8NRHNXtXAGp3/vpSpIkSVINFro5eW3HTJacNP3oTl9e9AYotHs/XUmSJEmqwUI3JxFBT1cH92/c\nzcsHe0fXqX06LHSeriRJkiTVYqGbo2u6OjjU1889T+0afafBebp76hdMkiRJkiawuhW6EXFrROyM\niMerbP+DiFib/TweEX0RcVK2bXNEPJZte7heGfP2+tNO5MTpbaxat330nTqXO09XkiRJkmqo5xHd\nvwWurbYxpfT/ppSWpZSWAf8RuDul9EJJkyuy7d11zJir1kILV57dwQ+f2Mnhvv7RdVp8sfN0JUmS\nJKmGuhW6KaV7gBdGbFh0PfD1emVpZD1dHew90MtDm0b5VrVNK16Uynm6kiRJklRRa94BImI6xSO/\nHy5ZnYA7I6IP+JuU0i01+t8A3ADQ0dHBmjVr6ph27EVvoq0FvrT6EQ5tmTKqPp0s4rTn/gf33fld\n+lpn1DmhJEmSJE0suRe6wFuBH5Wdtrw8pbQ1IuYDqyPiiewI8TBZEXwLQHd3d1q5cmXdA4+1y7c+\nxPpt+7j88suJiJE7nNYCX/7/WbGkFV67su75JEmSJGkiaYSrLr+TstOWU0pbs8edwG3AxTnkGjc9\nXR1sfelV1m/bO7oOA/fT3VSx9pckSZKkSS3XQjciZgOXA98uWTcjImYNLAPXABWv3Nwsrjy7gwhY\nvX7H6Do4T1eSJEmSqqrn7YW+DvwYOCsitkTE+yPigxHxwZJmvwasSintL1nXAdwXET8DHgS+m1K6\nvV45G8HJs6Zw0ZITR1/oQvE2Q9sfhVdfql8wSZIkSZqA6jZHN6V0/Sja/C3F2xCVrnsauKA+qRpX\nT1cHN33/Cba+9CoL50wbuUPnCrj7z4r30z2r6l2cJEmSJGnSaYQ5ugKu6eoA4M7RHtVd9AYoTPF+\nupIkSZJUxkK3QZx+8kzOOHkGq9ZvH12HtqnO05UkSZKkCix0G0hP1wIeePoF9rx6eHQdnKcrSZIk\nScNY6DaQnq4OevsTa57cOboOncsh9cMvf1zfYJIkSZI0gVjoNpALF89h3swprDrqebqevixJkiRJ\nA2oWuhFxZcny0rJtv16vUJNVS0vQ0zWfu5/cxcHevpE7tE2FxRd7QSpJkiRJKjHSEd3/UrL8zbJt\nfzzGWUTx9OWXD/by41/sHl2HzuWwzXm6kiRJkjRgpEI3qixXeq4x8MYz5jG9vcDq0Z6+3LkcSM7T\nlSRJkqTMSIVuqrJc6bnGwNS2Am8+82Tu3LCD/v5RvMULu4vzdDd5+rIkSZIkAbSOsP30iPgOxaO3\nA8tkz5dW76bj0dPVwe3rtvPY1j1csHhO7cbO05UkSZKkIUYqdN9WsvxfyraVP9cYufLs+RRagtXr\nd4xc6ELx9OU1N8GrL8K0E+sfUJIkSZIaWM1Tl1NKd5f+APcDe4EN2XPVwYkz2nlD54msWr99dB06\nVwAJnnGeriRJkiSNdHuhz0fEudnybOBnwFeAf46I68ch36TV07WAp3a8zDO794/ceOHroXWq99OV\nJEmSJEa+GNWKlNK6bPm9wFMppfOA1wP/oa7JJrlrujoARnf15bapsOgNztOVJEmSJEYudA+VLPcA\n3wJIKY3ynFodq8UnTefsBbNYNerbDK2A7Y/BKy/UN5gkSZIkNbiRCt2XIuJXI+JC4E3A7QAR0QpM\nq3e4ye6arg4e3vwCL+w/NHJj76crSZIkScDIhe5vAx8GvgR8rORI7lXAd+sZTMV5uv0JfrBhFEd1\nF3U7T1eSJEmSGOH2Qimlp4BrK6y/A7ijXqFU9LqFJ3DK7KmsXr+Dd3Qvrt24dYr305UkSZIkRih0\nI+IztbanlH53bOOoVERw9Tkd/M9HtnDgcB9T2wq1O3SugLv+c3Ge7vSTxiekJEmSJDWYkU5d/iCw\nHHgOeBh4pOxHddbT1cGrh/u47+fPj9zYebqSJEmSNGKhewpwC/ArwHuANuDbKaUvp5S+XO9wgktP\nn8usKa2ju83QwP10N3n6siRJkqTJq2ahm1LanVL6fErpCor30Z0DrI+I94w0cETcGhE7I+LxKttX\nRsSeiFib/fxJybZrI+LJiNgYETce5WtqKu2tLaw8ez53bthBX3+q3Xhwnq4XpJIkSZI0eY10RBeA\niLgI+CjwbuD7jO605b+lwoWsytybUlqW/Xwq21cB+CxwHdAFXB8RXaPJ2ax6ujrYvf8Q//zLF0du\n3Plm2PG499OVJEmSNGnVLHQj4lMR8Qjw+8DdQHdK6f0ppfUjDZxSugc4lmrrYmBjSunplNIh4BvA\n245hnKax8qyTaSvE6E5fHpin+8z9dc8lSZIkSY2o5lWXgT8GNgEXZD//OSIAAkgppfOPc/9vjIhH\nga3Ax1NK64CFwLMlbbYAl1QbICJuAG4A6OjoYM2aNccZqTG9dk7w7Yc3cdn02sVu9B9meUs72+77\nBht3zByndJIkSZLUOEYqdJfWcd8/BZaklF6OiLcA3wLOPNpBUkq3ULxgFt3d3WnlypVjGrJRPDtl\nM//nt9exqKub18wfoYB99jIWvbKZRU36XkiSJElSLSNdjOqZSj8Uj7guP54dp5T2ppRezpa/B7RF\nxDyKR3cXlzRdlK2b1K7u6gBg1frtIzfuXAE7HnOeriRJkqRJaaQ5uidExH+MiJsj4poo+gjwNPAb\nx7PjiFgQA+dBR1ycZdkNPAScGRFLI6IdeCfwnePZVzM4ZfY0zls4+yjm6QLP/Ki+oSRJkiSpAY10\n1eX/DpwFPAZ8ALgLeDvwr1JKNS8QFRFfB34MnBURWyLi/RHxwYj4YNbk7cDjEfEz4DPAO1NRL/Bh\n4A5gA/D32dzdSa+nq4O1z77Ezn0Hajdc+HponeZthiRJkiRNSiPN0T09pXQeQER8EdhGcV7tCJUW\npJSuH2H7zcDNVbZ9D/jeSPuYbHq6OvjL1U/xgw07uf7iJdUbtrbDkkssdCVJkiRNSiMd0T08sJBS\n6gO2jKbIVX2cvWAWi0+aNvrTl72friRJkqRJaKRC94KI2Jv97APOH1iOiL3jEVBHRAQ95yzgvo3P\ns/9gb+3GnSuKj87TlSRJkjTJjHTV5UJK6YTsZ1ZKqbVk+YTxCqkjero6ONTbzz1P7ard8NSLivN0\nN907PsEkSZIkqUGMdERXDeYNnScyZ3rbyKcvO09XkiRJ0iRloTvBtBZauPLs+fzwyZ309vXXbty5\nAnaug/27xyecJEmSJDUAC90J6JquDl565TAPbX6xdkPn6UqSJEmahCx0J6AVZ55Me2sLq9Zvr93w\n1AuhbbqnL0uSJEmaVCx0J6AZU1pZ/pp5rF6/g5RS9Yat7bDYebqSJEmSJhcL3Qmqp6uDLS++yhPb\n99Vu2Lk8m6f7/PgEkyRJkqScWehOUFedM58IRr76svN0JUmSJE0yFroT1PxZU7lw8ZyRC92FFzlP\nV5IkSdKkYqE7gfV0LeCxrXt47qVXqzcqtMGSSy10JUmSJE0aFroTWE9XBwB3bhjp9OXlsHO983Ql\nSZIkTQoWuhPYa+bP5PR5M5ynK0mSJEklLHQnuJ5zO/jJ07vZe+Bw9UYD99PddO/4BZMkSZKknFjo\nTnDXdHVwuC+x5sld1Rs5T1eSJEnSJGKhO8EtW3wi82a2s2rd9toNO1fArg3wco2CWJIkSZKagIXu\nBFdoCa46u4O7n9zFod7+6g2dpytJkiRpkrDQbQI9XR3sO9jLT57eXb3RqcugbYanL0uSJElqeha6\nTWD5mfOY1laoffVl5+lKkiRJmiTqVuhGxK0RsTMiHq+y/V0R8WhEPBYR90fEBSXbNmfr10bEw/XK\n2CymthV482vnsXr9DlJK1Rt2LneeriRJkqSmV88jun8LXFtj+ybg8pTSecCfAreUbb8ipbQspdRd\np3xNpadrAdv3HuCxrXuqNxqcp+tRXUmSJEnNq26FbkrpHuCFGtvvTym9mD39CbCoXlkmgyvPnk9L\nUPv05VOXQftMT1+WJEmS1NQaZY7u+4HvlzxPwJ0R8UhE3JBTpgnlpBntdHee5DxdSZIkSZNea94B\nIuIKioXu8pLVy1NKWyNiPrA6Ip7IjhBX6n8DcANAR0cHa9asqXfkhnXGlMN8fdMh/sf3fsjJ0yv/\nDWNx/6mcsetOfrTqWxxunzPOCSVJkiSp/nItdCPifOCLwHUppcF746SUtmaPOyPiNuBioGKhm1K6\nhWx+b3d3d1q5cmW9Yzes03e/wtefuIu9JyzlHcuXVm60ZSY8/RXetDDBuSvHNZ8kSZIkjYfcTl2O\niCXAPwDvSSk9VbJ+RkTMGlgGrgEqXrlZQy2ZO52zOmaxat326o1OuaA4T3fTveMXTJIkSZLGUd2O\n6EbE14GVwLyI2AJ8AmgDSCl9HvgTYC7wuYgA6M2usNwB3JatawW+llK6vV45m01PVwefW7ORF/cf\n4sQZ7cMbFNpgyWXO05UkSZLUtOpW6KaUrh9h+weAD1RY/zRwwfAeGo2erg5uvmsjP3xiJ//69VUu\nZN25HO78BLy8E2bOH9+AkiRJklRnjXLVZY2R8xbOZsEJU2tffXngfroe1ZUkSZLUhCx0m0xLS3B1\n13zu+fkuDhzuq9xoYJ6uha4kSZKkJmSh24R6uhbwyqE+frTx+coNCq3O05UkSZLUtCx0m9Clp5/E\nzCmtI5y+vByefxL21WgjSZIkSROQhW4TmtJa4PKzTubODTvp70+VGy3N5uk+41FdSZIkSc3FQrdJ\nXdPVwfMvH+Sfn32pcoMFF0D7LE9fliRJktR0LHSb1Mqz5tPaEtVPXy60wmnO05UkSZLUfCx0m9Ts\naW1cevpcVq3fXr1R53J4/inn6UqSJElqKha6Taynq4Ond+3nF7tertygc3nx0Xm6kiRJkpqIhW4T\nu7qrA6D66csD83Q33TuOqSRJkiSpvix0m9jCOdN43cITRpin+0bn6UqSJElqKha6Ta7nnAX89Jcv\nsmvfwcoNOpfD7p/DvhpzeSVJkiRpArHQbXI9XR2kBD/YUOWo7sA8XY/qSpIkSWoSFrpN7pxTZrFw\nzrQa83TPhyknWOhKkiRJahoWuk0uIujp6uC+jc/zyqHe4Q0KrbDE++lKkiRJah4WupPANed2cLC3\nn3ueer5yg4F5unu3jW8wSZIkSaqD1rwDqP4u7jyJ2dPaWL1+B9e+bsHwBktXFB+f+RGc9/bxDSdJ\nkqSjk1L20w+UL/dXeZ5G2F6t/SjaDrartq1SP0YxZuk2RjFm+bYar6HiGNR4rZXGYJTtSl5zxXa1\nliu9hrLlapmqvd5q/3YkWPZv4Y0fGbP/VPNkoTsJtBZauPLs+fzgiR309vXTWig7kD84T/deC11J\nko5V6Rf9IV90+4euH/Klvb9Gn/6hY1bcVrJcc3uFL+9Vt5ftt2a2SsVGtddU7bWUt62Qr2L7CsXK\nMbctL6ZGeF3lhd1oisaK/wZlhd+I42TPNU4CIrLHluJytBxZP7jcAsEo2pUvl41dcznKlkcYv6W8\n7yjzTJ83vm9xHVnoThI9XR3c9s9befiZF7n09LlDN7YUvJ+uJFVT6Qt/xZ9KbfpG2D6aMSq06S8Z\nd6Tio9IPtV7TSMVKpfZVth1T4TSKog5qv2+VCoyq+6TKtlpFZJX3RUdnSJFQUhwMWR9V1lcpJqqO\nUV48lPcv61u1zQj7rTRWaSFUc6zS54ywvUoRNFLb4xl7pPeg4uustY8KRdao+422+Kz2/lRrV/7f\niyYyC91J4s2vPZn2Qgur1+8YXuhCcZ7uU7cX5+mecMr4B5SaSemX4NKCZLDwSTW2VShkygum/mqF\n1EjbSvY94vZx7FutKOs/miLxGMavtH3Y+54VMioaVgTUKFQGvji2FKoUAFFjHEa3j5ZC2bZKBUa1\nfKXbC0P3WXNMRhivwms4rvEG8tUq+Cq8r7X2Oer3ZrT/XrX2Xa3wtIiQVF8WupPEzCmtvOk1c1m9\nfgd//C/OIco/YAbup/tPH4MTl0JrOxSmQKH9yHJre/H54PIUKLRB65Sy7e1H1pVub2kZ/xeu4Qa/\n0PdmX+r7io9Dlnsrr0/Ztv7+kuVaY/RXGK9k/ZAxKmSqWPT0lRROfUPHGlJU9Q3tM6QIrLatb/i+\nBovO8m01irJJURiVFDCDP4UjX2JrbSstUmp9ER/Sb6B9W4U2VfqPavxKYxQqLI92jCrbh2Q5xjGO\nps2wIqZW32qFUXkbCxNJ0sRRt0I3Im4FfhXYmVJ6XYXtAXwaeAvwCvBbKaWfZtuuzbYVgC+mlG6q\nV87JpKdoXcdRAAAWAUlEQVRrAXfd9hhP7XiZsxbMGrpxwfmw5I2w5WF45n7oPQh9B8c2QEtrjUK4\n/cjjsOWB4npKcQxgsJBIFQqKwXWV2pSvq0ebNLSgqlYc1SrCKhVbY9Wn0U+vGyhuBguDQjbPpKTo\naSmULLdU7jOk2CrZNlAsDRmjvJgq31eUjV86ZgxdN2ScWtuqFFLDirNRbKtYVFIhc8l7U7Nv1Nhm\n0SNJkhpfPY/o/i1wM/CVKtuvA87Mfi4B/itwSUQUgM8CPcAW4KGI+E5KaX0ds04KV58znz+6DVat\n2z680G0pwPu+P3RdSsUjbL0Hoe9Q8WdgudK6o95+uFhMD9l+CA6/VHzsO5Rtzx77DhfXkX3BHvyi\nXfKFu3xdxTaMos1oxqnSpmoRUlroVCrcyouwUfapVNRVK6bKC7+BYq+ltSRDa8n6bNtA35bWCn1L\n17eUjVdtP1nbIeO1HskrSZIkHYe6FboppXsiorNGk7cBX0kpJeAnETEnIk4BOoGNKaWnASLiG1lb\nC93jNP+EqSxbPIfVG3bwkavOHLlDRPGIa6Gt/uEkSZIkaYy05LjvhcCzJc+3ZOuqrdcY6Onq4NEt\ne9i+50DeUSRJkiSpLib8xagi4gbgBoCOjg7WrFmTb6AGd+L+4vzMz337Xq5c4pFaSZIkSc0nz0J3\nK7C45PmibF1blfUVpZRuAW4B6O7uTitXrhzzoM0kpcQXnribzb3TWbny4rzjSJIkSdKYy/PU5e8A\nvxlFlwJ7UkrbgIeAMyNiaUS0A+/M2moMRAQ9XR38+BfPs/fA4bzjSJIkSdKYq1uhGxFfB34MnBUR\nWyLi/RHxwYj4YNbke8DTwEbgC8D/DpBS6gU+DNwBbAD+PqW0rl45J6Oerg4O9yXufnJX3lEkSZIk\naczV86rL14+wPQG/U2Xb9ygWwqqDi5acyNwZ7axev4O3XnBq3nEkSZIkaUzleeqyclJoCa46Zz53\nPbmTw339eceRJEmSpDFloTtJ9XQtYN+BXh54+oW8o0iSJEnSmLLQnaSWv2YeU9taWLV+e95RJEmS\nJGlMWehOUtPaC6w482TuXL+D4nRpSZIkSWoOFrqTWE9XB8/tOcC65/bmHUWSJEmSxoyF7iR21dnz\naQlYtX5H3lEkSZIkacxY6E5ic2dOofu0k1htoStJkiSpiVjoTnI9XR1s2LaXZ194Je8okiRJkjQm\nLHQnuZ6uDgCP6kqSJElqGha6k1znvBmcOX+mha4kSZKkpmGhK645t4MHN7/AS68cyjuKJEmSJB03\nC13R07WAvv7EXU/uzDuKJEmSJB03C11x/sLZzJ81hVXrPH1ZkiRJ0sRnoStaWoKruzq4+6ldHDjc\nl3ccSZIkSTouFroCildffuVQHzf/cCM/eXo32/ccIKWUdyxJkiRJOmqteQdQY3jjGXNZOGcaN9+1\nkZvv2gjAtLYCp82dTufcGXTOm8HSedM5be4Mls6bwfxZU4iInFNLkiRJ0nAWugJgSmuBe/7DFTz3\n0qts3r2fzc/vZ9Pzr/DM7v08tXMfP3hiB4f7jhzhHSiCl84rFsGdWUG8dN4MTrYIliRJkpQjC10N\nKrQEi0+azuKTprPizJOHbOvt62fbngNsen5/Vgi/wubd+3ly+z5Wr99Bb/+RInh6eyE78psdAZ47\nY7AotgiWJEmSVG8WuhqV1kLLYBH8ZoYXwc+9dIBNu/fzzO79xWL4+f08sW0fq9ZVL4I7584YPC26\nc950Tp5pESxJkiTp+Fno6ri1FlpYMnc6S+ZOhwpF8NaXXmXz7ley06GLxfCGCkXwjMEiuHgEuDgv\nuLhsESxJkiRptCx0VVethRZOmzuD0+bO4PLXVi6CB44Ab95dPB163XN7uH3ddvpKiuCZU1o5be50\nFp04jZlT2pg5pcCMKa3MmNLKzCmtTG8vMDN7Xmlde6sXGJckSZImi7oWuhFxLfBpoAB8MaV0U9n2\nPwDeVZLlHODklNILEbEZ2Af0Ab0ppe56ZtX4Ky2COWvotsN9/Wx98dXi6dBZEbzp+f08vWs/+w/2\nsv9QH/sP9g45IlxLe6GFGVMKTG9vzYrfQklB3GrhLEmSJDWRqNe9UiOiADwF9ABbgIeA61NK66u0\nfyvweymlK7Pnm4HulNLzo91nd3d3evjhh483uiaIlBIHe/uLhe/BPvYf6mX/wV5eHnh+sLdkXfZ8\ncF1f1q6kz6G+IUeRa6lVOE9tK9BeaKGtNWgvFGhrDaYUWmhvbaGt7HHKwHKhhbbW4mN7yWNbIYrP\ny7YVWsJTuSVJkjSpRMQjoz0AWs8juhcDG1NKT2ehvgG8DahY6ALXA1+vYx41mYhgaluBqW0F5s48\n/vHKC+eXD/byyqGhhfORdX1DiuRXDvWx70Av2/cc4EBvH4d6+znclzjU28+hvn4O9fYff8ASEcVi\ne7D4rVREDxbbR9YNFMqthaC1pVgwt7ZEyWNxW8X1A88LVdaXbB8cu1CjXUsLhcLQ/Vi8S5IkaSzU\ns9BdCDxb8nwLcEmlhhExHbgW+HDJ6gTcGRF9wN+klG6pV1AJxr5wLpVS4nBf4nBW9B7u6+dgVgQP\nW1dSJA9sO9jXz+GB9iXF86GSvkeep8F2Bw73s/fV3rLt/fT1J3r7U/ZYfF56n+S8tAS0trTQ0gKF\nCFqyIrglij+FkvXF50FLUNYm6xcU17UEhZL1LVEydgQtLQz2OzL2kTHL9z/YL4rtomS5pSWIgf1m\nj1GyPNB+IPeQvlHat3T7wOs4yvbZ9ggIsv4caTfQN2Bo29LxqdyWYNj+YGimYl/844UkScpFo1yM\n6q3Aj1JKL5SsW55S2hoR84HVEfFESume8o4RcQNwA0BHRwdr1qwZl8BSPbVlPxUVsp/2Yxm5Jfup\nrD8l+hP0JYqP/dlj+frB7UfWl64b1i4Vx+5L0N9fZf3A82x7Son+LFPK9l98DikdyTq4LsvaT7F/\n78D2VPyr2eBySZ/i2FQYm8F9D19XfJ448qjaigVytlysk488Mnw9ZP+VxsD2ONK/pE3pOJTuI/uf\n0jGO7Cdq7Ieq+ynNOGw5G2BwP0O2Dc9bdczy7dkfKaq9R6Wvq3z/Q/ZR0mdIv5J9QwzJUi3v8Pdz\n+OuptZ+h+WJUfYfsH2rmHL6PEfqWdIyyx2G5y9YN61eeocq60vXlr2NonyPvT/nfi4ZlqJSrwj6G\nPK/y+oZkqTDe8LxReX2VMSuPcXTrh6yr9jpHsd/SjeX/PQ7dNvL+K41fc1uFlSONX/O1VNnf0DxV\nGkhNqp6F7lZgccnzRdm6St5J2WnLKaWt2ePOiLiN4qnQwwrd7EjvLVCco7ty5crjDi5Jo5UGi+VU\nVjynkkI5Df4RIJW066/Yt2R7f+3xao3R159lG5IREkfGSGV5y9sOX3fkEcr/EJCt60/ZHxaKbQcy\nlLZNJbmP/MFgaPuhmQCK78dgOyhrnwb/oDGwTEnmof2GLveXLDMk+9B+VBpnMH9xYTA7pWMcGXtY\nlv5UMnb5mEfGoGzbwHszMCalOcvHGdh3hf0cydQ/9A835a+9bExJzaPiH14q/BGotG1xfeUqfDTt\naxXw1fZdfR+Vx63Zbtg+K/eqPV75tqN/feWG9Kv2Xh/F+MP2NIp+v9G9iBvefEbVjBNJPQvdh4Az\nI2IpxQL3ncC/LW8UEbOBy4F3l6ybAbSklPZly9cAn6pjVkk6JoOn9lb8W7vUvIYW75WK5zSkKB5Y\nV94HGFpAl/Wt1G5wT0MK8Rr9EkMfa4x/pF1Z37L+5eOWryvfX+nrHvL+DelTY58VM2ZjJ6pvKxtv\nYKyyXVZ8TUcyVN53pbyj2X/p62GkPjX6jvgahuWt8u9UFmLovqrs8yhf10j7Hqnd0DFrvIaj2Gfp\nhpH6H+3+yhtVe4/KX+PR5hjSt/y9H03GYdsqry/fOiz3KLPWGn80712xX6qx7dj6la6YN3NK+dYJ\nq26FbkqpNyI+DNxB8UTLW1NK6yLig9n2z2dNfw1YlVLaX9K9A7gt+8tCK/C1lNLt9coqSZKOzsBf\n/8tPtZUkqRHU7fZCefD2QpIkSZLUnI7m9kLVr0ojSZIkSdIEZKErSZIkSWoqFrqSJEmSpKZioStJ\nkiRJaioWupIkSZKkpmKhK0mSJElqKha6kiRJkqSm0lT30Y2IXcAzeeeoYR7wfN4hRtDoGRs9HzR+\nxkbPB2YcC42eDxo/Y6PnAzOOhUbPB42fsdHzQeNnbPR8YMax0Oj5oPEznpZSOnk0DZuq0G10EfHw\naG9wnJdGz9jo+aDxMzZ6PjDjWGj0fND4GRs9H5hxLDR6Pmj8jI2eDxo/Y6PnAzOOhUbPBxMj42h5\n6rIkSZIkqalY6EqSJEmSmoqF7vi6Je8Ao9DoGRs9HzR+xkbPB2YcC42eDxo/Y6PnAzOOhUbPB42f\nsdHzQeNnbPR8YMax0Oj5YGJkHBXn6EqSJEmSmopHdCVJkiRJTcVCd5xExF9FxL0R8em8s1QSEadG\nxE8j4kBEtOadp1xEXBIR90fEfRHxV3nnqSQiXpdlvDcivhQRkXemSiLi9yLivrxzVBIRnRGxIyLW\nRMSqvPNUExG/GRE/yHIuzDtPqYi4Nsu1JiK2RcS/yjtTuYiYHhHfzTJ+OyKm5J2pVES0RsQ3IuKu\niPjzvPOUqvS7OiL+IPvd+NWIaGukfBHRFhE/joiXI+I1eWYbUCHj0uz39j0R8bWIKDRYvnnZZ8vd\nEfGdiJiWZ75KGUvW/3pEPJtntgFV/r+yp+T340kNmK8nIn6Y5Xt9nvkqZYyIZSXv36aI+FiD5WuJ\niL/L/r98Z0TMyzNfpYzZur/O3sNbG+D3zbDv1430mXK8LHTHQURcBMxMKa0A2iPiDXlnquAF4Crg\nJ3kHqeIZ4MqU0nJgfkScl3egCp5MKb0x+3cGaLhLs2cFxbK8c4xgdUppZUrpmryDVJIVtpenlK7K\ncm7NO1OplNLtWa6VwC+BO3OOVMm1wANZxgez543k14CfpZSuAKZFxAV5Byox5Hd1RMwHrsh+Nz4K\n5P2HjfLPkl6Kmf5nbomGK8/4EvCrKaU3A5uAt+QVLFOe70VgeUrpcuAR4FfzClai2neGtwMNUehS\nOeNjA78fU0ov5JRrQPn/l6cBvw30ZPkeyTNcZkjGlNLaks+XR4F/yjEbDP83XgYcyv6//CXgXXkF\nK1H+7/wGoD17D9eR//+fy79fX05jfaYcFwvd8XEpsDpbvhO4LMcsFaWUDqSUXsw7RzUppe0ppQPZ\n08NAX555KkkpHS55epDG+bAv9X7gy3mHGMEV2dGV38s7SBW/AhSyI7p/nfdfY6uJiNOBHSmll/PO\nUsEvgBnZ8hxgd45ZKjmd4gc8wFrgjTlmGaLC7+puYE22nPvnS3m+VLQjz0zlKmR8MaW0J3ua++dL\nhXx9KaX+7GkB+Hk+yY6o9J0hIt5C8b/B/sq9xleV7zXnZJ8vN0Xke9ZVhXyXUXzvvh8R/z0iZlTp\nOm6qfTfMsi1IKW3MIdagCvm2Uvz/CDTIZ0uFjA31+VLh+/W5NNBnyvGy0B0fc4C92fKe7LmOQUSc\nD5ycUlqfd5ZKIuJfRsTjQAcN8Au2VHb6ycqU0g/zzlLDNuC1wBXA1dm/d6PpoPjX2KuAV4C35Zyn\nml8Hbss7RBU/By6LiHUUC7X7c85T7kng8mz5Chr7d7afL2MkIk4FeoCGmzYRERdHxMPAlRSPOjei\nfwf8Xd4hRnAm8GbgROCtOWcp1wGcAlxH8Xfib+cbp6brgNvzDlHB8xTPwtkAfAj4h5zzVFL6+XIl\nDfI7e+D7NcUzXJrmM8VCd3zsAU7Ilk+g+B+RjlI2n+ZmikclG1JK6TsppdcBW8j/dJRy7wG+lneI\nWlJKB1NK+1NKvRRPiXpd3pkq2APcnS3/EDgnxyy1vBX4Tt4hqvh3wD+mlM4Fvgu8O+c85f6R4pel\nH1A8O6OhjkiW8fNlDGTTOr4M/G/Z75+GklJ6MKXUTfGPV+/LO0+5iLgS+HFK6VDeWWpJKb2Qircb\n+RaN9/myB7gvpdRHY3+2QHF6RyMWkdcAu1JK5wCfBD6eb5zhUkprgccj4i6Kv7Nz/3wp+37dVJ8p\nFrrj48cUz88HuJrGnQfbsLIJ/H8HfDyltD3vPJXE0Avq7AVezStLFWcBH4qI24FzI+IjeQcqFxGz\nSp6+ieIpro3mfmDgSPMyGvDoSkQsoDhPqaHOKigRFOctQfEv8LNzzDJMdqroR7Kj9n3AHXlnquEh\njhwd8PPl2N0CfLYRzxaKiPaSp4342QLFovFflny+/Ke8A5WLiBklU00a8fPlIY4Utw352QKDZ4ed\nk1L6Wd5ZKmjoz5YBKaVPZdeA2E3xj725qfD9uqk+Uyx0x0FK6afAgYi4F+hLKT2Yd6ZyUbwy5p3A\nBcAdEXFJ3pnKvAN4A/Dn2ZXqGnHOwLXZVTHvpngKUkOd/pZS+sOU0q+klK4F1qWU/jrvTBWsiIhH\nIuJ+YGtK6YG8A5XL/hr7akSsofjfZCNdZGfA24Bv5x2ihq8Bv5G9h+8CvppvnKEiYmH2e+aHwP2N\ndMGx8t/VwFLgniheSX0ZxSNVDZMvu6Ln31M80vLliMj9VP8KGS+neKr/x7J/919rpHzAhdlny10U\nL9z2lTzzQcWMD6SUriz5fPnjfBNWzPg64KGIuAdYTM6/uyvkOx24O8v3XuDzeeaDqt8Nr6R4xDl3\nFd7D3RTnYa8BPgV8Lsd4QNXfiWuyM4YONcD3nCHfr4EzaKDPlOMVxTM4JEmSJElqDh7RlSRJkiQ1\nFQtdSZIkSVJTsdCVJEmSJDUVC11JkiRJUlOx0JUkSZIkNRULXUmSchARfRGxtuTnxjEcuzMiHh+r\n8SRJmmha8w4gSdIk9WpKaVneISRJakYe0ZUkqYFExOaI+POIeCwiHoyI12TrOyPihxHxaET8ICKW\nZOs7IuK2iPhZ9vPGbKhCRHwhItZFxKqImJa1/92IWJ+N842cXqYkSXVloStJUj6mlZ26/G9Ktu1J\nKZ0H3Az8f9m6vwa+nFI6H/gq8Jls/WeAu1NKFwAXAeuy9WcCn00pnQu8BPzrbP2NwIXZOB+s14uT\nJClPkVLKO4MkSZNORLycUppZYf1m4MqU0tMR0QZsTynNjYjngVNSSoez9dtSSvMiYhewKKV0sGSM\nTmB1SunM7PkfAm0ppf8UEbcDLwPfAr6VUnq5zi9VkqRx5xFdSZIaT6qyfDQOliz3ceS6HP8C+CzF\no78PRYTX65AkNR0LXUmSGs+/KXn8cbZ8P/DObPldwL3Z8g+ADwFERCEiZlcbNCJagMUppbuAPwRm\nA8OOKkuSNNH5V1xJkvIxLSLWljy/PaU0cIuhEyPiUYpHZa/P1n0E+FJE/AGwC3hvtv6jwC0R8X6K\nR24/BGyrss8C8HdZMRzAZ1JKL43ZK5IkqUE4R1eSpAaSzdHtTik9n3cWSZImKk9dliRJkiQ1FY/o\nSpIkSZKaikd0JUmSJElNxUJXkiRJktRULHQlSZIkSU3FQleSJEmS1FQsdCVJkiRJTcVCV5IkSZLU\nVP4XiY0JpC9m980AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x12c871c18>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "train_rmse = [t for e, t, v in model.scbd]\n",
    "test_rmse = [v for e, t, v in model.scbd]\n",
    "epochs = [e for e, t, v in model.scbd]\n",
    "plt.figure(figsize=((16,4)))\n",
    "plt.plot(epochs, train_rmse, label=\"Train\")\n",
    "plt.plot(epochs, test_rmse, label=\"Test\")\n",
    "plt.legend()\n",
    "plt.xticks(epochs, fontsize=8)\n",
    "plt.xlabel(\"Epochs\")\n",
    "plt.ylabel(\"RMSE\")\n",
    "plt.grid(axis=\"y\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Starting training SGD with lambda = 0.1 ...\n",
      "latent dimention 20\n",
      "learning rate 0.01\n",
      "lambda SGD 0.1\n",
      "lambda ALS users 0.1\n",
      "lambda ALS items 0.1\n",
      "Confidence ALS 1\n",
      "Epochs 10\n",
      "[Epoch 0] : train RMSE = 1.0485\n",
      "[Epoch 0] : validation RMSE = 1.0978\n",
      "[Epoch 1] : train RMSE = 0.9747\n",
      "[Epoch 2] : train RMSE = 0.9483\n",
      "[Epoch 3] : train RMSE = 0.9366\n",
      "[Epoch 4] : train RMSE = 0.9288\n",
      "[Epoch 5] : train RMSE = 0.9227\n",
      "[Epoch 5] : validation RMSE = 0.9432\n",
      "[Epoch 6] : train RMSE = 0.9145\n",
      "[Epoch 7] : train RMSE = 0.9051\n",
      "[Epoch 8] : train RMSE = 0.8975\n",
      "[Epoch 9] : train RMSE = 0.8910\n",
      "Starting training SGD with lambda = 1.0 ...\n",
      "latent dimention 20\n",
      "learning rate 0.01\n",
      "lambda SGD 1.0\n",
      "lambda ALS users 0.1\n",
      "lambda ALS items 0.1\n",
      "Confidence ALS 1\n",
      "Epochs 10\n",
      "[Epoch 0] : train RMSE = 1.6217\n",
      "[Epoch 0] : validation RMSE = 1.5736\n",
      "[Epoch 1] : train RMSE = 1.5688\n",
      "[Epoch 2] : train RMSE = 1.5529\n",
      "[Epoch 3] : train RMSE = 1.5459\n",
      "[Epoch 4] : train RMSE = 1.5345\n",
      "[Epoch 5] : train RMSE = 1.5228\n",
      "[Epoch 5] : validation RMSE = 1.4556\n",
      "[Epoch 6] : train RMSE = 1.4938\n",
      "[Epoch 7] : train RMSE = 1.4520\n",
      "[Epoch 8] : train RMSE = 1.4123\n",
      "[Epoch 9] : train RMSE = 1.3907\n",
      "Starting training SGD with lambda = 10.0 ...\n",
      "latent dimention 20\n",
      "learning rate 0.01\n",
      "lambda SGD 10.0\n",
      "lambda ALS users 0.1\n",
      "lambda ALS items 0.1\n",
      "Confidence ALS 1\n",
      "Epochs 10\n",
      "[Epoch 0] : train RMSE = 3.2010\n",
      "[Epoch 0] : validation RMSE = 3.0572\n",
      "[Epoch 1] : train RMSE = 3.2005\n",
      "[Epoch 2] : train RMSE = 3.2003\n",
      "[Epoch 3] : train RMSE = 3.1996\n",
      "[Epoch 4] : train RMSE = 3.1985\n",
      "[Epoch 5] : train RMSE = 3.1999\n",
      "[Epoch 5] : validation RMSE = 3.0554\n",
      "[Epoch 6] : train RMSE = 3.1997\n",
      "[Epoch 7] : train RMSE = 3.2006\n",
      "[Epoch 8] : train RMSE = 3.1991\n",
      "[Epoch 9] : train RMSE = 3.2003\n",
      "Starting training SGD with lambda = 100.0 ...\n",
      "latent dimention 20\n",
      "learning rate 0.01\n",
      "lambda SGD 100.0\n",
      "lambda ALS users 0.1\n",
      "lambda ALS items 0.1\n",
      "Confidence ALS 1\n",
      "Epochs 10\n",
      "[Epoch 0] : train RMSE = 3.7139\n",
      "[Epoch 0] : validation RMSE = 3.5561\n",
      "[Epoch 1] : train RMSE = 3.7145\n",
      "[Epoch 2] : train RMSE = 3.7146\n",
      "[Epoch 3] : train RMSE = 3.7151\n",
      "[Epoch 4] : train RMSE = 3.7146\n",
      "[Epoch 5] : train RMSE = 3.7143\n",
      "[Epoch 5] : validation RMSE = 3.5564\n",
      "[Epoch 6] : train RMSE = 3.7146\n",
      "[Epoch 7] : train RMSE = 3.7139\n",
      "[Epoch 8] : train RMSE = 3.7148\n",
      "[Epoch 9] : train RMSE = 3.7145\n",
      "Starting training SGD with lambda = 1000.0 ...\n",
      "latent dimention 20\n",
      "learning rate 0.01\n",
      "lambda SGD 1000.0\n",
      "lambda ALS users 0.1\n",
      "lambda ALS items 0.1\n",
      "Confidence ALS 1\n",
      "Epochs 10\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/berryweinstein/anaconda3/lib/python3.6/site-packages/ipykernel/__main__.py:105: RuntimeWarning: overflow encountered in multiply\n",
      "/Users/berryweinstein/anaconda3/lib/python3.6/site-packages/ipykernel/__main__.py:104: RuntimeWarning: overflow encountered in multiply\n",
      "/Users/berryweinstein/anaconda3/lib/python3.6/site-packages/ipykernel/__main__.py:105: RuntimeWarning: invalid value encountered in subtract\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Epoch 0] : train RMSE = nan\n",
      "[Epoch 0] : validation RMSE = nan\n",
      "[Epoch 1] : train RMSE = nan\n",
      "[Epoch 2] : train RMSE = nan\n",
      "[Epoch 3] : train RMSE = nan\n",
      "[Epoch 4] : train RMSE = nan\n",
      "[Epoch 5] : train RMSE = nan\n",
      "[Epoch 5] : validation RMSE = nan\n",
      "[Epoch 6] : train RMSE = nan\n",
      "[Epoch 7] : train RMSE = nan\n",
      "[Epoch 8] : train RMSE = nan\n",
      "[Epoch 9] : train RMSE = nan\n"
     ]
    }
   ],
   "source": [
    "# 2\n",
    "import math\n",
    "lambda_l = [0.1, 1.0, 10.0, 100.0, 1000.0]\n",
    "log_lambda_l = [math.log(k) for k in lambda_l]\n",
    "min_test_rmse = []\n",
    "for l in lambda_l:\n",
    "    model = MFModel(X_train, X_test, users, items, hidden_dim=20, lr=0.01, gamma=l, reg_u=0.1, reg_i=0.1, epochs=10, optimizer='sgd')\n",
    "    print(\"Starting training SGD with lambda = %r ...\" % (l))\n",
    "    model.print_hyper_params()\n",
    "    model.train_model()\n",
    "    test_rmse = sorted(model.scbd, key=lambda x: x[2])\n",
    "    min_test_rmse.append(min(test_rmse))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA7sAAAEJCAYAAABVBzChAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xl8XHW9//H3J3uazHRNk0l3umShlNKGVcSyCBRQRLjI\nJsrV26vXXXH/eb3XKygKKouAuFxAUa4LrpRFlrIXoagFmqQt3dukbbpk3+fz+2NOQpomodtkksnr\n+Xjkwcw53znzCY/H6cw753s+X3N3AQAAAACQTFISXQAAAAAAAEcaYRcAAAAAkHQIuwAAAACApEPY\nBQAAAAAkHcIuAAAAACDpEHYBAAAAAEmHsAsAAAAASDqEXQAAAABA0iHsAgAAAACSTlqiCzhYEyZM\n8OnTpye6DAAAAABAHKxYsaLG3fMO9zjDLuxOnz5dL7/8cqLLAAAAAADEgZltPBLHYRozAAAAACDp\nEHYBAAAAAEmHsAsAAAAASDqEXQAAAABA0iHsAgAAAACSDmEXAAAAAJB0CLsAAAAAgKQz7NbZBQAA\nABJlV0OrVlXVqbyqTlPH5ejcuQWJLglAPwi7AAAAQC+dUdfGXY1aVVWnVdti4XZVVZ2217V2j7m0\nbDJhFxjC4hZ2zSxL0tOSMoP3+a27f73XmEWS/ihpfbDpAXf/RrxqAgAAAHprautQRXV9LNBui4Xa\niqp6Nbd3SpLSUkyzJubqbTMnqLQwrJJI7GdcTkaCKwcwkHhe2W2VdIa7N5hZuqRnzewhd1/ea9wz\n7n5BHOsAAAAA5O7aWd+q13tdrV1f0yj32JhQVppKI2G97/gpKi0MqzQS1uz8XGWmpSa2eAAHLW5h\n191dUkPwND348Xi9HwAAANClozOq9TVvTkPuus+2pqGte8zksdkqjYT17mMLVRKJBdvJY7NlZgms\nHMCREtd7ds0sVdIKSbMk/dDdX+xj2ClmtlLSVknXuvvr8awJAAAAyaW+pX2/aciV1fVq7YhKkjJS\nUzQ7P1enF03svlpbHAlrdHZ6gisHEE9xDbvu3ilpvpmNkfR7M5vr7q/1GPKKpKnBVOfzJP1B0uze\nxzGzJZKWSFJ+fr6WLVsWz7IBAAAwBLm7dre4NtVHtakuqk31UW2uj2pH05uTB3PTpanhFC2anKKp\noTRNDacqkmNKS+mUtEdq36PmTdLfNyXu9wAwOMx9cGYWm9l/Smpy9xsHGLNBUpm71/Q3pqyszF9+\n+eU4VAgAAIChoq0jqrU7Grrvq+26Ylvb3N49Zvr4Ud1XaksiYZUWhlUQzmIaMjDMmdkKdy873OPE\nsxtznqR2d99rZtmS3inphl5jCiRtd3c3sxMkpUjaFa+aAAAAMPTUNrXHAm1wX+2qbXVas6Ne7Z2x\nizKZaSkqLgjpvGMKVBqE2qKCsHIzWUUTQP/i+S9ERNI9wX27KZJ+7e5/MbOPSJK73ynpEkkfNbMO\nSc2SLvPButQMAACAQeXu2rKnWa9vq9tn/dqte5u7x0zIzVBp4WidNidPJZGQji4Ma/r4HKWlpiSw\ncgDD0aBNYz5SmMYMAAAw9LW0d2rN9n2nIZdX1am+tUOSlGLSjAk5Ki0c3X21tiQS0sRQVoIrB5Bo\nQ34aMwAAAEaGXQ2tKq+q16qq2th/t9Vp7c4GdUZjF1VGZaSqJBLWhccVqjQyOjYNOT+k7AzWrgUQ\nP4RdAAAAHJBo1LVhV2N3sI1dra1XdV1L95iCcJZKC8N6Z2l+d9OoaeNGKSWFplEABhdhFwAAAPtp\nbutURfW+TaMqquvV1NYpSUpNMc3Ky9XJM8f3mIYc1ricjARXDgAxhF0AAIARbkd9S/fSPl331q6v\naVQwC1mhzDSVFIZ1admU7mA7a2KustKZhgxg6CLsAgAAjBAdnVGtr2nsXuanK9jWNLR1j5k0Jlul\nhWFdMK+wew3byWOzWbsWwLBD2AUAAEhCDa0dqqjafxpya0dUkpSeapqTH9KioolvTkMuCGv0qPQE\nVw4ARwZhFwAAYBhzd1XVtnQH2q6rtht3NXWPGTMqXaWRsN5/0rTuplEz83KVkcbatQCSF2EXAABg\nmGjvjGrtjobu6cddwXZvU3v3mOnjR6k0EtYlCyZ3N42KjM5iGjKAEYewCwAAMATVNrfvc7W2vKpO\na7Y3qK0zNg05My1FxQUhLZ5bELtaGwmrOBJWbiZf7wBAIuwCAAAklLtry57m7oZRXf/dure5e8yE\n3AyVRMK65tTpsftrI2HNmJCjtFSmIQNAfwi7AAAAg6S1o1Nrtjfsc29teVWd6ls6JElm0lETcnTc\n1DG68qSp3Y2jJoayElw5AAw/hF0AAIA42N3Ytt805LU7GtQRLF47KiNVxQUhXTi/sHsaclFBSKMy\n+HoGAEcC/5oCAAAchmjUtXF3075No7bVqbqupXtMQThLJZGQziyZqNLIaJUWhjVt3CilpNA0CgDi\nhbALAABwgJrbOlW5vT64Wlur8qp6lVfVqamtU5KUmmKalZerk2eOV0kkpNLIaJVEQhqfm5ngygFg\n5CHsAgAA9GFHfYvKq+p7NI2q1fqaRgWzkBXKTFNJJKxLy6aoNBJb4md2fq6y0lMTWzgAQBJhFwAA\njHCdUdf6mga9vq0uFm6Dacg1Da3dYyaNyVZJJKzz5xWqNBLW0YVhTR6bzdq1ADCEEXYBAMCI0dDa\nocrqHkv8VNWrsrpOLe2xtWvTU02zJ4a0qCivu2lUaSSs0aPSE1w5AOBgEXYBAEDScXdV17Xs1zRq\nw66m7jGjs9NVGgnryhOndS/xMzMvVxlprF0LAMmAsAsAAIa19s6o3tgZrF27rU7lwZXbPU3t3WOm\njR+l0khY710wuTvYRkZnMQ0ZAJIYYRcAAAwbtc3tquhxpXZVVZ3WbG9QW2dsGnJGWoqKC0I65+gC\nlRbGmkYVF4QUymIaMgCMNIRdAAAw5Li7tuxp1qqqYBpyEGy37GnuHjM+J0OlhWFd87bpKi2M3Vs7\nY0KO0lKZhgwAIOwCAIAEa+3o1JrtDd1Xa7vusa1v6ZAkmUkzJuRo/pQxuvyEqSotDOvoSFh5oUym\nIQMA+kXYBQAAg2ZPY9s+DaNWVdVp7Y4GdQSL12anp6o4EtK7jy3cZxryqAy+sgAADg6fHAAA4IiL\nRl2bdjftd7W2qrale0x+OFOlkbDOKJ7YPQ152vgcpaZwtRYAcPgIuwAA4LC0tHeqsrp+n6u1FVV1\namzrlCSlpphm5uXoxBnjuq/WlkTCmpCbmeDKAQDJjLALAAAO2M761v2aRq3b2aBgFrJyM9NUEgnp\nkoWTg6u1ozU7P1dZ6amJLRwAMOIQdgEAwH46o671NY37TUPeWd/aPWbSmGyVRMI6b25Bd7CdPDZb\nKUxDBgAMAYRdAABGuMbWDlVU12lVVX331drK6jq1tMfWrk1PNc2aGNJps/OCacghlUbCGjMqI8GV\nAwDQP8IuAAAjhLtre12rVlXVBldrY/fZbtjVKA+mIY/OTldpJKwrTpjW3TRq1sRcZaSxdi0AYHgh\n7AIAkKSiUdejq7ZrxcbdwX229drd2Na9f+q4USqNhHXRcZNUEgmrtDCswtFZrF0LAEgKhF0AAJLQ\n82tr9M0Hy7Wqqk4ZaSkqyg/pnSX5sau1hbG1a0NZ6YkuEwCAuIlb2DWzLElPS8oM3ue37v71XmNM\n0s2SzpPUJOmD7v5KvGoCACDZrdler289VKEnKnZo0phs3XzZfJ1/TERpqUxDBgCMLPG8stsq6Qx3\nbzCzdEnPmtlD7r68x5jFkmYHPydKuiP4LwAAOAg76lv0g8fW6P6/bVJOZpq+vLhYHzhlOkv+AABG\nrLiFXXd3SQ3B0/Tgx3sNu1DSvcHY5WY2xswi7l4Vr7oAAEgmzW2d+skz63TnU2+otSOqq0+erk+e\nOVvjcuiUDAAY2eJ6z66ZpUpaIWmWpB+6+4u9hkyStLnH8y3BNsIuAAAD6Iy6Hnhli258tFLb61p1\n7tEF+uLiYs2YkJPo0gAAGBLiGnbdvVPSfDMbI+n3ZjbX3V872OOY2RJJSyQpPz9fy5YtO7KFAgAw\njLxW06n/q2zT5vqojhqdoq+cmKU5Y+u18bWXtDHRxQEAMEQMSjdmd99rZk9KOldSz7C7VdKUHs8n\nB9t6v/4uSXdJUllZmS9atCh+xQIAMERVVtfr+qXlemr1Tk0em61bLy/WBfMiLBUEAEAf4tmNOU9S\nexB0syW9U9INvYb9SdLHzex+xRpT1XK/LgAA+9pR16Lv/XW1fv3yZuVmpumr55Xo6lOmKTON5lMA\nAPQnnld2I5LuCe7bTZH0a3f/i5l9RJLc/U5JSxVbdmitYksPXRPHegAAGFaa2jp019PrdNfT69Te\nGdUHT5mhT5wxS2NpPgUAwFuKZzfmlZKO62P7nT0eu6SPxasGAACGo86o67crNuumR1drR32rzjum\nQF88t1jTxtN8CgCAAzUo9+wCAIAD89TqnfrW0nJVVNfruKljdMdVC7Rw2rhElwUAwLBD2AUAYAgo\nr6rT9UvL9cyaGk0dN0q3X7lAi+cW0HwKAIBDRNgFACCBtte16KZHK/WbFVsUzkrX1y4o1VUnTaX5\nFAAAh4mwCwBAAjS2duhHT6/Tj59ep86o68OnztDHT5+t0aPSE10aAABJgbALAMAg6uiM6jcrtuim\nR1erpqFVF8yL6AvnFGvq+FGJLg0AgKRC2AUAYBC4u5YFzadWb29Q2bSxuuvqhVowdWyiSwMAICkR\ndgEAiLPXt9XqW0sr9OzaGk0fP0p3XrVA5xxN8ykAAOKJsAsAQJxU1TbrpkdX63evbNHo7HR9/V2l\nuvLEacpIS0l0aQAAJD3CLgAAR1hDa4d+9NQb+vEz6xSNSkvefpT+4/RZGp1N8ykAAAYLYRcAgCOk\nozOq+1/arB88tlo1DW1697GF+vw5RZoyjuZTAAAMNsIuAACHyd31RMUOfeuhCq3d0aATpo/TTz5Q\novlTxiS6NAAARizCLgAAh+G1rbW67sFyvbBul2ZMyNGP3r9QZ5fm03wKAIAEI+wCAHAItu1t1o2P\nVOqBv2/VuJwM/fe7j9YVJ05VeirNpwAAGAoIuwAAHIT6lnbdsewN/fTZ9XJJH100Ux9dNFPhLJpP\nAQAwlBB2AQA4AO2dUd3/t036wWNrtKuxTRcdN0mfO3uOJo+l+RQAAEMRYRcAgAG4ux4r36FvPVSu\ndTsbdeKMcfrf80s0bzLNpwAAGMoIuwAA9GPllr267sFyvbh+t47Ky9FPri7TmSUTaT4FAMAwQNgF\nAKCXLXuadOMjlfrDP7ZpfE6G/uc9c3XZ8VNoPgUAwDBC2AUAIFDX0q7bn3xDP3tuvUzSx06fqY+8\nY6ZCNJ8CAGDYIewCAEa89s6ofvniJv3gsdXa09Su9y6YpGvPLlLhmOxElwYAAA4RYRcAMGK5ux5d\ntV3ffqhC62sadfJR4/XV80s0d9LoRJcGAAAOE2EXADAi/WPzXl3/YLn+tmG3Zk3M1c8+WKbTi2g+\nBQBAsiDsAgBGlM27m/TdRyr1p39u04TcDF130Vy9r2yK0mg+BQBAUiHsAgBGhNqmdv1w2Vrd/dwG\npaRInzhjlv79HTOVm8lHIQAAyajfT3gzK3b3iuBxpru39th3krsvH4wCAQA4HG0dUf1i+Ubd8sQa\n1Ta36+IFk/W5s+coMprmUwAAJLOB/pz9S0kLgscv9HgsSbf3eg4AwJDi7nr4tWrd8HCFNuxq0ttm\njddXzivR0YU0nwIAYCQYKOxaP4/7eg4AwJDxyqY9uu7Bcq3YuEdz8nP1v9ccr0Vz8mg+BQDACDJQ\n2PV+Hvf1HACAhNu0q0k3PFKhB1dWKS+UqW+/9xhdsnAyzacAABiBBgq7k83sFsWu4nY9VvB8Utwr\nAwDgAO1tatNtT6zVPS9sUFpKij515mwtOe0o5dB8CgCAEWugbwGf7/H45V77ej8HAGDQtXZ06ucv\nbNStT6xVXUu7Ll04RZ89e47yw1mJLg0AACRYv2HX3e/pvc3Mxkra6+5MYwYAJIy7a+mrseZTm3Y3\n6bQ5efry4mKVRMKJLg0AAAwRAy099J+Sfu3uFWaWKekhSfMldZjZFe7+2GAVCQBAlxUbd+u6B8v1\nyqa9Ki4I6Z5/PUHvmJOX6LIAAMAQM9A05vdJ+p/g8QcUu1c3T9IcSfdIGjDsmtkUSfdKylesodVd\n7n5zrzGLJP1R0vpg0wPu/o2D+xUAACPBhppGfeeRCi19tVoTQ5n6zsXzdPHCyUpNocMyAADY30Bh\nt63HdOVzJN3v7p2Sys3sQDp+dEj6nLu/YmYhSSvM7K/uvqrXuGfc/YKDLx0AMBLsaWzTrU+s1c+X\nb1B6aoo+c9Yc/dtpMzQqg+ZTAACgfwN9U2g1s7mStks6XdK1PfaNeqsDu3uVpKrgcb2ZlSvWxbl3\n2AUAYD+tHZ269/mNuvWJNWpo7dD7jp+iz5w1RxNpPgUAAA7AQGH305J+q9jU5e+7+3pJMrPzJP39\nYN7EzKZLOk7Si33sPsXMVkraKulad3+9j9cvkbREkvLz87Vs2bKDeXsAwDDi7nqxulO/Xd2mmmbX\nvAmpunRhtiaHdmvVK8v5iykAADggFu/GymaWK+kpSde5+wO99oUlRd29IQjRN7v77IGOV1ZW5i+/\nzMpHAJCMXtqwW998sFz/3LxXJZGwvnJesd4+m+ZTAACMJGa2wt3LDvc4A3Vj/uxAL3T3773Vwc0s\nXdLvJN3XO+gGx6jr8Xipmd1uZhPcveatjg0ASB7raxp1w0MVevj1auWHM/XdS+bpvQtoPgUAAA7d\nQNOYb5T0D8WWHGpVrBvzATMzk/RTSeX9BWMzK5C03d3dzE6QlCJp18G8DwBg+Nrd2KZbHl+jXyzf\nqMy0FH3unXP04bcfpeyM1ESXBgAAhrmBwu5xki6XdL6kFZJ+JelxP/B5z2+T9H5Jr5rZP4JtX5E0\nVZLc/U5Jl0j6qJl1SGqWdNlBHB8AMEy1tHfqnuc36LYn16qxtUOXnzBVnz5rjvJCmYkuDQAAJIkD\numfXzE5RLPieJemL7v6neBfWH+7ZBYDhKxp1/XnlNn3n4Upt3dusM4on6suLizU7P5To0gAAwBAR\n93t2e7xRnmJXeY+RtEXSjsN9UwDAyPPiul26bmm5Vm6p1dGFYX33knk6ZdaERJcFAACS1EANqv5V\n0qWSshRbguhSdyfoAgAOyhs7G/Tthyr011XbFRmdpe9deqzeM3+SUmg+BQAA4migK7s/kfSapI2S\nzpF0dqznVIy7vzu+pQEAhrNdDa26+fE1uu/FTcpOT9XnzynSh06doax0mk8BAID4Gyjsnj5oVQAA\nkkZLe6d+9tx63f7kG2pu79QVJ0zVp86arQm5NJ8CAACDp9+w6+5P9bfPzN4Wn3IAAMNVNOr64z+3\n6rsPV2pbbYvOKsnXlxYXa9bE3ESXBgAARqCB7tlNVeye3UmSHnb318zsAsWWD8pWrGkVAAB6/o0a\nXb+0XK9trdMxk0brpkvn6+SZ4xNdFgAAGMEGmsb8U0lTJP1N0i1mtk1SmaQvufsfBqM4AMDQtnZH\nvb79UIUeK9+hSWOy9YP3zde7jy2k+RQAAEi4gcJumaR57h41syxJ1ZJmuvuuwSkNADBU1TS06geP\nrdav/rZZo9JT9cVzi3XN26bTfAoAAAwZA4XdNnePSpK7t5jZOoIuAIxszW2x5lN3LHtDLe2duurE\nqfrkmbM1nuZTAABgiBko7Bab2crgsUmaGTw3Se7u8+JeHQBgSIhGXb//+1bd+GilqmpbdHZpvr64\nuFgz82g+BQAAhqaBwm7JoFUBABiynltbo+seLNeqqjodO3m0fvC++TrxKJpPAQCAoW2gpYc2DmYh\nAIChZfX2en1rabmerNypSWOydfNl8/WueTSfAgAAw8NAV3YBACPQjvoWff+va/R/L21STmaavnJe\nsa4+meZTAABgeCHsAgAkSU1tHfrJM+t151NvqK0jqqtPnq5PnTlbY3MyEl0aAADAQTugsGtmeZLk\n7jvjWw4AYLB1Rl2/e2WLbnq0UtvrWrV4boG+cG6xZkzISXRpAAAAh6zfsGtmJunrkj4uKSXY1CHp\nVnf/xiDVBwCIo2fW7NR1D5arorpe86eM0Q+vWKCy6eMSXRYAAMBhG+jK7mckvU3S8e6+XpLM7ChJ\nd5jZZ9z9+4NRIADgyKusrtf1S8v11OqdmjIuW7ddcZzOPyai2N85AQAAhr+Bwu77Jb3T3Wu6Nrj7\nOjO7StKjkgi7ADDM7Khr0ff+ulq/fnmzcjPT9P/OL9H7T56mzDSaTwEAgOQyUNhN7xl0u7j7TjNL\nj2NNAIAjrKmtQ3c9vU53Pb1O7Z1RXfO2GfrEGbM0ZhTNpwAAQHIaKOy2HeI+AMAQ0Rl1/XbFZt30\n6GrtqG/V+cdE9IVzizRtPM2nAABAchso7B5rZnV9bDdJWXGqBwBwhDy1eqeuf7BcldvrtWDqGN1x\n1QItnEbzKQAAMDL0G3bdnRu4AGAYKq+q0/VLy/XMmhpNHTdKt1+5QIvnFtB8CgAAjCgHtM6uJJnZ\nJEldAXibu3fEpyQAwKGorm3R9/5aqd+s2KJwVrq+dkGp3n/SNGWkpSS6NAAAgEE30Dq7X1asSVXX\nmrovSKqVlC7pHknfin95AIC30tjaoR89vU4/fnqdOqOuD586Qx8/fbZGj6KXIAAAGLkGurL7L5Le\n3uP5Lnc/zsxSJT0lwi4AJFRHZ1S/WbFFNz26WjUNrXrXsYX6wjlFmjJuVKJLAwAASLgBpzG7e2OP\npzcH2zrNLDuuVQEA+uXuWla5U9cvLdeaHQ0qmzZWP756oY6bOjbRpQEAAAwZA4XdXDNLd/d2SXL3\nuyXJzDIlhQehNgBAL69vq9X1S8v13Npdmj5+lO68aoHOOZrmUwAAAL0NFHZ/K+lHZvZxd2+SJDPL\nkXRbsA8AMEiqapt14yOr9cDft2hMdrr+612luuJEmk8BAAD0Z6Cw+zVJ10naZGYbFVtfd4qknwb7\nAABx1tDaoTuXvaGfPLtO0ai05LSj9B+LZml0Ns2nAAAABjLQOrudkr5kZv8taVawea27Nw9KZQAw\ngnV0RnX/S5v1g8dWq6ahTRfOL9S1Z9N8CgAA4EANtPTQ8ZI2u3u1pFfN7GpJ3wyu8v6Xu+8erCIB\nYKRwdz1RsUPfeqhCa3c06IQZ4/TTD5To2CljEl0aAADAsDLQzV4/ktQmSWZ2mqRvS7pXsbV273qr\nA5vZFDN70sxWmdnrZvapPsaYmd1iZmvNbKWZLTi0XwMAhr/Xttbqih+/qA/d87KiUddd71+o/1ty\nEkEXAADgEAx0z25qj6u375N0l7v/TtLvzOwfB3DsDkmfc/dXzCwkaYWZ/dXdV/UYs1jS7ODnREl3\nBP8FgBFj295m3fhIpR74+1aNy8nQNy48WpefMFXpqTSfAgAAOFQDhl0zS3P3DklnSlpygK+TJLl7\nlaSq4HG9mZVLmiSpZ9i9UNK97u6SlpvZGDOLBK8FgKRW39KuO5a9oZ8+u14u6aOLZuqji2YqnEXz\nKQAAgMM1UGj9laSnzKxGUrOkZyTJzGYpNpX5gJnZdEnHSXqx165Jkjb3eL4l2LZP2DWzJQrCdn5+\nvpYtW3Ywbw8AQ0pH1PXUlg79YW2b6tukkwtTdfHsDE3IqtYry6sTXR4AAEBSGKgb83Vm9rikiKRH\ng6uvUuw+308c6BuYWa6k30n6tLvXHUqR7n6XgvuEy8rKfNGiRYdyGABIKHfXY+U79K2HyrVuZ5tO\nOmqcvnpeqY6ZPDrRpQEAACSdAacju/vyPratPtCDm1m6YkH3Pnd/oI8hWxVbu7fL5GAbACSVlVv2\n6roHy/Xi+t06Ki9HP7m6TGeWTJSZJbo0AACApPSW994eKot9g/uppHJ3/14/w/4k6eNmdr9ijalq\nuV8XQDLZsqdJNz5SqT/8Y5vG52Tof94zV5cdP4XmUwAAAHEWt7Ar6W2S3q/YGr1d3Zu/ImmqJLn7\nnZKWSjpP0lpJTZKuiWM9ADBo6lradfuTb+hnz62XSfrY6TP1kXfMVIjmUwAAAIMibmHX3Z+VNOD8\nvOA+4I/FqwYAGGztnVHdt3yjbn58jfY2t+ui4ybp2rOLVDgmO9GlAQAAjCjxvLILACOGu+uR17fr\nhocrtL6mUafMHK+vnFeiuZNoPgUAAJAIhF0AOEz/2LxX1z24Si9t2KPZE3P1vx88XouK8mg+BQAA\nkECEXQA4RJt3N+k7j1Tqz//cpgm5Gbr+omN0adlkpdF8CgAAIOEIuwBwkGqb2vXDZWt193MblJIi\nffKMWVryjpnKzeSfVAAAgKGCb2YAcIDaOqL6xfKNuuWJNaptbtclCybrc2cXqWB0VqJLAwAAQC+E\nXQB4C+6uh1+r1rcfrtDGXU06ddYEfeW8EpUWhhNdGgAAAPpB2AWAAbyyaY+ue7BcKzbu0Zz8XN19\nzfF6xxyaTwEAAAx1hF0A6MOmXU264ZEKPbiySnmhTH37vcfokoU0nwIAABguCLsA0MPepjbd9sRa\n3fPCBqWlpOhTZ87WktOOUg7NpwAAAIYVvr0BgKTWjk79/IWNuvWJtapradelC6fos2fPUX6Y5lMA\nAADDEWEXwIjm7lr6arVueLhCm3Y36bQ5efry4mKVRGg+BQAAMJwRdgGMWCs27tY3HyzX3zftVXFB\nSPf+6wk6bU5eossCAADAEUDYBTDibKhp1HceqdDSV6s1MZSp71w8TxcvnKzUFDosAwAAJAvCLoAR\nY09jm255Yo1+sXyj0lNT9Nl3ztGH3z5DozL4pxAAACDZ8A0PQNJr7ejUPc9v0K1PrFVja4fed/xU\nfeadszUxRPMpAACAZEXYBZC03F1/Xlml7zxcoS17mnV6UZ6+fF6J5uSHEl0aAAAA4oywCyAp/W39\nbl23tFz/3LxXJZGwfvGheTp19oRElwUAAIBBQtgFkFTW7WzQDQ9X6JHXt6sgnKUb/+VYXXTcJJpP\nAQAAjDCEXQBJYXdjm255PNZ8KjMtRdeePUcfOvUoZWekJro0AAAAJABhF8Cw1tLeqbuf36AfPrFW\nTe2duuzY8ezDAAASjklEQVT4Kfr0WXOUF8pMdGkAAABIIMIugGEpGnX9eeU2fefhSm3d26wziyfq\nS4uLNZvmUwAAABBhF8AwtHzdLl2/tFwrt9Tq6MKwvnvJPJ0yi+ZTAAAAeBNhF8Cw8cbOBn37oQr9\nddV2RUZn6XuXHqv3zJ+kFJpPAQAAoBfCLoAhb1dDq25+fI3ue3GTstNT9flzivShU2coK53mUwAA\nAOgbYRfAkNXS3qmfPbdetz/5hprbO3XFCVP1qbNma0IuzacAAAAwMMIugCEnGnX98Z9b9d2HK7Wt\ntkVnleTrS4uLNWtibqJLAwAAwDBB2AUwpDz/Ro2uX1qu17bW6ZhJo3XTpfN18szxiS4LAAAAwwxh\nF8CQsGZ7vW54uEKPle/QpDHZuvmy+XrXvEKaTwEAAOCQEHYBDKqW9k6t3dGg8qo6VVbXqyL4qWlo\nVSgzTV9aXKwPnjKd5lMAAAA4LIRdAHERjbq27m3uFWrrtL6mUVGPjclMS1FRQUinF+WpJBLWhfML\nNZ7mUwAAADgCCLsADlttc3sQaOtioTYIuI1tnd1jpo4bpeKCkM6fV6jigpCKC0KaNj5HqUxTBgAA\nQBzELeya2c8kXSBph7vP7WP/Ikl/lLQ+2PSAu38jXvUAOHztnVGt29m4X6jdVtvSPWZ0drqKC0K6\nZOFkFUfCKioIqSg/pJxM/rYGAACAwRPPb593S7pN0r0DjHnG3S+IYw0ADoG7a3tdq8qrgynIVbFw\n+8bOBrV3xuYgp6eaZubl6oQZ47pDbUlBWPnhTJlxtRYAAACJFbew6+5Pm9n0eB0fwJHR2Nqhyu31\n+4Taiup61Ta3d48pHJ0Vu7e2eGIwBTmsGRNylJGWksDKAQAAgP4lel7hKWa2UtJWSde6++sJrgdI\nWp1R18Zdjd1htqKqTpXb67VxV1P3mJyMVBUVhHT+vEh3qC3KD2n0qPQEVg4AAAAcvESG3VckTXX3\nBjM7T9IfJM3ua6CZLZG0RJLy8/O1bNmyQSsSGI7q2lxb6qPaXB/VluBna0NUbdHYfpNUkGOaHErR\nwtnpmpyboimhFI3PNqVYu6RdUusuNW2U/r4xkb8JAAAAcGjM3eN38Ng05r/01aCqj7EbJJW5e81A\n48rKyvzll18+IvUBw13XmrU9r9SWV8XWrO0yITezu/txUUFIJZGwZk3MZR1bAAAADElmtsLdyw73\nOAm7smtmBZK2u7ub2QmSUiTtSlQ9wFDm7tqyp7k71FZsj/13w64mdQaL1mampWhOfmzN2q5QW1QQ\n0gTWrQUAAMAIFM+lh34laZGkCWa2RdLXJaVLkrvfKekSSR81sw5JzZIu83heZgaGib7WrF29vUEN\nrR3dY7rXrD0m0t0JeTpr1gIAAADd4tmN+fK32H+bYksTASNS7zVru7oh916ztqggpIsXTFJRQVjF\nkZDm5IeUy5q1AAAAwID4xgzEWdeatT1DbXlVXb9r1naF2uKCkArCWaxZCwAAABwCwi5wBDW2dmj1\n9vp9Qm3l9nrtbXpzzdrI6CwVF4S0qGiiSiKxplFHTchlzVoAAADgCCLsAoega83ayup6lQ+wZu2c\ngpAWz43EQm1+bN1a1qwFAAAA4o+wC7yFXQ2t3aG2MpiKvHp7vVraY4vWppg0fUKO5haO1sULJgfL\n/IQ1eWy2UmgYBQAAACQEYRcI9FyztivUVlTXa2d9zzVrM1RcENaVJ07rDrWz81mzFgAAABhqCLsY\ncXquWVtZXRdcsa3X+prG/dasfcecvO5QW1QQUl6INWsBAACA4YCwi6TWtWZtz1BbWV2/z5q1U8Zl\nq7ggrPPmFnR3QmbNWgAAAGB4I+wiKbR3RrW+pjHW/bj6zW7IW/c2d48JZ6WpOBLWexdM6r5SW1TA\nmrUAAABAMuJbPoYVd9eO+tZ9Qm1Fdb3e2NGgts5Yw6i0FNOsibkqmz5WVxUE99ZGWLMWAAAAGEkI\nuxiymto6uqcdx0JtrGlU7zVriwp63FsbYc1aAAAAAIRdDAG916zt6oS8aXeTPNYvSqMyUlVUENLi\nuQXdU5CLC0IaMyojscUDAAAAGJIIuxhUXWvW9rxS29eatUcXhnXxgskqKgiphDVrAQAAABwkwi7i\nomvN2soeobb3mrXjczJUHAnpyhOndYda1qwFAAAAcCQQdnFYutas7R1qe65Zm5GWojn5uaxZCwAA\nAGDQEHZxwOpaYmvWVlS9GWpXV9ervteatUX54X3urZ0+fpTSUmkYBQAAAGDwEHaxn641ayt6BNs+\n16wtCOuiBZOCZlFh1qwFAAAAMGSQTEawrjVru0JtVzfk3mvWzsyLrVl7ZcFUlQShNjKaNWsBAAAA\nDF2E3RGiqa1Dq7c39JiCHAu3e3qsWVsQzlJxJKTT5kzoDrUz81izFgAAAMDwQ9hNMp1R16bdTfuF\n2o291qydkx/SuXMLVJQfUnEkzJq1AAAAAJIKYXcY293YFuuAXFXf3Q159fYGNbd3SgrWrB2fo9LC\nsC46brKKIyEVF4Q0Zewo1qwFAAAAkNQIu8NAa0dszdqKqnpVbq9XeXB/7Y4+1qy9/ISp3aF29sSQ\nsjNYsxYAAADAyEPYHULcXVv3Nu8Xatf1sWbt22fnqSQS6u6EzJq1AAAAAPAmwm6CdK9Z26MTcmWv\nNWsnj81WcUE4dm9tEGpZsxYAAAAA3hphN856r1nbFXB7rlkbykpTSUFY7zluUvcU5Dn5IYWy0hNY\nOQAAAAAMX4TdI8TdtbO+VeXV9aoMmkZVVNdrbR9r1i6cNlZXnjRVxcHVWtasBQAAAIAji7B7CLrW\nrK2srlN5j07IvdesLSoI6e1zJnSH2qPycpSZRsMoAAAAAIg3wu5BeOGNXfryAyv3WbM2Oz1VRQUh\nnXN0QSzUsmYtAAAAACQcYfcgTMjNUEkktmZtUUFIJRHWrAUAAACAoYiwexBm54d0x1ULE10GAAAA\nAOAtsIYNAAAAACDpEHYBAAAAAEknbmHXzH5mZjvM7LV+9puZ3WJma81spZktiFctAAAAAICRJZ5X\ndu+WdO4A+xdLmh38LJF0RxxrAQAAAACMIHELu+7+tKTdAwy5UNK9HrNc0hgzi8SrHgAAAADAyJHI\ne3YnSdrc4/mWYBsAAAAAAIdlWCw9ZGZLFJvqrPz8fC1btiyxBQEAAAAAhrREht2tkqb0eD452LYf\nd79L0l2SVFZW5osWLYp7cQAAAACA4SuRYfdPkj5uZvdLOlFSrbtXvdWLVqxYUWNmG+NeXf8mSKpJ\n4PsDyYzzC4gPzi0gPji3gPgoOhIHiVvYNbNfSVokaYKZbZH0dUnpkuTud0paKuk8SWslNUm65kCO\n6+558aj3QJnZy+5elsgagGTF+QXEB+cWEB+cW0B8mNnLR+I4cQu77n75W+x3SR+L1/sDAAAAAEau\nRHZjBgAAAAAgLgi7B++uRBcAJDHOLyA+OLeA+ODcAuLjiJxbFptNDAAAAABA8uDKLgAAAAAg6RB2\nezGzJWa2PPi5oo/9vzazp8zsWTMrCrbNDZ4/Z2bzgm1fCsa9ZGYXDfbvAQwlZvZ9M3vGzG7utb3Q\nzJ4ws+fN7KxgW8jM/hycT1cH29LM7OfBefalRPwOwFBjZicG586zZvb9Xvv2+wzq59w6O3j9cjO7\nLhG/BzBUmdlnzOzZXtuyzOynwWfXrcG2vs6tRWa20cyWmdm9iagfGIrM7Gozezw4Nyb12D4/OIee\nMbO3B9v6OrcKgtc+ZWY/e8v3YxrzvsxsurtvMLN0ScvdfWGv/enu3m5m75B0qbt/zMx+L+mTkqKS\nbnf3C3uMy5X0mLufNPi/DZB4ZrZA0kfd/d/M7A5JP3P3l4J9t0j6P0n/lPQXd19kZp+VtF3S/ZKe\nlHSWpAsklbj7dWb2F0kfdvfqRPw+wFBhZgWS9rp7i5ndJ+nb7v5qsG+/z6B+zi139/bgNU8q9rm2\nMyG/EDCEmFmmYvcMznT3U3ts/4KkFe7+eI9tfZ1bp0g6y93/36AWDgxhQbj9hrt/qI99f5L0H5J2\nS3rA3c/t59z6uKQad7/XzH4s6TZ3/2d/78mV3V7cfUPwsCP46b2/PXiYK2ll8Hisu292962SxvQa\nly3ptbgVDAx9J0n6a/D4MUkn99h3jKTn3b1BUr2ZhbvGu3unYiG4uNcxnpR0wmAUDgxl7l7t7i3B\n03ZJnT329fUZtN+51SPopkqqllQ3GLUDw8CHJN3Tx/ZFkt4dXFl6d7Ctr88tSbo8uEo14HKcwAhy\njqTU4MrurcFnT5ex7r7F3Zsk5ZhZtvo+t1ZLGh28JiRp70BvSNjt30ck/bH3RjPLCKa03CppebC5\n5/9H6zH2dsUC8RNxrBMY6sbozS/QtcHzLqn+5vSSrn19jR/oGMCIFtw+k+fuq3pt7/0Z1Od5ZGZL\nJFVK2uXurYNSNDCEBbP7Frl7X9/fZkp6UNL5kr5mZmnq+9x6WbEv5udK+qSZ5cW9cGDoy5eU4e5n\nSmqSdGGPfTuDW0PzJM1V/9///iZpiZmVS2pz940DveGIDbs95nv3/Lk/2HeipPMk3dD7de7eFkxn\n+RdJ3+ja3GNItMfY/1DsH7qvxuv3AIaBWknh4HFY+/4FLtrjcde+vsYPdAxgxDKzcZJuU+wq1D76\n+Azq8zxy97skzZE02cyOi3fNwDDwfkm/7GdfraSn3L1R0lrFvrzvd265e4O7twfjnpY0O841A8NB\nraSngsdPSCrpse9Lkr4n6U7F/lBbo74/tz6n2FToEkm1ZnbaQG84YsNuMP1rUa+fy4K55DdJ+kBw\nybybxaQHT+skNQePd5vZZDMrDLZ33euhYAzTwjCSvSDpzODxWXpzRoQkrTSzk80sR1LY3eu6xgdT\nW+ZLquh1jNMlvTQolQNDWHBF6ReSru19D3s/n0H7nVtd49w9KqlRb36uASNZkaSPmtnDko42s0/0\n2Pe8pHnBeTRd0k71fW6Fpe5bBI6XtGHwygeGrOclzQsez5e0vmuHu69297Ml/bukTcFtNn19JzTF\n7uuVpF16c0pzn2hQ1YuZ/UjSGZK2BpsWK/aP3kJJ90l6WLEruS7pY+5eHkwhuyMY/zF3/4eZ3anY\nX9QzJN3o7g8M4q8BDCkW68K8QNI/3P0TZnZr8N/Jku5V7L7Cr7v7o8EXhF9KGifpLne/O/gj092S\npkla6u7XJ+Y3AYaO4D7AWyS9Hmz6sqQrgnNrv8+gfs6tj0i6TFKqpGXu/rVB/0WAIczMnnX3U3t8\nbkUUu5c3LOnH7v7Tfs6tD0taotgMpl+5+839vgkwgpjZjZLKFLtye4Wkm4Jz60OSrlLsj64fc/f1\n/Zxb0/Xm/fS7JF3m7m39vh9hFwAAAACQbEbsNGYAAAAAQPIi7AIAAAAAkg5hFwAAAACQdAi7AAAA\nAICkQ9gFAAAAACQdwi4AAAfBzBqG6jHN7L/M7NoDGHe3mV1yJN4TAIChirALAAAAAEg6hF0AAA6T\nmU03syfMbKWZPW5mU4PtM81suZm9ambffKsruGaWG7z+leA1F/Y4fkVwRXa1md1nZmeZ2XNmtsbM\nTuhxmGPN7IVg+78Frzczu83MKs3sMUkTe7znf5rZS2b2mpndZWZ25P8PAQAw+Ai7AAAcvlsl3ePu\n8yTdJ+mWYPvNkm5292MkbTmA47RIusjdF0g6XdJNPcLnLEk3SSoOfq6QdKqkayV9pccx5kk6Q9LJ\nkv7TzAolXSSpSFKppKslndJj/G3ufry7z5WULemCg/nFAQAYqgi7AAAcvpMl/TJ4/HPFQmjX9t8E\nj3/Z+0V9MEnXm9lKSY9JmiQpP9i33t1fdfeopNclPe7uLulVSdN7HOOP7t7s7jWSnpR0gqTTJP3K\n3TvdfZukJ3qMP93MXjSzVxULyUcf6C8NAMBQlpboAgAAQLcrJeVJWuju7Wa2QVJWsK+1x7hoj+dR\n7ft57r2O2ft5NzPLknS7pDJ332xm/9Xj/QAAGNa4sgsAwOF7XtJlweMrJT0TPF4u6eLg8WW9X9SH\n0ZJ2BEH3dEnTDqGWC80sy8zGS1ok6SVJT0t6n5mlmllEsSnS0pvBtsbMciXRoRkAkDS4sgsAwMEZ\nZWY977/9nqRPSPpfM/u8pJ2Srgn2fVrSL8zsq5IellT7Fse+T9KfgynFL0uqOIT6Vio2fXmCpP9x\n921m9nvFpiivkrRJ0guS5O57zezHkl6TVK1YMAYAIClY7HYfAABwpJnZKEnN7u5mdpmky939wkTX\nBQDASMCVXQAA4mehpNuCjsp7Jf1rgusBAGDE4MouAAAAACDp0KAKAAAAAJB0CLsAAAAAgKRD2AUA\nAAAAJB3CLgAAAAAg6RB2AQAAAABJh7ALAAAAAEg6/x/Lx0u1CGLhIgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1510ca54e0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(figsize=((16,4)))\n",
    "plt.plot(log_lambda_l, [i[2] for i in min_test_rmse])\n",
    "plt.xticks(log_lambda_l, fontsize=8)\n",
    "plt.xlabel(\"Log lambda\")\n",
    "plt.ylabel(\"SGD RMSE\")\n",
    "plt.grid(axis=\"y\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Starting training SGD with K (hidden) = 2 ...\n",
      "latent dimention 2\n",
      "learning rate 0.01\n",
      "lambda SGD 0.1\n",
      "lambda ALS users 0.1\n",
      "lambda ALS items 0.1\n",
      "Confidence ALS 1\n",
      "Epochs 10\n",
      "[Epoch 0] : train RMSE = 1.0500\n",
      "[Epoch 0] : validation RMSE = 1.1032\n",
      "[Epoch 1] : train RMSE = 0.9735\n",
      "[Epoch 2] : train RMSE = 0.9473\n",
      "[Epoch 3] : train RMSE = 0.9351\n",
      "[Epoch 4] : train RMSE = 0.9271\n",
      "[Epoch 5] : train RMSE = 0.9217\n",
      "[Epoch 5] : validation RMSE = 0.9447\n",
      "[Epoch 6] : train RMSE = 0.9181\n",
      "[Epoch 7] : train RMSE = 0.9128\n",
      "[Epoch 8] : train RMSE = 0.9087\n",
      "[Epoch 9] : train RMSE = 0.9043\n",
      "Starting training SGD with K (hidden) = 4 ...\n",
      "latent dimention 4\n",
      "learning rate 0.01\n",
      "lambda SGD 0.1\n",
      "lambda ALS users 0.1\n",
      "lambda ALS items 0.1\n",
      "Confidence ALS 1\n",
      "Epochs 10\n",
      "[Epoch 0] : train RMSE = 1.0473\n",
      "[Epoch 0] : validation RMSE = 1.0981\n",
      "[Epoch 1] : train RMSE = 0.9725\n",
      "[Epoch 2] : train RMSE = 0.9469\n",
      "[Epoch 3] : train RMSE = 0.9342\n",
      "[Epoch 4] : train RMSE = 0.9263\n",
      "[Epoch 5] : train RMSE = 0.9181\n",
      "[Epoch 5] : validation RMSE = 0.9416\n",
      "[Epoch 6] : train RMSE = 0.9106\n",
      "[Epoch 7] : train RMSE = 0.9021\n",
      "[Epoch 8] : train RMSE = 0.8928\n",
      "[Epoch 9] : train RMSE = 0.8869\n",
      "Starting training SGD with K (hidden) = 10 ...\n",
      "latent dimention 10\n",
      "learning rate 0.01\n",
      "lambda SGD 0.1\n",
      "lambda ALS users 0.1\n",
      "lambda ALS items 0.1\n",
      "Confidence ALS 1\n",
      "Epochs 10\n",
      "[Epoch 0] : train RMSE = 1.0480\n",
      "[Epoch 0] : validation RMSE = 1.0973\n",
      "[Epoch 1] : train RMSE = 0.9733\n",
      "[Epoch 2] : train RMSE = 0.9482\n",
      "[Epoch 3] : train RMSE = 0.9365\n",
      "[Epoch 4] : train RMSE = 0.9289\n",
      "[Epoch 5] : train RMSE = 0.9219\n",
      "[Epoch 5] : validation RMSE = 0.9434\n",
      "[Epoch 6] : train RMSE = 0.9137\n",
      "[Epoch 7] : train RMSE = 0.9056\n",
      "[Epoch 8] : train RMSE = 0.8967\n",
      "[Epoch 9] : train RMSE = 0.8894\n",
      "Starting training SGD with K (hidden) = 20 ...\n",
      "latent dimention 20\n",
      "learning rate 0.01\n",
      "lambda SGD 0.1\n",
      "lambda ALS users 0.1\n",
      "lambda ALS items 0.1\n",
      "Confidence ALS 1\n",
      "Epochs 10\n",
      "[Epoch 0] : train RMSE = 1.0483\n",
      "[Epoch 0] : validation RMSE = 1.0968\n",
      "[Epoch 1] : train RMSE = 0.9744\n",
      "[Epoch 2] : train RMSE = 0.9484\n",
      "[Epoch 3] : train RMSE = 0.9367\n",
      "[Epoch 4] : train RMSE = 0.9294\n",
      "[Epoch 5] : train RMSE = 0.9233\n",
      "[Epoch 5] : validation RMSE = 0.9429\n",
      "[Epoch 6] : train RMSE = 0.9152\n",
      "[Epoch 7] : train RMSE = 0.9080\n",
      "[Epoch 8] : train RMSE = 0.9004\n",
      "[Epoch 9] : train RMSE = 0.8937\n",
      "Starting training SGD with K (hidden) = 40 ...\n",
      "latent dimention 40\n",
      "learning rate 0.01\n",
      "lambda SGD 0.1\n",
      "lambda ALS users 0.1\n",
      "lambda ALS items 0.1\n",
      "Confidence ALS 1\n",
      "Epochs 10\n",
      "[Epoch 0] : train RMSE = 1.0495\n",
      "[Epoch 0] : validation RMSE = 1.0978\n",
      "[Epoch 1] : train RMSE = 0.9744\n",
      "[Epoch 2] : train RMSE = 0.9489\n",
      "[Epoch 3] : train RMSE = 0.9374\n",
      "[Epoch 4] : train RMSE = 0.9299\n",
      "[Epoch 5] : train RMSE = 0.9237\n",
      "[Epoch 5] : validation RMSE = 0.9433\n",
      "[Epoch 6] : train RMSE = 0.9177\n",
      "[Epoch 7] : train RMSE = 0.9111\n",
      "[Epoch 8] : train RMSE = 0.9033\n",
      "[Epoch 9] : train RMSE = 0.8952\n",
      "Starting training SGD with K (hidden) = 50 ...\n",
      "latent dimention 50\n",
      "learning rate 0.01\n",
      "lambda SGD 0.1\n",
      "lambda ALS users 0.1\n",
      "lambda ALS items 0.1\n",
      "Confidence ALS 1\n",
      "Epochs 10\n",
      "[Epoch 0] : train RMSE = 1.0490\n",
      "[Epoch 0] : validation RMSE = 1.0977\n",
      "[Epoch 1] : train RMSE = 0.9751\n",
      "[Epoch 2] : train RMSE = 0.9499\n",
      "[Epoch 3] : train RMSE = 0.9383\n",
      "[Epoch 4] : train RMSE = 0.9302\n",
      "[Epoch 5] : train RMSE = 0.9257\n",
      "[Epoch 5] : validation RMSE = 0.9444\n",
      "[Epoch 6] : train RMSE = 0.9198\n",
      "[Epoch 7] : train RMSE = 0.9128\n",
      "[Epoch 8] : train RMSE = 0.9046\n",
      "[Epoch 9] : train RMSE = 0.8979\n",
      "Starting training SGD with K (hidden) = 70 ...\n",
      "latent dimention 70\n",
      "learning rate 0.01\n",
      "lambda SGD 0.1\n",
      "lambda ALS users 0.1\n",
      "lambda ALS items 0.1\n",
      "Confidence ALS 1\n",
      "Epochs 10\n",
      "[Epoch 0] : train RMSE = 1.0499\n",
      "[Epoch 0] : validation RMSE = 1.0978\n",
      "[Epoch 1] : train RMSE = 0.9738\n",
      "[Epoch 2] : train RMSE = 0.9497\n",
      "[Epoch 3] : train RMSE = 0.9376\n",
      "[Epoch 4] : train RMSE = 0.9310\n",
      "[Epoch 5] : train RMSE = 0.9248\n",
      "[Epoch 5] : validation RMSE = 0.9445\n",
      "[Epoch 6] : train RMSE = 0.9200\n",
      "[Epoch 7] : train RMSE = 0.9133\n",
      "[Epoch 8] : train RMSE = 0.9059\n",
      "[Epoch 9] : train RMSE = 0.8979\n",
      "Starting training SGD with K (hidden) = 100 ...\n",
      "latent dimention 100\n",
      "learning rate 0.01\n",
      "lambda SGD 0.1\n",
      "lambda ALS users 0.1\n",
      "lambda ALS items 0.1\n",
      "Confidence ALS 1\n",
      "Epochs 10\n",
      "[Epoch 0] : train RMSE = 1.0484\n",
      "[Epoch 0] : validation RMSE = 1.0965\n",
      "[Epoch 1] : train RMSE = 0.9742\n",
      "[Epoch 2] : train RMSE = 0.9491\n",
      "[Epoch 3] : train RMSE = 0.9382\n",
      "[Epoch 4] : train RMSE = 0.9304\n",
      "[Epoch 5] : train RMSE = 0.9263\n",
      "[Epoch 5] : validation RMSE = 0.9455\n",
      "[Epoch 6] : train RMSE = 0.9213\n",
      "[Epoch 7] : train RMSE = 0.9167\n",
      "[Epoch 8] : train RMSE = 0.9091\n",
      "[Epoch 9] : train RMSE = 0.9021\n",
      "Starting training SGD with K (hidden) = 200 ...\n",
      "latent dimention 200\n",
      "learning rate 0.01\n",
      "lambda SGD 0.1\n",
      "lambda ALS users 0.1\n",
      "lambda ALS items 0.1\n",
      "Confidence ALS 1\n",
      "Epochs 10\n",
      "[Epoch 0] : train RMSE = 1.0488\n",
      "[Epoch 0] : validation RMSE = 1.0974\n",
      "[Epoch 1] : train RMSE = 0.9752\n",
      "[Epoch 2] : train RMSE = 0.9504\n",
      "[Epoch 3] : train RMSE = 0.9386\n",
      "[Epoch 4] : train RMSE = 0.9312\n",
      "[Epoch 5] : train RMSE = 0.9272\n",
      "[Epoch 5] : validation RMSE = 0.9450\n",
      "[Epoch 6] : train RMSE = 0.9228\n",
      "[Epoch 7] : train RMSE = 0.9198\n",
      "[Epoch 8] : train RMSE = 0.9145\n",
      "[Epoch 9] : train RMSE = 0.9068\n"
     ]
    }
   ],
   "source": [
    "# 3 + 4\n",
    "import time\n",
    "best_lambda = 0.1\n",
    "k_l = [2, 4, 10, 20, 40, 50, 70, 100, 200]\n",
    "time_meas = []\n",
    "min_test_rmse = []\n",
    "for k in k_l:\n",
    "    model = MFModel(X_train, X_test, users, items, hidden_dim=k, lr=0.01, gamma=best_lambda, reg_u=0.1, reg_i=0.1, epochs=10, optimizer='sgd')\n",
    "    print(\"Starting training SGD with K (hidden) = %d ...\" % (k))\n",
    "    model.print_hyper_params()\n",
    "    start = time.time()\n",
    "    model.train_model()\n",
    "    end = time.time()\n",
    "    time_meas.append(end - start)\n",
    "    test_rmse = sorted(model.scbd, key=lambda x: x[2])\n",
    "    min_test_rmse.append(min(test_rmse))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA8AAAAEJCAYAAABbrk7OAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3XuY3HV99//ne3d2Z5PMJpDT5gDIKQIJhiABVASCQEQM\nQtu7B353K9ZWbntrrfbWu/TqXW17X/Zo21tbW6RK0V9Vequ1QKQSQANaAQEJgQQC4ZzTbgIk2Rz2\nOJ/7j5lNJsueIDOZ2Z3n47q4dr6H+c47s8vuvvbz+by/kVJCkiRJkqSJrqHaBUiSJEmSdCQYgCVJ\nkiRJdcEALEmSJEmqCwZgSZIkSVJdMABLkiRJkuqCAViSJEmSVBcMwJIkSZKkumAAliRJkiTVBQOw\nJEmSJKkuZKpdwJEwc+bMdPzxx1e7DEmSJElSBTz88MM7UkqzRjuvLgLw8ccfz0MPPVTtMiRJkiRJ\nFRARL4zlPKdAS5IkSZLqggFYkiRJklQXDMCSJEmSpLpgAJYkSZIk1QUDsCRJkiSpLhiAJUmSJEl1\nwQAsSZIkSaoLBuAq27xzP3+zagMvvLy32qVIkiRJ0oRmAK6yHZ3dfOEHG3lm+55qlyJJkiRJE5oB\nuMpyLRkAOrv6qlyJJEmSJE1sFQvAEXFjRHRExOPDHD81Iu6LiO6I+OSgY5dFxIaI2BgR15Xs/98R\nsTYi1kTEqoiYV6n6j5TWbCEA7+k2AEuSJElSJVVyBPgm4LIRjr8CfAz4XOnOiGgEvgi8B1gIXB0R\nC4uH/yqltDiltARYCXy63EUfaQMjwHscAZYkSZKkiqpYAE4p3Ush5A53vCOl9CDQO+jQOcDGlNKz\nKaUe4GbgyuJzdpecNwVI5a36yJvU1EhDOAIsSZIkSZWWqXYBQ5gPvFSyvQk4d2AjIj4LvB/YBVx0\nZEsrv4hgSjbjGmBJkiRJqrBaDMAjSin9AfAHEfH7wEeBzwx1XkRcC1wL0NbWxurVq49Yja9XE/1s\nfGETq1dvr3YpkiRJkjRh1WIA3gwcW7J9THHfYF8HbmeYAJxSugG4AWDp0qVp2bJl5a2yjGY+cg+5\no3IsW3ZWtUuRJEmSpAmrFm+D9CCwICJOiIhm4FeAWwEiYkHJeVcCT1ahvrLLZTPs7XEKtCRJkiRV\nUsVGgCPim8AyYGZEbKIwUtsEkFK6PiLmAA8BU4F8RHwcWJhS2h0RHwXuABqBG1NK64qX/fOIOAXI\nAy8AH65U/UdSrqWJ3fsH9wKTJEmSJJVTxQJwSunqUY5vozC9eahjt1OY3jx4/y+Up7ra0prNsGXn\n/mqXIUmSJEkTWi1Oga47uWzG+wBLkiRJUoUZgGtAriXjfYAlSZIkqcIMwDUgly0E4Hw+VbsUSZIk\nSZqwDMA1oLWlsBTbTtCSJEmSVDkG4BqQyxYCsNOgJUmSJKlyDMA1IFccAbYRliRJkiRVjgG4BgyM\nAHc6AixJkiRJFWMArgEHpkA7AixJkiRJFWMArgEHpkA7AixJkiRJFWMArgGOAEuSJElS5RmAa0Br\ntglwBFiSJEmSKskAXAOmZBsBA7AkSZIkVZIBuAZkGhuY1NRoAJYkSZKkCjIA14hcS4ZO1wBLkiRJ\nUsUYgGtEazbjCLAkSZIkVZABuEbkWjLs6eqtdhmSJEmSNGEZgGtEzhFgSZIkSaooA3CNyGVdAyxJ\nkiRJlWQArhG5FkeAJUmSJKmSDMA1wiZYkiRJklRZBuAaMSWbYU9XHymlapciSZIkSRNSxQJwRNwY\nER0R8fgwx0+NiPsiojsiPjno2GURsSEiNkbEdSX7/yoinoyItRHx3Yg4qlL1H2m5lgx9+UR3X77a\npUiSJEnShFTJEeCbgMtGOP4K8DHgc6U7I6IR+CLwHmAhcHVELCwevhM4PaW0GHgK+P0y11w1rdkM\ngI2wJEmSJKlCKhaAU0r3Ugi5wx3vSCk9CAy++e05wMaU0rMppR7gZuDK4nNWpZQGEuL9wDHlr7w6\nci2FALzXdcCSJEmSVBGZahcwhPnASyXbm4Bzhzjvg8C/DneRiLgWuBagra2N1atXl7HE8nu+oxB8\nV//n/Rw/rbHK1UiSJEnSxFOLAXhUEfEHQB/w9eHOSSndANwAsHTp0rRs2bIjU9wblH3mZT7/s/s5\n5fQlvP2kGdUuR5IkSZImnFoMwJuBY0u2jynuAyAiPgCsAC5OE6hlcmtxCrS3QpIkSZKkyqjF2yA9\nCCyIiBMiohn4FeBWKHSHBv4n8L6U0r4q1lh2uexAAB68JFqSJEmSVA4VGwGOiG8Cy4CZEbEJ+AzQ\nBJBSuj4i5gAPAVOBfER8HFiYUtodER8F7gAagRtTSuuKl/17IAvcGREA96eUPlypf8ORNNAEa49d\noCVJkiSpIioWgFNKV49yfBvDdHFOKd0O3D7E/pPLU13tGRgB7nQKtCRJkiRVRC1Oga5L2UwDTY3h\nCLAkSZIkVYgBuEZEBLlsxiZYkiRJklQhBuAaMiWbcQRYkiRJkirEAFxDctmMa4AlSZIkqUIMwDWk\ntcURYEmSJEmqFANwDcllM+ztMQBLkiRJUiUYgGtIrqXJEWBJkiRJqhADcA1xDbAkSZIkVY4BuIa4\nBliSJEmSKscAXENy2Qz7e/vp689XuxRJkiRJmnAMwDUkl80AsLe7v8qVSJIkSdLEYwCuIbmWQgDu\n7O6tciWSJEmSNPEYgGtIa3EEeI+NsCRJkiSp7AzANWRgBNhGWJIkSZJUfgbgGjKwBthbIUmSJElS\n+RmAa8hAAHYEWJIkSZLKzwBcQw5MgXYEWJIkSZLKzgBcQxwBliRJkqTKMQDXkCnNjgBLkiRJUqUY\ngGtIQ0OQy2YMwJIkSZJUARULwBFxY0R0RMTjwxw/NSLui4juiPjkoGOXRcSGiNgYEdeV7P/FiFgX\nEfmIWFqp2qspl804BVqSJEmSKqCSI8A3AZeNcPwV4GPA50p3RkQj8EXgPcBC4OqIWFg8/Djw88C9\n5S62VuRaHAGWJEmSpEqoWABOKd1LIeQOd7wjpfQg0Dvo0DnAxpTSsymlHuBm4Mric55IKW2oVM21\nIJfNeB9gSZIkSaqAWlwDPB94qWR7U3FfXWhtybCna/DfBCRJkiRJhytT7QIqJSKuBa4FaGtrY/Xq\n1dUtaIz27+6ifW9+3NQrSZIkSeNFLQbgzcCxJdvHFPe9LimlG4AbAJYuXZqWLVtWluIq7XvbH2XL\nxh2Ml3olSZIkabyoxSnQDwILIuKEiGgGfgW4tco1HTG5FtcAS5IkSVIlVGwEOCK+CSwDZkbEJuAz\nQBNASun6iJgDPARMBfIR8XFgYUppd0R8FLgDaARuTCmtK17z54C/A2YB34uINSmld1fq31ANA/cB\nTikREdUuR5IkSZImjIoF4JTS1aMc30ZhevNQx24Hbh9i/3eB75alwBqVy2ZICfb19DMlW4sz1CVJ\nkiRpfKrFKdB1LddSCL3eC1iSJEmSyssAXGNyxVHfzi4DsCRJkiSVkwG4xrQWR4D3OgIsSZIkSWVl\nAK4xuWwT4BRoSZIkSSq3YQNwRJxa8jg76NjbKllUPXMKtCRJkiRVxkgjwN8oeXzfoGP/UIFaxMEp\n0I4AS5IkSVJ5jRSAY5jHQ22rTAZGgPd09Va5EkmSJEmaWEYKwGmYx0Ntq0wG7v3rCLAkSZIklVdm\nhGPHRMQXKIz2DjymuD2/4pXVqeZMA9lMA50GYEmSJEkqq5EC8KdKHj806NjgbZVRa0uGPTbBkiRJ\nkqSyGjYAp5S+OnhfRBwN7EwpOQW6gnLZjFOgJUmSJKnMRroN0qcHboUUEdmI+AHwDNAeEZccqQLr\n0ZSsI8CSJEmSVG4jNcH6ZWBD8fE1FNb+zgIuBP60wnXVtVw24xpgSZIkSSqzkQJwT8lU53cDN6eU\n+lNKTzDy2mEdJtcAS5IkSVL5jRSAuyPi9IiYBVwErCo5NrmyZdU31wBLkiRJUvmNNJL7ceDbFKY9\n/21K6TmAiLgceOQI1Fa3ci0Z9hqAJUmSJKmsRuoCfT9w6hD7bwdur2RR9S6XbXINsCRJkiSV2bAB\nOCJ+d6QnppT+pvzlCAprgHv68nT39ZPNNFa7HEmSJEmaEEaaAv05YA3wH0A3hS7QOgJy2cKnZW+3\nAViSJEmSymWkAHwmcDXwXuBh4JvA3SWdoVUhAwF4T1cf06c0V7kaSZIkSZoYhu0CnVJ6NKV0XUpp\nCfAV4EpgfUS874hVV6dyLYUA3NndW+VKJEmSJGniGOk2SAAUb4N0JvAWYBPQMZYLR8SNEdEREY8P\nc/zUiLgvIroj4pODjl0WERsiYmNEXFeyf3pE3BkRTxc/Hj2WWsab1pIRYEmSJElSeQwbgCPigxHx\nfeBbFNb//lJK6dJid+ixuAm4bITjrwAfo7DWuPR1G4EvAu8BFgJXR8TC4uHrKEzDXgDcXdyecAZG\ngL0XsCRJkiSVz0gjwF8G5gGdwLuBL0fErQP/jXbhlNK9FELucMc7UkoPAoPn+Z4DbEwpPZtS6gFu\npjD9muLHrxYffxW4arQ6xqMpWQOwJEmSJJXbSE2wLjpiVRxqPvBSyfYm4Nzi47aU0tbi421A23AX\niYhrgWsB2traWL16dfkrrZCdXXkAHl67nmk7n65yNZIkSZI0MQwbgFNK9wx3LCLOq0w5Y5dSShEx\nbEfqlNINwA0AS5cuTcuWLTtSpR22fT19sPoO5r3pRJZdeFK1y5EkSZKkCWGkNcCNEXF1RHwyIk4v\n7lsRET8B/r6CNW0Gji3ZPqa4D6A9IuYWa5nLGBtyjTeTmhppCJtgSZIkSVI5jbQG+CvAbwIzgC9E\nxL9QaFj1lymlMytY04PAgog4ISKagV8BBtYc3wpcU3x8DXBLBeuomoggl824BliSJEmSymikNcBL\ngcUppXxEtFBYc3tSSunlsVw4Ir4JLANmRsQm4DNAE0BK6fqImAM8BEwF8hHxcWBhSml3RHwUuANo\nBG5MKa0rXvbPgf8bEb8BvAD80uv7544frS1NBmBJkiRJKqORAnBPSikPkFLqiohnxxp+i8+5epTj\n2yhMbx7q2O3A7UPsfxm4eKw1jGe5bMYp0JIkSZJURiMF4FMjYm3xcQAnFbeDQg+qxRWvro7lWpwC\nLUmSJEnlNFIAPu2IVaHXyGUz7Nw/+BbJkiRJkqQ3aqTbIL1wJAvRoXItGTa9uq/aZUiSJEnShDFS\nF2hVUatdoCVJkiSprAzANcomWJIkSZJUXmMKwBExKyJmVboYHZRrybC3p5/+fKp2KZIkSZI0IQwb\ngKPgjyJiB7ABeCoitkfEp49cefUrly0sz97b4yiwJEmSJJXDSCPAnwDOA85OKU1PKR0NnAucFxGf\nOCLV1bGBAOw0aEmSJEkqj5EC8K8BV6eUnhvYkVJ6FvhV4P2VLqze5VqKAdhGWJIkSZJUFiMF4KaU\n0o7BO1NK24GmypUkODgC3OkIsCRJkiSVxUgBuOcNHlMZtBZHgPc6AixJkiRJZZEZ4dgZEbF7iP0B\ntFSoHhXlsoVBdqdAS5IkSVJ5DBuAU0qNR7IQHerAGmCnQEuSJElSWYw0AnyIiJgPDITiLSklk1kF\nHVgD7AiwJEmSJJXFsAE4In6fQiOsPynuug/YRaEB1leBP6t8efXL2yBJkiRJUnmN1ATrF4G/Ltl+\nOaX0FmAR8N6KViUaG4LJzY3s6e6tdimSJEmSNCGMFIBJKe0t2fx8cV8/MKmSRakgl83YBEuSJEmS\nymSkAJyLiAP3+00p3QQQEVlgaoXrEoVGWN4HWJIkSZLKY6QA/G3gSxExeWBHREwBri8eU4W1OgIs\nSZIkSWUzUgD+Q6ADeDEiHo6InwHPA+3FY6qwKdmMTbAkSZIkqUyGDcAppf6U0nXAscAHgGuA41JK\n143lFkgRcWNEdETE48Mcj4j4QkRsjIi1EfHWkmO/ExGPR8S6iPh4yf4zIuK+iHgsIm6LiAk9Fds1\nwJIkSZJUPsMG4Ig4OyLmpJT2p5QeA84Ebi6G1uljuPZNwGUjHH8PsKD437XAPxZf93TgQ8A5wBnA\niog4uficLwPXFbtRfxf41BjqGLdcAyxJkiRJ5TPSFOgvAT0AEXEB8OfA1yjcC/iG0S6cUroXeGWE\nU64EvpYK7geOioi5wGnAAymlfcWR5nuAny8+583AvcXHdwK/MFod45lrgCVJkiSpfEYKwI0ppYEA\n+8vADSml76SU/hA4eYTnjdV84KWS7U3FfY8D50fEjGIDrsspTMMGWEchOEPhPsXHMoHlWjLs7e4j\npVTtUiRJkiRp3MuMcKwxIjLFUdiLKUxTHsvzDktK6YmI+AtgFbAXWAP0Fw9/EPhCRPwhcCvFEeqh\nRMS1FGtua2tj9erVlSq5Yjo299CXT9z5g9U0N0a1y5EkSZKkcW2kIPtN4J6I2AHsB34EUFyPu6sM\nr72ZQ0dwjynuI6X0FeArxdf7Uwqjw6SUngSWF/e/GXjvcBdPKd1Acar20qVL07Jly8pQ8pH1UssL\nfOupxznznHcwqzVb7XIkSZIkaVwbqQv0Z4H/QaGZ1TvTwXm4DcBvl+G1bwXeX+wG/TZgV0ppK0BE\nzC5+PI7C+t9vDNrfAPwvCvcknrBas4W/T7gOWJIkSZIO34hTmYvNqQbve2osF46IbwLLgJkRsQn4\nDNBUvMb1wO0U1vduBPYBv17y9O9ExAygF/hISmlncf/VEfGR4uN/A/55LLWMV7mBAGwnaEmSJEk6\nbJVcy3v1KMcT8JFhjp0/zP7PA58//OrGh1xL4dPT2d1b5UokSZIkafwbqQu0qswRYEmSJEkqHwNw\nDWttcQ2wJEmSJJWLAbiGTbEJliRJkiSVjQG4hg1Mge50CrQkSZIkHTYDcA3LZhpoagxHgCVJkiSp\nDAzANSwiyGUzNsGSJEmSpDIwANe4XEuGvY4AS5IkSdJhMwDXuFy2iU4DsCRJkiQdNgNwjWt1CrQk\nSZIklYUBuMblWjI2wZIkSZKkMjAA17hc1gAsSZIkSeVgAK5xuZaM9wGWJEmSpDIwANe41myGPd29\n1S5DkiRJksY9A3CNy2UzdPXm6e3PV7sUSZIkSRrXDMA1LteSAfBewJIkSZJ0mAzANW5KthCAXQcs\nSZIkSYfHAFzjWosB2E7QkiRJknR4DMA1bmAKtAFYkiRJkg6PAbjG5QZGgJ0CLUmSJEmHxQBc41od\nAZYkSZKksjAA17hctgkwAEuSJEnS4apYAI6IGyOiIyIeH+Z4RMQXImJjRKyNiLeWHPudiHg8ItZF\nxMdL9i+JiPsjYk1EPBQR51Sq/loxsAZ4y879Va5EkiRJksa3So4A3wRcNsLx9wALiv9dC/wjQESc\nDnwIOAc4A1gREScXn/OXwB+nlJYAny5uT2hTmht5+4kz+OIPN/KdhzdVuxxJkiRJGrcqFoBTSvcC\nr4xwypXA11LB/cBRETEXOA14IKW0L6XUB9wD/PzAZYGpxcfTgC2Vqb52RARfvmYpbz9pBv/jW4/y\ntfuer3ZJkiRJkjQuZar42vOBl0q2NxX3PQ58NiJmAPuBy4GHiud8HLgjIj5HIby/Y7iLR8S1FEaW\naWtrY/Xq1eWu/4i65oTEvt2NfPqWdax94ilWnNhc7ZIkSZIkaVypZgAeUkrpiYj4C2AVsBdYA/QX\nD/8W8ImU0nci4peArwCXDHOdG4AbAJYuXZqWLVtW6dIr7qJleT75rUf59potzJ53HJ969ylERLXL\nkiRJkqRxoZpdoDcDx5ZsH1PcR0rpKymls1JKFwCvAk8Vz7kG+Lfi429RWCdcN5oaG/jbX1rC/3fu\ncfzD6mf4zK3ryOdTtcuSJEmSpHGhmgH4VuD9xW7QbwN2pZS2AkTE7OLH4yis//1G8TlbgAuLj98F\nPH1kS66+hobgs1edzn+74ES+dt8LfPLbj9LXn692WZIkSZJU8yo2BToivgksA2ZGxCbgM0ATQErp\neuB2Cut7NwL7gF8vefp3imuAe4GPpJR2Fvd/CPh8RGSALoprfOtNRHDde06ltSXD51Y9xb7ufj5/\n9RKymcZqlyZJkiRJNStSmvhTaJcuXZoeeuih0U8ch/75P5/jj29bzwVvnsWXfvUsJjUbgqV6llKi\nP5/o6c/T2BD+YUySJNWFiHg4pbR0tPNqrgmWXp9fP+8EpjRnuO7f1vL+Gx/gKx84m6ktTdUuS5rw\n+vOJnr48Pf35Ax97B2339OXpLfnY3TfwONHT11/4OMz5B/YP7OvP09uX6B7iuoPrGPi7ZnNjA28/\naQbLF7Vx6WltzJ7aUt03TZIkqcocAZ4gVq7dwsdvXsOpc1v52gfPZfoUb5Okgzq7ernriXZWPrqV\nNS/tpLEhaGpsoDnTQFNj4XFTYwPNjQ00ZaLkcfFj6TmDnpPNNBx43NQYxeMl2weeM/D8KDl+6Gtm\nGuKQzub54khm90iBbyBQ9vfT05dGCZ6HBsqD5xfD6LDBM/+aOsrdf27gvTr4nhfe24H3buB9b840\nFt/TOHDewDmDz9/e2c1dT7Tzwsv7AFhy7FFcurCNdy9q46RZObvIS5KkCWOsI8AG4Ankh0928OF/\neZjjpk/mX37zXNoc7alr+3r6+MGTHdz26BZ+uGE7PX155k5r4fwFM2lsCHr6Er39hwbKwnY6JCgO\nbB96XmGabblFFLqdNwQVeY1MQyGgHwiJxaA4EMKbB4XJpkOODzp/4HGmgeaS4F96fnZQoG3OHHrd\ngT8YNDc2VCyMppR4umMPq9Zt48717Ty6aRcAJ8ycwvKFbVy6sI0zjzuaxgbDsCRJGr8MwCXqJQAD\n3P/sy/zGTQ8yI5fl6795LsdOn1ztknQEdfX2s3rDdlau3cLdT3Swv7efWa1Z3vuWuVxxxlzOPPZo\nGsoUdPrzB4PxcKG5+8D2QIBOh273J3pfMzKbyKdUHBFtPCSYlgbU0u1DA+1rnzdwrFz/9vFs264u\n7nyinVXrtnH/sy/T25+YmWvmktMKYfi8k2fS0uS6YUmSNL4YgEvUUwAGWPPSTq658ae0NDXw9d88\nl5Nnt1a7JFVQT1+eH2/czspHt7JqfTt7uvuYPqWZy06fwxWL53HOCdMd3dOQdnf1snrDdlat28bq\nDdvZ093H5OZGLlgwi+WL2njXqbM5arLLKSRJUu0zAJeotwAM8OS23fzql39KPiW+9sFzOH3+tGqX\npDLq689z37Mvc9ujW7hjXTu79vcytSXDZafPYcXiebzjpBlkGqt5m2+NN919/dz/7Cvcub4wVbp9\ndzeNDcE5x08vNNFa2MYxRzujRJIk1SYDcIl6DMAAz+3Yy69++QF27+/ln3/9bJYeP73aJekw9OcT\nP33uFVau3cL3H9/Gy3t7yGUzXLqwjRWL53L+glk0Zwy9Onz5fOKxzbtYtX4bq9a183THHgAWzp16\nIAwvnDvVJlqSJKlmGIBL1GsABtiycz+/+uUH2LqrixvefxbnL5hV7ZL0OuTziUdeepXbHt3K7Y9t\npaOzm0lNjVx82mxWLJ7HslNmuV5TFffcjr0HRoYfeuFVUoL5R03i0oVtLF/UxjnHT3fGgSRJqioD\ncIl6DsAA2zu7+bWvPMCz2/fyhavP5LLT51S7JI0gpcTaTbtYuXYL31u7lS27umjONHDRKbNYsXge\nF582m8nN3sJb1bFjTzd3P9HOnevbuffpHfT05Zk2qYmLT53N8kVtnL9gFlOyfn1KkqQjywBcot4D\nMMCufb1c888/5bHNu/jcLy7m5848ptolqURKiSe2drJy7RZWrt3Ki6/so6kxOH/BLFYsnsulC9to\nbWmqdpnSIfb19HHvUztYtX4bP3iyg537emnONHD+yTO5dGEbF5/WxqzWbLXLlCRJdcAAXMIAXLCn\nu48PffUh7nv2Zf73Vafza297U7VLqnsbOzq57dGt3LZ2C89u30tjQ/COk2ZwxeJ5vHvRHKZNNvRq\nfOjrz/Pg86+yqjhVetOr+4mAtx53NMsXtrF80RxOmDml2mVKkqQJygBcwgB8UFdvPx/5+s+4+8kO\nfu+yU/mtZSdVu6S68/yOvQdGep/c1kkEnHvCdFYsnsd7Tp/DjJwjZhrfBmY03Lm+nVXrt7Fuy24A\nTp6dY/nCQhOtM445yvsyS5KksjEAlzAAH6q3P8/v/t9Hue3RLXzkopP45PJT7OZaYZte3cf31m5l\n5dqtPLZ5FwBnveloViyey+VvmUvb1JYqVyhVzqZX93HX+nZWrW/ngedeoT+fmN2a5ZKFbSxf2Mbb\nT5pBNmMzN0mSjpTuvn527+9j1/5ednf1snt/L7u7+ti9v7dkX9+BY02NDdz4gbOrXfaIxhqA7VRS\nh5oaG/g/v7yEKc2NfPGHz7Cnq4/PXLHI0Zgy27ari+89tpWVa7fwyIs7ATjjmGn8weWncfniucw/\nalKVK5SOjGOOnswHzjuBD5x3Ajv39fDDDR3cub6df39kM9944EVy2QwXnjKL5QvbWHbKbKZNcuq/\nJEkj6e3P09lVDLCDAuvI+woht7svP+L1mxqDaZOamDqpiaktTbRNnTgzFB0BrmMpJT77vSf48o+f\n431nzOMTl77ZNXqHaceebv7jsa3ctnYrDz7/CinBaXOnsmLxXFYsnsubZvj+SgO6evv5yTM7uHN9\nO3eu72DHnm4yDcHbT5rB8oVtXLKwjbnT/EORJGni6c8n9nSNPbAe3Fc4vq+nf8TrNzYEU1syTJ3U\nVAiyLU1MnZQpfhzYlzkQcKdOypSc10Q20zDuZog6BbqEAXh4KSX+7gcb+du7niKlwgjllUvms+KM\nucxudVruWOzc18P3H9/GyrVb+ckzO8inwlrHKxbPY8UZczlpVq7aJUo1r3DP652FJlrr2nl2x14A\nFh8zjUtPKzTRenNbbtz9MJYkTUz5fGJvTx+7u/rYtW9s04h37e+ls3i8s7tvxOtHQGv29QTY4r7i\nOZObG+vuZ6YBuIQBeHRbd+1n5aNb+fc1m1m3ZTcNAeedPJMrl8zn3Yu8Bc9gu7t6WbWunZVrt/Dj\np3fQl08cP2MyK4qh95S21rr7piOV08aOPQeaaA0sIThu+uQDTbSWHj+dRpdtSJLeoJQS+3v7h582\nPFSILRmNGRwNAAAT4ElEQVSF7ezqJT9KjMplMwdD6qAQe3B68RCjsJOayDVnXJ74OhmASxiAX5+N\nHZ3csmYLt6zZwouv7CObaeCS09q4csk8LjxlVt02q9nb3cddT7Szcu1W7tmwnZ7+PPOPmsSKxXO5\n4ox5LJo31dArVUDH7i7ueqKDVeu38ZONL9PTn2f6lGYuPnU2ly5s4/wFs5jUXJ/flySpnnX19h8I\nqEM1cxo8rfhgqC0c7xslwU5qanzN1ODSwFo64jp4Xy6bIdPYcITeCYEB+BAG4DcmpcKUxFse2czK\ntVt5eW8P0yY1cflb5nDlkvmcc/z0Cf+Xqa7efn74ZAcr127l7ifb6erN0zY1y3vfUhjpPfPYowy9\n0hG0p7uPezZsZ9X6bfzgyQ46u/poaWrg/AWFJloXn9bG9CnN1S5TkjQGvf35MU0bLl0HW9hXON4z\nSiOn5saG4ujr0CF2pKnFrS1NNGcMsOOJAbiEAfjw9fbn+c+NO7hlzRbuWLeNfT39zJ3WwvvOmMeV\nS+Zz2tyJM+W3u6+fe5/awcq1W7hrfTt7e/qZmWvmPacXGlmdXQfBXxoPevvzPPDsK9y5fhur1rez\ndVcXDQFLj5/O8oVtLF84h+NmTK52mZI0YfXnE52vs/tw6TTi/b0jN3LKNMSBwDptiGnEU4eYRjxt\n0sHHLU3ODqonVQ/AEXEjsALoSCmdPsTxAD4PXA7sAz6QUvpZ8djvAB8CAvinlNL/Ke7/V+CU4iWO\nAnamlJaMVosBuLz29fRx1xMd3PLIZu55ajt9+cSC2TmuOnM+7ztjHsdOH3+/cA4E/JVrt3LHum10\ndvVx1OQmLls0hyvOmMe5J0x3GotUw1JKrNuym1XrCmH4yW2dAJw6p5VLi2H49PkuU5CkUvl8Yk/P\nwHThsXUf3l1s5LRrfy97xtDI6TUdhodaBzvMNOJJTfXXyElvXC0E4AuAPcDXhgnAlwO/TSEAnwt8\nPqV0bkScDtwMnAP0AN8HPpxS2jjo+X8N7Eop/clotRiAK+eVvT3c/thWblmzmQeffxWAs950NFct\nmcd7F8+r6amI/fnEA8++zG1rt/D9x7fx6r5eWrMZli+aw4oz5vLOk2fSZOiVxqUXX95X6Ci9vp0H\nn3+FfIK501oOhOFzT5zu/9+Sxr2UEvt6+g8dcR1LR+LieZ3dfYwWBQY6EQ/VsGm4Zk7TJhe2p9jI\nSUdQ1QNwsYjjgZXDBOAvAatTSt8sbm8AlgHvBC5LKf1Gcf8fAt0ppb8seW4ALwLvSik9PVodBuAj\n46VX9nHb2i3c8sgWNrR3kmkIzl8wk6vOnM+lC9uY3Jypdonk84mHXniVlWu3cPtj29ixp5vJzY1c\nclobKxbP5YI3z3K6jDTBvLK3hx882cGqddu49+ntdPXmaW3J8K5iE60L3zzLTveSqqart79kpPXg\nKOtoo7BjbeQ0ublxyPu8jr4OtolcS8aO+xo3xkMAXgn8eUrpx8Xtu4HfA/YCtwBvB/YDdwMPpZR+\nu+S5FwB/M5Z/IBiAq+HJbbv590e2cOuazWzZ1cWkpkaWL2rjqiXzeeeCIzuymlJizUs7Wbl2K99b\nu5Vtu7vIZhp416mzueKMeVx0ymw7yEp1Yn9PPz/euINV67Zx95MdvLK3h+bGBt5x8gwuXdjGpae1\nMXuq90CXNHY9ffkx3wP20GZOY2vklM00jLgOdri1sdMmNdHaknG2i+rGuA3AKaWHIuI3gP9OIQyv\nozAC/PGS5/4jsDGl9NcjvPa1wLUAbW1tZ918881l+3dp7PIp8fSree7b2seD2/rY2wutTXD23Axv\nn5vh5KMaKrK2I6XEi515Htjaz0+39bFjfyIT8JZZjZwzJ8OS2Y1MyvgXTameDXx/eqSjj5919NOx\nr/Dz8MRpDby1rZG3zs4wL+cvjtJE159P7O+DfX2Jfb2JfX2wt/fg48LHwvbeQ7YLz+kZuY8TjQGT\nm2ByJpjSFEzOxIHtyU3B5AyFj6WPi+dOykBzo7+vSGNx0UUX1XwAHnIKdEpp66Dz/hTYlFL6h+J2\nBtgMnJVS2jSWOhwBrg09fXnueWo7t6zZzJ3r2+nuy3PM0ZO4csk8rloynwVtrYf9Ghu2dbJy7RZW\nrt3Kczv2kmkIzjt5JisWz2X5ojlMm+Q0R0mvlVLiqfY9BzpKr920C4ATZ07h0kVtLF/YxpnHHu1a\nNqkG5fOJzu7XjqwO1cxp9xDTiEdr5NQQDH3f19JR2MnDTyNuaarMH/slHWo8jAC/F/goB5tgfSGl\ndE7x2OyUUkdEHAesAt6WUtpZPHYZ8PsppQvHWocBuPbs6e7jjse3ccujW/jx09vJJzht7lSuWjKP\n9y2Zx9xpk8Z8rWe372Hl2q3c9ugWnu7YQ0PA206cwRVnzOOyRXM4uoYbcUmqTVt37eeu9e2sWt/O\nfc+8TF8+MTOX5ZLTZrN8URvvOGmm/QKkMhlo5HTItOFh170O0ZF4LI2cWkqD6aEdh0dcGzupiSnN\ndiKWxoOqB+CI+CaFplYzgXbgM0ATQErp+mIjq78HLqNwG6RfTyk9VHzuj4AZQC/wuymlu0uuexNw\nf0rp+rHWYgCubds7u1m5dgu3rNnCmpd2EgHnHD+dq86cz+Wnz2Xa5NeO2r70yr4DoXf91t1A4Tkr\nzpjLe06fy6zW7JH+Z0iaoHbt72X1hg7uXN/O6g3b2dPdx+TmRi588yyWL2rjXae0Dfl9SqoXKSW6\n+/Kv+x6wpaOw/aM0cprS3Dj0KGxp9+Fh1sbmsjZykupB1QNwLTEAjx/P79jLLWu2cMuazTy7Yy9N\njcGyU2Zz1ZL5LJo3lbueaOe2tVt59KWdACw59ihWLJ7LexfPfV2jxpL0RnT39XP/s6+wal3hFksd\nnd00NgTnnjC90ERrYRvHHD3+7oUuDTRyGmtg3bW/l86SsNvTP3Ijp5amhkFTg0e6nc6h+1pbMmRs\n5CRpFAbgEgbg8SelxOObd3PLms3c+ugWOjq7Dxw7ff5UViyex3vfMpdjp/uLpqTqyOcTazfvOhCG\nn+7YA8CieVMP3G/4tLmtTp3UEdHXn6ezGEyH7j48ckfirt6RA2xTYxyYJtw6TIgdbhpxa0uGbMYl\nA5IqywBcwgA8vvXnEw88+zIb2jtZdspsTpg5pdolSdJrPLdjb6GJ1rp2Hn7xVVKC+UdNYvmiwsjw\nOcdPdxRLwypt5DRUYH1tM6fSsNvL3lFaETc2xCGBdahpxCPdYiebsZGTpNpmAC5hAJYkHUnbO7v5\nwZPtrFrXzo827qCnL89Rk5t41ymFJloXvHkWk5sz1S5TZZRSYm9P/8EAO+o04l52lTR72jNKI6cI\naM2OFGCHGIUtCbGTbeQkaYIzAJcwAEuSqmVvdx8/eno7q9a3c/cTHeza30s208A7T57J8kVtXHxa\nGzNzNu6rtpQSXb350acND2wPMQo7Sh8nctnMsA2bRhyFndxErjnjbbgkaQQG4BIGYElSLejrz/Pg\n86+yqjhVevPO/UTAWccdXZwqPcdlHoehu6//dXcfPji9uJfe/pF/J5rU1DjEfV5HWgd7cF8uayMn\nSaokA3AJA7AkqdaklHhiayer1heaaK3bUril24LZuUITrUVzWDx/Wl2N+vUWGzm9nnvADnQk3r2/\nl+6+kRs5NTc2vGZq8NRBIXa4tbGtLU00ZwywklSrDMAlDMCSpFq36dV93LW+nVXr23nguVfozydm\nt2YPhOG3nzij5gNYfz6xp2vwCOzI3YdLR2z3jdLIKdMQr+kw/JppxK8JtAePtTTZiViSJioDcAkD\nsCRpPNm5r4cfbuhg1bp27nlqO/t6+sllMyw7ZRaXLmzjolNnM7Wlqeyvm1JiT3ffIVODhwqspete\nS8/t7O4b8foRHBpYhxtxnTz0NOJJTTZykiQNzQBcwgAsSRqvunr7+ckzO7hzfTt3rm9nx54emhqD\nt504g+UL27hkYRtzp00CCgF2f2//ECOuxZHWoaYRDxqhHa2R00An4tYRRmGHWxs7xUZOkqQKMQCX\nMABLkiaC/nxizUuvsmp9O3eua+fZHXuBwv2Gu3r7x9TIaXJz45D3eR15HWxhX64lQ6MBVpJUgwzA\nJQzAkqSJaGPHHlat38aGbZ2FEdlBgXXw1OKpk5poshOxJGkCGmsAzhyJYiRJUvmdPDvHybNPrnYZ\nkiSNG/4ZWJIkSZJUFwzAkiRJkqS6YACWJEmSJNUFA7AkSZIkqS4YgCVJkiRJdcEALEmSJEmqCwZg\nSZIkSVJdMABLkiRJkupCpJSqXUPFRcR24IVq1zGKmcCOwzhe6devBeOhxvFgPLyP46HG8cD3sX74\nuZYkVdJ4+DnzppTSrNFOqosAPB5ExEMppaVv9HilX78WjIcax4Px8D6OhxrHA9/H+uHnWpJUSRPp\n54xToCVJkiRJdcEALEmSJEmqCwbg2nHDYR6v9OvXgvFQ43gwHt7H8VDjeOD7WD/8XEuSKmnC/Jxx\nDbAkSZIkqS44AixJkiRJqgsG4CqLiHMj4icR8eOI+NsRzvtERPy4jK87LyJ+FhFdEZEp7vtUsY6v\nR0RTuV7rMGp8zXtTazWOJ6VfQ7X2PkbE8RHRHhGrI2JVLdY4HkTEZcX3cHVEbI2Iq3wfJ46xft+O\niP9a/N65MiKmVrdqSdJ4Mdbfvcf7zxkDcPW9ALwrpfROYHZEvGXwCRGRBZaU+XVfAS4G7i++xmzg\nomIda4Gryvx6b8Tg9+ZCaq/GcaH0a6hGP9cAd6aUlqWUltdwjTUtpfT94nu4DHgR+Bm+jxPJqN+3\ni7+cfBi4APj/gf9WpVolSePPqL97T4SfMwbgKkspbUspdRU3e4H+IU77DeCrZX7drpTSqyW7lgKr\ni4/vAt5eztd7I4Z4bxZRYzWOI6VfQzX3uS66KCJ+FBGfoHZrHBci4kSgHTgd38cJY4zftxcAj6WU\n+vBzLkl6Hcb4u/e4/zljAK4REbEYmJVSWj9ofxOwLKX0gwqXcBSwu/h4V3G7Jgy8N8BOarTGWjbE\n11Atfq63Am8GLgIuofCLfa3VOJ78PPBdavNzrfIZ6vPr51ySdFhG+d173P+cMQDXgIiYDvw9hVG6\nwX4N+MYRKGMXMDCHfyqFL/iqG/Te1GSN48Dgr6Gaex9TSt0ppb3FvyauBJ6hxmocZ64AbqUGP9cq\nq6E+v37OJUlv2Bh+9x73P2cMwFVWbGTyL8AnU0rbhjjlFOC3IuL7wKKI+O0KlfIgcGHx8SUU15hV\n0xDvTc3VOE4c8jVEYXS1pt7HiGgt2TwP2EiN1TheRMQcoCel9DL+PzPRDfX5fQo4PSIa8XMuSXod\nxvi797j/OWMArr5fBM4G/rLYufWQefQppd9LKb07pXQZsC6l9HfleNGIaIqIu4AzgDuAE4B7i12C\nlwD/Xo7XOUyHvDfASdRejTVviK+hP6b23sfzI+LhiPgJsDml9AC1V+N4cSVwC0BKqQPfxwljLN+3\nU0q9wD8BPwKuAb5UrXolSePOqL97T4SfM5FSqnYNkiRJkiRVnCPAkiRJkqS6YACWJEmSJNUFA7Ak\nSZIkqS4YgCVJkiRJdcEALEmSJEmqCwZgSZJeh4jY8zrOXRYR7ziM1zoqIv77GM/9o4j4ZPHxn0TE\nJW/0dcf4eh+IiHkl21+OiIWVfE1Jkg6XAViSpMpZBrzhAAwcBYwpAJdKKX06pXTXYbzuWHwAOBCA\nU0q/mVJaX+HXlCTpsBiAJUk6TBFxRUQ8EBGPRMRdEdEWEccDHwY+ERFrIuL8iJgVEd+JiAeL/51X\nfP4fRcSNEbE6Ip6NiI8VL/3nwEnF5//VEK/7BxHxVET8GDilZP9NEfFfio+fj4g/K17joYh4a0Tc\nERHPRMSHS57zqWJNayPij4v7jo+IJyLinyJiXUSsiohJxWsvBb5evO6kYu1Li8+7OiIei4jHI+Iv\nSl5jT0R8NiIejYj7I6KtrJ8ISZJGYQCWJOnw/Rh4W0rpTOBm4H+mlJ4Hrgf+NqW0JKX0I+Dzxe2z\ngV8AvlxyjVOBdwPnAJ+JiCbgOuCZ4vM/VfqCEXEW8CvAEuBy4OwR6nsxpbQE+BFwE/BfgLcBA0F3\nObCg+NpLgLMi4oLicxcAX0wpLQJ2Ar+QUvo28BDwX4u17S+pax7wF8C7itc6OyKuKh6eAtyfUjoD\nuBf40Ag1S5JUdplqFyBJ0gRwDPCvETEXaAaeG+a8S4CFETGwPTUicsXH30spdQPdEdEBjDY6ej7w\n3ZTSPoCIuHWEcweOPQbkUkqdQGdEdEfEUcDy4n+PFM/LUQi+LwLPpZTWFPc/DBw/Sl1nA6tTStuL\ndX0duAD4d6AHWFlyrUtHuZYkSWVlAJYk6fD9HfA3KaVbI2IZ8EfDnNdAYaS4q3RnMRB3l+zqp7w/\noweunR/0Ovni6wTwZymlLw2q6/gh6pp0GHX0ppRSybX8PUSSdEQ5BVqSpMM3DdhcfHxNyf5OoLVk\nexXw2wMbEbFklOsOfn6pe4GriutvW4ErXlfFh7oD+ODAaHREzI+I2W+wtp8CF0bEzIhoBK4G7jmM\n2iRJKhsDsCRJr8/kiNhU8t/vUhjx/VZEPAzsKDn3NuDnBppgAR8DlhYbTa2n0CRrWCmll4H/LDaT\n+qtBx34G/CvwKPAfwINv9B+UUloFfAO4LyIeA77N8MF7wE3A9QNNsEqutZXC2uUfFmt7OKV0yxut\nTZKkcoqDM5EkSZIkSZq4HAGWJEmSJNUFA7AkSZIkqS4YgCVJkiRJdcEALEmSJEmqCwZgSZIkSVJd\nMABLkiRJkuqCAViSJEmSVBcMwJIkSZKkuvD/AOZy54mrYLEgAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1510c26e80>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(figsize=((16,4)))\n",
    "plt.plot(k_l, [i[2] for i in min_test_rmse])\n",
    "plt.xticks(k_l, fontsize=8)\n",
    "plt.xlabel(\"Latent dimention\")\n",
    "plt.ylabel(\"SGD RMSE\")\n",
    "plt.grid(axis=\"y\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA7YAAAEJCAYAAACg+bsRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xl4nNdh3/vvwb6DAEHslEhKXESQEglTsrxIpmRZmyVL\npNLGbpImN268xLWz2WmS+tZOUndPctPmPm2c1Nc3vY2TtAJoSbYleaMlL7ItARRFUqJ2iRgABHeA\nxD5z7h8YUiBNihDJwWCA7+d58HDmnRczP8xAmPnpvOe8IcaIJEmSJEm5Ki/bASRJkiRJuhgWW0mS\nJElSTrPYSpIkSZJymsVWkiRJkpTTLLaSJEmSpJxmsZUkSZIk5TSLrSRJkiQpp1lsJUmSJEk5zWIr\nSZIkScppBdkOcDHq6urismXLsh1DkiRJkpQBTz311MEY45Lz7ZfTxXbZsmU8+eST2Y4hSZIkScqA\nEMJrM9kvY4cihxC+FEIYCCHsmrbt70MIO9Jfr4YQdky77fdDCC+GEPaGEG7LVC5JkiRJ0vySyRHb\nLwN/AfzNyQ0xxp8/eTmE8CfAsfTltcAHgTagGfhWCGFVjDGZwXySJEmSpHkgYyO2McbHgMNnuy2E\nEIB/DHwlveke4O9ijGMxxleAF4HrMpVNkiRJkjR/ZGtV5BuA/THGF9LXW4B9027vSW+TJEmSJOlN\nZWvxqA/xxmjtWxJC+AjwEYCGhga2b99+CWNJkiRJknLNrBfbEEIBsBV427TNCWDptOut6W0/I8b4\nReCLAJs2bYqbN2/OTFBJkiRJUk7IxqHItwDPxRh7pm17APhgCKE4hLAcWAn8JAvZJEmSJEk5JpOn\n+/kK8CNgdQihJ4Tw4fRNH+SMw5BjjLuBfwD2AA8Dn3BFZEmSJEnKjGf7Bvk3X3+Wg8fHsh3lksjY\nocgxxg+dY/uvnGP7F4AvZCqPJEmSJC1kA0OjPLCjl/u7EjzbN0hBXmDT5TXc2taY7WgXLVuLR0mS\nJEmSMmxkPMmje/rp6Erw+AsHSEW4prWaP/xAG3dd3cTiiuJsR7wkLLaSJEmSNI+kUpEnXjlER1eC\nbzzTx4nxJC2LSvn45ivYsrGVK+srsh3xkrPYSpIkSdI88OLAEB1dCbZ1J+g9NkpFcQHvv7qJLRtb\nefvyWvLyQrYjZozFVpIkSZJy1KHjYzzwdC+d3Ql29hwjPy9ww8o6fu/Oq3jfVQ2UFuVnO+KssNhK\nkiRJUg4ZnUjy7WcH6OzuYfveA0ymImubqvjs+6/iAxuaqa8syXbEWWexlSRJkqQ5LsbIk68doaOr\nh4d29jE0OklDVTEffvdytrS3sKaxKtsRs8piK0mSJElz1KsHT9DRnaCzu4d9h0coLcznjnWNbGlv\n4Z1X1JE/j+fNvhUWW0mSJEmaQ44Oj/PQzj46unroev0oIcC7rqjjN9+7itvXNVJebI07k8+IJEmS\nJGXZ+GSK7+4doLMrwXeeG2A8mWJVQwW/d8ca7tnQTFN1abYjzmkWW0mSJEnKghgjO/YdpaMrwUM7\nezkyPEFdRRG/eP3lbG1voa25ihA81HgmLLaSJEmSNIv2HR5mW3eCzu4ELx88QXFBHu9b28B97a3c\nsLKOgvy8bEfMORZbSZIkScqwwdEJvvFMH/d3JfjJK4cBePvyWj76nhXcsb6JqpLCLCfMbRZbSZIk\nScqAiWSKx184QEdXgm/u2c/YZIoVdeV8+tZV3LOhhaW1ZdmOOG9YbCVJkiTpEokxsrt3kPu7enjw\n6V4OHh9nUVkhP3/tUrZsbGHD0kXOm80Ai60kSZIkXaS+YyNs6+6ls7uH5/cfpyg/j5vX1LO1vYXN\nq+spKnDebCZZbCVJkiTpApwYm+ThXf10dPfww5cOESO87fIa/vW967jr6iYWlRVlO+KCYbGVJEmS\npBlKpiI/fOkgHV0JHt7Vz8hEkqW1pXzq5pVs2djCsrrybEdckCy2kiRJknQez/UP0tmVYNuOBPsH\nx6gsKeDejS3c197C2y6vcd5slllsJUmSJOksBoZGeWBHLx1dCfb0DVKQF9i8egmfu7uVm9fUU1KY\nn+2ISrPYSpIkSVLayHiSR/f009GV4PEXDpCKcE1rNZ+/ey13X9PM4oribEfUWVhsJUmSJC1oqVTk\nx68cpqOrh2/s6uf42CTN1SV8fPMVbNnYypX1FdmOqPOw2EqSJElakF4cOE5ndw/buntJHB2hvCif\nO9c3sbW9lbcvryUvz3mzucJiK0mSJGnBOHR8jAef7qWzO8HTPcfIC3DDyiX87u2ruXVtI6VFzpvN\nRRZbSZIkSfPa6ESS7zw3QEdXD9v3HmAyFVnbVMVn338VH7immfqqkmxH1EWy2EqSJEmad2KMPPXa\nEe7vSvC1nb0Mjk5SX1nMr757OVs2tnBVU1W2I+oSsthKkiRJmjdeO3SCjq4End0JXj88TGlhPrev\na2TLxhbedWUd+c6bnZcstpIkSZJy2rHhCR56Zup8s0+9doQQ4J1XLOY33ruS29Y1UlFs7ZnvfIUl\nSZIk5ZzxyRTb9w7Q2Z3g288OMJ5MsbK+gn9x+xru3dhMU3VptiNqFllsJUmSJOWEGCNP9xyjo6uH\nB5/u5cjwBHUVRfzC9ZdxX3srbc1VhOChxguRxVaSJEnSnNZzZJht3Qk6uhK8fPAERQV53Lq2ga3t\nLdywcgmF+XnZjqgss9hKkiRJmnOGRif4xjP93N/Vw49fOQzAdctr+eh7VnDH+iaqSgqznFBzicVW\nkiRJ0pwwmUzx+AsH6ehO8OjufsYmUyyvK+d33reKeze2sLS2LNsRNUdZbCVJkiRlTYyR3b2DdHQl\neODpXg4eH2NRWSH/eNNStra3sGHpIufN6rwstpIkSZJmXf+xUbbtSNDZlWDv/iEK8wPvXdPAlvYW\nblpdT1GB82Y1cxZbSZIkSbPixNgkj+zup6MrwQ9eOkiM0H7ZIv743nXctb6JmvKibEdUjrLYSpIk\nScqYZCryw5cO0tmV4OHd/QyPJ1laW8onb17Jlo0tLK8rz3ZEzQMWW0mSJEmX3N7+ITq6e9jWnWD/\n4BiVJQXcs6GZre2tbLq8xnmzuqQyVmxDCF8C7gIGYozrpm3/JPAJIAl8Lcb4uyGEZcCzwN70bk/E\nGD+WqWySJEmSLr2BoVEe2NFLZ3eC3b2DFOQF3rNqCf/qrlbee1U9JYX52Y6oeSqTI7ZfBv4C+JuT\nG0IINwH3ANfEGMdCCPXT9n8pxrghg3kkSZIkXWKjE0ke3bOfjq4eHn/hIMlU5OrWaj5391ruvqaZ\nuoribEfUApCxYhtjfCw9Ejvdx4F/F2McS+8zkKnHlyRJkpQZqVTkx68cprO7h68/08/xsUmaqkv4\n6I0r2NrewpX1ldmOqAVmtufYrgJuCCF8ARgFPh1j/Gn6tuUhhB3AMeCzMcbHZzmbJEmSpDfx0oHj\ndHYl6OxOkDg6QnlRPnesb2JrewvXL19MXp7zZpUds11sC4Ba4HrgWuAfQggrgD7gshjjoRDC24Bt\nIYS2GOPgmXcQQvgI8BGAhoYGtm/fPmvhJUmSpIVmaDzy475Jftg7ycvHUgRgXV0+d11dTHtDPsX5\nRxjfd4TH9mU7qRay2S62PUBHjDECPwkhpIC6GOMB4OThyU+FEF5ianT3yTPvIMb4ReCLAJs2bYqb\nN2+ereySJEnSgjA2meQ7zw5wf1eC7XsHmExFrmqq4l++q4V7NjRTX1WS7YjSaWa72G4DbgK+G0JY\nBRQBB0MIS4DDMcZkegR3JfDyLGeTJEmSFqwYI12vH+H+rgRf29nHsZEJ6iuL+dV3L2fLxhauaqrK\ndkTpnDJ5up+vAJuBuhBCD/A54EvAl0IIu4Bx4JdjjDGEcCPwRyGECSAFfCzGeDhT2SRJkiRNee3Q\nCTq7p+bNvnZomJLCPG5va2RreyvvurKOfOfNKgdkclXkD53jpl88y773A/dnKoskSZKkNxwbnuCh\nZ3rp7Erw5GtHCAHesWIxn7x5Jbeva6SieLYP7JQujr+xkiRJ0gIwPpnie88foLO7h2/tGWA8meLK\n+gp+9/bV3LuhheZFpdmOKF0wi60kSZI0T8UY2dlzjI6uHh7c2cfhE+MsLi/iF66/jK0bW1nXUkUI\nHmqs3GexlSRJkuaZxNERtnUnuL+rh5cPnKCoII/3rW3gvvYWbli5hML8vGxHlC4pi60kSZI0DwyN\nTvCNZ/rp6O7hiZen1mG9blktH7lhBXesb6K6tDDLCaXMsdhKkiRJOWoymeLxFw/S2ZXgkd39jE2m\nWF5Xzm+/bxVbNrawtLYs2xGlWWGxlSRJknJIjJE9fYN0dCX46o5eDh4fY1FZIf9401K2tLewceki\n581qwbHYSpIkSTlg/+Ao27oTdHQl2Lt/iML8wM1r6tmysZWb1iyhuCA/2xGlrLHYSpIkSXPU8Pgk\nj+zup6MrwQ9ePEgqwsbLFvHH967jrvVN1JQXZTuiNCdYbCVJkqQ5JJmK/OilQ3R09/Dwrn6Gx5O0\n1pTyz2+6ki3trSyvK892RGnOsdhKkiRJc8Dz+4e4v6uHr3b30j84SmVJAR+4ppmt7a1suryGvDzn\nzUrnYrGVJEmSsuTA0BgPPN1LZ3cPuxKD5OcFNq9awmfvuopbrmqgpNB5s9JMWGwlSZKkWTQ6keSb\ne/bT0dXDYy8cJJmKrG+p5nN3r+Xua5qpqyjOdkQp51hsJUmSpAxLpSI/ffUwHV0Jvv5MH0NjkzRV\nl/CRG1ewdWMLKxsqsx1RymkWW0mSJClDXj5wnM70KXoSR0coL8rn9nVN3NfewttXLCbfebPSJWGx\nlSRJki6hIyfGeXBnLx1dCXbsO0pegHevXMJnblvNrW0NlBX5EVy61PyvSpIkSbpIY5NJvvvcAPd3\nJdi+d4CJZGRNYyV/cOca7tnQQkNVSbYjSvOaxVaSJEm6ADFGul4/SkdXDw/t7OPYyARLKov5lXcu\nY8vGVtY2V2U7orRgWGwlSZKkt+D1Q8N0difo7O7h1UPDlBTmcVtbI1vbW3nXFYspyM/LdkRpwbHY\nSpIkSedxbGSCr+3so7O7h5++eoQQ4B0rFvOJm67kjvVNVBT7sVrKJv8LlCRJks5iIpnie3sP0NHd\nw7eeHWB8MsWV9RV85rbV3LuxhZZFpdmOKCnNYitJkiSlxRh5JnGMjq4EDzzdy+ET49SWF/FPrruM\nre0trG+pJgRP0SPNNRZbSZIkLXiJoyNs607Q0dXDSwdOUFSQx/uuamBrews3rlpCofNmpTnNYitJ\nkqQFaWh0gm/s6qezK8ETrxwiRrhuWS3/7IYV3Lm+ierSwmxHlDRDFltJkiQtGJPJFN9/8SAdXQke\n3dPP6ESKZYvL+K1bVrFlYwtLa8uyHVHSBbDYSpIkad7b0ztIR1cPX326lwNDY1SXFvJzb2tly8ZW\n2i9b5LxZKcedt9iGEK4HdscYh9LXq4CrYow/znQ4SZIk6ULtHxzlqzsSdHQleK5/iML8wE2r69na\n3spNa5ZQXJCf7YiSLpGZjNj+V6B92vXjZ9kmSZIkZd3w+CSP7O6noyvBD148SCrChqWL+ON72rjr\n6mZqyouyHVFSBsyk2IYYYzx5JcaYCiF4CLMkSZLmhGQq8sTLh7i/q4eHd/UzPJ6ktaaUT9x0JVs2\ntrBiSUW2I0rKsJkU1JdDCJ9iapQW4NeBlzMXSZIkSTq/5/cP0dGV4Ks7EvQdG6WyuIAPXNPMlo0t\nXLuslrw8581KC8VMiu3HgP8MfBaIwLeBj2QylCRJknQ2B4+P8cCOXjq6e9iVGCQ/L/CeVUv4l++/\niluuaqCk0Hmz0kJ03mIbYxwAPjgLWSRJkqSfMTqR5Jt79tPZneB7zx8gmYqsa6niX921lruvaWZJ\nZXG2I0rKspmsiryKqcOQG2KM60IIVwMfiDH+64ynkyRJ0oKUSkV++uphOrsTfG1nH0NjkzRWlfBr\nN6xga3sLqxoqsx1R0hwyk0OR/wr4DPCXADHGnSGEvwUstpIkSbqkXj5wnM7uBJ3dCXqOjFBWlM8d\n65rY2t7C9SsWk++8WUlnMZNiWxZj/MkZJ62ezFAeSZIkLTBHTozz0M5e7u9KsGPfUfICvOvKOj59\n62pubWugrMgTckh6czP5K3EwhHAFUwtHEUL4OaAvo6kkSZI0r41NJvnucwfo6Orhu3sHmEhG1jRW\n8gd3ruGeDS00VJVkO6KkHDKTYvsJ4IvAmhBCAngF+MWMppIkSdK8E2Ok6/WjdHb38ODTfRwbmWBJ\nZTG//I5lbG1vZW1zVbYjSspRM1kV+WXglhBCOZAXYxzKfCxJkiTNF68fGk7Pm+3h1UPDlBTmceva\nRra2t/DuK+soyM/LdkRJOW4mqyL/BvD/AEPAX4UQ2oHfizE+ep7v+xJwFzAQY1w3bfsnmRoFTgJf\nizH+bnr77wMfTm//VIzxkQv7kSRJkpRtx0Ym+PozfXR09fDTV48A8I4Vi/n1m67kjnWNVJYUZjmh\npPlkJoci/2qM8c9DCLcBi4FfAv4H8KbFFvgy8BfA35zcEEK4CbgHuCbGOBZCqE9vX8vUuXLbgGbg\nWyGEVTHG5Fv8eSRJkpQlE8kUjz1/gI6uBN98dj/jkymuWFLOZ25bzb0bW2hZVJrtiJLmqZkU25PL\nId8J/E2McXc4Y4nks4kxPhZCWHbG5o8D/y7GOJbeZyC9/R7g79LbXwkhvAhcB/xoBvkkSZKUJTFG\nnkkco6MrwYNP93LoxDi15UX8k+suY2t7C+tbqpnBR0dJuigzKbZPhRAeBZYDvx9CqARSF/h4q4Ab\nQghfAEaBT8cYfwq0AE9M268nve1nhBA+AnwEoKGhge3bt19gFEmSJF2oQyMpftQ7yQ97J+k9ESkI\nsKE+n19cXcz6unwK8g5w+MUDfO/FbCeVtBDMpNh+GNgAvBxjHA4hLAb+j4t4vFrgeuBa4B9CCCve\nyh3EGL/I1CrNbNq0KW7evPkCo0iSJOmtOD42yTee6aOzO8GPXj5EjHDtshr++a2tvH99E9VlzpuV\nlB0zWRU5BXRNu34IOHSBj9cDdMQYI/CTEEIKqAMSwNJp+7Wmt0mSJCmLJpMpfvDSITq6enhkdz+j\nEykuX1zGb753FVs2tnDZ4rJsR5SkGY3YXkrbgJuA74YQVgFFwEHgAeBvQwh/ytTiUSuBn8xyNkmS\nJKU92zdIR1cP23b0cmBojKqSAu5rb2Vreyvtly1y3qykOSVjxTaE8BVgM1AXQugBPgd8CfhSCGEX\nMA78cnr0dncI4R+APcAk8AlXRJYkSZpdA4OjfHVHL/d39fBc/xCF+YHNq+u5r72Fm9bUU1yQn+2I\nknRWYapXnmenEPKBBqYV4Rjj6xnMNSObNm2KTz75ZLZjSJIk5azh8Uke3b2fju4E33/hAKkIG5Yu\nYmt7C3dd3UxteVG2I0pawEIIT8UYN51vv/OO2IYQPsnUaOt+3lgNOQJXX1RCSZIkZUUqFXni5UPc\n35Xg4V19nBhP0rKolF/ffCVb2lu4YklFtiNK0lsyk0ORfwNYnV40SpIkSTnqhf1DdHQn2NadoO/Y\nKJXFBdx1dTNb2lu4blkteXnOm5WUm2ZSbPcBxzIdRJIkSZfeweNjPPh0Lx1dCZ5JHCM/L3Djyjr+\n4M6reN/aBkoKnTcrKffNpNi+DGwPIXwNGDu5Mcb4pxlLJUmSpAs2OpHkW8/up7MrwfbnD5BMRda1\nVPF/3rWWD1zTzJLK4mxHlKRLaibF9vX0V1H6S5IkSXNMKhV58rUjdHT18LVn+hganaSxqoRfu2EF\nW9tbWNVQme2IkpQx5y22McY/nI0gkiRJeuteOXiCzq4eOnck2Hd4hLKifG5f18jWja2844rF5Dtv\nVtICcM5iG0L4v2KMvxlCeJCpVZBPE2P8QEaTSZIk6ayODo/z4M4+Orp66H79KCHAu6+s47fft4rb\n2hopK5rJQXmSNH+82V+9/5H+9z/NRhBJkiSd2/hkiu/uHaCjq4fvPDfARDKyuqGS379jDfdsaKGx\nuiTbESUpa85ZbGOMT6X//d7sxZEkSdJJMUa69x2lsyvBgzt7OTo8QV1FMf/0HcvY2t7C2qYqQvBQ\nY0nyOBVJkqQ5Zt/hYTq7E3R2J3jl4AmKC/K4ra2RLe0t3HBlHQX5edmOKElzisVWkiRpDhgcneDr\nO/vo6Erwk1cPA3D9ilo+vvkK7ljXSGVJYZYTStLcNeNiG0IoizEOZzKMJEnSQjKRTPH4Cwe4vyvB\nN/fsZ3wyxYol5XzmttXcs6GZ1pqybEeUpJxw3mIbQngn8NdABXBZCOEa4KMxxl/PdDhJkqT5JsbI\nrsQgHd09PLCjl0MnxqkpK+RD1y5la3srV7dWO29Wkt6imYzY/hlwG/AAQIzx6RDCjRlNJUmSNM/0\nHh1h244EnV0JXhg4TlF+HresrWfLxlbes2oJRQXOm5WkCzWjQ5FjjPvO+D+HyczEkSRJmj+Oj03y\n8K5+Ort7+OFLh4gRNl1ew7/Zsp73r2+iusx5s5J0Kcyk2O5LH44cQwiFwG8Az2Y2liRJUm5KpiI/\nePEgHV09PLJ7PyMTSS6rLeM33ruSLRtbuHxxebYjStK8M5Ni+zHgz4EWIAE8Cnwik6EkSZJyzbN9\ng3R2J9jWnWBgaIyqkgK2tLdwX3sL7ZfVOG9WkjLovMU2xngQ+IVZyCJJkpRTBoZGeWBHL/d3JXi2\nb5CCvMDm1fXc197CTWvqKSnMz3ZESVoQZrIq8nLgk8Cy6fvHGD+QuViSJElz08h4kkf39NPRleDx\nFw6QinDN0kX84QfauPuaZmrLi7IdUZIWnJkcirwN+O/Ag0Aqs3EkSZLmnlQq8sQrh+joSvCNZ/o4\nMZ6kZVEpv775Su7d2MKV9RXZjihJC9pMiu1ojPE/ZzyJJEnSHPPiwBAdXVPzZnuPjVJRXMD7r25i\na3sr1y2rJS/PebOSNBfMpNj+eQjhc0wtGjV2cmOMsStjqSRJkrLk0PExHny6l47uBDt7jpGfF7hh\nZR2/d+dVvO+qBkqLnDcrSXPNTIrteuCXgJt541DkmL4uSZKU80Ynknz72QE6u3vYvvcAk6lIW3MV\nn33/VXxgQzP1lSXZjihJehMzKbb/CFgRYxzPdBhJkqTZEmPkydeO0NHVw0M7+xganaShqpgP37Cc\nrRtbWd1Yme2IkqQZmkmx3QUsAgYynEWSJCnjXj14go70+WZfPzxMaWE+d6xrZGt7K++4YjH5zpuV\npJwzk2K7CHguhPBTTp9j6+l+JElSTjg6PM5DO/vo6Oqh6/WjhADvuqKO37xlJbe1NVJePJOPRJKk\nuWomf8U/l/EUkiRJl9j4ZIrv7h2gsyvBd54bYDyZYlVDBb93xxru2dBMU3VptiNKki6R8xbbGOP3\nZiOIJEnSxYoxsmPfUTq7Ezz4dC9Hhieoqyjil95xOVs2ttDWXEUIHmosSfPNOYttCOH7McZ3hxCG\nmFoF+dRNQIwxVmU8nSRJ0gzsOzzMtu4End0JXj54guKCPG5ta2TrxhZuWFlHQX5etiNKkjLozUZs\nywFijC4JKEmS5pzB0Qm+8Uwf93cl+MkrhwF4+/JaPvaeK7h9fSNVJYVZTihJmi1vVmzjm9wmSZI0\n6yaSKR5/4QAdXQm+uWc/Y5MpViwp59O3ruKeDS0srS3LdkRJUha8WbGtDyH89rlujDH+aQbySJIk\nnSbGyO7eQTq6EjzwdIKDx8epKSvkg9cuZUt7K9e0VjtvVpIWuDcrtvlABVNzaiVJkmZV37ERtnX3\n0tndw/P7j1OUn8d7r6pny8YWNq+up6jAebOSpClvVmz7Yox/NGtJJEnSgndibJKHd/XT0d3DD186\nRIzwtstr+MKWddy1vpnqMufNSpJ+1psVW0dqJUlSxiVTkR++dJCOrgQP7+pnZCLJZbVlfOrmlWzZ\n2MKyuvJsR5QkzXFvVmzfO2spJEnSgvNc/yCdXQm27Uiwf3CMqpIC7t3Ywn3tLbzt8hrnzUqSZuyc\nxTbGePhi7jiE8CXgLmAgxrguve3zwK8BB9K7/UGM8eshhGXAs8De9PYnYowfu5jHlyRJc8/A0CgP\n7OiloyvBnr5BCvICm1fX87m7W7h5TT0lhfnZjihJykFvNmJ7sb4M/AXwN2ds/7MY4386y/4vxRg3\nZDCPJEnKgpHxJI/u6aezO8Fjzx8gFeGa1mr+8ANt3HV1E4srirMdUZKU4zJWbGOMj6VHYiVJ0gKT\nSkV+/MphOrp6+Maufo6PTdJcXcLHN1/Blo2tXFlfke2IkqR5JJMjtufyyRDCPwWeBH4nxngkvX15\nCGEHcAz4bIzx8SxkkyRJF+HFgeN0dvewrbuXxNERKooLuGNdI1vbW3n78lry8pw3K0m69Ga72P5X\n4I+BmP73T4BfBfqAy2KMh0IIbwO2hRDaYoyDZ95BCOEjwEcAGhoa2L59+2xllyRJZzE0Hvlx3yQ/\n6J3klWMpArC+Lp+7ry5mY0M+xflHGNt3hMf2ZTupJGm+CjHGzN351KHID51cPOot3LYd+HSM8ck3\nu/9NmzbFJ598010kSVIGjE4k+c5zA3R0Jdi+d4DJVGRtUxVb21v4wIZm6itLsh1RkjQPhBCeijFu\nOt9+szpiG0JoijH2pa9uAXalty8BDscYkyGEFcBK4OXZzCZJkt5cjJGnXjvC/V0Jvrazl8HRSeor\ni/nwu5ezpb2FNY1V2Y4oSVqgMlZsQwhfATYDdSGEHuBzwOYQwgamDkV+FfhoevcbgT8KIUwAKeBj\nF3u6IUmSdGm8dugEHV0JOrsTvH54mNLCfG5f18jW9hbeeUUd+c6blSRlWSZXRf7QWTb/93Psez9w\nf6aySJKkt+bY8AQPPTN1vtmnXjtCCPCuK+r4jfeu5PZ1jZQXZ2P9SUmSzs53JUmSBMD4ZIrtewfo\n7E7w7WcHGE+mWFlfwb+4fQ33bmymqbo02xElSTori60kSQtYjJGne47R0dXDg0/3cmR4grqKIn7x\n+svZ2t5CW3MVIXiosSRpbrPYSpK0APUcGWZbd4KOrgQvHzxBUUEet65t4L72Vt69so7C/LxsR5Qk\nacYstpJU8+heAAAXeElEQVQkLRBDoxN845l+7u/q4cevTK3ReN3yWj76nhXcsb6JqpLCLCeUJOnC\nWGwlSZrHJpMpHn/hIB3dCR7d3c/YZIoVdeX8zvtWce/GFpbWlmU7oiRJF81iK0nSPBNjZHfvIB1d\nCR54upeDx8dYVFbIz1+7lC0bW9iwdJHzZiVJ84rFVpKkeaL/2CjbdiTo7Eqwd/8QRfl53Lymnq3t\nLWxeXU9RgfNmJUnzk8U2g0Ynkhwfm6SuojjbUSRJ89SJsUke2d1PR1eCH7x0kBih/bJF/Ot713HX\n1U0sKivKdkRJkjLOYptB7/2T73Hd8lr+7Oc3ZDuKJGkeSaYiP3zpIJ1dCR7e3c/weJKltaV88uaV\nbNnYwvK68mxHlCRpVllsM2hNYyW7e49lO4YkaZ7Y2z9ER3cP27oT7B8co7KkgHs2tLC1vYVNl9c4\nb1aStGBZbDNobXMV258/wOhEkpLC/GzHkSTloIGhUR7Y0Utnd4LdvYMU5AU2r17C5+5u5eY19b6/\nSJKExTaj2pqrSKYie/uHuGbpomzHkSTNYYOjE+w7PMy+wyP0HBmm58gILwwM8cTLh0mmIle3VvP5\nu9dy9zXNLHbtBkmSTmOxzaC25moAdvcOWmwlaYEbHp+k58gI+w4Pn/7vkWH2HR5mcHTytP0rigtY\nWlvGR29cwdb2Fq6sr8xSckmS5j6LbQa11pRSWVLgPFtJWgDGJpMkjoxMK6tTI6/7jozQc3iYQyfG\nT9u/pDCP1poyltaU0n5ZDUtrS1laUza1rbaU6tJC58xKkjRDFtsMCiGwtqmKPX2D2Y4iSbpIk8kU\nfcdG2XdkmJ7DU+X15MjrviPDDAyNEeMb+xfmB1oWldJaU8atbQ201pTRWlPK0toyltaUUVdRZHGV\nJOkSsdhmWFtzNV/5yeskU5H8PD/ASNJclUpF9g+NvlFWT424ThXYvmOjJFNvNNe8AE3VpbTWlHLD\nyiXp0dap4tpaU0pDVYl/9yVJmiUW2wxb21zFyESSVw4ed36UJGVRjJGDx8ffODz4yOkLNSWOjDCe\nTJ32PQ1VxbTWlLHp8ppThwifPFy4aVEJhfl5WfppJEnSdBbbDGtrrgKmFpCy2EpS5sQYOTYycdaF\nmXrSc19HJpKnfU9teRFLa0pZ21zFrW0NLK0pOzXi2rKo1FPpSJKUIyy2GXZlfQVF+Xns6R3kng0t\n2Y4jSTntxNjk6QszTZvr2nN4mKGx01cWriwpYGlNGSuWlHPjqiUsrSlNj7xOldfyYt8GJUmaD3xH\nz7DC/DxWNVawu9cFpCTpfEYnkqdGWk+W1emLNB0Znjht/9LCfJbWTpXV65bVnCqsJ8trdWlhln4S\nSZI0myy2s6CtqZpvPrufGKMrYEpa0CaSKXqPjpy2mvAbl0c4MDR22v5F+XlThwXXlLJufdPU4kzT\nRlwXl7uysCRJstjOiraWKv7+yX30D47SVF2a7TiSlDHJVKR/cDQ90nr6XNeew8P0D44ybWFh8vMC\nTdUlLK0p46bVS35mgab6ymLyXFlYkiSdh8V2FqxtSi8glRi02ErKaTFGDgyNTVtVePi0Q4d7j44w\nkXyjuYYADZUlLK0t5foVi6cOE06fx7W1ppSm6hIKXFlYkiRdJIvtLLiqqYoQYE/fILesbch2HEk6\npxgjR4YnTjsVzr4zToszNnn6KXHqKoporSljfUs1d65vOu18rs2LSigucGVhSZKUWRbbWVBeXMDy\nxeXs7j2W7SiSxNDoxGmrCU+Nur5x+cT46afEqS4tZGltKSvrK7l5Tf2p+a1La8poqSmlrMi3EkmS\nlF1+GpklVzVXsbPnaLZjSFoARsaTp0ZaTy3MdHiEnqNT/x4bOX1l4bKi/PSCTG8cLrz05OHCtaVU\nlbiysCRJmtsstrOkrbmKr+3s49jIhKefkHRRxiaT9B4d/Zn5rSdHXg8eHz9t/6KCvFMjrBuWLppa\noCldZFtryqgpK3RlYUmSlNMstrOkrbkagD29g7zjisVZTiMpm2KMnBhPcmxkgsGRCY6lv05eHhyd\nZPC06ydvn+TYyAQjE6cfKlyQF2heVMrS2lJuuarh1IjryTJbV+HKwpIkaX6z2M6Skysj7+mz2Erz\nwUQydVoRPXbWIvpGGT1t2+gkyennvDmLypICqkoKqS4tpKq0gOV15aeuV5cW0rSo9FSBbawqId/i\nKkmSFjCL7SxZUllMfWWxC0hJc0SMkeH0qOmZo6VnltTpBfXktuEzFlg6U1F+HlXpUlpdWkhNWRHL\nFpefuj69pFad/De9raKkwKIqSZL0FlhsZ1FbcxV7egezHUOaNyaSKYbOMVp67IzR0sGRnz3Ud/J8\no6bFBelyWkh1aQGXLy47o4gWUF1WOG1k9Y2yWlyQ57xVSZKkWWKxnUVrm6t4/IWDjE4kKSn0vI7S\nyVHTM+eQnmtu6eAZh/SeeVqaMxXmh1MjoVWlhVSXFXHZ4nKqSwt+poxOP+y3urSQiuICCvLzZumZ\nkCRJ0sWw2M6ituZqJlORF/YfZ31rdbbjSBk3Mp7kxYHj7N0/xAv7h3hh4DiHToyfNnp6vlHTiuKp\nollZUpA+n2oZ685SRM9WUksKHTWVJElaCCy2s6iteWoBqd29xyy2mlfGJpO8fOAEz+8f4vn9Q+zt\nP84LA0O8fniYmO6tRQV5rKgrp76qhKU1pW86WnryemWJo6aSJEk6P4vtLFpaU0ZFcQF7+pxnq9w0\nkUzx2qET7O0/fqrEPr9/iFcPDZ9a5bcgL7C8rpx1LdVs3djK6sYKVjZUcnltmSVVkiRJGWGxnUV5\neYG1TVXsdgEpzXHJVGTf4eFp5XWqyL584ATjyRQAIcCyxeWsaqjgzvVNrGqoZFVDJcvryikqsMBK\nkiRp9lhsZ9na5ir+4cl9JFPR03ko62KMJI6O8ML+qXmwJ4vsiwPHGZ1IndqvtaaUVQ2VbF5dz6qG\nClY1VHJlfYWLoEmSJGlOyFixDSF8CbgLGIgxrktv+zzwa8CB9G5/EGP8evq23wc+DCSBT8UYH8lU\ntmxa21zF8HiS1w6dYMWSimzH0QIRY2RgaCw9/3XoVJF9ceA4x8cmT+3XWFXCyoYKfvHtl0+NwDZO\nFdiKYv8fmCRJkuauTH5a/TLwF8DfnLH9z2KM/2n6hhDCWuCDQBvQDHwrhLAqxvjm5/LIQW8sIDVo\nsVVGHD4xPlVeB04vscdGJk7ts7i8iFUNldzX3sKqxkpWN1Sysr6S6rLCLCaXJEmSLkzGim2M8bEQ\nwrIZ7n4P8HcxxjHglRDCi8B1wI8yFC9rVtZXUpgf2N07yN3XNGc7jnJEKhU5OjLBoeNjHDoxzuET\n42dcHufA8TFePnCcg8fHT31fVUkBqxsref/VTVPlNX0YcV1FcRZ/GkmSJOnSysbxhZ8MIfxT4Eng\nd2KMR4AW4Ilp+/Skt/2MEMJHgI8ANDQ0sH379symzYCmssD3d7/K9tL+bEdRlqRi5PgEDI1HhsYj\ng+l/z3X5+Dic62yv5YVQWRSoKgpcVZ1Ha2sRLRWBloo8FhUHQhgHDsH4Icb3wa59s/mTSpIkSZk3\n28X2vwJ/zNRn9D8G/gT41bdyBzHGLwJfBNi0aVPcvHnzJY6YeW8/8DTf3TvAe97zHkJwAan5IJmK\nHB1Oj56mR1APnxibdnmcQyfGTl0+MjxO6hxNdVFZIbXlxSyuKmJZeTG1FUUsLp/6qq0onrpcUURt\neRE1ZUUUegodSZIkLXCzWmxjjPtPXg4h/BXwUPpqAlg6bdfW9LZ5qa25iv/1VA8DQ2M0VJVkO47O\n4tIX1aliesWSCq5dblGVJEmSLqVZLbYhhKYYY1/66hZgV/ryA8DfhhD+lKnFo1YCP5nNbLNpbXM1\nAHt6By22s8SiKkmSJM1fmTzdz1eAzUBdCKEH+BywOYSwgalDkV8FPgoQY9wdQvgHYA8wCXxiPq6I\nfNJVTZUA7O49xk1r6rOcJjdZVCVJkiSdlMlVkT90ls3//U32/wLwhUzlmUsqSwq5fHEZu3sHsx1l\nzrCoSpIkSbpQ2VgVWUzNs13IxXYimeInrxzm4V39fOe5AfqOjVhUJUmSJF0Qi22WtDVX8/Vn+hkc\nnaCqpDDbcWbF6ESSx184yMO7+vn2c/s5OjxBaWE+N66qY2t7i0VVkiRJ0gWx2GbJ2qYqAJ7tHeTt\nKxZnOU3mDI1O8J3nBnh0936+u3eA4fEkVSUF3LK2gdvaGrlx5RJKi/KzHVOSJElSDrPYZklb81Sx\n3dM3/4rtoeNjfHPPfh7Z3c8PXjzEeDLFkspitmxs4fZ1jVy/YrGjsJIkSZIuGYttltRXlVBXUTxv\n5tn2Hh3hkd39PLyrn5++ephUhKW1pfzyOy/n9nWNbFxaQ15eyHZMSZIkSfOQxTaL2pqreGR3P7Xl\nRdy5volrWqsJIXfK30sHjvPwrn4e2d3Pzp5jAKxuqOSf37yS29oaWNtUlVM/jyRJkqTcFGI8x1K0\nOWDTpk3xySefzHaMC/Zs3yD/4eHn+P6LB5lIRloWlXLHukbuvLqJDa2L5twIZ4yR3b2DPLyrn4d3\n9/PiwHEANixdxO3rGrmtrZHldeVZTilJkiRpvgghPBVj3HTe/Sy22XdseIJvPbufrz/Tx+MvHGQ8\nmaKpuoQ71jVx5/pG2i/L3mG8yVTkqdeOnBqZTRwdIT8v8PbltdzW1sitbQ00VZdmJZskSZKk+c1i\nm6MGRyf49rP7+drOfh574QDjkykaq0q4fV0jd65vYtPlmS+545MpfvjSQR7ZvZ9v7unn4PFxigry\nuOHKOm5b18gtVzVQW16U0QySJEmSZLGdB06eKudrO/vY/vxUya2vLOaOdY3csb6Ja5fVkn+JSu7w\n+CSPPX8gfY7ZAYZGJykvyuemNfXcvq6RzavrqSh2SrYkSZKk2WOxnWeOj03ynecG+PrOPr67d4Cx\nyRR1FcXcvq6BO9c3cd2yWgre4il0jg1P8O3n9vPwrqnR4dGJFDVlhbxvbQO3r2vknVfUUVLoOWYl\nSZIkZYfFdh47MTbJd/cO8PVn+vjOcwOMTqRYXF7Ebesaef/6Jt6+/Nwld2BolEd3T51j9kcvHWIy\nFU8d6nxbWyPXLqt5ywVZkiRJkjLBYrtADI9Psn3vgVMld3g8SW15Ebe1TY3kXr9iMX1HR3lk99Ti\nT0+9foQYYXld+akye3VL9ZxbgVmSJEmSLLYL0Mh4ku89P8DXn+nn28/u58R4krKifIbHk8DUeXNv\na2vk9nWNrKyv8ByzkiRJkua0mRZbVwOaR0qL8rl9XRO3r2tidCLJY88fYPvzB1hRV85tbY0srS3L\ndkRJkiRJuuQstvNUSWE+t7Y1cmtbY7ajSJIkSVJGuUqQJEmSJCmnWWwlSZIkSTnNYitJkiRJymkW\nW0mSJElSTrPYSpIkSZJymsVWkiRJkpTTLLaSJEmSpJxmsZUkSZIk5bQQY8x2hgsWQjgAvJbtHOdR\nBxy8iNtnI8NckAsZc0EuPI+5kDEX+DwuHL7WkqRMmuvvM5fHGJecb6ecLra5IITwZIxx04XePhsZ\n5oJcyJgLcuF5zIWMucDnceHwtZYkZdJ8eZ/xUGRJkiRJUk6z2EqSJEmScprFNvO+eJG3z0aGuSAX\nMuaCXHgecyFjLvB5XDh8rSVJmTQv3mecYytJkiRJymmO2EqSJEmScprFNkNCCG8PIfwwhPD9EMKf\nvcl+vxVC+P4lfNzmEEJXCGE0hFCQ3vaZdI7/GUIovFSPdREZf+a5mWsZc8n036G59jyGEJaFEPaH\nELaHEB6dixlzQQjh9vRzuD2E0BdCuNfncX6Z6d/uEMIvpP9+PhRCqMpuaklSLpjpZ+9cf4+x2GbO\na8DNMcZ3A/UhhPVn7hBCKAY2XOLHPQy8F3gi/Rj1wE3pHDuBey/x412IM5+b9zD3MuaE6b9Dc/S1\nBvhmjHFzjPHWOZxxTosxPpx+DjcDrwNd+DzON+f9253+4PEx4EbgfwAfzVJWSVJuOe9n7/nwHmOx\nzZAYY3+McTR9dQJInmW3DwP/7yV+3NEY45FpmzYB29OXvwW841I+3oU4y3PTxhzLmEOm/w7Nudc6\n7aYQwuMhhN9i7mbMCSGEFcB+YB0+j/PKDP92rwSeiTFO4usuSZqhGX72zvn3GItthoUQrgaWxBj3\nnLG9ENgcY/xOhiMsAgbTl4+lr88JJ58b4ChzNONcdpbfobn4WvcBq4CbgFuY+rA+1zLmkq1AJ3Pz\ntdaldbbX2NddknTBzvPZO+ffYyy2GRRCqAX+gqlRtTP9EvC3sxDjGHDyGPkqpn6Rs+6M52ZOZswB\nZ/4OzbnnMcY4FmM8kf6/fw8BLzHHMuaYu4EHmIOvtS65s73Gvu6SpAsyg8/eOf8eY7HNkPTiH/8f\n8OkYY/9ZdlkNfDyE8DDQFkL4ZIai/BR4T/ryLaTnb2XTWZ6bOZcxR5z2O8TUaOiceh5DCJXTrr4L\neJE5ljFXhBAagfEY4yH8b2YhONtr/DywLoSQj6+7JGmGZvjZO+ffYyy2mfOPgGuB/5BeyfS049Rj\njP8ixnhbjPF2YHeM8b9cigcNIRSGEL4FXAM8AiwHHkuvmrsB2HYpHucinfbcAFcw9zLOeWf5HfpD\n5t7zeEMI4akQwg+BRIzxx8y9jLniHuCrADHGAXwe55WZ/O2OMU4AfwU8Dvwy8JfZyitJyinn/ew9\nH95jQowx2xkkSZIkSbpgjthKkiRJknKaxVaSJEmSlNMstpIkSZKknGaxlSRJkiTlNIutJEmSJCmn\nWWwlSQJCCMffwr6bQwjvvIjHWhRC+PUZ7vv5EMKn05f/KIRwy4U+7gwf71dCCM3Trv91CGFtJh9T\nkqSLZbGVJOmt2wxccLEFFgEzKrbTxRj/VYzxWxfxuDPxK8CpYhtj/Gcxxj0ZfkxJki6KxVaSpHMI\nIdwdQvhxCKE7hPCtEEJDCGEZ8DHgt0IIO0IIN4QQloQQ7g8h/DT99a70938+hPClEML2EMLLIYRP\npe/63wFXpL//P57lcf9lCOH5EML3gdXTtn85hPBz6cuvhhD+bfo+ngwhtIcQHgkhvBRC+Ni07/lM\nOtPOEMIfprctCyE8G0L4qxDC7hDCoyGE0vR9bwL+Z/p+S9PZN6W/70MhhGdCCLtCCP9+2mMcDyF8\nIYTwdAjhiRBCwyV9ISRJOg+LrSRJ5/Z94PoY40bg74DfjTG+Cvw34M9ijBtijI8Df56+fi1wH/DX\n0+5jDXAbcB3wuRBCIfB7wEvp7//M9AcMIbwN+CCwAbgTuPZN8r0eY9wAPA58Gfg54HrgZIG9FViZ\nfuwNwNtCCDemv3cl8H/HGNuAo8B9Mcb/DTwJ/EI628i0XM3AvwduTt/XtSGEe9M3lwNPxBivAR4D\nfu1NMkuSdMkVZDuAJElzWCvw9yGEJqAIeOUc+90CrA0hnLxeFUKoSF/+WoxxDBgLIQwA5xvNvAHo\njDEOA4QQHniTfU/e9gxQEWMcAoZCCGMhhEXAremv7vR+FUwV2teBV2KMO9LbnwKWnSfXtcD2GOOB\ndK7/CdwIbAPGgYem3df7znNfkiRdUhZbSZLO7b8AfxpjfCCEsBn4/Dn2y2NqZHd0+sZ00R2btinJ\npX3vPXnfqTMeJ5V+nAD82xjjX56Ra9lZcpVeRI6JGGOcdl9+vpAkzSoPRZYk6dyqgUT68i9P2z4E\nVE67/ijwyZNXQggbznO/Z37/dI8B96bnt1YCd7+lxKd7BPjVk6PHIYSWEEL9BWb7CfCeEEJdCCEf\n+BDwvYvIJknSJWOxlSRpSlkIoWfa128zNUL7v0IITwEHp+37ILDl5OJRwKeATekFmvYwtbjUOcUY\nDwE/SC/C9B/PuK0L+HvgaeAbwE8v9AeKMT4K/C3woxDCM8D/5tyF+qQvA//t5OJR0+6rj6m5wd9N\nZ3sqxvjVC80mSdKlFN44ckiSJEmSpNzjiK0kSZIkKadZbCVJkiRJOc1iK0mSJEnKaRZbSZIkSVJO\ns9hKkiRJknKaxVaSJEmSlNMstpIkSZKknGaxlSRJkiTltP8f1fjA9zNT07gAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1510c21cf8>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(figsize=((16,4)))\n",
    "plt.plot(k_l, time_meas)\n",
    "plt.xticks(k_l, fontsize=8)\n",
    "plt.xlabel(\"Latent dimention\")\n",
    "plt.ylabel(\"Time in sec\")\n",
    "plt.grid(axis=\"y\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}