14_recurrent_neural_networks.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Redes neurais Recorrentes (RNN)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Bibliotecas básicas"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-07T12:17:40.218123Z",
     "start_time": "2021-02-07T12:17:02.810944Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2.1.0\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "from functools import partial\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "\n",
    "import tensorflow as tf\n",
    "print(tf.__version__)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-07T12:17:40.240234Z",
     "start_time": "2021-02-07T12:17:40.236715Z"
    }
   },
   "outputs": [],
   "source": [
    "tf.compat.v1.disable_eager_execution()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### RNN básica - implementação manual"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-02T10:29:28.503661Z",
     "start_time": "2021-02-02T10:29:21.872195Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/liliane-hop/anaconda3/lib/python3.7/site-packages/tensorflow_core/python/ops/resource_variable_ops.py:1635: calling BaseResourceVariable.__init__ (from tensorflow.python.ops.resource_variable_ops) with constraint is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "If using Keras pass *_constraint arguments to layers.\n"
     ]
    }
   ],
   "source": [
    "tf.compat.v1.reset_default_graph()\n",
    "\n",
    "n_inputs = 3\n",
    "n_neurons = 5\n",
    "\n",
    "X0 = tf.compat.v1.placeholder(tf.float32, [None, n_inputs])\n",
    "X1 = tf.compat.v1.placeholder(tf.float32, [None, n_inputs])\n",
    "\n",
    "Wx = tf.Variable(tf.compat.v1.random_normal(shape=[n_inputs, n_neurons], dtype=tf.float32))\n",
    "Wy = tf.Variable(tf.compat.v1.random_normal(shape=[n_neurons, n_neurons], dtype=tf.float32))\n",
    "b = tf.Variable(tf.zeros([1, n_neurons], dtype=tf.float32))\n",
    "\n",
    "Y0 = tf.math.tanh(tf.matmul(X0, Wx) + b)\n",
    "Y1 = tf.math.tanh(tf.matmul(Y0, Wy) + tf.matmul(X1, Wx) + b)\n",
    "\n",
    "init = tf.compat.v1.global_variables_initializer()\n",
    "\n",
    "X0_batch = np.array([[0, 1, 2], [3, 4, 5], [6, 7, 8], [9, 0, 1]]) # t = 0\n",
    "X1_batch = np.array([[9, 8, 7], [0, 0, 0], [6, 5, 4], [3, 2, 1]]) # t = 1\n",
    "\n",
    "with tf.compat.v1.Session() as sess:\n",
    "    init.run()\n",
    "    Y0_val, Y1_val = sess.run([Y0, Y1], feed_dict={X0: X0_batch, X1: X1_batch})"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Saída da rede em ambos intervalos para todos os neurônios e todas as instâncias no minilote:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-02T10:29:28.521095Z",
     "start_time": "2021-02-02T10:29:28.510076Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[-0.9292309   0.21474057 -0.12915571  0.48339853 -0.12168597]\n",
      " [-0.9453479   0.9924072  -0.9528377   0.9893008  -0.32115355]\n",
      " [-0.9578743   0.9999551  -0.9984886   0.999834   -0.4956883 ]\n",
      " [ 1.          0.9999924  -0.9999964   0.99982053 -0.81004506]]\n"
     ]
    }
   ],
   "source": [
    "print(Y0_val)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-02T10:29:28.753582Z",
     "start_time": "2021-02-02T10:29:28.534933Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ 0.4526714   0.99998546 -0.9997825   0.99997705 -0.98249745]\n",
      " [-0.8092     -0.99118733  0.9938392  -0.9163951  -0.8390757 ]\n",
      " [ 0.27587575  0.98118156 -0.1959674   0.9657002  -0.92443734]\n",
      " [ 0.8012901   0.9927326   0.9985852  -0.05828086  0.9561478 ]]\n"
     ]
    }
   ],
   "source": [
    "print(Y1_val)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Desenrolamento estático através do tempo"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-02T10:29:29.711228Z",
     "start_time": "2021-02-02T10:29:28.761657Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From <ipython-input-6-f473b0140bcd>:9: BasicRNNCell.__init__ (from tensorflow.python.ops.rnn_cell_impl) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "This class is equivalent as tf.keras.layers.SimpleRNNCell, and will be replaced by that in Tensorflow 2.0.\n",
      "WARNING:tensorflow:From <ipython-input-6-f473b0140bcd>:10: static_rnn (from tensorflow.python.ops.rnn) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `keras.layers.RNN(cell, unroll=True)`, which is equivalent to this API\n",
      "WARNING:tensorflow:From /home/liliane-hop/anaconda3/lib/python3.7/site-packages/tensorflow_core/python/ops/rnn_cell_impl.py:456: Layer.add_variable (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `layer.add_weight` method instead.\n",
      "WARNING:tensorflow:From /home/liliane-hop/anaconda3/lib/python3.7/site-packages/tensorflow_core/python/ops/rnn_cell_impl.py:460: calling Zeros.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Call initializer instance with the dtype argument instead of passing it to the constructor\n"
     ]
    }
   ],
   "source": [
    "tf.compat.v1.reset_default_graph()\n",
    "\n",
    "n_inputs = 3\n",
    "n_neurons = 5\n",
    "\n",
    "X0 = tf.compat.v1.placeholder(tf.float32, [None, n_inputs])\n",
    "X1 = tf.compat.v1.placeholder(tf.float32, [None, n_inputs])\n",
    "\n",
    "basic_cell = tf.compat.v1.nn.rnn_cell.BasicRNNCell(num_units=n_neurons)\n",
    "output_seqs, states = tf.compat.v1.nn.static_rnn(basic_cell, [X0, X1], dtype=tf.float32)\n",
    "Y0, Y1 = output_seqs\n",
    "\n",
    "init = tf.compat.v1.global_variables_initializer()\n",
    "\n",
    "X0_batch = np.array([[0, 1, 2], [3, 4, 5], [6, 7, 8], [9, 0, 1]])\n",
    "X1_batch = np.array([[9, 8, 7], [0, 0, 0], [6, 5, 4], [3, 2, 1]])\n",
    "\n",
    "with tf.compat.v1.Session() as sess:\n",
    "    init.run()\n",
    "    Y0_val, Y1_val = sess.run([Y0, Y1], feed_dict={X0: X0_batch, X1: X1_batch})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-02T10:29:29.729328Z",
     "start_time": "2021-02-02T10:29:29.717402Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[-0.75895983  0.3400113   0.94230473  0.52180266 -0.24049175]\n",
      " [-0.99932796  0.8492658   0.99848884  0.9842553  -0.4880398 ]\n",
      " [-0.9999983   0.97337997  0.99996144  0.99959934 -0.67597884]\n",
      " [-0.9995692  -0.7197753  -0.9989893   0.9990812   0.34861642]]\n"
     ]
    }
   ],
   "source": [
    "print(Y0_val)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-02T10:29:30.000663Z",
     "start_time": "2021-02-02T10:29:29.737252Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[-0.9999998   0.9668398   0.9989235   0.99952376 -0.716678  ]\n",
      " [-0.9176656  -0.4178842   0.1343207  -0.6269127  -0.20180708]\n",
      " [-0.9999971   0.7315453   0.9682632   0.9812145  -0.51459897]\n",
      " [-0.9713243   0.6028014   0.6579353   0.9050441   0.28244194]]\n"
     ]
    }
   ],
   "source": [
    "print(Y1_val)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Desenrolamento dinâmico através do tempo"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-02T10:29:31.198418Z",
     "start_time": "2021-02-02T10:29:30.009091Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From <ipython-input-9-d4f8e4306e00>:10: dynamic_rnn (from tensorflow.python.ops.rnn) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `keras.layers.RNN(cell)`, which is equivalent to this API\n",
      "[[[ 0.51700026 -0.53753775  0.80512583 -0.28706634 -0.55653304]\n",
      "  [-0.7321642  -0.99993515  0.23379648 -0.74560183 -0.9999998 ]]\n",
      "\n",
      " [[ 0.39513594 -0.9901547   0.94781274 -0.705881   -0.99836606]\n",
      "  [-0.13300234  0.6276108  -0.86745507  0.33607927 -0.67801356]]\n",
      "\n",
      " [[ 0.25757223 -0.9998373   0.9867889  -0.89813447 -0.9999953 ]\n",
      "  [-0.82658786 -0.990005   -0.8405862  -0.5952307  -0.99999267]]\n",
      "\n",
      " [[-0.9453138  -0.9930309  -0.99824226 -0.9850142  -0.99621993]\n",
      "  [-0.9082471  -0.77422476 -0.8328735  -0.87709147 -0.9963527 ]]]\n"
     ]
    }
   ],
   "source": [
    "n_steps = 2\n",
    "n_inputs = 3\n",
    "n_neurons = 5\n",
    "\n",
    "tf.compat.v1.reset_default_graph()\n",
    "\n",
    "X = tf.compat.v1.placeholder(tf.float32, [None, n_steps, n_inputs])\n",
    "\n",
    "basic_cell = tf.compat.v1.nn.rnn_cell.BasicRNNCell(num_units=n_neurons)\n",
    "outputs, states = tf.compat.v1.nn.dynamic_rnn(basic_cell, X, dtype=tf.float32)\n",
    "\n",
    "init = tf.compat.v1.global_variables_initializer()\n",
    "\n",
    "X_batch = np.array([\n",
    "        [[0, 1, 2], [9, 8, 7]], # instância 1\n",
    "        [[3, 4, 5], [0, 0, 0]], # instância 2\n",
    "        [[6, 7, 8], [6, 5, 4]], # instância 3\n",
    "        [[9, 0, 1], [3, 2, 1]], # instância 4\n",
    "    ])\n",
    "\n",
    "with tf.compat.v1.Session() as sess:\n",
    "    init.run()\n",
    "    outputs_val = outputs.eval(feed_dict={X: X_batch})\n",
    "\n",
    "print(outputs_val)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Manipulando sentenças de entrada de comprimento variável"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-02T10:29:31.917007Z",
     "start_time": "2021-02-02T10:29:31.204513Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[[ 0.93458766 -0.7193072   0.31245053 -0.76001716 -0.923558  ]\n",
      "  [ 1.         -0.9561068   0.97031444 -0.9994637  -1.        ]]\n",
      "\n",
      " [[ 0.9999978  -0.95408326  0.7066738  -0.9949294  -0.9999828 ]\n",
      "  [ 0.          0.          0.          0.          0.        ]]\n",
      "\n",
      " [[ 1.         -0.9932587   0.8932454  -0.9999052  -1.        ]\n",
      "  [ 0.9999999  -0.69140786  0.9328227  -0.933578   -0.99999595]]\n",
      "\n",
      " [[ 0.9999765   0.9915666  -0.9528005  -0.99862623 -0.9998361 ]\n",
      "  [ 0.99659544 -0.6414234   0.6648374  -0.6759344  -0.9990536 ]]]\n"
     ]
    }
   ],
   "source": [
    "n_steps = 2\n",
    "n_inputs = 3\n",
    "n_neurons = 5\n",
    "\n",
    "tf.compat.v1.reset_default_graph()\n",
    "\n",
    "X = tf.compat.v1.placeholder(tf.float32, [None, n_steps, n_inputs])\n",
    "seq_length = tf.compat.v1.placeholder(tf.int32, [None])\n",
    "\n",
    "basic_cell = tf.compat.v1.nn.rnn_cell.BasicRNNCell(num_units=n_neurons)\n",
    "outputs, states = tf.compat.v1.nn.dynamic_rnn(basic_cell, X, dtype=tf.float32, sequence_length=seq_length)\n",
    "\n",
    "init = tf.compat.v1.global_variables_initializer()\n",
    "\n",
    "X_batch = np.array([\n",
    "        #step 0     step 1\n",
    "        [[0, 1, 2], [9, 8, 7]], # instância 1\n",
    "        [[3, 4, 5], [0, 0, 0]], # instância 2 (preenchido com vetor de zeros)\n",
    "        [[6, 7, 8], [6, 5, 4]], # instância 3\n",
    "        [[9, 0, 1], [3, 2, 1]], # instância 4\n",
    "    ])\n",
    "\n",
    "seq_length_batch = np.array([2, 1, 2, 2])\n",
    "\n",
    "with tf.compat.v1.Session() as sess:\n",
    "    init.run()\n",
    "    outputs_val, states_val = sess.run([outputs, states], feed_dict={X: X_batch, seq_length: seq_length_batch})\n",
    "\n",
    "print(outputs_val)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Treinando um classificador de sequência"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-03T11:00:02.897639Z",
     "start_time": "2021-02-03T11:00:01.024239Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From <ipython-input-6-905f5fa067ca>:13: BasicRNNCell.__init__ (from tensorflow.python.ops.rnn_cell_impl) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "This class is equivalent as tf.keras.layers.SimpleRNNCell, and will be replaced by that in Tensorflow 2.0.\n",
      "WARNING:tensorflow:From <ipython-input-6-905f5fa067ca>:14: dynamic_rnn (from tensorflow.python.ops.rnn) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `keras.layers.RNN(cell)`, which is equivalent to this API\n",
      "WARNING:tensorflow:From /home/liliane-hop/anaconda3/lib/python3.7/site-packages/tensorflow_core/python/ops/rnn_cell_impl.py:456: Layer.add_variable (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `layer.add_weight` method instead.\n",
      "WARNING:tensorflow:From /home/liliane-hop/anaconda3/lib/python3.7/site-packages/tensorflow_core/python/ops/resource_variable_ops.py:1635: calling BaseResourceVariable.__init__ (from tensorflow.python.ops.resource_variable_ops) with constraint is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "If using Keras pass *_constraint arguments to layers.\n",
      "WARNING:tensorflow:From /home/liliane-hop/anaconda3/lib/python3.7/site-packages/tensorflow_core/python/ops/rnn_cell_impl.py:460: calling Zeros.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Call initializer instance with the dtype argument instead of passing it to the constructor\n",
      "WARNING:tensorflow:From <ipython-input-6-905f5fa067ca>:16: dense (from tensorflow.python.layers.core) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use keras.layers.Dense instead.\n",
      "WARNING:tensorflow:From /home/liliane-hop/anaconda3/lib/python3.7/site-packages/tensorflow_core/python/layers/core.py:187: Layer.apply (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `layer.__call__` method instead.\n"
     ]
    }
   ],
   "source": [
    "tf.compat.v1.reset_default_graph()\n",
    "\n",
    "n_steps = 28\n",
    "n_inputs = 28\n",
    "n_neurons = 150\n",
    "n_outputs = 10\n",
    "\n",
    "learning_rate = 0.001\n",
    "\n",
    "X = tf.compat.v1.placeholder(tf.float32, [None, n_steps, n_inputs])\n",
    "y = tf.compat.v1.placeholder(tf.int32, [None])\n",
    "\n",
    "basic_cell = tf.compat.v1.nn.rnn_cell.BasicRNNCell(num_units=n_neurons)\n",
    "outputs, states = tf.compat.v1.nn.dynamic_rnn(basic_cell, X, dtype=tf.float32)\n",
    "\n",
    "logits = tf.compat.v1.layers.dense(states, n_outputs)\n",
    "xentropy = tf.compat.v1.nn.sparse_softmax_cross_entropy_with_logits(labels=y, logits=logits)\n",
    "loss = tf.reduce_mean(xentropy)\n",
    "\n",
    "optimizer = tf.compat.v1.train.AdamOptimizer(learning_rate=learning_rate)\n",
    "training_op = optimizer.minimize(loss)\n",
    "\n",
    "correct = tf.compat.v1.nn.in_top_k(logits, y, 1)\n",
    "accuracy = tf.reduce_mean(tf.cast(correct, tf.float32))\n",
    "\n",
    "init = tf.compat.v1.global_variables_initializer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-03T11:00:03.898112Z",
     "start_time": "2021-02-03T11:00:02.929731Z"
    }
   },
   "outputs": [],
   "source": [
    "def shuffle_batch(X, y, batch_size):\n",
    "    rnd_idx = np.random.permutation(len(X))\n",
    "    n_batches = len(X) // batch_size\n",
    "    for batch_idx in np.array_split(rnd_idx, n_batches):\n",
    "        X_batch, y_batch = X[batch_idx], y[batch_idx]\n",
    "        yield X_batch, y_batch\n",
    "        \n",
    "(X_train, y_train), (X_test, y_test) = tf.keras.datasets.mnist.load_data()\n",
    "X_train = X_train.astype(np.float32).reshape(-1, 28*28) / 255.0\n",
    "X_test = X_test.astype(np.float32).reshape(-1, 28*28) / 255.0\n",
    "y_train = y_train.astype(np.int32)\n",
    "y_test = y_test.astype(np.int32)\n",
    "X_valid, X_train = X_train[:5000], X_train[5000:]\n",
    "y_valid, y_train = y_train[:5000], y_train[5000:]\n",
    "\n",
    "X_test = X_test.reshape((-1, n_steps, n_inputs))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-02T10:42:28.954420Z",
     "start_time": "2021-02-02T10:29:35.619106Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 Last batch accuracy: 0.91333336 Test accuracy: 0.9212\n",
      "10 Last batch accuracy: 1.0 Test accuracy: 0.9741\n",
      "20 Last batch accuracy: 0.9866667 Test accuracy: 0.9739\n",
      "30 Last batch accuracy: 1.0 Test accuracy: 0.9793\n",
      "40 Last batch accuracy: 0.98 Test accuracy: 0.977\n",
      "50 Last batch accuracy: 1.0 Test accuracy: 0.9786\n",
      "60 Last batch accuracy: 0.99333334 Test accuracy: 0.9773\n",
      "70 Last batch accuracy: 0.97333336 Test accuracy: 0.9777\n",
      "80 Last batch accuracy: 1.0 Test accuracy: 0.9791\n",
      "90 Last batch accuracy: 0.9866667 Test accuracy: 0.9771\n"
     ]
    }
   ],
   "source": [
    "n_epochs = 100\n",
    "batch_size = 150\n",
    "\n",
    "with tf.compat.v1.Session() as sess:\n",
    "    init.run()\n",
    "    for epoch in range(n_epochs):\n",
    "        for X_batch, y_batch in shuffle_batch(X_train, y_train, batch_size):\n",
    "            X_batch = X_batch.reshape((-1, n_steps, n_inputs))\n",
    "            sess.run(training_op, feed_dict={X: X_batch, y: y_batch})\n",
    "        if epoch % 10 == 0:\n",
    "            acc_batch = accuracy.eval(feed_dict={X: X_batch, y: y_batch})\n",
    "            acc_test = accuracy.eval(feed_dict={X: X_test, y: y_test})\n",
    "            print(epoch, 'Last batch accuracy:', acc_batch, 'Test accuracy:', acc_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-01T11:10:57.345785Z",
     "start_time": "2021-02-01T11:10:57.342794Z"
    }
   },
   "source": [
    "### Prevendo Séries temporais"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-03T10:19:05.799223Z",
     "start_time": "2021-02-03T10:19:05.782394Z"
    }
   },
   "outputs": [],
   "source": [
    "t_min, t_max = 0, 30\n",
    "resolution = 0.1\n",
    "\n",
    "def time_series(t):\n",
    "    return t * np.sin(t) / 3 + 2 * np.sin(t*5)\n",
    "\n",
    "def next_batch(batch_size, n_steps):\n",
    "    t0 = np.random.rand(batch_size, 1) * (t_max - t_min - n_steps * resolution)\n",
    "    Ts = t0 + np.arange(0., n_steps + 1) * resolution\n",
    "    ys = time_series(Ts)\n",
    "    return ys[:, :-1].reshape(-1, n_steps, 1), ys[:, 1:].reshape(-1, n_steps, 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-02T10:49:28.154646Z",
     "start_time": "2021-02-02T10:49:15.836134Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 \tMSE: 20.400679\n",
      "100 \tMSE: 0.6562308\n",
      "200 \tMSE: 0.2516394\n",
      "300 \tMSE: 0.099226914\n",
      "400 \tMSE: 0.079162315\n",
      "500 \tMSE: 0.060888298\n",
      "600 \tMSE: 0.049327023\n",
      "700 \tMSE: 0.053943075\n",
      "800 \tMSE: 0.04837853\n",
      "900 \tMSE: 0.05036985\n",
      "1000 \tMSE: 0.044962913\n",
      "1100 \tMSE: 0.051766038\n",
      "1200 \tMSE: 0.042916216\n",
      "1300 \tMSE: 0.043102656\n",
      "1400 \tMSE: 0.046322674\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([[[-3.4285395 ],\n",
       "        [-2.4873607 ],\n",
       "        [-1.1441462 ],\n",
       "        [ 0.64267945],\n",
       "        [ 2.1547513 ],\n",
       "        [ 3.0455828 ],\n",
       "        [ 3.5083144 ],\n",
       "        [ 3.4149559 ],\n",
       "        [ 2.8399527 ],\n",
       "        [ 2.1580863 ],\n",
       "        [ 1.6326289 ],\n",
       "        [ 1.5457834 ],\n",
       "        [ 1.9189045 ],\n",
       "        [ 2.7590277 ],\n",
       "        [ 3.8841624 ],\n",
       "        [ 5.085305  ],\n",
       "        [ 6.0726795 ],\n",
       "        [ 6.618793  ],\n",
       "        [ 6.60153   ],\n",
       "        [ 6.0408335 ]]], dtype=float32)"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf.compat.v1.reset_default_graph()\n",
    "\n",
    "n_steps = 20\n",
    "n_inputs = 1\n",
    "n_neurons = 100\n",
    "\n",
    "X = tf.compat.v1.placeholder(tf.float32, [None, n_steps, n_inputs])\n",
    "y = tf.compat.v1.placeholder(tf.float32, [None, n_steps, n_outputs])\n",
    "\n",
    "cell = tf.compat.v1.nn.rnn_cell.BasicRNNCell(num_units=n_neurons, activation=tf.nn.relu)\n",
    "rnn_outputs, states = tf.compat.v1.nn.dynamic_rnn(cell, X, dtype=tf.float32)\n",
    "\n",
    "n_outputs = 1\n",
    "learning_rate = 0.001\n",
    "\n",
    "stacked_rnn_outputs = tf.reshape(rnn_outputs, [-1, n_neurons])\n",
    "stacked_outputs = tf.compat.v1.layers.dense(stacked_rnn_outputs, n_outputs)\n",
    "outputs = tf.reshape(stacked_outputs, [-1, n_steps, n_outputs])\n",
    "\n",
    "loss = tf.reduce_mean(tf.square(outputs - y))\n",
    "optimizer = tf.compat.v1.train.AdamOptimizer(learning_rate=learning_rate)\n",
    "training_op = optimizer.minimize(loss)\n",
    "\n",
    "init = tf.compat.v1.global_variables_initializer()\n",
    "saver = tf.compat.v1.train.Saver()\n",
    "\n",
    "n_iterations = 1500\n",
    "batch_size = 50\n",
    "t_instance = np.linspace(12.2, 12.2 + resolution * (n_steps + 1), n_steps + 1)\n",
    "\n",
    "with tf.compat.v1.Session() as sess:\n",
    "    init.run()\n",
    "    for iteration in range(n_iterations):\n",
    "        X_batch, y_batch = next_batch(batch_size, n_steps)\n",
    "        sess.run(training_op, feed_dict={X: X_batch, y: y_batch})\n",
    "        if iteration % 100 == 0:\n",
    "            mse = loss.eval(feed_dict={X: X_batch, y: y_batch})\n",
    "            print(iteration, '\\tMSE:', mse)\n",
    "    \n",
    "    X_new = time_series(np.array(t_instance[:-1].reshape(-1, n_steps, n_inputs)))\n",
    "    y_pred = sess.run(outputs, feed_dict={X: X_new})    \n",
    "    saver.save(sess, 'model/my_time_series_model')\n",
    "    \n",
    "y_pred"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Rnn criativa"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-02T10:50:56.546754Z",
     "start_time": "2021-02-02T10:50:55.553424Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from model/my_time_series_model\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAfAAAAEWCAYAAACZscV5AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nO2dd5hV5bX/P4uuFEFBOowUUZpKHYrYsPcajSXmJmquxp9JzDUmpt3cmJsYNaYYvZhYcu1gwd5RVBQEBgEp0kbaKFhBOsz7+2PtfecwTDkzs/tZn+c5zz7n7DN7r73Pmf3da71rrVeccxiGYRiGkS4axW2AYRiGYRh1xwTcMAzDMFKICbhhGIZhpBATcMMwDMNIISbghmEYhpFCTMANwzAMI4WYgBupR0Raich7IjIpoO0dLiKLg9hWA2y4U0R+0YC//1pEegVpk1FBQ7+fWrYtIjJNRB4IcJsfiMiRQW3PSAZideBGEIjIWOAmYACwC1gI/MA5914E+74LeBAYBmx0zt0Z9j6ThIi8DtzvnPtH3LYY+eMJ6v3OuW6V3r8Y/T8qAv7snHsneuuMNNAkbgOM9CMibYBngH8HHgWaAYcD26LYv3PuMu/plCj2FzYi0tg5tytuO4zYaAb8GmiD/h8ZRpVYCN0IggMBnHMPOed2Oee2OOdecs7N9T8gIv8mIgtF5AsReVFEeuasO1ZEFonIVyLyNxF5Q0S+6637tYjcn/PZIhFxItLEe72PiPxTRMpEZI2I/FZEGnvrLhWRt0TkZm+/K0TkxJxt7Ssi94jIWm/9k977R4rI6pzPXS8iy0Rko4gsEJEzc9b18ez9SkQ+FZFHqjtJIjLWC41+KSKrRORS7/17ReQOEXlORDYBR3nv/dZb305EnhGR9Z6dz4hIN2/djehF/m9e2Pxv3vvOs22kiHzsnxNv3ZkiMtd7PkJE3vFsKvPOfzNvnYjIn0RknYhsEJF5IjKwmmOr8lx66y4TkaUi8rmIPCUiXXLWORG5UkSWeOf3v0Skt3eeNojIo749Veyztt/G697vYZp3bp4Wkf1E5AFv2++JSFHO3//Z+142iMgsEalWPCt9P0eKyGoRudY7V2Ui8u2cz57k/W42er/RH4tIS+B5oItn29ci0kVERgDfBT4G3vd+C81ytuVE5Hve+fpSRG4XEal0rhfm/FaHeO+Xisj42r5zI2U45+xhjwY9UE/hM+A+4ESgXaX1pwNLgYPRqM/PgWneuvbARuAcoCnwQ2An8F1v/a/RMKO/rSLAAU28108A/wO0BPYHZgBXeOsuBXYAlwGN0QjBWiqGjp4FHgHaefs+wnv/SGB1zj7PBbqgN7zfADYBnb11DwE3eOtaAGOrOUc9veO8wNvXfsCh3rp7ga+AMTnbuRf4rbd+P+BsYG+gNTAReDJn26/75yvnPQf08Z4vA47NWTcRuN57PhQo9r6XIiqGPgCOB2YBbQHxvr/O1RxfdefyaOBTYAjQHPgrMLWSnZPR39AANGrzKtAL2AdYAHyrmn3W9tt4Hf3d9c7Z1ofAeO94/wXck/P3F3nnuglwLSqiLarZd+73cyT6m/2Nd+wnAZvx/g+AMuBw73k7YEhVv7Pavo+c8/WM9530ANYDJ+T8TtcAw73vqw/Q01tXCozPZx/2SM8jdgPskY2Hd3G/F1jtXcyeAjp6654HvpPz2UbeBa4ncAnwbs468bZRq4ADHb0L/l456y8ApnjPLwWW5qzb2/vbTkBnoJxKNxve5/a4sFZaPwc43Xv+L2AC0K2W8/NT4Ilq1t0L/KuK935bzecPBb7Ief06NQv4b4G7veet0RuQntVs+we+naj4fuhd7BvVcGw1nct/AjflvG6F3lQV5dg5Jmf9LOAnOa9vAW6rZr/V/jZyzssNlbb1fM7rU4E5NRzXF8AhNXxnuQK+xd+v9946oNh7vhK4AmhTl99Z5e8j53yNzXn9KBU3Yy8C11SznVI8Aa9tH/ZIz8NC6EYgOOcWOucudZqQMxD1WG/zVvcE/uyF7L4EPkeFuqv3uVU523G5r2uhJ+rxlOVs+39QT9zn45xtb/aetgK6A587576obScicomIzMnZx0A0cgBwnXcsM0Qzff+tms10Rz3h6qj2mEVkbxH5HxH5SEQ2AFOBtrlh8Vp4EDhLRJoDZwGznXMfeds+0AvJf+xt+3f+sTnnXgP+BtwOrBORCaL5DlUdW3Xnsgvwkf/COfc1Gq3pmvOZT3Keb6nidas8j7Mq8t62F9peKDoc8iXqtbcnPz5zzu3Meb05Z9tno175R6LDLaOq20hN30cOH+c8z91Pbb+xuuzDSAEm4EbgOOcWoR6KP166Cg1rt8157OWcm4aGF7v7f+uN53XP2dwm1HP26ZTzfBXqgbfP2W4b59yAPMxcBewrIm1r+pDoWP1dwPeB/ZxzbYH5qGjjnPvYOXeZc64L6mX9XUT6VLO/3jXsqqZykGuBfsBI51wbYJxvXh5/i3NuASqiJwLfRAXd5w5gEdDX2/bPcraLc+4vzrmhQH801+E/qthFTedyLXqjpQbr2O9+aKi3odT026gT3nj3dcB5aCShLTqsITX+YR44595zzp2O3lg+iXrNUPX3VuP3UQu1/caC2IeRIEzAjQYjIgd5CTx+YlV3NJT9rveRO4GfisgAb/0+InKut+5ZYICInOUlH/0/dr8QzwHGiUgPEdkHDUUD4JwrA14CbhGRNiLSyEuAOqI2m72/fR4V3HYi0lRExlXx0ZbohXa9Z/u3qbgxQUTO9Y8bDbk6NJxcmQeA8SJynog08ZKpDq3NTo/WqLf4pYjsC/yq0vpP0DHjmngQuAYV/4mVtr0B+FpEDkLzBAAQkeGiSXBNUbHcWtWx1XIuHwK+LSKHehGA3wHTnXOleRx3bVT726gHrdGhn/VAExH5JTou3yBEpJmIXCgi+zjndqDn2j+HnwD7ebbn2lHl95EH/wB+LCJDRekjOcmiAe3DSBAm4EYQbARGAtNFs6jfRb3UawGcc08AfwAe9kJ281FvEOfcp2jyze/R0Gpf4G1/w865l9HkqLno+OgzlfZ9CVp2swAV0EnomGw+XIyOxy5Cxyx/UPkDnvd6C/AOesEdlGsfmjA0XUS+Rsf9r3HOLa9iOyvRMOq16BDCHOCQPO28DdgLTQZ7F3ih0vo/A+eIZn//pZptPAQcAbzmnXOfH6Ne+UY00pCbRd/Ge+8L1IP/DPhjNduv8lw6514BfgE8hkZbegPn13y4+ZHHb6MuvIie1w/RY91K/kM5tXExUOr99r8HXAj/F6l6CFjuDc90oebvo0accxOBG9GbtY2ot79vFR+t9z6MZGGNXIzEIdaYxDAMo1bMAzcMwzCMFGICbhiGYRgpxELohmEYhpFCzAM3DMMwjBSSqslM2rdv74qKiuI2wzAMwzAiYdasWZ865zpUtS5VAl5UVMTMmTPjNsMwDMMwIkFEPqpunYXQDcMwDCOFxCrgItJWRCaJTiW5sKYewYZhGIZhVBB3CP3PwAvOuXO8+Wj3ru0PDMMwDMOIUcC9/r/j0Ckfcc5tB7bHZY9hGIZhpIk4Q+gHoBMH3CMiJSLyD2+mot0QkctFZKaIzFy/fn30VhqGYRhGAolTwJsAQ4A7nHOHobMdXV/5Q865Cc65Yc65YR06VJlJbxiGYRgFR5wCvhpY7Zyb7r2ehAq6YRiGYRi1ENsYuHPuYxFZJSL9nHOLgWPQKSENwzAMA4BHH4UVK2DcOBhldUq7EXcW+tXAA14G+nLg2zHbYxiGERjbt0OzZnFbkV4+/hguuADKy6F9e1i7Fpo2jduq5BBrHbhzbo43vj3YOXeGc+6LOO0xDMMIiocegjZt4B82q329mThRxfvGG+HTT+HVV+O2KFlYJzbDMIyA2bEDfv5z2LULLrsMHnssbovSyUMPweDBcO210LatvjYqMAE3DMMImPvvh+XL1YPs2hUefDBui9LHypXwzjtw/vnQvDmcdRY8/jhs3Rq3ZcnBBNwwDCNgJk2CAw+E00+HU0+FF1804akr776ry+OP1+Vpp8HXX8OsWfHZlDRMwA3DMAKmpASKi0FEhWfTJnj99bitShfz50OjRnDwwfp65EhdTp9e/d8UGibghmEYAfLJJ1BWBoceqq+POgpatoSnnorXrrQxbx707Qt77aWvO3WCnj0rPHPDBNwwDCNQ5szR5WGH6bJFC61hfvPN+GxKI/PmwaBBu79XXGweeC4m4IZhGAFSUqLLQw6peG/kSPjgA9iwIR6b0samTZoEWFnAR47U5LaysnjsShom4IZhGAFSUgJFRdCuXcV7xcXgHMycGZtZqWLBAj1fVXngYF64jwm4YRhGgMyZUzH+7TNihC5t/DY/5s3T5cCBu79/yCGaGOgPUxQ6JuCGYRgBsXOnhn4POmj399u1g379zHPMl8WLtQVtr167v7/33prYNnduPHYlDRNwwzCMgFi7VkX8gAP2XDdypHrgzkVvV9ooLdWM88aN91w3eLAJuI8JuGEYRkCUluqyqGjPdcXFsG4dfPRRlBalk9LSqs8hqIAvW6ZNXQodE3DDMIyAqEnA/UYkNg5eO7UJOGijl0LHBNwwDCMgfAHv0WPPdYMGaVMSGwevmc2bNVJRm4D7iW6FjAm4YRhGQJSWQpcu2rylMk2bwrBh5oHXhj/EUJ2A9+wJrVvbODiYgBuGYQRGTaFf0DB6SQls2xaVRemjpmEI0P7ogwaZgIMJuGEYRmDUJuDFxSre778flUXpozYBh4pM9ELP6DcBNwzDCICdO2HVqto9cLBx8JooLdUa8E6dqv/M4MHw5ZewenVkZiUSE3DDMIwA8GvAaxLwbt2ga1cbB68Jvwa8UQ3q5CeyFXoY3QTcMAwjAFat0mX37jV/buRI88Br4qOPqs7iz8VvsWoCbhiGYTSYtWt12aVLzZ8rLtZGJOvXh29TGlmzpvaboH320UiHCbhhGIbRYPwpLjt3rvlz/jj4jBnh2pNGdu3S89i1a+2ftZaqJuCGYRiBsHat1nrvt1/Nnxs6FJo0gTffjMauNPHJJyri+Qr44sWwdWv4diUVE3DDqALnKsY0DSMfyso0c7qm5CuAli3hiCPgqaeisStNrFmjy27dav/soEEq9gsXhmtTkjEBN4wq+MlPNJHm6KOtVKUurFunoWE/nFxIrF1b+/i3zxlnqPB8+GG4NqUN/38tXw8cCjuMbgJuGJV4+mn44x/hmGPgnXfgl7+M26J0sGsXjBunY7wHHVR4s0WVldU+/u1z2mm6nDw5PHvSSF088D59tGWtCbhhGP/Hr38N/fvDs8/Cd74D999fkWFsVM8jj+iY5GWXwYYNhRcirosH3qMHDBkCDz5o3cRyWb1a8wjat6/9s02awIABJuCGYXgsXgyzZ6sINW8OP/qRepZ/+1vcliWb8nK48Ua9oN5xh5YBPfhg3FZFx9at8MUX+Qs4wFVXwZw58Pzz4dmVNtas0XNYWx6BT6FnopuAG0YODz0EInDeefq6Vy8YPx6efDJeu5LOzJmwYAFcey00bgwXXAAvvgiffhq3ZdGQbwlZLhdfrJ74f/6ndnAzVMDzCZ/7DB6seReffBKeTUnGBNwwcnj4YTjyyN09qeOP14Qjy0qvnsmTVbhPP11fn3mmitLUqfHaFRW+gNfFA2/aVMV7xgw9bzNmmJCvXp1fApvP0KG6LNTOdibghuGxerWG0E89dff3jz1Wly+/HL1NaeHJJ7U0at999fWgQbr84IP4bIoSP0eiLh44wKWXwp13arRi5Eho1w6uuKIwhdw59cDrIuAjRmgi25Qp4dmVZEzADcPDb6xxxBG7vz9woNb3moBXzZIlGj73vW/QWudevWD+/PjsipL6eOA+V1yhwvXII3D22TBhAlx9dbD2pYEvv4TNm+sWQm/eHEaPhtdfD82sRGMCbhgeU6dC69ZwyCG7vy+iXvirr1rGcFX4pVC5Ag7qhReKgK9dq1nRtXVhq46OHTXv4t57NY/gzjsLr9WqX0JWFw8cdMjr/ffh888DNynxxC7gItJYREpE5Jm4bTEKm6lTYexYHcutzKhROvnEypXR25V0nnwSDjtMp4DMZeBAbVSybVs8dkWJXwOeb/Z0TfziF+pZ3n9/w7eVJupSA57LkUfqjXWh5FvkEruAA9cABdwMz0gCn36qYeBx46pe7yfLzJoVnU1pYN06mDZtT+8bVMB37iyMbmNr19Z9/Ls69tlHG708/DDs2BHMNtNAXbqw5TJihEbOCrFSJFYBF5FuwMnAP+K0wzD8cOXo0VWvHzxYQ6Qm4Lvz9NPq/Zxxxp7r/DmbCyGMXlZWv/Hv6rjoIo34vPJKcNtMOr4HXtfz2Lw5fOMbMGlS4XX/i9sDvw24Diiv7gMicrmIzBSRmettAt1Q2batsO74c5k9W5eHHlr1+hYttEmJCfjuPPooHHBARV/qXA48UG96CiETPUgPHLR0sWnTwgoLr14N++8PzZrV/W8vvRQ2bYLHHgvcrEQTm4CLyCnAOudcjZdE59wE59ww59ywDh06RGRdYfHZZ3DJJVoC1KUL/Pa3hZesVVKivZXbtKn+M0OHqoAX2rmpjjVr1EO86CJN9KtMs2Y6nllaGrlpkbJtmyZQBemBN2+uN0XvvRfcNpNOXUvIchk9Wv9///SnwnJC4vTAxwCniUgp8DBwtIgUWNpG/CxZomNIjzyinaFGjNAkmgkT4rYsWkpKNBGrJoYO1bFya+ii3H+/tlC95JLqP9OjR/YT/+rThS0fhg3TDneFcsPYEAEXgd//XrPR//CHYO1KMrEJuHPup865bs65IuB84DXn3EVx2VOILFsGRx2lE0+8/rqWrjz9NJxwAlxzjYp7IfDll7BihU4uURO+wM+ZE75NSWfnTrj7bhgzRj2f6igkAQ/SAwcYPhy++kr/TwuB1avrnoGey9lnawvfX/9aHZJCIO4xcCMmSkt1rustW7S+edQofb9RI7jnHr3r/8tfYjUxMnxBrs0D95Oy5s0L15408K9/aXb5j35U8+d69NAL865d0dgVB/XtwlYbw4bpshDC6Fu36lBefT1wnwkTNJx+4YXa3S7rJELAnXOvO+dOiduOQmH+fO02tnGjjmFWTkDq1EmzOu+9V73zrFNSosvaBLx1a03YKnQB/+or+NWvtPXnmWfW/NkePVS8fS81i4Tlgffvr8mTM2cGu90kUt8a8Mq0agXPPQcHH6xDglmfBjgRAm5Ew65dOtXj6NGa6PHqq9WL1tVXa0nG//5vtDbGwezZevHdf//aP1vo0xdu3AgnnqizP916a9XJa7n06KHLLIfR/S5s+cxhXReaNtWoTyH83urbha0qWrXS6oivv4af/7zh20syJuAFQkmJCveVV+rY2owZNXucw4erB1AIZRn5JLD5DBqkoeOtW8O1KYksWqRe94wZ2mSkupr5XApBwMvKNGoVRBe2yvTtWxhj4PVt4lIdBx+sYfRHHsl2FNEEPONs3669lYcN03HvBx7QsHk+oarTTtM61C++CN3M2NiyRYWpLgK+a5dOL1pIrFkDxxyj45QvvQRnnZXf33XvrsssC3jQNeC59OkDH32k/8dZxv99+Dd8QfDd7+rkKA8/HNw2k4YJeIb56iudhOPWW+Gyy1SovvnN2sOePqedpmL1wgvh2hkn8+bpMeYr4H6+QCGNgzsH55yjnswrr2jyY760aQNt22ZbwIPuwpZLnz5aqvfRR+FsPymsXKl9KFq1Cm6bI0boEMR99wW3zaRhAp5RduyAc8/VPtX3368lYu3a1W0bI0bouPBTT4VjYxLwE9hqKyHz6dNHm2wUwrikzyuvwLvvwm23VczzXReyXkoWtgcOsHRpONtPCitXVkRrgkIETj5Zs/izOuRlAp5RfvlLnb96wgQdC6oPjRtrS8csT6NZUqI3NpVn0qqOJk20pWoheeB/+YveyF1Uzy4N3btnV8C3bdNhhTA9cMi+gK9aFWz43Ke4WJ0Z/0Y9a5iAZ5AFC+Dmm7U/8Le/3bBtHXGETqqwaFEgpiWO2bO1/3m+wwqgXmiheOClpfDss3DFFRp5qA9du2a3jOzjj3UZlgfeoYOWL2ZdwFeuDEfAR47U5bvvBr/tJGACnkF++EP9p7/ppoZv64gjdPnGGw3fVtLYuVM96XzHv30GDdIL96efhmNXknj+eY2+XHxx/bfRpYtOO5rFRCy/zjgsD1wEevfOtoBv2KDdEMMQ8M6ddbsm4EYqmDlTs4R/+lO9e28ovXvrxSmLAr5okY6N1VXACymRbcoUrVioqV1qbfji5nurWSKsPui59OmTbQH35xYIQ8BBw+jTp4ez7bgxAc8YN90E++yjIc8gEIFx41TAszYOnm8Htsr4iVxZD6OXl6uAH3103YYYKuMLeBa7YoXtgQP06qW9+surnXQ53fj5EUEnsfmMHKlZ/OvWhbP9ODEBzxArV2rjle99r+ZpMevKuHHqaaxYEdw2k0BJibaq7Nevbn/XsaNGN7LugX/wgQ4THHVUw7bjN+fIooCXlWmyZ5gzHXfvrolY69eHt484CdsDP/hgXX74YTjbjxMT8Axx3316lx6U9+3jT3SStXGkkhINhzdpUre/EymMRLYpU3TZUAHPugceVhc2H98zzeo0titX6k1QWMMQffvqMouzK5qAZ4Tycp185KijdMKNIBk4EFq2zJaAO6ezkNU1fO4zeLB6qFmeZWvWLBWnfEvsqqN9e71JyqKAl5WFO/4NFV0T/XajWWPFCj3Gut5I50tRkW7bBNxILG+9BcuXN7xsrCqaNNHe6FkS8NJSzXytr4APGqRtGpcvD9SsRDFv3p4z1dWHRo1U5LIo4GvXhjv+Ddn3wJcurfCSw6BJE3VqTMCNxPLII7DXXvn3qK4rxcUact6yJZztR01dO7BVxk9ky+o4+M6d2k+gPp3XqqJLl4oZp7JEFB54+/bQrFl2PfAlSxpW5ZAPffuagBsJZdcuTV47+WQNdYfBqFF6UZ89O5ztR01JiY671VegBgzQsfCsjoMvXapdxoIU8Kx54Fu3apJf2B54o0YaYs6igH/+uU6WFKYHDrr9pUuzV0ljAp4B3n5b52c+55zw9uF3NHrnnfD2ESUlJZqd2qJF/f5+773Va8iqB+4fVxAhdNBM9KwJuF/+VFQU/r66dctmCN2vb4/CA9+0KXsdAU3AM8CkSSpEJ50U3j46dtRxpKyMg9dlDvDqGDw4uwI+d65GKPwSnIbSpYvmHGzeHMz2kkBpqS6jEvAseuB+WDsKAc/dX1YwAU85zulsYePHa/vUMBk1KhsC/skn6g0GIeBLl8LGjcHYlSTmzdOLXn0jFJXxw8xZ8oD8KT6jEPDu3VXAs9bMZelSHYrq1Svc/fg3CMuWhbufqDEBTzkffKAXklNPDX9fxcWaiJT2UF59O7BVZsQIvYF6772G25Q0Fi2C/v2D214Wa8FLSzXDOewxcFAPPIvNXJYu1ZuToG4Uq8NvJpS1REoT8JTz9NO6POWU8PeVlYYu06frXf/QoQ3bTlZnOtq1S8vjgkwsyqqAd+8eXv1yLlktJYsiAx10Jr3998/eMIQJeMp5+mkVoii8gMGD9U457YI1fbpmkTd0yKFdOzjooOwk9vmsWqXeXhgCniUPqLQ0mvA5ZPMGqLxcSxUPOiia/XXtagJuJIh161RMo/C+QWtRhw5Nt2A5p+esuDiY7fl5AVkqTwkjsahtW+1TkCUBikPAs5RDsHy55o80dCgrX7KYCGgCnmL8uZqjGP/2GTVKa8G3bYtun0GyZInWnfrh74YyapTWAmcpOSaM0h6RbNWCb9umYhqVgHfsqOcwK+cPKnpK1LeZUl0xATcSxdNP60Uxqn8AUM912zbtI55G/HmBg/LAR4/WZZbmS1+6VL3loDuMZUnAV63Sm+eoBLxJEx3Dzcr5A00mbdJEh7OioFs3bRyTlW6SYAKeWrZtgxdf1PB5Q+Zqriu+8KV1HPydd3TsO6j65v79VZhefjmY7SWBpUuhd+/gZ9jKkoD7PfCjEnDQ85elEHpJiYp38+bR7M+fFCZLeRgm4CnljTfg66+jG//26dpVM2LTKuBvvqlec+PGwWxPBI47TgU8KzOThZUZ7At4FvIF5s/XZZCldrWRpQlhnNMQepTRwyzO6mYCnlImT9Z2nuPHR7/v4uJ0JrJ99pleeMeNC3a7xx2noTm/vjzN7Nql4/lhCfimTdlofDNvnk612r59dPvMkge+apXWtEeVwAYm4EZC8LuvHXecjlVGzahR2jwmbReTt97SZdAC7t9EvfBCsNuNg7Iy2L5dQ+hBk6VmGvPmBTfRS7506aJdBHfujHa/YfDSS7o88sjo9un//kzAjVgpKdEf4WmnxbN/fxzcTwhLC1On6njb8OHBbrdDBz0njz8e7HbjIMz+3lmpZd61SzsgRi3gnTvrzfsnn0S73zB4/nn1iAcOjG6fLVtqOWOWmuGYgKeQp57Ssdeox799DjsMmjZNXxh96lQV2jCSZs47T2+s0j5Zggl47SxdqlOJBjVTW75kpRZ8xw545RU48cRoE3BBvfC0n79cTMBTyOTJmojVoUM8+2/RQpNP0pTItnGjJs0EHT738adynTgxnO1HhS/gPXsGv22/LC3tAu7PQBeHBw7pP3/TpsGGDSrgUdOpE3z8cfT7DYvYBFxEuovIFBFZICIfiMg1cdmSJlau1Brs00+P147iYp3EY8eOeO3Il2nTtHVjWALevbveVD36aDjbj4rSUm0aEkZuRatW0KZN+gXo/fe1xC6oUsR8yYoHft998SXgmoAHx07gWudcf6AYuEpEIizKSCdPPaXLuMa/fYqLtSFCWubDnjpVS8eCauBSFeedpxf3xYvD20fYhN0eNAu14C+9pDPRRZ1AmoVubOvXw4MPwre+Ff70x1XhC3gWShkhRgF3zpU552Z7zzcCC4GucdmTFp54Avr100ecpG1msqlTtY97q1bh7SMLYfSwBbxr13QL0CefwIwZ8eSfZKEb2113aROq738/nv136qSORxZKGSEhY+AiUgQcBuyR1ywil4vITBGZuT5rk+HWkbIymDIFvvGNuC2BHj30nyENidqcWCUAACAASURBVGxbtuhFN6zwuU/XrjB2bHrD6Lt26RBN2B54msvInntOl3ElkKa5FnzHDvj73+HYY6NtgJNLp066zEoYPXYBF5FWwGPAD5xzGyqvd85NcM4Nc84N6xBX1lZCeOQRDf1ccEHclmgoz5+JK+nMmKG1zYcfHv6+zjtPhxUWLgx/X0FTVqYX2ShC6GkNYU6erOVPUWeg+6R5COKJJ/Tm7eqr47PBBDxARKQpKt4POOcyUEUbLg8+qNnfUc2fWxvFxVpS8+mncVtSM2++qcuxY8Pf19ln681NGr3wMEvIfLp00ZuEzz4Lbx9hMWOG5qBceGH05U8+nTun1wP/61+hVy846aT4bDABDwgREeCfwELn3K1x2ZEWlizRrO9vfjNuSypIy8Qmb76pDSP23Tf8fXXpop6+CXjVpLUWfNs2uPJKFYCf/Sw+O9LajW35cu2EePnlwc1DUB9MwINjDHAxcLSIzPEeMd6bJZuHHtK7/iSMf/sMGwbNmiV7Ks2dO7WELOzx71zOOw8WLNBuXWkizBpwnzQK+LZtGlmZNUu9yDZt4rMlrd3Y/Bva88+P145999VkQBPwBuKce8s5J865wc65Q73Hc3HZk2Sc0/D5EUdUNORPAnvvDWPGaFelpDJnjs7aFsX4t89ZZ+nyuZT9msOsAffx+1GnScB/8Qt49lm4804V8jhJay34o4/CyJHh3hzmQ6NG+hs3ATciY9YsrS1OUvjcZ/x4FcmkFgj4499RCnjnztC3L7z9dnT7DIKwS8igIoSZlkz0WbPgllvgssvgiivitiadEYxly7TNcFKih507m4AbEXLHHertnntu3Jbsid9N6bXX4rWjOt58UxNnukbcYWDMGB3zS1O2dRQC3ry5TsGZFgH/zW/U3ptuitsSxW+nmiYP3B9ii6N1alVkqRubCXjC+fxzDZ9fdJHOpJM0hg6FffZJZhjdORXwKL1vn7FjNdM6LV3ZysvDrwH36dmzYrw9yXz9Nbz4oo7bJuV/L43d2KZN07HnuJtP+ZiAG5Fx770689GVV8ZtSdU0bgxHHw0vv5w8b3PRIi1xi0vAIT1h9ChqwH1699awatJ56SVNYDvjjLgtqaBJExXxtAn46NHxld5VplMnWLdOGxelnRoFXEQaicjoqIwxdqe8XDsXjRkDhxwStzXVM348fPSRlookiTjGv30OPFBDr2kR8ChKyHx699b9Jb0U6okn1HOM4/dTE2mqBf/8c21qNDpBKtKpk4p3GnsRVKZGAXfOlQO3R2SLUYmXXlJP5aqr4rakZvxx8JdfjteOyrz9tvaO7ts3+n2LaNOd99+Pft/1IWoB37kTVq0Kf1/1xTkNn590knq9SSJN3dj8VstJE3BIz01QTeQTQn9VRM72Gq8YEXL77SpAcZeu1EbfvjqdZtLGwWfN0lmj4vrlDhqkteBJ9zQhmhpwn969dZnkMHpZmVZWjBwZtyV7kiYPfMYMLd0aNixuSyrIUjOXfAT8CmAisF1ENojIRhHZo2e5ESylpVp7etll2iwlyYjoBAWvvZaccaXNmzV0d9hh8dkweLCOoS5dGp8N+bJiRfg14D5pEHA/chJXz/OaSFM3tgULtAqkZcu4LamgoATcOdfaOdfIOdfUOdfGex1jL6LC4M47VRiTUHuaD+PHwxdfaL1nEpg3T3MI4hTwQYN0OXdufDbky7JleqGNgq5dtZwsaTkTufjfWVIFPC3d2BYuhIMPjtuK3enYUZcFIeAAInKaiNzsPWKaSK9w2LoV/vEPOP10DU2ngaOP1mVSwuj+jcSQIfHZcPDBmqU/b158NuTLsmUVnnHYNGoEBxyQfA+8R4/klI/lkpZa8J074cMPkyfgrVrpoyAEXER+D1wDLPAe14jIf4dtWCEzcaJmSCY9eS2Xjh3VW0lKIltJCbRrpxfhuGjRQrPRk+6Bb9sGq1dHJ+CQ/FKy999PbuVHWrqxLV+upYlJE3DITi14Ph74ScCxzrm7nXN3AycAJ4drVmFz++3a9MD3atPC+PHafWzz5rgtUQE/7LD4a08HD06+gJeWakg2SgHv10/r9JM4jrt1qzbgSWL4HNLjgS9cqEsT8PDIt5FLbiBpnzAMMZRZs2D6dG3cErf41JXx42H79vhrn3fsUNGMc/zb56CDtEZ+69a4Lake3xOOUsCHDNFz4l/kk8TixZqM6ecwJI20dGNbtEiXBx0Urx1VUUgC/t9AiYjcKyL3AbOAG8M1q3C5/XbN2PzWt+K2pO6MGwdNm8Y/Dr5okYaFkyDgffuqd5vkhC3ftigFfOhQXc6aFd0+82XFCl1GeT7qQlq6sS1cqOH+fRLo8hWMgDvnHgKKgceBx4BRzrlHwjasEPn8c533+6KLkvmjr42WLbVhQ9zj4ElIYPPp00eXSS4lW7ZMv7v9949unwceqIlESRRwvyb+gANiNaNG0lALvnhxcvqfV6ZTJ/jyy2RHxvKhWgEXkSH+A+gMrPYeXbz3jIC5555k9z3Ph/HjVUA//TQ+G0pKdPa2Aw+MzwaftAh4r17RDtk0aqQRkiQK+IoVenOx775xW1I9aejGtmJFdKWJdcWvBU9DKV5N1NQk8JYa1jkgZSlWyaa8XKcNHTs2uckz+TB+PPziF9rU5bzz4rGhpETPYePG8ew/l/3202z4JUvitqR6li2L52ZnyBCYMEET2ZLUrtSfVjXJOShduiTz5sdnyxYVx6RGMXKbuUTRfTAsqvXAnXNH1fAw8Q6YF19MR9/z2hg2DFq3hilT4tl/eXlFBnpS6NMnuR74zp1qWxwCPny4XujnzIl+3zWxYkVyhcenc+dkd2P76CNdRtFbvz5kpRtbvo1cBorIeSJyif8I27BC4+9/18SUs86K25KG0aSJjoP7M4FFzYoVsGFDsgS8b9/kCvjy5Vo50L9/9Ps+/niNkjz2WPT7rg7nKjzwJNOjh9q6Zk3cllSNnwiY1BuhghFwEfkV8FfvcRRwE3BayHYVFCtWpKfveT4cfrhO4hHHdH1JSmDz6dMHVq7UzPikEWetbvv2cNRR2rgoKXPJf/45bNyYXOHx8cO+fsJd0ohydrv64CdsZl7AgXOAY4CPnXPfBg7BasED5c47NaknLX3Pa8OfPzmOevCSEo0CDBwY/b6ro08fDe37XkmSWLBAl3HV6p57rg4dJSWMnnTh8fHtS6qAr1ih/e59TzdpNG2qN5CFIOBbvXnBd4pIG2AdkJIO3cln61b45z+173m3bnFbEwwjRmgk4a23ot93SYmGg5s3j37f1eFn4iZRwBcu1MlF2sQ0PdGZZ+p3dfPN8ey/MmkoIQOdI0EkuQJeWqpRgkb5tgqLgSzUgtdURna7iIwFZohIW+AutInLbOCdiOzLPPffr6Hm738/bkuCo0ULTWaLYxx89uxkjX9DhbfkJ/YkiQUL4hn/9unQAa67Dh58EF5/PT47fKKcF70hNG+umehJFvCkRzEyLeDAh8AfgVOAnwHTgWOBb3mhdKOBlJfDLbfoeO2RR8ZtTbAcfjjMnBltX/SyMs3MTZqAd+6sIbukXWzLy7VrXdy9qq+/Xi/2l10GX38dry1lZXoDmsRZyCpTVJS835RPWjL5k94MpzZqKiP7s3NuFDAO+Ay4G3gBOFNE+kZkX6Z59lm9gP74x8muOa0Phx+uJS7Tp0e3zyQmsIGGEXv0SN7FduVK2LQpfgHfe2+4914dC//BD+K1paxML+xp+H9MqoBv2qSNnJIexfA98KQkUNaHfFqpfuSc+4Nz7jDgAuAMYFHolhUAN9+sF/ZzzonbkuAZM0YvglGG0X0BT+I0kEm82L73ni79vuRxcsQR8B//ofkgH3wQnx2+gKeBoiJYtSp5teCrV+uye8IzpTp10sqQr76K25L6k08ZWRMROVVEHgCeBxYDKa9Wjp8ZM2DqVPjhDzW8mjXattXZnKIW8D594kvIqomiouSNgb/7ro6lJuWG57rrYK+94NZb47MhbQK+a1fyasHTJOCQ7nHwmpLYjhWRu9H+55cBzwK9nXPnO+cmR2VgVvn973XCku98J25LwuPww+Gdd6LzEJKYwOZTVKQXii1b4rakgnffVe87Kb0H9tsPvv1tTeyM66KaNgGH5FU3+AKe9KqaTAs48FNgGnCwc+4059yDzrlNEdmVaaZNgyeegB/9SNuOZpWxY3U8bO7c8Pf15Zd6IUuqgPvjgStXxmuHz/btesNTXBy3Jbvzve+pbc89F/2+t2zRcGpaBNyf7jRpffZ9Ae/aNV47aiPTAu6cO9o59w/n3BdRGpR1ysvh2mv1InHttXFbEy6jR+syioYufiOQpCWw+SSt8cbcudqDYOTIuC3ZnYEDtaXwa69Fv28/IzktAt6zpyYA+s14ksLq1dokpUWLuC2pmUwLuBEOf/ubhi7/+791DuYs06OHhtGmTQt/X34CW1I98KTVgvu5CUnzwEXg6KNVwKPODk6bgDdqpBUESRTwpIfPQWcJbNrUBNzIk7lzteb1pJPgkgKZDmb06Gg88JISbWzh9zhOGl26aIvXpHjgkyert9ujR9yW7MnRR6uYLl4c7X7TJuCgTXjizNqvitWrk5/ABnqzmPZmLrEKuIicICKLRWSpiFwfpy1hs3QpnHAC7Lsv3HVXOupMg2DMGC11WbUq3P0kOYENdNat7t2TIeDr16sHfsYZcVtSNUcdpcuow+hpFfA1a5JVCrVqVTo8cDABrzci0hi4HTgR6A9cICIxNnUMh82btbZ12DCtOXzxRfXGCgV/HDzMMPqWLdoQJ6nj3z5JqQV/+mnNxTjzzLgtqZpevTSSMnNmtPstK9MoSfv20e63IfhtcP1Z5eJmyxZtDW0CHg1NYtz3CGCpc245gIg8DJwORDKiU50HfO65we3jyy81fLx5syYLPfRQ8tsLBs0hh2iizbRp8I1vhLOPefO0HjbJHjiogL/4YtxW6O+wR4/kni8RGDAg+rHdsjJNoEvyBByVGTBAlx98kIx8Br8mPU0CPmNG3FbUnzgFvCuQG1hdDeyREysilwOXA/SIYMBu/vzgttWihda1nnMOjBuXrgtDUDRtqrOThTkOnvQENp+iIli7ViMxcc2WNmsWvPKK9iFI8jDOgAFw332ayBaVnWmqAfcpKtLrTJDXrYaQlhpwn06ddEhp1y4d5kobcQp4XjjnJgATAIYNGxZ6XmrSMjqzwJgxKhibNoWTeT97tmaUJr33cm4teN+YZhP4wx+0gdC//3s8+8+XAQNg40YdT40q0e7jj5OZ1FcTjRvr8FwUlR754Oe6pEnAy8tVxJM6d3lNxOkTrmH3ecW7ee9FgnNVP4zgGT1a73DDClWVlMChhybbo4T4a8FnzoSJE+Gqq5LZbjaX3NBwVJSVpfMiPm6cRlbinskN0umBQ3rHweMU8PeAviJygIg0A84HnorRHiMkRo3SZRhews6dOgae9PA5xFsL7hxcc40mh/3kJ9Hvv65ELeA7d6oXlrYQOqiA79qlbYvjZtUqbYm7995xW5IfJuD1xDm3E/g+8CKwEHjUOZewikYjCNq102zZMMbB58/XjmLDhwe/7aDp2lVDnnF44E8+qTdQv/td8r1v0HLLTp2iE/BPPtGbnDQK+OjR+ruKcuKg6khTCRmkX8BjHQN3zj0HxND12IiaMWM0fFteHmwynx+WHzEiuG2GRZMmenGLWsCdg9/8Rsfdv/WtaPfdEKLMRE9jDbhP69ZaQvn663FbogKehiYuPh076jKtAl6AedFGHIwerWV1QderTp+uIbu0lOfFUQv+zDPaK/6GG/QmIi307h3dTFtpFnDQJlFvvx3/1KJp6cLm07Kl3gCZgBtGDYTV0GXGDPW+k57A5hPHvOD//Kc2D7rwwmj321CKinRcOorkrLQL+CWXaHTr/vvjs2Hz5nQ1cfHp1Kni+08bJuBGJPTtqx2ughwH37hRx0iTNqNWTRQVqZe0fXs0+/vqK3j+eTjvvHR531ARVYnihse/gPsh1bTRp49O33vPPfFV0/gZ6GnywEFzU3zb04YJuBEJIuqFv/VWcNucNUsvVmlIYPPp2VNtDrs3vM/kyXqzEFYXvDDxs/ajCKOXlekNZrNm4e8rLP7t33QCmJdfjmf/aRXwXr1g2bK4ragfJuBGZBxzjP6jBHVBfv11TYjzw/NpIOpa8IkTtTlJmqIUPr4HHsW5SmMXtspceKGK569+FY8XnrYmLj69e2sVQhLq6OuKCbgRGccdp8ugPIQpUzT7tm3bYLYXBVHWgu/aBW+8ASeemJ4cgVz23x/22is6DzztAt6smSYqvvtuPD330yzgAMuXx2tHfTABNyKjXz/1EF56qeHb2rxZG1f4U0+mhW7dNGoQhVc5b57mCRx+ePj7CgOR6LL2syDgoHMv9OwZjxe+ejV06KC92dOECbhh5IGIeuGvvqreYUOYNg127ICjjw7Gtqho2lSTZqIQJT/fIK0CDtEIeHm5lhFlQcB9L3zGDHjhhWj3nbYmLj6+gKdxHNwE3IiU44/XevCGJrO98opmVY8dG4xdURKVV/nWWxrxSNsEHbkUFYUfQv/sM22lmgUBB23W06MH3HJLtPtNWxMXn3bt9GECbhi1cOKJOq45cWL9t+EcPPaYhs9btQrOtqiIohbcOW2tmcYbnFwOOAC++ELL4cLCb+KRFQFv1kzrwqdMibZBSdqauOTSu7cJuGHUSqtWcPLJMGlS/cPoc+bA0qVa25xGior0YrdjR3j7WLNG5x73J5JJK74ghFl2l/YmLlVx/vk6NDBpUjT727RJb7TSGEKH9JaSmYAbkXPuuVq2Ud/JFx59VCdvOPPMYO2KiqIivbiG2Tzi/fd1mYZZ2moiSgFP41Si1TFgAAwcCA89FM3+/O8nrR54nz46rLVtW9yW1A0TcCNyTj5ZS7/+/ve6/+327doucvx47YGeRnr21GWY4+C+gA8eHN4+osA88Ppz1llaqfHll+HvK+0CfuihGhGcNy9uS+qGCbgROS1bwve+p+PYdS3duO8+9Vx/+MNwbIuCKGrB58zR8eM0TB1aE126aNld2ALeurX+LrPEuHGaCxHFPOF+NCmtIfRhw3Q5a1a8dtQVE3AjFq6+WsPgv/99/n+zZYvOZz1iREVTmDTSvbuW1IXtgR9ySHjbj4omTdQzDlvAs+Z9AxQX6/mLYp5w//vp2jX8fYVBUZHOQT9zZtyW1A0TcCMWunSB738f7rorv85szsHll6vo3XhjOjuL+TRrFm4t+KZNsGRJNgQc9IbHBLzutGypnQqjEvCOHaF58/D3FQYiMHSoeeCGkTc33ggHH6w9nGuaZnTlSs2qvf9++O1vdfw77fTsGZ6Az5+vNzyHHhrO9qPGBLz+HH64NnXZujXc/axend7wuc+wYToGHva5CpKUTTBoZIm99oInntCktnHjdLKTgw/WdZ9/Dp9+qmPkixer13rjjfDTn8Zrc1AUFQU7tWou8+frctCgcLYfNd27w9NP601J0JEX57It4GPHakOXkpJwSwpXrdIpg9PM0KHa0GfOHB1+SAPmgRux0q8fvPceXHedXgTuvlsfb76ppWb9++s4+cKF8LOfpTt0nktRkR7vzp3Bb3vhQg1l+slyaad7d/WKPvss+G1v3Kh99bMq4H4Z4dy54e2jvFxrqHv1Cm8fUXDEEZozMHly3Jbkj3ngRuy0a6fJab/7XdyWREdRkZatrFlTUVYWFIsW6Y1R48bBbjcuckvJ2rcPdttZLSHz6dFDKxHCFPA1a/QGK+0eePv2cOyx8PDDei1Kg7NgHrhhxIAv2mH0+V64EA46KPjtxkWYteBZF3AR7QUQpoAvWaLLtAs4aK5NaSlMnx63JflhAm4YMeAL7MKFwW53yxa9KfBzCbJAmAKe9vrlfPAFPKzpRbMk4GecodOh3nZb3Jbkhwm4YcRAt27ajS5oz2jJEr1QZ8kD79hRp2ENQ8BXrtRlWjuI5cPgwbBhQ8WxBs2SJZpzkYWboDZt4Prr4ZFH4Nln47amdkzADSMGwgpt+h59ljzwRo20bj4MAV+1Slvy7r138NtOCn473bDC6EuW6GxejTKiJtdfr8mz3/lOeDc9QZGRU24Y6WPwYK07LS8PbpuLFunNwYEHBrfNJBBWLXha57CuCwMH6jIsAV+6NBvhc5/mzXW6461b4ZRTkl0XbgJuGDExeLCWMQXZE33hQs1w32uv4LaZBMIS8JUrsy/grVtreHvx4uC37ZeQZUnAQT3wBx/UG+xbb43bmuoxATeMmAgjtLloUbbC5z7du2u5UpDRCtCbgh49gt1mEunXLxwB96fgzJqAA5x0kk5Z/Lvf6W8viZiAG0ZMDBigS3/qz4aya5depLOUwObTvTvs2KHNfYJi40adajPrHjhUCHjQmeh+7/AhQ4LdblL44x+10c8//xm3JVVjAm4YMdGqlY5PTp0azPZWrtTxuqx64BBsGN3fVqF44F99BevWBbvdmTO1QiArbXsr07u3tqOdODFuS6rGBNwwYuS44+Ctt/Quv6H4GehZ9cAhHAEvFA8cgg+jv/eeznqX1lnI8uHcc3V+gaB7NgSBCbhhxMixx+oYYhBTPmaxhMwnDAEvhBpwnzAEvLxcQ+jDhwe3zSRy9tla2ZFEL9wE3DBiZNw4nWktnznRa2PRIujQQeuas8Z++2mHrKAFvFEjnZs+6/TooecvSAFfskQbxAwbFtw2k0iXLnqMr70WtyV7EouAi8gfRWSRiMwVkSdEpG0cdhhG3Oy9t46xPflkw2cmy1oP9FxEgi8lW7wYDjhAx3CzTqNGmim+aFFw23znHV1mXcBBp2J9771wZg9sCHF54C8DA51zg4EPgYzM8mwYdefqq7WW9r776r8N52DBgmyGz32CFvAFC7Tet1AIupTs4Yd1Uh6/UUyWKS7WPJX58+O2ZHdiEXDn3EvOOf9e5l0gA110DaN+nH46jBwJv/wlTJmi4fTf/Q7+8z/zv+CuXQtffJHdbGAIVsB37IAPPyw8AV+xArZvb/i21q7V3+nFF2enhWpNFBfr0o86JIUknPp/A56vbqWIXC4iM0Vk5vr16yM0yzCiQQT++lcNzx19tGam33AD/OY3KjBPPVX7NubN02XWBbysLJgw5rJlKuKFJuC7dumxN5R779Uktksuafi20kBREey/P7z7btyW7E6TsDYsIq8AnapYdYNzbrL3mRuAncAD1W3HOTcBmAAwbNiwkCbEM4x4GT4cli/XsfCOHWHoUM1OP+UU9XJmzqy521WhCHh5uXp/Da3dXrBAl34znUIgNxO9LkMtmzZpI5Np0/TvNm+Gm2+GE07IZge2qhDRcfCCEXDn3Pia1ovIpcApwDHOhTVTrWGkh5Yt4cILd3/v8cdVlG+4AR59tPq/nTdPs2X33TdcG+Mkt5QsKAHPatJfVdSnlGznTjjmGJg+XX9fjzyi7596Kjz0UPA2JpnDDtNo2ObNyZm9Lq4s9BOA64DTnHMBtLAwjGzSowdceSVMmqRlO9Uxb162vW+oWy34hx/CZZfBFVdU/fkFCzQs2rJloCYmmn320ehOXQT8tttUvO+9V/uBb96s7Wefeqqwzh1otMa5YDP5G0pcY+B/A1oDL4vIHBG5MyY7DCPxXHON1orffHPV63fu1BIyE3Bl+3Y45xydTeq+++CCC3TsN5fZswsrfO5z0EH5C/iGDZpYedppFWPde+2lNwKFiJ9tn6RM9Liy0Ps457o75w71Ht+Lww7DSAOdOmlo/YEHdAKOyixapOPlWRfwffbRqTFrE/A//EEjEg8/rGO3b7+9+83PsmUqYuNrHOTLJnUpJXv2WdiyBa67TseAC50+ffRG+oMP4rakgiRkoRuGUQvf+Y4mE1U1Dv7227ocPTpam+KgtlKy8nK44w44+WQdp/3mN+GMM+C//ksz2AGeeUaXp54avr1Jo18/+OwzfdTG44/rzeOoUeHblQaaNNHzZwJuGEadGDVKw59VTWv41ls6ttm7d/R2RU1tAv7OOyrUfjKgiE4JuX07XH+9vvf005pNXQjnqzL5JrJt2QLPPac3P4VQ550vAwdaCN0wjDoiol74O+/sOSvSW29pO9ZCCHP27KnNSKqrW5k0SWfGOvnkivf69IH/+A/417/0HL7xRmF635C/gL/8siasnXlm+DaliQED4KOP4Ouv47ZEMQE3jJRwySUaxrv77or3Vq+G0lIV8ELg4IPh88+hqp5OzsFjj8Hxx0ObNruv+81v1Ju8+24YMQJ+8INo7E0aRUXa+702AX/uOZ2v/sgjo7AqPfiJj0mZWtQE3DBSwv77q+f4r39pFzGA11/XZaEIuN85za/jzuWDDzS8ftppe65r3Fjrll95BaZOhc6dw7UzqTRpohGJmgTcOXj+eU3ya9YsOtvSQFjzqtcXE3DDSBHf/S6sWwf33KMX2j//GXr10iYThUBNAu5PyXrccVX/bYsW2pSkceNwbEsLtWWiL1igU62edFJ0NqWF3r3195OUWvDQOrEZhhE8J54IRxwBP/mJlo7NnAl33VU4otS1q5aSVSXgL72k4uTXixtV06+flojt3KkeeWWee06XJ54YrV1poFkzvWE2D9wwjDojAv/zP5ol/P/+n3ZqK5QJJUCPv3//Pccgt23T5LRjj43HrjTRr58OwaxYUfX6KVM016CbzRFZJUFPy9oQTMANI2X4tagvvqg14IU2Ttm//54e+Ntv602NCXjt+P3fq0rE2rVLqxqOOCJam9JEv37a1rhyd784MAE3jBTSu7eO9Rail9S/P3z8sWaj+zz9tJaPHXNMfHalhcGDtbZ71qw9173/vnb7O/zw6O1KC/36wdatmicQNybghmGkiqFDdfnmm7p0DiZPVvEutAk26kPLlloONWPGnuv8c2oCXj1+BCMJYXQTcMMwUsWYMZrI9uyz+nrBAh3Prap8zKia4cPhZd9yJwAACZJJREFUvff2bIgzdSoccIAlAtZEkkrJTMANw0gVzZpps5ZnnlEBmjRJ3z/llHjtShMjRmg/9NxEtp07NYHNxr9rpkMHaNvWBNwwDKNenHKK9jx/4QW49VZ93bVr3Falh+HDdfneexXvTZsGX3xhN0K1IaJeeBJqwU3ADcNIHSedBHvvrT3PN22Cm26K26J0MWiQNrbxZ7IDjWg0bWqZ/PmQlFIyE3DDMFJHhw4wfbqOe//qV1q3bORP06ZwwgkwcaKGzkEz+Y88cs8+8saeHHQQrF2rGftxYp3YDMNIJQMHwpNPxm1Fern4Yj1/r76qSYGLFsFVV8VtVTrwE9k+/LCiKiIOTMANwzAKkJNPhnbt4E9/0oS2Ll3g0kvjtiod+AK+aJEJuGEYhhExzZvDD38Iv/ylvv7f/9UpRI3a6dNHm+HEPQ5uAm4YhlGg/OIXcNZZMGcOXHBB3Nakh+bNdW71uDPRTcANwzAKmAED9GHUjYEDYd68eG2wLHTDMAzDqCNDhmgI/euv47PBBNwwDMMw6shhh2knwLlz47PBBNwwDMMw6siQIbqcPTs+G0zADcMwDKOOdO2qDYVKSuKzwQTcMAzDMOqIiIbRTcANwzAMI2UMHaqZ6Bs26Ovy8mj3bwJuGIZhGPXg5JO1l7w/N/0PfgBnn73nPOthYQJuGIZhGPVg1Cjo3FnnpN+wAe65R7vZiUSzfxNwwzAMw6gHjRppJ7vnn4fbbtOa8KuvjnD/0e3KMAzDMLLFhRfC1q06rW1xMQwbFt2+rZWqYRiGYdSTUaN0bvoHHoBvfCPafccq4CJyLXAz0ME592mcthiGYRhGfRg+XB9RE1sIXUS6A8cBK+OywTAMwzDSSpxj4H8CrgMiSrg3DMMwjOwQi4CLyOnAGufc+3l89nIRmSkiM9evXx+BdYZhGIaRfEIbAxeRV4BOVay6AfgZGj6vFefcBGACwLBhw8xbNwzDMAxCFHDn3Piq3heRQcABwPui1e7dgNkiMsI593FY9hiGYRhGlog8C905Nw/Y338tIqXAMMtCNwzDMIz8sUYuhmEYhpFCYm/k4pwritsGwzAMw0gb4qKaNiUARGQ98FGAm2wPFHLovpCPv5CPHez4C/n4C/nYIX3H39M516GqFakS8KARkZnOuQg71yaLQj7+Qj52sOMv5OMv5GOHbB2/jYEbhmEYRgoxATcMwzCMFFLoAj4hbgNippCPv5CPHez4C/n4C/nYIUPHX9Bj4IZhGIaRVgrdAzcMwzCMVGICbhiGYRgppGAFXEROEJHFIrJURK6P256wEZFSEZknInNEZKb33r4i8rKILPGW7eK2MyhE5G4RWSci83Peq/J4RfmL91uYKyJD4rM8GKo5/l+LyBrvNzBHRE7KWfdT7/gXi8jx8VgdDCLSXUSmiMgCEflARK7x3i+I77+G48/89y8iLURkhoi87x37f3rvHyAi071jfEREmnnvN/deL/XWF8Vpf51xzhXcA2gMLAN6Ac2A94H+cdsV8jGXAu0rvXcTcL33/HrgD3HbGeDxjgOGAPNrO17gJOB5QIBiYHrc9od0/L8GflzFZ/t7/wPN0YmGlgGN4z6GBhx7Z2CI97w18KF3jAXx/ddw/Jn//r3vsJX3vCkw3ftOHwXO996/E/h37/mVwJ3e8/OBR+I+hro8CtUDHwEsdc4td85tBx4GTo/Zpjg4HbjPe34fcEaMtgSKc24q8Hmlt6s73tOBfznlXaCtiHSOxtJwqOb4q+N04GHn3Dbn3ApgKfo/kkqcc2XOudne843AQqArBfL913D81ZGZ79/7Dr/2Xjb1Hg44GpjkvV/5u/d/E5OAY8SbJjMNFKqAdwVW5bxeTc0/8CzggJdEZJaIXO6919E5V+Y9/xjoGI9pkVHd8RbS7+H7Xpj47pwhk8wevxcSPQz1xAru+690/FAA37+INBaROcA64GU0ovClc26n95Hc4/u/Y/fWfwXsF63F9adQBbwQGeucGwKcCFwlIuNyVzqNIRVMTWGhHa/HHUBv4FCgDLglXnPCRURaAY8BP3DObchdVwjffxXHXxDfv3Nul3PuUKAbGkk4KGaTQqNQBXwN0D3ndTfvvczinFvjLdcBT6A/7E/8UKG3XBefhZFQ3fEWxO/BOfeJd3ErB+6iIkyaueMXkaaoeD3gnHvce7tgvv+qjr+Qvn8A59yXwBRgFDos4s++mXt8/3fs3vp9gM8iNrXeFKqAvwf09TITm6HJC0/FbFNoiEhLEWntPweOA+ajx/wt72PfAibHY2FkVHe8TwGXeNnIxcBXOaHWzFBpXPdM9DcAevznexm5BwB9gRlR2xcU3hjmP4GFzrlbc1YVxPdf3fEXwvcvIh1EpK33fC/gWDQHYApwjvexyt+9/5s4B3jNi86kg7iz6OJ6oJmnH6LjIzfEbU/Ix9oLzTJ9H/jAP150rOdVYAnwCrBv3LYGeMwPoWHCHeiY13eqO140c/V277cwDxgWt/0hHf//esc3F71wdc75/A3e8S8GTozb/gYe+1g0PD4XmOM9TiqU77+G48/89w8MBkq8Y5wP/NJ7vxd6U7IUmAg0995v4b1e6q3vFfcx1OVhrVQNwzAMI4UUagjdMAzDMFKNCbhhGIZhpBATcMMwDMNIISbghmEYhpFCTMANwzAMI4U0qf0jhmGkHRHxS6gAOgG7gPXe6xFO5wQwDCNFWBmZYRQYIvJr4Gvn3M1x22IYRv2xELphFCgiMlRE3vAmuHkxp83o6yLyJxGZKSILRWS4iDzuzaP9W+8zRSKySEQe8D4zSUT29tYdIyIlovPP3y0izeM8TsPIKibghlGYCPBX4Bzn3FDgbuDGnPXbnXPD0LmTJwNXAQOBS71wPEA/4O/OuYOBDcCVItICuBf4hnNuEDpM9+8RHI9hFBwm4IZRmDRHBfllb+rFn6OTPPj4cwPMAz5wOsf0NmA5FRNfrHLOve09vx9t4dkPWOGc+9B7/z5gt5nvDMMIBktiM4zCRFBhHlXN+m3esjznuf/av25UTqCxhBrDiBDzwA2jMNkGdBCRUaDTT4rIgDpuo4f/98A3gbfQyTCKRKSP9/7FwBtBGGwYxu6YgBtGYVKOTp/4BxF5H52xanQdt7EYuEpEFgLtgDucc1uBbwMTRWSet587gzPbMAwfKyMzDKPOiEgR8IxzbmDMphhGwWIeuGEYhmGkEPPADcMwDCOFmAduGIZhGCnEBNwwDMMwUogJuGEYhmGkEBNwwzAMw0ghJuCGYRiGkUL+P+v3Ci0wCejuAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 576x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "t = np.linspace(t_min, t_max, int((t_max - t_min) / resolution))\n",
    "\n",
    "with tf.compat.v1.Session() as sess:\n",
    "    saver.restore(sess, 'model/my_time_series_model')\n",
    "\n",
    "    sequence = [0.] * n_steps\n",
    "    for iteration in range(300):\n",
    "        X_batch = np.array(sequence[-n_steps:]).reshape(1, n_steps, 1)\n",
    "        y_pred = sess.run(outputs, feed_dict={X: X_batch})\n",
    "        sequence.append(y_pred[0, -1, 0])\n",
    "        \n",
    "        \n",
    "plt.figure(figsize=(8,4))\n",
    "plt.plot(np.arange(len(sequence)), sequence, 'b-')\n",
    "plt.plot(t[:n_steps], sequence[:n_steps], 'b-', linewidth=3)\n",
    "plt.xlabel('Tempo')\n",
    "plt.ylabel('Valor')\n",
    "plt.title('Sequências criativas com uma instância')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### MultiRNNCell"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-02T10:54:02.435886Z",
     "start_time": "2021-02-02T10:54:02.093363Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From <ipython-input-19-114349a1dd8b>:11: MultiRNNCell.__init__ (from tensorflow.python.ops.rnn_cell_impl) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "This class is equivalent as tf.keras.layers.StackedRNNCells, and will be replaced by that in Tensorflow 2.0.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(2, 5, 100)"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf.compat.v1.reset_default_graph()\n",
    "\n",
    "n_inputs = 2\n",
    "n_steps = 5\n",
    "n_neurons = 100\n",
    "n_layers = 3\n",
    "\n",
    "X = tf.compat.v1.placeholder(tf.float32, [None, n_steps, n_inputs])\n",
    "\n",
    "layers = [tf.compat.v1.nn.rnn_cell.BasicRNNCell(num_units=n_neurons) for layer in range(n_layers)]\n",
    "multi_layer_cell = tf.compat.v1.nn.rnn_cell.MultiRNNCell(layers)\n",
    "outputs, states = tf.compat.v1.nn.dynamic_rnn(multi_layer_cell, X, dtype=tf.float32)\n",
    "\n",
    "init = tf.compat.v1.global_variables_initializer()\n",
    "X_batch = np.random.rand(2, n_steps, n_inputs)\n",
    "                                    \n",
    "with tf.compat.v1.Session() as sess:\n",
    "    init.run()\n",
    "    outputs_val, states_val = sess.run([outputs, states], feed_dict={X: X_batch})                                \n",
    "\n",
    "outputs_val.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Distribuindo a Rnn em vários dispositivos"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-02T10:59:13.197479Z",
     "start_time": "2021-02-02T10:59:13.181413Z"
    }
   },
   "outputs": [],
   "source": [
    "class DeviceCellWrapper(tf.compat.v1.nn.rnn_cell.RNNCell):\n",
    "    def __init__(self, device, cell):\n",
    "        self._cell = cell\n",
    "        self._device = device\n",
    "\n",
    "    @property\n",
    "    def state_size(self):\n",
    "        return self._cell.state_size\n",
    "\n",
    "    @property\n",
    "    def output_size(self):\n",
    "        return self._cell.output_size\n",
    "\n",
    "    def __call__(self, inputs, state, scope=None):\n",
    "        with tf.compat.v1.device(self._device):\n",
    "            return self._cell(inputs, state, scope)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-02T10:59:22.811983Z",
     "start_time": "2021-02-02T10:59:22.489155Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[[-0.01024527  0.07738794  0.08025952 ...  0.04309902  0.06226901\n",
      "    0.17012335]\n",
      "  [-0.10363811  0.07130952  0.06840975 ... -0.07937907  0.15951197\n",
      "    0.1989244 ]\n",
      "  [-0.22273828  0.3853796   0.16764209 ... -0.10228992 -0.08637314\n",
      "    0.55426055]\n",
      "  ...\n",
      "  [-0.35417774  0.43440124  0.589108   ...  0.24293865 -0.29663736\n",
      "    0.67660517]\n",
      "  [-0.20899323  0.76148987  0.5711088  ...  0.33963224 -0.40183413\n",
      "    0.7646412 ]\n",
      "  [-0.26786032  0.78920597  0.7057781  ...  0.4577092  -0.20650735\n",
      "    0.7924192 ]]\n",
      "\n",
      " [[ 0.04816885  0.01231821  0.10429928 ...  0.01952618  0.17488609\n",
      "    0.24252082]\n",
      "  [ 0.02891437  0.06223075  0.04770555 ...  0.00692777  0.00486861\n",
      "    0.18008871]\n",
      "  [-0.2625509   0.03996788  0.02360102 ... -0.12231635  0.01540499\n",
      "    0.6129265 ]\n",
      "  ...\n",
      "  [ 0.00960766  0.5359742   0.65483624 ...  0.04890637 -0.30614293\n",
      "    0.66255796]\n",
      "  [-0.2794039   0.63816893  0.5565458  ...  0.07458118 -0.38590625\n",
      "    0.7760383 ]\n",
      "  [-0.40246484  0.6730824   0.68278944 ...  0.15472138 -0.2615908\n",
      "    0.7825849 ]]]\n"
     ]
    }
   ],
   "source": [
    "tf.compat.v1.reset_default_graph()\n",
    "\n",
    "n_inputs = 5\n",
    "n_steps = 20\n",
    "n_neurons = 100\n",
    "\n",
    "X = tf.compat.v1.placeholder(tf.float32, shape=[None, n_steps, n_inputs])\n",
    "\n",
    "devices = ['/cpu:0', '/cpu:0', '/cpu:0']\n",
    "cells = [DeviceCellWrapper(dev,tf.compat.v1.nn.rnn_cell.BasicRNNCell(num_units=n_neurons)) for dev in devices]\n",
    "\n",
    "multi_layer_cell = tf.compat.v1.nn.rnn_cell.MultiRNNCell(cells)\n",
    "outputs, states = tf.compat.v1.nn.dynamic_rnn(multi_layer_cell, X, dtype=tf.float32)\n",
    "\n",
    "init = tf.compat.v1.global_variables_initializer()\n",
    "\n",
    "with tf.compat.v1.Session() as sess:\n",
    "    init.run()\n",
    "    print(sess.run(outputs, feed_dict={X: np.random.rand(2, n_steps, n_inputs)}))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Dropout"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-02T11:03:58.964636Z",
     "start_time": "2021-02-02T11:03:28.796557Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 Training MSE: 18.4021\n",
      "100 Training MSE: 4.9550595\n",
      "200 Training MSE: 4.0437274\n",
      "300 Training MSE: 3.4854348\n",
      "400 Training MSE: 3.5706801\n",
      "500 Training MSE: 3.703458\n",
      "600 Training MSE: 3.7963727\n",
      "700 Training MSE: 3.2222974\n",
      "800 Training MSE: 3.5320468\n",
      "900 Training MSE: 3.281272\n",
      "1000 Training MSE: 3.546964\n",
      "1100 Training MSE: 4.739721\n",
      "1200 Training MSE: 3.359603\n",
      "1300 Training MSE: 4.3955126\n",
      "1400 Training MSE: 3.721893\n"
     ]
    }
   ],
   "source": [
    "tf.compat.v1.reset_default_graph()\n",
    "\n",
    "n_inputs = 1\n",
    "n_neurons = 100\n",
    "n_layers = 3\n",
    "n_steps = 20\n",
    "n_outputs = 1\n",
    "\n",
    "X = tf.compat.v1.placeholder(tf.float32, [None, n_steps, n_inputs])\n",
    "y = tf.compat.v1.placeholder(tf.float32, [None, n_steps, n_outputs])\n",
    "\n",
    "keep_prob = tf.compat.v1.placeholder_with_default(1.0, shape=())\n",
    "\n",
    "cells = [tf.compat.v1.nn.rnn_cell.BasicRNNCell(num_units=n_neurons) for layer in range(n_layers)]\n",
    "cells_drop = [tf.compat.v1.nn.rnn_cell.DropoutWrapper(cell, input_keep_prob=keep_prob) for cell in cells]\n",
    "\n",
    "multi_layer_cell = tf.compat.v1.nn.rnn_cell.MultiRNNCell(cells_drop)\n",
    "rnn_outputs, states = tf.compat.v1.nn.dynamic_rnn(multi_layer_cell, X, dtype=tf.float32)\n",
    "\n",
    "\n",
    "learning_rate = 0.01\n",
    "\n",
    "stacked_rnn_outputs = tf.reshape(rnn_outputs, [-1, n_neurons])\n",
    "stacked_outputs = tf.compat.v1.layers.dense(stacked_rnn_outputs, n_outputs)\n",
    "outputs = tf.reshape(stacked_outputs, [-1, n_steps, n_outputs])\n",
    "\n",
    "loss = tf.reduce_mean(tf.square(outputs - y))\n",
    "optimizer = tf.compat.v1.train.AdamOptimizer(learning_rate=learning_rate)\n",
    "training_op = optimizer.minimize(loss)\n",
    "\n",
    "init = tf.compat.v1.global_variables_initializer()\n",
    "saver = tf.compat.v1.train.Saver()\n",
    "\n",
    "n_iterations = 1500\n",
    "batch_size = 50\n",
    "train_keep_prob = 0.5\n",
    "\n",
    "with tf.compat.v1.Session() as sess:\n",
    "    init.run()\n",
    "    for iteration in range(n_iterations):\n",
    "        X_batch, y_batch = next_batch(batch_size, n_steps)\n",
    "        _, mse = sess.run([training_op, loss], feed_dict={X: X_batch, y: y_batch, keep_prob: train_keep_prob})\n",
    "        if iteration % 100 == 0:\n",
    "            print(iteration, 'Training MSE:', mse)\n",
    "    \n",
    "    saver.save(sess, 'model/my_dropout_time_series_model')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-02T11:07:17.694940Z",
     "start_time": "2021-02-02T11:07:17.138706Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from model/my_dropout_time_series_model\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXIAAAEXCAYAAACwHc/gAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nO3de3xU1b338c9vYjQHQkoELVYuwbQWKAnhomC94a0S5bHaRixHaLGtVLDHykOPpdVWi/XVq5cq7anIsV5KbTUHrcWmVCp4qUINEowCoumDB2ijXDQSYjAJ6/ljz8RJSELI7MzsPfN9v177NZk9a+9Zswm/WVn7t9Yy5xwiIhJekVRXQEREEqNALiIScgrkIiIhp0AuIhJyCuQiIiGnQC4iEnIK5JJ2zOybZrY11fXoTT35jGa21cy+2UtVkhRSIBcAzMwdYrsvgXPfZGav+FhdEYlzRKorIIFxXNzPU4F72u17P7nVEZHuUotcAHDO1cY24N0O9p1hZuvMrNHM/p+Z3WJmR8aON7PPmdnLZva+me0xs6fN7KNmNgu4EfhUXOt+VvSY/xs9Zp+Z7TCzJWbWP+6cs8ys3szOMbNXouVWmdnw+Lqb2XVmVhst+wCQ2+71iJl918y2mdl+M6s2s892dT16eMx9ZrbczL4VrU+dmf0oeq6bzOzt6P5vtTtuqJk9amZ7o9syMxt8OJ8xWuYKM9sY/TfaYmbzzKzT/+PdeV8JCeecNm1tNqDM+9VofX4+8B5wBVAInAW8Bvws+vog4ANgPlAAjAa+CnwU+DfgZ8DmaLlBwL9Fj7sWODt6zJnAy8CDce87C2gCVgInA8XAemBFXJlp0ff+GnAicH20rlvjysyL7vv3aJmFQAtQ0sU16Mkx90WP+RUwApgOHAD+DPwwep6rAAeMjx4TiX6m54EJ0W0NUAnYYXzGK4F/Rf/thgP/B6gFvh5XZivwze6+r7bwbCmvgLbgbR0E8meA77YrczFQDxgwLhqchnVyvpuAV7rxvlOA/UAk+nxW9LyfjCtzebRMLMg9D9zT7jwr2wW5HcD32pVZDfymi7r05Jj7gG1AVty+SmBDu3LxAfW86BdEQdzrJ0S/AM49jM/4v8DMdmWuBTb29H21hWdT14p0x3jg+uif9fVmVg/8FuiL18LegBdYXjGz/zGzOWZ2zKFOamZnm9mTZrbdzPYCy4Ajo+eM2e+cey3u+T+jZfKjz0cCL7Q7detzM8sDPgb8rV2Z54BRndTrsI+Js9E51xL3/C2g/Y3et4Bj4+r/T+fc1tiLzrl/4H3OUXFluvqMxwBDgLvb/Rv9CO8vqI50530lJHSzU7ojAnwfeKSD13Y651rM7DPAJOAzwFeAH5rZmc65DR2d0MyGAU/g3VT9HrAbr2X/EF6gjmlud2hsuk4/GiE9mfrzUMc0dVC+o33dqX936xc711V4rfdEaUrUkFGLXLrjJWCEc+6NDrZm8PphnHMvOOe+D5yE17K7LHr8B0BWu3NOwAvY86LHbcFrBR+uTXhfIPFanzvn3ovW5dR2ZU4DNnZ0wp4ck4BNwMfMrCC2w8xOwLsWG+PKdPUZ38Krb2FH/0YJvK+EhFrk0h0LgeVm9ibwMF4reTRwsnPuOjObBJwLrMDrNhiL96d+LCBsBYaZ2Ti8vty9wOt4DYlrzWwZXmC6tgd1+znwgJm9iNeHXQZMBPbElfkpsNDMXgfWATOA0/H+AuhMT47piZV4N3mXmtk3ovvuwvvyfCr6vDuf8UbgLjN7F/gTkB2t6/HOuR/28H0lLFLdSa8teBvtbnZG930GeBZowMuYqCSaEYHX31qBF8T3A28A18UdexRQDryD92f7rOj+a/BuKr4P/BUvO8MRvQGHd7Ozvl09JkfLDIzb923gbbybr7/Fu7m6Ne71CPBdvBuRHwDVwMWHuAY9OeY+YHm7fcuB+9rtW0M04yf6fCjwGN4X3F7gUWBwu2O6/IzRMtPxAnFj9Fo/B3wh7vWtRG92dvd9tYVji935FxGRkFIfuYhIyCmQi4iEnAK5iEjIKZCLiIRcStIPBw4c6AoKClLx1iIiobVu3bpdzrmDRk2nJJAXFBRQWVmZircWEQmt6FiOg6hrRUQk5BTIRURCToFcRCTkAjPXSlNTE9u3b6exsTHVVQmlnJwcBg8eTHZ2dqqrIhJINTVw663w4LJm6q/eRO4vRjLzc0cwfz4UdjbZrw/HJkNgAvn27dvp168fBQUFmFmqqxMqzjl2797N9u3bGT58+KEPEMkwFRVQVgZNTdA0eRecvpv6Z3axZMkg7r8fysuhtNT/Y5MlMF0rjY2NDBgwQEG8B8yMAQMG6K8ZkQ7U1HiBuKHBC8aU1novlNbS1OTtLyvzyvl5bDIFpkUOKIgnQNdOpGNnvlhFwxPvfrjjg+j/ldF1sGo14E3pObmyP9sKS3w7NpkC0yI/HDU1MHcu5OVBJOI9zp2b+m/FeLW1tSxevDjV1RDJeHvuGgaNcaHuSNf2EaAxwp47h/l6bDKFLpBXVEBxMSxZAnv3gnPe45Il3v6Kip6f+9Of/nSPjnvsscfYuLHtoio33XQTa9as4Zlnnulxfb73ve+xcuXKHh8vItD4Qj58u6htQG5TIAILimhck3/wSwkcm0yhCuQH9VfF8aO/6vnne7bcYftA3tDQwIUXXsjdd9/Nrl27elYZYOHChZx77rk9Pl5EIDcXqMqH74+C/e1C3v6It39DvlfOx2OTKVSB/NZbDw7g7TU1we239+z8udF/jdWrVzN58mTKysoYMWIEl19+eWxFFRYsWMCoUaMoLi7mm9/8Js8//zyPP/44//mf/0lJSQk1NTUsXbqUhQsXMmHCBJYuXUpDQwMAs2bN4pprruHTn/40J5xwAuXl5a3v/eMf/5iioiLGjBnDggULWsvHyixcuJCTTjqJ0aNHM3v2bLQgiEj3zJgB2dlAbjO04G2NkQ9/zm0mOxtmzvT32KRKxbJE48ePd+1t3LjxoH3t9evnnNeZ0vWWl3fIU3Wob9++zjnnVq1a5fLy8ty2bdtcS0uLmzRpknv22Wfdrl273IknnugOHDjgnHPunXfecc4596Uvfck98sgjrefZtWtX68/XX3+9u/POO1vLlZWVuZaWFvfqq6+6wsJC55xzf/rTn9wpp5zi9u3b55xzbvfu3QedN7bPOedmzJjhHn/88YPq351rKJJp3njDuT59nOO29Y6/rnLc/aJj/G7v8a+rHLeud336eOX8PLY3AJWug5gaqhZ5fb2/5bpy8sknM3jwYCKRCCUlJWzdupWPfOQj5OTk8JWvfIVly5bRp0+fDo995ZVXOP300ykqKmLp0qW8+uqrra9dfPHFRCIRRo0axVtvvQXAypUrueKKK1rPd/TRRx90zlWrVjFx4kSKiop46qmn2pxTRDpXWOjlekcas4gsLoSrxsO6o2HOeCL3nEBkfxbl5R0P7Enk2GQKVSDvbj+UH/1VRx11VOvPWVlZNDc3c8QRR/D3v/+dsrIyli9fzpQpUzo8dtasWSxatIjq6mpuvPHGNvnd8ed13eweaWxsZO7cuZSXl1NdXc2VV16pnHGRw1BaClumFTFn4BDy+pmX7ZZrzBkwlC3Tiroc0JPIsckSqkDe2l/Vhd7sr6qvr6euro4LLriA22+/nQ0bNgDQr18/9u7d21pu7969HHfccTQ1NbF06dJDnve8887j17/+dWtf+p49e9q8HgvaAwcOpL6+vk3fuoh0T2EhLFoEdXXQ0uI9LlrUvdZ0IsfGq2tu5pLqauqam3v2IToRqkA+f373Avm8eb3z/nv37mXq1KkUFxdz2mmncdtttwHwhS98gZ/+9KeMHTuWmpoabr75ZiZOnMipp57KiBEjDnneKVOmcNFFFzFhwgRKSkr42c9+1ub1/v37c+WVVzJ69GjOP/98TjrppF75fCJBFxtD0m9QM3ZzNf0GNQduDElXHt+1i8d27+aPCWSzdcS6++e9nyZMmODaLyyxadMmRo4cechj28x7EJfBkp3tbUGY9yBVunsNRcKo7ZwntfCdzXDLCLKfHhSa//tnVVWx+t13Oat/f54qOfyRoGa2zjk3of3+QA3R747SUnj5ZS/F8MEHvRububled8q8eam/6SAi/osfQwK0nfNk5SCamrzXX345WDHg3Koq/vruh0P8j4xOpfG3ujps9erW/ef078/KHgT2GF8CuZn1B5YAowEHfNk594If5+5IrL9q0aLeegcRCZKwzHnS3vXDhvHCe+/RcOAAAB9Ee0A+iOsJ6ROJcMOwxIb4+9VH/nPgz865EcAYYJNP5xURCc2cJ+2dlZ/P8qIi+kQ6DrV9IhGeKCpicn5iQ/wTDuRm9hHgDOC/AZxzHzjn3u36KBGR7gvLnCcdOSs/n9+PGkVOu2CeE4nw+1GjEg7i4E+LfDiwE/i1ma03syVm1rd9ITObbWaVZla5c+dOH95WRDJFWOY86cy7zc0cgRdw/y0SIYLXr/2uT2mIfgTyI4BxwH8558YC+4AF7Qs55xY75yY45yYcc8wxPrxt7+VkikiwhGbOk078d20tDQcOMCY3lz+MHs2Y3FwaDhzg3tpaX87vRyDfDmx3zq2NPi/HC+y9rrdyMtsrKChIaBZDEUlM6xiSC2oh5wDU5MINo73HnANQWturY0gS9ZGsLH5aWEjl+PGcd/TRvDh+PD854QTysrJ8OX/CWSvOuVoz22Zmn3TOvQacA2w81HF+iH2b3Vtby4xBg5LxliKSArE5T6ZWZcHiQg48PBicwZx8ItO2QXFdIOY86cxjRUVtnmeZMX/oUOb7dH6/8sj/A1hqZkcC/wCu8Om8bSQjJ/Piiy9m27ZtNDY28o1vfIPZs2e3vrZgwQKGDBnC1VdfDXiLR+Tm5jJ//nyuu+46KioqMDNuuOEGLrvssh69v4h0rLQUtpxYxO074MF+sTEkxswBQ5k3LbhBPBl8CeTOuSrgoNFGfktGTua9997L0Ucfzfvvv89JJ53E5z//+dbXLrvsMq699trWQP7www+zYsUKli1bRlVVFRs2bGDXrl2cdNJJnHHGGRx33HE9roeIHExjSDoWqrlWkpGTeeeddzJmzBgmTZrEtm3beP3111tfGzt2LG+//Tb//Oc/2bBhA/n5+QwZMoTnnnuO6dOnk5WVxUc/+lHOPPNMXnzxxR7XQUTkcIRuiH4sJ/PSjRtpjLbMwZ+czNWrV7Ny5UpeeOEF+vTpw+TJkw+aLvbSSy+lvLyc2tpadZ+ISCCEqkUe01s5mXV1deTn59OnTx82b97MmjVrDipz2WWX8bvf/Y7y8nIuvfRSAE4//XR+//vf09LSws6dO3nmmWc4+eSTE6qLiEh3hTKQ91ZO5pQpU2hubmbkyJEsWLCASZMmHVTmU5/6FHv37uX4449v7QO/5JJLKC4uZsyYMZx99tn85Cc/YZCyaEQkSUI3jS3AxdXVnNG/P9cOHkzEjBbnuGPbNp6tqzsozSeTaBpbkfSWNtPYQu/nZIqIhEkou1ZERORDCuQiIiGnQC4iSRX2dTeDSIFcRJKmogKKi2HJEqgv3gWn76a+aBdLlnj7KypSXcNwUiAXkaSIX3ezqYm26242efvLytQy7wkF8l6yevVqpk6dCsDjjz/Oj370oy7L33HHHUyaNIlLL72U6urqZFRRJKm8dTdXe2tsrloNn6rzXoitu7lqNQ1PrGZyZVXqKhlSoUw/BKClxfs7bP16GDvWmxrNp7l9u37bFrIO830uuugiLrrooi7LXHvttVx77bWJVE0k0PbcNQy++543fzh0ve6mZr84LOFskbe0wPnnw/TpcOON3uP553v7E7B161ZGjBjB5ZdfzsiRIykrK6OhoYGCggK+9a1vMW7cOB555BH+8pe/cMoppzBu3DguvfRS6uvrAfjzn//MiBEjGDduHMuWLWs973333cfXv/51AN566y0uueQSxowZQ0lJCZWVldTX13POOecwbtw4ioqK+MMf/tB67G233cbo0aMZPXo0d9xxR0KfTySVwrzuZtCFM5BXVMDatd6ExM55j2vX+nKn5LXXXmPu3Lls2rSJvLw8fvnLXwIwYMAAXnrpJc4991x+8IMfsHLlSl566SUmTJjAbbfdRmNjI1deeSV//OMfWbduHbWdTBdwzTXXcPbZZ7NhwwYqKys58cQTycnJ4dFHH+Wll15i1apVzJ8/H+cc69at49e//jVr165lzZo13HPPPaxfvz7hzyiSCmFfdzPIwhnI16+Hffva7tu3D6oS71sbMmQIp556KgAzZszgueeeA2id6XDNmjVs3LiRU089lZKSEu6//37efPNNNm/ezPDhw/nEJz6BmTFjxowOz//UU0/xta99DYAjjjiCvLw8nHN85zvfobi4mHPPPZcdO3bw1ltv8dxzz3HJJZfQt29fcnNz+dznPsezzz6b8GcUSYWwr7sZZOHsIx87Fvr29VriMX37Qg9XBYpn0VWH2j/v27cvAM45zjvvPB566KE25aoS+BJZunQpO3fuZN26dWRnZ1NQUHDQ9LkiYTd/Ptx/PzTF1t18IxcWnwCz/wEfr/fW3Xx+UGDX3QyycLbIS0th4kTvbzUz73HiRG9/gv73f/+XF154AYDf/va3nHbaaW1enzRpEn/729944403ANi3bx9btmxhxIgRbN26lZpo7lT7QB9zzjnncPfddwPQ3NzMe++9R11dHcceeyzZ2dmsWrWKN998E/Cmx33sscdoaGhg3759PProo5x++ukJf0aRVIituxlpzCKyuBCuGg/rjoY544nccwKR/VmBXnczyMIZyLOyYMUKeOghWLjQe1yxwpeslU9+8pP84he/YOTIkbzzzjvMmTOnzevHHHMM9913H9OnT6e4uJhTTjmFzZs3k5OTw+LFi7nwwgsZN24cxx57bIfn//nPf86TTz7J8ccfz7hx43j99de5/PLLqayspKioiAceeIARI0YAMG7cOGbNmsXJJ5/MxIkT+epXv8rYsWMT/owiqVJaClumFTFn4BDy+hmRCOTlGnMGDGXLtCI/2mIZKZTT2PaWrVu3MnXqVF555ZVef6/nn3+e1157jSuu8G+d6iBcQxHpPZ1NYxvOFnnIPfTQQ3zxi188qD9eRKQnwnmzs5cUFBQkpTU+ffp0pk+f3uvvIyKZIVAt8lR086QLXTuRzBWYQJ6Tk8Pu3bsVkHrAOcfu3bvJyclJdVVEJAUC07UyePBgtm/fzs6dO1NdlVDKyclh8ODBqa6GiKRAYAJ5dnY2w4cPT3U1RERCJzBdKyIi0jMK5CIiIadALiIScgrkIiIh51sgN7MsM1tvZsv9OqeIiByany3ybwCbfDyfiIh0gy+B3MwGAxcCS/w4n4gEV00NzJ0L/QY1YzdX029QM3PnevslNfxqkd8BXAcc6KyAmc02s0ozq9SgH5FwqqiA4mJYsgTqi3fB6bupL9rFkiXefh9WW5QeSDiQm9lU4G3n3LquyjnnFjvnJjjnJhxzzDGJvq2IJFlNDZSVQUMDNDUBpdF1aUtraWry9peVqWWeCn6M7DwVuMjMLgBygDwz+41zruNFK0UklM58sYqGJ979cMcH0WmYR9fBqtUANACTK/uzrTDxZRel+xJukTvnvu2cG+ycKwC+ADylIC6SfvbcNcxbLDnmSNf2EaAxwp47hyW3YqI8chHpnsYX8uHbRW2DeZsCEVhQROOa/ORWTPwN5M651c65qX6eU0SCITcXqMqH74+C/e1Cx/6It39DvldOkkotcvGVUtPS14wZkJ0N5DZDC97WGPnw59xmsrNh5syUVjMjKZCLb5Salt7mz48G8gtqIecA1OTCDaO9x5wDUFpLdjbMm5fqmmYeBXLxhR+paWrNB1thIZSXQ6Qxi8jiQrhqPKw7GuaMJ3LPCUT2Z1Fe7pXrVS0tsHw53Hyz99jS0stvGHyBWVhCwi3R1LSKCi/QNzVB0+Roa/6ZXSxZMoj77/cCSGlp738O6VppKWw5sYjbd8CD/aC+HnJzjZkDhjJvWpKC+Pnnw9q1sG8f9O0LEyfCihWQldXLbx5capGLLxJJTdNAk3ApLIRFi6CuzourdXXe814P4uB9469d632DOOc9rl2b8f12apGLL1pT035Y7fWXHlQgmppWfXBqmgaaSLetX++1xOPt2wdVVTA1cxPm1CIXXySSmqaBJtJtY8d63Snx+vaFksz+glcgF18kkpqmgSbSbaWlXp94bi6YeY8TJ2b8DRQFcmmjp5kjiaSmaaCJdFtWlndj86GHYOFC7zHDb3SCArnESSQPPJHUNA00kcOSleX1h99wg/eY4UEcFMglyo/MkdJS2DKtiDkDh5DXz4hEIC/XmDNgKFumFXX6168GmkjSpGkOujnnDl3KZxMmTHCVlZVJf1/p3ODfVbFjULvMkSPdh4+xcm/1Z9tl/t9YqqiAqVXVsKE/Bx4eDM4g4ohM2wbFdSwv6fyLALwvmFtvhQeXNVN/9SZyfzGSmZ87gvnzk5QWJ8GXBjnoZrbOOTeh/X61yAVIfeZIT1vzoKkBpJvSOAddgVyAYGSO9GSgiQYTSbd1lYMechoQJICXObI3ljly00Y4Km5QT3zmSF7q6tgRDSaSbovloNfXf7gvTXLQ1SIXILyZI6nuEgqrjJygLI1z0BXIBQhv5kgQuoTCJhD3FFKRPZJoDnqAM16UtSKtEs0cSYW8PNi7F5i0q+MuoZtGwZqB5OV5fe6ZrqbGC9YNDdEdt1XB2Hfhpf4w3+ti6NMHXn65F7N9wpg9EpA6d5a1oj5yaZXyKUp7YMYMr2XZFN8l1BSB7AOB7hJKlUDcU4jPHoG22SNBnfgq4HVW14q0kdIpSnsgrF1CqRKIewphzB4JeJ0VyCXUArNqTUgE4p5CGGcwDHidFcgl9BIZTJRpAjFBWRizR/yocy/eLFUfuaSFWJfQokWprkmwBeKeQix7pKLC65ooKfECYlBvdELide7lm6XKWhHJIK1ZKz+ogjHvwhu5sPgEmP0P+Hg9VPWnz3dLejdrJRMtXw7Tp7cdjJSb66VAHsbNUs21ItKJTBoco3sKKdLLN0sVyCWjBWJwTJLpnkIK9PLNUnWtSMYKxOAYyQw+9ZFrQFAG0dzc3ROIwTGSGXr5Bq+6VtJMJnYV9FQgBsdI5ujFJeoUyNOI5uY+PIEYHCPiA3WtpBHfugpaWrym+/r13k2aoOf49lBY52AXaS/hFrmZDTGzVWa20cxeNbNv+FExOXy+dBXEbspMnw433ug9nn9+oKbs9EtY52CPyaS0SemaH10rzcB859woYBJwtZmN8uG8cph86SpI43UN2wvzhFu6FyLxEg7kzrl/Oedeiv68F9gEHJ/oeeXw+TKPhh8DFwI8AX+8sA6O0b0Qac/XPnIzKwDGAms7eG02MBtg6NChfr6tRPkyj0ai6xoGZAL+7vJjDvZkp3sG4l5IhtxHCQvfBgSZWS7wNHCLc25ZV2U1IKh3+DKPRqKB2Kc5JcKiosJr/TY1QdPkWvjOZrhlBNlPDyI722vx+z1Sss+p7/D+d6u97p/ONEboc3MR+/7WSTdaIv/OIfuyTie9OteKmWUD/wMsPVQQl97jS1dBousaBnwCfj+lqosj5fdCMug+Slj4kbViwH8Dm5xztyVeJUmEL/NoJDJwIeAT8PvJ6+JY7XVnrFoNn4ouChrr4li1moYnVjO5svMvsZ5knqT8XkgGfVmHhR8t8lOBmcDZZlYV3S7w4bzSQyldri2Miwb0UKLpnj3NPPElbTKRL9wM+rIOCz+yVp5zzplzrtg5VxLd/uRH5SSEEu2aCZFEujgS6ZbxJW0ykS/cDPqyDgvNfijSQ3l5sHcvMGlXxyNDbxoFawaSl+f9VRRv8O+q2DGoXebJke7Dx1i5t/qz7bKDW7oVFTC1qho29OfAw4PBGUQckWnboLiO5SXd6EaLZZ70dMWbMK3wkyY0+6GIzxJJ99xz1zD47nsfZp501S1z2cHH+5E22XovpCfZRIkcK75Ti1ykhxJJ94xEwI15B37YSRphtFsmUp0f1PFUkgJa6k3CISSjQiGxdM9ArGYvaUNdKxIciQ40ScFow552cQRiNXtJG+pakeBIZFRoyEYbajV76Ql1rUjwJTLQJGSjDcM6YZcEkwJ5ugpRX3OrRAaahHC0oVazF7+ojzwdhayboVVsoEn7encnoiU6a2OKxEbhLlqU6ppImCmQp6P4bgZo280Q5LzfRFYaT+RLQCTkFMjTUVfdDEEO5NDzgSaJfAmIhJwCeToKaTdDwjTaUDKUbnamI01qJJJRFMgDKqEV0jNoBkIR0YCgQErF8mEiEnwaEBQSWiFdRA6XbnYGjG8rpItIxlCLPGDilw+LtLRw4brnueGBB7hw3fNEYqMzu1g+TEQyj1rkARNbPizygypWfO86Jm7aRJ/GRhpyclg7ciTnL/wZB64vorG6ixXSRSSjqEUeMLF5qkvn/4uJGzfR7/33yXKOfu+/z8SNmyid/y/NUy0ibSiQB0xshfSxTVX02d/Y5rU++xspadqgeapFpA0F8oCJrZC+/qzBNOTktHmt4aijqDrr+EOvkC4iGUWBPGBi81SvGH4Gf+83jr3k0oKxl1z+/pHxrBh+huapFpE2dLMzgEpLYfOJJdyx42nuvq+CTzZW8VpOCR/9XCmbv5ClIC4ibWhkp4hISGhkp4hImlIgFxEJOfWRi4RZS4s3y9r69d489FpMIyMpkIuEVVjXZhXfqWtFJKzi12Z1ru3arJJRfAnkZjbFzF4zszfMbIEf5xRJqpYWWL4cbr7Ze4xNUBZkXa3NKhkl4a4VM8sCfgGcB2wHXjSzx51zGxM9t0hShLWLIlPXZpWD+NEiPxl4wzn3D+fcB8DvgM/6cF6R5AhrF4XWZpUoP252Hg9si3u+HZjYvpCZzQZmAwwdOtSHtxXxSVddFFOnpqZO3RFbm7WiwqtrSYmyVjJU0rJWnHOLgcXgjexM1vuKHFKYuyiysrwvmyB/4Uiv86NrZQcwJO754Og+gXDeRMs06qKQkPOjRf4i8AkzG44XwL8A/LsP5w2/sN5EyzTqopCQSziQO+eazezrwAogC7pvusQAAAtTSURBVLjXOfdqwjVLB/E30aDtTTT9KRws6qKQEPMlj9w59yfn3InOuULn3C1+nDMtKM9XRJJAIzt7U+wmWryw3EQTkdBQIO9NuokmIkmgSbN6k26iiUgSqEXei2pqYO5/ZNHvq1Mw91n6fXUKc/8ji5qaVNdMRNKJAnkvqaiA4mJYsgTqi3fB6bupL9rFkiXe/qCP/pbDpPECkkLqWukFNTVQVgYNDdEdpbWtj00rB9HU5L3+8stoIeV0oPECkmIK5L3gzBeraHji3Q93fGDe4+g6WLUagAZgcmV/thUqgyX0NF5AUkxdK71gz13DoDHu0h7p2j4CNEbYc+ew5FZMekei4wXULSMJUou8FzS+kA/fLoIfVkPOgQ4KRGBBEY3V+cmvnPgvkUm31C0jPlCLvBfk5gJV+fD9UbC/3SXeH/H2b8j3ykn4JTJeIKxzoUugKJD3ghkzIDsbyG2GFrytMfLhz7nNZGfDzJkprab4JTZe4KGHYOFC77G7LWpN4yA+UCDvBfPnRwP5BbVe10pNLtww2nvMOQCltWRnw7x5qa6p+CY26dYNN3iP3e0W0TQO4gMF8l5QWAjl5RBpzCKyuBCuGg/rjoY544nccwKR/VmUlyv1UNA0DuIL3ezsJaWlsOXEIm7fAQ/287o+c3ONmQOGMm+agrhEaRoH8YE5l/xV1yZMmOAqKyuT/r491tLi/Udbv977U1j/0UQkBcxsnXNuQvv9apEfitLDRCTg1Ed+KEoPE5GAUyA/FKWHiUjAKZAfitLDRCTgFMgPRelhIhJwutl5KEoPE5GAUyDvjtioPU1JKiIBpK4VEZGQUyAXEQm5zOha0chMEUlj6R/INTJTRNJc+netaGSmiKS59A/kGpkpImku/QO5RmaKSJpL/0CukZkikuYSCuRm9lMz22xmL5vZo2bW36+K+SaR9RSBmhqYOxf6DWrGbq6m36Bm5s719ouIBEFCC0uY2WeAp5xzzWb2YwDn3LcOdVxYFpaoqICyMmhqgqbJtfCdzXDLCLKfHkR2trecmxr2IpIsnS0skVCL3Dn3F+dcc/TpGmBwIucLkpoaL4g3NHiBnNJa74XSWpqavP1lZWqZi0jq+ZlH/mXg9z6eL6XOfLGKhife/XDHB+Y9jq6DVasBaAAmV/ZnW6FunIpI6hyyRW5mK83slQ62z8aVuR5oBpZ2cZ7ZZlZpZpU7d+70p/a9aM9dw6Ax7vIc6do+AjRG2HPnsORWTESknYQXXzazWcDXgHOccw3dOSYMfeSRCLgx78APqyHnwMEFGiOwoIhIdT4tLcmvn4hknl7pIzezKcB1wEXdDeJhkZsLVOXD90fB/naXaX/E278h3ysnIpJCieaRLwL6AU+aWZWZ/cqHOgXCjBmQnQ3kNkML3tYY+fDn3Gays2HmzJRWU0Qk4ayVjzvnhjjnSqLbVX5VLNXmz48G8gtqva6Vmly4YbT3mHMASmvJzoZ581JdUxHJdOk/srOHCgu9PPFIYxaRxYVw1XhYdzTMGU/knhOI7M+ivNwrJyKSSuk/jW0CSkthy4lF3L4DHuznTZyYm2vMHDCUedMUxEUkGBLOWumJMGStiIgETa9krYiISOopkIuIhJwCuYhIyCmQi4iEnAK5iEjIKZCLiIRc2gdyrfAjIukurQN5RQUUF8OSJVBfvAtO30190S6WLPH2V1SkuoYiIolL20CuFX5EJFOk7RB9rfAjIpkibVvkWuFHRDJF2gbyxhfy4dtFbYN5mwLeCj+Na/KTWzEREZ+lbSDXCj8ikinSNpBrhR8RyRShCOSxXPC8PG9R5Lw8DpkLrhV+RCRTBH4+8ooKL02wqSmaRhiVne1t5eXeAhCdHTu1qho29OfAw4PBGUQckWnboLiO5SVFnR4rIhI0oZyP/KBc8DjdyQUvLYUt04qYM3AIef3Ma83nGnMGDGXLNAVxEUkPgQ7kt956cABvr6kJbr+989cLC2HRIqirg5YW73HRIi3TJiLpI9CB/De/6V4gf/DB5NRHRCSIAh3I6+v9LSciko4CHci7m+OtXHARyWSBDuStueBdUC64iGS6QAfy1lzwLigXXEQyXaADeWGhlyfep8/BAT0729tfXq4MFBHJbIEO5ODlgr/8Msye3XZk5+zZ3n7lgotIpgv8yE4REfGEcmSniIgcmgK5iEjIKZCLiIRcSvrIzWwn8GbS3zg1BgK7Ul2JANP1OTRdo65l0vUZ5pw7pv3OlATyTGJmlR3dnBCPrs+h6Rp1TddHXSsiIqGnQC4iEnIK5L1vcaorEHC6Poema9S1jL8+6iMXEQk5tchFREJOgVxEJOQUyHvIzO41s7fN7JW4fT81s81m9rKZPWpm/Ts5dquZVZtZlZml5aQznVyfm6PXpsrM/mJmH+vk2C+Z2evR7UvJq3VyJXiNWqJlqszs8eTVOnk6uj5xr803M2dmAzs5NiN+h2LUR95DZnYGUA884JwbHd33GeAp51yzmf0YwDn3rQ6O3QpMcM6l7SCGTq5PnnPuvejP1wCjnHNXtTvuaKASmAA4YB0w3jn3TjLrnww9vUbR1+qdc2m9NlZH1ye6fwiwBBiB97uxq91xGfM7FKMWeQ85554B9rTb9xfnXHP06RpgcNIrFhCdXJ/34p72xftP1t75wJPOuT3R/3hPAlN6raIplMA1yggdXZ+o24Hr6PzaZMzvUMwRqa5AGvsy8PtOXnPAX8zMAXc75zImfcrMbgG+CNQBZ3VQ5HhgW9zz7dF9GaMb1wggJ9ot1wz8yDn3WLLql0pm9llgh3Nug5l1VizjfofUIu8FZnY93n+wpZ0UOc05Nw4oBa6O/gmZEZxz1zvnhuBdm6+nuj5B1M1rNCw6LP3fgTvMLO3XyTKzPsB3gO+lui5Bo0DuMzObBUwFLned3IBwzu2IPr4NPAqcnLQKBsdS4PMd7N8BDIl7Pji6LxN1do3if4f+AawGxiavWilTCAwHNkTvMw0GXjKzQe3KZdzvkAK5j8xsCl7f3UXOuYZOyvQ1s36xn4HPAAfdlU9HZvaJuKefBTZ3UGwF8BkzyzezfLzrsyIZ9QuC7lyj6LU5KvrzQOBUYGNyapg6zrlq59yxzrkC51wBXpfJOOdcbbuimfc75JzT1oMNeAj4F9CE9wv1FeANvL65quj2q2jZjwF/iv58ArAhur0KXJ/qz5LE6/M/eF9aLwN/BI6Plp0ALIk79svRa/kGcEWqP0vQrhHwaaA6+jtUDXwl1Z8lWden3etbgYGZ/DsU25R+KCIScupaEREJOQVyEZGQUyAXEQk5BXIRkZBTIBcRCTkN0Ze0YmYDgL9Gnw4CWoCd0ecnO+c+SEnFRHqR0g8lbZnZTUC9c+5nqa6LSG9S14qkPTMbb2ZPm9k6M1thZsdF9682s9vNrNLMNpnZSWa2LDqH9Q+iZQqic8wvjZYpj875gZmdY2bro3PL3xsbbSmSbArkku4MuAsoc86NB+4Fbol7/QPnTT71K+APwNXAaGBWtJsG4JPAL51zI4H3gLlmlgPcB1zmnCvC66ack4TPI3IQBXJJd0fhBeYnzawKuIG288THVtepBl51zv3LObcf+AcfTry0zTn3t+jPvwFOwwvu/885tyW6/34gY2axlGDRzU5Jd4YXoE/p5PX90ccDcT/Hnsf+f7S/kaQbSxIoapFLutsPHGNmpwCYWbaZfeowzzE0djze/N/PAa8BBWb28ej+mcDTflRY5HApkEu6OwCUAT82sw14s1J++jDP8RreAiCbgHzgv5xzjcAVwCNmVh19n1/5V22R7lP6oUgXzKwAWO7iFv8VCRq1yEVEQk4tchGRkFOLXEQk5BTIRURCToFcRCTkFMhFREJOgVxEJOT+P0z+7EL4XqRWAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "with tf.compat.v1.Session() as sess:\n",
    "    saver.restore(sess, 'model/my_dropout_time_series_model')\n",
    "\n",
    "    X_new = time_series(np.array(t_instance[:-1].reshape(-1, n_steps, n_inputs)))\n",
    "    y_pred = sess.run(outputs, feed_dict={X: X_new})\n",
    "    \n",
    "    \n",
    "plt.title('Testando o modelo', fontsize=14)\n",
    "plt.plot(t_instance[:-1], time_series(t_instance[:-1]), 'bo', markersize=10, label='instância')\n",
    "plt.plot(t_instance[1:], time_series(t_instance[1:]), 'c*', markersize=10, label='alvo')\n",
    "plt.plot(t_instance[1:], y_pred[0,:,0], 'r.', markersize=10, label='predição')\n",
    "plt.legend(loc='upper left')\n",
    "plt.xlabel('Tempo')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### LSTM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-03T11:19:42.035487Z",
     "start_time": "2021-02-03T11:10:06.863785Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 Last batch accuracy: 0.97333336 Test accuracy: 0.9542\n",
      "1 Last batch accuracy: 0.97333336 Test accuracy: 0.9688\n",
      "2 Last batch accuracy: 0.97333336 Test accuracy: 0.9781\n",
      "3 Last batch accuracy: 0.99333334 Test accuracy: 0.9818\n",
      "4 Last batch accuracy: 0.9866667 Test accuracy: 0.9848\n",
      "5 Last batch accuracy: 0.99333334 Test accuracy: 0.9857\n",
      "6 Last batch accuracy: 1.0 Test accuracy: 0.9878\n",
      "7 Last batch accuracy: 1.0 Test accuracy: 0.9862\n",
      "8 Last batch accuracy: 0.99333334 Test accuracy: 0.9873\n",
      "9 Last batch accuracy: 1.0 Test accuracy: 0.9875\n"
     ]
    }
   ],
   "source": [
    "tf.compat.v1.reset_default_graph()\n",
    "\n",
    "n_steps = 28\n",
    "n_inputs = 28\n",
    "n_neurons = 150\n",
    "n_outputs = 10\n",
    "n_layers = 3\n",
    "\n",
    "learning_rate = 0.001\n",
    "\n",
    "X = tf.compat.v1.placeholder(tf.float32, [None, n_steps, n_inputs])\n",
    "y = tf.compat.v1.placeholder(tf.int32, [None])\n",
    "\n",
    "lstm_cells = [tf.compat.v1.nn.rnn_cell.BasicLSTMCell(num_units=n_neurons) for layer in range(n_layers)]\n",
    "multi_cell = tf.compat.v1.nn.rnn_cell.MultiRNNCell(lstm_cells)\n",
    "\n",
    "outputs, states = tf.compat.v1.nn.dynamic_rnn(multi_cell, X, dtype=tf.float32)\n",
    "top_layer_h_state = states[-1][1]\n",
    "\n",
    "logits = tf.compat.v1.layers.dense(top_layer_h_state, n_outputs, name='softmax')\n",
    "xentropy = tf.compat.v1.nn.sparse_softmax_cross_entropy_with_logits(labels=y, logits=logits)\n",
    "loss = tf.reduce_mean(xentropy, name='loss')\n",
    "\n",
    "optimizer = tf.compat.v1.train.AdamOptimizer(learning_rate=learning_rate)\n",
    "training_op = optimizer.minimize(loss)\n",
    "\n",
    "correct = tf.compat.v1.nn.in_top_k(logits, y, 1)\n",
    "accuracy = tf.reduce_mean(tf.cast(correct, tf.float32))\n",
    "    \n",
    "init = tf.compat.v1.global_variables_initializer()\n",
    "\n",
    "n_epochs = 10\n",
    "batch_size = 150\n",
    "\n",
    "with tf.compat.v1.Session() as sess:\n",
    "    init.run()\n",
    "    for epoch in range(n_epochs):\n",
    "        for X_batch, y_batch in shuffle_batch(X_train, y_train, batch_size):\n",
    "            X_batch = X_batch.reshape((-1, n_steps, n_inputs))\n",
    "            sess.run(training_op, feed_dict={X: X_batch, y: y_batch})\n",
    "        acc_batch = accuracy.eval(feed_dict={X: X_batch, y: y_batch})\n",
    "        acc_test = accuracy.eval(feed_dict={X: X_test, y: y_test})\n",
    "        print(epoch, 'Last batch accuracy:', acc_batch, 'Test accuracy:', acc_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Embeddings"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Busque os dados"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-04T10:04:33.649259Z",
     "start_time": "2021-02-04T10:04:33.638865Z"
    }
   },
   "outputs": [],
   "source": [
    "from six.moves import urllib\n",
    "import errno\n",
    "import os\n",
    "import zipfile\n",
    "\n",
    "WORDS_PATH = 'datasets/words'\n",
    "WORDS_URL = 'http://mattmahoney.net/dc/text8.zip'\n",
    "\n",
    "def mkdir_p(path):\n",
    "    try:\n",
    "        os.makedirs(path)\n",
    "    except OSError as exc:\n",
    "        if exc.errno == errno.EEXIST and os.path.isdir(path):\n",
    "            pass\n",
    "        else:\n",
    "            raise\n",
    "\n",
    "\n",
    "def fetch_words_data(words_url=WORDS_URL, words_path=WORDS_PATH):\n",
    "    os.makedirs(words_path, exist_ok=True)\n",
    "    zip_path = os.path.join(words_path, 'words.zip')\n",
    "    if not os.path.exists(zip_path):\n",
    "        urllib.request.urlretrieve(words_url, zip_path)\n",
    "    with zipfile.ZipFile(zip_path) as f:\n",
    "        data = f.read(f.namelist()[0])\n",
    "    return data.decode('ascii').split()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-04T10:11:37.422273Z",
     "start_time": "2021-02-04T10:04:50.338399Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['anarchism', 'originated', 'as', 'a', 'term']"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "words = fetch_words_data()\n",
    "words[:5]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Construa o dicionário"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-04T10:14:40.920399Z",
     "start_time": "2021-02-04T10:14:28.537334Z"
    }
   },
   "outputs": [],
   "source": [
    "from collections import Counter\n",
    "\n",
    "vocabulary_size = 50000\n",
    "\n",
    "vocabulary = [('UNK', None)] + Counter(words).most_common(vocabulary_size - 1)\n",
    "vocabulary = np.array([word for word, _ in vocabulary])\n",
    "dictionary = {word: code for code, word in enumerate(vocabulary)}\n",
    "data = np.array([dictionary.get(word, 0) for word in words])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-04T10:14:47.253926Z",
     "start_time": "2021-02-04T10:14:47.240982Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('anarchism originated as a term of abuse first used',\n",
       " array([5234, 3081,   12,    6,  195,    2, 3134,   46,   59]))"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "' '.join(words[:9]), data[:9]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-04T10:14:52.315352Z",
     "start_time": "2021-02-04T10:14:52.302034Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'cycles originated as a term of abuse first used'"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\" \".join([vocabulary[word_index] for word_index in [5241, 3081, 12, 6, 195, 2, 3134, 46, 59]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-04T10:14:57.381472Z",
     "start_time": "2021-02-04T10:14:57.373486Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('culottes', 0)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "words[24], data[24]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Gere lotes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-04T10:15:02.626380Z",
     "start_time": "2021-02-04T10:15:02.570723Z"
    }
   },
   "outputs": [],
   "source": [
    "from collections import deque\n",
    "\n",
    "def generate_batch(batch_size, num_skips, skip_window):\n",
    "    global data_index\n",
    "    assert batch_size % num_skips == 0\n",
    "    assert num_skips <= 2 * skip_window\n",
    "    batch = np.ndarray(shape=[batch_size], dtype=np.int32)\n",
    "    labels = np.ndarray(shape=[batch_size, 1], dtype=np.int32)\n",
    "    span = 2 * skip_window + 1\n",
    "    buffer = deque(maxlen=span)\n",
    "    for _ in range(span):\n",
    "        buffer.append(data[data_index])\n",
    "        data_index = (data_index + 1) % len(data)\n",
    "    for i in range(batch_size // num_skips):\n",
    "        target = skip_window\n",
    "        targets_to_avoid = [ skip_window ]\n",
    "        for j in range(num_skips):\n",
    "            while target in targets_to_avoid:\n",
    "                target = np.random.randint(0, span)\n",
    "            targets_to_avoid.append(target)\n",
    "            batch[i * num_skips + j] = buffer[skip_window]\n",
    "            labels[i * num_skips + j, 0] = buffer[target]\n",
    "        buffer.append(data[data_index])\n",
    "        data_index = (data_index + 1) % len(data)\n",
    "    return batch, labels\n",
    "\n",
    "\n",
    "np.random.seed(42)\n",
    "data_index = 0\n",
    "batch, labels = generate_batch(8, 2, 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-04T10:15:07.770553Z",
     "start_time": "2021-02-04T10:15:07.763271Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([3081, 3081,   12,   12,    6,    6,  195,  195], dtype=int32),\n",
       " ['originated', 'originated', 'as', 'as', 'a', 'a', 'term', 'term'])"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "batch, [vocabulary[word] for word in batch]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-04T10:15:12.861705Z",
     "start_time": "2021-02-04T10:15:12.854727Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([[  12],\n",
       "        [5234],\n",
       "        [   6],\n",
       "        [3081],\n",
       "        [  12],\n",
       "        [ 195],\n",
       "        [   2],\n",
       "        [   6]], dtype=int32),\n",
       " ['as', 'anarchism', 'a', 'originated', 'as', 'term', 'of', 'a'])"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "labels, [vocabulary[word] for word in labels[:, 0]]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Construa o modelo"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-04T10:17:23.390924Z",
     "start_time": "2021-02-04T10:17:22.119360Z"
    }
   },
   "outputs": [],
   "source": [
    "batch_size = 128\n",
    "embedding_size = 128\n",
    "skip_window = 1\n",
    "num_skips = 2\n",
    "valid_size = 16\n",
    "valid_window = 100\n",
    "valid_examples = np.random.choice(valid_window, valid_size, replace=False)\n",
    "num_sampled = 64\n",
    "learning_rate = 0.01\n",
    "\n",
    "tf.compat.v1.reset_default_graph()\n",
    "\n",
    "\n",
    "train_labels = tf.compat.v1.placeholder(tf.int32, shape=[batch_size, 1])\n",
    "valid_dataset = tf.constant(valid_examples, dtype=tf.int32)\n",
    "\n",
    "vocabulary_size = 50000\n",
    "embedding_size = 150\n",
    "\n",
    "init_embeds = tf.compat.v1.random_uniform([vocabulary_size, embedding_size], -1.0, 1.0)\n",
    "embeddings = tf.Variable(init_embeds)\n",
    "\n",
    "train_inputs = tf.compat.v1.placeholder(tf.int32, shape=[None])\n",
    "embed = tf.compat.v1.nn.embedding_lookup(embeddings, train_inputs)\n",
    "\n",
    "nce_weights = tf.Variable(tf.compat.v1.truncated_normal([vocabulary_size, embedding_size], stddev=1.0 / np.sqrt(embedding_size)))\n",
    "nce_biases = tf.Variable(tf.zeros([vocabulary_size]))\n",
    "\n",
    "loss = tf.reduce_mean(tf.compat.v1.nn.nce_loss(nce_weights, nce_biases, train_labels, embed, num_sampled, vocabulary_size))\n",
    "\n",
    "optimizer = tf.compat.v1.train.AdamOptimizer(learning_rate)\n",
    "training_op = optimizer.minimize(loss)\n",
    "\n",
    "norm = tf.compat.v1.sqrt(tf.reduce_sum(tf.square(embeddings), axis=1, keepdims=True))\n",
    "normalized_embeddings = embeddings / norm\n",
    "valid_embeddings = tf.compat.v1.nn.embedding_lookup(normalized_embeddings, valid_dataset)\n",
    "similarity = tf.matmul(valid_embeddings, normalized_embeddings, transpose_b=True)\n",
    "\n",
    "init = tf.compat.v1.global_variables_initializer()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Treine o modelo"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-04T10:35:11.720920Z",
     "start_time": "2021-02-04T10:19:17.239644Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Iteration: 0\tAverage loss at step  0 :  279.81097412109375\n",
      "Nearest to often: chew, hatches, awarded, humanae, munich, documentary, sharpe, beale,\n",
      "Nearest to s: mitigated, carolina, devastation, seamless, tenji, orientational, chromosomal, privileged,\n",
      "Nearest to his: jalal, discworld, hijacked, designer, betrayed, gallbladder, uat, twenties,\n",
      "Nearest to three: quarters, executioner, applauded, blaine, openbsd, lowell, catalonia, divergences,\n",
      "Nearest to years: vil, march, vicinity, adjudication, publicize, cassius, strathclyde, eduard,\n",
      "Nearest to united: fetching, sayyaf, lima, archons, saumur, admirable, goldfish, outlets,\n",
      "Nearest to may: rapcore, dendrites, showcased, monoamine, cohorts, heinkel, playoff, iai,\n",
      "Nearest to some: nec, solids, claudine, sparing, preserve, patience, westinghouse, parting,\n",
      "Nearest to that: colonised, deceive, black, bing, kinsmen, upgrading, anhalt, asquith,\n",
      "Nearest to its: helped, spiny, concepci, deutscher, buick, tuba, formidable, sgi,\n",
      "Nearest to these: flooding, massachusetts, doublet, skeletons, disagreement, tailback, turtledove, transl,\n",
      "Nearest to people: painterly, characterises, yat, signification, duopoly, duddy, wants, croats,\n",
      "Nearest to been: trichomes, hur, nominee, rehearsal, partner, cups, cleanup, gepids,\n",
      "Nearest to between: loudness, appleseed, offs, optimize, marl, hp, swampy, substrates,\n",
      "Nearest to will: toolkits, gillian, epicurean, disoriented, munster, drunkenness, mitigation, dhamma,\n",
      "Nearest to also: melanesia, maryland, correlated, archons, yom, numbered, addition, ninhursag,\n",
      "Iteration: 2000\tAverage loss at step  2000 :  131.25713835334778\n",
      "Iteration: 4000\tAverage loss at step  4000 :  62.98237965631485\n",
      "Iteration: 6000\tAverage loss at step  6000 :  41.620014220237735\n",
      "Iteration: 8000\tAverage loss at step  8000 :  30.98470211648941\n",
      "Iteration: 10000\tAverage loss at step  10000 :  25.65026391005516\n",
      "Nearest to often: gathered, sparing, setter, aziz, antisymmetric, lenoir, wines, reflectance,\n",
      "Nearest to s: and, the, zero, four, ik, recall, arif, of,\n",
      "Nearest to his: the, zf, gravitation, fabian, delegates, tillich, hephaestus, paradox,\n",
      "Nearest to three: two, zero, one, four, eight, five, nine, six,\n",
      "Nearest to years: terrace, four, decades, urchin, craters, two, messagepad, howard,\n",
      "Nearest to united: u, empiricist, state, nationale, actinium, tr, confiscated, guyana,\n",
      "Nearest to may: is, markov, americana, habit, overseas, benny, boeing, granular,\n",
      "Nearest to some: zeros, aalborg, acupuncture, ankh, electrolysis, cook, imprisonment, nuclei,\n",
      "Nearest to that: actinium, this, aruba, which, flex, tarkovsky, and, kano,\n",
      "Nearest to its: the, mileva, alarms, copenhagen, nen, seq, facilities, cahiers,\n",
      "Nearest to these: rebuked, aryans, ambient, condoms, are, skeleton, argon, antigua,\n",
      "Nearest to people: lindsay, bellows, cola, chadic, actinium, mouthwash, qutb, legality,\n",
      "Nearest to been: conformation, grass, aegean, alarms, have, sampras, pus, candidiasis,\n",
      "Nearest to between: adobe, quests, hephaestion, slovene, deuterostomes, upheld, ataxia, nine,\n",
      "Nearest to will: can, clarendon, reciprocating, centered, if, clothing, bohemia, markov,\n",
      "Nearest to also: a, and, baku, palestine, who, asparagus, aalborg, sometimes,\n"
     ]
    }
   ],
   "source": [
    "num_steps = 10001\n",
    "\n",
    "with tf.compat.v1.Session() as session:\n",
    "    init.run()\n",
    "    average_loss = 0\n",
    "    \n",
    "    for step in range(num_steps):\n",
    "        print('\\rIteration: {}'.format(step), end='\\t')\n",
    "        batch_inputs, batch_labels = generate_batch(batch_size, num_skips, skip_window)\n",
    "        feed_dict = {train_inputs : batch_inputs, train_labels : batch_labels}\n",
    "\n",
    "        _, loss_val = session.run([training_op, loss], feed_dict=feed_dict)\n",
    "        average_loss += loss_val\n",
    "\n",
    "        if step % 2000 == 0:\n",
    "            if step > 0:\n",
    "                average_loss /= 2000\n",
    "            print('Average loss at step ', step, ': ', average_loss)\n",
    "            average_loss = 0\n",
    "\n",
    "        if step % 10000 == 0:\n",
    "            sim = similarity.eval()\n",
    "            for i in range(valid_size):\n",
    "                valid_word = vocabulary[valid_examples[i]]\n",
    "                top_k = 8\n",
    "                nearest = (-sim[i, :]).argsort()[1:top_k+1]\n",
    "                log_str = 'Nearest to %s:' % valid_word\n",
    "                for k in range(top_k):\n",
    "                    close_word = vocabulary[nearest[k]]\n",
    "                    log_str = '%s %s,' % (log_str, close_word)\n",
    "                print(log_str)\n",
    "\n",
    "    final_embeddings = normalized_embeddings.eval()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Salve o modelo"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-04T10:35:18.494130Z",
     "start_time": "2021-02-04T10:35:18.296359Z"
    }
   },
   "outputs": [],
   "source": [
    "np.save('model/my_final_embeddings.npy', final_embeddings)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Plote os embeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-04T10:40:13.099883Z",
     "start_time": "2021-02-04T10:40:10.667273Z"
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYYAAAD6CAYAAAClF+DrAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nOzdeVyU5fr48c8DIqAouCaoJ9SjiGwDuIA4riWWkuaS5pLosb5qC+qRr3ps4bR6fnJcyMqvncxySUpLM03NhULFEhBwAReIcsHjFiQEyHL//gAmEFDQmXlmhvv9evGCueeZea4ZcS6ee7luRQiBJEmSJFWwUjsASZIkybTIxCBJkiRVIRODJEmSVIVMDJIkSVIVMjFIkiRJVcjEIEmSJFUhE4MkGZCiKK6Kopysx/EDFUXpa8iYJOleFHNZx9C6dWvh6uqqdhiSVC+FhYWcP38eDw+POh1/+fJlrKysaNeunYEjkxqChISE60KINvV9XCNDBGMIrq6uxMfHqx2GJNVLZmYmw4YNw83NjcTERDw8PPj000/p0aMH8fHxtG7dmvj4eObPn8+6desICAhAURSKi4t599130Wq1ar8EyYwpivLL/TxOdiVJkoGdOXOG2bNnk5qaSvPmzXn//fdrPM7V1ZWZM2cyd+5ckpKSZFKQVCMTgyQZWMeOHQkKCgJg8uTJHDp0SOWIJOnuZGKQJANTFKXa7UaNGlFaWgpAQUGBGmFJUq1kYpAkA/v111+Ji4sDYNOmTfTr1w9XV1cSEhIA2Lp1q+7YZs2acevWLVXilKQKekkMiqKsVRTlauVpeYqiRCiKcklRlKTyr8cr3bdIUZTziqKcURQlWB8xSOarb9+y2ZmZmZls2rRJ5Wj0z83Njffeew93d3d+++03Zs2axWuvvUZYWBg9e/bE2tpad2xISAhfffUVGo2G2NhYFaOWGjK9TFdVFKU/kAt8KoTwLG+LAHKFEJF3HNsD+AzoDbgA+4BuQoiSu52jZ8+eQs5KsmwxMTFERkbyzTffqB2K0ezM2MnKxJVcybtCu6btCPMLY3jn4WqHJVkIRVEShBA96/s4vVwxCCF+AG7W8fCRwGYhRKEQ4mfgPGVJQmqgHBwcAFi4cCGxsbFoNBqWL1/OqVOn6N27NxqNBm9vb86dO6dypPq1M2MnEUciyMrLQiDIyssi4kgEOzN2qh2a1MAZeozhBUVRUsq7mlqUt7UHLlQ65mJ5WzWKojynKEq8oijx165dM3Co+pOXl8fw4cPx8fHB09OT6Oho9u/fj6+vL15eXkyfPp3CwkKgbIriokWL0Gg09OzZk8TERIKDg+nSpQurV6/WPefSpUvp1asX3t7evPbaa2q9NINasmQJWq2WpKQk5s6dy+rVqwkLCyMpKYn4+Hg6dOigdoh6tTJxJQUlVQeeC0oKWJm4UqWIJKmMIRPDB0AXQANkAf+u7xMIIdYIIXoKIXq2aVPvxXuq2b17Ny4uLiQnJ3Py5EmGDRtGaGgo0dHRnDhxguLiYj744APd8X/5y19089ZDQ0PZsmULR48e1SWAvXv3cu7cOX766SeSkpJISEjghx9+UOvlGU1gYCBvv/02//rXv/jll1+wt7dXOyS9upJ3pV7tkmQsBksMQoj/CiFKhBClwIf82V10CehY6dAO5W0Ww8vLi++++44FCxYQGxtLZmYmnTp1olu3bgBMnTq1ygf7E088oXtcnz59aNasGW3atMHW1pbs7Gz27t3L3r178fX1xc/Pj7S0NIvrVqnJxIkT+frrr7G3t+fxxx/nwIEDaoekV+2a1lz2orZ2STIWgyUGRVGcK918EqiYsfQ1MEFRFFtFUToBXYGfDBWHGrp160ZiYiJeXl68/PLLbNu27a7H29raAmBlZaX7ueJ2cXExQggWLVpEUlISSUlJnD9/nr/97W8GfQ1quHOqZkZGBp07d+all15i5MiRpKSkqBid/oX5hWFnbVelzc7ajjC/MJUikqQyeqmVpCjKZ8BAoLWiKBeB14CBiqJoAAFkAv8DIIQ4pSjK58BpoBh4/l4zkszN5cuXadmyJZMnT8bJyYlVq1aRmZnJ+fPn+etf/8r69esZMGBAnZ8vODiYV155hUmTJuHg4MClS5ewsbGhbdu2BnwVxuft7Y21tTU+Pj6EhoZSWFjI+vXrsbGxoV27dvzjH/9QO0S9qph9JGclSaZGL4lBCPF0Dc0f3eX4t4C39HFuU3TixAnCw8OxsrLCxsaGDz74gJycHMaNG0dxcTG9evVi5syZdX6+oUOHkpqaSmBgIFA2i2fDhg0Wkxhyc3MBsLGx4cCBA6TGHiR286eU3rhO2MDeaCc8g7t2kMpRGsbwzsNlIpBMjtmU3W6o6xiyrmwnIz2SgsIs7Gyd6dxlPs7tRqodlsGkxh5k75pVFN8u1LU1amzL0OdesNjkIEmGouo6Bskwsq5sJy1tMQWFlwFBQeFl0tIWk3Vlu9qhGUzs5k+rJAWA4tuFxG7+VKWIJKnhkYnBhGWkR1Jaml+lrbQ0n4z0yFoeYf5u3bher3ZJkvRPJgYTVlCYVa92S9CsVet6tUvqGDVqFP7+/nh4eLBmzRqgbOxr8eLF+Pj4EBAQwH//+1+Vo5Tul0wMJszO1rle7ZZAO+EZGjW2rdLWqLEt2gnPqBSRVJO1a9eSkJBAfHw8UVFR3Lhxg7y8PAICAkhOTqZ///58+OGHaocp3SeZGExY5y7zsbKqutrXysqezl3mqxSR4blrBzH0uRdo1roNKArNWreRA88mKCoqSndlcOHCBc6dO0fjxo0ZMWIEAP7+/mRmZqobpHTfzGbP54aoYvZRQ5qVBGXJQSYC0xUTE8O+ffuIi4ujSZMmDBw4kIKCAmxsbHSbEllbW1NcXKxypNL9konBxDm3G2nxiUAyLzk5ObRo0YImTZqQlpbG0aNH1Q5J0jPZlSSZvezsbN5//32g7K/Ziu6Munr11VfZt2+fIUKzSMOGDaO4uBh3d3cWLlxIQECA2iFJeiavGCSzV5EYZs+efV+Pf/3112tsLykpqbK7mlTG1taWb7/9tmpjyufkvuEKEU7g2IGxQ15l7Lp1aoQn6YG8YpDM3sKFC0lPT0ej0RAeHk5ubi5jx46le/fuTJo0iYrV/QkJCQwYMAB/f3+Cg4PJyiqb9ltR6hzK9sdYsGABfn5+fPHFF6q9JrOS8jnseAlyLgCi7PuOl8raJbMkrxgks7dkyRJOnjxJUlISMTExjBw5klOnTuHi4kJQUBCHDx+mT58+vPjii2zfvp02bdoQHR3N4sWLWbt2bbXna9WqFYmJiSq8EjO1/3UoqroQk6L8snbvp9SJSXogMjFIFqd379663d40Gg2ZmZk4OTlx8uRJHn30UaCsm8jZueb1IOPHjzdarBYh52L92iWTJxODZHEq72lRMW1SCIGHhwdxcXH3fHzTpk0NGZ7lcexQ3o1UQ7tkluQYg2T27tzgpyZubm5cu3ZNlxiKioo4deqUMcKzfENeBZs7tl21sS9rl8ySvGKQzF6rVq0ICgrC09MTe3t7HnrooWrHNG7cmC1btvDSSy+Rk5NDcXExc+bMwcPDQ4WILUzFOML+18u6jxw7lCUFOb5gtuR+DFKDd/bHK8RtTyf3ZiEOLW0JHNmFbn3kvsuS+bvf/RjkFYPUoJ398QoHN6ZRfLsUgNybhRzcmAYgk4PUYMkxBqlBi9uerksKFYpvlxK3PV2liCRJfTIxSA1a7s3CerVLUkMgE4MFq1xDCO6vjpClc2hpW692SWoIZGKwYHcmhgdliWWUA0d2oVHjqv8NGjW2InBkF5UiqqquBQJnzJjB6dOnjRmaZMFkYrAgy5Ytw9PTE09PT1asWFGthhBQ7zpCAwcOZM6cOfTs2ZOVK1eq9toMpVufdgya1F13heDQ0pZBk7qbzMBzXZP7f/7zH3r06GGEiKQGQQhhFl/+/v5Cql18fLzw9PQUubm54tatW6JHjx4iMTFReHh46I45ePCgaN68ubhw4YIoKSkRAQEBIjY2Vty+fVsEBgaKq1evCiGE2Lx5s5g2bZoQQogBAwaIWbNmqfKaJCHGjx8v7OzshI+Pj+jZs6cYMGCAGDNmjHBzcxMTJ04UpaWlQoiyf6djx46J4uJiMXXqVOHh4SE8PT3FsmXLVH4FkpqAeHEfn7dyuqqFOHToEE8++aSunMPo0aOJjY2tdtz91BGStYPUU5cCgf369dMdn5SUxKVLlzh58iRQdsUhSfWll64kRVHWKopyVVGUk5XaWiqK8p2iKOfKv7cob1cURYlSFOW8oigpiqL46SMGqW7uVkcoKSmJpKQkTpw4wd69e3XHydpBpqMisVtZWekSe2WdO3cmIyODF198kd27d9O8eXN1ApXMmr7GGNYBw+5oWwjsF0J0BfaX3wZ4DOha/vUc8IGeYmjQtFot27Zt448//iAvL4+vvvqKoKCge9YQAllHyJzUlNgra9GiBcnJyQwcOJDVq1czY8YMY4coWQC9dCUJIX5QFMX1juaRwMDynz8BYoAF5e2flvd/HVUUxUlRFGchRJY+Ymmo/Pz8CA0NpXfv3kDZLBV/f39dDaHHHnuM4cOH1/hYWUfIdNWlQGBl169fp3HjxowZMwY3NzcmT55swOgkS2XIMYaHKn3YXwEqKpu1ByrX6L1Y3lYtMSiK8hxlVxX85S9/MVykFmLevHnMmzevStumTZuq3B44cKDu51WrVul+1mg0/PDDDwDk7NjB1eUrSF22nA+dnWmbJXO2WupSILCyS5cuMW3aNEpLy1Zzv/POO8YIU7IweiuiV37F8I0QwrP8drYQwqnS/b8JIVooivINsEQIcai8fT+wQAhx1wp5soieceTs2EHWK68iCgp0bYqdHc5vvI5jSIiKkRlOdnY2mzZtYvbs2cTExBAZGck333yjdlj1VpHQi7OyaOTsTNu5cyz230yqm/stomfIdQz/VRTFGaD8+9Xy9ktAx0rHdShvk0zA1eUrqiQFAFFQwNXlK1SKyPD0vRBQDRUJvfjyZRCC4suXyXrlVXJ27FA7NMkMGTIxfA1MLf95KrC9Uvsz5bOTAoAcOb5gOopr6Taqrd0S3LkQsL6LAE1BQ0zokuHoa7rqZ0Ac4KYoykVFUf4GLAEeVRTlHPBI+W2AXUAGcB74EJitjxjMQd++fQHIzMys1vdvKhrVsg9ybe2WYMmSJXTp0oWkpCSWLl3K8ePHWbFiBadPnyYjI4PDhw9TVFTEiy++yJYtW0hISGD69OksXrxY7dB1GmJClwxHX7OSnq7lriE1HCuA5/VxXnNz5MgR4M/EMHHiRJUjqq7t3Dk1jjG0nTtHxaiM634WAaqtkbNzWTdSDe2SVF+yVpIROTg4AGVdF7GxsWg0GpYvX65yVFU5hoTg/MbrNHJxAUWhkYuL6gPPUVFRuLu7M2nSJKOc734WAaqt7dw5KHZ2VdqMkdDrWuSvNuvWreNyDQlNUpcsiaGCJUuWmPTMF8eQEJOazfL++++zb98+3V/x+laXtQKVFwEGBgZSVFTE2bNnTWatR8W/l7FnJVUkhtmz769HeN26dXh6euLi4qLnyKQHIRODZFKWLVvG2rVrgbJFemlpaWRkZPDYY48xffp05s6dq/dz1mWtgDksAlQjoVceuLexsaFp06aMHTuWkydP4u/vz4YNG1AUhddff50dO3aQn59P3759+b//+z+2bt1KfHw8kyZNwt7enri4OOzt7Y0av1SL+6m8p8aXJVRXbdq0qRCirMrp8OHDVY7G9NRWIfbhhx8W165dUze45GghlnkI8Zpj2ffkaHXjMRE///yzroJvbdV7hRDixo0busdMnjxZfP3110KIP6vCSobBfVZXlWMMKqhvmYOGonKFWAcHh1orxBpdyuew4yXIuQCIsu87Xiprl6qorcjfwYMH6dOnD15eXhw4cEDW4jJxMjGowNvbG2tra3x8fExu8Fmqwf7XoSi/altRflm7VEVNA/cFBQXMnj2bLVu2cOLECZ599lkK7lhzIZkWmRiMICUlheXLlzN//nyWL19OamoqBw4cIDk52SB95uaqpgqxWq1W7bAg52L92huQulz9ViSB1q1bk5uby5YtW+r1eMn45OCzgaWkpLBjxw6KiooAyMnJYUd5mQJvb281QzM5NVWI9fX1VTkqwLFDeTdSDe0NXF0G7p2cnHj22Wfx9PSkXbt29OrVS3dfaGgoM2fOlIPPJkZvRfQMzVyL6C1fvpycnJxq7Y6OjvJqoRY7M3ayMnElV/Ku0K5pO8L8whjeueaS4UZRMcZQuTvJxh5CosD7KfXiMlPbjl9i6Z4zXM7Ox8XJnvBgN0b5tlc7LIt0v0X05BWDgdWUFO7W3tDtzNhJxJEICkrKuh+y8rKIOBIBoF5yqPjw3/96WfeRYwcY8qpMCvdh2/FLLPryBPlFJQBcys5n0ZcnAGRyMCEyMRiYo6NjrVcMUnUrE1fqkkKFgpICViauVPeqwfspmQj0YOmeM7qkUCG/qISle87IxGBC5OCzgQ0ZMgQbG5sqbTY2NgwZUq2MlARcybtSr3bJvFzOzq9Xu6SOBpMY1KrJ4u3tTUhIiO4KwdHRkZCQEDnwXIt2TdvVq91UZWZm4unpWa391VdfZd++fXo9V0UNrjutXr2aTz/9tNbH3U9towfl4lTz4HJt7ZI6GkxXkpo1Wby9vWUiqKMwv7AqYwwAdtZ2hPmFqRiV/rz+uvHWPsycOdNo56qr8GC3KmMMAPY21oQHu6kYlXQns71iyMzMxN3dnWeffRYPDw+GDh1Kfn4+SUlJBAQE4O3tzZNPPslvv/3Gli1bdDVZNBoN+fnystVUDe88nIi+ETg3dUZBwbmpMxF9I9QdX7hPJSUl1X4/Q0NDdfP4Fy5cSI8ePfD29mb+/Pm1Ps/SpUuJiooCYO7cuQwePBiAAwcO6CrOLl68GB8fHwICAvjvf/8LQEREBJGRkQCcP3+eRx55BB8fH/z8/EhPTweodVMiQxnl2553RnvR3skeBWjvZM87o73k+IKpuZ86Gmp83Vkr6eeffxbW1tbi+PHjQgghxo0bJ9avXy+8vLxETEyMEEKIV155RYSFhQkhZE0Wybhq+/2cOnWq+OKLL8T169dFt27dRGlpqRBCiN9++63W54qLixNjx44VQgjRr18/0atXL3H79m0REREhVq9eLQBd7aHw8HDxxhtvCCGEeO2118TSpUuFEEL07t1bfPnll0IIIfLz80VeXt5daxtJ1eXm5orHH39ceHt7Cw8PD7F582a1Q7onGmKtpE6dOqHRaADw9/cnPT2d7OxsBgwYAMDUqVP54Ycf1AxRasDu/P2sqBsEZWNNdnZ2/O1vf+PLL7+kSZMmtT6Pv78/CQkJ/P7779ja2hIYGEh8fDyxsbFotVoaN26sGyu48zwAt27d4tKlSzz55JMA2NnZ6c5XW20jqbrdu3fj4uJCcnIyJ0+eZNiwYWqHZDBmnRjurMuSnZ2tYjSSuRk1ahT+/v54eHiwZs0avT9/TXWDKjRq1IiffvqJsWPH8s0339z1Q8bGxoZOnTqxbt06+vbti1ar5eDBg5w/fx53d3dsbGxQFKXG8zxIjFJVXl5efPfddyxYsIDY2FiLnnJu1onhTo6OjrRo0UJXkXP9+vW6qwdj12R50J2tJMNbu3YtCQkJxMfHExUVxY0bN4x27tzcXHJycnj88cdZvnw5ycnJdz1eq9USGRlJ//790Wq1rF69Gl9fX11CuJtmzZrRoUMHtm3bBkBhYSF//PGHXl5HQ9KtWzcSExPx8vLi5ZdfNupEAmOzqMQA8MknnxAeHo63tzdJSUm8+uqrwJ81WYw1+Fw5MUimKSoqSjdge+HCBc6dO2e0c9+6dYsRI0bg7e1Nv379WLZs2V2P12q1ZGVlERgYyEMPPYSdnV29CgyuX7+eqKgovL296du3L1euyHUh9XX58mWaNGnC5MmTCQ8PJzExUe2QDMaiayVlXdlORnokBYVZ2Nk607nLfJzbjTRQhFVNmDCB7du34+bmptvZqnXr1tV2tkpISGDevHnk5ubSunVr1q1bxx9//MG4ceN0v3jnzp1j/PjxFv2LaGwxMTG8/PLL7N27lyZNmjBw4EAiIiIYOHCgwc9tKrWCzv54hbjt6eTeLMShpS2BI7vQrY95rRcxpj179hAeHo6VlRU2NjZ88MEH9OxZ7zJERiVrJd0h68p20tIWU1padnVQUHiZtLTFAEZJDkuWLOHkyZMkJSURExPDyJEjOXXqFC4uLgQFBXH48GH69OnDiy++yPbt22nTpg3R0dEsXryYtWvX4ujoSFJSEhqNho8//php06YZPOaGJCcnhxYtWtCkSRPS0tI4evSoUc5rKrWCzv54hYMb0yi+XQpA7s1CDm5MA5DJ4Q6psQeJ3fwpt25c5wWtP9oJz+CuHaR2WAZlcV1JFTLSI3VJoUJpaT4Z6ZGqxFPT7I8zZ85w8uRJHn30UTQaDW+++SYXL5bV+J8xYwYff/wxJSUlREdHM3HiRFXitlTDhg2juLgYd3d3Fi5cSEBAgFHOe7daQcYUtz1dlxQqFN8uJW57ulHjMHWpsQfZu2YVt65fAyG4df0ae9esIjX2oNqhGZTFXjEUFGbVq93Qapr9IYTAw8ODuLi4asePGTOGf/7znwwePBh/f39atWplzHAtVt7xq/y+J5OS7ELWDvgnzd92palvW6Od31RqBeXeLKxXe0MVu/lTim9XfU+KbxcSu/lTi75qMPgVg6IomYqinFAUJUlRlPjytpaKonynKMq58u8t9H1eO1vnerXrW11mQbm5uXHt2jVdYigqKtLthWtnZ0dwcDCzZs2S3Uh6knf8KtlfnqMku+w/ekl2IdlfniPv+FWjxWAqtYIcWtrWq72hunXjer3aLYWxupIGCSE0lQZBFgL7hRBdgf3lt/Wqc5f5WFlV/c9mZWVP5y61lx7Qp8o7W4WHh9d4TOPGjdmyZQsLFizAx8cHjUbDkSNHdPdPmjQJKysrhg4dapSYLd3vezIRRVW7T0RRKb/vyTRaDOHBbtjbWFdpU6NWUODILjRqXPW/f6PGVgSO7GLUOExds1at69VuKdTqShoJDCz/+RMgBligzxNUDDCrNSsJYNOmTTW2r1q1SvezRqOptjo7JSWF/fv38+2339KjRw9OnToli/DpQcWVQl3bDaFigFntWUkVA8xyVtLdaSc8w941q6p0JzVqbIt2wjMqRmV4Bp+uqijKz8BvgAD+TwixRlGUbCGEU/n9CvBbxe07Hvsc8BzAX/7yF/9ffvnFoLGagoo9ojds2MDNmzeZOnVqgyzV7eDgQG5url6fM2vJTzUmAWsnW5wX9tbruSTLUXlWUrNWrc1qVtL9Tlc1RmJoL4S4pChKW+A74EXg68qJQFGU34QQdx1nMNc9n+tL7hFdxhCJoWKMoXJ3kmJjhdPorkYdgJYkY7nfxGDwMQYhxKXy71eBr4DewH8VRXEGKP9uvNE/E2dJe0TXVIvIwcGhxhLRP//8M4GBgbpyA4bQ1LctTqO7Yu1UNsBq7WQrk4Ik1cCgiUFRlKaKojSr+BkYCpwEvgamlh82FdhuyDjMSW2FucyxYFdNtYjy8vIICAggOTmZ/v378+GHHwIQFhbGrFmzOHHiBM7Ohps51tS3Lc4Le9NhiRbnhb1lUpCkGhj6iuEh4JCiKMnAT8BOIcRuYAnwqKIo54BHym9LWNYe0TXVIqqtRPThw4d5+umnAZgyZYpaIUuShIFnJQkhMgCfGtpvAOb3SWcEFQPM+/fvJycnB0dHR4YMGWJ2A88xMTHs27ePuLg4XS2igoKCu5aIrkulUEmSDM9iVz6bM0vYI7q+tYiCgoLYvHkzkydPZuPGjUaKUpKkmlhsrSRJXfWtRbRy5Uree+89vLy8uHTpkpGilCSpJhZddlsyDzk7dnB1+QqKs7Jo5OxM27lzcAwJUTssSTJ7JjtdVVKfq6sr16+bZm2XnB07yHrlVYovXwYhKL58maxXXiVnxw61Q5OkBksmBklVV5evQBQUVGkTBQVcXb5CpYgkSZKJwcLcbYP7vLw8hg8fjo+PD56enkRHRwNlM6B8fX3x8vJi+vTpFBYar3ZQcVbNZdBra5ckyfBkYrAwd9vgfvfu3bi4uJCcnMzJkycZNmwYBQUFhIaGEh0dzYkTJyguLuaDDz4wWryNalnMVlu7JEmGJxODhbnbBvdeXl589913LFiwgNjYWBwdHTlz5gydOnWiW7duAEydOrVatVdDajt3DoqdXZU2xc6OtnPnGC0GSZKqkonBglReVJacnIyvry8Flfrvu3XrRmJioq4e0euvv65itGUcQ0JwfuN1Grm4gKLQyMUF5zdel7OSJElFMjFYkHstKrt8+TJNmjRh8uTJhIeHk5iYiJubG5mZmZw/fx6A9evXM2DAAKPG7RgSQtcD+3FPPU3XA/vNNik8/vjjZGdn3/WYgQMHUtO066SkJHbt2mWo0CSpXmRisCD3WlR24sQJevfujUaj4Z///Ccvv/wydnZ2fPzxx4wbNw4vLy+srKyYOXOmSq+guszMTDw9Pau0RUREEBkZSWhoKO3bt9cNll+/fh1XV9caH/fhhx/i7+/Pb7/9ZrBYd+3ahZNTtW1F6kQmhvtXUlKidggWRyYGC2Jra8u3335Lamoq27ZtIyYmhoEDB5KZmUnr1q0JDg4mJSWFpKQkjh07Rs+ePUlJSSElJYWRI0cyffp05syZg62t+ez7a21tzdq1a+96zPr163n33XfZs2cPLVroZ3vxDRs26JLs//zP/1BSUlJlvcgbb7yBm5sb/fr14+mnnyYyMlL32C+++ILevXvTrVs3YmNjuX37Nq+++irR0dFoNBrdbDFjmTFjBqdPnzbqOSssXbqUqKgoAObOncvgwYMBOHDgAJMmTWLv3r0EBgbi5+fHuHHjdHt0uLq6smDBAvz8/Pjiiy9qPU66PzIxNGAVu8VV7PWQk5PDjh07SElJUTmyupszZw7Lly+vUoyvss8//5wlS5awd+9eWrfWzz69qampREdHc/jwYZKSkrC2tq5S3+nYsWNs3bqV5ORkvv3224vWONAAACAASURBVGpdR8XFxfz000+sWLGCf/7znzRu3JjXX3+d8ePHk5SUxPjx4/USZ1395z//oUePHkY9ZwWtVktsbCwA8fHx5ObmUlRURGxsLN7e3rz55pvs27ePxMREevbsybJly3SPbdWqFYmJiTzyyCN3PU6qP5kYGrD9+/dTVFRUpa2oqIj9+/erFFH9/eUvf6Ffv36sX7++2n2//PILL7zwAnv37qVdO/3tZbx//34SEhLo1asXGo2G/fv3k5GRobv/8OHDjBw5Ejs7O5o1a0bIHWMmo0ePBqqWHTeWmtayVIx7/PLLL3Tt2pXr169TWlqKVqtl7969Bo3H39+fhIQEfv/9d2xtbQkMDCQ+Pp7Y2Fjs7e05ffo0QUFBaDQaPvnkEypv71uRQI8ePXrX46T6k9VVGzBz2C2utlLcldsXLVrEyJEjGT58eJVj2rRpQ8uWLfn888/1ui2qEIKpU6fyzjvvVGlft25dnR5f0VV3Z9lxY6hYy7Jz506g7N+6Yt3Kww8/zIIFC5g1axa9e/emR48eDB061KDx2NjY0KlTJ9atW0ffvn3x9vbm4MGDnD9/nk6dOvHoo4/y2Wef1fjYpk2bAmX/Hnc7Tqo/ecXQgJnDbnGtWrWqNmB88+bNKt1CXbt2RaPR8Pnnn1c5rkmTJuzatYvVq1frtZT3kCFD2LJlC1evXtXFU/kv1KCgIHbs2EFBQQG5ubl8880393zOZs2acevWLb3FWJua1rJUNmPGDH7//XdWr15dZVzEkLRaLZGRkfTv3x+tVsvq1avx9fUlICCAw4cP62bM5eXlcfbs2WqPr+txUt3JxNCAmcNucQ4ODjg7O3PgwAGg7EN49+7d9OvXr8pxixcvrvGDrG3btuzevZt//OMf7NmzRy8x9ejRgzfffJOhQ4fi7e3No48+SlalEh69evXiiSeewNvbm8ceewwvL697JttBgwZx+vRpgw8+32styx9//MHFixcBjDaAq9VqycrKIjAwkIceegg7Ozu0Wi1t2rRh3bp1PP3003h7exMYGEhaWlq1x9f1OKnuZNntBi4lJcXkd4s7ffo0zz//vO7KITw8nEmTJhEaGsqIESMYO3YsUNZ3n5iYSGZmJpmZmYwYMYKTJ08CkJyczOOPP85XX31F7969DR5zbm4uDg4O/PHHH/Tv3581a9bg5+dX5Rg1yo1fvnyZli1bYmdnxzfffMN//vMfsrOziYyMpGfPnrz44os4Ozvz8MMP89lnn9Xpakdt245fYumeM1zOzsfFyZ7wYDdG+bZXOyyTcL9lt2VikCyOKXxQTJw4kdOnT1NQUMDUqVNZtGhRlfsryo1Xriyr2NkZfNX3nj17CA8Px8rKChsbGz744APmz59PZGQkeXl5LFiwgMOHD2Ntbc3o0aMJCQlh2rRpBovnQW07folFX54gv+jPtQz2Nta8M9pLJgdkYqiTvLw8nnrqKS5evEhJSQmvvPIKf/3rX5k3bx65ubm0bt2adevWkZOTwzPPPMNPP/0ElC2WCgkJ4cSJEyQkJFQ73tnZmYEDB9KnTx8OHjxIdnY2H330EVqtVh8vXaoHc/mgODd4SNkeFHdo5OJC1wPqzArbmbGTlYkruZJ3hXZN2xHmF8bwzsPv/UAVBS05wKXs/Grt7Z3sObxwsAoRmRa5UU8d1FRd9MUXX2TLli0kJCQwffp0Fi9eTPfu3bl9+zY///wzANHR0YwfP56ioqIaj69w5/x0yfiW7jlTJSkA5BeVsHTPGZUiqpmplRvfmbGTiCMRZOVlIRBk5WURcSSCnRk7VYmnri7XkBTu1q62qKgo3N3dadGiBUuWLKnz4zIzM9m0aZMBI6uqQU1X9fLy4u9//zsLFixgxIgRtGjRgpMnT/Loo48CZUvrncvLPT/11FNER0ezcOFCoqOjiY6O5syZM7UeD+rOT5fKmMsHRSNn55qvGFQqN74ycSUFJVU3TCooKWBl4kqTvmpwcbKv8YrBxclehWju7f3332ffvn106NChxvuLi4tp1Kj6x3JFYpg4caKhQwQaWGKomJGxa9cuXn75ZQYPHoyHhwdxcXHVjh0/fjzjxo1j9OjRKIpC165dOXHiRK3Hg7rz06Uy5vJB0XbunBrHGNQqN34l70q92k1FeLBbjV2H4cFuKkZVs5kzZ5KRkcFjjz3G9OnTSU9PZ9WqVYSGhmJnZ8fx48cJCgpi5MiRhIWFAWXrdX744QcWLlxIamoqGo2GqVOn6nVdTk0aVFfSndVFf/zxR65du6b7oC8qKuLUqVMAdOnSBWtra9544w3dCks3N7daj5dMQ3iwG/Y21lXaTPGDwtTKjbdrWvPK8NraTcUo3/a8M9qL9k72KJSNLZjaeFKF1atX4+LiwsGDB6vV7Lp48SJHjhxh2bJlREZG8t5775GUlKRbAb5kyRK0Wi1JSUkGTwqg4hWDoijDgJWANfAfIUTdO9zu04kTJ6rNyGjUqBEvvfQSOTk5FBcXM2fOHDw8PICyq4bw8HDdWEPjxo3ZsmVLrcdL6qv4QFB7VlJdOIaEmEyJ8TC/MCKORFTpTrKztiPML0zFqOpmlG97k/z3rY9x48ZhbV32B01QUBDz5s1j0qRJjB49utZuJ0NSJTEoimINvAc8ClwEjimK8rUQwqAlHoODgwkODq7WXtuOZfPnz2f+/PlV2jQaTbXjc3bs4EMra4qnPMO58vnocoxBPZbwQWFsFeMI5jYryVJUlPcAWLhwIcOHD2fXrl0EBQXpbWFmfah1xdAbOC+EyABQFGUzMBJQp/bvA7hzPnrx5ctkvfIqgMn8NShJdTG883CZCExAeno6Xl5eeHl5cezYMdLS0ujYsaNRSqZUUGuMoT1wodLti+VtVSiK8pyiKPGKosRfu3bNaMHVx9XlK6oMIAKIggKuLl+hUkTSg1q2bBmenp54enqyYsUKMjMzcXd359lnn8XDw4OhQ4eSn182wJ2ens6wYcPw9/dHq9XKUgzSA1uxYgWenp54e3tjY2PDY489hre3N9bW1vj4+LB8+XKDx6DKAjdFUcYCw4QQM8pvTwH6CCFeqO0xprryOdW9B9T0HioK7qlmdwHU4CUkJBAaGsrRo0cRQtCnTx82bNhAr169iI+PR6PR8NRTT/HEE08wefJkhgwZwurVq+natSs//vgjixYt0tV1uh+vvvoq/fv355FHHtHjq5LMUd7xq/y+J5OS7EKsnWxpHuxKU9+29XqO+13gplZX0iWgY6XbHcrbzI6pzUeXHsyhQ4d48skndX2+o0ePJjY2lk6dOqHRaIA/16nk5uZy5MgRxo0bp3t8xTaj9+vOonZSw5R3/CrZX55DFJUCUJJdSPaX5wDqnRzuh1pdSceAroqidFIUpTEwAfhapVgeSNu5c1Ds7Kq0qTkfXTKMytudVqxTKS0txcnJiaSkJN1XampqnZ6vtu6p0NBQtmzZApRtX/naa6/h5+eHl5eXrpsqLy+P6dOn07t3b3x9fdm+fbv+X7Ckqt/3ZOqSQgVRVMrvezKNcn5VEoMQohh4AdgDpAKfCyHMckGAqc1Hr0xukl5/Wq2Wbdu28ccff5CXl8dXX31Va82r5s2b06lTJ7744gugbMOY5OTkOp/r3LlzPP/885w6dQonJye2bt1a7ZjWrVuTmJjIrFmzdGXF33rrLQYPHsxPP/3EwYMHCQ8PJy8v7z5erWSqSrJrvvKsrV3fVFvgJoTYJYToJoToIoR4S6049MExJISuB/bjnnqargf2Gy0pjBo1Cn9/fzw8PFizZg1Qtn/B3//+d3x8fIiLi6tx03qpdn5+foSGhtK7d2/69OnDjBkzqi1Gqmzjxo189NFH+Pj44OHhUa+/3mvqnrpTTWVW9u7dy5IlS9BoNAwcOJCCggJ+/fXXur9IyeRZO9nWq13fGlRJDEuzdu1aWrZsSX5+Pr169WLMmDHk5eXRp08f/v3vf5Oamsq//vUvDh8+jI2NDbNnz2bjxo0888wzaodu0ubNm8e8efOqtFXs6wDo1rac/fEKcdsvE+L6vzj42RI4sgvd+tR9pfCd3VMVM51qOqZymRUhBFu3bsXNzbRWc0v60zzYtcoYA4BiY0XzYFejnF8mBjMWFRXFV199BcCFCxc4d+4c1tbWjBkzBqi6aT1Afn4+bdsafuCqITj74xUObkyj+HbZf9zcm4Uc3Fg2BlCf5HA/goODeffdd3n33XdRFIXjx4/j6+tr0HNKxlUxwPygs5Lul0wMZiomJoZ9+/YRFxdHkyZNdF0KdnZ2uqX1tW1aLz24uO3puqRQofh2KXHb0w2eGF555RXmzJmDt7c3paWldOrUySx2WpPqp6lvW6MlgjvJxGCmcnJyaNGiBU2aNCEtLY2jR49WO2bIkCGMHDmSuXPn0rZtW27evMmtW7d4+OGHVYi4uuzsbDZt2sTs2bOJiYkhMjLSbD7gcm/WPAhYW/udXF1da+yeqqzymEPPnj2JiYlh65WbvJORxaUJs2g/9SUWdXZmTLuW9Qteku6hQVVXtSTDhg2juLgYd3d3Fi5cSEBAQLVj7rVpvdqys7N5//336/UYUxk8d2hZ8yBgbe36sPXKTeafucDFwiIEcLGwiPlnLrD1yk2DnbOCqbzvknE0qK09Gxp9rJw0pAkTJrB9+3bc3NywsbGhadOmtG7dmpMnT+Lv78+GDRtQFAVXV1fGjx/Pd999x//+7//SsmVLXnvtNQoLC+nSpQsff/wxDg4OtW67agh3jjEANGpsxaBJ3Q3WldTzyCkuFhZVa+9ga0N837pV+F29ejWrV68Gyq46XV1dWbRoUY3v553vuxCCt99+GyEEw4cP51//+pdeX5+kf3JrT6mKipWTFfOeK1ZO5h2/qnJkf1qyZAldunQhKSmJpUuXcvz4cVasWMHp06fJyMjg8OHDumNbtWpFYmIijzzyCG+++Sb79u0jMTGRnj17smzZsntuu6pv3fq0Y9Ck7rorBIeWtgZNCgCXakgKd2uvycyZM0lKSuLYsWN06NCB6dOn1/h+Vqh43/v378+CBQs4cOCA7vHbtm174NdkSEIISktL732gVI0cY7BQd1s5+SBXDX379uXIkSMPGl6Nevfuras9r9FoyMzMpF+/fgC6zZKOHj3K6dOnCQoKAuD27dsEBgbec9tVQ+jWp53BB5ora29rU+MVQ3tbm3o/V1hYGIMHD6ZFixY1vp8VKt73Y8eOMXDgQNq0aQPApEmT+OGHHxg1atT9vBS9WbZsGWvXrgVgxowZjBo1iuDgYPr06UNCQgK7du0ymTE1cyITg4Uy1MpJQyUFqLnsRIWK2kVCCB599FE+++yzKo+917arlmBRZ2fmn7lAfumf3b/2VgqLOtcvAa5bt45ffvmFVatWsXPnzhrfzwqV9wkwNQkJCXz88cf8+OOPuoKHAwYM4Ny5c3zyySc1jrtJdSO7kiyUoVZOOjg4AJCVlUX//v3RaDR4enoSGxtb7+dq1qxZvWvMBwQEcPjwYc6fPw+U1Q06e/Zsg9h2dUy7lkS6daSDrQ0KZWMLkW4d6zUrKSEhgcjISDZs2ICVlVWt7+edevfuzffff8/169cpKSnhs88+Y8CAAfp6afelcsFDBwcHXcHDhx9+WCaFBySvGCyUoVdObtq0ieDgYBYvXkxJSQl//PFHvZ+jVatWBAUF4enpib29PQ899NA9H9OmTRvWrVvH008/ratk+uabb9KtW7cGse3qmHYtH2h66qpVq7h58yaDBg0CyqbB1vZ+Vubs7MySJUsYNGiQbvB55MiR9/9CDMiUr3LMhZyVZMEMMSvJwcGB3NxcfvjhB6ZPn87kyZMZNWqUruaPWlJSUti/fz85OTk4OjoyZMgQvL29VY3JkmRd2U5GeiQFhVnY2TrTuct8nNupmxgSExOr7Z2xfv16pkyZUmWNSENmbvsxSEZgyJWT/fv354cffmDnzp2EhoYyb9481WowpaSksGPHDoqKygZmc3Jy2LFjB4BBkkNUVBQffPABfn5+bNy4Ue/Pb2qyrmwnLW0xpaVltZwKCi+TllY240vN5FC54CFwz4KHUt3JKwapXiquGH755Rc6dOiAtbU1q1at4vz586xYoc52psuXLycnJ6dau6OjI3PnztX7+bp3786+fft0M6juhxACIQRWVqY/zHf4sJaCwuqbUdnZuhAUVP+xJUPQrQgvLKK9rY1cEV5OrmOQjComJgYfHx98fX2Jjo4mLCxMtVhqSgp3a38QM2fOJCMjg8cee4x///vfjBo1Cm9vbwICAkhJSQEgIiJCt3cCgKenJ5mZmWRmZuLm5sYzzzyDp6cnFy5cqO00JqWgsObV8rW1G5uaK8ItlUwMUr3kHlkLyz2Z+nMYJ/8Gxz9ZpNv6Ui2Ojo71an8Qq1evxsXFhYMHD5KZmYmvry8pKSm8/fbbdepKO3fuHLNnz+bUqVNmM7/ezrbm6bC1tRvbOxlZVabwAuSXCt7JMI3EZY5kYpDqLuVz2PES5FwARNn3HS+VtatoyJAh2NhUXeRlY2PDkCFDDHreQ4cOMWXKFAAGDx7MjRs3+P333+/6GHOcStm5y3ysrOyrtFlZ2dO5S/XCf2rQx4pwqSqZGKS62/86FN2xmUxRflm7iry9vQkJCdFdITg6OhISEqLarKRGjRpVKcVQUFCg+9kcp1I6txtJ9+5vYWfrAijY2brQvftbqs9KqlDbyu/7WREulZGzkgwkMzOTESNGWNa0uZyL9Ws3Im9vb6MnAq1Wy8aNG3nllVeIiYmhdevWNG/eHFdXV1358MTERH7++WejxmUIzu1GmkwiuJO+VoRLf5JXDFLdOdYyC6e2dgsXERFBQkIC3t7eLFy4kE8++QSAMWPGcPPmTTw8PFi1alW1xWIPIioqCnd3d1q0aMGSJUv09rzmTB8rwqWq5HRVA8nMzGTYsGH4+/uTmJiIh4cHn376KampqUYrDa13FWMMlbuTbOwhJAq8n1IvLlOX8nlZd1vOxbIkOuTV+36/9DFVVmo45HRVE3TmzBlmz55NamoqzZs357333jNqaWi9836qLAk4dgSUsu8mlBTuZ+Mfg9PjgH3lqbLLly/nhRdeICcnh4cfflg3ppGXl0fHjh0pKioiPT1d98eJVqslLS1Nzy9OslQyMRhQx44ddeWMJ0+ezJ49e3SloTUaDW+++SYXL6rfP18v3k/B3JMQkV323USSAphoYtDjgH3lqbIVK3wdHR3RaDR8//33AHzzzTcEBwdjY2PDc889x7vvvqsrnDd79uwHfjlSwyAHnw1IUZQqt5s1a2bxpaHVtHDhQtLT09FoNPj6+vLkk0/yxBNP8OSTT9KiRQvWrl3L2rVrSU9P56233qpWy3/OnDn6D8oIA/bjx48nOjqaQYMGsXnzZmbPnk1ubi5Hjhxh3LhxuuMqiuRJ0r3IKwYD+vXXX3VJYNOmTQQEBFh8aWg1Vd4RLjg4WFcK/NKlS5w+fRqA2NhY+vfvX6WW/9GjR/nwww85fvy4/oMywoD9E088we7du7l58yYJCQkMHjyY0tJSnJycSEpK0n2lpqbq7ZySZTNYYlAUJUJRlEuKoiSVfz1e6b5FiqKcVxTljKIowYaKQW1ubm689957uLu789tvv+nGFxYsWICPjw8ajcagG980ZFqtltjYWE6fPk2PHj146KGHyMrKIi4ujr59+9Zay1/vhrxaNkBfmY19WbueODg40KtXL8LCwhgxYgTW1tY0b96cTp068cUXXwBltZmSk5P1dk7Jshm6K2m5ECKycoOiKD2ACYAH4ALsUxSlmxCixMCxGJWrq2u1wb6dGTtZeX4l2X/Lpl3TdoT5hTG883CVIrRs7du3Jzs7m927d9O/f39u3rzJ559/joODA82aNTNeIBVjMHqalVSb8ePHM27cOGJiYnRtGzduZNasWbz55psUFRUxYcIEfHx89HpeyTIZbLqqoigRQG4NiWERgBDinfLbe4AIIcRdO97NbbrqnXZm7CTiSAQFJX+ugrWztiOib4RMDnpy48YN/Pz8+OWXXwAIDQ3lwIEDHDhwgBs3bjB27FjGjh3L8uXLa63l7+vrq/Kr0B9D7MchmRdTna76gqIoKYqirFUUpaJQenugclnJi+Vt1SiK8pyiKPGKosRfu3bNwKEa1srElVWSAkBBSQErE1eqFJHlqbwjXHh4OFqtluLiYv7617/i5+fHzZs30Wq1QNVa/n369GHGjBkWlxSyvzyn2+O7JLuQ7C/PkXf8qsqRSebgga4YFEXZB7Sr4a7FwFHgOiCANwBnIcR0RVFWAUeFEBvKn+Mj4FshxJa7ncvcrxi8P/FGUP29VlBImZqiQkQN17bjl1i65wyXs/NxcbInPNiNUb41/m1itrKW/KRLCpVZO9nivLC3ChGZF0vZjEmVHdyEEI/U5ThFUT4Evim/eQnoWOnuDuVtFq1d03Zk5VUvA9yuaU15VTKUbccvsejLE+QXlQ1pXcrOZ9GXJwAsKjnUlBTu1i5V9f7779d5hXlxcTGNGlnWzH9DzkqqXOfhSaCimtzXwARFUWwVRekEdAV+MlQcpiLMLww7a7sqbXbWdoT5qbfBTUO0dM8ZXVKokF9UwtI9Z1SKyDCsnWzr1S79qa6bMU2ZMoWgoCBd6XVLYsgxhv+nKMoJRVFSgEHAXAAhxCngc+A0sBt43tJmJNVkeOfhRPSNwLmpMwoKzk2d5cCzCi5n59er3Vw1D3ZFsan631uxsaJ5sKs6AZmRum7GdPr0afbt28dnn32mYrSGYbDrHyFErWlUCPEW8Jahzm2qhnceLhOBylyc7LlUQxJwcbKv4WjzVTH7SM5KejCHDh1i69atQPXNmJ544gns7S3r96aCZXWMSdI9hAe7VRljALC3sSY82E3FqAyjqW9bmQgMyBw3XaorWRJDuqtRo0bh7++Ph4cHa9asoaSkhNDQUDw9PfHy8mL58uVqh1gvo3zb885oL9o72aMA7Z3seWe0l0UNPEv6U7EZE1BlMyZLJ68YpLtau3YtLVu2JD8/n169euHv78+lS5d0O9NlZ2erHGH9jfJtLxOBVCcRERFMnz4db29vmjRpotuMydLJxCDdVVRUFF999RUAFy5c4Pbt22RkZPDiiy8yfPhwhg4dqnKEkqR/mZmZup+3bdum+/nsj1f45B+HaXOzPw4tbTn74xW69bG8KeeyK0mqVUxMDPv27SMuLo7k5GR8fX0pLCwkOTmZgQMHsnr1ambMmKF2mJJkFGd/vMLBjWnk3ixbC5J7s5CDG9M4++MVlSPTP3nFINUqJyeHFi1a0KRJE9LS0jh69CjXr1+ntLSUMWPG4ObmxuTJk9UOU5KMIm57OsW3S6u0Fd8uJW57usVdNcjEINVq2LBhrF69Gnd3d9zc3AgICODSpUsMHDhQt5XkO++8o3KUkmQcFVcKdW03ZzIxSLWytbXl22+/BcpLhieu5KO8j2j3D1kyXGp4HFra1pgEHFpa3mpyOcYg3VNFyfCsvCwEgqy8LCKORLAzY6faoZmkvn37qh2CyXj88cfJzs6uth93TEwMI0aMUDGy+gsc2YVGjat+ZDZqbEXgyC4qRWQ4MjFI9yRLhteP3JXvT7t27cLJyalaYjBH3fq0Y9Ck7rorBIeWtgya1N3ixhdAJoY6y8zMxNPTU+0wVHElr+ZZF7W1N3QODg5A2V/FAwcOZOzYsXTv3p1JkyZhqI2x1LJ06VKioqIAmDt3LoMHDwbgwIEDTJo0CVdXV65fv87ChQtJT09Ho9EQHh4OQG5urtm9N936tGPq20E8v3owU98OssikADIxSHVQW2lwWTL83o4fP86KFSs4ffo0GRkZHD58WO2Q9Kpib22A+Ph4cnNzKSoqIjY2lv79++uOW7JkCV26dCEpKYmlS5cClv/emDOZGOqhpKSEZ599Fg8PD4YOHUp+fj4ffvghvXr1wsfHhzFjxvDHH3+Qk5PDww8/rJu5k5eXR8eOHSkqKiI9PZ1hw4bh7++PVqutti+0KZIlw+9f79696dChA1ZWVmg0mioLpyyBv78/CQkJ/P7779ja2hIYGEh8fDyxsbG63fJqY+nvjTmTiaEezp07x/PPP8+pU6dwcnJi69atjB49mmPHjpGcnIy7uzsfffQRjo6OaDQavv/+ewC++eYbgoODsbGx4bnnnuPdd98lISGByMhIZs+erfKrujdZMvz+2dr+OWPF2tqa4uJiFaPRPxsbGzp16sS6devo27cvWq2WgwcPcv78edzd3e/6WEt/b8yZnK5aD506dUKj0QBlfyllZmZy8uRJXn75ZbKzs8nNzSU4OBiA8ePHEx0dzaBBg9i8eTOzZ88mNzeXI0eOMG7cON1zFhaaxxxoWTJcqo1WqyUyMpK1a9fi5eXFvHnz8Pf3R1EU3THNmjXj1q1bKkYp1Ye8YqiHmv7CCQ0NZdWqVZw4cYLXXnuNgoKy2TtPPPEEu3fv5ubNmyQkJDB48GBKS0txcnIiKSlJ95WamqrWy5EaIENMotBqtWRlZREYGMhDDz2EnZ1dtW6kVq1aERQUhKenp27wWTJd8orhAd26dQtnZ2eKiorYuHEj7duXVe10cHCgV69ehIWFMWLECKytrWnevDmdOnXiiy++YNy4cQghSElJwcfHR+VX0TC8/fbb/OMf/zD4eXJzc0mNPcjZLz5lkIPCmuenoZ3wDKtWrTL4udUwZMgQioqKdLfPnj2r+7nyuMGmTZv+PObHK4zzXsR7Mw/g0NKWl6a8bLEzfMyRvGJ4QG+88QZ9+vQhKCiI7t27V7lv/PjxbNiwgfHjx+vaNm7cyEcffYSPjw8eHh5s377d2CE3WG+//bZRzpMae5C9a1Zx6/o1EIJb16+xd80qUmMPGuX891JcXMykSZNwd3dn7Nix7Nq1i1GjRunu/+6773jyyScNdv6GVIzOXCnmMHcYoGfPFGxB2QAAEtNJREFUniI+Pl7tMB7I1is3eScji0uFRbS3tWFRZ2fGtGupdlgWadSoUVy4cIGCggLCwsLIyMhg6dKleHl54eHhodt8xRDWPD+tLCncoVnrNjz33scGO29dZGZm0qlTJw4dOkRQUBDTp0/XTZqIjY2lTZs2TJw4kaeffpqQkBCDxPDJPw7XWlpi6ttBBjlnQ6UoSoIQomd9Hye7koxk65WbzD9zgfzSskR8sbCI+WcuAMjkYAB3bjD0/fffs2rVKpKSkgx+7ls3rter3dg6duxIUFDZB/DkyZOJiopiypQpbNiwgWnTphEXF8enn35qsPM3pGJ05komBiN5JyNLlxQq5JcK3snIkonBAO7cYOjcuXNGO3ezVq1rvmJo1dpoMdxN5dlCFbenTZtGSEgIdnZ2jBs3jkaNDPfR0JCK0ZkrOcZgJJcKi+rVLt2/mjYYqpgtZgzaCc/QqHHVD7lGjW3RTnjGaDHcza+//kpcXBxQNiDcr18/XFxccHFx4c0332TatGkGPX9DKkZnrmRiMJL2tjb1aldDVFQU7u7uTJo0Se1QHkhNGwxB2WKsyrNnDMVdO4ihz71As9ZtQFFo1roNQ597AXftIIOfuy7c3Nx47733cHd357fffmPWrFkATJo0iY4dO95zYdqDakjF6MyVHHw2kjvHGADsrRQi3TqaTFdS9+7d2bdvHx06dNC1FRcXG7RbwRAKCwsZNWoUmZmZuLm5kZ2dTUREBN9++y1ff/01fn5+Bh18NieVJ0SUrPp/jA0K4P15L6kdlqQn9zv4/EBXDIqijFMU5ZSiKKWKovS8475FiqKcVxTljKIowZXah5W3nVcUZeGDnN+cjGnXkki3jnSwtUEBOtjamFRSmDlzJhkZGTz22GM4OjoyZcoUgoKCmDJlCpmZmQwePBhvb2+GDBnCr7/+CkBoaCizZs0iICCAzp07ExMTo5vlEhoaqtprqdhg6Ms17/N4+xaEtGvG2S8+IXTEMFJTU2VSKFfxx8rFwiKu/89Ess+lscsrkK1XbqodmqSyB7piUBTFHSgF/g+YL4SIL2/vAXwG9AZcgH1At/KHnQUeBS4Cx4CnhRCn73Uuc79iMAeurq7Ex8ezatUqduzYwaFDh7C3tyckJISxY8cydepU1q5dy9dff822bdsIDQ2loKCAzz77jK+//popU6Zw+PBhPDw86NWrFx999JGuhIixVawlKL795yBno8a2JtWlo7aeR05xsYYxrg62NsT39VAhIknfVLliEEKkCiHO1HDXSGCzEKJQCPEzcJ6yJNEbOC+EyBBC3AY2lx8rmZgnnngCe3t7AOLi4pg4cSIAU6ZM4dChQ7rjQkJCUBQFLy8vHnroIby8vLCyssLDw0PVapmxmz+tkhQAim8XErvZcNMwzY2cECHVxlCDz+2BC5VuXyxvq629RoqiPKcoSryiKPHXrlWf/icZTtOmTet0XEX9KCsrqyq1pKysrFStlmnqawlMgTlMiJDUcc/EoCjKPkVRTtbwZfC/9IUQa4QQPYUQPdu0aWPo00m16Nu3L5s3bwbKSnrcq86+KahtzYCprCUwBYs6O2NvVXVNg72VwqLOzipFJJmKe043EUI8ch/PewnoWOl2h/I27tIumah3332XadOmsXTpUtq0acPHH6tb1qEutBOeqXGMwVTWEpiCiokPskyLdCe9TFdVFCWGqoPPHsAm/hx83g90BRTKBp+HUJYQjgEThRCn7nUOcxt8zszMZMSIEZw8efK+Hu/g4EBubq6eozKMrCvbyUiPpKAwCztbZzp3mY9zO/WHjlJjDxK7+VNu3bhOs1at0U54Rg48Sw2KKrWSFEV5EngXaAPsVBQlSQgRLIQ4pSjK58BpoBh4XghRUv6YF4A9gDWwti5JQTJdWVe2k5a2mNLSfAAKCi+TlrYYQPXk4K4dJBOBJN2HB52V9JUQooMQ/7+9+w+OqjwXOP59gLjBTBIkIL/US8K0DXWTJUQNYe+inXQAcWqKBQXHkdyLl6nWwSFDZnJHp1NpuVcloNZ2/FUY2pq0QgsYtAwtAS9BByG5JkEUr0TpAEPDbZkkCCGXCe/9Y0/CbtwN+bHZc876fGZ22H3PJnme3RyezTnveR/jMcZMMMbMC9m21hgzzRjzLWPMrpDxPxljvmltWzvYnz0cDUdirb89ogG++OILCgsLycnJ4amnnrI58v77vLmipyh0u3Klg8+bK2yKSCk1VLokxjDqb49ogCeeeIJHH32UI0eOMGmSe07+Xeo8M6BxpZTzubowDOQT+datW/F6vfh8PubMmROX+KL1iA4EAuTk5FBZWcnRo8Ejae+99x5Lly4FgtcKuEWyJ3IRizaulHI+VxeGgXwiX7NmDbt376axsZHq6uq4xDeQHtHw1eWQ3SBr2mpGjBgdNjZixGiypq22KSKl1FC5ujAM5BO53++npKSE119/na6uLtti7t0jupvf7w+7VsAtJk0sJjt7LcmeyYCQ7JlMdvZa2088K6UGz13LZvbS+xN5R0cHJSUl7NixA5/Px+bNm3n33XcBeOWVV/jggw945513yM/Pp76+noyMjLjH3N0jevz48RQUFHD+/HkAXnzxRR588EGeffZZiovd9Z/qpInFWgiUSiCuLgyR9P5EPmVKcMWN5uZmCgoKKCgoYNeuXZw8eXJYC8PUqVPDrmFYvfrqoZXu9e+7/fFv5/jPMxc5/R8vM8WTRF7WJL782c+GLTallOqLqw8lRdL9idzv95Odnd0zXlZWRk5ODl6vl9mzZ+Pz+WyM8qrQpY8NV3tB69LH7rJhwwa8Xi9er5cXXniBEydOMH369K9MjlDKDRK/UU/TFqhZA22nIP0mKPox5N4f+wAHSZc+dr/6+npKSko4ePAgxhgKCgp44403uP3226mrq2PGjBncf//93HvvvTz00ENxjW3dunV4PB5WrlzJqlWraGxsZO/evezdu5eNGzeSlpbG4cOH6ejoYNGiRTz99NMAlJeXU11dzahRo5g7dy4VFXpdihvZcuWz4zVtgZ0r4bL1Sa3tZPAxOKY46NLH7nfgwAEWLlzYsyLtfffdR21tbcTJEfEWCARYv349K1eupK6ujs7OTi5fvkxtbS1z5sxh8eLFjB07lq6uLoqKimhqamLKlCls376dY8eOISK0trbGPW5lr4Q7lBSmZs3VotDtckdw3CF06ePEFWm6crx1T7Rob2/H4/FQWFhIXV0dtbW1BAIBtmzZwsyZM8nLy+Po0aN8/PHHpKenk5yczPLly9m2bRvXX3993ONW9krswtB2amDjNtClj90vEAiwY8cOLl68yIULF9i+fbtjliZPSkoiMzOTzZs3M3v2bAKBAPv27eP48eOMHj2aiooKampqaGpq4p577uHSpUuMGjWKQ4cOsWjRIt5++23mz59vdxoqzhL7UFL6TcHDR5HGHUKXPna/mTNnUlJSwh133AHAI488wg033GBzVFcFAgEqKirYtGkTOTk5lJaWkp+fT3t7OykpKaSnp9PS0sKuXbu46667+PLLL7l48SILFizA7/eTlZVldwoqzhK7MBT9OPwcA0DS6OC4g/xg4lgtBC5XWlpKaWlpz+NPavex8s7bWb/ke6RmjOMeG5f8DgQCrF27lsLCQlJSUkhOTiYQCODz+cjLyyM7O5ubb74Zv98PBKd8FxcXc+nSJYwxbNiwwZa4lX10VpJyrNbWVqqqqnjsscfsDmVAPqndF7FJ0NwVjzt+GXCn9tZQgzPYWUmJXxiUaw212ZFdXvvRv3D+71/tUZ46bjwrfunc7ne9e2tAcN0rXeLEvQZbGBL75LNytfLycpqbm5kxYwZlZWWUlZXh9XrJycnhzTfftDu8qM7/4+8DGneK4e6t4YYeKipIC4NyrGeeeYZp06bR0NDArFmzaGhooLGxkT179lBWVsaZM87s+ZCaMW5A406hvTVUNy0MyhUOHDjA0qVLGTlyJBMmTODOO+/k8OHDdocVUWDJw4y6zhM2Nuo6D4ElD9sUUf/Eo7dGpB4qzc3NzJ8/n/z8fAKBAMeOHYvZz1ODo4VBqRibHvgOc1c8Tuq48SBC6rjxrjjxHI/eGpF6qKxYsYKXXnqJ+vp6KioqXDfZIBEl9nRV5Wqpqak9y5IHAgFeffVVli1bxrlz59i/fz/r1q2zOcLopge+4/hC0Fv3CebhnJUUaZmQ999/n8WLF/c8p7OzM9qXqzjRwqAcKyMjA7/fj9fr5e677yY3Nxefz4eI8NxzzzFx4kS7Q0w4w91bo/cyIS0tLYwZM4aGhoZh+5lq4PRQknK0qqoqqqqqmDx5MikpKSxfvpzKykoeeOABu0NTMZCWlkZmZiZbt24FwBhDY2OjzVEpLQzK0Zqamti5cydtbW0AtLW1sXPnTpqammyOTMVKZWUlGzduxOfzceutt/LWW2/ZHdLXnl7gphzt+eef7ykKodLT01m1apUNEalYuvDhWdp3n6CrtZORYzykzZtKSt6NdoeVMLQfg0pIkYpCX+PKPS58eJbWbZ9hLl8BoKu1k9ZtnwFocbDZkA4lichiETkqIldE5LaQ8aki0iEiDdbtlZBt+SJyRESOi8jPRUQif3elgn8ZDGRcuUf77hM9RaGbuXyF9t0n7AlI9RjqOYaPgPuA/RG2NRtjZli3H4aMvwz8G/AN66aLvauoioqKSEoKb1qUlJREUVGRTRGpWOlqjTwtNdq4ip8hHUoyxnwC0N8P/SIyCUgzxhy0Hv8G+D6wayhxqMSVm5sLQE1NDW1tbaSnp1NUVNQzrtxr5BhPxCIwcownwrNVPA3nOYZMEfkQaAeeMsbUAlOA0PZpp6yxiERkBbAC4JZbbhnGUJWT5ebmaiFIQGnzpoadYwCQpBGkzZtqX1AK6EdhEJE9QKQriZ40xkSbV3YGuMUY8w8RyQd2iMitAw3OGPMa8BoEZyUN9OuVUs7VfYJZZyU5zzULgzHmuwP9psaYTqDTul8vIs3AN4HTQGhfzZusMaXU11BK3o1aCBxoWC5wE5HxIjLSup9F8CTz58aYM0C7iMyyZiM9DOjVLEop5SBDna66UEROAYXAOyKy29o0B2gSkQbgD8APjTHnrG2PAb8CjgPN6IlnpZRyFL3yWSmlEpS29lRKKRUTWhiUUkqFcc2hJBH5X+CvcfyR4wBnd2/vv0TJJVHygMTJJVHygMTJJTSPfzLGjB/oN3BNYYg3EakbzLE5J0qUXBIlD0icXBIlD0icXGKRhx5KUkopFUYLg1JKqTBaGKJ7ze4AYihRckmUPCBxckmUPCBxchlyHnqOQSmlVBj9i0EppVQYLQxKKaXCaGEgcVqURsvD2vbvVqyfisi8kPH51thxESmPf9TXJiI/EZHTIe/DgpBtEfNyKje83n0RkRPW732DiNRZY2NF5C8i8pn17w12xxmJiGwSkbMi8lHIWMTYJejn1vvUJCIz7Ys8XJQ8YruPGGO+9jdgOvAt4F3gtpDxqcBHUb7mEDALEIILAd7t4Dy+DTQCHiCT4OKFI61bM5AFXGc959t25xEhr58AqyOMR8zL7nj7yMMVr/c1cjgBjOs19hxQbt0vB561O84osc8BZobu09FiBxZY+7VY+/kHdsd/jTxiuo/oXwwEW5QaYz7t7/NDW5Sa4Kvf3aLUVn3kUQz83hjTaYz5guDKtndYt+PGmM+NMf8H/N56rltEy8up3P56R1MM/Nq6/2scsC9EYozZD5zrNRwt9mLgNyboIDDG2u9tFyWPaAa1j2hhuLZMEflQRP5LRALW2IBalDrAFOBkyOPueKONO9Hj1p/0m0IOVbgpfnBfvJEY4M8iUm+13gWYYIK9VgD+BkywJ7RBiRa7G9+rmO0jw9nz2VHsbFEaS4PMw/H6ygt4Gfgpwf+UfgqsB/41ftGpEP9sjDktIjcCfxGRY6EbjTFGRFw5B97NsRPjfeRrUxhMgrQoHUweBGO7OeRxaLzRxuOqv3mJyOvA29bDvvJyIrfF+xXGmNPWv2dFZDvBwxItIjLJGHPGOtxy1tYgByZa7K56r4wxLd33Y7GP6KGkPkjitCitBpaIiEdEMgnmcQg4DHxDRDJF5DpgifVcR+l1bHch0D0bI1peTuWK1zsaEUkRkdTu+8Bcgu9FNbDMetoynL0v9BYt9mrgYWt20iygLeSQk+PEfB+x+wy7E27WC3mK4F8HLcBua/wHwFGgAfhv4HshX3Ob9eI3A7/AuorciXlY2560Yv2UkBlUBGdf/I+17Um7c4iS12+BI0CT9Ys+6Vp5OfXmhte7j9izCM5wabT2iyet8QygBvgM2AOMtTvWKPH/juDh4cvWfrI8WuwEZyP90nqfjhAyy8/uW5Q8YrqP6JIYSimlwuihJKWUUmG0MCillAqjhUEppVQYLQxKKaXCaGFQSikVRguDUkqpMFoYlFJKhfl/DKtYZ3BD3TYAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "from sklearn.manifold import TSNE\n",
    "\n",
    "def plot_with_labels(low_dim_embs, labels):\n",
    "    assert low_dim_embs.shape[0] >= len(labels), plt.figure(figsize=(25, 18))\n",
    "    for i, label in enumerate(labels):\n",
    "        x, y = low_dim_embs[i,:]\n",
    "        plt.scatter(x, y)\n",
    "        plt.annotate(label, xy=(x, y), xytext=(5, 2), textcoords='offset points', ha='right', va='bottom')\n",
    "        \n",
    "\n",
    "tsne = TSNE(perplexity=30, n_components=2, init='pca', n_iter=5000)\n",
    "plot_only = 50\n",
    "low_dim_embs = tsne.fit_transform(final_embeddings[:plot_only,:])\n",
    "labels = [vocabulary[i] for i in range(plot_only)]\n",
    "plot_with_labels(low_dim_embs, labels)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Exercícios do capítulo"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "*7*\n",
    "\n",
    "Exercício: Gramáticas Reber incorporadas"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Gera strings que respeitam a gramática"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-07T12:17:40.407877Z",
     "start_time": "2021-02-07T12:17:40.259881Z"
    }
   },
   "outputs": [],
   "source": [
    "np.random.seed(42)\n",
    "\n",
    "default_reber_grammar = [\n",
    "    [(\"B\", 1)],\n",
    "    [(\"T\", 2), (\"P\", 3)],\n",
    "    [(\"S\", 2), (\"X\", 4)],\n",
    "    [(\"T\", 3), (\"V\", 5)],\n",
    "    [(\"X\", 3), (\"S\", 6)],\n",
    "    [(\"P\", 4), (\"V\", 6)],\n",
    "    [(\"E\", None)]]\n",
    "\n",
    "embedded_reber_grammar = [\n",
    "    [(\"B\", 1)],\n",
    "    [(\"T\", 2), (\"P\", 3)],\n",
    "    [(default_reber_grammar, 4)],\n",
    "    [(default_reber_grammar, 5)],\n",
    "    [(\"T\", 6)],\n",
    "    [(\"P\", 6)],\n",
    "    [(\"E\", None)]]\n",
    "\n",
    "def generate_string(grammar):\n",
    "    state = 0\n",
    "    output = []\n",
    "    while state is not None:\n",
    "        index = np.random.randint(len(grammar[state]))\n",
    "        production, state = grammar[state][index]\n",
    "        if isinstance(production, list):\n",
    "            production = generate_string(grammar=production)\n",
    "        output.append(production)\n",
    "    return ''.join(output)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-07T12:17:40.519678Z",
     "start_time": "2021-02-07T12:17:40.425212Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "BTXXTTVPXTVPXTTVPSE BPVPSE BTXSE BPVVE BPVVE BTSXSE BPTVPXTTTVVE BPVVE BTXSE BTXXVPSE BPTTTTTTTTVVE BTXSE BPVPSE BTXSE BPTVPSE BTXXTVPSE BPVVE BPVVE BPVVE BPTTVVE BPVVE BPVVE BTXXVVE BTXXVVE BTXXVPXVVE "
     ]
    }
   ],
   "source": [
    "for _ in range(25):\n",
    "    print(generate_string(default_reber_grammar), end=' ')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-07T12:17:40.673585Z",
     "start_time": "2021-02-07T12:17:40.539524Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "BTBPVVETE BTBTSSSSSSSXXVVETE BPBTSSSXXTTTTVPSEPE BTBPTTVVETE BPBTXXTVVEPE BTBTXSETE BPBTSSSSSXXTTVPXVPXTTTVVEPE BPBTSSXXTVPSEPE BPBPTTTTTTTVPSEPE BTBTSXSETE BPBPTVPXVVEPE BPBPVVEPE BPBPTVVEPE BTBPTTVPXTTVPSETE BTBTSSXSETE BTBTXXTTVVETE BPBTSXSEPE BPBPTVPSEPE BTBPVVETE BPBTXXTTTVPXTVVEPE BPBPTTVPXTVVEPE BTBPVVETE BPBPTVPXVPXTVVEPE BTBPVVETE BPBTSXSEPE "
     ]
    }
   ],
   "source": [
    "for _ in range(25):\n",
    "    print(generate_string(embedded_reber_grammar), end=' ')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Gera strings que não respeitam a gramática"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-07T12:17:40.779235Z",
     "start_time": "2021-02-07T12:17:40.692421Z"
    }
   },
   "outputs": [],
   "source": [
    "def generate_corrupted_string(grammar, chars='BEPSTVX'):\n",
    "    good_string = generate_string(grammar)\n",
    "    index = np.random.randint(len(good_string))\n",
    "    good_char = good_string[index]\n",
    "    bad_char = np.random.choice(sorted(set(chars) - set(good_char)))\n",
    "    return good_string[:index] + bad_char + good_string[index + 1:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-07T12:17:40.928733Z",
     "start_time": "2021-02-07T12:17:40.794822Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "BTTTXXVVETE BPBTXXSPXTVVEPE BTBTXSPTE BPTTSXXTVPXVVEPE PPBPVPSEPE BTBPTVETE BPTTSSSSSXSEPE BPBSVPSEPE BTBPVVESE BPBTXSEPS BEBTXSETE XPBTXXTVPSEPE BTBPVVEPE BTXPTVVETE BTBPVXETE BVBTXSETE BPTTXXVPXVPSEPE BTBPXVPSETE STBPTTVPXVPXTVPSETE BPBPTVPSESE BPBPVEEPE ETBTXSETE BTBTXSVTE BPBTXXVPSEPP BTBTXXVPSETS "
     ]
    }
   ],
   "source": [
    "\n",
    "for _ in range(25):\n",
    "    print(generate_corrupted_string(embedded_reber_grammar), end=' ')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Converte em uma sequência de vetores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-07T12:17:41.056989Z",
     "start_time": "2021-02-07T12:17:40.944895Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[1, 0, 0, 0, 0, 0, 0],\n",
       "       [0, 0, 0, 0, 1, 0, 0],\n",
       "       [1, 0, 0, 0, 0, 0, 0],\n",
       "       [0, 0, 0, 0, 1, 0, 0],\n",
       "       [0, 0, 0, 0, 0, 0, 1],\n",
       "       [0, 0, 0, 1, 0, 0, 0],\n",
       "       [0, 1, 0, 0, 0, 0, 0],\n",
       "       [0, 0, 0, 0, 1, 0, 0],\n",
       "       [0, 1, 0, 0, 0, 0, 0],\n",
       "       [0, 0, 0, 0, 0, 0, 0],\n",
       "       [0, 0, 0, 0, 0, 0, 0],\n",
       "       [0, 0, 0, 0, 0, 0, 0]], dtype=int32)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def string_to_one_hot_vectors(string, n_steps, chars='BEPSTVX'):\n",
    "    char_to_index = {char: index for index, char in enumerate(chars)}\n",
    "    output = np.zeros((n_steps, len(chars)), dtype=np.int32)\n",
    "    for index, char in enumerate(string):\n",
    "        output[index, char_to_index[char]] = 1.\n",
    "    return output\n",
    "\n",
    "string_to_one_hot_vectors(\"BTBTXSETE\", 12)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Gera o conjunto de dados com 50% de strings boas e 50% de strings ruins"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-07T12:17:42.403945Z",
     "start_time": "2021-02-07T12:17:41.075390Z"
    }
   },
   "outputs": [],
   "source": [
    "def generate_dataset(size):\n",
    "    good_strings = [generate_string(embedded_reber_grammar) for _ in range(size // 2)]\n",
    "    bad_strings = [generate_corrupted_string(embedded_reber_grammar) for _ in range(size - size // 2)]\n",
    "    all_strings = good_strings + bad_strings\n",
    "    \n",
    "    n_steps = max([len(string) for string in all_strings])\n",
    "    \n",
    "    X = np.array([string_to_one_hot_vectors(string, n_steps) for string in all_strings])\n",
    "    seq_length = np.array([len(string) for string in all_strings])\n",
    "    \n",
    "    y = np.array([[1] for _ in range(len(good_strings))] + [[0] for _ in range(len(bad_strings))])\n",
    "    rnd_idx = np.random.permutation(size)\n",
    "    return X[rnd_idx], seq_length[rnd_idx], y[rnd_idx]\n",
    "\n",
    "X_train, l_train, y_train = generate_dataset(10000)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Cria a RNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-07T12:22:08.606284Z",
     "start_time": "2021-02-07T12:22:08.060692Z"
    }
   },
   "outputs": [],
   "source": [
    "tf.compat.v1.reset_default_graph()\n",
    "\n",
    "possible_chars = 'BEPSTVX'\n",
    "n_inputs = len(possible_chars)\n",
    "n_neurons = 30\n",
    "n_outputs = 1\n",
    "\n",
    "learning_rate = 0.02\n",
    "momentum = 0.95\n",
    "\n",
    "X = tf.compat.v1.placeholder(tf.float32, [None, None, n_inputs], name='X')\n",
    "seq_length = tf.compat.v1.placeholder(tf.int32, [None], name='seq_length')\n",
    "y = tf.compat.v1.placeholder(tf.float32, [None, 1], name='y')\n",
    "\n",
    "gru_cell = tf.compat.v1.nn.rnn_cell.GRUCell(num_units=n_neurons)\n",
    "outputs, states = tf.compat.v1.nn.dynamic_rnn(gru_cell, X, dtype=tf.float32, sequence_length=seq_length)\n",
    "\n",
    "logits = tf.compat.v1.layers.dense(states, n_outputs, name='logits')\n",
    "y_pred = tf.cast(tf.greater(logits, 0.), tf.float32, name='y_pred')\n",
    "y_proba = tf.compat.v1.nn.sigmoid(logits, name='y_proba')\n",
    "\n",
    "xentropy = tf.compat.v1.nn.sigmoid_cross_entropy_with_logits(labels=y, logits=logits)\n",
    "loss = tf.reduce_mean(xentropy, name='loss')\n",
    "optimizer = tf.compat.v1.train.MomentumOptimizer(learning_rate=learning_rate, momentum=momentum, use_nesterov=True)\n",
    "training_op = optimizer.minimize(loss)\n",
    "\n",
    "correct = tf.equal(y_pred, y, name='correct')\n",
    "accuracy = tf.reduce_mean(tf.cast(correct, tf.float32), name='accuracy')\n",
    "\n",
    "init = tf.compat.v1.global_variables_initializer()\n",
    "saver = tf.compat.v1.train.Saver()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-07T12:27:40.162979Z",
     "start_time": "2021-02-07T12:25:08.564013Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   0  Train loss: 0.6667, accuracy: 68.00%  Validation accuracy: 61.52%\n",
      "   1  Train loss: 0.6514, accuracy: 72.00%  Validation accuracy: 61.36%\n",
      "   2  Train loss: 0.5895, accuracy: 78.00%  Validation accuracy: 74.70%\n",
      "   3  Train loss: 0.4444, accuracy: 84.00%  Validation accuracy: 80.38%\n",
      "   4  Train loss: 0.4608, accuracy: 84.00%  Validation accuracy: 82.34%\n",
      "   5  Train loss: 0.3261, accuracy: 88.00%  Validation accuracy: 86.80%\n",
      "   6  Train loss: 0.2872, accuracy: 64.00%  Validation accuracy: 60.48%\n",
      "   7  Train loss: 0.1410, accuracy: 100.00%  Validation accuracy: 95.16%\n",
      "   8  Train loss: 0.0688, accuracy: 100.00%  Validation accuracy: 96.24%\n",
      "   9  Train loss: 0.1132, accuracy: 100.00%  Validation accuracy: 98.46%\n",
      "  10  Train loss: 0.0411, accuracy: 100.00%  Validation accuracy: 99.18%\n",
      "  11  Train loss: 0.0235, accuracy: 100.00%  Validation accuracy: 99.38%\n",
      "  12  Train loss: 0.0130, accuracy: 100.00%  Validation accuracy: 98.90%\n",
      "  13  Train loss: 0.0174, accuracy: 100.00%  Validation accuracy: 99.44%\n",
      "  14  Train loss: 0.1110, accuracy: 100.00%  Validation accuracy: 99.04%\n",
      "  15  Train loss: 0.0365, accuracy: 98.00%  Validation accuracy: 98.26%\n",
      "  16  Train loss: 0.0073, accuracy: 100.00%  Validation accuracy: 99.62%\n",
      "  17  Train loss: 0.0063, accuracy: 100.00%  Validation accuracy: 99.18%\n",
      "  18  Train loss: 0.0153, accuracy: 100.00%  Validation accuracy: 99.20%\n",
      "  19  Train loss: 0.0077, accuracy: 100.00%  Validation accuracy: 99.44%\n",
      "  20  Train loss: 0.0057, accuracy: 100.00%  Validation accuracy: 99.96%\n",
      "  21  Train loss: 0.0030, accuracy: 100.00%  Validation accuracy: 99.64%\n",
      "  22  Train loss: 0.0019, accuracy: 100.00%  Validation accuracy: 99.92%\n",
      "  23  Train loss: 0.2048, accuracy: 98.00%  Validation accuracy: 97.54%\n",
      "  24  Train loss: 0.0159, accuracy: 100.00%  Validation accuracy: 99.90%\n",
      "  25  Train loss: 0.0018, accuracy: 100.00%  Validation accuracy: 99.98%\n",
      "  26  Train loss: 0.0015, accuracy: 100.00%  Validation accuracy: 100.00%\n",
      "  27  Train loss: 0.0012, accuracy: 100.00%  Validation accuracy: 100.00%\n",
      "  28  Train loss: 0.0011, accuracy: 100.00%  Validation accuracy: 100.00%\n",
      "  29  Train loss: 0.0009, accuracy: 100.00%  Validation accuracy: 100.00%\n",
      "  30  Train loss: 0.0008, accuracy: 100.00%  Validation accuracy: 100.00%\n",
      "  31  Train loss: 0.0007, accuracy: 100.00%  Validation accuracy: 100.00%\n",
      "  32  Train loss: 0.0007, accuracy: 100.00%  Validation accuracy: 100.00%\n",
      "  33  Train loss: 0.0006, accuracy: 100.00%  Validation accuracy: 100.00%\n",
      "  34  Train loss: 0.0006, accuracy: 100.00%  Validation accuracy: 100.00%\n",
      "  35  Train loss: 0.0005, accuracy: 100.00%  Validation accuracy: 100.00%\n",
      "  36  Train loss: 0.0005, accuracy: 100.00%  Validation accuracy: 100.00%\n",
      "  37  Train loss: 0.0005, accuracy: 100.00%  Validation accuracy: 100.00%\n",
      "  38  Train loss: 0.0004, accuracy: 100.00%  Validation accuracy: 100.00%\n",
      "  39  Train loss: 0.0004, accuracy: 100.00%  Validation accuracy: 100.00%\n",
      "  40  Train loss: 0.0004, accuracy: 100.00%  Validation accuracy: 100.00%\n",
      "  41  Train loss: 0.0004, accuracy: 100.00%  Validation accuracy: 100.00%\n",
      "  42  Train loss: 0.0004, accuracy: 100.00%  Validation accuracy: 100.00%\n",
      "  43  Train loss: 0.0003, accuracy: 100.00%  Validation accuracy: 100.00%\n",
      "  44  Train loss: 0.0003, accuracy: 100.00%  Validation accuracy: 100.00%\n",
      "  45  Train loss: 0.0003, accuracy: 100.00%  Validation accuracy: 100.00%\n",
      "  46  Train loss: 0.0003, accuracy: 100.00%  Validation accuracy: 100.00%\n",
      "  47  Train loss: 0.0003, accuracy: 100.00%  Validation accuracy: 100.00%\n",
      "  48  Train loss: 0.0003, accuracy: 100.00%  Validation accuracy: 100.00%\n",
      "  49  Train loss: 0.0003, accuracy: 100.00%  Validation accuracy: 100.00%\n"
     ]
    }
   ],
   "source": [
    "X_val, l_val, y_val = generate_dataset(5000)\n",
    "\n",
    "n_epochs = 50\n",
    "batch_size = 50\n",
    "\n",
    "with tf.compat.v1.Session() as sess:\n",
    "    init.run()\n",
    "    for epoch in range(n_epochs):\n",
    "        X_batches = np.array_split(X_train, len(X_train) // batch_size)\n",
    "        l_batches = np.array_split(l_train, len(l_train) // batch_size)\n",
    "        y_batches = np.array_split(y_train, len(y_train) // batch_size)\n",
    "        for X_batch, l_batch, y_batch in zip(X_batches, l_batches, y_batches):\n",
    "            loss_val, _ = sess.run([loss, training_op], feed_dict={X: X_batch, seq_length: l_batch, y: y_batch})\n",
    "        acc_train = accuracy.eval(feed_dict={X: X_batch, seq_length: l_batch, y: y_batch})\n",
    "        acc_val = accuracy.eval(feed_dict={X: X_val, seq_length: l_val, y: y_val})\n",
    "        print('{:4d}  Train loss: {:.4f}, accuracy: {:.2f}%  Validation accuracy: {:.2f}%'.format(epoch, loss_val, 100 * acc_train, \n",
    "                                                                                                  100 * acc_val))\n",
    "        saver.save(sess, 'models/my_reber_classifier')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Testa a rede"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-07T12:28:57.767417Z",
     "start_time": "2021-02-07T12:28:57.371228Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from models/my_reber_classifier\n",
      "\n",
      "Estimated probability that these are Reber strings:\n",
      "BPBTSSSSSSSXXTTVPXVPXTTTTTVVETE: 0.04%\n",
      "BPBTSSSSSSSXXTTVPXVPXTTTTTVVEPE: 99.75%\n"
     ]
    }
   ],
   "source": [
    "test_strings = [\n",
    "    'BPBTSSSSSSSXXTTVPXVPXTTTTTVVETE',\n",
    "    'BPBTSSSSSSSXXTTVPXVPXTTTTTVVEPE']\n",
    "\n",
    "l_test = np.array([len(s) for s in test_strings])\n",
    "max_length = l_test.max()\n",
    "X_test = [string_to_one_hot_vectors(s, n_steps=max_length) for s in test_strings]\n",
    "\n",
    "with tf.compat.v1.Session() as sess:\n",
    "    saver.restore(sess, 'models/my_reber_classifier')\n",
    "    y_proba_val = y_proba.eval(feed_dict={X: X_test, seq_length: l_test})\n",
    "\n",
    "print()\n",
    "print('Estimated probability that these are Reber strings:')\n",
    "for index, string in enumerate(test_strings):\n",
    "    print('{}: {:.2f}%'.format(string, 100 * y_proba_val[index][0]))"
   ]
  }
 ],
 "metadata": {
  "hide_input": false,
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.5"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}