14_recurrent_neural_networks_MachineTranslation.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "hide_input": false,
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.7.5"
    },
    "toc": {
      "base_numbering": 1,
      "nav_menu": {},
      "number_sections": true,
      "sideBar": true,
      "skip_h1_title": false,
      "title_cell": "Table of Contents",
      "title_sidebar": "Contents",
      "toc_cell": false,
      "toc_position": {},
      "toc_section_display": true,
      "toc_window_display": false
    },
    "varInspector": {
      "cols": {
        "lenName": 16,
        "lenType": 16,
        "lenVar": 40
      },
      "kernels_config": {
        "python": {
          "delete_cmd_postfix": "",
          "delete_cmd_prefix": "del ",
          "library": "var_list.py",
          "varRefreshCmd": "print(var_dic_list())"
        },
        "r": {
          "delete_cmd_postfix": ") ",
          "delete_cmd_prefix": "rm(",
          "library": "var_list.r",
          "varRefreshCmd": "cat(var_dic_list()) "
        }
      },
      "types_to_exclude": [
        "module",
        "function",
        "builtin_function_or_method",
        "instance",
        "_Feature"
      ],
      "window_display": false
    },
    "colab": {
      "name": "14_recurrent_neural_networks_MachineTranslation.ipynb",
      "provenance": [],
      "collapsed_sections": []
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "hgwau10cXdWj"
      },
      "source": [
        "# Redes neurais Recorrentes (RNN)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "U2EE__yBXrmh",
        "outputId": "c9eac832-e3f6-4863-b646-6ea2d2955401"
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Mounted at /content/drive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "e4oQyzcmXdWu"
      },
      "source": [
        "### Bibliotecas básicas"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "ExecuteTime": {
          "end_time": "2021-02-06T12:21:46.275001Z",
          "start_time": "2021-02-06T12:21:43.582289Z"
        },
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "R-W9C1s6XdWv",
        "outputId": "5b77274c-9945-48cb-cb10-eb9f4a5681f2"
      },
      "source": [
        "import re\n",
        "import os\n",
        "import io\n",
        "import time\n",
        "import string\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "from string import digits\n",
        "import matplotlib.pyplot as plt\n",
        "import matplotlib.ticker as ticker\n",
        "from sklearn.model_selection import train_test_split\n",
        "\n",
        "%matplotlib inline\n",
        "\n",
        "import warnings\n",
        "warnings.filterwarnings('ignore')\n",
        "\n",
        "import tensorflow as tf\n",
        "print(tf.__version__)"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "2.4.1\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "vLHt6y4_XdWy"
      },
      "source": [
        "## Tradução automática neural com mecanismo de atenção"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Jd2m3v_aXdWz"
      },
      "source": [
        "#### Carrega os dados"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "ExecuteTime": {
          "end_time": "2021-02-06T12:22:00.626393Z",
          "start_time": "2021-02-06T12:21:59.977989Z"
        },
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 195
        },
        "id": "UDLmesX2XdW0",
        "outputId": "e42bca53-2914-4cf1-d5d6-44f0fe0804a5"
      },
      "source": [
        "data_path = '/content/drive/MyDrive/Colab Notebooks/spa-eng/spa.txt'\n",
        "lines_raw = pd.read_table(data_path, names = ['source', 'target', 'comments'])\n",
        "lines_raw.sample(5)"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>source</th>\n",
              "      <th>target</th>\n",
              "      <th>comments</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>54051</th>\n",
              "      <td>Let me just go talk to Tom.</td>\n",
              "      <td>Solo déjame ir a hablar con Tom.</td>\n",
              "      <td>CC-BY 2.0 (France) Attribution: tatoeba.org #3...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>93875</th>\n",
              "      <td>There's a pub just around the corner.</td>\n",
              "      <td>Hay un pub al torcer la esquina.</td>\n",
              "      <td>CC-BY 2.0 (France) Attribution: tatoeba.org #2...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>91102</th>\n",
              "      <td>Tom bought a camera to give to Mary.</td>\n",
              "      <td>Tom compró una cámara para regalársela a Mary.</td>\n",
              "      <td>CC-BY 2.0 (France) Attribution: tatoeba.org #1...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>56216</th>\n",
              "      <td>You must perform your duty.</td>\n",
              "      <td>Debes cumplir con tu deber.</td>\n",
              "      <td>CC-BY 2.0 (France) Attribution: tatoeba.org #1...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>22197</th>\n",
              "      <td>I don't like movies.</td>\n",
              "      <td>No me gusta ver películas.</td>\n",
              "      <td>CC-BY 2.0 (France) Attribution: tatoeba.org #4...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                      source  ...                                           comments\n",
              "54051            Let me just go talk to Tom.  ...  CC-BY 2.0 (France) Attribution: tatoeba.org #3...\n",
              "93875  There's a pub just around the corner.  ...  CC-BY 2.0 (France) Attribution: tatoeba.org #2...\n",
              "91102   Tom bought a camera to give to Mary.  ...  CC-BY 2.0 (France) Attribution: tatoeba.org #1...\n",
              "56216            You must perform your duty.  ...  CC-BY 2.0 (France) Attribution: tatoeba.org #1...\n",
              "22197                   I don't like movies.  ...  CC-BY 2.0 (France) Attribution: tatoeba.org #4...\n",
              "\n",
              "[5 rows x 3 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 3
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "ExecuteTime": {
          "end_time": "2021-02-06T12:22:00.712808Z",
          "start_time": "2021-02-06T12:22:00.706507Z"
        },
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "-kxdQKftXdW1",
        "outputId": "bff79e0a-087a-41a2-a747-4383025d27f6"
      },
      "source": [
        "print(f'Linha: {lines_raw.shape[0]} | Coluna: {lines_raw.shape[1]}')"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Linha: 128084 | Coluna: 3\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "ExecuteTime": {
          "end_time": "2021-02-06T12:22:00.938088Z",
          "start_time": "2021-02-06T12:22:00.780985Z"
        },
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ih4sF_0cXdW2",
        "outputId": "f8e9a827-476c-4bc4-8316-bf1c0d8a1932"
      },
      "source": [
        "lines_raw.info()"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "<class 'pandas.core.frame.DataFrame'>\n",
            "RangeIndex: 128084 entries, 0 to 128083\n",
            "Data columns (total 3 columns):\n",
            " #   Column    Non-Null Count   Dtype \n",
            "---  ------    --------------   ----- \n",
            " 0   source    128084 non-null  object\n",
            " 1   target    128084 non-null  object\n",
            " 2   comments  128084 non-null  object\n",
            "dtypes: object(3)\n",
            "memory usage: 2.9+ MB\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "v9pOLJROXdW3"
      },
      "source": [
        "#### Pré-processa as frases fonte e alvo para ter pares de palavras no formato: [INGLÊS, ESPANHOL]"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "ExecuteTime": {
          "end_time": "2021-02-06T12:22:06.622553Z",
          "start_time": "2021-02-06T12:22:02.252786Z"
        },
        "id": "J0WUNMa-XdW3"
      },
      "source": [
        "def preprocess_sentence(sentence):\n",
        "    num_digits= str.maketrans('', '', digits)    \n",
        "    sentence= sentence.lower()\n",
        "    sentence= re.sub(\" +\", \" \", sentence)\n",
        "    sentence= re.sub(\"'\", '', sentence)\n",
        "    sentence= sentence.translate(num_digits)\n",
        "    sentence= re.sub(r\"([?.!,¿])\", r\" \\1 \", sentence)\n",
        "    sentence = sentence.rstrip().strip()\n",
        "    sentence=  'start_ ' + sentence + ' _end'    \n",
        "    return sentence\n",
        "\n",
        "def create_dataset(path, num_examples):\n",
        "    lines = io.open(path, encoding='UTF-8').read().strip().split('\\n')\n",
        "    word_pairs = [[preprocess_sentence(w) for w in l.split('\\t')]  for l in lines[:num_examples]]\n",
        "    return zip(*word_pairs)\n",
        "\n",
        "sample_size=60000\n",
        "source, target, _ = create_dataset(data_path, sample_size)"
      ],
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "ExecuteTime": {
          "end_time": "2021-02-06T12:00:27.703409Z",
          "start_time": "2021-02-06T12:00:27.698102Z"
        },
        "id": "P9Ir4jyBXdW4"
      },
      "source": [
        "#### Tokeniza e converte o texto em uma sequência de inteiros"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "ExecuteTime": {
          "end_time": "2021-02-06T12:22:11.154723Z",
          "start_time": "2021-02-06T12:22:06.856815Z"
        },
        "id": "EODWHYGRXdW5"
      },
      "source": [
        "source_sentence_tokenizer= tf.keras.preprocessing.text.Tokenizer(filters='')\n",
        "source_sentence_tokenizer.fit_on_texts(source)\n",
        "source_tensor = source_sentence_tokenizer.texts_to_sequences(source)\n",
        "source_tensor = tf.keras.preprocessing.sequence.pad_sequences(source_tensor, padding='post')\n",
        "\n",
        "target_sentence_tokenizer= tf.keras.preprocessing.text.Tokenizer(filters='')\n",
        "target_sentence_tokenizer.fit_on_texts(target)\n",
        "target_tensor = target_sentence_tokenizer.texts_to_sequences(target)\n",
        "target_tensor= tf.keras.preprocessing.sequence.pad_sequences(target_tensor,padding='post' )"
      ],
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "ExecuteTime": {
          "end_time": "2021-02-06T12:06:42.620962Z",
          "start_time": "2021-02-06T12:06:42.612662Z"
        },
        "id": "cZT4fGPdXdW6"
      },
      "source": [
        "#### Separação treino e teste"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "ExecuteTime": {
          "end_time": "2021-02-06T12:22:11.409646Z",
          "start_time": "2021-02-06T12:22:11.392956Z"
        },
        "id": "-eljg7JvXdW6"
      },
      "source": [
        "source_train_tensor, source_test_tensor, target_train_tensor, target_test_tensor = train_test_split(source_tensor, \n",
        "                                                                                                    target_tensor, \n",
        "                                                                                                    test_size=0.2)"
      ],
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "sGVo8eZaXdW7"
      },
      "source": [
        "#### Cria dados na memoria"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "ExecuteTime": {
          "end_time": "2021-02-06T12:24:39.447632Z",
          "start_time": "2021-02-06T12:24:39.429794Z"
        },
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "7Bq6XIlbXdW7",
        "outputId": "9114f0a4-f32d-4a78-f3f9-537f3ef9e61e"
      },
      "source": [
        "BATCH_SIZE = 64\n",
        "\n",
        "dataset = tf.data.Dataset.from_tensor_slices((source_train_tensor, target_train_tensor)).shuffle(BATCH_SIZE)\n",
        "dataset = dataset.batch(BATCH_SIZE, drop_remainder=True)\n",
        "source_batch, target_batch = next(iter(dataset))\n",
        "print(source_batch.shape)\n",
        "print(target_batch.shape)"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "(64, 13)\n",
            "(64, 20)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "ExecuteTime": {
          "end_time": "2021-02-06T12:22:12.268269Z",
          "start_time": "2021-02-06T12:22:04.393Z"
        },
        "id": "9niSofpdXdW8"
      },
      "source": [
        "#### Cria o modelo"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "ExecuteTime": {
          "end_time": "2021-02-06T12:27:20.317741Z",
          "start_time": "2021-02-06T12:27:20.312179Z"
        },
        "id": "wSFu99bVXdW9"
      },
      "source": [
        "BUFFER_SIZE = len(source_train_tensor)\n",
        "steps_per_epoch= len(source_train_tensor)//BATCH_SIZE\n",
        "embedding_dim=256\n",
        "units=1024\n",
        "source_vocab_size= len(source_sentence_tokenizer.word_index)+1\n",
        "target_vocab_size= len(target_sentence_tokenizer.word_index)+1"
      ],
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Zcnf3KTxXdW9"
      },
      "source": [
        "Encoder"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "ExecuteTime": {
          "end_time": "2021-02-06T12:30:33.369642Z",
          "start_time": "2021-02-06T12:30:33.359885Z"
        },
        "id": "sXpnrA8KXdW-"
      },
      "source": [
        "class Encoder(tf.keras.Model):\n",
        "    def __init__(self, vocab_size, embedding_dim, encoder_units, batch_size):\n",
        "        super(Encoder, self).__init__()\n",
        "        self.batch_size= batch_size\n",
        "        self.encoder_units=encoder_units\n",
        "        self.embedding=tf.keras.layers.Embedding(vocab_size, embedding_dim)\n",
        "        self.gru= tf.keras.layers.GRU(encoder_units,return_sequences=True, return_state=True, recurrent_initializer='glorot_uniform')\n",
        "    \n",
        "    def call(self, x, hidden):\n",
        "        x = self.embedding(x)\n",
        "        output, state = self.gru(x, initial_state=hidden)\n",
        "        return output, state\n",
        "    \n",
        "    def initialize_hidden_state(self):\n",
        "        return tf.zeros((self.batch_size, self.encoder_units))"
      ],
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "ExecuteTime": {
          "end_time": "2021-02-06T12:32:34.118378Z",
          "start_time": "2021-02-06T12:32:33.936384Z"
        },
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "9SNvL9aZXdW_",
        "outputId": "ef642a1f-8eee-4225-c81b-426bd8e50414"
      },
      "source": [
        "encoder = Encoder(source_vocab_size, embedding_dim, units, BATCH_SIZE)\n",
        "sample_hidden = encoder.initialize_hidden_state()\n",
        "sample_output, sample_hidden= encoder(source_batch, sample_hidden)\n",
        "\n",
        "print ('Encoder output shape: (batch size, sequence length, units) {}'.format(sample_output.shape))\n",
        "print ('Encoder Hidden state shape: (batch size, units) {}'.format(sample_hidden.shape))"
      ],
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Encoder output shape: (batch size, sequence length, units) (64, 13, 1024)\n",
            "Encoder Hidden state shape: (batch size, units) (64, 1024)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "EyAPrJfbXdXA"
      },
      "source": [
        "Camada de atenção"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "ExecuteTime": {
          "end_time": "2021-02-06T12:39:23.114929Z",
          "start_time": "2021-02-06T12:39:23.095804Z"
        },
        "id": "mf6NLadBXdXA"
      },
      "source": [
        "class BahdanauAttention(tf.keras.layers.Layer):\n",
        "    def __init__(self, units):\n",
        "        super(BahdanauAttention, self).__init__()\n",
        "        self.W1 = tf.keras.layers.Dense(units)\n",
        "        self.W2 = tf.keras.layers.Dense(units)\n",
        "        self.V = tf.keras.layers.Dense(1)\n",
        "    \n",
        "    def call(self, query, values):\n",
        "        hidden_with_time_axis = tf.expand_dims(query, 1)\n",
        "        score= self.V(tf.nn.tanh(self.W1(values) + self.W2(hidden_with_time_axis)))    \n",
        "        attention_weights= tf.nn.softmax(score, axis=1)\n",
        "        context_vector = attention_weights * values\n",
        "        context_vector = tf.reduce_sum(context_vector, axis=1)\n",
        "        return context_vector, attention_weights"
      ],
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "ExecuteTime": {
          "end_time": "2021-02-06T12:39:24.590821Z",
          "start_time": "2021-02-06T12:39:24.088514Z"
        },
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "WXEFiZ_cXdXB",
        "outputId": "91258796-e1fc-41b5-ebbe-27a44c15475e"
      },
      "source": [
        "attention_layer= BahdanauAttention(10)\n",
        "attention_result, attention_weights = attention_layer(sample_hidden, sample_output)\n",
        "print('Attention result shape: (batch size, units) {}'.format(attention_result.shape))\n",
        "print('Attention weights shape: (batch_size, sequence_length, 1) {}'.format(attention_weights.shape))"
      ],
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Attention result shape: (batch size, units) (64, 1024)\n",
            "Attention weights shape: (batch_size, sequence_length, 1) (64, 13, 1)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "OU2qqwqFXdXC"
      },
      "source": [
        "Decoder"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "ExecuteTime": {
          "end_time": "2021-02-06T12:43:17.353996Z",
          "start_time": "2021-02-06T12:43:17.341619Z"
        },
        "id": "QAQGYkjiXdXD"
      },
      "source": [
        "class Decoder(tf.keras.Model):\n",
        "    def __init__(self, vocab_size, embedding_dim, dec_units, batch_sz):\n",
        "        super(Decoder, self).__init__()\n",
        "        self.batch_sz = batch_sz\n",
        "        self.dec_units = dec_units\n",
        "        self.embedding = tf.keras.layers.Embedding(vocab_size, embedding_dim)\n",
        "        self.gru = tf.keras.layers.GRU(self.dec_units,\n",
        "                                       return_sequences=True,\n",
        "                                       return_state=True,\n",
        "                                       recurrent_initializer='glorot_uniform')\n",
        "        self.fc = tf.keras.layers.Dense(vocab_size)\n",
        "        self.attention = BahdanauAttention(self.dec_units)\n",
        "\n",
        "    def call(self, x, hidden, enc_output):\n",
        "        context_vector, attention_weights = self.attention(hidden, enc_output)\n",
        "        x = self.embedding(x)\n",
        "        x = tf.concat([tf.expand_dims(context_vector, 1), x], axis=-1)\n",
        "        output, state = self.gru(x)\n",
        "        output = tf.reshape(output, (-1, output.shape[2]))\n",
        "        x = self.fc(output)\n",
        "        return x, state, attention_weights"
      ],
      "execution_count": 15,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "ExecuteTime": {
          "end_time": "2021-02-06T12:44:19.964800Z",
          "start_time": "2021-02-06T12:44:19.400557Z"
        },
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "UH6F4osGXdXE",
        "outputId": "ef88275c-1a9e-4952-9049-bba420f7c1a2"
      },
      "source": [
        "decoder = Decoder(target_vocab_size, embedding_dim, units, BATCH_SIZE)\n",
        "sample_decoder_output, _, _ = decoder(tf.random.uniform((BATCH_SIZE, 1)), sample_hidden, sample_output)\n",
        "print ('Decoder output shape: (batch_size, vocab size) {}'.format(sample_decoder_output.shape))"
      ],
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Decoder output shape: (batch_size, vocab size) (64, 15349)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5A24-aG3XdXF"
      },
      "source": [
        "Otimizador e função de perda"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "ExecuteTime": {
          "end_time": "2021-02-06T12:46:57.993483Z",
          "start_time": "2021-02-06T12:46:57.986203Z"
        },
        "id": "9qSWTVNJXdXG"
      },
      "source": [
        "optimizer = tf.keras.optimizers.Adam()\n",
        "loss_object = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True, reduction='none')\n",
        "\n",
        "def loss_function(real, pred):\n",
        "    mask = tf.math.logical_not(tf.math.equal(real, 0))\n",
        "    loss_ = loss_object(real, pred)\n",
        "    mask = tf.cast(mask, dtype=loss_.dtype)\n",
        "    loss_ *= mask\n",
        "    return tf.reduce_mean(loss_)"
      ],
      "execution_count": 17,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "RRxiWWoZXdXH"
      },
      "source": [
        "Checkpoint"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "ExecuteTime": {
          "end_time": "2021-02-06T12:48:07.033305Z",
          "start_time": "2021-02-06T12:48:07.028224Z"
        },
        "id": "G_7LCMtBXdXH"
      },
      "source": [
        "checkpoint_dir = '/content/drive/MyDrive/Colab Notebooks/training_checkpoints'\n",
        "checkpoint_prefix = os.path.join(checkpoint_dir, 'ckpt')\n",
        "checkpoint = tf.train.Checkpoint(optimizer=optimizer, encoder=encoder, decoder=decoder)"
      ],
      "execution_count": 18,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "anrSxRLyXdXH"
      },
      "source": [
        "#### Treina o modelo"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "ExecuteTime": {
          "end_time": "2021-02-06T12:51:46.705863Z",
          "start_time": "2021-02-06T12:51:46.685004Z"
        },
        "id": "VD_UMfcGXdXI"
      },
      "source": [
        "@tf.function\n",
        "def train_step(inp, targ, enc_hidden):\n",
        "    loss = 0\n",
        "\n",
        "    with tf.GradientTape() as tape:\n",
        "        enc_output, enc_hidden = encoder(inp, enc_hidden)\n",
        "        dec_hidden = enc_hidden\n",
        "        dec_input = tf.expand_dims([target_sentence_tokenizer.word_index['start_']] * BATCH_SIZE, 1)\n",
        "        for t in range(1, targ.shape[1]):\n",
        "            predictions, dec_hidden, _ = decoder(dec_input, dec_hidden, enc_output)\n",
        "            loss += loss_function(targ[:, t], predictions)\n",
        "            dec_input = tf.expand_dims(targ[:, t], 1)\n",
        "\n",
        "    batch_loss = (loss / int(targ.shape[1]))\n",
        "    variables = encoder.trainable_variables + decoder.trainable_variables\n",
        "    gradients = tape.gradient(loss, variables)\n",
        "    optimizer.apply_gradients(zip(gradients, variables))\n",
        "    return batch_loss"
      ],
      "execution_count": 19,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "ExecuteTime": {
          "start_time": "2021-02-06T12:53:00.482Z"
        },
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "T_qqUMojXdXJ",
        "outputId": "d1420a5e-2fb2-4fd2-9d83-f1f0d57ee88c"
      },
      "source": [
        "EPOCHS = 5\n",
        "\n",
        "for epoch in range(EPOCHS):\n",
        "    start = time.time()\n",
        "    enc_hidden = encoder.initialize_hidden_state()\n",
        "    total_loss = 0\n",
        "\n",
        "    for (batch, (inp, targ)) in enumerate(dataset.take(steps_per_epoch)):\n",
        "        batch_loss = train_step(inp, targ, enc_hidden)\n",
        "        total_loss += batch_loss\n",
        "        if batch % 100 == 0:\n",
        "            print('Epoch {} Batch {} loss {}'.format(epoch + 1,batch, batch_loss.numpy()))\n",
        "  \n",
        "    if (epoch + 1) % 2 == 0:\n",
        "        checkpoint.save(file_prefix = checkpoint_prefix)\n",
        "\n",
        "    print('Epoch {} Loss {:.4f}'.format(epoch + 1, total_loss / steps_per_epoch))\n",
        "    print('Time taken for 1 epoch {} sec\\n'.format(time.time() - start))"
      ],
      "execution_count": 20,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch 1 Batch 0 loss 3.0799362659454346\n",
            "Epoch 1 Batch 100 loss 1.5914896726608276\n",
            "Epoch 1 Batch 200 loss 1.4999147653579712\n",
            "Epoch 1 Batch 300 loss 1.3425962924957275\n",
            "Epoch 1 Batch 400 loss 1.268446922302246\n",
            "Epoch 1 Batch 500 loss 1.1572431325912476\n",
            "Epoch 1 Batch 600 loss 1.0980063676834106\n",
            "Epoch 1 Batch 700 loss 1.0921634435653687\n",
            "Epoch 1 Loss 1.3365\n",
            "Time taken for 1 epoch 4904.994888782501 sec\n",
            "\n",
            "Epoch 2 Batch 0 loss 0.9378819465637207\n",
            "Epoch 2 Batch 100 loss 0.960283100605011\n",
            "Epoch 2 Batch 200 loss 0.8661623001098633\n",
            "Epoch 2 Batch 300 loss 0.9111055731773376\n",
            "Epoch 2 Batch 400 loss 0.7920030355453491\n",
            "Epoch 2 Batch 500 loss 0.7733731269836426\n",
            "Epoch 2 Batch 600 loss 0.7268795967102051\n",
            "Epoch 2 Batch 700 loss 0.6980582475662231\n",
            "Epoch 2 Loss 0.8219\n",
            "Time taken for 1 epoch 4849.427884340286 sec\n",
            "\n",
            "Epoch 3 Batch 0 loss 0.6220558881759644\n",
            "Epoch 3 Batch 100 loss 0.5717284679412842\n",
            "Epoch 3 Batch 200 loss 0.5847135782241821\n",
            "Epoch 3 Batch 300 loss 0.5980767011642456\n",
            "Epoch 3 Batch 400 loss 0.5335487723350525\n",
            "Epoch 3 Batch 500 loss 0.46015700697898865\n",
            "Epoch 3 Batch 600 loss 0.4592013955116272\n",
            "Epoch 3 Batch 700 loss 0.3936856687068939\n",
            "Epoch 3 Loss 0.5331\n",
            "Time taken for 1 epoch 4918.433731079102 sec\n",
            "\n",
            "Epoch 4 Batch 0 loss 0.37284359335899353\n",
            "Epoch 4 Batch 100 loss 0.47166892886161804\n",
            "Epoch 4 Batch 200 loss 0.3820625841617584\n",
            "Epoch 4 Batch 300 loss 0.367127001285553\n",
            "Epoch 4 Batch 400 loss 0.35914483666419983\n",
            "Epoch 4 Batch 500 loss 0.2982313334941864\n",
            "Epoch 4 Batch 600 loss 0.2948380708694458\n",
            "Epoch 4 Batch 700 loss 0.29430481791496277\n",
            "Epoch 4 Loss 0.3565\n",
            "Time taken for 1 epoch 4807.492107152939 sec\n",
            "\n",
            "Epoch 5 Batch 0 loss 0.2518405616283417\n",
            "Epoch 5 Batch 100 loss 0.30646634101867676\n",
            "Epoch 5 Batch 200 loss 0.2310340851545334\n",
            "Epoch 5 Batch 300 loss 0.25044649839401245\n",
            "Epoch 5 Batch 400 loss 0.2112617939710617\n",
            "Epoch 5 Batch 500 loss 0.19522975385189056\n",
            "Epoch 5 Batch 600 loss 0.21190741658210754\n",
            "Epoch 5 Batch 700 loss 0.21978750824928284\n",
            "Epoch 5 Loss 0.2453\n",
            "Time taken for 1 epoch 4756.022970199585 sec\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1bmXpee9XdXL"
      },
      "source": [
        "#### Tradução"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "ExecuteTime": {
          "start_time": "2021-02-06T13:01:32.856Z"
        },
        "id": "lPX3bJ9HXdXL"
      },
      "source": [
        "max_target_length= max(len(t) for t in  target_tensor)\n",
        "max_source_length= max(len(t) for t in source_tensor)\n",
        "\n",
        "def evaluate(sentence):\n",
        "    attention_plot = np.zeros((max_target_length, max_source_length))\n",
        "\n",
        "    sentence = preprocess_sentence(sentence)\n",
        "    inputs = [source_sentence_tokenizer.word_index[i] for i in sentence.split(' ')]\n",
        "    inputs = tf.keras.preprocessing.sequence.pad_sequences([inputs], maxlen=max_source_length, padding='post')\n",
        "    inputs = tf.convert_to_tensor(inputs)\n",
        "\n",
        "    result = ''\n",
        "\n",
        "    hidden = [tf.zeros((1, units))]\n",
        "    enc_out, enc_hidden = encoder(inputs, hidden)\n",
        "\n",
        "    dec_hidden = enc_hidden\n",
        "    dec_input = tf.expand_dims([target_sentence_tokenizer.word_index['start_']], 0)\n",
        "\n",
        "    for t in range(max_target_length):\n",
        "        predictions, dec_hidden, attention_weights = decoder(dec_input, dec_hidden, enc_out)\n",
        "\n",
        "    attention_weights = tf.reshape(attention_weights, (-1, ))\n",
        "    attention_plot[t] = attention_weights.numpy()\n",
        "\n",
        "    predicted_id = tf.argmax(predictions[0]).numpy()\n",
        "\n",
        "    result += target_sentence_tokenizer.index_word[predicted_id] + ' '\n",
        "\n",
        "    if target_sentence_tokenizer.index_word[predicted_id] == '_end':\n",
        "        return result, sentence, attention_plot\n",
        "\n",
        "    dec_input = tf.expand_dims([predicted_id], 0)\n",
        "    return result, sentence, attention_plot"
      ],
      "execution_count": 26,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-r2DzZroXdXM"
      },
      "source": [
        "#### Plota os pesos"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "ExecuteTime": {
          "start_time": "2021-02-06T13:01:33.944Z"
        },
        "id": "CNjdn8hZXdXN"
      },
      "source": [
        "def plot_attention(attention, sentence, predicted_sentence):\n",
        "    fig = plt.figure(figsize=(10,10))\n",
        "    ax = fig.add_subplot(1, 1, 1)\n",
        "    ax.matshow(attention, cmap='viridis')\n",
        "    fontdict = {'fontsize': 14}\n",
        "    ax.set_xticklabels([''] + sentence, fontdict=fontdict, rotation=90)\n",
        "    ax.set_yticklabels([''] + predicted_sentence, fontdict=fontdict)\n",
        "    ax.xaxis.set_major_locator(ticker.MultipleLocator(1))\n",
        "    ax.yaxis.set_major_locator(ticker.MultipleLocator(1))\n",
        "    plt.show()\n",
        "\n",
        "\n",
        "def translate(sentence):\n",
        "    result, sentence, attention_plot = evaluate(sentence)  \n",
        "    print('Input: %s' % (sentence))\n",
        "    print('Predicted translation: {}'.format(result))\n",
        "    attention_plot = attention_plot[:len(result.split(' ')), :len(sentence.split(' '))]\n",
        "    plot_attention(attention_plot, sentence.split(' '), result.split(' '))"
      ],
      "execution_count": 38,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "TGRmCyc6XdXO"
      },
      "source": [
        "#### Restaura o último ponto de verificação e testa"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "ExecuteTime": {
          "start_time": "2021-02-06T13:01:34.966Z"
        },
        "id": "OFuXrAVJXdXO",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "4f7e233e-6fb0-4652-f9b8-3e96c6dec128"
      },
      "source": [
        "checkpoint.restore(tf.train.latest_checkpoint(checkpoint_dir))"
      ],
      "execution_count": 28,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tensorflow.python.training.tracking.util.CheckpointLoadStatus at 0x7eff69973b38>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 28
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "gmewzroAXdXP"
      },
      "source": [
        "### Traduções finais com gráficos de atenção"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "ExecuteTime": {
          "start_time": "2021-02-06T13:01:35.589Z"
        },
        "id": "oELcT4PpXdXP",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 256
        },
        "outputId": "92d51ca7-9652-4a1a-e4cc-be985fde1e4c"
      },
      "source": [
        "translate(u'I am going to work.')"
      ],
      "execution_count": 39,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Input: start_ i am going to work . _end\n",
            "Predicted translation: . \n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAkcAAADMCAYAAAB5qwfFAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAQsUlEQVR4nO3debCddX3H8feXJCQsBYZIAUWWioK4IDVhsSYIaAuD8ge11Q6LpaOUAgWdodDSVi1qoRQ7LFOhtMjYMqVUCpbNDtUCDgyLBUVAdgjU0jahBcMiYfHbP37PkR8nT8LNdp7nnvN+zdzJvb/n5s6X35B7Pue3RmYiSZKkYr2uC5AkSeoTw5EkSVLFcCRJklQxHEmSJFUMR5IkSRXDkSRJUsVwJEmSVDEcSZIkVSYqHEXEXRHx5q7rkCRJ/TVR4QjYHpjVdRGSJKm/ZnZdgKR+iIhHgbb7hBJ4AXgIuCAzrxhpYZI0YpM2ciRpxS4ENgceBC5qPh5s2q4AXgEui4iPdVahJI2AI0eSBn4BOC0zT6sbI+JEYJfMPDgiTgZ+H7ikiwIlaRQis20UfTxFxDPArpn5SNe1SH0TEUuBX8zMh4badwTuyMxNImIn4PbM3LiTIiVpBBw5kjTwPLCAsraotqB5BjAD+Mkoi1L/RMTumXnbCp4dmpkXjbom9UdEbDvV783Mx9dlLatrLMJRRBwOXJKZy4ba1wc+npl/2zT9NvA/o65PmibOAr4SEfOA7zZt84HfBL7QfL0/8P3Rl6aeuSoiFmbmfXVjRBwGnEdZr6bJtYj2zR1tZqzDOlbbWEyrRcQrwNaZuXiofS6wODN72flS30TEx4HjgJ2bpvuAszLzkub5BkBm5gsdlageiIiTgGOA92Xmj5q2w4FzgY9l5lVd1qduRcR7qy/fBpxOCc03N217UQYrTsrMi0dc3pSMSzj6KbBlZi4Zat8N+HZmbt5NZZI0niLiDODDwPuBAynB6Ncy8+pOC1OvRMQNwDmZeelQ+0eB4zNzQTeVrdy0nlaLiLsoQ3cJ3BARL1ePZwDbAdd0UZs0nUXEZgwd9ZGZ/9dROeqhzDyhGZ2/FdgK+Ghm+vtWw3YHftDS/gPgvS3tvTCtwxEwSKLvBK4Gnq2evUiZ9/ynEdckTUsRsR1l6PsDwPr1I8obEKenJ1hEHNzSfA2wH3AxMGfwPZl52ShrU68tAo4GPj3UfjTw2MirmaJpP60WETMpc5ffyMz/7LoeabqKiH8DNgPOAJ5gaEFlZt7QRV3qh2b5wlSk6zw1EBH7A5dTgtAtTfMelOu8Ds7Mb3ZU2kpN+3AEEBEvADtn5qKua5Gmq4h4FtgzM+/uuhZJ4yMitqGMFA02etwLnJeZ/9FdVSs33afVBu4EdqQM30laPY8Cs7suQv0WEbOAG4HDM/P+rutR/zU7Gk/uuo5VMS4jRwcApwGfA24Hnqufu5BUen0RsS/lapCjh0/JlmoRsRh4f2Y+0HUt6r+I2BB4D/DzLL/Ro5fr08YlHNVz4fV/UOD8tzQlzfU6sykLr5cB9e5PMnOTLupS/0TEnwNk5u91XYv6LSI+SFmwP7flcW9fn8dlWm2frguQxsCxXRegaWMj4JCI+BDto/XHdVKV+ugsym7ykzPzia6LmaqxGDmSVldEzGH5Yd7nV/DtkoCIuG4ljzMz9x1ZMeq1iHgOeHdmPtx1LatiXEaOAIiINwLb8tozWsjM73RTkfqoOc/nbMqI40Yt39LLYd51ISI2H6zJi4iVniTv2j0NZKaj9Zqqm4CdAMPRqDWh6O+BhZQ1R4ND6wYm5sVOU3IRMAf4XcpFxJM8fLokIgb3Ej5Je194CKRaNSOvO1L+/3jYO/fU4jzgjOZ1+i7gpfphZt7RSVWvYyzCEXAm8AqwC+U28f2BLYFTgM90WJf6aTdgfmbe23UhPbAvMBgRcjRAU9Js5/9Tyjq19SkBellEnAP8YWa+tLK/r4kyuMni/JZnvX3TNS7haG/gwMy8LyISWJKZN0XEMuALwL92W5565k5gC8pBZBOtPvXaE7C1Cv4M+A3gKMqZRwALgFMpa/hO6Kgu9c8OXRewOsZiQXZELKUs+FoUEYuAQzPzxojYAbgnMzfstkL1SUS8g7Lm6GzgbpYf5n28i7r6ICJmA4dQRmETuAe4ODOXdVqYeiUi/hv4reGLZiPiQOBvMnPrbiqT1o71Xv9bpoX7ePVY8u8DRzWLbo8BvG9Nw9ajTLteDjxAORn6UcoJ6492V1a3ImIX4EHgLyh3H+1JmbJ+ICLe3mVt6p1NaV9g+zDlfj7pZyLigIi4KiJ+GBFvbto+GRH7dV3bioxLODoL2Kr5/BTgl4FHKHe5TKsjyzUSXwMWAx+hhIDdm4/5zZ+T6izge8C2mbkgMxdQdn/eSQlJ0sCdQNtZRsdT3qBKAETEIcA/Ut547QDMah7NAE7sqq7XMxbTasOao8p3Bh7PzCe7rkf9EhHPA+/x6oPXavplfmbeM9T+LuCWzGw79mAiuCvrtSJiIXANZWR+cNP6nsAbgQMy88YV/V1Nloi4Ezg1M/+hOYV/18x8JCJ2Ba7NzC07LrHVWIwcRcRnm0AElEP8mu2Bz0XEZzssTf10G9N0keA69gLtUyKbNs8mTkTMbK7KeIoyWnIX8FREnN7s2JpIzdlxbwO+DmzcfHwd2MlgpCFvBW5uaX8W6O2VROOyW+1zlLMUhk823rB5dsrIK1KfnQucGRFfZhqduzECVwJ/HRGf4tXRgL2AvwKu6Kyqbp2Ou7KWExHXAtdRRo8+n5kvv85f0eR6ghKkHxtqX0iPD4Yci2m15uLZLTNzyVD7Byk7bbbopjL10dBFxcN6exHiuhYRm1HWY32Ecm4YlHUB/wwckZlPd1VbV9yV1S4ivkg5QmU+5c3FzcD1zcdthiUNRMSJwBHAJ4F/AT4MbA+cQQnWf9lddSs2rcNRM3+ZlCsgnmf5U7HnAOdl5jEdlKeeanYyrlBmDr/DmSgRsSMw2J12b2Y+1GU9XYqIn1DWp90/1L4z8L3M3KCbyvohIjYA3gd8oPnYA3ghM3s7XaLRi4gvUQ5kntM0LQPOyMw/rr5nG+CJzFzZm9eRme7h6BOUk1m/Cnwa+HH1+EVgUWa2zXVqwkXETMrOtOG7+DIz/66bqroVEV9dwaOkrDl6CLhkOt2svaYi4hbg9uE3WBFxLiU07dVNZf0QEVtSQtG+lBPWtwFu9e41DWvWBe9CmY7+YWY+O/R8KeXf1CNd1DdsWoejgYg4BvhOZt7VfP0h4BOUA+xOz8xXVvb3NVmad/1XUhZlB2UKaSZlemDZpL7rjYgrKetpfko5HBPgnZQ+uh14B2Xh7YLMnIjt2u7KahcRX6GEou2AW4EbKFNqt3hgqFZHvZOt61pgTHarAYdRfnHTHDD1DWBzyiGQX+ywLvXTmZQX+00p07FvB+ZRzmf51Q7r6tpNwDeBbTJzYWYupIwEXANcS3khvBr4cncljtwiymLSSxnalQVM7EnqlAXqc4HTKGfVnJKZNxiMNC7GZeToaWD3zHwgIj4DHJSZ+0TEPsCFmbl9txWqTyLif4G9M/PuiPgx5f+d+yNib+CczHx3xyV2IiL+C9h3+ELe5uTsb2fm1hGxG/CtzJzbSZEjFhGvAFtn5uKh9rnA4glevP8WXl1ntDfwc5TdfNcB10/wjk+tJkeO1o0ZlDVGAPtR3ulC2SbYywOm1Kng1WMflgBvaj7/EeWgv0m1MdC2+2qr5hnAUsbnCJCpCF670WNgYyb07CeAzHw4My/IzMMyc1vKkQ9LKCNJ3+22OmnNjcsvubuB34mIqyjh6A+a9jcBnpCtYXcDu1KumLkNOKkZIfgUZdHxpLocuKDZejt4gZtPOevnsubr3Sn30Y21iDi7+TSBU5vTwwdmUPphItZdtYmI9ShT0ftQRo9+ibIT6XbK2iNpVfVqGmtcwtFJlHVGJwBfGyzMBg6ivPhNlIi4Ajg0M5c2n69QZh40orL65EuU4x8A/oiyjuY6SpD+9a6K6oGjKJfOXsSrvxtepuwGHRx2eC8lRI67dzV/BmVN2ovVsxeBOyjntEyqp4HZlH64nrKO78bMfK7LovosIu4F3pqZ4/K6u7ZF1wXUxmLNEUBEzAA2ycynqrbtgeeH1wuMu4i4EDguM59pPl+hzDxiRGX1WkRsDjyV4/IPYg1ExEbAW5ovH57kF7zm38/xmbm061r6JCJ+BcPQKomIY4G5mfknXdfSR81mqif6srt8bMKRJEnS2jAuC7IlSZLWCsORJElSZWzDUUQc2XUNfWOftLNf2tkv7eyX5dkn7eyXdtOhX8Y2HAG97/wO2Cft7Jd29ks7+2V59kk7+6Vd7/tlnMORJEnSKluru9XWj9k552fHx3TrJZYxi9ldl9Er9kk7+6Wd/dLOflmefdLOfmnXp355hqeezMwthtvX6mFUc9iIPWK/tfkjJUmS1olv5aWPtbU7rSZJklQxHEmSJFUMR5IkSRXDkSRJUsVwJEmSVDEcSZIkVQxHkiRJFcORJElSxXAkSZJUMRxJkiRVDEeSJEkVw5EkSVLFcCRJklQxHEmSJFUMR5IkSRXDkSRJUsVwJEmSVDEcSZIkVQxHkiRJFcORJElSxXAkSZJUMRxJkiRVDEeSJEkVw5EkSVLFcCRJklQxHEmSJFUMR5IkSRXDkSRJUsVwJEmSVDEcSZIkVQxHkiRJFcORJElSxXAkSZJUmbmmPyAijgSOBJjDhmtckCRJUpfWeOQoM8/PzHmZOW8Ws9dGTZIkSZ1xWk2SJKkypXAUEcdGxH3ruhhJkqSuTXXk6A3ATuuyEEmSpD6YUjjKzM9nZqzrYiRJkrrmmiNJkqSK4UiSJKliOJIkSaoYjiRJkiqGI0mSpIrhSJIkqWI4kiRJqhiOJEmSKoYjSZKkiuFIkiSpYjiSJEmqGI4kSZIqhiNJkqSK4UiSJKliOJIkSaoYjiRJkiqGI0mSpIrhSJIkqWI4kiRJqhiOJEmSKoYjSZKkiuFIkiSpYjiSJEmqGI4kSZIqhiNJkqSK4UiSJKliOJIkSaoYjiRJkiqGI0mSpIrhSJIkqWI4kiRJqhiOJEmSKoYjSZKkiuFIkiSpYjiSJEmqGI4kSZIqhiNJkqSK4UiSJKliOJIkSaoYjiRJkiqGI0mSpIrhSJIkqWI4kiRJqhiOJEmSKoYjSZKkiuFIkiSpYjiSJEmqGI4kSZIqhiNJkqSK4UiSJKliOJIkSaoYjiRJkiqGI0mSpIrhSJIkqWI4kiRJqhiOJEmSKoYjSZKkiuFIkiSpYjiSJEmqGI4kSZIqhiNJkqSK4UiSJKliOJIkSarMXNMfEBFHAkcCzGHDNS5IkiSpS2s8cpSZ52fmvMycN4vZa6MmSZKkzjitJkmSVDEcSZIkVQxHkiRJFcORJElSxXAkSZJUMRxJkiRVDEeSJEkVw5EkSVLFcCRJklQxHEmSJFUMR5IkSRXDkSRJUsVwJEmSVDEcSZIkVQxHkiRJFcORJElSxXAkSZJUMRxJkiRVDEeSJEkVw5EkSVLFcCRJklQxHEmSJFUMR5IkSRXDkSRJUsVwJEmSVDEcSZIkVQxHkiRJFcORJElSxXAkSZJUMRxJkiRVDEeSJEkVw5EkSVLFcCRJklSJzFx7PyxiCfDYWvuBa+YNwJNdF9Ez9kk7+6Wd/dLOflmefdLOfmnXp37ZLjO3GG5cq+GoTyLi3zNzXtd19Il90s5+aWe/tLNflmeftLNf2k2HfnFaTZIkqWI4kiRJqoxzODq/6wJ6yD5pZ7+0s1/a2S/Ls0/a2S/tet8vY7vmSJIkaXWM88iRJEnSKjMcSZIkVQxHkiRJFcORJElSxXAkSZJU+X/0U38gS2d3uAAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 720x720 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    }
  ]
}