16_reinforcement_learning.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "hide_input": false,
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.7.5"
    },
    "toc": {
      "base_numbering": 1,
      "nav_menu": {},
      "number_sections": true,
      "sideBar": true,
      "skip_h1_title": false,
      "title_cell": "Table of Contents",
      "title_sidebar": "Contents",
      "toc_cell": false,
      "toc_position": {},
      "toc_section_display": true,
      "toc_window_display": false
    },
    "varInspector": {
      "cols": {
        "lenName": 16,
        "lenType": 16,
        "lenVar": 40
      },
      "kernels_config": {
        "python": {
          "delete_cmd_postfix": "",
          "delete_cmd_prefix": "del ",
          "library": "var_list.py",
          "varRefreshCmd": "print(var_dic_list())"
        },
        "r": {
          "delete_cmd_postfix": ") ",
          "delete_cmd_prefix": "rm(",
          "library": "var_list.r",
          "varRefreshCmd": "cat(var_dic_list()) "
        }
      },
      "types_to_exclude": [
        "module",
        "function",
        "builtin_function_or_method",
        "instance",
        "_Feature"
      ],
      "window_display": false
    },
    "colab": {
      "name": "16_reinforcement_learning.ipynb",
      "provenance": [],
      "collapsed_sections": []
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "UlG5vv_NuM7o"
      },
      "source": [
        "# Aprendizado por reforço"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "C6aVQmXNuM7z"
      },
      "source": [
        "### Bibliotecas básicas"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "ExecuteTime": {
          "end_time": "2021-02-15T12:42:14.718459Z",
          "start_time": "2021-02-15T12:42:14.451446Z"
        },
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "SDw5EDOquM70",
        "outputId": "cb6fdf1e-5e9e-4463-911e-ddbb49648df1"
      },
      "source": [
        "import numpy as np\n",
        "import pandas as pd\n",
        "from functools import partial\n",
        "\n",
        "%matplotlib nbagg\n",
        "import matplotlib\n",
        "import matplotlib.animation as animation\n",
        "import matplotlib.pyplot as plt\n",
        "plt.rcParams['axes.labelsize'] = 14\n",
        "plt.rcParams['xtick.labelsize'] = 12\n",
        "plt.rcParams['ytick.labelsize'] = 12\n",
        "\n",
        "import warnings\n",
        "warnings.filterwarnings('ignore')\n",
        "\n",
        "import gym\n",
        "\n",
        "import tensorflow as tf\n",
        "print(tf.__version__)"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "2.4.1\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "ExecuteTime": {
          "end_time": "2021-02-15T12:30:19.961914Z",
          "start_time": "2021-02-15T12:30:19.958384Z"
        },
        "id": "KETDAJUeuM73"
      },
      "source": [
        "tf.compat.v1.disable_eager_execution()"
      ],
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "RuOdKcJluM74"
      },
      "source": [
        "### Introdução OpenAI gym"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "ExecuteTime": {
          "end_time": "2021-02-15T12:30:20.662502Z",
          "start_time": "2021-02-15T12:30:19.967844Z"
        },
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "8bz45b6IuM74",
        "outputId": "62d37efa-7acf-4a96-a47f-42e2df7d376c"
      },
      "source": [
        "env = gym.make('MsPacman-v0')\n",
        "observacao = env.reset()\n",
        "print(observacao.shape, '\\n')\n",
        "img = env.render(mode='rgb_array')"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "(210, 160, 3) \n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "ExecuteTime": {
          "end_time": "2021-02-15T12:30:30.382224Z",
          "start_time": "2021-02-15T12:30:30.201601Z"
        },
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 357
        },
        "id": "Ftgb5o1fuM75",
        "outputId": "edca10a7-2b0d-4b14-f795-58c9d0f95b4b"
      },
      "source": [
        "plt.figure(figsize=(10, 6))\n",
        "plt.imshow(img)\n",
        "plt.axis('off')\n",
        "plt.show()"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAQYAAAFUCAYAAADGc5LAAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO2deXgcxZn/v9Xdc2lGGt0+5ANjYxtsA+YMdiCBhBAScmcJ2c1Blg1Jfpuw5CDkZDeEBEjClYPlSAg5yRKWhCyEgLEBA+YwGHzfl2Td0mhmNHdPd/3+GBvjeauFZ9TS6Hg/z8MDlKR5u2qqvl3d9da3hJQSDMMwb0Sr9AUwDDP2YGFgGIbAwsAwDIGFgWEYAgsDwzAEFgaGYQjGUD/83Kmn8lomw0xg7nzlFaEqH1IYrjn7bFImJWDKwr8nKh4N0BTNlbcBawLXe6KjC8BQzJFtCZj26F/PaCEE4BGFfx8tQwqDE1eurcXGqKecPx0X3HZ6FKfUm6T8l7uC+MPeqgpcEeMGnzo2hc8elyTl6yIefPnl2gpc0ehwUp2Jn50eLelvyhKGzrSOvYmy/nQcIJHJq6V1IKtN4HpPfCJZ9featgT2JnQAJdxSxxHNfqvkv+GXjwzDEFgYGIYhsDAwDENgYWAYhsDCwDAMwdVX7HVeG2EPXRBO5gV6szoNLiRaqizyLlgC6EjpMCV9S9zksxA0aDJB3BSI5GgMnyYxLUDfytoADqR02IoYpVKl22j203rnJdCe0iEVNZwesOHVaD36shoSearX1YaNBh+NkbMFOtIait+oC0jMqLKgK6rXldGRsY6+3gHdxpQRrl/IsNFYYv1aqiwYimp0ZzSkreHf8/SD/VP1SR1pHTmbBq/3Wqjx0Hon8gJ9ijHgERLTHcZAe0pHXtE/m/0WqnQaI2ZqGMi5c693VRi+MD+By+bRdeK/t/vxpbV1pLylysKj7+iDr6gD5WyB965qxB7F0uC3lwzifTPSpPw3e4K4dkMNKT8+bOKBt/WTho+ZGs5/ohH9ii+rVJY15XDnWwZI+YGUjgtWNiFTpEuGAH55VgQLavLkb65aF8aDrTRX4qIZGVx3coyUb4p68KGnGlA8pKoMif85J4IpiqWqT6+px7M9vqEr9QbOaDRxz1kRUt6V1vGulY1IFi3vagDueMsAFoVpLsi3Xg3jf/bT+l0wPYMfnULrty3uwfufbCCJZX5d4r6zI5iuEP1/e74OT3b736RWb06d18ZDb+9HTdHNzpbAR1Y3YMOAl/zNFQsT+OSxKVL+UFsAX3mF5krMClp45Lw+eIrGQMYSePfKRrSl6Bj4rxPjuGB6hpTfvTOIGzbTMVAOrgqDoQE+xTjzOIiYAODVJPkbobgHHY5Bfx8ozD6UMQTgVWQy+lxMYdSEut5erVAX1fq416GtnPReE+p6q+7KgHPbSln686MGWahLUTWcYkOoYwNQzmAOlZdSv0M/U7ahS+kITm1oD9GGzmPAuX/6dEnGiIR0zFT0OMTQXXwx4KowpPMCAzlam+I7yiFsCcRyGtJF0yLTFo6px0mHGGmHqXHeBqI5QRp50NRcS+s2bSivKW6qJU4CGDTV9VBNTw+Vq35/0FT3BgkgbmrwKaac+RLrbUqBqEljx5zaUBauS3W9WYfUY+f6OY1ygZipoSpHL8B0aMNSkSjU0S6aj0kJ5RQfAFKOY0D9PVkSiOY0GEXCkbUE7FLHgMM4KwcxlOdj+5VXkh9KCXz46Qas7afTqCrdhl/REXO2UD5XapAIe+XBu+obYkAgZgrl83/IUD+7ZiyBlOK50hCSTAUPxYjmVANX4v6zI1jWnCN/c/3GavxiR4iUezSJaoPGsF8fUDRG2COhK2Y5ibymFAefJhFUxMhLgbhDjFqvhAYaY9DUlO9vnPAIiWpFGw5VvxqPVM7iknkNWUX9vJpESFE/Sxb6grJ+HglNEcOpfl9aMIirFydI+TM9Xnz8mXoSQxxsw+L+CRQEw1LEcBoDWVsoxUEThXpANQZyArbixlLqGDizMYsHzokoZyAtt95a+iaqUklZGlIlZF/aUCvfUKgEZijyUv1S0k1Mu9QYhzr70ZO1BbIlxoiW2LZOmCW34SGxOnpyZbShahbjJrKM/lnyGJACkREeA+XAy5UMwxBYGBiGIbAwMAxDYGFgGIbAwsAwDKGiriM+TeKEsFmS5ZQTvRlNmSU2NpE4PpxHQLGsxRxJ2hLYGjMwXkxUZlbl0aRIHy8VWwJbYh7l0vWxoTxqvTRGd0ZDu0tjoKIjaVrAwv3n9CuzuErl3t1VuGZ9ePgfNAoYAvjp6VFlSjRzJNtiBi5c1ThuvDYvPy6JT8+lKdGlkrWAdzzRhNYkHaLfWDyIdytSou/cEcQPNo3BlOhyEMK9FNbxhMDkrHepuDGbHE3c6s9D1dux77jYVvyOgWEYAgsDwzAEFgaGYQgsDAzDEFgYGIYhVHxVYqRp9Fl4bwtd2snZAn9pC5RkcebEzKo8zpuaJeWJvMBf2wLK7bnM8DCExAdnppU2fyu7fDjgwnp+QC/EUG1xfqTdr7RqmyhMeGGYGbTw/ZPjZHknlhN4vNOHjDX8L/f4cB4/WBon5W1JHX9v9zuayDDl49Ekrlo0iJYqmuhz4Lk6V4QhZNj4zpI4wt4jhcGWwPoBz4QWBn6UYBiGMOFnDLGchsc7fST3I5UXjjZqpdKT0fBYBzVX7cuqXX6Y4WNJgdXdPtQrnKV7s+7c77K2wKouH6qKHlekLDg4VYrXBjxK56odcfeG84QXhj0JA//2fP2IxnhtwIvLRjgGcyQ5W+CqdSN7QnXc1JTu5pXmF9upvaDb8KMEwzAEFgaGYQgsDAzDEFgYGIYhsDAwDENgYWAYhlDR5cqBnIabt1QrT2QqlfWKA0bL5b59VXimh37eC33uxLAkcM/uIBq8JZxMMknpz+qOR7WVyvN9Pty4iZa7aQm4otOP7vTw77eWFIi6dHJ1OVRUGGKmhtsVR75VlsIeipFEQuCPe+mJz8zI8nK/Fy8rjlZ0k6e7fXi6++hPEh+r8KMEwzAEFgaGYQgsDAzDEFgYGIYhsDAwDENweVViqHUlp+3HTn9T6u9XMoab11TJGE5MlPqNxb4zWjFKw1Vh+MScFN6jsFF7sc+L27ZVk/Ipfgs/OiUGT5F1Vl4KXL0ujM40dcj54oIkljVRG7XHOvz4zZ4gKZ9Xncf3ToqR5krmBb76Si3iZF+9xHeWDOKEsEk+60/7qvC3A3Qp89R6E189YZCU92Z1XPVKmPg+6ELih0tjmFlF8xju2BHC6h663PWOqVlcNi9JyvcmDHzntRrIohr6dYmbTo2iTnGU2fWbqrExevTLdifVmbh6Ea1fJKvha+tqiT2eBonrTo7jmBA9aeuXu4JY1eUn5W9rzuFz8xOkvDWp41uvhmEX1c+nSfz41CgaFX4MP95SjVcjtH4fnJnBxbPpKVGboh78cFM1igdV2GPjplOj1I8BwDWvhbE7QYfPpXNTeNc0Ogae6/Upt0tPD1i44ZQYjKJcnpwt8PV1YfRk6Bj48vEJnN6QI+UPHwjgj/vcWQZ3VRiOrbZwzhR6wXTwFQjoEsubs/AX1T1nAVUO5zouDJvKGLsG1VWp9tg4uzmntHYrFqRDnFhrYlkzjfGcYsACQIPPVl5TW1I/mLx1ZHCBgpgsDNOB82CrOulpakDdtnXewpAprokhJM5szGFq4MiBIyVwh7e0jKF6b6ENi09H6kxpyvpBAKc05LC4ltbvYYWwAsAUh/ptjRmFuEWXrB2sn8ra7Ve71GdHzqzKK2M4nfrk1STOasoprd2qPeoY86rVMfodzGOqDImzm7PwFP04YxXEXcUJYXWMzTGP8vfLgd8xMAxDcHXG8FCbH9ti9CPbUmrTzL6shqvXhaEXKbYtC3ZpKn6/pwqrFZllOx1mDPsTOr76Spg8SuRsgYTDTOb2HUE80ErvbBsG1Iq8KerBV16mB+omHezjLAn8eHM1woppvlNm3vO9PmWMgZwG1b0rYwl8b0ON8kTt7SVagG2NefDVV2jsVF4gqzC6lRK4aUu18jHGqX4v9XmV9YvmNGVKdM4W+P6GGjLNBwqnRKt4osuPbsXUXFUGAIOmhm+/FiYu0RKFRxwVD7YGlP1kv8Pvd2c0XPVKmMxoLek8y/j17iqs6KRjYJuL1m5CSudpZfuVV5IfSgl8+OkGrB3h1FKGYdzhzMYsHjgnonxkarn1VuWDFD9KMAxDYGFgGIbAwsAwDIGFgWEYAgsDwzCEstY3NCFLcl2SEiRzrVw0SOXbVVuCZP+5HaNUCvUG3EhTFZBkSWs84t73JKHBOTmppE9ysX86fU+V7J/l9JuyhOEnp8aQyh99tNU9PvxgY005oQhfXzSIcxUnSz/UFnDJDUrihlNiOKmOpkSXSldax+Uv1CGrTpIriQtbMviPhTRleLxx69YQHu0YvkOWTwN+edYAmv3Dt8d7NeLFN16tgRsC/v8WJPH+GWlSvqrLhxs3uzMGvr1kEG9tpmPACVWux5tRsjAIAcwJlfZl7FXklJfLjKCFRYpU2xf73PNPPMYhRqnUeOTBMwaH3+HqvLYr11Rp6kpMx3ZCExLza0xlSnSpRFz0VpwWUPcdN8+VnOVS/xwKfsfAMAyBhYFhGAILA8MwBBYGhmEILAwMwxAqeuCMLiQafXQFWaKwJduSw3+b7xESDQqXHxtAX0Zzbf26NArX5FGEjpoacUSaDAR0G2EPXbEwX99+PPptokGi0W+Tu6dEwb3KdKF/jsYYKIeKCkNLlYX/O7ef7Hc3beADTzW6ssx5fK2JP50dIQ0fNwUuXNmISE69T34k0QXw67MGcFwNXXL65qth/HWET8Iai7x7ehY/XBoj5TviBj78dAMsl46pK4V6n42Hz+1DdZFgSQl87Jn6kuzxnJgdtPDQ2/tgFKlP1hK46MkGHHDx+LxSqKgwaACChq20dnPrGUcXQMig2Wi2dCdrrhwECkknxR0OAPH+myx4NHV7lJOc4xYCQFDxPdkSxFyoXDQBhDySWLt5tMpmulZUGGwUXICK7wamDaUrUVkxZCFGsQikLDG02e4IIgGkLYGkInu0UlPHSpO3oWyPdAUfqw59T0bRxE5KuHbQri0L9S4WhqxViFMpKioM7SkdF6xsUj49Olm7lcqWmAfnrWgi5RIFW7RKYEngsufrYCgqPpCbnMLwjw4/XuijdmV5iYo8RgCF9wjve7JR2T97HWzXSqU1qeNdT6j7p1tjoBwqKgyWFEqLeDcxbYGOEY5ROkJpCz6ZSVkaUnSLQUWxMfL9My/HYv/k5UqGYRSwMDAMQ2BhYBiGwMLAMAyBhYFhGMKQqxKqU6XKod3hJKqxSmtKR70Lde/KaLBdykuI5jTXvo9KEjXdaQ8pBXYPGhg0h5/x0uZwStRYpT3lXl9ocSgf8tMvWNnoSvBKJmqUTuGUYbeyCdxag/97ux//6KCnRI833EoMytjAJ5+rd+WzCpc0fvJHrt1Q41rWrpMP1JDCMFmz8Ny6y7uJBM0QdYN313bhxKo4Kd+QrME/YlPdD+gaI9Me4wEbI5+1O/7npkxZ6LARNvJYForgXbU95OdBLY+XkvWI5Q1Y/Cpq0sHf+CRlji+FP817CeeGe5U/P6+mF/fNewnH+FKjfGXMWIBnDJOQd4a7cVJVHCE9j+cT9diZKdjuL62K4YRAHH8ZmI6UXXghd3ZNH47JprAy3lzJS2ZGGRaGSYfERbVdeEv1AADgyVgTHo5OhU/YEM37Mc+fwO96ZyGS98Cn2bjr2FfRY/qwMt6E8fSCjhkeLAwMGowcfnbMejR5Dh9icm64D/8+ZTcajRx6TLrrkZnYuCoMJ9bmlK5EB1I6XlRsqQ0aNs6fliXH3dlSYEWnD4n88F+B1HttnDs1Q8pNW+CxDj+yNjXVOqc5hybFCUeboh5sj3tI+bSAhWVN9GSgVF7g8U4/Wd0RkHjntCxqPHQN/uV+L/Yn6dcyJ5THKfU5Uh7NaVjZ5UPx3dwQEhdMz8Cv09fXUwJHxjWlhg2pMBZVxTHdU2irftOL11K1WB7qx9SAhY/Molsf05bA4x1+5BX1O29qFrVeWr91Ea/SmWt2MI/TGmj9YqaGlZ0+crybLiTeNS2jNHJ5rteHLsWOxYU1JhbV0hPGejI6nunxorgNfVqhDT1FDmMSwJNdfle27VcbNt45LXvwYKLDWAfHQFIxBt7SmEVLFe2fO+IebIzS/lkOrgrDB2dlcPlxSVL+8AG/UhiafDZ+cmpU6eB0/hNNSCSG3/CzQ3ncclqMuOHEcgLP9XqRzdIO9MUFCSxrpp30+o3VSmFYUmvittOpLVlbUsdT3T5iNqIL4OpFg1gYpiJ65dqwUhiWNWVx4yl0WXHjgIEnu3zE2MavS3zvpDimFomAlBJ9B0xkD75TFAIYtAzc2DEf/9a8D5c0HICAxGupMDamanDP3FdwYmMeZ58QhShaPO9MaXi624d8kcGKEMBViwaxWHFa0tdfCSuF4czGHG4+jbbh1lihfsVLk15N4r9OiitPovr0c3VKYTh/WgZXL6bH/D3T48UzPTQnosZj4/qlMYS91MHp/U8aGMgN39ptSsDGzadFiVFLxgLOW9GkFIbL5qVwYQu92f33juDYFAZm/HFp0358sK4DANDsycKvWfjJ7E0wpYAAMMuXBjD5PCgnO64Kw96Ejmd7qIpuczi3L2MJPN/rI1O1vO2epdegKbCml15TMi9gkseIApuiHqW13AGH1O5IVlPWuzejdvmVAF6NeNCncAFyMnDpTqvbdm/CUOa6WFJgbb8XdYrp/JSc9vpQb/Fm0OI98u4zP3DkXXUgp2FbL53xRZxcjCXwWsSLqGKq3eXgStSdUbdha1JXZs7aUuDlfi/2Jmj9nKb4B1KGMsZmh7usaQu82OcljysSwKALj7lA4XFzTa+XeEiatnB0C98aM1DtofXYl3AvtVvIIfKVxcwbS8yvGurXVZV8s4+nf/PzMwbwwZl0GnXPripcsz7sSozRqIe7MUr7/dtmb8CZB1cljoY1g/X4yv4lrsWfGG2u/pvrTo7h0rk09+MvrX58aW2dKzFKr4czsu1q5R+4/ChR6l1+NJa/yokxGvUY6RhOvy+HkU1byjVUqn6VjlEqo1GP0uHMR4ZhCCwMDMMQWBgmKd2mDzd1zMPGVA3iloHbOudizWA9crbAHd1z8Fi0GbYEftM7Cw9GpoOzHicXvFw5CUnYBlqzATwQaUGLNwOvsPGXyHToQmK2L4X/G5iK04JRnBKM4rHoFOzJBit9ycwow8Iw6RD4QfsCCBTebf939xwYQiIjNdzf34K/RqYjaetYFW/Cc4P1SNncRSYjQ37rX5hPs8TKYfeggcc7R9Z9aEmt6dr1usWgKfCnfVUkZbgcTgibeNsUmnY98rgb86luH7bGhp+dZwiJjx+TQkhx3mUlWaxIuXabC6ZncGzIyXvJHYYUhm8vGXQlyMMH/CMuDKc3mji9ceS/lFJoS+r439YA8i4kay2tz7n2fVSSWE5zRRg8msQXFyaUKdETnY/OSitTot2EXz4yDENgYWAYhsDCwDAMgYWBYRgCCwPDMAQWBoZhCBXNXmnyWfjPk+LwFK3m5WXhtJ1uB2+C8Y4mJL67ZBDTA9Se697dVXhe4XblxP6Ejh9uqh7p80dKQgPwrSWDmBWk9XNiWVMWnz6WblduT+m4bmN14ZCVCchUv4VrToxTPwYJ/Nf6GvQpHMZGg4oKQ9CQePf0jNLa7eYt1eiuzGWNOBqAs5uzSmu3FZ2lGa/GTYFH2/1jauBoopBjUAozqiy8dwZdm98aMyA2VY/4yUuVIuSRuLAlo7R2+9Hmarfzy44afpRgGIZQ0RlDJKfhuo01MIpudpaE0vZsomBJ4PYdQdR76W1w/YA7Zp7jjVcjHvzX+hpSHskK1w7CHYv0ZjRcu6GGPErkJTBQwTFQUWGImxru3T35du5JCDzYWlXpyxhT7Bz0YOfg5BPFmKnh12NwDEzc2zLDMGXDwsAwDIGFgWEYAgsDwzAEFgaGYQhDrkoMcRaNI2KE82ykLO+6KsE4ucxxi4TgvvDGGC4GGVIYPvx0Q0kf9pm5SbxfcUqUm9yytRq/3TP2lndUZC0g69JRe8yRZC2By1+ohW+czHn7RyEn4aE2P35T4th4yaF8SGFY21/aab4jbTcFAHsSBvaMLWtHpgLYENgwMPzTpicSnRm95DHrxDjRW4ZhRhMWBoZhCCwMDMMQWBgYhiGwMDAMQ6jo7koNEtUeSSxGJAqnOKnMR6p0G16FnGVtIG3RH+hCotqgC7wSBZMTVfSQIclWcABIWwJZm/7AIySCihj2wRj0QNjCNRVvtQWAlCWQU8SY6Hg1iSqdtqElgcG8ug1rPFJ5Z0vkhfL0L58mEVDEMCWQVMQQB2Oovo3BvICliBHQbeUSas4GUor+Wc4YGA0qKgwtVRb+92398BU5OJk28NGnG7AvSS/vmhMHlcui9+0N4IbNdD//CeE8frc8QhKv4qbAB55sQCRHrbNuPS2qPNXqtq0h3KPYIru8OYvbTo+R8o6Uhg893YhMkcOZLoB7lw9gXjV1cPruazX424EAKZ/oXDg9g2tPjpPynXEDH3umHlbRePbrwP3n9GNagJ5E9aWXarG6hzphffLYJL60MEnKX+zz4vIXakl5vc/GQ2/vR3XRMXhSAp94rh6bonSb+JXHJ3DJMWlS/vd2P775apiUzw5ZeOCcfhhFmpG1CnlEB1KVGaIVFQZdFBpfZe2mupsCQMhjo8FHO0OV4o4NAIYmUe+zoRV9niGEY5ZmjUcqY6juNgDg1aD8/VReQECC3omAsEM9vNo4SeVzGZ+ubvMer/oIOgGJOq+6DT0ObRhwiFHtUcfQANR5bYSLDHVsWTg7U0XQUMcIGeoYh8aAytqtuM+OJhUVBgnAtAW0ooTRnC0cU0jztkBO4TGqmtYBBXU3bToRHWq6bko4xFD/vu3w+6YNxaPK4Z+p/ma0T2LUAQiFQkopcfRWrsNnqDZ0wnToC059x5bq388P1RdsgVzRF28PESPv8L06HWws5cH+LhVjoIL3iIoKQ0dKx3tXNSqfr9pSanfc6zdV42fbQqQ8ZqobfmvMg3c90UTKbQDRnPrd61Wv1CpnB/0Ov7+m14vzFTFMWZgSFpOXwOUv1CnflfRkRvd98GWzm3FuI30EW9UXx937e0btOh7r8OPVCM3ay9lqQc5YAv/ybL3yXVBnWt2Gf9xXhX900MOV0w5p65Gchg893aB8j3HAoX/+fHsIv1OkJRfek1BaUzouXKkeA53pyrmkV1QYTCmwJ1HaJXRn9JLco7O2wO6SYgh0lPiFpCwNuxOlDGiBtgo9Ox4ipGs4pTaIxTUBzK6iz+OLqwM4p6Ear0STSFojP48ZzGsYLKENJQT2K95BDcVATsOAg7irsKTA3hL7Z19WR18Jzs6mXfoYGA3G3hUxo8J0vxc/OH4mNIBMYwHgtNogltYGcdmre7AzOfJ7YJixBQvDJOTf50zBqeEgNAB/ONCP5wcGAQDvnVKH8xprcO32dsTyeQgAn5zZiK6Midv3TdRTPhgVnOA0CZkX9GNhdQBCCLSms9gYTyNjSViy8M+mwRT2pbLIS2BByI+5wdIOwWHGPzxjYFDv0fHTJbNRpWtIHXyfsLy+Gt84bjoEgAPpXGUvkBl1eMbAIJa3cP3ODjzdP/h62auxJL6/ox3dWZroxUx8hpwxzArSzLyhqHFIFHGTBp+lTD8ei+Rtgc605pjLMBao9Rho8BrYlkjjpJoqaEJgqs+DiJnH5nga6TF6DJSAxLSADWOcJIQl8wL9I3xAbY3HLnnMOjGkMKw8v7ekDys+tXok+M6SQVw0g6acjkXakzrevaqJpESPJT47uxn/OquQg2EIAV0APzvxGEhZSArzaAJdmbH3KOHXJf50dj+mVY3hxn0Df2sL4Kuv0LRrN/nY7DQ+PMudsTGkMATG4Cn0Hk2OyetS4dOhTIkeS3g0AU/R9flG2tHXJXz6+OkLo5HqbmjuvTTkdwwMwxBYGBiGIbAwMAxD4DyGSYqUEpYsbO0VAPlvIQp3DQvOu0qZiQvPGCYp+9M5XPrqbqzuH0TEzOPy9Xvw184BZGyJKzftwz37e2ED+O7WNty0u7PSl8uMMhWdMVTpNpY15YghhS0LW5lVVlgTAQGJs5pyCCnyMTZFPSXv7iyVTfEU9ho69qayWBdNYjBvYU8qi/XxFKo9OnYls9CFwLP9g9iRyIxKktP0gIXFtTTOYF7ghV7vmM4FGQ5Bw8ZZTTlyh7Yl8FyvV2lXOBpUVBia/TbueMuA0sHp/CeaStzKPH7QBXDtSXEsDNNklCvXhvFAa9WIxv9V6+H8lAc6I6//94reGFb0Fizq1kaTWBulNmgjxVubs7j5NGqPtzVm4N0rGyfs48y0gI273zKgdHA6b0UTWpOVGQMTc+QxDDMsKjpjyNoCGwc8RC3zsuD6PFGRALbHDWQUzkGlGIkAQMCQOLHOHFMna2tw9sd0IpLV8FqEmqvuS+rj5kTrcshYwPoBD3GiytlAroIHIldUGDrTmuOJ2hO4L8CSwBdfUqfHllrvuSELfzu3f/gX5TKlduknunxY2aXe3j2R+8KBlI4PPTX2xkCFlyudTV8nNu7VW4ixnHBdCtwXxhL8joFhGAILA8MwBBYGhmEILAwMwxBYGBiGIQy5KvHbPe5k4G0aoOvTbrMtZuClfnqSUSUZyGpDHn9WCjvjHte+j0qyc9CdhbC8FHiwNUDOlaw0ZzTklBmtbvJUtxe9WXfu6d90KB/yW/qW4nTescqaXi+uWT9+rrdUXur3jojwaUJCHHFAq4B18KxPTZOwbNUR9JXHtAVuVJxuXmmuOzk24sLwh730CLxyKUsYmInP59+xH+ctOpwglTY1/MdvFmHpMXF87h378fU/Ho99feN/psKUBgvDJCXky2PpnBhOmh3H/GmHN0tlTA1vXRBBXdBEV9SHE2fFEfLnsenA2Ls7MyMHv3ycpMxoyOCnn9qMs46Lvl4mJeAzbFx38Q7MaUrjyt8twife2o4vnL8fEz5VrC0AAB8MSURBVDsxmSmGZwzM6/zfumY8sakR3/voDixfEMHtn9mEe1fPwJ6eIMbiewZm5OAZwyQlY2rY2h7ClvYQdnZVwbaBjKkjlvZASoEp4RzOnDeA3d1BbOsIVfpymVGGZwyTlH29AXzi9qUAgCnhLP7y5Zfx0TM78ZEzOomjFjP5cFUY3j8jjbdNyZLy9QMe/HYPXWJp9Fm46oRBGEXzFksCP95Sjd7M8C3OZgXzuGJhgkyE05bADZuqkcgXT5okvjA/iXnVdMnpkXY/VnX5SfmisIl/nUfdjiI5DT/aXA2zKJdBg8RXT0hgaoCeonTfviq8rFiWPKsxi4/OpqcMHUjpuHVriFif+TSJbyweVB4beOfOIHbEPTh0+lw06cH3/3ocDE3CZ9i44t37UBM4XP+FNSY+exytXyyn4cbN1cgW1U9A4svHJ9CiOCXqz/sDeKGPbq8+vSGHS45JkfLOtI6bt4RgF9XPq0lcvWgQtV5av1/tCmJLjObOvHNqBhe2ZEj5zriBO3bSx6Vqw8bViweJt4QE8NOtIbSmhj98mv0WvnbCIPSiDmrawI82VyOSo2Pg0rlJLFHY4D3Z7cPDBwLDvibAZWE4ud7Ex46hnTdoSKUwhAyJj8xOK63d7twRQmkH5Klp8Nm4eHaa3AVjOYFbtoaQUCw5nzsli2XN9Fi2PYMGVnXR32+pspT1bksWOrVZ1OE0AVwwPaNc736+16sUhmOr88oYGwcM3LY1RF4NejSJ981IY2rgyIEjJfBQWwA74ofL0qaOR16dAgCo8ubx2fNaUfOG/jUtYOHi2WkUH1DVmdJw09YQFQYBvGt6Botraf1e6ffihT5SjDkhdf22xgzcsjVE3n3qQuKiGWm0VFFh+Hu7XykMx4fV/fOZHu9BYTgSvy7xoZlpkkRlS+D3e6rQSnWsZGo8Ev80O620drt9RwgRxemAy5tySoGL5DTXhIHfMTAMQ3B1xvBUlw/xHH1AdUqDjeY03La1Gro4UpEtWVA/N+hM6bhpS4g8SmRtgVRe/TB9//4A1vTSu/aLDpmHuwYN/GQzfUEXNzXyGAEU7ji/2V2FBh+9221W3OkAYMOAVxmjJ6MrFxJztsCdO4LKk8H3JY+cooV8eXx8WQc8ug2PIRHyHfkIsDdh4KYtNHYiryGnqJ+UwG/3BNHso48SG6Lq+m2KepT168tqUB24bdoCd+0Modqgbbgnoe5vL/b58JPNtLw1pX5kTeYFfr49BJ9GHyU6XXLy7s9quGVriNyh81Ig5jAGHmrzY3OU1vGViHuZsUIOYagnZt445havf37GAD44k06j7tlVNaFTot1GExJ+T2HgTgnncN8X16GqSKgsG7jkZ6dgeyevShwt150cw6Vz6TPGX1r9+NLaugpc0dDItquVd0delZikHNucwi8+swmakNA1Cb/iJR4zeWFhmKQYukRzTRa6wxPb3t4AXtkTRsyFN+/M+IO/9UmKlIBlCwghj1ixkbLwDuSVPWFc+5f5lbtApqLwqsQkZV9vAP/8i6V4asuR1uUZU8P/+/US3LVqVoWujBkL8IxhkpLN69jRGcLLe8PwveHNfsbUsKU9hFhq5M11mLELC8Mk5/fPtuD3z7YUlXJO9GRnaGu35ZGhfnzUvNTnxc+3j5clL4lvLh7E8YqsxPv3BfBw+/AzyzQhcd1JccwI0nX+u3YG8WyP+kSmUvDrEj86JYraEbQ+i2Q1fGNdGBkX7OvObs4q067bkjq++1oNSYkuh/fPSCvTyjdHDdy4uRrjRRCvWDiI0xpG9gTyIYXhvKl030M5OCUSjVWW1pnKlOgXFUlP5aABOKNR7Q34tza6F6McDCGxrClHUqLdpDOlQdck4IIwTAtYyv62NWYUUrFd0LfZwbwyhkcbc+k6Q7KkVl0PN+GXjwzDEFgYGIYhsDAwDENgYWAYhsDCwDAMoaJ5DAKSuOMcIm0J4kpUDhok/IoY8mAMN5aodCHJ1txCDIG0BVdiGELCq4hhQyDjUgz3kPDrhbYvJmcL5KUb11roO6pPytoC1gjHyFjClSXU0RgD5VBRYWipsvDHt0bgK2oY0xb4l2frsT85/Ms7PpzHr84aKDptCRg0NfzT6gYMKPwjSmV5Uw4/PjVKyjvSOj6+ugEZF1YML5qRwTcXx0n5tpgHl66pG1Pm7hqAu94ygAU1dK39h5tq8FDb8HNB/Drwp7MjSnu8r7xci+d6h58L0uC18T/nRFBdZI9nS4HLnq9TukSVyuyghd+/NUKWTLOWwCXPNKDDJd+HUqmoMBgCmF5lKa3dDJeE0qtLTK+yFNZuEppwZzj5dam0GLOlOChIw69M0LCVMSJZCy4t87uHKHgZqq436HB3LD2ExNSAOoZqhlgOmijkV6is3VSzt3IwtMINUmXtZlQwv6LiKdG2FLCKzGJcTck5uFuwuIlVrkDDCAFL8XnuxhAOMcbSI8RB5KHvVfkj13CK4SY26HcrFf1pOFgSKNYAW4qKqn1FhaEjreNDTzUQk1EpgTYHu61S2RY3cNGTjaTclgVrOTd4oc+L966iMUy78CzqBv9o92OD4tTwwrPu2MIGcMXaWuWdu92l7zVjCVy6po7caQFgf8KdGAM5DR9b3aC009896E6MtqSODz5F+46UQJcLLunlUlFhyNnC0ePQLdKWhk3RkV18GTRHPkYkp7nmgznyCOxy6bh7JyQEtsdHtu/kpXDlPcJQZG2BTQ4+mJVkvPQ0hmFGERYGhmEILAwMwxBYGBiGIbAwMAxDKPnVsZRANCdglrB+HjPdW2uP5zT0ZKie0cNpJzcShVOORvLk6r6shiHOK5qUDJpC2T/jpnv9M+YQwwmPJlHrkSQtYCjKWlP64to6rOs/+iWWUkTkzbh2Yw2u31RNylVHpU1mknmBjzzdMKKZ9jaAlEt5GhOFW7dW43aFjaGbY+C7r9XgeyV83GkNOfx2+UBJMcoShlReYLBCd+iMJZAZUxuGxioCiXFmqTcRyNqCnP7tNmmrtLFXjnjz/JthGAILA8MwBBYGhmEILAwMwxBYGBiGIbi6Be6YYB4tVdRRpy+rKXfC+XWJpXU5stZuS+C1AY/y7ev8ahNNfrrRuCOtY2+CVidk2DixziTrGHkJrOv3urKMVO+1cXyYuhVlLIFXIx5XLMCm+C3Mq6YH1CTyAusHPCg2g9GFxCn1ptJQZHPUg2gJ6+q1XhuLFPXL2gLrIh6FJ4TESXUmQgaNvWvQQLcL24k1FOpX7P4FAFtiHgy4sBPVo0mcUp8jpkESwIYBjzJ35thQHtMUrlK9GQ07BukYCOg2Tq43yR3alsC6iFe5wrGwxkSDj46B9pSOfS64ngEuC8On5qbw2Xn0mLFH2v34/It1pHyq38Lv3hqBr6hVTBs4/4km7E7Qhr9iYQIfmJkh5b/eXYVr1odJ+XE1efzp7AgZmjFT4G2PN6E/O/xOelpDDr86i64TH0jpOG9F40FvyeHxzmkZ3LCUWrttjBq4aFUj8WQI6BL/feYApihE9F+ercfqEo7BW1qnXgfvTGs4d0UTkkXLopoAblgaw+JaKmRXrwvjj/uqjjq2Ez5d4hdnDmC64qStS9fUYWXX8E/0qvXYuOesAdR4jhQfCeD9TzbgtQF6Mtln5iVx6bEpUv6XNj+uWEvHQEuVjT+8NQJPURfJ2sB5K5rQqhjoXzkhgQun0zFwx84gfrCx5k1qdXS4vmm+lOyqsv5GuBPD7ZXmcq7JjRhvFlZlguNW/KFiizK/p1KpRIyh2lAofv9Qeal/49bvl4OrwtCb0bAjTj+y08HQ0rQLhh5exYwh52BL1JXWlTF6HaanGUtgZ9wgDZkw3XISLkznVdfUldZcs16L5dRtuz9pKB3AbAnsTRiIm7QhS014SeY17FQYr/RmNLV9nSxcV/H3CriXHi+lwN6EgWSe1q94BlMulhTYnTDII5GUzs5cPSWOgZwN7IwbMDRabjokSnWk1TH6SkiTfjNcFYY7dwRx984gKXcS2Pa0hosUlmhA4R2Aius3VeMGRUq0k73ZtpiBC1aWFqNUnu/14l1PjGyMR9r9+EcHnR5LqNs3ZQl8/Jl65WeV6pO4tt9TUv1sAP/vxVrlXdItj8aMDXziWXfq50QkJ/DhpxpKivHz7SFlSrTTJbUmdbynxDFw7YYaZdu66THqqjDYECVenCh54JR6l5dlxCiVsRnDvWsqJ/ZIm7S6WT83Y9iyVA/OkR8D5cDLlQzDEFgYGIYhsDAwDENgYWAYhsDCwDAMoaIHzgQNG+dNzbqiTnsSOjZGaSbaWMSWwKouH7Yp1qLdOoHLEBLvmJaFfwTPP0xbAqu6fK6cXt2a1PHXNroc257Sx5V93Im1OcwJ0ZToUrEO9pFUiaYsblFRYWjy2bjltCg51LYc7tlVNX6EAQI/3ORO6qoTfl3iByfHMFWRMuwWnSkNb1/RhLwLCUUv9PnwQt/wT6iuNBcfk8alc2lKdKlkrEMp0ZURBn6UYBiGwMLAMAyBhYFhGAILA8MwBBYGhmEILAwMwxBcXa68YFoGb2nKkvItMQ/+vH/4rj3l0FKVx7/NS6F442vGEvj59hCSLhycs6DGxCXH0CWqaE7D7dtDxD5Og8Tn5ifR7Kfr3X9tC2C9whno1PocLpqRJuWdaR137wxCjqFDeAQkLj8uiakKi7P/OxDAugit38l1OXxgJq1fT0bHHTto/TyaxL8vSCDsocux9+2tUtqolUrIsPHFBQmFfZzAL3cG0e7gsTDSfGx2CgsVVntren1Y0Tl85yrAZWE4symHzx5HB8jDB/wVE4Zmv43L5iWJr2QsJ/DLXUEkqftYycwOWsp6tyV13LUzCLPI1EMTwEdmpbEwTINvjnqUwrAwbCpjbBww8MudQcf9/pVACOBDs9JKa7edcY9SGObX5JX12xozcNfOINnGbQiJjx+TQksVFYZne3yuCENAl/jksSmEvUcGtyXwUJu/YsLwzmlZXNhCrd3yUrgmDPwowTAMwdUZw/O9XmX66pZY5RIse9KFu3bxRDtjCaRdsgDbl9Rx5w7qXBXNCeQV9ly2BP68P4BmhVGryrILALbGPMoYnWltTM0WgIL12f/uD+C5Hlq/rQ712x43lPXrdrCPy9sCf9hbhbCH/rA16c6dPGUJ3Ls7CH/Ro4RE4RGnUjze4VPW8YVe9zJ/XR2xKzr9rk1l3KI9reM6l5xzndgR9+D7G49+6mpD4M6d1P5rKNZFvMop+FhEQuDuXaXVb/2AV/kI5YQpBX66jVr8uUkyr+HHW0Y2Rjn8uXXkH8v5UYJhGAILA8MwBBYGhmEILAwMwxBYGBiGIbAwMAxDqKiD03jjg7PSWFxHU1Hd5De7qyaEk1GpnNWYxadccD4aiuMUp4WPRy4/LoGl9bQfrujw48G2gCsxWBhK4PhwHscr0pjdZGXn5BMFAJgZtPC+GTTNl6Gc3mAqU6IPpHTAJWHgRwmGYQgsDAzDEFgYGIYhsDAwDENgYWAYhjDhVyUEJAyV/EnAlIXfGA9okNAV9ZASyI+5ekgYomDYUoxlF3aXjg8kPALKps3bGFOuWW4z4YVhYTiP288YIJ00YWr4xHP1iObGx5f73hkZfPWEQVK+I+7B516oHVOeDBqA/z5zAMfV0KXdn2yuxsPt7iypjTT1Xhu/XT6AUJF9nJQCX3ixFtviw3eJGqtMeGHw6xJzqy2FtZsNXYyl4TQ0NR4b86qph2I6LyBQ7GhZYUQhL0F1vTUKY5Wxii6AOaG80tqt2LxlojHhheHQiCl2lhqPX6vKHWus1kNK9fWOR8Za35FwaFsXL2zCC8POQQMXr64n5XkpEMuNn3evKzr92D1Iv65kXsPIHVtbHrYEvr4ujKBBe+qexPjpctGchkvX1CtnlrsU38Vo8ZMtIdyzi7o4dbhoTjt+vqUySeS1CbH3oCejV9RnsDTEuDl5fChMKbC2f+zVY8covNsYP7dMhmFGDRYGhmEILAwMwxBYGBiGIbAwMAxDqOiqhA0gbmrIFh9MWAYZy70MxmReIFahjEhTcXJVOUgAg6aGwAgm4sTzmmu5Cjm7cm2edOlEMqCQcOZGPbK2UJ7ANVpUVBgOpHSc/0SjK5/lpjD8x8u18GiV+VYSpjuTuFRe4MNPN0AbwexOWwqkXWr3R9v9eKanMkuDbokxANy6LYQ7dtKj9sphIFu5CX1lZwxSoD871tbmBeLm+Ng/MRQSAgPjZB8IULhDZsdcXyidZF5z5QT1SsPvGBiGIbAwMAxDYGFgGIbAwsAwDIGFgWEYgqurEtMDFpr81JwjmtOwP0lDeTWJBTUmMVGREtge9yCrWEaaFcyjzks3GvdldLQrtp1W6Tbm1eSJCZclBbbFDORl8U8k5oYs4toDAJ1p9Q7HGo+NOSH6KjpnF2JQCzCJBTV5pdlHa9LAgGI7eIPXwowgbdtUXmDnoIFi/zENEgvDeeWy6+5BA4n80d8Tqg0bxypOcTIP1o9atUnMr84joNh23ZbUEcnRNqzz2pgVpDHSlsCOuLp+C8J5eBX125swEFcs+zb7LUwL0DYcNDXsSegkhiEKbVi87VoC2BU3kLJojJaAhUbFGBjIaWhVjAHfwTFQ7DBmHxwDOcUYOCaYR1gxBnoyOjpd2nrtqjBcNi+Jzx6XJOWPtPvxhRfrSPn0gIUH39YPX1FdcjZwwRNN2K3Yu3/VCYP4wEx6Cs+vd1fhP9eHSfmCmjz++vZ+0vAxU+Dtjzcpl0t/sDSGZU05Un7DpmrcviNEys9szOFXZw2Q8raUjneuaCRr/YYAfn5GFAsU1mdfeTmMB1rpXvt3t2Rx/dIYKd8Y9eB9qxqIJ0OVIfGbZRFMCdAO9Iln67G65+i3op/akMNvl9P6daY1nLeiiSQIaQK49fQoFtfS+l29Loz79tH6vXNaBjedSuu3LWbgwlWNKM6B8+kS95w1gJYqOgg/s6YOK7v8pPzi2Wl8fRG1x3u2x4t/fpZ6dtR5bfzp7H7iOmVL4ANPNWD9AM27+Pz8BD6tOGrvr21+XLGWjoGZQQt/eXs/PEUak7EE3vlEo1JMvrlkEBdOp2Pgjh1B/HBTDSkvB1eFQQiQuz8wtE2p6m+GupeVHEOUHkNDaTGEw+8PWQ+Hv3FGOsRwTmBS1bucTMVDfqjF4jpkG5b4PTm1h8pQ9nAMdZs4IRzacKgYjtfl9PtljAFVWw2VmObYP11MW3FVGKI5gbYkvQP3O2Rw5SXQntLhLfqxaR9ycKb0ZzVljKiDG1POEjiQ0skXM2gK2OQxokCPQ4y4Q+ps2lLXuzOtQSpiSABdGQ1Bg/5NyiGTMJlXX1N3RldKgy0Ljz6qrL4MvckOSeZgGxbTk3FIiZZAd1pHWOHv6JR+nMyr27Arra6fhEBHWld+h05ZsIOmug17M+q+Y8mCK1K8qDPagHKKDxQeGUoZA6ZdyAA2ij4uawN5hxh9Dv3TTUcyIYe4hbRfeSX5oZTAh59uUDrbeISEoXjms6RQNqSAhE8v/PuIGBDIWmp7bq8mlVZbeVvAVHQSDRI+xbO8hDg4QOjzsVeDMoZpC8U7iYK6+xT1llIgY6tj+DT1XSFnC1iKGLqQyudpWwpkHWL4FW0LAFlLlGTh7tiGla6fBghFDKf6GUIq37k49c+h2jBjCWX/dBoDeSmUIi0Oxig2bHRzDJzZmMUD50SUM4qWW29VdgRXZwymFDBLyJ13HpzOFL7Ao/99G6Xm8wvklB1xiBgl7xlw6uzOWGXEKLVtnSinDUelfiXGyEuB/Ai3YTljID3CY6AceLmSYRgCCwPDMAQWBoZhCCwMDMMQWBgYhiGUtSpxWkMONYqUYYZhxh6qDNs3o2RhEAL49hKaVsowzMSBHyUYhiEMOWPY2NMzWtfBMEwFaHEoHzIlWtcqZJXMMMyoYNnqDRlDzhhstw4NOMgll0yDEMB993XioouasHDhYZvtdNrGPfccQDpd/ktNTQM+/ekWRCImHnqoBx/72FTMnHl4+200mse997Yjn2e9Y8rj+GAQFzYWjjxozWTwQHd3ha9oZBgV+3ifT6CmxoNFi0IQAmhq8uKMM8I466zD+9MTiTwefbQXPT05JBIlbv8DEAhoCIcNLFlSje7uLNas8WLZsjqccMJh/4Tu7iwefbQXkYg5LAFiJif1hoHZfj+WVlcDAEK6jiaPBxKAJSUG8hPAN/4gQz5KCNXWtTJYvrwWV199LK65ZiekBK699jj4fBp0/fAsRkqJbNbG3/7Wg7vvPlByjIsuasK//usMXHXVdrS0+PC1r82B13tkDNsuxLj33nY8+ODEVHpmZPAKgZ8tXIgdqRR+0dYGAFgUCuE7c+YAANoyGXx5+3aUfkurLFLlC4BRmjG0tWVw//1dOPHEajQ1eREIaFi9egD9/Tl84ANToOsCQgj4/To8xVY2R8nOnSk8+GA3li+vxaxZAfj9Gh57rA+WJfGe9zRBCAFNEwgEdBjFm98Z5k2wpMTf+/rQm8shYxdmm3nbhl/T8NTAAA5kMvjnadPwUiyG7Snq4DTeGBVhaG3N4Pe/78DNNy/EkiWFadhLL0Wxa1cKF13UDMuSrz/353LlTfG3b09i9+4U7rxzEWbNCkBKidWrB2CaNt7zniZkszasg/5g/I6BKRULwEO9vcqfPX1QGG5ZsACxfB77M5nXxWO8UrEj6j73uVmwLAmPR+APf+jAww8XGj2Tca9Br756zusOQ3ff3YZnny34FqZS423Cx4x1OrNZfH7rVnx0yhTcNH8+rtqxA6lxLA4VE4Y9e1Lo6MgCADZtSqC/33Q9xrZtydc/d/v25IjEYCYnbwmHMd3nw2P9/ejJ5eDXdZxaU4M5gQDqPR4INw0YK0DFhOGRR3rx1FOREY1x//1d2LCB07cZd9GFwMemTkXasvCtXbsgAMz2+3HFrFkwhMCAaeKQGf14fWit6GnXDDPeOL2mBp+ePh1/7OrC3oMvGS+fMQOn1dTgkD1rtWHgxvnz8Y++Psf3EmOdignDsccGkEgUPPDb2jLo7qbnOAyXBQuC8HoLU7o9e9KIRPhRghke1YaBuYEAmjwe9BkGOnM5TPV6MdN/OJHOEOL1R4rxSsWE4ZJLpuGSS6YBAO66qw0PPOBuXoEQAp/97IzX//+GG/Zg1aqRfXRhJgdCCHx+xgysTyTwjZ07K305I0LFhOGNL2cuvLAJS5cWZg9r1kTxyCPuTL/eGOPii6fhHe9oAAD84x99eOYZerISwzhhCIErZs3CvEAAQKFvHRsI4Ptz5+LFWAxr43H8+8yZ0IVAIp/Hba2t2JNOV/iqy2dUhaG9PYO6Og9aWnxHDNpZswKYNStw8HeyZX++lIWcCa9Xw9SpRx7BNnduFebOLRyNtn49v5BkSkMHcGIohKm+w/2qxjBwejiMR/r6sC+dxu5UCroQiOfzeDkeR5qXK4+Om2/eh0WLQrj55oUj8vmWJfH97+/C8uV1uOaaeSMSg2FUdOVyuHL79tf/f/xKQoFRNWqREjhwIINbbtmH7duPPPw2nbZwxx2teOqp/mHFsG1gx44kbrllH9rajjz4Mxo18bOf7cfLL9PDUxlmKEwp8euODjzW1/d6WVsmg9v273/9kcF+wz/jnVF/xxCN5vHoo32YNy+IhobDb20TCQuPP96HwcHhZyV2d+fw97/34tRTaxAIaEeUP/poL0yngzEZxgEbhdTnvJQ4tabwPmxPOo1H+4d3IxurjMruShVerzhi5yMA17dC+3watDfMiaR0N+WamXzoALwHO5UtJbIue5aMNk67KysmDAzDVB4nYWAzWIZhCCwMDMMQWBgYhiHwJiqmIpBdyXL87kSciLAwMKNObciD6z5zPPzewxPW5zZF8OvHWit4Vcwb4UcJZlSZPSWAU4+rxYKZIVQHDCTSFuZMDWLxnBqcNr8WVT79zT+EGXFYGJhR5eK3teA/P7UAfq+Ox17uwTd+uRkDCROnL6jFTz6/GDOaApW+RAacx8CMEtPqfbjiQ3Nx3Iwg/F4dN/15F3a1J9Hel8bSebVYtrge/3TOdGzYE8eru2L41aP7K33Jk4KK2sczTJXfwFmL6qFrAr3RLF7cOoBUtpD+/srOKJrrCjtuT5obRtbk7NRKw48SDMMQWBgYhiGwMDCjgmVJROI5ZHIWNE2grtqDwMHlytqQByG/DiklYkkT8dTEOQNyvMIvH5lRQRNAwKfjPz48Fxec3ox0xsLvVx7AA6vbcddXTsa0ej+8hoYrfrEB29sSyJR5IhlTGvzykakotgSSGQvPbOxHKmPh/cum4rT5tfDoAk1hHw70ZbBmUz/aezMsCmMAnjEwo05DjRf3fG0pqvyHk5keW9uDn/x5VwWvanLCfgzMmEETQFOt74j9EumshViS3y2MNiwMDMMQ2KiFYZijhoWBYRgCCwPDMAQWBoZhCCwMDMMQWBgYhiGwMDAMQ2BhYBiGwMLAMAyBhYFhGAILA8MwBBYGhmEILAwMwxBYGBiGIbAwMAxDYGFgGIbAwsAwDIGFgWEYAgsDwzAEFgaGYQgsDAzDEFgYGIYhsDAwDEMY8lwJhmEmJzxjYBiGwMLAMAyBhYFhGAILA8MwBBYGhmEILAwMwxD+P4QNUplRgsAyAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 720x432 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "sLMWLSo5uM76"
      },
      "source": [
        "#### Função auxiliar para traçar um ambiente"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "ExecuteTime": {
          "end_time": "2021-02-15T12:32:01.924040Z",
          "start_time": "2021-02-15T12:32:01.911677Z"
        },
        "id": "n3_nA9BXuM77"
      },
      "source": [
        "def plot_environment(env, figsize=(10, 6)):\n",
        "    plt.close()\n",
        "    plt.figure(figsize=figsize)\n",
        "    img = env.render(mode='rgb_array')\n",
        "    plt.imshow(img)\n",
        "    plt.axis('off')\n",
        "    plt.show()"
      ],
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "ExecuteTime": {
          "end_time": "2021-02-15T12:32:37.993626Z",
          "start_time": "2021-02-15T12:32:37.988643Z"
        },
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Ss0uJx_YuM77",
        "outputId": "83462cac-f950-4ce6-8412-f3ae7c3f0abd"
      },
      "source": [
        "print(f'Espaço de ação: {env.action_space}')"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Espaço de ação: Discrete(9)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "ExecuteTime": {
          "end_time": "2021-02-15T12:34:16.991052Z",
          "start_time": "2021-02-15T12:34:16.697837Z"
        },
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 357
        },
        "id": "ajDrB1XAuM78",
        "outputId": "75be475a-64ee-46c2-8831-9779070b9efb"
      },
      "source": [
        "env.reset()\n",
        "for step in range(110):\n",
        "    env.step(3) #esquerda\n",
        "for step in range(40):\n",
        "    env.step(8) #inferior esquerda\n",
        "    \n",
        "plot_environment(env)"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAQYAAAFUCAYAAADGc5LAAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO2deZxcVZn3f+feW1tXdVevSTqddMgCScgCgbAkyCKCiCJuM4CjDvg64izqy6jIuAwzo6jgwjI6DoIi+io6iigOsi9hC0tIQvaks/e+1r7e7bx/VBZS59wi1X27q7rzfD8fP5hTVf3cc+45v3vOPc95HsY5B0EQxFtRKn0BBEFUHyQMBEEIkDAQBCFAwkAQhAAJA0EQAiQMBEEIaKU+/MyZZ9JeJkFMYX6yfj2TlZcUhpvPP18o4xwweOG/UxWPAiiS5jJtwJrC9Z7qqAzQJHNkmwOGPfHXM1EwBnhY4b/HS0lhcOKGdfXYEvOM5qeTgrvOiuGMRkMo/+meIH69v6YCV0S4wd/Oy+DTJ6eF8g0RD/75jfoKXNHEcFqDgR+eFSvrN6MShr6siv2p0j8NhPyobQgK5UbeRHQwPhqzEwRHzpRLazSvvG29ieolkpff16zFsD+lAijjkTqJmOa3yv7NuPXy5tYGnLxirlAeH05g/bPVLAwEQbi+K6FqKpa/YzFmndwq/TwYrsEZFy9Fw7Sw26YJgnAJV4XBH/ShvrkW9c21UFQF8eEELLMwjeGcIxVLI5fOI9xUi3BzrXSpQRBE5XFVGGYtaMXy8xdD9agY7BrGxjXbkM/qRz7ftWEfdq3fBwCYu2Q2lpx7ChTZ63+CICqK60sJxhjYoX0R2+bo2Lgfnbt6j3yeSWax7ZUOxEeSbpsmCMIlxu3lo+bREAj5kU3lkAvlAAC+gA96zkAyloapmwgGPJgTssCtYzeROYDejAqDi7OJFp+FoCY6EyQMhoiuCuU+haM1IL6VtQF0Z1TYEhvlUqPamOYXN8JNDvRkVHDhbTfHzIANryLWYzivIGWKel2r2WjyiTZ0m6E3q6D4jToDx6waC6qkev05FTnr+OsdUG1MH+f6hTQbzWXWr63GgiapxkBOQdYa+zNPZQUbsr/Um1Wh26LxRq+FOo9Y75TJMJwX+6eHccyssSQtWGhbU9I/p/kt1KiijbihIKq786wfN2GYMacF09ubC/84VLdTz1lQqDEApjBMRxr3XjwMD44duLrN8L5nm7FPsjX4tWVJvH9WVij/xb4gvrG5TihfHDbw4IUjQsPHDQWXPt2MEcnNKpfVLTp+cm5UKO/OqLjsmRbkinRJY8BPV0WwsM4UfnPjhjAe6hR9Ja6YlcMtp4u7OVtjHnxoTROKh1SNxvE/F0QwXbJVde3aRrw06CtdqbdwdrOB+1ZFhPL+rIp3P9OMdNH2rgLg7nOjWBIWfUG+ujGM/zko1u+ymTl89wyxfjsTHlz5XJPgWOZXOX5zfgQzJaL/d6804LkB/9vU6u1p8Np4+KIR1HmObV2bAx95oQmbo17hN59flMIn5mWE8oe7AvjCetFXoj1o4S8XD8NTJKI5i+E9zzSjKyOOgX9fnsBlM3NC+b27g7h1mzgGRsO4CQNTGFixyhe5XjEOeFWO4uZlkmfQYTSFwycZyxqTuyQyBnglnow+F10YFQbpNXmVQl1k++NeRf4bJ71XmLzesqcyDln0StqK8/LXjwp4oS5F1XCyDSa3DUA6gzlcXk79Dn8mbUOXXls5taFdog01h/taPPCP2GCAT+XwFP1BDu7oqehxsKG6+GJg3ITBsmzYlgXNo4ExBs75kR0KVVPBGIPFgbiuQMOxjWbYzNH1OG0yRHWxxbIOU2PTBmI6Exo5aSiuuXUbNqTXlDDkEscBJA15PWTT08Plsu8nDXlv4AAShgKfZMpplllvgzPEDNF23KkNeeG6ZNebd3A9dq6f0yhniBsKanTxAgyHNiwXjkId7aL5GOeQTvEBIOPQP9OS5RNQcLGP6Qq0IuHIWwx2uWPAwTFvNLBSMR97brhB+JBz4MPPN2HdiDiNWnDaSWhfOLPw27392L+9G2e8cwlqQgFwzrH5pZ0A51h+/mIwxpBPZbHzuQ2SdwwMcYNJ1/8hTb52zVkMGcm6UmNcmAoethHTZQOX43fnR7B6mi785jtbavFfHSGh3KNw1GqiDfvIgBJthD0cqmSWkzIVqTj4FI6gxIbJGRIONuq9HApEG0lDkb6/ccLDOGolbViqfnUeLp3FpU0FeUn9vApHSFI/ixf6grR+Hg5FYsOpfp9bmMRNS1NC+YuDXnz0xUbBBjvUhkzShnFDgSWxUaPa8EvEOG8zqTgorFAPFNngYIjrDLbkwVLuGDinOY8HL4hIZyBtd95Z/iGq0WBZNvoPDELPGZg2qwnDPZEjS4hgXQAA0L27D02tDQAKbsZ2GQove3FVCpPLX0q6iWGXa+NwZz9+8jZDvkwbMclTZTQYZbfhYbE6fvRRtKFsFuMmHPIncykyloJMGR7INmeIlGmj3DEwGly3YJsW9m/rgmlYWLB8Dvr2D2LP5oPYs/kgmtsa0dLWhD2bDyIZEw+zEARRHbg+Y9C8Gs5451JoHhVMYVj+jsWwbRsAg7+msPw4+92nwxfwQM+Jb60Jgqg8rs8YuM2RSWaPDPpsKgfTsFBT60cimkIikkJNrR+ZRBaRgdiUjutAEJMV14XBNC3seH0PevcPgtscuzbsO+L5uG9LJ/Zu6QQAdHb0omPDflDCG4KoPlxdSvTs6cNQ9whMw8JQ9wiS0TTyWR2mYWLjmm1IxTMABzau2YZ0QnQCIQiiOnBVGLLpPLLpPAAgn9WPHKCyLRuxocSR7x3+/z6F49SwUVbIKSeGcorUS6w64VgcNhGQbGsRx5K1GHbENUyWICqza0y0SNzHy8XmwPa4R7p1PS9kot4r2hjIKehxaQxUdCS1Biz87oIRqRdXudy/twY3b5ocMR40BvznWTGpSzRxLDvjGi5/tnnSxNq8/uQ0rp0/9tlw3gLe9XQLOtPiEP2XpUm8R+IS/ZOOIL61tcpdoo8XxtxzYZ1MMJyY9S4XN2aTE4lb/blUvR37jottRXklCIIQIGEgCEKAhIEgCAESBoIgBEgYCIIQqPiuxHjT7LPwvjZxa0e3Gf7YFSgrxJkTs2tMXDwjL5SnTIY/dQWkx3OJsaExjg/OzkrD/D3T70O3C/v5AbVgQ3bE+S89fmmotqnClBeG2UEL3zw9IWzvxHWGJ/t8yFljv7mLwya+tSIhlHelVTza43cMIkOMHo/CceOSJNpqREef7pcbXBGGkGbj68sSCHuPFQabA5uiniktDLSUIAhCYMrPGOK6gif7fILvR8ZkjmHUymUwp+CJXjG46nBeHuWHGDsWZ3hhwIdGSWTpobw7z7u8zfBsvw81RcsVzgsRnCrFm1GPNHJVR8K94TzlhWFfSsPfvdI4rjbejHrxqXG2QRyLbjPcuGF8M1QnDAWfW9cwrjZGw3/tEsMLug0tJQiCECBhIAhCgISBIAgBEgaCIARIGAiCECBhIAhCoKLblVFdwe3ba6UZmcplkyTB6Gj5zYEavDgo/r1Xh92xYXHgvr1BNHnLyExygjKSVx1TtZXLK8M+3LZVLHczJOBTfX4MZMf+vLU4Q8ylzNWjoaLCEDcU/FiS8q2yFM5QjCccDA/sFzM+E+PLGyNevCFJregmzw/48PzA8WcSr1ZoKUEQhAAJA0EQAiQMBEEIkDAQBCFAwkAQhIDLuxKl9pWcjh87/abc71fShpvXVEkbTkyV+lVj35koG+XhqjB8fG4G75WEUXtt2Iu7dtYK5dP9Fr57RhyeotBZJme4aUMYfVkxQs5nF6axukUMo/ZErx+/2BcUyhfUmviP0+JCc6VNhi+ur0dCOFfP8fVlSZwaNoS/9dsDNfhzt7iVeWajgS+emhTKh/IqblwfFuI+qIzj2yvimF0j+jHc3RHCC4Pidte7ZuTxqQVpoXx/SsPX36wDL6qhX+X4wZkxNEhSmX1nay22xI5/2+60BgM3LRHrF8kr+NKGeiE8ngKOW05P4KSQmGnrp3uCeLbfL5RfOE3HZ05JCeWdaRVf3RiGXVQ/n8LxvTNjaJbEY/je9lpsjIj1++DsHK6aI2aJ2hrz4Ntba1E8qMIeGz84MybGYwBw85th7E2Jw+e6+Rm8u1UcAy8P+aTHpWcGLNx6RhxakS+PbjN8eUMYgzlxDPzz4hTOatKF8ke6A3jggDvb4K4Kw7xaCxdMFy9YHHwFAirHedPy8BfVXbeAGoe8jovChtTGnqS8KrUeG+dP06Wh3YoF6TDL6w2snibaeFkyYAGgyWdLr6krrR5y3jrWOENBTBaFxYHzUKfc6WlGQN62Dd7CkCmuicY4zmnWMSNw7MDhHLjbW57HUKO30IbF2ZH6Moq0fmDAGU06ltaL9XtEIqwAMN2hfjviWsFu0SUrh+onC+32sz3y3JGza0ypDaesT16FY1WLLg3tVuuR21hQK7cx4hA8pkbjOH9aHp6ij3NWQdxlnBqW29gW90i/PxroHQNBEAKuzhge7vJjZ1z8k10ZedDM4byCmzaEoRYpts0L4dJk/GpfDV6QeJbtdpgxHEyp+OL6sLCU0G2GlMNM5scdQTzYKT7ZNkflirw15sEX3hAT6qYdwsdZHPjetlqEJdN8J8+8V4Z8UhtRXYHs2ZWzGP5jc500o/auMkOA7Yh78MX1ou2MyZCXBLrlHPjB9lrpMsapfq8Pe6X1i+mK1CVatxm+ublOmOYDhSzRMp7u92NAMjWXlQFA0lDwtTfDQpRojsISR8ZDnQFpPzno8P2BnIIb14eFGa3FnWcZP99bg6f6xDGw08XQboxz52llzw03CB9yDnz4+SasG2fXUoIg3OGc5jwevCAiXTK13XmndCFFSwmCIARIGAiCECBhIAhCgISBIAgBEgaCIARGtb+hMF5W1CXOIXiujRYFXPp21eYQvP/ctlEuhXoDbripMnBhS2sy4t594lDg7JxU1l9ysX863adK9s/R9JtRCcP3z4wjYx6/tRcGffjWlrrRmBL48pIk3inJLP1wV8ClaFAct54Rx2kNokt0ufRnVVz/agPycie5sri8LYf/u0h0GZ5s3LkjhMd6xx4hy6cAP10VxTT/2MPjbYx48S8b6+CGgP/jwjSunJUVyp/t9+G2be6Mga8tS+Id08Qx4ITM1+PtKFsYGAPmhsq7GfslPuWjZVbQwhKJq+1rw+7FTzzJwUa51Hn4oRyDY+9wDV7blWuqNA1lumM7oTCOU+oMqUt0uURcjK3YGpD3HTfzSra71D9LQe8YCIIQIGEgCEKAhIEgCAESBoIgBEgYCIIQqGjCGZVxNPvEHWSOwpFsi4/9bb6HcTRJovzYAIZzimv71+VRuCaPxHTMUISISCcCAdVG2CPuWBhHjh9PfJso4Gj228LTk6MQvcpwoX9OxBgYDRUVhrYaC//7zhHhvLthAx9Y0+zKNufiegO/PT8iNHzCYLj8mWZEdPk5+fFEZcDPV0Vxcp245fSVjWH8aZwzYVUj75mZx7dXxIXyjoSGDz/fBMulNHXl0Oiz8cg7h1FbJFicA1e/2FhWeDwn5gQtPHzRMLQi9clbDFc814RuF9PnlUNFhUEBENRsaWg3t9Y4KgNCmuiNZnN3vOZGA0PB6aS4wwEQYv+dKHgUeXuMxjnHLRiAoOQ+2RxCcKHRojAg5OFCaDePUllP14oKg41CFKDip4FhQxqVaFQ2eMFGsQhkLFY62O44wgFkLYa0xHu0UlPHSmPakLZHtoLLqsP3SSua2HEO1xLt2rxQ72JhyFsFO5WiosLQk1Fx2TMt0tWjU2i3ctke9+Dip1qEco5CWLRKYHHgU680QJNUPKqfmMLweK8frw6L4cpMjoosI4DCe4T3P9cs7Z9DDmHXyqUzreLdT8v7p1tjYDRUVBgszqQh4t3EsBl6x9lG+TBpWPATmYylICMeMagoNsa/f5q8GvsnbVcSBCGBhIEgCAESBoIgBEgYCIIQIGEgCEKg5K6ELKvUaOhxyERVrXRmVDS6UPf+nALbJb+EmK64dj8qScxwpz04Z9ib1JA0xu7x0uWQJapa6cm41xfaHMpL/vXLnml2xXglHTXKp5Bl2C1vArf24B/t8ePxXjFL9GTDLcegnA184uVGV/5W4ZImj//INzbXuea16xQHqqQwnKheeG495d2EQ/QQPbE5cdvDxvh77dI7BoIgBEgYCIIQIGEgCEKAhIEgCAESBoIgBFzdGF9er0ujEnVnVLwmOVIb1Gxc2poX0t3ZnOGpPh9S5th1q9Fr450zckK5YTM80etH3haDal0wTUeLJMPR1pgHuxIeobw1YGF1i5gZKGMyPNnnF3Z3GDguac2jziPuwb8x4sXBtHhb5oZMnNGoC+UxXcEz/T4Ub7dpjOOymTn4VfH19UuDPgyUcbpzut+SZj7KWgxP9vphSup38Yw86r1i/TZEvNLIXHOCJlY2ifWLGwqe6fMJ6d1UxvHu1pw0kMvLQz70S04sLqozsKRezDA2mFPx4qAXxW3oUwpt6CmKMMYBPNfvd+XYfq1m45LW/KHEREexDo2BtGQMnNucR1uN2D87Eh5siYn9czS4KgwfbM/h+pPTQvkj3X6pMLT4bHz/zJg0gtOlT7cglRp7w88JmbhjZVyIhhPXGV4e8iKfFzvQZxemsHqa2Em/s6VWKgzL6g3cdZYYlqwrrWLNgE8INqIy4KYlSSwKiyJ6w7qwVBhWt+Rx2xkJoXxLVMNz/T4hsI1f5fiP0xKYETj2E86Bj73UWJYwnBo2cOfKuLB33pdR8PyAD2ZRgBXGgBuXJLFUki3py+vDUmE4p1nH7SvFNtwRL9SveGvSq3D8+2kJaSaqa19ukArDpa053LRUTPP34qAXLw6KPhF1HhvfWRFH2CtGcLryOQ1Rfeyh3aYHbNy+MiYEaslZwMVPtUiF4VMLMri8TXzY/XdH0DVhoKUEQRACrs4Y9qdUvDQoquhOh7x9OYvhlSGfMFUzbfdCeiUNhrVD4jWlTQZDWEYU2BrzSEPLdTu4dkfyirTeQzl5lF8OYGPEg2FJFCCnAC4DWXnb7k9pUl8XizOsG/GiQTKdj5UZJSqqK3hZ0oYRpyjGHHgz4kVMMtXud4hKNJCTt2FnWpV6ztqc4Y0RL/anxPo5TfG7M5rUxjaHp6xhM7w27BWWKxxA0oVlLlBYbq4d8goxJA2bOUYL3xHXUOsR63Eg5Z5rN+Ml/JXZ7NvK9K8q9XVZJd/uz4u/+dHZUXxwtjiNum9PDW7eFHbFxkTUw10bbl2TE1OlfuNv45bT47hufkYo/2OnH59b1+CKjfLr4Qzvukn6A5dP5ZT7lJ8I1+PR2JiIeoy3DTfbdqrU70Ttn+VD7xgIghAgYSAIQoCEgSAIARIGgiAESBgIghAouSvxD6eIXmKjYW9Sw5N94xt9aFm94dr1ukXSYPjtgRrBZXg0nBo2cOF00S15srFmwIcd8bF752mM46MnZRCS5LusJEslLtduc9nMHOaFnGIvuUNJYfjasqQrRh7p9o+7MJzVbOCs5vG/KeXQlVbxh84ATBectVY06q7dj0oS1xVXhMGjcHx2UUrqEj3V+av2rNQl2k1oKUEQhAAJA0EQAiQMBEEIkDAQBCFAwkAQhAAJA0EQAhXNedbis/BvpyXgKdrNM3kh2045UYYmEwrj+NdlScwMiOG57t9bg1ck0a6cOJhS8e2tteOdf6QsFABfXZZEe1CsnxOrW/K4dp54XLkno+KWLbWFJCtTkBl+CzcvT4jxGDjw75vqMCyJMDYRVFQYghrHe2bmpKHdbt9ei4HKXNa4owA4f1peGtrtqb7jFwUASBgMj/X4q2rgKKzgY1AOs2osvG+WuDe/I66Bba0d98xLlSLk4bi8LScN7fbdbbVAhXzaaClBEIRARWcMEV3BLVvqoBU97CwOadizqYLFgR93BNHoFR+Dm6LuBPOcbGyMePDvm+qE8kieuZYItxoZyin4xuY6YSlhciBawTFQUWFIGAru3xus5CVUBA6GhzprKn0ZVcXupAe7kyeeKMYNBT+vwjEwdR/LBEGMGhIGgiAESBgIghAgYSAIQoCEgSAIgZK7EiVy0ThSnN/QbTgf3XVVgklymZMWDkZ94a02XDRSUhg+/HxTWX/sk/PTuFKSJcpN7thRi1/uq77tHRl5C8i7lGqPOJa8xXD9q/XwTZI578gE+CQ83OXHL8ocG687lJcUhnUj5WXzHe9wUwCwL6VhX3WFdiQqgA2GzdGxZ5ueSvTl1LLHrBOTRG8JgphIKur5SEwOQvU1mLNollBu6Cb2vLkf9lT2WT5BIWEgSuILeFHbEMK02U1gRW+W81kdvXv7kcvqMPXxDWdOTCy0lCCcYcCy1Qux8Mx5gigAgNfvwcpLlmPW/BkVuDhiPCFhIKTUNYawYPlJ8Af9yKZy2P3mAaQT2SOfD3QN48D2bgBAU2s95i1th+qZmoF1TkQqupRQwFHr4UKIEY5CFidZ8JEa1YZXImd5G8ha4gcq46jVxDUwRyHIicx6SOPCUXAAyFoMeVv8wMM4ghIb9iEbkNio1bhw1BYAMhaDLrEx0QTDNWhfOBMAkIym0NXRi3BTCL6AB6qmItIfw0hfFK0nTUOoPohgOIievf2wjOOP2vRWvApHjSq2ocWBpClvwzoPlz7ZUiaTZv/yKRwBiQ2DA2mJDXbIhuxuJE0GS2IjoNrSLVTdBjKS/jmaMTARVFQY2mos/OHCEfiKHjSGDfzV8004kBYv7+blSem26G/2B3DrNvE8/6lhE//vvIjgeJUwGD7wXBMiuviUu3NlTJrV6q4dIdwnOSJ73rQ87jorLpT3ZhR86Plm5IrGisqA+8+LYkGtuC7/1zfr8OfugFBeDex4Yy9q64NYcdESAICeM7DuqU1oX9SGtjEuJy6fmcM3Tk8I5bsTGq5+sRFW0Xj2q8DvLhhBa0DMRPW51+vxwqAYCesT89L43KK0UP7asBfXv1ovlDf6bDx80Qhqi9LgcQ58/OVGbI2Jx8RvWJzCNSdlhfJHe/z4ysawUD4nZOHBC0agFWlG3ir4EXVnKjNEKyoMKis0viy0m+xpCgAhj40mn9gZaiRPbADQFI5Gnw2l6O9pjDl6adZ5uNSG7GkDAF4F0u9nTAYGDvFJBIQd6uFVqu8Nvz/ow6yTWwEUXkYCQOP0MFStcONC4bHHlvCp8jYf9MpT0DFwNHjlbehxaMOAg41aj9yGAqDBayNcFFDH5oXcmTKCmtxGSJPbODwGZKHdivvsRFJRYeAADJtBKXIY1W3m6EJq2gy6ZLYqm9YBBXU3bHEiWmq6bnA42JB/33b4vmFDslQ5+pnsN9WYiTFYV4NTVsw9pmx6ewumt7cc+bc5yiXEYUq1oROGQ19w6js2l3/fLNUXbAa96MbbJWyYDvfVKbEx54f6O5eMgQo+IyoqDL0ZFe97tlm6vurKyF9kfWdrLX64MySUxw15w++Ie/Dup1uEchtATJe/e71xfb10djDi8P21Q15cKrFh8MKUsBiTA9e/2iB9VzKYOzHfBz/R68fGiOi1p9tyQc5ZDB97qVH6LqgvK2/DBw7U4PFeMbly1sFtPaIr+NDzTdL3GN0O/fNHu0L4fxK35MJ7EpHOjIrLn5GPgb5s5V7mVlQYDM6wL1XeJQzk1LKiR+dthr1l2WDoLfOGZCwFe1PlDGiGrgqtHUdDPqcjPpxEfXMtvP7C4E1GU9BzBhqm10NxYc6bNBUky2hDDoaDkndQpYjqCqIO4i7D4gz7y+yfw3kVw2VEdjbs8sfARHBiPp6I44ZzjlQ0ja1rdyEZTYNzDs45uvf0Y+cbe2FbljANJiY/1SdVRNXAOcfON/YCAFZctAT9B4dwcGcPAKB5ZiOaZzZi69pdmN7egpZZ5Z3EJaobmjEQJeE2h+ZRUd9SB01TYds2bNtGIORHuKn2UHwMmjFMNWjGQJRk8dkLAACMMSw4/STh89MvPBUAYJnVuJ9CjBYSBqIk+7Z0QvNqaF84E527emHqJuYta0ff/kGkYmnMP+0kjPRFMdg1DIMOUk0ZSgpDe7C8G13n4CjiJk0+S+p+XI2YNkNfVnH0ZahmLNNCNpXDUE8EXr8H02Y1YaQ3Cj1voHXuNEQGYogNJTBz3nTEhhIY7BqZ0Otj4GgN2NCq0CFMRtpkGBnnBLV1HrvsMetESWF45tKhsv5Ycdbq8eDry5K4YpboclqN9KRVvOfZFsElejIw2D2C4d4obMtGJpnFa0+8CdsqCP/rT26CbdsAB954enNFHHH8Ksdvzx9Ba83kaNw/dwXwxfWi27WbXD0niw+3uzM2SgpDoAoPy3kUXpXXJcOnQuoSPSngOCIEQIn/X8EgLT518vSFiXB11xT33g3QrgRBEAIkDARBCJAwEAQhQMJAEIQACQNBEAIVdXCqUW2sbtGFgBQ2LxxlloXCmgowcKxq0RGS+GNsjXnKPt05FZgZsLC0XoyalTQZXh3yTkpfkOMhqNlY1aILT2ibAy8PeaXhCieCigrDNL+Nu8+NSiM4Xfp0S5lHmScPKgO+cVoCi8KiM8oN68J4sHPsEZEmG++YlsftK8XweDviGt7zTLNjkJzJTmvAxr3nRqURnC5+qgWd6cqMgak58giCGBMVnTHkbYYtUY+gliYvRH2eqnAAuxIacpLIQeUEEgGAgMaxvMGoqszaCpzjYzoRySt4MyIGVz2QVidNRuvRkLOATVGPEIlKtwG9ggmRKyoMfVnFMaP2FO4LsDjw2dfl7rHl1nt+yMKf3zmx5xSOh3K79NP9PjzTL0Z2BqZ2X+jOqPjQmuobAxU+Xekc9HVq4169GZuUDtcSqC9UE/SOgSAIARIGgiAESBgIghAgYSAIQoCEgSAIgZK7Er/c544H3taouD/tNjvjGl4fETMZVZJoXimZ/qwcdic8rt2PSrI76c5GmMkZHuoMCHklK83ZTbrUo9VN1gx4MZR355n+FYfyknfpq5LsvNXK2iEvbt40ea63XF4f8Vad8FUSw2a4TZLdvNLccnp83IXh1/vFFHijxUkYaClBEDvy7j4AAB7ASURBVIQACQNBEAIkDARBCJAwEAQhQMJAEIQACQNBEAKunq68clYWF07PC+Wboh78cp+4xdLss3DjqUloRfJkceB722sxlDvxQpxVI4vqDHz65LRQHtcV3LatFvkiXw0Gjn9enEKbJEvU7w8G8OqweLz6rCYd15yUEcr7sipu3x6CXXSG1Ktw3LQkiXqvGLjjZ3uC2B4XfWcumZHD5W05oXx3QsPdu4MoPqdaq9m4aWlSiC3BAfznjhA6M2MfPtP8Fr50ahJqkbuLYQPf3VaLiC6Ogevmp7FMEgbvuQEfHukOjPmaAJeF4fRGA1efJKbICmpcKgwhjeMjc7LS0G4/6QihvAR5xHjRGrBw1ZwsWFHn7cso+MGOkCgMDHj3zByW1ov7+etHvHh1WLQxN2RK+86OuIY7doSE4AQq47hiVhZtNaIwPNrjlwrD4rC8f7446D0kDMfiVzk+NDsrOFHZHPjVvhp0ijpWNnUejr+ek5WGdvtxRwgRXfzNeS26VOAiuuKaMNBSgiAIAVdnDGv6fUjooguwkxtsTFdw145aqOxYRbZ4Qf2I6mB/SsMPtoeE8pSpQJe4fHMO/HJfENN84lJic0zuHr815sH3t4k2hvMKZOkxDZvhnt0h1GrijGFfSt7fXhv24fvbxPLOjHzJmjYZfrQrBJ8iLiX6XIrkPZJXcMeOkPCENjlD3GEMPNzlx7aYWMf1Efc8YxkvEVCPzb6tuhzRAfzo7Cg+OFucRt23p2ZKu0QTk4NbTo/juvniGuOPnX58bl1DBa6oNLzrJulhHnosEwQhQMJAEIQACQNBEAIkDARBCJAwEAQhQMJAEIRA6dBu50VcMfL6sBc/2iXuUVcnHF9ZmsRiSRSe3x0I4JGesXuWKYzjltMSmBUU9/nv2R3ES4PyjEzl4Fc5vntGDPXjGPosklfwLxvCyLkQvu78aXmp23VXWsW/vlknuESPhitnZfFXc0TPx20xDbdtq8VkSd3z+UVJrGwSXaLdpKQwXDxDPPcwGjLm5Gjww6xoMLB6muiL+tqQOw4kCoCzm+WxAf/c5XfFhsY4VrfomBEYvySgfRkFqsIBF4ShNWBJ+9uOuFZwxXZB3+YETakNj1J17jolWVYvr4eb0FKCIAgBEgaCIARIGAiCECBhIAhCgISBIAgBV49dlwsDF6LjHCZrMfBJsn2kMi4czQUADoasBbixDaYxDq/Ehg2GnEs23IPDrwKKZCtBtxlM7sa1FvqO7C/lbQZrnG3kLObKFmq1joGKCkNbjYUH3hGBr6hhDJvhYy814mC6opd33JzXouN7Z8aE8t6sio++0IScCzuGV8zK4StLE0L5zrgH161tcGM3zzUUAPecG8XCOnGv/dtb6/Bw19h9Qfwq8NvzI5gREH1BvvBGPV4eGrsvSJPXxv9cEEGt59gbaHOGT73SII0SVS5zghZ+9Y6IsGWatxiuebEJvS7FfSiXio48jQEzayxpaDetmh6Ab4Nf5dIQYzZnYIzDjad5ULOlNiJ5Cy5t87sHK8QylF1v0OHpWL4JjhkBuQ2/SzYUVvCvkIV2k83eRoOmFB6QstBuWgX9Kyr+SLY5g1UULGb8XHLGB45C1KliZJGHRm+DOdioQgXlh++r9CPXcLLhJjbEe8u5u/WwOFCsATZnFVX7igpDb1bFh9Y0CUFGOQe6HMJtVSOvDnvxvmebhXLDLqxF3eDxHj82S7KGF9a61YUN4PPr6qVP7h6X7mvOYrhubYPwpAWAgyl3bER1BVe/0ARFcgv3Jt2x0ZVW8cE1Yt/hHOivYJT0igqDbjNsc2GdVmmShoKtsfHd4InoyiSKg8mwx6V0905wMOxKjG/fMTlz5T1CKfI2w1aHOJiVZLL0NIIgJhASBoIgBEgYCIIQIGEgCEKAhIEgCIGyXx1zDsR0BqOM/fO44d5ee0JXMJgT9Sxlksa9FY5CliPZVptbDOcVlMhXdEKSNJi0fyYM9/pn3MGGEx6Fo97DBbeAUoxqT+mz6xqwYeT4t1jKEZG34xtb6vCdrbVCuSxV2olM2mT4yPNN4+ppbwPIuOSnMVW4c0ctfiwJY+jmGPjXN+vwH2X8uZVNOn55XrQsG6MShozJkKzQEzpnMeSq6sBQtcKQmmQh9aYCeZsJ2b/dJmuVN/ZGI940/yYIQoCEgSAIARIGgiAESBgIghAgYSAIQqDi8RimAo1eG4vDYrSinMWwMeJxJQTYdL+FBbVigpqUybAp6kFxMBiVcZzRaEgDimyLeRArY1+93mtjiaR+eZthQ8QjiQnBcVqDgZAm2t6T1DDgwnFiBYX6FUf/AoDtcQ+iLpxE9SgcZzTqQtAgDmBz1CP1nZkXMtEqiSo1lFPQkRS3+AOqjdMbDeEJbXNgQ8Qr3eFYVGegyScetu/JqDjgUtQzEgYXWNmk42erxH3i7oyKi59qRtaFvf5LWnO4dYUY2m1LTMMVzzYLMRkCKsd/nxPFdL/YgT72UiNeKCMN3ooG+T54X1bBO59qQbpoW1RhwK0r4lhaLwrZTRvCeOBAzXHbdsKncvzXOVHMlGTaum5tA57pH3tGr3qPjftWRVHnOVZ8OIArn2vCm1ExM9knF6Rx3byMUP7HLj8+v65BKG+rsfHrd0TgKeoieRu4+KkWdEoG+hdOTeHymTmh/O7dQXxrS93b1Or4IGFwiXK8yty08XZmZUFw3LJfyjZjlWuT8bZRqg2Z5PuHy8v9jVvfHw0kDC6QMhk6EmJT9mcV10KvxXVFauNgWpNGALM5sD+lIWGIT9RyHV7SpoLdksArQzlFHr6OF67LK5nNu+UezznD/pSGtCnWr3gGM1oszrA3pQlLIs6dI3MN5uT3qc8hqKtuA7sTGjRFLDccHKV6s3Ibw2W4Sb8dJAwu8MqQF+9+WgzPBQCmS2cJ/tLjx+O94vSYQx4aMGMxfPTFRunfKjdO4roRT1n1swH842v10qekWzEaczbw8ZfcqZ8TEZ3hw2uayrLxo10hqUu00yV1plW8VxIWEHDuO9/YXCdtWzdjjJIwuAAHc00A3LPh3jWNxvZ4B2l1s35u2rB5uTE4y7fhTs6M0tB2JUEQAiQMBEEIkDAQBCFAwkAQhAAJA0EQAhXdlQhqNi6ekXdFnfalVGyJiZ5o1YjNgWf7fdgp2Yt2KwOXxjje1ZqHfxzzH2Ythmf7fa5kr+5Mq/hTl7gd25NRJ1X4uOX1OuaGRJfocrEO9ZFMmUFZ3KKiwtDis3HHypiQ1HY03LenZvIIAxi+vdUd11Un/CrHt06PY4bEZdgt+jIKLnqqBaYLDkWvDvvw6vDYM1RXmqtOyuK6+aJLdLnkrMMu0ZURBlpKEAQhQMJAEIQACQNBEAIkDARBCJAwEAQhQMJAEITAlD9d2VZj4u8WZFB88DVnMfxoVwhpFxLnLKwzcM1J4hZVTFfw410hIQuRAo7PnJLGNL+43/2nrgA2SSIDndmo44pZWaG8L6vi3t1B8CpKwsPAcf3JacyQhDj73+4ANkTE+p3eoOMDs8X6DeZU3N0h1s+jcPzTwhTCHnE79jf7a6Rh1MolpNn47MKUJHwcw093B9HjEGNhvLl6TgaLJKH21g758FTf2CNXASeAMEzz2/jUgrSQwzGuM/x0TxBpMfpY2cwJWvj0yaIwdKVV3LM7CKMoqIfCgI+0Z7EoLBrfFvNIhWFR2JDa2BLV8NPdQcfz/pWAMeBD7VlpaLfdCY9UGE6pM6X12xHXcM/uoHCMW2McHz0pg7YaURheGvS5IgwBleMT8zIIe481bnPg4S5/xYThktY8Lm8TQ7uZnLkmDLSUIAhCYMrPGAazhad28UQ7ZzFkXQoBdiCt4icdQaE8pjOYkvBcNgd+fzCAaZJArbKQXQCwI+6R2ujLKlU1WwAKoc/+cDCAlwfF+u1wqN+uhCat34BD+DjTZvj1/hqEPeKHnWl3nuQZi+H+vUH4i5YSHIUlTqV4stcnreOrQ+55/k55YejJqrjFpci5TnQkPPjmluOfutpg+MluMfxXKTZEvNIpeDXCwXDvnvLqtynqlS6hnDA4w3/uFLOeu0naVPC97eNrYzT8vnPsUbbfDlpKEAQhQMJAEIQACQNBEAIkDARBCJAwEAQhQMJAEITAlN+udJMPtmextEF0RXWTX+ytmRKRjMplVXMef+tC5KNSnCzJFj4Zuf7kFFY0iv3wqV4/HuoKuGKDhKEMFodNLJa4MbvJM30nnigAwOyghffPEt18CZGzmgypS3R3RgVcEgZaShAEIUDCQBCEAAkDQRACJAwEQQiQMBAEITDldyUYODSZ/HHA4IVvTAYUcKiSenAOmFVXDw6NFQK2FGPZhdOlkwMOD4O0aU0bVRU1y22mvDAsCpv48dlRoZOmDAUff7kRMX1y3Nz3zcrhi6cmhfKOhAefebW+qmIyKAD++5woTq4Tt3a/v60Wj/S4s6U23jR6bfzyvChCReHjOGf4h9fqsTMx9ihR1cqUFwa/yjG/1pKEdrOhsmoaTqWp89hYUCvGUMyaDAzFES0rDCv4Jciut04SWKVaURkwN2RKQ7sVB2+Zakx5YTg8YooTo07G2ypL7lqt9eBcfr2TkWrrOxwObevihU15Ydid1HDVC41CuckZ4vrkeff6VJ8fe5Pi7UqbCsYvbe3osDnw5Q1hBDWxp+5LTZ4uF9MVXLe2UTqz3CO5FxPF97eHcN8eMYpTr4vBaSfPXRolKVOZEmcPBnNqReMMlgebNJnHS2FwhnUj1VePjgl4tzF5HpkEQUwYJAwEQQiQMBAEIUDCQBCEAAkDQRACFd2VsAEkDAX54sSEoyBnuefBmDYZ4hXyiDQkmatGAweQNBQExtERJ2Eqrvkq6Hbl2jztUkYyoOBw5kY98jaTZuCaKCoqDN0ZFZc+3ezK33JTGP7vG/XwKJW5KynDnUlcxmT48PNNUMbRu9PmDFmX2v2xHj9eHKzM1qBbYgwAd+4M4e7dYqq90RDNV25C76owrJgTx8KZqSP/tmyGxzdNQ32NgfMWRvD01mYMJ4/6FNicYSRfbXvzDAljcpyfKAUHQ3SSnAMBCk/IfNX1hfJJm4orGdQrjUvCwKEpHJcuG8bH39FzpFQ3GTbsD2PBjAxuvGIf9vQHEU17YdlAdZ0GJAjirbgyV2mtz+OBz27EFWcMHFPuUTnuunYbVp0cwV/fdQY+uroXt/z1TjdMEgQxjox5xnBqWxJLZyUxf3oGHpXDMBle21uP6eE8Tp6RQXtTDiNJL+Y0ZxHPaEhkp7wXNkFMesY8Y/jkhV34+of2wHPo7Xc6r+Kr/7MIf1w3A0DhFNjpcxK48xPb8b8bp+POx+eBlhEEUd2M62tPw2L4+u8X4u5n2sfTDEEQLjPu8/qsriDvsAXnVTgW1hlCEBXOgV0JD/KSbaT2oIkGr3jQeDinokdy7LRGtbGgzhTmKBZn2BnXYPLiTzjmhywhag8A9GXlJxzrPDbmhsRX0bpdsCGGAONYWGdKg310pjVEJcfBm7wWZgXFwCcZk2F3UkPxLEwBx6KwKd123ZvUkDKP/5lQq9mYJ8niZByqnxiqjeOUWhMBybHrrrSKiC62YYPXRntQtJG1GDoS8votDJvwSuq3P6UhIelz0/wWWgNiGyYNBftSqmBDY4U2LD52zQHsSWjIWKKNtoCFZr9oI6or6EyLw813aAwURxizD40BXTIGTgqaCEvGwGBORZ9LR6/HVRg0heP7H9vh+PnMgIWHLhyBr6guug1c9nQL9krO7t94ahIfmC1m4fn53hr826awUL6wzsSfLhoRGj5uMFz0ZIt0u/RbK+JY3aIL5bdurcWPO0JC+TnNOn62KiqUd2VUXPJUs7DXrzHgR2fHsFAS+uwLb4TxYKd41v49bXl8Z0VcKN8S8+D9zzYJMRlqNI5frI5gekDsQB9/qREvDB7/UfQzm3T88jyxfn1ZBRc/1SI4CCkMuPOsGJbWi/W7aUMYvzkg1u+S1hx+cKZYv51xDZc/24xiHzifynHfqijaasRB+Mm1DXim3y+UXzUniy8vEcPjvTToxd+8JMbsaPDa+O35I0LUKZsDH1jThE1R0e/i709J4VpJqr0/dfnx+XUNQvnsoIU/XjQCT5HG5CyGS55ulorJV5YlcflMcQzc3RHEt7fWCeWjYVyFgbGjGuzkIccYhBlDqWeZ7PtAibcWrHwbCsqzwRy+X7IeDr9xhjvYcHZgktV7NJ6Kh+OhFotryTYs8z45tYcsoOxRG/I2cYI5tGEpG47X5fT9cvsn5G1VyjHNsX+6+OrOVWGIZzT0x32wOZDKa+iN+jCtTofm4JZrcqAno8Jb1MMM+3AEZ5GRvIKutPiUjzlEY9Ithu6MKtyYpMFgC8uIAoMONhIOrrNZi0m/35dVwCU2OID+nIKgJv4m4+BJmDbl1zSQU6XSYPPC0kfm1ZcTH7IlyR1qw2IGcw4u0RwYyKoIS+I7Orkfp015G/Zn5fXjYOjNqtJ76OQFmzTkbTiUk/cdixeiIiWKOqMNSKf4QGHJILMx4uDFaNgFD2Ct6M/lbcB0sDHs0D/djEjGeIlHSM8NNwgfcg58+PmmI5Ftvvc323HZ8mEAwJ2PzcWv185E3lCgKhw1PgsP/NNGzGnOgXPgup+cho0Hjk73GTh8auG/x9gAQ96Sh+f2Klwaasu0GQxJJ1HA4ZMIEwc7NEDE9bFXgdSGYTPJO4mCuvska13OGXJSZy4OnyJ/Kug2gyWxoTIuXU/bnCHvYMMvaVsAyFusrBDujm1Y6fopAJPYcKqfxrj0nYvFmcNAd27DnMWk/dPDCs5+xZicSUWaHbJRHLDRzTFwTnMeD14Qkc4o2u68U9oRXJ0xGBZD3igomWUz5HRIn5iHcR6czhRu4PF/30a5/vwMepmemeWfGXDq7M5Yo7BRbts6MZo2nJD6lWnD5AzmOLehwRmMMmxwMGTHeQyMhjHPPbK6inReLbl2NSyGVE6F5eJhFYIgxo8xC8P3HpmHf7hvaUmVXNvRgA/cvhLbe8Q3+gRBVB9jXkokcx50Dgfw4Out6Og79ripzRke29SCwbjvmFOVBEFUN668Y4ikvbj1zwuEcstm+O+nT3LDBEEQEwiFdiMIQmBUM4aVTTrqJC7DBEFUHzIP27ejbGFgDPjaMtGtlCCIqQMtJQiCECg5Y9gyODhR10EQRAVocygv6RKtKhUKlUwQxIRg2XKvw5IzBtutpAGHuOaaVjAG/OY3fbjiihYsWnTU7yGbtXHffd3IZkf/UlNRgGuvbUMkYuDhhwdx9dUzMHv20eO3sZiJ++/vgWmS3hGjY3EwiMubCykPOnM5PDgw8Da/mJxMSABGn4+hrs6DJUtCYAxoafHi7LPDWLXq6Pn0VMrEY48NYXBQRypV5vE/AIGAgnBYw7JltRgYyGPtWi9Wr27Aqace9bYcGMjjsceGEIkYYxIg4sSkUdMwx+/HitpaAEBIVdHi8YADsDhH1JwCceMPUXIpwWRH10bBeefV46ab5uHmm3eDc+Ab3zgZPp8CVT06i+GcI5+38ec/D+Lee7vLtnHFFS34P/9nFm68cRfa2nz40pfmwus91oZtF2zcf38PHnpoaio9MT54GcMPFy1CRyaD/+rqAgAsCYXw9blzAQBduRz+edculP9Iqyzc4ZTjhMwYurpy+N3v+rF8eS1aWrwIBBS88EIUIyM6PvCB6VBVBsYY/H4VnuJQNsfJ7t0ZPPTQAM47rx7t7QH4/QqeeGIYlsXx3ve2gDEGRWEIBFRoxYffCeJtsDjHo8PDGNJ15OzCbNO0bfgVBWuiUXTncvib1la8Ho9jV0aM4DTZmBBh6OzM4Ve/6sXtty/CsmWFadjrr8ewZ08GV1wxDZbFj6z7dX10U/xdu9LYuzeDn/xkCdrbA+Cc44UXojAMG+99bwvyeRvWofhg9I6BKBcLwMNDQ9LPnj8kDHcsXIi4aeJgLndEPCYrFUvy8JnPtMOyODwehl//uhePPFJo9FzOvQa96aa5R46D33tvF156qRC3MJOZbBM+otrpy+fx9zt24K+mT8cPTjkFN3Z0IDOJxaFiwrBvXwa9vXkAwNatKYyMGK7b2LkzfeTv7tqVHhcbxInJueEwZvp8eGJkBIO6Dr+q4sy6OswNBNDo8YC5GYCxAlRMGP7ylyGsWRMZVxu/+10/Nm8m923CXVTGcPWMGchaFr66Zw8YgDl+Pz7f3g6NMUQNA4eD0U/WRSvliyOIMjirrg7XzpyJB/r7sf/QS8brZ83Cyro6HA7PWqtpuO2UU/D48LDje4lqp2LCMG9eAKlUIQZ+V1cOAwNiHoexsnBhEF5vYUq3b18WkQgtJYixUatpmB8IoMXjwbCmoU/XMcPrxWz/UUc6jbEjS4rJSsWE4ZprWnHNNa0AgHvu6cKDD7rrV8AYw6c/PevIv2+9dR+efXZ8ly7EiQFjDH8/axY2pVL4l927K30540LFhOGtL2cuv7wFK1YUZg9r18bwl7+4M/16q42rrmrFu97VBAB4/PFhvPiimFmJIJzQGMPn29uxIBAAUOhb8wIBfHP+fLwWj2NdIoF/mj0bKmNImSbu6uzEvmy2wlc9eiZUGHp6cmho8KCtzXfMoG1vD6C9PXDoO/lR/33OCz4TXq+CGTOOjTE5f34N5s8vpEbbtIleSBLloQJYHgphhu9ov6rTNJwVDuMvw8M4kM1ibyYDlTEkTBNvJBLI0nbl8XH77QewZEkIt9++aFz+vmVxfPObe3DeeQ24+WYxBiVBjBf9uo4bdu068u/JKwkFJjRQC+dAd3cOd9xxALt2pY/5LJu1cPfdnVizZmRMNmwb6OhI4447DqCr69jEn7GYgR/+8CDeeENMnkoQpTA4x897e/HE8PCRsq5cDncdPHhkyWC/5X+TnQl/xxCLmXjssWEsWBBEU9PRt7aplIUnnxxGMjl2r8SBAR2PPjqEM8+sQyCgHFP+2GNDMJwSYxKEAzYKrs8m5zizrvA+bF82i8dGxvYgq1Ym5HSlDK+XHXPyEYDrR6F9PgXKW+ZEnLvrck2ceKgAvIc6lc058i7HLJlonE5XVkwYCIKoPE7CQMFgCYIQIGEgCEKAhIEgCAE6REU4csQHjTufEix5uric35X4LjHxkDAQUs5f1oRr3z0bALBlfwJ3PbRP+E5DyINvfnIx/F75xHPrgSTu/MNeobw+5MEtRb97eWsEP3+i06WrJ8YKCQNxDAoDls2tw2nz6rBwdiEMXzQpnko9aXoNFrQFsXB2CCMJHb0jR53JGIDFc2rBObDylHrs7kkhni5EUJ4zPYAFM0NYODuESEJHXySPZXPrEE+bWHlKPbYfTCKTpwhblYaEgTgGn1fFv/3tIjSHvSW/d9VFbbji3OlgjOGJNwbx88ePPu0VBtz7xRVYODuE2/9hKW68Zxte21E4tHbVhW14/6oZR373wLPd+NVXVuKshfU485R6XH/7m+joTo1rHYm3h14+EseQNyx864EO/OHF3rf97tuFL2OMHflOa6MP3/nUqThncQMSGRM3378DT60fgm7Y+M4DHfj9C71QGPD5D83Dpy6f40pdiNFDMwbiGGwbWN8RQ2ujr+T3huN5HBzIYHZLAA0hD+bOqDnymaIweD0KMjkL/dEcsnkLNX4Nq5Y0QlUYhmJ5vLYjemTJsH53DNMaCiduT5sfRt4g79RKQ8JAjIqfP9GJR18bwC9uOgMfWN2K96+a8ZZPGVQFWLcrhi/fsw2WzTF/ZtDxbxHVBwkDMSouPaMFZy9qgMej4NUdEazddjQ6lsIYPnbJbMyZXoMbPjIfDx3HsoSoLkgYiFFx+oJ6XHbWdADAzq4UHl7bf+QzhQFXrJqBU2aF8MHzWvHS1hEMRPKIJHTU1mhQFIaGWg8458jqNupDHoT8KjjnSGRMJDJTJwfkZIVePhITQudgBp+4dT2ee3MYDbUe/OyLK/CRC9rg9yr4z88uw/VXnATOga/dtx23/baj0pd7wkMzBuIIy+fVYfm8MABg0eyjWcJnNvvx8UsKzk7RpI5HXxvAqzsi0E0bV66ageXzwkc+BwpejY21HvQMZ/HsxiH0DudgcyCds/DilhFkchauXD0DK0+ph0dlaAn70D2cw9qtI+gZyiE3yjSFhHvQsWviCH976Wx84tLZJb+ztzeNf7xrE2wONIe9+NmXVqDGp0q/u25XDF/92XahvKnOi/u+tAI1/qO/e2LdIL7/+z1jqwBRNhSPgXhbQgEVoUDpSaRhcowkCjlAFAa01Pscz0vkdRvRlOg1KftdNm8d8Y4kJg4SBoIgBChQC0EQxw0JA0EQAiQMBEEIkDAQBCFAwkAQhAAJA0EQAiQMBEEIkDAQBCFAwkAQhAAJA0EQAiQMBEEIkDAQBCFAwkAQhAAJA0EQAiQMBEEIkDAQBCFAwkAQhAAJA0EQAiQMBEEIkDAQBCFAwkAQhAAJA0EQAiQMBEEIlMwrQRDEiQnNGAiCECBhIAhCgISBIAgBEgaCIARIGAiCECBhIAhC4P8DiN4cEFcrNEsAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 720x432 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "ExecuteTime": {
          "end_time": "2021-02-15T12:34:45.992467Z",
          "start_time": "2021-02-15T12:34:45.977988Z"
        },
        "id": "WIJKtkAXuM79"
      },
      "source": [
        "obs, reward, done, info = env.step(0)"
      ],
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Vd5LWzfkuM7-"
      },
      "source": [
        "A observação diz ao agente como é o ambiente:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "ExecuteTime": {
          "end_time": "2021-02-15T12:34:53.460315Z",
          "start_time": "2021-02-15T12:34:53.448322Z"
        },
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "6r5EB2nguM7-",
        "outputId": "9a3c60c5-8999-491e-e0a1-087e9e6f29b1"
      },
      "source": [
        "obs.shape"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(210, 160, 3)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_2ViLzabuM7_"
      },
      "source": [
        "O ambiente também informa ao agente quanta recompensa ele obteve durante a última etapa:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "ExecuteTime": {
          "end_time": "2021-02-15T12:35:02.928857Z",
          "start_time": "2021-02-15T12:35:02.921610Z"
        },
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "wS3qmSE6uM7_",
        "outputId": "10bdfddf-7476-4fe4-ef39-8924207538e9"
      },
      "source": [
        "reward"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.0"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9REPymm0uM8A"
      },
      "source": [
        "Quando o jogo acaba, o ambiente retorna **done=True**:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "ExecuteTime": {
          "end_time": "2021-02-15T12:35:05.981011Z",
          "start_time": "2021-02-15T12:35:05.969405Z"
        },
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "XL625gywuM8B",
        "outputId": "58bb77da-3ad3-4b0b-c052-4c7487c7b4bc"
      },
      "source": [
        "done"
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "False"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 11
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "mgZEOyEzuM8B"
      },
      "source": [
        "O info é um dicionário específico do ambiente que pode fornecer algumas informações extras sobre o estado interno do ambiente. Isso é útil para depuração, mas seu agente não deve usar essas informações para aprendizado (seria trapaça)."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "ExecuteTime": {
          "end_time": "2021-02-15T12:35:10.489902Z",
          "start_time": "2021-02-15T12:35:10.484332Z"
        },
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "1Eo3ca2xuM8C",
        "outputId": "dcaf2166-d53b-43cd-a848-654ac9611f91"
      },
      "source": [
        "info"
      ],
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'ale.lives': 3}"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "NW-UFpBYuM8C"
      },
      "source": [
        "#### Jogo completo com 3 vidas, movendo-nos em direções aleatórias por 10 etapas de cada vez, gravando cada quadro"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UrIyey2fuM8D"
      },
      "source": [
        "def update_scene(num, frames, patch):\n",
        "    patch.set_data(frames[num])\n",
        "    return patch,\n",
        "\n",
        "def plot_animation(frames, repeat=False, interval=40):\n",
        "    plt.close()\n",
        "    fig = plt.figure()\n",
        "    patch = plt.imshow(frames[0])\n",
        "    plt.axis('off')\n",
        "    return animation.FuncAnimation(fig, update_scene, fargs=(frames, patch), frames=len(frames), repeat=repeat, interval=interval)"
      ],
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "ExecuteTime": {
          "end_time": "2021-02-15T12:42:46.373015Z",
          "start_time": "2021-02-15T12:42:45.305907Z"
        },
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 247
        },
        "id": "NzfOPh5luM8D",
        "outputId": "c6e00d2f-fed0-42fd-d240-9f9fa3ea7934"
      },
      "source": [
        "frames = []\n",
        "\n",
        "n_max_steps = 1000\n",
        "n_change_steps = 10\n",
        "\n",
        "obs = env.reset()\n",
        "for step in range(n_max_steps):\n",
        "    img = env.render(mode='rgb_array')\n",
        "    frames.append(img)\n",
        "    if step % n_change_steps == 0:\n",
        "        action = env.action_space.sample()\n",
        "    obs, reward, done, info = env.step(action)\n",
        "    if done:\n",
        "        break\n",
        "\n",
        "\n",
        "video = plot_animation(frames)\n",
        "plt.show()"
      ],
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAALQAAADnCAYAAAC313xrAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO2deXgUZZ7Hv1V9pruTdBKSgIBcQUi4Ua6AIIoHynrtPDPO7HrNOJ7rtTujrquM1+rsnI7H6jiuqDPjrowjiLcoCsgpKodAIAnInfvoTjp9VNW7fwS66q10qrq6u5JO+X6ex0feVHW971v17er3+L3flyOEgMGwCnx/F4DByCRM0AxLwQTNsBRM0AxLwQTNsBR2rYMcx7EhEEbWQQjhejumKejDd9yR+dIwGCaiKWieo78Ik98uRSDW65ejX7i7vAN3lnfE05/WuXD9xoJ+LNHA4dW5LZhfGo2nf7fHh6eqfP1Yop7kOyXsWNKQ9PmaglYjEEDo/W3fL0iqNEH2lTFbUbcnpSx8voJkrDysU8iwFEzQDEvBBM2wFIba0GqendmK8XlCPP3UPh/eOpITT583OIz7Jwbj6cOdNly/qTCedvIE75/bRF1z0ceDQCC3m5af3Ywil9xSvverfGxrcSZdxkEuEa+f3RJPSwDO/7iYOuf9cxvhVHy1r91YgKMh+dY8MCmAhaWRePrNIzl4dp/cearIj+HpGW3xdFDgcPlngzTL9dY5TfDZ5VbsbVv9qAo44unbxwVx+fBwPP1JnQuPf5MXT5/uFbBsTms8HZWAxWvoeq1e1Ei9sb6/vhDNEZtmuZTMLIriiWnt8XRjhMdV64viaR4EqxfRz++iNYMQU7R7X65swXCPGE8/tisXn9a74+krh3fhtnFyp35Pux23f5F6pz4tQZ/uFTEuXxa030F30XIdhDrOq4a1OYA6fupvyrNG5woYnCNf12M3NjRu5+k8RHUvEsAZeQJciufsVP1uDcmh61nSKFLH3Ta6nm1R/Y5MWa6AfKdcF7eNrldpjkRdc087/aicqnqF6SIB6K6XTVEUu8H+ntdOlyEvpLoxXOLnp2SUT8CYXLlwuQ66nn4nnUeHkF6nNC1BP7A9D7kKgdV20Jfb0ODEVevkN3JIpAsbk0AdB3qOWty6pQBOXs7jm3YHjNAS4ak8En0drt5QSL3JTnTRb7Gnqnz434OeePq46nh1wE7lISTxnfvJpgJKYDVB+t69VOPFB8fkN1lDhBbTsRBdrwTfU/xofSElsNaosRbm9lb6+UVUIw4S6fn8YqqC/Os2P3IUX9aqAF3PD467sV/xt0B/CnpHq/ZPf2PEhsbG3n/iJHD4vNGleY2tzck3LxIRlfTz2KhzfF/AgX0ax4MCr5uHms1N2ucf6LDjQEfvj6dL1M9zg8EyqWmN6uWhf2+/1GkeHu+y9XhBpAPrFDIsBRM0w1IwQTMsRVptaD3KcmNYNDiif6KCP1Z7qWG7vuCnZR3UaMBARCDAizV9G4fBgeCmsZ2GPrO6zo1aRQd4kj+GucWyRurDNqxQDP0axVRBT/QLeGByUP9EBS9UexOORJjJfROD1LDdQCQs9oOgORh+vse7bJSgZxRFqWt82exIS9CsycGwFEzQDEvBBM2wFKa2oTPBuLwYNaN2qNOGDiH576GdIxiXR0/P7jY42zhQmZAfo9L7AnZD8c4+u4QRXnnaWiDdk0zZTNYL+q/zWqhYjh+tL8S6huRnwAa5JXyoCKARJWDEiiEZLWO28t55TdTozZnvlqA+nHzvd0ZRFH+eJwdAnQjxmPF+aSaLmHGyXtANYRuUbmWRREELGkik+0GcQuzjIcH+pK6Lp9qUosHho4jEUffOyJchWToFOo+mSHqt4KwX9MVrtMMw9WgI27L+rWIWs9Ks98ZGl+n37vVDHrx+yKN/YpKwTiHDUjBBMywFEzTDUpjehjbdfpqknwdBH5TTZFIpfibqnW33jdMyPD92113UwYpVpQjEjL7UjdbY6ChEoutn4hoDkXTrncoIkLnPN98hYfel9dTfhj75ZGpWYJnB7GGyTFz/uzOUR2O9e8fa0AxLwQTNsBRM0AxLkVYb+i9zmzGxQA78eWJXLjXrc/FpXXh8eiCePhi04Yq18syfkyfYejHtLDn9nRJIinbZ6kWNKHbL8923bPZjk2LF9C1ndOCmM+RVExsanLhtq2xUUuIW8ZEilkMiwPR36dmvrYvr4VTM6l72aREOdcq35r+mteHCofKqitcOevCr3bnx9GR/FK8qYh4CUQ7zPyqBFusvbKA8Kv7580J80yYH/tw3MYCrRnbF0+8dc+P+r/Pj6VE+ASvOaY6nI2LPmcGvLqkHr2jinr96EBoVRjPPzWrFnGLZffS5fV78sVpeJDC3OIJnZ8kGOvVdPC78RDaz4UHwlcoZdOZ7JYgq7A7eOqcJI3xygNO9X+Xjw+OyPcOPRoZwj8KMaGerA9dsoK0RjJCWoPOdBIMUrkZqsxSnDdTx1gjdgeBAH09EgVOiznGoflM8droMaiMTnqPzSGQ0U+SSqBUr6uVYPgedh9dOX8TO03nYOf2OUqFTooxm7CoTHp+6Xqo8bap6JTKaKXJJVF14VbHyHPS9zVE/P54uQ0ydB6f//Pyq5+fi6TzcNjqPfIfBYB11kdIZthvmEeBWCKExzKNdcdxnl6hIuagEHFa8+TgQylUHAGqCNih7ziO9AuwKER8P8QiJ8h8KnSIKXXIxOwWOMoqxcQSjFG8IAlBLgABgjE8ApwpRVdpZlbpF6ovSFuXQpHjTuXmCYYowS5EABzU8NQBgtE+gBHak00YZuRS7RErwwRhHBQc5eEKFdhLS0+inLJcOmz3YYYOoCB89LUeknKiaIzxlRuOxSTjNIz8/QQK+7VTmQVCmen61QRu1JvR0r0A5UdV18VT4b75Don6BwyIoGzajw3Z9MA7NYKSOUUEzdTIsBRM0w1IwQTMshaFRjgcnBaghGTVvH3XrmhBSmXMED08JUH97YHueIaOZ8waHca4BMxsJwIPb83XPU/KDESFMLojpn2giO1odWG4wEP6xqe2GJqY/PuGivJv14EDw2FT6+S3dkUd1PPWYUxzBkqHhXo87eWOxIoYE/cNRXZrH9wfshgRt44Brx4Sovz24Pc9QuMvUwliPa2ghSsYFPb80gsuG937T+4IVh92GBX316JAhR6jGMG9M0Ame38M78wwt9SrPEww9Pz1Yk4NhKZigGZaCCZphKUyNhx7sFjFJ0ZnqFDhdt3w180sicCmmZL9qdqA5mtnl9OcNDlOzdhsanNRsZDbitUuoVMRhiARYU5d8+zcZilwiphfKzy8sclhvwBMFACqLI/AqZiN3tjpMsUM4hamCnl0cxTMz5eCW6oANC1drB+2o+d1ZbQmMZjJ7Q16Y3UrFcsz/sBgHOrJb0ENyJCyrlAOiwiJQtjKzBjqT/TEqj1SMZp6Y1k6FN9y6xY9VR1N3F9XDVEG3RTnsaJUjyI50Ghfi3nYH6sOyoNPdJSkRu9ocVNBTJEGgT7YREUHd22h6MT0JCcZ4Ko+msPEv+b6Ag4rdaDO4cZFRTBX0Z/VufGZgGCgRV6cRSpgsensKZiNHQnZckqYJjx7bWpxp53Hj5tT3HEyF7P5dZTAMwgTNsBRM0AxLodmGTrQKQguj7paZQJCMlVMyEGcg58EZvheZJqYRQ9MbEZHrsR21FkZiMDKFSIzrTAvNAH9u+H+ZKlEXT1B7RR31t9P/PphaUziwIVCP63Q/O2vUj+cIDl9JP7/RKwZrBrBlAnLk3v40mvmuQuDgCNZPWEf9dd4389G9MMoaos42WBvaJNychPUT1oEovPcIAdZPWAcXZ8KgMQMAE7Qp5PIxfFqxHjHCYc7uBZizewEIASpP/vuD8g3w8YL+hRiGYU0Ok+A4AKS7YfH5hLXxVeVrK9bBaaCjxjBGWoJ+c0ETpiiCjx7akYc/H/TG05cO68Lvz5JjOWqDdlygMCpJhk0XNaDELXeDr9tYSAXI3DU+iNvHd8TTa+td+PEmeXZxsFvEhotkMxSJAGPfomMe9l5aR62MWPRxMWVD8IcZbVgyVF7c8HKtF4/uyounpxVE8cYC2fSFSCKaDpz8N4Bz9szHuorutvSivWdjdfnn2L6kHhwv/0BeubYIO1qd8fRDk9tx9Wg58H3V0Rzcvc0fT4/xCfhoUWM8HZE4VKwaTNWr+rITVNDVnA9K0KAIDHq5sgVnl8irfZ7c68PT+2QDnQWlEbw0pyWeruuyYe6HxmJxPl7UiFE++dfozi/8eOeYHMtx7ehOLJ0sr3r5usWJ760rMpSHkrQE7eBBBfWojUx4jj7uMLic5tRnlNdQd6VsqjLY1Y0oVRkSGc04dfKwc0SVB10PTpXHqSzsIPik/PP43z8++W8XJ8HJE/BaearrxWnnSRKs83HaaNMcdR7qe6te3cKDPm50OVSiPNQasWVAI0rSGrbLdUjUHoIhgaPMUpw8oUIHRQLK1yOZYbt8h0TdhGCMo/bac9sI5fgTk0AFw3Ag8DvparSqAmQKnLTK26McVQavXaLMUiIiqPBSG0eQpzCiyeVjWF62CVp8r6YSnZL8PgnEOGoc2GOj3ZyiEtCpqBfPEeQ7jNWrLcpR6zV9dokKyuoSOYRF+biDI/Ap8pAIKCOhZIbt1M+vU+Co4y6eUGY3AukOitLCtGE7vYyjEodoNL3hqXadPMKqh6CGgEOrThnUQlDTKfDo1DguEjoPgdfva7dFeXRIvZ8XEnmENCYcJJJ+vfQ2MI0lkYcees8vInGIpJmHEjbKwbAUTNAmEZF4XF87HRIBrqk5E4QA19acCZEAN9ROQ0gyb9XGdxk2bGcCIcmG27+dgqquXNx6cCr2h3245eBU7Av7cOvBqdjdZcx7hJE8moJ+pbJF63APXj3gwSdprmt7eW6rqTsrSQCu32hs0cCtZ3Rg1qCo/ok9aNX5f/JsanTieYV3czIsq2wx9Sc4CddgXc4fEsY/j8qcL4emoM8bkrwjEQCsqTO2gDIRRlyQUiHRsJ0eE/wxw/ci0wRixtVz7uCIIaOZ/mC4R8zovWVtaIalYIJmWAomaIalMHWUY7RPwIJSuX3UFuWx4oh5ngypcs3oTqqt+ffDOZo7Fexus2NLk7PX45lgTnEU5fm9R+TlOyRcebocXyIS4NUD3l7P7y+uPD1EzWh+Vu/S3a4jHUwV9OSCGB5V2K1WB2xZKehfTA5Q08xr612agt7c5MQvdhhzMDXKY1PbNQVd7JaoexsWs1PQd47v6GE0M2AFfaKLxwfHXIp0dk4mfHTCDYci+KfTBDObTNMhcNS9jfXDesBkWN/gQnVAFrTZGjBV0FuaXNhiwC+6v7hlS9+aoWSCui4bbthsvglPujxg0Is7XVinkGEpmKAZloIJmmEpNNvQjTpuk36n1GOrYiOQJPLINP1hhtNfNIX5HitEsp2YpO9QOlTjmKagp72r7QX89sImTCtMfXeoqMTp5sFInTPfG3j3dmerA5fpuMFqvZNYk4NhKZigGZaCCZphKUxesZKotaPupajP0TuuPifd48mek21kol5m3Lv+vbemCvry4WE8PUM2mqkJ2rFwtWw04+IJai6nl8GPeJO2Mdh2cQNK3XJU/j99Xoh1CqOZfyvvwF3lstHMp/UuXKPYxmJwjoQvFstGMyIBRq6gjWZqL6+jbAoWfFSMAybGG2SCslwRn56vNJrpuWnQoSvrqJ/gs94roXag+uu8FswvkVfi/HaPD09WyUYzC0sjeHWuYtOgLh4zFZsG8RzBIZUNxZiVtI3B2gsaMdonT33ftnUAbxoEqJfp9Py2JrOMR/McTv8ayuO9uXBlYjlRX5P2vTN4PNGpmShDJjFV0G8dcePdo7I9lVpLEQkY9SZtX6VeITX7/RLqRgqqi/x+jw9P7ZXX2qk/X9fF98hDzfi36OOxDI1V2zngk8py6m/nbtybkbHwmqBNt15lK7Trdc2GQuoNri7XZ/UuKg91sSXS8/mp8zhvdbHm88s0pgqagNMRh95xUC5JiZDAQUozDzMi1Vw8h/dmj4dDZTrz0ZxyXLy5ChHtQidB+vUSCQct8/y+eH6Zho1ymIDPxuOtmePg4DhctKkKF22qAiEEizdV4Yqt+7H8rLHw2titN4Ps7vkMUDgOyHPYEJMIgoKIlTPPAAAEBBF/nzEWg5z2rB9DGaiw14TJEAC37jwYT//Lrm8RTbu5wegNzTf0R+c1ah2mfH9TwckTvLOwKa1rGEUCcJFBj+pUsXPAK9PGxNOvTBsDAsDRRxFDH5zX2OdvrEvWDEqrT1KeL+jqTgtNQVf4zd02geuDPNSkYjSTKhzHYaxPdpIq86XnKmWU8nyhz41mTu1ckCoeO0lLE6zJwbAUTNAMS8EEbRIxScIvq49DIgSP7z8GcvL/IiH4VfVxhFnH0BRMHbab6I9RZihNYR7/vd+Yg2ZfcP/EALU3y7NVXjRHU19uHxYJnj5Yj7frWpHDc3invg0+uw3v1LfBY+Pxdn1r2rOFg1wibh0n7ysgSMDj3+RpfKJ/uG1cB4pccsfl74dysLvdYVp+pgq6LFfAjWPlm14dsGWloH9S1kkZzfzlgAfNqbjnniRGCN443m1F/PrJ///fse5dspYfN2ZR3Bt+J6HubVjMTkF/f0SIMprZ3uIYuIKuDdrxp2pPPN0UyU6jmWW1XtgUUUvtOta1E/IF3FCmtetK+lTkay9ta4ty1L3t6ynmZPnbIQ+KXLKga4PmzuWZevVdbQ7sautbo5FUeGyXsTfb7OIoZhen8QrPAE0RGx7emf339pl9ffuLzDqFDEvBBM2wFEzQDEuh2Ybe226sia230WMyGM3TKKkM/x4P2Uwvlx7HUnDtrGq3m240o2X5mwytUd7wvdUymklra+R0SWZr5IENwbDCcDx1tMWNYYVhHGtxW2Jbt2S2RjYD07ZGZmhBMMQfwXv3fBH/y6LHZ+F/btyJH78wGcctIupsg7WhTcLtkPDhfVsBAIQA7SE7Vv/7Flz19DS8cvMOeF1ai58YqcIEbSKhKI+uk/2KC56YBQJg7YObccmvZ6Ajwn4czSCtu2rj6B9NiYBq/3IgPeJxjc5o2VW+AyIB9VPNg1AdH4LuxZ/Kv9h1yqDOo3tlsnxOj3oCkIh2PcMxG2YvnQeOI9j22Of4/KGN1DV068UR6m3TJ/XKyudnjLQEvfKcZsp99D++zsMrio1rLhsexjMzZaOZ6oANC1eXGMpj8+IGDM6Rg1t+tJ42mrm7ogN3K4xm1tT1NJrZdrHCaEYCRqiMZvZdVkfFcsz/kDaaeWpGGy4bLnfu/qfGQ20aNK0whlULm+PptiiHiW93L+8nhMOZ/3E2AGD7E+vij3LHknrkO+WHvWRNEba3yjtrPTIlgOvGyFsGrzjsxu1fyFtnlOWK+OwCeWVHWOxpNFN7RR0lyDPfpY1mXp3bggWl8oznb3bTRjPnlEbw53kKo5kQjxnvG3M0/eT8xh6bBimNZq4bE8Ijis2PvmzWdx/VIu3fPb19uenjqX3zNPMgxsrQ26nG6mH0eOKDWp8h0C93MnuiZ7JeqQ55pVsGI6Q5bJeeN1pyw3bZ4G2X+nGOI9j++PqT/+4+MuPBuYgk3DZu4NQLSHbYLhP1oDFx2E7vjZuJYal080imDJnPw+0QsfmRDd1HE358YNbLOH2RhwzraptIolm6GQ/MRURgg0tmwe6sSYRjPCofqqT+NmvpKTGzCRWzYG9o0+DQEbZhrkLU3WPSTMxmoinory6pN3SxX36Ti+WHPPonZpCbz+igliJtaHBSw1vJsPmiBjhtcufkis+KcKgz+e/6JH8Uryh8lFPh6s8LDS1NGukT8OYCeagwInKY84GxIdFnZ7ZijmKhwnP7vfhTdd8G5P9wZAg/nxDM2PU0n1qJ25grS44tg+MvSeK1E6qcyrHdZCl2i9Q4tFFzFgdv/F71vIaxcts5Os9wCjPpfqdEXcPbD88vx0bSvndKWBuaYSmYoBmWggmaYSlMHeWYNSiCG8rkeIQTXTyW7jC2UvnX09vgV7SLf7/Xhz0GOk9+p4RfT2+PpyUC3LTFWKfxp2M7MLNIjln5uM6F1781t/P7w5EhnDs4Ek9vaXLixRqvxid68sdZrdRY+M++zEd7whnKxEz0x3DneDlOpjXK4Z6v/IbK8OiUdioW54VqL75odmp8Ij1MFfSQHAmLh8pBPdUB48uIFg6OUDfkzweMCcltI1QZUnEfnVoQo65xvMv8H7YJfjrP7k6fMUFfNDRMdXAf2J6HdgM7WRe7RKoMJ0LG6z2/NEIFJ7191FwHVlMFvbPVgQe3y54XepuSJ+LXu3Phsctv6BqDRiWBKEeVIZVAmOWHcrBN8VbZ0wfrC9856qZMWVIxaPnFjjxq1DugY6CjZn/QTt27TsH4GPofqnzId8g3fVebea5JgMmCPtBhT3u/v9fTHNcOiTyW1Rp7s6lZW+/GWmND8mmzucmFzU0u/RM1eDnNeh8L2bGsNr3n9+bhvp2XYJ1ChqVggmZYCiZohqUw1EBaW++EoOG5cDRkbBRDIsAnJ+h24kC0AW+P8T3qYZSAgeG0bIEkeH5GjXyOhGya987OE2qZmB6GBH3LloKM3vgY4XDtxkL9E7Oc2qDdEvUwCkH6z2/1CTdWn+h9KC/fIWH3pcn3yAfea4HB0IAJmmEpmKAZliKtUfNch0SZnYQEDhFFp9HJE3gVs3wiUXd+CApU8cutUQ7KVR35DomKRwjGOENmJxwIFQvSnYex77HXLsGp+EhE7J6wOYWNI8hTzIYR6M+K+p1Sj1k8pcGKxyZRMdpRCeg0uBaxwEnP87dFOUN+eg6OwKeol0SgigUx/vw6BY5aFe7iCTUTLBAgmEY/LS1BvzavRdNo5uKh2kYzLh7Y9Q90g7/bxkDmk/MbNY1m9ChNwmhGj19Nb9c0mplS0LvRTG9suLBB02jm/klBTaOZZNi+pF7TaEaPeSXaRjM81/P5ddsYyOlVC5s0jWb+aVQWGc3EpO631SnUQzYSoY/HEgz5RXRWWsQkjjrH8LAe0S5jMgiELoN66JKo8kjGTra7XvRbncpTdW9T2RQoKqKHzZYRJHDa9SLGn5/6/otJaMQImkYzx+66izpYsao068ZL/60iqGkFlgy1l5/QtALLRspyBV0rMD3+Oq9Z0wosG0g0bDf0ySd7VX12qZPBSBMmaIalYIJmWAomaIalMLXnc+mwLvzurDb9ExWcsbLvNw2qWEUPsUUNLtOaWhDFGwrTl1S4cm0RdrYmv9auNmjDmBXaQ4Nmw4Ng/+V1+icquPMLP949lqN/YoqYKmieA9zZub03RSTNoaJM1NPoTyUBh0jm/FlSI4V6GzXxMQprcjAsBRM0w1IwQTMsRVpt6N+d2YYz8uStcZ/f78U7GW7wvzq3BYWKIJsHd+Th65bkO09FLhGvVMrxCBKASz+lYwVWLGiCQ/HVvmlzAbUV8T0TAphfIs+orTrqxgsmu3TefEYHlig8MT6rd+E3e+RZvGEeAc/PkjvcUQm4ci1dr7cXNlHd62s2FKAlmnyj98zCKB6eIsdZNEV4XJfhhQyXDuui3GOrAnb87EtjZjZK0hL02DwBUxXBSUWuzPdSKvJjVHBSrt1YRIKDB1XGREYzkwti1NS3S+XCOcIrUtf4ssVcbwkAGOah8zzYQQvRbaPrlch9dHJBjOqEOQz+Huc5JCqPVIxm9BjkovMQ01yDl5agH92Zh3zF27PKBAOWf93mpwS2u81YHq0RHtdvlKPUEoWu3Li5gAriqVM5Iz2/34eVR+RfnsOd5g/d/OWAB2vr5ajCE6rN60900fVKJISfbCyg9ncxavSzs81B5REWMz9EsfqEC0dCch6pmBEpSUuBW030KDuFkVDRREQkTnPNGgB8Uqd9fFebw3THHzVVAQeqAr3n2SnwuvX6WKdeejRHbFh9wtwv75GQHUdCmXsRsk4hw1IwQTMsBRM0w1JkdxR7Cgz1iLhmdKf+iRqsPJKTdQsZ1OQ7JFw2vCuta5yW099z58D4vBhmDpKHRBvDPN4/nvrQr+UEPS5PwOPTAvonavB5gyvrBV3sltKuZzZQWRztsaYwHUFn91NjMAzCBM2wFEzQDEuR9W3okV4BdsXX7niIp0xe+oJSt4hcheFKW5RDU8TcCYdil0j5dgRjnCFPjUzgsUk4zSN3HAUJ+NbADrv9QXaXDsAbC5rTMprJBEsnBzSNZszgzvKOtI1m0mXWoKim0Uw2kvWCbo3y1Bs61g8jTR0xDk0RuRBGLblSylOg8wz2QZ5qohJdhpY04ywSERbpPIxsO5eIrBf0+R8X93cRcO/Xftz7dd/m+ctv8vDLb/L0TzSRDY0uTH3H3Dfya9968FoG93xknUKGpWCCZlgKJmiGpTC9DZ3Kzq3ZmEc2luG7Wm8tTHYfTaW2RldFmHVHjZQjU2XojzzTKQNg9jM26j5q8hu6LxyQ+tZlKTH9UYZsqDeQPeXohrWhGZaCCZphKZigGZYirTb0ssoWTMyXPRV+tScXfzskz/pcdFoYj05pj6cPdtrx/XVF8bSTJ9hwobyhDwDMfL+E2qnpvXObUOySTSf+5Qs/tjTJsRw3ju3AT8vkFSobGl24a5tsVFLiFvHuwqZ4WgSH2e/LGxcB3Rv4OHm5c/OP64pwWBGE859T23HBEDmW4/++9eC3e2XTl0n+GF6a0xJPB2I8ztOZ4VxzfiNy7fI8/nUbC7G7XV7l/fOKIL4/Qo7leP+4G0sV8SMjvQL+Nl92PI1IHOZ9SNdry+J66o21eM0gKqjq6RmtmK1YLfLHah9erJE3faosjuAPCvfY+rANSxQmPTwItiymn1/lhyXUPilvzG/GCK9sRnT/9nxqtfoPRoTws4pgPL2zzYGfbErdzCYtQQ9ySRiiiMbyqAxa3DZCHe8QaDcUDqCOn/qb8iolbpEKTnKpflNyHXQehSqzG56j80hkNDM4R6SMZuyqfo7fSdczT7VdmoOny5AT1e/5l7rpaDoHT38mX5Wneos2O0/XK5HRzOAciTKaUTt/Fqqen89O5+FS1asHXOLnp6TYLffgBBAAAAWjSURBVNL3RqURr53O43hXesE6aQ3bjfYJVAFPdPGU1VS+Q8Iwj3ynIxJQE5TfQhwIKvLlby8A7G63Q3lbxuXFKIEd6rShQxGoU+IWUexSfmk4HFK8Xe0cwbg8dR6030VFfox6ENVBO7Xj0zCPgHxF+GhzlEedwvglxyZhtE+up0ig6akBAOV5Mcrc5kCHDV2KsNghOSJlgdYe43BU4V/h5AnG5sr1IgD2qOo1QfHrCQD7AnZqN60RXgE+hRNVQ5hHo+IN7rNLGOGV6yUQYB9VL4IJque3p91O/cKOzY1RezweDdmoAKRCp4ghihdWl8hRGzYZHbYb8LtgMawN2wWL8Z2GCZphKZigGZbC0CjHzyqCae9HwmAYwcUbixUxJOgfl4X0T2Iw+hFNQb+0fXtflYPBSJoHNY5pDttxHGc4NrCsrHumMByWUF7ePevU2hrDtm3J21YtWlSETz9txtlnF8Lh6G7ibNrUho6OBLMHjD7Bw/Oo9PsRIwRrW1v1P2AihJC+CR8dPToHc+f6AXDw++1YsqR7KvbgwRCi0cPYuTOofQEAlZV+VFb6EYlIuPvukfB4ugf6n3vuMFavbkIwyETd13h4HnP8flT6/YhKEqKSBAJgc3u77mf7moyOckyalAtBICgsdGDWLD++/ba7zT1qlAc33TQ8qWtcemkJ3n23EffeOxo1NZ2IntzW9ZZbTsfw4ebtQMronXy7HVNzc/HIgQP47yNH8MDo0fiH4mJMzc3V/3Afk1FBv/VWA1wuHhdfXIx9+zqxdm0rgkEBe/d24ODB5Kxf77tvPx59dCxcLh6vvXYCXV0iampC2Lu3A11d7O3cH5yIRvHbQ4fi6agkYWltLa4ZMgTjPJmzIMgEpq1YmTIlF8OGubFmTTOeeeZwSte49dbT0doqYOnSajQ2RvU/wDAVG4DT3G4cDochEoKnDh/Gc+XlWPx1H5uWaGCaoD/7rAVPPXVI/0QN7rprL2szZwk2AGM8Htw2fDjuqKrCMJcLz1dUgBCCUqcT9dHseOGYNlPocvEoKLDD603dYNDvd6CgwA6ezWf2O6UuF34xZgweqKmBi+fx4oQJAACO4/DKxIn9XDoZ06RywQWDsHz5NNx//2h4vTbk5BjP6qWXJmH58mkYNcoDr9fGhN1P8Oge6ShyOPCnigp4bTZ0CN1ho4SQ+L+zgYxLRBAIBEGOb50504+VK6fjN78Zn/Q1IhEJyvHx55+fgJUrp2P8eHO3I2YkZmRODp4pLwcA5NnteLGiAlft2oWwJCEiSfjezp39XEKZjAv61VePY9myYxAVW5sSQqi0Hldc8TU6O0VK1KJIoDUJxDAPAkA8ee8JIRBO/nf59u24fMeO/i2ciozPFJ7ikkuKcccdIwAAO3YEcc89+wxf4403piI3t7vfevPNu5Me+mNkntNcLrxYUYGAIOCqXbv6tSxaM4WmCZrBMAstQbNuFsNSMEEzLAUTNMNSZP2WFAyZ2eUFeOS67uGzww1duOG32TPlnC2wN/QA4Zwpg/DwteNRfawD9724G2VDvVj282n9Xaysgwl6AHDhWSW45wdl+ObbIP79xT3YWRvAnc/uxKghTNRqWJNjAOB28sj1OBAVJARC3dPMbR0x2HgOhXnOfi5ddsHe0AxLwQQ9AFi/qxkvfXAIk0fn4a5/HIPTS3Kw9OrxaA5Ece8Lu/u7eFkFa3IMAFqCMazaWAcQ4HvzT8O4YT7ke+146JUqVB3p6O/iZRVs6nsAUZTnxKRR3bvLhiICtla16XzCmrBYDoalYLEcjO8MTNAMS8EEzbAUTNAMS8EEzbAUTNAMS8EEzbAUTNAMS8EEzbAUTNAMS8EEzbAUTNAMS8EEzbAUTNAMS8EEzbAUmvHQDMZAg72hGZaCCZphKZigGZaCCZphKZigGZaCCZphKf4fTFtUZFlYIHAAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aNtfcAKYuM8E"
      },
      "source": [
        "env.close()"
      ],
      "execution_count": 15,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "O2b-u_Yju3a0"
      },
      "source": [
        "### Um ambiente simples: the Cart-Pole"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "DYLy9XvLu-Bk",
        "outputId": "08d60864-0150-414b-a905-94234a97e308"
      },
      "source": [
        "env = gym.make('CartPole-v0')\n",
        "obs = env.reset()\n",
        "obs"
      ],
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([-0.03205541,  0.00267524, -0.04580721,  0.04114281])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 16
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2dhOGxJMvgL0"
      },
      "source": [
        "#### Corrigindo o problema de renderização"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rxx_bgPpvgV7"
      },
      "source": [
        "from PIL import Image, ImageDraw\n",
        "\n",
        "try:\n",
        "    from pyglet.gl import gl_info\n",
        "    openai_cart_pole_rendering = True\n",
        "except Exception:\n",
        "    openai_cart_pole_rendering = False\n",
        "\n",
        "def render_cart_pole(env, obs):\n",
        "    if openai_cart_pole_rendering:\n",
        "        # use OpenAI gym's rendering function\n",
        "        return env.render(mode='rgb_array')\n",
        "    else:\n",
        "        # rendering for the cart pole environment (in case OpenAI gym can't do it)\n",
        "        img_w = 600\n",
        "        img_h = 400\n",
        "        cart_w = img_w // 12\n",
        "        cart_h = img_h // 15\n",
        "        pole_len = img_h // 3.5\n",
        "        pole_w = img_w // 80 + 1\n",
        "        x_width = 2\n",
        "        max_ang = 0.2\n",
        "        bg_col = (255, 255, 255)\n",
        "        cart_col = 0x000000 # Blue Green Red\n",
        "        pole_col = 0x669acc # Blue Green Red\n",
        "\n",
        "        pos, vel, ang, ang_vel = obs\n",
        "        img = Image.new('RGB', (img_w, img_h), bg_col)\n",
        "        draw = ImageDraw.Draw(img)\n",
        "        cart_x = pos * img_w // x_width + img_w // x_width\n",
        "        cart_y = img_h * 95 // 100\n",
        "        top_pole_x = cart_x + pole_len * np.sin(ang)\n",
        "        top_pole_y = cart_y - cart_h // 2 - pole_len * np.cos(ang)\n",
        "        draw.line((0, cart_y, img_w, cart_y), fill=0)\n",
        "        draw.rectangle((cart_x - cart_w // 2, cart_y - cart_h // 2, cart_x + cart_w // 2, cart_y + cart_h // 2), fill=cart_col) # draw cart\n",
        "        draw.line((cart_x, cart_y - cart_h // 2, top_pole_x, top_pole_y), fill=pole_col, width=pole_w) # draw pole\n",
        "        return np.array(img)\n",
        "\n",
        "def plot_cart_pole(env, obs):\n",
        "    plt.close()  # or else nbagg sometimes plots in the previous cell\n",
        "    img = render_cart_pole(env, obs)\n",
        "    plt.imshow(img)\n",
        "    plt.axis('off')\n",
        "    plt.show()"
      ],
      "execution_count": 17,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 247
        },
        "id": "kkz9ycIKvuvN",
        "outputId": "cd57455a-6315-4a04-e89a-d0314804fc01"
      },
      "source": [
        "plot_cart_pole(env, obs)"
      ],
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAVQAAADnCAYAAABBu67aAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAFn0lEQVR4nO3cT2tcVRzH4XMnk7R1jEarWOrCP6AGBBdCF+26vgA3vojuuy8uunTbVbvyNUgWrpSuAiK4EEHQVpCGlpiYtGMmmbnu0k4b2nTmmzmleZ7dOTNwfovhw+XemzRt2xYAptepPQDAy0JQAUIEFSBEUAFCBBUgpPuMz70CAPCk5qBNV6gAIYIKECKoACGCChAiqAAhggoQIqgAIYIKECKoACGCChAiqAAhggoQIqgAIYIKECKoACGCChAiqAAhggoQIqgAIYIKECKoACGCChAiqAAhggoQIqgAIYIKECKoACGCChAiqAAhggoQIqgAIYIKECKoACGCChAiqAAhggoQIqgAIYIKECKoACGCChAiqAAhggoQIqgAIYIKECKoACGCChAiqAAhggoQIqgAIYIKECKoACGCChAiqAAhggoQIqgAIYIKECKoACGCChAiqAAhggoQIqgAIYIKECKoACGCChAiqAAhggoQIqgAIYIKECKoACGCChAiqAAhggoQIqgAIYIKECKoACGCChAiqAAhggoQIqgAIYIKECKoACGCChAiqAAhggoQIqgAIYIKECKoACGCChAiqAAhggoQIqgAIYIKECKoACGCChAiqAAhggoQIqgAIYIKECKoACGCChAiqAAhggoQIqgAIYIKECKoACGCChAiqAAhggoQIqgAIYIKECKoACGCChAiqAAhggoQIqgAIYIKECKoACGCChAiqAAhggoQIqgAIYIKECKoACGCChAiqAAhggoQIqgAIYIKECKoACGCChAiqAAhggoQIqgAIYIKECKoACGCChDSrT0Ax9dw0C+7/a39ddPplIVXT5emaSpOBZMTVKr554+fyq0fv91fz596rXz61ddlbv5ExalgcoJKPW1bSjt6ZDmsOAxMzz1UgBBBBQgRVIAQQaWaxbOflPneG/vr4U6/bNz6ueJEMB1BpZqFxdNlbv7k/rod7ZWdzbWKE8F0BBUgRFABQgQVIERQAUIEFSBEUAFCBBUgRFABQgSVippycunM2M7Ov3fLaLhbaR6YjqBSTdM05fTH58f2Nv78uQx3HlSaCKYjqAAhggoQIqgAIYIKECKoACGCChAiqAAhggoQIqhU1TSdUprm4UZbSjsa1hsIpiCoVLX47nJ55a339tej4aCs/fJ9xYlgcoJKVZ25+dLpdMf2Rns7laaB6QgqQIigAoQIKkCIoAKECCpAiKAChAgqQIigAoQIKtUtffj52Hr7zu9l8GCj0jQwOUGlut7b74+t/9tcK8Odfp1hYAqCChAiqAAhggoQIqgAIYIKECKoACGCChAiqAAhgkp1c/OnytyJ3sONtpTB1r16A8GEBJXqTr15tiye+eiRnbbc/fWHavPApAQVIERQAUIEFSBEUAFCBBUgRFABQgQVIERQAUIElRdD0zy20Za2bauMApMSVF4I73z2xVhUt/7+rTy4d7viRPD8BJUXQvfRv+UvpYz2BqUd7laaBiYjqAAhggoQ0q09AC+vfr9fBoPBob472Np6Ym97e7sMNzcPfV6v1yvdrp809TTPeJLqMSsTu3z5crlx48ahvvt6b6F8ef6DsYf9363eLmsb/UOft7KyUs6dO/e8Y8IkHn8tpZTiCpUjdP/+/bK+vn6o766vl/LNX3emOm9310Ms6nIPFSBEUAFCBBUgRFABQp76UOqwr7zAQUaj0UzP29vb85tlJhYWFg7cf2pQr1y5chSzcEysrq7O9Lzr16+XlZWVmZ7J8XT16tUD972HypG5dOlSuXbt2szOu3nzZrlw4cLMzuNYO/A9VPdQAUIEFSBEUAFCBBUgRFABQvxzFI7M8vJyuXjx4szOW1pamtlZcBCvTQE8P69NARwlQQUIedY91AMvawF4kitUgBBBBQgRVIAQQQUIEVSAEEEFCPkfiIex/w0YiIIAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "2oMTUShPvyXk",
        "outputId": "b8ac4d71-6a52-4c87-988a-7dfd97233721"
      },
      "source": [
        "print(f'Espaço de ação: {env.action_space}')"
      ],
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Espaço de ação: Discrete(2)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "T9BlzCrNwBJl"
      },
      "source": [
        "Empurrando o carrinho"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 265
        },
        "id": "qYyZQxusvyaS",
        "outputId": "cc2fe8c0-aed4-477c-a0e6-ec8975584b2c"
      },
      "source": [
        "obs = env.reset()\n",
        "while True:\n",
        "    obs, reward, done, info = env.step(0)\n",
        "    if done:\n",
        "        break\n",
        "\n",
        "plt.close()\n",
        "img = render_cart_pole(env, obs)\n",
        "plt.imshow(img)\n",
        "plt.axis('off')"
      ],
      "execution_count": 20,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(-0.5, 599.5, 399.5, -0.5)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 20
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAVQAAADnCAYAAABBu67aAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAIvklEQVR4nO3dTW8cBxnA8Wd217uOHadJbGxEW0gdUAqoKiqqhGgPSARQv0A/Ax+glx4qVT30Y3DkiMQtSBTUAhUtBYE49C1NobRp2iZuXhyvX3Z3OATaTmwltf3MTuL9/W5+Ns4+h9U/k9nZnaIsywBg/1pNLwBwUAgqQBJBBUgiqABJBBUgSec2j7sEAGC7YqehI1SAJIIKkERQAZIIKkASQQVIIqgASQQVIImgAiQRVIAkggqQRFABkggqQBJBBUgiqABJBBUgiaACJBFUgCSCCpBEUAGSCCpAEkEFSCKoAEkEFSCJoAIkEVSAJIIKkERQAZIIKkASQQVIIqgASQQVIImgAiQRVIAkggqQRFABkggqQBJBBUgiqABJBBUgiaACJBFUgCSCCpBEUAGSCCpAEkEFSCKoAEkEFSCJoAIkEVSAJIIKkERQAZIIKkASQQVIIqgASQQVIImgAiQRVIAkggqQRFABkggqQBJBBUgiqABJBBUgiaACJBFUgCSCCpBEUAGSCCpAEkEFSCKoAEkEFSCJoAIkEVSAJIIKkERQAZIIKkASQQVIIqgASQQVIImgAiQRVIAkggqQRFABkggqQBJBBUgiqABJBBUgiaACJBFUgCSCCpBEUAGSCCpAEkEFSCKoAEkEFSCJoAIkEVSAJIIKkERQAZIIKkASQQVIIqgASQQVIImgAiQRVIAkggqQRFABkggqQBJBBUgiqABJBBUgiaACJBFUgCSCCpBEUAGSCCpAEkEFSCKoAEkEFSCJoAIkEVSAJIIKkERQAZIIKkASQQVIIqgASQQVIImgAiQRVIAkggqQRFABkggqQBJBBUgiqABJBBUgiaACJBFUgCSCyliVo1GMBltNrwG16DS9AJNl49rFOP/ar2Nm4esxf+qH0e70omh3oij8287dryjL8laP3/JB2K2Vs6/Gu7//RUSU0Tl0JIqiFSd/8vOYXXyg6dVgN4qdhg4LGKtP3/1r/P/f6UH/apSjYbSmes0uBUkElbEZbq3HYGOtMptdfCAOHftaQxtBLkFlbPor52P1w7cqs8704Ya2gXyCytiMBhs3TYpYeuh0I7tAHQSVsbnw9zOVn4tWK8K7+xwgXs2MRVmWUY4GldnRBx6J6XsWG9oI8gkqY7H64duxdvG9yqzV6UbRaje0EeQTVMZiuLUeo8HmZz8X7U4snHqswY0gn6BSu7IsY+XsnyuzIlrRPTzf0EZQD0FlDMror5yvTObufTDa3UMN7QP1EFRq17/0fgzWVyuzmfn7o+0TUhwwgkrt1i79Jwbr1z77uWhPxaHj9za4EdRDUKlVORrF+uULlVl7ajqO3P/dhjaC+ggqtSpHg1g5+0pl1j18zNf1cSB5VVOrwcZalKNRZbZw6vFodboNbQT1EVRqdemtlyvnT6NoRdGZam4hqJGgUpsbHzcdVma9ufk4vvxoQxtBvQSV2gw3rscnr79UHRYtHzflwBJUalOWo21f2bf00OmIYse7R8BdT1Cpzebqp9vekOpMz0UhqBxQgkptLr35pyiHn98yumhP+XQUB5qgUotyNIzRcLMym5m/L+bu/XZDG0H9BJVabFy7FCvv/KUyK1qdhraB8RBU6lGOtp0//erDP3P+lANNUKnFR//8bURZDWprarqhbWA8BJVa3Px1fbNLJ2Nm/r6GtoHxEFTSrV++ENc/OleZdXozvlCaA09QSTfYXIut/pXK7OiJRxraBsZHUEl37f3Xb5oUMbt4oolVYKwElXRXP6gGdfroUrS7Mw1tA+MjqKTaWruy/Q2pxeXozh5taCMYH0El1fWL/471yx9+YVJEd87topkMgkqasixj0L9WmRXtTiyceryhjWC8BJVE5Y0L+r+gPdXz6SgmhqCSphyNtn06av7UY9E5dKShjWC8BJU0l9/9W6xf+bgyK1ptR6hMDEElzWi4VTlCbfdmnT9loggqKUbDrbj4xh8qs6JoRac329BGMH6CSo6yjM3VTyuj4998NFpuGc0EEVRSDLc2orzpDanePUvucMpEEVRSXHrr5Rj0r35hUogpE0dQ2beyLKMcDiqz7tx8HD/5aEMbQTMElX0bbq7FxTf/WJkVRStabedPmSyCyr6Vo1EMNtYqs69850cRhZcXk8Urnn1beefVGA2qt4zuHj7mgn4mjqCybxtXPq5c0N+dW4iZhW80uBE0Q1DZl8H6alz/5F+V2dT0XHQPH29mIWiQoLIvg43rsXbxvcpsdulkQ9tAswSVfemvnI+IsjI7euJh50+ZSILKvqycfSWi/Dyo7d6s+0cxsQSVPRsNNmO4tV6ZHV5cjpn5+xraCJolqOxZf+WDuHbTHU7b3UMNbQPNE1T2bDTc2jZbevinDWwCdwZBZc8u/OM31UHR8oUoTDRBZU/KsozR1kZldvTE96J3ZLGhjaB5gsqeXP/onVi7VL3+tD01Ha12p6GNoHmCyp4MN/uVI9Si1Y75b/2gwY2geYLKrpVlGSvnXqsOi1b07vHffSab/58RERFbW1uxtrZ2+z8YN4L68Xtvx/raZkz3OtFpt2Lq2P2x2t+M9uDKl/o7er1eTE9P72dluOMUZVne6vFbPsjB8cILL8STTz75pf/8cLMfUY7ix99fjge/vhBX14fxy9+98aV//6mnnoqnn356L6vCnWDHz1Y7QiUibhyhrqys7Pr3fvXS69EqipiZnorV/ubtf+F/+v3+rp8L7nTOobJvo7LcVUzhoBJUgCSCCpBEUAGS3PJNqc1N58UmxWAwGOvzDYdDry/uWt1ud8f5LYP67LPP1rELd6Bz586N9flefPFFry/uWs8///yOc9ehEhERZ86ciSeeeGJsz/fMM8/Ec889N7bng2Q7XofqHCpAEkEFSCKoAEkEFSCJoAIk8eUoRETE/Px8nD59emzPt7y8PLbngnFx2RTA7rlsCqBOggqQ5HbnUHc8rAVgO0eoAEkEFSCJoAIkEVSAJIIKkERQAZL8F1Zzu5DfCS25AAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 247
        },
        "id": "99PdGmuHwbBY",
        "outputId": "e5f1e953-87ff-4f30-e114-e0b58543240e"
      },
      "source": [
        "obs = env.reset()\n",
        "while True:\n",
        "    obs, reward, done, info = env.step(1)\n",
        "    if done:\n",
        "        break\n",
        "\n",
        "plot_cart_pole(env, obs)"
      ],
      "execution_count": 21,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAVQAAADnCAYAAABBu67aAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAIoUlEQVR4nO3dT28chRnA4Xf2j9d24thATGjUEhK1ShEUIg7tBRWprXopn4BPw4kzn6O3HjhQVa2oaKugIgqUEqIUEhCBxsZx4tjJ2rs7PQSSTJISh7zJdGee5zZvVtr3sPlpvTs7U5RlGQDcu07dCwA0haACJBFUgCSCCpBEUAGS9O7w704BALhVcbuhd6gASQQVIImgAiQRVIAkggqQRFABkggqQBJBBUgiqABJBBUgiaACJBFUgCSCCpBEUAGSCCpAEkEFSCKoAEkEFSCJoAIkEVSAJIIKkERQAZIIKkASQQVIIqgASQQVIImgAiQRVIAkggqQRFABkggqQBJBBUgiqABJBBUgiaACJBFUgCSCCpBEUAGSCCpAEkEFSCKoAEkEFSCJoAIkEVSAJIIKkERQAZIIKkASQQVIIqgASQQVIImgAiQRVIAkggqQRFABkggqQBJBBUgiqABJBBUgiaACJBFUgCSCCpBEUAGSCCpAEkEFSCKoAEkEFSCJoAIkEVSAJIIKkERQAZIIKkASQQVIIqgASQQVIImgAiQRVIAkggqQRFABkggqQBJBBUgiqABJBBUgiaACJBFUgCSCCpBEUAGSCCpAEkEFSCKoAEkEFSCJoAIkEVSAJIIKkERQAZIIKkASQQVIIqgASQQVIImgAiQRVIAkggqQRFABkggqQBJBBUgiqABJBBUgiaACJBFUgCSCCpBEUAGSCCpAEkEFSCKoAEkEFSCJoAIkEVSAJIIKkERQAZIIKkASQQVIIqgASQQVIImgAiQRVIAkggqQRFABkggqQBJBBUgiqABJBBUgiaACJBFUgCSCCpBEUAGSCCpAEkEFSCKoAEkElcYrJ+PYubwR5z9+OybjUd3r0GC9uheA+2Vz9Uxc/OyD2N48H2un3oreYD72ff+piK6XPfeHVxaNtXXudJz9+++uHW+PhrF68i9x4Olf1rgVTeZPfhpr6fBzMbP34euDsozJzrC+hWg8QaWxeoP5KDrVP8LWTr0Vo+FWTRvRdIJKcxWdePhHP6uMtjfXopyMa1qIphNUGqsoiph76GBlVo5HceGz92vaiKYTVBptdumx6M8vXTsuJ+PYWjld30I0mqDSaLNL34uZvQ9Vh2VEWZb1LESjCSqtc/6Tt2P70lrda9BAgkrjLT/5QuV4NNyKcuIXU+QTVBqtKIqYWdhfHZaTWDt1vJ6FaDRBpfEGi8sxe9O3/ZfXzta0DU0mqDTezPxSDBYeqcyurH8Z25vrNW1EUwkqrTC//ETl+Mr6F7Gzeb6eZWgsQaUVHjr8XEQUda9BwwkqrdDpzkRvbqEyW/nXGzVtQ1MJKq0w2Lc/Fn/wdGU23FitaRuaSlBpreHFc3F57fO616BBBJXWWDh4NKK4/pLf2boQw0tf1bgRTSOotMbCwR9H0elWZlsrp/2unzSCSmt0uv0Y7FuuzM5/8k5ECCo5BJXW6M3uiaVDz1Zmk9F2jC5fqmkjmkZQaZXe7J6I4vr5qNsbqy44TRpBpVX2H30+eoM9da9BQwkqLXPrr6U2zn7kPlOkEFRapeh0Y37/ocpMUMkiqLRKp9ePxcefqcwm450YXlypaSOaRFBpnf78vii6/WvH4+FmrJ95t8aNaApBpXWWDj0bM3uqN+4bDTejnExq2oimEFRaqdMfVI5XT7wZo+FmTdvQFIJK+xSdeOyZX9e9BQ0kqLROURRRdHuVWTkZxdbqmZo2oikElVbqzy9Gd2b+2nE5HsWFT/1iinsjqLTSnkePxGDx0cpsZ+tCTEbbNW1EEwgqrdWf21c5Xj/zrjuhck8ElVYqiiIO/ORXt8wnO1dq2IamEFRaq+h0K1fwj3ISX773+/oWYuoJKq01v/zE1dui3MBv+rkXgkprdbq96HSqp0/tbK47wZ/vTFBptZmFRyrHm+c+juGFczVtw7QTVFpt+cmfx83XSN25fLGeZZh6gkqrFZ1udHr9yuw/7/+hpm2YdoJKqw0WD8TS4eeqw8nEl1N8J4JKqxVFcfX0qRtsrnwSG1+crGkjppmg0nqPPvWLKG74tr+cjKMce4fK3RNUWu/mW0tHRAw3VqMsy5o2YloJKq3XG+yNpSeOVWarH/65pm2YZoJK63V6/Rgs7K/MysnYlae4a4IKEbd8MXXlwpexfvqdmrZhWgkqRMTyky9Ed7CnMisnY5+jclcEFeLqn/3FTV9MnfvnH6OcjGraiGkkqBARnd4g9h99vjIbDS9FeIPKXRBUiIii04n+nqXKbHTlUqyf+UdNGzGNBBW+tnDwaPRmF64dl+NRbG98VeNGTBtBha/NLh6I7sxsZTYZj3wxxa4JKtyoqP6XWPnwjRgPt2pahmkjqPCNonPLjfuuntzvHSq7I6jwtaIoonfTuaiT0TBWT7xZ00ZMG0GFG+x97IcxWDxwfVCWsb21Xt9CTBVBhRv05vZFd2auMts4+5HborArvTs/BKbf5uZmjEa7+9VT7+HDESunYzIpY2u4E5eufBrLX63EYGH3n6UuLCxEp+P9StsUdzglxKfxNMJLL70Ur7/++q4e+8zh5fjNTx+Pf39+Pn77pw9iPJlEd2b+lmum/i/9fj+OHz8ehw4dupeV+f922xeDd6i0wsWLF2NtbW1Xj/3rxoX423unYrhzw1X7N4e7fq5+vx9jV/xvJUGFm2zviCHfjQ95AJIIKkASQQVI8q2foW5vu6cOzfCgL3AyGo38/2mwmZmZ286/Nagvv/zy/dgFHriTJ08+sOcaj8fx6quvxtLS0p0fzFR65ZVXbjt3Hiqt8OKLL8Zrr732QJ6r3+/HiRMn4siRIw/k+ajFbc9D9RkqQBJBBUgiqABJBBUgiaACJPFbflrh2LFjMRzu/gIn96LX68Xc3NydH0jjOG0K4O45bQrgfhJUgCR3+gx1d5coB8A7VIAsggqQRFABkggqQBJBBUgiqABJ/gtkULEsRvddcgAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "xLmYLGpUwpFb"
      },
      "source": [
        "Definindo uma política"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 247
        },
        "id": "vUsK9PW3wrxD",
        "outputId": "e8688dc1-b34d-47f0-b9b9-08ed31a4af61"
      },
      "source": [
        "frames = []\n",
        "\n",
        "n_max_steps = 1000\n",
        "n_change_steps = 10\n",
        "\n",
        "obs = env.reset()\n",
        "for step in range(n_max_steps):\n",
        "    img = render_cart_pole(env, obs)\n",
        "    frames.append(img)\n",
        "\n",
        "    # hard-coded policy\n",
        "    position, velocity, angle, angular_velocity = obs\n",
        "    if angle < 0:\n",
        "        action = 0\n",
        "    else:\n",
        "        action = 1\n",
        "\n",
        "    obs, reward, done, info = env.step(action)\n",
        "    if done:\n",
        "        break\n",
        "\n",
        "video = plot_animation(frames)\n",
        "plt.show()"
      ],
      "execution_count": 22,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAVQAAADnCAYAAABBu67aAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAFiElEQVR4nO3dz2scZRzH8Wd2t0mk0VpIURDEeKi9ePIk6K1HA57zb9RToVB66H/hX2Eu9dBLz6J4KlI9KFrEJFBTk272x3iQxq4N6az9dGfavF63eWbg+R6WN8NkdlPVdV0AeH69tgcAeFUIKkCIoAKECCpAiKAChAyecd4rAABPq45bdIcKECKoACGCChAiqAAhggoQIqgAIYIKECKoACGCChAiqAAhggoQIqgAIYIKECKoACGCChAiqAAhggoQIqgAIYIKECKoACGCChAiqAAhggoQIqgAIYIKECKoACGCChAiqAAhggoQIqgAIYIKECKoACGCChAiqAAhggoQIqgAIYIKECKoACGCChAiqAAhggoQIqgAIYIKECKoACGCChAiqAAhggoQIqgAIYIKECKoACGCChAiqAAhggoQIqgAIYIKECKoACGCChAiqAAhggoQIqgAIYIKECKoACGCChAiqAAhggoQIqgAIYIKECKoACGCChAiqAAhggoQIqgAIYIKECKoACGCChAiqAAhggoQIqgAIYIKECKoACGCChAiqAAhggoQIqgAIYIKECKoACGCChAiqAAhggoQIqgAIYIKECKoACGCChAiqAAhggoQIqgAIYIKECKoACGCChAiqAAhggoQIqgAIYIKECKoACGCChAiqAAhggoQIqgAIYIKECKoACGCChAiqAAhggoQIqgAIYIKECKoACGCChAiqAAhggoQIqgAIYIKECKoACGCChAiqAAhggoQIqgAIYIKECKoACGCChAiqAAhggoQIqgAIYIKECKoACGCChAiqAAhg7YHgMdGB3tlcnhwdNzrnylLq+dbnAjmI6h0xv1vvip/3L1zdLz61vvl4mdflKqqWpwKmhNUOqOup6XU03+Pp9MTrobu8QwVIERQAUIEFSBEUAFCBJXOOPfuh6Xq9Y+Oh3vb5WD31xYngvkIKp1x9sL6TFDHB3+Ww73tFieC+QgqQIigAoQIKkCIoAKECCpAiKAChAgqQIigAoQIKkCIoAKECCrdUVWlN1ieWZqMHpW6rlsaCOYjqHTGYGW1rF36dGbt9++/LqUIKi8HQaUzqqqa+XGUUkqZTsZ6yktDUAFCBBUgRFABQgQVIERQAUIEFSBEUAFCBBUgRFABQgSVTumdWSqlVG2PAf+LoNIpax98UgavvX50PH70sOzv/NziRNCcoNIp//0u/2T4V9nf/qWlaWA+ggoQIqgAIYIKECKoACGCChAiqAAhggoQIqgAIYIKECKoACGCSqdUvX55451LM2t7v939599JQ8cJKp3S6w/K6tsXZ9b27v9Q6umkpYmgOUEFCBFUgBBBBQgRVIAQQQUIEVSAEEEFCBFUgBBBBQgRVIAQQaVzqn6/lFI9sVKXeuq7/HSfoNI559c/KsvnLhwdjw8elu27d1qcCJoRVDqn1z9TqurJj2ZdpuNRa/NAU4IKECKoACGCChAyaHsATofRaFT29/cbXVvX0zKZTGfWhsNhefDgQeP9lpeXy8rKylwzwvOq6ro+6fyJJ6GpW7dulc3NzUbXVlVVPv/4vfLm2aWjte9+3Cnf/rTdeL+rV6+WK1euzD0nNFQdt+gOlYUYjUZld3e38fVfbu08135N74YhyTNUgBBBBQgRVIAQQQUIOfGPUoeHh4uag1fceLzYHzeZTqc+v7wwS0tLx66fGNTr16+/iFk4he7du7fQ/W7fvl2Gw+FC9+T0uHnz5rHr3kNlIba2tsrGxsbC9rtx40a5du3awvbj1Dn2PVTPUAFCBBUgRFABQgQVIERQAUL8OAoLsba2Vi5fvryw/dbX1xe2FzzmtSmA+XltCuBFElSAkGc9Qz32thaAp7lDBQgRVIAQQQUIEVSAEEEFCBFUgJC/AcDyrIue8Pr2AAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "AhUSkYalfXRp"
      },
      "source": [
        "### Políticas de rede neural"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jVKkJYn4fV6R",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 320
        },
        "outputId": "ea0cef69-ecc0-48c2-c4eb-beef8e39e3eb"
      },
      "source": [
        "n_inputs = env.observation_space.shape[0]\n",
        "n_hidden = 4\n",
        "n_outputs = 1\n",
        "\n",
        "initializer = tf.compat.v1.variance_scaling_initializer()\n",
        "\n",
        "\n",
        "X = tf.compat.v1.placeholder(tf.float32, shape=[None, n_inputs])\n",
        "hidden = tf.compat.v1.layers.dense(X, n_hidden, activation=tf.nn.elu, kernel_initializer=initializer)\n",
        "outputs = tf.compat.v1.layers.dense(hidden, n_outputs, activation=tf.nn.sigmoid, kernel_initializer=initializer)\n",
        "\n",
        "p_left_and_right = tf.concat(axis=1, values=[outputs, 1 - outputs])\n",
        "action = tf.compat.v1.multinomial(tf.math.log(p_left_and_right), num_samples=1)\n",
        "\n",
        "init = tf.compat.v1.global_variables_initializer()\n",
        "\n",
        "\n",
        "n_max_steps = 1000\n",
        "frames = []\n",
        "\n",
        "with tf.compat.v1.Session() as sess:\n",
        "    init.run()\n",
        "    obs = env.reset()\n",
        "    for step in range(n_max_steps):\n",
        "        img = render_cart_pole(env, obs)\n",
        "        frames.append(img)\n",
        "        action_val = action.eval(feed_dict={X: obs.reshape(1, n_inputs)})\n",
        "        obs, reward, done, info = env.step(action_val[0][0])\n",
        "        if done:\n",
        "            break\n",
        "\n",
        "env.close()\n",
        "\n",
        "\n",
        "video = plot_animation(frames)\n",
        "plt.show()"
      ],
      "execution_count": 23,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/util/dispatch.py:201: multinomial (from tensorflow.python.ops.random_ops) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Use `tf.random.categorical` instead.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAVQAAADnCAYAAABBu67aAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAFnElEQVR4nO3dP29bVRzH4WPHdZo0raB/QKioDCyUSu3EUlViiMTWpStvplMX3kR31IkVxMLGgApDiVRVUIQQ0KRNawXq2PeypVgN8S351vdCn2c7x7b8G6yPbm6O7F5d1wWAw+u3PQDA/4WgAoQIKkCIoAKECCpAyGDO444AADyvt9+mK1SAEEEFCBFUgBBBBQgRVIAQQQUIEVSAEEEFCBFUgBBBBQgRVIAQQQUIEVSAEEEFCBFUgBBBBQgRVIAQQQUIEVSAEEEFCBFUgBBBBQgRVIAQQQUIEVSAEEEFCBFUgBBBBQgRVIAQQQUIEVSAEEEFCBFUgBBBBQgRVIAQQQUIEVSAEEEFCBFUgBBBBQgRVIAQQQUIEVSAEEEFCBFUgBBBBQgRVIAQQQUIEVSAEEEFCBFUgBBBBQgRVIAQQQUIEVSAEEEFCBFUgBBBBQgRVIAQQQUIEVSAEEEFCBFUgBBBBQgRVIAQQQUIEVSAEEEFCBFUgBBBBQgRVIAQQQUIEVSAEEEFCBFUgBBBBQgRVIAQQQUIEVSAEEEFCBFUgBBBBQgRVIAQQQUIEVSAEEEFCBFUgBBBBQgRVIAQQQUIEVSAEEEFCBFUgBBBBQgRVIAQQQUIEVSAEEEFCBFUgBBBBQgRVIAQQQUIEVSAEEEFCBFUgBBBBQgRVIAQQQUIEVSAEEEFCBFUgBBBBQgRVIAQQQUIEVSAEEEFCBFUgBBBBQgRVIAQQQUIEVSAEEEFCBFUgBBBBQgRVIAQQQUIEVSAEEEFCBFUgBBBBQgRVIAQQQUIEVSAEEEFCBFUgBBBBQgZtD0A/F1d12U82ix1Ve3tHVk5XpaGKy1OBc0IKp1STcZl47NPyu7O4729d658XE6/d6XFqaAZQaVz6qoqpX52hVrXdYvTQHPuoQKECCpAiKAChAgqndJfGpST734ws/fw3telmu62NBE0J6h0Sq+/VI6+9tbM3h8Pf5k5RgVdJagAIYIKECKoACGCChAiqAAhggoQIqgAIYIKECKodM5w7WTpD5b31vV0Up4+/r3FiaAZQaVzTpw9X4Zrr++tp+Od8uiHb1qcCJoRVIAQQQUIEVSAEEEFCBFUgBBBBQgRVIAQQQUIEVS6p1dKrz+Y2aqraanruqWBoBlBpYN65c1LH83sPPj+qzL5c9TSPNCMoNI5vV6vLB05OrNXTZ6WUvuhPrpNUAFCBBUgRFABQgQVIERQAUIEFSBEUAFCBBUgRFDppJVTb5flE2f21tV0Uh79eLvFiWA+QaWTltdOlcHKiWcbdVV2HtxvbyBoQFABQgQVIERQAUIEFSBEUAFCBBUgRFABQgQVIERQ6azl46dn1rs726WajFuaBuYTVDrrzPsfzqy3f/qujEdbLU0D8wkqQIigAoQIKkCIoAKECCpAiKAChAgqQIigAoQIKv8pdV21PQL8I0Gls1ZPnSvHz55/tlHX5ddvP29vIJhDUOms/uBI6Q+GM3uTp6OWpoH5BBUgRFABQgZtD8CrZTqdltGo+Z/tk93Jc+vt7e3Grx8MBuXYsWONnw+H0avr+qDHD3wQXtTdu3fL5cuXy3Q6bfT89Utny7k31vbW938blS9u/9z4/a5evVpu3rz5omPCPL39Nl2hslDT6bRsbW01DuqnXx7u+0+fPHlyqNfDi3APFSBEUAFCBBUgRFABQg78p9R47BcmyZpMJvOfFFRVlc8xccPhcN/9A4N6/fr1lzELr7DNzc1SVYv7gpM7d+74HBN348aNffedQ2WhNjY2yoULFxofmzqsa9eulVu3bi3kvXil7HsO1T1UgBBBBQgRVIAQQQUIEVSAEF+OwkKtrq6W9fX1hR2dunjx4kLeB0pxbArg33BsCuBlElSAkHn3UPe9rAXgea5QAUIEFSBEUAFCBBUgRFABQgQVIOQveCXInmjavv0AAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "gYcXF5grp_SR"
      },
      "source": [
        "### Política de gradiente"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "R6PT8V8qsQQv",
        "outputId": "8f75ea8a-1e71-483c-b86a-e9509bfede69"
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "execution_count": 24,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JEx7ZItwi4NJ",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "863d296d-a83e-445e-a938-7b18663c19e4"
      },
      "source": [
        "n_inputs = 4\n",
        "n_hidden = 4\n",
        "n_outputs = 1\n",
        "learning_rate = 0.01\n",
        "\n",
        "initializer = tf.compat.v1.variance_scaling_initializer()\n",
        "\n",
        "\n",
        "X = tf.compat.v1.placeholder(tf.float32, shape=[None, n_inputs])\n",
        "hidden = tf.compat.v1.layers.dense(X, n_hidden, activation=tf.nn.elu, kernel_initializer=initializer)\n",
        "logits = tf.compat.v1.layers.dense(hidden, n_outputs)\n",
        "outputs = tf.compat.v1.nn.sigmoid(logits)\n",
        "\n",
        "p_left_and_right = tf.concat(axis=1, values=[outputs, 1 - outputs])\n",
        "action = tf.compat.v1.multinomial(tf.math.log(p_left_and_right), num_samples=1)\n",
        "\n",
        "init = tf.compat.v1.global_variables_initializer()\n",
        "\n",
        "y = 1. - tf.compat.v1.to_float(action)\n",
        "cross_entropy = tf.compat.v1.nn.sigmoid_cross_entropy_with_logits(labels=y, logits=logits)\n",
        "optimizer = tf.compat.v1.train.AdamOptimizer(learning_rate)"
      ],
      "execution_count": 25,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:From <ipython-input-25-05eb347131e0>:19: to_float (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Use `tf.cast` instead.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "V-upht5_rLMF"
      },
      "source": [
        "grads_and_vars = optimizer.compute_gradients(cross_entropy)\n",
        "gradients = [grad for grad, variable in grads_and_vars if grad != None]\n",
        "\n",
        "gradient_placeholders = []\n",
        "grads_and_vars_feed = []\n",
        "for grad, variable in grads_and_vars:\n",
        "  if grad != None:\n",
        "    gradient_placeholder = tf.compat.v1.placeholder(tf.float32, shape=grad.get_shape())\n",
        "    gradient_placeholders.append(gradient_placeholder)\n",
        "    grads_and_vars_feed.append((gradient_placeholder, variable))\n",
        "\n",
        "training_op = optimizer.apply_gradients(grads_and_vars_feed)\n",
        "init = tf.compat.v1.global_variables_initializer()\n",
        "saver = tf.compat.v1.train.Saver()"
      ],
      "execution_count": 26,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ugLsIHHrsCIL"
      },
      "source": [
        "def discount_rewards(rewards, discount_rate):\n",
        "    discounted_rewards = np.zeros(len(rewards))\n",
        "    cumulative_rewards = 0\n",
        "    for step in reversed(range(len(rewards))):\n",
        "        cumulative_rewards = rewards[step] + cumulative_rewards * discount_rate\n",
        "        discounted_rewards[step] = cumulative_rewards\n",
        "    return discounted_rewards\n",
        "\n",
        "def discount_and_normalize_rewards(all_rewards, discount_rate):\n",
        "    all_discounted_rewards = [discount_rewards(rewards, discount_rate) for rewards in all_rewards]\n",
        "    flat_rewards = np.concatenate(all_discounted_rewards)\n",
        "    reward_mean = flat_rewards.mean()\n",
        "    reward_std = flat_rewards.std()\n",
        "    return [(discounted_rewards - reward_mean)/reward_std for discounted_rewards in all_discounted_rewards]"
      ],
      "execution_count": 27,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "SKYe9FUcsEcz",
        "outputId": "1c864212-d7d7-4e42-c79e-2a060c158ed1"
      },
      "source": [
        "env = gym.make('CartPole-v0')\n",
        "\n",
        "n_games_per_update = 10\n",
        "n_max_steps = 1000\n",
        "n_iterations = 250\n",
        "save_iterations = 10\n",
        "discount_rate = 0.95\n",
        "\n",
        "with tf.compat.v1.Session() as sess:\n",
        "    init.run()\n",
        "    for iteration in range(n_iterations):\n",
        "        print('\\rIteration: {}'.format(iteration), end='')\n",
        "        all_rewards = []\n",
        "        all_gradients = []\n",
        "        for game in range(n_games_per_update):\n",
        "            current_rewards = []\n",
        "            current_gradients = []\n",
        "            obs = env.reset()\n",
        "            for step in range(n_max_steps):\n",
        "                action_val, gradients_val = sess.run([action, gradients], feed_dict={X: obs.reshape(1, n_inputs)})\n",
        "                obs, reward, done, info = env.step(action_val[0][0])\n",
        "                current_rewards.append(reward)\n",
        "                current_gradients.append(gradients_val)\n",
        "                if done:\n",
        "                    break\n",
        "            all_rewards.append(current_rewards)\n",
        "            all_gradients.append(current_gradients)\n",
        "\n",
        "        all_rewards = discount_and_normalize_rewards(all_rewards, discount_rate=discount_rate)\n",
        "        feed_dict = {}\n",
        "        for var_index, gradient_placeholder in enumerate(gradient_placeholders):\n",
        "            mean_gradients = np.mean([reward * all_gradients[game_index][step][var_index]\n",
        "                                      for game_index, rewards in enumerate(all_rewards)\n",
        "                                          for step, reward in enumerate(rewards)], axis=0)\n",
        "            feed_dict[gradient_placeholder] = mean_gradients\n",
        "        sess.run(training_op, feed_dict=feed_dict)\n",
        "        if iteration % save_iterations == 0:\n",
        "            saver.save(sess, '/content/drive/MyDrive/Colab Notebooks/my_policy_net_pg.ckpt')\n",
        "\n",
        "env.close()"
      ],
      "execution_count": 28,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Iteration: 249"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bfEXtr8xzgIV"
      },
      "source": [
        "def render_policy_net(model_path, action, X, n_max_steps = 1000):\n",
        "    frames = []\n",
        "    env = gym.make('CartPole-v0')\n",
        "    obs = env.reset()\n",
        "    with tf.compat.v1.Session() as sess:\n",
        "        saver.restore(sess, model_path)\n",
        "        for step in range(n_max_steps):\n",
        "            img = render_cart_pole(env, obs)\n",
        "            frames.append(img)\n",
        "            action_val = action.eval(feed_dict={X: obs.reshape(1, n_inputs)})\n",
        "            obs, reward, done, info = env.step(action_val[0][0])\n",
        "            if done:\n",
        "                break\n",
        "    env.close()\n",
        "    return frames"
      ],
      "execution_count": 29,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 265
        },
        "id": "hPKJ63MEsLSG",
        "outputId": "8746586b-0150-4656-a326-e4a0c9c8f30a"
      },
      "source": [
        "frames = render_policy_net('/content/drive/MyDrive/Colab Notebooks/my_policy_net_pg.ckpt', action, X, n_max_steps=1000)\n",
        "video = plot_animation(frames)\n",
        "plt.show()"
      ],
      "execution_count": 30,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Restoring parameters from /content/drive/MyDrive/Colab Notebooks/my_policy_net_pg.ckpt\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAVQAAADnCAYAAABBu67aAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAEzklEQVR4nO3cv25bZRyA4S+xG5GqSKhCiAmGShVTmdi6M7B1ZeitdOoNdOYasnRmLHfQskClSgxIEITcFMgfm62ianCc8NaOyPNs5zvH+n6D9ero+Mhbi8ViAPDfbW96AID/C0EFiAgqQERQASKCChCZnnHeKwAAb9s6bdEdKkBEUAEiggoQEVSAiKACRAQVICKoABFBBYgIKkBEUAEiggoQEVSAiKACRAQVICKoABFBBYgIKkBEUAEiggoQEVSAiKACRAQVICKoABFBBYgIKkBEUAEiggoQEVSAiKACRAQVICKoABFBBYgIKkBEUAEiggoQEVSAiKACRAQVICKoABFBBYgIKkBEUAEiggoQEVSAiKACRAQVICKoABFBBYgIKkBEUAEiggoQEVSAiKACRAQVICKoABFBBYgIKkBEUAEiggoQEVSAiKACRAQVICKoABFBBYgIKkBEUAEiggoQEVSAiKACRAQVICKoABFBBYgIKkBEUAEiggoQEVSAiKACRAQVICKoABFBBYgIKkBEUAEiggoQEVSAiKACRAQVICKoABFBBYgIKkBEUAEiggoQEVSAiKACRAQVICKoABFBBYgIKkBEUAEiggoQEVSAiKACRAQVICKoABFBBYgIKkBEUAEiggoQEVSAiKACRAQVICKoABFBBYgIKkBEUAEiggoQEVSAiKACRAQVICKoABFBBYgIKkBEUAEiggoQEVSAiKACRAQVICKoABFBBYgIKkBEUAEiggoQEVSAiKACRAQVICKoABFBBYhMNz0A/JvDg9/G/Pjo9fFkZ3dc231/gxPBcoLKpfX822/Gy59/eH384Wd3x6d3v97gRLCcoHJpLRbzMRbzfy5sbhhYgWeoABFBBYgIKkBEUAEiggoQEVSAiKACRAQVICKoABFBBYgIKkBEUAEiggoQEVSAiKACRAQVICKoABFBBYgIKkBEUAEiggoQEVSAiKACRAQVICKoABFBBYgIKkBEUAEigsqlNbn23qZHgHMRVC6tjz//8o3jV7+8GEd/zjY0DZxNULm0tibTN45f/fpinPx5sKFp4GyCChARVICIoAJEBBUgIqgAEUEFiAgqQERQASKCChARVICIoAJEBBUgIqgAEUEFiAgqQERQASKCChARVICIoAJEBBUgIqgAEUEFiAgqQERQASKCChARVICIoAJEBBUgMt30AFwtBwcH4/j4eKVr/3j58s2FxRiz2Wz8tbW78n43btwYk8nkPCPChW0tFotl55eehPO6f//+ePz48UrXfvTB7vjqi09eHy8WY+x993z8fnC40uen0+l48uTJuHXr1oVmhSW2Tlt0h8pazWazsb+/v9K1+/tjfP/jTxfeazKZjJOTkwt/Hs7LM1SAiKACRAQVICKoAJGlP0odHq72ayqsaj6fr3W/o6Mj32NyOzs7p64vDeqDBw/exSxcYc+ePVvbXvP5fDx69GjcvHlzbXtyNTx8+PDUde+hslb37t0be3t7a9lrMpmMp0+fjtu3b69lP66UU99D9QwVICKoABFBBYgIKkBEUAEi/hyFtbpz586YzWZr2Wt7e3tcv359LXvBGF6bArgIr00BvEuCChA56xnqqbe1ALzNHSpARFABIoIKEBFUgIigAkQEFSDyN0vPeaBSpTw1AAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "MQ_RS_QwxtRi"
      },
      "source": [
        "### Cadeias de Markov"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YP9m0qhLxuWy",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "15095b3d-47e1-4ed9-df13-a3807bf83cc7"
      },
      "source": [
        "transition_probabilities = [\n",
        "        [0.7, 0.2, 0.0, 0.1],  # de s0 para s0, s1, s2, s3\n",
        "        [0.0, 0.0, 0.9, 0.1],  # de s1 para s1, s2, s3\n",
        "        [0.0, 1.0, 0.0, 0.0],  # de s2 para s2, s3\n",
        "        [0.0, 0.0, 0.0, 1.0],  # de s3 para s3\n",
        "]\n",
        "\n",
        "n_max_steps = 50\n",
        "\n",
        "def print_sequence(start_state=0):\n",
        "    current_state = start_state\n",
        "    print('States:', end=' ')\n",
        "    for step in range(n_max_steps):\n",
        "        print(current_state, end=' ')\n",
        "        if current_state == 3:\n",
        "            break\n",
        "        current_state = np.random.choice(range(4), p=transition_probabilities[current_state])\n",
        "    else:\n",
        "        print('...', end='')\n",
        "    print()\n",
        "\n",
        "for _ in range(10):\n",
        "    print_sequence()"
      ],
      "execution_count": 32,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "States: 0 0 0 0 0 3 \n",
            "States: 0 0 0 0 0 0 1 3 \n",
            "States: 0 3 \n",
            "States: 0 0 0 0 1 3 \n",
            "States: 0 0 1 2 1 2 1 2 1 2 1 2 1 2 1 2 1 2 1 2 1 2 1 2 1 2 1 2 1 3 \n",
            "States: 0 1 2 1 2 1 2 1 2 1 2 1 3 \n",
            "States: 0 0 0 3 \n",
            "States: 0 3 \n",
            "States: 0 0 0 1 2 1 2 1 2 1 2 1 2 1 3 \n",
            "States: 0 3 \n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "oXdPUy9Ype8q"
      },
      "source": [
        "### Processo de decisão de Markov"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "xrcxB6DfpjZe",
        "outputId": "21e1e426-b718-47da-83c0-1821f4e26216"
      },
      "source": [
        "transition_probabilities = [\n",
        "        [[0.7, 0.3, 0.0], [1.0, 0.0, 0.0], [0.8, 0.2, 0.0]], # em s0, se a ação a0, então prob 0,7 p/ o estado s0 e 0,3 p/ o estado s1...\n",
        "        [[0.0, 1.0, 0.0], None, [0.0, 0.0, 1.0]],\n",
        "        [None, [0.8, 0.1, 0.1], None],\n",
        "    ]\n",
        "\n",
        "rewards = [\n",
        "        [[+10, 0, 0], [0, 0, 0], [0, 0, 0]],\n",
        "        [[0, 0, 0], [0, 0, 0], [0, 0, -50]],\n",
        "        [[0, 0, 0], [+40, 0, 0], [0, 0, 0]],\n",
        "    ]\n",
        "\n",
        "possible_actions = [[0, 1, 2], [0, 2], [1]]\n",
        "\n",
        "def policy_fire(state):\n",
        "    return [0, 2, 1][state]\n",
        "\n",
        "def policy_random(state):\n",
        "    return np.random.choice(possible_actions[state])\n",
        "\n",
        "def policy_safe(state):\n",
        "    return [0, 0, 1][state]\n",
        "\n",
        "class MDPEnvironment(object):\n",
        "    def __init__(self, start_state=0):\n",
        "        self.start_state=start_state\n",
        "        self.reset()\n",
        "    def reset(self):\n",
        "        self.total_rewards = 0\n",
        "        self.state = self.start_state\n",
        "    def step(self, action):\n",
        "        next_state = np.random.choice(range(3), p=transition_probabilities[self.state][action])\n",
        "        reward = rewards[self.state][action][next_state]\n",
        "        self.state = next_state\n",
        "        self.total_rewards += reward\n",
        "        return self.state, reward\n",
        "\n",
        "def run_episode(policy, n_steps, start_state=0, display=True):\n",
        "    env = MDPEnvironment()\n",
        "    if display:\n",
        "        print('States (+rewards):', end=' ')\n",
        "    for step in range(n_steps):\n",
        "        if display:\n",
        "            if step == 10:\n",
        "                print('...', end=' ')\n",
        "            elif step < 10:\n",
        "                print(env.state, end=' ')\n",
        "        action = policy(env.state)\n",
        "        state, reward = env.step(action)\n",
        "        if display and step < 10:\n",
        "            if reward:\n",
        "                print('({})'.format(reward), end=' ')\n",
        "    if display:\n",
        "        print('Total rewards =', env.total_rewards)\n",
        "    return env.total_rewards\n",
        "\n",
        "for policy in (policy_fire, policy_random, policy_safe):\n",
        "    all_totals = []\n",
        "    print(policy.__name__)\n",
        "    for episode in range(1000):\n",
        "        all_totals.append(run_episode(policy, n_steps=100, display=(episode<5)))\n",
        "    print('Summary: mean={:.1f}, std={:1f}, min={}, max={}'.format(np.mean(all_totals), np.std(all_totals), np.min(all_totals), \n",
        "                                                                   np.max(all_totals)))\n",
        "    print()"
      ],
      "execution_count": 33,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "policy_fire\n",
            "States (+rewards): 0 1 (-50) 2 (40) 0 (10) 0 1 (-50) 2 (40) 0 (10) 0 (10) 0 (10) ... Total rewards = 250\n",
            "States (+rewards): 0 (10) 0 (10) 0 1 (-50) 2 (40) 0 (10) 0 (10) 0 (10) 0 (10) 0 (10) ... Total rewards = -90\n",
            "States (+rewards): 0 1 (-50) 2 (40) 0 1 (-50) 2 (40) 0 (10) 0 (10) 0 (10) 0 ... Total rewards = 140\n",
            "States (+rewards): 0 (10) 0 (10) 0 (10) 0 (10) 0 1 (-50) 2 2 2 (40) 0 (10) ... Total rewards = -80\n",
            "States (+rewards): 0 (10) 0 1 (-50) 2 2 1 (-50) 2 (40) 0 1 (-50) 2 (40) ... Total rewards = 20\n",
            "Summary: mean=122.0, std=128.364635, min=-350, max=550\n",
            "\n",
            "policy_random\n",
            "States (+rewards): 0 0 0 0 (10) 0 0 1 1 1 1 (-50) ... Total rewards = -130\n",
            "States (+rewards): 0 (10) 0 0 0 0 0 (10) 0 0 0 0 ... Total rewards = -50\n",
            "States (+rewards): 0 0 0 (10) 0 1 1 1 1 1 (-50) 2 (40) ... Total rewards = 120\n",
            "States (+rewards): 0 0 1 (-50) 2 (40) 0 (10) 0 (10) 0 0 0 (10) 0 ... Total rewards = 140\n",
            "States (+rewards): 0 0 1 (-50) 2 (40) 0 (10) 0 0 0 0 0 ... Total rewards = -30\n",
            "Summary: mean=-31.9, std=92.705853, min=-380, max=200\n",
            "\n",
            "policy_safe\n",
            "States (+rewards): 0 (10) 0 (10) 0 (10) 0 (10) 0 1 1 1 1 1 ... Total rewards = 40\n",
            "States (+rewards): 0 (10) 0 (10) 0 (10) 0 (10) 0 (10) 0 (10) 0 (10) 0 (10) 0 (10) 0 (10) ... Total rewards = 150\n",
            "States (+rewards): 0 1 1 1 1 1 1 1 1 1 ... Total rewards = 0\n",
            "States (+rewards): 0 1 1 1 1 1 1 1 1 1 ... Total rewards = 0\n",
            "States (+rewards): 0 (10) 0 1 1 1 1 1 1 1 1 ... Total rewards = 10\n",
            "Summary: mean=22.5, std=26.389390, min=0, max=180\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4uDuKPObriVC"
      },
      "source": [
        "### Q-Values"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qDbXv9c1riej"
      },
      "source": [
        "nan = np.nan #representa ações impossíveis\n",
        "\n",
        "T = np.array([ #shape=[s, a, s']\n",
        "        [[0.7, 0.3, 0.0], [1.0, 0.0, 0.0], [0.8, 0.2, 0.0]],\n",
        "        [[0.0, 1.0, 0.0], [nan, nan, nan], [0.0, 0.0, 1.0]],\n",
        "        [[nan, nan, nan], [0.8, 0.1, 0.1], [nan, nan, nan]]\n",
        "    ])\n",
        "\n",
        "R = np.array([\n",
        "        [[+10, 0, 0], [0, 0, 0], [0, 0, 0]],\n",
        "        [[0, 0, 0], [nan, nan, nan], [0, 0, -50]],\n",
        "        [[nan, nan, nan], [+40, 0, 0], [nan, nan, nan]]\n",
        "    ])\n",
        "\n",
        "possible_actions = [[0, 1, 2], [0, 2], [1]]\n",
        "\n",
        "Q = np.full((3, 3), -np.inf) #-inf para ações impossíveis\n",
        "\n",
        "for state, actions in enumerate(possible_actions):\n",
        "  Q[state, actions] = 0.0 #Valor inicial - 0.0 ṕ/ todas as ações possíveis\n",
        "\n",
        "discount_rate = 0.95\n",
        "n_iterations = 100\n",
        "\n",
        "for interation in range(n_iterations):\n",
        "  Q_prev = Q.copy()\n",
        "  for s in range(3):\n",
        "    for a in possible_actions[s]:\n",
        "      Q[s, a] = np.sum([T[s, a, sp] * (R[s, a, sp] + discount_rate * np.max(Q_prev[sp])) for sp in range(3)])"
      ],
      "execution_count": 43,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "8PPgBgh7t7jH",
        "outputId": "e3e0e15c-1e27-46ab-aa9f-7b4ae8126796"
      },
      "source": [
        "Q"
      ],
      "execution_count": 44,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([[21.88646117, 20.79149867, 16.854807  ],\n",
              "       [ 1.10804034,        -inf,  1.16703135],\n",
              "       [       -inf, 53.8607061 ,        -inf]])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 44
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Zi-bTalnt7mr",
        "outputId": "77d2825c-1b27-40ff-a1c2-81b94643986e"
      },
      "source": [
        "np.argmax(Q, axis=1) #no estado S0, escolha a0, no S1, escolha a2 e no S2, escolha a1."
      ],
      "execution_count": 45,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([0, 2, 1])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 45
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_Q7RVS3DqtBW"
      },
      "source": [
        "### Q-Learning"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "3yBjC2Szqt9x",
        "outputId": "4aaf7f79-dba9-4315-f0b0-06d822a94bcc"
      },
      "source": [
        "n_states = 3\n",
        "n_actions = 3\n",
        "n_steps = 20000\n",
        "alpha = 0.01\n",
        "gamma = 0.99\n",
        "\n",
        "exploration_policy = policy_random\n",
        "\n",
        "q_values = np.full((n_states, n_actions), -np.inf)\n",
        "\n",
        "for state, actions in enumerate(possible_actions):\n",
        "    q_values[state][actions]=0\n",
        "\n",
        "env = MDPEnvironment()\n",
        "\n",
        "for step in range(n_steps):\n",
        "    action = exploration_policy(env.state)\n",
        "    state = env.state\n",
        "    next_state, reward = env.step(action)\n",
        "    next_value = np.max(q_values[next_state]) # política gananciosa\n",
        "    q_values[state, action] = (1-alpha)*q_values[state, action] + alpha*(reward + gamma * next_value)\n",
        "\n",
        "def optimal_policy(state):\n",
        "    return np.argmax(q_values[state])\n",
        "q_values"
      ],
      "execution_count": 34,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([[43.40390802, 41.52537991, 37.64793212],\n",
              "       [23.38243823,        -inf, 24.52213512],\n",
              "       [       -inf, 75.87584985,        -inf]])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 34
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "wl2qBFw1q8Dr",
        "outputId": "bbecb52c-099c-478e-d97b-fa10446ddd00"
      },
      "source": [
        "all_totals = []\n",
        "for episode in range(1000):\n",
        "    all_totals.append(run_episode(optimal_policy, n_steps=100, display=(episode<5)))\n",
        "print('Summary: mean={:.1f}, std={:1f}, min={}, max={}'.format(np.mean(all_totals), np.std(all_totals), np.min(all_totals), np.max(all_totals)))\n",
        "print()"
      ],
      "execution_count": 35,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "States (+rewards): 0 (10) 0 1 (-50) 2 (40) 0 (10) 0 (10) 0 (10) 0 (10) 0 (10) 0 (10) ... Total rewards = 90\n",
            "States (+rewards): 0 (10) 0 1 (-50) 2 (40) 0 1 (-50) 2 (40) 0 1 (-50) 2 (40) ... Total rewards = 300\n",
            "States (+rewards): 0 (10) 0 (10) 0 1 (-50) 2 (40) 0 (10) 0 (10) 0 (10) 0 (10) 0 (10) ... Total rewards = 60\n",
            "States (+rewards): 0 (10) 0 (10) 0 (10) 0 (10) 0 1 (-50) 2 1 (-50) 2 (40) 0 ... Total rewards = 120\n",
            "States (+rewards): 0 (10) 0 (10) 0 1 (-50) 2 (40) 0 (10) 0 1 (-50) 2 (40) 0 ... Total rewards = -10\n",
            "Summary: mean=122.9, std=139.806854, min=-400, max=610\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}