DL_P2_MultiClassClassification.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "DL_P2_MultiClassClassification.ipynb",
      "version": "0.3.2",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "[View in Colaboratory](https://colab.research.google.com/github/ObaidAshraf/Keras_models/blob/master/DL_P2_MultiClassClassification.ipynb)"
      ]
    },
    {
      "metadata": {
        "id": "BiREgZGQ4LlA",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "## **Step 1. Project Description**"
      ]
    },
    {
      "metadata": {
        "id": "6zFgcOdK0G6C",
        "colab_type": "code",
        "colab": {
          "resources": {
            "http://localhost:8080/nbextensions/google.colab/files.js": {
              "data": "Ly8gQ29weXJpZ2h0IDIwMTcgR29vZ2xlIExMQwovLwovLyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKLy8geW91IG1heSBub3QgdXNlIHRoaXMgZmlsZSBleGNlcHQgaW4gY29tcGxpYW5jZSB3aXRoIHRoZSBMaWNlbnNlLgovLyBZb3UgbWF5IG9idGFpbiBhIGNvcHkgb2YgdGhlIExpY2Vuc2UgYXQKLy8KLy8gICAgICBodHRwOi8vd3d3LmFwYWNoZS5vcmcvbGljZW5zZXMvTElDRU5TRS0yLjAKLy8KLy8gVW5sZXNzIHJlcXVpcmVkIGJ5IGFwcGxpY2FibGUgbGF3IG9yIGFncmVlZCB0byBpbiB3cml0aW5nLCBzb2Z0d2FyZQovLyBkaXN0cmlidXRlZCB1bmRlciB0aGUgTGljZW5zZSBpcyBkaXN0cmlidXRlZCBvbiBhbiAiQVMgSVMiIEJBU0lTLAovLyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KLy8gU2VlIHRoZSBMaWNlbnNlIGZvciB0aGUgc3BlY2lmaWMgbGFuZ3VhZ2UgZ292ZXJuaW5nIHBlcm1pc3Npb25zIGFuZAovLyBsaW1pdGF0aW9ucyB1bmRlciB0aGUgTGljZW5zZS4KCi8qKgogKiBAZmlsZW92ZXJ2aWV3IEhlbHBlcnMgZm9yIGdvb2dsZS5jb2xhYiBQeXRob24gbW9kdWxlLgogKi8KKGZ1bmN0aW9uKHNjb3BlKSB7CmZ1bmN0aW9uIHNwYW4odGV4dCwgc3R5bGVBdHRyaWJ1dGVzID0ge30pIHsKICBjb25zdCBlbGVtZW50ID0gZG9jdW1lbnQuY3JlYXRlRWxlbWVudCgnc3BhbicpOwogIGVsZW1lbnQudGV4dENvbnRlbnQgPSB0ZXh0OwogIGZvciAoY29uc3Qga2V5IG9mIE9iamVjdC5rZXlzKHN0eWxlQXR0cmlidXRlcykpIHsKICAgIGVsZW1lbnQuc3R5bGVba2V5XSA9IHN0eWxlQXR0cmlidXRlc1trZXldOwogIH0KICByZXR1cm4gZWxlbWVudDsKfQoKLy8gTWF4IG51bWJlciBvZiBieXRlcyB3aGljaCB3aWxsIGJlIHVwbG9hZGVkIGF0IGEgdGltZS4KY29uc3QgTUFYX1BBWUxPQURfU0laRSA9IDEwMCAqIDEwMjQ7Ci8vIE1heCBhbW91bnQgb2YgdGltZSB0byBibG9jayB3YWl0aW5nIGZvciB0aGUgdXNlci4KY29uc3QgRklMRV9DSEFOR0VfVElNRU9VVF9NUyA9IDMwICogMTAwMDsKCmZ1bmN0aW9uIF91cGxvYWRGaWxlcyhpbnB1dElkLCBvdXRwdXRJZCkgewogIGNvbnN0IHN0ZXBzID0gdXBsb2FkRmlsZXNTdGVwKGlucHV0SWQsIG91dHB1dElkKTsKICBjb25zdCBvdXRwdXRFbGVtZW50ID0gZG9jdW1lbnQuZ2V0RWxlbWVudEJ5SWQob3V0cHV0SWQpOwogIC8vIENhY2hlIHN0ZXBzIG9uIHRoZSBvdXRwdXRFbGVtZW50IHRvIG1ha2UgaXQgYXZhaWxhYmxlIGZvciB0aGUgbmV4dCBjYWxsCiAgLy8gdG8gdXBsb2FkRmlsZXNDb250aW51ZSBmcm9tIFB5dGhvbi4KICBvdXRwdXRFbGVtZW50LnN0ZXBzID0gc3RlcHM7CgogIHJldHVybiBfdXBsb2FkRmlsZXNDb250aW51ZShvdXRwdXRJZCk7Cn0KCi8vIFRoaXMgaXMgcm91Z2hseSBhbiBhc3luYyBnZW5lcmF0b3IgKG5vdCBzdXBwb3J0ZWQgaW4gdGhlIGJyb3dzZXIgeWV0KSwKLy8gd2hlcmUgdGhlcmUgYXJlIG11bHRpcGxlIGFzeW5jaHJvbm91cyBzdGVwcyBhbmQgdGhlIFB5dGhvbiBzaWRlIGlzIGdvaW5nCi8vIHRvIHBvbGwgZm9yIGNvbXBsZXRpb24gb2YgZWFjaCBzdGVwLgovLyBUaGlzIHVzZXMgYSBQcm9taXNlIHRvIGJsb2NrIHRoZSBweXRob24gc2lkZSBvbiBjb21wbGV0aW9uIG9mIGVhY2ggc3RlcCwKLy8gdGhlbiBwYXNzZXMgdGhlIHJlc3VsdCBvZiB0aGUgcHJldmlvdXMgc3RlcCBhcyB0aGUgaW5wdXQgdG8gdGhlIG5leHQgc3RlcC4KZnVuY3Rpb24gX3VwbG9hZEZpbGVzQ29udGludWUob3V0cHV0SWQpIHsKICBjb25zdCBvdXRwdXRFbGVtZW50ID0gZG9jdW1lbnQuZ2V0RWxlbWVudEJ5SWQob3V0cHV0SWQpOwogIGNvbnN0IHN0ZXBzID0gb3V0cHV0RWxlbWVudC5zdGVwczsKCiAgY29uc3QgbmV4dCA9IHN0ZXBzLm5leHQob3V0cHV0RWxlbWVudC5sYXN0UHJvbWlzZVZhbHVlKTsKICByZXR1cm4gUHJvbWlzZS5yZXNvbHZlKG5leHQudmFsdWUucHJvbWlzZSkudGhlbigodmFsdWUpID0+IHsKICAgIC8vIENhY2hlIHRoZSBsYXN0IHByb21pc2UgdmFsdWUgdG8gbWFrZSBpdCBhdmFpbGFibGUgdG8gdGhlIG5leHQKICAgIC8vIHN0ZXAgb2YgdGhlIGdlbmVyYXRvci4KICAgIG91dHB1dEVsZW1lbnQubGFzdFByb21pc2VWYWx1ZSA9IHZhbHVlOwogICAgcmV0dXJuIG5leHQudmFsdWUucmVzcG9uc2U7CiAgfSk7Cn0KCi8qKgogKiBHZW5lcmF0b3IgZnVuY3Rpb24gd2hpY2ggaXMgY2FsbGVkIGJldHdlZW4gZWFjaCBhc3luYyBzdGVwIG9mIHRoZSB1cGxvYWQKICogcHJvY2Vzcy4KICogQHBhcmFtIHtzdHJpbmd9IGlucHV0SWQgRWxlbWVudCBJRCBvZiB0aGUgaW5wdXQgZmlsZSBwaWNrZXIgZWxlbWVudC4KICogQHBhcmFtIHtzdHJpbmd9IG91dHB1dElkIEVsZW1lbnQgSUQgb2YgdGhlIG91dHB1dCBkaXNwbGF5LgogKiBAcmV0dXJuIHshSXRlcmFibGU8IU9iamVjdD59IEl0ZXJhYmxlIG9mIG5leHQgc3RlcHMuCiAqLwpmdW5jdGlvbiogdXBsb2FkRmlsZXNTdGVwKGlucHV0SWQsIG91dHB1dElkKSB7CiAgY29uc3QgaW5wdXRFbGVtZW50ID0gZG9jdW1lbnQuZ2V0RWxlbWVudEJ5SWQoaW5wdXRJZCk7CiAgaW5wdXRFbGVtZW50LmRpc2FibGVkID0gZmFsc2U7CgogIGNvbnN0IG91dHB1dEVsZW1lbnQgPSBkb2N1bWVudC5nZXRFbGVtZW50QnlJZChvdXRwdXRJZCk7CiAgb3V0cHV0RWxlbWVudC5pbm5lckhUTUwgPSAnJzsKCiAgY29uc3QgcGlja2VkUHJvbWlzZSA9IG5ldyBQcm9taXNlKChyZXNvbHZlKSA9PiB7CiAgICBpbnB1dEVsZW1lbnQuYWRkRXZlbnRMaXN0ZW5lcignY2hhbmdlJywgKGUpID0+IHsKICAgICAgcmVzb2x2ZShlLnRhcmdldC5maWxlcyk7CiAgICB9KTsKICB9KTsKCiAgY29uc3QgY2FuY2VsID0gZG9jdW1lbnQuY3JlYXRlRWxlbWVudCgnYnV0dG9uJyk7CiAgaW5wdXRFbGVtZW50LnBhcmVudEVsZW1lbnQuYXBwZW5kQ2hpbGQoY2FuY2VsKTsKICBjYW5jZWwudGV4dENvbnRlbnQgPSAnQ2FuY2VsIHVwbG9hZCc7CiAgY29uc3QgY2FuY2VsUHJvbWlzZSA9IG5ldyBQcm9taXNlKChyZXNvbHZlKSA9PiB7CiAgICBjYW5jZWwub25jbGljayA9ICgpID0+IHsKICAgICAgcmVzb2x2ZShudWxsKTsKICAgIH07CiAgfSk7CgogIC8vIENhbmNlbCB1cGxvYWQgaWYgdXNlciBoYXNuJ3QgcGlja2VkIGFueXRoaW5nIGluIHRpbWVvdXQuCiAgY29uc3QgdGltZW91dFByb21pc2UgPSBuZXcgUHJvbWlzZSgocmVzb2x2ZSkgPT4gewogICAgc2V0VGltZW91dCgoKSA9PiB7CiAgICAgIHJlc29sdmUobnVsbCk7CiAgICB9LCBGSUxFX0NIQU5HRV9USU1FT1VUX01TKTsKICB9KTsKCiAgLy8gV2FpdCBmb3IgdGhlIHVzZXIgdG8gcGljayB0aGUgZmlsZXMuCiAgY29uc3QgZmlsZXMgPSB5aWVsZCB7CiAgICBwcm9taXNlOiBQcm9taXNlLnJhY2UoW3BpY2tlZFByb21pc2UsIHRpbWVvdXRQcm9taXNlLCBjYW5jZWxQcm9taXNlXSksCiAgICByZXNwb25zZTogewogICAgICBhY3Rpb246ICdzdGFydGluZycsCiAgICB9CiAgfTsKCiAgaWYgKCFmaWxlcykgewogICAgcmV0dXJuIHsKICAgICAgcmVzcG9uc2U6IHsKICAgICAgICBhY3Rpb246ICdjb21wbGV0ZScsCiAgICAgIH0KICAgIH07CiAgfQoKICBjYW5jZWwucmVtb3ZlKCk7CgogIC8vIERpc2FibGUgdGhlIGlucHV0IGVsZW1lbnQgc2luY2UgZnVydGhlciBwaWNrcyBhcmUgbm90IGFsbG93ZWQuCiAgaW5wdXRFbGVtZW50LmRpc2FibGVkID0gdHJ1ZTsKCiAgZm9yIChjb25zdCBmaWxlIG9mIGZpbGVzKSB7CiAgICBjb25zdCBsaSA9IGRvY3VtZW50LmNyZWF0ZUVsZW1lbnQoJ2xpJyk7CiAgICBsaS5hcHBlbmQoc3BhbihmaWxlLm5hbWUsIHtmb250V2VpZ2h0OiAnYm9sZCd9KSk7CiAgICBsaS5hcHBlbmQoc3BhbigKICAgICAgICBgKCR7ZmlsZS50eXBlIHx8ICduL2EnfSkgLSAke2ZpbGUuc2l6ZX0gYnl0ZXMsIGAgKwogICAgICAgIGBsYXN0IG1vZGlmaWVkOiAkewogICAgICAgICAgICBmaWxlLmxhc3RNb2RpZmllZERhdGUgPyBmaWxlLmxhc3RNb2RpZmllZERhdGUudG9Mb2NhbGVEYXRlU3RyaW5nKCkgOgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAnbi9hJ30gLSBgKSk7CiAgICBjb25zdCBwZXJjZW50ID0gc3BhbignMCUgZG9uZScpOwogICAgbGkuYXBwZW5kQ2hpbGQocGVyY2VudCk7CgogICAgb3V0cHV0RWxlbWVudC5hcHBlbmRDaGlsZChsaSk7CgogICAgY29uc3QgZmlsZURhdGFQcm9taXNlID0gbmV3IFByb21pc2UoKHJlc29sdmUpID0+IHsKICAgICAgY29uc3QgcmVhZGVyID0gbmV3IEZpbGVSZWFkZXIoKTsKICAgICAgcmVhZGVyLm9ubG9hZCA9IChlKSA9PiB7CiAgICAgICAgcmVzb2x2ZShlLnRhcmdldC5yZXN1bHQpOwogICAgICB9OwogICAgICByZWFkZXIucmVhZEFzQXJyYXlCdWZmZXIoZmlsZSk7CiAgICB9KTsKICAgIC8vIFdhaXQgZm9yIHRoZSBkYXRhIHRvIGJlIHJlYWR5LgogICAgbGV0IGZpbGVEYXRhID0geWllbGQgewogICAgICBwcm9taXNlOiBmaWxlRGF0YVByb21pc2UsCiAgICAgIHJlc3BvbnNlOiB7CiAgICAgICAgYWN0aW9uOiAnY29udGludWUnLAogICAgICB9CiAgICB9OwoKICAgIC8vIFVzZSBhIGNodW5rZWQgc2VuZGluZyB0byBhdm9pZCBtZXNzYWdlIHNpemUgbGltaXRzLiBTZWUgYi82MjExNTY2MC4KICAgIGxldCBwb3NpdGlvbiA9IDA7CiAgICB3aGlsZSAocG9zaXRpb24gPCBmaWxlRGF0YS5ieXRlTGVuZ3RoKSB7CiAgICAgIGNvbnN0IGxlbmd0aCA9IE1hdGgubWluKGZpbGVEYXRhLmJ5dGVMZW5ndGggLSBwb3NpdGlvbiwgTUFYX1BBWUxPQURfU0laRSk7CiAgICAgIGNvbnN0IGNodW5rID0gbmV3IFVpbnQ4QXJyYXkoZmlsZURhdGEsIHBvc2l0aW9uLCBsZW5ndGgpOwogICAgICBwb3NpdGlvbiArPSBsZW5ndGg7CgogICAgICBjb25zdCBiYXNlNjQgPSBidG9hKFN0cmluZy5mcm9tQ2hhckNvZGUuYXBwbHkobnVsbCwgY2h1bmspKTsKICAgICAgeWllbGQgewogICAgICAgIHJlc3BvbnNlOiB7CiAgICAgICAgICBhY3Rpb246ICdhcHBlbmQnLAogICAgICAgICAgZmlsZTogZmlsZS5uYW1lLAogICAgICAgICAgZGF0YTogYmFzZTY0LAogICAgICAgIH0sCiAgICAgIH07CiAgICAgIHBlcmNlbnQudGV4dENvbnRlbnQgPQogICAgICAgICAgYCR7TWF0aC5yb3VuZCgocG9zaXRpb24gLyBmaWxlRGF0YS5ieXRlTGVuZ3RoKSAqIDEwMCl9JSBkb25lYDsKICAgIH0KICB9CgogIC8vIEFsbCBkb25lLgogIHlpZWxkIHsKICAgIHJlc3BvbnNlOiB7CiAgICAgIGFjdGlvbjogJ2NvbXBsZXRlJywKICAgIH0KICB9Owp9CgpzY29wZS5nb29nbGUgPSBzY29wZS5nb29nbGUgfHwge307CnNjb3BlLmdvb2dsZS5jb2xhYiA9IHNjb3BlLmdvb2dsZS5jb2xhYiB8fCB7fTsKc2NvcGUuZ29vZ2xlLmNvbGFiLl9maWxlcyA9IHsKICBfdXBsb2FkRmlsZXMsCiAgX3VwbG9hZEZpbGVzQ29udGludWUsCn07Cn0pKHNlbGYpOwo=",
              "ok": true,
              "headers": [
                [
                  "content-type",
                  "application/javascript"
                ]
              ],
              "status": 200,
              "status_text": ""
            }
          },
          "base_uri": "https://localhost:8080/",
          "height": 72
        },
        "outputId": "dc7b7f71-c608-4dff-9fea-310b2576ca84"
      },
      "cell_type": "code",
      "source": [
        "from google.colab import files\n",
        "uploaded = files.upload() "
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "\n",
              "     <input type=\"file\" id=\"files-6fefa4e1-fa2a-4317-ba90-c29e0ac48238\" name=\"files[]\" multiple disabled />\n",
              "     <output id=\"result-6fefa4e1-fa2a-4317-ba90-c29e0ac48238\">\n",
              "      Upload widget is only available when the cell has been executed in the\n",
              "      current browser session. Please rerun this cell to enable.\n",
              "      </output>\n",
              "      <script src=\"/nbextensions/google.colab/files.js\"></script> "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "Saving iris.csv to iris.csv\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "MIveQq_i4MzP",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "## **Step 2. Making Preparations**"
      ]
    },
    {
      "metadata": {
        "id": "GHgArP7706Ue",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "import numpy\n",
        "import pandas\n",
        "import keras\n",
        "from keras import layers\n",
        "from keras.models import Sequential\n",
        "from keras.layers import Dense\n",
        "from keras.wrappers.scikit_learn import KerasClassifier\n",
        "from keras.utils import np_utils\n",
        "from sklearn.model_selection import cross_val_score\n",
        "from sklearn.model_selection import KFold\n",
        "from sklearn.preprocessing import LabelEncoder\n",
        "from sklearn.model_selection import StratifiedKFold\n",
        "from sklearn.preprocessing import StandardScaler\n",
        "from sklearn.pipeline import Pipeline\n",
        "\n",
        "# fix random seed for reproducibility\n",
        "seed = 7\n",
        "numpy.random.seed(seed)\n",
        "\n",
        "# load dataset\n",
        "dataframe = pandas.read_csv(\"iris.csv\", header=None)\n",
        "dataset = dataframe.values\n",
        "X = dataset[:,0:4].astype(float)\n",
        "Y = dataset[:,4]\n",
        "\n",
        "le = LabelEncoder()\n",
        "encoded_Y = le.fit(Y)\n",
        "encoded_Y = le.transform(Y)\n",
        "encoded_Y = np_utils.to_categorical(encoded_Y)\n",
        "train_data = X\n",
        "train_targets = encoded_Y"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "to0uKHP64jiv",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "## **Step 3: Define the Neural Network Baseline Model**"
      ]
    },
    {
      "metadata": {
        "id": "ZnB8z8Vj39G1",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def baseline_model():\n",
        "  model = Sequential()\n",
        "  model.add(Dense(8, activation = 'relu', input_shape=(4,)))\n",
        "  model.add(Dense(3, activation = 'softmax'))\n",
        "  \n",
        "  model.compile(loss = 'categorical_crossentropy',\n",
        "                optimizer = 'adam',\n",
        "                metrics = ['accuracy'])\n",
        "  \n",
        "  return model"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "Ki3Wwhaa4pR_",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "## **Step 4. Evaluate The Model with k-Fold Cross Validation**"
      ]
    },
    {
      "metadata": {
        "id": "kku_-9jQ4rBP",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "e857986c-6410-4d01-b10b-a3d8f115214f"
      },
      "cell_type": "code",
      "source": [
        "estimator = KerasClassifier(build_fn=baseline_model, epochs=200, batch_size=5, verbose=0)\n",
        "kfold = KFold(n_splits=10, shuffle=True, random_state=seed)\n",
        "results = cross_val_score(estimator, X, encoded_Y, cv=kfold)\n",
        "print(\"Results: %.2f%% (%.2f%%)\" % (results.mean()*100, results.std()*100))\n",
        "\n"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Results: 97.33% (4.42%)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "JugihWuM6VaL",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "## Step 5.1. Evaluate a Smaller Network"
      ]
    },
    {
      "metadata": {
        "id": "drFGecn56clB",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def smaller_model():\n",
        "  model = Sequential()\n",
        "  model.add(Dense(8, activation = 'relu', input_shape=(4,)))\n",
        "  model.add(Dense(4, activation = 'relu'))\n",
        "  model.add(Dense(3, activation = 'softmax'))\n",
        "  \n",
        "  model.compile(loss = 'categorical_crossentropy',\n",
        "                optimizer = 'adam',\n",
        "                metrics = ['accuracy'])\n",
        "  \n",
        "  return model"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "K2mT9RyP6pIQ",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "fe724284-db56-4f6c-988f-9aafe67f336c"
      },
      "cell_type": "code",
      "source": [
        "estimator = KerasClassifier(build_fn=smaller_model, epochs=200, batch_size=5, verbose=0)\n",
        "kfold = KFold(n_splits=10, shuffle=True, random_state=seed)\n",
        "results = cross_val_score(estimator, X, encoded_Y, cv=kfold)\n",
        "print(\"Results: %.2f%% (%.2f%%)\" % (results.mean()*100, results.std()*100))\n",
        "\n"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Results: 96.67% (4.47%)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "MXqR5I5S-D1T",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "## Step 5.2. Evaluate a Larger Network"
      ]
    },
    {
      "metadata": {
        "id": "LF1UY5T2-GS-",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def larger_model():\n",
        "  model = Sequential()\n",
        "  model.add(Dense(8, activation = 'relu', input_shape=(4,)))\n",
        "  model.add(Dense(8, activation = 'relu'))\n",
        "  model.add(Dense(4, activation = 'relu'))\n",
        "  model.add(Dense(3, activation = 'softmax'))\n",
        "  \n",
        "  model.compile(loss = 'categorical_crossentropy',\n",
        "                optimizer = 'adam',\n",
        "                metrics = ['accuracy'])\n",
        "  \n",
        "  return model"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "tlGM-CyA-Jvy",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "02d0936f-cad8-48b2-dd31-18c448d42684"
      },
      "cell_type": "code",
      "source": [
        "estimator = KerasClassifier(build_fn=larger_model, epochs=200, batch_size=5, verbose=0)\n",
        "kfold = KFold(n_splits=10, shuffle=True, random_state=seed)\n",
        "results = cross_val_score(estimator, X, encoded_Y, cv=kfold)\n",
        "print(\"Results: %.2f%% (%.2f%%)\" % (results.mean()*100, results.std()*100))\n",
        "\n"
      ],
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Results: 92.00% (13.60%)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "N8aU3M3eC3BS",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "## Step 6. Really Scaling up: developing a model that overfits & Step 7. Tuning the Model"
      ]
    },
    {
      "metadata": {
        "id": "tYPDPmIbC6uh",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def tuned_model():\n",
        "  model = Sequential()\n",
        "  model.add(Dense(8, activation = 'relu', input_shape=(4,)))\n",
        "  model.add(Dense(8, activation = 'relu'))\n",
        "  model.add(Dense(8, activation = 'relu'))\n",
        "  model.add(Dense(5, activation = 'relu'))\n",
        "  model.add(Dense(3, activation = 'softmax'))\n",
        "  \n",
        "  model.compile(loss = 'categorical_crossentropy',\n",
        "                optimizer = 'adam',\n",
        "                metrics = ['accuracy'])\n",
        "  \n",
        "  return model"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "hSqAhjrJC97A",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "f64981dd-9212-4a43-ad9e-12be91008188"
      },
      "cell_type": "code",
      "source": [
        "estimator = KerasClassifier(build_fn=tuned_model, epochs=250, batch_size=5, verbose=0)\n",
        "kfold = KFold(n_splits=25, shuffle=True, random_state=seed)\n",
        "results = cross_val_score(estimator, X, encoded_Y, cv=kfold)\n",
        "print(\"Results: %.2f%% (%.2f%%)\" % (results.mean()*100, results.std()*100))\n",
        "\n"
      ],
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Results: 92.67% (20.59%)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "-wRKr-J9MCSF",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "## Step 8. Rewriting the code using the Keras Functional API"
      ]
    },
    {
      "metadata": {
        "id": "AgTkYFDWMETw",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def create_func_api_model():\n",
        "  inputs = keras.Input(shape=(4,))\n",
        "  x = layers.Dense(8, activation = 'relu')(inputs)\n",
        "  x = layers.Dense(8, activation = 'relu')(x)\n",
        "  x = layers.Dense(8, activation = 'relu')(x)\n",
        "  x = layers.Dense(5, activation = 'relu')(x)\n",
        "  outputs = layers.Dense(3, activation = 'softmax')(x)\n",
        "  \n",
        "  model = keras.Model(inputs, outputs)\n",
        "  model.compile(loss = 'categorical_crossentropy',\n",
        "                optimizer = 'adam',\n",
        "                metrics = ['accuracy'])\n",
        "  return model"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "ybMtHssAMU-y",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "bbc14470-d90f-4ca9-cd50-6810650172cf"
      },
      "cell_type": "code",
      "source": [
        "estimator = KerasClassifier(build_fn=create_func_api_model, epochs=250, batch_size=5, verbose=0)\n",
        "kfold = KFold(n_splits=25, shuffle=True, random_state=seed)\n",
        "results = cross_val_score(estimator, X, encoded_Y, cv=kfold)\n",
        "print(\"Results: %.2f%% (%.2f%%)\" % (results.mean()*100, results.std()*100))\n",
        "\n"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Results: 96.00% (7.12%)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "wle3zxqzaNwr",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "## Step 9. Rewriting the code by doing Model Subclassing"
      ]
    },
    {
      "metadata": {
        "id": "AQzBOTscaO-r",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "# TODO"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "Eu93ugxaZvhO",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "## Step 10. Rewriting the code without using scikit-learn"
      ]
    },
    {
      "metadata": {
        "id": "tv_g1ptkZyse",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def baseline_model():\n",
        "  model = Sequential()\n",
        "  model.add(Dense(8, activation = 'relu', input_shape=(4,)))\n",
        "  model.add(Dense(8, activation = 'relu'))\n",
        "  model.add(Dense(8, activation = 'relu'))\n",
        "  model.add(Dense(5, activation = 'relu'))\n",
        "  model.add(Dense(3, activation = 'softmax'))\n",
        "  \n",
        "  model.compile(loss = 'categorical_crossentropy',\n",
        "                optimizer = 'adam',\n",
        "                metrics = ['accuracy'])\n",
        "  \n",
        "  return model"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "D9YyBEakZ2br",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 272
        },
        "outputId": "bb724a5e-830b-42b5-8762-9269d18b318d"
      },
      "cell_type": "code",
      "source": [
        "import numpy as np\n",
        "\n",
        "k=5\n",
        "num_val_samples = len(train_data) // k\n",
        "num_epochs = 100\n",
        "all_scores = []\n",
        "\n",
        "num_epochs = 500\n",
        "batch_size_val = num_val_samples // k\n",
        "all_mae_histories = []\n",
        "for i in range(k):\n",
        "    print('processing fold #', i)\n",
        "    val_data = train_data[i * num_val_samples: (i + 1) * num_val_samples]\n",
        "    val_targets = train_targets[i * num_val_samples: (i + 1) * num_val_samples]\n",
        "    \n",
        "    partial_train_data = np.concatenate(\n",
        "        [train_data[:i * num_val_samples],\n",
        "         train_data[(i + 1) * num_val_samples:]],\n",
        "        axis=0)\n",
        "    \n",
        "    partial_train_targets = np.concatenate(\n",
        "        [train_targets[:i * num_val_samples],\n",
        "         train_targets[(i + 1) * num_val_samples:]],\n",
        "        axis=0)\n",
        "    \n",
        "    model = baseline_model()\n",
        "    history = model.fit(partial_train_data, partial_train_targets,\n",
        "                    validation_data=(val_data, val_targets),\n",
        "                    epochs=num_epochs, batch_size=batch_size_val, verbose=0)\n",
        "    print(\"Training Accuracy:\", history.history['acc'][num_epochs-1])\n",
        "    print(\"Validation Accuracy:\", history.history['val_acc'][num_epochs-1])\n"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "processing fold # 0\n",
            "Training Accuracy: 0.9749999970197678\n",
            "Validation Accuracy: 1.0\n",
            "processing fold # 1\n",
            "Training Accuracy: 0.9833333343267441\n",
            "Validation Accuracy: 1.0\n",
            "processing fold # 2\n",
            "Training Accuracy: 1.0\n",
            "Validation Accuracy: 0.8333333373069763\n",
            "processing fold # 3\n",
            "Training Accuracy: 0.9833333313465118\n",
            "Validation Accuracy: 0.9666666626930237\n",
            "processing fold # 4\n",
            "Training Accuracy: 1.0\n",
            "Validation Accuracy: 0.8\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}