diabetes_pima_spark.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "source": [
        "### Install Dependencies"
      ],
      "metadata": {
        "id": "z92Souaq32oU"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Sw9DJF2w3hC4",
        "outputId": "b656581d-a852-4b14-a0cb-d1e26d059f36"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting pyspark\n",
            "  Downloading pyspark-3.5.1.tar.gz (317.0 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m317.0/317.0 MB\u001b[0m \u001b[31m2.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Requirement already satisfied: py4j==0.10.9.7 in /usr/local/lib/python3.10/dist-packages (from pyspark) (0.10.9.7)\n",
            "Building wheels for collected packages: pyspark\n",
            "  Building wheel for pyspark (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for pyspark: filename=pyspark-3.5.1-py2.py3-none-any.whl size=317488493 sha256=250c45feeec8dce2e38c5cea554d2c5c371f017be016f0d3e1adf7b2a770145c\n",
            "  Stored in directory: /root/.cache/pip/wheels/80/1d/60/2c256ed38dddce2fdd93be545214a63e02fbd8d74fb0b7f3a6\n",
            "Successfully built pyspark\n",
            "Installing collected packages: pyspark\n",
            "Successfully installed pyspark-3.5.1\n"
          ]
        }
      ],
      "source": [
        "#!pip install pyspark"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Run a SparkSession"
      ],
      "metadata": {
        "id": "ArxROdBX35Pf"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from pyspark.sql import SparkSession\n",
        "\n",
        "spark = SparkSession.builder.appName(\"spark\").getOrCreate()"
      ],
      "metadata": {
        "id": "g5HZR9FJ38i7"
      },
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Clone Diabetes Dataset"
      ],
      "metadata": {
        "id": "_imHI-mg4fjU"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "! git clone https://github.com/education454/diabetes_dataset"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "30Gy2hFY4JKW",
        "outputId": "b72b57d0-b6ce-4286-b721-de29f9a38ac0"
      },
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Cloning into 'diabetes_dataset'...\n",
            "remote: Enumerating objects: 6, done.\u001b[K\n",
            "remote: Counting objects: 100% (6/6), done.\u001b[K\n",
            "remote: Compressing objects: 100% (5/5), done.\u001b[K\n",
            "remote: Total 6 (delta 0), reused 0 (delta 0), pack-reused 0\u001b[K\n",
            "Receiving objects: 100% (6/6), 13.02 KiB | 1.86 MiB/s, done.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "! ls diabetes_dataset"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ZWEK2__M4oxy",
        "outputId": "bbf63557-05b3-4af1-c9db-ee4c295d07de"
      },
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "diabetes.csv  new_test.csv\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df = spark.read.csv('/content/diabetes_dataset/diabetes.csv', header=True, inferSchema=True)"
      ],
      "metadata": {
        "id": "1TTJyXPG4t4S"
      },
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df.show()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Zk83Jq0p5NQN",
        "outputId": "144bb782-2687-4e78-ee51-f5876db633bc"
      },
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+-----------+-------+-------------+-------------+-------+----+------------------------+---+-------+\n",
            "|Pregnancies|Glucose|BloodPressure|SkinThickness|Insulin| BMI|DiabetesPedigreeFunction|Age|Outcome|\n",
            "+-----------+-------+-------------+-------------+-------+----+------------------------+---+-------+\n",
            "|          2|    138|           62|           35|      0|33.6|                   0.127| 47|      1|\n",
            "|          0|     84|           82|           31|    125|38.2|                   0.233| 23|      0|\n",
            "|          0|    145|            0|            0|      0|44.2|                    0.63| 31|      1|\n",
            "|          0|    135|           68|           42|    250|42.3|                   0.365| 24|      1|\n",
            "|          1|    139|           62|           41|    480|40.7|                   0.536| 21|      0|\n",
            "|          0|    173|           78|           32|    265|46.5|                   1.159| 58|      0|\n",
            "|          4|     99|           72|           17|      0|25.6|                   0.294| 28|      0|\n",
            "|          8|    194|           80|            0|      0|26.1|                   0.551| 67|      0|\n",
            "|          2|     83|           65|           28|     66|36.8|                   0.629| 24|      0|\n",
            "|          2|     89|           90|           30|      0|33.5|                   0.292| 42|      0|\n",
            "|          4|     99|           68|           38|      0|32.8|                   0.145| 33|      0|\n",
            "|          4|    125|           70|           18|    122|28.9|                   1.144| 45|      1|\n",
            "|          3|     80|            0|            0|      0| 0.0|                   0.174| 22|      0|\n",
            "|          6|    166|           74|            0|      0|26.6|                   0.304| 66|      0|\n",
            "|          5|    110|           68|            0|      0|26.0|                   0.292| 30|      0|\n",
            "|          2|     81|           72|           15|     76|30.1|                   0.547| 25|      0|\n",
            "|          7|    195|           70|           33|    145|25.1|                   0.163| 55|      1|\n",
            "|          6|    154|           74|           32|    193|29.3|                   0.839| 39|      0|\n",
            "|          2|    117|           90|           19|     71|25.2|                   0.313| 21|      0|\n",
            "|          3|     84|           72|           32|      0|37.2|                   0.267| 28|      0|\n",
            "+-----------+-------+-------------+-------------+-------+----+------------------------+---+-------+\n",
            "only showing top 20 rows\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df.printSchema()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "b5A511BR5VFl",
        "outputId": "c303745e-af76-485d-c061-ab76ce4c3bbf"
      },
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "root\n",
            " |-- Pregnancies: integer (nullable = true)\n",
            " |-- Glucose: integer (nullable = true)\n",
            " |-- BloodPressure: integer (nullable = true)\n",
            " |-- SkinThickness: integer (nullable = true)\n",
            " |-- Insulin: integer (nullable = true)\n",
            " |-- BMI: double (nullable = true)\n",
            " |-- DiabetesPedigreeFunction: double (nullable = true)\n",
            " |-- Age: integer (nullable = true)\n",
            " |-- Outcome: integer (nullable = true)\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print((df.count(), len(df.columns)))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "QeRwP6-U5dC0",
        "outputId": "9c32b9fc-bff4-4a55-ff57-c517db1d59d8"
      },
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "(2000, 9)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df.groupby('Outcome').count().show()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "oCKbJ94-5sa5",
        "outputId": "516d4b23-5077-47e3-85d5-e8c69cf71b17"
      },
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+-------+-----+\n",
            "|Outcome|count|\n",
            "+-------+-----+\n",
            "|      1|  684|\n",
            "|      0| 1316|\n",
            "+-------+-----+\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df.describe().show()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "nLhFjBWA56lE",
        "outputId": "54d59e60-5b7d-45e2-becd-1b4c4143bbf9"
      },
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+-------+-----------------+------------------+------------------+-----------------+-----------------+------------------+------------------------+------------------+------------------+\n",
            "|summary|      Pregnancies|           Glucose|     BloodPressure|    SkinThickness|          Insulin|               BMI|DiabetesPedigreeFunction|               Age|           Outcome|\n",
            "+-------+-----------------+------------------+------------------+-----------------+-----------------+------------------+------------------------+------------------+------------------+\n",
            "|  count|             2000|              2000|              2000|             2000|             2000|              2000|                    2000|              2000|              2000|\n",
            "|   mean|           3.7035|          121.1825|           69.1455|           20.935|           80.254|32.192999999999984|     0.47092999999999974|           33.0905|             0.342|\n",
            "| stddev|3.306063032730656|32.068635649902916|19.188314815604098|16.10324290992682|111.1805335457595| 8.149900701279762|      0.3235525586811429|11.786423106049496|0.4744982342297426|\n",
            "|    min|                0|                 0|                 0|                0|                0|               0.0|                   0.078|                21|                 0|\n",
            "|    max|               17|               199|               122|              110|              744|              80.6|                    2.42|                81|                 1|\n",
            "+-------+-----------------+------------------+------------------+-----------------+-----------------+------------------+------------------------+------------------+------------------+\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Cleaning Data"
      ],
      "metadata": {
        "id": "IyijDUrc6ec4"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "#find for null values\n",
        "for col in df.columns:\n",
        "  print(col+\":\", df[df[col].isNull()].count())"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "7DSuclTG6Azg",
        "outputId": "610d7ce9-5b97-448f-eb45-a51a5cdc7ed3"
      },
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Pregnancies: 0\n",
            "Glucose: 0\n",
            "BloodPressure: 0\n",
            "SkinThickness: 0\n",
            "Insulin: 0\n",
            "BMI: 0\n",
            "DiabetesPedigreeFunction: 0\n",
            "Age: 0\n",
            "Outcome: 0\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# find total number of 0 values in columns: Glucose, Bloodpressure, SkinThickness, Insulin and BMI\n",
        "def count_zeros():\n",
        "  columns_list = ['Glucose', 'BloodPressure', 'SkinThickness', 'Insulin', 'BMI']\n",
        "  for i in columns_list:\n",
        "    print(i+\":\", df[df[i]==0].count())"
      ],
      "metadata": {
        "id": "JPaelr0j7BlK"
      },
      "execution_count": 14,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "count_zeros()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "9od_l_Ps72Tw",
        "outputId": "2bdbf006-6a1c-4aeb-c80c-0f3075874e3f"
      },
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Glucose: 13\n",
            "BloodPressure: 90\n",
            "SkinThickness: 573\n",
            "Insulin: 956\n",
            "BMI: 28\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from pyspark.sql.functions import *\n",
        "for i in df.columns[1:6]:\n",
        "  data = df.agg({i: 'mean'}).first()[0]\n",
        "  print(\"Mean value for {} is {}\".format(i, int(data)))\n",
        "  df = df.withColumn(i, when(df[i]==0, int(data)).otherwise(df[i]))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "hlrsvIkx8BAI",
        "outputId": "6e0b7777-7d22-47b2-a386-23db3f00dd67"
      },
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Mean value for Glucose is 121\n",
            "Mean value for BloodPressure is 69\n",
            "Mean value for SkinThickness is 20\n",
            "Mean value for Insulin is 80\n",
            "Mean value for BMI is 32\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df.show()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Nd5ya-2r9H9u",
        "outputId": "e4498cc7-9957-4190-8b8c-92957d119c6f"
      },
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+-----------+-------+-------------+-------------+-------+----+------------------------+---+-------+\n",
            "|Pregnancies|Glucose|BloodPressure|SkinThickness|Insulin| BMI|DiabetesPedigreeFunction|Age|Outcome|\n",
            "+-----------+-------+-------------+-------------+-------+----+------------------------+---+-------+\n",
            "|          2|    138|           62|           35|     80|33.6|                   0.127| 47|      1|\n",
            "|          0|     84|           82|           31|    125|38.2|                   0.233| 23|      0|\n",
            "|          0|    145|           69|           20|     80|44.2|                    0.63| 31|      1|\n",
            "|          0|    135|           68|           42|    250|42.3|                   0.365| 24|      1|\n",
            "|          1|    139|           62|           41|    480|40.7|                   0.536| 21|      0|\n",
            "|          0|    173|           78|           32|    265|46.5|                   1.159| 58|      0|\n",
            "|          4|     99|           72|           17|     80|25.6|                   0.294| 28|      0|\n",
            "|          8|    194|           80|           20|     80|26.1|                   0.551| 67|      0|\n",
            "|          2|     83|           65|           28|     66|36.8|                   0.629| 24|      0|\n",
            "|          2|     89|           90|           30|     80|33.5|                   0.292| 42|      0|\n",
            "|          4|     99|           68|           38|     80|32.8|                   0.145| 33|      0|\n",
            "|          4|    125|           70|           18|    122|28.9|                   1.144| 45|      1|\n",
            "|          3|     80|           69|           20|     80|32.0|                   0.174| 22|      0|\n",
            "|          6|    166|           74|           20|     80|26.6|                   0.304| 66|      0|\n",
            "|          5|    110|           68|           20|     80|26.0|                   0.292| 30|      0|\n",
            "|          2|     81|           72|           15|     76|30.1|                   0.547| 25|      0|\n",
            "|          7|    195|           70|           33|    145|25.1|                   0.163| 55|      1|\n",
            "|          6|    154|           74|           32|    193|29.3|                   0.839| 39|      0|\n",
            "|          2|    117|           90|           19|     71|25.2|                   0.313| 21|      0|\n",
            "|          3|     84|           72|           32|     80|37.2|                   0.267| 28|      0|\n",
            "+-----------+-------+-------------+-------------+-------+----+------------------------+---+-------+\n",
            "only showing top 20 rows\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "for col in df.columns:\n",
        "  print(\"correlation to outcome for {} is {}\".format(col, df.stat.corr('Outcome', col)))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "wp6G46CO9aC5",
        "outputId": "024f824a-ed70-46ed-9b1f-5f9deefd541a"
      },
      "execution_count": 20,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "correlation to outcome for Pregnancies is 0.22443699263363961\n",
            "correlation to outcome for Glucose is 0.48796646527321064\n",
            "correlation to outcome for BloodPressure is 0.17171333286446713\n",
            "correlation to outcome for SkinThickness is 0.1659010662889893\n",
            "correlation to outcome for Insulin is 0.1711763270226193\n",
            "correlation to outcome for BMI is 0.2827927569760082\n",
            "correlation to outcome for DiabetesPedigreeFunction is 0.1554590791569403\n",
            "correlation to outcome for Age is 0.23650924717620253\n",
            "correlation to outcome for Outcome is 1.0\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#convert all columns in one column \"features\"\n",
        "from pyspark.ml.feature import VectorAssembler\n",
        "assembler = VectorAssembler(inputCols=['Pregnancies',\n",
        "                                       'Glucose',\n",
        "                                       'BloodPressure',\n",
        "                                       'SkinThickness',\n",
        "                                       'Insulin',\n",
        "                                       'BMI',\n",
        "                                       'DiabetesPedigreeFunction',\n",
        "                                       'Age'], outputCol='features')\n",
        "output_data = assembler.transform(df)"
      ],
      "metadata": {
        "id": "Nm7_YTu4-K_n"
      },
      "execution_count": 23,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "output_data.printSchema()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "kLgGb7nr_C3T",
        "outputId": "d6aa6967-d58b-4a04-ef9a-cf156332b808"
      },
      "execution_count": 25,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "root\n",
            " |-- Pregnancies: integer (nullable = true)\n",
            " |-- Glucose: integer (nullable = true)\n",
            " |-- BloodPressure: integer (nullable = true)\n",
            " |-- SkinThickness: integer (nullable = true)\n",
            " |-- Insulin: integer (nullable = true)\n",
            " |-- BMI: double (nullable = true)\n",
            " |-- DiabetesPedigreeFunction: double (nullable = true)\n",
            " |-- Age: integer (nullable = true)\n",
            " |-- Outcome: integer (nullable = true)\n",
            " |-- features: vector (nullable = true)\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "output_data.show()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "RondALE8_Stl",
        "outputId": "e3519325-3e86-49f5-86f0-81f9fae6c71d"
      },
      "execution_count": 26,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+-----------+-------+-------------+-------------+-------+----+------------------------+---+-------+--------------------+\n",
            "|Pregnancies|Glucose|BloodPressure|SkinThickness|Insulin| BMI|DiabetesPedigreeFunction|Age|Outcome|            features|\n",
            "+-----------+-------+-------------+-------------+-------+----+------------------------+---+-------+--------------------+\n",
            "|          2|    138|           62|           35|     80|33.6|                   0.127| 47|      1|[2.0,138.0,62.0,3...|\n",
            "|          0|     84|           82|           31|    125|38.2|                   0.233| 23|      0|[0.0,84.0,82.0,31...|\n",
            "|          0|    145|           69|           20|     80|44.2|                    0.63| 31|      1|[0.0,145.0,69.0,2...|\n",
            "|          0|    135|           68|           42|    250|42.3|                   0.365| 24|      1|[0.0,135.0,68.0,4...|\n",
            "|          1|    139|           62|           41|    480|40.7|                   0.536| 21|      0|[1.0,139.0,62.0,4...|\n",
            "|          0|    173|           78|           32|    265|46.5|                   1.159| 58|      0|[0.0,173.0,78.0,3...|\n",
            "|          4|     99|           72|           17|     80|25.6|                   0.294| 28|      0|[4.0,99.0,72.0,17...|\n",
            "|          8|    194|           80|           20|     80|26.1|                   0.551| 67|      0|[8.0,194.0,80.0,2...|\n",
            "|          2|     83|           65|           28|     66|36.8|                   0.629| 24|      0|[2.0,83.0,65.0,28...|\n",
            "|          2|     89|           90|           30|     80|33.5|                   0.292| 42|      0|[2.0,89.0,90.0,30...|\n",
            "|          4|     99|           68|           38|     80|32.8|                   0.145| 33|      0|[4.0,99.0,68.0,38...|\n",
            "|          4|    125|           70|           18|    122|28.9|                   1.144| 45|      1|[4.0,125.0,70.0,1...|\n",
            "|          3|     80|           69|           20|     80|32.0|                   0.174| 22|      0|[3.0,80.0,69.0,20...|\n",
            "|          6|    166|           74|           20|     80|26.6|                   0.304| 66|      0|[6.0,166.0,74.0,2...|\n",
            "|          5|    110|           68|           20|     80|26.0|                   0.292| 30|      0|[5.0,110.0,68.0,2...|\n",
            "|          2|     81|           72|           15|     76|30.1|                   0.547| 25|      0|[2.0,81.0,72.0,15...|\n",
            "|          7|    195|           70|           33|    145|25.1|                   0.163| 55|      1|[7.0,195.0,70.0,3...|\n",
            "|          6|    154|           74|           32|    193|29.3|                   0.839| 39|      0|[6.0,154.0,74.0,3...|\n",
            "|          2|    117|           90|           19|     71|25.2|                   0.313| 21|      0|[2.0,117.0,90.0,1...|\n",
            "|          3|     84|           72|           32|     80|37.2|                   0.267| 28|      0|[3.0,84.0,72.0,32...|\n",
            "+-----------+-------+-------------+-------------+-------+----+------------------------+---+-------+--------------------+\n",
            "only showing top 20 rows\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from pyspark.ml.classification import LogisticRegression\n",
        "final_data = output_data.select('features', 'Outcome')"
      ],
      "metadata": {
        "id": "_Ol-4wjD_yLn"
      },
      "execution_count": 28,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "final_data.printSchema()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "5A6SWikuAJSw",
        "outputId": "4628bf98-56d1-4596-8f07-d71319ea1f2d"
      },
      "execution_count": 29,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "root\n",
            " |-- features: vector (nullable = true)\n",
            " |-- Outcome: integer (nullable = true)\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "train, test = final_data.randomSplit([0.7, 0.3])\n",
        "models = LogisticRegression(labelCol='Outcome')\n",
        "model = models.fit(train)"
      ],
      "metadata": {
        "id": "SECxG0wqALQD"
      },
      "execution_count": 30,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "summary = model.summary"
      ],
      "metadata": {
        "id": "wR4PLK02AeE6"
      },
      "execution_count": 31,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "summary.predictions.describe().show()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "U87UkVf5AkNk",
        "outputId": "b83e2f10-a327-48bf-b245-c12398184547"
      },
      "execution_count": 32,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+-------+------------------+-------------------+\n",
            "|summary|           Outcome|         prediction|\n",
            "+-------+------------------+-------------------+\n",
            "|  count|              1406|               1406|\n",
            "|   mean|0.3314366998577525|0.24395448079658605|\n",
            "| stddev|0.4708971507614184|0.42961839649833605|\n",
            "|    min|               0.0|                0.0|\n",
            "|    max|               1.0|                1.0|\n",
            "+-------+------------------+-------------------+\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Evaluation & Test Model"
      ],
      "metadata": {
        "id": "uSZOmHVMA35c"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from pyspark.ml.evaluation import BinaryClassificationEvaluator\n",
        "predictions = model.evaluate(test)"
      ],
      "metadata": {
        "id": "jqIApQHNAreo"
      },
      "execution_count": 33,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "predictions.predictions.show(30)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "zmpCmtDmBHtu",
        "outputId": "06a21fd3-839f-4b48-a9e4-e59170a241a8"
      },
      "execution_count": 36,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+--------------------+-------+--------------------+--------------------+----------+\n",
            "|            features|Outcome|       rawPrediction|         probability|prediction|\n",
            "+--------------------+-------+--------------------+--------------------+----------+\n",
            "|[0.0,67.0,76.0,20...|      0|[2.23856234057849...|[0.90365936954836...|       0.0|\n",
            "|[0.0,73.0,69.0,20...|      0|[4.07619412877912...|[0.98331130359106...|       0.0|\n",
            "|[0.0,78.0,88.0,29...|      0|[2.72737596900438...|[0.93862284062239...|       0.0|\n",
            "|[0.0,84.0,64.0,22...|      0|[2.47327835795519...|[0.92224717280921...|       0.0|\n",
            "|[0.0,86.0,68.0,32...|      0|[2.64179687819792...|[0.93350359190717...|       0.0|\n",
            "|[0.0,91.0,68.0,32...|      0|[2.26120310496104...|[0.90561252083396...|       0.0|\n",
            "|[0.0,91.0,80.0,20...|      0|[2.36081769741827...|[0.91379024357165...|       0.0|\n",
            "|[0.0,93.0,60.0,20...|      0|[2.36432634115091...|[0.91406624523050...|       0.0|\n",
            "|[0.0,93.0,60.0,20...|      0|[2.36432634115091...|[0.91406624523050...|       0.0|\n",
            "|[0.0,93.0,60.0,25...|      0|[2.70577408779405...|[0.93736650110534...|       0.0|\n",
            "|[0.0,93.0,100.0,3...|      0|[1.02497115391416...|[0.73593978823573...|       0.0|\n",
            "|[0.0,94.0,69.0,20...|      0|[2.59762810424199...|[0.93070877238876...|       0.0|\n",
            "|[0.0,94.0,69.0,20...|      0|[2.59762810424199...|[0.93070877238876...|       0.0|\n",
            "|[0.0,95.0,64.0,39...|      0|[1.66882728270544...|[0.84141940485977...|       0.0|\n",
            "|[0.0,95.0,80.0,45...|      0|[2.24154547623227...|[0.90391876614332...|       0.0|\n",
            "|[0.0,97.0,64.0,36...|      0|[1.89270137085703...|[0.86906323243121...|       0.0|\n",
            "|[0.0,98.0,82.0,15...|      0|[3.00321392474627...|[0.95271911019829...|       0.0|\n",
            "|[0.0,98.0,82.0,15...|      0|[3.00321392474627...|[0.95271911019829...|       0.0|\n",
            "|[0.0,99.0,69.0,20...|      0|[2.99543073095335...|[0.95236727511392...|       0.0|\n",
            "|[0.0,100.0,70.0,2...|      0|[2.21636181900813...|[0.90170921571668...|       0.0|\n",
            "|[0.0,100.0,88.0,6...|      0|[0.73478981012282...|[0.67585548211240...|       0.0|\n",
            "|[0.0,101.0,62.0,2...|      0|[3.00375418575083...|[0.95274344052381...|       0.0|\n",
            "|[0.0,101.0,62.0,2...|      0|[3.00375418575083...|[0.95274344052381...|       0.0|\n",
            "|[0.0,101.0,76.0,2...|      0|[2.12138508046842...|[0.89296438618067...|       0.0|\n",
            "|[0.0,101.0,76.0,2...|      0|[2.12138508046842...|[0.89296438618067...|       0.0|\n",
            "|[0.0,102.0,52.0,2...|      0|[3.01406385857244...|[0.95320545619704...|       0.0|\n",
            "|[0.0,102.0,64.0,4...|      0|[1.58613094188900...|[0.83007105717458...|       0.0|\n",
            "|[0.0,102.0,75.0,2...|      0|[2.13549972106658...|[0.89430598576641...|       0.0|\n",
            "|[0.0,102.0,78.0,4...|      0|[2.23987029729666...|[0.90377317899649...|       0.0|\n",
            "|[0.0,104.0,64.0,3...|      1|[1.95509478004189...|[0.87600111390338...|       0.0|\n",
            "+--------------------+-------+--------------------+--------------------+----------+\n",
            "only showing top 30 rows\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#model acuracy\n",
        "evaluator = BinaryClassificationEvaluator(rawPredictionCol='rawPrediction', labelCol='Outcome')\n",
        "evaluator.evaluate(model.transform(test))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "anWBeh6UBTLC",
        "outputId": "c2c445b1-ba90-463a-cef1-915504b612a1"
      },
      "execution_count": 37,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.8565415772008592"
            ]
          },
          "metadata": {},
          "execution_count": 37
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "model.save(\"model\")"
      ],
      "metadata": {
        "id": "V0zkyfmoCM2X"
      },
      "execution_count": 38,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#load saved model back\n",
        "from pyspark.ml.classification import LogisticRegressionModel\n",
        "model = LogisticRegressionModel.load(\"model\")"
      ],
      "metadata": {
        "id": "WzOrrIBACVLi"
      },
      "execution_count": 39,
      "outputs": []
    }
  ]
}