Spark_StrokePrediction.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "e31bc851",
   "metadata": {
    "id": "gUxZnsqrmynW"
   },
   "source": [
    "### Create a spark session and import the required libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "ab3ba311",
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyspark.sql import SparkSession\n",
    "from pyspark.sql.functions import *\n",
    "from pyspark.sql.types import *\n",
    "from pyspark.sql import functions as F"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "37b641ca",
   "metadata": {},
   "outputs": [],
   "source": [
    "spark = SparkSession.builder.appName(\"First Project\").getOrCreate()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "843b8468",
   "metadata": {
    "id": "gUxZnsqrmynW"
   },
   "source": [
    "### Create a data schema programatically"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "8616fe16",
   "metadata": {},
   "outputs": [],
   "source": [
    "customSchema = StructType([StructField('id', IntegerType(), True),\n",
    "                           StructField('gender', StringType(), True),\n",
    "                           StructField('age', DoubleType(), True),\n",
    "                           StructField('hypertension', IntegerType(), True),\n",
    "                           StructField('heart_disease', IntegerType(), True),\n",
    "                           StructField('ever_married', StringType(), True),\n",
    "                           StructField('work_type', StringType(), True),\n",
    "                           StructField('Residence_type', StringType(), True),\n",
    "                           StructField('avg_glucose_level', DoubleType(), True),\n",
    "                           StructField('bmi', DoubleType(), True),\n",
    "                           StructField('smoking_status', StringType(), True),\n",
    "                           StructField('stroke', IntegerType(), True)])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2d00718f",
   "metadata": {
    "id": "gUxZnsqrmynW"
   },
   "source": [
    "### Read the data using the standard DataReader (Key,Value) pairs format"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "61fcc4cd",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = (spark.read.format('csv')\n",
    "    .schema(customSchema)\n",
    "    .option('header','true')\n",
    "    .load(\"healthcare-dataset-stroke-data.csv\")\n",
    ")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "974bc193",
   "metadata": {
    "id": "gUxZnsqrmynW"
   },
   "source": [
    "### Explore the data "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "353b6f34",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "root\n",
      " |-- id: integer (nullable = true)\n",
      " |-- gender: string (nullable = true)\n",
      " |-- age: double (nullable = true)\n",
      " |-- hypertension: integer (nullable = true)\n",
      " |-- heart_disease: integer (nullable = true)\n",
      " |-- ever_married: string (nullable = true)\n",
      " |-- work_type: string (nullable = true)\n",
      " |-- Residence_type: string (nullable = true)\n",
      " |-- avg_glucose_level: double (nullable = true)\n",
      " |-- bmi: double (nullable = true)\n",
      " |-- smoking_status: string (nullable = true)\n",
      " |-- stroke: integer (nullable = true)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df.printSchema()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "ae2be32b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-----+------+----+------------+-------------+------------+-------------+--------------+-----------------+----+---------------+------+\n",
      "|   id|gender| age|hypertension|heart_disease|ever_married|    work_type|Residence_type|avg_glucose_level| bmi| smoking_status|stroke|\n",
      "+-----+------+----+------------+-------------+------------+-------------+--------------+-----------------+----+---------------+------+\n",
      "| 9046|  Male|67.0|           0|            1|         Yes|      Private|         Urban|           228.69|36.6|formerly smoked|     1|\n",
      "|51676|Female|61.0|           0|            0|         Yes|Self-employed|         Rural|           202.21|null|   never smoked|     1|\n",
      "|31112|  Male|80.0|           0|            1|         Yes|      Private|         Rural|           105.92|32.5|   never smoked|     1|\n",
      "|60182|Female|49.0|           0|            0|         Yes|      Private|         Urban|           171.23|34.4|         smokes|     1|\n",
      "| 1665|Female|79.0|           1|            0|         Yes|Self-employed|         Rural|           174.12|24.0|   never smoked|     1|\n",
      "|56669|  Male|81.0|           0|            0|         Yes|      Private|         Urban|           186.21|29.0|formerly smoked|     1|\n",
      "|53882|  Male|74.0|           1|            1|         Yes|      Private|         Rural|            70.09|27.4|   never smoked|     1|\n",
      "|10434|Female|69.0|           0|            0|          No|      Private|         Urban|            94.39|22.8|   never smoked|     1|\n",
      "|27419|Female|59.0|           0|            0|         Yes|      Private|         Rural|            76.15|null|        Unknown|     1|\n",
      "|60491|Female|78.0|           0|            0|         Yes|      Private|         Urban|            58.57|24.2|        Unknown|     1|\n",
      "+-----+------+----+------------+-------------+------------+-------------+--------------+-----------------+----+---------------+------+\n",
      "only showing top 10 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df.show(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "66d40eed",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+---+------+---+------------+-------------+------------+---------+--------------+-----------------+---+--------------+------+\n",
      "| id|gender|age|hypertension|heart_disease|ever_married|work_type|Residence_type|avg_glucose_level|bmi|smoking_status|stroke|\n",
      "+---+------+---+------------+-------------+------------+---------+--------------+-----------------+---+--------------+------+\n",
      "|  0|     0|  0|           0|            0|           0|        0|             0|                0|201|             0|     0|\n",
      "+---+------+---+------------+-------------+------------+---------+--------------+-----------------+---+--------------+------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df.select([count(when(isnull(column),1)).alias(column)for column in df.columns]).show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "6fb198f0",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "977b1754",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<AxesSubplot:xlabel='stroke'>"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAEGCAYAAACJnEVTAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAARxUlEQVR4nO3df6ye5V3H8feHtrb8VKAHUnqYraZu/JAfcuxwLNnqMdK5xWKEpG6jnSM7kWDYookBZzRqGjEzC2kiRDIZJU6bZupotjBHuuI2UsFT2YTCkGYwOGlDu+KPTsOPwtc/zg0+tg89p+X0OZTr/Uqe3Pfzva/rPtednHx693ru5zqpKiRJbThhtgcgSRocQ1+SGmLoS1JDDH1JaoihL0kNmTvbA5jKwoULa8mSJbM9DEk6rmzfvv0HVTV0cP0tH/pLlixhfHx8tochSceVJN/vV5/W9E6Sp5M8kuTbSca72hlJ7kvyZLc9vaf9zUl2JnkiyZU99cu68+xMsj5J3uyFSZKm70jm9FdU1SVVNdK9vwnYUlXLgC3de5KcD6wGLgBWArclmdP1uR0YA5Z1r5Vv/hIkSdP1Zj7IXQVs6PY3AFf11DdW1YtV9RSwE1ieZBFwWlVtq8mvAd/d00eSNADTndMv4GtJCviLqroDOLuqdgNU1e4kZ3VtFwP/1NN3oqu93O0fXD9EkjEm/0fAO97xjkOOv/zyy0xMTPDCCy9Mc/hvHwsWLGB4eJh58+bN9lAkHYemG/pXVNWuLtjvS/Ldw7TtN09fh6kfWpz8R+UOgJGRkUPaTExMcOqpp7JkyRJa+ligqti3bx8TExMsXbp0tocj6Tg0remdqtrVbfcAfw8sB57rpmzotnu65hPAuT3dh4FdXX24T/2IvfDCC5x55plNBT5AEs4888wm/4cjaWZMGfpJTk5y6mv7wC8CjwKbgbVds7XAPd3+ZmB1kvlJljL5ge1D3VTQ/iSXd0/trOnpc8RaC/zXtHrdkmbGdKZ3zgb+vgubucBfV9VXk/wzsCnJdcAzwDUAVbUjySbgMeAAcENVvdKd63rgLuBE4N7uJUkakClDv6q+B1zcp74PGH2DPuuAdX3q48CFRz7Mw1ty01dm9HxP3/LBGT3fkbr11lsZGxvjpJNOmtVxSHr7ect/I7dFt956Kx/96EcNfc2Imb4pat1s3xS+WS64dpTuvvtuLrroIi6++GKuvfZavv/97zM6OspFF13E6OgozzzzDAAf+9jH+OIXv/h6v1NOOQWA+++/n/e///1cffXVvOtd7+IjH/kIVcX69evZtWsXK1asYMWKFbNybZLevrzTPwo7duxg3bp1PPDAAyxcuJDnn3+etWvXsmbNGtauXcudd97JjTfeyJe+9KXDnufhhx9mx44dnHPOOVxxxRU88MAD3HjjjXz2s59l69atLFy4cDAXJKkZ3ukfha9//etcffXVr4fyGWecwbZt2/jwhz8MwLXXXsu3vvWtKc+zfPlyhoeHOeGEE7jkkkt4+umnj+WwJcnQPxpVNeWjk68dnzt3Lq+++urr/V566aXX28yfP//1/Tlz5nDgwIFjMFpJ+j+G/lEYHR1l06ZN7Nu3D4Dnn3+e97znPWzcuBGAL3zhC7z3ve8FJpeG3r59OwD33HMPL7/88pTnP/XUU9m/f/8xGr2klr0t5vQH/Wn6BRdcwKc//Wne9773MWfOHC699FLWr1/Pxz/+cT7zmc8wNDTE5z//eQA+8YlPsGrVKpYvX87o6Cgnn3zylOcfGxvjAx/4AIsWLWLr1q3H+nIkNSSTC16+dY2MjNTBf0Tl8ccf57zzzpulEc2+1q9fR8ZHNmfW8fLIZpLtPUvhv87pHUlqiKEvSQ05bkP/rT4tday0et2SZsZxGfoLFixg3759zQXga+vpL1iwYLaHIuk4dVw+vTM8PMzExAR79+6d7aEM3Gt/OUuSjsZxGfrz5s3zL0dJ0lE4Lqd3JElHx9CXpIYY+pLUEENfkhpi6EtSQwx9SWqIoS9JDTH0Jakhhr4kNcTQl6SGGPqS1BBDX5IaYuhLUkMMfUlqiKEvSQ0x9CWpIYa+JDXE0Jekhhj6ktSQaYd+kjlJHk7y5e79GUnuS/Jktz29p+3NSXYmeSLJlT31y5I80h1bnyQzezmSpMM5kjv9TwKP97y/CdhSVcuALd17kpwPrAYuAFYCtyWZ0/W5HRgDlnWvlW9q9JKkIzKt0E8yDHwQ+FxPeRWwodvfAFzVU99YVS9W1VPATmB5kkXAaVW1raoKuLunjyRpAKZ7p38r8DvAqz21s6tqN0C3PaurLwae7Wk30dUWd/sH1w+RZCzJeJLxvXv3TnOIkqSpTBn6ST4E7Kmq7dM8Z795+jpM/dBi1R1VNVJVI0NDQ9P8sZKkqcydRpsrgF9O8kvAAuC0JH8FPJdkUVXt7qZu9nTtJ4Bze/oPA7u6+nCfuiRpQKa806+qm6tquKqWMPkB7der6qPAZmBt12wtcE+3vxlYnWR+kqVMfmD7UDcFtD/J5d1TO2t6+kiSBmA6d/pv5BZgU5LrgGeAawCqakeSTcBjwAHghqp6petzPXAXcCJwb/eSJA3IEYV+Vd0P3N/t7wNG36DdOmBdn/o4cOGRDlKSNDP8Rq4kNcTQl6SGGPqS1BBDX5IaYuhLUkMMfUlqiKEvSQ0x9CWpIYa+JDXE0Jekhhj6ktQQQ1+SGmLoS1JDDH1JaoihL0kNMfQlqSGGviQ1xNCXpIYY+pLUEENfkhpi6EtSQwx9SWqIoS9JDTH0Jakhhr4kNcTQl6SGGPqS1BBDX5IaYuhLUkMMfUlqiKEvSQ0x9CWpIYa+JDXE0JekhkwZ+kkWJHkoyXeS7Ejyh139jCT3JXmy257e0+fmJDuTPJHkyp76ZUke6Y6tT5Jjc1mSpH6mc6f/IvDzVXUxcAmwMsnlwE3AlqpaBmzp3pPkfGA1cAGwErgtyZzuXLcDY8Cy7rVy5i5FkjSVKUO/Jv2wezuvexWwCtjQ1TcAV3X7q4CNVfViVT0F7ASWJ1kEnFZV26qqgLt7+kiSBmBac/pJ5iT5NrAHuK+qHgTOrqrdAN32rK75YuDZnu4TXW1xt39wvd/PG0synmR87969R3A5kqTDmVboV9UrVXUJMMzkXfuFh2neb56+DlPv9/PuqKqRqhoZGhqazhAlSdNwRE/vVNV/APczORf/XDdlQ7fd0zWbAM7t6TYM7Orqw33qkqQBmc7TO0NJfqzbPxH4BeC7wGZgbddsLXBPt78ZWJ1kfpKlTH5g+1A3BbQ/yeXdUztrevpIkgZg7jTaLAI2dE/gnABsqqovJ9kGbEpyHfAMcA1AVe1Isgl4DDgA3FBVr3Tnuh64CzgRuLd7SZIGZMrQr6p/BS7tU98HjL5Bn3XAuj71ceBwnwdIko4hv5ErSQ0x9CWpIYa+JDXE0Jekhhj6ktQQQ1+SGmLoS1JDDH1JaoihL0kNMfQlqSGGviQ1xNCXpIYY+pLUEENfkhpi6EtSQwx9SWqIoS9JDTH0Jakhhr4kNcTQl6SGGPqS1BBDX5IaYuhLUkMMfUlqiKEvSQ0x9CWpIYa+JDXE0Jekhhj6ktQQQ1+SGmLoS1JDDH1JaoihL0kNmTL0k5ybZGuSx5PsSPLJrn5GkvuSPNltT+/pc3OSnUmeSHJlT/2yJI90x9YnybG5LElSP9O50z8A/HZVnQdcDtyQ5HzgJmBLVS0DtnTv6Y6tBi4AVgK3JZnTnet2YAxY1r1WzuC1SJKmMGXoV9XuqvqXbn8/8DiwGFgFbOiabQCu6vZXARur6sWqegrYCSxPsgg4raq2VVUBd/f0kSQNwBHN6SdZAlwKPAicXVW7YfIfBuCsrtli4NmebhNdbXG3f3BdkjQg0w79JKcAfwt8qqr+63BN+9TqMPV+P2ssyXiS8b179053iJKkKUwr9JPMYzLwv1BVf9eVn+umbOi2e7r6BHBuT/dhYFdXH+5TP0RV3VFVI1U1MjQ0NN1rkSRNYTpP7wT4S+Dxqvpsz6HNwNpufy1wT099dZL5SZYy+YHtQ90U0P4kl3fnXNPTR5I0AHOn0eYK4FrgkSTf7mq/C9wCbEpyHfAMcA1AVe1Isgl4jMknf26oqle6ftcDdwEnAvd2L0nSgEwZ+lX1LfrPxwOMvkGfdcC6PvVx4MIjGaAkaeb4jVxJaoihL0kNMfQlqSGGviQ1xNCXpIYY+pLUEENfkhpi6EtSQwx9SWqIoS9JDTH0Jakhhr4kNcTQl6SGGPqS1BBDX5IaYuhLUkMMfUlqiKEvSQ0x9CWpIYa+JDXE0Jekhhj6ktQQQ1+SGmLoS1JDDH1JaoihL0kNMfQlqSGGviQ1xNCXpIYY+pLUEENfkhpi6EtSQwx9SWqIoS9JDZky9JPcmWRPkkd7amckuS/Jk9329J5jNyfZmeSJJFf21C9L8kh3bH2SzPzlSJIOZzp3+ncBKw+q3QRsqaplwJbuPUnOB1YDF3R9bksyp+tzOzAGLOteB59TknSMTRn6VfUN4PmDyquADd3+BuCqnvrGqnqxqp4CdgLLkywCTquqbVVVwN09fSRJA3K0c/pnV9VugG57VldfDDzb026iqy3u9g+u95VkLMl4kvG9e/ce5RAlSQeb6Q9y+83T12HqfVXVHVU1UlUjQ0NDMzY4SWrd0Yb+c92UDd12T1efAM7taTcM7Orqw33qkqQBOtrQ3wys7fbXAvf01FcnmZ9kKZMf2D7UTQHtT3J599TOmp4+kqQBmTtVgyR/A7wfWJhkAvgD4BZgU5LrgGeAawCqakeSTcBjwAHghqp6pTvV9Uw+CXQicG/3kiQN0JShX1W/9gaHRt+g/TpgXZ/6OHDhEY1OkjSj/EauJDXE0Jekhhj6ktQQQ1+SGmLoS1JDDH1JaoihL0kNMfQlqSGGviQ1xNCXpIYY+pLUEENfkhpi6EtSQwx9SWqIoS9JDTH0Jakhhr4kNcTQl6SGGPqS1BBDX5IaYuhLUkMMfUlqiKEvSQ0x9CWpIYa+JDXE0Jekhsyd7QG8XSy56SuzPYS3jadv+eBsD0F62/JOX5IaYuhLUkMMfUlqiKEvSQ0x9CWpIYa+JDXE0Jekhgw89JOsTPJEkp1Jbhr0z5eklg009JPMAf4c+ABwPvBrSc4f5BgkqWWDvtNfDuysqu9V1UvARmDVgMcgSc0a9DIMi4Fne95PAO8+uFGSMWCse/vDJE8MYGwtWAj8YLYHMZX86WyPQLPE38+Z9eP9ioMO/fSp1SGFqjuAO479cNqSZLyqRmZ7HFI//n4OxqCndyaAc3veDwO7BjwGSWrWoEP/n4FlSZYm+RFgNbB5wGOQpGYNdHqnqg4k+U3gH4A5wJ1VtWOQY2icU2Z6K/P3cwBSdciUuiTpbcpv5EpSQwx9SWqIod+AJHcm2ZPk0dkei3Qwl2YZLEO/DXcBK2d7ENLBXJpl8Az9BlTVN4DnZ3scUh8uzTJghr6k2dRvaZbFszSWJhj6kmbTtJZm0cwx9CXNJpdmGTBDX9JscmmWATP0G5Dkb4BtwDuTTCS5brbHJMHk0izAa0uzPA5scmmWY8tlGCSpId7pS1JDDH1JaoihL0kNMfQlqSGGviQ1xNCXDpLkU0lOOsI+S1zFVMcDQ1861KeAvqHfrQopHbcMfTUtyclJvpLkO0keTfIHwDnA1iRbuzY/TPJHSR4Efi7Jb3VtH03yqT7n/IkkDyf52SQ/meSrSbYn+WaSdw32CqX/b6B/GF16C1oJ7KqqDwIk+VHg14EVVfWDrs3JwKNV9ftJLuuOv5vJxcIeTPKPwL93/d/J5PLAv15V306yBfiNqnoyybuB24CfH+D1Sf+P38hV05L8FJNLAGwCvlxV30zyNDDyWugnOQDMr6pXknwSOLOqfr879sfAXibXi3mQyfD/1arakeSU7tgTPT9yflWdN6DLkw7hnb6aVlX/1t29/xLwJ0m+1qfZC1X1Srffbyng1/wnk2vDXwHsYHL69D+q6pIZHLL0pjinr6YlOQf4n6r6K+DPgJ8B9gOnvkGXbwBXJTkpycnArwDf7I69BFwFrEny4ar6L+CpJNd0PytJLj52VyNNzTt9te6ngc8keRV4Gbge+Dng3iS7q2pFb+Oq+pckdwEPdaXPVdXDSZZ0x/87yYeA+5L8N/AR4PYkvwfMY3K+/zsDuC6pL+f0JakhTu9IUkMMfUlqiKEvSQ0x9CWpIYa+JDXE0Jekhhj6ktSQ/wWNza17B+GHaQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "dfPandas = df.groupBy('stroke').count().toPandas()\n",
    "dfPandas.plot.bar(x='stroke', y='count', rot=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "50caffe0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<AxesSubplot:xlabel='gender', ylabel='age'>"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX4AAAEGCAYAAABiq/5QAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAAVD0lEQVR4nO3df5Bd5X3f8feHlQ0CYhvJi6oIY8UjDbZrxxCvXWwyrm1QqiStgYmx8cRl3ZBqMuMKtdM2JWkat03rkjRNApokjSYmXtqUmFAw1ENlC9nYk8Z1LX7Y/K62GDBCQes1/sEPQyS+/eMemZW0khbQuWe15/2a2Tn3PPeec77LHD776LnnPCdVhSSpP47pugBJ0nAZ/JLUMwa/JPWMwS9JPWPwS1LPLOq6gLl49atfXStXruy6DEk6qtx6663fqqrR/duPiuBfuXIl27Zt67oMSTqqJHlotnaHeiSpZwx+SeoZg1+Sesbgl6SeMfglqWcMfknqGYNfknrmqLiOfyHYuHEjk5OTXZfBjh07AFixYkWndaxatYr169d3WoOeNx/Oz/lybsLCPz8N/p55+umnuy5BmpXn5vDkaHgQy9jYWHnn7pGxYcMGAC6//PKOK5H25bl55CW5tarG9m9vdYw/yT9JcneSu5JcneS4JEuSbEmyvVme1GYNkqR9tRb8SVYAlwBjVfUmYAS4ELgU2FpVq4GtzbokaUjavqpnEbA4ySLgeOBR4Fxgonl/Ajiv5RokSTO0FvxVtQP4beBhYCfw3ar6HLCsqnY2n9kJnDzb9knWJdmWZNvU1FRbZUpS77Q51HMSg979jwE/CpyQ5MNz3b6qNlXVWFWNjY4eMJ20JOlFanOo5xzgG1U1VVV/DVwHvBN4LMlygGa5q8UaJEn7aTP4HwbOTHJ8kgBnA/cCNwLjzWfGgRtarEGStJ9Wr+NP8m+ADwK7gduBXwROBK4BTmXwx+GCqvr2ofbzUq/jnw93Jc4Xe/87rFq1quNK5oeu79D03Hye5+a+jsS5ebDr+Fu9c7eqPgZ8bL/mZxj0/odmcnKSO+66lz3HLxnmYeelY54d/KG/9YHHOq6keyNPHbK/MRSTk5Nsv/t2Tj1xT9eldO7lfz0YgHjmIW/WfPiJkVb335spG/Ycv4SnX/8zXZeheWTxfTd1XQIAp564h1/9ie91XYbmkY/f9opW9+/snJLUMwa/JPWMwS9JPWPwS1LPGPyS1DMGvyT1jMEvST1j8EtSzxj8ktQzBr8k9YzBL0k9Y/BLUs/0ZpI2aT7asWMHT35/pPVJuXR0eej7I5ywY0dr+7fHL0k901qPP8lpwKdmNL0O+HXgqqZ9JfAg8IGqerytOqT5bMWKFTyze6fTMmsfH7/tFRy7YkVr+28t+KvqfuB0gCQjwA7geuBSYGtVXZbk0mb9X7RVBwz+OT3y1Hfnzfzrmh9Gnppmx47dXZchDd2whnrOBv5fVT0EnAtMNO0TwHlDqkGSxPC+3L0QuLp5vayqdgJU1c4kJ7d98BUrVvBXzyzyCVzax+L7bmLFimVdlyENXes9/iQvB94H/PkL3G5dkm1Jtk1NTbVTnCT10DCGen4auK2q9j7d+7EkywGa5a7ZNqqqTVU1VlVjo6OjQyhTkvphGMH/IZ4f5gG4ERhvXo8DNwyhBklSo9XgT3I8sAa4bkbzZcCaJNub9y5rswZJ0r5a/XK3qp4Clu7XNs3gKh9JUge8c1eSesbgl6SeMfglqWcMfknqGYNfknrG4JeknjH4JalnDH5J6hmDX5J6xuCXpJ4x+CWpZwx+SeoZg1+Sesbgl6SeMfglqWcMfknqmbafwPWqJNcmuS/JvUnekWRJki1JtjfLk9qsQZK0r7Z7/JcDm6vq9cBbgHuBS4GtVbUa2NqsS5KGpLXgT/IK4F3AJwCq6tmq+g5wLjDRfGwCOK+tGiRJB2qzx/86YAr4kyS3J/njJCcAy6pqJ0CzPHm2jZOsS7ItybapqakWy5Skfmkz+BcBPwH8YVWdATzJCxjWqapNVTVWVWOjo6Nt1ShJvbOoxX0/AjxSVV9p1q9lEPyPJVleVTuTLAd2tVjDD4089W0W33fTMA41rx3zg+8B8Nxxr+i4ku6NPPVtYFnXZUhD11rwV9VfJflmktOq6n7gbOCe5mccuKxZ3tBWDXutWrWq7UMcNSYnvw/AqtcZeLDMc0O91GaPH2A98KdJXg48APwDBsNL1yS5GHgYuKDlGli/fn3bhzhqbNiwAYDLL7+840okdaXV4K+qO4CxWd46u83jSpIOru0ev6TDePiJET5+m9+5PPbU4FqTZcc/13El3Xv4iRFWt7h/g1/qkN8xPO/ZyUkAjn2t/01W0+65YfBLHfL7p+f5/dPwOEmbJPWMwS9JPWPwS1LPGPyS1DMGvyT1jMEvST1j8EtSzxj8ktQzBr8k9YzBL0k9Y/BLUs8Y/JLUM61O0pbkQeD7wB5gd1WNJVkCfApYCTwIfKCqHm+zDknS84bR439PVZ1eVXsfyHIpsLWqVgNbeQEPYJckvXRdDPWcC0w0ryeA8zqoQZJ6q+3gL+BzSW5Nsq5pW1ZVOwGa5ckt1yBJmqHtB7GcVVWPJjkZ2JLkvrlu2PyhWAdw6qmntlWfJPVOqz3+qnq0We4CrgfeDjyWZDlAs9x1kG03VdVYVY2Njo62WaYk9UprwZ/khCQ/svc18FPAXcCNwHjzsXHghrZqkCQdqM2hnmXA9Un2Hue/VdXmJF8FrklyMfAwcEGLNUiS9tNa8FfVA8BbZmmfBs5u67iSpEPzzl1J6hmDX5J6xuCXpJ4x+CWpZwx+SeoZg1+Sesbgl6SeMfglqWdeUPA3Uy9Iko5icwr+JO9Mcg9wb7P+liR/0GplkqRWzLXH/7vA3wGmAarqa8C72ipKktSeOQ/1VNU392vac4RrkSQNwVwnaftmkncCleTlwCU0wz6SpKPLXIP/l4DLgRXAI8DngI+2VdRCtHHjRiYnJ7su44c1bNiwodM6Vq1axfr16zutQeqrOQV/VX0L+PmWa9EQLF68uOsSJHVsTsGf5IpZmr8LbKsqn6A1B/ZuJc0Xc/1y9zjgdGB78/PjwBLg4iS/d6gNk4wkuT3JZ5r1JUm2JNneLE960dVLkl6wuQb/KuC9VbWxqjYC5wBvAM5n8CzdQ9nAvl8EXwpsrarVwNZmXZI0JHMN/hXAzLt2TwB+tKr2AM8cbKMkpwA/C/zxjOZzgYnm9QRw3lyLlSS9dHO9que3gDuS3AKEwc1bH2+mcLj5ENv9HvDLwI/MaFtWVTsBqmpnkpNn2zDJOmAdwKmnnjrHMiVJhzOnHn9VfQI4C7gPuB74NeD/VtWTVfXPZ9smyd8FdlXVrS+msKraVFVjVTU2Ojr6YnYhSZrFXK/q+UUGY/WnAHcAZwJfBt57iM3OAt6X5GcYfDn8iiT/FXgsyfKmt78c2PUS6pckvUBzHePfALwNeKiq3gOcAUwdaoOq+pWqOqWqVgIXAp+vqg8DNwLjzcfGAS8HlaQhmmvw/6CqfgCQ5Niqug847UUe8zJgTZLtwJpmXZI0JHP9cveRJK8CPg1sSfI48OhcD1JVtwC3NK+ngbNfSJGSpCNnrlM2nN+8/NdJvgC8EtjcWlWSpNbMtcf/Q1X1xTYKkSQNh8/claSeMfglqWcMfknqGYNfknrG4JeknjH4JalnDH5J6hmDX5J6xuCXpJ4x+CWpZwx+SeqZFzxXj6SFZ+PGjUxOTnZaw97jb9iwodM6AFatWsX69eu7LqM1Br+keWHx4sVdl9AbrQV/kuOALwHHNse5tqo+lmQJ8ClgJfAg8IGqerytOiQd3kLu3epAbY7xPwO8t6reApwOrE1yJnApsLWqVgNbm3VJ0pC0Fvw18ESz+rLmp4BzgYmmfQI4r60aJEkHavWqniQjSe4AdgFbquorwLKq2gnQLE9uswZJ0r5aDf6q2lNVpwOnAG9P8qa5bptkXZJtSbZNTU21VqMk9c1QruOvqu8weNj6WuCxJMsBmuWug2yzqarGqmpsdHR0GGVKUi+0FvxJRpO8qnm9GDgHuA+4ERhvPjYO3NBWDZKkA7V5Hf9yYCLJCIM/MNdU1WeSfBm4JsnFwMPABS3WIEnaT2vBX1VfB86YpX0aOLut40qSDs25eiSpZwx+SeoZg1+Sesbgl6SeMfglqWcMfknqGYNfknrG4JeknjH4JalnDH5J6hmDX5J6xuCXpJ4x+CWpZwx+SeoZg1+Sesbgl6SeafPRi69J8oUk9ya5O8mGpn1Jki1JtjfLk9qqQZJ0oDZ7/LuBf1pVbwDOBD6a5I3ApcDWqloNbG3WJUlD0lrwV9XOqrqtef194F5gBXAuMNF8bAI4r60aJEkHGsoYf5KVDJ6/+xVgWVXthMEfB+Dkg2yzLsm2JNumpqaGUaYk9ULrwZ/kROC/A/+4qr431+2qalNVjVXV2OjoaHsFSlLPtBr8SV7GIPT/tKqua5ofS7K8eX85sKvNGiRJ+2rzqp4AnwDurarfmfHWjcB483ocuKGtGiRJB1rU4r7PAv4+cGeSO5q2XwUuA65JcjHwMHBBizVIkvbTWvBX1V8AOcjbZ7d1XEnSoXnnriT1jMEvST1j8EtSzxj8ktQzBr8k9YzBL2lemJ6e5pJLLmF6errrUhY8g1/SvDAxMcGdd97JVVdd1XUpC57BL6lz09PTbN68mapi8+bN9vpbZvBL6tzExATPPfccAHv27LHX3zKDX1Lnbr75Znbv3g3A7t272bJlS8cVLWwGv6TOnXPOOSxaNJhBZtGiRaxZs6bjihY2g19S58bHxznmmEEcjYyMcNFFF3Vc0cJm8Evq3NKlS1m7di1JWLt2LUuXLu26pAWtzWmZJWnOxsfHefDBB+3tD4HBL2leWLp0KVdccUXXZfRCm0/gujLJriR3zWhbkmRLku3N8qS2ji9Jml2bY/yfBNbu13YpsLWqVgNbm3VJ0hC1FvxV9SXg2/s1nwtMNK8ngPPaOr4kaXbDvqpnWVXtBGiWJw/5+JLUe/P2cs4k65JsS7Jtamqq63IkacEYdvA/lmQ5QLPcdbAPVtWmqhqrqrHR0dGhFShJC92wg/9GYLx5PQ7cMOTjS1LvtXk559XAl4HTkjyS5GLgMmBNku3AmmZdkjRErd3AVVUfOshbZ7d1TEnS4c3bL3clSe0w+CWpZwx+SeoZg1+Sesbgl6SeMfglqWcMfknqGYNfknrG4JeknjH4JalnDH5J6hmDX5J6xuCXNC9MT09zySWXMD093XUpC57BL2lemJiY4M477+Sqq67qupQFz+CX1Lnp6Wk2b95MVbF582Z7/S0z+CV1bmJigueeew6APXv22OtvWSfBn2RtkvuTTCa5tIsaJM0fN998M7t37wZg9+7dbNmypeOKFrahB3+SEeD3gZ8G3gh8KMkbh12HpPnjnHPOYdGiwQMBFy1axJo1azquaGHrosf/dmCyqh6oqmeBPwPO7aAOSfPE+Pg4xxwziKORkREuuuiijita2LoI/hXAN2esP9K07SPJuiTbkmybmpoaWnGShm/p0qWsXbuWJKxdu5alS5d2XdKC1kXwZ5a2OqChalNVjVXV2Ojo6BDKktSl8fFx3vzmN9vbH4JFHRzzEeA1M9ZPAR7toA5J88jSpUu54oorui6jF7ro8X8VWJ3kx5K8HLgQuLGDOiSpl4be46+q3Un+EfBZYAS4sqruHnYdktRXXQz1UFU3ATd1cWxJ6jvv3JWknknVARfUzDtJpoCHuq5jAXk18K2ui5Bm4bl5ZL22qg64LPKoCH4dWUm2VdVY13VI+/PcHA6HeiSpZwx+SeoZg7+fNnVdgHQQnptD4Bi/JPWMPX5J6hmDX5J6xuBfIJJUkv8yY31RkqkknznMdu8+3GekuUiyJ8kdM35WtnisB5O8uq39L3SdTNmgVjwJvCnJ4qp6GlgD7Oi4JvXL01V1etdF6PDs8S8s/xP42eb1h4Cr976R5O1J/jLJ7c3ytP03TnJCkiuTfLX5nE9G00uS5K1Jvpjk1iSfTbK8ab8lye8m+VKSe5O8Lcl1SbYn+Xcztv90s+3dSdYd5BgfTvJ/mn9l/FHzeFcdgsG/sPwZcGGS44AfB74y4737gHdV1RnArwMfn2X7fwl8vqreBrwH+I9JTmi5Zi0ci2cM81yf5GXARuD9VfVW4Erg38/4/LNV9S7gPwM3AB8F3gR8JMneR3D9QrPtGHDJjHYAkrwB+CBwVvOvjT3Az7f3Ky4MDvUsIFX19WZc9UMcOPvpK4GJJKsZPPHsZbPs4qeA9yX5Z836ccCpwL3tVKwFZp+hniRvYhDkW5LAYBr2nTM+v/c5HHcCd1fVzma7Bxg8rGmaQdif33zuNcDqpn2vs4G3Al9tjrEY2HVEf6sFyOBfeG4Efht4NzCzd/QbwBeq6vzmj8Mts2wb4Oeq6v6Wa1Q/hEGgv+Mg7z/TLJ+b8Xrv+qIk7wbOAd5RVU8luYVBZ2T/Y0xU1a8cqaL7wKGehedK4N9W1Z37tb+S57/s/chBtv0ssD5N1ynJGa1UqL64HxhN8g6AJC9L8jdfwPavBB5vQv/1wJmzfGYr8P4kJzfHWJLktS+18IXO4F9gquqRqrp8lrd+C/gPSf4Xg39yz+Y3GAwBfT3JXc269KJU1bPA+4HfTPI14A7gnS9gF5sZ9Py/zuBc/N+zHOMe4NeAzzWf2wIsf4mlL3hO2SBJPWOPX5J6xuCXpJ4x+CWpZwx+SeoZg1+Sesbgl46wJJ9M8v6u65AOxuCXOpbEO+g1VJ5w6rUk/4rBpF7fBL4F3ApcD/w+MAo8BfzDqrovySeB7zGYMOxvAL9cVdc2dzpvBN4LfIPBNAJ79/9W4HeAE5v9f6SqdjbTD/wlcBaDaTb+U+u/rNQw+NVbScaAnwPOYPD/wm0Mgn8T8EtVtT3J3wL+gEGow+Cu0J8EXs8gsK8FzgdOA94MLAPuAa6cMTvluVU1leSDDGan/IVmX6+qqr/d+i8q7cfgV5/9JHBD8+AakvwPBpOAvRP482bKIoBjZ2zz6ap6DrgnybKm7V3A1VW1B3g0yeeb9tM49OyUnzryv5J0eAa/+iyztB0DfOcQT5KaOYvkzO1nm/vkcLNTPnnYCqUW+OWu+uwvgL+X5LgkJzJ4etlTwDeSXACQgbccZj9fYvAAnJHmCVPvadpf6uyUUisMfvVWVX2VwTj914DrgG3Adxl82XtxM6Pk3cDhHkF5PbCdwQNF/hD4YrP/lzo7pdQKZ+dUryU5saqeSHI8g577uqq6reu6pDY5xq++25TkjQy+1J0w9NUH9vglqWcc45eknjH4JalnDH5J6hmDX5J6xuCXpJ75/yrmp0RNaXBfAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "dfPandas = df.select('gender','age','stroke').toPandas()\n",
    "dfPandas = dfPandas[dfPandas['stroke']==1]\n",
    "sns.boxplot(x='gender',y='age',data=dfPandas)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "ed40fb11",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------+------------------+\n",
      "|summary|               age|\n",
      "+-------+------------------+\n",
      "|  count|              5110|\n",
      "|   mean|43.226614481409015|\n",
      "| stddev| 22.61264672311348|\n",
      "|    min|              0.08|\n",
      "|    25%|              25.0|\n",
      "|    50%|              45.0|\n",
      "|    75%|              61.0|\n",
      "|    max|              82.0|\n",
      "+-------+------------------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df.select('age').summary().show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9cf74e5b",
   "metadata": {
    "id": "HQFPKukRLzbB"
   },
   "source": [
    "### Remove rows with age below 2 years old\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "cc99c0be",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------+------------------+\n",
      "|summary|               age|\n",
      "+-------+------------------+\n",
      "|  count|              4990|\n",
      "|   mean| 44.23907815631262|\n",
      "| stddev|21.908077881917443|\n",
      "|    min|               2.0|\n",
      "|    25%|              27.0|\n",
      "|    50%|              45.0|\n",
      "|    75%|              61.0|\n",
      "|    max|              82.0|\n",
      "+-------+------------------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df = df.filter(df.age >= 2)\n",
    "df.select('age').summary().show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "da7b22b9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+------+---------------+\n",
      "|gender| smoking_status|\n",
      "+------+---------------+\n",
      "|  Male|formerly smoked|\n",
      "|Female|   never smoked|\n",
      "|  Male|   never smoked|\n",
      "|Female|         smokes|\n",
      "|Female|   never smoked|\n",
      "|  Male|formerly smoked|\n",
      "|  Male|   never smoked|\n",
      "|Female|   never smoked|\n",
      "|Female|        Unknown|\n",
      "|Female|        Unknown|\n",
      "|Female|   never smoked|\n",
      "|Female|         smokes|\n",
      "|Female|         smokes|\n",
      "|  Male|        Unknown|\n",
      "|Female|   never smoked|\n",
      "|Female|   never smoked|\n",
      "|  Male|         smokes|\n",
      "|  Male|         smokes|\n",
      "|Female|   never smoked|\n",
      "|  Male|        Unknown|\n",
      "+------+---------------+\n",
      "only showing top 20 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df.select('gender','smoking_status').show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "925a5ecd",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+------+\n",
      "|gender|\n",
      "+------+\n",
      "|Female|\n",
      "| Other|\n",
      "|  Male|\n",
      "+------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df.select('gender').distinct().show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "9dfe35a4",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+---------------+\n",
      "| smoking_status|\n",
      "+---------------+\n",
      "|         smokes|\n",
      "|        Unknown|\n",
      "|   never smoked|\n",
      "|formerly smoked|\n",
      "+---------------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df.select('smoking_status').distinct().show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "850bdfe1",
   "metadata": {
    "id": "HQFPKukRLzbB"
   },
   "source": [
    "### Change the string values of the columns into numerical values as follows:\n",
    "1. Gender column:\n",
    "  * Male = 1 \n",
    "  * Female = 0 \n",
    "  * Other = 0 \n",
    "2. Smoking Status column:\n",
    "  * never smoked = 0\n",
    "  * Unknown = 0.5\n",
    "  * formerly smoked = 0.75\n",
    "  * smokes = 1.0\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "1c2a3bd3",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+------+\n",
      "|gender|\n",
      "+------+\n",
      "|  Male|\n",
      "|Female|\n",
      "|  Male|\n",
      "|Female|\n",
      "|Female|\n",
      "+------+\n",
      "only showing top 5 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df.select('gender').show(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "355f2ef0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+------+\n",
      "|gender|\n",
      "+------+\n",
      "|     1|\n",
      "|     0|\n",
      "|     1|\n",
      "|     0|\n",
      "|     0|\n",
      "+------+\n",
      "only showing top 5 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df = df.withColumn(\"gender\",F.when(col('gender')=='Male',1).otherwise(0))\n",
    "df.select('gender').show(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "9423c5e0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+---------------+\n",
      "| smoking_status|\n",
      "+---------------+\n",
      "|formerly smoked|\n",
      "|   never smoked|\n",
      "|   never smoked|\n",
      "|         smokes|\n",
      "|   never smoked|\n",
      "+---------------+\n",
      "only showing top 5 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df.select('smoking_status').show(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "45a991c3",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------------+\n",
      "|smoking_status|\n",
      "+--------------+\n",
      "|          0.75|\n",
      "|           0.0|\n",
      "|           0.0|\n",
      "|           1.0|\n",
      "|           0.0|\n",
      "+--------------+\n",
      "only showing top 5 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df = df.withColumn(\"smoking_status\",F.when(col('smoking_status')=='never smoked',0)\n",
    "                   .when(col('smoking_status')=='Unknown',0.5)\n",
    "                   .when(col('smoking_status')=='formerly smoked',0.75)\n",
    "                   .otherwise(1))\n",
    "df.select('smoking_status').show(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "7df0cf29",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+------+--------------+\n",
      "|gender|smoking_status|\n",
      "+------+--------------+\n",
      "|     1|          0.75|\n",
      "|     0|           0.0|\n",
      "|     1|           0.0|\n",
      "|     0|           1.0|\n",
      "|     0|           0.0|\n",
      "|     1|          0.75|\n",
      "|     1|           0.0|\n",
      "|     0|           0.0|\n",
      "|     0|           0.5|\n",
      "|     0|           0.5|\n",
      "|     0|           0.0|\n",
      "|     0|           1.0|\n",
      "|     0|           1.0|\n",
      "|     1|           0.5|\n",
      "|     0|           0.0|\n",
      "|     0|           0.0|\n",
      "|     1|           1.0|\n",
      "|     1|           1.0|\n",
      "|     0|           0.0|\n",
      "|     1|           0.5|\n",
      "+------+--------------+\n",
      "only showing top 20 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df.select('gender','smoking_status').show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "89ce3b77",
   "metadata": {},
   "source": [
    "### Deal with null values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "d5854671",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------+------------------+\n",
      "|summary|               bmi|\n",
      "+-------+------------------+\n",
      "|  count|              4795|\n",
      "|   mean|29.140855057351413|\n",
      "| stddev| 7.765248721877016|\n",
      "|    min|              11.3|\n",
      "|    25%|              23.9|\n",
      "|    50%|              28.3|\n",
      "|    75%|              33.2|\n",
      "|    max|              97.6|\n",
      "+-------+------------------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df.select(\"bmi\").summary().show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "4d78cb87",
   "metadata": {},
   "outputs": [],
   "source": [
    "bmi_50 = float(df.select(\"bmi\").summary().collect()[5][1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "c14d1ca1",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df.fillna(bmi_50, subset=['bmi'])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "053c429a",
   "metadata": {},
   "source": [
    "### Split the data into training and test dataframes:\n",
    "- 80% training and 20% test.\n",
    "- seed = 42.\n",
    "- Save each dataframe as a parquet file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "0236ea8b",
   "metadata": {},
   "outputs": [],
   "source": [
    "trainDF,testDF = df.randomSplit([0.8,0.2],seed=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "710eadb4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4037"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trainDF.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "0ac0e022",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "953"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "testDF.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "cf045944",
   "metadata": {},
   "outputs": [],
   "source": [
    "trainDF.write.parquet(\"trainDF\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "ec3f29ae",
   "metadata": {},
   "outputs": [],
   "source": [
    "testDF.write.parquet(\"testDF\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "78fa24da",
   "metadata": {},
   "source": [
    "### Read the saved Train and Test DataFrame:\n",
    "- Use the dataframes you read in the subsequent steps."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "c3f5c1e3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4037"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trainDF1 = spark.read.parquet(\"trainDF\")\n",
    "trainDF1.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "77ae6495",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "953"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "testDF1 = spark.read.parquet(\"testDF\")\n",
    "testDF1.count()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "987b4b40",
   "metadata": {},
   "source": [
    "### Create the model:\n",
    "- Perform feature engineering steps.\n",
    "- Create the logistic regression classifier.\n",
    "- Build the pipeline model that uses all feature engineering steps and the model.\n",
    "- Train the pipeline model using the trainig dataset."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "353c5d9a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['hypertension',\n",
       " 'heart_disease',\n",
       " 'ever_married',\n",
       " 'work_type',\n",
       " 'Residence_type']"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "categoricalCols = ['hypertension','heart_disease',\n",
    "                   'ever_married', 'work_type', 'Residence_type']\n",
    "categoricalCols"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "3ec0842c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['gender', 'age', 'avg_glucose_level', 'bmi', 'smoking_status']"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "numericCols = ['gender','age','avg_glucose_level',\n",
    "               'bmi','smoking_status']\n",
    "numericCols"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "3518527f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['hypertension_Index',\n",
       " 'heart_disease_Index',\n",
       " 'ever_married_Index',\n",
       " 'work_type_Index',\n",
       " 'Residence_type_Index']"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "indexOutputCols = [x + \"_Index\" for x in categoricalCols]\n",
    "indexOutputCols"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "02dc06d0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['hypertension_OHE',\n",
       " 'heart_disease_OHE',\n",
       " 'ever_married_OHE',\n",
       " 'work_type_OHE',\n",
       " 'Residence_type_OHE']"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "oheOutputCols = [x + \"_OHE\" for x in categoricalCols]\n",
    "oheOutputCols"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "9c9e4dc2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['hypertension_OHE',\n",
       " 'heart_disease_OHE',\n",
       " 'ever_married_OHE',\n",
       " 'work_type_OHE',\n",
       " 'Residence_type_OHE',\n",
       " 'gender',\n",
       " 'age',\n",
       " 'avg_glucose_level',\n",
       " 'bmi',\n",
       " 'smoking_status']"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "assemblerInputs = oheOutputCols + numericCols\n",
    "assemblerInputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "4e41b352",
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyspark.ml.feature import StringIndexer, VectorAssembler,OneHotEncoder\n",
    "from pyspark.ml import Pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "106453d7",
   "metadata": {},
   "outputs": [],
   "source": [
    "stringIndexer = StringIndexer(inputCols=categoricalCols,\n",
    "                             outputCols=indexOutputCols,\n",
    "                             handleInvalid='skip')\n",
    "oheEncoder = OneHotEncoder(inputCols=indexOutputCols,\n",
    "                          outputCols=oheOutputCols)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "05956f65",
   "metadata": {},
   "outputs": [],
   "source": [
    "vecAssembler = VectorAssembler(inputCols=assemblerInputs,outputCol='features')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "1529c24b",
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyspark.ml.classification import LogisticRegression\n",
    "lr = LogisticRegression(featuresCol = 'features', labelCol = 'stroke')\n",
    "pipeline = Pipeline(stages=[stringIndexer,oheEncoder,vecAssembler,lr])\n",
    "\n",
    "pipelineModel = pipeline.fit(trainDF1)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a05c2677",
   "metadata": {},
   "source": [
    "### Perform predictions on Test DataFrame"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "38b8524f",
   "metadata": {},
   "outputs": [],
   "source": [
    "predictions = pipelineModel.transform(testDF1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "95d8b779",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-----------------------------------------------------------+\n",
      "|features                                                   |\n",
      "+-----------------------------------------------------------+\n",
      "|[1.0,1.0,1.0,1.0,0.0,0.0,0.0,1.0,1.0,55.0,89.17,31.5,0.0]  |\n",
      "|(13,[0,1,3,7,9,10,11],[1.0,1.0,1.0,1.0,24.0,97.55,26.2])   |\n",
      "|(13,[0,1,2,3,9,10,11],[1.0,1.0,1.0,1.0,33.0,86.97,42.2])   |\n",
      "|[1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,43.0,88.23,37.6,0.5]  |\n",
      "|[1.0,1.0,1.0,1.0,0.0,0.0,0.0,1.0,1.0,79.0,198.79,24.9,0.0] |\n",
      "|(13,[0,1,2,3,9,10,11],[1.0,1.0,1.0,1.0,43.0,110.32,28.4])  |\n",
      "|(13,[1,2,4,8,9,10,11],[1.0,1.0,1.0,1.0,78.0,75.19,27.6])   |\n",
      "|[1.0,1.0,1.0,0.0,1.0,0.0,0.0,1.0,0.0,74.0,74.96,26.6,0.0]  |\n",
      "|(13,[0,1,6,9,10,11,12],[1.0,1.0,1.0,14.0,92.22,22.8,0.5])  |\n",
      "|[1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,1.0,69.0,101.52,26.8,1.0] |\n",
      "|[1.0,1.0,1.0,0.0,0.0,1.0,0.0,0.0,1.0,52.0,87.26,40.1,1.0]  |\n",
      "|(13,[3,9,10,11,12],[1.0,82.0,84.03,26.5,0.75])             |\n",
      "|(13,[0,1,3,8,9,10,11],[1.0,1.0,1.0,1.0,17.0,81.77,44.7])   |\n",
      "|[1.0,1.0,1.0,0.0,1.0,0.0,0.0,0.0,0.0,73.0,79.69,28.3,0.75] |\n",
      "|(13,[1,2,3,9,10,11],[1.0,1.0,1.0,59.0,99.06,23.4])         |\n",
      "|[1.0,1.0,1.0,0.0,0.0,1.0,0.0,1.0,0.0,60.0,80.86,31.0,1.0]  |\n",
      "|(13,[0,1,3,7,9,10,11],[1.0,1.0,1.0,1.0,14.0,83.42,28.7])   |\n",
      "|[0.0,1.0,1.0,1.0,0.0,0.0,0.0,1.0,1.0,69.0,229.21,30.1,1.0] |\n",
      "|[1.0,1.0,1.0,0.0,0.0,1.0,0.0,1.0,1.0,70.0,202.55,28.3,0.75]|\n",
      "|(13,[0,1,3,7,9,10,11],[1.0,1.0,1.0,1.0,29.0,71.89,27.6])   |\n",
      "+-----------------------------------------------------------+\n",
      "only showing top 20 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "predictions.select(\"features\").show(truncate=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "7e71ec92",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "root\n",
      " |-- id: integer (nullable = true)\n",
      " |-- gender: integer (nullable = true)\n",
      " |-- age: double (nullable = true)\n",
      " |-- hypertension: integer (nullable = true)\n",
      " |-- heart_disease: integer (nullable = true)\n",
      " |-- ever_married: string (nullable = true)\n",
      " |-- work_type: string (nullable = true)\n",
      " |-- Residence_type: string (nullable = true)\n",
      " |-- avg_glucose_level: double (nullable = true)\n",
      " |-- bmi: double (nullable = true)\n",
      " |-- smoking_status: double (nullable = true)\n",
      " |-- stroke: integer (nullable = true)\n",
      " |-- ever_married_Index: double (nullable = false)\n",
      " |-- heart_disease_Index: double (nullable = false)\n",
      " |-- Residence_type_Index: double (nullable = false)\n",
      " |-- work_type_Index: double (nullable = false)\n",
      " |-- hypertension_Index: double (nullable = false)\n",
      " |-- ever_married_OHE: vector (nullable = true)\n",
      " |-- heart_disease_OHE: vector (nullable = true)\n",
      " |-- Residence_type_OHE: vector (nullable = true)\n",
      " |-- hypertension_OHE: vector (nullable = true)\n",
      " |-- work_type_OHE: vector (nullable = true)\n",
      " |-- features: vector (nullable = true)\n",
      " |-- rawPrediction: vector (nullable = true)\n",
      " |-- probability: vector (nullable = true)\n",
      " |-- prediction: double (nullable = false)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "predictions.printSchema()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "7a32b742",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+------+----------+\n",
      "|stroke|prediction|\n",
      "+------+----------+\n",
      "|     0|       0.0|\n",
      "|     0|       0.0|\n",
      "|     0|       0.0|\n",
      "|     0|       0.0|\n",
      "|     0|       0.0|\n",
      "|     0|       0.0|\n",
      "|     0|       0.0|\n",
      "|     1|       0.0|\n",
      "|     0|       0.0|\n",
      "|     0|       0.0|\n",
      "|     0|       0.0|\n",
      "|     1|       0.0|\n",
      "|     0|       0.0|\n",
      "|     0|       0.0|\n",
      "|     0|       0.0|\n",
      "|     0|       0.0|\n",
      "|     0|       0.0|\n",
      "|     0|       0.0|\n",
      "|     0|       0.0|\n",
      "|     0|       0.0|\n",
      "+------+----------+\n",
      "only showing top 20 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "predictions.select(\"stroke\",\"prediction\").show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "fc303d6b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+------+----------+\n",
      "|stroke|prediction|\n",
      "+------+----------+\n",
      "|     1|       0.0|\n",
      "|     1|       0.0|\n",
      "|     1|       0.0|\n",
      "|     1|       0.0|\n",
      "|     1|       0.0|\n",
      "|     1|       0.0|\n",
      "|     1|       0.0|\n",
      "|     1|       0.0|\n",
      "|     1|       0.0|\n",
      "|     1|       0.0|\n",
      "|     1|       0.0|\n",
      "|     1|       0.0|\n",
      "|     1|       0.0|\n",
      "|     1|       0.0|\n",
      "|     1|       0.0|\n",
      "|     1|       0.0|\n",
      "|     1|       0.0|\n",
      "|     1|       0.0|\n",
      "|     1|       0.0|\n",
      "|     1|       0.0|\n",
      "+------+----------+\n",
      "only showing top 20 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "predictions.select(\"stroke\",\"prediction\").filter(col(\"stroke\")==1).show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "3ea267eb",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "predictions.select(\"stroke\",\"prediction\").filter(col(\"prediction\")==1).count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "480d892e",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "953"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "predictions.select(\"stroke\",\"prediction\").filter(col(\"prediction\")==0).count()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "daec120f",
   "metadata": {},
   "source": [
    "### Evaluate Model Performance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "a7b5ad25",
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyspark.ml.evaluation import BinaryClassificationEvaluator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "44d3a1e6",
   "metadata": {},
   "outputs": [],
   "source": [
    "evaluatorROC = BinaryClassificationEvaluator(rawPredictionCol='prediction'\n",
    "                                             ,labelCol='stroke')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "3adbe41f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.5"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "evaluatorROC.evaluate(predictions)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "40206589",
   "metadata": {},
   "source": [
    "### Improve Model Performance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "6c323fd5",
   "metadata": {},
   "outputs": [],
   "source": [
    "total_count = trainDF1.count()\n",
    "stroke_count = trainDF1.filter(col('stroke')==1).count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "be76262d",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_weight(x):\n",
    "    if x==1:\n",
    "        return 1-(stroke_count*1.0/total_count)\n",
    "    else:\n",
    "        return stroke_count*1.0/total_count"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "0ce8ab43",
   "metadata": {},
   "outputs": [],
   "source": [
    "get_weightUDF = udf(lambda z:get_weight(z))   \n",
    "trainDF1 = trainDF1.withColumn(\"weightCol\", get_weightUDF(F.col(\"stroke\")))\n",
    "trainDF1 = trainDF1.withColumn(\"weightCol\",col(\"weightCol\").cast(DoubleType()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "id": "4fdb5f45",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+------+-------------------+\n",
      "|stroke|          weightCol|\n",
      "+------+-------------------+\n",
      "|     0|0.05003715630418628|\n",
      "|     0|0.05003715630418628|\n",
      "|     0|0.05003715630418628|\n",
      "|     0|0.05003715630418628|\n",
      "|     0|0.05003715630418628|\n",
      "|     0|0.05003715630418628|\n",
      "|     0|0.05003715630418628|\n",
      "|     0|0.05003715630418628|\n",
      "|     0|0.05003715630418628|\n",
      "|     0|0.05003715630418628|\n",
      "|     1| 0.9499628436958137|\n",
      "|     0|0.05003715630418628|\n",
      "|     0|0.05003715630418628|\n",
      "|     0|0.05003715630418628|\n",
      "|     0|0.05003715630418628|\n",
      "|     0|0.05003715630418628|\n",
      "|     0|0.05003715630418628|\n",
      "|     0|0.05003715630418628|\n",
      "|     0|0.05003715630418628|\n",
      "|     0|0.05003715630418628|\n",
      "+------+-------------------+\n",
      "only showing top 20 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "trainDF1.select(\"stroke\",\"weightCol\").show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "00e034e9",
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyspark.ml.classification import LogisticRegression\n",
    "lr = LogisticRegression(featuresCol = 'features', labelCol = 'stroke',weightCol='weightCol')\n",
    "pipeline = Pipeline(stages=[stringIndexer,oheEncoder,vecAssembler,lr])\n",
    "\n",
    "pipelineModel = pipeline.fit(trainDF1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "id": "66efa0c6",
   "metadata": {},
   "outputs": [],
   "source": [
    "predictions_w = pipelineModel.transform(testDF1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "id": "b3cfb670",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7757777671252577"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "evaluatorROC.evaluate(predictions_w)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}