Automatic Mean Imputation.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Untitled3.ipynb",
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "MC2q4G2wP9Jp",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 448
        },
        "outputId": "f78795cb-cd8b-413e-ad7b-4f4027daf22c"
      },
      "source": [
        "!apt-get update\r\n",
        "!apt-get install openjdk-8-jdk-headless -qq > /dev/null\r\n",
        "!wget -q htt  https://archive.apache.org/dist/spark/spark-2.3.1/spark-2.3.1-bin-hadoop2.7.tgz\r\n",
        "!tar xf spark-2.3.1-bin-hadoop2.7.tgz\r\n",
        "!pip install -q findspark\r\n",
        "\r\n",
        "import os\r\n",
        "os.environ[\"JAVA_HOME\"] = \"/usr/lib/jvm/java-8-openjdk-amd64\"\r\n",
        "os.environ[\"SPARK_HOME\"] = \"/content/spark-2.3.1-bin-hadoop2.7\"\r\n",
        "\r\n",
        "import findspark\r\n",
        "findspark.init()\r\n",
        "from pyspark.sql import SparkSession\r\n",
        "spark = SparkSession.builder.getOrCreate()\r\n",
        "spark"
      ],
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\r0% [Working]\r            \rGet:1 http://security.ubuntu.com/ubuntu bionic-security InRelease [88.7 kB]\n",
            "\r0% [Connecting to archive.ubuntu.com (91.189.88.152)] [1 InRelease 14.2 kB/88.7\r0% [Waiting for headers] [Connected to cloud.r-project.org (65.8.180.16)] [Wait\r                                                                               \rIgn:2 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64  InRelease\n",
            "\r0% [Waiting for headers] [Connected to cloud.r-project.org (65.8.180.16)] [Wait\r0% [1 InRelease gpgv 88.7 kB] [Waiting for headers] [Connected to cloud.r-proje\r                                                                               \rIgn:3 https://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1804/x86_64  InRelease\n",
            "\r0% [1 InRelease gpgv 88.7 kB] [Waiting for headers] [Waiting for headers] [Wait\r                                                                               \rHit:4 http://ppa.launchpad.net/c2d4u.team/c2d4u4.0+/ubuntu bionic InRelease\n",
            "\r0% [1 InRelease gpgv 88.7 kB] [Waiting for headers] [Waiting for headers] [Conn\r                                                                               \rHit:5 http://archive.ubuntu.com/ubuntu bionic InRelease\n",
            "\r0% [1 InRelease gpgv 88.7 kB] [Waiting for headers] [Waiting for headers] [Conn\r                                                                               \rHit:6 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64  Release\n",
            "Get:7 https://cloud.r-project.org/bin/linux/ubuntu bionic-cran40/ InRelease [3,626 B]\n",
            "Hit:8 https://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1804/x86_64  Release\n",
            "Get:9 http://archive.ubuntu.com/ubuntu bionic-updates InRelease [88.7 kB]\n",
            "Hit:10 http://ppa.launchpad.net/cran/libgit2/ubuntu bionic InRelease\n",
            "Hit:11 http://ppa.launchpad.net/graphics-drivers/ppa/ubuntu bionic InRelease\n",
            "Get:12 http://archive.ubuntu.com/ubuntu bionic-backports InRelease [74.6 kB]\n",
            "Fetched 256 kB in 2s (128 kB/s)\n",
            "Reading package lists... Done\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "\n",
              "            <div>\n",
              "                <p><b>SparkSession - in-memory</b></p>\n",
              "                \n",
              "        <div>\n",
              "            <p><b>SparkContext</b></p>\n",
              "\n",
              "            <p><a href=\"http://acaf6fed03c0:4040\">Spark UI</a></p>\n",
              "\n",
              "            <dl>\n",
              "              <dt>Version</dt>\n",
              "                <dd><code>v2.3.1</code></dd>\n",
              "              <dt>Master</dt>\n",
              "                <dd><code>local[*]</code></dd>\n",
              "              <dt>AppName</dt>\n",
              "                <dd><code>pyspark-shell</code></dd>\n",
              "            </dl>\n",
              "        </div>\n",
              "        \n",
              "            </div>\n",
              "        "
            ],
            "text/plain": [
              "<pyspark.sql.session.SparkSession at 0x7f4e087ada20>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 15
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "na3dsBE35N_r",
        "outputId": "152f52da-e084-4fd2-8ee5-4657d915b422"
      },
      "source": [
        "df=spark.read.format(\"csv\").option(\"header\",\"true\").option(\"inferschema\",\"true\").load(\"/content/mgo.csv\");\r\n",
        "df.show()"
      ],
      "execution_count": 55,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "+-------+----+------+---------+\n",
            "|Country| Age|Salary|Purchased|\n",
            "+-------+----+------+---------+\n",
            "| France|  44| 72000|       No|\n",
            "|  Spain|  27| 48000|      Yes|\n",
            "|Germany|  30| 54000|       No|\n",
            "|  Spain|  38| 61000|       No|\n",
            "|Germany|  40|  null|      Yes|\n",
            "| France|  35| 58000|      Yes|\n",
            "|  Spain|null| 52000|       No|\n",
            "| France|  48| 79000|      Yes|\n",
            "|Germany|  50| 83000|       No|\n",
            "| France|  37| 67000|      Yes|\n",
            "+-------+----+------+---------+\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "GBv27qX051rn",
        "outputId": "c13667b2-1c52-494f-ce53-97abe0e4196c"
      },
      "source": [
        "df.printSchema()"
      ],
      "execution_count": 56,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "root\n",
            " |-- Country: string (nullable = true)\n",
            " |-- Age: integer (nullable = true)\n",
            " |-- Salary: integer (nullable = true)\n",
            " |-- Purchased: string (nullable = true)\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "n1lAT0SY6xm_",
        "outputId": "2d53a23d-d8a2-4b29-ddfe-28bdf00c567d"
      },
      "source": [
        "(df.schema['Country'].dataType)"
      ],
      "execution_count": 68,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "StringType"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 68
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XH1P3Ns2Ym22"
      },
      "source": [
        "ms=df.columns;\r\n",
        "gs=[];\r\n",
        "for i in ms:\r\n",
        "  gs.append(df.filter(df[i].isNull()).count());"
      ],
      "execution_count": 58,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Kh9ZkI56ZOzj",
        "outputId": "97214077-039b-4c70-8e31-d89484c0ddad"
      },
      "source": [
        "gs"
      ],
      "execution_count": 59,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[0, 1, 1, 0]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 59
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TcZvKLWs8BnX"
      },
      "source": [
        "from pyspark.sql.functions import *"
      ],
      "execution_count": 60,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8r_R3oIu69DA"
      },
      "source": [
        "ms=df.columns;\r\n",
        "input_columns=[];\r\n",
        "for i in ms:\r\n",
        "  if str(df.schema[i].dataType) in ('IntegerType','DoubleType'):\r\n",
        "    input_columns.append(i);"
      ],
      "execution_count": 61,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "xGKDTK9v8r0L",
        "outputId": "692cb162-5b9e-48a4-d2ce-3003620b418d"
      },
      "source": [
        "input_columns"
      ],
      "execution_count": 62,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['Age', 'Salary']"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 62
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "b58QcsI58tiV"
      },
      "source": [
        "def fill_using_mean(dfa,cola):\r\n",
        "  for c in dfa.columns:\r\n",
        "    if c in cola:\r\n",
        "      ms=dfa.select(mean(c).alias(\"Ammao\")).collect()[0].asDict()['Ammao'];\r\n",
        "      dfa=dfa.na.fill({c:ms});\r\n",
        "  return dfa;"
      ],
      "execution_count": 63,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "B9wbpFmsWYYE",
        "outputId": "fc278c3d-5a7a-45a8-ab5c-c9648aa3882c"
      },
      "source": [
        "dfa=fill_using_mean(df,input_columns)"
      ],
      "execution_count": 64,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "38.77777777777778\n",
            "63777.77777777778\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "3mFULBL-bbfs",
        "outputId": "7c2ddfa1-5a2b-40e5-d015-a475d1d284b6"
      },
      "source": [
        "dfa.show()"
      ],
      "execution_count": 65,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "+-------+---+------+---------+\n",
            "|Country|Age|Salary|Purchased|\n",
            "+-------+---+------+---------+\n",
            "| France| 44| 72000|       No|\n",
            "|  Spain| 27| 48000|      Yes|\n",
            "|Germany| 30| 54000|       No|\n",
            "|  Spain| 38| 61000|       No|\n",
            "|Germany| 40| 63777|      Yes|\n",
            "| France| 35| 58000|      Yes|\n",
            "|  Spain| 38| 52000|       No|\n",
            "| France| 48| 79000|      Yes|\n",
            "|Germany| 50| 83000|       No|\n",
            "| France| 37| 67000|      Yes|\n",
            "+-------+---+------+---------+\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6gcYYjhgdMN_"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}