Outliers_&_Skewness_Handling.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Outliers & Skewness Handling.ipynb",
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "MC2q4G2wP9Jp",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 500
        },
        "outputId": "5858ebf5-94ef-4e19-c535-d25bb0db0ab5"
      },
      "source": [
        "!apt-get update\r\n",
        "!apt-get install openjdk-8-jdk-headless -qq > /dev/null\r\n",
        "!wget -q htt  https://archive.apache.org/dist/spark/spark-2.3.1/spark-2.3.1-bin-hadoop2.7.tgz\r\n",
        "!tar xf spark-2.3.1-bin-hadoop2.7.tgz\r\n",
        "!pip install -q findspark\r\n",
        "\r\n",
        "import os\r\n",
        "os.environ[\"JAVA_HOME\"] = \"/usr/lib/jvm/java-8-openjdk-amd64\"\r\n",
        "os.environ[\"SPARK_HOME\"] = \"/content/spark-2.3.1-bin-hadoop2.7\"\r\n",
        "\r\n",
        "import findspark\r\n",
        "findspark.init()\r\n",
        "from pyspark.sql import SparkSession\r\n",
        "spark = SparkSession.builder.getOrCreate()\r\n",
        "spark"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\r0% [Working]\r            \rGet:1 http://ppa.launchpad.net/c2d4u.team/c2d4u4.0+/ubuntu bionic InRelease [15.9 kB]\n",
            "\r0% [Waiting for headers] [Connecting to security.ubuntu.com (91.189.91.39)] [Co\r                                                                               \rHit:2 http://archive.ubuntu.com/ubuntu bionic InRelease\n",
            "\r0% [Waiting for headers] [Connecting to security.ubuntu.com (91.189.91.39)] [Co\r                                                                               \rGet:3 http://archive.ubuntu.com/ubuntu bionic-updates InRelease [88.7 kB]\n",
            "\r                                                                               \rGet:4 https://cloud.r-project.org/bin/linux/ubuntu bionic-cran40/ InRelease [3,626 B]\n",
            "\r0% [3 InRelease 47.5 kB/88.7 kB 54%] [Connecting to security.ubuntu.com (91.189\r0% [2 InRelease gpgv 242 kB] [3 InRelease 47.5 kB/88.7 kB 54%] [Connecting to s\r                                                                               \rHit:5 http://ppa.launchpad.net/cran/libgit2/ubuntu bionic InRelease\n",
            "\r0% [2 InRelease gpgv 242 kB] [3 InRelease 47.5 kB/88.7 kB 54%] [Connecting to s\r0% [2 InRelease gpgv 242 kB] [Waiting for headers] [Connecting to security.ubun\r                                                                               \rGet:6 http://archive.ubuntu.com/ubuntu bionic-backports InRelease [74.6 kB]\n",
            "\r0% [2 InRelease gpgv 242 kB] [6 InRelease 8,396 B/74.6 kB 11%] [Waiting for hea\r                                                                               \rHit:7 http://ppa.launchpad.net/graphics-drivers/ppa/ubuntu bionic InRelease\n",
            "\r0% [2 InRelease gpgv 242 kB] [6 InRelease 17.1 kB/74.6 kB 23%] [Waiting for hea\r                                                                               \r0% [2 InRelease gpgv 242 kB] [Waiting for headers] [Waiting for headers]\r                                                                        \rGet:8 http://security.ubuntu.com/ubuntu bionic-security InRelease [88.7 kB]\n",
            "Ign:9 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64  InRelease\n",
            "Ign:10 https://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1804/x86_64  InRelease\n",
            "Hit:11 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64  Release\n",
            "Hit:12 https://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1804/x86_64  Release\n",
            "Get:13 https://cloud.r-project.org/bin/linux/ubuntu bionic-cran40/ Packages [41.5 kB]\n",
            "Get:14 http://ppa.launchpad.net/c2d4u.team/c2d4u4.0+/ubuntu bionic/main Sources [1,704 kB]\n",
            "Get:15 http://ppa.launchpad.net/c2d4u.team/c2d4u4.0+/ubuntu bionic/main amd64 Packages [873 kB]\n",
            "Fetched 2,890 kB in 2s (1,396 kB/s)\n",
            "Reading package lists... Done\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "\n",
              "            <div>\n",
              "                <p><b>SparkSession - in-memory</b></p>\n",
              "                \n",
              "        <div>\n",
              "            <p><b>SparkContext</b></p>\n",
              "\n",
              "            <p><a href=\"http://5f698fca66f7:4040\">Spark UI</a></p>\n",
              "\n",
              "            <dl>\n",
              "              <dt>Version</dt>\n",
              "                <dd><code>v2.3.1</code></dd>\n",
              "              <dt>Master</dt>\n",
              "                <dd><code>local[*]</code></dd>\n",
              "              <dt>AppName</dt>\n",
              "                <dd><code>pyspark-shell</code></dd>\n",
              "            </dl>\n",
              "        </div>\n",
              "        \n",
              "            </div>\n",
              "        "
            ],
            "text/plain": [
              "<pyspark.sql.session.SparkSession at 0x7ff5de1d6a20>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 1
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "c3arKlNZyp0d"
      },
      "source": [
        "import pandas as pd"
      ],
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "na3dsBE35N_r",
        "outputId": "b72c4d03-871f-4682-8f03-b5389cc2e21b"
      },
      "source": [
        "farm = pd.DataFrame()\r\n",
        "farm['Price'] = [632541, 425618, 356471, 7412512]\r\n",
        "farm['Rooms'] = [2, 5, 3, 100]\r\n",
        "farm['Square_Feet'] = [1600, 2850, 1780, 90000]\r\n",
        "print(farm)"
      ],
      "execution_count": 21,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "     Price  Rooms  Square_Feet\n",
            "0   632541      2         1600\n",
            "1   425618      5         2850\n",
            "2   356471      3         1780\n",
            "3  7412512    100        90000\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "i4Rgooq5ylUd",
        "outputId": "6cfa04ee-40b4-49f3-dedb-4c676a61809a"
      },
      "source": [
        "df=spark.createDataFrame(farm);\r\n",
        "df.show()"
      ],
      "execution_count": 22,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "+-------+-----+-----------+\n",
            "|  Price|Rooms|Square_Feet|\n",
            "+-------+-----+-----------+\n",
            "| 632541|    2|       1600|\n",
            "| 425618|    5|       2850|\n",
            "| 356471|    3|       1780|\n",
            "|7412512|  100|      90000|\n",
            "+-------+-----+-----------+\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DraZTS3_t59s"
      },
      "source": [
        "ms=df.columns;\r\n",
        "input_columns_Number_Type=[];\r\n",
        "input_columns_String_Type=[];\r\n",
        "for i in ms:\r\n",
        "  if str(df.schema[i].dataType)=='StringType':\r\n",
        "    input_columns_String_Type.append(i);\r\n",
        "  else:\r\n",
        "    input_columns_Number_Type.append(i);"
      ],
      "execution_count": 23,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "OYQGLfsLy1ZL",
        "outputId": "e8d4d23f-7e21-4e74-e533-255d75fdc469"
      },
      "source": [
        "input_columns_String_Type"
      ],
      "execution_count": 24,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 24
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "j0jEb86cy3lY",
        "outputId": "e862b0bb-15e3-4805-e611-f314b83bdd1f"
      },
      "source": [
        "input_columns_Number_Type"
      ],
      "execution_count": 25,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['Price', 'Rooms', 'Square_Feet']"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 25
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "n1T3uJGnvT1i"
      },
      "source": [
        "# **Computing percentiles for all Numeric Column**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "D5vVRjaouiDf"
      },
      "source": [
        "d={};\r\n",
        "for c in input_columns_Number_Type:\r\n",
        "  d[c]=df.approxQuantile(c,[0.01,0.99],0.25);"
      ],
      "execution_count": 26,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "uxR2DDx2y6Od",
        "outputId": "db842b05-8bbc-4655-8ebb-b8ba437fa3bb"
      },
      "source": [
        "d"
      ],
      "execution_count": 27,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'Price': [356471.0, 7412512.0],\n",
              " 'Rooms': [2.0, 100.0],\n",
              " 'Square_Feet': [1600.0, 90000.0]}"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 27
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "tjeHZXBCviPQ"
      },
      "source": [
        "# **Check skewness for all numeric columns**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DYo9tzoZNyJ2"
      },
      "source": [
        "from pyspark.sql.functions import *"
      ],
      "execution_count": 28,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "E6K4Qd0evmoH"
      },
      "source": [
        "for col in input_columns_Number_Type:\r\n",
        "  skew_value_for_the_column=df.select(skewness(c).alias(\"sk\")).collect()[0].asDict()['sk'];\r\n",
        "  if skew_value_for_the_column>1:\r\n",
        "    df=df.withColumn(col,log(when(df[col]<d[col][0],d[col][0]).when(df[col]>d[col][1],d[col][1]).otherwise(df[col])+1));\r\n",
        "  elif skew_value_for_the_column<-1:\r\n",
        "    df=df.withColumn(col,exp(when(df[col]<d[col][0],d[col][0]).when(df[col]>d[col][1],d[col][1]).otherwise(df[col])));"
      ],
      "execution_count": 31,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "FmPhkvlCy-yh",
        "outputId": "e65dd32a-431b-4580-acc9-7e6b6f74fab0"
      },
      "source": [
        "df.show()"
      ],
      "execution_count": 32,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "+------------------+------------------+------------------+\n",
            "|             Price|             Rooms|       Square_Feet|\n",
            "+------------------+------------------+------------------+\n",
            "|12.784010974377479|1.0986122886681098|7.3783837129967145|\n",
            "|12.784010974377479|1.0986122886681098|7.3783837129967145|\n",
            "|12.784010974377479|1.0986122886681098|7.3783837129967145|\n",
            "|12.784010974377479|1.7254630513334037|7.3783837129967145|\n",
            "+------------------+------------------+------------------+\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cvNZBB3M1-1B"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}