Limit_and_offset_concept_in_API.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Limit and offset concept in API.ipynb",
      "provenance": [],
      "authorship_tag": "ABX9TyPrUB0oUz1Pw4dIAfHDKZZu",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/SatadruMukherjee/Data-Preprocessing-Models/blob/main/Limit_and_offset_concept_in_API.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# **Importing the required module**"
      ],
      "metadata": {
        "id": "25T2PoqR4C9j"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "OcVjbLjzSd3L"
      },
      "outputs": [],
      "source": [
        "import requests\n",
        "import uuid\n",
        "import json"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# **Defining the limit & initial offset**"
      ],
      "metadata": {
        "id": "z3wq3q8A4G8D"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "offset=0\n",
        "limit=1000"
      ],
      "metadata": {
        "id": "W2CocuMtSfsC"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# **Creating the folder where jsons for each iterations will be stored**"
      ],
      "metadata": {
        "id": "iYHhwQY_3vLP"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!mkdir /content/extracted_data"
      ],
      "metadata": {
        "id": "e6yUmztgSkn6"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# **Pagination Code**"
      ],
      "metadata": {
        "id": "dIzR4Bn333Ow"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "while True:\n",
        "  url=\"http://jsonplaceholder.typicode.com/photos?_start={}&_limit={}\".format(offset,limit)\n",
        "  print(\"URL for the current iteration : {}\".format(url))\n",
        "  result=requests.get(url);\n",
        "  data_extracted=result.json();\n",
        "  if(len(data_extracted)==0):\n",
        "    break;\n",
        "  else:\n",
        "    file_name=\"/content/extracted_data/\"+str(uuid.uuid4())+\".json\"\n",
        "    with open(file_name, 'w') as fp:\n",
        "      json.dump(data_extracted, fp)\n",
        "    offset=offset+limit"
      ],
      "metadata": {
        "id": "273wmNGDShOY"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# **Installing Spark**"
      ],
      "metadata": {
        "id": "5-tGQhQl36Bd"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!apt-get update\n",
        "!apt-get install openjdk-8-jdk-headless -qq > /dev/null\n",
        "!wget -q htt  https://archive.apache.org/dist/spark/spark-2.3.1/spark-2.3.1-bin-hadoop2.7.tgz\n",
        "!tar xf spark-2.3.1-bin-hadoop2.7.tgz\n",
        "!pip install -q findspark\n",
        "\n",
        "import os\n",
        "os.environ[\"JAVA_HOME\"] = \"/usr/lib/jvm/java-8-openjdk-amd64\"\n",
        "os.environ[\"SPARK_HOME\"] = \"/content/spark-2.3.1-bin-hadoop2.7\"\n",
        "\n",
        "import findspark\n",
        "findspark.init()\n",
        "from pyspark.sql import SparkSession"
      ],
      "metadata": {
        "id": "0X0Wr_dISmdy"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# **Build the sparkSession**"
      ],
      "metadata": {
        "id": "AS7qSOQk431X"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "spark=SparkSession.builder.appName(\"Hello\").getOrCreate()"
      ],
      "metadata": {
        "id": "nTrSQKQqSoPR"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from pyspark.sql.functions import *\n",
        "from pyspark.sql.types import *"
      ],
      "metadata": {
        "id": "uA73I4tISpwq"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# **Reading all the json files created in the earlier steps**"
      ],
      "metadata": {
        "id": "EvA5M2x74Tj4"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "df1 = spark.read.option(\"multiline\", True).option(\"inferSchema\", False).json(\"/content/extracted_data/\")"
      ],
      "metadata": {
        "id": "FIMLHu20SqJX"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df1.show()"
      ],
      "metadata": {
        "id": "Aid63Q4iSsEr"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# **Reading the csv file processed using https://jsonformatter.org/405ecb# from http://jsonplaceholder.typicode.com/photos**"
      ],
      "metadata": {
        "id": "LZx7SkCd4eQC"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "df2 = spark.read.format('csv').option(\"header\", True).option(\"inferSchema\", False).load(\"/content/items_processed_total.csv\")"
      ],
      "metadata": {
        "id": "9cg6sVRiXu2X"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df2.show()"
      ],
      "metadata": {
        "id": "mp_QdcRPcz_y"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df1.columns"
      ],
      "metadata": {
        "id": "uIjVIC7Sc1l6"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# **Make the column order same for both dataframes**"
      ],
      "metadata": {
        "id": "jH5Fr3r148nz"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "df3=df2.select(['albumId', 'id', 'thumbnailUrl', 'title', 'url'])"
      ],
      "metadata": {
        "id": "okmqrPF3c5Rs"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "(df1.subtract(df3)).count()"
      ],
      "metadata": {
        "id": "vZ5L2TRMc-so"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "(df3.subtract(df1)).count()"
      ],
      "metadata": {
        "id": "KiuCdjc6dJFX"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}