Access_s3_data_in_Spark_outside_AWS_Account.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Access s3 data in Spark outside AWS Account.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "authorship_tag": "ABX9TyNW/xSE6ZskIr+jxyhmn2Nb",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/SatadruMukherjee/Data-Preprocessing-Models/blob/main/Access_s3_data_in_Spark_outside_AWS_Account.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "2Q2Sk2W9QAxY"
      },
      "outputs": [],
      "source": [
        "!apt-get update\n",
        "!apt-get install openjdk-8-jdk-headless -qq > /dev/null\n",
        "!wget -q htt  https://archive.apache.org/dist/spark/spark-2.3.1/spark-2.3.1-bin-hadoop2.7.tgz\n",
        "!tar xf spark-2.3.1-bin-hadoop2.7.tgz\n",
        "!pip install -q findspark\n",
        "\n",
        "import os\n",
        "os.environ[\"JAVA_HOME\"] = \"/usr/lib/jvm/java-8-openjdk-amd64\"\n",
        "os.environ[\"SPARK_HOME\"] = \"/content/spark-2.3.1-bin-hadoop2.7\"\n",
        "\n",
        "import findspark\n",
        "findspark.init()\n",
        "from pyspark.sql import SparkSession"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# **If you are using PySpark to access S3 buckets, you must pass the Spark engine the right packages to use, specifically aws-java-sdk and hadoop-aws.**"
      ],
      "metadata": {
        "id": "RhXgp-U2Sa_Z"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "fs.s3a.impl--The implementation class of the S3A Filesystem\n",
        "\n",
        "fs.s3a.access.key--AWS access key ID used by S3A file system\n",
        "\n",
        "fs.s3a.secret.key--AWS secret key used by S3A file system\n"
      ],
      "metadata": {
        "id": "v3o3jdf6S3M2"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "access_key=''\n",
        "secret_key=''"
      ],
      "metadata": {
        "id": "mRQ0rN5aoMVd"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "spark = SparkSession \\\n",
        ".builder \\\n",
        ".config(\"spark.jars.packages\", \"org.apache.hadoop:hadoop-aws:2.7.3,com.amazonaws:aws-java-sdk:1.7.4\") \\\n",
        ".config(\"fs.s3a.impl\",\"org.apache.hadoop.fs.s3a.S3AFileSystem\") \\\n",
        ".config(\"fs.s3a.access.key\", access_key) \\\n",
        ".config(\"fs.s3a.secret.key\", secret_key) \\\n",
        ".getOrCreate()"
      ],
      "metadata": {
        "id": "-uTNWYTvQG0I"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "sdf = spark.read.option(\"header\", \"true\").option(\"inferschema\", \"true\").csv(\"{}\")"
      ],
      "metadata": {
        "id": "7hUl9rZhQeC4"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "sdf.show()"
      ],
      "metadata": {
        "id": "XkM0kSkJQ4Fh"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "sdf.printSchema()"
      ],
      "metadata": {
        "id": "E08iqiCpvlWO"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df1=sdf.groupBy('CLASS_NAME').count()"
      ],
      "metadata": {
        "id": "P4XBgByGvlQR"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df1.show()"
      ],
      "metadata": {
        "id": "WDiJaEf7vvdV"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df1.coalesce(1).write.format('csv').option(\"header\", \"true\").save('{}')"
      ],
      "metadata": {
        "id": "jD2ua2nvvza5"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# **in EMR**"
      ],
      "metadata": {
        "id": "xdH7aAm1sdHc"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "spark-submit --packages com.amazonaws:aws-java-sdk:1.7.4,org.apache.hadoop:hadoop-aws:2.7.3"
      ],
      "metadata": {
        "id": "l5hmc1lysjUb"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "access_key=''\n",
        "secret_key=''"
      ],
      "metadata": {
        "id": "9r1_VBL6xCRN"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "spark = SparkSession \\\n",
        ".builder \\\n",
        ".config(\"fs.s3a.impl\",\"org.apache.hadoop.fs.s3a.S3AFileSystem\") \\\n",
        ".config(\"fs.s3a.access.key\", access_key) \\\n",
        ".config(\"fs.s3a.secret.key\", secret_key) \\\n",
        ".getOrCreate()"
      ],
      "metadata": {
        "id": "sGm63hv8saSJ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "sdf = spark.read.option(\"inferschema\", \"true\").option(\"header\", \"true\").csv(\"{}\")"
      ],
      "metadata": {
        "id": "ZB0ArLEjw3To"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "sdf.coalesce(1).write.format('csv').option(\"header\", \"true\").save('{}')"
      ],
      "metadata": {
        "id": "-aiHzJ4csei8"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}