tutorials/profiles.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Profiles Basic Tutorial\n",
    "\n",
    "This Jupyter notebook will give a basic tutorial on how to use PyDeequ's Profiles module."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyspark.sql import SparkSession, Row, DataFrame\n",
    "import json\n",
    "import pandas as pd\n",
    "import sagemaker_pyspark\n",
    "\n",
    "import pydeequ\n",
    "\n",
    "classpath = \":\".join(sagemaker_pyspark.classpath_jars())\n",
    "\n",
    "spark = (SparkSession\n",
    "    .builder\n",
    "    .config(\"spark.driver.extraClassPath\", classpath)\n",
    "    .config(\"spark.jars.packages\", pydeequ.deequ_maven_coord)\n",
    "    .config(\"spark.jars.excludes\", pydeequ.f2j_maven_coord)\n",
    "    .getOrCreate())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### We will be using the Amazon Product Reviews dataset -- specifically the Electronics subset. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "root\n",
      " |-- marketplace: string (nullable = true)\n",
      " |-- customer_id: string (nullable = true)\n",
      " |-- review_id: string (nullable = true)\n",
      " |-- product_id: string (nullable = true)\n",
      " |-- product_parent: string (nullable = true)\n",
      " |-- product_title: string (nullable = true)\n",
      " |-- star_rating: integer (nullable = true)\n",
      " |-- helpful_votes: integer (nullable = true)\n",
      " |-- total_votes: integer (nullable = true)\n",
      " |-- vine: string (nullable = true)\n",
      " |-- verified_purchase: string (nullable = true)\n",
      " |-- review_headline: string (nullable = true)\n",
      " |-- review_body: string (nullable = true)\n",
      " |-- review_date: date (nullable = true)\n",
      " |-- year: integer (nullable = true)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df = spark.read.parquet(\"s3a://amazon-reviews-pds/parquet/product_category=Electronics/\")\n",
    "\n",
    "df.printSchema()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pydeequ.profiles import *\n",
    "\n",
    "result = ColumnProfilerRunner(spark) \\\n",
    "    .onData(df) \\\n",
    "    .run()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Column review_id:\n",
      "completeness: 1.0\n",
      "approx distinct: 3010972\n",
      "datatype: String\n",
      "\n",
      "Statistics of customer_id\n",
      "Min:10005.0\n",
      "Max:53096582.0\n",
      "Mean:28806032.68895954\n",
      "StandardDeviation:15415072.111267326\n",
      "\n",
      "Column review_date:\n",
      "completeness: 1.0\n",
      "approx distinct: 5898\n",
      "datatype: Unknown\n",
      "\n",
      "Statistics of helpful_votes\n",
      "Min:0.0\n",
      "Max:12786.0\n",
      "Mean:1.865194053838942\n",
      "StandardDeviation:21.296393520562624\n",
      "\n",
      "Statistics of star_rating\n",
      "Min:1.0\n",
      "Max:5.0\n",
      "Mean:4.036143941340712\n",
      "StandardDeviation:1.3866747032700206\n",
      "\n",
      "Statistics of year\n",
      "Min:1999.0\n",
      "Max:2015.0\n",
      "Mean:2012.8595236432125\n",
      "StandardDeviation:2.464162689284542\n",
      "\n",
      "Column product_title:\n",
      "completeness: 1.0\n",
      "approx distinct: 164112\n",
      "datatype: String\n",
      "\n",
      "Column review_headline:\n",
      "completeness: 0.9999987183340393\n",
      "approx distinct: 1694860\n",
      "datatype: String\n",
      "\n",
      "Column product_id:\n",
      "completeness: 1.0\n",
      "approx distinct: 169835\n",
      "datatype: String\n",
      "\n",
      "Statistics of total_votes\n",
      "Min:0.0\n",
      "Max:12944.0\n",
      "Mean:2.3798239503636407\n",
      "StandardDeviation:22.457108543167916\n",
      "\n",
      "Statistics of product_parent\n",
      "Min:6478.0\n",
      "Max:999998189.0\n",
      "Mean:510045457.18261784\n",
      "StandardDeviation:286789206.496994\n",
      "\n",
      "Column review_body:\n",
      "completeness: 0.9999724441818453\n",
      "approx distinct: 3024295\n",
      "datatype: String\n",
      "\n",
      "Column vine:\n",
      "completeness: 1.0\n",
      "approx distinct: 2\n",
      "datatype: String\n",
      "\n",
      "Column marketplace:\n",
      "completeness: 1.0\n",
      "approx distinct: 5\n",
      "datatype: String\n",
      "\n",
      "Column verified_purchase:\n",
      "completeness: 1.0\n",
      "approx distinct: 2\n",
      "datatype: String\n",
      "\n"
     ]
    }
   ],
   "source": [
    "for col, profile in result.profiles.items():\n",
    "    print(profile)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### For more info ... look at full list of profiles in `docs/profiles.md` "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "conda_python3",
   "language": "python",
   "name": "conda_python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}