scd_type2.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "!conda install --yes --prefix {sys.prefix} boto3\n",
    "!{sys.executable} -m pip install git+git://github.com/moj-analytical-services/etl_manager.git#egg=etl_manager"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Remove metastore and data used in this example (Want to start a blank slate)\n",
    "import os\n",
    "import shutil\n",
    "\n",
    "if os.path.isdir('metastore_db') :\n",
    "    shutil.rmtree('metastore_db')\n",
    "    \n",
    "if os.path.isdir('data/scd2') :\n",
    "    shutil.rmtree('data/scd2')\n",
    "    \n",
    "if os.path.isdir('data/scd2_update') :\n",
    "    shutil.rmtree('data/scd2_update')\n",
    "\n",
    "if os.path.isfile('derby.log') :\n",
    "    os.remove('derby.log')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'/home/jovyan/Documents/projects/spark_testing'"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import boto3\n",
    "\n",
    "# from dataengineeringutils.utils import read_json\n",
    "\n",
    "import pyspark\n",
    "from pyspark import SparkContext, SparkConf\n",
    "from pyspark.sql import SparkSession, functions as F\n",
    "from pyspark.storagelevel import StorageLevel\n",
    "\n",
    "# Import own function library\n",
    "from pyspark.sql.types import *\n",
    "\n",
    "os.environ['PYSPARK_SUBMIT_ARGS'] = '--packages com.amazonaws:aws-java-sdk:1.10.34,org.apache.hadoop:hadoop-aws:2.6.0 pyspark-shell'\n",
    "\n",
    "session = boto3.Session()\n",
    "credentials = session.get_credentials()\n",
    "\n",
    "\n",
    "spark = SparkSession \\\n",
    "    .builder \\\n",
    "    .master(\"local[*]\") \\\n",
    "    .config(\"hive.exec.dynamic.partition\", \"true\") \\\n",
    "    .config(\"hive.exec.dynamic.partition.mode\", \"nonstrict\") \\\n",
    "    .config(\"spark.hadoop.fs.s3.impl\", \"org.apache.hadoop.fs.s3native.NativeS3FileSystem\") \\\n",
    "    .config(\"spark.hadoop.fs.s3.awsAccessKeyId\", credentials.access_key) \\\n",
    "    .config(\"spark.hadoop.fs.s3.awsSecretAccessKey\", credentials.secret_key) \\\n",
    "    .enableHiveSupport() \\\n",
    "    .getOrCreate()\n",
    "    \n",
    "os.getcwd()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Example: \n",
    "\n",
    "Want to create a dataset that has SCD2 using spark. Creating a dataset where each record has a start and end date making it easy to query the data based on a date :\n",
    "\n",
    "```sql\n",
    "SELECT * FROM database.table where start_date >= 2018-01-01 AND end_date < 2018-01-01\n",
    "```\n",
    "In my example below we have 3 days worth of data. The data itself has rows that update over time rather than a row being created. We will ingest new data each day and recalculate the start and end date for each record in a partition.\n",
    "\n",
    "Our dummy in data has an id (our primary key) and a dummy_contents. The dummy_content will always be a when the row is new and then iterates through the alphabet each time it is updated.\n",
    "\n",
    "_Note: This is just to keep track of how many times each record has changed. This code can be condenced quite a lot by changing some of the SQL code with sparks df API. However SQL looks better to be more specific when writing dataframes._"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Test new update method\n",
    "df_partitions = ['partition_bin']\n",
    "in_schema = StructType([StructField('id', IntegerType(), True), StructField('dummy_contents', StringType())])\n",
    "\n",
    "# Day one we get our first set of data\n",
    "delta_day_1 = [\n",
    "    (0, 'a'),\n",
    "    (1, 'a'),\n",
    "    (2, 'a'),\n",
    "    (3, 'a'),\n",
    "    (4, 'a'),\n",
    "    (5, 'a'),\n",
    "    (6, 'a'),\n",
    "    (7, 'a'),\n",
    "    (8, 'a'),\n",
    "    (9, 'a')]\n",
    "\n",
    "# Data 2 we mostly get new data with some old data and updated records\n",
    "delta_day_2 = [\n",
    "    (7, 'b'),\n",
    "    (8, 'b'),\n",
    "    (9, 'b'),\n",
    "    (10, 'a'),\n",
    "    (11, 'a'),\n",
    "    (12, 'a'),\n",
    "    (13, 'a')]\n",
    "\n",
    "# Day 3 we get loads of updated old data \n",
    "delta_day_3 = [\n",
    "    (1, 'b'),\n",
    "    (2, 'b'),\n",
    "    (3, 'b'),\n",
    "    (4, 'b'),\n",
    "    (5, 'b'),\n",
    "    (6, 'c'),\n",
    "    (7, 'c'),\n",
    "    (8, 'c'),\n",
    "    (9, 'c'),\n",
    "    (10, 'b'),\n",
    "    (11, 'b'),\n",
    "    (12, 'b'),\n",
    "    (13, 'b'),\n",
    "    (14, 'a'),\n",
    "    (15, 'a')\n",
    "]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Day 1 (2018-01-01)\n",
    "Dataset is created and day1 is written into dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DataFrame[]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "spark.createDataFrame(delta_day_1, in_schema).createOrReplaceTempView('df_1')\n",
    "\n",
    "# Create a view of our df_1 data with added start_date, end_date and partition_bin columns\n",
    "spark.sql(\"\"\"\n",
    "CREATE OR REPLACE TEMPORARY VIEW df_1\n",
    "AS SELECT *,\n",
    "to_date('2018-01-01') AS start_date,\n",
    "to_date('2999-01-01') AS end_date,\n",
    "FLOOR(id/5) AS partition_bin\n",
    "FROM df_1\n",
    "\"\"\")\n",
    "\n",
    "# spark.sql(\"\"\"\n",
    "# SELECT *, to_date('2018-01-01')\n",
    "# \"\"\")\n",
    "# df_1.withColumn('start_date', '2018-01-01')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DataFrame[]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "spark.sql(\"\"\"\n",
    "CREATE TABLE IF NOT EXISTS scd2 (\n",
    "    id INT,\n",
    "    dummy_contents STRING,\n",
    "    start_date DATE,\n",
    "    end_date DATE,\n",
    "    partition_bin INT\n",
    ")\n",
    "USING HIVE OPTIONS(fileFormat 'PARQUET')\n",
    "PARTITIONED BY (partition_bin)\n",
    "LOCATION 'data/scd2'\n",
    "\"\"\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DataFrame[]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "spark.sql(\"\"\"\n",
    "INSERT OVERWRITE TABLE scd2 PARTITION (partition_bin)\n",
    "SELECT * FROM df_1\n",
    "\"\"\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+---+--------------+----------+----------+-------------+\n",
      "| id|dummy_contents|start_date|  end_date|partition_bin|\n",
      "+---+--------------+----------+----------+-------------+\n",
      "|  0|             a|2018-01-01|2999-01-01|            0|\n",
      "|  1|             a|2018-01-01|2999-01-01|            0|\n",
      "|  2|             a|2018-01-01|2999-01-01|            0|\n",
      "|  3|             a|2018-01-01|2999-01-01|            0|\n",
      "|  4|             a|2018-01-01|2999-01-01|            0|\n",
      "|  5|             a|2018-01-01|2999-01-01|            1|\n",
      "|  6|             a|2018-01-01|2999-01-01|            1|\n",
      "|  7|             a|2018-01-01|2999-01-01|            1|\n",
      "|  8|             a|2018-01-01|2999-01-01|            1|\n",
      "|  9|             a|2018-01-01|2999-01-01|            1|\n",
      "+---+--------------+----------+----------+-------------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "spark.sql(\"SELECT * from scd2 ORDER BY id, start_date\").show(50)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Day 2 (2018-01-02)\n",
    "\n",
    "We add new day_2 data and apply SCD to current table\n",
    "\n",
    "**First add our calculated columns to df_2**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+---+--------------+----------+----------+-------------+\n",
      "| id|dummy_contents|start_date|  end_date|partition_bin|\n",
      "+---+--------------+----------+----------+-------------+\n",
      "|  7|             b|2018-02-01|2999-01-01|            1|\n",
      "|  8|             b|2018-02-01|2999-01-01|            1|\n",
      "|  9|             b|2018-02-01|2999-01-01|            1|\n",
      "| 10|             a|2018-02-01|2999-01-01|            2|\n",
      "| 11|             a|2018-02-01|2999-01-01|            2|\n",
      "| 12|             a|2018-02-01|2999-01-01|            2|\n",
      "| 13|             a|2018-02-01|2999-01-01|            2|\n",
      "+---+--------------+----------+----------+-------------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "spark.createDataFrame(delta_day_2, in_schema).createOrReplaceTempView('df_2')\n",
    "\n",
    "# Create a view of our df_1 data with added start_date, end_date and partition_bin columns\n",
    "spark.sql(\"\"\"\n",
    "CREATE OR REPLACE TEMPORARY VIEW df_2\n",
    "AS SELECT *,\n",
    "to_date('2018-02-01') AS start_date,\n",
    "to_date('2999-01-01') AS end_date,\n",
    "FLOOR(id/5) AS partition_bin\n",
    "FROM df_2\n",
    "\"\"\")\n",
    "\n",
    "spark.sql(\"SELECT * FROM df_2\").show(50)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Next append data from scd2 that matches any partitions in df_2 and recalculate SCD**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+---+--------------+----------+----------+-------------+\n",
      "| id|dummy_contents|start_date|  end_date|partition_bin|\n",
      "+---+--------------+----------+----------+-------------+\n",
      "| 12|             a|2018-02-01|2999-01-01|            2|\n",
      "| 13|             a|2018-02-01|2999-01-01|            2|\n",
      "|  6|             a|2018-01-01|2999-01-01|            1|\n",
      "|  5|             a|2018-01-01|2999-01-01|            1|\n",
      "|  9|             a|2018-01-01|2018-02-01|            1|\n",
      "|  9|             b|2018-02-01|2999-01-01|            1|\n",
      "|  8|             a|2018-01-01|2018-02-01|            1|\n",
      "|  8|             b|2018-02-01|2999-01-01|            1|\n",
      "|  7|             a|2018-01-01|2018-02-01|            1|\n",
      "|  7|             b|2018-02-01|2999-01-01|            1|\n",
      "| 10|             a|2018-02-01|2999-01-01|            2|\n",
      "| 11|             a|2018-02-01|2999-01-01|            2|\n",
      "+---+--------------+----------+----------+-------------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# CONCAT DF_2 and SCD2 (FOR ONLY MATCHING PARTITIONS)\n",
    "spark.sql(\"\"\"\n",
    "CREATE OR REPLACE TEMPORARY VIEW df_2_update AS\n",
    "SELECT scd2.*\n",
    "FROM scd2,\n",
    "(\n",
    "    SELECT DISTINCT partition_bin\n",
    "    FROM df_2\n",
    ") as p\n",
    "WHERE scd2.partition_bin = p.partition_bin\n",
    "UNION ALL\n",
    "SELECT df_2.*\n",
    "FROM df_2\n",
    "\"\"\")\n",
    "\n",
    "# RECALCULATE SCD2 FOR OUR UPDATE TABLE\n",
    "spark.sql(\"\"\"\n",
    "CREATE OR REPLACE TEMPORARY VIEW df_2_update AS\n",
    "SELECT id, dummy_contents, start_date,\n",
    "to_date(lead(start_date, 1, '2999-01-01') OVER (PARTITION BY id ORDER BY start_date)) AS end_date,\n",
    "partition_bin\n",
    "FROM df_2_update\n",
    "\"\"\")\n",
    "          \n",
    "spark.sql(\"SELECT * FROM df_2_update\").show(50)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Save the output to a temp location (causing spark to actually do an action - will allow us to write back to scd2)**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DataFrame[]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "spark.sql(\"DROP TABLE IF EXISTS scd2_update\")\n",
    "spark.sql(\"\"\"\n",
    "CREATE TABLE scd2_update\n",
    "USING HIVE OPTIONS(fileFormat 'PARQUET')\n",
    "PARTITIONED BY (partition_bin)\n",
    "LOCATION 'data/scd2_update'\n",
    "SELECT * FROM df_2_update\n",
    "\"\"\") # Doesn't need to be hive as will always want full overwrite of table"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Write temp location data into scd2_update patitions (note how full table isn't overwritten - only matching partitions)**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DataFrame[]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "spark.sql(\"\"\"\n",
    "INSERT OVERWRITE TABLE scd2 PARTITION (partition_bin)\n",
    "SELECT * FROM scd2_update\n",
    "\"\"\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+---+--------------+----------+----------+-------------+\n",
      "| id|dummy_contents|start_date|  end_date|partition_bin|\n",
      "+---+--------------+----------+----------+-------------+\n",
      "|  0|             a|2018-01-01|2999-01-01|            0|\n",
      "|  1|             a|2018-01-01|2999-01-01|            0|\n",
      "|  2|             a|2018-01-01|2999-01-01|            0|\n",
      "|  3|             a|2018-01-01|2999-01-01|            0|\n",
      "|  4|             a|2018-01-01|2999-01-01|            0|\n",
      "|  5|             a|2018-01-01|2999-01-01|            1|\n",
      "|  6|             a|2018-01-01|2999-01-01|            1|\n",
      "|  7|             a|2018-01-01|2018-02-01|            1|\n",
      "|  7|             b|2018-02-01|2999-01-01|            1|\n",
      "|  8|             a|2018-01-01|2018-02-01|            1|\n",
      "|  8|             b|2018-02-01|2999-01-01|            1|\n",
      "|  9|             a|2018-01-01|2018-02-01|            1|\n",
      "|  9|             b|2018-02-01|2999-01-01|            1|\n",
      "| 10|             a|2018-02-01|2999-01-01|            2|\n",
      "| 11|             a|2018-02-01|2999-01-01|            2|\n",
      "| 12|             a|2018-02-01|2999-01-01|            2|\n",
      "| 13|             a|2018-02-01|2999-01-01|            2|\n",
      "+---+--------------+----------+----------+-------------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "spark.sql(\"SELECT * FROM scd2 ORDER BY id, start_date\").show(50)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Day 3 (2018-01-03)\n",
    "**We add new day_3 data and apply SCD to current table (basically the same process for day 2 but more condenced and generalised code)**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DataFrame[]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# CREATE df of data from delta 3\n",
    "delta_start_date = '2018-03-01'\n",
    "default_end_date = '2999-01-01'\n",
    "\n",
    "spark.createDataFrame(delta_day_3, in_schema).createOrReplaceTempView('df_delta')\n",
    "spark.sql(f\"\"\"\n",
    "CREATE OR REPLACE TEMPORARY VIEW df_delta\n",
    "AS SELECT *,\n",
    "to_date('{delta_start_date}') AS start_date,\n",
    "to_date('2999-01-01') AS end_date,\n",
    "FLOOR(id/5) AS partition_bin\n",
    "FROM df_delta\n",
    "\"\"\")\n",
    "\n",
    "# Pull and chunks of data in scd2 that share a partition with df_delta_update and recalculate start and end dates\n",
    "spark.sql(f\"\"\"\n",
    "CREATE OR REPLACE TEMPORARY VIEW df_delta AS\n",
    "SELECT id, dummy_contents, start_date,\n",
    "to_date(lead(start_date, 1, '{default_end_date}') OVER (PARTITION BY id ORDER BY start_date)) AS end_date,\n",
    "partition_bin\n",
    "FROM\n",
    "(\n",
    "    SELECT scd2.*\n",
    "    FROM scd2,\n",
    "    (\n",
    "        SELECT DISTINCT partition_bin\n",
    "        FROM df_delta\n",
    "    ) as p\n",
    "    WHERE scd2.partition_bin = p.partition_bin\n",
    "    UNION ALL\n",
    "    SELECT df_delta.*\n",
    "    FROM df_delta\n",
    ")\n",
    "\"\"\")\n",
    "\n",
    "# scd2_update write to temp_update location\n",
    "spark.sql(\"DROP TABLE IF EXISTS scd2_update\")\n",
    "\n",
    "spark.sql(\"\"\"\n",
    "CREATE TABLE scd2_update\n",
    "USING HIVE OPTIONS(fileFormat 'PARQUET')\n",
    "PARTITIONED BY (partition_bin)\n",
    "LOCATION 'data/scd2_update'\n",
    "SELECT * FROM df_delta\n",
    "\"\"\")\n",
    "\n",
    "# Overwrite back into scd2 database\n",
    "spark.sql(\"\"\"\n",
    "INSERT OVERWRITE TABLE scd2 PARTITION (partition_bin)\n",
    "SELECT * FROM scd2_update\n",
    "\"\"\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+---+--------------+----------+----------+-------------+\n",
      "| id|dummy_contents|start_date|  end_date|partition_bin|\n",
      "+---+--------------+----------+----------+-------------+\n",
      "|  0|             a|2018-01-01|2999-01-01|            0|\n",
      "|  1|             a|2018-01-01|2018-03-01|            0|\n",
      "|  1|             b|2018-03-01|2999-01-01|            0|\n",
      "|  2|             a|2018-01-01|2018-03-01|            0|\n",
      "|  2|             b|2018-03-01|2999-01-01|            0|\n",
      "|  3|             a|2018-01-01|2018-03-01|            0|\n",
      "|  3|             b|2018-03-01|2999-01-01|            0|\n",
      "|  4|             a|2018-01-01|2018-03-01|            0|\n",
      "|  4|             b|2018-03-01|2999-01-01|            0|\n",
      "|  5|             a|2018-01-01|2018-03-01|            1|\n",
      "|  5|             a|2018-01-01|2999-01-01|            1|\n",
      "|  5|             b|2018-03-01|2999-01-01|            1|\n",
      "|  6|             a|2018-01-01|2999-01-01|            1|\n",
      "|  6|             a|2018-01-01|2018-03-01|            1|\n",
      "|  6|             c|2018-03-01|2999-01-01|            1|\n",
      "|  7|             a|2018-01-01|2018-02-01|            1|\n",
      "|  7|             a|2018-01-01|2018-02-01|            1|\n",
      "|  7|             b|2018-02-01|2018-03-01|            1|\n",
      "|  7|             b|2018-02-01|2999-01-01|            1|\n",
      "|  7|             c|2018-03-01|2999-01-01|            1|\n",
      "|  8|             a|2018-01-01|2018-02-01|            1|\n",
      "|  8|             a|2018-01-01|2018-02-01|            1|\n",
      "|  8|             b|2018-02-01|2018-03-01|            1|\n",
      "|  8|             b|2018-02-01|2999-01-01|            1|\n",
      "|  8|             c|2018-03-01|2999-01-01|            1|\n",
      "|  9|             a|2018-01-01|2018-02-01|            1|\n",
      "|  9|             a|2018-01-01|2018-02-01|            1|\n",
      "|  9|             b|2018-02-01|2018-03-01|            1|\n",
      "|  9|             b|2018-02-01|2999-01-01|            1|\n",
      "|  9|             c|2018-03-01|2999-01-01|            1|\n",
      "| 10|             a|2018-02-01|2018-03-01|            2|\n",
      "| 10|             a|2018-02-01|2999-01-01|            2|\n",
      "| 10|             b|2018-03-01|2999-01-01|            2|\n",
      "| 11|             a|2018-02-01|2018-03-01|            2|\n",
      "| 11|             a|2018-02-01|2999-01-01|            2|\n",
      "| 11|             b|2018-03-01|2999-01-01|            2|\n",
      "| 12|             a|2018-02-01|2999-01-01|            2|\n",
      "| 12|             a|2018-02-01|2018-03-01|            2|\n",
      "| 12|             b|2018-03-01|2999-01-01|            2|\n",
      "| 13|             a|2018-02-01|2018-03-01|            2|\n",
      "| 13|             a|2018-02-01|2999-01-01|            2|\n",
      "| 13|             b|2018-03-01|2999-01-01|            2|\n",
      "| 14|             a|2018-03-01|2999-01-01|            2|\n",
      "| 15|             a|2018-03-01|2999-01-01|            3|\n",
      "+---+--------------+----------+----------+-------------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "spark.sql(\"SELECT * FROM scd2_update ORDER by id, start_date\").show(50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}