Merge pull request DataExpert-io#195 from divakaivan/main

Week 3 Spark Fundamentals 1st notebook 1st cell fix missing column
tskprog · Dec 14, 2024 · a3a98a7 · a3a98a7
2 parents 8707ce0 + 98ef5e0
commit a3a98a7
Showing 1 changed file with 6 additions and 3 deletions.
diff --git a/bootcamp/materials/3-spark-fundamentals/notebooks/event_data_pyspark.ipynb b/bootcamp/materials/3-spark-fundamentals/notebooks/event_data_pyspark.ipynb
@@ -2,7 +2,7 @@
  "cells": [
   {
    "cell_type": "code",
-   "execution_count": 1,
+   "execution_count": null,
    "id": "81cca085-dba2-42eb-a13b-fa64b6e86583",
    "metadata": {},
    "outputs": [
@@ -53,7 +53,11 @@
     "\n",
     "spark\n",
     "\n",
-    "df = spark.read.option(\"header\", \"true\").csv(\"/home/iceberg/data/events.csv\").withColumn(\"event_date\", expr(\"DATE_TRUNC('day', event_time)\"))\n",
+    "events = spark.read.option(\"header\", \"true\").csv(\"/home/iceberg/data/events.csv\").withColumn(\"event_date\", expr(\"DATE_TRUNC('day', event_time)\"))\n",
+    "devices = spark.read.option(\"header\",\"true\").csv(\"/home/iceberg/data/devices.csv\")\n",
+    "\n",
+    "df = events.join(devices,on=\"device_id\",how=\"left\")\n",
+    "df = df.withColumnsRenamed({'browser_type': 'browser_family', 'os_type': 'os_family'})\n",
     "\n",
     "df.show()"
    ]
@@ -570,7 +574,6 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "faaed2df",
    "metadata": {
     "collapsed": false,
     "jupyter": {