Skip to content

Commit

Permalink
Merge pull request #179 from ishawakankar/report-changes
Browse files Browse the repository at this point in the history 
Report changes for reading from redis
  • Loading branch information
@SanthoshVasabhaktula
SanthoshVasabhaktula authored Aug 7, 2020
2 parents bd1c924 + 4156f43 commit 94a8ed0
Show file tree
Hide file tree
Showing 5 changed files with 8 additions and 8 deletions.
4 changes: 2 additions & 2 deletions data-products/src/main/scala/org/sunbird/analytics/job/report/AssessmentMetricsJobV2.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -136,7 +136,7 @@ object AssessmentMetricsJobV2 extends optional.Application with IJob with BaseRe

def getUserData(spark: SparkSession, loadData: (SparkSession, Map[String, String], String, StructType) => DataFrame): DataFrame = {
val schema = Encoders.product[UserData].schema
loadData(spark, Map("keys.pattern" -> "user:*","infer.schema" -> "true", "key.column"-> "userid"), "org.apache.spark.sql.redis", schema)
loadData(spark, Map("table" -> "user","infer.schema" -> "true", "key.column"-> "userid"), "org.apache.spark.sql.redis", schema)
.withColumn("username",concat_ws(" ", col("firstname"), col("lastname")))
}

Expand All @@ -160,7 +160,7 @@ object AssessmentMetricsJobV2 extends optional.Application with IJob with BaseRe
col("active"))

val userDenormDF = userCoursesDF
.join(userDF, concat(lit("user:"), userCoursesDF.col("userid")) === userDF.col("userid"), "inner")
.join(userDF, Seq("userid"), "inner")
.withColumn(UserCache.externalid, when(userCoursesDF.col("channel") === userDF.col(UserCache.userchannel), userDF.col(UserCache.externalid)).otherwise(""))
.withColumn(UserCache.schoolname, when(userCoursesDF.col("channel") === userDF.col(UserCache.userchannel), userDF.col(UserCache.schoolname)).otherwise(""))
.withColumn(UserCache.block, when(userCoursesDF.col("channel") === userDF.col(UserCache.userchannel), userDF.col(UserCache.block)).otherwise(""))
Expand Down
4 changes: 2 additions & 2 deletions data-products/src/main/scala/org/sunbird/analytics/job/report/CourseMetricsJobV2.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -145,7 +145,7 @@ object CourseMetricsJobV2 extends optional.Application with IJob with ReportGene

def getUserData(spark: SparkSession, loadData: (SparkSession, Map[String, String], String, StructType) => DataFrame): DataFrame = {
val schema = Encoders.product[UserData].schema
loadData(spark, Map("keys.pattern" -> "user:*","infer.schema" -> "true", "key.column"-> "userid"),"org.apache.spark.sql.redis", schema)
loadData(spark, Map("table" -> "user","infer.schema" -> "true", "key.column"-> "userid"),"org.apache.spark.sql.redis", schema)
.withColumn("username",concat_ws(" ", col("firstname"), col("lastname")))
}

Expand Down Expand Up @@ -187,7 +187,7 @@ object CourseMetricsJobV2 extends optional.Application with IJob with ReportGene
)
// userCourseDenormDF lacks some of the user information that need to be part of the report here, it will add some more user details
val reportDF = userCourseDenormDF
.join(userDF, concat(lit("user:"), userCourseDenormDF.col("userid")) === userDF.col("userid"), "inner")
.join(userDF, Seq("userid"), "inner")
.withColumn(UserCache.externalid, when(userCourseDenormDF.col("channel") === userDF.col(UserCache.userchannel), userDF.col(UserCache.externalid)).otherwise(""))
.withColumn(UserCache.schoolname, when(userCourseDenormDF.col("channel") === userDF.col(UserCache.userchannel), userDF.col(UserCache.schoolname)).otherwise(""))
.withColumn(UserCache.block, when(userCourseDenormDF.col("channel") === userDF.col(UserCache.userchannel), userDF.col(UserCache.block)).otherwise(""))
Expand Down
2 changes: 1 addition & 1 deletion data-products/src/test/resources/course-metrics-updaterv2/user_data.json
View file
Original file line number Diff line number Diff line change
@@ -1 +1 @@
[{"id": "c7ef3848-bbdb-4219-8344-817d5b8103fa","userchannel": "channel-2","email": "AosVy8IPstwqbXkGBN/ZtAXLSNR1m7xh/XKA/9npuSNY5p+d2lfl3zZpDHigOxjbujIDqFab8S4+\\n1vAxVXbou4IoAl6IHiZKvWygxnxPXCrmw5MlvURIR3WtJfCDG1yV0tkGrkZd9iMoUJfHXS9T1HeO\\nb2UyB38tfEb9BT5hlBQ=","emailverified": "True","firstname": "SudhirFOrRootOrg","framework": "{'board': ['State (Uttar Pradesh)'], 'gradeLevel': ['Class 1'], 'id': ['NCF'], 'medium': ['Hindi'], 'subject': ['Tamil']}","lastname": "V","schoolname": "School-1","district": "MPPS","orgname": "BAYYARAM","block": "SERA","externalid": "c98456789-fdcvbn","schooludisecode": "20","state": "GPPS","maskedemail": "su********************@gmail.com","maskedphone": "","phone": "","phoneverified": "False","rootorgid": "01268867935250841620619","status": 1,"userid": "user:c7ef3848-bbdb-4219-8344-817d5b8103fa","username": "AosVy8IPstwqbXkGBN/ZtBLSwkj+QWMmybV+eJejN2LU2WPAAAaK5ddrtpTfhC+rTa8manhhg3am\\nFLsIZO2PqDTTwflzFlGhuCmNVYEhkp9yLtGEIRl343/esZ3Xe4V0T6a+wzaAmCWueMEdPmZuRg==","usertype": "OTHER","webpages": ""}, {"id": "user021","userchannel": "channel-2","email": "AosVy8IPstwqbXkGBN/ZtAXLSNR1m7xh/XKA/9npuSNY5p+d2lfl3zZpDHigOxjbujIDqFab8S4+\\n1vAxVXbou4IoAl6IHiZKvWygxnxPXCrmw5MlvURIR3WtJfCDG1yV0tkGrkZd9iMoUJfHXS9T1HeO\\nb2UyB38tfEb9BT5hlBQ=","emailverified": "True","firstname": "SudhirFOrRootOrg","framework": "{'board': ['State (Uttar Pradesh)'], 'gradeLevel': ['Class 1'], 'id': ['NCF'], 'medium': ['Hindi'], 'subject': ['Tamil']}","lastname": "V","schoolname": "School-1","district": "MPPS","orgname": "BAYYARAM","block": "SERA","externalid": "c98456789-fdcvbn","schooludisecode": "20","state": "GPPS","maskedemail": "su********************@gmail.com","maskedphone": "","phone": "","phoneverified": "False","rootorgid": "01268867935250841620619","status": 1,"userid": "user:user021","username": "AosVy8IPstwqbXkGBN/ZtBLSwkj+QWMmybV+eJejN2LU2WPAAAaK5ddrtpTfhC+rTa8manhhg3am\\nFLsIZO2PqDTTwflzFlGhuCmNVYEhkp9yLtGEIRl343/esZ3Xe4V0T6a+wzaAmCWueMEdPmZuRg==","usertype": "OTHER","webpages": ""}, {"id": "f3dd58a4-a56f-4c1d-95cf-3231927a28e9","userchannel": "channel-2","email": "AosVy8IPstwqbXkGBN/ZtAXLSNR1m7xh/XKA/9npuSNY5p+d2lfl3zZpDHigOxjbujIDqFab8S4+\\n1vAxVXbou4IoAl6IHiZKvWygxnxPXCrmw5MlvURIR3WtJfCDG1yV0tkGrkZd9iMoUJfHXS9T1HeO\\nb2UyB38tfEb9BT5hlBQ=","emailverified": "True","firstname": "SudhirFOrRootOrg","framework": "{'board': ['State (Uttar Pradesh)'], 'gradeLevel': ['Class 1'], 'id': ['NCF'], 'medium': ['Hindi'], 'subject': ['Tamil']}","lastname": "V","schoolname": "School-2","district": "BG","orgname": "MOG","block": "BLOCK","externalid": "df09619-fdcvbn","schooludisecode": "10","state": "KA","maskedemail": "ka****@gmail.com","maskedphone": "","phone": "","phoneverified": "False","rootorgid": "01268867935250841620619","status": 1,"userid": "user:f3dd58a4-a56f-4c1d-95cf-3231927a28e9","username": "AosVy8IPstwqbXkGBN/ZtBLSwkj+QWMmybV+eJejN2LU2WPAAAaK5ddrtpTfhC+rTa8manhhg3am\\nFLsIZO2PqDTTwflzFlGhuCmNVYEhkp9yLtGEIRl343/esZ3Xe4V0T6a+wzaAmCWueMEdPmZuRg==","usertype": "OTHER","webpages": ""}, {"id": "f3dd58a4-a56f-4c1d-95cf-3231927a28e9","userchannel": "channel-2","email": "AosVy8IPstwqbXkGBN/ZtAXLSNR1m7xh/XKA/9npuSNY5p+d2lfl3zZpDHigOxjbujIDqFab8S4+\\n1vAxVXbou4IoAl6IHiZKvWygxnxPXCrmw5MlvURIR3WtJfCDG1yV0tkGrkZd9iMoUJfHXS9T1HeO\\nb2UyB38tfEb9BT5hlBQ=","emailverified": "True","firstname": "SudhirFOrRootOrg","framework": "{'board': ['State (Uttar Pradesh)'], 'gradeLevel': ['Class 1'], 'id': ['NCF'], 'medium': ['Hindi'], 'subject': ['Tamil']}","lastname": "V","schoolname": "School-2","district": "BG","orgname": "MOG","block": "BLOCK","externalid": "df09619-fdcvbn","schooludisecode": "10","state": "KA","maskedemail": "ka****@gmail.com","maskedphone": "","phone": "","phoneverified": "False","rootorgid": "01268867935250841620619","status": 1,"userid": "user:user026","username": "AosVy8IPstwqbXkGBN/ZtBLSwkj+QWMmybV+eJejN2LU2WPAAAaK5ddrtpTfhC+rTa8manhhg3am\\nFLsIZO2PqDTTwflzFlGhuCmNVYEhkp9yLtGEIRl343/esZ3Xe4V0T6a+wzaAmCWueMEdPmZuRg==","usertype": "OTHER","webpages": ""}, {"id": "f3dd58a4-a56f-4c1d-95cf-3231927a28e9","userchannel": "channel-2","email": "AosVy8IPstwqbXkGBN/ZtAXLSNR1m7xh/XKA/9npuSNY5p+d2lfl3zZpDHigOxjbujIDqFab8S4+\\n1vAxVXbou4IoAl6IHiZKvWygxnxPXCrmw5MlvURIR3WtJfCDG1yV0tkGrkZd9iMoUJfHXS9T1HeO\\nb2UyB38tfEb9BT5hlBQ=","emailverified": "True","firstname": "SudhirFOrRootOrg","framework": "{'board': ['State (Uttar Pradesh)'], 'gradeLevel': ['Class 1'], 'id': ['NCF'], 'medium': ['Hindi'], 'subject': ['Tamil']}","lastname": "V","schoolname": "School-2","district": "BG","orgname": "MOG","block": "BLOCK","externalid": "df09619-fdcvbn","schooludisecode": "10","state": "KA","maskedemail": "ka****@gmail.com","maskedphone": "","phone": "","phoneverified": "False","rootorgid": "01268867935250841620619","status": 1,"userid": "user:user025","username": "AosVy8IPstwqbXkGBN/ZtBLSwkj+QWMmybV+eJejN2LU2WPAAAaK5ddrtpTfhC+rTa8manhhg3am\\nFLsIZO2PqDTTwflzFlGhuCmNVYEhkp9yLtGEIRl343/esZ3Xe4V0T6a+wzaAmCWueMEdPmZuRg==","usertype": "OTHER","webpages": ""}]
[{"id": "c7ef3848-bbdb-4219-8344-817d5b8103fa","userchannel": "channel-2","email": "AosVy8IPstwqbXkGBN/ZtAXLSNR1m7xh/XKA/9npuSNY5p+d2lfl3zZpDHigOxjbujIDqFab8S4+\\n1vAxVXbou4IoAl6IHiZKvWygxnxPXCrmw5MlvURIR3WtJfCDG1yV0tkGrkZd9iMoUJfHXS9T1HeO\\nb2UyB38tfEb9BT5hlBQ=","emailverified": "True","firstname": "SudhirFOrRootOrg","framework": "{'board': ['State (Uttar Pradesh)'], 'gradeLevel': ['Class 1'], 'id': ['NCF'], 'medium': ['Hindi'], 'subject': ['Tamil']}","lastname": "V","schoolname": "School-1","district": "MPPS","orgname": "BAYYARAM","block": "SERA","externalid": "c98456789-fdcvbn","schooludisecode": "20","state": "GPPS","maskedemail": "su********************@gmail.com","maskedphone": "","phone": "","phoneverified": "False","rootorgid": "01268867935250841620619","status": 1,"userid": "c7ef3848-bbdb-4219-8344-817d5b8103fa","username": "AosVy8IPstwqbXkGBN/ZtBLSwkj+QWMmybV+eJejN2LU2WPAAAaK5ddrtpTfhC+rTa8manhhg3am\\nFLsIZO2PqDTTwflzFlGhuCmNVYEhkp9yLtGEIRl343/esZ3Xe4V0T6a+wzaAmCWueMEdPmZuRg==","usertype": "OTHER","webpages": ""}, {"id": "user021","userchannel": "channel-2","email": "AosVy8IPstwqbXkGBN/ZtAXLSNR1m7xh/XKA/9npuSNY5p+d2lfl3zZpDHigOxjbujIDqFab8S4+\\n1vAxVXbou4IoAl6IHiZKvWygxnxPXCrmw5MlvURIR3WtJfCDG1yV0tkGrkZd9iMoUJfHXS9T1HeO\\nb2UyB38tfEb9BT5hlBQ=","emailverified": "True","firstname": "SudhirFOrRootOrg","framework": "{'board': ['State (Uttar Pradesh)'], 'gradeLevel': ['Class 1'], 'id': ['NCF'], 'medium': ['Hindi'], 'subject': ['Tamil']}","lastname": "V","schoolname": "School-1","district": "MPPS","orgname": "BAYYARAM","block": "SERA","externalid": "c98456789-fdcvbn","schooludisecode": "20","state": "GPPS","maskedemail": "su********************@gmail.com","maskedphone": "","phone": "","phoneverified": "False","rootorgid": "01268867935250841620619","status": 1,"userid": "user021","username": "AosVy8IPstwqbXkGBN/ZtBLSwkj+QWMmybV+eJejN2LU2WPAAAaK5ddrtpTfhC+rTa8manhhg3am\\nFLsIZO2PqDTTwflzFlGhuCmNVYEhkp9yLtGEIRl343/esZ3Xe4V0T6a+wzaAmCWueMEdPmZuRg==","usertype": "OTHER","webpages": ""}, {"id": "f3dd58a4-a56f-4c1d-95cf-3231927a28e9","userchannel": "channel-2","email": "AosVy8IPstwqbXkGBN/ZtAXLSNR1m7xh/XKA/9npuSNY5p+d2lfl3zZpDHigOxjbujIDqFab8S4+\\n1vAxVXbou4IoAl6IHiZKvWygxnxPXCrmw5MlvURIR3WtJfCDG1yV0tkGrkZd9iMoUJfHXS9T1HeO\\nb2UyB38tfEb9BT5hlBQ=","emailverified": "True","firstname": "SudhirFOrRootOrg","framework": "{'board': ['State (Uttar Pradesh)'], 'gradeLevel': ['Class 1'], 'id': ['NCF'], 'medium': ['Hindi'], 'subject': ['Tamil']}","lastname": "V","schoolname": "School-2","district": "BG","orgname": "MOG","block": "BLOCK","externalid": "df09619-fdcvbn","schooludisecode": "10","state": "KA","maskedemail": "ka****@gmail.com","maskedphone": "","phone": "","phoneverified": "False","rootorgid": "01268867935250841620619","status": 1,"userid": "f3dd58a4-a56f-4c1d-95cf-3231927a28e9","username": "AosVy8IPstwqbXkGBN/ZtBLSwkj+QWMmybV+eJejN2LU2WPAAAaK5ddrtpTfhC+rTa8manhhg3am\\nFLsIZO2PqDTTwflzFlGhuCmNVYEhkp9yLtGEIRl343/esZ3Xe4V0T6a+wzaAmCWueMEdPmZuRg==","usertype": "OTHER","webpages": ""}, {"id": "f3dd58a4-a56f-4c1d-95cf-3231927a28e9","userchannel": "channel-2","email": "AosVy8IPstwqbXkGBN/ZtAXLSNR1m7xh/XKA/9npuSNY5p+d2lfl3zZpDHigOxjbujIDqFab8S4+\\n1vAxVXbou4IoAl6IHiZKvWygxnxPXCrmw5MlvURIR3WtJfCDG1yV0tkGrkZd9iMoUJfHXS9T1HeO\\nb2UyB38tfEb9BT5hlBQ=","emailverified": "True","firstname": "SudhirFOrRootOrg","framework": "{'board': ['State (Uttar Pradesh)'], 'gradeLevel': ['Class 1'], 'id': ['NCF'], 'medium': ['Hindi'], 'subject': ['Tamil']}","lastname": "V","schoolname": "School-2","district": "BG","orgname": "MOG","block": "BLOCK","externalid": "df09619-fdcvbn","schooludisecode": "10","state": "KA","maskedemail": "ka****@gmail.com","maskedphone": "","phone": "","phoneverified": "False","rootorgid": "01268867935250841620619","status": 1,"userid": "user026","username": "AosVy8IPstwqbXkGBN/ZtBLSwkj+QWMmybV+eJejN2LU2WPAAAaK5ddrtpTfhC+rTa8manhhg3am\\nFLsIZO2PqDTTwflzFlGhuCmNVYEhkp9yLtGEIRl343/esZ3Xe4V0T6a+wzaAmCWueMEdPmZuRg==","usertype": "OTHER","webpages": ""}, {"id": "f3dd58a4-a56f-4c1d-95cf-3231927a28e9","userchannel": "channel-2","email": "AosVy8IPstwqbXkGBN/ZtAXLSNR1m7xh/XKA/9npuSNY5p+d2lfl3zZpDHigOxjbujIDqFab8S4+\\n1vAxVXbou4IoAl6IHiZKvWygxnxPXCrmw5MlvURIR3WtJfCDG1yV0tkGrkZd9iMoUJfHXS9T1HeO\\nb2UyB38tfEb9BT5hlBQ=","emailverified": "True","firstname": "SudhirFOrRootOrg","framework": "{'board': ['State (Uttar Pradesh)'], 'gradeLevel': ['Class 1'], 'id': ['NCF'], 'medium': ['Hindi'], 'subject': ['Tamil']}","lastname": "V","schoolname": "School-2","district": "BG","orgname": "MOG","block": "BLOCK","externalid": "df09619-fdcvbn","schooludisecode": "10","state": "KA","maskedemail": "ka****@gmail.com","maskedphone": "","phone": "","phoneverified": "False","rootorgid": "01268867935250841620619","status": 1,"userid": "user025","username": "AosVy8IPstwqbXkGBN/ZtBLSwkj+QWMmybV+eJejN2LU2WPAAAaK5ddrtpTfhC+rTa8manhhg3am\\nFLsIZO2PqDTTwflzFlGhuCmNVYEhkp9yLtGEIRl343/esZ3Xe4V0T6a+wzaAmCWueMEdPmZuRg==","usertype": "OTHER","webpages": ""}]
4 changes: 2 additions & 2 deletions ...products/src/test/scala/org/sunbird/analytics/job/report/TestAssessmentMetricsJobV2.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -112,7 +112,7 @@ class TestAssessmentMetricsJobV2 extends BaseReportSpec with MockFactory {

val schema = Encoders.product[UserData].schema
(reporterMock.loadData _)
.expects(spark, Map("keys.pattern" -> "user:*","infer.schema" -> "true", "key.column"-> "userid"),"org.apache.spark.sql.redis", schema)
.expects(spark, Map("table" -> "user","infer.schema" -> "true", "key.column"-> "userid"),"org.apache.spark.sql.redis", schema)
.anyNumberOfTimes()
.returning(userInfoDF)

Expand Down Expand Up @@ -145,7 +145,7 @@ class TestAssessmentMetricsJobV2 extends BaseReportSpec with MockFactory {

val schema = Encoders.product[UserData].schema
(reporterMock.loadData _)
.expects(spark, Map("keys.pattern" -> "user:*","infer.schema" -> "true", "key.column"-> "userid"),"org.apache.spark.sql.redis", schema)
.expects(spark, Map("table" -> "user","infer.schema" -> "true", "key.column"-> "userid"),"org.apache.spark.sql.redis", schema)
.anyNumberOfTimes()
.returning(userInfoDF)

Expand Down
2 changes: 1 addition & 1 deletion data-products/src/test/scala/org/sunbird/analytics/job/report/TestCourseMetricsJobV2.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,7 @@ class TestCourseMetricsJobV2 extends BaseReportSpec with MockFactory with BaseRe

val schema = Encoders.product[UserData].schema
(reporterMock.loadData _)
.expects(spark, Map("keys.pattern" -> "user:*","infer.schema" -> "true", "key.column"-> "userid"),"org.apache.spark.sql.redis", schema)
.expects(spark, Map("table" -> "user","infer.schema" -> "true", "key.column"-> "userid"),"org.apache.spark.sql.redis", schema)
.anyNumberOfTimes()
.returning(userDF)

Expand Down

0 comments on commit 94a8ed0

Please sign in to comment.