diff --git a/data-products/src/main/scala/org/sunbird/analytics/job/report/AssessmentMetricsJobV2.scala b/data-products/src/main/scala/org/sunbird/analytics/job/report/AssessmentMetricsJobV2.scala index 1c8d1beba..15dbd4ca4 100644 --- a/data-products/src/main/scala/org/sunbird/analytics/job/report/AssessmentMetricsJobV2.scala +++ b/data-products/src/main/scala/org/sunbird/analytics/job/report/AssessmentMetricsJobV2.scala @@ -136,7 +136,7 @@ object AssessmentMetricsJobV2 extends optional.Application with IJob with BaseRe def getUserData(spark: SparkSession, loadData: (SparkSession, Map[String, String], String, StructType) => DataFrame): DataFrame = { val schema = Encoders.product[UserData].schema - loadData(spark, Map("keys.pattern" -> "user:*","infer.schema" -> "true", "key.column"-> "userid"), "org.apache.spark.sql.redis", schema) + loadData(spark, Map("table" -> "user","infer.schema" -> "true", "key.column"-> "userid"), "org.apache.spark.sql.redis", schema) .withColumn("username",concat_ws(" ", col("firstname"), col("lastname"))) } @@ -160,7 +160,7 @@ object AssessmentMetricsJobV2 extends optional.Application with IJob with BaseRe col("active")) val userDenormDF = userCoursesDF - .join(userDF, concat(lit("user:"), userCoursesDF.col("userid")) === userDF.col("userid"), "inner") + .join(userDF, Seq("userid"), "inner") .withColumn(UserCache.externalid, when(userCoursesDF.col("channel") === userDF.col(UserCache.userchannel), userDF.col(UserCache.externalid)).otherwise("")) .withColumn(UserCache.schoolname, when(userCoursesDF.col("channel") === userDF.col(UserCache.userchannel), userDF.col(UserCache.schoolname)).otherwise("")) .withColumn(UserCache.block, when(userCoursesDF.col("channel") === userDF.col(UserCache.userchannel), userDF.col(UserCache.block)).otherwise("")) diff --git a/data-products/src/main/scala/org/sunbird/analytics/job/report/CourseMetricsJobV2.scala b/data-products/src/main/scala/org/sunbird/analytics/job/report/CourseMetricsJobV2.scala index 65bb1a988..8a1ae2b50 100644 --- a/data-products/src/main/scala/org/sunbird/analytics/job/report/CourseMetricsJobV2.scala +++ b/data-products/src/main/scala/org/sunbird/analytics/job/report/CourseMetricsJobV2.scala @@ -145,7 +145,7 @@ object CourseMetricsJobV2 extends optional.Application with IJob with ReportGene def getUserData(spark: SparkSession, loadData: (SparkSession, Map[String, String], String, StructType) => DataFrame): DataFrame = { val schema = Encoders.product[UserData].schema - loadData(spark, Map("keys.pattern" -> "user:*","infer.schema" -> "true", "key.column"-> "userid"),"org.apache.spark.sql.redis", schema) + loadData(spark, Map("table" -> "user","infer.schema" -> "true", "key.column"-> "userid"),"org.apache.spark.sql.redis", schema) .withColumn("username",concat_ws(" ", col("firstname"), col("lastname"))) } @@ -187,7 +187,7 @@ object CourseMetricsJobV2 extends optional.Application with IJob with ReportGene ) // userCourseDenormDF lacks some of the user information that need to be part of the report here, it will add some more user details val reportDF = userCourseDenormDF - .join(userDF, concat(lit("user:"), userCourseDenormDF.col("userid")) === userDF.col("userid"), "inner") + .join(userDF, Seq("userid"), "inner") .withColumn(UserCache.externalid, when(userCourseDenormDF.col("channel") === userDF.col(UserCache.userchannel), userDF.col(UserCache.externalid)).otherwise("")) .withColumn(UserCache.schoolname, when(userCourseDenormDF.col("channel") === userDF.col(UserCache.userchannel), userDF.col(UserCache.schoolname)).otherwise("")) .withColumn(UserCache.block, when(userCourseDenormDF.col("channel") === userDF.col(UserCache.userchannel), userDF.col(UserCache.block)).otherwise("")) diff --git a/data-products/src/test/resources/course-metrics-updaterv2/user_data.json b/data-products/src/test/resources/course-metrics-updaterv2/user_data.json index 806c271b0..506726051 100644 --- a/data-products/src/test/resources/course-metrics-updaterv2/user_data.json +++ b/data-products/src/test/resources/course-metrics-updaterv2/user_data.json @@ -1 +1 @@ -[{"id": "c7ef3848-bbdb-4219-8344-817d5b8103fa","userchannel": "channel-2","email": "AosVy8IPstwqbXkGBN/ZtAXLSNR1m7xh/XKA/9npuSNY5p+d2lfl3zZpDHigOxjbujIDqFab8S4+\\n1vAxVXbou4IoAl6IHiZKvWygxnxPXCrmw5MlvURIR3WtJfCDG1yV0tkGrkZd9iMoUJfHXS9T1HeO\\nb2UyB38tfEb9BT5hlBQ=","emailverified": "True","firstname": "SudhirFOrRootOrg","framework": "{'board': ['State (Uttar Pradesh)'], 'gradeLevel': ['Class 1'], 'id': ['NCF'], 'medium': ['Hindi'], 'subject': ['Tamil']}","lastname": "V","schoolname": "School-1","district": "MPPS","orgname": "BAYYARAM","block": "SERA","externalid": "c98456789-fdcvbn","schooludisecode": "20","state": "GPPS","maskedemail": "su********************@gmail.com","maskedphone": "","phone": "","phoneverified": "False","rootorgid": "01268867935250841620619","status": 1,"userid": "user:c7ef3848-bbdb-4219-8344-817d5b8103fa","username": "AosVy8IPstwqbXkGBN/ZtBLSwkj+QWMmybV+eJejN2LU2WPAAAaK5ddrtpTfhC+rTa8manhhg3am\\nFLsIZO2PqDTTwflzFlGhuCmNVYEhkp9yLtGEIRl343/esZ3Xe4V0T6a+wzaAmCWueMEdPmZuRg==","usertype": "OTHER","webpages": ""}, {"id": "user021","userchannel": "channel-2","email": "AosVy8IPstwqbXkGBN/ZtAXLSNR1m7xh/XKA/9npuSNY5p+d2lfl3zZpDHigOxjbujIDqFab8S4+\\n1vAxVXbou4IoAl6IHiZKvWygxnxPXCrmw5MlvURIR3WtJfCDG1yV0tkGrkZd9iMoUJfHXS9T1HeO\\nb2UyB38tfEb9BT5hlBQ=","emailverified": "True","firstname": "SudhirFOrRootOrg","framework": "{'board': ['State (Uttar Pradesh)'], 'gradeLevel': ['Class 1'], 'id': ['NCF'], 'medium': ['Hindi'], 'subject': ['Tamil']}","lastname": "V","schoolname": "School-1","district": "MPPS","orgname": "BAYYARAM","block": "SERA","externalid": "c98456789-fdcvbn","schooludisecode": "20","state": "GPPS","maskedemail": "su********************@gmail.com","maskedphone": "","phone": "","phoneverified": "False","rootorgid": "01268867935250841620619","status": 1,"userid": "user:user021","username": "AosVy8IPstwqbXkGBN/ZtBLSwkj+QWMmybV+eJejN2LU2WPAAAaK5ddrtpTfhC+rTa8manhhg3am\\nFLsIZO2PqDTTwflzFlGhuCmNVYEhkp9yLtGEIRl343/esZ3Xe4V0T6a+wzaAmCWueMEdPmZuRg==","usertype": "OTHER","webpages": ""}, {"id": "f3dd58a4-a56f-4c1d-95cf-3231927a28e9","userchannel": "channel-2","email": "AosVy8IPstwqbXkGBN/ZtAXLSNR1m7xh/XKA/9npuSNY5p+d2lfl3zZpDHigOxjbujIDqFab8S4+\\n1vAxVXbou4IoAl6IHiZKvWygxnxPXCrmw5MlvURIR3WtJfCDG1yV0tkGrkZd9iMoUJfHXS9T1HeO\\nb2UyB38tfEb9BT5hlBQ=","emailverified": "True","firstname": "SudhirFOrRootOrg","framework": "{'board': ['State (Uttar Pradesh)'], 'gradeLevel': ['Class 1'], 'id': ['NCF'], 'medium': ['Hindi'], 'subject': ['Tamil']}","lastname": "V","schoolname": "School-2","district": "BG","orgname": "MOG","block": "BLOCK","externalid": "df09619-fdcvbn","schooludisecode": "10","state": "KA","maskedemail": "ka****@gmail.com","maskedphone": "","phone": "","phoneverified": "False","rootorgid": "01268867935250841620619","status": 1,"userid": "user:f3dd58a4-a56f-4c1d-95cf-3231927a28e9","username": "AosVy8IPstwqbXkGBN/ZtBLSwkj+QWMmybV+eJejN2LU2WPAAAaK5ddrtpTfhC+rTa8manhhg3am\\nFLsIZO2PqDTTwflzFlGhuCmNVYEhkp9yLtGEIRl343/esZ3Xe4V0T6a+wzaAmCWueMEdPmZuRg==","usertype": "OTHER","webpages": ""}, {"id": "f3dd58a4-a56f-4c1d-95cf-3231927a28e9","userchannel": "channel-2","email": "AosVy8IPstwqbXkGBN/ZtAXLSNR1m7xh/XKA/9npuSNY5p+d2lfl3zZpDHigOxjbujIDqFab8S4+\\n1vAxVXbou4IoAl6IHiZKvWygxnxPXCrmw5MlvURIR3WtJfCDG1yV0tkGrkZd9iMoUJfHXS9T1HeO\\nb2UyB38tfEb9BT5hlBQ=","emailverified": "True","firstname": "SudhirFOrRootOrg","framework": "{'board': ['State (Uttar Pradesh)'], 'gradeLevel': ['Class 1'], 'id': ['NCF'], 'medium': ['Hindi'], 'subject': ['Tamil']}","lastname": "V","schoolname": "School-2","district": "BG","orgname": "MOG","block": "BLOCK","externalid": "df09619-fdcvbn","schooludisecode": "10","state": "KA","maskedemail": "ka****@gmail.com","maskedphone": "","phone": "","phoneverified": "False","rootorgid": "01268867935250841620619","status": 1,"userid": "user:user026","username": "AosVy8IPstwqbXkGBN/ZtBLSwkj+QWMmybV+eJejN2LU2WPAAAaK5ddrtpTfhC+rTa8manhhg3am\\nFLsIZO2PqDTTwflzFlGhuCmNVYEhkp9yLtGEIRl343/esZ3Xe4V0T6a+wzaAmCWueMEdPmZuRg==","usertype": "OTHER","webpages": ""}, {"id": "f3dd58a4-a56f-4c1d-95cf-3231927a28e9","userchannel": "channel-2","email": "AosVy8IPstwqbXkGBN/ZtAXLSNR1m7xh/XKA/9npuSNY5p+d2lfl3zZpDHigOxjbujIDqFab8S4+\\n1vAxVXbou4IoAl6IHiZKvWygxnxPXCrmw5MlvURIR3WtJfCDG1yV0tkGrkZd9iMoUJfHXS9T1HeO\\nb2UyB38tfEb9BT5hlBQ=","emailverified": "True","firstname": "SudhirFOrRootOrg","framework": "{'board': ['State (Uttar Pradesh)'], 'gradeLevel': ['Class 1'], 'id': ['NCF'], 'medium': ['Hindi'], 'subject': ['Tamil']}","lastname": "V","schoolname": "School-2","district": "BG","orgname": "MOG","block": "BLOCK","externalid": "df09619-fdcvbn","schooludisecode": "10","state": "KA","maskedemail": "ka****@gmail.com","maskedphone": "","phone": "","phoneverified": "False","rootorgid": "01268867935250841620619","status": 1,"userid": "user:user025","username": "AosVy8IPstwqbXkGBN/ZtBLSwkj+QWMmybV+eJejN2LU2WPAAAaK5ddrtpTfhC+rTa8manhhg3am\\nFLsIZO2PqDTTwflzFlGhuCmNVYEhkp9yLtGEIRl343/esZ3Xe4V0T6a+wzaAmCWueMEdPmZuRg==","usertype": "OTHER","webpages": ""}] \ No newline at end of file +[{"id": "c7ef3848-bbdb-4219-8344-817d5b8103fa","userchannel": "channel-2","email": "AosVy8IPstwqbXkGBN/ZtAXLSNR1m7xh/XKA/9npuSNY5p+d2lfl3zZpDHigOxjbujIDqFab8S4+\\n1vAxVXbou4IoAl6IHiZKvWygxnxPXCrmw5MlvURIR3WtJfCDG1yV0tkGrkZd9iMoUJfHXS9T1HeO\\nb2UyB38tfEb9BT5hlBQ=","emailverified": "True","firstname": "SudhirFOrRootOrg","framework": "{'board': ['State (Uttar Pradesh)'], 'gradeLevel': ['Class 1'], 'id': ['NCF'], 'medium': ['Hindi'], 'subject': ['Tamil']}","lastname": "V","schoolname": "School-1","district": "MPPS","orgname": "BAYYARAM","block": "SERA","externalid": "c98456789-fdcvbn","schooludisecode": "20","state": "GPPS","maskedemail": "su********************@gmail.com","maskedphone": "","phone": "","phoneverified": "False","rootorgid": "01268867935250841620619","status": 1,"userid": "c7ef3848-bbdb-4219-8344-817d5b8103fa","username": "AosVy8IPstwqbXkGBN/ZtBLSwkj+QWMmybV+eJejN2LU2WPAAAaK5ddrtpTfhC+rTa8manhhg3am\\nFLsIZO2PqDTTwflzFlGhuCmNVYEhkp9yLtGEIRl343/esZ3Xe4V0T6a+wzaAmCWueMEdPmZuRg==","usertype": "OTHER","webpages": ""}, {"id": "user021","userchannel": "channel-2","email": "AosVy8IPstwqbXkGBN/ZtAXLSNR1m7xh/XKA/9npuSNY5p+d2lfl3zZpDHigOxjbujIDqFab8S4+\\n1vAxVXbou4IoAl6IHiZKvWygxnxPXCrmw5MlvURIR3WtJfCDG1yV0tkGrkZd9iMoUJfHXS9T1HeO\\nb2UyB38tfEb9BT5hlBQ=","emailverified": "True","firstname": "SudhirFOrRootOrg","framework": "{'board': ['State (Uttar Pradesh)'], 'gradeLevel': ['Class 1'], 'id': ['NCF'], 'medium': ['Hindi'], 'subject': ['Tamil']}","lastname": "V","schoolname": "School-1","district": "MPPS","orgname": "BAYYARAM","block": "SERA","externalid": "c98456789-fdcvbn","schooludisecode": "20","state": "GPPS","maskedemail": "su********************@gmail.com","maskedphone": "","phone": "","phoneverified": "False","rootorgid": "01268867935250841620619","status": 1,"userid": "user021","username": "AosVy8IPstwqbXkGBN/ZtBLSwkj+QWMmybV+eJejN2LU2WPAAAaK5ddrtpTfhC+rTa8manhhg3am\\nFLsIZO2PqDTTwflzFlGhuCmNVYEhkp9yLtGEIRl343/esZ3Xe4V0T6a+wzaAmCWueMEdPmZuRg==","usertype": "OTHER","webpages": ""}, {"id": "f3dd58a4-a56f-4c1d-95cf-3231927a28e9","userchannel": "channel-2","email": "AosVy8IPstwqbXkGBN/ZtAXLSNR1m7xh/XKA/9npuSNY5p+d2lfl3zZpDHigOxjbujIDqFab8S4+\\n1vAxVXbou4IoAl6IHiZKvWygxnxPXCrmw5MlvURIR3WtJfCDG1yV0tkGrkZd9iMoUJfHXS9T1HeO\\nb2UyB38tfEb9BT5hlBQ=","emailverified": "True","firstname": "SudhirFOrRootOrg","framework": "{'board': ['State (Uttar Pradesh)'], 'gradeLevel': ['Class 1'], 'id': ['NCF'], 'medium': ['Hindi'], 'subject': ['Tamil']}","lastname": "V","schoolname": "School-2","district": "BG","orgname": "MOG","block": "BLOCK","externalid": "df09619-fdcvbn","schooludisecode": "10","state": "KA","maskedemail": "ka****@gmail.com","maskedphone": "","phone": "","phoneverified": "False","rootorgid": "01268867935250841620619","status": 1,"userid": "f3dd58a4-a56f-4c1d-95cf-3231927a28e9","username": "AosVy8IPstwqbXkGBN/ZtBLSwkj+QWMmybV+eJejN2LU2WPAAAaK5ddrtpTfhC+rTa8manhhg3am\\nFLsIZO2PqDTTwflzFlGhuCmNVYEhkp9yLtGEIRl343/esZ3Xe4V0T6a+wzaAmCWueMEdPmZuRg==","usertype": "OTHER","webpages": ""}, {"id": "f3dd58a4-a56f-4c1d-95cf-3231927a28e9","userchannel": "channel-2","email": "AosVy8IPstwqbXkGBN/ZtAXLSNR1m7xh/XKA/9npuSNY5p+d2lfl3zZpDHigOxjbujIDqFab8S4+\\n1vAxVXbou4IoAl6IHiZKvWygxnxPXCrmw5MlvURIR3WtJfCDG1yV0tkGrkZd9iMoUJfHXS9T1HeO\\nb2UyB38tfEb9BT5hlBQ=","emailverified": "True","firstname": "SudhirFOrRootOrg","framework": "{'board': ['State (Uttar Pradesh)'], 'gradeLevel': ['Class 1'], 'id': ['NCF'], 'medium': ['Hindi'], 'subject': ['Tamil']}","lastname": "V","schoolname": "School-2","district": "BG","orgname": "MOG","block": "BLOCK","externalid": "df09619-fdcvbn","schooludisecode": "10","state": "KA","maskedemail": "ka****@gmail.com","maskedphone": "","phone": "","phoneverified": "False","rootorgid": "01268867935250841620619","status": 1,"userid": "user026","username": "AosVy8IPstwqbXkGBN/ZtBLSwkj+QWMmybV+eJejN2LU2WPAAAaK5ddrtpTfhC+rTa8manhhg3am\\nFLsIZO2PqDTTwflzFlGhuCmNVYEhkp9yLtGEIRl343/esZ3Xe4V0T6a+wzaAmCWueMEdPmZuRg==","usertype": "OTHER","webpages": ""}, {"id": "f3dd58a4-a56f-4c1d-95cf-3231927a28e9","userchannel": "channel-2","email": "AosVy8IPstwqbXkGBN/ZtAXLSNR1m7xh/XKA/9npuSNY5p+d2lfl3zZpDHigOxjbujIDqFab8S4+\\n1vAxVXbou4IoAl6IHiZKvWygxnxPXCrmw5MlvURIR3WtJfCDG1yV0tkGrkZd9iMoUJfHXS9T1HeO\\nb2UyB38tfEb9BT5hlBQ=","emailverified": "True","firstname": "SudhirFOrRootOrg","framework": "{'board': ['State (Uttar Pradesh)'], 'gradeLevel': ['Class 1'], 'id': ['NCF'], 'medium': ['Hindi'], 'subject': ['Tamil']}","lastname": "V","schoolname": "School-2","district": "BG","orgname": "MOG","block": "BLOCK","externalid": "df09619-fdcvbn","schooludisecode": "10","state": "KA","maskedemail": "ka****@gmail.com","maskedphone": "","phone": "","phoneverified": "False","rootorgid": "01268867935250841620619","status": 1,"userid": "user025","username": "AosVy8IPstwqbXkGBN/ZtBLSwkj+QWMmybV+eJejN2LU2WPAAAaK5ddrtpTfhC+rTa8manhhg3am\\nFLsIZO2PqDTTwflzFlGhuCmNVYEhkp9yLtGEIRl343/esZ3Xe4V0T6a+wzaAmCWueMEdPmZuRg==","usertype": "OTHER","webpages": ""}] \ No newline at end of file diff --git a/data-products/src/test/scala/org/sunbird/analytics/job/report/TestAssessmentMetricsJobV2.scala b/data-products/src/test/scala/org/sunbird/analytics/job/report/TestAssessmentMetricsJobV2.scala index ccd3feab7..9558d7efd 100644 --- a/data-products/src/test/scala/org/sunbird/analytics/job/report/TestAssessmentMetricsJobV2.scala +++ b/data-products/src/test/scala/org/sunbird/analytics/job/report/TestAssessmentMetricsJobV2.scala @@ -112,7 +112,7 @@ class TestAssessmentMetricsJobV2 extends BaseReportSpec with MockFactory { val schema = Encoders.product[UserData].schema (reporterMock.loadData _) - .expects(spark, Map("keys.pattern" -> "user:*","infer.schema" -> "true", "key.column"-> "userid"),"org.apache.spark.sql.redis", schema) + .expects(spark, Map("table" -> "user","infer.schema" -> "true", "key.column"-> "userid"),"org.apache.spark.sql.redis", schema) .anyNumberOfTimes() .returning(userInfoDF) @@ -145,7 +145,7 @@ class TestAssessmentMetricsJobV2 extends BaseReportSpec with MockFactory { val schema = Encoders.product[UserData].schema (reporterMock.loadData _) - .expects(spark, Map("keys.pattern" -> "user:*","infer.schema" -> "true", "key.column"-> "userid"),"org.apache.spark.sql.redis", schema) + .expects(spark, Map("table" -> "user","infer.schema" -> "true", "key.column"-> "userid"),"org.apache.spark.sql.redis", schema) .anyNumberOfTimes() .returning(userInfoDF) diff --git a/data-products/src/test/scala/org/sunbird/analytics/job/report/TestCourseMetricsJobV2.scala b/data-products/src/test/scala/org/sunbird/analytics/job/report/TestCourseMetricsJobV2.scala index ff7677e15..bf4e41518 100644 --- a/data-products/src/test/scala/org/sunbird/analytics/job/report/TestCourseMetricsJobV2.scala +++ b/data-products/src/test/scala/org/sunbird/analytics/job/report/TestCourseMetricsJobV2.scala @@ -61,7 +61,7 @@ class TestCourseMetricsJobV2 extends BaseReportSpec with MockFactory with BaseRe val schema = Encoders.product[UserData].schema (reporterMock.loadData _) - .expects(spark, Map("keys.pattern" -> "user:*","infer.schema" -> "true", "key.column"-> "userid"),"org.apache.spark.sql.redis", schema) + .expects(spark, Map("table" -> "user","infer.schema" -> "true", "key.column"-> "userid"),"org.apache.spark.sql.redis", schema) .anyNumberOfTimes() .returning(userDF)