Merge pull request #43 from openeduhub/sodix_spider

Sodix spider
hpi-schul-cloud · Jul 4, 2022 · a346f84 · a346f84
2 parents a2f2c8b + b8028eb
commit a346f84
Show file tree

Hide file tree

Showing 5 changed files with 418 additions and 9 deletions.
diff --git a/converter/.env.example b/converter/.env.example
@@ -34,3 +34,7 @@ YOUTUBE_API_KEY = ""
 
 # only for oeh spider: select the sources you want to fetch from oeh (comma seperated)
 # OEH_IMPORT_SOURCES = 'oeh,wirlernenonline_spider,serlo_spider,youtube_spider'
+
+# Sodix Spider login data
+# SODIX_SPIDER_USERNAME = ""
+# SODIX_SPIDER_PASSWORD = ""
diff --git a/converter/constants.py b/converter/constants.py
@@ -4,7 +4,10 @@ class Constants:
     LICENSE_CC_BY_SA_40 = "https://creativecommons.org/licenses/by-sa/4.0/"
     LICENSE_CC_BY_30 = "https://creativecommons.org/licenses/by/3.0/"
     LICENSE_CC_BY_40 = "https://creativecommons.org/licenses/by/4.0/"
+    LICENSE_CC_BY_NC_40 = "https://creativecommons.org/licenses/by-nc/4.0/"
+    LICENSE_CC_BY_ND_40 = "https://creativecommons.org/licenses/by-nd/4.0/"
     LICENSE_CC_BY_NC_SA_30 = "https://creativecommons.org/licenses/by-nc-sa/3.0/"
+    LICENSE_CC_BY_NC_SA_40 = "https://creativecommons.org/licenses/by-nc-sa/4.0/"
     LICENSE_CC_BY_NC_ND_30 = "https://creativecommons.org/licenses/by-nc-nd/3.0/"
     LICENSE_CC_BY_NC_ND_40 = "https://creativecommons.org/licenses/by-nc-nd/4.0/"
     LICENSE_PDM = "https://creativecommons.org/publicdomain/mark/1.0/"

diff --git a/converter/es_connector.py b/converter/es_connector.py
@@ -288,7 +288,13 @@ def transformItem(self, uuid, spider, item):
             spaces["cclom:general_keyword"] = None
         if "technical" in item["lom"]:
             if "duration" in item["lom"]["technical"]:
-                spaces["cclom:duration"] = item["lom"]["technical"]["duration"]
+                duration = item["lom"]["technical"]["duration"]
+                try:
+                    # edusharing requries milliseconds
+                    duration = int(float(duration) * 1000)
+                except:
+                    pass
+                spaces["cclom:duration"] = duration
 
         # TODO: this does currently not support multiple values per role
         if "lifecycle" in item["lom"]:
@@ -358,13 +364,12 @@ def transformItem(self, uuid, spider, item):
         }
         for key in item["valuespaces"]:
             spaces[valuespaceMapping[key]] = item["valuespaces"][key]
-        if "typicalagerange" in item["lom"]["educational"]:
-            spaces["ccm:educationaltypicalagerange_from"] = item["lom"]["educational"][
-                "typicalagerange"
-            ]["fromRange"]
-            spaces["ccm:educationaltypicalagerange_to"] = item["lom"]["educational"][
-                "typicalagerange"
-            ]["toRange"]
+        if "typicalAgeRange" in item["lom"]["educational"]:
+            tar = item["lom"]["educational"]["typicalAgeRange"]
+            if "fromRange" in tar:
+                spaces["ccm:educationaltypicalagerange_from"] = tar["fromRange"]
+            if "toRange" in tar:
+                spaces["ccm:educationaltypicalagerange_to"] = tar["toRange"]
         # intendedEndUserRole = Field(output_processor=JoinMultivalues())
         # discipline = Field(output_processor=JoinMultivalues())
         # educationalContext = Field(output_processor=JoinMultivalues())

diff --git a/converter/spiders/base_classes/lom_base.py b/converter/spiders/base_classes/lom_base.py
@@ -73,7 +73,10 @@ def hasChanged(self, response=None) -> bool:
         db = EduSharing().findItem(self.getId(response), self)
         changed = db == None or db[1] != self.getHash(response)
         if not changed:
-            logging.info("Item " + db[0] + " has not changed")
+            logging.info(
+                "Item " + self.getId(response) +
+                "(uuid: " + db[0] + ") has not changed"
+             )
         return changed
 
     # you might override this method if you don't want to import specific entries