Add process for package upload with integrated title data (#274)

hbz · Aug 17, 2020 · 5473bc5 · 5473bc5
1 parent 285974b
commit 5473bc5
Show file tree

Hide file tree

Showing 6 changed files with 45 additions and 25 deletions.
diff --git a/grails-app/controllers/ygor/EnrichmentController.groovy b/grails-app/controllers/ygor/EnrichmentController.groovy
@@ -335,19 +335,12 @@ class EnrichmentController implements ControllersHelper{
     // Upload following.
     def gokbUsername = params.gokbUsername
     def gokbPassword = params.gokbPassword
-    // send titles
-    String uri = getDestinationUri(grailsApplication, Enrichment.FileType.TITLES, enrichment.addOnly)
+    // send package with integrated title data
+    String uri = getDestinationUri(grailsApplication, Enrichment.FileType.PACKAGE, enrichment.addOnly)
     def locale = RequestContextUtils.getLocale(request).toString()
-    SendTitlesThreadGokb sendTitlesThread = new SendTitlesThreadGokb(enrichment, uri, gokbUsername, gokbPassword,
-        locale)
-    UploadJob uploadJob = new UploadJob(Enrichment.FileType.TITLES, sendTitlesThread)
-    uploadJob.start()
-    watchUpload(uploadJob, Enrichment.FileType.TITLES, fileName)
-    // send package
-    uri = getDestinationUri(grailsApplication, Enrichment.FileType.PACKAGE, enrichment.addOnly)
     SendPackageThreadGokb sendPackageThreadGokb = new SendPackageThreadGokb(grailsApplication, enrichment, uri,
-        gokbUsername, gokbPassword, locale)
-    uploadJob = new UploadJob(Enrichment.FileType.PACKAGE, sendPackageThreadGokb)
+        gokbUsername, gokbPassword, locale, true)
+    UploadJob uploadJob = new UploadJob(Enrichment.FileType.PACKAGE, sendPackageThreadGokb)
     uploadJob.start()
     watchUpload(uploadJob, Enrichment.FileType.PACKAGE, fileName)
   }

diff --git a/grails-app/controllers/ygor/StatisticController.groovy b/grails-app/controllers/ygor/StatisticController.groovy
@@ -395,7 +395,7 @@ class StatisticController implements ControllersHelper{
       }
       else if (fileType.equals(Enrichment.FileType.PACKAGE)){
         SendPackageThreadGokb sendPackageThread = new SendPackageThreadGokb(grailsApplication, enrichment, uri,
-            gokbUsername, gokbPassword, RequestContextUtils.getLocale(request).toString())
+            gokbUsername, gokbPassword, RequestContextUtils.getLocale(request).toString(), false)
         uploadJob = new UploadJob(Enrichment.FileType.PACKAGE, sendPackageThread)
       }
       if (uploadJob != null){

diff --git a/grails-app/domain/ygor/Enrichment.groovy b/grails-app/domain/ygor/Enrichment.groovy
@@ -37,6 +37,7 @@ class Enrichment{
     JSON,
     PACKAGE,
     TITLES,
+    PACKAGE_WITH_TITLEDATA,
     RAW
   }
 

diff --git a/src/groovy/de/hbznrw/ygor/export/GokbExporter.groovy b/src/groovy/de/hbznrw/ygor/export/GokbExporter.groovy
@@ -41,10 +41,15 @@ class GokbExporter {
       case FileType.ORIGIN:
         return new File(enrichment.originPathName)
       case FileType.PACKAGE:
-        ObjectNode result = GokbExporter.extractPackage(enrichment)
+        ObjectNode result = GokbExporter.extractPackage(enrichment, false)
         def file = new File(enrichment.enrichmentFolder + ".package.json")
         file.write(JSON_OBJECT_MAPPER.writerWithDefaultPrettyPrinter().writeValueAsString(result), "UTF-8")
         return file
+      case FileType.PACKAGE_WITH_TITLEDATA:
+        ObjectNode result = GokbExporter.extractPackage(enrichment, true)
+        def file = new File(enrichment.enrichmentFolder + ".packageWithTitleData.json")
+        file.write(JSON_OBJECT_MAPPER.writerWithDefaultPrettyPrinter().writeValueAsString(result), "UTF-8")
+        return file
       case FileType.TITLES:
         return extractTitles(enrichment)
       case FileType.RAW:
@@ -55,24 +60,30 @@ class GokbExporter {
   }
 
 
-  static ObjectNode extractPackage(Enrichment enrichment) {
+  static ObjectNode extractPackage(Enrichment enrichment, boolean integrateWithTitleData) {
     ObjectNode pkg = new ObjectNode(NODE_FACTORY)
     log.debug("extracting package ...")
     pkg.set("packageHeader", extractPackageHeader(enrichment))
-    pkg.set("tipps", extractTipps(enrichment))
+    pkg.set("tipps", extractTipps(enrichment, integrateWithTitleData))
     log.debug("extracting package finished")
     pkg
   }
 
 
-  static ArrayNode extractTipps(Enrichment enrichment) {
+  static ArrayNode extractTipps(Enrichment enrichment, boolean integrateWithTitleData) {
     log.debug("extracting tipps ...")
     ArrayNode tipps = new ArrayNode(NODE_FACTORY)
     for (String recId in enrichment.dataContainer.records) {
       Record record = Record.load(enrichment.dataContainer.enrichmentFolder, enrichment.resultHash, recId,
           enrichment.dataContainer.mappingsContainer)
       if (record.isValid()){
-        ObjectNode tipp = JsonToolkit.getTippJsonFromRecord("gokb", record, FORMATTER)
+        ObjectNode tipp
+        if (integrateWithTitleData){
+          tipp = JsonToolkit.getCombinedTitleTippJsonFromRecord("gokb", record, FORMATTER)
+        }
+        else{
+          tipp = JsonToolkit.getTippJsonFromRecord("gokb", record, FORMATTER)
+        }
         tipp = postProcessIssnIsbn(tipp, record, FileType.PACKAGE)
         tipp = removeEmptyFields(tipp)
         tipp = removeEmptyIdentifiers(tipp, FileType.PACKAGE)

diff --git a/src/groovy/de/hbznrw/ygor/processing/SendPackageThreadGokb.groovy b/src/groovy/de/hbznrw/ygor/processing/SendPackageThreadGokb.groovy
@@ -18,9 +18,11 @@ class SendPackageThreadGokb extends UploadThreadGokb{
   final static Pattern INT_FROM_MESSAGE_REGEX = Pattern.compile("with (\\d+) TIPPs")
   String gokbJobId
   Map gokbStatusResponse
+  boolean integrateWithTitleData
 
   SendPackageThreadGokb(def grailsApplication, @Nonnull Enrichment enrichment, @Nonnull String uri,
-                        @Nonnull String user, @Nonnull String password, @Nonnull locale){
+                        @Nonnull String user, @Nonnull String password, @Nonnull locale,
+                        boolean integrateWithTitleData){
     this.grailsApplication = grailsApplication
     this.enrichment = enrichment
     this.uri = uri
@@ -31,12 +33,19 @@ class SendPackageThreadGokb extends UploadThreadGokb{
     result = []
     gokbStatusResponse = [:]
     this.locale = locale
+    this.integrateWithTitleData = integrateWithTitleData
   }
 
 
   @Override
   void run(){
-    def json = enrichment.getAsFile(Enrichment.FileType.PACKAGE, true)
+    def json
+    if (integrateWithTitleData){
+      json = enrichment.getAsFile(Enrichment.FileType.PACKAGE_WITH_TITLEDATA, true)
+    }
+    else{
+      json = enrichment.getAsFile(Enrichment.FileType.PACKAGE, true)
+    }
     log.info("exportFile: " + enrichment.resultHash + " -> " + uri)
     result << GokbExporter.sendText(uri, json.getText(), user, password, locale)
   }

diff --git a/src/groovy/de/hbznrw/ygor/tools/JsonToolkit.groovy b/src/groovy/de/hbznrw/ygor/tools/JsonToolkit.groovy
@@ -66,22 +66,28 @@ class JsonToolkit {
 
 
   static ObjectNode getTippJsonFromRecord(String target, Record record, YgorFormatter formatter) {
-    getJsonFromRecord("\$TIPP", target, record, formatter)
+    getJsonFromRecord(new ArrayList(Arrays.asList("\$TIPP")), target, record, formatter)
   }
 
 
   static ObjectNode getTitleJsonFromRecord(String target, Record record, YgorFormatter formatter) {
-    getJsonFromRecord("\$TITLE", target, record, formatter)
+
+    getJsonFromRecord(new ArrayList(Arrays.asList("\$TITLE")), target, record, formatter)
+  }
+
+
+  static ObjectNode getCombinedTitleTippJsonFromRecord(String target, Record record, YgorFormatter formatter) {
+    getJsonFromRecord(new ArrayList(Arrays.asList("\$TITLE", "\$TIPP")), target, record, formatter)
   }
 
 
-  private static ObjectNode getJsonFromRecord(String typeFilter, String target, Record record,
+  private static ObjectNode getJsonFromRecord(List<String> typeFilter, String target, Record record,
                                               YgorFormatter formatter) {
     ObjectNode result = MAPPER.createObjectNode()
     for (MultiField multiField in record.multiFields.values()) {
       if (multiField.keyMapping == null) {
         def value = multiField.getFirstPrioValue()
-        ArrayList concatKey = Arrays.asList(typeFilter)
+        ArrayList concatKey = new ArrayList<>(typeFilter)
         Iterator it = multiField.fields.iterator()
         if (it.hasNext()){
           concatKey.addAll(it.next().key)
@@ -92,15 +98,15 @@ class JsonToolkit {
         Set qualifiedKeys = multiField.keyMapping."${target}"
         qualifiedKeys.each { qualifiedKey ->
           ArrayList splitKey = qualifiedKey.split("\\.") as ArrayList
-          if (splitKey.size() > 1 && splitKey[0].equals(typeFilter)) {
+          if (splitKey.size() > 1 && splitKey[0] in typeFilter) {
             def value = multiField.getFirstPrioValue()
             upsertIntoJsonNode(result, splitKey, value, multiField.type, formatter,
                 multiField.keyMapping.keepIfEmpty)
           }
         }
       }
     }
-    if (typeFilter.equals("\$TITLE") && record.historyEvents.size() > 0){
+    if ("\$TITLE" in typeFilter && record.historyEvents.size() > 0){
       ArrayNode historyEvents = MAPPER.createArrayNode()
       for (HistoryEvent he in record.historyEvents){
         if (he.isValid()){
-Original file line number
+Diff line change
@@ Expand Up / @@ -37,6 +37,7 @@ class Enrichment{ @@
         JSON,
         PACKAGE,
         TITLES,
+        PACKAGE_WITH_TITLEDATA,
         RAW
       }
@@ Expand Down @@