Merge to master: Release 2.4.2 - Multilingual & Submission compressio…

…n (#96) * fix json serializer nil exception (#79) if hash was missing @context key the json serializer throwed a nil exception * Merge pull request #87 from ontoportal-lirmm/fix/achived-submission-space-optimization Fix: Archived submission space optimization * Merge pull request #71 from ontoportal-lirmm/feature/support-multilingual-read-one-language-from-request-parameter Feature: Support multilingual - Add read one language from request parameter tests * Merge pull request #73 from ontoportal-lirmm/feature/add-request-language-to-context Feature/add request language to context * fix json serializer nil exception (#79) if hash was missing @context key the json serializer throwed a nil exception * Merge pull request #83 from ontoportal-lirmm/feature/add-request-language-to-context Feature: add request language to context follow up * Merge pull request #93 from ontoportal-lirmm/support-multi-lang-part-02 Feature: Add request all languages unit test * Merge pull request #94 from ontoportal-lirmm/feature/support-multi-langual-search Feature: Support multilangual search --------- Co-authored-by: Raimi Solorzano Niederhausen <[email protected]>
EarthPortal · Dec 5, 2023 · aa5081d · aa5081d
1 parent 7d3359b
commit aa5081d
Show file tree

Hide file tree

Showing 24 changed files with 1,681 additions and 153 deletions.
diff --git a/Gemfile b/Gemfile
@@ -19,6 +19,7 @@ gem 'rest-client'
 gem 'rsolr', '~> 1.0'
 gem 'rubyzip', '~> 1.0'
 gem 'thin'
+gem 'request_store'
 
 # Testing
 group :test do

diff --git a/Gemfile.lock b/Gemfile.lock
@@ -1,6 +1,6 @@
 GIT
   remote: https://github.com/ontoportal-lirmm/goo.git
-  revision: bd7154217438c3b9160e0e9b495c7c718b55fbf8
+  revision: 74ea47defc7f6260b045a6c6997bbe6a59c7bf62
   branch: master
   specs:
     goo (0.0.2)
@@ -36,18 +36,16 @@ GEM
       public_suffix (>= 2.0.2, < 6.0)
     ansi (1.5.0)
     ast (2.4.2)
-    base64 (0.1.1)
-    bcrypt (3.1.19)
+    bcrypt (3.1.20)
     builder (3.2.4)
     coderay (1.1.3)
     concurrent-ruby (1.2.2)
     connection_pool (2.4.1)
     cube-ruby (0.0.3)
     daemons (1.4.1)
-    date (3.3.3)
+    date (3.3.4)
     docile (1.4.0)
-    domain_name (0.5.20190701)
-      unf (>= 0.0.5, < 1.0.0)
+    domain_name (0.6.20231109)
     email_spec (2.2.2)
       htmlentities (~> 4.3.3)
       launchy (~> 2.1)
@@ -84,13 +82,13 @@ GEM
       domain_name (~> 0.5)
     i18n (0.9.5)
       concurrent-ruby (~> 1.0)
-    json (2.6.3)
-    json_pure (2.6.3)
+    json (2.7.1)
+    json_pure (2.7.1)
     language_server-protocol (3.17.0.3)
     launchy (2.5.2)
       addressable (~> 2.8)
     libxml-ruby (2.9.0)
-    logger (1.5.3)
+    logger (1.6.0)
     macaddr (1.7.2)
       systemu (~> 2.6.5)
     mail (2.8.1)
@@ -112,12 +110,12 @@ GEM
     multi_json (1.15.0)
     multipart-post (2.3.0)
     net-http-persistent (2.9.4)
-    net-imap (0.4.1)
+    net-imap (0.4.7)
       date
       net-protocol
     net-pop (0.1.2)
       net-protocol
-    net-protocol (0.2.1)
+    net-protocol (0.2.2)
       timeout
     net-smtp (0.4.0)
       net-protocol
@@ -136,20 +134,22 @@ GEM
     pry (0.14.2)
       coderay (~> 1.1)
       method_source (~> 1.0)
-    public_suffix (5.0.3)
-    racc (1.7.1)
+    public_suffix (5.0.4)
+    racc (1.7.3)
     rack (1.6.13)
     rack-test (0.8.3)
       rack (>= 1.0, < 3)
     rainbow (3.1.1)
     rake (10.5.0)
     rdf (1.0.8)
       addressable (>= 2.2)
-    redis (5.0.7)
-      redis-client (>= 0.9.0)
-    redis-client (0.17.0)
+    redis (5.0.8)
+      redis-client (>= 0.17.0)
+    redis-client (0.18.0)
       connection_pool
-    regexp_parser (2.8.2)
+    regexp_parser (2.8.3)
+    request_store (1.5.1)
+      rack (>= 1.4)
     rest-client (2.1.0)
       http-accept (>= 1.7.0, < 2.0)
       http-cookie (>= 1.0.2, < 2.0)
@@ -158,19 +158,18 @@ GEM
     rexml (3.2.6)
     rsolr (1.1.2)
       builder (>= 2.1.2)
-    rubocop (1.57.1)
-      base64 (~> 0.1.1)
+    rubocop (1.58.0)
       json (~> 2.3)
       language_server-protocol (>= 3.17.0)
       parallel (~> 1.10)
       parser (>= 3.2.2.4)
       rainbow (>= 2.2.2, < 4.0)
       regexp_parser (>= 1.8, < 3.0)
       rexml (>= 3.2.5, < 4.0)
-      rubocop-ast (>= 1.28.1, < 2.0)
+      rubocop-ast (>= 1.30.0, < 2.0)
       ruby-progressbar (~> 1.7)
       unicode-display_width (>= 2.4.0, < 3.0)
-    rubocop-ast (1.29.0)
+    rubocop-ast (1.30.0)
       parser (>= 3.2.1.0)
     ruby-progressbar (1.13.0)
     ruby2_keywords (0.0.5)
@@ -192,16 +191,15 @@ GEM
       eventmachine (~> 1.0, >= 1.0.4)
       rack (>= 1, < 3)
     thread_safe (0.3.6)
-    timeout (0.4.0)
+    timeout (0.4.1)
     tzinfo (0.3.62)
-    unf (0.1.4)
-      unf_ext
-    unf_ext (0.0.8.2)
     unicode-display_width (2.5.0)
     uuid (2.3.9)
       macaddr (~> 1.0)
 
 PLATFORMS
+  x86_64-darwin-21
+  x86_64-darwin-23
   x86_64-linux
 
 DEPENDENCIES
@@ -224,6 +222,7 @@ DEPENDENCIES
   rack (~> 1.0)
   rack-test (~> 0.6)
   rake (~> 10.0)
+  request_store
   rest-client
   rsolr (~> 1.0)
   rubocop

diff --git a/config/solr/term_search/schema.xml b/config/solr/term_search/schema.xml
@@ -128,21 +128,39 @@
     <field name="resource_id"           type="string"             indexed="true"  stored="true"  multiValued="false" required="true" />
 
     <!-- searchable fields -->
-    <field name="prefLabel"             type="text_general"       indexed="true"  stored="true"  multiValued="false" />
-    <field name="prefLabelExact"        type="string_ci"          indexed="true"  stored="true"  multiValued="false" />
-    <field name="prefLabelSuggest"      type="text_suggest"       indexed="true"  stored="true"  omitNorms="true" />
-    <field name="prefLabelSuggestEdge"  type="text_suggest_edge"  indexed="true"  stored="true" />
-    <field name="prefLabelSuggestNgram" type="text_suggest_ngram" indexed="true"  stored="true"  omitNorms="true" />
+    <!-- all languages fields -->
+    <field name="prefLabel"             type="text_general"       indexed="true"  stored="true"  multiValued="true" />
+    <field name="prefLabelExact"        type="string_ci"          indexed="true"  stored="true"  multiValued="true" />
+    <field name="prefLabelSuggest"      type="text_suggest"       indexed="true"  stored="true"  multiValued="true"  omitNorms="true" />
+    <field name="prefLabelSuggestEdge"  type="text_suggest_edge"  indexed="true"  stored="true"  multiValued="true"  />
+    <field name="prefLabelSuggestNgram" type="text_suggest_ngram" indexed="true"  stored="true"  multiValued="true"  omitNorms="true" />
+
+    <!--  language specific fields  (e.g prefLabel_en or prefLabel_fr) -->
+    <dynamicField name="prefLabel_*" type="text_general" indexed="true" stored="true" multiValued="true" />
+    <dynamicField name="prefLabelExact_*" type="string_ci" indexed="true" stored="true" multiValued="true" />
+    <dynamicField name="prefLabelSuggest_*" type="text_suggest" indexed="true" stored="true" omitNorms="true" multiValued="true" />
+    <dynamicField name="prefLabelSuggestEdge_*" type="text_suggest_edge" indexed="true" stored="true" multiValued="true" />
+    <dynamicField name="prefLabelSuggestNgram_*" type="text_suggest_ngram" indexed="true" stored="true" omitNorms="true" multiValued="true" />
+
 
     <field name="synonym"               type="text_general"       indexed="true"  stored="true"  multiValued="true" />
     <field name="synonymExact"          type="string_ci"          indexed="true"  stored="true"  multiValued="true" />
     <field name="synonymSuggest"        type="text_suggest"       indexed="true"  stored="true"  omitNorms="true" multiValued="true" />
     <field name="synonymSuggestEdge"    type="text_suggest_edge"  indexed="true"  stored="true"  multiValued="true" />
     <field name="synonymSuggestNgram"   type="text_suggest_ngram" indexed="true"  stored="true"  omitNorms="true" multiValued="true" />
 
+    <dynamicField name="synonym_*" type="text_general" indexed="true" stored="true" multiValued="true" />
+    <dynamicField name="synonymExact_*" type="string_ci" indexed="true" stored="true" multiValued="true" />
+    <dynamicField name="synonymSuggest_*" type="text_suggest" indexed="true" stored="true" omitNorms="true" multiValued="true" />
+    <dynamicField name="synonymSuggestEdge_*" type="text_suggest_edge" indexed="true" stored="true" multiValued="true" />
+    <dynamicField name="synonymSuggestNgram_*" type="text_suggest_ngram" indexed="true" stored="true" omitNorms="true" multiValued="true" />
+
+
     <field name="notation"              type="text_general"       indexed="true"  stored="true"  multiValued="false" />
 
-    <field name="definition"            type="string"             indexed="true"  stored="true"  multiValued="true" />
+    <field name="definition" type="string" indexed="true" stored="true" multiValued="true" />
+    <dynamicField name="definition_*" type="string" indexed="true" stored="true" multiValued="true" />
+
     <field name="submissionAcronym"     type="string"             indexed="true"  stored="true"  multiValued="false" />
     <field name="parents"               type="string"             indexed="true"  stored="true"  multiValued="true" />
     <field name="ontologyType"          type="ontologyType"       indexed="true"  stored="true"  multiValued="false" />
@@ -251,6 +269,17 @@
     <copyField source="synonym" dest="synonymSuggestEdge" />
     <copyField source="synonym" dest="synonymSuggestNgram" />
 
+    <copyField source="prefLabel_*" dest="prefLabelExact_*" />
+    <copyField source="prefLabel_*" dest="prefLabelSuggest_*" />
+    <copyField source="prefLabel_*" dest="prefLabelSuggestEdge_*" />
+    <copyField source="prefLabel_*" dest="prefLabelSuggestNgram_*" />
+
+
+    <copyField source="synonym_*" dest="synonymExact_*" />
+    <copyField source="synonym_*" dest="synonymSuggest_*" />
+    <copyField source="synonym_*" dest="synonymSuggestEdge_*" />
+    <copyField source="synonym_*" dest="synonymSuggestNgram_*" />
+
     <copyField source="notation" dest="_text_"/>
 
 

diff --git a/lib/ontologies_linked_data/concerns/mappings/mapping_creator.rb b/lib/ontologies_linked_data/concerns/mappings/mapping_creator.rb
@@ -81,8 +81,8 @@ def create_mapping_process(mapping_process_hash, source_uri, object_uri, user)
           process.relation = relations_array
           process.creator = user
 
-          process.subject_source_id = RDF::URI.new(source_uri || mapping_process_hash[:subject_source_id])
-          process.object_source_id = RDF::URI.new(object_uri || mapping_process_hash[:object_source_id])
+          process.subject_source_id = create_uri(source_uri || mapping_process_hash[:subject_source_id])
+          process.object_source_id = create_uri(object_uri || mapping_process_hash[:object_source_id])
           process.date = mapping_process_hash[:date] ? DateTime.parse(mapping_process_hash[:date]) : DateTime.now
           process_fields = %i[source source_name comment name source_contact_info]
           process_fields.each do |att|
@@ -92,6 +92,9 @@ def create_mapping_process(mapping_process_hash, source_uri, object_uri, user)
         end
 
         private
+        def create_uri(value)
+          RDF::URI.new(value) unless value.nil?
+        end
 
         def save_rest_mapping(classes, process)
           LinkedData::Mappings.create_rest_mapping(classes, process)