Edit of additional checks messages, fixes #105

README.md

@@ -1,4 +1,7 @@
-# openownership-cove-bods-alpha
+# Open Ownership BODS Cove
+
+Checks data complies with the Beneficial Ownership Data Standard (BODS) versions 0.1-0.4.
+Based on: https://github.com/OpenDataServices/cove
 
 ## Dev installation
 

cove_bods/process.py

@@ -5,6 +5,7 @@
 from libcovebods.config import LibCoveBODSConfig
 from libcovebods.jsonschemavalidate import JSONSchemaValidator
 from libcovebods.additionalfields import AdditionalFields
+from libcovebods.schema_dir import schema_registry
 import libcovebods.run_tasks
 import libcovebods.data_reader
 from typing import List
@@ -23,6 +24,9 @@
 from libcoveweb2.utils import get_file_type_for_flatten_tool
 from libcoveweb2.utils import group_data_list_by
 
+from logging import getLogger
+
+logger = getLogger(__name__)
 
 def create_error_file(directory: str, name: str, data: dict):
     """Create temporary error file"""
@@ -234,7 +238,11 @@ def process(self, process_data: dict) -> dict:
             process_data["json_data_filename"], sample_mode=process_data['sample_mode']
         )
         process_data['config'] = LibCoveBODSConfig()
-        process_data['schema'] = SchemaBODS(process_data['data_reader'], process_data['config'])
+        try:
+            process_data['schema'] = SchemaBODS(process_data['data_reader'], process_data['config'])
+        except json.decoder.JSONDecodeError:
+            raise ValueError("JSON: Data parsing error")
+        logger.info("Schema version:", process_data['schema'].schema_version)
         # Save some to disk for templates
         if not os.path.exists(self.data_filename):
             save_data = {
@@ -294,13 +302,18 @@ def process(self, process_data: dict) -> dict:
 
         os.makedirs(self.output_dir, exist_ok=True)
 
+        if os.path.isdir(process_data['schema'].pkg_schema_url):
+            schema = schema_registry(process_data['schema'].pkg_schema_url).contents("urn:statement")
+        else:
+            schema = process_data['schema'].pkg_schema_url
+
         flatten_kwargs = {
             "output_name": self.output_dir,
             "root_list_path": "there-is-no-root-list-path",
             "root_id": "statementID",
             "id_name": "statementID",
             "root_is_list": True,
-            "schema": process_data['schema'].pkg_schema_url,
+            "schema": schema,
         }
 
         try:

cove_bods/templates/cove_bods/additional_fields_table.html

@@ -0,0 +1,93 @@
+{% load i18n %}
+
+<!-- <p>{{ additional_fields }}</p> -->
+
+<table class="table">
+    <thead>
+    <tr>
+        <th class="">{% trans 'Field Name' %}</th>
+        <th class="">{% trans 'Field Path' %}</th>
+        <th class="">{% trans 'Usage Count' %}</th> 
+        <th class="">{% trans 'First 3 Values' %}</th>
+        <th class="">{% trans 'Child Fields' %}</th>
+    </tr>
+    </thead>
+    <tbody>
+    {% for full_path, info in additional_fields.items %}
+        {% if info.root_additional_field %}
+            <tr>
+                <td>
+                    {{ info.field_name }}
+                </td>
+                <td>
+                    {{ full_path }}
+                </td>
+                <td>
+                    {{ info.count }}
+                </td>
+                <td>
+                    <ul class="list-unstyled">
+                    {% for example in info.examples|slice:":3" %}
+                    <li>
+                        {{ example }}
+                    </li>
+                    {% endfor %}
+                    </ul>
+                </td>
+                <td>
+                    {% if info.additional_field_descendance %}
+                        {{info.additional_field_descendance|length}}
+                        <a data-toggle="modal" data-target="#additional-field-{{full_path|slugify}}">{% trans "(See child fields)" %}</a>
+                    {% endif %}
+                </td>
+            </tr>
+         {% endif %}
+    {% endfor %}
+    </tbody>
+</table>
+
+{% for parent_full_path, parent_info in additional_fields.items %}
+  {% if parent_info.root_additional_field and parent_info.additional_field_descendance %}
+    <div class="modal fade" id="additional-field-{{parent_full_path|slugify}}" tabindex="-1" role="dialog">
+      <div class="modal-dialog modal-lg" role="document">
+        <div class="modal-content">
+          <div class="modal-header">
+            <button type="button" class="close" data-dismiss="modal" aria-label="Close"><span aria-hidden="true">&times;</span></button>
+            <h4>{% blocktrans %}Child fields of {{parent_full_path}}{% endblocktrans %} </h4>
+
+          </div>
+          <div class="modal-body">
+            <table class="table">
+               <thead>
+                 <tr>
+                   <th> {% trans 'Field' %}</th>
+                   <th> {% trans 'Path to Field' %}</th>
+                   <!-- <th> {% trans 'Usage Count' %}</th> -->
+                   <th> {% trans 'Value' %}</th>
+                 </tr>
+               </thead>
+               <tbody>
+               {% for full_path, info in parent_info.additional_field_descendance.items %}
+                 <tr>
+                    <td>{{info.field_name}}</td>
+                    <td>{{info.path}}</td>
+                    <!-- <td>{{info.count}}</td> -->
+                    <!-- <td>
+                      {% for example in info.examples %}
+                        {{example}} <br />
+                      {% endfor %}
+                    </td> -->
+                    <td>
+                        {{ info.examples|first }}
+                    </td>
+                 </tr>
+               {% endfor %}
+               </tbody>
+            </table>
+          </div>
+        </div>
+      </div>
+    </div>
+  {% endif %}
+{% endfor %}
+

cove_bods/templates/cove_bods/explore.html

@@ -142,7 +142,7 @@ <h4 class="panel-title">
 
     <div class="row">
 
-        {% if additional_fields_count %}
+        {% if additional_fields_count or any_additional_fields_exist %}
 
               <a name="additional-fields" class="anchor"></a>
               <div class="panel panel-danger">
@@ -152,7 +152,7 @@ <h4 class="panel-title">
                   </h4>
                 </div>
                 <div id="additionalFieldsTable" class="collapse in panel-body">
-                  {% include "additional_fields_table.html" %}
+                  {% include "cove_bods/additional_fields_table.html" %}
                 </div>
               </div>
         {% else %}

cove_project/urls.py

@@ -1,12 +1,11 @@
-from django.conf.urls import url
 from django.conf.urls.static import static
 from django.conf import settings
 from libcoveweb2.urls import urlpatterns
 import cove_bods.views
 from django.urls import re_path
 
 urlpatterns += [re_path(r"^$", cove_bods.views.NewInput.as_view(), name="index")]
-urlpatterns += [url(r'^data/(.+)$', cove_bods.views.ExploreBODSView.as_view(), name='explore')]
+urlpatterns += [re_path(r'^data/(.+)$', cove_bods.views.ExploreBODSView.as_view(), name='explore')]
 
 urlpatterns += static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT)
 

requirements.in

@@ -1,6 +1,6 @@
 dealer
 sentry-sdk
-Django>3.2,<3.3
+Django>5.0,<5.1
 jsonschema
 libcovebods>=0.15.0
 libcoveweb2>=0.1.0

requirements.txt

@@ -8,8 +8,6 @@ amqp==5.1.1
     # via kombu
 asgiref==3.7.2
     # via django
-async-timeout==4.0.2
-    # via redis
 attrs==23.1.0
     # via jsonschema
 backports-datetime-fromisoformat==2.0.0
@@ -46,7 +44,7 @@ dealer==2.1.0
     # via -r requirements.in
 defusedxml==0.7.1
     # via odfpy
-django==3.2.19
+django==5.0.7
     # via
     #   -r requirements.in
     #   django-bootstrap3
@@ -59,7 +57,8 @@ django-environ==0.10.0
     # via libcoveweb2
 et-xmlfile==1.1.0
     # via openpyxl
-flattentool==0.20.1
+#flattentool==0.20.1
+flattentool @ git+https://github.com/OpenDataServices/flatten-tool.git@handle_bods_0.4
     # via -r requirements.in
 gunicorn==20.1.0
     # via -r requirements.in
@@ -71,17 +70,20 @@ ijson==3.2.0.post0
     #   libcovebods
 jsonref==1.1.0
     # via flattentool
-jsonschema==4.9.1
+jsonschema==4.23.0
     # via
     #   -r requirements.in
     #   libcovebods
 kombu==5.2.4
     # via celery
-libcove2==0.1.0
+#libcove2==0.1.0
+libcove2 @ git+https://github.com/OpenDataServices/lib-cove-2.git@handle_new_jsonschema
     # via libcovebods
-libcovebods==0.15.0
+#libcovebods==0.15.0
+libcovebods @ git+https://github.com/openownership/lib-cove-bods.git@add_bods_0.4
     # via -r requirements.in
-libcoveweb2==0.1.0
+#libcoveweb2==0.1.0
+libcoveweb2 @ git+https://github.com/OpenDataServices/lib-cove-web-2.git@handle_bods_0.4
     # via -r requirements.in
 lxml==4.9.2
     # via flattentool
@@ -106,7 +108,6 @@ python-dateutil==2.8.2
 pytz==2023.3
     # via
     #   celery
-    #   django
     #   flattentool
     #   libcovebods
 redis==4.5.5
@@ -121,7 +122,7 @@ rfc3987==1.3.8
     # via libcovebods
 schema==0.7.5
     # via flattentool
-sentry-sdk==1.24.0
+sentry-sdk==2.14.0
     # via
     #   -r requirements.in
     #   libcoveweb2

requirements_dev.txt

@@ -16,10 +16,6 @@ async-generator==1.10
     # via
     #   trio
     #   trio-websocket
-async-timeout==4.0.2
-    # via
-    #   -r requirements.txt
-    #   redis
 attrs==23.1.0
     # via
     #   -r requirements.txt
@@ -89,7 +85,7 @@ defusedxml==0.7.1
     # via
     #   -r requirements.txt
     #   odfpy
-django==3.2.19
+django==5.0.7
     # via
     #   -r requirements.txt
     #   django-bootstrap3
@@ -130,19 +126,21 @@ jsonref==1.1.0
     # via
     #   -r requirements.txt
     #   flattentool
-jsonschema==4.9.1
+jsonschema==4.23.0
     # via
     #   -r requirements.txt
     #   libcovebods
 kombu==5.2.4
     # via
     #   -r requirements.txt
     #   celery
-libcove2==0.1.0
+#libcove2==0.1.0
+libcove2 @ git+https://github.com/OpenDataServices/lib-cove-2.git@handle_new_jsonschema
     # via
     #   -r requirements.txt
     #   libcovebods
-libcovebods==0.15.0
+#libcovebods==0.15.0
+libcovebods @ git+https://github.com/openownership/lib-cove-bods.git@add_bods_0.4
     # via -r requirements.txt
 libcoveweb2==0.1.0
     # via -r requirements.txt
@@ -213,7 +211,6 @@ pytz==2023.3
     # via
     #   -r requirements.txt
     #   celery
-    #   django
     #   flattentool
     #   libcovebods
 redis==4.5.5
@@ -239,7 +236,7 @@ schema==0.7.5
     #   flattentool
 selenium==4.8.0
     # via -r requirements_dev.in
-sentry-sdk==1.24.0
+sentry-sdk==2.14.0
     # via
     #   -r requirements.txt
     #   libcoveweb2