Merge pull request #1377 from nationalarchives/feature/awaiting-repar…

…se-report

Add report on documents needing re-parsing

ds_caselaw_editor_ui/templates/reports/awaiting_enrichment.html

@@ -4,7 +4,7 @@
   <div class="standard-text-template">
     <h1>Documents awaiting enrichment</h1>
     <p>
-      These documents have not yet been enriched with the target version of the enrichment engine, version {{ target_enrichment_version }}.
+      These documents have not yet been enriched with the latest version of the enrichment engine, version <b>{{ target_enrichment_version }}</b>, and have not recently had an enrichment attempt.
     </p>
     <p>
       There are <b>{{ documents|length|intcomma }}</b> documents waiting.

ds_caselaw_editor_ui/templates/reports/awaiting_parse.html

@@ -0,0 +1,27 @@
+{% extends "layouts/base.html" %}
+{% load i18n humanize %}
+{% block content %}
+  <div class="standard-text-template">
+    <h1>Documents awaiting parsing</h1>
+    <p>
+      These documents have not yet been parsed with the latest version of the parser, version <b>{{ target_parser_version }}</b>, and have not recently had an parsing attempt.
+    </p>
+    <p>
+      There are <b>{{ documents|length|intcomma }}</b> documents waiting.
+    </p>
+    <table class="table">
+      <tr>
+        <th>#</th>
+        <th>Document</th>
+        <th>Parsed with</th>
+        <th>Last sent for parsing</th>
+      </tr>
+      {% for document in documents %}
+        <tr>
+          <td>{{ forloop.counter }}</td>
+          {% for cell in document %}<td>{{ cell }}</td>{% endfor %}
+        </tr>
+      {% endfor %}
+    </table>
+  </div>
+{% endblock content %}

ds_caselaw_editor_ui/templates/reports/index.html

@@ -4,6 +4,9 @@
   <div class="standard-text-template">
     <h1>Reports</h1>
     <ul>
+      <li>
+        <a href="{% url 'report_awaiting_parse' %}">Awaiting parsing</a>
+      </li>
       <li>
         <a href="{% url 'report_awaiting_enrichment' %}">Awaiting enrichment</a>
       </li>

judgments/management/commands/enrich_next_in_reenrichment_queue.py

@@ -2,7 +2,6 @@
 
 from judgments.utils import api_client
 
-TARGET_ENRICHMENT_VERSION = 6
 NUMBER_TO_ENRICH = 1
 
 
@@ -11,7 +10,7 @@ class Command(BaseCommand):
 
     def handle(self, *args, **options):
         document_details_to_enrich = api_client.get_pending_enrichment_for_version(
-            TARGET_ENRICHMENT_VERSION,
+            api_client.get_highest_enrichment_version(),
         )
 
         for document_details in document_details_to_enrich[1 : NUMBER_TO_ENRICH + 1]:

judgments/tests/test_reports.py

@@ -36,3 +36,25 @@ def test_awaiting_enrichment_view(self, mock_api_client):
         assert "enrich_version_string" not in decoded_response
 
         assert response.status_code == 200
+
+    @patch("judgments.views.reports.api_client")
+    def test_awaiting_parse_view(self, mock_api_client):
+        self.client.force_login(User.objects.get_or_create(username="testuser")[0])
+
+        mock_api_client.get_pending_parse_for_version.return_value = [
+            ["uri", "parser_version_string", "minutes_since_parse_request"],
+            ["/test/123", "1.2.3", 45],
+        ]
+
+        response = self.client.get(reverse("report_awaiting_parse"))
+
+        decoded_response = response.content.decode("utf-8")
+        assert "Documents awaiting parsing" in decoded_response
+
+        assert "/test/123" in decoded_response
+        assert "1.2.3" in decoded_response
+        assert "45" in decoded_response
+
+        assert "parser_version_string" not in decoded_response
+
+        assert response.status_code == 200

judgments/urls.py

@@ -74,6 +74,11 @@
     path("labs", Labs.as_view(), name="labs"),
     # Reports
     path("reports", reports.Index.as_view(), name="reports"),
+    path(
+        "reports/awaiting-parse",
+        reports.AwaitingParse.as_view(),
+        name="report_awaiting_parse",
+    ),
     path(
         "reports/awaiting-enrichment",
         reports.AwaitingEnrichment.as_view(),

judgments/views/reports.py

@@ -14,16 +14,37 @@ def get_context_data(self, **kwargs):
         return context
 
 
+class AwaitingParse(TemplateView):
+    template_name = "reports/awaiting_parse.html"
+
+    def get_context_data(self, **kwargs):
+        context = super().get_context_data(**kwargs)
+
+        target_parser_version = api_client.get_highest_parser_version()
+
+        context["page_title"] = "Documents awaiting parsing"
+        context[
+            "target_parser_version"
+        ] = f"{target_parser_version[0]}.{target_parser_version[1]}"
+        context["documents"] = api_client.get_pending_parse_for_version(
+            target_parser_version,
+        )[1:]
+
+        return context
+
+
 class AwaitingEnrichment(TemplateView):
     template_name = "reports/awaiting_enrichment.html"
 
     def get_context_data(self, **kwargs):
         context = super().get_context_data(**kwargs)
 
-        target_enrichment_version = 6
+        target_enrichment_version = api_client.get_highest_enrichment_version()
 
         context["page_title"] = "Documents awaiting enrichment"
-        context["target_enrichment_version"] = target_enrichment_version
+        context[
+            "target_enrichment_version"
+        ] = f"{target_enrichment_version[0]}.{target_enrichment_version[1]}"
         context["documents"] = api_client.get_pending_enrichment_for_version(
             target_enrichment_version,
         )[1:]

requirements/base.txt

@@ -16,7 +16,7 @@ xmltodict~=0.13.0
 requests-toolbelt~=1.0.0
 lxml~=5.1.0
 wsgi-basic-auth~=1.1.0
-ds-caselaw-marklogic-api-client==19.1.0
+ds-caselaw-marklogic-api-client==20.0.0
 ds-caselaw-utils~=1.3.3
 rollbar
 django-stronghold==0.4.0