Fix ground truth issues (#2534)

app/grandchallenge/reader_studies/forms.py

@@ -12,7 +12,6 @@
     Layout,
     Submit,
 )
-from django.contrib.auth import get_user_model
 from django.core.exceptions import ObjectDoesNotExist, ValidationError
 from django.forms import (
     BooleanField,
@@ -49,7 +48,6 @@
 from grandchallenge.reader_studies.models import (
     ANSWER_TYPE_TO_INTERFACE_KIND_MAP,
     CASE_TEXT_SCHEMA,
-    Answer,
     CategoricalOption,
     Question,
     ReaderStudy,
@@ -405,21 +403,6 @@ def add_or_remove_user(self, *, obj):
         permission_request.save()
 
 
-class AnswersRemoveForm(Form):
-    user = ModelChoiceField(
-        queryset=get_user_model().objects.all().order_by("username"),
-        required=True,
-    )
-
-    def remove_answers(self, *, reader_study):
-        user = self.cleaned_data["user"]
-        Answer.objects.filter(
-            question__reader_study=reader_study,
-            creator=user,
-            is_ground_truth=False,
-        ).delete()
-
-
 class ReaderStudyPermissionRequestUpdateForm(PermissionRequestUpdateForm):
     class Meta(PermissionRequestUpdateForm.Meta):
         model = ReaderStudyPermissionRequest
@@ -451,14 +434,7 @@ def clean_ground_truth(self):
         headers = rdr.fieldnames
         if sorted(
             filter(lambda x: not x.endswith("__explanation"), headers)
-        ) != sorted(
-            ["case"]
-            + list(
-                self.reader_study.questions.values_list(
-                    "question_text", flat=True
-                )
-            )
-        ):
+        ) != sorted(self.reader_study.ground_truth_file_headers):
             raise ValidationError(
                 f"Fields provided do not match with reader study. Fields should "
                 f"be: {','.join(self.reader_study.ground_truth_file_headers)}"

app/grandchallenge/reader_studies/models.py

@@ -506,6 +506,8 @@ def add_ground_truth(self, *, data, user):  # noqa: C901
                     continue
                 question = self.questions.get(question_text=key)
                 _answer = json.loads(gt[key])
+                if _answer is None and question.required is False:
+                    continue
                 if question.answer_type == Question.AnswerType.CHOICE:
                     try:
                         option = question.options.get(title=_answer)
@@ -602,18 +604,28 @@ def get_progress_for_user(self, user):
             "diff": questions - hangings,
         }
 
+    @cached_property
+    def questions_with_ground__truth(self):
+        return self.questions.annotate(
+            gt_count=Count("answer", filter=Q(answer__is_ground_truth=True))
+        ).filter(gt_count__gte=1)
+
     def score_for_user(self, user):
         """Returns the average and total score for answers given by ``user``."""
+
         return Answer.objects.filter(
-            creator=user, question__reader_study=self, is_ground_truth=False
+            creator=user,
+            question__in=self.questions_with_ground__truth,
+            is_ground_truth=False,
         ).aggregate(Sum("score"), Avg("score"))
 
     @cached_property
     def scores_by_user(self):
         """The average and total scores for this ``ReaderStudy`` grouped by user."""
         return (
             Answer.objects.filter(
-                question__reader_study=self, is_ground_truth=False
+                question__in=self.questions_with_ground__truth,
+                is_ground_truth=False,
             )
             .order_by("creator_id")
             .values("creator__username")
@@ -1341,6 +1353,11 @@ def assign_permissions(self):
             self.question.reader_study.editors_group,
             self,
         )
+        assign_perm(
+            f"delete_{self._meta.model_name}",
+            self.question.reader_study.editors_group,
+            self,
+        )
         assign_perm(f"view_{self._meta.model_name}", self.creator, self)
         assign_perm(f"change_{self._meta.model_name}", self.creator, self)
 

app/grandchallenge/reader_studies/static/reader_studies/js/remove_answers.js

@@ -0,0 +1,26 @@
+let user;
+
+$(document).ready(() => {
+    $('#warningModal').on('show.bs.modal', function (event) {
+        const button = $(event.relatedTarget);
+        const modal = $(this);
+        modal.find(".warning-text").text(button.data("warning"));
+        modal.find(".modal-action").text(button.data("action"));
+        $("#warningModalLabel").text(button.data("title"));
+        user = button.data("user");
+    });
+    $('#proceed').on('click', (e) => {
+        const target = $(e.currentTarget);
+        htmx.ajax(
+            "DELETE",
+            `${target.data("baseurl")}remove-answers/${user}/`,
+            {
+                values: {csrfmiddlewaretoken: target.data("csrf")},
+                headers: {
+                    'X-CSRFToken': target.data("csrf"),
+                    'Content-Type': 'application/json'
+                },
+            }
+        )
+    });
+});

app/grandchallenge/reader_studies/templates/reader_studies/readerstudy_detail.html

@@ -295,35 +295,42 @@ <h2>Ground Truth</h2>
                 </p>
                 <p>
                     The first row of the CSV file must contain the headings
-                    <code>images</code>, followed by the question text for
-                    each of the questions in the reader study.
+                    <code>case</code>, followed by the question text for
+                    each of the questions in the reader study (excluding those
+                    of type <code>heading</code>).
                 </p>
                 <p>
-                    The consecutive lines contain the images per hanging. For
-                    multiple images separate the image names with a <code>;</code>.
+                    The consecutive lines contain the id per case.
                     Then include the answer for the question
                     corresponding to the question text defined in the header for that
                     column. All answers must be enclosed in single quotes. Strings
                     must be enclosed in both single and double quotes, i.e.
                     <code>'"answer"'</code>. For choice type questions, the
-                    options text(s) must be provided as a string.
+                    options text(s) must be provided as a string. For optional
+                    questions, use <code>null</code> if you do not widh to
+                    provide ground truth for that question.
                 </p>
                 <p>
                     An example of the first two lines of the csv for this reader study is:
                     <code>
                         {{ example_ground_truth|linebreaks }}
                     </code>
                 </p>
-                <p>
-                    <a class="btn btn-primary"
-                       href="{% url 'reader-studies:add-ground-truth' slug=object.slug %}">
-                        <i class="fas fa-upload"></i> Upload Ground Truth
-                    </a>
-                    <a class="btn btn-secondary"
-                       href="{% url 'reader-studies:example-ground-truth' slug=object.slug %}">
-                        <i class="fas fa-download"></i> Download Example CSV
-                    </a>
-                </p>
+                <div class="d-flex justify-content-between">
+                    <div>
+                        <a class="btn btn-primary"
+                           href="{% url 'reader-studies:add-ground-truth' slug=object.slug %}">
+                            <i class="fas fa-upload"></i> Upload Ground Truth
+                        </a>
+                        <a class="btn btn-secondary"
+                           href="{% url 'reader-studies:example-ground-truth' slug=object.slug %}">
+                            <i class="fas fa-download"></i> Download Example CSV
+                        </a>
+                    </div>
+                    <button class="btn btn-danger" hx-delete="{% url 'reader-studies:ground-truth-remove' slug=object.slug %}" hx-headers='{"X-CSRFToken": "{{ csrf_token }}" }'>
+                        <i class="fas fa-trash-alt"></i> Delete ground truth
+                    </button>
+                </div>
 
             </div>
 

app/grandchallenge/reader_studies/templates/reader_studies/readerstudy_progress.html

@@ -67,7 +67,7 @@ <h2>Users Progress for {{ reader_study.title }}</h2>
                                     data-target="#warningModal"
                                     data-title="Delete Answers"
                                     data-warning="This will delete all answers for user '{{ user.obj }}'. This action cannot be undone."
-                                    data-user="{{ user.obj.id }}"
+                                    data-user="{{ user.obj }}"
                                     data-action="Delete Answers">
                                 <i class="fa fa-trash-alt"></i>&nbsp;Delete User's Answers
                             </button>
@@ -104,7 +104,7 @@ <h5 class="modal-title" id="warningModalLabel"></h5>
                             data-dismiss="modal">Cancel
                     </button>
                     <button type="button" class="btn btn-danger"
-                            id="proceed">
+                            id="proceed" data-baseurl="{{ reader_study.get_absolute_url }}" data-csrf="{{ csrf_token }}">
                         <i class="fa fa-trash"></i> <span class="modal-action"></span>
                     </button>
                 </div>
@@ -116,33 +116,6 @@ <h5 class="modal-title" id="warningModalLabel"></h5>
 
 {% block script %}
     {{ block.super }}
-    <script>
-        window.drf = {
-            csrfHeaderName: "{{ csrf_header_name|default:'X-CSRFToken' }}",
-            csrfToken: "{% if request %}{{ csrf_token }}{% endif %}"
-        };
-
-        let user;
-
-        $(document).ready(() => {
-            $('#warningModal').on('show.bs.modal', function (event) {
-                const button = $(event.relatedTarget);
-                const modal = $(this);
-                modal.find(".warning-text").text(button.data("warning"));
-                modal.find(".modal-action").text(button.data("action"));
-                $("#warningModalLabel").text(button.data("title"));
-                user = button.data("user");
-            });
-            $('#proceed').on('click', () => {
-                $.post({
-                    url: "{% url 'reader-studies:answers-remove' slug=reader_study.slug %}",
-                    data: {user: user, csrfmiddlewaretoken: window.drf.csrfToken},
-                    success: () => {
-                        window.location.replace(window.location.href);
-                    }
-                })
-            });
-        });
-    </script>
+    <script src="{% static 'reader_studies/js/remove_answers.js' %}"></script>
     <script src="{% static "rest_framework/js/csrf.js" %}"></script>
 {% endblock %}

app/grandchallenge/reader_studies/templatetags/reader_study_tags.py

@@ -9,7 +9,11 @@
 def get_ground_truth(reader_study, display_set, question):
     """Get the ground truth value for the display_set/question combination in reader_study."""
     ground_truths = reader_study.statistics["ground_truths"]
-    return ground_truths[display_set][question]
+    try:
+        return ground_truths[display_set][question]
+    except KeyError:
+        # No gt exists for this display set or question yet
+        return ""
 
 
 @register.filter

app/grandchallenge/reader_studies/urls.py

@@ -5,7 +5,8 @@
     AddGroundTruthToReaderStudy,
     AddImagesToReaderStudy,
     AddQuestionToReaderStudy,
-    AnswersRemove,
+    AnswersRemoveForUser,
+    AnswersRemoveGroundTruth,
     DisplaySetDetail,
     DisplaySetUpdate,
     EditorsUpdate,
@@ -71,10 +72,15 @@
     ),
     path("<slug>/copy/", ReaderStudyCopy.as_view(), name="copy"),
     path(
-        "<slug>/remove-answers/",
-        AnswersRemove.as_view(),
+        "<slug>/remove-answers/<username>/",
+        AnswersRemoveForUser.as_view(),
         name="answers-remove",
     ),
+    path(
+        "<slug>/remove-ground-truth/",
+        AnswersRemoveGroundTruth.as_view(),
+        name="ground-truth-remove",
+    ),
     path(
         "<slug>/ground-truth/add/",
         AddGroundTruthToReaderStudy.as_view(),