Buttons to open readerstudy at specific display set (#2460)

app/config/settings.py

@@ -1072,6 +1072,7 @@
 WORKSTATIONS_ARCHIVE_ITEM_QUERY_PARAM = "archiveItem"
 WORKSTATIONS_CONFIG_QUERY_PARAM = "config"
 WORKSTATIONS_USER_QUERY_PARAM = "viewAsUser"
+WORKSTATIONS_DISPLAY_SET_QUERY_PARAM = "displaySet"
 # The name of the network that the workstations will be attached to
 WORKSTATIONS_NETWORK_NAME = os.environ.get(
     "WORKSTATIONS_NETWORK_NAME", "grand-challengeorg_workstations"
@@ -1119,6 +1120,10 @@
 READER_STUDY_VIEW_AS_USER_FEATURE = strtobool(
     os.environ.get("READER_STUDY_VIEW_AS_USER_FEATURE", "False")
 )
+# Feature flag for reader study display set view
+READER_STUDY_DISPLAY_SET_VIEW_FEATURE = strtobool(
+    os.environ.get("READER_STUDY_DISPLAY_SET_VIEW_FEATURE", "True")
+)
 
 CELERY_BEAT_SCHEDULE = {
     "ping_google": {

app/grandchallenge/reader_studies/models.py

@@ -643,16 +643,18 @@ def statistics(self):
         )
 
         scores_by_case = (
-            Answer.objects.filter(
-                question__reader_study=self, is_ground_truth=False
-            )
-            .order_by("display_set")
-            .values("display_set_id")
+            DisplaySet.objects.filter(reader_study=self)
+            .select_related("reader_study__workstation__config")
             .annotate(
-                Sum("score"),
-                Avg("score"),
+                sum=Sum(
+                    "answers__score", filter=Q(answers__is_ground_truth=False)
+                ),
+                avg=Avg(
+                    "answers__score", filter=Q(answers__is_ground_truth=False)
+                ),
             )
-            .order_by("score__avg")
+            .order_by("avg")
+            .all()
         )
 
         options = {}

app/grandchallenge/reader_studies/templates/reader_studies/readerstudy_display_sets_row.html

@@ -53,15 +53,23 @@ <h5 class="mb-0">
                     {{ content.title }}
                   {% endif %}
                 </td>
-                <td>
-                  {% if content.selected_image %}
-                    <a href="{% url 'workstations:workstation-session-create' slug=reader_study.workstation.slug %}?{% workstation_query image=content.selected_image config=reader_study.workstation_config %}">
-                      <span class="badge badge-primary">
-                          <i class="fa fa-eye"></i> View Case
-                      </span>
-                    </a>
-                  {% endif %}
-                </td>
+                {% if reader_study_display_set_view_feature %}
+                    <td>
+                        <a href="{% url 'workstations:workstation-session-create' slug=reader_study.workstation.slug %}?{% workstation_query display_set=object %}">
+                          <span class="badge badge-primary">
+                              <i class="fa fa-eye"></i> View Case
+                          </span>
+                        </a>
+                    </td>
+                {% elif  content.selected_image %}
+                    <td>
+                        <a href="{% url 'workstations:workstation-session-create' slug=reader_study.workstation.slug %}?{% workstation_query image=content.selected_image config=reader_study.workstation_config %}">
+                          <span class="badge badge-primary">
+                              <i class="fa fa-eye"></i> View Image
+                          </span>
+                        </a>
+                    </td>
+                {% endif %}
               </tr>
             {% endfor %}
             <tr>

app/grandchallenge/reader_studies/templates/reader_studies/readerstudy_statistics.html

@@ -27,18 +27,30 @@ <h2>Statistics per case</h2>
                     {% for question in object.statistics.questions %}
                         <th>{{ question }} (GT)</th>
                     {% endfor %}
+                    {% if reader_study_display_set_view_feature %}
+                        <th>View case</th>
+                    {% endif %}
                 </tr>
             </thead>
             <tbody>
                 {% for entry in object.statistics.scores_by_case %}
                     <tr>
-                        <td data-order="{{ entry.display_set_id }}">{{ entry.display_set_id }}</td>
-                        <td data-order="{{ entry.score__sum|stringformat:'020f' }}">{{ entry.score__sum }} / {{ object.statistics.max_score_cases }}</td>
-                        <td data-order="{{ entry.score__avg|stringformat:'020f' }}">{{ entry.score__avg|floatformat:4 }}</td>
+                        <td data-order="{{ entry.id }}">{{ entry.id }}</td>
+                        <td data-order="{{ entry.sum|stringformat:'020f' }}">{{ entry.sum }} / {{ object.statistics.max_score_cases }}</td>
+                        <td data-order="{{ entry.avg|stringformat:'020f' }}">{{ entry.avg|floatformat:4 }}</td>
                         {% for question in object.statistics.questions %}
-                            {% get_ground_truth object entry.display_set_id question as ground_truth %}
+                            {% get_ground_truth object entry.id question as ground_truth %}
                             <td data-order="{{ ground_truth }}">{{ ground_truth }}</td>
                         {% endfor %}
+                    {% if reader_study_display_set_view_feature %}
+                        <td data-order="{{ entry.id }}">
+                            <a href="{% url 'workstations:workstation-session-create' slug=object.workstation.slug %}?{% workstation_query display_set=entry %}">
+                                <span class="badge badge-primary">
+                                    <i class="fa fa-eye"></i>
+                                </span>
+                            </a>
+                        </td>
+                    {% endif %}
                     </tr>
                 {% endfor %}
             </tbody>

app/grandchallenge/reader_studies/views.py

@@ -352,6 +352,15 @@ class ReaderStudyStatistics(
     # TODO: this view also contains the ground truth answer values.
     # If the permission is changed to 'read', we need to filter these values out.
 
+    def get_context_data(self, *args, **kwargs):
+        context = super().get_context_data(*args, **kwargs)
+        context.update(
+            {
+                "reader_study_display_set_view_feature": settings.READER_STUDY_DISPLAY_SET_VIEW_FEATURE,
+            }
+        )
+        return context
+
 
 class ReaderStudyDisplaySetList(
     LoginRequiredMixin, ObjectPermissionRequiredMixin, PaginatedTableListView
@@ -379,7 +388,12 @@ def get_permission_object(self):
 
     def get_context_data(self, **kwargs):
         context = super().get_context_data(**kwargs)
-        context.update({"reader_study": self.reader_study})
+        context.update(
+            {
+                "reader_study": self.reader_study,
+                "reader_study_display_set_view_feature": settings.READER_STUDY_DISPLAY_SET_VIEW_FEATURE,
+            }
+        )
         return context
 
     def get_queryset(self):

app/grandchallenge/workstations/templatetags/workstations.py

@@ -6,14 +6,15 @@
 
 
 @register.simple_tag()
-def workstation_query(
+def workstation_query(  # noqa: C901
     image=None,
     overlay=None,
     reader_study=None,
     algorithm_job=None,
     archive_item=None,
     config=None,
     user=None,
+    display_set=None,
 ):
     """
     Generate the workstation query string.
@@ -34,6 +35,8 @@ def workstation_query(
                     )
                 }
             )
+    elif display_set:
+        query = {settings.WORKSTATIONS_DISPLAY_SET_QUERY_PARAM: display_set.pk}
     elif reader_study:
         query = {
             settings.WORKSTATIONS_READY_STUDY_QUERY_PARAM: reader_study.pk
@@ -68,5 +71,11 @@ def workstation_query(
                 settings.WORKSTATIONS_CONFIG_QUERY_PARAM: reader_study.workstation_config.pk
             }
         )
+    elif display_set and display_set.reader_study.workstation_config:
+        query.update(
+            {
+                settings.WORKSTATIONS_CONFIG_QUERY_PARAM: display_set.reader_study.workstation_config.pk
+            }
+        )
 
     return urlencode(query)

app/tests/reader_studies_tests/test_models.py

@@ -267,9 +267,9 @@ def test_statistics(reader_study_with_gt, settings):
     assert len(scores) == rs.display_sets.count()
     ds = set(rs.display_sets.values_list("pk", flat=True))
     for score in scores:
-        ds -= {score["display_set_id"]}
-        assert score["score__sum"] == 3.0
-        assert score["score__avg"] == 1.0
+        ds -= {score.id}
+        assert score.sum == 3.0
+        assert score.avg == 1.0
     assert ds == set()
 
     with capture_on_commit_callbacks(execute=True):

app/tests/workstations_tests/test_templatetags.py

@@ -4,41 +4,19 @@
 from tests.algorithms_tests.factories import AlgorithmJobFactory
 from tests.archives_tests.factories import ArchiveItemFactory
 from tests.factories import ImageFactory, UserFactory, WorkstationConfigFactory
-from tests.reader_studies_tests.factories import ReaderStudyFactory
+from tests.reader_studies_tests.factories import (
+    DisplaySetFactory,
+    ReaderStudyFactory,
+)
 
 
-def test_workstation_query(settings):
-    image, overlay = ImageFactory.build_batch(2)
+def test_workstation_query_for_reader_studies(settings):
     reader_study = ReaderStudyFactory.build(
         workstation_config=WorkstationConfigFactory.build()
     )
-    algorithm_job = AlgorithmJobFactory.build()
     config = WorkstationConfigFactory.build()
-    archive_item = ArchiveItemFactory.build()
     user = UserFactory.build()
 
-    qs = workstation_query(image=image)
-    assert "&" not in qs
-    assert f"{settings.WORKSTATIONS_BASE_IMAGE_QUERY_PARAM}={image.pk}" in qs
-    assert (
-        f"{settings.WORKSTATIONS_OVERLAY_QUERY_PARAM}={overlay.pk}" not in qs
-    )
-    assert f"{settings.WORKSTATIONS_CONFIG_QUERY_PARAM}={config.pk}" not in qs
-
-    qs = workstation_query(image=image, overlay=overlay)
-    assert "&" in qs
-    assert f"{settings.WORKSTATIONS_BASE_IMAGE_QUERY_PARAM}={image.pk}" in qs
-    assert f"{settings.WORKSTATIONS_OVERLAY_QUERY_PARAM}={overlay.pk}" in qs
-    assert f"{settings.WORKSTATIONS_CONFIG_QUERY_PARAM}={config.pk}" not in qs
-
-    qs = workstation_query(image=image, config=config)
-    assert "&" in qs
-    assert f"{settings.WORKSTATIONS_BASE_IMAGE_QUERY_PARAM}={image.pk}" in qs
-    assert (
-        f"{settings.WORKSTATIONS_OVERLAY_QUERY_PARAM}={overlay.pk}" not in qs
-    )
-    assert f"{settings.WORKSTATIONS_CONFIG_QUERY_PARAM}={config.pk}" in qs
-
     qs = workstation_query(reader_study=reader_study, user=user)
     assert "&" in qs
     assert (
@@ -85,21 +63,53 @@ def test_workstation_query(settings):
     )
     assert f"{settings.WORKSTATIONS_CONFIG_QUERY_PARAM}" not in qs
 
-    qs = workstation_query(algorithm_job=algorithm_job)
-    assert "&" not in qs
+
+def test_workstation_query_for_display_sets(settings):
+    reader_study = ReaderStudyFactory.build(
+        workstation_config=WorkstationConfigFactory.build()
+    )
+    config = WorkstationConfigFactory.build()
+    display_set = DisplaySetFactory.build(reader_study=reader_study)
+
+    qs = workstation_query(display_set=display_set)
+    assert "&" in qs
     assert (
-        f"{settings.WORKSTATIONS_ALGORITHM_JOB_QUERY_PARAM}={algorithm_job.pk}"
+        f"{settings.WORKSTATIONS_DISPLAY_SET_QUERY_PARAM}={display_set.pk}"
+        in qs
+    )
+    assert (
+        f"{settings.WORKSTATIONS_CONFIG_QUERY_PARAM}={reader_study.workstation_config.pk}"
         in qs
     )
+    assert f"{settings.WORKSTATIONS_CONFIG_QUERY_PARAM}={config.pk}" not in qs
 
-    qs = workstation_query(algorithm_job=algorithm_job, config=config)
+    qs = workstation_query(display_set=display_set, config=config)
     assert "&" in qs
     assert (
-        f"{settings.WORKSTATIONS_ALGORITHM_JOB_QUERY_PARAM}={algorithm_job.pk}"
+        f"{settings.WORKSTATIONS_DISPLAY_SET_QUERY_PARAM}={display_set.pk}"
         in qs
     )
+    assert (
+        f"{settings.WORKSTATIONS_CONFIG_QUERY_PARAM}={reader_study.workstation_config.pk}"
+        not in qs
+    )
     assert f"{settings.WORKSTATIONS_CONFIG_QUERY_PARAM}={config.pk}" in qs
 
+    reader_study.workstation_config = None
+
+    qs = workstation_query(display_set=display_set)
+    assert "&" not in qs
+    assert (
+        f"{settings.WORKSTATIONS_DISPLAY_SET_QUERY_PARAM}={display_set.pk}"
+        in qs
+    )
+    assert f"{settings.WORKSTATIONS_CONFIG_QUERY_PARAM}" not in qs
+
+
+def test_workstation_query_for_archive_items(settings):
+    config = WorkstationConfigFactory.build()
+    archive_item = ArchiveItemFactory.build()
+
     qs = workstation_query(archive_item=archive_item, config=config)
     assert "&" in qs
     assert (
@@ -115,3 +125,50 @@ def test_workstation_query(settings):
         in qs
     )
     assert f"{settings.WORKSTATIONS_CONFIG_QUERY_PARAM}" not in qs
+
+
+def test_workstation_query_for_algorithms(settings):
+    algorithm_job = AlgorithmJobFactory.build()
+    config = WorkstationConfigFactory.build()
+
+    qs = workstation_query(algorithm_job=algorithm_job)
+    assert "&" not in qs
+    assert (
+        f"{settings.WORKSTATIONS_ALGORITHM_JOB_QUERY_PARAM}={algorithm_job.pk}"
+        in qs
+    )
+
+    qs = workstation_query(algorithm_job=algorithm_job, config=config)
+    assert "&" in qs
+    assert (
+        f"{settings.WORKSTATIONS_ALGORITHM_JOB_QUERY_PARAM}={algorithm_job.pk}"
+        in qs
+    )
+    assert f"{settings.WORKSTATIONS_CONFIG_QUERY_PARAM}={config.pk}" in qs
+
+
+def test_workstation_query_for_images(settings):
+    image, overlay = ImageFactory.build_batch(2)
+    config = WorkstationConfigFactory.build()
+
+    qs = workstation_query(image=image)
+    assert "&" not in qs
+    assert f"{settings.WORKSTATIONS_BASE_IMAGE_QUERY_PARAM}={image.pk}" in qs
+    assert (
+        f"{settings.WORKSTATIONS_OVERLAY_QUERY_PARAM}={overlay.pk}" not in qs
+    )
+    assert f"{settings.WORKSTATIONS_CONFIG_QUERY_PARAM}={config.pk}" not in qs
+
+    qs = workstation_query(image=image, overlay=overlay)
+    assert "&" in qs
+    assert f"{settings.WORKSTATIONS_BASE_IMAGE_QUERY_PARAM}={image.pk}" in qs
+    assert f"{settings.WORKSTATIONS_OVERLAY_QUERY_PARAM}={overlay.pk}" in qs
+    assert f"{settings.WORKSTATIONS_CONFIG_QUERY_PARAM}={config.pk}" not in qs
+
+    qs = workstation_query(image=image, config=config)
+    assert "&" in qs
+    assert f"{settings.WORKSTATIONS_BASE_IMAGE_QUERY_PARAM}={image.pk}" in qs
+    assert (
+        f"{settings.WORKSTATIONS_OVERLAY_QUERY_PARAM}={overlay.pk}" not in qs
+    )
+    assert f"{settings.WORKSTATIONS_CONFIG_QUERY_PARAM}={config.pk}" in qs