incorporate first prototype of auto-ir-metadata

application/setup.cfg

@@ -28,7 +28,9 @@ install_requires =
     PyGithub==1.59.1
     django-extensions
     discourse-client-in-disraptor==0.0.10
-    tira>=0.0.143
+    # tira>=0.0.143
+    tira @ git+https://github.com/tira-io/tira.git@pyterrier-artifacts#egg=tira&subdirectory=python-client
+    auto-ir-metadata @ git+https://github.com/webis-de/auto-ir-metadata@dev
     huggingface-hub
     djangorestframework==3.15.1
     django-filter==24.2

application/src/tira_app/endpoints/v1/_anonymous.py

@@ -4,9 +4,10 @@
 import zipfile
 from pathlib import Path
 
+from auto_ir_metadata import load_ir_metadata
 from django.conf import settings
 from django.core.cache import cache
-from django.http import FileResponse, HttpResponseServerError
+from django.http import FileResponse, HttpResponse, HttpResponseServerError
 from django.urls import path
 from rest_framework.decorators import api_view
 from rest_framework.request import Request
@@ -34,9 +35,20 @@ def read_anonymous_submission(request: Request, submission_uuid: str) -> Respons
     """
     try:
         ret = modeldb.AnonymousUploads.objects.get(uuid=submission_uuid)
-
-        return Response({"uuid": ret.uuid, "dataset_id": ret.dataset.dataset_id, "created": ret.created})
-    except:
+        ret = {
+            "uuid": ret.uuid,
+            "dataset_id": ret.dataset.dataset_id,
+            "created": ret.created,
+            "has_metadata": ret.has_metadata,
+            "metadata_git_repo": ret.metadata_git_repo,
+            "metadata_has_notebook": ret.metadata_has_notebook,
+        }
+        if not ret["has_metadata"]:
+            del ret["metadata_git_repo"]
+            del ret["metadata_has_notebook"]
+
+        return Response(ret)
+    except Exception as e:
         return HttpResponseServerError(
             json.dumps({"status": 1, "message": f"Run with uuid {html.escape(submission_uuid)} does not exist."})
         )
@@ -72,7 +84,6 @@ def download_anonymous_submission(request: Request, submission_uuid: str) -> Res
 
     result_dir = Path(settings.TIRA_ROOT) / "data" / "anonymous-uploads" / submission_uuid
     format = json.loads(upload.dataset.format)[0]
-    format = secure_filename(format)
     status_code, message = check_format(result_dir, format)
 
     if status_code != _fmt.OK:
@@ -111,7 +122,6 @@ def claim_submission(request: Request, vm_id: str, submission_uuid: str) -> Resp
     body = json.loads(body)
     result_dir = Path(settings.TIRA_ROOT) / "data" / "anonymous-uploads" / submission_uuid
     format = json.loads(upload.dataset.format)[0]
-    format = secure_filename(format)
     status_code, message = check_format(result_dir, format)
 
     if status_code != _fmt.OK:
@@ -152,8 +162,48 @@ def chunks(self):
     return Response({"upload_group": body["upload_group"], "status": "0"})
 
 
+@api_view(["GET"])
+def render_notebook_of_submission(request: Request, submission_uuid: str) -> Response:
+    """Read information about an anonymous submission identified by the ownership uuid.
+
+    Args:
+        request (Request): The request that triggered the REST API call.
+        submission_uuid (str): The ownership uuid of the anonymous submission
+
+    Returns:
+        Response: The rendered jupyter notebook.
+    """
+    try:
+        upload = modeldb.AnonymousUploads.objects.get(uuid=submission_uuid)
+    except:
+        return HttpResponseServerError(
+            json.dumps({"status": 1, "message": f"Run with uuid {html.escape(submission_uuid)} does not exist."})
+        )
+
+    if not upload.has_metadata or not upload.metadata_has_notebook:
+        return HttpResponseServerError(
+            json.dumps(
+                {"status": 1, "message": f"Run with uuid {html.escape(submission_uuid)} has no jupyter notebook."}
+            )
+        )
+    metadata = load_ir_metadata(upload.get_path_in_file_system())
+
+    if not metadata or "notebook_html" not in metadata:
+        return HttpResponseServerError(
+            json.dumps(
+                {
+                    "status": 1,
+                    "message": f"Could not render Jupyter Notebook of run with uuid {html.escape(submission_uuid)}.",
+                }
+            )
+        )
+
+    return HttpResponse(metadata["notebook_html"])
+
+
 endpoints = [
     path("claim/<str:vm_id>/<str:submission_uuid>", claim_submission),
+    path("view/<str:submission_uuid>/jupyter-notebook.html", render_notebook_of_submission),
     path("<str:submission_uuid>.zip", download_anonymous_submission),
     path("<str:submission_uuid>", read_anonymous_submission),
 ]

application/src/tira_app/endpoints/vm_api.py

@@ -8,6 +8,7 @@
 from http import HTTPStatus
 from pathlib import Path
 
+from auto_ir_metadata import load_ir_metadata
 from discourse_client_in_disraptor.discourse_api_client import get_disraptor_user
 from django.conf import settings
 from django.core.cache import cache
@@ -558,10 +559,30 @@ def anonymous_upload(request, dataset_id):
             HttpResponseServerError(json.dumps({"status": 1, "message": message}))
         from .. import model as modeldb
 
-        (Path(settings.TIRA_ROOT) / "data" / "anonymous-uploads").mkdir(exist_ok=True, parents=True)
-        shutil.move(result_dir / "extracted", Path(settings.TIRA_ROOT) / "data" / "anonymous-uploads" / upload_id)
+        anon_uploads_dir = Path(settings.TIRA_ROOT) / "data" / "anonymous-uploads"
+        (anon_uploads_dir).mkdir(exist_ok=True, parents=True)
+        upload_dir = anon_uploads_dir / upload_id
+        shutil.move(result_dir / "extracted", upload_dir)
+        has_metadata = False
+        metadata_git_repo = None
+        metadata_has_notebook = False
+
+        try:
+            metadata = load_ir_metadata(upload_dir)
+            has_metadata = True
+            metadata_git_repo = metadata["git_url"] if "git_url" in metadata else None
+            metadata_has_notebook = "notebook" in metadata and "notebook_html" in metadata
+        except:
+            pass
+
         dataset = modeldb.Dataset.objects.get(dataset_id=dataset_id)
-        modeldb.AnonymousUploads.objects.create(uuid=upload_id, dataset=dataset)
+        modeldb.AnonymousUploads.objects.create(
+            uuid=upload_id,
+            dataset=dataset,
+            has_metadata=has_metadata,
+            metadata_git_repo=metadata_git_repo,
+            metadata_has_notebook=metadata_has_notebook,
+        )
 
         return JsonResponse({"status": 0, "message": "ok", "uuid": upload_id})
     else:

application/src/tira_app/migrations/0014_anonymousuploads_has_metadata_and_more.py

@@ -0,0 +1,28 @@
+# Generated by Django 5.0.9 on 2025-01-30 09:52
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("tira", "0013_alter_dataset_description"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="anonymousuploads",
+            name="has_metadata",
+            field=models.BooleanField(default=False),
+        ),
+        migrations.AddField(
+            model_name="anonymousuploads",
+            name="metadata_git_repo",
+            field=models.CharField(default=None, max_length=500, null=True),
+        ),
+        migrations.AddField(
+            model_name="anonymousuploads",
+            name="metadata_has_notebook",
+            field=models.BooleanField(default=False),
+        ),
+    ]

application/src/tira_app/model.py

@@ -227,6 +227,12 @@ class AnonymousUploads(models.Model):
     uuid = models.CharField(max_length=150, primary_key=True)
     dataset = models.ForeignKey(Dataset, on_delete=models.CASCADE)
     created = models.DateField(auto_now_add=True)
+    has_metadata = models.BooleanField(default=False)
+    metadata_git_repo = models.CharField(max_length=500, default=None, null=True)
+    metadata_has_notebook = models.BooleanField(default=False)
+
+    def get_path_in_file_system(self):
+        return Path(settings.TIRA_ROOT) / "data" / "anonymous-uploads" / self.uuid
 
 
 class DockerSoftware(models.Model):

application/test/api_access_matrix.py

@@ -2092,6 +2092,17 @@
             ADMIN: 500,
         },
     ),
+    route_to_test(
+        url_pattern="v1/anonymous/view/<str:submission_uuid>/jupyter-notebook.html",
+        params={"submission_uuid": "12345"},
+        group_to_expected_status_code={
+            GUEST: 500,
+            PARTICIPANT: 500,
+            ORGANIZER_WRONG_TASK: 500,
+            ORGANIZER: 500,
+            ADMIN: 500,
+        },
+    ),
     route_to_test(
         url_pattern="v1/anonymous/<str:submission_uuid>.zip",
         params={"submission_uuid": "12345"},

frontend/src/ClaimSubmission.vue

@@ -34,6 +34,9 @@
           </span>
           for task <a :href="'/task-overview/' + dataset.default_task">{{ dataset.default_task }}</a> <span v-if="download_link !== undefined">(<a :href="download_link" target="_blank">Download</a></span>).
         </p>
+        <p v-if="has_metadata && metadata_git_repo !== undefined && jupyter_notebook_link !== undefined">
+          Metadata on the run in the <a target='_blank' href="https://github.com/irgroup/ir_metadata">ir_metadata</a> format is available, including <a :href="metadata_git_repo" target='_blank' >the git repository</a> and the <a :href="jupyter_notebook_link" target='_blank' >jupyter notebook</a> that produced this run.
+        </p>
 
         <div class="py-2"></div>
 
@@ -157,6 +160,15 @@ export default {
   },
   computed: {
     link_chatnoir() { return  chatNoirUrl(this.dataset) },
+    has_metadata() { return this.submissionToClaim && this.submissionToClaim.has_metadata },
+    metadata_git_repo() { if (this.submissionToClaim && this.has_metadata && this.submissionToClaim.metadata_git_repo) {
+      return this.submissionToClaim.metadata_git_repo
+    }},
+    jupyter_notebook_link() {
+      if (this.submissionToClaim && this.has_metadata && this.submissionToClaim.metadata_has_notebook) {
+        return this.rest_url + `/v1/anonymous/view/` + this.uuid + '/jupyter-notebook.html'
+      }
+    },
     download_link() { 
       if (this.submissionToClaim && this.submissionToClaim.dataset_id && this.dataset) {
         return this.rest_url + `/v1/anonymous/` + this.submissionToClaim.uuid + `.zip`

frontend/src/utils.ts

@@ -29,6 +29,9 @@ export interface ClaimSubmissionInfo {
     uuid: string;
     dataset_id: string;
     created: string;
+    has_metadata: boolean | undefined;
+    metadata_git_repo: string | undefined;
+    metadata_has_notebook: string | undefined;
 }
 
 export interface UserContext {