Merge pull request #671 from xchem/staging

Bulk tag loader changes (1263)

api/urls.py

@@ -113,13 +113,17 @@
     "site_observations", viewer_views.SiteObservationView, basename='site_observations'
 )
 router.register("canon_sites", viewer_views.CanonSiteView, basename='canon_sites')
+router.register("experiments", viewer_views.ExperimentView, basename='experiments')
 router.register(
     "canon_site_confs", viewer_views.CanonSiteConfView, basename='canon_site_confs'
 )
 router.register(
     "xtalform_sites", viewer_views.XtalformSiteView, basename='xtalform_sites'
 )
 router.register("poses", viewer_views.PoseView, basename='poses')
+router.register(
+    "metadata_upload", viewer_views.UploadMetadataView, basename='metadata_upload'
+)
 
 # Squonk Jobs
 router.register(

viewer/download_structures.py

@@ -8,7 +8,6 @@
 import json
 import logging
 import os
-import re
 import shutil
 import uuid
 import zipfile
@@ -20,19 +19,15 @@
 
 import pandoc
 from django.conf import settings
-from django.db.models import CharField, Exists, F, OuterRef, Subquery, Value
+from django.db.models import Exists, F, OuterRef, Value
+from django.db.models.fields import CharField
 from django.db.models.functions import Concat
 
-from viewer.models import (
-    DownloadLinks,
-    SiteObservation,
-    SiteObservationTag,
-    SiteObvsSiteObservationTag,
-    TagCategory,
-)
+from viewer.models import DownloadLinks, SiteObservation
 from viewer.utils import clean_filename
 
 from .serializers import DownloadStructuresSerializer
+from .tags import get_metadata_fields
 
 logger = logging.getLogger(__name__)
 
@@ -68,71 +63,6 @@
     'readme': (''),
 }
 
-TAG_CATEGORIES = (
-    'ConformerSites',
-    'CanonSites',
-    'CrystalformSites',
-    'Quatassemblies',
-    'Crystalforms',
-)
-CURATED_TAG_CATEGORIES = ('Series', 'Forum', 'Other')
-
-
-class TagSubquery(Subquery):
-    """Annotate SiteObservation with tag of given category"""
-
-    def __init__(self, category):
-        # fmt: off
-        query = SiteObservationTag.objects.filter(
-            pk=Subquery(
-                SiteObvsSiteObservationTag.objects.filter(
-                    site_observation=OuterRef(OuterRef('pk')),
-                    site_obvs_tag__category=TagCategory.objects.get(
-                        category=category,
-                    ),
-                ).values('site_obvs_tag')[:1]
-            )
-        ).annotate(
-            combitag=Concat(
-                F('tag_prefix'),
-                Value(' - '),
-                F('tag'),
-                output_field=CharField(),
-            ),
-        ).values('combitag')[0:1]
-        super().__init__(query)
-        # fmt: on
-
-
-class UploadTagSubquery(Subquery):
-    """Annotate SiteObservation with tag of given category"""
-
-    def __init__(self, category):
-        # fmt: off
-        query = SiteObservationTag.objects.filter(
-            pk=Subquery(
-                SiteObvsSiteObservationTag.objects.filter(
-                    site_observation=OuterRef(OuterRef('pk')),
-                    site_obvs_tag__category=TagCategory.objects.get(
-                        category=category,
-                    ),
-                ).values('site_obvs_tag')[:1]
-            )
-        ).values('upload_name')[0:1]
-        super().__init__(query)
-        # fmt: on
-
-
-class CuratedTagSubquery(Exists):
-    """Annotate SiteObservation with tag of given category"""
-
-    def __init__(self, tag):
-        query = SiteObvsSiteObservationTag.objects.filter(
-            site_observation=OuterRef('pk'),
-            site_obvs_tag=tag,
-        )
-        super().__init__(query)
-
 
 @dataclass(frozen=True)
 class ArchiveFile:
@@ -452,49 +382,7 @@ def _metadata_file_zip(ziparchive, target, site_observations):
     """Compile and add metadata file to archive."""
     logger.info('+ Processing metadata')
 
-    annotations = {}
-    values = [
-        'code',
-        'longcode',
-        'experiment__code',
-        'cmpd__compound_code',
-        'smiles',
-        'canon_site_conf__canon_site__centroid_res',
-        'downloaded',
-    ]
-    header = [
-        'Code',
-        'Long code',
-        'Experiment code',
-        'Compound code',
-        'Smiles',
-        'Centroid res',
-        'Downloaded',
-    ]
-
-    for category in TagCategory.objects.filter(category__in=TAG_CATEGORIES):
-        tag = f'tag_{category.category.lower()}'
-        upload_tag = f'upload_tag_{category.category.lower()}'
-        values.append(tag)
-        header.append(f'{category.category} alias')
-        annotations[tag] = TagSubquery(category.category)
-        values.append(upload_tag)
-        header.append(f'{category.category} upload name')
-        annotations[upload_tag] = UploadTagSubquery(category.category)
-
-    values.append('pose__display_name')
-    header.append('Pose')
-
-    pattern = re.compile(r'\W+')  # non-alphanumeric characters
-    for tag in SiteObservationTag.objects.filter(
-        category__in=TagCategory.objects.filter(category__in=CURATED_TAG_CATEGORIES),
-        target=target,
-    ):
-        # for reasons unknown, mypy thinks tag is a string
-        tagname = f'tag_{pattern.sub("_", tag.tag).strip().lower()}'  # type: ignore[attr-defined]
-        values.append(tagname)
-        header.append(f'[{tag.category}] {tag.tag}')  # type: ignore[attr-defined]
-        annotations[tagname] = CuratedTagSubquery(tag)
+    header, annotations, values = get_metadata_fields(target)
 
     # fmt: off
     qs = SiteObservation.filter_manager.by_target(
@@ -508,7 +396,11 @@ def _metadata_file_zip(ziparchive, target, site_observations):
                 pk=OuterRef('pk'),
             ),
         )
-    ).annotate(**annotations).values_list(*values)
+    ).annotate(
+        **annotations
+    ).values_list(
+        *values
+    )
     # fmt: on
 
     buff = StringIO()

viewer/filters.py

@@ -7,6 +7,7 @@
     CanonSite,
     CanonSiteConf,
     Compound,
+    Experiment,
     Pose,
     QuatAssembly,
     SiteObservation,
@@ -60,6 +61,12 @@ class Meta:
         fields = ("target",)
 
 
+class ExperimentFilter(TargetFilterMixin):
+    class Meta:
+        model = Experiment
+        fields = ("target",)
+
+
 class CanonSiteConfFilter(TargetFilterMixin):
     class Meta:
         model = CanonSiteConf

viewer/migrations/0064_auto_20240918_1256.py

@@ -0,0 +1,40 @@
+# Generated by Django 3.2.25 on 2024-09-18 12:56
+
+import django.contrib.postgres.fields
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('viewer', '0063_merge_20240906_1243'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='experiment',
+            name='cif_info_source_file',
+            field=models.TextField(null=True),
+        ),
+        migrations.AddField(
+            model_name='experiment',
+            name='map_info_source_files',
+            field=django.contrib.postgres.fields.ArrayField(base_field=models.TextField(null=True), null=True, size=None),
+        ),
+        migrations.AddField(
+            model_name='experiment',
+            name='mtz_info_source_file',
+            field=models.TextField(null=True),
+        ),
+        migrations.AddField(
+            model_name='experiment',
+            name='pdb_info_source_file',
+            field=models.TextField(null=True),
+        ),
+        migrations.AlterField(
+            model_name='siteobservation',
+            name='pose',
+            field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='site_observations', to='viewer.pose'),
+        ),
+    ]

viewer/models.py

@@ -215,13 +215,17 @@ class Experiment(models.Model):
     pdb_info = models.FileField(
         upload_to="target_loader_data/", null=True, max_length=255
     )
+    pdb_info_source_file = models.TextField(null=True)
     mtz_info = models.FileField(
         upload_to="target_loader_data/", null=True, max_length=255
     )
+    mtz_info_source_file = models.TextField(null=True)
     cif_info = models.FileField(
         upload_to="target_loader_data/", null=True, max_length=255
     )
+    cif_info_source_file = models.TextField(null=True)
     map_info = ArrayField(models.FileField(max_length=255), null=True)
+    map_info_source_files = ArrayField(models.TextField(null=True), null=True)
     type = models.PositiveSmallIntegerField(null=True)
     pdb_sha256 = models.TextField(null=True)
     prefix_tooltip = models.TextField(null=True)
@@ -500,7 +504,7 @@ class SiteObservation(Versionable, models.Model):
     canon_site_conf = models.ForeignKey(CanonSiteConf, on_delete=models.CASCADE)
     pose = models.ForeignKey(
         Pose,
-        on_delete=models.CASCADE,
+        on_delete=models.SET_NULL,
         null=True,
         related_name="site_observations",
     )
@@ -1348,7 +1352,7 @@ class SiteObservationTag(Tag):
     )
 
     def __str__(self) -> str:
-        return f"{self.site_observations}"
+        return f"{self.id}"
 
     def __repr__(self) -> str:
         return "<SiteObservationTag %r %r>" % (self.id, self.site_observations)

viewer/serializers.py

@@ -931,6 +931,12 @@ class Meta:
         fields = '__all__'
 
 
+class ExperimentReadSerializer(serializers.ModelSerializer):
+    class Meta:
+        model = models.Experiment
+        fields = '__all__'
+
+
 class CanonSiteConfReadSerializer(serializers.ModelSerializer):
     class Meta:
         model = models.CanonSiteConf
@@ -1132,3 +1138,9 @@ class Meta:
             'site_observations',
             'main_site_observation_cmpd_code',
         )
+
+
+class MetadataUploadSerializer(serializers.Serializer):
+    filename = serializers.FileField()
+    target = serializers.CharField()
+    target_access_string = serializers.CharField()