Merge latest staging with Job execution

fragalysis/settings.py

@@ -655,3 +655,7 @@
 FE_IMAGE_TAG: str = os.environ.get("FE_IMAGE_TAG", "undefined")
 STACK_NAMESPACE: str = os.environ.get("STACK_NAMESPACE", "undefined")
 STACK_VERSION: str = os.environ.get("STACK_VERSION", "undefined")
+
+
+# XChem Align data format
+XCA_DATA_FORMAT_VERSION = "2.2"

viewer/filters.py

@@ -8,6 +8,7 @@
     CanonSiteConf,
     Compound,
     Experiment,
+    ExperimentUpload,
     Pose,
     QuatAssembly,
     SiteObservation,
@@ -125,3 +126,15 @@ class AssemblyFilter(TargetFilterMixin):
     class Meta:
         model = QuatAssembly
         fields = ("target",)
+
+
+class ExperimentUploadFilter(filters.FilterSet):
+    class Meta:
+        model = ExperimentUpload
+        fields = (
+            "target",
+            "project",
+            "committer",
+            "data_version_major",
+            "data_version_minor",
+        )

viewer/managers.py

@@ -413,3 +413,34 @@ def filter_qs(self):
 
     def by_target(self, target):
         return self.get_queryset().filter_qs().filter(target=target.id)
+
+
+class ExperimentUploadQueryset(QuerySet):
+    def annotated_qs(self):
+        ExperimentUpload = apps.get_model("viewer", "ExperimentUpload")
+        qs = ExperimentUpload.objects.prefetch_related(
+            "target",
+            "project",
+        ).annotate(
+            target_name=F("target__title"),
+            proposal_number=F("project__title"),
+            committer_name=F("committer__username"),
+        )
+
+        return qs
+
+
+class ExperimentUploadDataManager(Manager):
+    def get_queryset(self):
+        return ExperimentUploadQueryset(self.model, using=self._db)
+
+    def annotated_qs(self):
+        return (
+            self.get_queryset()
+            .annotated_qs()
+            .order_by(
+                'project',
+                'target__title',
+                'upload_version',
+            )
+        )

viewer/migrations/0086_auto_20241213_1042.py

@@ -0,0 +1,23 @@
+# Generated by Django 3.2.25 on 2024-12-13 10:42
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('viewer', '0085_target_alias_order'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='experimentupload',
+            name='data_version_major',
+            field=models.PositiveSmallIntegerField(default=0),
+        ),
+        migrations.AddField(
+            model_name='experimentupload',
+            name='data_version_minor',
+            field=models.PositiveSmallIntegerField(default=0),
+        ),
+    ]

viewer/models.py

@@ -20,6 +20,7 @@
     CompoundDataManager,
     CompoundIdentifierDataManager,
     ExperimentDataManager,
+    ExperimentUploadDataManager,
     PoseDataManager,
     QuatAssemblyDataManager,
     SessionActionsDataManager,
@@ -196,6 +197,11 @@ class ExperimentUpload(models.Model):
     )
     upload_data_dir = models.TextField(null=True)
     upload_version = models.PositiveSmallIntegerField(default=1)
+    data_version_major = models.PositiveSmallIntegerField(default=0)
+    data_version_minor = models.PositiveSmallIntegerField(default=0)
+
+    objects = models.Manager()
+    upload_manager = ExperimentUploadDataManager()
 
     def __str__(self) -> str:
         return f"{self.project}"

viewer/serializers.py

@@ -895,13 +895,51 @@ class Meta:
 
 
 class TargetExperimentReadSerializer(ValidateProjectMixin, serializers.ModelSerializer):
+    tarball = serializers.SerializerMethodField()
+    target_name = serializers.CharField()
+    proposal_number = serializers.CharField()
+    committer_name = serializers.CharField()
+
+    def get_tarball(self, obj):
+        request = self.context.get('request')
+        path = (
+            Path(settings.MEDIA_URL)
+            .joinpath(settings.TARGET_LOADER_MEDIA_DIRECTORY)
+            .joinpath(obj.target.zip_archive.name)
+            .joinpath(obj.file.name)
+        )
+        if request:
+            return request.build_absolute_uri(path)
+        else:
+            return None
+
     class Meta:
         model = models.ExperimentUpload
-        fields = '__all__'
+        fields = (
+            'target',
+            'target_name',
+            'project',
+            'proposal_number',
+            'tarball',
+            'commit_datetime',
+            'committer',
+            'committer_name',
+            'task_id',
+            'neighbourhood_transforms',
+            'conformer_site_transforms',
+            'reference_structure_transforms',
+            'upload_data_dir',
+            'upload_version',
+            'data_version_major',
+            'data_version_minor',
+        )
 
 
 class TargetExperimentWriteSerializer(serializers.ModelSerializer):
     target_access_string = serializers.CharField(label='Target Access String')
+    file = serializers.FileField(required=False)
+    data_version = serializers.CharField(required=False)
+    target_name = serializers.CharField(required=False)
 
     def validate(self, data):
         """Verify TAS is correctly formed."""
@@ -915,6 +953,8 @@ class Meta:
         fields = (
             'target_access_string',
             'file',
+            'data_version',
+            'target_name',
         )
 
 

viewer/target_loader.py

@@ -445,6 +445,76 @@ def wrapper_create_objects(
     return wrapper_create_objects
 
 
+def split_version(version_number: str) -> tuple[int, int]:
+    splits = version_number.split('.')
+
+    if len(splits) != 2:
+        raise ValueError("Unrecognised data format, should be <major>.<minor>")
+
+    try:
+        major = int(splits[0])
+        minor = int(splits[1])
+    except ValueError as exc:
+        raise ValueError(f"Non-numeric version number: {version_number}") from exc
+
+    return major, minor
+
+
+def validate_data_version(
+    major: int,
+    minor: int,
+    o_major: int | None = None,
+    o_minor: int | None = None,
+    target_name: str | None = None,
+    project_name: str | None = None,
+) -> Tuple[bool, str]:
+    logger.debug('major: %s; minor: %s', major, minor)
+    logger.debug('o_major: %s; o_minor: %s', o_major, o_minor)
+
+    s_major, s_minor = [int(k) for k in settings.XCA_DATA_FORMAT_VERSION.split('.')]
+    logger.debug('s_major: %s; s_minor: %s', s_major, s_minor)
+
+    if major != s_major:
+        return (
+            False,
+            f"Data major version mismatch: '{s_major}' "
+            + f"expected, '{major}' uploaded",
+        )
+
+    # alternatively, if target- and project name are given (likely pre-upload check):
+    if target_name and project_name and not o_major and not o_minor:
+        previous_uploads = ExperimentUpload.objects.filter(
+            target__title=target_name,
+            project__title=project_name,
+        )
+        if previous_uploads.exists():
+            last_upload = previous_uploads.order_by('upload_version').last()
+            o_major = last_upload.data_version_major
+            o_minor = last_upload.data_version_minor
+
+    if o_major and o_major < major:
+        return False, (
+            f"Incoming data major version '{major}' does not match previous upload: "
+            + f"'{o_major}'. Please delete the target and prepare new upload"
+        )
+
+    if minor != s_minor:
+        return (
+            True,
+            f"Data minor version mismatch: {settings.XCA_DATA_FORMAT_VERSION} "
+            + f"expected, {major}.{minor} uploaded",
+        )
+
+    if o_minor and o_minor < minor:
+        return True, (
+            f"Incoming data minor version '{minor}' does not match previous upload: "
+            + f"'{o_minor}'"
+        )
+
+    # absolutely nothing went wrong
+    return True, ''
+
+
 class TargetLoader:
     def __init__(
         self,
@@ -1558,6 +1628,39 @@ def process_bundle(self):
         self.version_dir = meta["version_dir"]
         self.previous_version_dirs = meta["previous_version_dirs"]
         prefix_tooltips = meta.get("code_prefix_tooltips", {})
+        data_format_version = str(meta["data_format_version"])
+
+        try:
+            major, minor = split_version(data_format_version)
+        except ValueError as exc:
+            self.report.log(logging.ERROR, exc.args[0])
+            # throw a fatal error, but assign version numbers to
+            # see if more errors are caught
+            major = 0
+            minor = 0
+
+        # check for previous uploads
+        previous_uploads = ExperimentUpload.objects.filter(
+            target=self.target,
+            project=self.project,
+        )
+        if previous_uploads.exists():
+            last_upload = previous_uploads.order_by('upload_version').last()
+
+            version_validated, ver_val_msg = validate_data_version(
+                major,
+                minor,
+                o_major=last_upload.data_version_major,
+                o_minor=last_upload.data_version_minor,
+            )
+        else:
+            version_validated, ver_val_msg = validate_data_version(major, minor)
+
+        if not version_validated:
+            self.report.log(logging.ERROR, ver_val_msg)
+
+        if version_validated and ver_val_msg:
+            self.report.log(logging.WARNING, ver_val_msg)
 
         # TODO: is it here where I can figure out if this has already been uploaded?
         if self._is_already_uploaded(target_created, project_created):
@@ -1609,6 +1712,8 @@ def process_bundle(self):
         )
         self.experiment_upload.upload_data_dir = self.version_dir
         self.experiment_upload.upload_version = self.version_number
+        self.experiment_upload.data_version_major = major
+        self.experiment_upload.data_version_minor = minor
         self.experiment_upload.save()
 
         (  # pylint: disable=unbalanced-tuple-unpacking

viewer/views.py

@@ -41,6 +41,7 @@
     Squonk2AgentRv,
     get_squonk2_agent,
 )
+from viewer.target_loader import split_version, validate_data_version
 from viewer.utils import (
     CSV_TO_DICT_DOWNLOAD_ROOT,
     create_csv_from_dict,
@@ -1567,7 +1568,6 @@ def create(self, request, *args, **kwargs):
         logger.debug("User=%s", self.request.user)
 
         target_access_string = serializer.validated_data['target_access_string']
-        filename = serializer.validated_data['file']
 
         if settings.AUTHENTICATE_UPLOAD:
             if self.request.user.username == 'asap-service':
@@ -1618,6 +1618,38 @@ def create(self, request, *args, **kwargs):
                         status=status.HTTP_403_FORBIDDEN,
                     )
 
+        if 'data_version' in serializer.validated_data.keys():
+            try:
+                major, minor = split_version(serializer.validated_data['data_version'])
+            except ValueError as exc:
+                return Response(
+                    {
+                        'success': False,
+                        'message': exc.args[0],
+                    },
+                    status=status.HTTP_200_OK,
+                )
+            try:
+                target_name = serializer.validated_data['target_name']
+            except KeyError:
+                return Response(
+                    {'success': False, 'message': 'Target name not given'},
+                    status=status.HTTP_200_OK,
+                )
+
+            val_result, msg = validate_data_version(
+                major, minor, target_name=target_name, project_name=target_access_string
+            )
+            return Response(
+                {
+                    'success': val_result,
+                    'message': msg,
+                },
+                status=status.HTTP_200_OK,
+            )
+
+        filename = serializer.validated_data['file']
+
         # memo to self: cannot use TemporaryDirectory here because task
         temp_path = Path(settings.MEDIA_ROOT).joinpath('tmp')
         temp_path.mkdir(exist_ok=True)
@@ -1802,10 +1834,10 @@ def create(self, request, *args, **kwargs):
 
 
 class ExperimentUploadView(ISPyBSafeQuerySet):
-    queryset = models.ExperimentUpload.objects.all()
+    queryset = models.ExperimentUpload.upload_manager.annotated_qs()
     serializer_class = serializers.TargetExperimentReadSerializer
     permission_class = [permissions.IsAuthenticated]
-    filterset_fields = ("target", "project")
+    filterset_class = filters.ExperimentUploadFilter
     filter_permissions = "target__project"
     http_method_names = ('get',)