Merge pull request #109 from DeepMicroscopy/TIFFzStack

Merged TIFF ZStacks

exact/exact/annotations/migrations/0023_auto_20180428_1756.py

@@ -4,8 +4,8 @@
 
 import datetime
 from django.db import migrations, models
-from django.utils.timezone import utc
-
+import datetime
+utc = datetime.timezone.utc
 
 class Migration(migrations.Migration):
 

exact/exact/annotations/static/annotations/js/exact-image-viewer.js

@@ -556,15 +556,15 @@ class EXACTViewer {
             }
             this.frameSlider = new Slider("#frameSlider", {
                 ticks_snap_bounds: 1,
-                value: 1,
+                value: 0,
                 formatter: function(val){ return value_formatter (labels,val) },
-                min: 1,
+                min: 0,
                 tooltip: 'always',
-                max: frames
+                max: frames-1
             });
             this.frameSlider.on('change', this.onFrameSliderChanged.bind(this));
         }
-        else if (objectivePower > 1) {
+        if (objectivePower > 1) {
 
             const default_ticks = [0, 1, 2, 5, 10, 20, 40, 80, 160];
             const default_names = ["0x", "1x", "2x", "5x", "10x", "20x", "40x", "80x", "160x"];

exact/exact/images/api_views.py

@@ -148,7 +148,7 @@ def view_image_tile(self, request, pk, z_dimension, frame, level, tile_path):
         try:
             slide = image_cache.get(file_path)
 
-            tile = slide.get_tile(level, (col, row))
+            tile = slide.get_tile(level, (col, row), frame=frame)
 
             buf = PILBytesIO()
             tile.save(buf, format, quality=90)
@@ -210,7 +210,7 @@ def update_image_cache(self, request, pk=None):
                     cache_key = f"{pk}/{z_dimension}/{frame}/{level}/{col}/{row}"
                     if tiles_cache.has_key(cache_key) == False:
 
-                        tile = slide.get_tile(level, (col, row))
+                        tile = slide.get_tile(level=level, address=(col, row), frame=frame)
 
                         buf = PILBytesIO()
                         tile.save(buf, "jpeg", quality=90)
@@ -383,7 +383,7 @@ def create(self, request):
             f.seek(0)  # reset file cursor to the beginning of the file
 
             file_list = {}
-            print('Magic number: ',hex(magic_number))
+#            print('Magic number: ',hex(magic_number))
             if magic_number == b'PK\x03\x04':
                 zipname = ''.join(random.choice(string.ascii_uppercase +
                                                 string.ascii_lowercase +

exact/exact/images/migrations/0033_image_stack_setfilename.py

@@ -0,0 +1,52 @@
+# Manual migration to set filename field of images correctly, since EXACT now opens many z Stacks natively and does not require the filename change any more.
+from django.db import migrations, models
+from openslide import OpenSlide
+import openslide
+from pathlib import Path
+
+from exact.images.models import Image
+from django.conf import settings
+import os
+from util.slide_server import getSlideHandler
+
+def remove_natively_handled_files_auxfiles(apps, schema_editor):
+    db_alias = schema_editor.connection.alias
+
+    for image in Image.objects.all():
+        try:
+            path = os.path.join(settings.IMAGE_PATH, image.image_set.path, image.name)
+            if image.frames > 1:
+                print('Path: ',path)
+                if not os.path.exists(path):
+                    continue
+                slide = getSlideHandler(path)
+                if slide is None:
+                    continue
+                if image.depth==1:
+                    rp = image.image_set.root_path()
+                    if not os.path.exists(rp / Path(image.filename)):
+                        print('Setting image filename from: ',image.filename,'to:',image.name)
+                        image.filename = image.name
+                        image.save()
+                    for frame in range(image.frames):
+                        filepath = rp / Path(Path(image.name).stem) / "{}_{}_{}".format(1, frame+1, image.name)
+                        if os.path.exists(filepath):
+                            print('Removing: ', filepath)
+                            os.unlink(filepath)
+
+        except Exception as e:
+            print('Exception: ',e,image)
+            raise
+            continue
+
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('images', '0032_rename_descreption_framedescription_description_and_more'),
+    ]
+
+    operations = [
+        migrations.RunPython(remove_natively_handled_files_auxfiles)
+    ]

exact/exact/images/models.py

@@ -12,7 +12,7 @@
 from django.db.models.signals import post_delete, post_save, m2m_changed
 from django.dispatch import receiver
 from django.utils.functional import cached_property
-
+from util.slide_server import getSlideHandler
 import logging
 
 import math
@@ -39,12 +39,9 @@
 
 logger = logging.getLogger('django')
 
+from util.enums import FrameType
 
 class FrameDescription(models.Model):
-    class FrameType:
-        ZSTACK = 0
-        TIMESERIES = 1
-        UNDEFINED = 255
 
     FRAME_TYPES = (
         (FrameType.ZSTACK, 'z Stack'),
@@ -93,7 +90,7 @@ class ImageSourceTypes:
     image_type = models.IntegerField(choices=SOURCE_TYPES, default=ImageSourceTypes.DEFAULT)
 
     def get_file_name(self, depth=1, frame=1): 
-        if depth > 1 or frame > 1 or self.frames > 1 or self.depth > 1:
+        if depth > 1 or self.depth > 1:
             return str(Path(Path(self.name).stem) / "{}_{}_{}".format(depth, frame, self.name))
         else:
             return self.filename
@@ -127,12 +124,30 @@ def save(self, *args, **kwargs):
     def save_file(self, path:Path):
 
         try:
-            # check if the file can be opened by OpenSlide if not convert it
+            # check if the file can be opened natively, if not convert it
             try:
-                osr = OpenSlide(str(path))
+                osr = getSlideHandler(str(path))
                 self.filename = path.name
-            except:
-                print('Unable to open image with OpenSlide')
+                self.save()
+                if (osr.nFrames>1):
+                    for frame_id in range(osr.nFrames):
+                        # save FrameDescription object for each frame
+                        FrameDescription.objects.create(
+                                Image=self,
+                                frame_id=frame_id,
+                                file_path=self.filename,
+                                description=osr.frame_descriptors[frame_id],                                    
+                                frame_type=osr.frame_type,
+                        )
+                    print('Added',osr.nFrames,'frames')
+                    self.frames=osr.nFrames
+                    if openslide.PROPERTY_NAME_OBJECTIVE_POWER in osr.properties:
+                        self.objectivePower = osr.properties[openslide.PROPERTY_NAME_OBJECTIVE_POWER]
+                    if openslide.PROPERTY_NAME_MPP_X in osr.properties:
+                        self.mpp = osr.properties[openslide.PROPERTY_NAME_MPP_X]
+
+            except Exception as e:
+                print('Unable to open image with OpenSlide',e)
                 import pyvips
                 old_path = path
 
@@ -149,7 +164,7 @@ def save_file(self, path:Path):
                     self.save() # initially save
                     for frame_id in range(self.frames):
                         height, width = reader.dimensions 
-                        np_image = np.array(reader.read_region(location=(0,0), size=(reader.dimensions), level=0, zLevel=frame_id))[:,:,0]
+                        np_image = np.array(reader.read_region(location=(0,0), size=(reader.dimensions), level=0, frame=frame_id))[:,:,0]
                         linear = np_image.reshape(height * width * self.channels)
                         vi = pyvips.Image.new_from_memory(np.ascontiguousarray(linear.data), height, width, self.channels, 'uchar')
 
@@ -387,7 +402,7 @@ def save_file(self, path:Path):
                     vi.tiffsave(str(path), tile=True, compression='lzw', bigtiff=True, pyramid=True, tile_width=256, tile_height=256)
                     self.filename = path.name
 
-            osr = OpenSlide(self.path())
+            osr = getSlideHandler(self.path())
             self.width, self.height = osr.level_dimensions[0]
             try:
                 mpp_x = osr.properties[openslide.PROPERTY_NAME_MPP_X]
@@ -747,22 +762,6 @@ def rotation_angle(self):
         return - math.atan2(self.transformation_matrix["t_01"], self.transformation_matrix["t_00"]) * 180 / math.pi
 
 
-    def fixedCvInvert(self, H):
-        """[If the determinate is zero, OpenCV returns a wrong inverted matrix. ]
-
-        Args:
-            H ([type]): [3x3 Matrix]
-
-        Returns:
-            [type]: [Inverted matrix]
-        """   
-        if (cv2.determinant(H) != 0.0):
-            return cv2.invert(H)[1]
-        else:
-            return np.array([[1., 0., -H[0,-1]],
-                          [0., 1., -H[1,-1]],
-                          [0., 0., 0.]])
-
     @cached_property
     def inv_matrix(self):
 
@@ -772,7 +771,8 @@ def inv_matrix(self):
                         [t["t_20"], t["t_21"], t["t_22"]]])
 
 
-        M = self.fixedCvInvert(H)
+        # Using numpys pseudo-inverse as this is the generalization for singular matrices
+        M = np.linalg.pinv(H)
 
         return {
             "t_00": M [0,0], 
@@ -801,7 +801,7 @@ def get_matrix_without_rotation(self):
                         [0.         ,             0, 1]])
 
 
-        inv_rot = self.fixedCvInvert(H)
+        inv_rot = np.linalg.pinv(rot)
 
         M = H@inv_rot
         return M
@@ -817,7 +817,7 @@ def get_scale(self):
     def get_inv_scale(self):
 
         M = self.get_matrix_without_rotation
-        M = self.fixedCvInvert(M)
+        M = np.linalg.pinv(M)
         return M[0][0], M[1][1]
 
     def __str__(self):

exact/exact/images/views.py

@@ -487,8 +487,8 @@ def view_image_navigator_overlay_tile(request, image_id, z_dimension, frame, lev
     file_path = os.path.join(settings.IMAGE_PATH, image.path())
     slide = image_cache.get(file_path)
 
-    tile = slide.get_tile(level, (col, row))
-
+    tile = slide.get_tile(level, (col, row), frame=frame)
+ 
     # replace with databse call to imageset.product
     for product in image.image_set.product_set.all():
         for plugin in plugin_finder.filter_plugins(product_name=product.name, navigation_view_policy=ViewPolicy.RGB_IMAGE):
@@ -501,7 +501,7 @@ def view_image_navigator_overlay_tile(request, image_id, z_dimension, frame, lev
     return response
 
 @login_required
-@cache_page(60 * 60 * 24 * 30)
+#@cache_page(60 * 60 * 24 * 30)
 def view_image_tile(request, image_id, z_dimension, frame, level, tile_path):
     """
     This view is to authenticate direct access to the images via nginx auth_request directive
@@ -531,15 +531,15 @@ def view_image_tile(request, image_id, z_dimension, frame, level, tile_path):
     try:
         slide = image_cache.get(file_path)
 
-        tile = slide.get_tile(level, (col, row))
+        tile = slide.get_tile(level, (col, row),frame=frame)
 
         buf = PILBytesIO()
         tile.save(buf, format, quality=90)
         buffer = buf.getvalue()
 
         load_from_drive_time = timer() - start
 
-        logger.info(f"{load_from_drive_time:.4f};{request.path}")
+        logger.info(f"{load_from_drive_time:.4f};{request.path};NC")
 
         if hasattr(cache, "delete_pattern"):
             tiles_cache.set(cache_key, buffer, 7*24*60*60)

exact/requirements.txt

@@ -5,39 +5,40 @@
 #    pip-compile --output-file requirements.txt requirements.in
 #
 
-confusable-homoglyphs==3.2.0
+confusable-homoglyphs==3.3.1
 drf-flex-fields==1.0.2
-Django==3.2.11
-django-appconf==1.0.5
+django-appconf==1.0.6
 django-extensions==3.2.3
-django-filter==23.2
+django-filter==23.5
 django-friendly-tag-loader==1.3.1
-django-project-version==0.18.1
-django-redis==5.3.0
+django-project-version==0.19.0
+django-redis==5.4.0
 django-registration==3.4
 django-templated-mail==1.1.1
-django-widget-tweaks==1.4.12
+django-widget-tweaks==1.5.0
 djangorestframework==3.14.0
-djangorestframework-simplejwt==5.2.2
-djoser==2.2.0
-uritemplate==3.0.1
+djangorestframework-simplejwt==5.3.1
+djoser==2.2.2
+zarr==2.15.0
+xmltodict==0.13.0
+uritemplate==4.1.1
 fasteners==0.15
-monotonic==1.5
-Pillow==10.0.0
-psycopg2-binary==2.9.6
-pytz==2023.3
+monotonic==1.6
+pillow==10.2.0
+psycopg2-binary==2.9.9
+pytz==2024.1
 gunicorn==21.2.0
-opencv-python==4.8.0.74
-openslide-python==1.3.0
-tifffile>=2023.7.0
-pyvips==2.2.1
-pydicom==2.4.2
-ptvsd==4.3.2
+opencv-python>=4.9.0.80
+openslide-python==1.3.1
+tifffile==2023.2.28
+pyvips>=2.2.1
+pydicom>=2.4.4
+ptvsd>=4.3.2
 czifile==2019.7.2
 gitpython==3.1.11
-gdown==4.7.1
-pandas==2.0.3
-qt-wsi-registration>=0.0.6
+gdown==5.1.0
+pandas==2.2.1
+qt-wsi-registration==0.0.11
 EXCAT-Sync>=0.0.38
 pyyaml>=6.0.1
-aicsimageio==4.11.0
+aicsimageio==4.14.0