Merge pull request #45 from LCOGT/fix/raw_data_32

Fix raw_data endpoint to work with 32 bit images with negative values

datalab/datalab_session/analysis/get_tif.py

@@ -1,5 +1,5 @@
-from datalab.datalab_session.file_utils import create_tif
-from datalab.datalab_session.s3_utils import key_exists, add_file_to_bucket, get_s3_url, get_fits
+from datalab.datalab_session.utils.file_utils import create_tif
+from datalab.datalab_session.utils.s3_utils import key_exists, add_file_to_bucket, get_s3_url, get_fits
 
 def get_tif(input: dict):
   """

datalab/datalab_session/analysis/line_profile.py

@@ -3,8 +3,8 @@
 from astropy.wcs import WcsError
 from astropy import coordinates
 
-from datalab.datalab_session.file_utils import scale_points, get_hdu
-from datalab.datalab_session.s3_utils import get_fits
+from datalab.datalab_session.utils.file_utils import scale_points, get_hdu
+from datalab.datalab_session.utils.s3_utils import get_fits
 
 # For creating an array of brightness along a user drawn line
 def line_profile(input: dict):

datalab/datalab_session/analysis/raw_data.py

@@ -1,8 +1,8 @@
 import numpy as np
 import math
 from PIL import Image
-from datalab.datalab_session.s3_utils import get_fits
-from datalab.datalab_session.file_utils import get_hdu
+from datalab.datalab_session.utils.s3_utils import get_fits
+from datalab.datalab_session.utils.file_utils import get_hdu
 from fits2image.scaling import extract_samples, calc_zscale_min_max
 
 # TODO: This analysis endpoint assumes the image to be of 16 bitdepth. We should make this agnositc to bit depth in the future
@@ -23,27 +23,36 @@ def raw_data(input: dict):
     image = Image.fromarray(image_data)
     newImage = image.resize((max_size, max_size), Image.LANCZOS)
     bitpix = abs(int(sci_hdu.header.get('BITPIX', 16)))
+    max_value = int(sci_hdu.header.get('SATURATE', 0))  # If saturate header is present, use that as max value
     match bitpix:
         case 8:
             datatype = np.uint8
-            max_value = np.iinfo(datatype).max
+            if not max_value:
+                max_value = np.iinfo(datatype).max
         case 16:
             datatype = np.float16
-            max_value = np.finfo(datatype).max
+            if not max_value:
+                max_value = np.finfo(datatype).max
         case 32:
             datatype = np.float32
-            max_value = np.finfo(datatype).max
+            if not max_value:
+                max_value = np.finfo(datatype).max
     scaled_array = np.asarray(newImage).astype(datatype)
     scaled_array_flipped = np.flip(scaled_array, axis=0)
 
+    # Set the zmin/zmax to integer values for calculating bins
+    zmin = math.floor(zmin)
+    zmax = math.ceil(zmax)
+
     # Here we do a crazy histogram scaling to stretch the points in between zmin and zmax since that is where most detail is
     # We have 10 bins before zmin, 100 between zmin and zmax and 10 after zmax.
+    zero_point = math.floor(min(np.min(samples), 0))  # This is for images whose values go below 0
     lower_bound = int(zmin * 0.8)  # Increase resolution slightly below zmin
     upper_bound = int(zmax*1.2)  # Increase resolution slightly beyond zmax
-    lower_step = int(lower_bound / 10)
-    upper_step = int((max_value - upper_bound) / 10)
-    step = int((upper_bound - lower_bound) / 100)
-    bins = np.arange(0, lower_bound, lower_step).tolist()
+    lower_step = int(abs(lower_bound / 10))
+    upper_step = int(abs((max_value - upper_bound) / 10))
+    step = int(abs((upper_bound - lower_bound) / 100))
+    bins = np.arange(zero_point, lower_bound, lower_step).tolist()
     bins += np.arange(lower_bound, upper_bound, step).tolist()
     bins += np.arange(upper_bound, max_value, upper_step).tolist()
     histogram, bin_edges = np.histogram(samples, bins=bins)
@@ -67,7 +76,7 @@ def raw_data(input: dict):
             'width': scaled_array.shape[1],
             'histogram': hist,
             'bins': bin_middles,
-            'zmin': int(median),
-            'zmax': int(zmax),
+            'zmin': round(median),
+            'zmax': round(zmax),
             'bitdepth': bitpix
         }

datalab/datalab_session/analysis/source_catalog.py

@@ -1,7 +1,7 @@
 import numpy as np
 
-from datalab.datalab_session.file_utils import get_hdu, scale_points
-from datalab.datalab_session.s3_utils import get_fits
+from datalab.datalab_session.utils.file_utils import get_hdu, scale_points
+from datalab.datalab_session.utils.s3_utils import get_fits
 
 # Source catalog Function Definition
 # ARGS: input (dict)

datalab/datalab_session/data_operations/data_operation.py

@@ -5,9 +5,9 @@
 from django.core.cache import cache
 import numpy as np
 
-from datalab.datalab_session.s3_utils import get_fits
+from datalab.datalab_session.utils.s3_utils import get_fits
 from datalab.datalab_session.tasks import execute_data_operation
-from datalab.datalab_session.file_utils import get_hdu
+from datalab.datalab_session.utils.file_utils import get_hdu
 
 CACHE_DURATION = 60 * 60 * 24 * 30  # cache for 30 days
 

datalab/datalab_session/data_operations/fits_output_handler.py

@@ -2,8 +2,8 @@
 import numpy as np
 from astropy.io import fits
 
-from datalab.datalab_session.file_utils import create_jpgs
-from datalab.datalab_session.s3_utils import save_fits_and_thumbnails
+from datalab.datalab_session.utils.file_utils import create_jpgs
+from datalab.datalab_session.utils.s3_utils import save_fits_and_thumbnails
 
 
 class FITSOutputHandler():

datalab/datalab_session/data_operations/input_data_handler.py

@@ -1,7 +1,7 @@
 from astropy.io import fits
 
-from datalab.datalab_session.s3_utils import get_fits
-from datalab.datalab_session.file_utils import get_hdu
+from datalab.datalab_session.utils.s3_utils import get_fits
+from datalab.datalab_session.utils.file_utils import get_hdu
 
 class InputDataHandler():
   """A class to read FITS files and provide access to the data.

datalab/datalab_session/data_operations/median.py

@@ -6,7 +6,7 @@
 from datalab.datalab_session.data_operations.fits_output_handler import FITSOutputHandler
 from datalab.datalab_session.data_operations.data_operation import BaseDataOperation
 from datalab.datalab_session.exceptions import ClientAlertException
-from datalab.datalab_session.file_utils import crop_arrays
+from datalab.datalab_session.utils.file_utils import crop_arrays
 
 log = logging.getLogger()
 log.setLevel(logging.INFO)

datalab/datalab_session/data_operations/rgb_stack.py

@@ -1,13 +1,12 @@
 import logging
 
-from astropy.io import fits
 import numpy as np
 
 from datalab.datalab_session.data_operations.input_data_handler import InputDataHandler
 from datalab.datalab_session.data_operations.data_operation import BaseDataOperation
 from datalab.datalab_session.data_operations.fits_output_handler import FITSOutputHandler
 from datalab.datalab_session.exceptions import ClientAlertException
-from datalab.datalab_session.file_utils import crop_arrays, create_jpgs
+from datalab.datalab_session.utils.file_utils import crop_arrays, create_jpgs
 
 log = logging.getLogger()
 log.setLevel(logging.INFO)

datalab/datalab_session/data_operations/stacking.py

@@ -6,7 +6,7 @@
 from datalab.datalab_session.data_operations.fits_output_handler import FITSOutputHandler
 from datalab.datalab_session.data_operations.data_operation import BaseDataOperation
 from datalab.datalab_session.exceptions import ClientAlertException
-from datalab.datalab_session.file_utils import crop_arrays
+from datalab.datalab_session.utils.file_utils import crop_arrays
 
 log = logging.getLogger()
 log.setLevel(logging.INFO)

datalab/datalab_session/data_operations/subtraction.py

@@ -6,7 +6,7 @@
 from datalab.datalab_session.data_operations.data_operation import BaseDataOperation
 from datalab.datalab_session.data_operations.fits_output_handler import FITSOutputHandler
 from datalab.datalab_session.exceptions import ClientAlertException
-from datalab.datalab_session.file_utils import crop_arrays
+from datalab.datalab_session.utils.file_utils import crop_arrays
 
 log = logging.getLogger()
 log.setLevel(logging.INFO)

datalab/datalab_session/tests/test_operations.py

@@ -168,7 +168,7 @@ def tearDown(self):
         self.clean_test_dir()
         return super().tearDown()
 
-    @mock.patch('datalab.datalab_session.file_utils.tempfile.NamedTemporaryFile')
+    @mock.patch('datalab.datalab_session.utils.file_utils.tempfile.NamedTemporaryFile')
     @mock.patch('datalab.datalab_session.data_operations.input_data_handler.get_fits')
     @mock.patch('datalab.datalab_session.data_operations.fits_output_handler.save_fits_and_thumbnails')
     @mock.patch('datalab.datalab_session.data_operations.fits_output_handler.create_jpgs')
@@ -223,7 +223,7 @@ def tearDown(self):
 
     @mock.patch('datalab.datalab_session.data_operations.fits_output_handler.save_fits_and_thumbnails')
     @mock.patch('datalab.datalab_session.data_operations.fits_output_handler.create_jpgs')
-    @mock.patch('datalab.datalab_session.file_utils.tempfile.NamedTemporaryFile')
+    @mock.patch('datalab.datalab_session.utils.file_utils.tempfile.NamedTemporaryFile')
     @mock.patch('datalab.datalab_session.data_operations.input_data_handler.get_fits')
     def test_operate(self, mock_get_fits, mock_named_tempfile, mock_create_jpgs, mock_save_fits_and_thumbnails):
 
@@ -264,7 +264,7 @@ def tearDown(self):
         self.clean_test_dir()
         return super().tearDown()
 
-    @mock.patch('datalab.datalab_session.file_utils.tempfile.NamedTemporaryFile')
+    @mock.patch('datalab.datalab_session.utils.file_utils.tempfile.NamedTemporaryFile')
     @mock.patch('datalab.datalab_session.data_operations.input_data_handler.get_fits')
     @mock.patch('datalab.datalab_session.data_operations.fits_output_handler.save_fits_and_thumbnails')
     @mock.patch('datalab.datalab_session.data_operations.fits_output_handler.create_jpgs')

datalab/datalab_session/tests/test_utils.py

@@ -1,5 +1,5 @@
-from datalab.datalab_session.file_utils import *
-from datalab.datalab_session.s3_utils import *
+from datalab.datalab_session.utils.file_utils import *
+from datalab.datalab_session.utils.s3_utils import *
 from datalab.datalab_session.tests.test_files.file_extended_test_case import FileExtendedTestCase
 
 class FileUtilsTestClass(FileExtendedTestCase):

datalab/datalab_session/file_utils.py → datalab/datalab_session/utils/file_utils.py

@@ -6,7 +6,7 @@
 from fits2image.conversions import fits_to_jpg, fits_to_tif
 
 from datalab.datalab_session.exceptions import ClientAlertException
-from datalab.datalab_session.s3_utils import save_fits_and_thumbnails
+from datalab.datalab_session.utils.s3_utils import save_fits_and_thumbnails
 
 log = logging.getLogger()
 log.setLevel(logging.INFO)