From 94026c3f953b25d60e389145d578f32ff8344dc0 Mon Sep 17 00:00:00 2001 From: Jon Date: Mon, 16 Dec 2024 21:27:28 -0100 Subject: [PATCH 1/3] Fix raw_data endpoint to work with 32 bit images with negative values --- datalab/datalab_session/analysis/raw_data.py | 27 +++++++++++++------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/datalab/datalab_session/analysis/raw_data.py b/datalab/datalab_session/analysis/raw_data.py index 38c43b9..4601e94 100644 --- a/datalab/datalab_session/analysis/raw_data.py +++ b/datalab/datalab_session/analysis/raw_data.py @@ -23,27 +23,36 @@ def raw_data(input: dict): image = Image.fromarray(image_data) newImage = image.resize((max_size, max_size), Image.LANCZOS) bitpix = abs(int(sci_hdu.header.get('BITPIX', 16))) + max_value = int(sci_hdu.header.get('SATURATE', 0)) # If saturate header is present, use that as max value match bitpix: case 8: datatype = np.uint8 - max_value = np.iinfo(datatype).max + if not max_value: + max_value = np.iinfo(datatype).max case 16: datatype = np.float16 - max_value = np.finfo(datatype).max + if not max_value: + max_value = np.finfo(datatype).max case 32: datatype = np.float32 - max_value = np.finfo(datatype).max + if not max_value: + max_value = np.finfo(datatype).max scaled_array = np.asarray(newImage).astype(datatype) scaled_array_flipped = np.flip(scaled_array, axis=0) + # Set the zmin/zmax to integer values for calculating bins + zmin = math.floor(zmin) + zmax = math.ceil(zmax) + # Here we do a crazy histogram scaling to stretch the points in between zmin and zmax since that is where most detail is # We have 10 bins before zmin, 100 between zmin and zmax and 10 after zmax. + zero_point = math.floor(min(np.min(samples), 0)) # This is for images whose values go below 0 lower_bound = int(zmin * 0.8) # Increase resolution slightly below zmin upper_bound = int(zmax*1.2) # Increase resolution slightly beyond zmax - lower_step = int(lower_bound / 10) - upper_step = int((max_value - upper_bound) / 10) - step = int((upper_bound - lower_bound) / 100) - bins = np.arange(0, lower_bound, lower_step).tolist() + lower_step = int(abs(lower_bound / 10)) + upper_step = int(abs((max_value - upper_bound) / 10)) + step = int(abs((upper_bound - lower_bound) / 100)) + bins = np.arange(zero_point, lower_bound, lower_step).tolist() bins += np.arange(lower_bound, upper_bound, step).tolist() bins += np.arange(upper_bound, max_value, upper_step).tolist() histogram, bin_edges = np.histogram(samples, bins=bins) @@ -67,7 +76,7 @@ def raw_data(input: dict): 'width': scaled_array.shape[1], 'histogram': hist, 'bins': bin_middles, - 'zmin': int(median), - 'zmax': int(zmax), + 'zmin': round(median), + 'zmax': round(zmax), 'bitdepth': bitpix } From 2a89e24f8bc4c2117a906606129e6ff9349d8994 Mon Sep 17 00:00:00 2001 From: Lloyd Dakin Date: Wed, 8 Jan 2025 11:13:35 -0800 Subject: [PATCH 2/3] moving util files --- datalab/datalab_session/analysis/get_tif.py | 4 ++-- datalab/datalab_session/analysis/line_profile.py | 4 ++-- datalab/datalab_session/analysis/raw_data.py | 4 ++-- datalab/datalab_session/analysis/source_catalog.py | 4 ++-- datalab/datalab_session/data_operations/data_operation.py | 4 ++-- .../datalab_session/data_operations/fits_output_handler.py | 4 ++-- datalab/datalab_session/data_operations/input_data_handler.py | 4 ++-- datalab/datalab_session/data_operations/median.py | 2 +- datalab/datalab_session/data_operations/rgb_stack.py | 3 +-- datalab/datalab_session/data_operations/stacking.py | 2 +- datalab/datalab_session/data_operations/subtraction.py | 2 +- datalab/datalab_session/tests/test_utils.py | 4 ++-- datalab/datalab_session/{ => utils}/file_utils.py | 2 +- datalab/datalab_session/{ => utils}/s3_utils.py | 0 14 files changed, 21 insertions(+), 22 deletions(-) rename datalab/datalab_session/{ => utils}/file_utils.py (98%) rename datalab/datalab_session/{ => utils}/s3_utils.py (100%) diff --git a/datalab/datalab_session/analysis/get_tif.py b/datalab/datalab_session/analysis/get_tif.py index 211ecac..f96866e 100644 --- a/datalab/datalab_session/analysis/get_tif.py +++ b/datalab/datalab_session/analysis/get_tif.py @@ -1,5 +1,5 @@ -from datalab.datalab_session.file_utils import create_tif -from datalab.datalab_session.s3_utils import key_exists, add_file_to_bucket, get_s3_url, get_fits +from datalab.datalab_session.utils.file_utils import create_tif +from datalab.datalab_session.utils.s3_utils import key_exists, add_file_to_bucket, get_s3_url, get_fits def get_tif(input: dict): """ diff --git a/datalab/datalab_session/analysis/line_profile.py b/datalab/datalab_session/analysis/line_profile.py index 02c10cb..45bffff 100644 --- a/datalab/datalab_session/analysis/line_profile.py +++ b/datalab/datalab_session/analysis/line_profile.py @@ -3,8 +3,8 @@ from astropy.wcs import WcsError from astropy import coordinates -from datalab.datalab_session.file_utils import scale_points, get_hdu -from datalab.datalab_session.s3_utils import get_fits +from datalab.datalab_session.utils.file_utils import scale_points, get_hdu +from datalab.datalab_session.utils.s3_utils import get_fits # For creating an array of brightness along a user drawn line def line_profile(input: dict): diff --git a/datalab/datalab_session/analysis/raw_data.py b/datalab/datalab_session/analysis/raw_data.py index 4601e94..c1e2a25 100644 --- a/datalab/datalab_session/analysis/raw_data.py +++ b/datalab/datalab_session/analysis/raw_data.py @@ -1,8 +1,8 @@ import numpy as np import math from PIL import Image -from datalab.datalab_session.s3_utils import get_fits -from datalab.datalab_session.file_utils import get_hdu +from datalab.datalab_session.utils.s3_utils import get_fits +from datalab.datalab_session.utils.file_utils import get_hdu from fits2image.scaling import extract_samples, calc_zscale_min_max # TODO: This analysis endpoint assumes the image to be of 16 bitdepth. We should make this agnositc to bit depth in the future diff --git a/datalab/datalab_session/analysis/source_catalog.py b/datalab/datalab_session/analysis/source_catalog.py index b4f8bc9..1e5b3e0 100644 --- a/datalab/datalab_session/analysis/source_catalog.py +++ b/datalab/datalab_session/analysis/source_catalog.py @@ -1,7 +1,7 @@ import numpy as np -from datalab.datalab_session.file_utils import get_hdu, scale_points -from datalab.datalab_session.s3_utils import get_fits +from datalab.datalab_session.utils.file_utils import get_hdu, scale_points +from datalab.datalab_session.utils.s3_utils import get_fits # Source catalog Function Definition # ARGS: input (dict) diff --git a/datalab/datalab_session/data_operations/data_operation.py b/datalab/datalab_session/data_operations/data_operation.py index e87d158..574f373 100644 --- a/datalab/datalab_session/data_operations/data_operation.py +++ b/datalab/datalab_session/data_operations/data_operation.py @@ -5,9 +5,9 @@ from django.core.cache import cache import numpy as np -from datalab.datalab_session.s3_utils import get_fits +from datalab.datalab_session.utils.s3_utils import get_fits from datalab.datalab_session.tasks import execute_data_operation -from datalab.datalab_session.file_utils import get_hdu +from datalab.datalab_session.utils.file_utils import get_hdu CACHE_DURATION = 60 * 60 * 24 * 30 # cache for 30 days diff --git a/datalab/datalab_session/data_operations/fits_output_handler.py b/datalab/datalab_session/data_operations/fits_output_handler.py index f0831c1..1e2bbca 100644 --- a/datalab/datalab_session/data_operations/fits_output_handler.py +++ b/datalab/datalab_session/data_operations/fits_output_handler.py @@ -2,8 +2,8 @@ import numpy as np from astropy.io import fits -from datalab.datalab_session.file_utils import create_jpgs -from datalab.datalab_session.s3_utils import save_fits_and_thumbnails +from datalab.datalab_session.utils.file_utils import create_jpgs +from datalab.datalab_session.utils.s3_utils import save_fits_and_thumbnails class FITSOutputHandler(): diff --git a/datalab/datalab_session/data_operations/input_data_handler.py b/datalab/datalab_session/data_operations/input_data_handler.py index 4b03c2f..b0f1c1b 100644 --- a/datalab/datalab_session/data_operations/input_data_handler.py +++ b/datalab/datalab_session/data_operations/input_data_handler.py @@ -1,7 +1,7 @@ from astropy.io import fits -from datalab.datalab_session.s3_utils import get_fits -from datalab.datalab_session.file_utils import get_hdu +from datalab.datalab_session.utils.s3_utils import get_fits +from datalab.datalab_session.utils.file_utils import get_hdu class InputDataHandler(): """A class to read FITS files and provide access to the data. diff --git a/datalab/datalab_session/data_operations/median.py b/datalab/datalab_session/data_operations/median.py index c0351ae..2bd3d44 100644 --- a/datalab/datalab_session/data_operations/median.py +++ b/datalab/datalab_session/data_operations/median.py @@ -6,7 +6,7 @@ from datalab.datalab_session.data_operations.fits_output_handler import FITSOutputHandler from datalab.datalab_session.data_operations.data_operation import BaseDataOperation from datalab.datalab_session.exceptions import ClientAlertException -from datalab.datalab_session.file_utils import crop_arrays +from datalab.datalab_session.utils.file_utils import crop_arrays log = logging.getLogger() log.setLevel(logging.INFO) diff --git a/datalab/datalab_session/data_operations/rgb_stack.py b/datalab/datalab_session/data_operations/rgb_stack.py index 04c693a..ff6b3b1 100644 --- a/datalab/datalab_session/data_operations/rgb_stack.py +++ b/datalab/datalab_session/data_operations/rgb_stack.py @@ -1,13 +1,12 @@ import logging -from astropy.io import fits import numpy as np from datalab.datalab_session.data_operations.input_data_handler import InputDataHandler from datalab.datalab_session.data_operations.data_operation import BaseDataOperation from datalab.datalab_session.data_operations.fits_output_handler import FITSOutputHandler from datalab.datalab_session.exceptions import ClientAlertException -from datalab.datalab_session.file_utils import crop_arrays, create_jpgs +from datalab.datalab_session.utils.file_utils import crop_arrays, create_jpgs log = logging.getLogger() log.setLevel(logging.INFO) diff --git a/datalab/datalab_session/data_operations/stacking.py b/datalab/datalab_session/data_operations/stacking.py index 655e0f3..0292f22 100644 --- a/datalab/datalab_session/data_operations/stacking.py +++ b/datalab/datalab_session/data_operations/stacking.py @@ -6,7 +6,7 @@ from datalab.datalab_session.data_operations.fits_output_handler import FITSOutputHandler from datalab.datalab_session.data_operations.data_operation import BaseDataOperation from datalab.datalab_session.exceptions import ClientAlertException -from datalab.datalab_session.file_utils import crop_arrays +from datalab.datalab_session.utils.file_utils import crop_arrays log = logging.getLogger() log.setLevel(logging.INFO) diff --git a/datalab/datalab_session/data_operations/subtraction.py b/datalab/datalab_session/data_operations/subtraction.py index ccfcea6..450f766 100644 --- a/datalab/datalab_session/data_operations/subtraction.py +++ b/datalab/datalab_session/data_operations/subtraction.py @@ -6,7 +6,7 @@ from datalab.datalab_session.data_operations.data_operation import BaseDataOperation from datalab.datalab_session.data_operations.fits_output_handler import FITSOutputHandler from datalab.datalab_session.exceptions import ClientAlertException -from datalab.datalab_session.file_utils import crop_arrays +from datalab.datalab_session.utils.file_utils import crop_arrays log = logging.getLogger() log.setLevel(logging.INFO) diff --git a/datalab/datalab_session/tests/test_utils.py b/datalab/datalab_session/tests/test_utils.py index a03ea4b..375d13a 100644 --- a/datalab/datalab_session/tests/test_utils.py +++ b/datalab/datalab_session/tests/test_utils.py @@ -1,5 +1,5 @@ -from datalab.datalab_session.file_utils import * -from datalab.datalab_session.s3_utils import * +from datalab.datalab_session.utils.file_utils import * +from datalab.datalab_session.utils.s3_utils import * from datalab.datalab_session.tests.test_files.file_extended_test_case import FileExtendedTestCase class FileUtilsTestClass(FileExtendedTestCase): diff --git a/datalab/datalab_session/file_utils.py b/datalab/datalab_session/utils/file_utils.py similarity index 98% rename from datalab/datalab_session/file_utils.py rename to datalab/datalab_session/utils/file_utils.py index a2730ac..0137879 100644 --- a/datalab/datalab_session/file_utils.py +++ b/datalab/datalab_session/utils/file_utils.py @@ -6,7 +6,7 @@ from fits2image.conversions import fits_to_jpg, fits_to_tif from datalab.datalab_session.exceptions import ClientAlertException -from datalab.datalab_session.s3_utils import save_fits_and_thumbnails +from datalab.datalab_session.utils.s3_utils import save_fits_and_thumbnails log = logging.getLogger() log.setLevel(logging.INFO) diff --git a/datalab/datalab_session/s3_utils.py b/datalab/datalab_session/utils/s3_utils.py similarity index 100% rename from datalab/datalab_session/s3_utils.py rename to datalab/datalab_session/utils/s3_utils.py From 2179ee33a7c46d8afb4bae477ab642c017f99ca3 Mon Sep 17 00:00:00 2001 From: Lloyd Dakin Date: Wed, 8 Jan 2025 11:54:19 -0800 Subject: [PATCH 3/3] fixed tests from file utils move --- datalab/datalab_session/tests/test_operations.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/datalab/datalab_session/tests/test_operations.py b/datalab/datalab_session/tests/test_operations.py index 00ffd76..2743de1 100644 --- a/datalab/datalab_session/tests/test_operations.py +++ b/datalab/datalab_session/tests/test_operations.py @@ -168,7 +168,7 @@ def tearDown(self): self.clean_test_dir() return super().tearDown() - @mock.patch('datalab.datalab_session.file_utils.tempfile.NamedTemporaryFile') + @mock.patch('datalab.datalab_session.utils.file_utils.tempfile.NamedTemporaryFile') @mock.patch('datalab.datalab_session.data_operations.input_data_handler.get_fits') @mock.patch('datalab.datalab_session.data_operations.fits_output_handler.save_fits_and_thumbnails') @mock.patch('datalab.datalab_session.data_operations.fits_output_handler.create_jpgs') @@ -223,7 +223,7 @@ def tearDown(self): @mock.patch('datalab.datalab_session.data_operations.fits_output_handler.save_fits_and_thumbnails') @mock.patch('datalab.datalab_session.data_operations.fits_output_handler.create_jpgs') - @mock.patch('datalab.datalab_session.file_utils.tempfile.NamedTemporaryFile') + @mock.patch('datalab.datalab_session.utils.file_utils.tempfile.NamedTemporaryFile') @mock.patch('datalab.datalab_session.data_operations.input_data_handler.get_fits') def test_operate(self, mock_get_fits, mock_named_tempfile, mock_create_jpgs, mock_save_fits_and_thumbnails): @@ -264,7 +264,7 @@ def tearDown(self): self.clean_test_dir() return super().tearDown() - @mock.patch('datalab.datalab_session.file_utils.tempfile.NamedTemporaryFile') + @mock.patch('datalab.datalab_session.utils.file_utils.tempfile.NamedTemporaryFile') @mock.patch('datalab.datalab_session.data_operations.input_data_handler.get_fits') @mock.patch('datalab.datalab_session.data_operations.fits_output_handler.save_fits_and_thumbnails') @mock.patch('datalab.datalab_session.data_operations.fits_output_handler.create_jpgs')