diff --git a/astroquery/esa/integral/core.py b/astroquery/esa/integral/core.py index 788c4e8109..c83d598ca1 100644 --- a/astroquery/esa/integral/core.py +++ b/astroquery/esa/integral/core.py @@ -13,6 +13,7 @@ from astroquery.query import BaseQuery, BaseVOQuery from astroquery import log import pyvo +from requests import HTTPError from . import conf import time @@ -348,12 +349,13 @@ def download_science_windows(self, *, science_windows=None, observation_id=None, params = self.__get_science_window_parameter(science_windows, observation_id, revolution, proposal) params['RETRIEVAL_TYPE'] = 'SCW' try: + return esautils.download_file(url=conf.ISLA_DATA_SERVER, session=self.tap._session, filename=output_file, params=params, verbose=True) except Exception as e: log.error('No science windows have been found with these inputs. {}'.format(e)) - def get_timeline(self, ra, dec, *, radius=14, plot=False, plot_revno=False, plot_distance=False): + def get_timeline(self, ra, dec, *, radius=14): """Retrieve the INTEGRAL timeline associated to coordinates and radius Parameters @@ -364,13 +366,6 @@ def get_timeline(self, ra, dec, *, radius=14, plot=False, plot_revno=False, plot Declination radius: float or quantity, optional, default value 14 degrees radius in degrees (int, float) or quantity of the cone_search - plot: boolean, optional, default value False - show the timeline using matplotlib - plot_revno: boolean, optional, default value False - If plot is True, show in the X-Axis the revolution number - instead of the date - plot_distance: boolean, optional, default value False - If plot is True, show in the distance plot concatenated to the timeline Returns ------- @@ -406,20 +401,6 @@ def get_timeline(self, ra, dec, *, radius=14, plot=False, plot_revno=False, plot "scwOffAxis": data["scwOffAxis"] }) - # Plot the timeline if required - if plot: - x = timeline['scwRevs'] if plot_revno else timeline['scwTimes'] - x_label = 'Revolutions' if plot_revno else 'Calendar Dates' - if plot_distance: - esautils.plot_concatenated_results(x, timeline['scwExpo'] / 1000, timeline['scwOffAxis'], - x_label, 'Effective Exposure (ks)', 'Distance (deg)', - 'Observations', x_label='Pointing (Ks)', - y_label='Off-axis (deg)') - else: - esautils.plot_result(x, timeline['scwExpo'] / 1000, - x_label, 'Effective Exposure (ks)', - 'Observations') - return {'total_items': total_items, 'fraFC': fraFC, 'totEffExpo': totEffExpo, 'timeline': timeline} def get_epochs(self, *, target_name=None, instrument=None, band=None): @@ -447,76 +428,8 @@ def get_epochs(self, *, target_name=None, instrument=None, band=None): query = conf.ISLA_EPOCH_QUERY.format(instrument_oid, band_oid) return self.query_tap(query) - def get_long_term_timeseries(self, target_name, *, instrument=None, band=None, plot=False): - """Retrieve the INTEGRAL long term timeseries associated to the target and instrument pr bamd - - Parameters - ---------- - target_name : str, mandatory - target name to be requested, mandatory - instrument : str, optional - Possible values are in isla.instruments object - band : str, optional - Possible values are in isla.bandsobject - plot: boolean - show the long term timeseries using matplotlib - - Returns - ------- - An object containing: - source_id: id of the source - aggregation_value - total_items: total number of elements in the timeseries - aggregation_unit - detectors: a list of the detector available for that instrument - timeseries_list: A list of astropy.table object containing the long term timeseries - """ - - value = self.__get_instrument_or_band(instrument=instrument, band=band) - - query_params = { - 'REQUEST': 'long_timeseries', - "source": target_name, - "instrument_oid": self.instrument_band_map[value]['instrument_oid'] - } - try: - # Execute the request to the servlet - request_result = esautils.execute_servlet_request(url=conf.ISLA_SERVLET, - tap=self.tap, - query_params=query_params) - if 'detectors' not in request_result or len(request_result['detectors']) == 0: - raise ValueError('Please try with different input parameters.') - # Parse the long term timeseries - source_id = request_result['sourceId'] - aggregation_value = request_result['aggregationValue'] - total_items = request_result['totalItems'] - aggregation_unit = request_result['aggregationUnit'] - detectors = request_result['detectors'] - # Retrieve all the timeseries for each detector - timeseries_list = [] - for i in range(0, len(detectors)): - timeseries_list.append(Table({ - "time": [datetime.fromisoformat(timeseries_time) for timeseries_time in request_result["time"][i]], - "rates": request_result["rates"][i], - "ratesError": request_result["ratesError"][i], - })) - - # Plot the long term timeseries if required - if plot: - for i, timeseries in enumerate(timeseries_list): - esautils.plot_result(timeseries['time'], timeseries['rates'], - 'Time', 'Rate (cps)', - f"Long Term Timeseries ({detectors[i]})", error_y=timeseries['ratesError']) - - return {'source_id': source_id, 'aggregation_value': aggregation_value, - 'total_items': total_items, 'aggregation_unit': aggregation_unit, - 'detectors': detectors, 'timeseries_list': timeseries_list} - except ValueError as valueErr: - log.error('No long term timeseries have been found with these inputs. {}'.format(valueErr)) - except Exception as e: - log.error('Problem when retrieving long term timeseries. {}'.format(e)) - - def download_long_term_timeseries(self, target_name, *, instrument=None, band=None, output_file=None): + def get_long_term_timeseries(self, target_name, *, instrument=None, band=None, path='', filename=None, + cache=False, read_fits=True): """Method to download long term timeseries associated to an epoch and instrument or band Parameters @@ -527,13 +440,19 @@ def download_long_term_timeseries(self, target_name, *, instrument=None, band=No Possible values are in isla.instruments object band : str Possible values are in isla.bandsobject - - output_file: str, optional - File name and path for the downloaded file + path: str, optional + Path for the downloaded file + filename: str, optional + Filename for the downloaded file + cache: bool, optional, default False + Flag to determine if the file is stored in the cache or not + read_fits: bool, optional, default True + Open the downloaded file and parse the existing FITS files Returns ------- - The path and filename of the file with long term timeseries + If read_fits=True, a list with objects containing filename, path and FITS file opened with long + term timeseries. If read_fits=False, return the path of the downloaded file """ value = self.__get_instrument_or_band(instrument=instrument, band=band) @@ -542,84 +461,20 @@ def download_long_term_timeseries(self, target_name, *, instrument=None, band=No 'source': target_name, 'instrument_oid': self.instrument_band_map[value]['instrument_oid']} try: - return esautils.download_file(url=conf.ISLA_DATA_SERVER, session=self.tap._session, filename=output_file, - params=params, verbose=True) - except Exception as e: - log.error('No long term timeseries have been found with these inputs. {}'.format(e)) - - def get_short_term_timeseries(self, target_name, epoch, instrument=None, band=None, *, plot=False): - """Retrieve the INTEGRAL short term timeseries associated to the target and instrument or band - - Parameters - ---------- - target_name : str, mandatory - target name to be requested, mandatory - epoch : str, mandatory - reference epoch for the short term timeseries - instrument : str, optional - Possible values are in isla.instruments object - band : str, optional - Possible values are in isla.bandsobject - plot: boolean, optional - show the long term timeseries using matplotlib - - Returns - ------- - An object containing: - source_id: id of the source - total_items: total number of elements in the timeseries - detectors: a list of the detector available for that instrument - timeseries_list: A list of astropy.table object containing the short term timeseries - """ - - self.__validate_epoch(target_name=target_name, epoch=epoch, - instrument=instrument, band=band) - - value = self.__get_instrument_or_band(instrument=instrument, band=band) - - query_params = { - 'REQUEST': 'short_timeseries', - "source": target_name, - "band_oid": self.instrument_band_map[value]['band_oid'], - "epoch": epoch - } - try: - # Execute the request to the servlet - request_result = esautils.execute_servlet_request(url=conf.ISLA_SERVLET, - tap=self.tap, - query_params=query_params) - - if 'detectors' not in request_result or len(request_result['detectors']) == 0: - raise ValueError('Please try with different input parameters.') - - # Parse the short term timeseries - source_id = request_result['sourceId'] - total_items = request_result['totalItems'] - detectors = request_result['detectors'] - # Retrieve all the timeseries for each detector - timeseries_list = [] - for i in range(0, len(detectors)): - timeseries_list.append(Table({ - "time": [datetime.fromisoformat(timeseries_time) for timeseries_time in request_result["time"][i]], - "rates": request_result["rates"][i], - "rates_error": request_result["ratesError"][i], - })) - - # Plot the short term timeseries if required - if plot: - for i, timeseries in enumerate(timeseries_list): - esautils.plot_result(timeseries['time'], timeseries['rates'], - 'Time', 'Rate (cps)', - f"Light curve ({detectors[i]})", error_y=timeseries['rates_error']) - - return {'source_id': source_id, 'total_items': total_items, 'detectors': detectors, - 'timeseries_list': timeseries_list} - except ValueError as valueErr: - log.error('No short term timeseries have been found with these inputs. {}'.format(valueErr)) + downloaded_file = esautils.download_file(url=conf.ISLA_DATA_SERVER, session=self.tap._session, + params=params, path=path, filename=filename, + cache=cache, cache_folder=self.cache_location, verbose=True) + if read_fits: + return esautils.read_downloaded_fits([downloaded_file]) + else: + return downloaded_file + except HTTPError as err: + log.error('No long term timeseries have been found with these inputs. {}'.format(err)) except Exception as e: - log.error('Problem when retrieving short term timeseries. {}'.format(e)) + log.error('Problem when retrieving long term timeseries. {}'.format(e)) - def download_short_term_timeseries(self, target_name, epoch, *, instrument=None, band=None, output_file=None): + def get_short_term_timeseries(self, target_name, epoch, instrument=None, band=None, + path='', filename=None, cache=False, read_fits=True): """Method to download short term timeseries associated to an epoch and instrument or band Parameters @@ -632,12 +487,19 @@ def download_short_term_timeseries(self, target_name, epoch, *, instrument=None, Possible values are in isla.instruments object band : str, optional Possible values are in isla.bandsobject - output_file: str, optional - File name and path for the downloaded file + path: str, optional + Path for the downloaded file + filename: str, optional + Filename for the downloaded file + cache: bool, optional, default False + Flag to determine if the file is stored in the cache or not + read_fits: bool, optional, default True + Open the downloaded file and parse the existing FITS files Returns ------- - The path and filename of the file with short term timeseries + If read_fits=True, a list with objects containing filename, path and FITS file opened with short + term timeseries. If read_fits=False, return the path of the downloaded file """ @@ -650,37 +512,48 @@ def download_short_term_timeseries(self, target_name, epoch, *, instrument=None, 'source': target_name, 'band_oid': self.instrument_band_map[value]['band_oid'], 'epoch': epoch} + try: - return esautils.download_file(url=conf.ISLA_DATA_SERVER, session=self.tap._session, filename=output_file, - params=params, - verbose=True) + downloaded_file = esautils.download_file(url=conf.ISLA_DATA_SERVER, session=self.tap._session, + params=params, path=path, filename=filename, + cache=cache, cache_folder=self.cache_location, verbose=True) + + if read_fits: + return esautils.read_downloaded_fits([downloaded_file]) + else: + return downloaded_file + except HTTPError as err: + log.error('No short term timeseries have been found with these inputs. {}'.format(err)) except Exception as e: - log.error('No short term timeseries have been found with these inputs. {}'.format(e)) - return + log.error('Problem when retrieving short term timeseries. {}'.format(e)) - def get_spectra(self, target_name, epoch, instrument=None, band=None, *, plot=False, - show_warning=True): - """Retrieve the INTEGRAL spectra associated to the target and instrument or band + def get_spectra(self, target_name, epoch, instrument=None, band=None, *, path='', filename=None, + cache=False, read_fits=True): + """Method to download mosaics associated to an epoch and instrument or band Parameters ---------- target_name : str, mandatory target name to be requested, mandatory - instrument : str, optional - Possible values are in isla.instruments object - band : str, optional - Possible values are in isla.bandsobject epoch : str, mandatory reference epoch for the short term timeseries - plot: boolean, optional, default False - show the long term timeseries using matplotlib - show_warning: boolean, optional, default True - show the warning message about the nature of this product + instrument : str + Possible values are in isla.instruments object + band : str + Possible values are in isla.bandsobject + path: str, optional + Path for the downloaded file + filename: str, optional + Filename for the downloaded file + cache: bool, optional, default False + Flag to determine if the file is stored in the cache or not + read_fits: bool, optional, default True + Open the downloaded file and parse the existing FITS files Returns ------- - spectrum: a list of objects containing the parameters of the spectra and - an astropy.table object containing the spectra + If read_fits=True, a list with objects containing filename, path and FITS file opened with spectra. + If read_fits=False, return a list of paths of the downloaded files """ self.__validate_epoch(target_name=target_name, epoch=epoch, @@ -705,98 +578,48 @@ def get_spectra(self, target_name, epoch, instrument=None, band=None, *, plot=Fa raise ValueError('Please try with different input parameters.') # Parse the spectrum - spectrum = [] + downloaded_files = [] for element in request_result: - spectra_element = {} - - spectra_element['spectra_oid'] = element['spectraOid'] - spectra_element['file_name'] = element['fileName'] - spectra_element['metadata'] = element['metadata'] - spectra_element['date_start'] = element['dateStart'] - spectra_element['date_stop'] = element['dateStop'] - spectra_element['detector'] = element['detector'] - # Retrieve all the timeseries for each detector - spectra_element['spectra'] = Table({"energy": element['energy'], - "energy_error": element["energyError"], - 'rate': element["rate"], - "rate_error": element["rateError"], - }) - - spectrum.append(spectra_element) - - # Plot the spectrum if required - if plot: - esautils.plot_result(spectra_element['spectra']['energy'], spectra_element['spectra']['rate'], - 'Energy (keV)', 'Counts s⁻¹ keV⁻¹', - 'Spectrum', error_x=spectra_element['spectra']['energy_error'], - error_y=spectra_element['spectra']['rate_error'], log_scale=True) - - self.__log_warning_message('get_spectra', 'download_spectra', show_warning) - return spectrum - except ValueError as valueErr: - log.error('Spectra are not available with these inputs. {}'.format(valueErr)) + params = {'RETRIEVAL_TYPE': 'spectras', + 'spectra_oid': element['spectraOid']} + downloaded_files.append( + esautils.download_file(url=conf.ISLA_DATA_SERVER, session=self.tap._session, + params=params, path=path, filename=filename, + cache=cache, cache_folder=self.cache_location, verbose=True)) + + if read_fits: + return esautils.read_downloaded_fits(downloaded_files) + else: + return downloaded_files + except ValueError as err: + log.error('Spectra are not available with these inputs. {}'.format(err)) except Exception as e: log.error('Problem when retrieving spectra. {}'.format(e)) - def download_spectra(self, target_name, epoch, *, instrument=None, band=None, output_file=None): + def get_mosaic(self, epoch, instrument=None, band=None, *, path='', filename=None, cache=False, read_fits=True): """Method to download mosaics associated to an epoch and instrument or band Parameters ---------- - target_name : str, mandatory - target name to be requested, mandatory epoch : str, mandatory reference epoch for the short term timeseries instrument : str Possible values are in isla.instruments object band : str Possible values are in isla.bandsobject - output_file: str, optional - File name and path for the downloaded file + cache: bool, optional, default False + Flag to determine if the file is stored in the cache or not + path: str, optional + Path for the downloaded file + filename: str, optional + Filename for the downloaded file + read_fits: bool, optional, default True + Open the downloaded file and parse the existing FITS files Returns ------- - A list of paths and filenames of the files with spectras - """ - - self.__validate_epoch(target_name=target_name, epoch=epoch, - instrument=instrument, band=band) - - spectrum = self.get_spectra(target_name=target_name, epoch=epoch, instrument=instrument, band=band, plot=False, - show_warning=False) - downloaded_files = [] - try: - for spectra in spectrum: - params = {'RETRIEVAL_TYPE': 'spectras', - 'spectra_oid': spectra['spectra_oid']} - downloaded_files.append( - esautils.download_file(url=conf.ISLA_DATA_SERVER, session=self.tap._session, filename=output_file, - params=params, - verbose=True)) - return downloaded_files - except Exception as e: - log.error('No spectra files have been found with these inputs. {}'.format(e)) - - def get_mosaic(self, epoch, instrument=None, band=None, *, plot=False, show_warning=True): - """Retrieve the INTEGRAL mosaics associated to the instrument or band - - Parameters - ---------- - epoch : str, mandatory - reference epoch for the short term timeseries - instrument : str, optional - Possible values are in isla.instruments object - band : str, optional - Possible values are in isla.bandsobject - plot: boolean, optional, default False - show the long term timeseries using matplotlib - show_warning: boolean, optional, default True - show the warning message about the nature of this product - - Returns - ------- - mosaics: a list of objects containing the parameters of the mosaic and - an astropy.table object containing the mosaic + If read_fits=True, a list with objects containing filename, path and FITS file opened with mosaics. + If read_fits=False, return a list of paths of the downloaded files """ self.__validate_epoch(epoch=epoch, @@ -819,69 +642,22 @@ def get_mosaic(self, epoch, instrument=None, band=None, *, plot=False, show_warn if len(request_result) == 0: raise ValueError('Please try with different input parameters.') - # Plot the mosaics if required - mosaics = [] + downloaded_files = [] for element in request_result: - mosaic_element = {} - - mosaic_element['file_name'] = element['fileName'] - mosaic_element['mosaic_oid'] = element['mosaicOid'] - mosaic_element['height'] = element['height'] - mosaic_element['width'] = element['width'] - mosaic_element['min_z_scale'] = element['minZScale'] - mosaic_element['max_z_scale'] = element['maxZScale'] - # Retrieve all the timeseries for each detector - mosaic_element['mosaic'] = Table({ - 'ra': np.array(element['ra'], dtype=float).flatten().flatten(), - 'dec': np.array(element['dec'], dtype=float).flatten().flatten(), - 'data': np.array(element['data'], dtype=float).flatten().flatten() - }) - - mosaics.append(mosaic_element) - if plot: - esautils.plot_image(z=mosaic_element['mosaic']['data'], - height=mosaic_element['height'], width=mosaic_element['width'], - plot_title='Mosaic') - - self.__log_warning_message('get_mosaic', 'download_mosaic', show_warning) - return mosaics - except ValueError as valErr: - log.error('Mosaics are not available for these inputs. {}'.format(valErr)) - except Exception as e: - log.error('Problem when retrieving mosaics. {}'.format(e)) - - def download_mosaic(self, epoch, *, instrument=None, band=None, output_file=None): - """Method to download mosaics associated to an epoch and instrument or band - - Parameters - ---------- - epoch : str, mandatory - reference epoch for the short term timeseries - instrument : str - Possible values are in isla.instruments object - band : str - Possible values are in isla.bandsobject - - output_file: str, optional - File name and path for the downloaded file - - Returns - ------- - A list of paths and filenames of the files with mosaics - """ - - mosaics = self.get_mosaic(epoch=epoch, instrument=instrument, band=band, plot=False, show_warning=False) - downloaded_files = [] - try: - for mosaic in mosaics: params = {'RETRIEVAL_TYPE': 'mosaics', - 'mosaic_oid': mosaic['mosaic_oid']} + 'mosaic_oid': element['mosaicOid']} downloaded_files.append( esautils.download_file(url=conf.ISLA_DATA_SERVER, session=self.tap._session, - filename=output_file, params=params, verbose=True)) - return downloaded_files + params=params, path=path, filename=filename, + cache=cache, cache_folder=self.cache_location, verbose=True)) + if read_fits: + return esautils.read_downloaded_fits(downloaded_files) + else: + return downloaded_files + except ValueError as err: + log.error('Mosaics are not available for these inputs. {}'.format(err)) except Exception as e: - log.error('No mosaics have been found with these inputs. {}'.format(e)) + log.error('Problem when retrieving mosaics. {}'.format(e)) def get_source_metadata(self, target_name): """Retrieve the metadata associated to an INTEGRAL target @@ -1055,14 +831,5 @@ def __get_science_window_parameter(self, science_windows, observation_id, revolu raise ValueError("Input parameters are wrong") - def __log_warning_message(self, get_method, download_method, show_warning=True): - if show_warning: - log.warning( - f"The plots and data provided by '{get_method}' have been developed using the automatic reduction " - f"pipeline at ISDC, using INTEGRAL OSA version 11.2. No manual scientific validation has been " - f"performed on these outputs. They should be examined and validated before being used for scientific " - f"purposes. To ensure accuracy and reliability, please use '{download_method}' to download " - f"the original source files directly.") - Integral = IntegralClass() diff --git a/astroquery/esa/integral/tests/data/lt.zip b/astroquery/esa/integral/tests/data/lt.zip new file mode 100644 index 0000000000..81fd39338b Binary files /dev/null and b/astroquery/esa/integral/tests/data/lt.zip differ diff --git a/astroquery/esa/integral/tests/data/mosaic.tar.gz b/astroquery/esa/integral/tests/data/mosaic.tar.gz new file mode 100644 index 0000000000..6879c325e3 Binary files /dev/null and b/astroquery/esa/integral/tests/data/mosaic.tar.gz differ diff --git a/astroquery/esa/integral/tests/data/spectra.tar b/astroquery/esa/integral/tests/data/spectra.tar new file mode 100644 index 0000000000..21fdb7b311 Binary files /dev/null and b/astroquery/esa/integral/tests/data/spectra.tar differ diff --git a/astroquery/esa/integral/tests/data/st.tar b/astroquery/esa/integral/tests/data/st.tar new file mode 100644 index 0000000000..21fdb7b311 Binary files /dev/null and b/astroquery/esa/integral/tests/data/st.tar differ diff --git a/astroquery/esa/integral/tests/mocks.py b/astroquery/esa/integral/tests/mocks.py index a66c48ba52..dc5e15ff86 100644 --- a/astroquery/esa/integral/tests/mocks.py +++ b/astroquery/esa/integral/tests/mocks.py @@ -113,7 +113,7 @@ def get_mock_mosaic(): }] -def get_mock_response(message): +def get_mock_response(): error_message = "Mocked HTTP error" mock_response = Mock() mock_response.raise_for_status.side_effect = HTTPError(error_message) diff --git a/astroquery/esa/integral/tests/setup_package.py b/astroquery/esa/integral/tests/setup_package.py new file mode 100644 index 0000000000..448d2cf2b6 --- /dev/null +++ b/astroquery/esa/integral/tests/setup_package.py @@ -0,0 +1,18 @@ +# Licensed under a 3-clause BSD style license - see LICENSE.rst +import os + + +# setup paths to the test data +# can specify a single file or a list of files +def get_package_data(): + paths = [os.path.join('data', '*.vot'), + os.path.join('data', '*.xml'), + os.path.join('data', '*.zip'), + os.path.join('data', '*.gz'), + os.path.join('data', '*.tar'), + os.path.join('data', '*.fits'), + os.path.join('data', '*.txt'), + ] # etc, add other extensions + # you can also enlist files individually by names + # finally construct and return a dict for the sub module + return {'astroquery.esa.integral.tests': paths} diff --git a/astroquery/esa/integral/tests/test_isla_tap.py b/astroquery/esa/integral/tests/test_isla_tap.py index ce8c3a83f0..438c54e27a 100644 --- a/astroquery/esa/integral/tests/test_isla_tap.py +++ b/astroquery/esa/integral/tests/test_isla_tap.py @@ -8,6 +8,8 @@ European Space Agency (ESA) """ +import os + from astropy.coordinates import SkyCoord from astroquery.esa.integral import IntegralClass @@ -24,6 +26,20 @@ def mock_instrument_bands(isla_module): isla_module.instruments, isla_module.bands, isla_module.instrument_band_map = mocks.get_instrument_bands() +def data_path(filename): + data_dir = os.path.join(os.path.dirname(__file__), 'data') + return os.path.join(data_dir, filename) + + +def close_file(file): + file.close() + + +def close_files(file_list): + for file in file_list: + close_file(file['fits']) + + class TestTap: def test_get_tables(self): @@ -100,9 +116,8 @@ def test_login_success(self, mock_post, instrument_band_mock): @patch('astroquery.esa.utils.utils.ESAAuthSession.post') def test_login_error(self, mock_post, instrument_band_mock): instrument_band_mock.return_value = mocks.get_instrument_bands() - error_message = "Mocked HTTP error" - mock_response = mocks.get_mock_response(error_message) + mock_response = mocks.get_mock_response() # Configure the mock post method to return the mock Response mock_post.return_value = mock_response @@ -137,7 +152,7 @@ def test_logout_error(self, mock_post, instrument_band_mock): instrument_band_mock.return_value = mocks.get_instrument_bands() error_message = "Mocked HTTP error" - mock_response = mocks.get_mock_response(error_message) + mock_response = mocks.get_mock_response() # Configure the mock post method to return the mock Response mock_post.return_value = mock_response @@ -328,37 +343,27 @@ def test_download_science_windows(self, instrument_band_mock, download_mock): assert kwargs['params']['RETRIEVAL_TYPE'] == 'SCW' @patch('astroquery.esa.integral.core.pyvo.dal.TAPService.capabilities', []) - @patch('astroquery.esa.utils.utils.plt.show') - @patch('astroquery.esa.utils.utils.plt.scatter') @patch('astroquery.esa.utils.utils.execute_servlet_request') @patch('astroquery.esa.integral.core.IntegralClass.get_instrument_band_map') - def test_get_timeline_no_distance(self, instrument_band_mock, servlet_mock, scatter_mock, plt_mock): + def test_get_timeline_no_distance(self, instrument_band_mock, servlet_mock): instrument_band_mock.return_value = mocks.get_instrument_bands() servlet_mock.return_value = mocks.get_mock_timeline() isla = IntegralClass() - isla.get_timeline(ra=83.63320922851562, dec=22.01447105407715, plot=True, plot_revno=True, plot_distance=False) + isla.get_timeline(ra=83.63320922851562, dec=22.01447105407715) args, kwargs = servlet_mock.call_args assert kwargs['query_params']['REQUEST'] == 'timelines' - scatter_mock.assert_called() - plt_mock.assert_called_with(block=False) - @patch('astroquery.esa.integral.core.pyvo.dal.TAPService.capabilities', []) - @patch('astroquery.esa.utils.utils.plt.show') - @patch('astroquery.esa.utils.utils.plt.scatter') @patch('astroquery.esa.utils.utils.execute_servlet_request') @patch('astroquery.esa.integral.core.IntegralClass.get_instrument_band_map') - def test_get_timeline_distance(self, instrument_band_mock, servlet_mock, scatter_mock, plt_mock): + def test_get_timeline_distance(self, instrument_band_mock, servlet_mock): instrument_band_mock.return_value = mocks.get_instrument_bands() servlet_mock.return_value = mocks.get_mock_timeline() isla = IntegralClass() - timeline = isla.get_timeline(ra=83.63320922851562, dec=22.01447105407715, plot=True, plot_revno=True, - plot_distance=True) - scatter_mock.assert_not_called() - plt_mock.assert_called_with(block=False) + timeline = isla.get_timeline(ra=83.63320922851562, dec=22.01447105407715) assert len(timeline['timeline']['scwRevs']) > 0 @@ -387,163 +392,157 @@ def test_get_epochs(self, instrument_band_mock, query_tap_mock): "(instrument_oid = id1 or band_oid = id2)") @patch('astroquery.esa.integral.core.pyvo.dal.TAPService.capabilities', []) + @patch('astroquery.esa.utils.utils.download_file') @patch('astroquery.esa.integral.core.log') - @patch('astroquery.esa.utils.utils.execute_servlet_request') @patch('astroquery.esa.integral.core.IntegralClass.get_instrument_band_map') - def test_get_long_term_timeseries_error(self, instrument_band_mock, servlet_mock, log_mock): + def test_get_long_term_timeseries_error(self, instrument_band_mock, log_mock, download_mock): instrument_band_mock.return_value = mocks.get_instrument_bands() - servlet_mock.return_value = {} + error_message = 'Error' + download_mock.side_effect = HTTPError(error_message) isla = IntegralClass() mock_instrument_bands(isla_module=isla) - isla.get_long_term_timeseries(target_name='J174537.0-290107', band='b1', plot=True) - log_mock.error.assert_called_with('No long term timeseries have been found with these inputs. ' - 'Please try with different input parameters.') - @patch('astroquery.esa.integral.core.pyvo.dal.TAPService.capabilities', []) - @patch('astroquery.esa.integral.core.log') - @patch('astroquery.esa.utils.utils.execute_servlet_request') - @patch('astroquery.esa.integral.core.IntegralClass.get_instrument_band_map') - def test_get_long_term_timeseries_exception(self, instrument_band_mock, servlet_mock, log_mock): - instrument_band_mock.return_value = mocks.get_instrument_bands() - error_message = "Mocked HTTP error" - mock_response = mocks.get_mock_response(error_message) - servlet_mock.side_effect = mock_response - - isla = IntegralClass() - mock_instrument_bands(isla_module=isla) - isla.get_long_term_timeseries(target_name='J174537.0-290107', band='b1', plot=True) - log_mock.error.assert_called_with("Problem when retrieving long term timeseries. " - "argument of type 'Mock' is not iterable") + isla.get_long_term_timeseries(target_name='J174537.0-290107', band='b1') + log_mock.error.assert_called_with('No long term timeseries have been found with these inputs. ' + error_message) @patch('astroquery.esa.integral.core.pyvo.dal.TAPService.capabilities', []) - @patch('astroquery.esa.utils.utils.execute_servlet_request') + @patch('astroquery.esa.utils.utils.download_file') + @patch('astroquery.esa.integral.core.log') @patch('astroquery.esa.integral.core.IntegralClass.get_instrument_band_map') - def test_get_long_term_timeseries(self, instrument_band_mock, servlet_mock): + def test_get_long_term_timeseries_exception(self, instrument_band_mock, log_mock, download_mock): instrument_band_mock.return_value = mocks.get_instrument_bands() - servlet_mock.return_value = mocks.get_mock_timeseries() + error_message = 'Error' + download_mock.side_effect = ValueError(error_message) isla = IntegralClass() mock_instrument_bands(isla_module=isla) - lt_timeseries_list = isla.get_long_term_timeseries(target_name='J174537.0-290107', band='b1', plot=True) - args, kwargs = servlet_mock.call_args - assert kwargs['query_params']['REQUEST'] == 'long_timeseries' - - assert len(lt_timeseries_list['timeseries_list']) == 2 + isla.get_long_term_timeseries(target_name='J174537.0-290107', band='b1') + log_mock.error.assert_called_with('Problem when retrieving long term timeseries. ' + error_message) @patch('astroquery.esa.integral.core.pyvo.dal.TAPService.capabilities', []) @patch('astroquery.esa.utils.utils.download_file') @patch('astroquery.esa.integral.core.IntegralClass.get_instrument_band_map') - def test_download_long_term_timeseries(self, instrument_band_mock, download_mock): + def test_get_long_term_timeseries(self, instrument_band_mock, download_mock): instrument_band_mock.return_value = mocks.get_instrument_bands() + download_mock.return_value = data_path('lt.zip') isla = IntegralClass() mock_instrument_bands(isla_module=isla) - isla.download_long_term_timeseries(target_name='J174537.0-290107', band='b1') + lt_timeseries_list_extracted = isla.get_long_term_timeseries(target_name='J174537.0-290107', band='b1') - args, kwargs = download_mock.call_args - assert kwargs['params']['RETRIEVAL_TYPE'] == 'long_timeseries' + assert len(lt_timeseries_list_extracted) == 2 + lt_timeseries_list_compressed = isla.get_long_term_timeseries(target_name='J174537.0-290107', band='b1', + read_fits=False) + + assert type(lt_timeseries_list_compressed) is str + close_files(lt_timeseries_list_extracted) @patch('astroquery.esa.integral.core.pyvo.dal.TAPService.capabilities', []) @patch('astroquery.esa.integral.core.log') - @patch('astroquery.esa.utils.utils.execute_servlet_request') + @patch('astroquery.esa.utils.utils.download_file') @patch('astroquery.esa.integral.core.IntegralClass.get_epochs') @patch('astroquery.esa.integral.core.IntegralClass.get_instrument_band_map') - def test_get_short_term_timeseries_error(self, instrument_band_mock, epoch_mock, servlet_mock, log_mock): + def test_get_short_term_timeseries_error(self, instrument_band_mock, epoch_mock, download_mock, log_mock): instrument_band_mock.return_value = mocks.get_instrument_bands() epoch_mock.return_value = {'epoch': ['time']} - servlet_mock.return_value = {} + error_message = 'Error' + download_mock.side_effect = HTTPError(error_message) isla = IntegralClass() mock_instrument_bands(isla_module=isla) isla.get_short_term_timeseries(target_name='target', - band='b1', epoch='time', plot=True) - log_mock.error.assert_called_with('No short term timeseries have been found with these inputs. ' - 'Please try with different input parameters.') + band='b1', epoch='time') + log_mock.error.assert_called_with( + 'No short term timeseries have been found with these inputs. {0}'.format(error_message)) @patch('astroquery.esa.integral.core.pyvo.dal.TAPService.capabilities', []) - @patch('astroquery.esa.integral.core.IntegralClass.get_epochs') @patch('astroquery.esa.integral.core.log') - @patch('astroquery.esa.utils.utils.execute_servlet_request') + @patch('astroquery.esa.utils.utils.download_file') + @patch('astroquery.esa.integral.core.IntegralClass.get_epochs') @patch('astroquery.esa.integral.core.IntegralClass.get_instrument_band_map') - def test_get_short_term_timeseries_exception(self, instrument_band_mock, servlet_mock, log_mock, epoch_mock): + def test_get_short_term_timeseries_exception(self, instrument_band_mock, epoch_mock, download_mock, log_mock): instrument_band_mock.return_value = mocks.get_instrument_bands() epoch_mock.return_value = {'epoch': ['time']} - error_message = "Mocked HTTP error" - mock_response = mocks.get_mock_response(error_message) - servlet_mock.side_effect = mock_response + error_message = 'Error' + download_mock.side_effect = ValueError(error_message) isla = IntegralClass() mock_instrument_bands(isla_module=isla) isla.get_short_term_timeseries(target_name='target', - band='b1', epoch='time', plot=True) - log_mock.error.assert_called_with("Problem when retrieving short term timeseries. " - "argument of type 'Mock' is not iterable") + band='b1', epoch='time') + log_mock.error.assert_called_with('Problem when retrieving short term timeseries. ' + error_message) @patch('astroquery.esa.integral.core.pyvo.dal.TAPService.capabilities', []) - @patch('astroquery.esa.utils.utils.execute_servlet_request') @patch('astroquery.esa.integral.core.IntegralClass.get_epochs') @patch('astroquery.esa.integral.core.IntegralClass.get_instrument_band_map') - def test_get_short_term_timeseries_epoch_error(self, instrument_band_mock, epoch_mock, servlet_mock): + def test_get_short_term_timeseries_epoch_error(self, instrument_band_mock, epoch_mock): + instrument_band_mock.return_value = mocks.get_instrument_bands() epoch_mock.return_value = {'epoch': ['time2']} - servlet_mock.return_value = mocks.get_mock_timeseries() isla = IntegralClass() mock_instrument_bands(isla_module=isla) with pytest.raises(ValueError) as err: isla.get_short_term_timeseries(target_name='target', - band='28_40', epoch='time', plot=True) + band='b1', epoch='time') assert 'Epoch time is not available for this target and instrument/band.' in err.value.args[0] @patch('astroquery.esa.integral.core.pyvo.dal.TAPService.capabilities', []) + @patch('astroquery.esa.utils.utils.download_file') @patch('astroquery.esa.integral.core.IntegralClass.get_epochs') - @patch('astroquery.esa.utils.utils.execute_servlet_request') @patch('astroquery.esa.integral.core.IntegralClass.get_instrument_band_map') - def test_get_short_term_timeseries(self, instrument_band_mock, servlet_mock, epoch_mock): + def test_get_short_term_timeseries(self, instrument_band_mock, epoch_mock, download_mock): + instrument_band_mock.return_value = mocks.get_instrument_bands() epoch_mock.return_value = {'epoch': ['time']} - servlet_mock.return_value = mocks.get_mock_timeseries() + download_mock.return_value = data_path('st.tar') isla = IntegralClass() mock_instrument_bands(isla_module=isla) - st_timeseries_list = isla.get_short_term_timeseries(target_name='target', - band='b1', epoch='time', plot=True) - args, kwargs = servlet_mock.call_args - assert kwargs['query_params']['REQUEST'] == 'short_timeseries' + st_timeseries_list_extracted = isla.get_short_term_timeseries(target_name='target', + band='b1', epoch='time') + assert len(st_timeseries_list_extracted) == 3 - assert len(st_timeseries_list['timeseries_list']) == 2 + st_timeseries_list_compressed = isla.get_short_term_timeseries(target_name='target', + band='b1', epoch='time', read_fits=False) + assert type(st_timeseries_list_compressed) is str + + close_files(st_timeseries_list_extracted) @patch('astroquery.esa.integral.core.pyvo.dal.TAPService.capabilities', []) + @patch('astroquery.esa.integral.core.log') + @patch('astroquery.esa.utils.utils.execute_servlet_request') @patch('astroquery.esa.integral.core.IntegralClass.get_epochs') - @patch('astroquery.esa.utils.utils.download_file') @patch('astroquery.esa.integral.core.IntegralClass.get_instrument_band_map') - def test_download_short_term_timeseries(self, instrument_band_mock, download_mock, epoch_mock): + def test_get_spectra_error_server(self, instrument_band_mock, epoch_mock, servlet_mock, log_mock): instrument_band_mock.return_value = mocks.get_instrument_bands() - epoch_mock.return_value = {'epoch': ['today']} + epoch_mock.return_value = {'epoch': ['time']} + error_message = 'Error' + servlet_mock.side_effect = HTTPError(error_message) + isla = IntegralClass() mock_instrument_bands(isla_module=isla) - isla.download_short_term_timeseries(target_name='J174537.0-290107', epoch='today', band='b1') - - args, kwargs = download_mock.call_args - assert kwargs['params']['RETRIEVAL_TYPE'] == 'short_timeseries' - download_mock.assert_called() + isla.get_spectra(target_name='target', + band='b1', epoch='time') + log_mock.error.assert_called_with('Problem when retrieving spectra. ' + 'Error') @patch('astroquery.esa.integral.core.pyvo.dal.TAPService.capabilities', []) @patch('astroquery.esa.integral.core.log') @patch('astroquery.esa.utils.utils.execute_servlet_request') @patch('astroquery.esa.integral.core.IntegralClass.get_epochs') @patch('astroquery.esa.integral.core.IntegralClass.get_instrument_band_map') - def test_get_spectra_error(self, instrument_band_mock, epoch_mock, servlet_mock, log_mock): + def test_get_spectra_no_values(self, instrument_band_mock, epoch_mock, servlet_mock, log_mock): instrument_band_mock.return_value = mocks.get_instrument_bands() epoch_mock.return_value = {'epoch': ['time']} - servlet_mock.return_value = {} + servlet_mock.return_value = [] isla = IntegralClass() mock_instrument_bands(isla_module=isla) isla.get_spectra(target_name='target', - band='b1', epoch='time', plot=True) + band='b1', epoch='time') log_mock.error.assert_called_with('Spectra are not available with these inputs. ' 'Please try with different input parameters.') @@ -555,66 +554,52 @@ def test_get_spectra_error(self, instrument_band_mock, epoch_mock, servlet_mock, def test_get_spectra_exception(self, instrument_band_mock, servlet_mock, log_mock, epoch_mock): instrument_band_mock.return_value = mocks.get_instrument_bands() epoch_mock.return_value = {'epoch': ['time']} - error_message = "Mocked HTTP error" - mock_response = mocks.get_mock_response(error_message) + mock_response = mocks.get_mock_response() servlet_mock.side_effect = mock_response isla = IntegralClass() mock_instrument_bands(isla_module=isla) isla.get_spectra(target_name='target', - band='b1', epoch='time', plot=True) + band='b1', epoch='time') log_mock.error.assert_called_with("Problem when retrieving spectra. " "object of type 'Mock' has no len()") @patch('astroquery.esa.integral.core.pyvo.dal.TAPService.capabilities', []) + @patch('astroquery.esa.utils.utils.download_file') @patch('astroquery.esa.integral.core.IntegralClass.get_epochs') @patch('astroquery.esa.utils.utils.execute_servlet_request') @patch('astroquery.esa.integral.core.IntegralClass.get_instrument_band_map') - def test_get_spectra(self, instrument_band_mock, servlet_mock, epoch_mock): + def test_get_spectra(self, instrument_band_mock, servlet_mock, epoch_mock, download_mock): instrument_band_mock.return_value = mocks.get_instrument_bands() servlet_mock.return_value = mocks.get_mock_spectra() epoch_mock.return_value = {'epoch': ['today']} + download_mock.return_value = data_path('spectra.tar') isla = IntegralClass() mock_instrument_bands(isla_module=isla) - spectra_list = isla.get_spectra(target_name='target', - epoch='today', band='b1', plot=True) - args, kwargs = servlet_mock.call_args - assert kwargs['query_params']['REQUEST'] == 'spectra' - - assert len(spectra_list) == 1 - - @patch('astroquery.esa.integral.core.pyvo.dal.TAPService.capabilities', []) - @patch('astroquery.esa.integral.core.IntegralClass.get_epochs') - @patch('astroquery.esa.utils.utils.execute_servlet_request') - @patch('astroquery.esa.utils.utils.download_file') - @patch('astroquery.esa.integral.core.IntegralClass.get_instrument_band_map') - def test_download_spectra(self, instrument_band_mock, download_mock, servlet_mock, epoch_mock): - instrument_band_mock.return_value = mocks.get_instrument_bands() - servlet_mock.return_value = mocks.get_mock_spectra() - epoch_mock.return_value = {'epoch': ['today']} + spectra_list_extracted = isla.get_spectra(target_name='target', + epoch='today', band='b1') + assert len(spectra_list_extracted) == 3 - isla = IntegralClass() - mock_instrument_bands(isla_module=isla) - isla.download_spectra(target_name='J174537.0-290107', epoch='today', band='b1') + spectra_list_compressed = isla.get_spectra(target_name='target', + epoch='today', band='b1', read_fits=False) + assert type(spectra_list_compressed) is list - args, kwargs = download_mock.call_args - assert kwargs['params']['RETRIEVAL_TYPE'] == 'spectras' - download_mock.assert_called() + close_files(spectra_list_extracted) @patch('astroquery.esa.integral.core.pyvo.dal.TAPService.capabilities', []) @patch('astroquery.esa.integral.core.log') @patch('astroquery.esa.utils.utils.execute_servlet_request') @patch('astroquery.esa.integral.core.IntegralClass.get_epochs') @patch('astroquery.esa.integral.core.IntegralClass.get_instrument_band_map') - def test_get_mosaic_error(self, instrument_band_mock, epoch_mock, servlet_mock, log_mock): + def test_get_mosaic_no_values(self, instrument_band_mock, epoch_mock, servlet_mock, log_mock): instrument_band_mock.return_value = mocks.get_instrument_bands() epoch_mock.return_value = {'epoch': ['time']} - servlet_mock.return_value = {} + servlet_mock.return_value = [] isla = IntegralClass() mock_instrument_bands(isla_module=isla) - isla.get_mosaic(epoch='time', instrument='i1', plot=True) + isla.get_mosaic(epoch='time', instrument='i1') log_mock.error.assert_called_with('Mosaics are not available for these inputs. ' 'Please try with different input parameters.') @@ -626,52 +611,35 @@ def test_get_mosaic_error(self, instrument_band_mock, epoch_mock, servlet_mock, def test_get_mosaic_exception(self, instrument_band_mock, servlet_mock, log_mock, epoch_mock): instrument_band_mock.return_value = mocks.get_instrument_bands() epoch_mock.return_value = {'epoch': ['time']} - error_message = "Mocked HTTP error" - mock_response = mocks.get_mock_response(error_message) - servlet_mock.side_effect = mock_response + error_message = 'Error' + servlet_mock.side_effect = HTTPError(error_message) isla = IntegralClass() mock_instrument_bands(isla_module=isla) - isla.get_mosaic(epoch='time', instrument='i1', plot=True) + isla.get_mosaic(epoch='time', instrument='i1') log_mock.error.assert_called_with("Problem when retrieving mosaics. " - "object of type 'Mock' has no len()") + "Error") @patch('astroquery.esa.integral.core.pyvo.dal.TAPService.capabilities', []) - @patch('astroquery.esa.utils.utils.plt.imshow') + @patch('astroquery.esa.utils.utils.download_file') @patch('astroquery.esa.integral.core.IntegralClass.get_epochs') @patch('astroquery.esa.utils.utils.execute_servlet_request') @patch('astroquery.esa.integral.core.IntegralClass.get_instrument_band_map') - def test_get_mosaic(self, instrument_band_mock, servlet_mock, epoch_mock, plt_mock): + def test_get_mosaic(self, instrument_band_mock, servlet_mock, epoch_mock, download_mock): instrument_band_mock.return_value = mocks.get_instrument_bands() servlet_mock.return_value = mocks.get_mock_mosaic() epoch_mock.return_value = {'epoch': ['today']} + download_mock.return_value = data_path('mosaic.tar.gz') isla = IntegralClass() mock_instrument_bands(isla_module=isla) - mosaics = isla.get_mosaic(epoch='today', instrument='i1', plot=True) - args, kwargs = servlet_mock.call_args + mosaics_extracted = isla.get_mosaic(epoch='today', instrument='i1') + assert len(mosaics_extracted) == 2 - assert kwargs['query_params']['REQUEST'] == 'mosaics' - assert len(mosaics) == 1 - plt_mock.assert_called() + mosaics_compressed = isla.get_mosaic(epoch='today', instrument='i1', read_fits=False) - @patch('astroquery.esa.integral.core.pyvo.dal.TAPService.capabilities', []) - @patch('astroquery.esa.integral.core.IntegralClass.get_epochs') - @patch('astroquery.esa.utils.utils.execute_servlet_request') - @patch('astroquery.esa.utils.utils.download_file') - @patch('astroquery.esa.integral.core.IntegralClass.get_instrument_band_map') - def test_download_mosaic(self, instrument_band_mock, download_mock, servlet_mock, epoch_mock): - instrument_band_mock.return_value = mocks.get_instrument_bands() - servlet_mock.return_value = mocks.get_mock_mosaic() - epoch_mock.return_value = {'epoch': ['today']} - - isla = IntegralClass() - mock_instrument_bands(isla_module=isla) - isla.download_mosaic(epoch='today', instrument='i1') - - args, kwargs = download_mock.call_args - assert kwargs['params']['RETRIEVAL_TYPE'] == 'mosaics' - download_mock.assert_called() + assert type(mosaics_compressed) is list + close_files(mosaics_extracted) @patch('astroquery.esa.integral.core.pyvo.dal.TAPService.capabilities', []) @patch('astroquery.esa.utils.utils.execute_servlet_request') diff --git a/astroquery/esa/utils/tests/__init__.py b/astroquery/esa/utils/tests/__init__.py new file mode 100644 index 0000000000..49603ec083 --- /dev/null +++ b/astroquery/esa/utils/tests/__init__.py @@ -0,0 +1,9 @@ +""" +================ +UTILS Tests Init +================ + +European Space Astronomy Centre (ESAC) +European Space Agency (ESA) + +""" diff --git a/astroquery/esa/utils/tests/setup_package.py b/astroquery/esa/utils/tests/setup_package.py new file mode 100644 index 0000000000..291bc087e3 --- /dev/null +++ b/astroquery/esa/utils/tests/setup_package.py @@ -0,0 +1,18 @@ +# Licensed under a 3-clause BSD style license - see LICENSE.rst +import os + + +# setup paths to the test data +# can specify a single file or a list of files +def get_package_data(): + paths = [os.path.join('data', '*.vot'), + os.path.join('data', '*.xml'), + os.path.join('data', '*.zip'), + os.path.join('data', '*.gz'), + os.path.join('data', '*.tar'), + os.path.join('data', '*.fits'), + os.path.join('data', '*.txt'), + ] # etc, add other extensions + # you can also enlist files individually by names + # finally construct and return a dict for the sub module + return {'astroquery.esa.utils.tests': paths} diff --git a/astroquery/esa/utils/tests/test_utils.py b/astroquery/esa/utils/tests/test_utils.py new file mode 100644 index 0000000000..94485065aa --- /dev/null +++ b/astroquery/esa/utils/tests/test_utils.py @@ -0,0 +1,10 @@ +# Licensed under a 3-clause BSD style license - see LICENSE.rst +""" +=============== +ESA UTILS tests +=============== + +European Space Astronomy Centre (ESAC) +European Space Agency (ESA) + +""" diff --git a/astroquery/esa/utils/utils.py b/astroquery/esa/utils/utils.py index 2dda13eb93..2e6bffa242 100644 --- a/astroquery/esa/utils/utils.py +++ b/astroquery/esa/utils/utils.py @@ -7,18 +7,23 @@ European Space Agency (ESA) """ +import datetime import getpass import os +import binascii +import shutil -import numpy as np +import tarfile +import zipfile from astropy import log from astropy.coordinates import SkyCoord from astropy.units import Quantity +from astropy.io import fits from pyvo.auth.authsession import AuthSession from astroquery.utils import commons -import matplotlib.pyplot as plt +import requests from requests import Response @@ -111,7 +116,6 @@ def _send_login_request(self, username: str, password: str) -> Response: Returns: Response: The response object from the login request. """ - import requests # Use requests for handling HTTP interactions # Login payload payload = {"username": username, "password": password} @@ -247,141 +251,9 @@ def execute_post_request(url, tap, *, query_params=None): return response -def plot_result(x, y, x_title, y_title, plot_title, *, error_x=None, error_y=None, log_scale=False): +def download_file(url, session, *, params=None, path='', filename=None, cache=False, cache_folder=None, verbose=False): """ - Draw series in a 2D plot - - Parameters - ---------- - x: array of numbers, mandatory - values for the X series - y: array of numbers, mandatory - values for the Y series - x_title: str, mandatory - title of the X axis - y_title: str, mandatory - title of the Y axis - plot_title: str, mandatory - title of the plot - error_x: array of numbers, optional - error on the X series - error_y: array of numbers, optional - error on the Y series - log_scale: boolean, optional, default False - Draw X and Y axes using log scale - """ - - plt.figure(figsize=(8, 6)) - plt.scatter(x, y, color='blue') - plt.xlabel(x_title) - plt.ylabel(y_title) - - if log_scale: - plt.xscale('log') - plt.yscale('log') - - plt.title(plot_title) - - if error_x is not None or error_y is not None: - plt.errorbar(x, y, xerr=error_x, yerr=error_y, fmt='o') - - plt.show(block=False) - - -def plot_concatenated_results(x, y1, y2, x_title, y1_title, y2_title, plot_title, *, - x_label=None, y_label=None): - """ - Draw two plots concatenated on the X axis - - Parameters - ---------- - x: array of numbers, mandatory - values for the X series - y1: array of numbers, mandatory - values for the first Y series - y2: array of numbers, mandatory - values for the second Y series - x_title: str, mandatory - title of the X axis - y1_title: str, mandatory - title of the first Y axis - y2_title: str, mandatory - title of the second Y axis - plot_title: str, mandatory - title of the plot - x_label: str, optional - label for the X series - y_label: str, optional - label for the Y series - """ - # Create a figure with two subplots that share the x-axis - fig = plt.figure() - gs = fig.add_gridspec(2, hspace=0) - axs = gs.subplots(sharex=True, sharey=True) - - # Plot the first dataset on the first subplot - axs[0].scatter(x, y1, color='blue', label=x_label) - axs[0].set_ylabel(y1_title) - axs[0].grid(True, which='both') - - # Plot the second dataset on the second subplot - axs[1].scatter(x, y2, label=y_label, color='#DB4052') - axs[1].set_xlabel(x_title) - axs[1].set_ylabel(y2_title) - axs[1].grid(True, which='both') - - # Show the combined plot - fig.suptitle(plot_title) - for ax in axs: - ax.label_outer() - plt.show(block=False) - - -def plot_image(z, height, width, *, x_title=None, y_title=None, z_title=None, plot_title=None, z_min=2, z_max=10): - """ - Method to draw images and mosaics as heatmaps - - Parameters - ---------- - z: array of numbers, mandatory - values for the heatmap series - height: number, mandatory - height dimension for the heatmap - width: number, mandatory - width dimension for the heatmap - x_title: str, optional - title of the X axis - y_title: str, optional - title of the Y axis - z_title: str, optional - title of the Z axis (heatmap values) - plot_title: str, optional - title of the plot - z_min: number, optional, default 2 - Min value to show in the plot - z_max: number, optional, default 10 - Max value to show in the plot - """ - z_reshaped = z.reshape(width, height) - - if z_min is not None and z_max is not None: - z_clipped = np.clip(z_reshaped, z_min, z_max) - plt.imshow(z_clipped, origin='lower', cmap='hot') - else: - plt.imshow(z, origin='lower', cmap='hot') - - # Plot the heatmap - - plt.colorbar(label=z_title) - plt.xlabel(x_title) - plt.ylabel(y_title) - plt.title(plot_title) - plt.show(block=False) - - -def download_file(url, session, *, filename=None, params=None, verbose=False): - """ - Download a file in streaming mode using a existing session + Download a file in streaming mode using an existing session Parameters ---------- @@ -389,10 +261,16 @@ def download_file(url, session, *, filename=None, params=None, verbose=False): URL to be downloaded session: ESAAuthSession, mandatory session to download the file, including the cookies from ESA login - filename: str, optional - filename to be given to the final file params: dict, optional Additional params for the request + path: str, optional + Path where the file will be stored + filename: str, optional + filename to be given to the final file + cache: bool, optional, default False + flag to store the file in the Astroquery cache + cache_folder: str, optional + folder to store the cached file verbose: boolean, optional, default False Write the outputs in console @@ -403,7 +281,6 @@ def download_file(url, session, *, filename=None, params=None, verbose=False): if 'TAPCLIENT' not in params: params['TAPCLIENT'] = 'ASTROQUERY' - with session.get(url, stream=True, params=params) as response: response.raise_for_status() @@ -413,16 +290,146 @@ def download_file(url, session, *, filename=None, params=None, verbose=False): filename = content_disposition.split('filename=')[-1].strip('"') else: filename = os.path.basename(url.split('?')[0]) - + if cache: + filename = get_cache_filepath(filename, cache_folder) + path = '' # Open a local file in binary write mode if verbose: log.info('Downloading: ' + filename) - with open(filename, 'wb') as file: + file_path = os.path.join(path, filename) + with open(file_path, 'wb') as file: for chunk in response.iter_content(chunk_size=8192): file.write(chunk) if verbose: - log.info(f"File {filename} has been downloaded successfully") - return filename + log.info(f"File {file_path} has been downloaded successfully") + return file_path + + +def get_cache_filepath(filename=None, cache_path=None): + """ + Stores the content from a response as an Astroquery cache object. + + Parameters: + response (requests.Response): + The HTTP response object with iterable content. + filename: str, optional + filename to be given to the final file + cache_filename (str, optional): + The desired filename in the cache. If None, a default name is used. + + Returns: + str: Path to the cached file. + """ + # Determine the cache path + cache_file_path = os.path.join(cache_path, filename) + # Create the cache directory if it doesn't exist + os.makedirs(cache_path, exist_ok=True) + + return cache_file_path + + +def read_downloaded_fits(files): + extracted_files = [] + for file in files: + extracted_files.extend(extract_file(file)) + + fits_files = [] + for file in extracted_files: + fits_file = safe_open_fits(file) + if fits_file: + fits_files.append({ + 'filename': os.path.basename(file), + 'path': file, + 'fits': fits_file + }) + + return fits_files + + +def safe_open_fits(file_path): + """ + Safely open a FITS file using astropy.io.fits. + + Parameters: + file_path: string + The path to the file to be opened. + + Returns: + fits.HDUList or None + Returns the HDUList object if the file is a valid FITS file, otherwise None. + """ + try: + hdu_list = fits.open(file_path) + return hdu_list + except (OSError, fits.VerifyError) as e: + print(f"Skipping file {file_path}: {e}") + return None + + +def extract_file(file_path, output_dir=None): + """ + Extracts a .tar, .tar.gz, or .zip file. If the file is in a different format, + returns the path of the original file. + + Parameters: + file_path (str): + Path to the archive file (.tar, .tar.gz, or .zip). + output_dir (str, optional): + Directory to store the extracted files. If None, a directory + with the same name as the archive file (minus the extension) + is created. + + Returns: + list: List of paths to the extracted files. + """ + if not output_dir: + output_dir = os.path.abspath(file_path) + if tarfile.is_tarfile(file_path): + with tarfile.open(file_path, "r") as tar_ref: + return extract_from_tar(tar_ref, file_path, output_dir) + elif is_gz_file(file_path): + with tarfile.open(file_path, "r:gz") as tar: + return extract_from_tar(tar, file_path, output_dir) + elif zipfile.is_zipfile(file_path): + return extract_from_zip(file_path, output_dir) + elif not is_gz_file(file_path): + return [str(file_path)] + + +def is_gz_file(filepath): + with open(filepath, 'rb') as test_f: + return binascii.hexlify(test_f.read(2)) == b'1f8b' + + +def extract_from_tar(tar, file_path, output_dir=None): + """ + Extract files from a tar file (both .tar and .tar.gz formats). + """ + # Prepare the output directory + output_dir = prepare_output_dir(file_path) + + # Extract all files into the specified directory + tar.extractall(output_dir) + + # Get the paths of the extracted files + extracted_files = [os.path.join(output_dir, member.name) for member in tar.getmembers()] + return extracted_files + + +def extract_from_zip(file_path, output_dir=None): + """ + Handle extraction of .zip files. + """ + with zipfile.ZipFile(file_path, 'r') as zip_ref: + # Prepare the output directory + output_dir = prepare_output_dir(file_path) + + # Extract all files into the specified directory + zip_ref.extractall(output_dir) + + # Get the paths of the extracted files + extracted_files = [os.path.join(output_dir, file) for file in zip_ref.namelist()] + return extracted_files def check_rename_to_gz(filename): @@ -452,6 +459,25 @@ def check_rename_to_gz(filename): return os.path.basename(filename) +def prepare_output_dir(file_path): + """ + Prepare the output directory. If output_dir is provided, use it. Otherwise, + create a new directory with the name of the file (without the extension). + """ + # Create a directory based on the file name without extension + base_path = os.path.dirname(file_path) + base_name = os.path.basename(file_path) + file_name_without_extensions = base_name.split('.')[0] + current_time = datetime.datetime.now().strftime("%y%m%d%H%M%S") + + output_dir = os.path.join(base_path, f"{file_name_without_extensions}_{current_time}") + + if os.path.exists(output_dir): + shutil.rmtree(output_dir) + os.makedirs(output_dir, exist_ok=True) + return output_dir + + def resolve_target(url, session, target_name, target_resolver): """ Download a file in streaming mode using a existing session diff --git a/docs/esa/integral/integral.rst b/docs/esa/integral/integral.rst index a3e064708b..dfbb542bdb 100644 --- a/docs/esa/integral/integral.rst +++ b/docs/esa/integral/integral.rst @@ -1,5 +1,3 @@ -.. doctest-skip-all - .. _astroquery.esa.integral: ********************************************************************** @@ -115,20 +113,19 @@ The results can be exported to a file for further investigation or analysis. >>> isla.download_science_windows(science_windows=['008100430010', '033400230030'], output_file=None) # doctest: +IGNORE_OUTPUT ----------------------------------- -6. Timeline retrieval and plotting ----------------------------------- +--------------------- +6. Timeline retrieval +--------------------- This method enables the exploration of the observation timeline for a specific region in the sky. Users can provide right ascension (RA) and declination (Dec) coordinates and adjust the radius -to refine their search. Additionally, plots can be generated to illustrate details such as -revolution numbers or distances from the observation center. +to refine their search. .. doctest-remote-data:: >>> from astroquery.esa.integral import IntegralClass >>> isla = IntegralClass() - >>> timeline = isla.get_timeline(ra=83.63320922851562, dec=22.01447105407715, plot=True, plot_revno=True, plot_distance=True) + >>> timeline = isla.get_timeline(ra=83.63320922851562, dec=22.01447105407715) >>> timeline # doctest: +IGNORE_OUTPUT {'total_items': 8714, 'fraFC': 0.8510442965343126, 'totEffExpo': 16416293.994214607, 'timeline': scwExpo scwRevs scwTimes scwOffAxis @@ -176,144 +173,17 @@ This will perform an ADQL search to the Integral database and will return the ou ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ This method provides access to long-term timeseries data for a specified target. -Users can refine their results by selecting an instrument or energy band and may -generate a plot automatically to visualize them. +Users can refine their results by selecting an instrument or energy band. .. doctest-remote-data:: >>> from astroquery.esa.integral import IntegralClass >>> isla = IntegralClass() - >>> lt_timeseries_list = isla.get_long_term_timeseries(target_name='J174537.0-290107', instrument='jem-x', plot=True) - >>> lt_timeseries_list# doctest: +IGNORE_OUTPUT - {'source_id': 'J174537.0-290107', 'aggregation_value': -1, 'total_items': 863, 'aggregation_unit': 'raw', 'detectors': ['JEMX1', 'JEMX2'], 'timeseries_list': [
- time rates ratesError - object float64 float64 - ------------------- ------------------ ------------------ - 2007-03-02 01:01:31 10.909699440002441 2.010499954223633 - 2007-03-02 01:34:55 6.0742998123168945 0.805400013923645 - 2007-03-02 01:51:47 3.182499885559082 0.7900999784469604 - ... ... ... - 2016-03-08 08:55:04 14.921699523925781 2.262700080871582 - 2016-03-08 09:28:07 15.576800346374512 2.0703999996185303 - 2016-03-08 10:01:52 28.054500579833984 4.750199794769287,
- time rates ratesError - object float64 float64 - ------------------- ------------------ ------------------- - 2013-08-25 07:55:18 7.502500057220459 0.446399986743927 - 2013-10-15 03:54:50 4.342899799346924 0.3774000108242035 - 2015-09-19 22:55:18 5.5295000076293945 0.38519999384880066 - 2015-09-22 10:04:41 11.792900085449219 0.7631999850273132]} + >>> ltt = get_long_term_timeseries(target_name='J174537.0-290107', instrument='jem-x') # doctest: +IGNORE_OUTPUT 8.2. Retrieving Short-Term Timeseries ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Short-term time series data for a specific target and epoch can be accessed through this method. -Users may apply optional filters, such as instrument or energy band, -and can create a plot to visualize the extracted data. - -.. doctest-remote-data:: - - >>> from astroquery.esa.integral import IntegralClass - >>> isla = IntegralClass() - >>> st_timeseries_list = isla.get_short_term_timeseries(target_name='J011705.1-732636', band='28_40', epoch='0745_06340000001', plot=True) - >>> st_timeseries_list# doctest: +IGNORE_OUTPUT - {'source_id': 'J011705.1-732636', 'total_items': 82, 'detectors': ['ISGRI'], 'timeseries_list': [
- time rates rates_error - object float64 float64 - ------------------- ------------------- ------------------ - 2008-11-18 19:03:12 0.02567444182932377 0.7811741828918457 - 2008-11-18 19:19:52 0.7495294809341431 0.7880769371986389 - 2008-11-18 19:36:32 0.7943239808082581 0.7840375304222107 - ... ... ... - 2008-11-19 15:22:38 1.8685616254806519 0.7967823147773743 - 2008-11-19 15:39:18 3.6194331645965576 0.7971285581588745 - 2008-11-19 15:52:14 4.299497127532959 1.06647527217865]} - -8.3. Retrieving spectra -~~~~~~~~~~~~~~~~~~~~~~~ - -.. note:: - The plots and data provided by this method have been developed using - the automatic reduction pipeline at ISDC, using INTEGRAL OSA version 11.2. No manual - scientific validation has been performed on these outputs. They should be examined - and validated before being used for scientific purposes. To ensure accuracy and - reliability, please use dataset downloaded as described in section 9.3. - -Spectral data for a specified target and epoch can be obtained using this method. -Additional parameters enable users to filter results by instrument or energy band, -with the option to generate plots for spectral visualization. - -.. doctest-remote-data:: - - >>> from astroquery.esa.integral import IntegralClass - >>> isla = IntegralClass() - >>> spectras = isla.get_spectra(target_name='J011705.1-732636', instrument='ibis', epoch='0745_06340000001', plot=True) - >>> spectras # doctest: +IGNORE_OUTPUT - [{'spectra_oid': 46778, 'file_name': 'SMC_X-1_ISGRI_0745_06340000001_spectrum.fits', 'metadata': {}, 'date_start': '1858-11-17 00:00:00', 'date_stop': '1858-11-17 00:00:00', 'detector': 'ISGRI', 'spectra':
- energy energy_error rate rate_error - float64 float64 float64 float64 - ------- ------------ --------------------- --------------------- - 18.5 0.5 1.3224695920944214 0.18369388580322266 - 20.0 1.0 0.8658972978591919 0.05929631367325783 - 21.5 0.5 0.7667226791381836 0.057101089507341385 - ... ... ... ... - 125.75 3.75 0.004203930962830782 0.004149571992456913 - 134.5 5.0 0.0014642734313383698 0.0035532701294869184 - 143.25 3.75 0.004885881207883358 0.004115489777177572}] - - -8.4. Retrieving mosaics -~~~~~~~~~~~~~~~~~~~~~~~ - -.. note:: - The plots and data provided by this method have been developed using - the automatic reduction pipeline at ISDC, using INTEGRAL OSA version 11.2. No manual - scientific validation has been performed on these outputs. They should be examined - and validated before being used for scientific purposes. To ensure accuracy and - reliability, please use dataset downloaded as described in section 9.4. - -A mosaic image for a specific epoch can be retrieved with this method. -The query can be tailored by selecting an instrument or energy band. -Plots of the mosaic image can also be generated if visualization is required. - -.. doctest-remote-data:: - - >>> from astroquery.esa.integral import IntegralClass - >>> isla = IntegralClass() - >>> mosaics = isla.get_mosaic(epoch='0727_88601650001', instrument='ibis', plot=True) - >>> mosaics# doctest: +IGNORE_OUTPUT - [{'file_name': 'ISGRI_0727_88601650001_28_40_mosaic_smooth.fits.gz', 'mosaic_oid': 2126, 'height': 587, 'width': 589, 'min_z_scale': -2.5965349674224854, 'max_z_scale': 2.6038904190063477, 'mosaic':
- ra dec data - float64 float64 float64 - -------------- -------------- ------- - 22.6550755124 -36.5822127976 nan - 22.7390065307 -36.6532690339 nan - 22.8233738975 -36.7245035258 nan - ... ... ... - 184.770923897 44.6632117374 nan - 184.7323895179 44.5688775166 nan - 184.6941077098 44.474844758 nan}] - ----------------------------- -9. Downloading ISLA products ----------------------------- - -9.1. Downloading Long-Term Timeseries -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Long-term time series data can be downloaded using this method. Filters for instrument -and energy band can be applied to narrow down the data, and the results are saved to a -designated output file for further analysis. - -.. doctest-remote-data:: - - >>> from astroquery.esa.integral import IntegralClass - >>> isla = IntegralClass() - >>> isla.download_long_term_timeseries(target_name='J174537.0-290107', instrument='jem-x') # doctest: +IGNORE_OUTPUT - -9.2. Downloading Short-Term Timeseries -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - This method allows for the download of short-term time series data for a target and epoch of interest. Users can refine their search using instrument or energy band filters and the results are saved to a file for detailed examination. @@ -322,10 +192,10 @@ the results are saved to a file for detailed examination. >>> from astroquery.esa.integral import IntegralClass >>> isla = IntegralClass() - >>> isla.download_short_term_timeseries(target_name='J011705.1-732636', band='28_40', epoch='0745_06340000001') # doctest: +IGNORE_OUTPUT + >>> isla.get_short_term_timeseries(target_name='J011705.1-732636', band='28_40', epoch='0745_06340000001') # doctest: +IGNORE_OUTPUT -9.3. Downloading spectra -~~~~~~~~~~~~~~~~~~~~~~~~ +8.3. Retrieving spectra +~~~~~~~~~~~~~~~~~~~~~~~ This method allows users to download spectral data for a target and epoch. Users can apply filters, such as instrument or energy band, and save the results to an @@ -335,10 +205,11 @@ output file for further processing. >>> from astroquery.esa.integral import IntegralClass >>> isla = IntegralClass() - >>> isla.download_spectra(target_name='J011705.1-732636', instrument='ibis', epoch='0745_06340000001') # doctest: +IGNORE_OUTPUT + >>> isla.get_spectra(target_name='J011705.1-732636', instrument='ibis', epoch='0745_06340000001') # doctest: +IGNORE_OUTPUT + -9.4. Downloading mosaics -~~~~~~~~~~~~~~~~~~~~~~~~ +8.4. Retrieving mosaics +~~~~~~~~~~~~~~~~~~~~~~~ Mosaic images corresponding to a specified epoch can be downloaded using this method. Users can filter by instrument or energy band and save the resulting image to a file for later use. @@ -347,11 +218,11 @@ Users can filter by instrument or energy band and save the resulting image to a >>> from astroquery.esa.integral import IntegralClass >>> isla = IntegralClass() - >>> mosaics = isla.download_mosaic(epoch='0727_88601650001', instrument='ibis') # doctest: +IGNORE_OUTPUT + >>> mosaics = isla.get_mosaic(epoch='0727_88601650001', instrument='ibis') # doctest: +IGNORE_OUTPUT Reference/API ============= .. automodapi:: astroquery.esa.integral - :no-inheritance-diagram: + :no-inheritance-diagram: \ No newline at end of file