From 6c413d9dd56ff73472a50e0667f67be101890fb6 Mon Sep 17 00:00:00 2001 From: David Gonzalez Date: Mon, 22 Jul 2024 09:18:16 +0200 Subject: [PATCH 1/4] Method to get all products by program id (proposal id) --- astroquery/esa/jwst/__init__.py | 2 ++ astroquery/esa/jwst/core.py | 37 +++++++++++++++++++++++ astroquery/esa/jwst/tests/test_jwsttap.py | 8 +++++ docs/esa/jwst/jwst.rst | 9 ++++++ 4 files changed, 56 insertions(+) diff --git a/astroquery/esa/jwst/__init__.py b/astroquery/esa/jwst/__init__.py index 4d1680ab5b..609782fa0e 100644 --- a/astroquery/esa/jwst/__init__.py +++ b/astroquery/esa/jwst/__init__.py @@ -44,6 +44,8 @@ class Conf(_config.ConfigNamespace): JWST_OBSERVATION_TABLE_DEC = _config.ConfigItem("targetposition_coordinates_cval2", "Name of Dec parameter " "in table") + + JWST_ARCHIVE_TABLE = _config.ConfigItem("jwst.archive", "JWST archive table") conf = Conf() diff --git a/astroquery/esa/jwst/core.py b/astroquery/esa/jwst/core.py index 45293e2995..8749ebf6d7 100644 --- a/astroquery/esa/jwst/core.py +++ b/astroquery/esa/jwst/core.py @@ -66,6 +66,7 @@ class JwstClass(BaseQuery): TARGET_RESOLVERS = ['ALL', 'SIMBAD', 'NED', 'VIZIER'] CAL_LEVELS = ['ALL', 1, 2, 3, -1] REQUESTED_OBSERVATION_ID = "Missing required argument: 'observation_id'" + REQUESTED_PROPOSAL_ID = "Missing required argument: 'proposal_id'" def __init__(self, *, tap_plus_handler=None, data_handler=None, show_messages=True): if tap_plus_handler is None: @@ -1034,6 +1035,42 @@ def get_obs_products(self, *, observation_id=None, cal_level="ALL", files=files) return files + + def get_pro_products(self, *, proposal_id=None, product_type=None, verbose=False): + """Get JWST products given its proposal ID. + + Parameters + ---------- + proposal_id : str, mandatory + proposal ID of the product. + product_type : str or list, optional, default None + If the string or at least one element of the list is empty, + the value is replaced by None. + With None, all products will be downloaded. + Possible string values: 'thumbnail', 'preview', 'auxiliary', 'science' or 'info'. + Posible list values: any combination of string values. + verbose : bool, optional, default 'False' + flag to display information about the process + + Returns + ------- + allobs : list + Returns the observationsid included into the proposal_id. + """ + + if proposal_id is None: + raise ValueError(self.REQUESTED_PROPOSAL_ID) + query = (f"SELECT observationid " + f"FROM {str(conf.JWST_ARCHIVE_TABLE)} " + f"WHERE proposal_id='{str(proposal_id)}'") + if verbose: + print(query) + job = self.__jwsttap.launch_job_async(query=query, verbose=verbose) + allobs = set(JwstClass.get_decoded_string(job.get_results()['observationid'])) + for oid in allobs: + log.info(f"Downloading products for Observation ID: {oid}") + self.get_obs_products(observation_id=oid, product_type=product_type) + return list(allobs) def __check_file_number(self, output_dir, output_file_name, output_file_full_path, files): diff --git a/astroquery/esa/jwst/tests/test_jwsttap.py b/astroquery/esa/jwst/tests/test_jwsttap.py index 15f8605f8a..39acef9a07 100644 --- a/astroquery/esa/jwst/tests/test_jwsttap.py +++ b/astroquery/esa/jwst/tests/test_jwsttap.py @@ -684,6 +684,14 @@ def test_get_products_list_error(self): jwst.get_product_list(observation_id=observation_id, product_type='test') assert "product_type must be one of" in err.value.args[0] + def test_get_pro_products(self): + dummyTapHandler = DummyTapHandler() + jwst = JwstClass(tap_plus_handler=dummyTapHandler, data_handler=dummyTapHandler, show_messages=False) + + with pytest.raises(ValueError) as err: + jwst.get_pro_products() + assert "Missing required argument: 'proposal_id'" in err.value.args[0] + def test_get_obs_products(self): dummyTapHandler = DummyTapHandler() jwst = JwstClass(tap_plus_handler=dummyTapHandler, data_handler=dummyTapHandler, show_messages=False) diff --git a/docs/esa/jwst/jwst.rst b/docs/esa/jwst/jwst.rst index f0429aa652..566fc62b99 100644 --- a/docs/esa/jwst/jwst.rst +++ b/docs/esa/jwst/jwst.rst @@ -326,6 +326,15 @@ Using the observation ID as input parameter, this function will retrieve the obs 'jw02739001001_02105_00002_nrcblong', 'jw02739001001_02105_00003_nrcalong'] +To query the data products associated with a certain Proposal ID and filtered by product_type. + +.. doctest-remote-data:: + + >>> from astroquery.esa.jwst import Jwst + >>> observation_list = Jwst.get_pro_products(proposal_id='6651', product_type='preview') + >>> print(observation_list) # doctest: +IGNORE_OUTPUT + [np.str_('jw06651001001_05201_00001_nis'), np.str_('jw06651002001_05201_00001_nis')] + 1.5 Getting public tables ~~~~~~~~~~~~~~~~~~~~~~~~~ From 537bfd69a8d8500a5ea14a15f67b850e511c1f26 Mon Sep 17 00:00:00 2001 From: David Gonzalez Date: Mon, 22 Jul 2024 09:30:46 +0200 Subject: [PATCH 2/4] Fix PEP8 issues --- CHANGES.rst | 2 ++ astroquery/esa/jwst/__init__.py | 2 +- astroquery/esa/jwst/core.py | 2 +- astroquery/esa/jwst/tests/test_jwsttap.py | 2 +- 4 files changed, 5 insertions(+), 3 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 2953d907f5..556799d74a 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -35,6 +35,8 @@ esa.jwst - get_obs_products method supports product_type parameter as string or list [#2995] +- Add get_pro_products method to get all products by program id [#3073] + mpc ^^^ diff --git a/astroquery/esa/jwst/__init__.py b/astroquery/esa/jwst/__init__.py index 609782fa0e..39dbbb0564 100644 --- a/astroquery/esa/jwst/__init__.py +++ b/astroquery/esa/jwst/__init__.py @@ -44,7 +44,7 @@ class Conf(_config.ConfigNamespace): JWST_OBSERVATION_TABLE_DEC = _config.ConfigItem("targetposition_coordinates_cval2", "Name of Dec parameter " "in table") - + JWST_ARCHIVE_TABLE = _config.ConfigItem("jwst.archive", "JWST archive table") diff --git a/astroquery/esa/jwst/core.py b/astroquery/esa/jwst/core.py index 8749ebf6d7..0b614f6292 100644 --- a/astroquery/esa/jwst/core.py +++ b/astroquery/esa/jwst/core.py @@ -1035,7 +1035,7 @@ def get_obs_products(self, *, observation_id=None, cal_level="ALL", files=files) return files - + def get_pro_products(self, *, proposal_id=None, product_type=None, verbose=False): """Get JWST products given its proposal ID. diff --git a/astroquery/esa/jwst/tests/test_jwsttap.py b/astroquery/esa/jwst/tests/test_jwsttap.py index 39acef9a07..99822ab2f2 100644 --- a/astroquery/esa/jwst/tests/test_jwsttap.py +++ b/astroquery/esa/jwst/tests/test_jwsttap.py @@ -687,7 +687,7 @@ def test_get_products_list_error(self): def test_get_pro_products(self): dummyTapHandler = DummyTapHandler() jwst = JwstClass(tap_plus_handler=dummyTapHandler, data_handler=dummyTapHandler, show_messages=False) - + with pytest.raises(ValueError) as err: jwst.get_pro_products() assert "Missing required argument: 'proposal_id'" in err.value.args[0] From ee7158c7d4e95617929dfd9b53e9ca354bc36312 Mon Sep 17 00:00:00 2001 From: David Gonzalez Date: Mon, 22 Jul 2024 11:31:07 +0200 Subject: [PATCH 3/4] Rename the method get_pro_products to download_files_from_program to align it with hubble Fix the position of the proposal_id mandatory argument in the method definition --- CHANGES.rst | 2 +- astroquery/esa/jwst/core.py | 9 +++------ astroquery/esa/jwst/tests/test_jwsttap.py | 9 ++++----- docs/esa/jwst/jwst.rst | 2 +- 4 files changed, 9 insertions(+), 13 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 556799d74a..c13db98529 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -35,7 +35,7 @@ esa.jwst - get_obs_products method supports product_type parameter as string or list [#2995] -- Add get_pro_products method to get all products by program id [#3073] +- Add download_files_from_program method to get all products by program id [#3073] mpc ^^^ diff --git a/astroquery/esa/jwst/core.py b/astroquery/esa/jwst/core.py index 0b614f6292..221464ee6d 100644 --- a/astroquery/esa/jwst/core.py +++ b/astroquery/esa/jwst/core.py @@ -66,7 +66,6 @@ class JwstClass(BaseQuery): TARGET_RESOLVERS = ['ALL', 'SIMBAD', 'NED', 'VIZIER'] CAL_LEVELS = ['ALL', 1, 2, 3, -1] REQUESTED_OBSERVATION_ID = "Missing required argument: 'observation_id'" - REQUESTED_PROPOSAL_ID = "Missing required argument: 'proposal_id'" def __init__(self, *, tap_plus_handler=None, data_handler=None, show_messages=True): if tap_plus_handler is None: @@ -1036,13 +1035,13 @@ def get_obs_products(self, *, observation_id=None, cal_level="ALL", return files - def get_pro_products(self, *, proposal_id=None, product_type=None, verbose=False): + def download_files_from_program(self, proposal_id, *, product_type=None, verbose=False): """Get JWST products given its proposal ID. Parameters ---------- - proposal_id : str, mandatory - proposal ID of the product. + proposal_id : int, mandatory + Program or Proposal ID associated to the observations. product_type : str or list, optional, default None If the string or at least one element of the list is empty, the value is replaced by None. @@ -1058,8 +1057,6 @@ def get_pro_products(self, *, proposal_id=None, product_type=None, verbose=False Returns the observationsid included into the proposal_id. """ - if proposal_id is None: - raise ValueError(self.REQUESTED_PROPOSAL_ID) query = (f"SELECT observationid " f"FROM {str(conf.JWST_ARCHIVE_TABLE)} " f"WHERE proposal_id='{str(proposal_id)}'") diff --git a/astroquery/esa/jwst/tests/test_jwsttap.py b/astroquery/esa/jwst/tests/test_jwsttap.py index 99822ab2f2..19e7d822e8 100644 --- a/astroquery/esa/jwst/tests/test_jwsttap.py +++ b/astroquery/esa/jwst/tests/test_jwsttap.py @@ -684,13 +684,12 @@ def test_get_products_list_error(self): jwst.get_product_list(observation_id=observation_id, product_type='test') assert "product_type must be one of" in err.value.args[0] - def test_get_pro_products(self): + def test_download_files_from_program(self): dummyTapHandler = DummyTapHandler() jwst = JwstClass(tap_plus_handler=dummyTapHandler, data_handler=dummyTapHandler, show_messages=False) - - with pytest.raises(ValueError) as err: - jwst.get_pro_products() - assert "Missing required argument: 'proposal_id'" in err.value.args[0] + with pytest.raises(TypeError) as err: + jwst.download_files_from_program() + assert "missing 1 required positional argument: 'proposal_id'" in err.value.args[0] def test_get_obs_products(self): dummyTapHandler = DummyTapHandler() diff --git a/docs/esa/jwst/jwst.rst b/docs/esa/jwst/jwst.rst index 566fc62b99..fd1e85afe2 100644 --- a/docs/esa/jwst/jwst.rst +++ b/docs/esa/jwst/jwst.rst @@ -331,7 +331,7 @@ To query the data products associated with a certain Proposal ID and filtered by .. doctest-remote-data:: >>> from astroquery.esa.jwst import Jwst - >>> observation_list = Jwst.get_pro_products(proposal_id='6651', product_type='preview') + >>> observation_list = Jwst.download_files_from_program(proposal_id='6651', product_type='preview') >>> print(observation_list) # doctest: +IGNORE_OUTPUT [np.str_('jw06651001001_05201_00001_nis'), np.str_('jw06651002001_05201_00001_nis')] From 2019702e81a44df9f9c1cd2c15003bcb6cca1e69 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Brigitta=20Sip=C5=91cz?= Date: Mon, 22 Jul 2024 11:37:37 -0700 Subject: [PATCH 4/4] MAINT: fix style and beautify docs --- astroquery/esa/jwst/core.py | 2 +- astroquery/esa/jwst/tests/test_jwsttap.py | 2 +- docs/esa/jwst/jwst.rst | 9 ++++++--- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/astroquery/esa/jwst/core.py b/astroquery/esa/jwst/core.py index 221464ee6d..b9347e5f4d 100644 --- a/astroquery/esa/jwst/core.py +++ b/astroquery/esa/jwst/core.py @@ -1044,7 +1044,7 @@ def download_files_from_program(self, proposal_id, *, product_type=None, verbose Program or Proposal ID associated to the observations. product_type : str or list, optional, default None If the string or at least one element of the list is empty, - the value is replaced by None. + the value is replaced by None. With None, all products will be downloaded. Possible string values: 'thumbnail', 'preview', 'auxiliary', 'science' or 'info'. Posible list values: any combination of string values. diff --git a/astroquery/esa/jwst/tests/test_jwsttap.py b/astroquery/esa/jwst/tests/test_jwsttap.py index 19e7d822e8..d627c7b8e2 100644 --- a/astroquery/esa/jwst/tests/test_jwsttap.py +++ b/astroquery/esa/jwst/tests/test_jwsttap.py @@ -688,7 +688,7 @@ def test_download_files_from_program(self): dummyTapHandler = DummyTapHandler() jwst = JwstClass(tap_plus_handler=dummyTapHandler, data_handler=dummyTapHandler, show_messages=False) with pytest.raises(TypeError) as err: - jwst.download_files_from_program() + jwst.download_files_from_program() assert "missing 1 required positional argument: 'proposal_id'" in err.value.args[0] def test_get_obs_products(self): diff --git a/docs/esa/jwst/jwst.rst b/docs/esa/jwst/jwst.rst index fd1e85afe2..e0f5842a79 100644 --- a/docs/esa/jwst/jwst.rst +++ b/docs/esa/jwst/jwst.rst @@ -289,7 +289,7 @@ than get_product_list, it also supports product_type parameter as string or list Here product_type as list: .. doctest-remote-data:: - + >>> from astroquery.esa.jwst import Jwst >>> observation_id = 'jw01122001001_0210r_00001_nrs2' >>> results = Jwst.get_obs_products(observation_id=observation_id, cal_level=2, product_type=['science', 'preview']) @@ -331,9 +331,12 @@ To query the data products associated with a certain Proposal ID and filtered by .. doctest-remote-data:: >>> from astroquery.esa.jwst import Jwst - >>> observation_list = Jwst.download_files_from_program(proposal_id='6651', product_type='preview') + >>> observation_list = Jwst.download_files_from_program(proposal_id='6651', product_type='preview') # doctest: +IGNORE_OUTPUT + INFO: Query finished. [astroquery.utils.tap.core] + INFO: Downloading products for Observation ID: jw06651001001_05201_00001_nis [astroquery.esa.jwst.core] + INFO: Downloading products for Observation ID: jw06651002001_05201_00001_nis [astroquery.esa.jwst.core] >>> print(observation_list) # doctest: +IGNORE_OUTPUT - [np.str_('jw06651001001_05201_00001_nis'), np.str_('jw06651002001_05201_00001_nis')] + ['jw06651001001_05201_00001_nis', 'jw06651002001_05201_00001_nis'] 1.5 Getting public tables