From 45039bd0a5a2f22383a430cd52243001878e6385 Mon Sep 17 00:00:00 2001 From: kim Date: Fri, 23 Feb 2024 10:03:57 -0900 Subject: [PATCH 01/10] adds way to search via CMR keywords directly with ASFSearchOptions --- asf_search/ASFSearchOptions/validator_map.py | 3 ++- asf_search/ASFSearchOptions/validators.py | 20 +++++++++++++++++++- asf_search/CMR/translate.py | 7 +++++++ 3 files changed, 28 insertions(+), 2 deletions(-) diff --git a/asf_search/ASFSearchOptions/validator_map.py b/asf_search/ASFSearchOptions/validator_map.py index 72fdd0a4..bf937e81 100644 --- a/asf_search/ASFSearchOptions/validator_map.py +++ b/asf_search/ASFSearchOptions/validator_map.py @@ -3,7 +3,7 @@ from .validators import ( parse_string, parse_float, parse_wkt, parse_date, parse_string_list, parse_int_list, parse_int_or_range_list, - parse_float_or_range_list, + parse_float_or_range_list, parse_cmr_keywords_list, parse_session ) @@ -62,6 +62,7 @@ def validate(key, value): 'relativeBurstID': parse_int_list, 'fullBurstID': parse_string_list, 'dataset': parse_string_list, + 'cmr_keywords': parse_cmr_keywords_list, # Config parameters Parser 'session': parse_session, diff --git a/asf_search/ASFSearchOptions/validators.py b/asf_search/ASFSearchOptions/validators.py index 9164f3c3..1fbcea74 100644 --- a/asf_search/ASFSearchOptions/validators.py +++ b/asf_search/ASFSearchOptions/validators.py @@ -2,7 +2,7 @@ import datetime import requests -from typing import Union, Tuple, TypeVar, Callable, List, Type, Sequence +from typing import Dict, Union, Tuple, TypeVar, Callable, List, Type, Sequence import math from shapely import wkt, errors @@ -102,6 +102,24 @@ def parse_list(value: Sequence, h) -> List: except ValueError as exc: raise ValueError(f'Invalid {h.__name__} list: {exc}') from exc +def parse_cmr_keywords_list(value: Sequence[Dict]): + if not isinstance(value, Sequence): + value = [value] + + required_keys = ['key', 'fmt'] + + for idx, item in enumerate(value): + if not isinstance(item, dict): + raise ValueError(f"Expected item in umm search key list index {idx} to be dict, got value {item} of type {type(item)}") + for key in required_keys: + item_value = item.get(key) + if item_value is None: + raise ValueError(f"Expected key \"{key}\" in umm search key dict at index {idx}, got keys {item.keys()}") + elif not isinstance(item_value, str): + raise ValueError(f"Expected value at index {idx} for key \"{key}\" to be of type \"str\", got {type(item_value)} instead.") + + return value + # Parse and validate an iterable of strings: "foo,bar,baz" def parse_string_list(value: Sequence[str]) -> List[str]: return parse_list(value, str) diff --git a/asf_search/CMR/translate.py b/asf_search/CMR/translate.py index f6c79049..7982e74f 100644 --- a/asf_search/CMR/translate.py +++ b/asf_search/CMR/translate.py @@ -51,6 +51,10 @@ def translate_opts(opts: ASFSearchOptions) -> List: # convert the above parameters to a list of key/value tuples cmr_opts = [] + + # user provided umm fields + custom_cmr_keywords = dict_opts.pop('cmr_keywords', []) + for (key, val) in dict_opts.items(): # If it's "session" or something else CMR doesn't accept, don't send it: if key not in field_map: @@ -74,6 +78,9 @@ def translate_opts(opts: ASFSearchOptions) -> List: if should_use_asf_frame(cmr_opts): cmr_opts = use_asf_frame(cmr_opts) + for search_keyword in custom_cmr_keywords: + cmr_opts.append((search_keyword['key'], search_keyword['fmt'])) + additional_keys = [ ('page_size', CMR_PAGE_SIZE), ('options[temporal][and]', 'true'), From 8657e752362db2e281f4d9194df8bb5df2ded094 Mon Sep 17 00:00:00 2001 From: kim Date: Thu, 29 Feb 2024 13:28:19 -0900 Subject: [PATCH 02/10] update changelog --- CHANGELOG.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index e4526737..505b8354 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,11 @@ and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html). - --> +------ +## [v7.0.5](https://github.com/asfadmin/Discovery-asf_search/compare/v7.0.4...v7.0.5) +### Added +- Adds `cmr_keywords` search keyword, enables passing CMR format strings in search directly + ------ ## [v7.0.4](https://github.com/asfadmin/Discovery-asf_search/compare/v7.0.3...v7.0.4) ### Changed From 0a1d06cb31429d82351847d25b9e9cca205d02f0 Mon Sep 17 00:00:00 2001 From: Kim <33294735+SpicyGarlicAlbacoreRoll@users.noreply.github.com> Date: Wed, 6 Mar 2024 09:20:05 -0900 Subject: [PATCH 03/10] Update CHANGELOG.md --- CHANGELOG.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 52ff480d..3f75fc67 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,10 +25,15 @@ and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html). - --> + ------ -## [v7.0.5](https://github.com/asfadmin/Discovery-asf_search/compare/v7.0.4...v7.0.5) +## [v7.0.6](https://github.com/asfadmin/Discovery-asf_search/compare/v7.0.5...v7.0.6) ### Added - Adds `cmr_keywords` search keyword, enables passing CMR format strings in search directly + +------ +## [v7.0.5](https://github.com/asfadmin/Discovery-asf_search/compare/v7.0.4...v7.0.5) +### Added - Adds basic NISAR dataset search and product functionality for test data ------ From 097014f941d0f17759ab6617bd519e3dcc37bbe9 Mon Sep 17 00:00:00 2001 From: Kim <33294735+SpicyGarlicAlbacoreRoll@users.noreply.github.com> Date: Wed, 6 Mar 2024 09:22:06 -0900 Subject: [PATCH 04/10] Update CHANGELOG.md --- CHANGELOG.md | 3 --- 1 file changed, 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6e22709f..95ef803f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,9 +30,6 @@ and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [v7.0.6](https://github.com/asfadmin/Discovery-asf_search/compare/v7.0.5...v7.0.6) ### Added - Adds `cmr_keywords` search keyword, enables passing CMR format strings in search directly - ------- -## [v7.0.6](https://github.com/asfadmin/Discovery-asf_search/compare/v7.0.5...v7.0.6) ### Changed - Changed what collections the `NISAR` dataset and platform collections lists are pointed at. From 5662fcc83fb21c2d1ec2a415bc9eb6a2509dc602 Mon Sep 17 00:00:00 2001 From: kim Date: Thu, 21 Mar 2024 18:27:45 -0800 Subject: [PATCH 05/10] feature: adds shortName search keyword --- CHANGELOG.md | 5 +++++ asf_search/ASFSearchOptions/validator_map.py | 1 + asf_search/CMR/field_map.py | 1 + asf_search/search/geo_search.py | 1 + asf_search/search/search.py | 1 + asf_search/search/search_count.py | 1 + asf_search/search/search_generator.py | 1 + 7 files changed, 11 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index e2a30816..7c0526cb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,11 @@ and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html). - --> +------ +## [v7.0.7](https://github.com/asfadmin/Discovery-asf_search/compare/v7.0.6...v7.0.7) +### Added +- Adds `shortName` keyword, for use with lists of collection short names + ------ ## [v7.0.6](https://github.com/asfadmin/Discovery-asf_search/compare/v7.0.5...v7.0.6) ### Changed diff --git a/asf_search/ASFSearchOptions/validator_map.py b/asf_search/ASFSearchOptions/validator_map.py index 72fdd0a4..63390a3e 100644 --- a/asf_search/ASFSearchOptions/validator_map.py +++ b/asf_search/ASFSearchOptions/validator_map.py @@ -56,6 +56,7 @@ def validate(key, value): 'insarStackId': parse_string, 'instrument': parse_string, 'collections': parse_string_list, + 'shortName': parse_string_list, 'temporalBaselineDays': parse_string_list, 'operaBurstID': parse_string_list, 'absoluteBurstID': parse_int_list, diff --git a/asf_search/CMR/field_map.py b/asf_search/CMR/field_map.py index b9b2b10f..0754b4fd 100644 --- a/asf_search/CMR/field_map.py +++ b/asf_search/CMR/field_map.py @@ -35,6 +35,7 @@ 'relativeOrbit': {'key': 'attribute[]', 'fmt': 'int,PATH_NUMBER,{0}'}, 'temporal': {'key': 'temporal', 'fmt': '{0}'}, 'collections': {'key': 'echo_collection_id[]', 'fmt': '{0}'}, + 'shortName': {'key': 'shortName', 'fmt': '{0}'}, 'temporalBaselineDays': {'key': 'attribute[]', 'fmt': 'int,TEMPORAL_BASELINE_DAYS,{0}'}, # SLC BURST fields diff --git a/asf_search/search/geo_search.py b/asf_search/search/geo_search.py index af18f7ed..3da37652 100644 --- a/asf_search/search/geo_search.py +++ b/asf_search/search/geo_search.py @@ -43,6 +43,7 @@ def geo_search( temporalBaselineDays: Union[str, Sequence[str]] = None, operaBurstID: Union[str, Sequence[str]] = None, dataset: Union[str, Sequence[str]] = None, + shortName: Union[str, Sequence[str]] = None, maxResults: int = None, opts: ASFSearchOptions = None, ) -> ASFSearchResults: diff --git a/asf_search/search/search.py b/asf_search/search/search.py index 3113b87e..1dab9767 100644 --- a/asf_search/search/search.py +++ b/asf_search/search/search.py @@ -42,6 +42,7 @@ def search( temporalBaselineDays: Union[str, Sequence[str]] = None, operaBurstID: Union[str, Sequence[str]] = None, dataset: Union[str, Sequence[str]] = None, + shortName: Union[str, Sequence[str]] = None, maxResults: int = None, opts: ASFSearchOptions = None, ) -> ASFSearchResults: diff --git a/asf_search/search/search_count.py b/asf_search/search/search_count.py index 8b08c884..5120e851 100644 --- a/asf_search/search/search_count.py +++ b/asf_search/search/search_count.py @@ -44,6 +44,7 @@ def search_count( temporalBaselineDays: Union[str, Sequence[str]] = None, operaBurstID: Union[str, Sequence[str]] = None, dataset: Union[str, Sequence[str]] = None, + shortName: Union[str, Sequence[str]] = None, maxResults: int = None, opts: ASFSearchOptions = None, ) -> int: diff --git a/asf_search/search/search_generator.py b/asf_search/search/search_generator.py index 75e8a751..65e871f1 100644 --- a/asf_search/search/search_generator.py +++ b/asf_search/search/search_generator.py @@ -60,6 +60,7 @@ def search_generator( temporalBaselineDays: Union[str, Sequence[str]] = None, operaBurstID: Union[str, Sequence[str]] = None, dataset: Union[str, Sequence[str]] = None, + shortName: Union[str, Sequence[str]] = None, maxResults: int = None, opts: ASFSearchOptions = None, ) -> Generator[ASFSearchResults, None, None]: From da783f8f94c22752dfaf03bb9e97c032eb08aeb2 Mon Sep 17 00:00:00 2001 From: kim Date: Fri, 22 Mar 2024 10:39:34 -0800 Subject: [PATCH 06/10] makes cmr_keywords available in search methods directly, makes shortName and cmr_keywords dodge subquery system --- asf_search/CMR/subquery.py | 2 +- asf_search/search/geo_search.py | 3 ++- asf_search/search/search.py | 3 ++- asf_search/search/search_count.py | 3 ++- asf_search/search/search_generator.py | 1 + 5 files changed, 8 insertions(+), 4 deletions(-) diff --git a/asf_search/CMR/subquery.py b/asf_search/CMR/subquery.py index 4c4551d9..dfff6133 100644 --- a/asf_search/CMR/subquery.py +++ b/asf_search/CMR/subquery.py @@ -22,7 +22,7 @@ def build_subqueries(opts: ASFSearchOptions) -> List[ASFSearchOptions]: if params.get(chunked_key) is not None: params[chunked_key] = chunk_list(params[chunked_key], CMR_PAGE_SIZE) - list_param_names = ['platform', 'season', 'collections', 'dataset'] # these parameters will dodge the subquery system + list_param_names = ['platform', 'season', 'collections', 'dataset', 'cmr_keywords', 'shortName'] # these parameters will dodge the subquery system skip_param_names = ['maxResults']# these params exist in opts, but shouldn't be passed on to subqueries at ALL collections, aliased_keywords = get_keyword_concept_ids(params, opts.collectionAlias) diff --git a/asf_search/search/geo_search.py b/asf_search/search/geo_search.py index 3da37652..0b260912 100644 --- a/asf_search/search/geo_search.py +++ b/asf_search/search/geo_search.py @@ -1,4 +1,4 @@ -from typing import Tuple, Union, Sequence +from typing import Dict, Tuple, Union, Sequence import datetime from copy import copy @@ -44,6 +44,7 @@ def geo_search( operaBurstID: Union[str, Sequence[str]] = None, dataset: Union[str, Sequence[str]] = None, shortName: Union[str, Sequence[str]] = None, + cmr_keywords: Union[Dict[str, str], Sequence[Dict[str, str]]] = None, maxResults: int = None, opts: ASFSearchOptions = None, ) -> ASFSearchResults: diff --git a/asf_search/search/search.py b/asf_search/search/search.py index 1dab9767..83ea2ec6 100644 --- a/asf_search/search/search.py +++ b/asf_search/search/search.py @@ -1,4 +1,4 @@ -from typing import Union, Sequence, Tuple +from typing import Dict, Union, Sequence, Tuple from copy import copy import datetime @@ -43,6 +43,7 @@ def search( operaBurstID: Union[str, Sequence[str]] = None, dataset: Union[str, Sequence[str]] = None, shortName: Union[str, Sequence[str]] = None, + cmr_keywords: Union[Dict[str, str], Sequence[Dict[str, str]]] = None, maxResults: int = None, opts: ASFSearchOptions = None, ) -> ASFSearchResults: diff --git a/asf_search/search/search_count.py b/asf_search/search/search_count.py index 5120e851..b62b4b91 100644 --- a/asf_search/search/search_count.py +++ b/asf_search/search/search_count.py @@ -1,5 +1,5 @@ import datetime -from typing import Sequence, Tuple, Union +from typing import Dict, Sequence, Tuple, Union from copy import copy from asf_search.ASFSearchOptions import ASFSearchOptions from asf_search.CMR.subquery import build_subqueries @@ -45,6 +45,7 @@ def search_count( operaBurstID: Union[str, Sequence[str]] = None, dataset: Union[str, Sequence[str]] = None, shortName: Union[str, Sequence[str]] = None, + cmr_keywords: Union[Dict[str, str], Sequence[Dict[str, str]]] = None, maxResults: int = None, opts: ASFSearchOptions = None, ) -> int: diff --git a/asf_search/search/search_generator.py b/asf_search/search/search_generator.py index 65e871f1..0998a3b3 100644 --- a/asf_search/search/search_generator.py +++ b/asf_search/search/search_generator.py @@ -61,6 +61,7 @@ def search_generator( operaBurstID: Union[str, Sequence[str]] = None, dataset: Union[str, Sequence[str]] = None, shortName: Union[str, Sequence[str]] = None, + cmr_keywords: Union[Dict[str, str], Sequence[Dict[str, str]]] = None, maxResults: int = None, opts: ASFSearchOptions = None, ) -> Generator[ASFSearchResults, None, None]: From 088e4d1eff4283fdceda2f66a82ef773533c7fc3 Mon Sep 17 00:00:00 2001 From: kim Date: Fri, 22 Mar 2024 10:41:16 -0800 Subject: [PATCH 07/10] update changelog --- CHANGELOG.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0bcd87ec..17988f1b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,12 +29,11 @@ and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ------ ## [v7.0.7](https://github.com/asfadmin/Discovery-asf_search/compare/v7.0.6...v7.0.7) ### Added +- Adds `cmr_keywords` search keyword, enables passing CMR format strings in search directly - Adds `shortName` keyword, for use with lists of collection short names ------ ## [v7.0.6](https://github.com/asfadmin/Discovery-asf_search/compare/v7.0.5...v7.0.6) -### Added -- Adds `cmr_keywords` search keyword, enables passing CMR format strings in search directly ### Changed - timestamps while building queries and reading results from CMR now use UTC if no timezone is provided - Changed what collections the `NISAR` dataset and platform collections lists are pointed at. From d9dead058d27ed1065c7af88f1f09b2e2bcf5460 Mon Sep 17 00:00:00 2001 From: kim Date: Fri, 22 Mar 2024 11:32:14 -0800 Subject: [PATCH 08/10] write tests for shortName and cmr_keywords search keywords, makes cmr_keywords regular search() keyword, simplifies input format --- asf_search/ASFSearchOptions/validators.py | 22 ++++++++++----------- asf_search/CMR/translate.py | 5 ++--- asf_search/search/geo_search.py | 4 ++-- asf_search/search/search.py | 4 ++-- asf_search/search/search_count.py | 4 ++-- asf_search/search/search_generator.py | 2 +- tests/Search/test_search.py | 16 +++++++++------ tests/yml_tests/test_search.yml | 24 +++++++++++++++++++++++ 8 files changed, 53 insertions(+), 28 deletions(-) diff --git a/asf_search/ASFSearchOptions/validators.py b/asf_search/ASFSearchOptions/validators.py index 8a664029..b1a30a4e 100644 --- a/asf_search/ASFSearchOptions/validators.py +++ b/asf_search/ASFSearchOptions/validators.py @@ -109,21 +109,19 @@ def parse_list(value: Sequence, h) -> List: except ValueError as exc: raise ValueError(f'Invalid {h.__name__} list: {exc}') from exc -def parse_cmr_keywords_list(value: Sequence[Dict]): - if not isinstance(value, Sequence): +def parse_cmr_keywords_list(value: Sequence[Union[Dict, Sequence]]): + if not isinstance(value, Sequence) or (len(value) == 2 and isinstance(value[0], str)): # in case we're passed single key value pair as sequence value = [value] - required_keys = ['key', 'fmt'] - for idx, item in enumerate(value): - if not isinstance(item, dict): - raise ValueError(f"Expected item in umm search key list index {idx} to be dict, got value {item} of type {type(item)}") - for key in required_keys: - item_value = item.get(key) - if item_value is None: - raise ValueError(f"Expected key \"{key}\" in umm search key dict at index {idx}, got keys {item.keys()}") - elif not isinstance(item_value, str): - raise ValueError(f"Expected value at index {idx} for key \"{key}\" to be of type \"str\", got {type(item_value)} instead.") + if not isinstance(item, tuple) and not isinstance(item, Sequence): + raise ValueError(f"Expected item in cmr_keywords list index {idx} to be tuple pair, got value {item} of type {type(item)}") + if len(item) != 2: + raise ValueError(f"Expected item in cmr_keywords list index {idx} to be of length 2, got value {item} of length {len(item)}") + + search_key, search_value = item + if not isinstance(search_key, str) or not isinstance(search_value, str): + raise ValueError(f"Expected tuple pair of types: \"{type(str)}, {type(str)}\" in cmr_keywords at index {idx}, got value \"{str(item)}\" of types: \"{type(search_key)}, {type(search_value)}\"") return value diff --git a/asf_search/CMR/translate.py b/asf_search/CMR/translate.py index 8fb0b28d..1c57de58 100644 --- a/asf_search/CMR/translate.py +++ b/asf_search/CMR/translate.py @@ -78,9 +78,8 @@ def translate_opts(opts: ASFSearchOptions) -> List: if should_use_asf_frame(cmr_opts): cmr_opts = use_asf_frame(cmr_opts) - for search_keyword in custom_cmr_keywords: - cmr_opts.append((search_keyword['key'], search_keyword['fmt'])) - + cmr_opts.extend(custom_cmr_keywords) + additional_keys = [ ('page_size', CMR_PAGE_SIZE), ('options[temporal][and]', 'true'), diff --git a/asf_search/search/geo_search.py b/asf_search/search/geo_search.py index 0b260912..491901fd 100644 --- a/asf_search/search/geo_search.py +++ b/asf_search/search/geo_search.py @@ -1,4 +1,4 @@ -from typing import Dict, Tuple, Union, Sequence +from typing import Tuple, Union, Sequence import datetime from copy import copy @@ -44,7 +44,7 @@ def geo_search( operaBurstID: Union[str, Sequence[str]] = None, dataset: Union[str, Sequence[str]] = None, shortName: Union[str, Sequence[str]] = None, - cmr_keywords: Union[Dict[str, str], Sequence[Dict[str, str]]] = None, + cmr_keywords: Union[Tuple[str, str], Sequence[Tuple[str, str]]] = None, maxResults: int = None, opts: ASFSearchOptions = None, ) -> ASFSearchResults: diff --git a/asf_search/search/search.py b/asf_search/search/search.py index 83ea2ec6..95bcc146 100644 --- a/asf_search/search/search.py +++ b/asf_search/search/search.py @@ -1,4 +1,4 @@ -from typing import Dict, Union, Sequence, Tuple +from typing import Union, Sequence, Tuple from copy import copy import datetime @@ -43,7 +43,7 @@ def search( operaBurstID: Union[str, Sequence[str]] = None, dataset: Union[str, Sequence[str]] = None, shortName: Union[str, Sequence[str]] = None, - cmr_keywords: Union[Dict[str, str], Sequence[Dict[str, str]]] = None, + cmr_keywords: Union[Tuple[str, str], Sequence[Tuple[str, str]]] = None, maxResults: int = None, opts: ASFSearchOptions = None, ) -> ASFSearchResults: diff --git a/asf_search/search/search_count.py b/asf_search/search/search_count.py index b62b4b91..8de55742 100644 --- a/asf_search/search/search_count.py +++ b/asf_search/search/search_count.py @@ -1,5 +1,5 @@ import datetime -from typing import Dict, Sequence, Tuple, Union +from typing import Sequence, Tuple, Union from copy import copy from asf_search.ASFSearchOptions import ASFSearchOptions from asf_search.CMR.subquery import build_subqueries @@ -45,7 +45,7 @@ def search_count( operaBurstID: Union[str, Sequence[str]] = None, dataset: Union[str, Sequence[str]] = None, shortName: Union[str, Sequence[str]] = None, - cmr_keywords: Union[Dict[str, str], Sequence[Dict[str, str]]] = None, + cmr_keywords: Union[Tuple[str, str], Sequence[Tuple[str, str]]] = None, maxResults: int = None, opts: ASFSearchOptions = None, ) -> int: diff --git a/asf_search/search/search_generator.py b/asf_search/search/search_generator.py index 0998a3b3..3d484ba1 100644 --- a/asf_search/search/search_generator.py +++ b/asf_search/search/search_generator.py @@ -61,7 +61,7 @@ def search_generator( operaBurstID: Union[str, Sequence[str]] = None, dataset: Union[str, Sequence[str]] = None, shortName: Union[str, Sequence[str]] = None, - cmr_keywords: Union[Dict[str, str], Sequence[Dict[str, str]]] = None, + cmr_keywords: Union[Tuple[str, str], Sequence[Tuple[str, str]]] = None, maxResults: int = None, opts: ASFSearchOptions = None, ) -> Generator[ASFSearchResults, None, None]: diff --git a/tests/Search/test_search.py b/tests/Search/test_search.py index 6c06e716..3fda1992 100644 --- a/tests/Search/test_search.py +++ b/tests/Search/test_search.py @@ -117,12 +117,16 @@ def run_test_build_subqueries(params: ASFSearchOptions, expected: List): for key, actual_val in a: expected_val = getattr(b, key) if isinstance(actual_val, list): - if len(actual_val) > 0: # ASFSearchOptions leaves empty lists as None - expected_set = set(expected_val) - actual_set = set(actual_val) - - difference = expected_set.symmetric_difference(actual_set) - assert len(difference) == 0, f"Found {len(difference)} missing entries for subquery generated keyword: \"{key}\"\n{list(difference)}" + if key == 'cmr_keywords': + for idx, key_value_pair in enumerate(actual_val): + assert key_value_pair == expected_val[idx] + else: + if len(actual_val) > 0: # ASFSearchOptions leaves empty lists as None + expected_set = set(expected_val) + actual_set = set(actual_val) + + difference = expected_set.symmetric_difference(actual_set) + assert len(difference) == 0, f"Found {len(difference)} missing entries for subquery generated keyword: \"{key}\"\n{list(difference)}" else: assert actual_val == expected_val diff --git a/tests/yml_tests/test_search.yml b/tests/yml_tests/test_search.yml index bedc3617..f18370c1 100644 --- a/tests/yml_tests/test_search.yml +++ b/tests/yml_tests/test_search.yml @@ -570,6 +570,30 @@ tests: "C1244598379-ASFDEV", ] }] + + - test-search-build_subquery shortName: + params: + shortName: 'newShortName' + expected: + - shortName: ['newShortName'] + + - test-search-build_subquery shortName multiple: + params: + shortName: ['newShortName', 'oldShortName'] + expected: + - shortName: ['newShortName', 'oldShortName'] + + - test-search-build_subquery cmr_keywords: + params: + cmr_keywords: ['attribute[]', 'unique_value'] + expected: + - cmr_keywords: [['attribute[]', 'unique_value']] + - test-search-build_subquery multiple cmr_keywords: + params: + cmr_keywords: [['attribute[]', 'unique_value'],['special_key', 'special_value']] + expected: + - cmr_keywords: [['attribute[]', 'unique_value'],['special_key', 'special_value']] + - test-search-build_subquery configuration params: params: host: 'cmr.uat.earthdata.nasa.gov' From c7695e774eaf895e6d565788ed502b1fc2495367 Mon Sep 17 00:00:00 2001 From: kim Date: Fri, 22 Mar 2024 12:08:26 -0800 Subject: [PATCH 09/10] makes dataset and platform work in same search --- CHANGELOG.md | 2 ++ asf_search/search/search_generator.py | 3 --- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 17988f1b..2938be49 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,6 +31,8 @@ and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ### Added - Adds `cmr_keywords` search keyword, enables passing CMR format strings in search directly - Adds `shortName` keyword, for use with lists of collection short names +### Changed +- Allows using `dataset` and `platform` in same search ------ ## [v7.0.6](https://github.com/asfadmin/Discovery-asf_search/compare/v7.0.5...v7.0.6) diff --git a/asf_search/search/search_generator.py b/asf_search/search/search_generator.py index 3d484ba1..d99cc653 100644 --- a/asf_search/search/search_generator.py +++ b/asf_search/search/search_generator.py @@ -82,9 +82,6 @@ def search_generator( (getattr(opts, 'granule_list', False) or getattr(opts, 'product_list', False)): raise ValueError("Cannot use maxResults along with product_list/granule_list.") - if opts.dataset is not None and opts.platform is not None: - raise ValueError("Cannot use dataset along with platform keyword in search.") - preprocess_opts(opts) url = '/'.join(s.strip('/') for s in [f'https://{opts.host}', f'{INTERNAL.CMR_GRANULE_PATH}']) From bd5acc3f5e351bfb321a2cc8ee09c16d2a99d99f Mon Sep 17 00:00:00 2001 From: kim Date: Fri, 22 Mar 2024 12:53:17 -0800 Subject: [PATCH 10/10] fixes kml output failure on non-spatial results --- asf_search/export/kml.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/asf_search/export/kml.py b/asf_search/export/kml.py index 57f6d638..1486a1f8 100644 --- a/asf_search/export/kml.py +++ b/asf_search/export/kml.py @@ -126,7 +126,9 @@ def getItem(self, p): outerBondaryIs.append(linearRing) coordinates = ETree.Element('coordinates') - coordinates.text = '\n' + (14 * ' ') + ('\n' + (14 * ' ')).join([f"{c['Longitude']},{c['Latitude']},2000" for c in p['shape']]) + '\n' + (14 * ' ') + + if p.get('shape') is not None: + coordinates.text = '\n' + (14 * ' ') + ('\n' + (14 * ' ')).join([f"{c['Longitude']},{c['Latitude']},2000" for c in p.get('shape')]) + '\n' + (14 * ' ') linearRing.append(coordinates) self.indent(placemark, 3)