Merge pull request #284 from asfadmin/master

Release V7.0.7

CHANGELOG.md

@@ -25,6 +25,15 @@ and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 -
 
 -->
+
+------
+## [v7.0.7](https://github.com/asfadmin/Discovery-asf_search/compare/v7.0.6...v7.0.7)
+### Added
+- Adds `cmr_keywords` search keyword, enables passing CMR format strings in search directly
+- Adds `shortName` keyword, for use with lists of collection short names
+### Changed
+- Allows using `dataset` and `platform` in same search
+
 ------
 ## [v7.0.6](https://github.com/asfadmin/Discovery-asf_search/compare/v7.0.5...v7.0.6)
 ### Changed

asf_search/ASFSearchOptions/validator_map.py

@@ -3,7 +3,7 @@
 from .validators import (
     parse_string, parse_float, parse_wkt, parse_date,
     parse_string_list, parse_int_list, parse_int_or_range_list,
-    parse_float_or_range_list,
+    parse_float_or_range_list, parse_cmr_keywords_list,
     parse_session
 )
 
@@ -56,12 +56,14 @@ def validate(key, value):
     'insarStackId':           parse_string,
     'instrument':             parse_string,
     'collections':            parse_string_list,
+    'shortName':              parse_string_list,
     'temporalBaselineDays':   parse_string_list,
     'operaBurstID':           parse_string_list,
     'absoluteBurstID':        parse_int_list,
     'relativeBurstID':        parse_int_list,
     'fullBurstID':            parse_string_list,
     'dataset':                parse_string_list,
+    'cmr_keywords':           parse_cmr_keywords_list,
 
     # Config parameters       Parser
     'session':                parse_session,

asf_search/ASFSearchOptions/validators.py

@@ -2,7 +2,7 @@
 from datetime import datetime, timezone
 
 import requests
-from typing import Union, Tuple, TypeVar, Callable, List, Type, Sequence
+from typing import Dict, Union, Tuple, TypeVar, Callable, List, Type, Sequence
 
 import math
 from shapely import wkt, errors
@@ -109,6 +109,22 @@ def parse_list(value: Sequence, h) -> List:
     except ValueError as exc:
         raise ValueError(f'Invalid {h.__name__} list: {exc}') from exc
 
+def parse_cmr_keywords_list(value: Sequence[Union[Dict, Sequence]]):
+    if not isinstance(value, Sequence) or (len(value) == 2 and isinstance(value[0], str)): # in case we're passed single key value pair as sequence
+        value = [value]
+
+    for idx, item in enumerate(value):
+        if not isinstance(item, tuple) and not isinstance(item, Sequence):
+            raise ValueError(f"Expected item in cmr_keywords list index {idx} to be tuple pair, got value {item} of type {type(item)}")
+        if len(item) != 2:
+            raise ValueError(f"Expected item in cmr_keywords list index {idx} to be of length 2, got value {item} of length {len(item)}")
+
+        search_key, search_value = item
+        if not isinstance(search_key, str) or not isinstance(search_value, str):
+            raise ValueError(f"Expected tuple pair of types: \"{type(str)}, {type(str)}\" in cmr_keywords at index {idx}, got value \"{str(item)}\" of types: \"{type(search_key)}, {type(search_value)}\"")
+
+    return value
+
 # Parse and validate an iterable of strings: "foo,bar,baz"
 def parse_string_list(value: Sequence[str]) -> List[str]:
     return parse_list(value, str)

asf_search/CMR/field_map.py

@@ -35,6 +35,7 @@
     'relativeOrbit':        {'key': 'attribute[]',             'fmt': 'int,PATH_NUMBER,{0}'},
     'temporal':             {'key': 'temporal',                'fmt': '{0}'},
     'collections':          {'key': 'echo_collection_id[]',    'fmt': '{0}'},
+    'shortName':            {'key': 'shortName',               'fmt': '{0}'},
     'temporalBaselineDays': {'key': 'attribute[]',             'fmt': 'int,TEMPORAL_BASELINE_DAYS,{0}'},
 
     # SLC BURST fields

asf_search/CMR/subquery.py

@@ -22,7 +22,7 @@ def build_subqueries(opts: ASFSearchOptions) -> List[ASFSearchOptions]:
         if params.get(chunked_key) is not None:
             params[chunked_key] = chunk_list(params[chunked_key], CMR_PAGE_SIZE)
 
-    list_param_names = ['platform', 'season', 'collections', 'dataset']  # these parameters will dodge the subquery system
+    list_param_names = ['platform', 'season', 'collections', 'dataset', 'cmr_keywords', 'shortName']  # these parameters will dodge the subquery system
     skip_param_names = ['maxResults']# these params exist in opts, but shouldn't be passed on to subqueries at ALL
 
     collections, aliased_keywords = get_keyword_concept_ids(params, opts.collectionAlias)

asf_search/CMR/translate.py

@@ -51,6 +51,10 @@ def translate_opts(opts: ASFSearchOptions) -> List:
 
     # convert the above parameters to a list of key/value tuples
     cmr_opts = []
+
+    # user provided umm fields
+    custom_cmr_keywords = dict_opts.pop('cmr_keywords', [])
+
     for (key, val) in dict_opts.items():
         # If it's "session" or something else CMR doesn't accept, don't send it:
         if key not in field_map:
@@ -74,6 +78,8 @@ def translate_opts(opts: ASFSearchOptions) -> List:
     if should_use_asf_frame(cmr_opts):
             cmr_opts = use_asf_frame(cmr_opts)
 
+    cmr_opts.extend(custom_cmr_keywords)
+
     additional_keys = [
         ('page_size', CMR_PAGE_SIZE),
         ('options[temporal][and]', 'true'), 

asf_search/export/kml.py

@@ -126,7 +126,9 @@ def getItem(self, p):
         outerBondaryIs.append(linearRing)
 
         coordinates = ETree.Element('coordinates')
-        coordinates.text = '\n' + (14 * ' ') + ('\n' + (14 * ' ')).join([f"{c['Longitude']},{c['Latitude']},2000" for c in p['shape']]) + '\n' + (14 * ' ')
+
+        if p.get('shape') is not None:
+            coordinates.text = '\n' + (14 * ' ') + ('\n' + (14 * ' ')).join([f"{c['Longitude']},{c['Latitude']},2000" for c in p.get('shape')]) + '\n' + (14 * ' ')
         linearRing.append(coordinates)
 
         self.indent(placemark, 3)

asf_search/search/geo_search.py

@@ -43,6 +43,8 @@ def geo_search(
         temporalBaselineDays: Union[str, Sequence[str]] = None,
         operaBurstID: Union[str, Sequence[str]] = None,
         dataset: Union[str, Sequence[str]] = None,
+        shortName: Union[str, Sequence[str]] = None,
+        cmr_keywords: Union[Tuple[str, str], Sequence[Tuple[str, str]]] = None,
         maxResults: int = None,
         opts: ASFSearchOptions = None,
 ) -> ASFSearchResults:

asf_search/search/search.py

@@ -42,6 +42,8 @@ def search(
         temporalBaselineDays: Union[str, Sequence[str]] = None,
         operaBurstID: Union[str, Sequence[str]] = None,
         dataset: Union[str, Sequence[str]] = None,
+        shortName: Union[str, Sequence[str]] = None,
+        cmr_keywords: Union[Tuple[str, str], Sequence[Tuple[str, str]]] = None,
         maxResults: int = None,
         opts: ASFSearchOptions = None,
 ) -> ASFSearchResults:

asf_search/search/search_count.py

@@ -44,6 +44,8 @@ def search_count(
         temporalBaselineDays: Union[str, Sequence[str]] = None,
         operaBurstID: Union[str, Sequence[str]] = None,
         dataset: Union[str, Sequence[str]] = None,
+        shortName: Union[str, Sequence[str]] = None,
+        cmr_keywords: Union[Tuple[str, str], Sequence[Tuple[str, str]]] = None,
         maxResults: int = None,
         opts: ASFSearchOptions = None,
 ) -> int:

asf_search/search/search_generator.py

@@ -60,6 +60,8 @@ def search_generator(
         temporalBaselineDays: Union[str, Sequence[str]] = None,
         operaBurstID: Union[str, Sequence[str]] = None,
         dataset: Union[str, Sequence[str]] = None,
+        shortName: Union[str, Sequence[str]] = None,
+        cmr_keywords: Union[Tuple[str, str], Sequence[Tuple[str, str]]] = None,
         maxResults: int = None,
         opts: ASFSearchOptions = None,
         ) -> Generator[ASFSearchResults, None, None]:
@@ -80,9 +82,6 @@ def search_generator(
         (getattr(opts, 'granule_list', False) or getattr(opts, 'product_list', False)):
             raise ValueError("Cannot use maxResults along with product_list/granule_list.")
 
-    if opts.dataset is not None and opts.platform is not None:
-        raise ValueError("Cannot use dataset along with platform keyword in search.")
-
     preprocess_opts(opts)
 
     url = '/'.join(s.strip('/') for s in [f'https://{opts.host}', f'{INTERNAL.CMR_GRANULE_PATH}'])

tests/Search/test_search.py

@@ -117,12 +117,16 @@ def run_test_build_subqueries(params: ASFSearchOptions, expected: List):
         for key, actual_val in a:
             expected_val = getattr(b, key)
             if isinstance(actual_val, list):
-                if len(actual_val) > 0: # ASFSearchOptions leaves empty lists as None
-                    expected_set = set(expected_val)
-                    actual_set = set(actual_val)
-
-                    difference = expected_set.symmetric_difference(actual_set)
-                    assert len(difference) == 0, f"Found {len(difference)} missing entries for subquery generated keyword: \"{key}\"\n{list(difference)}"
+                if key == 'cmr_keywords':
+                    for idx, key_value_pair in enumerate(actual_val):
+                        assert key_value_pair == expected_val[idx]
+                else:
+                    if len(actual_val) > 0: # ASFSearchOptions leaves empty lists as None
+                        expected_set = set(expected_val)
+                        actual_set = set(actual_val)
+
+                        difference = expected_set.symmetric_difference(actual_set)
+                        assert len(difference) == 0, f"Found {len(difference)} missing entries for subquery generated keyword: \"{key}\"\n{list(difference)}"
             else:
                 assert actual_val == expected_val
 

tests/yml_tests/test_search.yml

@@ -570,6 +570,30 @@ tests:
             "C1244598379-ASFDEV",
           ]
         }]
+
+  - test-search-build_subquery shortName:
+      params:
+        shortName: 'newShortName'
+      expected:
+        - shortName: ['newShortName']
+
+  - test-search-build_subquery shortName multiple:
+      params:
+        shortName: ['newShortName', 'oldShortName']
+      expected:
+        - shortName: ['newShortName', 'oldShortName']
+
+  - test-search-build_subquery cmr_keywords:
+      params:
+        cmr_keywords: ['attribute[]', 'unique_value']
+      expected:
+        - cmr_keywords: [['attribute[]', 'unique_value']]
+  - test-search-build_subquery multiple cmr_keywords:
+      params:
+        cmr_keywords: [['attribute[]', 'unique_value'],['special_key', 'special_value']]
+      expected:
+        - cmr_keywords: [['attribute[]', 'unique_value'],['special_key', 'special_value']]
+
   - test-search-build_subquery configuration params:
       params:
         host: 'cmr.uat.earthdata.nasa.gov'