feat: per product_type download conf and earth_search_cog merged into earth_search

eodag/api/core.py

@@ -1024,7 +1024,9 @@ def _prepare_search(
         logger.debug(
             "Using plugin class for search: %s", search_plugin.__class__.__name__
         )
-        auth_plugin = self._plugins_manager.get_auth_plugin(search_plugin.provider)
+        auth_plugin = self._plugins_manager.get_auth_plugin(
+            search_plugin.provider, product_type
+        )
 
         # append auth to search plugin if needed
         if getattr(search_plugin.config, "need_auth", False):
@@ -1120,6 +1122,7 @@ def _do_search(self, search_plugin, count=True, raise_errors=False, **kwargs):
             # WARNING: this means an eo_product that has an invalid geometry can still
             # be returned as a search result if there was no search extent (because we
             # will not try to do an intersection)
+            auth_plugin = kwargs.get("auth", None)
             for eo_product in res:
                 # if product_type is not defined, try to guess using properties
                 if eo_product.product_type is None:
@@ -1146,13 +1149,15 @@ def _do_search(self, search_plugin, count=True, raise_errors=False, **kwargs):
                         pass
                     else:
                         eo_product.product_type = guesses[0]
+                        auth_plugin = self._plugins_manager.get_auth_plugin(
+                            eo_product.provider, eo_product.product_type
+                        )
+
                 if eo_product.search_intersection is not None:
                     download_plugin = self._plugins_manager.get_download_plugin(
                         eo_product
                     )
-                    eo_product.register_downloader(
-                        download_plugin, kwargs.get("auth", None)
-                    )
+                    eo_product.register_downloader(download_plugin, auth_plugin)
 
             results.extend(res)
             total_results = None if nb_res is None else total_results + nb_res
@@ -1335,7 +1340,9 @@ def deserialize_and_register(self, filename):
             if product.downloader is None:
                 auth = product.downloader_auth
                 if auth is None:
-                    auth = self._plugins_manager.get_auth_plugin(product.provider)
+                    auth = self._plugins_manager.get_auth_plugin(
+                        product.provider, product.product_type
+                    )
                 products[i].register_downloader(
                     self._plugins_manager.get_download_plugin(product), auth
                 )
@@ -1477,7 +1484,9 @@ def download(
         if product.downloader is None:
             auth = product.downloader_auth
             if auth is None:
-                auth = self._plugins_manager.get_auth_plugin(product.provider)
+                auth = self._plugins_manager.get_auth_plugin(
+                    product.provider, product.product_type
+                )
             product.register_downloader(
                 self._plugins_manager.get_download_plugin(product), auth
             )

eodag/api/product/metadata_mapping.py

@@ -295,7 +295,13 @@ def convert_remove_extension(string):
         @staticmethod
         def convert_get_group_name(string, pattern):
             try:
-                return re.search(pattern, str(string)).lastgroup
+                return ",".join(
+                    [
+                        k
+                        for k, v in re.search(pattern, str(string)).groupdict().items()
+                        if v is not None
+                    ]
+                )
             except AttributeError:
                 logger.warning(
                     "Could not extract property from %s using %s", string, pattern

eodag/cli.py

@@ -505,7 +505,9 @@ def download(ctx, **kwargs):
             if product.downloader is None:
                 auth = product.downloader_auth
                 if auth is None:
-                    auth = satim_api._plugins_manager.get_auth_plugin(product.provider)
+                    auth = satim_api._plugins_manager.get_auth_plugin(
+                        product.provider, product.product_type
+                    )
                 search_results[idx].register_downloader(
                     satim_api._plugins_manager.get_download_plugin(product), auth
                 )
@@ -525,7 +527,9 @@ def download(ctx, **kwargs):
             if product.downloader is None:
                 auth = product.downloader_auth
                 if auth is None:
-                    auth = satim_api._plugins_manager.get_auth_plugin(product.provider)
+                    auth = satim_api._plugins_manager.get_auth_plugin(
+                        product.provider, product.product_type
+                    )
                 search_results[idx].register_downloader(
                     satim_api._plugins_manager.get_download_plugin(product), auth
                 )

eodag/config.py

@@ -111,9 +111,21 @@ def from_yaml(cls, loader, node):
     def from_mapping(cls, mapping):
         """Build a :class:`~eodag.config.ProviderConfig` from a mapping"""
         cls.validate(mapping)
-        for key in ("api", "search", "download", "auth"):
+
+        plugin_keys = ("api", "search", "download", "auth")
+        for key in plugin_keys:
             if key in mapping:
                 mapping[key] = PluginConfig.from_mapping(mapping[key])
+        # per product_type conf
+        for pt in mapping.get("products", {}).keys():
+            for key in plugin_keys:
+                if (
+                    key in mapping["products"][pt]
+                    and mapping["products"][pt][key] is not None
+                ):
+                    mapping["products"][pt][key] = PluginConfig.from_mapping(
+                        mapping["products"][pt][key]
+                    )
         c = cls()
         c.__dict__.update(mapping)
         return c
@@ -257,6 +269,16 @@ def provider_config_init(provider_config):
                 param_value.outputs_prefix = tempfile.gettempdir()
             if not getattr(param_value, "delete_archive", None):
                 param_value.delete_archive = True
+        # per product_type configurations
+        for product_type, product_type_conf in getattr(
+            provider_config, "products", {}
+        ).items():
+            if param_name in product_type_conf.keys():
+                param_value = product_type_conf[param_name]
+                if not getattr(param_value, "outputs_prefix", None):
+                    param_value.outputs_prefix = tempfile.gettempdir()
+                if not getattr(param_value, "delete_archive", None):
+                    param_value.delete_archive = True
     # Set default priority to 0
     provider_config.__dict__.setdefault("priority", 0)
 

eodag/plugins/download/http.py

@@ -381,6 +381,7 @@ def _download_assets(
                             asset_url.replace(product.location, "")
                             .replace("https://", "")
                             .replace("http://", "")
+                            .strip("/")
                         )
                     asset_abs_path = os.path.join(fs_dir_path, asset_rel_path)
                     asset_abs_path_dir = os.path.dirname(asset_abs_path)

eodag/plugins/manager.py

@@ -185,30 +185,62 @@ def get_download_plugin(self, product):
         :rtype: :class:`~eodag.plugins.download.Download`
         """
         plugin_conf = self.providers_config[product.provider]
+        # use product type specific download plugin if it is configured
+        product_type = getattr(product, "product_type", None)
+        if product_type:
+            product_type_download_conf = (
+                getattr(plugin_conf, "products", {})
+                .get(product_type, {})
+                .get("download", {})
+            )
         try:
+            if not product_type_download_conf:
+                product_type_download_conf = plugin_conf.download
+                product_type_as_plugin_identifier = None
+            else:
+                product_type_as_plugin_identifier = product_type
             plugin_conf.download.priority = plugin_conf.priority
             plugin = self._build_plugin(
-                product.provider, plugin_conf.download, Download
+                product.provider,
+                product_type_download_conf,
+                Download,
+                product_type_as_plugin_identifier,
             )
             return plugin
         except AttributeError:
             plugin_conf.api.priority = plugin_conf.priority
             plugin = self._build_plugin(product.provider, plugin_conf.api, Api)
             return plugin
 
-    def get_auth_plugin(self, provider):
+    def get_auth_plugin(self, provider, product_type=None):
         """Build and return the authentication plugin for the given product_type and
         provider
 
         :param provider: The provider for which to get the authentication plugin
         :type provider: str
+        :param product_type: (optional) The product_type if it has a distinct
+                             configuration than the providers one
+        :type product_type: str
         :returns: The Authentication plugin for the provider
         :rtype: :class:`~eodag.plugins.authentication.Authentication`
         """
         plugin_conf = self.providers_config[provider]
         try:
+            plugin_auth_conf = plugin_conf.auth
+            # use product type specific auth plugin if it is configured
+            optional_args = {}
+            if product_type:
+                optional_args["product_type"] = product_type
+                product_type_conf = getattr(plugin_conf, "products", {}).get(
+                    product_type, {}
+                )
+                if "auth" in product_type_conf:
+                    plugin_auth_conf = product_type_conf["auth"]
+
             plugin_conf.auth.priority = plugin_conf.priority
-            plugin = self._build_plugin(provider, plugin_conf.auth, Authentication)
+            plugin = self._build_plugin(
+                provider, plugin_auth_conf, Authentication, **optional_args
+            )
             return plugin
         except AttributeError:
             # We guess the plugin being built is of type Api, therefore no need
@@ -255,11 +287,13 @@ def set_priority(self, provider, priority):
             # Sort the provider configs, taking into account the new priority order
             provider_configs.sort(key=attrgetter("priority"), reverse=True)
         # Update the priority of already built plugins of the given provider
-        for provider_name, topic_class in self._built_plugins_cache:
+        for provider_name, product_type, topic_class in self._built_plugins_cache:
             if provider_name == provider:
-                self._built_plugins_cache[(provider, topic_class)].priority = priority
+                self._built_plugins_cache[
+                    (provider, product_type, topic_class)
+                ].priority = priority
 
-    def _build_plugin(self, provider, plugin_conf, topic_class):
+    def _build_plugin(self, provider, plugin_conf, topic_class, product_type=None):
         """Build the plugin of the given topic with the given plugin configuration and
         registered as the given provider
 
@@ -269,20 +303,24 @@ def _build_plugin(self, provider, plugin_conf, topic_class):
         :type plugin_conf: :class:`~eodag.config.PluginConfig`
         :param topic_class: The type of plugin to build
         :type topic_class: :class:`~eodag.plugin.base.PluginTopic`
+        :param product_type: (optional) product_type if needed to identify the plugin
+        :type product_type: str
         :returns: The built plugin
         :rtype: :class:`~eodag.plugin.search.Search` or
                 :class:`~eodag.plugin.download.Download` or
                 :class:`~eodag.plugin.authentication.Authentication` or
                 :class:`~eodag.plugin.crunch.Crunch`
         """
         cached_instance = self._built_plugins_cache.setdefault(
-            (provider, topic_class.__name__), None
+            (provider, product_type, topic_class.__name__), None
         )
         if cached_instance is not None:
             return cached_instance
         plugin_class = EODAGPluginMount.get_plugin_by_class_name(
             topic_class, getattr(plugin_conf, "type")
         )
         plugin = plugin_class(provider, plugin_conf)
-        self._built_plugins_cache[(provider, topic_class.__name__)] = plugin
+        self._built_plugins_cache[
+            (provider, product_type, topic_class.__name__)
+        ] = plugin
         return plugin

eodag/resources/providers.yml

@@ -1716,6 +1716,9 @@
       # say the max is 10_000. In practice a too high number (e.g. 5_000) returns a 502 error ({"message": "Internal server error"}).
       # Let's set it to a more robust number: 500
       max_items_per_page: 500
+    metadata_mapping:
+      keywords: |
+        {$.collection#get_group_name((?=.*(?P<COG>-cog).*)?(?=.*(?P<S2>-s2).*)?(?=.*(?P<L8>landsat-8).*)?(?=.*(?P<L1C>-l1c).*)?(?=.*(?P<L2A>-l2a).*)?)}
   products:
     S2_MSI_L1C:
       productType: sentinel-s2-l1c
@@ -1737,6 +1740,14 @@
           $.properties."sentinel:product_id".`sub(/([S2AB]{3})_MSIL2A_([0-9]{4})([0-9]{2})([0-9]{2})(T.*)/, products/\\2/\\3/\\4/\\1_MSIL2A_\\2\\3\\4\\5)`.`sub(/\/0+/, \/)`
         tilePath: |
           $.assets.info.href.`sub(/.*/sentinel-s2-l2a\/(tiles\/.*)\/tileInfo\.json/, \\1)`
+    S2_MSI_L2A_COG:
+      productType: sentinel-s2-l2a-cogs
+      metadata_mapping:
+        assets: '$.assets'
+      download: !plugin
+        type: HTTPDownload
+        base_uri: 'https://sentinel-cogs.s3.us-west-2.amazonaws.com'
+      auth:
     L8_OLI_TIRS_C1L1:
       productType: landsat-8-l1-c1
     GENERIC_PRODUCT_TYPE:
@@ -1760,37 +1771,3 @@
           - tilePath
   auth: !plugin
     type: AwsAuth
-
----
-!provider
-  name: earth_search_cog
-  priority: 0
-  roles:
-    - host
-  description: Earth Search
-  url: https://www.element84.com/earth-search/
-  search: !plugin
-    type: StacSearch
-    api_endpoint: https://earth-search.aws.element84.com/v0/search
-    need_auth: false
-    pagination:
-      # Override the default next page url key path of StacSearch because the next link returned
-      # by Earth Search is invalid (as of 2021/04/29). Entry set to null (None) to avoid using the
-      # next page retrieval mechanism, `next_page_url_tpl` will be used instead (inherited from StacSearch)
-      # Remove that entry if Earth Search updates that and returns a valid link.
-      next_page_url_key_path: null
-      # 2021/04/28: Earth-Search relies on Sat-API whose docs (http://sat-utils.github.io/sat-api/#search-stac-items-by-simple-filtering-)
-      # say the max is 10_000. In practice a too high number (e.g. 5_000) returns a 502 error ({"message": "Internal server error"}).
-      # Let's set it to a more robust number: 500
-      max_items_per_page: 500
-    metadata_mapping:
-      platformSerialIdentifier: '$.id.`split(_, 0, -1)`'
-      polarizationMode: '$.id.`sub(/.{14}([A-Z]{2}).*/, \\1)`'
-  products:
-    S2_MSI_L2A_COG:
-      productType: sentinel-s2-l2a-cogs
-    GENERIC_PRODUCT_TYPE:
-      productType: '{productType}'
-  download: !plugin
-    type: HTTPDownload
-    base_uri: 'https://sentinel-cogs.s3.us-west-2.amazonaws.com'

eodag/resources/user_conf_template.yml

@@ -128,8 +128,3 @@ earth_search:
             aws_profile:
     download:
         outputs_prefix:
-earth_search_cog:
-    priority: # Lower value means lower priority (Default: 0)
-    search:   # Search parameters configuration
-    download:
-        outputs_prefix:

tests/context.py

@@ -33,6 +33,7 @@
 from eodag.api.search_result import SearchResult
 from eodag.cli import download, eodag, list_pt, search_crunch
 from eodag.config import load_default_config, merge_configs
+from eodag.plugins.authentication.aws_auth import AwsAuth
 from eodag.plugins.authentication.base import Authentication
 from eodag.plugins.crunch.filter_date import FilterDate
 from eodag.plugins.crunch.filter_latest_tpl_name import FilterLatestByName

tests/test_end_to_end.py

@@ -87,7 +87,7 @@
     [0.2563590566012408, 43.19555008715042, 2.379835675499976, 43.907759172380565],
 ]
 EARTH_SEARCH_COG_SEARCH_ARGS = [
-    "earth_search_cog",
+    "earth_search",
     "S2_MSI_L2A_COG",
     "2020-01-01",
     "2020-01-15",