diff --git a/eodag/api/product/_assets.py b/eodag/api/product/_assets.py index b07b58a869..af4a303042 100644 --- a/eodag/api/product/_assets.py +++ b/eodag/api/product/_assets.py @@ -17,10 +17,15 @@ # limitations under the License. from __future__ import annotations +import logging import re from collections import UserDict -from typing import TYPE_CHECKING, Any, Optional +from typing import TYPE_CHECKING, Any, Optional, Union +from typing_extensions import override + +from eodag.api.product.metadata_mapping import NOT_AVAILABLE, OFFLINE_STATUS +from eodag.utils import guess_file_type from eodag.utils.exceptions import NotAvailableError from eodag.utils.repr import dict_to_html_table @@ -29,6 +34,8 @@ from eodag.types.download_args import DownloadConf from eodag.utils import Unpack +logger = logging.getLogger("eodag.api.product.assets") + class AssetsDict(UserDict): """A UserDict object which values are :class:`~eodag.api.product._assets.Asset` @@ -56,12 +63,84 @@ class AssetsDict(UserDict): product: EOProduct + TECHNICAL_ASSETS = ["download_link", "quicklook", "thumbnail"] + def __init__(self, product: EOProduct, *args: Any, **kwargs: Any) -> None: self.product = product super(AssetsDict, self).__init__(*args, **kwargs) def __setitem__(self, key: str, value: dict[str, Any]) -> None: + if not self._check(key, value): + return super().__setitem__(key, Asset(self.product, key, value)) + self.sort() + + @override + def update(self, value: Union[dict[str, Any], AssetsDict]) -> None: # type: ignore + """Used to self update with exernal value""" + buffer: dict = {} + for key in value: + if self._check(key, value[key]): + buffer[key] = value[key] + super().update(buffer) + self.sort() + + def _check(self, asset_key: str, asset_values: dict[str, Any]) -> bool: + + # Asset must have href or order_link + href = asset_values.pop("href", None) + if href not in [None, "", NOT_AVAILABLE]: + asset_values["href"] = href + order_link = asset_values.pop("order_link", None) + if order_link not in [None, "", NOT_AVAILABLE]: + asset_values["order_link"] = order_link + + if "href" not in asset_values and "order_link" not in asset_values: + logger.warning( + "asset '{}' skipped ignored because neither href nor order_link is available".format( + asset_key + ), + ) + return False + + def target_url(asset: dict) -> Optional[str]: + target_url = None + if "href" in asset: + target_url = asset["href"] + elif "order_link" in asset: + target_url = asset["order_link"] + return target_url + + assets = self.as_dict() + used_urls = [] + for key in assets: + if key == asset_key: + # Duplicated key + return False + used_urls.append(target_url(assets[key])) + + # Prevent asset key / asset target url replication (out from technical ones) + # thumbnail and quicklook can share same url + url = target_url(asset_values) + if asset_key not in AssetsDict.TECHNICAL_ASSETS and (url in used_urls): + # Duplicated url + return False + + return True + + def sort(self): + """Used to self sort""" + sorted_assets = {} + data = self.as_dict() + # Keep technical assets first + for key in AssetsDict.TECHNICAL_ASSETS: + if key in data: + sorted_assets[key] = data.pop(key) + # Sort and add others + data = dict(sorted(data.items())) + for key in data: + sorted_assets[key] = data[key] + super().update(sorted_assets) def as_dict(self) -> dict[str, Any]: """Builds a representation of AssetsDict to enable its serialization @@ -165,6 +244,12 @@ class Asset(UserDict): """ product: EOProduct + + # Location + location: Optional[str] + remote_location: Optional[str] + + # File size: int filename: Optional[str] rel_path: str @@ -172,7 +257,42 @@ class Asset(UserDict): def __init__(self, product: EOProduct, key: str, *args: Any, **kwargs: Any) -> None: self.product = product self.key = key + self.location = None + self.remote_location = None super(Asset, self).__init__(*args, **kwargs) + self._update() + + def __setitem__(self, key, item): + super().__setitem__(key, item) + self._update() + + def _update(self): + + title = self.get("title", None) + if title is None: + super().__setitem__("title", self.key) + + # Order link behaviour require order:status state + orderlink = self.get("order_link", None) + orderstatus = self.get("order:status", None) + if orderlink is not None and orderstatus is None: + super().__setitem__("order:status", OFFLINE_STATUS) + + href = self.get("href", None) + if href is None: + super().__setitem__("href", "") + href = "" + + if href != "": + # Provide location and remote_location when undefined and href updated + if self.location is None: + self.location = href + if self.remote_location is None: + self.remote_location = href + # With order behaviour, href can be fill later + content_type = self.get("type", None) + if content_type is None: + super().__setitem__("type", guess_file_type(href)) def as_dict(self) -> dict[str, Any]: """Builds a representation of Asset to enable its serialization diff --git a/eodag/api/product/metadata_mapping.py b/eodag/api/product/metadata_mapping.py index 780bf2f167..9ecbc0bedf 100644 --- a/eodag/api/product/metadata_mapping.py +++ b/eodag/api/product/metadata_mapping.py @@ -24,6 +24,7 @@ from datetime import datetime, timedelta from string import Formatter from typing import TYPE_CHECKING, Any, AnyStr, Callable, Iterator, Optional, Union, cast +from urllib.parse import unquote import geojson import orjson @@ -189,6 +190,8 @@ def format_metadata(search_param: str, *args: Any, **kwargs: Any) -> str: - ``to_rounded_wkt``: simplify the WKT of a geometry - ``to_title``: Convert a string to title case - ``to_upper``: Convert a string to uppercase + - ``url_decode``: Convert a string url_encoded to decoded ones + - ``round``: Convert a string number to another one without decimal part :param search_param: The string to be formatted :param args: (optional) Additional arguments to use in the formatting process @@ -782,6 +785,28 @@ def convert_to_upper(string: str) -> str: """Convert a string to uppercase.""" return string.upper() + @staticmethod + def convert_url_decode(string: str): + return unquote(string) + + @staticmethod + def convert_round(string: Any) -> str: + """Convert a number string to integer string""" + if isinstance(string, float): + return str(int(string)) + elif isinstance(string, int): + return str(string) + elif isinstance(string, str): + formatted = "" + for i in range(0, len(string)): + char = string[i] + if char == ".": + break + if "0123456789".find(char) >= 0: + formatted += char + return formatted + return string + @staticmethod def convert_to_title(string: str) -> str: """Convert a string to title case.""" @@ -1518,10 +1543,17 @@ def format_query_params( if COMPLEX_QS_REGEX.match(provider_search_param): parts = provider_search_param.split("=") + if len(parts) == 1: - formatted_query_param = format_metadata( - provider_search_param, collection, **query_dict - ) + + # If part contains something to interprete + if parts[0].strip("{}").find("{") >= 0: + formatted_query_param = format_metadata( + provider_search_param, collection, **query_dict + ) + else: + formatted_query_param = "{" + parts[0].strip("{}") + "}" + formatted_query_param = formatted_query_param.replace("'", '"') if "{{" in provider_search_param: # retrieve values from hashes where keys are given in the param diff --git a/eodag/config.py b/eodag/config.py index cc46501668..a8fb841b21 100644 --- a/eodag/config.py +++ b/eodag/config.py @@ -580,7 +580,7 @@ class MetadataPreMapping(TypedDict, total=False): def __or__(self, other: Union[Self, dict[str, Any]]) -> Self: """Return a new PluginConfig with merged values.""" - new_config = self.__class__.from_mapping(self.__dict__) + new_config: Self = self.__class__.from_mapping(self.__dict__) new_config.update(other) return new_config diff --git a/eodag/plugins/apis/usgs.py b/eodag/plugins/apis/usgs.py index 8816897e11..760dfcf20c 100644 --- a/eodag/plugins/apis/usgs.py +++ b/eodag/plugins/apis/usgs.py @@ -134,6 +134,8 @@ def authenticate(self) -> None: os.remove(api.TMPFILE) continue raise AuthenticationError("Please check your USGS credentials.") from e + except Exception as e: + raise AuthenticationError("Authentication failure") from e def query( self, diff --git a/eodag/plugins/search/base.py b/eodag/plugins/search/base.py index 3ff83130e2..7db31229d8 100644 --- a/eodag/plugins/search/base.py +++ b/eodag/plugins/search/base.py @@ -18,16 +18,20 @@ from __future__ import annotations import logging -from typing import TYPE_CHECKING, Annotated, get_args +from typing import TYPE_CHECKING, Annotated, Callable, get_args import orjson from pydantic import AliasChoices +from jsonpath_ng import JSONPath +from jsonpath_ng.jsonpath import Child as JSONChild from pydantic import ValidationError as PydanticValidationError from pydantic.fields import Field, FieldInfo +from eodag.api.product import AssetsDict from eodag.api.product.metadata_mapping import ( NOT_AVAILABLE, NOT_MAPPED, + format_metadata, mtd_cfg_as_conversion_and_querypath, ) from eodag.api.search_result import SearchResult @@ -55,6 +59,7 @@ from mypy_boto3_s3 import S3ServiceResource from requests.auth import AuthBase + from eodag import EOProduct from eodag.config import PluginConfig logger = logging.getLogger("eodag.search.base") @@ -224,15 +229,65 @@ def get_metadata_mapping( self, collection: Optional[str] = None ) -> dict[str, Union[str, list[str]]]: """Get the plugin metadata mapping configuration (collection specific if exists) + :param collection: the desired collection + :returns: The collection metadata assets-mapping + """ + metadata_mapping = getattr(self.config, "metadata_mapping", {}) + if isinstance(collection, str): + # Special overfload for collection config + # "metadata_mapping_from_product" overloaded by "current" + collection_metadata_mapping: dict[str, Union[str, list[str]]] = {} + target_collection: Optional[str] = collection + watchdog = 10 + while target_collection is not None and watchdog > 0: + watchdog -= 1 + # Check if collection has a specific configuration + collection_config = self.config.products.get(target_collection, {}) + # Overload imported asset mapping configuration by current one + buffer = collection_config.get("metadata_mapping", {}) + buffer.update(collection_metadata_mapping) + collection_metadata_mapping = buffer + # This collection configuration can refer to another collection configuration + target_collection = collection_config.get( + "metadata_mapping_from_product", None + ) + metadata_mapping.update(collection_metadata_mapping) + if watchdog == 0: + logger.warning("get_assets_mapping watchdog triggered") + + return metadata_mapping + def get_assets_mapping( + self, collection: Optional[str] = None + ) -> Union[Any, dict[str, dict[str, Any]]]: + """Get the plugin asset mapping configuration (collection specific if exists) :param collection: the desired collection - :returns: The collection specific metadata-mapping + :returns: The collection specific assets-mapping """ - if collection: - return self.config.products.get(collection, {}).get( - "metadata_mapping", self.config.metadata_mapping - ) - return self.config.metadata_mapping + assets_mapping = getattr(self.config, "assets_mapping", {}) + if isinstance(collection, str): + # Special overfload for collection config + # "metadata_mapping_from_product" overloaded by "current" + collection_asset_mapping: dict[str, Union[str, list[str]]] = {} + target_collection: Optional[str] = collection + watchdog = 10 + while target_collection is not None and watchdog > 0: + watchdog -= 1 + # Check if collection has a specific configuration + collection_config = self.config.products.get(target_collection, {}) + # Overload imported asset mapping configuration by current one + buffer = collection_config.get("assets_mapping", {}) + buffer.update(collection_asset_mapping) + collection_asset_mapping = buffer + # This collection configuration can refer to another collection configuration + target_collection = collection_config.get( + "metadata_mapping_from_product", None + ) + assets_mapping.update(collection_asset_mapping) + if watchdog == 0: + logger.warning("get_assets_mapping watchdog triggered") + + return assets_mapping def get_sort_by_arg(self, kwargs: dict[str, Any]) -> Optional[SortByList]: """Extract the ``sort_by`` argument from the kwargs or the provider default sort configuration @@ -507,7 +562,9 @@ def queryables_from_metadata_mapping( queryables[k] = v return queryables - def get_assets_from_mapping(self, provider_item: dict[str, Any]) -> dict[str, Any]: + def get_assets_from_mapping( + self, provider_item: dict[str, Any], product: EOProduct + ) -> AssetsDict: """ Create assets based on the assets_mapping in the provider's config and an item returned by the provider @@ -515,26 +572,232 @@ def get_assets_from_mapping(self, provider_item: dict[str, Any]) -> dict[str, An :param provider_item: dict of item properties returned by the provider :returns: dict containing the asset metadata """ - assets_mapping = getattr(self.config, "assets_mapping", None) + + # Gather collection and product properties + collection = getattr(product, "collection", None) + properties = getattr(product, "properties", {}) + + # Complete given product properties with configuration by product + # (not always yet in product properties when current function called) + if isinstance(collection, str): + # Replace templated parameters by values from product properties and provider_item + collection_properties = MappingInterpretor.metadata_mapping_compute( + self.config.products.get(collection, {}), + properties=properties, + provider_item=provider_item, + ) + # have to redo to consider new parameters just added + # (some parameters are filled by other ones just defined) + collection_properties = MappingInterpretor.metadata_mapping_compute( + collection_properties, properties=collection_properties + ) + # extends base config + for field_name in collection_properties: + if field_name not in [ + "metadata_mapping", + "assets_mapping", + "metadata_mapping_from_product", + ]: + properties[field_name] = collection_properties[field_name] + + # Gather asset mapping + assets_mapping = self.get_assets_mapping(collection) if not assets_mapping: return {} - assets = {} - for key, values in assets_mapping.items(): - asset_href = values.get("href") - if not asset_href: - logger.warning( - "asset mapping %s skipped because no href is available", key + else: + # Compute asset mapping + assets_mapping = MappingInterpretor.metadata_mapping_compute( + assets_mapping, properties=properties, provider_item=provider_item + ) + + # Specifcs assets (like download_link, thumbnail or quicklook) + assets = AssetsDict(product) + assets.update(assets_mapping) + + # Global imported assets + imported_assets = {} + if "assets" in properties: + imported_assets = product.properties.pop("assets", {}) + + # Process local assets through product driver + driver = getattr(product, "driver", None) + if driver is not None: + asset_key_from_href = getattr(self.config, "asset_key_from_href", True) + computed_assets = {} + for asset_key in imported_assets: + # Local transformation from driver + asset = imported_assets[asset_key] + norm_key, asset["roles"] = driver.guess_asset_key_and_roles( + asset.get("href", "") if asset_key_from_href else asset_key, + product, ) - continue - json_url_path = string_to_jsonpath(asset_href) - if isinstance(json_url_path, str): - url_path = json_url_path - else: - url_match = json_url_path.find(provider_item) - if len(url_match) == 1: - url_path = url_match[0].value - else: - url_path = NOT_AVAILABLE - assets[key] = deepcopy(values) - assets[key]["href"] = url_path + asset["title"] = norm_key + computed_assets[asset_key] = asset + imported_assets = computed_assets + + assets.update(computed_assets) + return assets + + +class MappingInterpretor: + """Class to process configuration mapping""" + + @staticmethod + def metadata_mapping_compute( + metadata_mapping_data: Optional[dict] = None, + properties: Optional[dict] = None, + provider_item: Optional[dict] = None, + ): + """Mapping from configuration with product properties and provider_item""" + # patch json_path without {...} to tag it as interpretable + metadata_mapping_data = MappingInterpretor.update_json_path_as_interpretable( + metadata_mapping_data + ) + + # all interpretable in {...} + return MappingInterpretor.replace_interpretable( + metadata_mapping_data, + MappingInterpretor.metadata_substitution, + properties=properties, + provider_item=provider_item, + ) # typing: ignore[arg-type] + + @staticmethod + def update_json_path_as_interpretable(value: Any): + """Transform in crawled structure raw value '$.[...]' into '{$.[...]}'""" + if isinstance(value, str) and str != "": + if value.startswith("$."): + value = "{" + value + "}" + elif isinstance(value, dict): + for key in value: + value[key] = MappingInterpretor.update_json_path_as_interpretable( + value[key] + ) + elif isinstance(value, list): + for i in range(0, len(value)): + value[i] = MappingInterpretor.update_json_path_as_interpretable( + value[i] + ) + return value + + @staticmethod + def metadata_substitution( + value: str, + properties: Optional[dict[str, Any]] = None, + provider_item: Optional[dict[str, Any]] = None, + ) -> Any: + """Mapping rules + replace field {value#filter} by property if possible + replace field {$.jsonpath#filter} by provider_item value if possible + """ + + # Properties substitution "{field}" matching with "properties" key + if isinstance(value, str) and properties is not None: + # Extract filters + filters = value.strip("{}").split("#") + # Parameters subtitution + field_name = filters.pop(0) + if field_name in properties: + value = properties[field_name] + # apply filters + for filter in filters: + scheme = "{fieldname#" + filter + "}" + try: + value = format_metadata(scheme, fieldname=value) + except Exception as e: + logger.warning( + "Error during properties substitution template '{}': {}".format( + value, str(e) + ) + ) + + # Provider item substitution "{$.jsonpath}" matching with "provider_item" json path resolve + if isinstance(value, str) and provider_item is not None: + # Extract filters + filters = value.strip("{}").split("#") + # Parameters subtitution + field_name = filters.pop(0) + if field_name.startswith("$."): + # Is a josn path ? + json_path = string_to_jsonpath(field_name) + if isinstance(json_path, JSONPath) or isinstance(json_path, JSONChild): + match = json_path.find(provider_item) + if len(match) == 1: + value = match[0].value + else: + value = NOT_AVAILABLE + # Apply filters + for filter in filters: + scheme = "{fieldname#" + filter + "}" + try: + value = format_metadata(scheme, fieldname=value) + except Exception as e: + logger.warning( + "Error during provider_item substitution template {}: {}".format( + value, str(e) + ) + ) + + return value + + @staticmethod + def replace_interpretable(value: Any, on_interpretable: Callable, *args, **kwargs): + """Helper used to parse mapping parameters + + It extract sub element {...} from value, and call "on_interpretable" + to substitute this sub part and replace it in value + If value is not a string, ll try crawl by recursion to scan all data structure + + in_interpretable is called chen a sub element {...} is found to substitute it. + args and kwargs are pass through on_interpretable function + """ + + if isinstance(value, str) and len(value) > 0: + # Extract level 1 layer {} + level = 0 + start = 0 + end = 0 + i = 0 + while i < len(value): + char = value[i] + if char == "{": + level += 1 + if level == 1: + start = i + if char == "}": + level -= 1 + if level == 0: + end = i + 1 + segment = value[start:end] + if len(segment) >= 2 and segment[1:-1].find("{") >= 0: + # Has sublevels ? + result = ( + "{" + + MappingInterpretor.replace_interpretable( + segment[1:-1], on_interpretable, *args, **kwargs + ) + + "}" + ) + else: + # Direct interpretable substitution + result = on_interpretable(segment, *args, **kwargs) + if result != segment: + value = value[0:start] + str(result) + value[end:] + # Data moved, cursor is invalid + i = -1 + i += 1 + elif isinstance(value, list): + # Crawl by recursion + for i in range(0, len(value)): + value[i] = MappingInterpretor.replace_interpretable( + value[i], on_interpretable, *args, **kwargs + ) + elif isinstance(value, dict): + # Crawl by recursion + for key in value: + value[key] = MappingInterpretor.replace_interpretable( + value[key], on_interpretable, *args, **kwargs + ) + + return value diff --git a/eodag/plugins/search/build_search_result.py b/eodag/plugins/search/build_search_result.py index 489a02d0a0..53b987ded8 100644 --- a/eodag/plugins/search/build_search_result.py +++ b/eodag/plugins/search/build_search_result.py @@ -40,7 +40,6 @@ from eodag.api.product.metadata_mapping import ( DEFAULT_GEOMETRY, NOT_AVAILABLE, - OFFLINE_STATUS, STAGING_STATUS, format_metadata, mtd_cfg_as_conversion_and_querypath, @@ -436,8 +435,6 @@ def __init__(self, provider: str, config: PluginConfig) -> None: **{ "id": "$.id", "title": "$.id", - "order:status": OFFLINE_STATUS, - "eodag:download_link": "$.null", "geometry": ["feature", "$.geometry"], "eodag:default_geometry": "POLYGON((180 -90, 180 90, -180 90, -180 -90, 180 -90))", }, @@ -1281,6 +1278,9 @@ def normalize_results( **kwargs, ) + # "Technicals" assets as (downloadlink, quicklook, thumbnail) + product.assets.update(self.get_assets_from_mapping(result, product)) + # backup original register_downloader to register_downloader_only product.register_downloader_only = product.register_downloader # patched register_downloader that will also update properties diff --git a/eodag/plugins/search/cop_marine.py b/eodag/plugins/search/cop_marine.py index 3af94ba5fc..788ddca7fc 100644 --- a/eodag/plugins/search/cop_marine.py +++ b/eodag/plugins/search/cop_marine.py @@ -33,7 +33,6 @@ from dateutil.utils import today from eodag import EOProduct -from eodag.api.product import AssetsDict from eodag.api.search_result import SearchResult from eodag.config import PluginConfig from eodag.plugins.search import PreparedSearch @@ -227,9 +226,17 @@ def _create_product( "id": item_id, "title": item_id, "geometry": geometry, - "eodag:download_link": download_url, "dataset": dataset_item["id"], } + assets = { + "download_link": { + "title": "downloadlink", + "href": download_url, + "roles": ["archive", "data"], + "type": "application/x-netcdf", + } + } + if use_dataset_dates: dates = _get_dates_from_dataset_data(dataset_item) if not dates: @@ -276,22 +283,25 @@ def _create_product( _check_int_values_properties(properties) - properties["eodag:thumbnail"] = collection_dict["assets"]["thumbnail"]["href"] + assets["thumbnail"] = { + "title": "thumbnail", + "href": collection_dict["assets"]["thumbnail"]["href"], + "roles": ["thumbnail"], + "type": "image/jpeg", + } if "omiFigure" in collection_dict["assets"]: - properties["eodag:quicklook"] = collection_dict["assets"]["omiFigure"][ - "href" - ] - assets = { - "native": { - "title": "native", - "href": download_url, - "type": "application/x-netcdf", + assets["quicklook"] = { + "title": "quicklook", + "href": collection_dict["assets"]["omiFigure"]["href"], + "roles": ["overview"], + "type": "image/jpeg", } - } - additional_assets = self.get_assets_from_mapping(dataset_item) - assets.update(additional_assets) + + # List current asset urls product = EOProduct(self.provider, properties, collection=collection) - product.assets = AssetsDict(product, assets) + merged_assets = self.get_assets_from_mapping(dataset_item, product) + merged_assets.update(assets) + product.assets.update(merged_assets) return product def query( diff --git a/eodag/plugins/search/qssearch.py b/eodag/plugins/search/qssearch.py index 314022c6b8..208525a2dc 100644 --- a/eodag/plugins/search/qssearch.py +++ b/eodag/plugins/search/qssearch.py @@ -411,9 +411,6 @@ def __init__(self, provider: str, config: PluginConfig) -> None: other_collection_mtd_mapping = mtd_cfg_as_conversion_and_querypath( other_collection_def_params.get("metadata_mapping", {}) ) - else: - msg = f"Cannot reuse empty metadata_mapping from {other_product_for_mapping} for {collection}" - raise MisconfiguredError(msg) # update mapping for metadata, mapping in other_collection_mtd_mapping.items(): collection_metadata_mapping.pop(metadata, None) @@ -1271,7 +1268,6 @@ def normalize_results( % normalize_remaining_count ) products: list[EOProduct] = [] - asset_key_from_href = getattr(self.config, "asset_key_from_href", True) product_kwargs = deepcopy(kwargs) # collection alias as collection property for product if alias := getattr(self.config, "collection_config", {}).get("alias"): @@ -1284,21 +1280,10 @@ def normalize_results( ) product = EOProduct(self.provider, properties, **product_kwargs) - additional_assets = self.get_assets_from_mapping(result) - product.assets.update(additional_assets) - # move assets from properties to product's attr, normalize keys & roles - for key, asset in product.properties.pop("assets", {}).items(): - norm_key, asset["roles"] = product.driver.guess_asset_key_and_roles( - asset.get("href", "") if asset_key_from_href else key, - product, - ) - if norm_key: - product.assets[norm_key] = asset - # Normalize title with key - product.assets[norm_key]["title"] = norm_key - # sort assets - product.assets.data = dict(sorted(product.assets.data.items())) + # "Technicals" assets as (downloadlink, quicklook, thumbnail) + product.assets.update(self.get_assets_from_mapping(result, product)) products.append(product) + return products def count_hits(self, count_url: str, result_type: Optional[str] = "json") -> int: diff --git a/eodag/resources/providers.yml b/eodag/resources/providers.yml index 8577cd611b..a34d8e2208 100644 --- a/eodag/resources/providers.yml +++ b/eodag/resources/providers.yml @@ -40,13 +40,28 @@ start_datetime: '$.temporalCoverage.startDate' end_datetime: '$.temporalCoverage.endDate' published: '$.publishDate' - eodag:thumbnail: '$.browse[0].thumbnailPath' - eodag:quicklook: '$.browse[0].browsePath' - order:status: '{$.available#get_group_name((?PTrue)|(?PFalse))}' - eodag:download_link: 'https://earthexplorer.usgs.gov/download/external/options/{_collection}/{entityId}/M2M/' - # metadata needed for download usgs:entityId: '$.entityId' usgs:productId: '$.id' + assets_mapping: + download_link: + href: 'https://earthexplorer.usgs.gov/download/external/options/{_collection}/{$.entityId}/M2M/' + roles: + - 'archive' + - 'data' + type: 'application/octet-stream' + order:status: '{$.available#get_group_name((?PTrue)|(?PFalse))}' + quicklook: + href: '$.browse[0].browsePath' + title: "quicklook" + roles: + - 'overwiev' + type: 'image/jpeg' + thumbnail: + href: '$.browse[0].thumbnailPath' + title: "thumbnail" + roles: + - 'thumbnail' + type: 'image/jpeg' extract: True order_enabled: true max_workers: 2 @@ -110,8 +125,9 @@ geometry: - '{{"search":{{"shape": {geometry#to_geojson} }} }}' - '$.dataGeometry' - # order:status set to succeeded for consistency between providers - order:status: '{$.null#replace_str("Not Available","succeeded")}' + assets_mapping: + download_link: + order:status: 'succeeded' products: CBERS4_PAN10M_L2: instruments: PAN10M @@ -148,24 +164,47 @@ - '$.sunElevation' # Custom parameters (not defined in the base document referenced above) _aws_path: '$.downloadUrl' - eodag:download_link: 's3://cbers-pds/{_aws_path}' eodag:mtd_download_link: 's3://cbers-meta-pds/{_aws_path}' - _preview_basename: '{$.sceneID#replace_str("_L2","")}' - eodag:thumbnail: 'https://s3.amazonaws.com/cbers-meta-pds/{_aws_path}/{_preview_basename}_small.jpeg' - eodag:quicklook: 'https://s3.amazonaws.com/cbers-meta-pds/{_aws_path}/{_preview_basename}.jpg' id: - '{{"search":{{"sceneID":"{id}" }} }}' - '{title}' + assets_mapping: + download_link: + href: 's3://cbers-pds/{$.downloadUrl}' + roles: + - 'archive' + - 'data' + order:status: 'succeded' + quicklook: + href: 'https://s3.amazonaws.com/cbers-meta-pds/{$.downloadUrl}/{$.sceneID#replace_str("_L2","")}.jpg' + title: "quicklook" + roles: + - 'overwiev' + type: 'image/jpeg' + thumbnail: + href: 'https://s3.amazonaws.com/cbers-meta-pds/{$.downloadUrl}/{$.sceneID#replace_str("_L2","")}_small.jpeg' + title: "thumbnail" + roles: + - 'thumbnail' + type: 'image/jpeg' CBERS4_PAN10M_L4: instruments: PAN10M _collection: cbers4 processing:level: 4 metadata_mapping_from_product: CBERS4_PAN10M_L2 - metadata_mapping: - # Custom parameters (not defined in the base document referenced above) - _preview_basename: '{$.sceneID#replace_str("_L4","")}' - eodag:thumbnail: 'https://s3.amazonaws.com/cbers-meta-pds/{_aws_path}/{_preview_basename}_small.jpeg' - eodag:quicklook: 'https://s3.amazonaws.com/cbers-meta-pds/{_aws_path}/{_preview_basename}.jpg' + assets_mapping: + quicklook: + href: 'https://s3.amazonaws.com/cbers-meta-pds/{$.downloadUrl}/{$.sceneID#replace_str("_L4","")}.jpg' + title: "quicklook" + roles: + - 'overwiev' + type: 'image/jpeg' + thumbnail: + href: 'https://s3.amazonaws.com/cbers-meta-pds/{$.downloadUrl}/{$.sceneID#replace_str("_L4","")}_small.jpeg' + title: "thumbnail" + roles: + - 'thumbnail' + type: 'image/jpeg' CBERS4_PAN5M_L2: instruments: PAN5M _collection: cbers4 @@ -175,12 +214,7 @@ instruments: PAN5M _collection: cbers4 processing:level: 4 - metadata_mapping_from_product: CBERS4_PAN10M_L2 - metadata_mapping: - # Custom parameters (not defined in the base document referenced above) - _preview_basename: '{$.sceneID#replace_str("_L4","")}' - eodag:thumbnail: 'https://s3.amazonaws.com/cbers-meta-pds/{_aws_path}/{_preview_basename}_small.jpeg' - eodag:quicklook: 'https://s3.amazonaws.com/cbers-meta-pds/{_aws_path}/{_preview_basename}.jpg' + metadata_mapping_from_product: CBERS4_PAN10M_L4 CBERS4_MUX_L2: instruments: MUX _collection: cbers4 @@ -190,12 +224,7 @@ instruments: MUX _collection: cbers4 processing:level: 4 - metadata_mapping_from_product: CBERS4_PAN10M_L2 - metadata_mapping: - # Custom parameters (not defined in the base document referenced above) - _preview_basename: '{$.sceneID#replace_str("_L4","")}' - eodag:thumbnail: 'https://s3.amazonaws.com/cbers-meta-pds/{_aws_path}/{_preview_basename}_small.jpeg' - eodag:quicklook: 'https://s3.amazonaws.com/cbers-meta-pds/{_aws_path}/{_preview_basename}.jpg' + metadata_mapping_from_product: CBERS4_PAN10M_L4 CBERS4_AWFI_L2: instruments: AWFI _collection: cbers4 @@ -205,12 +234,7 @@ instruments: AWFI _collection: cbers4 processing:level: 4 - metadata_mapping_from_product: CBERS4_PAN10M_L2 - metadata_mapping: - # Custom parameters (not defined in the base document referenced above) - _preview_basename: '{$.sceneID#replace_str("_L4","")}' - eodag:thumbnail: 'https://s3.amazonaws.com/cbers-meta-pds/{_aws_path}/{_preview_basename}_small.jpeg' - eodag:quicklook: 'https://s3.amazonaws.com/cbers-meta-pds/{_aws_path}/{_preview_basename}.jpg' + metadata_mapping_from_product: CBERS4_PAN10M_L4 L8_OLI_TIRS_C1L1: _collection: landsat8 _on_amazon: true @@ -255,6 +279,25 @@ id: - '{{"search":{{"productID":"{id}" }} }}' - '{title}' + assets_mapping: + download_link: + href: 's3://landsat-pds/c1/L8/{_path:03.0f}/{_row:03.0f}/{title}/' + roles: + - 'archive' + - 'data' + order:status: 'succeded' + quicklook: + href: 'https://landsat-pds.s3.amazonaws.com/c1/L8/{_path:03.0f}/{_row:03.0f}/{title}/{title}_thumb_large.jpg' + title: "quicklook" + roles: + - 'overwiev' + type: 'image/jpeg' + thumbnail: + href: '$.thumbnail' + title: "thumbnail" + roles: + - 'thumbnail' + type: 'image/jpeg' MODIS_MCD43A4: _collection: modis metadata_mapping: @@ -279,15 +322,29 @@ - '{{"search":{{"date":{{"to":"{end_datetime}"}} }} }}' - '$.EndingDateTime' # Custom parameters (not defined in the base document referenced above) - _vertical_tile_nb: '$.verticalTileNumber' - _horizontal_tile_nb: '$.horizontalTileNumber' - _doy_date: '{$.sceneID#slice_str(9,16,1)}' - eodag:download_link: 's3://modis-pds/MCD43A4.006/{_horizontal_tile_nb:02.0f}/{_vertical_tile_nb:02.0f}/{_doy_date}/' - eodag:thumbnail: '$.thumbnail' - eodag:quicklook: '$.thumbnail' id: - '{{"search":{{"sceneID":"{id}" }} }}' - '{title}' + assets_mapping: + download_link: + href: 's3://modis-pds/MCD43A4.006/{$.horizontalTileNumber#round}/{$.verticalTileNumber#round}/{$.sceneID#slice_str(9,16,1)}/' + roles: + - 'archive' + - 'data' + type: 'application/octet-stream' + order:status: 'succeded' + quicklook: + href: '$.thumbnail' + title: "quicklook" + roles: + - 'overwiev' + type: 'image/jpeg' + thumbnail: + href: '$.thumbnail' + title: "thumbnail" + roles: + - 'thumbnail' + type: 'image/jpeg' NAIP: _collection: naip metadata_mapping: @@ -308,11 +365,17 @@ - '{{"search":{{"date":{{"to":"{end_datetime}"}} }} }}' - '$.date' # Custom parameters (not defined in the base document referenced above) - _aws_path: '$.awsPath' - eodag:download_link: 's3://naip-analytic/{_aws_path}' id: - '{{"search":{{"sceneID":"{id}" }} }}' - '{title}' + assets_mapping: + download_link: + href: 's3://naip-analytic/{$.awsPath}' + roles: + - 'archive' + - 'data' + type: 'application/octet-stream' + order:status: 'succeded' S1_SAR_GRD: product:type: GRD _collection: sentinel1 @@ -347,13 +410,29 @@ - '{{"search":{{"polarization":"{sar:polarizations#replace_tuple(((["HH"],"SH"),(["VV"],"SV"),(["HH","HV"], "DH"),(["VV","VH"],"DV")))}" }} }}' - '{$.polarization#replace_tuple((("SH",["HH"]),("SV",["VV"]),("DH",["HH","HV"]),("DV",["VV","VH"])))}' # Custom parameters (not defined in the base document referenced above) - _aws_path: '$.awsPath' - eodag:download_link: 's3://sentinel-s1-l1c/{_aws_path}' - eodag:thumbnail: 'https://render.eosda.com/S1/thumb/{title}.png' - eodag:quicklook: 'https://render.eosda.com/S1/thumb/{title}.png' id: - '{{"search":{{"sceneID":"{id}" }} }}' - '{title}' + assets_mapping: + download_link: + href: 's3://sentinel-s1-l1c/{$.awsPath}' + roles: + - 'archive' + - 'data' + type: 'application/octet-stream' + order:status: 'succeded' + quicklook: + href: 'https://render.eosda.com/S1/thumb/{title}.png' + title: "quicklook" + roles: + - 'overwiev' + type: 'image/png' + thumbnail: + href: 'https://render.eosda.com/S1/thumb/{title}.png' + title: "thumbnail" + roles: + - 'thumbnail' + type: 'image/png' S2_MSI_L1C: _collection: sentinel2 metadata_mapping: @@ -393,6 +472,26 @@ - '{title}' _processed_l2a: '$.null' _aws_path_l2a: '$.null' + assets_mapping: + download_link: + href: 's3://sentinel-s2-l1c/{$.awsPath}' + roles: + - 'archive' + - 'data' + type: 'application/octet-stream' + order:status: 'succeded' + quicklook: + href: '{$.thumbnail#replace_str("sentinel-s2-l1c.s3.amazonaws.com","roda.sentinel-hub.com/sentinel-s2-l1c")}' + title: "quicklook" + roles: + - 'overwiev' + type: 'image/jpeg' + thumbnail: + href: '{$.thumbnail#replace_str("sentinel-s2-l1c.s3.amazonaws.com","roda.sentinel-hub.com/sentinel-s2-l1c")}' + title: "thumbnail" + roles: + - 'thumbnail' + type: 'image/jpeg' S2_MSI_L2A: _collection: sentinel2 _processed_l2a: true @@ -447,10 +546,6 @@ - '{{"search":{{"zenithAngle":"{view:incidence_angle}" }} }}' - '$.zenithAngle' # Custom parameters (not defined in the base document referenced above) - eodag:thumbnail: '{$.thumbnail#replace_str("sentinel-s2-l1c.s3.amazonaws.com","roda.sentinel-hub.com/sentinel-s2-l1c")}' - eodag:quicklook: '{eodag:thumbnail}' - eodag:download_link: 's3://sentinel-s2-l2a/{_aws_path_l2a}' - _aws_path: '$.null' eodag:product_path: '$.null' eodag:product_info: 'https://roda.sentinel-hub.com/sentinel-s2-l2a/{_aws_path_l2a}/productInfo.json' id: @@ -460,7 +555,26 @@ - '{{"search":{{"processedL2A":"{_processed_l2a}" }} }}' - '$.processedL2A' _aws_path_l2a: '$.awsPathL2A' - + assets_mapping: + download_link: + href: 's3://sentinel-s2-l2a/{$.awsPathL2A}' + roles: + - 'archive' + - 'data' + type: 'application/octet-stream' + order:status: 'succeded' + quicklook: + href: '{$.thumbnail#replace_str("sentinel-s2-l1c.s3.amazonaws.com","roda.sentinel-hub.com/sentinel-s2-l1c")}' + title: "quicklook" + roles: + - 'overwiev' + type: 'image/jpeg' + thumbnail: + href: '{$.thumbnail#replace_str("sentinel-s2-l1c.s3.amazonaws.com","roda.sentinel-hub.com/sentinel-s2-l1c")}' + title: "thumbnail" + roles: + - 'thumbnail' + type: 'image/jpeg' download: !plugin type: AwsDownload ssl_verify: true @@ -613,18 +727,31 @@ geometry: - 'geometry={geometry#to_rounded_wkt}' - '$.geometry' - # The url of the quicklook - eodag:quicklook: '$.properties.quicklook' - # The url to download the product "as is" (literal or as a template to be completed either after the search result - # is obtained from the provider or during the eodag download phase) - eodag:download_link: '$.properties.services.download.url' - # order:status: must be one of succeeded, ordered, orderable - order:status: '{$.properties.storage.mode#get_group_name((?Pdisk|tier2)|(?Pstaging)|(?Punknown|tape|tier3))}' - - # Additional metadata provided by the providers but that don't appear in the reference spec - eodag:thumbnail: '$.properties.thumbnail' links: $.null services: $.null + assets_mapping: + download_link: + href: '$.properties.services.download.url' + roles: + - 'archive' + - 'data' + type: '$.properties.services.download.mimeType' + # order:status: must be one of succeeded, ordered, orderable + file:size: '$.properties.services.download.size' + file:checksum: '$.properties.services.download.checksum' + order:status: '{$.properties.storage.mode#get_group_name((?Pdisk|tier2)|(?Pstaging)|(?Punknown|tape|tier3))}' + quicklook: + href: '$.properties.quicklook' + title: "quicklook" + roles: + - 'overwiev' + type: "image/jpeg" + thumbnail: + href: '$.properties.thumbnail' + title: "thumbnail" + roles: + - 'thumbnail' + type: "image/jpeg" products: S1_SAR_OCN: product:type: OCN @@ -891,16 +1018,34 @@ geometry: - null - '{$.Footprint#from_ewkt}' - # The url to download the product "as is" (literal or as a template to be completed either after the search result - # is obtained from the provider or during the eodag download phase) - eodag:download_link: 'https://zipper.creodias.eu/odata/v1/Products({uid})/$value' - # order:status: must be one of succeeded, ordered, orderable - order:status: '{$.Online#get_group_name((?PTrue)|(?PFalse))}' collection: - null - $.null - eodag:quicklook: '$.Assets[?(@.Type="QUICKLOOK")].DownloadLink' - eodag:thumbnail: '$.Assets[?(@.Type="QUICKLOOK")].DownloadLink' + assets: '$.Assets' + assets_mapping: + download_link: + title: 'downloadlink' + href: 'https://zipper.creodias.eu/odata/v1/Products({uid})/$value' + roles: + - 'archive' + - 'data' + type: '$.ContentType' + file:checksum: '$.Checksum[?Algorithm="MD5"].Value' + file:size: '$.ContentLength' + # order:status: must be one of succeeded, ordered, orderable + order:status: '{$.Online#get_group_name((?PTrue)|(?PFalse))}' + quicklook: + href: '$.Assets[?(@.Type="QUICKLOOK")].DownloadLink' + title: "quicklook" + roles: + - 'overwiev' + type: "image/jpeg" + thumbnail: + href: '$.Assets[?(@.Type="QUICKLOOK")].DownloadLink' + title: "thumbnail" + roles: + - 'thumbnail' + type: "image/jpeg" download: !plugin type: HTTPDownload extract: true @@ -1337,6 +1482,19 @@ # Note: Sorting is intentionally disabled for this provider, as enabling it causes pagination to malfunction. metadata_mapping: assets: '{$.assets#from_alternate(s3)}' + assets_mapping: + quicklook: + href: '$.assets.reduced_resolution_browse.alternate.s3.href' + title: "quicklook" + roles: + - 'overwiev' + type: '$.assets.reduced_resolution_browse.type' + thumbnail: + href: '$.assets.thumbnail.alternate.s3.href' + title: "thumbnail" + roles: + - 'thumbnail' + type: '$.assets.thumbnail.type' products: LANDSAT_C2L1: _collection: landsat-c2l1 @@ -1421,6 +1579,16 @@ - '{{"query":{{"grid:code":{{"eq":"{grid:code}"}}}}}}' - '$.properties.grid:code' assets: '{$.assets#dict_filter($[?(href=~"^s3.*")])}' + assets_mapping: + download_link: + title: "downloadlink" + href: '$.links[?(@.rel="self")].href' + roles: + - 'archive' + - 'data' + type: 'aplication/json' + # order:status set to succeeded for consistency between providers + order:status: 'succeeded' products: S1_SAR_GRD: _collection: sentinel-1-grd @@ -1516,13 +1684,51 @@ mgrs:utm_zone: '$.properties."sentinel:utm_zone"' mgrs:latitude_band: '$.properties."sentinel:latitude_band"' mgrs:grid_square: '$.properties."sentinel:grid_square"' - eodag:download_link: 's3://gcp-public-data-sentinel-2/tiles/{mgrs:utm_zone}/{mgrs:latitude_band}/{mgrs:grid_square}/{title}.SAFE' + assets_mapping: + download_link: + href: 's3://gcp-public-data-sentinel-2/tiles/{mgrs:utm_zone}/{mgrs:latitude_band}/{mgrs:grid_square}/{title}.SAFE' + roles: + - 'archive' + - 'data' + type: 'application/octet-stream' + order:status: 'succeeded' + quicklook: + href: '$.assets.thumbnail.href' + title: "thumbnail" + roles: + - 'overwiev' + type: '$.assets.thumbnail.type' + thumbnail: + href: '$.assets.thumbnail.href' + title: "thumbnail" + roles: + - 'thumbnail' + type: '$.assets.thumbnail.type' L8_OLI_TIRS_C1L1: _collection: landsat-8-l1-c1 metadata_mapping: wrsPath: '$.properties."landsat:wrs_path"' wrsRow: '$.properties."landsat:wrs_row"' - eodag:download_link: 's3://gcp-public-data-landsat/LC08/01/{wrsPath:03d}/{wrsRow:03d}/{title}' + assets_mapping: + download_link: + href: 's3://gcp-public-data-landsat/LC08/01/{wrsPath:03d}/{wrsRow:03d}/{title}' + roles: + - 'archive' + - 'data' + type: 'application/octet-stream' + order:status: 'succeeded' + quicklook: + href: '$.assets.thumbnail.href' + title: "thumbnail" + roles: + - 'overwiev' + type: '$.assets.thumbnail.type' + thumbnail: + href: '$.assets.thumbnail.href' + title: "thumbnail" + roles: + - 'thumbnail' + type: '$.assets.thumbnail.type' GENERIC_COLLECTION: _collection: '{collection}' download: !plugin @@ -1562,10 +1768,16 @@ - 'area={geometry#to_nwse_bounds_str(/)}' - '$.geometry' eodag:default_geometry: 'POLYGON((180 -90, 180 90, -180 90, -180 -90, 180 -90))' - # order:status set to succeeded for consistency between providers - order:status: '{$.null#replace_str("Not Available","succeeded")}' qs: $.qs - eodag:download_link: 'https://apps.ecmwf.int/datasets/data/{dataset}?{qs#to_geojson}' + assets_mapping: + download_link: + href: 'https://apps.ecmwf.int/datasets/data/{dataset}?{qs#to_geojson}' + roles: + - 'archive' + - 'data' + type: 'application/octet-stream' + # order:status set to succeeded for consistency between providers + order:status: 'succeeded' products: # See Archive Catalog in https://apps.ecmwf.int/archive-catalogue/ # See available Public Datasets in https://apps.ecmwf.int/datasets/ @@ -1649,7 +1861,15 @@ - '{{"area": {geometry#to_nwse_bounds}}}' - $.geometry qs: $.qs - eodag:order_link: 'https://ads.atmosphere.copernicus.eu/api/retrieve/v1/processes/{dataset}/execution?{{"inputs": {qs#to_geojson}}}' + assets_mapping: + download_link: + title: "downloadlink" + href: "" + roles: + - archive + - data + type: "application/netcdf" + order_link: 'https://ads.atmosphere.copernicus.eu/api/retrieve/v1/processes/{dataset}/execution?{"inputs": {qs#to_geojson}}' products: # See available Public Datasets in https://ads.atmosphere.copernicus.eu/cdsapp#!/search?type=dataset CAMS_GAC_FORECAST: @@ -1818,7 +2038,15 @@ - '{{"area": {geometry#to_nwse_bounds}}}' - $.geometry qs: $.qs - eodag:order_link: 'https://cds.climate.copernicus.eu/api/retrieve/v1/processes/{dataset}/execution?{{"inputs": {qs#to_geojson}}}' + assets_mapping: + download_link: + title: "downloadlink" + href: "" + roles: + - archive + - data + type: "application/netcdf" + order_link: 'https://ads.atmosphere.copernicus.eu/api/retrieve/v1/processes/{dataset}/execution?{"inputs": {qs#to_geojson}}' products: # See available Public Datasets in https://cds.climate.copernicus.eu/cdsapp#!/search?type=dataset AG_ERA5: @@ -2038,15 +2266,34 @@ geometry: - 'geometry={geometry#to_rounded_wkt}' - '$.geometry' - # The url of the quicklook - eodag:quicklook: '$.properties.quicklook' - # The url to download the product "as is" (literal or as a template to be completed either after the search result - # is obtained from the provider or during the eodag download phase) - eodag:download_link: '$.properties.services.download.url' - # order:status set to succeeded for consistency between providers - order:status: '{$.null#replace_str("Not Available","succeeded")}' - # Additional metadata provided by the providers but that don't appear in the reference spec - eodag:thumbnail: '$.properties.thumbnail' + # Unset field from product + sara:quicklook: '$.null' + sara:thumbnail: '$.null' + assets_mapping: + download_link: + # The url to download the product "as is" (literal or as a template to be completed either after the search result + # is obtained from the provider or during the eodag download phase) + href: '$.properties.services.download.url' + roles: + - 'archive' + - 'data' + type: '{$.properties.services.download.mimeType#replace_str("application/unknown","application/octet-stream")}' + file:size: '$.properties.services.download.size' + file:checksum: '{$.properties.services.download.checksum#replace_str("md5=","")#to_lower' + # order:status set to succeeded for consistency between providers + order:status: 'succeeded' + quicklook: + href: '$.properties.quicklook' + title: "quicklook" + roles: + - 'overwiev' + type: "image/png" + thumbnail: + href: '$.properties.thumbnail' + title: "thumbnail" + roles: + - 'thumbnail' + type: "image/png" products: # Sentinel 1 S1_SAR_OCN: @@ -2234,9 +2481,7 @@ - '$.geometry' eodag:default_geometry: 'POLYGON((180 -90, 180 90, -180 90, -180 -90, 180 -90))' collection: '$.queries[0].domain' - order:status: '{$.requiresJobQueue#get_group_name((?PFalse)|(?PTrue))}' qs: $.qs - eodag:download_link: https://my.meteoblue.com/dataset/query?{qs#to_geojson} # Meteoblue specific parameters datapoints: '$.datapoints' requiresJobQueue: '$.requiresJobQueue' @@ -2253,7 +2498,15 @@ timeIntervalsAlignment: - '{{"timeIntervalsAlignment": {timeIntervalsAlignment#to_geojson} }}' - '$.timeIntervalsAlignment' - eodag:order_link: https://my.meteoblue.com/dataset/query?{qs#replace_str(r"^(.*)(\")(queries\")(.)",r"\1\2runOnJobQueue\2\4 true, \2\3\4")} + assets_mapping: + download_link: + href: https://my.meteoblue.com/dataset/query?{qs#to_geojson} + roles: + - 'archive' + - 'data' + type: 'application/octet-stream' + order:status: '{$.requiresJobQueue#get_group_name((?PFalse)|(?PTrue))}' + order_link: https://my.meteoblue.com/dataset/query?{qs#replace_str(r"^(.*)(\")(queries\")(.)",r"\1\2runOnJobQueue\2\4 true, \2\3\4")} products: NEMSGLOBAL_TCDC: product:type: NEMSGLOBAL @@ -2540,14 +2793,32 @@ - '{$.Footprint#from_ewkt}' # The url to download the product "as is" (literal or as a template to be completed either after the search result # is obtained from the provider or during the eodag download phase) - eodag:download_link: 'https://catalogue.dataspace.copernicus.eu/odata/v1/Products({uid})/$value' - # order:status: must be one of succeeded, ordered, orderable - order:status: '{$.Online#get_group_name((?PTrue)|(?PFalse))}' collection: - null - $.null - eodag:quicklook: '$.Assets[?(@.Type="QUICKLOOK")].DownloadLink' - eodag:thumbnail: '$.Assets[?(@.Type="QUICKLOOK")].DownloadLink' + assets_mapping: + download_link: + href: 'https://catalogue.dataspace.copernicus.eu/odata/v1/Products({uid})/$value' + roles: + - 'archive' + - 'data' + type: '$.ContentType' + file:checksum: '$.Checksum[?Algorithm="MD5"].Value' + file:size: '$.ContentLength' + # order:status: must be one of succeeded, ordered, orderable + order:status: '{$.Online#get_group_name((?PTrue)|(?PFalse))}' + quicklook: + href: '$.Assets[?(@.Type="QUICKLOOK")].DownloadLink' + title: "quicklook" + roles: + - 'overwiev' + type: "image/jpeg" + thumbnail: + href: '$.Assets[?(@.Type="QUICKLOOK")].DownloadLink' + title: "thumbnail" + roles: + - 'thumbnail' + type: "image/jpeg" download: !plugin type: HTTPDownload extract: true @@ -2968,6 +3239,20 @@ grid:code: - '{{"query":{{"s2:mgrs_tile":{{"eq":"{grid:code#replace_str("MGRS-","")}"}}}}}}' - '{$.properties."s2:mgrs_tile"#replace_str(r"^T?(.*)$",r"MGRS-\1")}' + assets: '$.assets' + assets_mapping: + quicklook: + href: '$.assets[?(@.rel == "preview")].href' + title: "quicklook" + roles: + - 'overwiev' + type: '$.assets[?(@.rel == "preview")].type' + thumbnail: + href: '$.assets[?(@.rel == "preview")].href' + title: "thumbnail" + roles: + - 'thumbnail' + type: '$.assets[?(@.rel == "preview")].type' products: S1_SAR_GRD: _collection: sentinel-1-grd @@ -3050,6 +3335,7 @@ spatial:scene_id: - '{{"query":{{"spatial:scene_id":{{"eq":{spatial:scene_id}}}}}}}' - '$.properties.spatial:scene_id' + assets: '$.assets' products: GENERIC_COLLECTION: _collection: '{collection}' @@ -3072,44 +3358,6 @@ - host description: WEkEO - Sentinel and some various Copernicus data url: https://www.wekeo.eu/ - # anchors to avoid duplications - anchor_s1_sar: &s1_sar_params - processing:level: - - '{{"processingLevel": "{processing:level}"}}' - - '$.id.`sub(/^[^_]([^_]+)_([^_]+)_([^_]+)_([0-4]+).*/, LEVEL\\4)`' - sar:instrument_mode: - - '{{"sensorMode": "{sar:instrument_mode}"}}' - - '$.id.`sub(/^[^_]([^_]+)_([^_]+)_.*/, \\2)`' - swath: - - '{{"swath": "{swath}"}}' - - '$.null' - polarisation: - - '{{"polarisation": "{polarisation}"}}' - - '$.null' - sat:relative_orbit: - - '{{"relativeOrbitNumber": "{sat:relative_orbit}"}}' - - '$.null' - missionTakeId: - - '{{"missionTakeId": "{missionTakeId}"}}' - - '$.null' - anchor_orbit_cycle: &orbit_cycle - sat:relative_orbit: - - '{{"relativeOrbitNumber": "{sat:relative_orbit}"}}' - - '$.null' - sat:absolute_orbit: - - '{{"orbit": "{sat:absolute_orbit}"}}' - - '$.null' - sat:orbit_cycle: - - '{{"cycle": "{sat:orbit_cycle}"}}' - - '$.null' - anchor_id_from_date: &id_from_date - id: - - | - {{ - "startdate": "{id#replace_str(r'^.*_([0-9][0-9][0-9][0-9])([0-9][0-9])([0-9][0-9])([0-9][0-9])([0-9][0-9])_.*$',r'\1-\2-\3T\4%3A\5%3A00Z')}", - "enddate": "{id#replace_str(r'^.*_([0-9][0-9][0-9][0-9])([0-9][0-9])([0-9][0-9])([0-9][0-9])([0-9][0-9])_.*$',r'\1-\2-\3T\4%3A\5%3A00Z')}" - }} - - '$.id' search: !plugin type: WekeoSearch api_endpoint: https://gateway.prod.wekeo2.eu/hda-broker/api/v1/dataaccess/search @@ -3146,11 +3394,7 @@ end_datetime: - '{{"enddate": "{end_datetime}"}}' - '$.properties.enddate' - eodag:download_link: '$.properties.location' - eodag:quicklook: '$.properties.thumbnail' - eodag:thumbnail: '$.properties.thumbnail' title: '$.id' - order:status: 'orderable' processing:level: - '{{"processingLevel": "{processing:level}"}}' - '$.null' @@ -3193,13 +3437,59 @@ satellite: - '{{"satellite": {satellite}}}' - '$.null' + assets_mapping: + download_link: + href: '$.properties.location' + roles: + - 'archive' + - 'data' + type: '' + file:size: '$.properties.size' + order:status: 'orderable' + quicklook: + href: '$.properties.thumbnail' + title: "quicklook" + roles: + - 'overwiev' + type: 'image/jpeg' + thumbnail: + href: '$.properties.thumbnail' + title: "thumbnail" + roles: + - 'thumbnail' + type: 'image/jpeg' products: S1_SAR_GRD: _collection: EO:ESA:DAT:SENTINEL-1 product:type: GRD metadata_mapping: - <<: *s1_sar_params - eodag:order_link: 'https://gateway.prod.wekeo2.eu/hda-broker/api/v1/dataaccess/download?{{"location": "{eodag:download_link}","product_id":"{id}", "dataset_id": "EO:ESA:DAT:SENTINEL-1"}}' + processing:level: + - '{{"processingLevel": "{processing:level}"}}' + - '$.id.`sub(/^[^_]([^_]+)_([^_]+)_([^_]+)_([0-4]+).*/, LEVEL\\4)`' + sar:instrument_mode: + - '{{"sensorMode": "{sar:instrument_mode}"}}' + - '$.id.`sub(/^[^_]([^_]+)_([^_]+)_.*/, \\2)`' + swath: + - '{{"swath": "{swath}"}}' + - '$.null' + polarisation: + - '{{"polarisation": "{polarisation}"}}' + - '$.null' + sat:relative_orbit: + - '{{"relativeOrbitNumber": "{sat:relative_orbit}"}}' + - '$.null' + missionTakeId: + - '{{"missionTakeId": "{missionTakeId}"}}' + - '$.null' + assets_mapping: + download_link: + href: '' + roles: + - 'archive' + - 'data' + type: 'application/octet-stream' + order:status: 'orderable' + order_link: 'https://gateway.prod.wekeo2.eu/hda-broker/api/v1/dataaccess/download?{"location": "{$.properties.location}","product_id":"{id}", "dataset_id": "{_collection}"}' S1_SAR_RAW: _collection: EO:ESA:DAT:SENTINEL-1 product:type: RAW @@ -3223,7 +3513,15 @@ eo:cloud_cover: - '{{"cloudCover": "{eo:cloud_cover}"}}' - '$.null' - eodag:order_link: 'https://gateway.prod.wekeo2.eu/hda-broker/api/v1/dataaccess/download?{{"location": "{eodag:download_link}","product_id":"{id}", "dataset_id": "EO:ESA:DAT:SENTINEL-2"}}' + assets_mapping: + download_link: + href: '' + roles: + - 'archive' + - 'data' + type: 'application/octet-stream' + order:status: 'orderable' + order_link: 'https://gateway.prod.wekeo2.eu/hda-broker/api/v1/dataaccess/download?{"location": "{$.properties.location}","product_id":"{id}", "dataset_id": "{__collection}"}' S2_MSI_L2A: _collection: EO:ESA:DAT:SENTINEL-2 product:type: S2MSI2A @@ -3237,7 +3535,15 @@ processing:level: - '{{"processingLevel": "{processing:level}"}}' - '2' - eodag:order_link: 'https://gateway.prod.wekeo2.eu/hda-broker/api/v1/dataaccess/download?{{"location": "{eodag:download_link}","product_id":"{id}", "dataset_id": "EO:ESA:DAT:SENTINEL-3"}}' + assets_mapping: + download_link: + href: '' + roles: + - 'archive' + - 'data' + type: 'application/octet-stream' + order:status: 'orderable' + order_link: 'https://gateway.prod.wekeo2.eu/hda-broker/api/v1/dataaccess/download?{"location": "{$.properties.location}","product_id":"{id}", "dataset_id": "EO:ESA:DAT:SENTINEL-3"}' S3_LAN_SI: _collection: EO:ESA:DAT:SENTINEL-3 product:type: SR_2_LAN_SI @@ -3273,38 +3579,42 @@ - '{{"sat": "{constellation}"}}' - '$.id.`sub(/^[^_]([^_]+)_.*/, Sentinel-\\1)`' platform: '$.id.`sub(/^[^_]([^_]+)_.*/, S\\1)`' - <<: *orbit_cycle - eodag:order_link: 'https://gateway.prod.wekeo2.eu/hda-broker/api/v1/dataaccess/download?{{"location": "{eodag:download_link}","product_id":"{id}", "dataset_id": "EO:EUM:DAT:SENTINEL-3:OL_1_EFR___"}}' + sat:relative_orbit: + - '{{"relativeOrbitNumber": "{sat:relative_orbit}"}}' + - '$.null' + sat:absolute_orbit: + - '{{"orbit": "{sat:absolute_orbit}"}}' + - '$.null' + sat:orbit_cycle: + - '{{"cycle": "{sat:orbit_cycle}"}}' + - '$.null' + assets_mapping: + download_link: + href: '' + roles: + - 'archive' + - 'data' + type: 'application/octet-stream' + order:status: 'orderable' + order_link: 'https://gateway.prod.wekeo2.eu/hda-broker/api/v1/dataaccess/download?{"location": "{$.properties.location}","product_id":"{id}", "dataset_id": "{_collection}"}' S3_ERR: _collection: EO:EUM:DAT:SENTINEL-3:OL_1_ERR___ metadata_mapping_from_product: S3_EFR - metadata_mapping: - eodag:order_link: 'https://gateway.prod.wekeo2.eu/hda-broker/api/v1/dataaccess/download?{{"location": "{eodag:download_link}","product_id":"{id}", "dataset_id": "EO:EUM:DAT:SENTINEL-3:OL_1_ERR___"}}' S3_OLCI_L2WFR: _collection: EO:EUM:DAT:SENTINEL-3:OL_2_WFR___ metadata_mapping_from_product: S3_EFR - metadata_mapping: - eodag:order_link: 'https://gateway.prod.wekeo2.eu/hda-broker/api/v1/dataaccess/download?{{"location": "{eodag:download_link}","product_id":"{id}", "dataset_id": "EO:EUM:DAT:SENTINEL-3:OL_2_WFR___"}}' S3_OLCI_L2WRR: _collection: EO:EUM:DAT:SENTINEL-3:OL_2_WRR___ metadata_mapping_from_product: S3_EFR - metadata_mapping: - eodag:order_link: 'https://gateway.prod.wekeo2.eu/hda-broker/api/v1/dataaccess/download?{{"location": "{eodag:download_link}","product_id":"{id}", "dataset_id": "EO:EUM:DAT:SENTINEL-3:OL_2_WRR___"}}' S3_SRA: _collection: EO:EUM:DAT:SENTINEL-3:SR_1_SRA___ metadata_mapping_from_product: S3_EFR - metadata_mapping: - eodag:order_link: 'https://gateway.prod.wekeo2.eu/hda-broker/api/v1/dataaccess/download?{{"location": "{eodag:download_link}","product_id":"{id}", "dataset_id": "EO:EUM:DAT:SENTINEL-3:SR_1_SRA___"}}' S3_SRA_A: _collection: EO:EUM:DAT:SENTINEL-3:SR_1_SRA_A_ metadata_mapping_from_product: S3_EFR - metadata_mapping: - eodag:order_link: 'https://gateway.prod.wekeo2.eu/hda-broker/api/v1/dataaccess/download?{{"location": "{eodag:download_link}","product_id":"{id}", "dataset_id": "EO:EUM:DAT:SENTINEL-3:SR_1_SRA_A_"}}' S3_SRA_BS: _collection: EO:EUM:DAT:SENTINEL-3:SR_1_SRA_BS metadata_mapping_from_product: S3_EFR - metadata_mapping: - eodag:order_link: 'https://gateway.prod.wekeo2.eu/hda-broker/api/v1/dataaccess/download?{{"location": "{eodag:download_link}","product_id":"{id}", "dataset_id": "EO:EUM:DAT:SENTINEL-3:SR_1_SRA_BS"}}' S3_SLSTR_L1RBT: _collection: EO:EUM:DAT:SENTINEL-3:SL_1_RBT___ product:type: SL_1_RBT___ @@ -3316,15 +3626,39 @@ - '{{"sat": "{constellation}"}}' - '$.id.`sub(/^[^_]([^_]+)_.*/, Sentinel-\\1)`' platform: '$.id.`sub(/^[^_]([^_]+)_.*/, S\\1)`' - <<: *orbit_cycle - eodag:order_link: 'https://gateway.prod.wekeo2.eu/hda-broker/api/v1/dataaccess/download?{{"location": "{eodag:download_link}","product_id":"{id}", "dataset_id": "EO:EUM:DAT:SENTINEL-3:SL_1_RBT___"}}' + sat:relative_orbit: + - '{{"relativeOrbitNumber": "{sat:relative_orbit}"}}' + - '$.null' + sat:absolute_orbit: + - '{{"orbit": "{sat:absolute_orbit}"}}' + - '$.null' + sat:orbit_cycle: + - '{{"cycle": "{sat:orbit_cycle}"}}' + - '$.null' + assets_mapping: + download_link: + href: '' + roles: + - 'archive' + - 'data' + type: 'application/octet-stream' + order:status: 'orderable' + order_link: 'https://gateway.prod.wekeo2.eu/hda-broker/api/v1/dataaccess/download?{"location": "{$.properties.location}","product_id":"{id}", "dataset_id": "{_collection}"}}' S3_WAT: _collection: EO:EUM:DAT:SENTINEL-3:SR_2_WAT___ metadata_mapping: id: - '{{"type": "SR_2_WAT___", "timeliness": {id#split_id_into_s3_params}["timeliness"], "sat": {id#split_id_into_s3_params}["sat"], "startdate": {id#split_id_into_s3_params}["startDate"], "enddate": {id#split_id_into_s3_params}["endDate"]}}' - '{$.id#remove_extension}' - eodag:order_link: 'https://gateway.prod.wekeo2.eu/hda-broker/api/v1/dataaccess/download?{{"location": "{eodag:download_link}","product_id":"{id}", "dataset_id": "EO:EUM:DAT:SENTINEL-3:SR_2_WAT___"}}' + assets_mapping: + download_link: + href: '' + roles: + - 'archive' + - 'data' + type: 'application/octet-stream' + order:status: 'orderable' + order_link: 'https://gateway.prod.wekeo2.eu/hda-broker/api/v1/dataaccess/download?{"location": "{$.properties.location}","product_id":"{id}", "dataset_id": "{_collection}"}}' S5P_L1B_IR_ALL: _collection: EO:ESA:DAT:SENTINEL-5P processing:level: L1B @@ -3335,12 +3669,21 @@ processingMode: - '{{"processingMode": "{processingMode}"}}' - '$.null' - eodag:order_link: 'https://gateway.prod.wekeo2.eu/hda-broker/api/v1/dataaccess/download?{{"location": "{eodag:download_link}","product_id":"{id}", "dataset_id": "EO:ESA:DAT:SENTINEL-5P"}}' + assets_mapping: + download_link: + href: '' + roles: + - 'archive' + - 'data' + type: 'application/octet-stream' + order:status: 'orderable' + order_link: 'https://gateway.prod.wekeo2.eu/hda-broker/api/v1/dataaccess/download?{"location": "{$.properties.location}","product_id":"{id}", "dataset_id": "{_collection}"}}' S5P_L2_IR_ALL: _collection: EO:ESA:DAT:SENTINEL-5P processing:level: L2 metadata_mapping_from_product: S5P_L1B_IR_ALL EEA_HRL_TCF: + _collection: EO:EEA:DAT:HRL:TCF product:type: EO:EEA:DAT:HRL:TCF metadata_mapping: start_datetime: @@ -3350,12 +3693,27 @@ resolution: - '{{"resolution": "{resolution}"}}' - '{$.id#replace_str(r"^.*_R([0-9]+m)_.*$",r"\1")}' - eodag:order_link: 'https://gateway.prod.wekeo2.eu/hda-broker/api/v1/dataaccess/download?{{"location": "{eodag:download_link}","product_id":"{id}", "dataset_id": "EO:EEA:DAT:HRL:TCF"}}' + assets_mapping: + download_link: + href: '' + roles: + - 'archive' + - 'data' + type: 'application/octet-stream' + order:status: 'orderable' + order_link: 'https://gateway.prod.wekeo2.eu/hda-broker/api/v1/dataaccess/download?{"location": "{$.properties.location}","product_id":"{id}", "dataset_id": "{_collection}"}}' COP_DEM_GLO30_DGED: _collection: EO:ESA:DAT:COP-DEM product:type: DGE_30 - metadata_mapping: - eodag:order_link: 'https://gateway.prod.wekeo2.eu/hda-broker/api/v1/dataaccess/download?{{"location": "{eodag:download_link}","product_id":"{id}", "dataset_id": "EO:ESA:DAT:COP-DEM"}}' + assets_mapping: + download_link: + href: '' + roles: + - 'archive' + - 'data' + type: 'application/octet-stream' + order:status: 'orderable' + order_link: 'https://gateway.prod.wekeo2.eu/hda-broker/api/v1/dataaccess/download?{"location": "{$.properties.location}","product_id":"{id}", "dataset_id": "{_collection}"}}' COP_DEM_GLO30_DTED: _collection: EO:ESA:DAT:COP-DEM product:type: DTE_30 @@ -3371,17 +3729,28 @@ CLMS_GLO_NDVI_333M: _collection: EO:JRC:DAT:CLMS product:type: vegetation_indices - _product_identifier: ndvi_global_300m_10daily_v1 metadata_mapping: _product_identifier: - - '{{"productIdentifier": "{_product_identifier}"}}' + - '{{"productIdentifier": "ndvi_global_300m_10daily_v1"}' - '$.null' id: '$.id' - eodag:order_link: 'https://gateway.prod.wekeo2.eu/hda-broker/api/v1/dataaccess/download?{{"location": "{eodag:download_link}","product_id":"{id}", "dataset_id": "EO:JRC:DAT:CLMS"}}' + assets_mapping: + download_link: + href: '' + roles: + - 'archive' + - 'data' + type: 'application/octet-stream' + order:status: 'orderable' + order_link: 'https://gateway.prod.wekeo2.eu/hda-broker/api/v1/dataaccess/download?{"location": "{$.properties.location}","product_id":"{id}", "dataset_id": "{_collection}"}}' CLMS_GLO_NDVI_1KM_LTS: _collection: EO:JRC:DAT:CLMS product:type: vegetation_indices - _product_identifier: ndvi-lts_global_1km_10daily_v2 + metadata_mapping: + _product_identifier: + - '{{"productIdentifier": "ndvi-lts_global_1km_10daily_v2"}' + - '$.null' + id: '$.id' metadata_mapping_from_product: CLMS_GLO_NDVI_333M CLMS_CORINE: _collection: EO:EEA:DAT:CORINE @@ -3392,37 +3761,64 @@ id: - '{{"format": "{id#get_group_name((?PgeoPackage)|(?Pfgdb)|(?Praster100m))}"}}' - '$.id' - start_datetime: '$.properties.startdate' - end_datetime: '$.properties.enddate' + start_datetime: + - $.null + - '$.properties.startdate' + end_datetime: + - $.null + - '$.properties.enddate' format: - '{{"format": "{format}"}}' - '{$.id#get_group_name((?PgeoPackage)|(?Pfgdb)|(?Praster100m))}' eodag:default_geometry: 'POLYGON((180 -90, 180 90, -180 90, -180 -90, 180 -90))' - eodag:order_link: 'https://gateway.prod.wekeo2.eu/hda-broker/api/v1/dataaccess/download?{{"location": "{eodag:download_link}","product_id":"{id}", "dataset_id": "EO:EEA:DAT:CORINE"}}' + assets_mapping: + download_link: + href: '' + roles: + - 'archive' + - 'data' + type: 'application/octet-stream' + order:status: 'orderable' + order_link: 'https://gateway.prod.wekeo2.eu/hda-broker/api/v1/dataaccess/download?{"location": "{$.properties.location}","product_id":"{id}", "dataset_id": "{_collection}"}}' CLMS_GLO_FCOVER_333M: _collection: EO:JRC:DAT:CLMS product:type: vegetation_properties - _product_identifier: fcover_global_300m_10daily_v1 + metadata_mapping: + _product_identifier: + - '{{"productIdentifier": "fcover_global_300m_10daily_v1"}' + - '$.null' metadata_mapping_from_product: CLMS_GLO_NDVI_333M CLMS_GLO_DMP_333M: _collection: EO:JRC:DAT:CLMS product:type: dry-gross_dry_matter_productivity - _product_identifier: dry-gross_dry_matter_productivity/dmp_global_300m_10daily_v1 + metadata_mapping: + _product_identifier: + - '{{"productIdentifier": "dry-gross_dry_matter_productivity/dmp_global_300m_10daily_v1"}' + - '$.null' metadata_mapping_from_product: CLMS_GLO_NDVI_333M CLMS_GLO_GDMP_333M: _collection: EO:JRC:DAT:CLMS product:type: dry-gross_dry_matter_productivity - _product_identifier: gdmp_global_300m_10daily_v1 + metadata_mapping: + _product_identifier: + - '{{"productIdentifier": "gdmp_global_300m_10daily_v1"}' + - '$.null' metadata_mapping_from_product: CLMS_GLO_NDVI_333M CLMS_GLO_FAPAR_333M: _collection: EO:JRC:DAT:CLMS product:type: vegetation_properties - _product_identifier: fapar_global_300m_10daily_v1 + metadata_mapping: + _product_identifier: + - '{{"productIdentifier": "fapar_global_300m_10daily_v1"}' + - '$.null' metadata_mapping_from_product: CLMS_GLO_NDVI_333M CLMS_GLO_LAI_333M: _collection: EO:JRC:DAT:CLMS product:type: vegetation_properties - _product_identifier: lai_global_300m_10daily_v1 + metadata_mapping: + _product_identifier: + - '{{"productIdentifier": "lai_global_300m_10daily_v1"}' + - '$.null' metadata_mapping_from_product: CLMS_GLO_NDVI_333M CLMS_HRVPP_ST: _collection: EO:EEA:DAT:CLMS_HRVPP_ST @@ -3430,28 +3826,60 @@ id: - '{{"uid": "{id}"}}' - '$.id' - eodag:order_link: 'https://gateway.prod.wekeo2.eu/hda-broker/api/v1/dataaccess/download?{{"location": "{eodag:download_link}","product_id":"{id}", "dataset_id": "EO:EEA:DAT:CLMS_HRVPP_ST"}}' + assets_mapping: + download_link: + href: '' + roles: + - 'archive' + - 'data' + type: 'application/octet-stream' + order:status: 'orderable' + order_link: 'https://gateway.prod.wekeo2.eu/hda-broker/api/v1/dataaccess/download?{"location": "{$.properties.location}","product_id":"{id}", "dataset_id": "{_collection}"}}' CLMS_HRVPP_ST_LAEA: _collection: EO:EEA:DAT:CLMS_HRVPP_ST-LAEA metadata_mapping: id: - '{{"uid": "{id}"}}' - '$.id' - eodag:order_link: 'https://gateway.prod.wekeo2.eu/hda-broker/api/v1/dataaccess/download?{{"location": "{eodag:download_link}","product_id":"{id}", "dataset_id": "EO:EEA:DAT:CLMS_HRVPP_ST-LAEA"}}' + assets_mapping: + download_link: + href: '' + roles: + - 'archive' + - 'data' + type: 'application/octet-stream' + order:status: 'orderable' + order_link: 'https://gateway.prod.wekeo2.eu/hda-broker/api/v1/dataaccess/download?{"location": "{$.properties.location}","product_id":"{id}", "dataset_id": "{_collection}"}}' CLMS_HRVPP_VPP: _collection: EO:EEA:DAT:CLMS_HRVPP_VPP metadata_mapping: id: - '{{"uid": "{id}"}}' - '$.id' - eodag:order_link: 'https://gateway.prod.wekeo2.eu/hda-broker/api/v1/dataaccess/download?{{"location": "{eodag:download_link}","product_id":"{id}", "dataset_id": "EO:EEA:DAT:CLMS_HRVPP_VPP"}}' + assets_mapping: + download_link: + href: '' + roles: + - 'archive' + - 'data' + type: 'application/octet-stream' + order:status: 'orderable' + order_link: 'https://gateway.prod.wekeo2.eu/hda-broker/api/v1/dataaccess/download?{"location": "{$.properties.location}","product_id":"{id}", "dataset_id": "{_collection}"}}' CLMS_HRVPP_VPP_LAEA: _collection: EO:EEA:DAT:CLMS_HRVPP_VPP-LAEA metadata_mapping: id: - '{{"uid": "{id}"}}' - '$.id' - eodag:order_link: 'https://gateway.prod.wekeo2.eu/hda-broker/api/v1/dataaccess/download?{{"location": "{eodag:download_link}","product_id":"{id}", "dataset_id": "EO:EEA:DAT:CLMS_HRVPP_VPP-LAEA"}}' + assets_mapping: + download_link: + href: '' + roles: + - 'archive' + - 'data' + type: 'application/octet-stream' + order:status: 'orderable' + order_link: 'https://gateway.prod.wekeo2.eu/hda-broker/api/v1/dataaccess/download?{"location": "{$.properties.location}","product_id":"{id}", "dataset_id": "{_collection}"}}' auth: !plugin type: TokenAuth matching_url: https://[-\w\.]+.wekeo2.eu @@ -3594,11 +4022,20 @@ product:type: - dataset_id - $.dataset - eodag:download_link: $.properties.location dataset: - dataset_id - $.dataset + eodag:download_link: $.properties.location eodag:order_link: 'https://gateway.prod.wekeo2.eu/hda-broker/api/v1/dataaccess/download?{{"location": "{eodag:download_link}","product_id":"{id}", "dataset_id": "{dataset}"}}' + assets_mapping: + download_link: + href: '' + roles: + - 'archive' + - 'data' + type: '$.ContentType' + "order_link": 'https://gateway.prod.wekeo2.eu/hda-broker/api/v1/dataaccess/download?{"location": "{$.properties.location#url_decode}","product_id":"{id}", "dataset_id": "{dataset}"}' + order:status: 'orderable' products: SATELLITE_CARBON_DIOXIDE: dataset: EO:ECMWF:DAT:SATELLITE_CARBON_DIOXIDE @@ -3874,10 +4311,16 @@ variable: - '{{"variable": "{variable}"}}' - '{$.properties.location#get_variables_from_path}' - eodag:download_link: '$.properties.location' title: '$.id' - order:status: 'orderable' - eodag:order_link: 'https://gateway.prod.wekeo2.eu/hda-broker/api/v1/dataaccess/download?{{"location": "{eodag:download_link}","product_id":"{id}", "cacheable": "true", "dataset_id": "productType"}}' + assets_mapping: + download_link: + href: '$.properties.location' + roles: + - 'archive' + - 'data' + type: 'application/octet-stream' + order:status: 'orderable' + order_link: 'https://gateway.prod.wekeo2.eu/hda-broker/api/v1/dataaccess/download?{"location": "{$.properties.location}","product_id":"{id}", "cacheable": "true", "dataset_id": "productType"}' products: GENERIC_COLLECTION: _collection: '{collection}' @@ -4150,16 +4593,30 @@ geometry: - null - '{$.Footprint#from_ewkt}' - # The url to download the product "as is" (literal or as a template to be completed either after the search result - # is obtained from the provider or during the eodag download phase) - eodag:download_link: '$.S3Path.`sub(/^(.*)$/, s3:/\\1)`' - # order:status: must be one of succeeded, ordered, orderable - order:status: '{$.Online#get_group_name((?PTrue)|(?PFalse))}' collection: - null - $.null - eodag:quicklook: '$.Assets[?(@.Type="QUICKLOOK")].DownloadLink' - eodag:thumbnail: '$.Assets[?(@.Type="QUICKLOOK")].DownloadLink' + assets_mapping: + download_link: + href: '$.S3Path.`sub(/^(.*)$/, s3:/\\1)`' + roles: + - 'archive' + - 'data' + type: 'application/octet-stream' + # order:status: must be one of succeeded, ordered, orderable + order:status: '{$.Online#get_group_name((?PTrue)|(?PFalse))}' + quicklook: + href: '$.Assets[?(@.Type="QUICKLOOK")].DownloadLink' + title: "quicklook" + roles: + - 'overwiev' + type: "image/jpeg" + thumbnail: + href: '$.Assets[?(@.Type="QUICKLOOK")].DownloadLink' + title: "thumbnail" + roles: + - 'thumbnail' + type: "image/jpeg" download: !plugin type: AwsDownload s3_endpoint: 'https://eodata.cloudferro.com' @@ -4592,7 +5049,16 @@ - dataset - $.dataset qs: $.qs - eodag:order_link: 'https://polytope.lumi.apps.dte.destination-earth.eu/api/v1/requests/destination-earth?{{"verb": "retrieve", "request": {qs#to_geojson} }}' + assets_mapping: + download_link: + title: "downloadlink" + href: '' + roles: + - 'archive' + - 'data' + type: 'application/octet-stream' + order_link: 'https://polytope.lumi.apps.dte.destination-earth.eu/api/v1/requests/destination-earth?{"verb": "retrieve", "request": {qs#to_geojson} }' + order:status: 'orderable' products: DT_CLIMATE_ADAPTATION: discover_queryables: @@ -4790,7 +5256,16 @@ - dataset - $.dataset qs: $.qs - eodag:order_link: 'https://polytope.mn5.apps.dte.destination-earth.eu/api/v1/requests/destination-earth?{{"verb": "retrieve", "request": {qs#to_geojson} }}' + assets_mapping: + download_link: + title: "downloadlink" + href: '' + roles: + - 'archive' + - 'data' + type: 'application/octet-stream' + order_link: 'https://polytope.mn5.apps.dte.destination-earth.eu/api/v1/requests/destination-earth?{{"verb": "retrieve", "request": {qs#to_geojson} }}' + order:status: 'orderable' products: DT_CLIMATE_G1_HIGHRESMIP_CONT_IFS_FESOM_R1: dataset: climate-dt @@ -4876,10 +5351,31 @@ asset_key_from_href: false metadata_mapping: eodag:default_geometry: 'POLYGON((180 -90, 180 90, -180 90, -180 -90, 180 -90))' - eodag:quicklook: '{eodag:thumbnail}' - order:status: '{$.properties."order:status"#get_group_name((?Psucceeded)|(?Pshipping)|(?Porderable))}' - eodag:download_link: '$.assets.downloadLink.href' - assets: '$.null' + assets: '$.assets' + assets_mapping: + download_link: + title: "downloadlink" + href: '$.assets.downloadLink.href' + roles: + - 'archive' + - 'data' + type: '$.assets.downloadLink.type' + alternate: '$.assets.downloadLink.alternate' + order:status: '{$.properties."order:status"#get_group_name((?Psucceeded)|(?Pshipping)|(?Porderable))}' + quicklook: + href: '$.assets["ql.jpg"].href' + title: "quicklook" + roles: + - 'overwiev' + type: '$.assets["ql.jpg"].type' + alternate: '$.assets["ql.jpg"].alternate' + thumbnail: + href: '$.assets["ql.jpg"].href' + title: "thumbnail" + roles: + - 'thumbnail' + type: $.assets["ql.jpg"].type + alternate: '$.assets["ql.jpg"].alternate' discover_collections: fetch_url: 'https://hda.data.destination-earth.eu/stac/v2/collections' result_type: json @@ -5500,13 +5996,6 @@ - 'geo={geometry#to_rounded_wkt}' - '$.geometry' eodag:default_geometry: 'POLYGON((180 -90, 180 90, -180 90, -180 -90, 180 -90))' - # The url of the quicklook - eodag:quicklook: '$.properties.links.previews[?(@.title="Quicklook")].href' - # The url to download the product "as is" (literal or as a template to be completed either after the search result - # is obtained from the provider or during the eodag download phase) - eodag:download_link: '$.properties.links.data[?(@.title="Product download")].href' - # order:status set to succeeded for consistency between providers - order:status: '{$.null#replace_str("Not Available","succeeded")}' assets: '{$.properties.links.sip-entries#assets_list_to_dict}' # Additional metadata provided by the providers but that don't appear in the reference spec size: '$.properties.productInformation.size' @@ -5515,6 +6004,27 @@ acquisitionInformation: '$.null' productInformation: '$.null' extraInformation: '$.null' + assets_mapping: + download_link: + title: 'downloadlink' + href: '$.properties.links.data[?(@.title="Product download")].href' + roles: + - 'archive' + - 'data' + type: '$.properties.links.data[?(@.title="Product download")].mediaType' + order:status: 'succeeded' + quicklook: + href: '$.properties.links.previews[?(@.title="Quicklook")].href' + title: "quicklook" + roles: + - 'overwiev' + type: "image/jpeg" + thumbnail: + href: '$.properties.links.previews[?(@.title="Quicklook")].href' + title: "thumbnail" + roles: + - 'thumbnail' + type: "image/jpeg" products: # S3 SRAL S3_SRA: @@ -6138,11 +6648,26 @@ - '{$.properties."grid:code"#replace_str(r"^T?(.*)$",r"MGRS-\1")}' sci:doi: '{$.properties.sci:doi#replace_str(r"^\[\]$","Not Available")}' published: '$.properties.datetime' - eodag:download_link: '$.assets[?(@.roles[0] == "data") & (@.type != "application/xml")].href' - eodag:quicklook: '$.assets[?(@.roles[0] == "overview")].href.`sub(/^(.*)$/, \\1?scope=gdh)`' - eodag:thumbnail: '$.assets[?(@.roles[0] == "overview")].href.`sub(/^(.*)$/, \\1?scope=gdh)`' - # order:status set to succeeded for consistency between providers - order:status: '{$.null#replace_str("Not Available","succeeded")}' + assets_mapping: + download_link: + title: 'downloadlink' + href: '$.assets[?(@.roles[0] == "data") & (@.type == "application/zip")].href' + roles: + - 'archive' + - 'data' + order:status: 'succeeded' + quicklook: + href: '$.assets[?(@.roles[0] == "overview")].href' + title: "quicklook" + roles: + - 'overwiev' + type: "image/jpeg" + thumbnail: + href: '$.assets[?(@.roles[0] == "overview")].href' + title: "thumbnail" + roles: + - 'thumbnail' + type: "image/jpeg" products: S1_SAR_OCN: product:type: OCN @@ -6272,11 +6797,26 @@ - '{$.properties."grid:code"#replace_str(r"^T?(.*)$",r"MGRS-\1")}' sci:doi: '{$.properties.sci:doi#replace_str(r"^\[\]$","Not Available")}' published: '$.properties.datetime' - eodag:download_link: '$.properties.endpoint_url' - eodag:quicklook: '$.assets[?(@.roles[0] == "overview")].href.`sub(/^(.*)$/, \\1?scope=gdh)`' - eodag:thumbnail: '$.assets[?(@.roles[0] == "overview")].href.`sub(/^(.*)$/, \\1?scope=gdh)`' - # order:status set to succeeded for consistency between providers - order:status: '{$.null#replace_str("Not Available","succeeded")}' + assets_mapping: + download_link: + href: '$.properties.endpoint_url' + roles: + - 'archive' + - 'data' + type: 'application/octet-stream' + order:status: 'succeeded' + quicklook: + href: '$.assets[?(@.roles[0] == "overview")].href.`sub(/^(.*)$/, \\1?scope=gdh)`' + title: "quicklook" + roles: + - 'overwiev' + type: 'image/jpeg' + thumbnail: + href: '$.assets[?(@.roles[0] == "overview")].href.`sub(/^(.*)$/, \\1?scope=gdh)`' + title: "thumbnail" + roles: + - 'thumbnail' + type: 'image/jpeg' products: S1_SAR_OCN: product:type: OCN @@ -6437,7 +6977,15 @@ - '{{"area": {geometry#to_nwse_bounds}}}' - $.geometry qs: $.qs - eodag:order_link: 'https://ewds.climate.copernicus.eu/api/retrieve/v1/processes/{dataset}/execution?{{"inputs": {qs#to_geojson}}}' + assets_mapping: + download_link: + href: '' + title: "downloadlink" + roles: + - 'archive' + - 'data' + type: 'application/octet-stream' + order_link: 'https://ewds.climate.copernicus.eu/api/retrieve/v1/processes/{dataset}/execution?{"inputs": {qs#to_geojson}}' products: EFAS_HISTORICAL: dataset: efas-historical diff --git a/eodag/resources/stac_provider.yml b/eodag/resources/stac_provider.yml index c60f97f3b0..d2d215b06a 100644 --- a/eodag/resources/stac_provider.yml +++ b/eodag/resources/stac_provider.yml @@ -108,11 +108,5 @@ search: eo:cloud_cover: - '{{"query":{{"eo:cloud_cover":{{"lte":{eo:cloud_cover}}}}}}}' - '$.properties."eo:cloud_cover"' - # eodag metadata - eodag:download_link: '$.links[?(@.rel="self")].href' - eodag:quicklook: '$.assets.quicklook.href' - eodag:thumbnail: '$.assets.thumbnail.href' - # order:status set to succeeded for consistency between providers - order:status: '{$.null#replace_str("Not Available","succeeded")}' # Normalization code moves assets from properties to product's attr assets: '$.assets' diff --git a/eodag/utils/s3.py b/eodag/utils/s3.py index 2efcf7f091..cfe1144077 100644 --- a/eodag/utils/s3.py +++ b/eodag/utils/s3.py @@ -516,8 +516,8 @@ def update_assets_from_s3( s3_endpoint: Optional[str] = None, content_url: Optional[str] = None, ) -> None: - """Update ``EOProduct.assets`` using content listed in its ``remote_location`` or given - ``content_url``. + """Update ``EOProduct.assets`` using content listed in its ``product.assets["downloadlink"].remote_location`` + or given ``content_url``. If url points to a zipped archive, its content will also be be listed. @@ -525,11 +525,13 @@ def update_assets_from_s3( :param auth: Authentication plugin :param s3_endpoint: s3 endpoint if not hosted on AWS :param content_url: s3 URL pointing to the content that must be listed (defaults to - ``product.remote_location`` if empty) + ``product.assets['downloadlink'].remote_location`` if empty) """ + if content_url is None and "download_link" in product.assets: + content_url = product.assets["download_link"].remote_location if content_url is None: - content_url = product.remote_location + return None bucket, prefix = get_bucket_name_and_prefix(content_url) @@ -541,7 +543,6 @@ def update_assets_from_s3( logger.debug("Listing assets in %s", prefix) s3_client = auth.get_s3_client() - if prefix.endswith(".zip"): # List prefix zip content assets_urls = [ @@ -562,18 +563,13 @@ def update_assets_from_s3( key, roles = product.driver.guess_asset_key_and_roles( out_of_zip_url, product ) - - if key and key not in product.assets: + if key is not None: product.assets[key] = { "title": key, # Normalize title with key "roles": roles, "href": asset_url, + "type": guess_file_type(asset_url), } - if mime_type := guess_file_type(asset_url): - product.assets[key]["type"] = mime_type - - # sort assets - product.assets.data = dict(sorted(product.assets.data.items())) # update driver product.driver = product.get_driver()