From e8a22fe20bd5af0fd9005bc3a5f9a7a6922b4ae7 Mon Sep 17 00:00:00 2001 From: Stefaan Lippens Date: Mon, 24 Aug 2026 20:18:06 +0200 Subject: [PATCH 1/3] _JobResultDownloader path templates: add auto_increment and extension to allow building stable download filenames, regardless of backend naming/id approach related to #931, eu-cdse/openeo-cdse-infra#1259 --- CHANGELOG.md | 2 +- openeo/_version.py | 2 +- openeo/rest/job.py | 29 +++++++++++++- tests/rest/test_job.py | 89 +++++++++++++++++++++++++++++++++++++++++- 4 files changed, 117 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4422cf232..b6ac02060 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,7 +12,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Added `AGENTS.md` with guidance for AI coding agents contributing to this project, including a request to disclose AI assistance in PRs ([#923](https://github.com/Open-EO/openeo-python-client/issues/923)) - Add a `py.typed` to indicate to type checkers that the package contains type annotations. - Support document based "derived_from" links in `openeo.testing.results` ([#928](https://github.com/Open-EO/openeo-python-client/issues/928)) -- Add `JobResults.download_as_collection()` (experimental) to download job results as a self-contained STAC collection with rewritten hrefs ([#931](https://github.com/Open-EO/openeo-python-client/issues/931)) +- Add `JobResults.download_as_collection()` (experimental) to download job results as a self-contained STAC collection with rewritten hrefs, including a template system to fine-tune file names and paths ([#931](https://github.com/Open-EO/openeo-python-client/issues/931)) ### Changed diff --git a/openeo/_version.py b/openeo/_version.py index 746f84b44..b4858a595 100644 --- a/openeo/_version.py +++ b/openeo/_version.py @@ -1 +1 @@ -__version__ = "0.52.0a4" +__version__ = "0.52.0a5" diff --git a/openeo/rest/job.py b/openeo/rest/job.py index be658ecbf..c7d350ca4 100644 --- a/openeo/rest/job.py +++ b/openeo/rest/job.py @@ -1,5 +1,6 @@ from __future__ import annotations +import collections import contextlib import copy import datetime @@ -418,6 +419,11 @@ def _filename_from_url(url: str, *, full: bool = False) -> str: return "/".join(parts) +def _filename_extension_from_url(url: str, *, fallback: str = "") -> str: + filename = _filename_from_url(url) + return "".join(Path(filename).suffixes) or fallback + + _MEDIA_TYPE_EXTENSION_MAP = { "image/tiff": ".tiff", "image/tiff; application=geotiff": ".tiff", @@ -758,8 +764,10 @@ def register(self, path: Path): class _JobResultDownloader: """ - Helper class to download batch job results as a STAC collection (openEO API 1.1 style): - recursively walking through items, assets and additional linked metadata. + Helper class to download batch job results + as a local, self-contained STAC collection (openEO API 1.1 style): + recursively walking through items, assets, additional linked metadata, etc., + and rewriting the related links accordingly. .. warning:: this is an experimental API, subject to change. @@ -805,6 +813,7 @@ def __init__( self._on_download_failure = on_download_failure self._path_templates = {**self.DEFAULT_PATH_TEMPLATES, **(path_templates or {})} self._redact = get_url_query_param_stripper(redact_url_logging) + self._auto_increment_counters: Dict[str, int] = collections.defaultdict(int) def _write_json_file(self, data: dict, path: Union[str, Path]) -> Path: path = Path(path) @@ -833,6 +842,8 @@ def build_path_collection(self, *, collection_id: str) -> Path: vars = { "job_id": _sanitize_filename(self._job.job_id), "collection_id": _sanitize_filename(collection_id), + "auto_increment": self._auto_increment_id("collection"), + "extension": ".json", } return self._root_dir / self._path_templates["collection"].format(**vars) @@ -841,6 +852,8 @@ def build_path_item(self, *, item_id: str) -> Path: vars = { "job_id": _sanitize_filename(self._job.job_id), "item_id": _sanitize_filename(item_id), + "auto_increment": self._auto_increment_id("item"), + "extension": ".json", } return self._root_dir / self._path_templates["item"].format(**vars) @@ -850,6 +863,10 @@ def build_path_asset(self, *, asset_key: str, asset_href: str, item_id: Optional "job_id": _sanitize_filename(self._job.job_id), "asset_key": _sanitize_filename(asset_key), "asset_filename": _filename_from_url(asset_href, full=False), + # TODO: separate pool for item- and collection-assets? + "auto_increment": self._auto_increment_id("asset"), + # TODO: also leverage media type to determine extension? + "extension": _filename_extension_from_url(asset_href), } if item_id: vars["item_id"] = _sanitize_filename(item_id) @@ -862,9 +879,17 @@ def build_path_generic_link(self, *, rel: str, href: str) -> Path: "job_id": _sanitize_filename(self._job.job_id), "rel": _sanitize_filename(rel), "filename": _filename_from_url(href, full=False), + "auto_increment": self._auto_increment_id("generic"), + # TODO: also leverage media type to determine extension? + "extension": _filename_extension_from_url(href), } return self._root_dir / self._path_templates["generic-link"].format(**vars) + def _auto_increment_id(self, pool: str) -> int: + """Generate auto-incrementing ID within a given pool of entity types.""" + self._auto_increment_counters[pool] += 1 + return self._auto_increment_counters[pool] + def _relative_to(self, target: Path, doc: Path) -> str: """Get relative reference to target to be used from given document""" return Path(os.path.relpath(target, start=doc.parent)).as_posix() diff --git a/tests/rest/test_job.py b/tests/rest/test_job.py index 0d60b25ab..878f3069b 100644 --- a/tests/rest/test_job.py +++ b/tests/rest/test_job.py @@ -20,6 +20,7 @@ BatchJob, JobResultDownloadException, ResultAsset, + _filename_extension_from_url, _filename_from_url, _JobResultDownloader, _sanitize_filename, @@ -1359,7 +1360,7 @@ def test_filename_from_url(): ("https://example.com/foo//bar.txt", True, "foo/bar.txt"), ], ) -def test_test_filename_from_url_invalid_parts(url, full, expected): +def test_filename_from_url_invalid_parts(url, full, expected): if isinstance(expected, Exception): with pytest.raises(type(expected), match=str(expected)): _filename_from_url(url, full=full) @@ -1367,6 +1368,22 @@ def test_test_filename_from_url_invalid_parts(url, full, expected): assert _filename_from_url(url, full=full) == expected +def test_filename_extension_from_url(): + assert _filename_extension_from_url("https://example.com/foo/bar.txt") == ".txt" + assert _filename_extension_from_url("foo/bar.txt") == ".txt" + assert _filename_extension_from_url("/foo/bar.txt") == ".txt" + assert _filename_extension_from_url("https://example.com/foo/bar.tiff") == ".tiff" + assert _filename_extension_from_url("https://example.com/foo/bar.tar.gz") == ".tar.gz" + assert _filename_extension_from_url("https://example.com/foo/bar.txt?q=1&r=2#frag") == ".txt" + assert _filename_extension_from_url("https://example.com/foo/ba%CF%83.%CF%84x%CF%84") == ".τxτ" + assert _filename_extension_from_url("https://example.com/foo/bar") == "" + assert _filename_extension_from_url("https://example.com/foo/bar/") == "" + assert _filename_extension_from_url("https://example.com/") == "" + + assert _filename_extension_from_url("https://example.com/foo/bar", fallback=".data") == ".data" + assert _filename_extension_from_url("https://example.com/foo/bar.txt", fallback=".data") == ".txt" + + class TestJobResultDownloader: @pytest.fixture @@ -1688,6 +1705,76 @@ def test_custom_file_tree_structure(self, result_mocker, tmp_path, path_template self.check_expected_downloads(downloaded=downloaded, expected=expected, tmp_path=tmp_path) + def test_custom_file_tree_structure_auto_increment(self, result_mocker, tmp_path): + job = result_mocker.setup_job_results( + items={ + "item11": { + "assets": { + "a11-1": {"path": "asset11-1.tiff", "content": b"DATA:11-1"}, + "a11-2": {"path": "asset11-2.tiff", "content": b"DATA:11-2"}, + } + }, + "item22": { + "assets": { + "a22-1": {"path": "asset22-1.tiff", "content": b"DATA:22-1"}, + } + }, + }, + linked_docs=[ + {"rel": "derived_from", "path": "derived_from8.json", "json": {"hello": "eight"}}, + {"rel": "derived_from", "path": "derived_from9.json", "json": {"hello": "nine"}}, + ], + ) + path_templates = { + "collection": "collection-{auto_increment}{extension}", + "item": "item-{auto_increment:02d}{extension}", + "asset": "asset-{auto_increment:03d}{extension}", + "collection-asset": "collection-asset-{auto_increment}{extension}", + "generic-link": "generic-{auto_increment}{extension}", + } + + downloader = _JobResultDownloader( + job=job, + target=tmp_path, + path_templates=path_templates, + ) + downloaded = downloader.download_collection(download_derived_from=True) + + expected = { + "collection-1.json": dirty_equals.IsPartialDict( + { + "links": [ + {"rel": "item", "href": "item-01.json"}, + {"rel": "item", "href": "item-02.json"}, + {"rel": "derived_from", "href": "generic-1.json"}, + {"rel": "derived_from", "href": "generic-2.json"}, + ] + } + ), + "item-01.json": dirty_equals.IsPartialDict( + { + "assets": { + "a11-1": dirty_equals.IsPartialDict(href="asset-001.tiff"), + "a11-2": dirty_equals.IsPartialDict(href="asset-002.tiff"), + }, + } + ), + "item-02.json": dirty_equals.IsPartialDict( + { + "assets": { + "a22-1": dirty_equals.IsPartialDict(href="asset-003.tiff"), + } + } + ), + "asset-001.tiff": b"DATA:11-1", + "asset-002.tiff": b"DATA:11-2", + "asset-003.tiff": b"DATA:22-1", + "generic-1.json": {"hello": "eight"}, + "generic-2.json": {"hello": "nine"}, + } + + self.check_expected_downloads(downloaded=downloaded, expected=expected, tmp_path=tmp_path) + def test_download_collision_default(self, result_mocker, tmp_path, caplog): """Download collisions are logged as warning by default (on_download_failure="warn")""" job = result_mocker.setup_job_results( From b06ae7955cca81e791ff6d9596e1fe7682f18972 Mon Sep 17 00:00:00 2001 From: Stefaan Lippens Date: Tue, 25 Aug 2026 10:22:33 +0200 Subject: [PATCH 2/3] JobResult.download_as_collection: rename `json_dump` less weird naming and more future proof for when allowing it to be a `json.dump` replacement callable ref #931 --- openeo/rest/job.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/openeo/rest/job.py b/openeo/rest/job.py index c7d350ca4..fb99529ed 100644 --- a/openeo/rest/job.py +++ b/openeo/rest/job.py @@ -693,7 +693,7 @@ def download_as_collection( rewrite_references: bool = True, download_derived_from: bool = False, download_collection_assets: bool = False, - json_dumping: Optional[dict] = None, + json_dump: Optional[dict] = None, on_download_failure: Literal["warn", "raise"] = "warn", path_templates: Optional[dict] = None, redact_url_logging: bool = True, @@ -715,9 +715,9 @@ def download_as_collection( additional "derived_from" documents linked from the STAC collection. :param download_collection_assets: whether to download the STAC Collection level assets in addition to assets from linked STAC Items. - :param json_dumping: kwargs to finetune json.dump when writing STAC metadata files. + :param json_dump: kwargs to finetune json.dump when writing STAC metadata files. :param on_download_failure: how to handle download failures, one of "warn" or "raise". - :param path_templates: optional template overrides for download paths. + :param path_templates: optional dictionary of template overrides for download paths. :param redact_url_logging: whether to redact (possibly sensitive) query parameters from URLs in logging, .. versionadded:: 0.52.0 @@ -726,7 +726,7 @@ def download_as_collection( job=self._job, target=target, rewrite_references=rewrite_references, - json_dumping=json_dumping, + json_dump=json_dump, on_download_failure=on_download_failure, path_templates=path_templates, redact_url_logging=redact_url_logging, @@ -796,7 +796,7 @@ def __init__( job: BatchJob, target: Union[Path, str, None] = None, rewrite_references: bool = True, - json_dumping: Optional[dict] = None, + json_dump: Optional[dict] = None, on_download_failure: Literal["warn", "raise"] = "warn", path_templates: Optional[dict] = None, redact_url_logging: bool = True, @@ -808,7 +808,7 @@ def __init__( raise OpenEoClientException(f"Download target {self._root_dir} exists but isn't a folder.") self._rewrite_references = rewrite_references # TODO: also support passing a `json.dump`-style callable to customize json dumping - self._json_dumping = {"ensure_ascii": False, **(json_dumping or {})} + self._json_dump = {"ensure_ascii": False, **(json_dump or {})} self._download_tracker = _DownloadTracker() self._on_download_failure = on_download_failure self._path_templates = {**self.DEFAULT_PATH_TEMPLATES, **(path_templates or {})} @@ -819,7 +819,7 @@ def _write_json_file(self, data: dict, path: Union[str, Path]) -> Path: path = Path(path) ensure_parent_dir_for(path) with open(path, mode="w", encoding="utf-8") as f: - json.dump(obj=data, fp=f, **self._json_dumping) + json.dump(obj=data, fp=f, **self._json_dump) return path @contextlib.contextmanager From 127002fb6e9f4ae5aa1abc4bd7e7fa7689c002cc Mon Sep 17 00:00:00 2001 From: Stefaan Lippens Date: Tue, 25 Aug 2026 15:15:52 +0200 Subject: [PATCH 3/3] JobResult.download_as_collection: add add_original_hrefs feature #931 --- openeo/rest/_testing.py | 7 +- openeo/rest/job.py | 33 +++++--- openeo/utils/datastructure.py | 18 +++++ tests/rest/test_job.py | 128 ++++++++++++++++++++++++++++++ tests/rest/test_testing.py | 56 +++++++++++++ tests/utils/test_datastructure.py | 14 ++++ 6 files changed, 239 insertions(+), 17 deletions(-) create mode 100644 openeo/utils/datastructure.py create mode 100644 tests/utils/test_datastructure.py diff --git a/openeo/rest/_testing.py b/openeo/rest/_testing.py index 84056a904..1dec3b282 100644 --- a/openeo/rest/_testing.py +++ b/openeo/rest/_testing.py @@ -537,7 +537,7 @@ def setup_job_results( collection_assets[f"{item_id}-{asset_key}"] = asset item_href = self.setup_item(job_id=job_id, item_id=item_id, item_data=item_data, assets=assets) - collection_links.append({"rel": "item", "href": item_href}) + collection_links.append({"rel": "item", "href": item_href, **item_data.get("link_extra", {})}) for doc in linked_docs: collection_links.append(self.setup_linked_document(job_id=job_id, doc=doc)) @@ -586,8 +586,7 @@ def setup_asset(self, *, job_id: str, asset_data: dict) -> dict: self.requests_mock.head(href, headers={"Content-Length": f"{len(content)}"}) self.requests_mock.get(href, content=content) return StacDummyBuilder.asset( - href=href, - type=asset_data.get("type", "image/tiff; application=geotiff"), + href=href, type=asset_data.get("type", "image/tiff; application=geotiff"), **asset_data.get("extra", {}) ) def setup_linked_document(self, *, job_id: str, doc: dict): @@ -599,4 +598,4 @@ def setup_linked_document(self, *, job_id: str, doc: dict): text = doc.get("text", "hello world") self.requests_mock.head(href, headers={"Content-Length": f"{len(text)}"}) self.requests_mock.get(href, text=text) - return {"rel": doc.get("rel", "doc"), "href": href} + return {"rel": doc.get("rel", "doc"), "href": href, **doc.get("link_extra", {})} diff --git a/openeo/rest/job.py b/openeo/rest/job.py index fb99529ed..dae7a060f 100644 --- a/openeo/rest/job.py +++ b/openeo/rest/job.py @@ -33,6 +33,7 @@ from openeo.rest.models.general import LogsResponse from openeo.rest.models.logs import log_level_name from openeo.util import ensure_dir, ensure_parent_dir_for +from openeo.utils.datastructure import make_new_key from openeo.utils.events import EVENTS from openeo.utils.http import ( HTTP_502_BAD_GATEWAY, @@ -691,6 +692,7 @@ def download_as_collection( target: Union[Path, str, None] = None, *, rewrite_references: bool = True, + add_original_hrefs: bool = False, download_derived_from: bool = False, download_collection_assets: bool = False, json_dump: Optional[dict] = None, @@ -711,6 +713,7 @@ def download_as_collection( :param rewrite_references: whether to rewrite (item/asset/...) HREFs in the downloaded STAC documents to point to the local files instead of the original URLs. + :param add_original_hrefs: whether to add the original HREF URLs as "alternate" HREFs in the metadata :param download_derived_from: whether to download additional "derived_from" documents linked from the STAC collection. :param download_collection_assets: whether to download @@ -726,6 +729,7 @@ def download_as_collection( job=self._job, target=target, rewrite_references=rewrite_references, + add_original_hrefs=add_original_hrefs, json_dump=json_dump, on_download_failure=on_download_failure, path_templates=path_templates, @@ -796,6 +800,7 @@ def __init__( job: BatchJob, target: Union[Path, str, None] = None, rewrite_references: bool = True, + add_original_hrefs: bool = False, json_dump: Optional[dict] = None, on_download_failure: Literal["warn", "raise"] = "warn", path_templates: Optional[dict] = None, @@ -807,6 +812,7 @@ def __init__( if self._root_dir.exists() and not self._root_dir.is_dir(): raise OpenEoClientException(f"Download target {self._root_dir} exists but isn't a folder.") self._rewrite_references = rewrite_references + self._add_original_hrefs = add_original_hrefs # TODO: also support passing a `json.dump`-style callable to customize json dumping self._json_dump = {"ensure_ascii": False, **(json_dump or {})} self._download_tracker = _DownloadTracker() @@ -923,10 +929,7 @@ def download_collection( with self._download_attempt_context(name=f"item {link=}"): path = self._download_item(href=link["href"]) if self._rewrite_references: - rel_path = self._relative_to(target=path, doc=result_metadata_path) - logger.debug(f"Rewriting link {self._redact(link)=} href to local {rel_path=}") - link["href"] = rel_path - + self._rewrite_href(obj=link, path=path, relative_to=result_metadata_path) elif link["rel"] in extra_rels: with self._download_attempt_context(name=f"link {link=}"): path = self.build_path_generic_link(rel=link["rel"], href=link["href"]) @@ -935,9 +938,7 @@ def download_collection( logger.debug(f"Downloaded link {self._redact(link)=} to {path=}") self._download_tracker.register(path) if self._rewrite_references: - rel_path = self._relative_to(target=path, doc=result_metadata_path) - logger.debug(f"Rewriting link {self._redact(link)=} href to local {rel_path=}") - link["href"] = rel_path + self._rewrite_href(obj=link, path=path, relative_to=result_metadata_path) if download_collection_assets: for asset_key, asset in result_metadata.get("assets", {}).items(): @@ -946,9 +947,7 @@ def download_collection( asset_key=asset_key, asset_href=asset["href"], asset_metadata=asset, item_id=None ) if self._rewrite_references: - rel_path = self._relative_to(target=path, doc=result_metadata_path) - logger.debug(f"Rewriting STAC Collection asset {asset_key=} href to local {rel_path=}") - asset["href"] = rel_path + self._rewrite_href(obj=asset, path=path, relative_to=result_metadata_path) if self._rewrite_references: # Rewrite the root collection metadata with updated references @@ -959,6 +958,16 @@ def download_collection( return self._download_tracker.paths + def _rewrite_href(self, obj: dict, path: Path, relative_to: Path): + """Rewrite (in-place) href of given object.""" + original = obj["href"] + rel_path = self._relative_to(target=path, doc=relative_to) + logger.debug(f"Rewriting href {rel_path=} (in {self._redact(obj)})") + obj["href"] = rel_path + if self._add_original_hrefs: + alternate = obj.setdefault("alternate", {}) + alternate[make_new_key(alternate, "original")] = {"href": original} + def _download_item(self, href: str) -> Path: item: dict = self._connection.get(href, expected_status=200).json() item_id = item["id"] @@ -973,9 +982,7 @@ def _download_item(self, href: str) -> Path: asset_key=asset_key, asset_href=asset["href"], asset_metadata=asset, item_id=item_id ) if self._rewrite_references: - rel_path = self._relative_to(target=asset_path, doc=metadata_path) - logger.debug(f"Rewriting asset {asset_key=} ({item_id=}) href to local {rel_path=}") - asset["href"] = rel_path + self._rewrite_href(obj=asset, path=asset_path, relative_to=metadata_path) if self._rewrite_references: logger.info(f"Update write of STAC Item {item_id!r} metadata to {metadata_path}") diff --git a/openeo/utils/datastructure.py b/openeo/utils/datastructure.py new file mode 100644 index 000000000..ea5949d1a --- /dev/null +++ b/openeo/utils/datastructure.py @@ -0,0 +1,18 @@ +""" +Generic data structure related utilities +""" + +import itertools +from typing import Container + + +def make_new_key(container: Container[str], key: str, *, start: int = 1): + """ + Construct a new key that doesn't exist yet in the given container (dict, set, ...), + based on given base key (to use directly if possible, or by appending an auto-increment-style suffix). + """ + if key not in container: + return key + for i in itertools.count(start): + if (k := f"{key}-{i}") not in container: + return k diff --git a/tests/rest/test_job.py b/tests/rest/test_job.py index 878f3069b..1531e4aab 100644 --- a/tests/rest/test_job.py +++ b/tests/rest/test_job.py @@ -1801,3 +1801,131 @@ def test_download_collision_with_raise(self, result_mocker, tmp_path): ) with pytest.raises(JobResultDownloadException, match=r"Download collision, already downloaded.*asset\.tiff"): downloader.download_collection() + + def test_add_original_hrefs(self, result_mocker, tmp_path): + job = result_mocker.setup_job_results( + items={ + "item1": { + "full_path": "original/path/to/item1.json", + "assets": {"asset1": {"full_path": "original/path/to/asset1.tiff"}}, + }, + }, + linked_docs=[ + {"rel": "derived_from", "full_path": "original/path/to/derived_from.json", "json": {"hello": "world"}}, + ], + ) + downloader = _JobResultDownloader(job=job, target=tmp_path, add_original_hrefs=True) + downloaded = downloader.download_collection(download_derived_from=True) + expected = { + "job-results.json": dirty_equals.IsPartialDict( + { + "id": "job-123-results", + "type": "Collection", + "stac_version": "1.1.0", + "links": [ + { + "rel": "item", + "href": "item1/item1.json", + "alternate": {"original": {"href": "https://oeo.test/original/path/to/item1.json"}}, + }, + { + "rel": "derived_from", + "href": "derived_from.json", + "alternate": {"original": {"href": "https://oeo.test/original/path/to/derived_from.json"}}, + }, + ], + } + ), + "item1/item1.json": dirty_equals.IsPartialDict( + { + "id": "item1", + "type": "Feature", + "stac_version": "1.1.0", + "assets": { + "asset1": dirty_equals.IsPartialDict( + { + "href": "asset1.tiff", + "alternate": {"original": {"href": "https://oeo.test/original/path/to/asset1.tiff"}}, + } + ), + }, + } + ), + "item1/asset1.tiff": b"TIFF-DUMMY-DATA", + "derived_from.json": {"hello": "world"}, + } + self.check_expected_downloads(downloaded=downloaded, expected=expected, tmp_path=tmp_path) + + def test_add_original_hrefs_preserve_existing_originals(self, result_mocker, tmp_path): + job = result_mocker.setup_job_results( + items={ + "item1": { + "full_path": "original/path/to/item1.json", + "assets": { + "asset1": { + "full_path": "original/path/to/asset1.tiff", + "extra": {"alternate": {"original": {"href": "https://example.com/asset.tiff"}}}, + } + }, + "link_extra": {"alternate": {"original": {"href": "https://example.com/item.json"}}}, + }, + }, + linked_docs=[ + { + "rel": "derived_from", + "full_path": "original/path/to/derived_from.json", + "json": {"hello": "world"}, + "link_extra": {"alternate": {"original": {"href": "https://example.com/df.json"}}}, + }, + ], + ) + downloader = _JobResultDownloader(job=job, target=tmp_path, add_original_hrefs=True) + downloaded = downloader.download_collection(download_derived_from=True) + expected = { + "job-results.json": dirty_equals.IsPartialDict( + { + "id": "job-123-results", + "type": "Collection", + "stac_version": "1.1.0", + "links": [ + { + "rel": "item", + "href": "item1/item1.json", + "alternate": { + "original": {"href": "https://example.com/item.json"}, + "original-1": {"href": "https://oeo.test/original/path/to/item1.json"}, + }, + }, + { + "rel": "derived_from", + "href": "derived_from.json", + "alternate": { + "original": {"href": "https://example.com/df.json"}, + "original-1": {"href": "https://oeo.test/original/path/to/derived_from.json"}, + }, + }, + ], + } + ), + "item1/item1.json": dirty_equals.IsPartialDict( + { + "id": "item1", + "type": "Feature", + "stac_version": "1.1.0", + "assets": { + "asset1": dirty_equals.IsPartialDict( + { + "href": "asset1.tiff", + "alternate": { + "original": {"href": "https://example.com/asset.tiff"}, + "original-1": {"href": "https://oeo.test/original/path/to/asset1.tiff"}, + }, + } + ), + }, + } + ), + "item1/asset1.tiff": b"TIFF-DUMMY-DATA", + "derived_from.json": {"hello": "world"}, + } + self.check_expected_downloads(downloaded=downloaded, expected=expected, tmp_path=tmp_path) diff --git a/tests/rest/test_testing.py b/tests/rest/test_testing.py index c505b2e4e..220060a17 100644 --- a/tests/rest/test_testing.py +++ b/tests/rest/test_testing.py @@ -169,3 +169,59 @@ def test_asset_error(self, requests_mock, con120): ) with pytest.raises(OpenEoRestError, match=re.escape("[500] Nope!")): con120.get("https://oeo.test/j/job-456/r/a/asset-678.tif") + + def test_link_extra(self, requests_mock, con120): + result_mocker = JobResultCollectionMocker(requests_mock=requests_mock, connection=con120) + result_mocker.setup_job_results( + job_id="job-456", + items={ + "item-567": { + "assets": { + "asset-678": { + "path": "asset-678.tif", + "extra": {"alternate": {"upstream": {"href": "https://example.com/upstream.tiff"}}}, + } + }, + "link_extra": {"alternate": {"upstream": {"href": "https://example.com/upstream.json"}}}, + } + }, + ) + + job = con120.job("job-456") + assert job.get_results().get_metadata() == dirty_equals.IsPartialDict( + { + "type": "Collection", + "stac_version": "1.1.0", + "id": "job-456-results", + "links": [ + { + "rel": "item", + "href": "https://oeo.test/j/job-456/r/i/item-567.json", + "alternate": {"upstream": {"href": "https://example.com/upstream.json"}}, + } + ], + "assets": { + "item-567-asset-678": { + "href": "https://oeo.test/j/job-456/r/a/asset-678.tif", + "roles": ["data"], + "type": "image/tiff; application=geotiff", + "alternate": {"upstream": {"href": "https://example.com/upstream.tiff"}}, + } + }, + } + ) + assert con120.get("https://oeo.test/j/job-456/r/i/item-567.json").json() == dirty_equals.IsPartialDict( + { + "type": "Feature", + "stac_version": "1.1.0", + "id": "item-567", + "assets": { + "asset-678": { + "href": "https://oeo.test/j/job-456/r/a/asset-678.tif", + "roles": ["data"], + "type": "image/tiff; application=geotiff", + "alternate": {"upstream": {"href": "https://example.com/upstream.tiff"}}, + } + }, + } + ) diff --git a/tests/utils/test_datastructure.py b/tests/utils/test_datastructure.py new file mode 100644 index 000000000..3de9b51c9 --- /dev/null +++ b/tests/utils/test_datastructure.py @@ -0,0 +1,14 @@ +from openeo.utils.datastructure import make_new_key + + +def test_make_new_key_on_dict(): + d = {"href": "https://example.com", "name": "example"} + assert make_new_key(d, "description") == "description" + assert make_new_key(d, "name") == "name-1" + assert d == {"href": "https://example.com", "name": "example"} + + assert make_new_key(d, "description") == "description" + assert make_new_key(d, "name") == "name-1" + + d[make_new_key(d, "name")] = "other" + assert d == {"href": "https://example.com", "name": "example", "name-1": "other"}