From 9d0fe4ebfe3f22d3425844f790f4eec28f14b40b Mon Sep 17 00:00:00 2001 From: Wei8007 <189627153@qq.com> Date: Mon, 13 Jul 2026 19:59:58 +0800 Subject: [PATCH] Validate load_stac properties with STAC queryables --- CHANGELOG.md | 1 + openeo/metadata.py | 6 +- openeo/rest/datacube.py | 88 ++++++++++++++++++++----- tests/rest/datacube/test_datacube100.py | 22 +++++++ tests/rest/test_connection.py | 54 ++++++++++++--- 5 files changed, 145 insertions(+), 26 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 25ad8d11f..ae9580ed2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- Validate `load_stac` property filters against discoverable STAC queryables. ([#884](https://github.com/Open-EO/openeo-python-client/issues/884)) - `DataCube.resample_spatial()` now supports parameterized `resolution` and `projection` arguments. ([#897](https://github.com/Open-EO/openeo-python-client/issues/897)) - Sanitize asset download filenames (e.g. strip slashes, (semi)colon, hash, ...), instead of blindly using the asset key as filename. ([#820](https://github.com/Open-EO/openeo-python-client/issues/820)) - Support parameters in `DataCube` apply- and band-math operations ([#903](https://github.com/Open-EO/openeo-python-client/issues/903)) diff --git a/openeo/metadata.py b/openeo/metadata.py index 84f57d529..18eccac8b 100644 --- a/openeo/metadata.py +++ b/openeo/metadata.py @@ -688,7 +688,11 @@ def metadata_from_stac(url: str) -> CubeMetadata: :param url: The URL to a static STAC catalog (STAC Item, STAC Collection, or STAC Catalog) or a specific STAC API Collection :return: A :py:class:`CubeMetadata` containing the DataCube band metadata from the url. """ - stac_object = pystac.read_file(href=url) + return metadata_from_stac_object(pystac.read_file(href=url)) + + +def metadata_from_stac_object(stac_object: pystac.STACObject) -> CubeMetadata: + """Build cube metadata from an already loaded PySTAC object.""" bands = _StacMetadataParser().bands_from_stac_object(stac_object) # At least assume there are spatial dimensions diff --git a/openeo/rest/datacube.py b/openeo/rest/datacube.py index 2e0ba56a1..6fe5bc938 100644 --- a/openeo/rest/datacube.py +++ b/openeo/rest/datacube.py @@ -31,6 +31,7 @@ ) import numpy as np +import pystac import requests import shapely.geometry import shapely.geometry.base @@ -50,7 +51,7 @@ Band, CollectionMetadata, CubeMetadata, - metadata_from_stac, + metadata_from_stac_object, ) from openeo.rest import ( DEFAULT_JOB_STATUS_POLL_CONNECTION_RETRY_INTERVAL, @@ -73,7 +74,14 @@ from openeo.rest.service import Service from openeo.rest.udp import RESTUserDefinedProcess from openeo.rest.vectorcube import VectorCube -from openeo.util import dict_no_none, guess_format, load_json, normalize_crs, rfc3339 +from openeo.util import ( + dict_no_none, + guess_format, + load_json, + load_json_resource, + normalize_crs, + rfc3339, +) if typing.TYPE_CHECKING: # Imports for type checking only (circular import issue at runtime). @@ -444,16 +452,10 @@ def load_stac( if bands is not None: arguments["bands"] = bands - properties = cls._build_load_properties_argument( - properties=properties, - # TODO: possible to detect queryables here too? - ) - if properties is not None: - arguments["properties"] = properties - - graph = PGNode("load_stac", arguments=arguments) + stac_object = None try: - metadata = metadata_from_stac(url) + stac_object = pystac.read_file(href=url) + metadata = metadata_from_stac_object(stac_object) # TODO: also apply spatial/temporal filters to metadata? if isinstance(bands, list): @@ -477,6 +479,15 @@ def load_stac( except Exception as e: log.warning(f"Failed to extract cube metadata from STAC URL {url}", exc_info=True) metadata = None + + properties = cls._build_load_properties_argument( + properties=properties, + queryables=_Queryables.build_stac(stac_object) if properties is not None else None, + ) + if properties is not None: + arguments["properties"] = properties + + graph = PGNode("load_stac", arguments=arguments) return cls(graph=graph, connection=connection, metadata=metadata) @classmethod @@ -3245,6 +3256,13 @@ def __init__(self, properties: Iterable[str], additional: bool = False): self.properties = set(properties) self.additional = bool(additional) + @classmethod + def _from_json_schema(cls, data: dict, *, source: str) -> _Queryables: + properties = list(data.get("properties", {}).keys()) + additional = data.get("additionalProperties", False) + log.debug(f"Queryables from {source!r}: {properties=} {additional=}") + return cls(properties=properties, additional=additional) + @classmethod def build(cls, *, collection_id: str, connection: Optional[Connection]) -> Union[_Queryables, None]: if connection and connection.capabilities().supports_endpoint("/collections/{collection_id}/queryables"): @@ -3252,11 +3270,49 @@ def build(cls, *, collection_id: str, connection: Optional[Connection]) -> Union try: resp = connection.get(path, allow_redirects=True) resp.raise_for_status() - data = resp.json() - properties = list(data.get("properties", {}).keys()) - additional = data.get("additionalProperties", False) - log.debug(f"Queryables from {path!r}: {properties=} {additional=}") - return cls(properties=properties, additional=additional) + return cls._from_json_schema(resp.json(), source=path) except Exception as e: log.warning(f"Failed to get/parse queryables of from {path}: {e!r}") return None + + @classmethod + def build_stac(cls, stac_object: Optional[pystac.STACObject]) -> Union[_Queryables, None]: + """Discover queryables advertised by a STAC object or its STAC API root.""" + if stac_object is None: + return None + + queryables_rels = ["http://www.opengis.net/def/rel/ogc/1.0/queryables", "queryables"] + queryables_link = next( + (link for rel in queryables_rels for link in stac_object.get_links(rel=rel)), + None, + ) + source = queryables_link.get_absolute_href() if queryables_link else None + + if source is None and isinstance(stac_object, pystac.Collection): + self_href = stac_object.get_self_href() + root_href = next( + ( + href + for link in stac_object.get_links("root") + if (href := link.get_absolute_href()) and href != self_href + ), + None, + ) + try: + root = pystac.read_file(root_href) if root_href else None + conformance = root.extra_fields.get("conformsTo", []) if root else [] + if ( + self_href + and isinstance(conformance, list) + and any("filter" in item.lower() for item in conformance if isinstance(item, str)) + ): + source = urllib.parse.urljoin(self_href.rstrip("/") + "/", "queryables") + except Exception as e: + log.debug(f"Failed to inspect STAC root for queryables: {e!r}") + + if source: + try: + return cls._from_json_schema(load_json_resource(source), source=source) + except Exception as e: + log.warning(f"Failed to get/parse STAC queryables from {source}: {e!r}") + return None diff --git a/tests/rest/datacube/test_datacube100.py b/tests/rest/datacube/test_datacube100.py index 40f7ef10f..3a209429f 100644 --- a/tests/rest/datacube/test_datacube100.py +++ b/tests/rest/datacube/test_datacube100.py @@ -15,6 +15,7 @@ import dirty_equals import pyproj +import pystac import pytest import requests import shapely.geometry @@ -30,6 +31,7 @@ from openeo.rest import OpenEoClientException from openeo.rest.connection import Connection from openeo.rest.datacube import THIS, UDF, DataCube, _Queryables +from openeo.testing.stac import StacDummyBuilder from openeo.utils.version import ComparableVersion from .. import get_download_graph @@ -2363,6 +2365,26 @@ def test_broken(self, con100, requests_mock, api_capabilities): queryables = _Queryables.build(collection_id="S2", connection=con100) assert queryables is None + def test_stac_api_filter_conformance(self, monkeypatch): + collection = pystac.Collection.from_dict(StacDummyBuilder.collection()) + collection.set_self_href("https://stac.test/collections/S2") + collection.add_link(pystac.Link(rel="root", target="https://stac.test/")) + root = pystac.Catalog(id="root", description="STAC API root") + root.extra_fields["conformsTo"] = ["https://api.stacspec.org/v1.0.0/item-search#filter"] + sources = [] + + monkeypatch.setattr(pystac, "read_file", lambda href: root) + monkeypatch.setattr( + "openeo.rest.datacube.load_json_resource", + lambda source: sources.append(source) or _build_queryables_doc(platform=True, additional=False), + ) + + queryables = _Queryables.build_stac(collection) + + assert sources == ["https://stac.test/collections/S2/queryables"] + assert queryables.properties == {"eo:cloud_cover", "platform"} + assert queryables.additional is False + @pytest.mark.parametrize("api_capabilities", [{"collection_queryables": True}]) @pytest.mark.parametrize( diff --git a/tests/rest/test_connection.py b/tests/rest/test_connection.py index e84987268..1d56147b4 100644 --- a/tests/rest/test_connection.py +++ b/tests/rest/test_connection.py @@ -3208,6 +3208,41 @@ def test_property_filtering(self, con120, properties, expected): } } + def test_property_filtering_with_queryables(self, con120, build_stac_ref, tmp_path, recwarn): + (tmp_path / "queryables.json").write_text( + json.dumps( + { + "type": "object", + "properties": {"platform": {"type": "string"}}, + "additionalProperties": False, + } + ), + encoding="utf8", + ) + stac_ref = build_stac_ref( + StacDummyBuilder.collection( + links=[ + { + "rel": "http://www.opengis.net/def/rel/ogc/1.0/queryables", + "href": "queryables.json", + } + ] + ) + ) + + cube = con120.load_stac( + stac_ref, + properties={ + "platform": lambda p: p == "S2A", + "unsupported": lambda p: p == "value", + }, + ) + + assert set(cube.flat_graph()["loadstac1"]["arguments"]["properties"]) == {"platform", "unsupported"} + assert [str(w.message) for w in recwarn] == [ + "Property filtering with unsupported properties ['unsupported'] (queryables: ['platform'])." + ] + def test_load_stac_from_job_canonical(self, con120, requests_mock): requests_mock.get( API_URL + "jobs/j0bi6/results", @@ -3489,18 +3524,20 @@ def test_load_stac_band_filtering_no_band_dimension( ): stac_ref = build_stac_ref(StacDummyBuilder.collection()) - # This is a temporary mock.patch hack to make metadata_from_stac return metadata without a band dimension + # This is a temporary mock.patch hack to return metadata without a band dimension # TODO #743: Do this properly through appropriate STAC metadata - from openeo.metadata import metadata_from_stac as orig_metadata_from_stac + from openeo.metadata import ( + metadata_from_stac_object as orig_metadata_from_stac_object, + ) - def metadata_from_stac(url: str): - metadata = orig_metadata_from_stac(url=url) + def metadata_from_stac_object(stac_object): + metadata = orig_metadata_from_stac_object(stac_object) assert metadata.has_band_dimension() metadata = metadata.drop_dimension("bands") assert not metadata.has_band_dimension() return metadata - with mock.patch("openeo.rest.datacube.metadata_from_stac", new=metadata_from_stac): + with mock.patch("openeo.rest.datacube.metadata_from_stac_object", new=metadata_from_stac_object): cube = dummy_backend.connection.load_stac(stac_ref, bands=bands) assert cube.metadata.has_band_dimension() == has_band_dimension @@ -3547,14 +3584,13 @@ def test_load_stac_band_filtering_custom_band_dimension( ): stac_ref = build_stac_ref(StacDummyBuilder.collection()) - # This is a temporary mock.patch hack to make metadata_from_stac return metadata with a custom band dimension + # This is a temporary mock.patch hack to return metadata with a custom band dimension # TODO #743: Do this properly through appropriate STAC metadata - from openeo.metadata import metadata_from_stac as orig_metadata_from_stac - def metadata_from_stac(url: str): + def metadata_from_stac_object(stac_object): return CubeMetadata(dimensions=[BandDimension(name="bandzz", bands=[Band("Bz1"), Band("Bz2")])]) - with mock.patch("openeo.rest.datacube.metadata_from_stac", new=metadata_from_stac): + with mock.patch("openeo.rest.datacube.metadata_from_stac_object", new=metadata_from_stac_object): cube = dummy_backend.connection.load_stac(stac_ref, bands=bands) assert cube.metadata.has_band_dimension()