Validate load_stac properties with STAC queryables

CHANGELOG.md

@@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Added
 
+- Validate `load_stac` property filters against discoverable STAC queryables. ([#884](https://github.com/Open-EO/openeo-python-client/issues/884))
 - `DataCube.resample_spatial()` now supports parameterized `resolution` and `projection` arguments.  ([#897](https://github.com/Open-EO/openeo-python-client/issues/897))
 - Sanitize asset download filenames (e.g. strip slashes, (semi)colon, hash, ...), instead of blindly using the asset key as filename. ([#820](https://github.com/Open-EO/openeo-python-client/issues/820))
 - Support parameters in `DataCube` apply- and band-math operations ([#903](https://github.com/Open-EO/openeo-python-client/issues/903))

openeo/metadata.py

@@ -688,7 +688,11 @@ def metadata_from_stac(url: str) -> CubeMetadata:
     :param url: The URL to a static STAC catalog (STAC Item, STAC Collection, or STAC Catalog) or a specific STAC API Collection
     :return: A :py:class:`CubeMetadata` containing the DataCube band metadata from the url.
     """
-    stac_object = pystac.read_file(href=url)
+    return metadata_from_stac_object(pystac.read_file(href=url))
+
+
+def metadata_from_stac_object(stac_object: pystac.STACObject) -> CubeMetadata:
+    """Build cube metadata from an already loaded PySTAC object."""
     bands = _StacMetadataParser().bands_from_stac_object(stac_object)
 
     # At least assume there are spatial dimensions

openeo/rest/datacube.py

@@ -31,6 +31,7 @@
 )
 
 import numpy as np
+import pystac
 import requests
 import shapely.geometry
 import shapely.geometry.base
@@ -50,7 +51,7 @@
     Band,
     CollectionMetadata,
     CubeMetadata,
-    metadata_from_stac,
+    metadata_from_stac_object,
 )
 from openeo.rest import (
     DEFAULT_JOB_STATUS_POLL_CONNECTION_RETRY_INTERVAL,
@@ -73,7 +74,14 @@
 from openeo.rest.service import Service
 from openeo.rest.udp import RESTUserDefinedProcess
 from openeo.rest.vectorcube import VectorCube
-from openeo.util import dict_no_none, guess_format, load_json, normalize_crs, rfc3339
+from openeo.util import (
+    dict_no_none,
+    guess_format,
+    load_json,
+    load_json_resource,
+    normalize_crs,
+    rfc3339,
+)
 
 if typing.TYPE_CHECKING:
     # Imports for type checking only (circular import issue at runtime).
@@ -444,16 +452,10 @@ def load_stac(
         if bands is not None:
             arguments["bands"] = bands
 
-        properties = cls._build_load_properties_argument(
-            properties=properties,
-            # TODO: possible to detect queryables here too?
-        )
-        if properties is not None:
-            arguments["properties"] = properties
-
-        graph = PGNode("load_stac", arguments=arguments)
+        stac_object = None
         try:
-            metadata = metadata_from_stac(url)
+            stac_object = pystac.read_file(href=url)
+            metadata = metadata_from_stac_object(stac_object)
             # TODO: also apply spatial/temporal filters to metadata?
 
             if isinstance(bands, list):
@@ -477,6 +479,15 @@ def load_stac(
         except Exception as e:
             log.warning(f"Failed to extract cube metadata from STAC URL {url}", exc_info=True)
             metadata = None
+
+        properties = cls._build_load_properties_argument(
+            properties=properties,
+            queryables=_Queryables.build_stac(stac_object) if properties is not None else None,
+        )
+        if properties is not None:
+            arguments["properties"] = properties
+
+        graph = PGNode("load_stac", arguments=arguments)
         return cls(graph=graph, connection=connection, metadata=metadata)
 
     @classmethod
@@ -3245,18 +3256,63 @@ def __init__(self, properties: Iterable[str], additional: bool = False):
         self.properties = set(properties)
         self.additional = bool(additional)
 
+    @classmethod
+    def _from_json_schema(cls, data: dict, *, source: str) -> _Queryables:
+        properties = list(data.get("properties", {}).keys())
+        additional = data.get("additionalProperties", False)
+        log.debug(f"Queryables from {source!r}: {properties=} {additional=}")
+        return cls(properties=properties, additional=additional)
+
     @classmethod
     def build(cls, *, collection_id: str, connection: Optional[Connection]) -> Union[_Queryables, None]:
         if connection and connection.capabilities().supports_endpoint("/collections/{collection_id}/queryables"):
             path = f"/collections/{collection_id}/queryables"
             try:
                 resp = connection.get(path, allow_redirects=True)
                 resp.raise_for_status()
-                data = resp.json()
-                properties = list(data.get("properties", {}).keys())
-                additional = data.get("additionalProperties", False)
-                log.debug(f"Queryables from {path!r}: {properties=} {additional=}")
-                return cls(properties=properties, additional=additional)
+                return cls._from_json_schema(resp.json(), source=path)
             except Exception as e:
                 log.warning(f"Failed to get/parse queryables of from {path}: {e!r}")
         return None
+
+    @classmethod
+    def build_stac(cls, stac_object: Optional[pystac.STACObject]) -> Union[_Queryables, None]:
+        """Discover queryables advertised by a STAC object or its STAC API root."""
+        if stac_object is None:
+            return None
+
+        queryables_rels = ["http://www.opengis.net/def/rel/ogc/1.0/queryables", "queryables"]
+        queryables_link = next(
+            (link for rel in queryables_rels for link in stac_object.get_links(rel=rel)),
+            None,
+        )
+        source = queryables_link.get_absolute_href() if queryables_link else None
+
+        if source is None and isinstance(stac_object, pystac.Collection):
+            self_href = stac_object.get_self_href()
+            root_href = next(
+                (
+                    href
+                    for link in stac_object.get_links("root")
+                    if (href := link.get_absolute_href()) and href != self_href
+                ),
+                None,
+            )
+            try:
+                root = pystac.read_file(root_href) if root_href else None
+                conformance = root.extra_fields.get("conformsTo", []) if root else []
+                if (
+                    self_href
+                    and isinstance(conformance, list)
+                    and any("filter" in item.lower() for item in conformance if isinstance(item, str))
+                ):
+                    source = urllib.parse.urljoin(self_href.rstrip("/") + "/", "queryables")
+            except Exception as e:
+                log.debug(f"Failed to inspect STAC root for queryables: {e!r}")
+
+        if source:
+            try:
+                return cls._from_json_schema(load_json_resource(source), source=source)
+            except Exception as e:
+                log.warning(f"Failed to get/parse STAC queryables from {source}: {e!r}")
+        return None

tests/rest/datacube/test_datacube100.py

@@ -15,6 +15,7 @@
 
 import dirty_equals
 import pyproj
+import pystac
 import pytest
 import requests
 import shapely.geometry
@@ -30,6 +31,7 @@
 from openeo.rest import OpenEoClientException
 from openeo.rest.connection import Connection
 from openeo.rest.datacube import THIS, UDF, DataCube, _Queryables
+from openeo.testing.stac import StacDummyBuilder
 from openeo.utils.version import ComparableVersion
 
 from .. import get_download_graph
@@ -2363,6 +2365,26 @@ def test_broken(self, con100, requests_mock, api_capabilities):
         queryables = _Queryables.build(collection_id="S2", connection=con100)
         assert queryables is None
 
+    def test_stac_api_filter_conformance(self, monkeypatch):
+        collection = pystac.Collection.from_dict(StacDummyBuilder.collection())
+        collection.set_self_href("https://stac.test/collections/S2")
+        collection.add_link(pystac.Link(rel="root", target="https://stac.test/"))
+        root = pystac.Catalog(id="root", description="STAC API root")
+        root.extra_fields["conformsTo"] = ["https://api.stacspec.org/v1.0.0/item-search#filter"]
+        sources = []
+
+        monkeypatch.setattr(pystac, "read_file", lambda href: root)
+        monkeypatch.setattr(
+            "openeo.rest.datacube.load_json_resource",
+            lambda source: sources.append(source) or _build_queryables_doc(platform=True, additional=False),
+        )
+
+        queryables = _Queryables.build_stac(collection)
+
+        assert sources == ["https://stac.test/collections/S2/queryables"]
+        assert queryables.properties == {"eo:cloud_cover", "platform"}
+        assert queryables.additional is False
+
 
 @pytest.mark.parametrize("api_capabilities", [{"collection_queryables": True}])
 @pytest.mark.parametrize(

tests/rest/test_connection.py

@@ -3208,6 +3208,41 @@ def test_property_filtering(self, con120, properties, expected):
             }
         }
 
+    def test_property_filtering_with_queryables(self, con120, build_stac_ref, tmp_path, recwarn):
+        (tmp_path / "queryables.json").write_text(
+            json.dumps(
+                {
+                    "type": "object",
+                    "properties": {"platform": {"type": "string"}},
+                    "additionalProperties": False,
+                }
+            ),
+            encoding="utf8",
+        )
+        stac_ref = build_stac_ref(
+            StacDummyBuilder.collection(
+                links=[
+                    {
+                        "rel": "http://www.opengis.net/def/rel/ogc/1.0/queryables",
+                        "href": "queryables.json",
+                    }
+                ]
+            )
+        )
+
+        cube = con120.load_stac(
+            stac_ref,
+            properties={
+                "platform": lambda p: p == "S2A",
+                "unsupported": lambda p: p == "value",
+            },
+        )
+
+        assert set(cube.flat_graph()["loadstac1"]["arguments"]["properties"]) == {"platform", "unsupported"}
+        assert [str(w.message) for w in recwarn] == [
+            "Property filtering with unsupported properties ['unsupported'] (queryables: ['platform'])."
+        ]
+
     def test_load_stac_from_job_canonical(self, con120, requests_mock):
         requests_mock.get(
             API_URL + "jobs/j0bi6/results",
@@ -3489,18 +3524,20 @@ def test_load_stac_band_filtering_no_band_dimension(
     ):
         stac_ref = build_stac_ref(StacDummyBuilder.collection())
 
-        # This is a temporary mock.patch hack to make metadata_from_stac return metadata without a band dimension
+        # This is a temporary mock.patch hack to return metadata without a band dimension
         # TODO #743: Do this properly through appropriate STAC metadata
-        from openeo.metadata import metadata_from_stac as orig_metadata_from_stac
+        from openeo.metadata import (
+            metadata_from_stac_object as orig_metadata_from_stac_object,
+        )
 
-        def metadata_from_stac(url: str):
-            metadata = orig_metadata_from_stac(url=url)
+        def metadata_from_stac_object(stac_object):
+            metadata = orig_metadata_from_stac_object(stac_object)
             assert metadata.has_band_dimension()
             metadata = metadata.drop_dimension("bands")
             assert not metadata.has_band_dimension()
             return metadata
 
-        with mock.patch("openeo.rest.datacube.metadata_from_stac", new=metadata_from_stac):
+        with mock.patch("openeo.rest.datacube.metadata_from_stac_object", new=metadata_from_stac_object):
             cube = dummy_backend.connection.load_stac(stac_ref, bands=bands)
 
         assert cube.metadata.has_band_dimension() == has_band_dimension
@@ -3547,14 +3584,13 @@ def test_load_stac_band_filtering_custom_band_dimension(
     ):
         stac_ref = build_stac_ref(StacDummyBuilder.collection())
 
-        # This is a temporary mock.patch hack to make metadata_from_stac return metadata with a custom band dimension
+        # This is a temporary mock.patch hack to return metadata with a custom band dimension
         # TODO #743: Do this properly through appropriate STAC metadata
-        from openeo.metadata import metadata_from_stac as orig_metadata_from_stac
 
-        def metadata_from_stac(url: str):
+        def metadata_from_stac_object(stac_object):
             return CubeMetadata(dimensions=[BandDimension(name="bandzz", bands=[Band("Bz1"), Band("Bz2")])])
 
-        with mock.patch("openeo.rest.datacube.metadata_from_stac", new=metadata_from_stac):
+        with mock.patch("openeo.rest.datacube.metadata_from_stac_object", new=metadata_from_stac_object):
             cube = dummy_backend.connection.load_stac(stac_ref, bands=bands)
 
         assert cube.metadata.has_band_dimension()