From 5765b53be3db173e7554cc774502f1497c24bd1d Mon Sep 17 00:00:00 2001 From: Daria Agafonova Date: Thu, 30 Jul 2026 10:33:39 +0700 Subject: [PATCH 1/2] Use schema manifest for bulk caching --- CHANGELOG.md | 14 ++ docs/api/schema.rst | 2 +- docs/user_guide.md | 18 +- hed/schema/hed_cache.py | 194 ++++++++++----- hed/schema/schema_version_manifest.py | 31 +++ tests/schema/test_hed_schema_io.py | 242 ++++++++++++++++++- tests/schema/test_schema_version_manifest.py | 25 ++ 7 files changed, 451 insertions(+), 75 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3981649e..da776200 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,17 @@ +# Release 1.3.0 (unreleased) + +## Schema cache cleanup + +- `cache_xml_versions()` now uses `schema_versions.json` for default version discovery instead of crawling GitHub's REST API. Released and prerelease schemas are still downloaded and cached as before. +- Added `manifest_url` to `cache_xml_versions()` and `get_available_hed_versions()` so a fork or mirror can publish a compatible manifest. +- Non-default `hed_base_urls`, `hed_library_urls`, and `skip_folders` values are still honored through the REST crawl, but now raise `DeprecationWarning`; these arguments are planned for removal in HEDTools 2.0. +- The listing and bulk-cache paths reuse the same ETag-aware manifest metadata cache. + +## Documentation and tests + +- Added `get_available_hed_versions()` to the schema API reference and clarified the scope of `skip_folders`. +- Added coverage for the default manifest path, custom REST fallback, custom manifests, deprecation warnings, unsupported manifest formats, and manifest-cache reuse. + # Release 1.2.0 July 18, 2026 ## New features diff --git a/docs/api/schema.rst b/docs/api/schema.rst index 4856bf75..d7ce3ee2 100644 --- a/docs/api/schema.rst +++ b/docs/api/schema.rst @@ -114,7 +114,7 @@ Cache management ~~~~~~~~~~~~~~~~ .. automodule:: hed.schema.hed_cache - :members: cache_xml_versions, get_hed_versions, set_cache_directory, get_cache_directory + :members: cache_xml_versions, get_hed_versions, get_available_hed_versions, set_cache_directory, get_cache_directory :undoc-members: Schema loader base class diff --git a/docs/user_guide.md b/docs/user_guide.md index 0184c79e..fcc5682a 100644 --- a/docs/user_guide.md +++ b/docs/user_guide.md @@ -406,7 +406,17 @@ If GitHub can't be reached (offline, rate-limited, etc.), `get_available_hed_ver `get_available_hed_versions()` is designed to be called often — e.g. every time a web page loads — without adding up to a lot of GitHub traffic. -For the standard hed-schemas repository it reads a single repository-level manifest (`schema_versions.json`) from GitHub's raw/CDN host in one request. That host is *not* subject to GitHub's REST API rate limit, so this stays cheap even for an unauthenticated, frequently-polling caller. If that manifest can't be read — for example when you point the function at a custom or forked URL set, or the fetch fails — it falls back to crawling the REST API directory listings, backed by its own small on-disk cache (separate from the downloaded schema content) checked in two increasingly cheap tiers before making a real request: +For the standard hed-schemas repository it reads a single repository-level manifest (`schema_versions.json`) from GitHub's raw/CDN host in one request. That host is *not* subject to GitHub's REST API rate limit, so this stays cheap even for an unauthenticated, frequently-polling caller. A fork or mirror can provide the same manifest format through the `manifest_url` argument: + +```python +get_available_hed_versions( + manifest_url="https://example.org/hed/schema_versions.json", +) +``` + +If the manifest can't be read or uses an unsupported format, HEDTools falls back to the existing REST API directory crawl. The deprecated `hed_base_urls`, `hed_library_urls`, and `skip_folders` arguments still select that crawl when they have non-default values, but they will be removed in HEDTools 2.0. + +The manifest and REST results share a small on-disk metadata cache (separate from the downloaded schema content), checked in two increasingly cheap tiers before making a real request: 1. **Recently checked** — if a given piece of information was checked within the last `cache_time_threshold` seconds (60 by default), it's reused with no network call at all. 2. **Confirmed unchanged** — otherwise, a conditional request is made using a stored ETag. If GitHub confirms nothing changed (a 304 response), the previous result is reused. @@ -424,6 +434,8 @@ get_available_hed_versions(force_refresh=True) GitHub's API allows 60 requests per hour per IP address for unauthenticated callers, versus 5,000 per hour for authenticated ones. Authentication also makes conditional (ETag) requests free of charge against that limit — for unauthenticated callers, even a confirmed-unchanged response still counts against the 60/hour budget. +The default manifest request does not need a GitHub token. A token is useful only when HEDTools has to use the REST fallback or when custom REST URLs are supplied. + If your use of HEDTools makes frequent GitHub calls — a web service checking for new versions, a CI pipeline, a container that restarts often — set a GitHub personal access token (no special scopes needed; it only needs to read a public repository) as an environment variable: ```bash @@ -447,6 +459,10 @@ set_cache_directory("/opt/hed_cache") # optional: a specific location to ship o cache_xml_versions() # downloads every discovered version's full content ``` +By default, `cache_xml_versions()` discovers released and prerelease schemas from the same manifest and then downloads their XML files. This avoids the GitHub REST API directory crawl and reuses a recently fetched manifest from `get_available_hed_versions()`. A fork or mirror can be selected with `manifest_url`. + +The old `hed_base_urls`, `hed_library_urls`, and `skip_folders` arguments still work for custom REST layouts during the 1.x deprecation period. `skip_folders` filters only top-level library folders; nested directories inside `hedxml` and `prerelease` are always ignored. + This is a much heavier operation than `get_available_hed_versions()` — it downloads every version it finds, not just a listing — so it's meant to be run once (e.g. during image build or setup), not on a request-handling hot path. It's throttled independently (won't re-run within 30 minutes of its last successful run in the same cache folder) to avoid accidental repeated use. ### Clearing the cache diff --git a/hed/schema/hed_cache.py b/hed/schema/hed_cache.py index 21912d72..60e8de6c 100644 --- a/hed/schema/hed_cache.py +++ b/hed/schema/hed_cache.py @@ -5,6 +5,7 @@ import shutil import os import time +import warnings import json from hashlib import sha1 @@ -18,6 +19,7 @@ from semantic_version import Version from hed.schema.hed_cache_lock import CacheError, CacheLock from hed.schema.schema_io.schema_util import url_to_file, make_url_request +from hed.schema.schema_version_manifest import MANIFEST_URL from pathlib import Path import urllib from urllib.error import URLError @@ -231,26 +233,44 @@ def cache_xml_versions( hed_library_urls=DEFAULT_LIBRARY_URL_LIST, skip_folders=DEFAULT_SKIP_FOLDERS, cache_folder=None, + manifest_url=MANIFEST_URL, ) -> float: - """Cache all schemas at the given URLs. + """Cache all released and prerelease schemas. Parameters: - hed_base_urls (str or list): Path or list of paths. These should point to a single folder. - hed_library_urls (str or list): Path or list of paths. These should point to folder containing library folders. - skip_folders (list): A list of subfolders to skip over when downloading. + hed_base_urls (str or list): Deprecated REST URL(s) for the standard schema folder. + Non-default values continue to use the REST crawl. + hed_library_urls (str or list): Deprecated REST URL(s) containing library schema folders. + Non-default values continue to use the REST crawl. + skip_folders (list): Deprecated list of top-level library folders to skip during a REST crawl. cache_folder (str): The folder holding the cache. + manifest_url (str): URL of a ``schema_versions.json`` manifest. Used when the deprecated + REST arguments have their default values. Returns: float: Returns -1 if cache failed for any reason, including having been cached too recently. Returns 0 if it successfully cached this time. Notes: - - The Default skip_folders is 'deprecated'. + - By default, version discovery uses the manifest rather than GitHub's REST API. + - Custom REST arguments are still honored and fall back to the existing directory crawl. + - ``skip_folders`` only filters top-level library folders. Directory entries inside a + schema's ``hedxml`` or ``prerelease`` folder are always ignored. - The HED cache folder defaults to HED_CACHE_DIRECTORY. - - The directories on GitHub are of the form: - https://api.github.com/repos/hed-standard/hed-schemas/contents/standard_schema """ + if isinstance(hed_base_urls, str): + hed_base_urls = [hed_base_urls] + else: + hed_base_urls = list(hed_base_urls) + if isinstance(hed_library_urls, str): + hed_library_urls = [hed_library_urls] + else: + hed_library_urls = list(hed_library_urls) + skip_folders = tuple(skip_folders) + use_manifest = _rest_arguments_are_default(hed_base_urls, hed_library_urls, skip_folders) + _warn_deprecated_rest_arguments(hed_base_urls, hed_library_urls, skip_folders) + if not cache_folder: cache_folder = HED_CACHE_DIRECTORY @@ -260,19 +280,37 @@ def cache_xml_versions( try: with CacheLock(cache_folder): - if isinstance(hed_base_urls, str): - hed_base_urls = [hed_base_urls] - if isinstance(hed_library_urls, str): - hed_library_urls = [hed_library_urls] - all_hed_versions = {} - for hed_base_url in hed_base_urls: - new_hed_versions = _get_hed_xml_versions_one_library(hed_base_url) - _merge_in_versions(all_hed_versions, new_hed_versions) - for hed_library_url in hed_library_urls: - new_hed_versions = _get_hed_xml_versions_from_url_all_libraries( - hed_library_url, skip_folders=skip_folders - ) - _merge_in_versions(all_hed_versions, new_hed_versions) + all_hed_versions = None + if use_manifest: + from hed.schema import schema_version_manifest as _manifest + + url_cache = _read_available_versions_cache(cache_folder) + cache_before = json.dumps(url_cache, sort_keys=True) + try: + manifest_json = _get_json_with_etag( + manifest_url, + url_cache, + force_refresh=False, + cache_time_threshold=AVAILABLE_VERSIONS_TIME_THRESHOLD, + ) + if _manifest.is_supported(manifest_json): + all_hed_versions = _manifest.all_version_infos(manifest_json, check_prerelease=True) + except Exception: + pass + finally: + if json.dumps(url_cache, sort_keys=True) != cache_before: + _write_available_versions_cache(cache_folder, url_cache) + + if all_hed_versions is None: + all_hed_versions = {} + for hed_base_url in hed_base_urls: + new_hed_versions = _get_hed_xml_versions_one_library(hed_base_url) + _merge_in_versions(all_hed_versions, new_hed_versions) + for hed_library_url in hed_library_urls: + new_hed_versions = _get_hed_xml_versions_from_url_all_libraries( + hed_library_url, skip_folders=skip_folders + ) + _merge_in_versions(all_hed_versions, new_hed_versions) for library_name, hed_versions in all_hed_versions.items(): for version, version_info in hed_versions.items(): @@ -293,39 +331,30 @@ def get_available_hed_versions( cache_folder=None, force_refresh=False, cache_time_threshold=AVAILABLE_VERSIONS_TIME_THRESHOLD, + manifest_url=MANIFEST_URL, ) -> Union[list, dict]: """List HED schema versions available on GitHub, without downloading or caching their content. - For the canonical hed-schemas URLs this reads a single repository-level manifest - (schema_versions.json) from the raw/CDN host in one request - see schema_version_manifest - - which is not subject to GitHub's REST API rate limit. If that manifest can't be read (a - custom/forked URL set, or any fetch/parse failure) the function falls back to crawling - GitHub's REST API directory listings. That fallback never fetches a schema file's actual XML - content, but listing everything can still add up to a couple dozen small JSON directory-listing - requests in one call: 1-2 for the standard schema (plus its prerelease folder), 1 to enumerate - the library folders, and 1-2 more per library folder found. That worst case only applies to - library_name="all"; passing library_name=None (the default) skips every library-related - request entirely, and passing a specific library name skips the standard-schema request and - restricts the library side to just that one library's folder. - - It's the live-from-GitHub counterpart to get_hed_versions() (which only reports what's already - bundled with hedtools or previously cached on disk, with zero network calls), and it's still - far cheaper than cache_xml_versions() (which makes those same listing calls AND then downloads - every version's full content - fine to do once for a version you're about to use, wasteful to - do just to show a list of names). The REST fallback caches its own results on disk (see Notes) - so that a caller polling it frequently - e.g. a web service handling many requests - doesn't - trip GitHub's API rate limits. Callers don't need to implement their own throttling on top of - this. + With the default arguments, this reads one repository-level ``schema_versions.json`` manifest + from GitHub's raw-content host. A compatible manifest from a fork or mirror can be selected + with ``manifest_url``. If the manifest cannot be read or parsed, the function falls back to + GitHub's REST API directory listings. + + This is the live counterpart to :func:`get_hed_versions`, which reports only bundled or + previously cached schemas without making a network request. This function lists versions but + never downloads their XML content. The manifest and REST results are cached on disk so callers + can safely use the function for frequently refreshed interfaces. Typical usage is: call this to populate something like a version-picker dropdown, then only fetch the one version the user actually selects, via load_schema_version() (which downloads and caches just that version, lazily, the first time it's needed). Parameters: - hed_base_urls (str or list): Path or list of paths for the standard schema folder(s). - hed_library_urls (str or list): Path or list of paths for folder(s) containing library - schema subfolders. - skip_folders (list): A list of library subfolders to skip. Default is 'deprecated'. + hed_base_urls (str or list): Deprecated REST URL(s) for the standard schema folder. + Non-default values continue to use the REST crawl. + hed_library_urls (str or list): Deprecated REST URL(s) containing library schema folders. + Non-default values continue to use the REST crawl. + skip_folders (list): Deprecated list of top-level library folders to skip during a REST crawl. library_name (str or None): None retrieves the standard schema only. Pass "all" to retrieve all standard and library schemas as a dict. Pass a specific library name to retrieve just that library. @@ -343,6 +372,8 @@ def get_available_hed_versions( Default is 60 seconds - short enough that new releases show up quickly, long enough that a caller polling this in a tight loop doesn't generate a request per call. + manifest_url (str): URL of a ``schema_versions.json`` manifest. Used when the deprecated + REST arguments have their default values. Returns: Union[list, dict]: List of version numbers, or {library_name: [versions]} if @@ -386,14 +417,18 @@ def get_available_hed_versions( ['8.5.0', '8.4.0', '8.3.0', ...] Notes: - - The manifest fast path is used only for the canonical hed-schemas URLs (the defaults - for hed_base_urls, hed_library_urls, and skip_folders). Any other URL set, or any - failure reading or parsing the manifest, transparently falls through to the REST crawl - described below, so behavior is never worse than before. + - The manifest path is used when ``hed_base_urls``, ``hed_library_urls``, and + ``skip_folders`` have their default values. Pass ``manifest_url`` to use a compatible + manifest from a fork or mirror. + - Non-default REST arguments are still honored and bypass the manifest path. Any failure + reading or parsing a manifest transparently falls through to the REST crawl. + - ``skip_folders`` only filters top-level library folders. Directory entries inside a + schema's ``hedxml`` or ``prerelease`` folder are always ignored. - The REST fallback caches per GitHub URL (there are several under the hood: the standard schema folder and its prerelease folder, the library-folder listing, and each library's own folder and prerelease folder), in a small metadata file (available_versions_cache.json) inside the cache folder, in two layers: + 1. If a given URL was checked within cache_time_threshold seconds (default 60), it's reused with no network call at all. 2. Otherwise, a conditional GET is made using the ETag from the last time that URL @@ -407,37 +442,36 @@ def get_available_hed_versions( fetched and failed. - This uses a much shorter threshold than the one in hed_cache_lock.py, which throttles cache_xml_versions()'s far more expensive per-version download step. - - Unlike cache_xml_versions(), this never writes schema content - the on-disk cache - used here holds only the same small directory-listing JSON GitHub itself returns - (version names, SHAs, and download URLs), never a schema file itself. It has no - interaction with get_hed_versions(), cache_local_versions(), or the schema files - cache_xml_versions() downloads. + - Unlike cache_xml_versions(), this never writes schema content. Its on-disk cache holds + only the manifest or small REST directory listings, never a schema file itself. - force_refresh=True skips layer 1 above but still uses layer 2 (the conditional GET), so it stays cheap when nothing has actually changed. """ if isinstance(hed_base_urls, str): hed_base_urls = [hed_base_urls] + else: + hed_base_urls = list(hed_base_urls) if isinstance(hed_library_urls, str): hed_library_urls = [hed_library_urls] + else: + hed_library_urls = list(hed_library_urls) + skip_folders = tuple(skip_folders) + use_manifest = _rest_arguments_are_default(hed_base_urls, hed_library_urls, skip_folders) + _warn_deprecated_rest_arguments(hed_base_urls, hed_library_urls, skip_folders) if not cache_folder: cache_folder = HED_CACHE_DIRECTORY url_cache = _read_available_versions_cache(cache_folder) cache_before = json.dumps(url_cache, sort_keys=True) - # Fast path: read the repo-level manifest in a single fetch from the raw/CDN host (not subject - # to the GitHub REST API rate limit) instead of crawling the API directory listings. Only used - # for the canonical hed-schemas URLs; any custom/forked URL set falls through to the crawl. Any - # failure (unreachable, malformed, or an unrecognized manifest format) also falls through. - if ( - list(hed_base_urls) == list(DEFAULT_URL_LIST) - and list(hed_library_urls) == list(DEFAULT_LIBRARY_URL_LIST) - and tuple(skip_folders) == tuple(DEFAULT_SKIP_FOLDERS) - ): + # Fast path: read one repo-level manifest from the raw/CDN host instead of crawling REST + # directory listings. A custom manifest URL works as long as the deprecated REST arguments + # retain their defaults. Any fetch, parse, or format failure falls through to the REST crawl. + if use_manifest: from hed.schema import schema_version_manifest as _manifest try: - manifest_json = _get_json_with_etag(_manifest.MANIFEST_URL, url_cache, force_refresh, cache_time_threshold) + manifest_json = _get_json_with_etag(manifest_url, url_cache, force_refresh, cache_time_threshold) if _manifest.is_supported(manifest_json): if json.dumps(url_cache, sort_keys=True) != cache_before: _write_available_versions_cache(cache_folder, url_cache) @@ -518,6 +552,35 @@ def get_available_hed_versions( return [] +def _rest_arguments_are_default(hed_base_urls, hed_library_urls, skip_folders): + """Return True when the deprecated REST discovery arguments have their default values.""" + return ( + list(hed_base_urls) == list(DEFAULT_URL_LIST) + and list(hed_library_urls) == list(DEFAULT_LIBRARY_URL_LIST) + and tuple(skip_folders) == tuple(DEFAULT_SKIP_FOLDERS) + ) + + +def _warn_deprecated_rest_arguments(hed_base_urls, hed_library_urls, skip_folders): + """Warn when a caller still relies on custom REST discovery arguments.""" + changed = [] + if list(hed_base_urls) != list(DEFAULT_URL_LIST): + changed.append("hed_base_urls") + if list(hed_library_urls) != list(DEFAULT_LIBRARY_URL_LIST): + changed.append("hed_library_urls") + if tuple(skip_folders) != tuple(DEFAULT_SKIP_FOLDERS): + changed.append("skip_folders") + if not changed: + return + + warnings.warn( + f"{', '.join(changed)} will be removed in HEDTools 2.0. " + "Publish a schema_versions.json manifest and pass manifest_url instead.", + DeprecationWarning, + stacklevel=3, + ) + + def _read_available_versions_cache(cache_folder): """Load the on-disk per-URL listing cache used by get_available_hed_versions(). @@ -829,7 +892,8 @@ def _get_hed_xml_versions_from_url_all_libraries( hed_base_library_url(str): A single GitHub API url to cache, which contains library schema folders The subfolders should be a schema folder containing hedxml and/or prerelease folders. library_name(str or None): If str, cache only the named library schemas. - skip_folders (list): A list of sub folders to skip over when downloading. + skip_folders (list): Top-level library folders to skip. This does not filter + directories inside ``hedxml`` or ``prerelease``. etag_cache (dict or None): Passed through to _get_json_with_etag() for every request this makes, including one per discovered library subfolder. None (the default) disables conditional/cached requests. @@ -840,7 +904,7 @@ def _get_hed_xml_versions_from_url_all_libraries( Union[list, dict]: List of version numbers or dictionary {library_name: [versions]}. Notes: - - The Default skip_folders is 'deprecated'. + - The default ``skip_folders`` value is ``("deprecated",)``. - The HED cache folder defaults to HED_CACHE_DIRECTORY. - The directories on GitHub are of the form: https://api.github.com/repos/hed-standard/hed-schemas/contents/standard_schema/hedxml diff --git a/hed/schema/schema_version_manifest.py b/hed/schema/schema_version_manifest.py index 74a3cc4b..612f5f4f 100644 --- a/hed/schema/schema_version_manifest.py +++ b/hed/schema/schema_version_manifest.py @@ -114,6 +114,37 @@ def available_versions(manifest, library_name=None, check_prerelease=False): return _versions_for_key(manifest, manifest_key, check_prerelease) +def all_version_infos(manifest, check_prerelease=True): + """Return manifest entries in the shape used by the schema cache. + + Parameters: + manifest (dict): A supported manifest (see :func:`is_supported`). + check_prerelease (bool): If True, include prerelease versions. + + Returns: + dict: ``{library_name_or_None: {version: (sha, download_url, prerelease)}}``. + Deprecated versions are never included. + """ + result = {} + ref = manifest.get("repo_commit") or "main" + for manifest_key, categories in manifest.get("libraries", {}).items(): + library_name = None if manifest_key == _STANDARD_MANIFEST_KEY else manifest_key + version_infos = {} + selected_categories = [("released", False)] + if check_prerelease: + selected_categories.append(("prerelease", True)) + for category, is_prerelease in selected_categories: + for entry in categories.get(category, []): + version_infos[entry["version"]] = ( + entry["sha"], + raw_url_for(entry["file"], ref), + is_prerelease, + ) + if version_infos: + result[library_name] = version_infos + return result + + def find_version_info(manifest, xml_version, library_name, ref=None): """Locate one version in ``manifest`` and return its download info, or None if absent. diff --git a/tests/schema/test_hed_schema_io.py b/tests/schema/test_hed_schema_io.py index b2a574a8..7a785c86 100644 --- a/tests/schema/test_hed_schema_io.py +++ b/tests/schema/test_hed_schema_io.py @@ -1,5 +1,5 @@ import unittest -from unittest.mock import patch +from unittest.mock import Mock, patch from hed.errors import HedFileError from hed.errors.error_types import SchemaErrors @@ -11,6 +11,7 @@ import json import math import tempfile +import warnings from urllib.error import URLError from semantic_version import Version from hed.errors import HedExceptions @@ -20,6 +21,42 @@ import shutil +SAMPLE_CACHE_MANIFEST = { + "manifest_format_version": 1, + "repo_commit": "abc123", + "libraries": { + "": { + "released": [ + { + "version": "8.4.0", + "file": "standard_schema/hedxml/HED8.4.0.xml", + "sha": "sha840", + } + ], + "prerelease": [ + { + "version": "8.5.0", + "file": "standard_schema/prerelease/HED8.5.0.xml", + "sha": "sha850", + } + ], + "deprecated": [], + }, + "score": { + "released": [ + { + "version": "2.1.0", + "file": "library_schemas/score/hedxml/HED_score_2.1.0.xml", + "sha": "shascore", + } + ], + "prerelease": [], + "deprecated": [], + }, + }, +} + + def _assert_valid_sorted_versions(test_case, versions): """Assert every entry is a valid semver string and the list is sorted newest-first. @@ -164,8 +201,11 @@ def test_cache_xml_versions_seeds_bundled_schemas_on_github_failure(self): cache always contains at least the bundled released schemas. """ with tempfile.TemporaryDirectory() as tmp_dir: - with patch.object( - hed_cache, "_get_hed_xml_versions_one_library", side_effect=URLError("simulated failure") + with ( + patch.object(hed_cache, "_get_json_with_etag", side_effect=URLError("simulated manifest failure")), + patch.object( + hed_cache, "_get_hed_xml_versions_one_library", side_effect=URLError("simulated REST failure") + ), ): result = hed_cache.cache_xml_versions(cache_folder=tmp_dir) @@ -182,6 +222,120 @@ def test_cache_xml_versions_seeds_bundled_schemas_on_github_failure(self): score = hed_cache.get_hed_versions(tmp_dir, library_name="score") self.assertTrue(score, "score schemas must be in cache after failed cache_xml_versions") + def test_cache_xml_versions_uses_manifest_for_default_urls(self): + """The default bulk-cache path should not call GitHub's REST discovery helpers.""" + with tempfile.TemporaryDirectory() as tmp_dir: + with ( + patch.object(hed_cache, "_get_json_with_etag", return_value=SAMPLE_CACHE_MANIFEST) as manifest_fetch, + patch.object(hed_cache, "_get_hed_xml_versions_one_library") as standard_rest, + patch.object(hed_cache, "_get_hed_xml_versions_from_url_all_libraries") as library_rest, + patch.object(hed_cache, "_cache_hed_version", return_value="cached") as cache_one, + ): + result = hed_cache.cache_xml_versions(cache_folder=tmp_dir) + + self.assertEqual(result, 0) + self.assertEqual(manifest_fetch.call_args.args[0], hed_cache.MANIFEST_URL) + standard_rest.assert_not_called() + library_rest.assert_not_called() + self.assertEqual(cache_one.call_count, 3) + + def test_cache_xml_versions_accepts_custom_manifest(self): + """A fork can use its own manifest for bulk caching.""" + custom_manifest_url = "https://example.test/schema_versions.json" + with tempfile.TemporaryDirectory() as tmp_dir: + with ( + patch.object(hed_cache, "_get_json_with_etag", return_value=SAMPLE_CACHE_MANIFEST) as manifest_fetch, + patch.object(hed_cache, "_cache_hed_version", return_value="cached"), + ): + result = hed_cache.cache_xml_versions( + cache_folder=tmp_dir, + manifest_url=custom_manifest_url, + ) + + self.assertEqual(result, 0) + self.assertEqual(manifest_fetch.call_args.args[0], custom_manifest_url) + + def test_cache_xml_versions_falls_back_when_manifest_is_unsupported(self): + """An unsupported manifest must leave the REST bulk-cache path available.""" + rest_versions = {None: {"8.4.0": ("sha840", "https://example.test/HED8.4.0.xml", False)}} + with tempfile.TemporaryDirectory() as tmp_dir: + with ( + patch.object( + hed_cache, + "_get_json_with_etag", + return_value={"manifest_format_version": 2, "libraries": {}}, + ), + patch.object( + hed_cache, + "_get_hed_xml_versions_one_library", + return_value=rest_versions, + ) as rest_fetch, + patch.object(hed_cache, "_get_hed_xml_versions_from_url_all_libraries", return_value={}), + patch.object(hed_cache, "_cache_hed_version", return_value="cached"), + ): + result = hed_cache.cache_xml_versions(cache_folder=tmp_dir) + + self.assertEqual(result, 0) + rest_fetch.assert_called_once() + + def test_cache_xml_versions_custom_urls_still_use_rest(self): + """Non-default REST arguments remain functional during the deprecation period.""" + custom_standard_url = "https://example.test/standard" + rest_versions = {None: {"8.4.0": ("sha840", "https://example.test/HED8.4.0.xml", False)}} + + with tempfile.TemporaryDirectory() as tmp_dir: + with ( + patch.object(hed_cache, "_get_json_with_etag") as manifest_fetch, + patch.object(hed_cache, "_get_hed_xml_versions_one_library", return_value=rest_versions) as rest_fetch, + patch.object(hed_cache, "_get_hed_xml_versions_from_url_all_libraries", return_value={}), + patch.object(hed_cache, "_cache_hed_version", return_value="cached"), + self.assertWarnsRegex(DeprecationWarning, "hed_base_urls"), + ): + result = hed_cache.cache_xml_versions( + hed_base_urls=(custom_standard_url,), + cache_folder=tmp_dir, + ) + + self.assertEqual(result, 0) + manifest_fetch.assert_not_called() + rest_fetch.assert_called_once_with(custom_standard_url) + + def test_default_rest_arguments_do_not_warn(self): + """Explicit or implicit default REST arguments should not produce deprecation noise.""" + with tempfile.TemporaryDirectory() as tmp_dir: + with ( + patch.object(hed_cache, "_get_json_with_etag", return_value=SAMPLE_CACHE_MANIFEST), + patch.object(hed_cache, "_cache_hed_version", return_value="cached"), + warnings.catch_warnings(record=True) as caught, + ): + warnings.simplefilter("always") + hed_cache.cache_xml_versions( + hed_base_urls=list(hed_cache.DEFAULT_URL_LIST), + hed_library_urls=list(hed_cache.DEFAULT_LIBRARY_URL_LIST), + skip_folders=list(hed_cache.DEFAULT_SKIP_FOLDERS), + cache_folder=tmp_dir, + ) + + self.assertFalse(any(item.category is DeprecationWarning for item in caught)) + + def test_manifest_cache_is_reused_by_bulk_cache(self): + """Listing versions and then caching them should fetch the manifest only once.""" + response = Mock() + response.read.return_value = json.dumps(SAMPLE_CACHE_MANIFEST).encode("utf-8") + response.headers = {"ETag": '"manifest-etag"'} + + with tempfile.TemporaryDirectory() as tmp_dir: + with ( + patch.object(hed_cache, "make_url_request", return_value=response) as request, + patch.object(hed_cache, "_cache_hed_version", return_value="cached"), + ): + versions = hed_cache.get_available_hed_versions(cache_folder=tmp_dir) + result = hed_cache.cache_xml_versions(cache_folder=tmp_dir) + + self.assertEqual(versions, ["8.4.0"]) + self.assertEqual(result, 0) + request.assert_called_once() + def test_get_hed_version_path_downloads_only_requested_version(self): """get_hed_version_path must download only the single requested version, never the full catalog. @@ -305,6 +459,77 @@ def test_get_available_hed_versions_lists_without_downloading(self): f"Unexpected files in cache folder: {cached_files}", ) + def test_get_available_hed_versions_accepts_custom_manifest(self): + """A fork can provide its own manifest without changing the deprecated REST arguments.""" + custom_manifest_url = "https://example.test/schema_versions.json" + with tempfile.TemporaryDirectory() as tmp_dir: + with patch.object( + hed_cache, + "_get_json_with_etag", + return_value=SAMPLE_CACHE_MANIFEST, + ) as manifest_fetch: + versions = hed_cache.get_available_hed_versions( + cache_folder=tmp_dir, + manifest_url=custom_manifest_url, + ) + + self.assertEqual(versions, ["8.4.0"]) + self.assertEqual(manifest_fetch.call_args.args[0], custom_manifest_url) + + def test_unsupported_manifest_falls_back_to_rest(self): + """An unknown manifest format must leave the existing REST fallback available.""" + rest_versions = {None: {"8.4.0": ("sha840", "https://example.test/HED8.4.0.xml", False)}} + with tempfile.TemporaryDirectory() as tmp_dir: + with ( + patch.object( + hed_cache, + "_get_json_with_etag", + return_value={"manifest_format_version": 2, "libraries": {}}, + ), + patch.object( + hed_cache, + "_get_hed_xml_versions_one_library", + return_value=rest_versions, + ) as rest_fetch, + ): + versions = hed_cache.get_available_hed_versions(cache_folder=tmp_dir) + + self.assertEqual(versions, ["8.4.0"]) + rest_fetch.assert_called_once() + + def test_manifest_and_rest_paths_return_the_same_versions(self): + """The transition must not change the version-list result.""" + rest_versions = { + None: { + "8.4.0": ("sha840", "https://example.test/HED8.4.0.xml", False), + "8.5.0": ("sha850", "https://example.test/HED8.5.0.xml", True), + } + } + with tempfile.TemporaryDirectory() as manifest_cache: + with patch.object(hed_cache, "_get_json_with_etag", return_value=SAMPLE_CACHE_MANIFEST): + manifest_result = hed_cache.get_available_hed_versions( + check_prerelease=True, + cache_folder=manifest_cache, + ) + + with tempfile.TemporaryDirectory() as rest_cache: + with ( + patch.object( + hed_cache, + "_get_hed_xml_versions_one_library", + return_value=rest_versions, + ), + self.assertWarns(DeprecationWarning), + ): + rest_result = hed_cache.get_available_hed_versions( + hed_base_urls=("https://example.test/standard",), + hed_library_urls=(), + check_prerelease=True, + cache_folder=rest_cache, + ) + + self.assertEqual(manifest_result, rest_result) + def test_get_available_hed_versions_caches_result(self): """A second call within the threshold should reuse the cached listing; force_refresh bypasses it.""" with tempfile.TemporaryDirectory() as tmp_dir: @@ -425,11 +650,12 @@ def test_get_available_hed_versions_degrades_on_malformed_response(self): """ real_file_url = hed_cache.DEFAULT_HED_LIST_VERSIONS_URL + "/hedxml/HED8.2.0.xml" try: - result = hed_cache.get_available_hed_versions( - hed_base_urls=(), - hed_library_urls=(real_file_url,), - library_name="all", - ) + with self.assertWarns(DeprecationWarning): + result = hed_cache.get_available_hed_versions( + hed_base_urls=(), + hed_library_urls=(real_file_url,), + library_name="all", + ) except Exception as ex: self.fail(f"get_available_hed_versions() should degrade gracefully, not raise: {ex!r}") self.assertEqual(result, {}) diff --git a/tests/schema/test_schema_version_manifest.py b/tests/schema/test_schema_version_manifest.py index fe0bf66a..2da0102f 100644 --- a/tests/schema/test_schema_version_manifest.py +++ b/tests/schema/test_schema_version_manifest.py @@ -120,6 +120,31 @@ def test_unknown_library_returns_empty(self): self.assertEqual(manifest.available_versions(SAMPLE_MANIFEST, "nosuch"), []) +class TestAllVersionInfos(unittest.TestCase): + def test_cache_shape_includes_released_and_prerelease(self): + result = manifest.all_version_infos(SAMPLE_MANIFEST) + + self.assertEqual( + result[None]["8.4.0"], + ( + "sha840", + "https://raw.githubusercontent.com/hed-standard/hed-schemas/" + "abc123def456/standard_schema/hedxml/HED8.4.0.xml", + False, + ), + ) + self.assertTrue(result[None]["8.5.0"][2]) + self.assertEqual(result["score"]["2.1.0"][0], "shasc") + self.assertNotIn("8.3.5", result[None]) + self.assertNotIn("2.0.0", result["score"]) + + def test_prerelease_can_be_excluded(self): + result = manifest.all_version_infos(SAMPLE_MANIFEST, check_prerelease=False) + + self.assertNotIn("8.5.0", result[None]) + self.assertNotIn("mouse", result) + + class TestFindVersionInfo(unittest.TestCase): def test_released_pins_to_repo_commit(self): info = manifest.find_version_info(SAMPLE_MANIFEST, "8.4.0", None) From ef3b73f0ac596935060aaa6e6d6dd75ddccd65c4 Mon Sep 17 00:00:00 2001 From: Daria Agafonova Date: Fri, 31 Jul 2026 00:56:35 +0700 Subject: [PATCH 2/2] Fix schema URLs for custom manifests --- docs/user_guide.md | 7 ++++- hed/schema/hed_cache.py | 12 +++++--- hed/schema/schema_version_manifest.py | 26 +++++++++++++++-- tests/schema/test_hed_schema_io.py | 16 +++++++++-- tests/schema/test_schema_version_manifest.py | 30 ++++++++++++++++++++ 5 files changed, 81 insertions(+), 10 deletions(-) diff --git a/docs/user_guide.md b/docs/user_guide.md index fcc5682a..f5201c27 100644 --- a/docs/user_guide.md +++ b/docs/user_guide.md @@ -414,6 +414,11 @@ get_available_hed_versions( ) ``` +For a manifest hosted on `raw.githubusercontent.com`, schema files are read from the same +owner/repository and pinned to the manifest's `repo_commit`. On other hosts, each manifest `file` +path is resolved relative to `manifest_url`, so a mirror should publish the manifest and schema tree +under the same base URL. + If the manifest can't be read or uses an unsupported format, HEDTools falls back to the existing REST API directory crawl. The deprecated `hed_base_urls`, `hed_library_urls`, and `skip_folders` arguments still select that crawl when they have non-default values, but they will be removed in HEDTools 2.0. The manifest and REST results share a small on-disk metadata cache (separate from the downloaded schema content), checked in two increasingly cheap tiers before making a real request: @@ -459,7 +464,7 @@ set_cache_directory("/opt/hed_cache") # optional: a specific location to ship o cache_xml_versions() # downloads every discovered version's full content ``` -By default, `cache_xml_versions()` discovers released and prerelease schemas from the same manifest and then downloads their XML files. This avoids the GitHub REST API directory crawl and reuses a recently fetched manifest from `get_available_hed_versions()`. A fork or mirror can be selected with `manifest_url`. +By default, `cache_xml_versions()` discovers released and prerelease schemas from the same manifest and then downloads their XML files. This avoids the GitHub REST API directory crawl and reuses a recently fetched manifest from `get_available_hed_versions()`. A fork or mirror can be selected with `manifest_url`; schema files follow the URL rules described above. The old `hed_base_urls`, `hed_library_urls`, and `skip_folders` arguments still work for custom REST layouts during the 1.x deprecation period. `skip_folders` filters only top-level library folders; nested directories inside `hedxml` and `prerelease` are always ignored. diff --git a/hed/schema/hed_cache.py b/hed/schema/hed_cache.py index 60e8de6c..0f8b8c4f 100644 --- a/hed/schema/hed_cache.py +++ b/hed/schema/hed_cache.py @@ -234,7 +234,7 @@ def cache_xml_versions( skip_folders=DEFAULT_SKIP_FOLDERS, cache_folder=None, manifest_url=MANIFEST_URL, -) -> float: +) -> int: """Cache all released and prerelease schemas. Parameters: @@ -248,8 +248,8 @@ def cache_xml_versions( REST arguments have their default values. Returns: - float: Returns -1 if cache failed for any reason, including having been cached too recently. - Returns 0 if it successfully cached this time. + int: Returns -1 if cache failed for any reason, including having been cached too recently. + Returns 0 if it successfully cached this time. Notes: - By default, version discovery uses the manifest rather than GitHub's REST API. @@ -294,7 +294,11 @@ def cache_xml_versions( cache_time_threshold=AVAILABLE_VERSIONS_TIME_THRESHOLD, ) if _manifest.is_supported(manifest_json): - all_hed_versions = _manifest.all_version_infos(manifest_json, check_prerelease=True) + all_hed_versions = _manifest.all_version_infos( + manifest_json, + check_prerelease=True, + manifest_url=manifest_url, + ) except Exception: pass finally: diff --git a/hed/schema/schema_version_manifest.py b/hed/schema/schema_version_manifest.py index 612f5f4f..cbf8eaf1 100644 --- a/hed/schema/schema_version_manifest.py +++ b/hed/schema/schema_version_manifest.py @@ -34,6 +34,8 @@ from __future__ import annotations +from urllib.parse import urljoin, urlsplit, urlunsplit + from semantic_version import Version # The manifest is served as raw file content (CDN-backed), not through the REST API, so it is not @@ -71,6 +73,24 @@ def raw_url_for(file_path: str, ref: str) -> str: return f"{RAW_CONTENT_BASE}/{ref}/{file_path}" +def content_url_for_manifest(file_path: str, ref: str, manifest_url: str = MANIFEST_URL) -> str: + """Build a schema-content URL from the location of its manifest. + + Raw GitHub manifests are pinned to ``ref`` in the same owner/repository. Other hosts resolve + manifest file paths relative to the manifest URL, which supports mirrors that publish the + manifest and schema tree together. + """ + parsed = urlsplit(manifest_url) + path_parts = [part for part in parsed.path.split("/") if part] + if parsed.netloc.lower() == "raw.githubusercontent.com": + if len(path_parts) < 4 or path_parts[-1] != MANIFEST_FILE: + raise ValueError(f"Invalid raw GitHub manifest URL: {manifest_url}") + repo_path = "/" + "/".join(path_parts[:2]) + repo_base = urlunsplit((parsed.scheme, parsed.netloc, repo_path, "", "")) + return f"{repo_base}/{ref}/{file_path}" + return urljoin(manifest_url, file_path) + + def _sort_versions(versions): """Sort version strings newest-first using semantic-versioning precedence.""" return sorted(versions, key=Version, reverse=True) @@ -114,12 +134,14 @@ def available_versions(manifest, library_name=None, check_prerelease=False): return _versions_for_key(manifest, manifest_key, check_prerelease) -def all_version_infos(manifest, check_prerelease=True): +def all_version_infos(manifest, check_prerelease=True, manifest_url=MANIFEST_URL): """Return manifest entries in the shape used by the schema cache. Parameters: manifest (dict): A supported manifest (see :func:`is_supported`). check_prerelease (bool): If True, include prerelease versions. + manifest_url (str): Location of the manifest. Schema URLs use the same GitHub repository + or resolve relative to this URL for other hosts. Returns: dict: ``{library_name_or_None: {version: (sha, download_url, prerelease)}}``. @@ -137,7 +159,7 @@ def all_version_infos(manifest, check_prerelease=True): for entry in categories.get(category, []): version_infos[entry["version"]] = ( entry["sha"], - raw_url_for(entry["file"], ref), + content_url_for_manifest(entry["file"], ref, manifest_url), is_prerelease, ) if version_infos: diff --git a/tests/schema/test_hed_schema_io.py b/tests/schema/test_hed_schema_io.py index 7a785c86..24b84756 100644 --- a/tests/schema/test_hed_schema_io.py +++ b/tests/schema/test_hed_schema_io.py @@ -240,12 +240,12 @@ def test_cache_xml_versions_uses_manifest_for_default_urls(self): self.assertEqual(cache_one.call_count, 3) def test_cache_xml_versions_accepts_custom_manifest(self): - """A fork can use its own manifest for bulk caching.""" - custom_manifest_url = "https://example.test/schema_versions.json" + """A fork manifest should also select schema content from that fork.""" + custom_manifest_url = "https://raw.githubusercontent.com/example/hed-schemas/dev/schema_versions.json" with tempfile.TemporaryDirectory() as tmp_dir: with ( patch.object(hed_cache, "_get_json_with_etag", return_value=SAMPLE_CACHE_MANIFEST) as manifest_fetch, - patch.object(hed_cache, "_cache_hed_version", return_value="cached"), + patch.object(hed_cache, "_cache_hed_version", return_value="cached") as cache_one, ): result = hed_cache.cache_xml_versions( cache_folder=tmp_dir, @@ -254,6 +254,16 @@ def test_cache_xml_versions_accepts_custom_manifest(self): self.assertEqual(result, 0) self.assertEqual(manifest_fetch.call_args.args[0], custom_manifest_url) + download_urls = [call.args[2][1] for call in cache_one.call_args_list] + self.assertTrue(download_urls) + self.assertTrue( + all( + url.startswith( + f"https://raw.githubusercontent.com/example/hed-schemas/{SAMPLE_CACHE_MANIFEST['repo_commit']}/" + ) + for url in download_urls + ) + ) def test_cache_xml_versions_falls_back_when_manifest_is_unsupported(self): """An unsupported manifest must leave the REST bulk-cache path available.""" diff --git a/tests/schema/test_schema_version_manifest.py b/tests/schema/test_schema_version_manifest.py index 2da0102f..fc555119 100644 --- a/tests/schema/test_schema_version_manifest.py +++ b/tests/schema/test_schema_version_manifest.py @@ -144,6 +144,28 @@ def test_prerelease_can_be_excluded(self): self.assertNotIn("8.5.0", result[None]) self.assertNotIn("mouse", result) + def test_custom_github_manifest_uses_same_repository(self): + result = manifest.all_version_infos( + SAMPLE_MANIFEST, + manifest_url="https://raw.githubusercontent.com/example/hed-schemas/dev/schema_versions.json", + ) + + self.assertEqual( + result[None]["8.4.0"][1], + "https://raw.githubusercontent.com/example/hed-schemas/abc123def456/standard_schema/hedxml/HED8.4.0.xml", + ) + + def test_mirror_manifest_resolves_schema_files_relative_to_manifest(self): + result = manifest.all_version_infos( + SAMPLE_MANIFEST, + manifest_url="https://schemas.example.org/hed/schema_versions.json", + ) + + self.assertEqual( + result[None]["8.4.0"][1], + "https://schemas.example.org/hed/standard_schema/hedxml/HED8.4.0.xml", + ) + class TestFindVersionInfo(unittest.TestCase): def test_released_pins_to_repo_commit(self): @@ -193,6 +215,14 @@ def test_raw_url_for(self): "https://raw.githubusercontent.com/hed-standard/hed-schemas/deadbeef/standard_schema/hedxml/HED8.4.0.xml", ) + def test_content_url_rejects_malformed_raw_github_manifest_url(self): + with self.assertRaisesRegex(ValueError, "Invalid raw GitHub manifest URL"): + manifest.content_url_for_manifest( + "standard_schema/hedxml/HED8.4.0.xml", + "deadbeef", + "https://raw.githubusercontent.com/schema_versions.json", + ) + if __name__ == "__main__": unittest.main()