From 702e21eb81ffff61fff997c7180e186196fb8b7d Mon Sep 17 00:00:00 2001 From: Ornella33 Date: Tue, 12 May 2026 23:53:24 +0200 Subject: [PATCH 1/3] Fix Discovery persistence and lookup handling Before, the Discovery service tried to persist its in-memory data structure directly. This could corrupt the storage file discovery_store.json and prevented reliable lookup after restart. This change persists only the AAS-to-asset-ID mapping in a JSON-safe format, rebuilds the reverse lookup index on load, creates the storage file when persistent mode is enabled, and fixes lookup responses to pass paging metadata correctly. --- .gitignore | 1 + server/app/interfaces/discovery.py | 61 ++++++++++++++----- server/app/services/run_discovery.py | 18 +++++- .../discovery_standalone/README.md | 50 ++++++++------- .../discovery_standalone/compose.yml | 8 +-- 5 files changed, 92 insertions(+), 46 deletions(-) diff --git a/.gitignore b/.gitignore index dab9383e..1efa31ac 100644 --- a/.gitignore +++ b/.gitignore @@ -36,3 +36,4 @@ server/example_configurations/repository_standalone/input/ server/example_configurations/repository_standalone/storage/ server/example_configurations/registry_standalone/input/ server/example_configurations/registry_standalone/storage/ +server/example_configurations/discovery_standalone/storage/ diff --git a/server/app/interfaces/discovery.py b/server/app/interfaces/discovery.py index a6e43361..a0b2f31f 100644 --- a/server/app/interfaces/discovery.py +++ b/server/app/interfaces/discovery.py @@ -5,6 +5,7 @@ """ import json +import os from typing import Dict, List, Set, Type import werkzeug.exceptions @@ -65,28 +66,58 @@ def _delete_aas_id_from_specific_asset_ids(self, asset_id: model.SpecificAssetId @classmethod def from_file(cls, filename: str) -> "DiscoveryStore": """ - Load the state of the `DiscoveryStore` from a local file. - Safely handles files that are missing expected keys. + Load a persisted discovery store from JSON. + The file stores only the AAS-to-asset-id mapping as the source of truth. + While loading, the reverse asset-id-to-AAS index is rebuilt in memory so + lookup by asset ID works without persisting duplicate state. """ with open(filename, "r") as file: data = json.load(file, cls=jsonization.ServerAASFromJsonDecoder) - discovery_store = DiscoveryStore() - discovery_store.aas_id_to_asset_ids = data.get("aas_id_to_asset_ids", {}) - discovery_store.asset_id_to_aas_ids = data.get("asset_id_to_aas_ids", {}) - return discovery_store + + discovery_store = DiscoveryStore() + + for aas_id, asset_ids in data.get("aas_id_to_asset_ids", {}).items(): + parsed_asset_ids = set() + + for asset_id in asset_ids: + if isinstance(asset_id, model.SpecificAssetId): + parsed_asset_id = asset_id + else: + parsed_asset_id = model.SpecificAssetId( + name=asset_id["name"], + value=asset_id["value"], + ) + + parsed_asset_ids.add(parsed_asset_id) + discovery_store._add_aas_id_to_specific_asset_id(parsed_asset_id, aas_id) + + discovery_store.aas_id_to_asset_ids[aas_id] = parsed_asset_ids + + return discovery_store def to_file(self, filename: str) -> None: """ - Write the current state of the `DiscoveryStore` to a local JSON file for persistence. + Persist the discovery store as JSON. + + Only the AAS-to-asset-id mapping is written because the reverse lookup + index can be rebuilt when the store is loaded. The data is written to a + temporary file first and then atomically moved into place to avoid + corrupting the existing store if serialization fails. """ - with open(filename, "w") as file: - data = { - "aas_id_to_asset_ids": self.aas_id_to_asset_ids, - "asset_id_to_aas_ids": self.asset_id_to_aas_ids, + data = { + "aas_id_to_asset_ids": { + aas_id: list(asset_ids) + for aas_id, asset_ids in self.aas_id_to_asset_ids.items() } + } + + temp_filename = f"{filename}.tmp" + with open(temp_filename, "w") as file: json.dump(data, file, cls=jsonization.ServerAASToJsonEncoder, indent=4) + os.replace(temp_filename, filename) + class DiscoveryAPI(BaseWSGIApp): def __init__(self, persistent_store: DiscoveryStore, base_path: str = "/api/v3.1"): @@ -160,8 +191,8 @@ def get_all_aas_ids_by_asset_link( aas_keys = self.persistent_store.search_aas_ids_by_asset_link(asset_link) matching_aas_keys.update(aas_keys) - paginated_slice, cursor = self._get_slice(request, list(matching_aas_keys)) - return response_t(list(paginated_slice), cursor=cursor) + paginated_slice, paging_metadata = self._get_slice(request, list(matching_aas_keys)) + return response_t(list(paginated_slice), paging_metadata=paging_metadata) def search_all_aas_ids_by_asset_link( self, request: Request, url_args: dict, response_t: Type[APIResponse], **_kwargs @@ -171,8 +202,8 @@ def search_all_aas_ids_by_asset_link( for asset_link in asset_links: aas_keys = self.persistent_store.search_aas_ids_by_asset_link(asset_link) matching_aas_keys.update(aas_keys) - paginated_slice, cursor = self._get_slice(request, list(matching_aas_keys)) - return response_t(list(paginated_slice), cursor=cursor) + paginated_slice, paging_metadata = self._get_slice(request, list(matching_aas_keys)) + return response_t(list(paginated_slice), paging_metadata=paging_metadata) def get_all_specific_asset_ids_by_aas_id( self, request: Request, url_args: dict, response_t: Type[APIResponse], **_kwargs diff --git a/server/app/services/run_discovery.py b/server/app/services/run_discovery.py index 7c47124c..5da8d464 100644 --- a/server/app/services/run_discovery.py +++ b/server/app/services/run_discovery.py @@ -12,15 +12,31 @@ wsgi_optparams["base_path"] = base_path +def load_discovery_store(path: str) -> DiscoveryStore: + storage_dir = os.path.dirname(path) + if storage_dir: + os.makedirs(storage_dir, exist_ok=True) + + if os.path.exists(path) and os.path.getsize(path) > 0: + return DiscoveryStore.from_file(path) + + discovery_store = DiscoveryStore() + discovery_store.to_file(path) + return discovery_store + + # Load DiscoveryStore from disk, if `storage_path` is set if storage_path: - discovery_store: DiscoveryStore = DiscoveryStore.from_file(storage_path) + discovery_store: DiscoveryStore = load_discovery_store(storage_path) else: discovery_store = DiscoveryStore() def persist_store(): if storage_path: + storage_dir = os.path.dirname(storage_path) + if storage_dir: + os.makedirs(storage_dir, exist_ok=True) discovery_store.to_file(storage_path) diff --git a/server/example_configurations/discovery_standalone/README.md b/server/example_configurations/discovery_standalone/README.md index 45dfde79..7bc2f132 100644 --- a/server/example_configurations/discovery_standalone/README.md +++ b/server/example_configurations/discovery_standalone/README.md @@ -1,7 +1,7 @@ # Eclipse BaSyx Python SDK - Discovery Service This is a Python-based implementation of the **BaSyx Asset Administration Shell (AAS) Discovery Service**. -It provides basic discovery functionality for AAS IDs and their corresponding assets, as specified in the official [Discovery Service Specification v3.1.0_SSP-001](https://app.swaggerhub.com/apis/Plattform_i40/DiscoveryServiceSpecification/V3.1.0_SSP-001). +It provides basic discovery functionality for AAS IDs and their corresponding assets, as specified in the official [Discovery Service Specification v3.1.1_SSP-001](https://app.swaggerhub.com/apis/Plattform_i40/DiscoveryServiceSpecification/V3.1.1_SSP-001). ## Overview @@ -11,38 +11,36 @@ The Discovery Service stores and retrieves relations between AAS identifiers and | Function | Description | Example URL | |------------------------------------------|----------------------------------------------------------|-----------------------------------------------------------------------| -| **search_all_aas_ids_by_asset_link** | Find AAS identifiers by providing asset link values | `POST http://localhost:8084/api/v3.0/lookup/shellsByAssetLink` | -| **get_all_specific_asset_ids_by_aas_id** | Return specific asset ids associated with an AAS ID | `GET http://localhost:8084/api/v3.0/lookup/shells/{aasIdentifier}` | -| **post_all_asset_links_by_id** | Register specific asset ids linked to an AAS | `POST http://localhost:8084/api/v3.0/lookup/shells/{aasIdentifier}` | -| **delete_all_asset_links_by_id** | Delete all asset links associated with a specific AAS ID | `DELETE http://localhost:8084/api/v3.0/lookup/shells/{aasIdentifier}` | -| +| **get_description** | Return the supported Discovery Service profiles | `GET http://localhost:8084/api/v3.1/description` | +| **get_all_aas_ids_by_asset_link** | Find AAS identifiers by asset link query parameter | `GET http://localhost:8084/api/v3.1/lookup/shells?assetIds={assetIds}` | +| **search_all_aas_ids_by_asset_link** | Find AAS identifiers by providing asset link values | `POST http://localhost:8084/api/v3.1/lookup/shellsByAssetLink` | +| **get_all_specific_asset_ids_by_aas_id** | Return specific asset ids associated with an AAS ID | `GET http://localhost:8084/api/v3.1/lookup/shells/{aasIdentifier}` | +| **post_all_asset_links_by_id** | Register specific asset ids linked to an AAS | `POST http://localhost:8084/api/v3.1/lookup/shells/{aasIdentifier}` | +| **delete_all_asset_links_by_id** | Delete all asset links associated with a specific AAS ID | `DELETE http://localhost:8084/api/v3.1/lookup/shells/{aasIdentifier}` | + ## Configuration -Add discovery_store as directory -The service can be configured to use either: +This example Docker compose configuration starts a discovery server. + +The container image can also be built and run via: +``` +$ docker compose up +``` -- **In-memory storage** (default): Temporary data storage that resets on service restart. -- **MongoDB storage**: Persistent backend storage using MongoDB. +## Persistence -### Configuration via Environment Variables +The discovery service can run in persistent or non-persistent mode. -| Variable | Description | Default | -|------------------|--------------------------------------------|-----------------------------| -| `STORAGE_TYPE` | `inmemory` or `mongodb` | `inmemory` | -| `MONGODB_URI` | MongoDB connection URI | `mongodb://localhost:27017` | -| `MONGODB_DBNAME` | Name of the MongoDB database | `basyx_registry` | +### Persistent Mode -## Deployment via Docker +Persistent mode configuration is provided in the `compose.yaml`. -A `Dockerfile` and `docker-compose.yml` are provided for simple deployment. -The container image can be built and run via: -```bash -docker compose up --build -``` -## Test +Only the AAS-to-asset-ID mapping is persisted. The reverse lookup index is rebuilt in memory when the service starts. + +### Non-Persistent Mode -Examples of asset links and specific asset IDs for testing purposes are provided as JSON files in the [storage](./storage) folder. +If `storage_path` is not set, the discovery service runs in memory only. -## Acknowledgments +## Notes +- Stop the service before manually editing `discovery_store.json`. -This Dockerfile is inspired by the [tiangolo/uwsgi-nginx-docker](https://github.com/tiangolo/uwsgi-nginx-docker) repository. diff --git a/server/example_configurations/discovery_standalone/compose.yml b/server/example_configurations/discovery_standalone/compose.yml index 27b9309e..0aea1510 100644 --- a/server/example_configurations/discovery_standalone/compose.yml +++ b/server/example_configurations/discovery_standalone/compose.yml @@ -6,7 +6,7 @@ services: dockerfile: server/docker/discovery/Dockerfile ports: - "8084:80" - #environment: - #- storage_path=/discovery_store.json - #volumes: - # - ./discovery_store.json:/discovery_store.json + environment: + storage_path: /storage/discovery_store.json + volumes: + - ./storage:/storage From c3185a09f649722fdbc2d023a20196162c05c0c2 Mon Sep 17 00:00:00 2001 From: s-heppner Date: Wed, 13 May 2026 09:02:52 +0200 Subject: [PATCH 2/3] Update server/app/interfaces/discovery.py --- server/app/interfaces/discovery.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/app/interfaces/discovery.py b/server/app/interfaces/discovery.py index a0b2f31f..a340a0e1 100644 --- a/server/app/interfaces/discovery.py +++ b/server/app/interfaces/discovery.py @@ -68,7 +68,7 @@ def from_file(cls, filename: str) -> "DiscoveryStore": """ Load a persisted discovery store from JSON. - The file stores only the AAS-to-asset-id mapping as the source of truth. + The file stores the AAS-to-asset-id mapping as the source of truth. While loading, the reverse asset-id-to-AAS index is rebuilt in memory so lookup by asset ID works without persisting duplicate state. """ From aa35c4be53a6ac617dc054466af66506e453b741 Mon Sep 17 00:00:00 2001 From: s-heppner Date: Wed, 13 May 2026 09:16:57 +0200 Subject: [PATCH 3/3] Simplify run_discovery storage persistence --- server/app/services/run_discovery.py | 27 ++++++++++----------------- 1 file changed, 10 insertions(+), 17 deletions(-) diff --git a/server/app/services/run_discovery.py b/server/app/services/run_discovery.py index 5da8d464..deead79a 100644 --- a/server/app/services/run_discovery.py +++ b/server/app/services/run_discovery.py @@ -8,35 +8,28 @@ wsgi_optparams = {} + if base_path is not None: wsgi_optparams["base_path"] = base_path -def load_discovery_store(path: str) -> DiscoveryStore: - storage_dir = os.path.dirname(path) - if storage_dir: - os.makedirs(storage_dir, exist_ok=True) - - if os.path.exists(path) and os.path.getsize(path) > 0: - return DiscoveryStore.from_file(path) - - discovery_store = DiscoveryStore() - discovery_store.to_file(path) - return discovery_store - - # Load DiscoveryStore from disk, if `storage_path` is set if storage_path: - discovery_store: DiscoveryStore = load_discovery_store(storage_path) + # If the storage file exists, we load it + if os.path.exists(storage_path) and os.path.getsize(storage_path) > 0: + discovery_store = DiscoveryStore.from_file(storage_path) + + # If the file doesn't exist at the given path, we initiate a new file + else: + os.makedirs(os.path.dirname(storage_path), exist_ok=True) + discovery_store = DiscoveryStore() + discovery_store.to_file(storage_path) else: discovery_store = DiscoveryStore() def persist_store(): if storage_path: - storage_dir = os.path.dirname(storage_path) - if storage_dir: - os.makedirs(storage_dir, exist_ok=True) discovery_store.to_file(storage_path)