feat: Milestone 2 — Structured Run Manifest System (#64)

.gitignore

@@ -31,6 +31,12 @@ share/python-wheels/
 *.egg
 MANIFEST
 
+# Windows-specific: MANIFEST above matches databusclient/manifest/ case-insensitively
+# on Windows, preventing the manifest module from being committed.
+!databusclient/manifest/
+!databusclient/manifest/**
+databusclient/manifest/__pycache__/
+
 # PyInstaller
 #  Usually these files are written by a python script from a template
 #  before PyInstaller builds the exe, so as to inject date/other infos into it.

README.md

@@ -18,6 +18,7 @@ Command-line and Python client for downloading and deploying datasets on DBpedia
   - [Download](#cli-download)
   - [Deploy](#cli-deploy)
   - [Delete](#cli-delete)
+  - [Manifest](#cli-manifest)
 - [Module Usage](#module-usage)
   - [Deploy](#module-deploy)
 - [Development & Contributing](#development--contributing)
@@ -556,6 +557,55 @@ databusclient delete https://databus.dbpedia.org/dbpedia/collections/dbpedia-sna
 docker run --rm -v $(pwd):/data dbpedia/databus-python-client delete https://databus.dbpedia.org/dbpedia/collections/dbpedia-snapshot-2022-12 --databus-key YOUR_API_KEY
 ```
 
+<a id="cli-manifest"></a>
+### Manifest
+
+All three commands support an optional `--manifest` flag that writes a structured JSON-LD record of the operation to disk:
+
+**Download**
+```bash
+# Python
+databusclient download https://databus.dbpedia.org/dbpedia/mappings/mappingbased-literals/2022.12.01/mappingbased-literals_lang=az.ttl.bz2 --manifest ./manifests/download-run.jsonld
+# Docker
+docker run --rm -v $(pwd):/data dbpedia/databus-python-client download https://databus.dbpedia.org/dbpedia/mappings/mappingbased-literals/2022.12.01/mappingbased-literals_lang=az.ttl.bz2 --manifest ./manifests/download-run.jsonld
+```
+
+**Deploy**
+```bash
+# Python
+databusclient deploy \
+  --version-id https://databus.dbpedia.org/user1/group1/artifact1/2022-05-18 \
+  --title "Client Testing" --abstract "Testing the client...." \
+  --description "Testing the client...." \
+  --license http://dalicc.net/licenselibrary/AdaptivePublicLicense10 \
+  --apikey YOUR_KEY --manifest ./manifests/deploy-run.jsonld \
+  'https://raw.githubusercontent.com/dbpedia/databus/master/server/app/api/swagger.yml|type=swagger'
+# Docker
+docker run --rm -v $(pwd):/data dbpedia/databus-python-client deploy \
+  --version-id https://databus.dbpedia.org/user1/group1/artifact1/2022-05-18 \
+  --title "Client Testing" --abstract "Testing the client...." \
+  --description "Testing the client...." \
+  --license http://dalicc.net/licenselibrary/AdaptivePublicLicense10 \
+  --apikey YOUR_KEY --manifest ./manifests/deploy-run.jsonld \
+  'https://raw.githubusercontent.com/dbpedia/databus/master/server/app/api/swagger.yml|type=swagger'
+```
+**Delete**
+```bash
+# Python
+databusclient delete https://databus.dbpedia.org/dbpedia/mappings/mappingbased-literals/2022.12.01 --databus-key YOUR_API_KEY --manifest ./manifests/delete-run.jsonld
+# Docker
+docker run --rm -v $(pwd):/data dbpedia/databus-python-client delete https://databus.dbpedia.org/dbpedia/mappings/mappingbased-literals/2022.12.01 --databus-key YOUR_API_KEY --manifest ./manifests/delete-run.jsonld
+```
+
+The manifest records input parameters, per-file URLs, checksums, byte sizes, timestamps, and success/failure status for each file. It uses the DataID vocabulary and is versioned via `dbus:schemaVersion`.
+
+- If the target path already exists, the manifest is written to an auto-suffixed path (e.g. `run_1.jsonld`) with a warning.
+- Sensitive fields (API keys, vault tokens) are never written.
+- If manifest writing fails, a warning is printed and the exit code reflects the actual operation result.
+- If the operation itself fails, a `dbus:operationError` block is recorded in the manifest capturing the error type, message, and traceback.
+
+Refer [examples/reproducible-download.md](examples/reproducible-download.md) for a full walkthrough.
+
 ## Module Usage
 
 <a id="module-deploy"></a>
@@ -675,4 +725,4 @@ Or to ensure compatibility with the `pyproject.toml` configured dependencies, ru
 
 ```bash
 poetry run pytest tests/
-```
+```

databusclient/api/delete.py

@@ -23,13 +23,16 @@ class DeleteQueue:
     Allows adding multiple databus URIs to a queue and executing their deletion in batch.
     """
 
-    def __init__(self, databus_key: str):
+    def __init__(self, databus_key: str, manifest_context=None):
         """Create a DeleteQueue bound to a given Databus API key.
 
         Args:
             databus_key: API key used to authenticate deletion requests.
+            manifest_context: Optional ManifestContext to record deletion
+                outcomes into. Passed through to _delete_list on execute().
         """
         self.databus_key = databus_key
+        self.manifest_context = manifest_context
         self.queue: set[str] = set()
 
     def add_uri(self, databusURI: str):
@@ -69,11 +72,13 @@ def execute(self):
         """Execute all queued deletions.
 
         Each queued URI will be deleted using `_delete_resource`.
+        Passes manifest_context through so deletions are recorded.
         """
         _delete_list(
             list(self.sorted_queue()),
             self.databus_key,
             force=True,
+            manifest_context=self.manifest_context,
         )
 
 
@@ -116,6 +121,7 @@ def _delete_resource(
     dry_run: bool = False,
     force: bool = False,
     queue: DeleteQueue = None,
+    manifest_context=None,
 ):
     """Delete a single Databus resource (version, artifact, group).
 
@@ -144,6 +150,8 @@ def _delete_resource(
 
     if dry_run:
         print(f"[DRY RUN] Would delete: {databusURI}")
+        if manifest_context is not None:
+            manifest_context.record_file(url=databusURI, status="dry_run")
         return
 
     if queue is not None:
@@ -156,6 +164,8 @@ def _delete_resource(
 
     if response.status_code in (200, 204):
         print(f"Successfully deleted: {databusURI}")
+        if manifest_context is not None:
+            manifest_context.record_file(url=databusURI, status="success")
     else:
         raise Exception(
             f"Failed to delete {databusURI}: {response.status_code} - {response.text}"
@@ -168,6 +178,7 @@ def _delete_list(
     dry_run: bool = False,
     force: bool = False,
     queue: DeleteQueue = None,
+    manifest_context=None,
 ):
     """Delete a list of Databus resources.
 
@@ -180,7 +191,7 @@ def _delete_list(
     """
     for databusURI in databusURIs:
         _delete_resource(
-            databusURI, databus_key, dry_run=dry_run, force=force, queue=queue
+            databusURI, databus_key, dry_run=dry_run, force=force, queue=queue, manifest_context=manifest_context
         )
 
 
@@ -190,6 +201,7 @@ def _delete_artifact(
     dry_run: bool = False,
     force: bool = False,
     queue: DeleteQueue = None,
+    manifest_context=None,
 ):
     """Delete an artifact and all its versions.
 
@@ -223,11 +235,11 @@ def _delete_artifact(
         else:
             # Delete all versions
             _delete_list(
-                version_uris, databus_key, dry_run=dry_run, force=force, queue=queue
+                version_uris, databus_key, dry_run=dry_run, force=force, queue=queue, manifest_context=manifest_context
             )
 
     # Finally, delete the artifact itself
-    _delete_resource(databusURI, databus_key, dry_run=dry_run, force=force, queue=queue)
+    _delete_resource(databusURI, databus_key, dry_run=dry_run, force=force, queue=queue,manifest_context=manifest_context)
 
 
 def _delete_group(
@@ -236,6 +248,7 @@ def _delete_group(
     dry_run: bool = False,
     force: bool = False,
     queue: DeleteQueue = None,
+    manifest_context=None,
 ):
     """Delete a group and all its artifacts and versions.
 
@@ -266,14 +279,14 @@ def _delete_group(
     # Delete all artifacts (which deletes their versions)
     for artifact_uri in artifact_uris:
         _delete_artifact(
-            artifact_uri, databus_key, dry_run=dry_run, force=force, queue=queue
+            artifact_uri, databus_key, dry_run=dry_run, force=force, queue=queue, manifest_context=manifest_context
         )
 
     # Finally, delete the group itself
-    _delete_resource(databusURI, databus_key, dry_run=dry_run, force=force, queue=queue)
+    _delete_resource(databusURI, databus_key, dry_run=dry_run, force=force, queue=queue,manifest_context=manifest_context)
 
 
-def delete(databusURIs: List[str], databus_key: str, dry_run: bool, force: bool):
+def delete(databusURIs: List[str], databus_key: str, dry_run: bool, force: bool, manifest_context=None):
     """Delete a dataset from the databus.
 
     Delete a group, artifact, or version identified by the given databus URI.
@@ -286,7 +299,7 @@ def delete(databusURIs: List[str], databus_key: str, dry_run: bool, force: bool)
         force: If True, skip confirmation prompt and proceed with deletion.
     """
 
-    queue = DeleteQueue(databus_key)
+    queue = DeleteQueue(databus_key, manifest_context=manifest_context)
 
     for databusURI in databusURIs:
         _host, _account, group, artifact, version, file = (
@@ -296,24 +309,24 @@ def delete(databusURIs: List[str], databus_key: str, dry_run: bool, force: bool)
         if group == "collections" and artifact is not None:
             print(f"Deleting collection: {databusURI}")
             _delete_resource(
-                databusURI, databus_key, dry_run=dry_run, force=force, queue=queue
+                databusURI, databus_key, dry_run=dry_run, force=force, queue=queue, manifest_context=manifest_context
             )
         elif file is not None:
             print(f"Deleting file is not supported via API: {databusURI}")
         elif version is not None:
             print(f"Deleting version: {databusURI}")
             _delete_resource(
-                databusURI, databus_key, dry_run=dry_run, force=force, queue=queue
+                databusURI, databus_key, dry_run=dry_run, force=force, queue=queue, manifest_context=manifest_context
             )
         elif artifact is not None:
             print(f"Deleting artifact and all its versions: {databusURI}")
             _delete_artifact(
-                databusURI, databus_key, dry_run=dry_run, force=force, queue=queue
+                databusURI, databus_key, dry_run=dry_run, force=force, queue=queue, manifest_context=manifest_context
             )
         elif group is not None and group != "collections":
             print(f"Deleting group and all its artifacts and versions: {databusURI}")
             _delete_group(
-                databusURI, databus_key, dry_run=dry_run, force=force, queue=queue
+                databusURI, databus_key, dry_run=dry_run, force=force, queue=queue, manifest_context=manifest_context
             )
         else:
             print(f"Deleting {databusURI} is not supported.")