Merge pull request #128 from eScienceLab/develop

version 1.1 API

Author: douglowe

README.md

@@ -21,6 +21,7 @@ This project presents a Flask-based API for validating RO-Crates.
 |------------|-----------|-------------------------|-----------------------------------------------------------------------|
 | root_path | optional  | string                 | Root path which contains the RO-Crate  |
 | webhook_url | optional  | string                 | Webhook to send validation result to  |
+| profile_name | optional | string                 | RO-Crate profile to validate against  |
 | minio_config | required  | dictionary                 | MinIO Configuration Details    |
 
 `minio_config`
@@ -167,12 +168,24 @@ curl -X 'POST' \
 
 2. Create the `.env` file for shared environment information. An example environment file is included (`example.env`), which can be copied for this purpose. But make sure to change any security settings (username and passwords).
 
-3. Build and start the services using Docker Compose:
+3. A directory containing RO-Crate profiles to replace the default RO-Crate profiles for validation may be provided. Note that this will need to contain all profile files, as the default profile data will not be used. An example of this is given in the `docker-compose-develop.yml` file, and described here:
+   1. Store the profiles in a convenient directory, e.g.: `./local/rocrate_validator_profiles`
+   2. Add a volume to the celery worker container for these, e.g.:
+   ```
+    volumes:
+      - ./local/rocrate_validator_profiles:/app/profiles:ro
+   ```
+   3. Provide the `PROFILES_PATH` environment to the flask container (not the celery worker container) to match the internal path, e.g.:
+   ```
+      - PROFILES_PATH=/app/profiles
+   ```
+
+4. Build and start the services using Docker Compose:
     ```bash
    docker compose up --build
    ```
 
-4. Set up the MinIO bucket
+5. Set up the MinIO bucket
    1. Open the MinIO web interface at `http://localhost:9000`.  
    2. Log in with your MinIO credentials.  
    3. Create a new bucket named `ro-crates`.  

app/ro_crates/routes/post_routes.py

@@ -7,7 +7,7 @@
 from apiflask import APIBlueprint, Schema
 from apiflask.fields import String, Boolean
 from marshmallow.fields import Nested
-from flask import Response
+from flask import Response, current_app
 
 from app.services.validation_service import (
     queue_ro_crate_validation_task,
@@ -81,7 +81,10 @@ def validate_ro_crate_via_id(json_data, crate_id) -> tuple[Response, int]:
     else:
         profile_name = None
 
-    return queue_ro_crate_validation_task(minio_config, crate_id, root_path, profile_name, webhook_url)
+    profiles_path = current_app.config["PROFILES_PATH"]
+
+    return queue_ro_crate_validation_task(minio_config, crate_id, root_path, profile_name,
+                                          webhook_url, profiles_path)
 
 
 @post_routes_bp.post("/validate_metadata")
@@ -108,4 +111,6 @@ def validate_ro_crate_metadata(json_data) -> tuple[Response, int]:
     else:
         profile_name = None
 
-    return queue_ro_crate_metadata_validation_task(crate_json, profile_name)
+    profiles_path = current_app.config["PROFILES_PATH"]
+
+    return queue_ro_crate_metadata_validation_task(crate_json, profile_name, profiles_path=profiles_path)

app/services/validation_service.py

@@ -25,7 +25,8 @@
 
 
 def queue_ro_crate_validation_task(
-    minio_config, crate_id, root_path=None, profile_name=None, webhook_url=None
+    minio_config, crate_id, root_path=None, profile_name=None, webhook_url=None,
+    profiles_path=None
 ) -> tuple[Response, int]:
     """
     Queues an RO-Crate for validation with Celery.
@@ -51,22 +52,24 @@ def queue_ro_crate_validation_task(
         raise InvalidAPIUsage(f"No RO-Crate with prefix: {crate_id}", 400)
 
     try:
-        process_validation_task_by_id.delay(minio_config, crate_id, root_path, profile_name, webhook_url)
+        process_validation_task_by_id.delay(minio_config, crate_id, root_path,
+                                            profile_name, webhook_url, profiles_path)
         return jsonify({"message": "Validation in progress"}), 202
 
     except Exception as e:
         return jsonify({"error": str(e)}), 500
 
 
 def queue_ro_crate_metadata_validation_task(
-    crate_json: str, profile_name=None, webhook_url=None
+    crate_json: str, profile_name=None, webhook_url=None, profiles_path=None
 ) -> tuple[Response, int]:
     """
     Queues an RO-Crate for validation with Celery.
 
     :param crate_id: The ID of the RO-Crate to validate.
     :param profile_name: The profile to validate against.
     :param webhook_url: The URL to POST the validation results to.
+    :param profiles_path: A path to the profile definition directory.
     :return: A tuple containing a JSON response and an HTTP status code.
     :raises: Exception: If an error occurs whilst queueing the task.
     """
@@ -88,7 +91,8 @@ def queue_ro_crate_metadata_validation_task(
         result = process_validation_task_by_metadata.delay(
                                                      crate_json,
                                                      profile_name,
-                                                     webhook_url
+                                                     webhook_url,
+                                                     profiles_path
                 )
         if webhook_url:
             return jsonify({"message": "Validation in progress"}), 202

app/tasks/validation_tasks.py

@@ -7,6 +7,7 @@
 import logging
 import os
 import shutil
+import json
 from typing import Optional
 
 from rocrate_validator import services
@@ -22,14 +23,14 @@
     find_validation_object_on_minio
 )
 from app.utils.webhook_utils import send_webhook_notification
-from app.utils.file_utils import build_metadata_only_rocrate
 
 logger = logging.getLogger(__name__)
 
 
 @celery.task
 def process_validation_task_by_id(
-    minio_config: dict, crate_id: str, root_path: str, profile_name: str | None, webhook_url: str | None
+    minio_config: dict, crate_id: str, root_path: str, profile_name: str | None,
+    webhook_url: str | None, profiles_path: str | None
 ) -> None:
     """
     Background task to process the RO-Crate validation by ID.
@@ -56,7 +57,7 @@ def process_validation_task_by_id(
         logging.info(f"Processing validation task for {file_path}")
 
         # Perform validation:
-        validation_result = perform_ro_crate_validation(file_path, profile_name)
+        validation_result = perform_ro_crate_validation(file_path, profile_name, profiles_path=profiles_path)
 
         if isinstance(validation_result, str):
             logging.error(f"Validation failed: {validation_result}")
@@ -97,32 +98,27 @@ def process_validation_task_by_id(
 
 @celery.task
 def process_validation_task_by_metadata(
-    crate_json: str, profile_name: str | None, webhook_url: str | None
+    crate_json: str, profile_name: str | None, webhook_url: str | None, profiles_path: Optional[str] = None
 ) -> ValidationResult | str:
     """
     Background task to process the RO-Crate validation for a given json metadata string.
 
     :param crate_json: A string containing the RO-Crate JSON metadata to validate.
     :param profile_name: The name of the validation profile to use. Defaults to None.
     :param webhook_url: The webhook URL to send notifications to. Defaults to None.
+    :param profiles_path: The path to the profiles definition directory. Defaults to None.
     :raises Exception: If an error occurs during the validation process.
 
     :todo: Replace the Crate ID with a more comprehensive system, and replace profile name with URI.
     """
 
-    skip_checks_list = ['ro-crate-1.1_12.1']
-    file_path = None
-
     try:
-        # Fetch the RO-Crate from MinIO using the provided ID:
-        file_path = build_metadata_only_rocrate(crate_json)
-
-        logging.info(f"Processing validation task for {file_path}")
+        logging.info("Processing validation task for provided metadata string")
 
         # Perform validation:
-        validation_result = perform_ro_crate_validation(file_path,
+        validation_result = perform_metadata_validation(crate_json,
                                                         profile_name,
-                                                        skip_checks_list
+                                                        profiles_path
                                                         )
 
         if isinstance(validation_result, str):
@@ -131,9 +127,9 @@ def process_validation_task_by_metadata(
             raise Exception(f"Validation failed: {validation_result}")
 
         if not validation_result.has_issues():
-            logging.info(f"RO Crate {file_path} is valid.")
+            logging.info("RO Crate metadata is valid.")
         else:
-            logging.info(f"RO Crate {file_path} is invalid.")
+            logging.info("RO Crate metadata is invalid.")
 
         if webhook_url:
             send_webhook_notification(webhook_url, validation_result.to_json())
@@ -147,25 +143,22 @@ def process_validation_task_by_metadata(
             send_webhook_notification(webhook_url, error_data)
 
     finally:
-        # Clean up the temporary file if it was created:
-        if file_path and os.path.exists(file_path):
-            shutil.rmtree(file_path)
-
         if isinstance(validation_result, str):
             return validation_result
         else:
             return validation_result.to_json()
 
 
 def perform_ro_crate_validation(
-    file_path: str, profile_name: str | None, skip_checks_list: Optional[list] = None
+    file_path: str, profile_name: str | None, skip_checks_list: Optional[list] = None, profiles_path: Optional[str] = None
 ) -> ValidationResult | str:
     """
     Validates an RO-Crate using the provided file path and profile name.
 
     :param file_path: The path to the RO-Crate file to validate
     :param profile_name: The name of the validation profile to use. Defaults to None. If None, the CRS4 validator will
         attempt to determine the profile.
+    :param profiles_path: The path to the profiles definition directory
     :param skip_checks_list: A list of checks to skip, if needed
     :return: The validation result.
     :raises Exception: If an error occurs during the validation process.
@@ -183,7 +176,41 @@ def perform_ro_crate_validation(
         settings = services.ValidationSettings(
             rocrate_uri=full_file_path,
             **({"profile_identifier": profile_name} if profile_name else {}),
-            **({"skip_checks": skip_checks_list} if skip_checks_list else {})
+            **({"skip_checks": skip_checks_list} if skip_checks_list else {}),
+            **({"profiles_path": profiles_path} if profiles_path else {})
+        )
+
+        return services.validate(settings)
+
+    except Exception as e:
+        logging.error(f"Unexpected error during validation: {e}")
+        return str(e)
+
+
+def perform_metadata_validation(
+    crate_json: str, profile_name: str | None, skip_checks_list: Optional[list] = None, profiles_path: Optional[str] = None
+) -> ValidationResult | str:
+    """
+    Validates only RO-Crate metadata provided as a json string.
+
+    :param crate_json: The JSON string containing the metadata
+    :param profile_name: The name of the validation profile to use. Defaults to None. If None, the CRS4 validator will
+        attempt to determine the profile.
+    :param profiles_path: The path to the profiles definition directory
+    :param skip_checks_list: A list of checks to skip, if needed
+    :return: The validation result.
+    :raises Exception: If an error occurs during the validation process.
+    """
+
+    try:
+        logging.info(f"Validating ro-crate metadata with profile {profile_name}")
+
+        settings = services.ValidationSettings(
+            **({"metadata_only": True}),
+            **({"metadata_dict": json.loads(crate_json)}),
+            **({"profile_identifier": profile_name} if profile_name else {}),
+            **({"skip_checks": skip_checks_list} if skip_checks_list else {}),
+            **({"profiles_path": profiles_path} if profiles_path else {})
         )
 
         return services.validate(settings)

app/utils/config.py

@@ -10,34 +10,32 @@
 from flask import Flask
 
 
+def get_env(name: str, default=None, required=False):
+    value = os.environ.get(name, default)
+    if required and value is None:
+        raise RuntimeError(f"Missing required environment variable: {name}")
+    return value
+
+
 class Config:
     """Base configuration class for the Flask application."""
 
-    SECRET_KEY = os.getenv("SECRET_KEY", "my_precious")
-
     # Celery configuration:
-    CELERY_BROKER_URL = os.getenv("CELERY_BROKER_URL")
-    CELERY_RESULT_BACKEND = os.getenv("CELERY_RESULT_BACKEND")
+    CELERY_BROKER_URL = get_env("CELERY_BROKER_URL", required=False)
+    CELERY_RESULT_BACKEND = get_env("CELERY_RESULT_BACKEND", required=False)
 
-    # MinIO configuration:
-    MINIO_ENDPOINT = os.getenv("MINIO_ENDPOINT")
-    MINIO_ACCESS_KEY = os.getenv("MINIO_ACCESS_KEY")
-    MINIO_SECRET_KEY = os.getenv("MINIO_SECRET_KEY")
-    MINIO_BUCKET_NAME = os.getenv("MINIO_BUCKET_NAME", "bucket-name")
+    # rocrate validator configuration:
+    PROFILES_PATH = get_env("PROFILES_PATH", required=False)
 
 
 class DevelopmentConfig(Config):
     """Development configuration class."""
-
     DEBUG = True
-    ENV = "development"
 
 
 class ProductionConfig(Config):
     """Production configuration class."""
-
     DEBUG = False
-    ENV = "production"
 
 
 class InvalidAPIUsage(Exception):
@@ -63,10 +61,13 @@ def make_celery(app: Flask = None) -> Celery:
     :param app: The Flask application to use.
     :return: The Celery instance.
     """
+    env = os.environ.get("FLASK_ENV", "development")
+    config_cls = ProductionConfig if env == "production" else DevelopmentConfig
+
     celery = Celery(
         app.import_name if app else __name__,
-        broker=os.getenv("CELERY_BROKER_URL"),
-        backend=os.getenv("CELERY_RESULT_BACKEND"),
+        broker=config_cls.CELERY_BROKER_URL,
+        backend=config_cls.CELERY_RESULT_BACKEND,
     )
 
     if app: