Production-readiness refactor: drop MinIO, make Crate IDs more robust, add async validation and offline cache

.github/workflows/build-with-profiles.yml

@@ -39,8 +39,13 @@ jobs:
         uses: docker/build-push-action@v6
         with:
           context: .
-          file: ./Dockerfile.fivesafes-profile
           push: true
+          # Add the five-safes profile (via EXTRA_PROFILES_PATH) and warm its
+          # cache, using the shared Dockerfile's build args.
+          build-args: |
+            FIVE_SAFES_PROFILE_VERSION=five-safes-0.7.4-beta
+            PROFILES_ARCHIVE_URL=https://github.com/eScienceLab/rocrate-validator/archive/refs/tags/five-safes-0.7.4-beta.tar.gz
+            EXTRA_PROFILES_PATH=/app/extra-profiles
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
 

.github/workflows/lint.yml

@@ -0,0 +1,29 @@
+name: Lint
+
+on:
+  pull_request:
+    branches: [ develop ]
+
+jobs:
+  ruff:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+
+      - name: Install ruff
+        run: |
+          python -m pip install --upgrade pip
+          pip install ruff
+
+      - name: Lint
+        run: ruff check .
+
+      - name: Format check
+        run: ruff format --check .

.github/workflows/test_docker.yml

@@ -20,16 +20,15 @@ jobs:
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install pytest requests minio docker
+          pip install pytest requests boto3
 
-      - name: Build Docker Compose Containers
+      - name: Run integration tests (brings up the compose stack)
         run: |
           cp example.env .env
-          docker compose -f docker-compose-develop.yml build 
+          pytest -s -v tests/test_integration.py
 
-      - name: Spin Up Docker Compose and Run Tests
-        run: pytest -s -v tests/test_integration.py
-
-      - name: Ensure that Docker Compose is Shutdown
+      - name: Ensure Docker Compose is shut down
         if: always()
-        run: docker compose down
+        run: >
+          docker compose -f docker-compose-develop.yml -p cratey_integration
+          --profile objectstore down -v || true

.github/workflows/unit_tests.yml

@@ -20,8 +20,7 @@ jobs:
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install -r requirements.txt
-          pip install pytest pytest-mock
+          pip install -r requirements-dev.txt
 
       - name: Run tests (excluding integration tests)
         run: |

Dockerfile

@@ -1,17 +1,46 @@
 FROM python:3.11-slim
 
-# Install required system packages, including git
-RUN apt-get update && apt-get install -y git && rm -rf /var/lib/apt/lists/*
+# git is needed by some dependencies; wget is only used when baking a profile.
+RUN apt-get update && apt-get install -y git wget && rm -rf /var/lib/apt/lists/*
 
 WORKDIR /app
 
 COPY requirements.txt .
 RUN pip install --upgrade pip
 RUN pip install --no-cache-dir -r requirements.txt
 
-COPY cratey.py LICENSE /app/
+COPY wsgi.py LICENSE /app/
 COPY app /app/app
 
+# Optionally fetch an extra RO-Crate profile into a normal directory. It is
+# *added* to the bundled profiles at runtime via EXTRA_PROFILES_PATH.
+# A plain build leaves PROFILES_ARCHIVE_URL empty
+# and skips this; the "with profiles" image build passes it as --build-arg.
+ARG PROFILES_ARCHIVE_URL=""
+ARG FIVE_SAFES_PROFILE_VERSION=""
+# Set EXTRA_PROFILES_PATH only for the profiles build (passed as a build arg).
+ARG EXTRA_PROFILES_PATH=""
+ENV EXTRA_PROFILES_PATH=${EXTRA_PROFILES_PATH}
+ENV CACHE_PATH=/app/.rocrate-cache
+RUN if [ -n "$PROFILES_ARCHIVE_URL" ]; then \
+        mkdir -p /app/extra-profiles && \
+        wget -O /tmp/profiles.tar.gz "$PROFILES_ARCHIVE_URL" && \
+        tar -xzf /tmp/profiles.tar.gz \
+            -C /app/extra-profiles \
+            --strip-components=3 \
+            "rocrate-validator-${FIVE_SAFES_PROFILE_VERSION}/rocrate_validator/profiles/five-safes-crate" && \
+        rm /tmp/profiles.tar.gz ; \
+    fi
+
+# Pre-populate the HTTP cache so opt-in offline validation
+# (VALIDATION_OFFLINE=true) works without network at runtime.
+RUN if [ -n "$EXTRA_PROFILES_PATH" ]; then \
+        rocrate-validator cache warm --all-profiles \
+            --extra-profiles-path "$EXTRA_PROFILES_PATH" --cache-path "$CACHE_PATH" ; \
+    else \
+        rocrate-validator cache warm --all-profiles --cache-path "$CACHE_PATH" ; \
+    fi
+
 RUN useradd -ms /bin/bash flaskuser
 RUN chown -R flaskuser:flaskuser /app
 
@@ -21,4 +50,5 @@ EXPOSE 5000
 
 CMD ["flask", "run", "--host=0.0.0.0"]
 
-LABEL org.opencontainers.image.source="https://github.com/eScienceLab/Cratey-Validator"
+LABEL org.opencontainers.image.source="https://github.com/eScienceLab/RO-Crate-Validation-Service"
+LABEL org.ro-crate-validation-service.five-safes-profile-version="${FIVE_SAFES_PROFILE_VERSION}"