test: set up itk nightly runs

.github/workflows/nightly.yaml

@@ -0,0 +1,38 @@
+name: Nightly ITK
+
+on:
+  schedule:
+    - cron: '0 2 * * *'  # 2:00 AM UTC daily
+  workflow_dispatch:     # Allow manual execution
+
+permissions:
+  contents: write 
+
+jobs:
+  nightly:
+    name: Nightly ITK Run
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v6
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v7
+
+      - name: Run Nightly ITK Tests
+        run: bash run_itk.sh
+        working-directory: itk
+        env:
+          A2A_SAMPLES_REVISION: itk-v.021-alpha
+          ITK_NIGHTLY_RUN: "True"
+
+      - name: Upload Results to Rolling Release
+        uses: softprops/action-gh-release@v2
+        with:
+          tag_name: "nightly-metrics"
+          prerelease: true
+          files: |
+            itk/itk_python.json
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

itk/process_results.py

@@ -0,0 +1,170 @@
+#!/usr/bin/env python3
+"""ITK Compatibility Metrics Processor.
+
+Compiles test outcomes from raw JSON results, retrieves and aggregates historical
+runs from GitHub Release assets, and outputs the updated historical metrics log.
+"""
+
+import datetime
+import json
+import logging
+import os
+import pathlib
+import sys
+import urllib.error
+import urllib.request
+
+
+# --- CONSTANTS ---
+RESULTS_FILE = 'raw_results.json'
+HISTORY_OUTPUT_FILE = 'itk_python.json'
+HISTORY_URL = 'https://github.com/a2aproject/a2a-python/releases/download/nightly-metrics/itk_python.json'
+SCENARIOS_FILE = 'scenarios.json'
+DEFAULT_HISTORY_LIMIT = 50
+
+HTTP_STATUS_OK = 200
+HTTP_STATUS_NOT_FOUND = 404
+
+# Configure logging to match standard ITK formatting
+logging.basicConfig(
+    level=logging.INFO,
+)
+logger = logging.getLogger(__name__)
+
+
+def load_raw_results(filepath: str) -> dict:
+    """Loads the raw compatibility results from raw_results.json."""
+    path = pathlib.Path(filepath)
+    if not path.exists():
+        logger.error('Results file %s not found.', filepath)
+        raise SystemExit(1)
+
+    try:
+        with path.open() as f:
+            return json.load(f)
+    except (OSError, json.JSONDecodeError):
+        logger.exception('Error loading results JSON')
+        raise SystemExit(1) from None
+
+
+def fetch_existing_history(url: str) -> list:
+    """Fetches the existing compatibility history from the GitHub release asset.
+
+    If the asset does not exist (HTTP 404), a fresh empty history list is returned.
+    For all other network or server errors, the script exits with a non-zero status
+    to prevent overwriting and losing historical metrics.
+    """
+    try:
+        req = urllib.request.Request(  # noqa: S310
+            url, headers={'User-Agent': 'Mozilla/5.0'}
+        )
+        with urllib.request.urlopen(req, timeout=15) as response:  # noqa: S310
+            if response.status == HTTP_STATUS_OK:
+                history = json.loads(response.read().decode('utf-8'))
+                logger.info(
+                    'Successfully retrieved history. Current entries: %d',
+                    len(history),
+                )
+                return history
+            logger.error(
+                'Unexpected HTTP status when downloading existing history: %d',
+                response.status,
+            )
+            raise SystemExit(1)  # noqa: TRY301
+    except urllib.error.HTTPError as e:
+        if e.code == HTTP_STATUS_NOT_FOUND:
+            logger.warning(
+                'No existing history found (HTTP %d). Initializing fresh history.',
+                e.code,
+            )
+            return []
+        logger.exception(
+            'HTTP error downloading existing history: %d. Aborting to preserve metrics.',
+            e.code,
+        )
+        raise SystemExit(1) from None
+    except Exception:
+        logger.exception(
+            'Failed to download existing history. Aborting to preserve metrics.'
+        )
+        raise SystemExit(1) from None
+
+
+def load_scenarios(filepath: str) -> list:
+    """Loads the list of tests from the scenarios.json definitions."""
+    path = pathlib.Path(filepath)
+    if not path.exists():
+        logger.error('Scenarios file %s not found.', filepath)
+        raise SystemExit(1)
+
+    try:
+        with path.open() as f:
+            data = json.load(f)
+        return data['tests']
+    except (OSError, json.JSONDecodeError, KeyError):
+        logger.exception('Failed to load scenarios.json definitions')
+        raise SystemExit(1) from None
+
+
+def save_history(filepath: str, history: list) -> None:
+    """Saves the updated history back to disk as a release asset candidate."""
+    path = pathlib.Path(filepath)
+    try:
+        with path.open('w') as f:
+            json.dump(history, f, indent=2)
+        logger.info(
+            'Successfully compiled and wrote nightly history to: %s',
+            filepath,
+        )
+    except (OSError, TypeError):
+        logger.exception('Error writing history file')
+        sys.exit(1)
+
+
+def main() -> None:
+    """Orchestrates nightly ITK metrics processing and compiles rolling history."""
+    # 1. Load raw compatibility results
+    data = load_raw_results(RESULTS_FILE)
+    all_passed = data.get('all_passed', False)
+    results = data.get('results', {})
+
+    # 2. Fetch existing history from rolling release
+    history = fetch_existing_history(HISTORY_URL)
+
+    # 3. Load scenarios list for metadata
+    scenarios_list = load_scenarios(SCENARIOS_FILE)
+
+    # Merge definitions with current outcomes
+    compiled_scenarios = []
+    for scenario in scenarios_list:
+        name = scenario.get('name')
+        passed = results.get(name, False)
+        combined = dict(scenario)
+        combined['passed'] = passed
+        compiled_scenarios.append(combined)
+
+    # 4. Compile new run metadata
+    new_run = {
+        'timestamp': datetime.datetime.now(datetime.timezone.utc).isoformat(),
+        'commit_sha': os.environ.get('GITHUB_SHA', 'local-dev'),
+        'github_run_id': os.environ.get('GITHUB_RUN_ID', '0'),
+        'all_passed': all_passed,
+        'scenarios': compiled_scenarios,
+    }
+
+    # 5. Merge and Prune rolling window
+    history.append(new_run)
+    history_limit = int(
+        os.environ.get('ITK_HISTORY_LIMIT', str(DEFAULT_HISTORY_LIMIT))
+    )
+    if len(history) > history_limit:
+        history = history[-history_limit:]
+        logger.info('Pruned history to last %d entries.', history_limit)
+
+    # 6. Save candidates back to disk
+    save_history(HISTORY_OUTPUT_FILE, history)
+    sys.exit(0)
+
+
+if __name__ == '__main__':
+    main()

itk/run_itk.sh

@@ -112,83 +112,18 @@ fi
 echo "ITK Service is up! Sending compatibility test request..."
 RESPONSE=$(curl -s -X POST http://127.0.0.1:8000/run \
   -H "Content-Type: application/json" \
-  -d '{
-    "tests": [
-      {
-        "name": "Star Topology (Full) - JSONRPC & GRPC",
-        "sdks": ["current", "python_v10", "python_v03", "go_v10", "go_v03"],
-        "traversal": "euler",
-        "edges": ["0->1", "0->2", "0->3", "0->4", "1->0", "2->0", "3->0", "4->0"],
-        "protocols": ["jsonrpc", "grpc"],
-        "behavior": "send_message"
-      },
-      {
-        "name": "Star Topology (No Go v03) - HTTP_JSON",
-        "sdks": ["current", "python_v10", "python_v03", "go_v10"],
-        "traversal": "euler",
-        "edges": ["0->1", "0->2", "0->3", "1->0", "2->0", "3->0"],
-        "protocols": ["http_json"],
-        "behavior": "send_message"
-      },
-      {
-        "name": "Star Topology (Full) - JSONRPC & GRPC (Streaming)",
-        "sdks": ["current", "python_v10", "python_v03", "go_v10", "go_v03"],
-        "traversal": "euler",
-        "edges": ["0->1", "0->2", "0->3", "0->4", "1->0", "2->0", "3->0", "4->0"],
-        "protocols": ["jsonrpc", "grpc"],
-        "streaming": true,
-        "behavior": "send_message"
-      },
-      {
-        "name": "Star Topology (No Go v03) - HTTP_JSON (Streaming)",
-        "sdks": ["current", "python_v10", "python_v03", "go_v10"],
-        "traversal": "euler",
-        "edges": ["0->1", "0->2", "0->3", "1->0", "2->0", "3->0"],
-        "protocols": ["http_json"],
-        "streaming": true,
-        "behavior": "send_message"
-      },
-      {
-        "name": "Push Notification Test - JSONRPC & GRPC",
-        "sdks": ["current", "python_v10", "python_v03", "go_v03"],
-        "traversal": "euler",
-        "edges": ["0->1", "0->2", "0->3", "1->0", "2->0", "3->0"],
-        "protocols": ["jsonrpc", "grpc"],
-        "behavior": "push_notification"
-      },
-      {
-        "name": "Push Notification Test - HTTP_JSON",
-        "sdks": ["current", "python_v10", "python_v03"],
-        "traversal": "euler",
-        "edges": ["0->1", "0->2", "1->0", "2->0"],
-        "protocols": ["http_json"],
-        "behavior": "push_notification"
-      },
-      {
-        "name": "Resubscribe Test - JSONRPC",
-        "sdks": ["current", "python_v10", "python_v03", "go_v10", "go_v03"],
-        "traversal": "euler",
-        "edges": ["0->1", "0->2", "0->3", "0->4", "1->0", "2->0", "3->0", "4->0"],
-        "protocols": ["jsonrpc"],
-        "streaming": true,
-        "behavior": "resubscribe"
-      },
-      {
-        "name": "Resubscribe Test - Python & Go Non-JSONRPC Protocols",
-        "sdks": ["current", "python_v10", "python_v03", "go_v10"],
-        "traversal": "euler",
-        "edges": ["0->1", "0->2", "0->3", "1->0", "2->0", "3->0"],
-        "protocols": ["grpc", "http_json"],
-        "streaming": true,
-        "behavior": "resubscribe"
-      }
-    ]
-  }')
-
-echo "--------------------------------------------------------"
-echo "ITK TEST RESULTS:"
-echo "--------------------------------------------------------"
-echo "$RESPONSE" | python3 -c "
+  -d @scenarios.json)
+
+if [ "${ITK_NIGHTLY_RUN^^}" = "TRUE" ]; then
+  echo "Nightly run detected. Saving raw results and running process_results.py..."
+  echo "$RESPONSE" > raw_results.json
+  python3 process_results.py
+  RESULT=$?
+else
+  echo "--------------------------------------------------------"
+  echo "ITK TEST RESULTS:"
+  echo "--------------------------------------------------------"
+  echo "$RESPONSE" | python3 -c "
 import sys, json
 try:
     data = json.load(sys.stdin)
@@ -206,7 +141,8 @@ except Exception as e:
     print(f'Raw response: {data if \"data\" in locals() else \"no data\"}')
     sys.exit(1)
 "
-RESULT=$?
+  RESULT=$?
+fi
 set -e
 
 if [ $RESULT -ne 0 ]; then

itk/scenarios.json

@@ -0,0 +1,72 @@
+{
+  "tests": [
+    {
+      "name": "Star Topology (Full) - JSONRPC & GRPC",
+      "sdks": ["current", "python_v10", "python_v03", "go_v10", "go_v03"],
+      "traversal": "euler",
+      "edges": ["0->1", "0->2", "0->3", "0->4", "1->0", "2->0", "3->0", "4->0"],
+      "protocols": ["jsonrpc", "grpc"],
+      "behavior": "send_message"
+    },
+    {
+      "name": "Star Topology (No Go v03) - HTTP_JSON",
+      "sdks": ["current", "python_v10", "python_v03", "go_v10"],
+      "traversal": "euler",
+      "edges": ["0->1", "0->2", "0->3", "1->0", "2->0", "3->0"],
+      "protocols": ["http_json"],
+      "behavior": "send_message"
+    },
+    {
+      "name": "Star Topology (Full) - JSONRPC & GRPC (Streaming)",
+      "sdks": ["current", "python_v10", "python_v03", "go_v10", "go_v03"],
+      "traversal": "euler",
+      "edges": ["0->1", "0->2", "0->3", "0->4", "1->0", "2->0", "3->0", "4->0"],
+      "protocols": ["jsonrpc", "grpc"],
+      "streaming": true,
+      "behavior": "send_message"
+    },
+    {
+      "name": "Star Topology (No Go v03) - HTTP_JSON (Streaming)",
+      "sdks": ["current", "python_v10", "python_v03", "go_v10"],
+      "traversal": "euler",
+      "edges": ["0->1", "0->2", "0->3", "1->0", "2->0", "3->0"],
+      "protocols": ["http_json"],
+      "streaming": true,
+      "behavior": "send_message"
+    },
+    {
+      "name": "Push Notification Test - JSONRPC & GRPC",
+      "sdks": ["current", "python_v10", "python_v03", "go_v03"],
+      "traversal": "euler",
+      "edges": ["0->1", "0->2", "0->3", "1->0", "2->0", "3->0"],
+      "protocols": ["jsonrpc", "grpc"],
+      "behavior": "push_notification"
+    },
+    {
+      "name": "Push Notification Test - HTTP_JSON",
+      "sdks": ["current", "python_v10", "python_v03"],
+      "traversal": "euler",
+      "edges": ["0->1", "0->2", "1->0", "2->0"],
+      "protocols": ["http_json"],
+      "behavior": "push_notification"
+    },
+    {
+      "name": "Resubscribe Test - JSONRPC",
+      "sdks": ["current", "python_v10", "python_v03", "go_v10", "go_v03"],
+      "traversal": "euler",
+      "edges": ["0->1", "0->2", "0->3", "0->4", "1->0", "2->0", "3->0", "4->0"],
+      "protocols": ["jsonrpc"],
+      "streaming": true,
+      "behavior": "resubscribe"
+    },
+    {
+      "name": "Resubscribe Test - Python & Go Non-JSONRPC Protocols",
+      "sdks": ["current", "python_v10", "python_v03", "go_v10"],
+      "traversal": "euler",
+      "edges": ["0->1", "0->2", "0->3", "1->0", "2->0", "3->0"],
+      "protocols": ["grpc", "http_json"],
+      "streaming": true,
+      "behavior": "resubscribe"
+    }
+  ]
+}