From 0961c8f0877dd0e0fdea51c8c93ca9dff751630e Mon Sep 17 00:00:00 2001 From: Krzysztof Dziedzic Date: Tue, 5 May 2026 11:56:36 +0000 Subject: [PATCH] test: set up itk nightly runs --- .github/workflows/nightly.yaml | 38 ++++++++ itk/process_results.py | 170 +++++++++++++++++++++++++++++++++ itk/run_itk.sh | 92 +++--------------- itk/scenarios.json | 72 ++++++++++++++ 4 files changed, 294 insertions(+), 78 deletions(-) create mode 100644 .github/workflows/nightly.yaml create mode 100644 itk/process_results.py create mode 100644 itk/scenarios.json diff --git a/.github/workflows/nightly.yaml b/.github/workflows/nightly.yaml new file mode 100644 index 000000000..3182f19f5 --- /dev/null +++ b/.github/workflows/nightly.yaml @@ -0,0 +1,38 @@ +name: Nightly ITK + +on: + schedule: + - cron: '0 2 * * *' # 2:00 AM UTC daily + workflow_dispatch: # Allow manual execution + +permissions: + contents: write + +jobs: + nightly: + name: Nightly ITK Run + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v6 + + - name: Install uv + uses: astral-sh/setup-uv@v7 + + - name: Run Nightly ITK Tests + run: bash run_itk.sh + working-directory: itk + env: + A2A_SAMPLES_REVISION: itk-v.021-alpha + ITK_NIGHTLY_RUN: "True" + + - name: Upload Results to Rolling Release + uses: softprops/action-gh-release@v2 + with: + tag_name: "nightly-metrics" + prerelease: true + files: | + itk/itk_python.json + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/itk/process_results.py b/itk/process_results.py new file mode 100644 index 000000000..26e689683 --- /dev/null +++ b/itk/process_results.py @@ -0,0 +1,170 @@ +#!/usr/bin/env python3 +"""ITK Compatibility Metrics Processor. + +Compiles test outcomes from raw JSON results, retrieves and aggregates historical +runs from GitHub Release assets, and outputs the updated historical metrics log. +""" + +import datetime +import json +import logging +import os +import pathlib +import sys +import urllib.error +import urllib.request + + +# --- CONSTANTS --- +RESULTS_FILE = 'raw_results.json' +HISTORY_OUTPUT_FILE = 'itk_python.json' +HISTORY_URL = 'https://github.com/a2aproject/a2a-python/releases/download/nightly-metrics/itk_python.json' +SCENARIOS_FILE = 'scenarios.json' +DEFAULT_HISTORY_LIMIT = 50 + +HTTP_STATUS_OK = 200 +HTTP_STATUS_NOT_FOUND = 404 + +# Configure logging to match standard ITK formatting +logging.basicConfig( + level=logging.INFO, +) +logger = logging.getLogger(__name__) + + +def load_raw_results(filepath: str) -> dict: + """Loads the raw compatibility results from raw_results.json.""" + path = pathlib.Path(filepath) + if not path.exists(): + logger.error('Results file %s not found.', filepath) + raise SystemExit(1) + + try: + with path.open() as f: + return json.load(f) + except (OSError, json.JSONDecodeError): + logger.exception('Error loading results JSON') + raise SystemExit(1) from None + + +def fetch_existing_history(url: str) -> list: + """Fetches the existing compatibility history from the GitHub release asset. + + If the asset does not exist (HTTP 404), a fresh empty history list is returned. + For all other network or server errors, the script exits with a non-zero status + to prevent overwriting and losing historical metrics. + """ + try: + req = urllib.request.Request( # noqa: S310 + url, headers={'User-Agent': 'Mozilla/5.0'} + ) + with urllib.request.urlopen(req, timeout=15) as response: # noqa: S310 + if response.status == HTTP_STATUS_OK: + history = json.loads(response.read().decode('utf-8')) + logger.info( + 'Successfully retrieved history. Current entries: %d', + len(history), + ) + return history + logger.error( + 'Unexpected HTTP status when downloading existing history: %d', + response.status, + ) + raise SystemExit(1) # noqa: TRY301 + except urllib.error.HTTPError as e: + if e.code == HTTP_STATUS_NOT_FOUND: + logger.warning( + 'No existing history found (HTTP %d). Initializing fresh history.', + e.code, + ) + return [] + logger.exception( + 'HTTP error downloading existing history: %d. Aborting to preserve metrics.', + e.code, + ) + raise SystemExit(1) from None + except Exception: + logger.exception( + 'Failed to download existing history. Aborting to preserve metrics.' + ) + raise SystemExit(1) from None + + +def load_scenarios(filepath: str) -> list: + """Loads the list of tests from the scenarios.json definitions.""" + path = pathlib.Path(filepath) + if not path.exists(): + logger.error('Scenarios file %s not found.', filepath) + raise SystemExit(1) + + try: + with path.open() as f: + data = json.load(f) + return data['tests'] + except (OSError, json.JSONDecodeError, KeyError): + logger.exception('Failed to load scenarios.json definitions') + raise SystemExit(1) from None + + +def save_history(filepath: str, history: list) -> None: + """Saves the updated history back to disk as a release asset candidate.""" + path = pathlib.Path(filepath) + try: + with path.open('w') as f: + json.dump(history, f, indent=2) + logger.info( + 'Successfully compiled and wrote nightly history to: %s', + filepath, + ) + except (OSError, TypeError): + logger.exception('Error writing history file') + sys.exit(1) + + +def main() -> None: + """Orchestrates nightly ITK metrics processing and compiles rolling history.""" + # 1. Load raw compatibility results + data = load_raw_results(RESULTS_FILE) + all_passed = data.get('all_passed', False) + results = data.get('results', {}) + + # 2. Fetch existing history from rolling release + history = fetch_existing_history(HISTORY_URL) + + # 3. Load scenarios list for metadata + scenarios_list = load_scenarios(SCENARIOS_FILE) + + # Merge definitions with current outcomes + compiled_scenarios = [] + for scenario in scenarios_list: + name = scenario.get('name') + passed = results.get(name, False) + combined = dict(scenario) + combined['passed'] = passed + compiled_scenarios.append(combined) + + # 4. Compile new run metadata + new_run = { + 'timestamp': datetime.datetime.now(datetime.timezone.utc).isoformat(), + 'commit_sha': os.environ.get('GITHUB_SHA', 'local-dev'), + 'github_run_id': os.environ.get('GITHUB_RUN_ID', '0'), + 'all_passed': all_passed, + 'scenarios': compiled_scenarios, + } + + # 5. Merge and Prune rolling window + history.append(new_run) + history_limit = int( + os.environ.get('ITK_HISTORY_LIMIT', str(DEFAULT_HISTORY_LIMIT)) + ) + if len(history) > history_limit: + history = history[-history_limit:] + logger.info('Pruned history to last %d entries.', history_limit) + + # 6. Save candidates back to disk + save_history(HISTORY_OUTPUT_FILE, history) + sys.exit(0) + + +if __name__ == '__main__': + main() diff --git a/itk/run_itk.sh b/itk/run_itk.sh index 5d4b21ef2..035c7851f 100755 --- a/itk/run_itk.sh +++ b/itk/run_itk.sh @@ -112,83 +112,18 @@ fi echo "ITK Service is up! Sending compatibility test request..." RESPONSE=$(curl -s -X POST http://127.0.0.1:8000/run \ -H "Content-Type: application/json" \ - -d '{ - "tests": [ - { - "name": "Star Topology (Full) - JSONRPC & GRPC", - "sdks": ["current", "python_v10", "python_v03", "go_v10", "go_v03"], - "traversal": "euler", - "edges": ["0->1", "0->2", "0->3", "0->4", "1->0", "2->0", "3->0", "4->0"], - "protocols": ["jsonrpc", "grpc"], - "behavior": "send_message" - }, - { - "name": "Star Topology (No Go v03) - HTTP_JSON", - "sdks": ["current", "python_v10", "python_v03", "go_v10"], - "traversal": "euler", - "edges": ["0->1", "0->2", "0->3", "1->0", "2->0", "3->0"], - "protocols": ["http_json"], - "behavior": "send_message" - }, - { - "name": "Star Topology (Full) - JSONRPC & GRPC (Streaming)", - "sdks": ["current", "python_v10", "python_v03", "go_v10", "go_v03"], - "traversal": "euler", - "edges": ["0->1", "0->2", "0->3", "0->4", "1->0", "2->0", "3->0", "4->0"], - "protocols": ["jsonrpc", "grpc"], - "streaming": true, - "behavior": "send_message" - }, - { - "name": "Star Topology (No Go v03) - HTTP_JSON (Streaming)", - "sdks": ["current", "python_v10", "python_v03", "go_v10"], - "traversal": "euler", - "edges": ["0->1", "0->2", "0->3", "1->0", "2->0", "3->0"], - "protocols": ["http_json"], - "streaming": true, - "behavior": "send_message" - }, - { - "name": "Push Notification Test - JSONRPC & GRPC", - "sdks": ["current", "python_v10", "python_v03", "go_v03"], - "traversal": "euler", - "edges": ["0->1", "0->2", "0->3", "1->0", "2->0", "3->0"], - "protocols": ["jsonrpc", "grpc"], - "behavior": "push_notification" - }, - { - "name": "Push Notification Test - HTTP_JSON", - "sdks": ["current", "python_v10", "python_v03"], - "traversal": "euler", - "edges": ["0->1", "0->2", "1->0", "2->0"], - "protocols": ["http_json"], - "behavior": "push_notification" - }, - { - "name": "Resubscribe Test - JSONRPC", - "sdks": ["current", "python_v10", "python_v03", "go_v10", "go_v03"], - "traversal": "euler", - "edges": ["0->1", "0->2", "0->3", "0->4", "1->0", "2->0", "3->0", "4->0"], - "protocols": ["jsonrpc"], - "streaming": true, - "behavior": "resubscribe" - }, - { - "name": "Resubscribe Test - Python & Go Non-JSONRPC Protocols", - "sdks": ["current", "python_v10", "python_v03", "go_v10"], - "traversal": "euler", - "edges": ["0->1", "0->2", "0->3", "1->0", "2->0", "3->0"], - "protocols": ["grpc", "http_json"], - "streaming": true, - "behavior": "resubscribe" - } - ] - }') - -echo "--------------------------------------------------------" -echo "ITK TEST RESULTS:" -echo "--------------------------------------------------------" -echo "$RESPONSE" | python3 -c " + -d @scenarios.json) + +if [ "${ITK_NIGHTLY_RUN^^}" = "TRUE" ]; then + echo "Nightly run detected. Saving raw results and running process_results.py..." + echo "$RESPONSE" > raw_results.json + python3 process_results.py + RESULT=$? +else + echo "--------------------------------------------------------" + echo "ITK TEST RESULTS:" + echo "--------------------------------------------------------" + echo "$RESPONSE" | python3 -c " import sys, json try: data = json.load(sys.stdin) @@ -206,7 +141,8 @@ except Exception as e: print(f'Raw response: {data if \"data\" in locals() else \"no data\"}') sys.exit(1) " -RESULT=$? + RESULT=$? +fi set -e if [ $RESULT -ne 0 ]; then diff --git a/itk/scenarios.json b/itk/scenarios.json new file mode 100644 index 000000000..956c3c111 --- /dev/null +++ b/itk/scenarios.json @@ -0,0 +1,72 @@ +{ + "tests": [ + { + "name": "Star Topology (Full) - JSONRPC & GRPC", + "sdks": ["current", "python_v10", "python_v03", "go_v10", "go_v03"], + "traversal": "euler", + "edges": ["0->1", "0->2", "0->3", "0->4", "1->0", "2->0", "3->0", "4->0"], + "protocols": ["jsonrpc", "grpc"], + "behavior": "send_message" + }, + { + "name": "Star Topology (No Go v03) - HTTP_JSON", + "sdks": ["current", "python_v10", "python_v03", "go_v10"], + "traversal": "euler", + "edges": ["0->1", "0->2", "0->3", "1->0", "2->0", "3->0"], + "protocols": ["http_json"], + "behavior": "send_message" + }, + { + "name": "Star Topology (Full) - JSONRPC & GRPC (Streaming)", + "sdks": ["current", "python_v10", "python_v03", "go_v10", "go_v03"], + "traversal": "euler", + "edges": ["0->1", "0->2", "0->3", "0->4", "1->0", "2->0", "3->0", "4->0"], + "protocols": ["jsonrpc", "grpc"], + "streaming": true, + "behavior": "send_message" + }, + { + "name": "Star Topology (No Go v03) - HTTP_JSON (Streaming)", + "sdks": ["current", "python_v10", "python_v03", "go_v10"], + "traversal": "euler", + "edges": ["0->1", "0->2", "0->3", "1->0", "2->0", "3->0"], + "protocols": ["http_json"], + "streaming": true, + "behavior": "send_message" + }, + { + "name": "Push Notification Test - JSONRPC & GRPC", + "sdks": ["current", "python_v10", "python_v03", "go_v03"], + "traversal": "euler", + "edges": ["0->1", "0->2", "0->3", "1->0", "2->0", "3->0"], + "protocols": ["jsonrpc", "grpc"], + "behavior": "push_notification" + }, + { + "name": "Push Notification Test - HTTP_JSON", + "sdks": ["current", "python_v10", "python_v03"], + "traversal": "euler", + "edges": ["0->1", "0->2", "1->0", "2->0"], + "protocols": ["http_json"], + "behavior": "push_notification" + }, + { + "name": "Resubscribe Test - JSONRPC", + "sdks": ["current", "python_v10", "python_v03", "go_v10", "go_v03"], + "traversal": "euler", + "edges": ["0->1", "0->2", "0->3", "0->4", "1->0", "2->0", "3->0", "4->0"], + "protocols": ["jsonrpc"], + "streaming": true, + "behavior": "resubscribe" + }, + { + "name": "Resubscribe Test - Python & Go Non-JSONRPC Protocols", + "sdks": ["current", "python_v10", "python_v03", "go_v10"], + "traversal": "euler", + "edges": ["0->1", "0->2", "0->3", "1->0", "2->0", "3->0"], + "protocols": ["grpc", "http_json"], + "streaming": true, + "behavior": "resubscribe" + } + ] +}