diff --git a/.pycodestylerc b/.pycodestylerc index 24fc83752..162bcd630 100644 --- a/.pycodestylerc +++ b/.pycodestylerc @@ -1,5 +1,5 @@ [pycodestyle] count = True max-line-length = 120 -exclude=test_diff.py,migrations,venv*,parse.py,config.py +exclude=test_diff.py,migrations,venv*,.venv*,parse.py,config.py ignore = E701 diff --git a/bootstrap_gunicorn.py b/bootstrap_gunicorn.py index 0cdfdd37a..30fd9d4f3 100644 --- a/bootstrap_gunicorn.py +++ b/bootstrap_gunicorn.py @@ -6,12 +6,12 @@ current_dir = path.dirname(path.abspath(__file__)) TIMEOUT = 120 # In seconds -# Arguments to start gunicorn args = [ - "gunicorn", "-w", "4", "--daemon", "--pid", "gunicorn.pid", "-b", "unix:sampleplatform.sock", "-m", "007", - "-g", "www-data", "-u", "www-data", f"--chdir={current_dir}", "--log-level", "debug", "--timeout", f"{TIMEOUT}", + "gunicorn", "-w", "4", "-b", "unix:sampleplatform.sock", "-m", "007", + "-g", "www-data", "-u", "www-data", f"--chdir={current_dir}", "--log-level", "debug", + "--timeout", f"{TIMEOUT}", "--access-logfile", f"{current_dir}/logs/access.log", "--capture-output", "--log-file", f"{current_dir}/logs/error.log", "run:app" ] -subprocess.Popen(args) +subprocess.run(args) diff --git a/install/install.sh b/install/install.sh index 75ed79623..d44cce1fc 100644 --- a/install/install.sh +++ b/install/install.sh @@ -264,10 +264,12 @@ chown -R www-data:www-data "${root_dir}" "${sample_repository}" echo "* Creating startup script" { - cp "${dir}/platform" /etc/init.d/platform - sed -i "s#BASE_DIR#${root_dir}#g" /etc/init.d/platform - chmod 755 /etc/init.d/platform - update-rc.d platform defaults + rm -f /etc/init.d/platform + update-rc.d platform remove || true + cp "${dir}/platform.service" /etc/systemd/system/platform.service + sed -i "s|#BASE_DIR#|${root_dir}|g" /etc/systemd/system/platform.service + systemctl daemon-reload + systemctl enable platform.service } >> "$install_log" 2>&1 echo "* Creating RClone config file" diff --git a/install/installation.md b/install/installation.md index 74575fdf9..4464febf0 100644 --- a/install/installation.md +++ b/install/installation.md @@ -172,7 +172,7 @@ sudo python3 bootstrap_gunicorn.py 1. Firstly check the Platform Installation log file in the install folder. Check for any errors, which may have been caused during platform installation on your system, and then try to resolve them accordingly. 2. Next check for nginx status by `service nginx status` command, if it is not active, check nginx error log file, possibly in `/var/log/nginx/error.log` file. 3. Next check for platform status by `service platform status` command, if it is not `active(running)` then check for platform logs in the `logs` directory of your project. - 4. In case of any gunicorn error try manually running `/etc/init.d/platform start` command and recheck the platform status. + 4. In case of any gunicorn error try manually running `sudo systemctl start platform.service` command and recheck the platform status. ### Setting Up The Bucket diff --git a/install/platform b/install/platform deleted file mode 100644 index 21c161f8e..000000000 --- a/install/platform +++ /dev/null @@ -1,27 +0,0 @@ -#!/bin/bash -# /etc/init.d/platform -# -# Carry out specific functions when asked to by the system -case "${1}" in - start) - echo "Starting Platform daemon..." - cd BASE_DIR - python bootstrap_gunicorn.py - ;; - stop) - echo "Stopping Platform daemon..." - pid=`cat "BASE_DIR/gunicorn.pid"` - kill "${pid}" - ;; - reload) - echo "Reloading Platform daemon..." - pid=`cat "BASE_DIR/gunicorn.pid"` - kill -HUP "${pid}" - ;; - *) - echo "Usage: /etc/init.d/platform {start|stop|reload}" - exit 1 - ;; -esac - -exit 0 \ No newline at end of file diff --git a/install/platform.service b/install/platform.service new file mode 100644 index 000000000..06b2c4ac2 --- /dev/null +++ b/install/platform.service @@ -0,0 +1,19 @@ +[Unit] +Description=CCExtractor Sample Platform (gunicorn) +After=network.target mysql.service +Wants=mysql.service + +[Service] +Type=simple +User=www-data +Group=www-data +WorkingDirectory=#BASE_DIR# +ExecStart=/usr/bin/gunicorn -w 4 -b unix:#BASE_DIR#/sampleplatform.sock -m 007 --timeout 120 --log-level debug --access-logfile #BASE_DIR#/logs/access.log --capture-output --log-file #BASE_DIR#/logs/error.log run:app +ExecReload=/bin/kill -s HUP $MAINPID +Restart=on-failure +RestartSec=5 +KillMode=control-group +TimeoutStopSec=30 + +[Install] +WantedBy=multi-user.target diff --git a/migrations/versions/d4f8e2a1b3c7_.py b/migrations/versions/d4f8e2a1b3c7_.py new file mode 100644 index 000000000..e84d0302e --- /dev/null +++ b/migrations/versions/d4f8e2a1b3c7_.py @@ -0,0 +1,44 @@ +"""Add api_token table for scoped API token auth. + +Revision ID: d4f8e2a1b3c7 +Revises: c8f3a2b1d4e5 +Create Date: 2026-06-11 03:00:00.000000 + +""" +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision = 'd4f8e2a1b3c7' +down_revision = 'c8f3a2b1d4e5' +branch_labels = None +depends_on = None + + +def upgrade(): + """Apply the migration.""" + op.add_column('user', sa.Column('github_login', sa.String(length=255), nullable=True)) + op.create_table( + 'api_token', + sa.Column('id', sa.Integer(), nullable=False, autoincrement=True), + sa.Column('user_id', sa.Integer(), nullable=False), + sa.Column('token_name', sa.String(length=50), nullable=False), + sa.Column('token_hash', sa.String(length=255), nullable=False), + sa.Column('token_prefix', sa.String(length=16), nullable=False), + sa.Column('scopes_json', sa.Text(), nullable=False), + sa.Column('created_at', sa.DateTime(timezone=True), nullable=False), + sa.Column('expires_at', sa.DateTime(timezone=True), nullable=False), + sa.Column('revoked_at', sa.DateTime(timezone=True), nullable=True), + sa.PrimaryKeyConstraint('id'), + sa.ForeignKeyConstraint(['user_id'], ['user.id'], onupdate='CASCADE', ondelete='CASCADE'), + sa.UniqueConstraint('user_id', 'token_name', name='uq_user_token_name'), + mysql_engine='InnoDB' + ) + op.create_index('ix_api_token_token_prefix', 'api_token', ['token_prefix']) + + +def downgrade(): + """Revert the migration.""" + op.drop_index('ix_api_token_token_prefix', table_name='api_token') + op.drop_table('api_token') + op.drop_column('user', 'github_login') diff --git a/mod_api/__init__.py b/mod_api/__init__.py new file mode 100644 index 000000000..3fb527b5c --- /dev/null +++ b/mod_api/__init__.py @@ -0,0 +1,42 @@ +""" +mod_api: JSON REST API blueprint for the CCExtractor CI platform. + +Registered at /api/v1. All endpoints return structured JSON, use scoped +Bearer token auth, and enforce per-client rate limiting. +""" + +from flask import Blueprint + +mod_api = Blueprint('api', __name__) + +# Middleware imports +from mod_api.middleware import auth # noqa: E402 +from mod_api.middleware import error_handler # noqa: E402 +from mod_api.middleware import rate_limit # noqa: E402 +from mod_api.middleware import security # noqa: E402 + +# Explicitly register before_request hooks in the exact order they should run +mod_api.before_request(auth.authenticate_request) +mod_api.before_request(rate_limit.check_rate_limit) +mod_api.before_request(auth.enforce_auth_error) + +# Explicitly register after_request hooks. +# NOTE: Flask executes after_request hooks in REVERSE registration order. +# Registration: security → rate_limit → (convert is app-level, see below) +# Execution: rate_limit → security +# This means rate-limit headers are added first, then security headers layer +# on top — both on the same response object. +mod_api.after_request(security.add_security_headers) +mod_api.after_request(rate_limit.add_rate_limit_headers) + +# Registered as after_app_request so it fires for ALL requests (including +# routing-level 404s/405s that never enter the blueprint). +mod_api.after_app_request(error_handler.convert_api_errors_to_json) + +# Route modules register themselves against the blueprint; the rest of +# the stack adds one module per PR. +from mod_api.routes import auth as auth_routes # noqa: E402, F401 +from mod_api.routes import results as results_routes # noqa: E402, F401 +from mod_api.routes import runs as runs_routes # noqa: E402, F401 +from mod_api.routes import samples as samples_routes # noqa: E402, F401 +from mod_api.routes import system as system_routes # noqa: E402, F401 diff --git a/mod_api/middleware/__init__.py b/mod_api/middleware/__init__.py new file mode 100644 index 000000000..860b3ce01 --- /dev/null +++ b/mod_api/middleware/__init__.py @@ -0,0 +1 @@ +"""mod_api.middleware: auth, rate limiting, validation, and error handling.""" diff --git a/mod_api/middleware/auth.py b/mod_api/middleware/auth.py new file mode 100644 index 000000000..0903c3d83 --- /dev/null +++ b/mod_api/middleware/auth.py @@ -0,0 +1,146 @@ +""" +Bearer token authentication and scope/role enforcement for API routes. + +Runs as a before_request hook on the api blueprint. Public endpoints +(token creation, health check) are exempted. On success, the authenticated +user and token are stored in flask.g for downstream handlers. + +HTTP semantics: + 401 = token missing, expired, revoked, or invalid + 403 = valid token but insufficient scope or role +""" + +import functools +from typing import Any, List + +from flask import g, request + +from mod_api.middleware.error_handler import make_error_response +from mod_api.models.api_token import TOKEN_PREFIX, ApiToken + +_AUTH_FAILED_MSG = 'Bearer token is missing, expired, or invalid.' + +# These endpoints bypass auth entirely. +_PUBLIC_ENDPOINTS = frozenset([ + 'api.create_token', # POST /auth/tokens (uses email/password body) + 'api.system_health', # GET /system/health (uptime monitoring) +]) + + +def _unauthorized(): + """Shorthand for a 401 response with the standard auth failure message.""" + return make_error_response( + 'unauthorized', _AUTH_FAILED_MSG, http_status=401) + + +def authenticate_request(): + """Validate Bearer token and attach user context to the request. + + If auth fails, sets g.auth_error instead of returning immediately, + so that subsequent hooks (like rate limiting) still run. + """ + if request.endpoint in _PUBLIC_ENDPOINTS: + g.api_user = None + g.api_token = None + return + + auth_header = request.headers.get('Authorization', '') + if not auth_header: + g.auth_error = _unauthorized() + return + + parts = auth_header.split(' ', 1) + # Auth scheme names are case-insensitive (RFC 7235 section 2.1). + if len(parts) != 2 or parts[0].lower() != 'bearer': + g.auth_error = _unauthorized() + return + + token_value = parts[1].strip() + if not token_value or not token_value.startswith(TOKEN_PREFIX): + g.auth_error = _unauthorized() + return + + # Look up by prefix, then verify the full hash against each candidate. + prefix = ApiToken.extract_prefix(token_value) + candidates = ApiToken.query.filter_by(token_prefix=prefix).all() + + if not candidates: + g.auth_error = _unauthorized() + return + + matched_token = None + for candidate in candidates: + if ApiToken.verify_token(token_value, candidate.token_hash): + matched_token = candidate + break + + if matched_token is None: + g.auth_error = _unauthorized() + return + + if not matched_token.is_valid: + g.auth_error = _unauthorized() + return + + g.api_token = matched_token + g.api_user = matched_token.user + + +def enforce_auth_error(): + """Return any stored auth errors after rate limiting.""" + if hasattr(g, 'auth_error') and g.auth_error is not None: + return g.auth_error + + +def require_scope(*scopes: str): + """Reject the request if the token lacks any of the ``scopes``.""" + def decorator(f): + @functools.wraps(f) + def decorated_function(*args, **kwargs): + token = getattr(g, 'api_token', None) + if token is None: + return _unauthorized() + + missing_scopes = [s for s in scopes if not token.has_scope(s)] + if missing_scopes: + return make_error_response( + 'forbidden', + 'Token lacks the required scopes for this operation.', + details={ + 'required_scopes': list(scopes), + 'missing_scopes': missing_scopes, + 'token_scopes': token.scopes, + }, + http_status=403, + ) + return f(*args, **kwargs) + return decorated_function + return decorator + + +def require_roles(roles: List[Any]): + """Reject the request if the user's role is not in ``roles``. + + Takes Role members (not strings), matching check_access_rights in + mod_auth. Typed loosely because DeclEnum members are EnumSymbol + instances at runtime but plain tuples to the type checker. + """ + def decorator(f): + @functools.wraps(f) + def decorated_function(*args, **kwargs): + user = getattr(g, 'api_user', None) + if user is None: + return _unauthorized() + if user.role not in roles: + return make_error_response( + 'forbidden', + 'Your role does not have permission for this operation.', + details={ + 'required_roles': [role.value for role in roles], + 'user_role': user.role.value, + }, + http_status=403, + ) + return f(*args, **kwargs) + return decorated_function + return decorator diff --git a/mod_api/middleware/error_handler.py b/mod_api/middleware/error_handler.py new file mode 100644 index 000000000..33b50842e --- /dev/null +++ b/mod_api/middleware/error_handler.py @@ -0,0 +1,160 @@ +"""Structured JSON error responses for API routes.""" + +from flask import current_app, jsonify, request +from marshmallow import ValidationError as MarshmallowValidationError +from sqlalchemy.exc import SQLAlchemyError + +from mod_api import mod_api + +_API_PREFIX = '/api/v1' + + +def make_error_response(code, message, details=None, http_status=400): + """Build a JSON error response conforming to the ErrorResponse schema.""" + body = { + 'code': code, + 'message': str(message)[:500], + 'details': details if details is not None else {}, + } + response = jsonify(body) + response.status_code = http_status + return response + + +@mod_api.errorhandler(400) +def handle_400(error): + """Bad request.""" + return make_error_response( + 'validation_error', + getattr(error, 'description', 'Bad request.'), + http_status=400, + ) + + +@mod_api.errorhandler(401) +def handle_401(error): + """Unauthorized.""" + return make_error_response( + 'unauthorized', + 'Bearer token is missing, expired, or invalid.', + http_status=401, + ) + + +@mod_api.errorhandler(403) +def handle_403(error): + """Forbidden.""" + return make_error_response( + 'forbidden', + 'Token does not have the required scope for this operation.', + http_status=403, + ) + + +@mod_api.errorhandler(404) +def handle_404(error): + """Not found.""" + return make_error_response( + 'not_found', + getattr(error, 'description', 'Resource not found.'), + http_status=404, + ) + + +@mod_api.errorhandler(405) +def handle_405(error): + """Handle method-not-allowed errors for API routes.""" + resp = make_error_response( + 'method_not_allowed', + 'Method not allowed.', + http_status=405, + ) + if hasattr(error, 'valid_methods') and error.valid_methods: + resp.headers['Allow'] = ', '.join(error.valid_methods) + return resp + + +@mod_api.errorhandler(422) +def handle_422(error): + """Unprocessable entity.""" + return make_error_response( + 'unprocessable', + getattr( + error, + 'description', + 'Request is valid JSON but semantically invalid.'), + http_status=422, + ) + + +@mod_api.errorhandler(429) +def handle_429(error): + """Rate limited. + + This is only a fallback for 429s raised outside the rate-limit + middleware. The live limiter (mod_api.middleware.rate_limit) returns + accurate per-bucket limit/retry_after/window values and the + Retry-After header; we deliberately don't hardcode numbers here that + would be wrong for the auth (5/15m) and write (20/min) buckets. + """ + return make_error_response( + 'rate_limited', + 'Rate limit exceeded.', + http_status=429, + ) + + +@mod_api.errorhandler(500) +def handle_500(error): + """Handle unexpected server errors for API routes.""" + current_app.logger.exception(error) + return make_error_response( + 'internal_error', + 'An unexpected error occurred.', + http_status=500, + ) + + +@mod_api.errorhandler(MarshmallowValidationError) +def handle_marshmallow_validation_error(error): + """Catch schema validation failures and return them as 400.""" + return make_error_response( + 'validation_error', + 'Request failed schema validation.', + details={'fields': error.messages}, + http_status=400, + ) + + +@mod_api.errorhandler(SQLAlchemyError) +def handle_sqlalchemy_error(error): + """Log database errors.""" + current_app.logger.exception(error) + return make_error_response( + 'internal_error', + 'An unexpected database error occurred.', + http_status=500, + ) + + +def convert_api_errors_to_json(response): + """Catch routing errors that were handled by global app handlers and convert them to JSON.""" + if request.path.startswith(_API_PREFIX): + if response.status_code >= 500 and not response.is_json: + new_resp = make_error_response( + 'internal_error', 'An unexpected error occurred.', http_status=response.status_code + ) + response.data = new_resp.data + response.mimetype = new_resp.mimetype + return response + if response.status_code == 404 and not response.is_json: + new_resp = make_error_response('not_found', 'Resource not found.', http_status=404) + response.data = new_resp.data + response.mimetype = new_resp.mimetype + return response + if response.status_code == 405 and not response.is_json: + new_resp = make_error_response('method_not_allowed', 'Method not allowed.', http_status=405) + response.data = new_resp.data + response.mimetype = new_resp.mimetype + return response + return response diff --git a/mod_api/middleware/rate_limit.py b/mod_api/middleware/rate_limit.py new file mode 100644 index 000000000..222dd0f5e --- /dev/null +++ b/mod_api/middleware/rate_limit.py @@ -0,0 +1,143 @@ +""" +Per-client fixed-window rate limiting for API endpoints. + +Limits: + POST /auth/tokens 5 req / 15 min (keyed by IP) + POST/DELETE/PUT/PATCH 20 req / min (keyed by token) + GET 120 req / min (keyed by token) + +Includes X-RateLimit-* headers on every response. + +Note: This is a fixed-window implementation (counter resets when the +window expires). For true sliding-window behavior, consider migrating +to Redis with a sorted-set approach. State is per-process, so multiple +Gunicorn workers enforce limits independently. +""" + +import threading +import time + +from flask import current_app, g, request + +from mod_api.middleware.error_handler import make_error_response + +_rate_limit_store = {} # key -> {'count': int, 'window_start': float} +_rate_limit_lock = threading.Lock() +_eviction_counter = 0 +_EVICTION_INTERVAL = 100 # run cleanup every N requests +_MAX_ENTRIES = 10000 # hard limit on stored keys to prevent memory exhaustion + + +def _evict_stale_entries(): + """Prune entries older than 15 min to bound memory usage.""" + global _eviction_counter + with _rate_limit_lock: + _eviction_counter += 1 + if _eviction_counter < _EVICTION_INTERVAL: + return + _eviction_counter = 0 + now = time.time() + stale_keys = [ + key for key, entry in _rate_limit_store.items() + if (now - entry['window_start']) > 900 + ] + for key in stale_keys: + del _rate_limit_store[key] + + if len(_rate_limit_store) > _MAX_ENTRIES: + # Sort by window_start (oldest first) and evict until we are at 90% capacity + sorted_keys = sorted( + _rate_limit_store.keys(), + key=lambda k: _rate_limit_store[k]['window_start'] + ) + keys_to_remove = len(_rate_limit_store) - int(_MAX_ENTRIES * 0.9) + for key in sorted_keys[:keys_to_remove]: + del _rate_limit_store[key] + + +def _get_client_ip(): + """Extract the real client IP (ProxyFix handles X-Forwarded-For securely).""" + return request.remote_addr + + +def _get_rate_limit_key(): + """Build the rate-limit bucket key for this request.""" + if request.endpoint == 'api.create_token': + return f'ip:{_get_client_ip()}' + token = getattr(g, 'api_token', None) + if token: + return f'token:{token.id}' + return f'ip:{_get_client_ip()}' + + +def _get_limits(): + """Return (max_requests, window_seconds) for the current endpoint.""" + if request.endpoint == 'api.create_token': + return 5, 900 + if request.method in ('POST', 'DELETE', 'PUT', 'PATCH'): + return 20, 60 + return 120, 60 + + +def check_rate_limit(): + """Apply rate limits based on client IP or API token.""" + if current_app.config.get('TESTING'): + return + + _evict_stale_entries() + + key = _get_rate_limit_key() + max_requests, window_seconds = _get_limits() + now = time.time() + + with _rate_limit_lock: + entry = _rate_limit_store.get(key) + + if entry is None or (now - entry['window_start']) >= window_seconds: + _rate_limit_store[key] = {'count': 1, 'window_start': now} + else: + entry['count'] += 1 + if entry['count'] > max_requests: + reset_at = int(entry['window_start'] + window_seconds) + retry_after = max(1, reset_at - int(now)) + + response = make_error_response( + 'rate_limited', + f'Rate limit exceeded. Retry after {retry_after} seconds.', + details={ + 'retry_after': retry_after, + 'limit': max_requests, + 'window': f'{window_seconds}s', + }, + http_status=429, + ) + response.headers['Retry-After'] = str(retry_after) + response.headers['X-RateLimit-Limit'] = str(max_requests) + response.headers['X-RateLimit-Remaining'] = '0' + response.headers['X-RateLimit-Reset'] = str(reset_at) + return response + + +def add_rate_limit_headers(response): + """Inject X-RateLimit-* headers based on the current window.""" + if current_app.config.get('TESTING') or response.status_code == 429: + return response + + key = _get_rate_limit_key() + max_requests, window_seconds = _get_limits() + now = time.time() + + with _rate_limit_lock: + entry = _rate_limit_store.get(key) + if entry: + remaining = max(0, max_requests - entry['count']) + reset_at = int(entry['window_start'] + window_seconds) + else: + remaining = max_requests + reset_at = int(now + window_seconds) + + response.headers['X-RateLimit-Limit'] = str(max_requests) + response.headers['X-RateLimit-Remaining'] = str(remaining) + response.headers['X-RateLimit-Reset'] = str(reset_at) + + return response diff --git a/mod_api/middleware/security.py b/mod_api/middleware/security.py new file mode 100644 index 000000000..c639b006c --- /dev/null +++ b/mod_api/middleware/security.py @@ -0,0 +1,10 @@ +"""Security headers middleware for API responses.""" + + +def add_security_headers(response): + """Attach security headers to all API responses.""" + response.headers['Strict-Transport-Security'] = 'max-age=31536000; includeSubDomains' + response.headers['Content-Security-Policy'] = "default-src 'none'; frame-ancestors 'none'" + response.headers['X-Content-Type-Options'] = 'nosniff' + response.headers['X-Frame-Options'] = 'DENY' + return response diff --git a/mod_api/middleware/validation.py b/mod_api/middleware/validation.py new file mode 100644 index 000000000..7922db568 --- /dev/null +++ b/mod_api/middleware/validation.py @@ -0,0 +1,309 @@ +""" +Request validation decorators for bodies, query params, and path IDs. + +All of these return 400 with field-level details on failure, so route +handlers can assume clean input. +""" + +from datetime import datetime, timezone +from functools import wraps + +from flask import request +from marshmallow import ValidationError as MarshmallowValidationError + +from mod_api.middleware.error_handler import make_error_response + +# Whitelist of allowed sort params. +ALLOWED_RUN_SORTS = frozenset([ + 'created_at', '-created_at', + 'run_id', '-run_id', +]) + + +def validate_body(schema_class): + """Validate the JSON body with a schema, pass result as ``validated_data``.""" + def decorator(f): + @wraps(f) + def decorated(*args, **kwargs): + content_type = request.content_type or '' + if content_type.split(';')[0].strip() != 'application/json': + return make_error_response( + 'validation_error', + 'Content-Type must be application/json.', + http_status=415, + ) + json_data = request.get_json(silent=True) + if json_data is None: + return make_error_response( + 'validation_error', + 'Request body must be valid JSON.', + http_status=400, + ) + schema = schema_class() + try: + validated = schema.load(json_data) + except MarshmallowValidationError as e: + return make_error_response( + 'validation_error', + 'Request failed schema validation.', + details={'fields': e.messages}, + http_status=400, + ) + kwargs['validated_data'] = validated + return f(*args, **kwargs) + return decorated + return decorator + + +def validate_offset_pagination(default_limit=50): + """Extract and validate ``limit`` and ``offset`` query params.""" + def decorator(f): + @wraps(f) + def decorated(*args, **kwargs): + if 'cursor' in request.args: + return make_error_response( + 'validation_error', + 'Cannot mix cursor and offset pagination.', + details={'fields': { + 'cursor': 'Cannot specify cursor when using offset pagination.'}}, + http_status=400, + ) + + try: + limit = int(request.args.get('limit', default_limit)) + except (ValueError, TypeError): + return make_error_response( + 'validation_error', + 'limit must be an integer.', + details={'fields': { + 'limit': 'Must be an integer between 1 and 100.'}}, + http_status=400, + ) + + try: + offset = int(request.args.get('offset', 0)) + except (ValueError, TypeError): + return make_error_response( + 'validation_error', + 'offset must be a non-negative integer.', + details={'fields': { + 'offset': 'Must be a non-negative integer.'}}, + http_status=400, + ) + + if limit < 1 or limit > 100: + return make_error_response( + 'validation_error', + 'limit must be between 1 and 100.', + details={'fields': {'limit': 'Must be between 1 and 100.'}}, + http_status=400, + ) + + if offset < 0: + return make_error_response( + 'validation_error', + 'offset must be non-negative.', + details={'fields': {'offset': 'Must be >= 0.'}}, + http_status=400, + ) + + if offset > 2147483647: + return make_error_response( + 'validation_error', + 'offset is too large.', + details={'fields': {'offset': 'Must be <= 2147483647.'}}, + http_status=400, + ) + + kwargs['limit'] = limit + kwargs['offset'] = offset + return f(*args, **kwargs) + return decorated + return decorator + + +def _parse_limit(default_limit): + try: + limit = int(request.args.get('limit', default_limit)) + except (ValueError, TypeError): + return None, make_error_response( + 'validation_error', + 'limit must be an integer.', + details={'fields': {'limit': 'Must be an integer between 1 and 100.'}}, + http_status=400, + ) + + if limit < 1 or limit > 100: + return None, make_error_response( + 'validation_error', + 'limit must be between 1 and 100.', + details={'fields': {'limit': 'Must be between 1 and 100.'}}, + http_status=400, + ) + return limit, None + + +def _parse_cursor(): + cursor = request.args.get('cursor') + if cursor is None: + return None, None + try: + cursor = int(cursor) + except (ValueError, TypeError): + return None, make_error_response( + 'validation_error', + 'cursor must be an integer.', + details={'fields': {'cursor': 'Must be an integer.'}}, + http_status=400, + ) + if cursor < 0: + return None, make_error_response( + 'validation_error', + 'cursor must be non-negative.', + details={'fields': {'cursor': 'Must be >= 0.'}}, + http_status=400, + ) + if cursor > 10_000_000: + return None, make_error_response( + 'validation_error', + 'cursor out of range.', + details={'fields': {'cursor': 'Must be <= 10000000.'}}, + http_status=400, + ) + return cursor, None + + +def validate_cursor_pagination(default_limit=50): + """Extract and validate ``limit`` and ``cursor`` query params.""" + def decorator(f): + @wraps(f) + def decorated(*args, **kwargs): + if 'offset' in request.args: + return make_error_response( + 'validation_error', + 'Cannot mix cursor and offset pagination.', + details={'fields': { + 'offset': 'Cannot specify offset when using cursor pagination.'}}, + http_status=400, + ) + + limit, err = _parse_limit(default_limit) + if err: + return err + + cursor, err = _parse_cursor() + if err: + return err + + kwargs['limit'] = limit + kwargs['cursor'] = cursor + return f(*args, **kwargs) + return decorated + return decorator + + +def validate_path_id(param_name): + """Ensure a URL path parameter is a positive integer.""" + def decorator(f): + @wraps(f) + def decorated(*args, **kwargs): + value = kwargs.get(param_name) + try: + int_value = int(value) + except (ValueError, TypeError): + return make_error_response( + 'validation_error', + f'{param_name} must be a positive integer.', + details={ + 'fields': { + param_name: 'Must be a positive integer.'}}, + http_status=400, + ) + if int_value < 1 or int_value > 2147483647: + return make_error_response( + 'validation_error', + f'{param_name} must be between 1 and 2147483647.', + details={ + 'fields': { + param_name: 'Must be between 1 and 2147483647. Out of bounds IDs are rejected.' + } + }, + http_status=400, + ) + kwargs[param_name] = int_value + return f(*args, **kwargs) + return decorated + return decorator + + +def _parse_iso8601_date(param_name, param_str): + if not param_str: + return None, None + try: + dt = datetime.fromisoformat(param_str.replace('Z', '+00:00')) + except ValueError: + return None, make_error_response( + 'validation_error', + f'{param_name} must be a valid ISO 8601 datetime.', + details={'fields': {param_name: 'Invalid ISO 8601 format.'}}, + http_status=400, + ) + if dt.tzinfo is None: + dt = dt.replace(tzinfo=timezone.utc) + return dt, None + + +def validate_date_range(f): + """Parse date query params and reject inverted ranges.""" + @wraps(f) + def decorated(*args, **kwargs): + created_after_str = request.args.get('created_after') + created_before_str = request.args.get('created_before') + + created_after, err = _parse_iso8601_date('created_after', created_after_str) + if err: + return err + + created_before, err = _parse_iso8601_date('created_before', created_before_str) + if err: + return err + + if created_after and created_before and created_after > created_before: + return make_error_response( + 'validation_error', + 'created_after cannot be later than created_before.', + details={'fields': { + 'created_after': 'Cannot be after created_before.'}}, + http_status=400, + ) + + kwargs['created_after'] = created_after + kwargs['created_before'] = created_before + return f(*args, **kwargs) + return decorated + + +def validate_sort(allowed=None): + """Validate the ``sort`` query param against a whitelist.""" + if allowed is None: + allowed = ALLOWED_RUN_SORTS + + def decorator(f): + @wraps(f) + def decorated(*args, **kwargs): + sort = request.args.get('sort', '-created_at') + if sort not in allowed: + return make_error_response( + 'validation_error', + f'sort must be one of: {", ".join(sorted(allowed))}', + details={ + 'fields': { + 'sort': f'Must be one of: {sorted(allowed)}' + } + }, + http_status=400, + ) + kwargs['sort'] = sort + return f(*args, **kwargs) + return decorated + return decorator diff --git a/mod_api/models/__init__.py b/mod_api/models/__init__.py new file mode 100644 index 000000000..dcb36537a --- /dev/null +++ b/mod_api/models/__init__.py @@ -0,0 +1 @@ +"""mod_api.models: database models for the API module.""" diff --git a/mod_api/models/api_token.py b/mod_api/models/api_token.py new file mode 100644 index 000000000..a4ec26471 --- /dev/null +++ b/mod_api/models/api_token.py @@ -0,0 +1,175 @@ +""" +ApiToken model: server-side storage for scoped API tokens. + +Tokens are opaque strings prefixed with 'spci_'. Only the SHA-256 hash +is persisted; the plaintext is returned exactly once at creation time. +A fast hash with a constant-time compare is sufficient here because the +tokens are 256-bit random secrets — a slow password KDF (argon2/bcrypt) +buys nothing against brute force on high-entropy values. +""" + +import hashlib +import hmac +import json +import secrets +from datetime import datetime, timedelta, timezone +from typing import List + +from sqlalchemy import (Column, DateTime, ForeignKey, Integer, String, Text, + UniqueConstraint) +from sqlalchemy.orm import relationship, validates + +from database import Base + + +class Scope: + """The scopes a token can carry. + + Reference these constants instead of writing the strings inline, so a + rename stays a single edit. Plain strings rather than a DeclEnum + because scopes cross the wire: they arrive in request bodies, are + stored as a JSON array, and are compared against what the client sent. + """ + + RUNS_READ = 'runs:read' + RUNS_WRITE = 'runs:write' + RESULTS_READ = 'results:read' + BASELINES_WRITE = 'baselines:write' + SYSTEM_READ = 'system:read' + TOKENS_MANAGE = 'tokens:manage' + + +VALID_SCOPES = frozenset([ + Scope.RUNS_READ, + Scope.RUNS_WRITE, + Scope.RESULTS_READ, + Scope.BASELINES_WRITE, + Scope.SYSTEM_READ, + Scope.TOKENS_MANAGE, +]) + +DEFAULT_SCOPES = [Scope.RUNS_READ, Scope.RESULTS_READ] + +TOKEN_PREFIX = 'spci_' +TOKEN_BYTE_LENGTH = 32 + + +class ApiToken(Base): + """Scoped API token bound to a user account.""" + + __tablename__ = 'api_token' + __table_args__ = ( + UniqueConstraint('user_id', 'token_name', name='uq_user_token_name'), + {'mysql_engine': 'InnoDB'}, + ) + + id = Column(Integer, primary_key=True) + user_id = Column( + Integer, + ForeignKey('user.id', onupdate='CASCADE', ondelete='CASCADE'), + nullable=False, + ) + user = relationship('User', uselist=False) + token_name = Column(String(50), nullable=False) + token_hash = Column(String(255), nullable=False) + token_prefix = Column(String(16), nullable=False, index=True) + scopes_json = Column(Text(), nullable=False) + created_at = Column(DateTime(timezone=True), nullable=False) + expires_at = Column(DateTime(timezone=True), nullable=False) + revoked_at = Column(DateTime(timezone=True), nullable=True) + + @validates('scopes_json') + def validate_scopes_json(self, key, value): + """Ensure scopes_json only contains known scopes.""" + try: + scopes = json.loads(value) + except json.JSONDecodeError: + raise ValueError("scopes_json must be a valid JSON string") + + if not isinstance(scopes, list): + raise ValueError("scopes_json must be a JSON array") + + for scope in scopes: + if scope not in VALID_SCOPES: + raise ValueError(f"Unknown scope: {scope}") + return value + + def __init__( + self, + user_id: int, + token_name: str, + token_hash: str, + token_prefix: str, + scopes: List[str], + expires_in_days: int = 7, + ) -> None: + self.user_id = user_id + self.token_name = token_name + self.token_hash = token_hash + self.token_prefix = token_prefix + self.scopes_json = json.dumps(scopes) + self.created_at = datetime.now(timezone.utc) + self.expires_at = self.created_at + timedelta(days=expires_in_days) + + def __repr__(self) -> str: + """Return a debug representation of the token.""" + return f'' + + @property + def scopes(self) -> List[str]: + """Parse the JSON scopes column into a list.""" + return json.loads(self.scopes_json) + + @property + def is_expired(self) -> bool: + """Check whether this token has passed its expiration time.""" + now = datetime.now(timezone.utc) + expires = self.expires_at + if expires is None: + return True + # MySQL DATETIME columns don't preserve tzinfo; treat naive as UTC. + if expires.tzinfo is None: + expires = expires.replace(tzinfo=timezone.utc) + return bool(now > expires) + + @property + def is_revoked(self) -> bool: + """Check whether this token has been explicitly revoked.""" + return bool(self.revoked_at is not None) + + @property + def is_valid(self) -> bool: + """Return True if the token is neither expired nor revoked.""" + return not self.is_expired and not self.is_revoked + + def has_scope(self, scope: str) -> bool: + """Return True if the token grants the given scope.""" + return scope in self.scopes + + def revoke(self) -> None: + """Mark this token as revoked with the current timestamp.""" + self.revoked_at = datetime.now(timezone.utc) + + @staticmethod + def generate_token() -> str: + """Create a new random token string with the spci_ prefix.""" + random_bytes = secrets.token_urlsafe(TOKEN_BYTE_LENGTH) + return f'{TOKEN_PREFIX}{random_bytes}' + + @staticmethod + def hash_token(plaintext: str) -> str: + """Hash a token securely using SHA-256.""" + return hashlib.sha256(plaintext.encode('utf-8')).hexdigest() + + @staticmethod + def verify_token(plaintext: str, token_hash: str) -> bool: + """Verify a token against its SHA-256 hash using constant-time comparison.""" + if not plaintext or not token_hash: + return False + expected_hash = ApiToken.hash_token(plaintext) + return hmac.compare_digest(expected_hash, token_hash) + + @staticmethod + def extract_prefix(token: str) -> str: + """Return the first 16 chars used for DB lookup.""" + return token[:16] if len(token) >= 16 else token diff --git a/mod_api/routes/__init__.py b/mod_api/routes/__init__.py new file mode 100644 index 000000000..eac65b967 --- /dev/null +++ b/mod_api/routes/__init__.py @@ -0,0 +1 @@ +"""mod_api.routes — Endpoint handlers for the API.""" diff --git a/mod_api/routes/auth.py b/mod_api/routes/auth.py new file mode 100644 index 000000000..d39d6e80d --- /dev/null +++ b/mod_api/routes/auth.py @@ -0,0 +1,213 @@ +""" +Token lifecycle: create, list, and revoke API tokens. + +POST /auth/tokens Authenticate with email/password, get a token +GET /auth/tokens List tokens (admin-only; ?all=true for all users) +DELETE /auth/tokens/current Revoke the token you're currently using +DELETE /auth/tokens/{id} Revoke a specific token by ID +""" + +from flask import g, request +from passlib.apps import custom_app_context as pwd_context +from sqlalchemy.exc import IntegrityError + +from mod_api import mod_api +from mod_api.middleware.auth import require_roles, require_scope +from mod_api.middleware.error_handler import make_error_response +from mod_api.middleware.validation import (validate_body, + validate_offset_pagination) +from mod_api.models.api_token import DEFAULT_SCOPES, ApiToken, Scope +from mod_api.schemas.auth import (ApiTokenItemSchema, AuthTokenSchema, + TokenCreateRequestSchema) +from mod_api.utils import paginated_response, single_response +from mod_auth.models import Role, User + +_DUMMY_HASH = pwd_context.hash('__dummy__') + + +@mod_api.route('/auth/tokens', methods=['POST']) +@validate_body(TokenCreateRequestSchema) +def create_token(validated_data=None): + """ + Authenticate with email + password and issue a scoped API token. + + The plaintext token value is returned exactly once in this response. + It's never stored or logged — only the SHA-256 hash is persisted + (see ApiToken: the token is a 256-bit random secret, so a fast hash + with constant-time compare is sufficient). + """ + email = validated_data['email'] + password = validated_data['password'] + token_name = validated_data['token_name'] + expires_in_days = validated_data.get('expires_in_days', 7) + scopes = validated_data.get('scopes') or DEFAULT_SCOPES + + user = User.query.filter_by(email=email).first() + + # Hash password even if user is not found to prevent timing attacks + if user is None: + try: + pwd_context.verify(password, _DUMMY_HASH) + except Exception: + pass + return make_error_response( + 'invalid_credentials', + 'Invalid email or password.', + http_status=401, + ) + + if not user.is_password_valid(password): + return make_error_response( + 'invalid_credentials', + 'Invalid email or password.', + http_status=401, + ) + + # Check role limitations + # Note: Plain 'user' role deliberately cannot request tokens:manage. They + # can create tokens with runs:write but cannot list them. They must revoke + # either the current token or by ID. + allowed_scopes = { + Scope.RUNS_READ, Scope.RUNS_WRITE, Scope.RESULTS_READ, + Scope.SYSTEM_READ, + } + if user.is_admin: + allowed_scopes.add(Scope.TOKENS_MANAGE) + allowed_scopes.add(Scope.BASELINES_WRITE) + + invalid_scopes = set(scopes) - allowed_scopes + if invalid_scopes: + return make_error_response( + 'forbidden', + f'Your current role ({user.role.value}) does not permit requesting ' + f'the following scopes: {", ".join(invalid_scopes)}.', + http_status=403, + ) + + plaintext = ApiToken.generate_token() + token_hash = ApiToken.hash_token(plaintext) + token_prefix = ApiToken.extract_prefix(plaintext) + + api_token = ApiToken( + user_id=user.id, + token_name=token_name, + token_hash=token_hash, + token_prefix=token_prefix, + scopes=scopes, + expires_in_days=expires_in_days, + ) + g.db.add(api_token) + + try: + g.db.commit() + except IntegrityError as e: + g.db.rollback() + error_msg = str(e).lower() + if 'uq_user_token_name' in error_msg or 'api_token.user_id, api_token.token_name' in error_msg: + # Names stay reserved even after revocation (the unique + # constraint spans revoked rows, kept for audit history), + # so "revoke and retry" would not free the name. + return make_error_response( + 'validation_error', + f'Token name "{token_name}" already exists for this user. ' + 'Names remain reserved after revocation; choose a new name.', + details={'fields': { + 'token_name': 'Already in use (including by revoked ' + 'tokens). Choose a different name.'}}, + http_status=400, + ) + raise + + return single_response( + { + 'token': plaintext, + 'token_type': 'bearer', + 'token_name': token_name, + 'scopes': scopes, + 'expires_at': api_token.expires_at, + }, + schema=AuthTokenSchema(), + http_status=201, + ) + + +@mod_api.route('/auth/tokens/current', methods=['DELETE']) +def revoke_current_token(): + """Revoke whatever token is in the Authorization header right now. + + Note: This endpoint is intentionally scope-free. Any valid token + is allowed to revoke itself regardless of its scopes. + """ + token = getattr(g, 'api_token', None) + if token is None: + return make_error_response( + 'unauthorized', + 'No token found in the current request.', + http_status=401, + ) + token.revoke() + g.db.add(token) + g.db.commit() + return '', 204 + + +@mod_api.route('/auth/tokens', methods=['GET']) +@require_roles([Role.admin]) +@require_scope(Scope.TOKENS_MANAGE) +@validate_offset_pagination() +def list_tokens(limit=50, offset=0): + """ + List API tokens, paginated. Admin-only. + + tokens:manage is an admin-only scope (see create_token), so the + require_roles guard above already rejects everyone else with 403. + Lists the caller's own tokens by default; pass ?all=true to list + every token in the system. + """ + want_all = request.args.get('all', 'false').lower() == 'true' + + if want_all: + query = ApiToken.query.order_by(ApiToken.created_at.desc()) + else: + query = ApiToken.query.filter_by( + user_id=g.api_user.id, + ).order_by(ApiToken.created_at.desc()) + + total = query.count() + tokens = query.offset(offset).limit(limit).all() + schema = ApiTokenItemSchema(many=True) + + return paginated_response(tokens, total, limit, offset, schema=schema) + + +@mod_api.route('/auth/tokens/', methods=['DELETE']) +def revoke_specific_token(token_id): + """ + Revoke a token by its numeric ID. + + Non-admins can only revoke their own tokens. Admins can revoke anyone's. + Already-revoked tokens are silently accepted (idempotent). + + Deliberately requires no extra scope: scopes gate data access, while + revocation is self-service credential hygiene. Any valid token may + revoke tokens belonging to its own user — plain users cannot obtain + tokens:manage (see create_token), yet must be able to clean up their + own credentials. + """ + is_admin = g.api_user.is_admin + token = ApiToken.query.filter_by(id=token_id).first() + + # Non-admins get a uniform 404 for both "doesn't exist" and "belongs to + # another user" to prevent token-ID enumeration. + is_own = token is not None and token.user_id == g.api_user.id + if not token or (not is_admin and not is_own): + return make_error_response('not_found', 'Token not found.', http_status=404) + + # Reaching here means the caller is either the owner or an admin (any other + # caller was already given a 404 above), so the revocation is authorized. + if not token.is_revoked: + token.revoke() + g.db.add(token) + g.db.commit() + + return '', 204 diff --git a/mod_api/routes/results.py b/mod_api/routes/results.py new file mode 100644 index 000000000..57a49078b --- /dev/null +++ b/mod_api/routes/results.py @@ -0,0 +1,475 @@ +""" +Expected/actual output, diffs, and baseline approval routes. + +GET /runs/{id}/samples/{sid}/regression-tests/{rid}/outputs/{oid}/expected +GET /runs/{id}/samples/{sid}/regression-tests/{rid}/outputs/{oid}/actual +GET /runs/{id}/samples/{sid}/regression-tests/{rid}/outputs/{oid}/diff +POST /runs/{id}/samples/{sid}/baseline-approval Approve a new baseline +""" + +import base64 +import os + +from flask import current_app, g, redirect, request, url_for + +from mod_api import mod_api +from mod_api.middleware.auth import require_roles, require_scope +from mod_api.middleware.error_handler import make_error_response +from mod_api.middleware.validation import validate_body, validate_path_id +from mod_api.models.api_token import Scope +from mod_api.schemas.results import (BaselineApprovalRequestSchema, + BaselineApprovalSchema, + OutputFileContentSchema) +from mod_api.services.diff_service import compute_diff, file_sha256, read_lines +from mod_api.services.status import is_dummy_row +from mod_api.services.storage import (get_test_results_base_path, + resolve_artifact) +from mod_api.utils import safe_resolve, single_response +from mod_auth.models import Role +from mod_regression.models import RegressionTestOutputFiles +from mod_test.models import Test, TestResultFile + +INVALID_PATH_MSG = 'Invalid file path.' +READ_ERROR_MSG = 'Failed to read file.' + + +def _find_result_file(run_id, regression_test_id, output_id=None): + """ + Look up the right TestResultFile row. + + Uses run_id + regression_test_id from the path. If output_id is + given as a query param, narrow to that specific output file. + """ + query = TestResultFile.query.filter_by( + test_id=run_id, + regression_test_id=regression_test_id, + ) + + if output_id is not None: + query = query.filter_by(regression_test_output_id=output_id) + + return query.first() + + +def _validate_result_file_access(run_id, sample_id, regression_id, output_id): + """Validate access to a result file and return it, or an error response.""" + test = Test.query.filter(Test.id == run_id).first() + if test is None: + return None, make_error_response('not_found', f'Run {run_id} not found.', http_status=404) + + result_file = _find_result_file(run_id, regression_id, output_id) + + if result_file is None: + return None, make_error_response( + 'not_found', + f'No result for regression test {regression_id}.', + http_status=404, + ) + + actual_sample_id = ( + result_file.regression_test.sample_id + if result_file.regression_test else None + ) + if actual_sample_id != sample_id: + return None, make_error_response( + 'not_found', + f'Regression test {regression_id} does not belong to sample {sample_id}.', + http_status=404, + ) + + return result_file, None + + +def _read_output_file(file_path, fmt, is_expected=True): + """Read an output file into the response dict. + + The sha256 in the result is always computed over the complete file, + even when content is truncated to the 1 MiB inline limit. + """ + if not os.path.isfile(file_path): + type_str = 'Expected' if is_expected else 'Actual' + return None, make_error_response( + 'not_found', + f'{type_str} output file not found on disk.', + http_status=404, + ) + + sha256 = file_sha256(file_path) + file_size = os.path.getsize(file_path) + truncated = False + download_url = None + + if file_size > 1048576: + truncated = True + filename = os.path.basename(file_path) + download_url, _ = resolve_artifact(f'TestResults/{filename}') + + if fmt == 'text': + try: + with open(file_path, 'r', encoding='utf-8', errors='replace') as f: + content = f.read(1048576) + encoding = 'utf-8' + except Exception: + return None, make_error_response('internal_error', READ_ERROR_MSG, http_status=500) + else: + try: + with open(file_path, 'rb') as f: + content = base64.b64encode(f.read(1048576)).decode('ascii') + encoding = 'base64' + except Exception: + return None, make_error_response('internal_error', READ_ERROR_MSG, http_status=500) + + return { + 'content': content, + 'encoding': encoding, + 'sha256': sha256, + 'truncated': truncated, + 'download_url': download_url, + }, None + + +@mod_api.route( + '/runs//samples//regression-tests//outputs//expected', + methods=['GET'] +) +@require_scope(Scope.RESULTS_READ) +@validate_path_id('run_id') +@validate_path_id('sample_id') +@validate_path_id('regression_id') +@validate_path_id('output_id') +def get_expected_output(run_id, sample_id, regression_id, output_id): + """Return the expected output file for a regression test result.""" + result_file, err = _validate_result_file_access( + run_id, sample_id, regression_id, output_id) + if err: + return err + + if is_dummy_row(result_file): + return make_error_response('not_found', 'Expected output not found.', http_status=404) + + base_path = get_test_results_base_path() + expected_filename = result_file.expected + ext = '' + if result_file.regression_test_output: + ext = result_file.regression_test_output.correct_extension + if ext: + ext = ext.replace('/', '').replace('\\', '').replace('..', '') + expected_filename += ext + + file_path = safe_resolve(base_path, expected_filename) + if file_path is None: + return make_error_response('forbidden', INVALID_PATH_MSG, http_status=403) + + fmt = request.args.get('format', 'base64') + + data, err = _read_output_file(file_path, fmt, is_expected=True) + if err: + return err + + content = data['content'] + encoding = data['encoding'] + sha256 = data['sha256'] + truncated = data['truncated'] + download_url = data['download_url'] + + _, storage_status = resolve_artifact(f'TestResults/{expected_filename}') + + return single_response({ + 'run_id': run_id, + 'sample_id': sample_id, + 'regression_id': result_file.regression_test_id, + 'output_id': result_file.regression_test_output_id, + 'filename': expected_filename, + 'content_type': 'application/octet-stream', + 'encoding': encoding, + 'content': content, + 'truncated': truncated, + 'download_url': download_url, + 'sha256': sha256, + 'storage_status': storage_status, + }, schema=OutputFileContentSchema()) + + +@mod_api.route( + '/runs//samples//regression-tests//outputs//actual', + methods=['GET'] +) +@require_scope(Scope.RESULTS_READ) +@validate_path_id('run_id') +@validate_path_id('sample_id') +@validate_path_id('regression_id') +@validate_path_id('output_id') +def get_actual_output(run_id, sample_id, regression_id, output_id): + """ + Return the actual output file for a regression test result. + + got=null in the DB means the output matched expected — not that it's + missing. We return 303 (redirect to expected) in that case. Missing + output (the dummy sentinel row) returns 404. + """ + result_file, err = _validate_result_file_access( + run_id, sample_id, regression_id, output_id) + if err: + return err + + if is_dummy_row(result_file): + return make_error_response( + 'missing_output', + 'Test produced no output when output was expected.', + http_status=404, + ) + + if result_file.got is None: + # Relative redirect: clients only resend Authorization headers on + # same-origin redirects, and an absolute URL would break behind a + # reverse proxy anyway. + return redirect(url_for( + 'api.get_expected_output', + run_id=run_id, + sample_id=sample_id, + regression_id=regression_id, + output_id=output_id, + format=request.args.get('format', 'base64'), + ), code=303) + + base_path = get_test_results_base_path() + actual_filename = result_file.got + if result_file.regression_test_output: + ext = result_file.regression_test_output.correct_extension + if ext: + ext = ext.replace('/', '').replace('\\', '').replace('..', '') + actual_filename += ext + + file_path = safe_resolve(base_path, actual_filename) + if file_path is None: + return make_error_response('forbidden', INVALID_PATH_MSG, http_status=403) + + fmt = request.args.get('format', 'base64') + + data, err = _read_output_file(file_path, fmt, is_expected=False) + if err: + return err + + content = data['content'] + encoding = data['encoding'] + sha256 = data['sha256'] + truncated = data['truncated'] + download_url = data['download_url'] + + _, storage_status = resolve_artifact(f'TestResults/{actual_filename}') + + return single_response({ + 'run_id': run_id, + 'sample_id': sample_id, + 'regression_id': result_file.regression_test_id, + 'output_id': result_file.regression_test_output_id, + 'filename': actual_filename, + 'content_type': 'application/octet-stream', + 'encoding': encoding, + 'content': content, + 'truncated': truncated, + 'download_url': download_url, + 'sha256': sha256, + 'storage_status': storage_status, + }, schema=OutputFileContentSchema()) + + +def _handle_missing_diff(result_file, format_type, diff_ids): + if is_dummy_row(result_file): + if format_type == 'unified': + return single_response({**diff_ids, 'format': 'unified', 'content': ''}) + return single_response({ + **diff_ids, + 'status': 'missing_actual', + 'format': 'structured', + 'summary': {'added_lines': 0, 'removed_lines': 0, 'changed_hunks': 0}, + 'hunks': [], + }) + + if result_file.got is None: + if format_type == 'unified': + return single_response({**diff_ids, 'format': 'unified', 'content': ''}) + return single_response({ + **diff_ids, + 'status': 'identical', + 'format': 'structured', + 'summary': {'added_lines': 0, 'removed_lines': 0, 'changed_hunks': 0}, + 'hunks': [], + }) + return None + + +@mod_api.route( + '/runs//samples//regression-tests//outputs//diff', + methods=['GET'] +) +@require_scope(Scope.RESULTS_READ) +@validate_path_id('run_id') +@validate_path_id('sample_id') +@validate_path_id('regression_id') +@validate_path_id('output_id') +def get_diff(run_id, sample_id, regression_id, output_id): + """Structured diff between expected and actual output.""" + result_file, err = _validate_result_file_access( + run_id, sample_id, regression_id, output_id) + if err: + return err + + diff_ids = { + 'run_id': run_id, + 'sample_id': sample_id, + 'regression_id': result_file.regression_test_id, + 'output_id': result_file.regression_test_output_id, + } + + format_type = request.args.get('format', 'structured') + + missing_response = _handle_missing_diff(result_file, format_type, diff_ids) + if missing_response: + return missing_response + + base_path = get_test_results_base_path() + ext = result_file.regression_test_output.correct_extension if result_file.regression_test_output else '' + if ext: + ext = ext.replace('/', '').replace('\\', '').replace('..', '') + expected_path = safe_resolve(base_path, result_file.expected + ext) + actual_path = safe_resolve(base_path, result_file.got + ext) + + if expected_path is None or actual_path is None: + return make_error_response('forbidden', INVALID_PATH_MSG, http_status=403) + + if not os.path.isfile(expected_path): + return make_error_response('not_found', 'Expected output file not found on disk.', http_status=404) + if not os.path.isfile(actual_path): + return make_error_response('not_found', 'Actual output file not found on disk.', http_status=404) + + max_diff_bytes = 10 * 1024 * 1024 # 10 MiB + if os.path.getsize(expected_path) > max_diff_bytes or os.path.getsize(actual_path) > max_diff_bytes: + return make_error_response('unprocessable', 'File too large for diff. Use download_url.', http_status=422) + + if format_type == 'unified': + import difflib + expected_lines = read_lines(expected_path) + actual_lines = read_lines(actual_path) + differ = list(difflib.unified_diff( + expected_lines, + actual_lines, + fromfile='expected', + tofile='actual', + lineterm='' + )) + if len(differ) > 10000: + differ = differ[:10000] + differ.append("\n... Diff truncated due to length ...") + unified_content = '\n'.join(differ) + return single_response({ + **diff_ids, + 'format': 'unified', + 'content': unified_content + }) + + context_lines = request.args.get('context_lines', 3, type=int) + context_lines = max(1, min(context_lines, 50)) + + diff_result = compute_diff( + expected_path, actual_path, context_lines=context_lines) + diff_result.update(diff_ids) + diff_result['format'] = 'structured' + return single_response(diff_result) + + +@mod_api.route('/runs//samples//baseline-approval', methods=['POST']) +@require_roles([Role.admin]) +@require_scope(Scope.BASELINES_WRITE) +@validate_path_id('run_id') +@validate_path_id('sample_id') +@validate_body(BaselineApprovalRequestSchema) +def create_baseline_approval(run_id, sample_id, validated_data=None): + """ + Record intent to approve actual output as the new expected baseline. + + WARNING: When remove_variants is set to true, this action will remove all + platform-specific variants, making this output the single source of truth + across all platforms. Care should be taken as this applies globally. + """ + test = Test.query.filter(Test.id == run_id).first() + if test is None: + return make_error_response('not_found', f'Run {run_id} not found.', http_status=404) + + regression_id = validated_data['regression_id'] + output_id = validated_data['output_id'] + + result_file = TestResultFile.query.filter_by( + test_id=run_id, + regression_test_id=regression_id, + regression_test_output_id=output_id, + ).first() + + if result_file is None: + return make_error_response('not_found', 'Result file not found.', http_status=404) + + actual_sample_id = ( + result_file.regression_test.sample_id + if result_file.regression_test else None + ) + if actual_sample_id != sample_id: + return make_error_response( + 'not_found', + f'Regression test {regression_id} does not belong to sample {sample_id}.', + http_status=404, + ) + + if is_dummy_row(result_file): + return make_error_response('unprocessable', 'Cannot approve a dummy row.', http_status=422) + + if result_file.got is None: + return make_error_response('unprocessable', 'Output already matches expected.', http_status=422) + + # The actual output file (named by its hash) is already in TestResults/. + # We just need to update the RegressionTestOutput to point to this new hash. + rto = result_file.regression_test_output + if rto is None: + return make_error_response('internal_error', 'No RegressionTestOutput linked.', http_status=500) + + new_baseline = result_file.got + + base_path = get_test_results_base_path() + ext = rto.correct_extension or '' + if ext: + ext = ext.replace('/', '').replace('\\', '').replace('..', '') + actual_filename = new_baseline + ext + file_path = safe_resolve(base_path, actual_filename) + if not file_path or not os.path.isfile(file_path): + return make_error_response('unprocessable', 'Actual output file not found in storage.', http_status=422) + + old_baseline = rto.correct + rto.correct = new_baseline + + remove_variants = validated_data.get('remove_variants', False) + if remove_variants: + RegressionTestOutputFiles.query.filter_by( + regression_test_output_id=rto.id).delete() + + g.db.commit() + + # Audit log: this mutates the global expected baseline (and optionally + # deletes all variants), so record who approved what. + approver = getattr(g, 'api_user', None) + current_app.logger.info( + 'Baseline approved by %s (user_id=%s): run=%s regression=%s ' + 'output=%s old_hash=%s new_hash=%s remove_variants=%s', + approver.name if approver else 'unknown', + approver.id if approver else None, + run_id, regression_id, output_id, old_baseline, new_baseline, + remove_variants) + + import datetime + return single_response({ + 'status': 'approved', + 'run_id': run_id, + 'sample_id': sample_id, + 'regression_id': regression_id, + 'output_id': output_id, + 'requested_by': getattr(g, 'api_user').name if getattr(g, 'api_user', None) else 'unknown', + 'created_at': datetime.datetime.now(datetime.timezone.utc) + }, schema=BaselineApprovalSchema()) diff --git a/mod_api/routes/runs.py b/mod_api/routes/runs.py new file mode 100644 index 000000000..1ff75a91d --- /dev/null +++ b/mod_api/routes/runs.py @@ -0,0 +1,677 @@ +""" +Test run routes. + +GET /runs List runs (filtered, paginated, sorted) +POST /runs Trigger a new run +GET /runs/{id} Single run details +GET /runs/{id}/summary Pass/fail/skip counts +GET /runs/{id}/progress Progress event timeline +GET /runs/{id}/config Run configuration and test matrix +POST /runs/{id}/cancel Cancel a queued or running test +""" + +from collections import defaultdict + +from flask import g, request +from sqlalchemy import func, or_ +from sqlalchemy.exc import IntegrityError +from sqlalchemy.orm import joinedload + +from mod_api import mod_api +from mod_api.middleware.auth import require_roles, require_scope +from mod_api.middleware.error_handler import make_error_response +from mod_api.middleware.validation import (validate_body, validate_date_range, + validate_offset_pagination, + validate_path_id, validate_sort) +from mod_api.models.api_token import Scope +from mod_api.schemas.runs import (ProgressEventSchema, RunCreateRequestSchema, + RunSchema, RunSummarySchema) +from mod_api.services.error_service import derive_errors_for_run +from mod_api.services.status import (batch_get_run_data, derive_run_status, + derive_sample_status) +from mod_api.utils import get_sort_column, paginated_response, single_response +from mod_auth.models import Role +from mod_customized.models import CustomizedTest +from mod_regression.models import RegressionTest, RegressionTestOutput +from mod_test.models import (Fork, Test, TestPlatform, TestProgress, + TestResult, TestResultFile, TestStatus, TestType) + + +def _serialize_run(test): + """Turn a Test row into the Run response shape the spec expects.""" + return _batch_serialize([test])[0] + + +def _batch_serialize(tests, statuses=None, timestamps=None): + if statuses is None or timestamps is None: + statuses, timestamps = batch_get_run_data(tests) + return [ + { + 'run_id': t.id, + 'status': statuses.get(t.id, 'queued'), + 'platform': t.platform.value, + 'test_type': 'pr' if t.test_type == TestType.pull_request else 'commit', + 'repository': t.fork.github_name if t.fork else 'unknown', + 'branch': t.branch, + 'commit_sha': t.commit, + 'pr_number': t.pr_nr if t.pr_nr and t.pr_nr > 0 else None, + 'created_at': timestamps.get(t.id, {}).get('created_at'), + 'queued_at': timestamps.get(t.id, {}).get('queued_at'), + 'started_at': timestamps.get(t.id, {}).get('started_at'), + 'completed_at': timestamps.get(t.id, {}).get('completed_at'), + 'github_link': t.github_link if t.fork else None, + } + for t in tests + ] + + +def _apply_repository_filter(query, repository): + repo_field = RunCreateRequestSchema().fields.get('repository') + if repo_field: + try: + repo_field.deserialize(repository) + except Exception as e: + return None, make_error_response( + 'validation_error', + 'Invalid repository format.', + details={'fields': {'repository': str(e)}}, + http_status=400, + ) + fork_url = f'https://github.com/{repository}.git' + return query.join(Fork).filter(Fork.github == fork_url), None + + +def _apply_date_filters(query, created_after, created_before): + first_progress = ( + g.db.query( + TestProgress.test_id, func.min( + TestProgress.timestamp).label('min_ts')) .group_by( + TestProgress.test_id) .subquery()) + # LEFT JOIN so queued runs (no TestProgress rows yet, hence no known + # creation time) aren't silently dropped — otherwise combining a date + # filter with ?status=queued always returns an empty page. Runs without + # timestamps are treated as matching any requested window. + query = query.outerjoin( + first_progress, Test.id == first_progress.c.test_id) + if created_after: + query = query.filter(or_(first_progress.c.min_ts >= created_after, + first_progress.c.min_ts.is_(None))) + if created_before: + query = query.filter(or_(first_progress.c.min_ts <= created_before, + first_progress.c.min_ts.is_(None))) + return query + + +def _apply_run_filters(query, created_after, created_before): + platform = request.args.get('platform') + if platform: + try: + platform_enum = TestPlatform.from_string(platform) + query = query.filter(Test.platform == platform_enum) + except Exception: + valid_platforms = ', '.join(TestPlatform.values()) + return None, make_error_response( + 'validation_error', + f'Invalid platform: {platform}. Must be one of: {valid_platforms}.', + http_status=400, + ) + + branch = request.args.get('branch') + if branch: + query = query.filter(Test.branch == branch) + + commit_sha = request.args.get('commit_sha') + if commit_sha: + query = query.filter(Test.commit == commit_sha) + + repository = request.args.get('repository') + if repository: + query, err = _apply_repository_filter(query, repository) + if err: + return None, err + + if created_after or created_before: + query = _apply_date_filters(query, created_after, created_before) + + return query, None + + +def _validate_run_permissions(user, target_repo, main_repo_full): + # GitHub owner/repo names are case-insensitive. + if target_repo.lower() == main_repo_full.lower(): + allowed = (Role.admin, Role.tester, Role.contributor) + if user.role not in allowed: + return make_error_response( + 'forbidden', + 'Only admins, testers, and contributors can trigger runs for the main repository.', + details={ + 'required_roles': [role.value for role in allowed], + 'repository': target_repo, + }, + http_status=403, + ) + else: + owner = target_repo.split('/')[0] + github_login = getattr(user, 'github_login', None) + + if not github_login and getattr(user, 'github_token', None): + from mod_auth.controllers import fetch_username_from_token + github_login = fetch_username_from_token(user) + if github_login: + user.github_login = github_login + g.db.add(user) + + github_login = github_login or '' + + if not github_login or owner.lower() != github_login.lower(): + return make_error_response( + 'forbidden', + f'You can only trigger runs for your own repository (expected owner: {github_login}) ' + 'or the main repository.', + details={ + 'repository': target_repo, + 'owner_required': github_login, + }, + http_status=403, + ) + return None + + +def _validate_regression_test_ids(regression_test_ids): + if regression_test_ids is not None: + if not regression_test_ids: + return None, make_error_response( + 'validation_error', + 'regression_test_ids cannot be empty.', + details={'fields': { + 'regression_test_ids': 'Must contain at least one ID.'}}, + http_status=400, + ) + active_tests = RegressionTest.query.filter( + RegressionTest.id.in_(regression_test_ids), + RegressionTest.active == True, # noqa: E712 + ).all() + active_ids = {t.id for t in active_tests} + inactive_ids = [ + tid for tid in regression_test_ids if tid not in active_ids] + if inactive_ids: + return None, make_error_response( + 'unprocessable', + 'Some regression test IDs are inactive or do not exist.', + details={'inactive_ids': inactive_ids}, + http_status=422, + ) + else: + active_tests = RegressionTest.query.filter_by(active=True).all() + regression_test_ids = [t.id for t in active_tests] + return regression_test_ids, None + + +@mod_api.route('/runs', methods=['GET']) +@require_scope(Scope.RUNS_READ) +@validate_offset_pagination() +@validate_sort() +@validate_date_range +def list_runs( + limit=50, + offset=0, + sort='-created_at', + created_after=None, + created_before=None): + """List runs with filters for platform, branch, commit, repo, status, and date range.""" + query, err = _apply_run_filters(Test.query, created_after, created_before) + if err: + return err + + sort_map = { + 'run_id': Test.id, + 'created_at': Test.id, # best proxy - Test has no created_at column + } + order = get_sort_column(sort, sort_map) + if order is not None: + query = query.order_by(order) + else: + query = query.order_by(Test.id.desc()) + + status_filter = request.args.get('status') + if status_filter: + if status_filter not in ('queued', 'running', 'canceled'): + return make_error_response( + 'validation_error', + f'Filtering by status "{status_filter}" is not supported. Supported: queued, running, canceled.', + http_status=400, + ) + + latest_progress_sq = ( + g.db.query(func.max(TestProgress.id).label('max_id')) + .group_by(TestProgress.test_id) + .subquery() + ) + + if status_filter == 'queued': + query = query.outerjoin(TestProgress).filter( + TestProgress.id.is_(None)) + elif status_filter == 'running': + query = query.join( + TestProgress, + TestProgress.test_id == Test.id) .filter( + TestProgress.id.in_(latest_progress_sq)) .filter( + TestProgress.status.in_( + [ + TestStatus.preparation, + TestStatus.testing])) + elif status_filter == 'canceled': + query = query.join(TestProgress, TestProgress.test_id == Test.id)\ + .filter(TestProgress.id.in_(latest_progress_sq))\ + .filter(TestProgress.status == TestStatus.canceled) + + total = query.count() + tests = query.offset(offset).limit(limit).all() + serialized = _batch_serialize(tests) + return paginated_response( + serialized, + total, + limit, + offset, + schema=RunSchema()) + + +def _get_or_create_fork(fork_url): + fork = Fork.query.filter(Fork.github == fork_url).first() + if fork is None: + fork = Fork(fork_url) + g.db.add(fork) + try: + g.db.flush() + except IntegrityError: + g.db.rollback() + fork = Fork.query.filter(Fork.github == fork_url).first() + if fork is None: + return None, make_error_response( + 'internal_error', 'Failed to create or resolve fork.', http_status=500) + return fork, None + + +def _ci_artifact_exists(commit_sha, platform): + """Return True if a CI build artifact exists for this commit + platform. + + The worker runs prebuilt binaries downloaded from GitHub Actions rather + than building from source, so a run can only execute if a build artifact + keyed to ``commit_sha`` exists on the main repo (this is also true for + fork PR commits, whose artifacts are produced by the main repo's PR + workflow). Mirrors verify_artifacts_exist() in the webhook path. + + Fails open (returns True) if GitHub can't be reached, so run creation + never depends on a successful artifact lookup — the cron still guards + against genuinely missing artifacts. + """ + from run import config, log + try: + from github import Auth, Github + + from mod_ci.controllers import find_artifact_for_commit + gh = Github(auth=Auth.Token(config.get('GITHUB_TOKEN', ''))) + repo = gh.get_repo( + f"{config.get('GITHUB_OWNER', '')}/{config.get('GITHUB_REPOSITORY', '')}") + return find_artifact_for_commit(repo, commit_sha, platform, log) is not None + except Exception: + log.exception( + 'create_run: artifact pre-check failed; allowing run to proceed') + return True + + +@mod_api.route('/runs', methods=['POST']) +@require_scope(Scope.RUNS_WRITE) +@validate_body(RunCreateRequestSchema) +def create_run(validated_data=None): + """Trigger a new test run for a commit + platform combination. + + CI worker pickup: the cron (run_cron.py) picks up any Test row that has + no 'completed'/'canceled' TestProgress, then runs the prebuilt GitHub + Actions artifact for that commit. We therefore reject up front any + commit+platform with no build artifact (see _ci_artifact_exists), so + the run isn't accepted only to fail asynchronously in the worker. + """ + commit_sha = validated_data['commit_sha'] + platform_str = validated_data['platform'] + branch = validated_data.get('branch', 'master') + repository = validated_data.get('repository') + pull_request = validated_data.get('pull_request') or 0 + regression_test_ids = validated_data.get('regression_test_ids') + + platform = TestPlatform.from_string(platform_str) + + # Main repo requires contributor+; forks allow any authenticated user. + from run import config + main_owner = config.get('GITHUB_OWNER', '') + main_repo = config.get('GITHUB_REPOSITORY', '') + main_repo_full = f'{main_owner}/{main_repo}' + # repository is a required field (RunCreateRequestSchema), so it is always + # present; a main-repo run passes the main repo's "owner/repo" explicitly. + target_repo = repository + + err = _validate_run_permissions(g.api_user, target_repo, main_repo_full) + if err: + return err + + # Reject commits with no CI build artifact — the worker runs prebuilt + # binaries, so such a run would be accepted but never execute. + if not _ci_artifact_exists(commit_sha, platform): + return make_error_response( + 'unprocessable', + f'No CI build artifact found for commit {commit_sha[:8]} on ' + f'{platform.value}. Ensure the build workflow has completed for ' + 'this commit before triggering a run.', + details={'commit_sha': commit_sha, 'platform': platform.value}, + http_status=422, + ) + + fork_url = f'https://github.com/{repository}.git' + + fork, err = _get_or_create_fork(fork_url) + if err: + return err + + # Validate regression test IDs against active tests only. + regression_test_ids, err = _validate_regression_test_ids( + regression_test_ids) + if err: + return err + + test_type = TestType.pull_request if pull_request else TestType.commit + + test = Test( + platform=platform, + test_type=test_type, + fork_id=fork.id, + branch=branch, + commit=commit_sha, + pr_nr=pull_request, + ) + g.db.add(test) + try: + g.db.flush() + except Exception: + g.db.rollback() + return make_error_response( + 'internal_error', + 'Failed to create run.', + http_status=500) + + for rt_id in regression_test_ids: + ct = CustomizedTest(test.id, rt_id) + g.db.add(ct) + try: + g.db.commit() + except Exception: + g.db.rollback() + return make_error_response( + 'internal_error', + 'Failed to finalize run.', + http_status=500) + + return single_response( + _serialize_run(test), + schema=RunSchema(), + http_status=202) + + +@mod_api.route('/runs/', methods=['GET']) +@require_scope(Scope.RUNS_READ) +@validate_path_id('run_id') +def get_run(run_id): + """Fetch a single run by ID.""" + test = Test.query.filter(Test.id == run_id).first() + if test is None: + return make_error_response( + 'not_found', + f'Run {run_id} not found.', + http_status=404) + + return single_response(_serialize_run(test), schema=RunSchema()) + + +def _run_regression_ids(test): + """Regression test IDs that belong to this run. + + Uses the customized selection when present; otherwise falls back to + every ACTIVE regression test, mirroring create_run's default. (The + model's get_customized_regressiontests() falls back to all tests + including inactive ones, which inflates total_samples/skipped_count + with tests the run could never execute.) + """ + if test.customized_tests: + return [ct.regression_id for ct in test.customized_tests] + return [rt.id for rt in + RegressionTest.query.filter_by(active=True).all()] + + +def _aggregate_run_statistics( + results, + files_by_result, + expected_outputs_by_rt): + pass_count = fail_count = skipped_count = missing_count = total_runtime = 0 + for result in results: + result_files = files_by_result.get(result.regression_test_id, []) + expected = expected_outputs_by_rt.get(result.regression_test_id) + status = derive_sample_status(result, result_files, expected) + + if status == 'pass': + pass_count += 1 + elif status == 'fail': + fail_count += 1 + elif status == 'missing_output': + missing_count += 1 + else: + skipped_count += 1 + + if result.runtime: + total_runtime += result.runtime + + return pass_count, fail_count, skipped_count, missing_count, total_runtime + + +@mod_api.route('/runs//summary', methods=['GET']) +@require_scope(Scope.RUNS_READ) +@validate_path_id('run_id') +def get_run_summary(run_id): + """ + Aggregate pass/fail/skip/missing/error counts from result rows. + + fail_count comes from TestResult rows, not from test.failed (which + only reflects cancellation status and is unreliable for this purpose). + """ + test = Test.query.filter(Test.id == run_id).first() + if test is None: + return make_error_response( + 'not_found', + f'Run {run_id} not found.', + http_status=404) + + results = TestResult.query.filter_by(test_id=run_id).all() + total_samples = len(_run_regression_ids(test)) + + # Preload TestResultFiles + + all_files = ( + TestResultFile.query.options( + joinedload(TestResultFile.regression_test_output) + .joinedload(RegressionTestOutput.multiple_files) + ) + .filter_by(test_id=run_id).all() if results else [] + ) + files_by_result = defaultdict(list) + for f in all_files: + files_by_result[f.regression_test_id].append(f) + + # Preload expected outputs + expected_outputs_by_rt = defaultdict(list) + if results: + all_expected = RegressionTestOutput.query.filter( + RegressionTestOutput.regression_id.in_([r.regression_test_id for r in results]) + ).all() + for rto in all_expected: + expected_outputs_by_rt[rto.regression_id].append(rto) + + pass_count, fail_count, skipped_count, missing_count, total_runtime = _aggregate_run_statistics( + results, files_by_result, expected_outputs_by_rt) + + # Reconcile skipped samples (those without any TestResult row) + if len(results) < total_samples: + skipped_count += (total_samples - len(results)) + + # Retrieve error_count from the error service + error_count = len( + derive_errors_for_run( + run_id, + expected_outputs_by_rt, + preloaded_results=results, + preloaded_files=all_files)) + + statuses, _ = batch_get_run_data([test]) + run_status = statuses.get(test.id, 'queued') + + return single_response({ + 'run_id': run_id, + 'status': run_status, + 'total_samples': total_samples, + 'pass_count': pass_count, + 'fail_count': fail_count, + 'skipped_count': skipped_count, + 'missing_output_count': missing_count, + 'error_count': error_count, + 'duration_ms': total_runtime if total_runtime > 0 else None, + }, schema=RunSummarySchema()) + + +@mod_api.route('/runs//progress', methods=['GET']) +@require_scope(Scope.RUNS_READ) +@validate_path_id('run_id') +@validate_offset_pagination() +def get_run_progress(run_id, limit=50, offset=0): + """ + Get the timeline of progress events for a run, paginated. + + Events come from TestProgress rows written by the CI worker. + """ + test = Test.query.filter(Test.id == run_id).first() + if test is None: + return make_error_response( + 'not_found', + f'Run {run_id} not found.', + http_status=404) + + query = TestProgress.query.filter_by(test_id=run_id) + + # Optional status filter. + status_filter = request.args.get('status') + if status_filter: + try: + status_enum = TestStatus.from_string(status_filter) + query = query.filter(TestProgress.status == status_enum) + except Exception: + return make_error_response( + 'validation_error', + f'Invalid status filter: {status_filter}.', + details={ + 'fields': { + 'status': 'Must be one of: queued, preparation, testing, completed, canceled, error.'}}, + http_status=400, + ) + + query = query.order_by(TestProgress.id.asc()) + total = query.count() + progress = query.offset(offset).limit(limit).all() + + events = [{ + 'timestamp': p.timestamp, + 'status': p.status.name, + 'message': p.message, + } for p in progress] + + schema = ProgressEventSchema() + return paginated_response(events, total, limit, offset, schema=schema) + + +@mod_api.route('/runs//config', methods=['GET']) +@require_scope(Scope.RUNS_READ) +@validate_path_id('run_id') +def get_run_config(run_id): + """Get the configuration that was used to launch this run.""" + test = Test.query.filter(Test.id == run_id).first() + if test is None: + return make_error_response( + 'not_found', + f'Run {run_id} not found.', + http_status=404) + + regression_ids = _run_regression_ids(test) + + return single_response({ + 'run_id': run_id, + 'platform': test.platform.value, + 'branch': test.branch, + 'commit_sha': test.commit, + 'regression_test_ids': regression_ids, + }) + + +@mod_api.route('/runs//cancel', methods=['POST']) +@require_roles([Role.admin, Role.contributor, Role.tester]) +@require_scope(Scope.RUNS_WRITE) +@validate_path_id('run_id') +def cancel_run(run_id): + """Cancel a running or queued test. + + Idempotent — canceling something already finished returns 202 + with status=no_op. + + Note: In this shared CI environment, any user with 'runs:write' + (admin, contributor, tester) can cancel any run on the platform, + regardless of ownership. This is intentional. + """ + test = Test.query.with_for_update().filter(Test.id == run_id).first() + if test is None: + return make_error_response( + 'not_found', + f'Run {run_id} not found.', + http_status=404) + + status = derive_run_status(test) + if status in ('pass', 'fail', 'canceled', 'error'): + return single_response({ + 'run_id': run_id, + 'action': 'cancel', + 'status': 'no_op', + 'message': f'Run is already in terminal state: {status}', + }, http_status=202) + + user = g.api_user + reason = None + if request.is_json and request.get_json(silent=True): + reason = request.get_json(silent=True).get('reason') + if reason: + reason_str = str(reason).strip() + if len(reason_str) < 5: + return make_error_response( + 'validation_error', + 'Cancel reason must be at least 5 characters.', + details={'fields': {'reason': 'Minimum length is 5.'}}, + http_status=400, + ) + reason = reason_str[:255] + + cancel_msg = f'Canceled by {user.name} via API' if user else 'Canceled via API' + if reason: + cancel_msg = f'{cancel_msg}: {reason}' + + progress = TestProgress(run_id, TestStatus.canceled, cancel_msg) + g.db.add(progress) + g.db.commit() + + return single_response({ + 'run_id': run_id, + 'action': 'cancel', + 'status': 'accepted', + 'message': 'Run has been canceled.', + }, http_status=202) diff --git a/mod_api/routes/samples.py b/mod_api/routes/samples.py new file mode 100644 index 000000000..9ebe7fb00 --- /dev/null +++ b/mod_api/routes/samples.py @@ -0,0 +1,647 @@ +""" +Sample and regression test routes. + +GET /runs/{id}/samples Per-run regression test results +GET /runs/{id}/samples/{sid} Single result in a run +GET /samples Media sample catalog +GET /samples/{id} Single media sample +GET /samples/{id}/history Cross-run history for a sample +GET /regression-tests Regression test definitions +""" + +from collections import defaultdict + +from flask import g, request +from sqlalchemy import func +from sqlalchemy.orm import joinedload, selectinload + +from mod_api import mod_api +from mod_api.middleware.auth import require_scope +from mod_api.middleware.error_handler import make_error_response +from mod_api.middleware.validation import (validate_date_range, + validate_offset_pagination, + validate_path_id) +from mod_api.models.api_token import Scope +from mod_api.schemas.samples import SampleHistoryEntrySchema +from mod_api.services.status import (batch_get_run_data, derive_output_status, + derive_sample_status, get_run_timestamps, + is_dummy_row) +from mod_api.utils import paginated_response, single_response +from mod_regression.models import (Category, RegressionTest, + RegressionTestOutput) +from mod_sample.models import Sample, Tag +from mod_test.models import (Test, TestPlatform, TestProgress, TestResult, + TestResultFile) + +# Valid per-sample status values accepted by the ?status filter. Limited to the +# statuses derive_sample_status can actually emit, so filtering can't silently +# return empty for a value that never occurs. +_VALID_SAMPLE_STATUSES = frozenset({ + 'pass', 'fail', 'missing_output', 'not_started', +}) + + +def _preload_expected_outputs(results): + """Map regression_test_id -> [RegressionTestOutput] for the given results. + + Lets per-sample status derivation use the same missing-output detection + as /runs/{id}/summary, so the two endpoints can't disagree. + """ + rt_ids = {r.regression_test_id for r in results} + expected_by_rt = defaultdict(list) + if rt_ids: + for rto in RegressionTestOutput.query.filter( + RegressionTestOutput.regression_id.in_(rt_ids)).all(): + expected_by_rt[rto.regression_id].append(rto) + return expected_by_rt + + +def _serialize_outputs(result_files): + outputs = [] + for rf in result_files: + if is_dummy_row(rf): + continue + outputs.append({ + 'output_id': rf.regression_test_output_id, + 'filename': ( + rf.regression_test_output.create_correct_filename(rf.expected) + if rf.regression_test_output else rf.expected + ), + 'status': derive_output_status(rf), + }) + return outputs + + +def _serialize_run_sample(result, result_files, expected_outputs=None): + """Build the per-regression-test result dict for a run.""" + status = derive_sample_status(result, result_files, expected_outputs) + outputs = _serialize_outputs(result_files) + + sample_name = None + sample_id = None + command = None + categories = [] + + if result.regression_test: + rt = result.regression_test + command = rt.command + if rt.sample: + sample_id = rt.sample_id + sample_name = rt.sample.original_name + if rt.categories: + categories = [c.name for c in rt.categories] + + return { + 'regression_test_id': result.regression_test_id, + 'sample_id': sample_id, + 'sample_name': sample_name, + 'status': status, + 'exit_code': result.exit_code, + 'expected_rc': result.expected_rc, + 'runtime_ms': result.runtime, + 'command': command, + 'categories': categories, + 'outputs': outputs, + } + + +def _filter_run_samples_by_tag(serialized, tag_filter): + tag_lower = tag_filter.lower() + tagged_sample_ids = set() + + valid_sample_ids = [s['sample_id'] + for s in serialized if s.get('sample_id')] + samples = Sample.query.options(joinedload(Sample.tags)).filter( + Sample.id.in_(valid_sample_ids)).all() if valid_sample_ids else [] + sample_map = {sample.id: sample for sample in samples} + + for s in serialized: + if s['sample_id']: + sample = sample_map.get(s['sample_id']) + if sample and any(tag_lower == t.name.lower() + for t in sample.tags): + tagged_sample_ids.add(s['sample_id']) + return [s for s in serialized if s.get('sample_id') in tagged_sample_ids] + + +def _apply_run_sample_filters(serialized, args): + status_filter = args.get('status') + if status_filter: + serialized = [s for s in serialized if s['status'] == status_filter] + + name_filter = args.get('name') + if name_filter: + name_lower = name_filter.lower() + serialized = [s for s in serialized if s.get( + 'sample_name') and name_lower in s['sample_name'].lower()] + + tag_filter = args.get('tag') + if tag_filter: + serialized = _filter_run_samples_by_tag(serialized, tag_filter) + + category_filter = args.get('category') + if category_filter: + cat_lower = category_filter.lower() + serialized = [ + s for s in serialized + if s.get('categories') and cat_lower in [ + c.lower() for c in s['categories'] + ] + ] + return serialized + + +@mod_api.route('/runs//samples', methods=['GET']) +@require_scope(Scope.RUNS_READ) +@validate_path_id('run_id') +@validate_offset_pagination() +def list_run_samples(run_id, limit=50, offset=0): + """ + List per-sample results for a run, with optional filters. + + Supports ?status, ?name, ?tag, ?category query params. + """ + # Validate the status filter up front, before any DB work. + status_filter = request.args.get('status') + if status_filter and status_filter not in _VALID_SAMPLE_STATUSES: + return make_error_response( + 'validation_error', + f"Invalid status: {status_filter}", + http_status=400 + ) + + test = Test.query.filter(Test.id == run_id).first() + if test is None: + return make_error_response( + 'not_found', + f'Run {run_id} not found.', + http_status=404) + + results = TestResult.query.options( + joinedload(TestResult.regression_test) + .joinedload(RegressionTest.sample), + joinedload(TestResult.regression_test) + .selectinload(RegressionTest.categories), + ).filter_by(test_id=run_id).all() + + # Preload TestResultFiles together with the expected-output rows that + # derive_sample_status compares against. + all_files = TestResultFile.query.options( + joinedload(TestResultFile.regression_test_output) + .joinedload(RegressionTestOutput.multiple_files) + ).filter_by(test_id=run_id).all() if results else [] + files_by_result = defaultdict(list) + for f in all_files: + files_by_result[f.regression_test_id].append(f) + + # Preload expected outputs so per-sample status matches /summary. + expected_by_rt = _preload_expected_outputs(results) + + # Serialize list to filter by derived status and joined fields + serialized = [] + for result in results: + result_files = files_by_result.get(result.regression_test_id, []) + serialized.append(_serialize_run_sample( + result, result_files, + expected_by_rt.get(result.regression_test_id))) + + # Apply query param filters. + serialized = _apply_run_sample_filters(serialized, request.args) + + total = len(serialized) + paged = serialized[offset:offset + limit] + return paginated_response(paged, total, limit, offset) + + +@mod_api.route('/runs//samples/', methods=['GET']) +@require_scope(Scope.RUNS_READ) +@validate_path_id('run_id') +@validate_path_id('regression_test_id') +def get_run_sample(run_id, regression_test_id): + """Get a single regression test result within a run.""" + test = Test.query.filter(Test.id == run_id).first() + if test is None: + return make_error_response( + 'not_found', + f'Run {run_id} not found.', + http_status=404) + + result = TestResult.query.options( + joinedload(TestResult.regression_test) + .joinedload(RegressionTest.sample), + joinedload(TestResult.regression_test) + .selectinload(RegressionTest.categories), + ).filter_by( + test_id=run_id, + regression_test_id=regression_test_id, + ).first() + if result is None: + return make_error_response( + 'not_found', + f'Regression test {regression_test_id} not found in run {run_id}.', + http_status=404, + ) + + result_files = TestResultFile.query.options( + joinedload(TestResultFile.regression_test_output) + .joinedload(RegressionTestOutput.multiple_files) + ).filter_by( + test_id=run_id, + regression_test_id=regression_test_id, + ).all() + + expected_by_rt = _preload_expected_outputs([result]) + return single_response(_serialize_run_sample( + result, result_files, expected_by_rt.get(result.regression_test_id))) + + +@mod_api.route('/samples', methods=['GET']) +@require_scope(Scope.RUNS_READ) +@validate_offset_pagination() +def list_samples(limit=50, offset=0): + """ + List media samples from the catalog. + + Supports ?name, ?extension, ?tag, ?sha256, + ?status (active/inactive) filters. + """ + query = Sample.query.options(joinedload(Sample.tags)) + + name = request.args.get('name') + if name: + # Escape LIKE wildcards to prevent unintended pattern matching. + # The explicit escape char makes the backslash escaping portable + # rather than relying on the backend's default. + safe_name = name.replace('\\', '\\\\').replace( + '%', '\\%').replace('_', '\\_') + query = query.filter( + Sample.original_name.ilike(f'%{safe_name}%', escape='\\')) + + extension = request.args.get('extension') + if extension: + query = query.filter(Sample.extension == extension) + + sha256_filter = request.args.get('sha256') + if sha256_filter: + query = query.filter(Sample.sha == sha256_filter) + + tag_filter = request.args.get('tag') + if tag_filter: + + query = query.filter(Sample.tags.any( + func.lower(Tag.name) == tag_filter.lower())) + + status_filter = request.args.get('status') + if status_filter: + if status_filter.lower() not in ('active', 'inactive'): + return make_error_response( + 'validation_error', + 'Invalid status: {status_filter}. ' + 'Must be active or inactive.'.format( + status_filter=status_filter), + http_status=400) + want_active = status_filter.lower() == 'active' + if want_active: + query = query.filter( + Sample.tests.any(RegressionTest.active == True) # noqa: E712 + ) # tests refers to RegressionTest + else: + query = query.filter( + ~Sample.tests.any(RegressionTest.active == True) # noqa: E712 + ) # tests refers to RegressionTest + + # Paginate at DB level without Python-side filters + total = query.count() + samples = query.offset(offset).limit(limit).all() + + # Batch load active regression test counts + sample_ids = [s.id for s in samples] + counts_list = g.db.query( + RegressionTest.sample_id, + func.count(RegressionTest.id) + ).filter( + RegressionTest.sample_id.in_(sample_ids), + RegressionTest.active == True # noqa: E712 + ).group_by(RegressionTest.sample_id).all() if sample_ids else [] + counts = dict(counts_list) + + serialized = [] + for s in samples: + active_count = counts.get(s.id, 0) + serialized.append({ + 'sample_id': s.id, + 'sha': s.sha, + 'extension': s.extension, + 'original_name': s.original_name, + 'filename': s.filename, + 'tags': [t.name for t in s.tags], + 'regression_test_count': active_count, + 'active': active_count > 0, + }) + + return paginated_response(serialized, total, limit, offset) + + +@mod_api.route('/samples/', methods=['GET']) +@require_scope(Scope.RUNS_READ) +@validate_path_id('sample_id') +def get_sample(sample_id): + """Get a single media sample by its ID.""" + sample = Sample.query.options(joinedload(Sample.tags)).filter( + Sample.id == sample_id).first() + if sample is None: + return make_error_response( + 'not_found', + f'Sample {sample_id} not found.', + http_status=404) + + active_count = RegressionTest.query.filter_by( + sample_id=sample.id, active=True + ).count() + + return single_response({ + 'sample_id': sample.id, + 'sha': sample.sha, + 'extension': sample.extension, + 'original_name': sample.original_name, + 'filename': sample.filename, + 'tags': [t.name for t in sample.tags], + 'regression_test_count': active_count, + 'active': active_count > 0, + }) + + +def _get_history_failure_signature(result, result_files, status): + if status == 'fail': + for rf in result_files: + if rf.got is not None and not is_dummy_row(rf): + return f'diff_mismatch:output:{rf.regression_test_output_id}' + if result.exit_code != result.expected_rc: + return f'exit_code_mismatch:rc:{result.exit_code}' + elif status == 'missing_output': + return 'missing_output' + return None + + +def _process_history_entries( + results, + files_by_result, + status_filter, + timestamps_map=None, + test_map=None, + expected_by_rt=None): + entries = [] + for result in results: + test = test_map.get(result.test_id) if test_map else result.test + if test is None: + continue + + result_files = files_by_result.get( + (result.test_id, result.regression_test_id), []) + expected = expected_by_rt.get( + result.regression_test_id) if expected_by_rt else None + status = derive_sample_status(result, result_files, expected) + + if status_filter and status != status_filter: + continue + + failure_sig = _get_history_failure_signature( + result, result_files, status) + if timestamps_map is not None and test.id in timestamps_map: + timestamps = timestamps_map[test.id] + else: + timestamps = get_run_timestamps(test) + + entries.append({ + 'run_id': test.id, + 'regression_test_id': result.regression_test_id, + 'status': status, + 'platform': test.platform.value, + 'branch': test.branch, + 'commit_sha': test.commit, + 'tested_at': timestamps.get('completed_at') or timestamps.get('started_at'), + 'failure_signature': failure_sig, + }) + return entries + + +def _apply_history_filters( + query, + branch, + platform, + created_after, + created_before): + if branch: + query = query.filter(Test.branch == branch) + + if platform: + try: + platform_enum = TestPlatform.from_string(platform) + query = query.filter(Test.platform == platform_enum) + except Exception: + valid_platforms = ', '.join(TestPlatform.values()) + return None, make_error_response( + 'validation_error', 'Invalid platform: {platform}. ' + 'Must be one of: {valid_platforms}.'.format( + platform=platform, valid_platforms=valid_platforms + ), + http_status=400, + ) + + if created_after or created_before: + + first_progress = ( + g.db.query(TestProgress.test_id, func.min( + TestProgress.timestamp).label('min_ts')) + .group_by(TestProgress.test_id) + .subquery() + ) + query = query.join(first_progress, Test.id == first_progress.c.test_id) + if created_after: + query = query.filter(first_progress.c.min_ts >= created_after) + if created_before: + query = query.filter(first_progress.c.min_ts <= created_before) + + return query, None + + +@mod_api.route('/samples//history', methods=['GET']) +@require_scope(Scope.RUNS_READ) +@validate_path_id('sample_id') +@validate_offset_pagination() +@validate_date_range +def get_sample_history( + sample_id, + limit=50, + offset=0, + created_after=None, + created_before=None): + """ + Show how a sample performed across different runs. + + Use failure_signature to tell apart genuine regressions from infra flakes. + """ + sample = Sample.query.options(joinedload(Sample.tags)).filter( + Sample.id == sample_id).first() + if sample is None: + return make_error_response( + 'not_found', + f'Sample {sample_id} not found.', + http_status=404) + + regression_tests = RegressionTest.query.filter_by( + sample_id=sample_id).all() + rt_ids = [rt.id for rt in regression_tests] + + if not rt_ids: + return paginated_response([], 0, limit, offset) + + # Validate the status filter up front, before any heavy query. + status_filter = request.args.get('status') + if status_filter and status_filter not in _VALID_SAMPLE_STATUSES: + return make_error_response( + 'validation_error', + f"Invalid status: {status_filter}", + http_status=400 + ) + + query = TestResult.query.filter( + TestResult.regression_test_id.in_(rt_ids) + ).join(Test, Test.id == TestResult.test_id) + + branch = request.args.get('branch') + platform = request.args.get('platform') + + query, err = _apply_history_filters( + query, branch, platform, created_after, created_before) + if err: + return err + + results = query.order_by(Test.id.desc()).all() + + # Preload TestResultFiles + test_ids = list({r.test_id for r in results}) + all_files = TestResultFile.query.options( + joinedload(TestResultFile.regression_test_output) + .joinedload(RegressionTestOutput.multiple_files) + ).filter( + TestResultFile.test_id.in_(test_ids)).all() if test_ids else [] + files_by_result = defaultdict(list) + for f in all_files: + files_by_result[(f.test_id, f.regression_test_id)].append(f) + + # Preload expected outputs so status matches /summary and /samples. + expected_by_rt = _preload_expected_outputs(results) + + # Batch load tests to avoid N+1 in _process_history_entries + unique_tests = Test.query.filter( + Test.id.in_(test_ids)).all() if test_ids else [] + test_map = {t.id: t for t in unique_tests} + + # Batch compute timestamps for all referenced tests + _, timestamps_map = batch_get_run_data(unique_tests) + + entries = _process_history_entries( + results, + files_by_result, + status_filter, + timestamps_map=timestamps_map, + test_map=test_map, + expected_by_rt=expected_by_rt) + + total = len(entries) + paged = entries[offset:offset + limit] + + return paginated_response( + paged, total, limit, offset, schema=SampleHistoryEntrySchema() + ) + + +def _serialize_rt(rt): + return { + 'regression_test_id': rt.id, + 'sample_id': rt.sample_id, + 'sample_name': rt.sample.original_name if rt.sample else None, + 'command': rt.command, + 'input_type': rt.input_type.value, + 'output_type': rt.output_type.value, + 'expected_rc': rt.expected_rc, + 'active': rt.active, + 'categories': [c.name for c in rt.categories], + 'description': rt.description, + } + + +@mod_api.route('/regression-tests', methods=['GET']) +@require_scope(Scope.RUNS_READ) +@validate_offset_pagination() +def list_regression_tests(limit=50, offset=0): + """ + List regression test definitions. + + Supports ?active, ?category, ?tag, ?sample_id filters. Note: when + ?active is omitted it defaults to true, so inactive regression tests + are hidden unless ?active=false is passed explicitly. + """ + query = RegressionTest.query.options( + joinedload(RegressionTest.sample), + selectinload(RegressionTest.categories), + ) + + active_filter = request.args.get('active') + if active_filter is not None: + value = active_filter.lower() + if value in ('true', '1', 'yes'): + is_active = True + elif value in ('false', '0', 'no'): + is_active = False + else: + # Reject garbage instead of silently treating it as false. + return make_error_response( + 'validation_error', + f'Invalid active filter: {active_filter}. ' + 'Must be true or false.', + details={'fields': {'active': 'Must be true or false.'}}, + http_status=400, + ) + else: + is_active = True + query = query.filter(RegressionTest.active == is_active) + + category = request.args.get('category') + if category: + query = query.join(RegressionTest.categories).filter( + Category.name == category) + + sample_id_filter = request.args.get('sample_id') + if sample_id_filter: + try: + sid = int(sample_id_filter) + if sid < 1 or sid > 2147483647: + raise ValueError("Out of bounds") + query = query.filter(RegressionTest.sample_id == sid) + except (ValueError, TypeError): + return make_error_response( + 'validation_error', + 'sample_id must be a positive integer ' + 'between 1 and 2147483647.', + details={ + 'fields': { + 'sample_id': 'Must be a positive integer ' + 'between 1 and 2147483647.'}}, + http_status=400, + ) + + tag_filter = request.args.get('tag') + if tag_filter: + query = query.filter( + RegressionTest.sample.has( + Sample.tags.any(func.lower(Tag.name) == tag_filter.lower()) + ) + ) + + # Paginate at DB level + total = query.count() + tests = query.offset(offset).limit(limit).all() + serialized = [_serialize_rt(rt) for rt in tests] + return paginated_response(serialized, total, limit, offset) diff --git a/mod_api/routes/system.py b/mod_api/routes/system.py new file mode 100644 index 000000000..6a9cd7b92 --- /dev/null +++ b/mod_api/routes/system.py @@ -0,0 +1,204 @@ +""" +System, health, and queue routes. + +GET /system/health Health check (unauthenticated) +GET /system/queue Queue status — active + queued runs +""" + +import os +from datetime import datetime, timezone + +from flask import g, jsonify, request +from sqlalchemy import text + +from mod_api import mod_api +from mod_api.middleware.auth import require_scope +from mod_api.middleware.error_handler import make_error_response +from mod_api.middleware.validation import validate_offset_pagination +from mod_api.models.api_token import Scope +from mod_api.schemas.common import DATETIME_FORMAT +from mod_api.services.status import batch_get_run_data +from mod_api.utils import paginated_response +from mod_test.models import Test, TestPlatform, TestProgress, TestStatus + + +@mod_api.route('/system/health', methods=['GET']) +def system_health(): + """ + Public health check — no auth required. + + Returns 200 when things are ok or degraded, 503 when the system is down. + Monitoring services and load balancers can hit this freely. + """ + now = datetime.now(timezone.utc) + dependencies = [] + overall = 'ok' + + # Database connectivity. + try: + g.db.execute(text('SELECT 1')) + dependencies.append( + {'name': 'database', 'status': 'ok', 'message': None}) + except Exception: + dependencies.append({'name': 'database', + 'status': 'down', + 'message': 'Database connection failed.'}) + overall = 'down' + + # Local sample storage. + try: + from run import config + sample_repo = config.get('SAMPLE_REPOSITORY', '') + if os.path.isdir(sample_repo): + dependencies.append( + {'name': 'local_storage', 'status': 'ok', 'message': None}) + else: + dependencies.append({ + 'name': 'local_storage', + 'status': 'degraded', + 'message': 'Local storage check failed.', + }) + if overall == 'ok': + overall = 'degraded' + except Exception: + dependencies.append({'name': 'local_storage', 'status': 'down', + 'message': 'Local storage check failed.'}) + overall = 'down' + + # Google Cloud Storage. + try: + from run import storage_client_bucket + if storage_client_bucket: + dependencies.append( + {'name': 'gcs', 'status': 'ok', 'message': None}) + else: + dependencies.append({'name': 'gcs', + 'status': 'degraded', + 'message': 'GCS client not initialized.'}) + if overall == 'ok': + overall = 'degraded' + except Exception: + dependencies.append({'name': 'gcs', 'status': 'degraded', + 'message': 'GCS connectivity check failed.'}) + if overall == 'ok': + overall = 'degraded' + + http_status = 503 if overall == 'down' else 200 + response = jsonify({ + 'status': overall, + 'checked_at': now.strftime(DATETIME_FORMAT), + 'dependencies': dependencies, + }) + response.status_code = http_status + return response + + +def _apply_queue_filters( + base_query, + running_subq, + queue_depth, + running_count, + status_filter): + if status_filter == 'queued': + query = base_query.filter(~Test.id.in_( + g.db.query(running_subq.c.test_id))) + total = queue_depth + elif status_filter == 'running': + query = base_query.filter(Test.id.in_( + g.db.query(running_subq.c.test_id))) + total = running_count + elif status_filter: + return None, None, make_error_response( + 'validation_error', 'Invalid status. Must be queued or running.', http_status=400) + else: + query = base_query + total = queue_depth + running_count + return query, total, None + + +@mod_api.route('/system/queue', methods=['GET']) +@require_scope(Scope.SYSTEM_READ) +@validate_offset_pagination() +def get_queue(limit=50, offset=0): + """ + Get queue summary and list of runs. + + Note: The `position` field is only populated when `?status=queued` is + explicitly provided. Otherwise, it will be null for all items. + + Excludes anything that's already completed or canceled. Supports + ?platform and ?status filters. + """ + terminal_subq = g.db.query( + TestProgress.test_id + ).filter( + TestProgress.status.in_([TestStatus.completed, TestStatus.canceled]) + ).group_by(TestProgress.test_id).subquery() + + running_subq = g.db.query( + TestProgress.test_id + ).filter( + TestProgress.status.in_([TestStatus.preparation, TestStatus.testing]) + ).group_by(TestProgress.test_id).subquery() + + base_query = Test.query.filter( + ~Test.id.in_(g.db.query(terminal_subq.c.test_id)) + ) + + platform_filter = request.args.get('platform') + if platform_filter: + try: + plat = TestPlatform.from_string(platform_filter) + base_query = base_query.filter(Test.platform == plat) + except Exception: + return make_error_response( + 'validation_error', + 'Invalid platform.', + http_status=400) + + running_count = base_query.filter(Test.id.in_( + g.db.query(running_subq.c.test_id))).count() + queue_depth = base_query.filter(~Test.id.in_( + g.db.query(running_subq.c.test_id))).count() + + status_filter = request.args.get('status') + query, total, err = _apply_queue_filters( + base_query, running_subq, queue_depth, running_count, status_filter) + if err: + return err + + query = query.order_by(Test.id.asc()) + paged_tests = query.offset(offset).limit(limit).all() + + statuses, timestamps = batch_get_run_data(paged_tests) + + paged_jobs = [] + queued_index = offset + 1 if status_filter == 'queued' else None + + for test in paged_tests: + status = statuses.get(test.id, 'queued') + ts = timestamps.get(test.id, {}) + + pos = None + if status == 'queued' and queued_index is not None: + pos = queued_index + queued_index += 1 + + paged_jobs.append({ + 'run_id': test.id, + 'status': status, + 'platform': test.platform.value, + # Same 'Z' format the schemas use, so the API emits one + # datetime style everywhere (timestamps are UTC). + 'queued_at': ts.get('queued_at').strftime(DATETIME_FORMAT) if ts.get('queued_at') else None, + 'started_at': ts.get('started_at').strftime(DATETIME_FORMAT) if ts.get('started_at') else None, + 'position': pos, + }) + + return paginated_response( + paged_jobs, total, limit, offset, + extra_meta={ + 'queue_depth': queue_depth, + 'running_count': running_count, + } + ) diff --git a/mod_api/schemas/__init__.py b/mod_api/schemas/__init__.py new file mode 100644 index 000000000..889960659 --- /dev/null +++ b/mod_api/schemas/__init__.py @@ -0,0 +1 @@ +"""mod_api.schemas: Marshmallow schemas for request/response validation.""" diff --git a/mod_api/schemas/auth.py b/mod_api/schemas/auth.py new file mode 100644 index 000000000..bbfc15546 --- /dev/null +++ b/mod_api/schemas/auth.py @@ -0,0 +1,67 @@ +"""Request/response schemas for the token endpoints.""" + +from marshmallow import RAISE, Schema, fields, validate + +from mod_api.models.api_token import VALID_SCOPES +from mod_api.schemas.common import DATETIME_FORMAT + + +class TokenCreateRequestSchema(Schema): + """Validates POST /auth/tokens bodies.""" + + email = fields.Email(required=True) + password = fields.String( + required=True, + validate=validate.Length(min=8, max=128), + ) + token_name = fields.String( + required=True, + validate=[ + validate.Length(min=1, max=50), + validate.Regexp( + r'^[a-zA-Z0-9_\-]+$', + error='token_name must match ^[a-zA-Z0-9_-]+$', + ), + ], + ) + expires_in_days = fields.Integer( + load_default=7, + validate=validate.Range(min=1, max=30), + ) + scopes = fields.List( + fields.String(validate=validate.OneOf(VALID_SCOPES)), + load_default=None, + validate=validate.Length(max=6), + ) + + class Meta: + """Reject unknown fields.""" + + unknown = RAISE + + +class AuthTokenSchema(Schema): + """The one-time response returned when a token is created.""" + + token = fields.String(required=True) + token_type = fields.String(dump_default='bearer') + token_name = fields.String(required=True) + scopes = fields.List(fields.String(), required=True) + expires_at = fields.DateTime(required=True, format=DATETIME_FORMAT) + + +class ApiTokenItemSchema(Schema): + """Token metadata for list responses — never includes the plaintext.""" + + id = fields.Integer(required=True) + user_id = fields.Integer(required=True) + token_name = fields.String(required=True) + scopes = fields.Method('get_scopes') + created_at = fields.DateTime(required=True, format=DATETIME_FORMAT) + expires_at = fields.DateTime(required=True, format=DATETIME_FORMAT) + is_revoked = fields.Boolean(required=True) + revoked_at = fields.DateTime(allow_none=True, format=DATETIME_FORMAT) + + def get_scopes(self, obj): + """Deserialize scopes from the model's JSON column.""" + return obj.scopes diff --git a/mod_api/schemas/common.py b/mod_api/schemas/common.py new file mode 100644 index 000000000..9e244bcbd --- /dev/null +++ b/mod_api/schemas/common.py @@ -0,0 +1,3 @@ +"""Constants shared across API schemas.""" + +DATETIME_FORMAT = "%Y-%m-%dT%H:%M:%SZ" diff --git a/mod_api/schemas/results.py b/mod_api/schemas/results.py new file mode 100644 index 000000000..fe054dc26 --- /dev/null +++ b/mod_api/schemas/results.py @@ -0,0 +1,64 @@ +"""Schemas for output file content and baseline approvals.""" + +from marshmallow import RAISE, Schema, fields, validate + +from mod_api.schemas.common import DATETIME_FORMAT + + +class OutputFileContentSchema(Schema): + """File content blob returned for expected or actual output.""" + + run_id = fields.Integer(allow_none=True) + sample_id = fields.Integer(required=True) + regression_id = fields.Integer(required=True) + output_id = fields.Integer(required=True) + filename = fields.String(required=True) + content_type = fields.String(required=True) + encoding = fields.String( + required=True, validate=validate.OneOf(['utf-8', 'base64'])) + content = fields.String(required=True) + # sha256 always covers the complete file, even when content is truncated. + sha256 = fields.String(allow_none=True) + truncated = fields.Boolean(load_default=False) + download_url = fields.String(allow_none=True) + storage_status = fields.String( + required=True, + validate=validate.OneOf(['ok', 'degraded', 'missing']), + ) + + +class BaselineApprovalRequestSchema(Schema): + """POST /runs/{id}/samples/{sid}/baseline-approval body.""" + + regression_id = fields.Integer( + required=True, + validate=validate.Range(min=1), + ) + output_id = fields.Integer( + required=True, + validate=validate.Range(min=1), + ) + + remove_variants = fields.Boolean( + load_default=False, + ) + + class Meta: + """Reject unknown fields.""" + + unknown = RAISE + + +class BaselineApprovalSchema(Schema): + """Response after a baseline approval is applied.""" + + status = fields.String( + required=True, + validate=validate.OneOf( + ['approved'])) + run_id = fields.Integer(required=True) + sample_id = fields.Integer(required=True) + regression_id = fields.Integer(required=True) + output_id = fields.Integer(required=True) + requested_by = fields.String(required=True) + created_at = fields.DateTime(required=True, format=DATETIME_FORMAT) diff --git a/mod_api/schemas/runs.py b/mod_api/schemas/runs.py new file mode 100644 index 000000000..a4ab5539c --- /dev/null +++ b/mod_api/schemas/runs.py @@ -0,0 +1,99 @@ +"""Schemas for runs, summaries, and progress events.""" + +from marshmallow import RAISE, Schema, fields, validate + +from mod_api.schemas.common import DATETIME_FORMAT + + +class ProgressEventSchema(Schema): + """A single progress event in a run's timeline.""" + + timestamp = fields.DateTime(required=True, format=DATETIME_FORMAT) + status = fields.String(required=True) + message = fields.String(required=True) + + +class RunSchema(Schema): + """Full run details.""" + + run_id = fields.Integer(required=True) + status = fields.String(required=True, validate=validate.OneOf([ + 'queued', 'running', 'pass', 'fail', 'canceled', 'incomplete', 'error' + ])) + platform = fields.String( + required=True, validate=validate.OneOf(['linux', 'windows'])) + test_type = fields.String(validate=validate.OneOf(['commit', 'pr'])) + repository = fields.String(required=True) + branch = fields.String(allow_none=True) + commit_sha = fields.String(required=True) + pr_number = fields.Integer(allow_none=True, load_default=None) + created_at = fields.DateTime(allow_none=True, format=DATETIME_FORMAT) + queued_at = fields.DateTime(allow_none=True, format=DATETIME_FORMAT) + started_at = fields.DateTime(allow_none=True, format=DATETIME_FORMAT) + completed_at = fields.DateTime(allow_none=True, format=DATETIME_FORMAT) + github_link = fields.String(allow_none=True) + + +class RunSummarySchema(Schema): + """Pass/fail/skip aggregate counts for a run.""" + + run_id = fields.Integer(required=True) + status = fields.String(required=True) + total_samples = fields.Integer(required=True) + pass_count = fields.Integer(required=True) + fail_count = fields.Integer(required=True) + skipped_count = fields.Integer(required=True) + missing_output_count = fields.Integer(required=True) + error_count = fields.Integer(load_default=0) + duration_ms = fields.Integer(allow_none=True) + + +class RunCreateRequestSchema(Schema): + """POST /runs request body.""" + + commit_sha = fields.String( + required=True, + validate=validate.Regexp( + r'^[a-fA-F0-9]{40}$', + error='commit_sha must be a 40-character hex string.', + ), + ) + platform = fields.String( + required=True, + validate=validate.OneOf(['linux', 'windows']), + ) + branch = fields.String( + load_default='master', + validate=[ + validate.Length(max=100), + validate.Regexp( + r'^[A-Za-z0-9._-]+(/[A-Za-z0-9._-]+)*$', + error='branch must match ^[A-Za-z0-9._-]+(/[A-Za-z0-9._-]+)*$', + ), + ], + ) + repository = fields.String( + required=True, + validate=[ + validate.Length(max=100), + validate.Regexp( + r'^[a-zA-Z0-9_.\-]+/[a-zA-Z0-9_.\-]+$', + error='repository must match owner/repo format.', + ), + ], + ) + pull_request = fields.Integer( + load_default=None, + allow_none=True, + validate=validate.Range(min=1, max=2147483647), + ) + regression_test_ids = fields.List( + fields.Integer(validate=validate.Range(min=1, max=2147483647)), + load_default=None, + validate=validate.Length(max=500), + ) + + class Meta: + """Reject unknown fields.""" + + unknown = RAISE diff --git a/mod_api/schemas/samples.py b/mod_api/schemas/samples.py new file mode 100644 index 000000000..0b43b012c --- /dev/null +++ b/mod_api/schemas/samples.py @@ -0,0 +1,18 @@ +"""Schemas for sample endpoints.""" + +from marshmallow import Schema, fields + +from mod_api.schemas.common import DATETIME_FORMAT + + +class SampleHistoryEntrySchema(Schema): + """One row in a sample's cross-run history.""" + + run_id = fields.Integer(required=True) + regression_test_id = fields.Integer(required=True) + status = fields.String(required=True) + platform = fields.String(required=True) + branch = fields.String(required=True) + commit_sha = fields.String(required=True) + tested_at = fields.DateTime(allow_none=True, format=DATETIME_FORMAT) + failure_signature = fields.String(allow_none=True) diff --git a/mod_api/services/__init__.py b/mod_api/services/__init__.py new file mode 100644 index 000000000..04182e587 --- /dev/null +++ b/mod_api/services/__init__.py @@ -0,0 +1 @@ +"""mod_api.services - Core business logic for the API.""" diff --git a/mod_api/services/diff_service.py b/mod_api/services/diff_service.py new file mode 100644 index 000000000..f527c57ba --- /dev/null +++ b/mod_api/services/diff_service.py @@ -0,0 +1,220 @@ +""" +Structured diff computation between expected and actual output files. + +Produces JSON hunks with line-level detail instead of the legacy HTML +diff output. Uses difflib.unified_diff internally. +""" + +import difflib +import hashlib +import os +import re +from typing import Any, Dict, List, Optional, Tuple + +from mod_api.services.storage import get_test_results_base_path + + +def compute_diff( + expected_path: str, + actual_path: str, + context_lines: int = 3, + max_hunks: int = 500, +) -> Dict[str, Any]: + """ + Compute a structured diff between two files. + + Returns a dict matching the Diff schema: status, summary (added_lines, + removed_lines, changed_hunks), and a list of hunks. + """ + context_lines = max(1, min(context_lines, 50)) + + if not os.path.isfile(expected_path): + return { + 'status': 'missing_expected', + 'summary': {'added_lines': 0, 'removed_lines': 0, 'changed_hunks': 0}, + 'hunks': [], + } + + if not os.path.isfile(actual_path): + return { + 'status': 'missing_actual', + 'summary': {'added_lines': 0, 'removed_lines': 0, 'changed_hunks': 0}, + 'hunks': [], + } + + expected_lines = read_lines(expected_path) + actual_lines = read_lines(actual_path) + + if expected_lines == actual_lines: + return { + 'status': 'identical', + 'summary': {'added_lines': 0, 'removed_lines': 0, 'changed_hunks': 0}, + 'hunks': [], + } + + hunks = _compute_hunks(expected_lines, actual_lines, + context_lines, max_hunks) + added = sum( + 1 for h in hunks for line in h['lines'] if line['kind'] == 'added') + removed = sum( + 1 for h in hunks for line in h['lines'] if line['kind'] == 'removed') + + return { + 'status': 'different', + 'summary': { + 'added_lines': added, + 'removed_lines': removed, + 'changed_hunks': len(hunks), + }, + 'hunks': hunks, + } + + +# Matches the @@ -a,b +c,d @@ header line from unified_diff. +_HUNK_RE = re.compile(r'^@@ -(\d+)(?:,\d+)? \+(\d+)(?:,\d+)? @@') + + +def _process_diff_line(line, current_hunk, expected_line_num, actual_line_num): + if line.startswith('+'): + current_hunk['lines'].append({ + 'kind': 'added', + 'expected_line': None, + 'actual_line': actual_line_num, + 'text': line[1:], + }) + actual_line_num += 1 + elif line.startswith('-'): + current_hunk['lines'].append({ + 'kind': 'removed', + 'expected_line': expected_line_num, + 'actual_line': None, + 'text': line[1:], + }) + expected_line_num += 1 + else: + content = line[1:] if line.startswith(' ') else line + current_hunk['lines'].append({ + 'kind': 'context', + 'expected_line': expected_line_num, + 'actual_line': actual_line_num, + 'text': content, + }) + expected_line_num += 1 + actual_line_num += 1 + return expected_line_num, actual_line_num + + +def _process_hunk_header( + line: str, + current_hunk: Optional[Dict[str, Any]], + hunks: List[Dict[str, Any]], + max_hunks: int +) -> Tuple[Optional[Dict[str, Any]], int, int, bool]: + if current_hunk and len(hunks) >= max_hunks: + return None, 0, 0, True + if current_hunk: + hunks.append(current_hunk) + + m = _HUNK_RE.match(line) + if m: + expected_line_num = int(m.group(1)) + actual_line_num = int(m.group(2)) + else: + expected_line_num = 0 + actual_line_num = 0 + + new_hunk = { + 'expected_start': expected_line_num, + 'actual_start': actual_line_num, + 'lines': [], + } + return new_hunk, expected_line_num, actual_line_num, False + + +def _compute_hunks( + expected_lines: List[str], + actual_lines: List[str], + context_lines: int, + max_hunks: int, +) -> List[Dict[str, Any]]: + """Parse unified_diff output into structured hunk dicts.""" + differ = difflib.unified_diff( + expected_lines, + actual_lines, + lineterm='', + n=context_lines, + ) + + hunks: List[Dict[str, Any]] = [] + current_hunk: Optional[Dict[str, Any]] = None + expected_line_num = 0 + actual_line_num = 0 + + for line in differ: + if line.startswith(('---', '+++')): + continue + + if line.startswith('@@'): + current_hunk, expected_line_num, actual_line_num, stop = _process_hunk_header( + line, current_hunk, hunks, max_hunks + ) + if stop: + break + continue + + if current_hunk is None: + continue + + expected_line_num, actual_line_num = _process_diff_line( + line, current_hunk, expected_line_num, actual_line_num) + + if current_hunk: + hunks.append(current_hunk) + + return hunks[:max_hunks] + + +def _enforce_safe_path(file_path: str) -> bool: + base = os.path.realpath(get_test_results_base_path()) + target = os.path.realpath(file_path) + return target.startswith(base + os.sep) or target == base + + +def read_lines(file_path: str, max_size_bytes: int = 10 * 1024 * 1024) -> List[str]: + """Read file lines with a cp1252 fallback, matching legacy behavior. + + Parameters + ---------- + file_path : str + Absolute path to the file to read. + max_size_bytes : int + Maximum file size in bytes. Raises ValueError if exceeded. + """ + if not _enforce_safe_path(file_path): + raise ValueError("Unsafe file path") + file_size = os.path.getsize(file_path) + if file_size > max_size_bytes: + raise ValueError( + f"File too large ({file_size} bytes > {max_size_bytes} limit)") + try: + with open(file_path, encoding='utf8') as f: + return [line.rstrip('\n\r') for line in f.readlines()] + except UnicodeDecodeError: + # errors='replace' because cp1252 leaves five byte values undefined — + # without it the fallback can itself raise on binary output files. + with open(file_path, encoding='cp1252', errors='replace') as f: + return [line.rstrip('\n\r') for line in f.readlines()] + + +def file_sha256(file_path: str) -> Optional[str]: + """Compute SHA-256 of a file. Returns None if the file can't be read.""" + if not _enforce_safe_path(file_path): + return None + try: + sha = hashlib.sha256() + with open(file_path, 'rb') as f: + for block in iter(lambda: f.read(8192), b''): + sha.update(block) + return sha.hexdigest() + except (OSError, IOError): + return None diff --git a/mod_api/services/error_service.py b/mod_api/services/error_service.py new file mode 100644 index 000000000..f8bcd85b1 --- /dev/null +++ b/mod_api/services/error_service.py @@ -0,0 +1,307 @@ +""" +Error derivation from TestResult and TestResultFile rows. + +Walks result data and produces structured ErrorItem dicts. There's no +dedicated error table — errors are inferred from: + exit_code_mismatch → exit code != expected + diff_mismatch → got != null and not in multiple correct files + missing_output → dummy (-1,-1,-1,'','error') row present +""" + +import logging +from collections import defaultdict +from typing import Any, Dict, List + +from sqlalchemy.orm import joinedload + +from mod_api.services.status import is_dummy_row +from mod_regression.models import RegressionTestOutput +from mod_test.models import (TestProgress, TestResult, TestResultFile, + TestStatus) + +_SEVERITY_ORDER = ('info', 'warning', 'error', 'critical') + + +def _is_output_acceptable(rf: TestResultFile) -> bool: + if not rf.regression_test_output: + return False + for multi in rf.regression_test_output.multiple_files: + if multi.file_hashes == rf.got: + return True + return False + + +def _check_exit_code_errors(result, test_id, occurred_at): + if result.exit_code != result.expected_rc: + return [{ + 'error_id': f'err_{test_id}_{result.regression_test_id}_rc', + 'run_id': test_id, + 'sample_id': _get_sample_id(result), + 'regression_id': result.regression_test_id, + 'type': 'exit_code_mismatch', + 'severity': 'error', + 'message': ( + f'Exit code {result.exit_code} != expected {result.expected_rc} ' + f'for regression test {result.regression_test_id}' + ), + 'occurred_at': occurred_at, + }] + return [] + + +def _check_missing_output_errors(result, result_files, test_id, occurred_at, expected_outputs): + errors = [] + actual_output_ids = {rf.regression_test_output_id for rf in result_files} + if expected_outputs is not None: + for rto in expected_outputs: + if not rto.ignore and rto.id not in actual_output_ids: + errors.append({ + 'error_id': f'err_{test_id}_{result.regression_test_id}_missing_{rto.id}', + 'run_id': test_id, + 'sample_id': _get_sample_id(result), + 'regression_id': result.regression_test_id, + 'type': 'missing_output', + 'severity': 'error', + 'message': ( + f'Regression test {result.regression_test_id} ' + f'produced no output for expected file {rto.id}' + ), + 'occurred_at': occurred_at, + }) + else: + for rf in result_files: + if is_dummy_row(rf): + errors.append({ + 'error_id': f'err_{test_id}_{result.regression_test_id}_missing', + 'run_id': test_id, + 'sample_id': _get_sample_id(result), + 'regression_id': result.regression_test_id, + 'type': 'missing_output', + 'severity': 'error', + 'message': ( + f'Regression test {result.regression_test_id} ' + f'produced no output when output was expected' + ), + 'occurred_at': occurred_at, + }) + return errors + + +def _check_diff_mismatch_errors(result, result_files, test_id, occurred_at): + errors = [] + for rf in result_files: + if is_dummy_row(rf): + continue + if rf.got is not None and not _is_output_acceptable(rf): + errors.append({ + 'error_id': f'err_{test_id}_{result.regression_test_id}_{rf.regression_test_output_id}', + 'run_id': test_id, + 'sample_id': _get_sample_id(result), + 'regression_id': result.regression_test_id, + 'type': 'diff_mismatch', + 'severity': 'warning', + 'message': ( + f'Output differs from expected for regression test ' + f'{result.regression_test_id}, output {rf.regression_test_output_id}' + ), + 'occurred_at': occurred_at, + }) + return errors + + +def _evaluate_test_result( + result, + result_files, + test_id, + occurred_at, + expected_outputs=None): + errors = [] + errors.extend(_check_exit_code_errors(result, test_id, occurred_at)) + errors.extend(_check_missing_output_errors(result, result_files, test_id, occurred_at, expected_outputs)) + errors.extend(_check_diff_mismatch_errors(result, result_files, test_id, occurred_at)) + return errors + + +def _group_result_files(test_id, results, preloaded_files=None): + """Map regression_test_id -> [TestResultFile], loading if not preloaded.""" + if preloaded_files is not None: + all_files = preloaded_files + elif results: + all_files = TestResultFile.query.options( + joinedload(TestResultFile.regression_test_output) + .joinedload(RegressionTestOutput.multiple_files) + ).filter_by(test_id=test_id).all() + else: + all_files = [] + files_by_result = defaultdict(list) + for f in all_files: + files_by_result[f.regression_test_id].append(f) + return files_by_result + + +def _load_expected_outputs(results): + """Map regression_test_id -> [RegressionTestOutput] for the given results. + + Missing-output detection must use the same RegressionTestOutput + comparison as /runs/{id}/summary — /errors and /summary have to agree. + id > 0 excludes the -1 sentinel some fixtures create to satisfy the + dummy TestResultFile row's foreign key; it is not a real expectation. + """ + if not results: + return {} + rt_ids = {r.regression_test_id for r in results} + expected_by_rt = defaultdict(list) + for rto in RegressionTestOutput.query.filter( + RegressionTestOutput.regression_id.in_(rt_ids), + RegressionTestOutput.id > 0).all(): + expected_by_rt[rto.regression_id].append(rto) + return expected_by_rt + + +def derive_errors_for_run(test_id: int, + expected_outputs_by_rt: Dict[int, + List[Any]] = None, + preloaded_results=None, + preloaded_files=None) -> List[Dict[str, + Any]]: + """Walk result rows and emit one ErrorItem per detected failure.""" + progress = TestProgress.query.filter_by(test_id=test_id).order_by( + TestProgress.timestamp.desc()).first() + occurred_at = progress.timestamp.isoformat( + ) if progress and progress.timestamp else None + + if preloaded_results is not None: + results = preloaded_results + else: + results = TestResult.query.filter_by(test_id=test_id).all() + + files_by_result = _group_result_files(test_id, results, preloaded_files) + + if expected_outputs_by_rt is None: + expected_outputs_by_rt = _load_expected_outputs(results) + + errors = [] + for result in results: + result_files = files_by_result.get(result.regression_test_id, []) + expected_outputs = expected_outputs_by_rt.get( + result.regression_test_id) if expected_outputs_by_rt else None + errors.extend(_evaluate_test_result( + result, result_files, test_id, occurred_at, expected_outputs)) + + return errors + + +def _aggregate_error_into_bucket(err, bucket): + bucket['count'] += 1 + + # Escalate severity to the worst we've seen. + try: + curr_idx = _SEVERITY_ORDER.index(bucket['severity']) + new_idx = _SEVERITY_ORDER.index(err['severity']) + if new_idx > curr_idx: + bucket['severity'] = err['severity'] + except ValueError: + # Fallback if unknown severity + if err['severity'] == 'error': + bucket['severity'] = 'error' + + err_time = err.get('occurred_at') + if err_time: + if bucket['first_seen_at'] is None or err_time < bucket['first_seen_at']: + bucket['first_seen_at'] = err_time + if bucket['last_seen_at'] is None or err_time > bucket['last_seen_at']: + bucket['last_seen_at'] = err_time + + sid = err.get('sample_id') + if sid and sid not in bucket['sample_ids'] and len( + bucket['sample_ids']) < 1000: + bucket['sample_ids'].append(sid) + + +def derive_error_summary( + test_id: int, group_by: str = 'type') -> List[Dict[str, Any]]: + """Group errors by the given key and return bucket counts.""" + errors = derive_errors_for_run(test_id) + buckets: Dict[str, Dict[str, Any]] = {} + + for err in errors: + key = str(err.get(group_by, 'unknown')) + + if key not in buckets: + buckets[key] = { + 'key': key, + 'group_by': group_by, + 'count': 0, + 'severity': err['severity'], + 'sample_ids': [], + 'first_seen_at': None, + 'last_seen_at': None, + } + + _aggregate_error_into_bucket(err, buckets[key]) + + return list(buckets.values()) + + +def derive_infrastructure_errors(test_id: int) -> List[Dict[str, Any]]: + """ + Best-effort infra error extraction from TestProgress messages. + + There's no structured error protocol from the CI worker yet, so we + do keyword matching against progress messages to guess the failure type. + """ + errors = [] + progress_rows = TestProgress.query.filter_by( + test_id=test_id, + status=TestStatus.canceled, + ).all() + + for p in progress_rows: + message = p.message or '' + # User-initiated cancellations (cancel_run writes "... via API") are not + # infrastructure failures, so they must not be reported here. + if 'via API' in message: + continue + error_type = _classify_infra_error(message.lower()) + errors.append({ + 'error_id': f'infra_{test_id}_{p.id}', + 'run_id': test_id, + 'sample_id': None, + 'regression_id': None, + 'type': error_type, + 'severity': 'critical', + 'message': p.message or 'Unknown infrastructure error', + 'location': None, + 'occurred_at': p.timestamp.isoformat() if p.timestamp else None, + }) + + return errors + + +def _classify_infra_error(message_lower: str) -> str: + """Guess the infra error type from progress message keywords.""" + if any(w in message_lower for w in ['provisioning', 'vm ', 'instance']): + return 'vm_provisioning' + if any(w in message_lower for w in ['checkout', 'git clone', 'fetch']): + return 'checkout' + if any(w in message_lower for w in ['merge', 'conflict']): + return 'merge' + if any(w in message_lower for w in ['build', 'compile', 'make']): + return 'build' + if any(w in message_lower for w in ['worker', 'timeout', 'connection']): + return 'worker' + if any(w in message_lower for w in ['storage', 'disk', 'gcs']): + return 'storage' + return 'worker' + + +def _get_sample_id(result: TestResult): + """Pull sample_id through the RegressionTest relationship, if available.""" + try: + if result.regression_test and result.regression_test.sample_id: + return result.regression_test.sample_id + except Exception: + logging.getLogger(__name__).exception( + f"Failed to fetch sample_id for TestResult {result.test_id}_{result.regression_test_id}" + ) + return None diff --git a/mod_api/services/status.py b/mod_api/services/status.py new file mode 100644 index 000000000..adaf6227b --- /dev/null +++ b/mod_api/services/status.py @@ -0,0 +1,267 @@ +""" +Status derivation from the raw data model. + +Normalizes TestProgress/TestResult/TestResultFile states into clean +strings for the API layer. This is the single source of truth for +status logic — route handlers must not inline their own derivation. + +Run statuses: queued, running, pass, fail, canceled, error, incomplete +Sample statuses: pass, fail, skipped, missing_output, running, not_started + +Things to watch out for: + - test.failed only checks for TestStatus.canceled — never use it + for determining whether regression tests actually passed + - TestResultFile.got = null means MATCH, not missing output + - Dummy row (-1,-1,-1,'','error') = test produced no output at all + - TestStatus.canceled covers both user cancels and infra failures +""" + +from collections import defaultdict +from typing import List, Optional + +from sqlalchemy.orm import joinedload + +from mod_regression.models import RegressionTestOutput +from mod_test.models import (Test, TestProgress, TestResult, TestResultFile, + TestStatus) + + +def derive_run_status(test: Test) -> str: + """ + Map the raw model state to one of the 7 normalized run statuses. + + Looks at the most recent TestProgress row and, for completed runs, + counts actual failures from TestResult rows. + """ + statuses, _ = batch_get_run_data([test]) + return statuses.get(test.id, 'queued') + + +def _check_output_acceptable(rf: TestResultFile) -> bool: + if rf.regression_test_output: + for multi in rf.regression_test_output.multiple_files: + if multi.file_hashes == rf.got: + return True + return False + + +def _has_missing_output(result_files: List[TestResultFile], expected_outputs: Optional[List] = None) -> bool: + if expected_outputs is not None: + actual_output_ids = {rf.regression_test_output_id for rf in result_files} + for rto in expected_outputs: + if not rto.ignore and rto.id not in actual_output_ids: + return True + return False + else: + for rf in result_files: + if is_dummy_row(rf): + return True + return False + + +def derive_sample_status( + test_result: Optional[TestResult], + result_files: List[TestResultFile], + expected_outputs: Optional[List] = None, +) -> str: + """Map a TestResult + its output files to a per-sample status string. + + Checks for missing output first (expected outputs with no matching + TestResultFile), then exit code, then output diffs against accepted + baselines. + + Parameters + ---------- + test_result : Optional[TestResult] + The TestResult row, or None if the test hasn't run. + result_files : List[TestResultFile] + Actual output file rows from the database. + expected_outputs : Optional[List] + RegressionTestOutput rows that define what outputs were expected. + When provided, missing-output detection compares these against + result_files. When None, legacy dummy-row detection is used as + a fallback. + """ + if test_result is None: + return 'not_started' + + if _has_missing_output(result_files, expected_outputs): + return 'missing_output' + + if test_result.exit_code != test_result.expected_rc: + return 'fail' + + if any(rf.got is not None and not _check_output_acceptable(rf) + for rf in result_files): + return 'fail' + + return 'pass' + + +def is_dummy_row(rf: TestResultFile) -> bool: + """ + Detect the sentinel TestResultFile row where regression_test_output_id == -1 and got == 'error'. + + This row means the test produced no output when output was expected. + The old test_id == -1 and regression_test_id == -1 checks were removed + because they are no longer populated as -1 in newer data. + (Verified against production DB on 2026-06-25 by a maintainer: + 0 legacy rows exist.) + It should never show up as a real file in API responses. + """ + return bool(rf.regression_test_output_id == -1 and rf.got == 'error') + + +def derive_output_status(rf: TestResultFile) -> str: + """Classify a single output file: pass, fail, or missing_output.""" + if is_dummy_row(rf): + return 'missing_output' + if rf.got is None: + return 'pass' + return 'fail' + + +def get_run_timestamps(test: Test) -> dict: + """ + Build a timestamp dict from TestProgress rows. + + Test doesn't have a created_at column, so we use the earliest + progress entry as a proxy. + """ + _, timestamps = batch_get_run_data([test]) + ts = timestamps.get(test.id, {}) + return { + 'created_at': ts.get('created_at'), + 'queued_at': ts.get('queued_at'), + 'started_at': ts.get('started_at'), + 'completed_at': ts.get('completed_at'), + } + + +def _compute_run_timestamps(t_prog): + ts = { + 'created_at': None, + 'queued_at': None, + 'started_at': None, + 'completed_at': None, + } + if t_prog: + ts['queued_at'] = t_prog[0].timestamp + ts['created_at'] = t_prog[0].timestamp + for p in t_prog: + if p.status == TestStatus.testing and ts['started_at'] is None: + ts['started_at'] = p.timestamp + if p.status in (TestStatus.completed, TestStatus.canceled): + ts['completed_at'] = p.timestamp + return ts + + +def _check_completed_run_status( + t_id, + results_by_test, + files_by_test_and_rt, + expected_outputs_by_rt): + results = results_by_test.get(t_id, []) + if not results: + # A run marked completed that produced zero TestResult rows is not + # a pass — the worker finished without reporting anything. + return 'error' + for r in results: + r_files = files_by_test_and_rt.get((t_id, r.regression_test_id), []) + expected = expected_outputs_by_rt.get( + r.regression_test_id) if expected_outputs_by_rt is not None else None + sample_status = derive_sample_status(r, r_files, expected) + if sample_status not in ('pass', 'not_started'): + return 'fail' + return 'pass' + + +def _compute_run_status( + t_prog, + results_by_test, + files_by_test_and_rt, + t_id, + expected_outputs_by_rt=None): + if not t_prog: + return 'queued' + + raw_status = t_prog[-1].status + + if raw_status in (TestStatus.preparation, TestStatus.testing): + return 'running' + if raw_status == TestStatus.canceled: + return 'canceled' + if raw_status == TestStatus.completed: + return _check_completed_run_status( + t_id, + results_by_test, + files_by_test_and_rt, + expected_outputs_by_rt) + return 'incomplete' + + +def batch_get_run_data(tests: list) -> tuple: + """ + Batch compute derive_run_status and get_run_timestamps for a list of tests. + + Returns (statuses_dict, timestamps_dict) + """ + if not tests: + return {}, {} + + test_ids = [t.id for t in tests] + + # Preload TestProgress + all_progress = TestProgress.query.filter(TestProgress.test_id.in_( + test_ids)).order_by(TestProgress.id.asc()).all() + progress_by_test = {tid: [] for tid in test_ids} + for p in all_progress: + progress_by_test[p.test_id].append(p) + + # Preload TestResult + all_results = TestResult.query.filter( + TestResult.test_id.in_(test_ids)).all() + results_by_test = {tid: [] for tid in test_ids} + for r in all_results: + results_by_test[r.test_id].append(r) + + # Preload TestResultFile + + all_files = TestResultFile.query.options( + joinedload(TestResultFile.regression_test_output) + .joinedload(RegressionTestOutput.multiple_files) + ).filter(TestResultFile.test_id.in_(test_ids)).all() + files_by_test_and_rt = {} + for f in all_files: + key = (f.test_id, f.regression_test_id) + if key not in files_by_test_and_rt: + files_by_test_and_rt[key] = [] + files_by_test_and_rt[key].append(f) + + # Preload expected outputs (RegressionTestOutput) for missing-output + # detection + all_rt_ids = set() + for tid in test_ids: + for r in results_by_test.get(tid, []): + all_rt_ids.add(r.regression_test_id) + + expected_outputs_by_rt = {} + if all_rt_ids: + all_expected = RegressionTestOutput.query.filter( + RegressionTestOutput.regression_id.in_(all_rt_ids) + ).all() + expected_outputs_by_rt = defaultdict(list) + for rto in all_expected: + expected_outputs_by_rt[rto.regression_id].append(rto) + + statuses = {} + timestamps_dict = {} + + for t in tests: + t_prog = progress_by_test[t.id] + timestamps_dict[t.id] = _compute_run_timestamps(t_prog) + statuses[t.id] = _compute_run_status( + t_prog, results_by_test, files_by_test_and_rt, t.id, + expected_outputs_by_rt=expected_outputs_by_rt) + + return statuses, timestamps_dict diff --git a/mod_api/services/storage.py b/mod_api/services/storage.py new file mode 100644 index 000000000..f9d56533e --- /dev/null +++ b/mod_api/services/storage.py @@ -0,0 +1,77 @@ +""" +Storage helpers for resolving artifact locations. + +Artifacts can live in local SAMPLE_REPOSITORY, GCS, or both. When both +exist, GCS is preferred and a signed URL is returned. When only local +exists, storage_status is 'degraded'. When neither exists, it's 'missing'. +""" + +import logging +import os +from datetime import timedelta +from typing import Optional, Tuple + +logger = logging.getLogger(__name__) + + +def resolve_artifact(relative_path: str) -> Tuple[Optional[str], str]: + """ + Look for an artifact in local storage and GCS. + + Returns (download_url_or_None, storage_status). + """ + from run import config, storage_client_bucket + + sample_repo = config.get('SAMPLE_REPOSITORY', '') + local_path = os.path.join(sample_repo, relative_path) + # Prevent path traversal: resolved path must stay within sample_repo + real_base = os.path.realpath(sample_repo) + real_path = os.path.realpath(local_path) + if not (real_path.startswith(real_base + os.sep) or real_path == real_base): + return None, 'missing' + local_exists = os.path.isfile(local_path) + + gcs_url = None + if storage_client_bucket: + try: + blob = storage_client_bucket.blob(relative_path) + if blob.exists(): + gcs_url = blob.generate_signed_url( + version='v4', + # int() guards against a string value in config, which + # would otherwise raise inside timedelta and be swallowed + # by the except below as a silent 'degraded'. + expiration=timedelta(minutes=int(config.get( + 'GCS_SIGNED_URL_EXPIRY_LIMIT', 60))), + method='GET', + ) + except Exception as e: + logger.warning(f"Failed to generate GCS signed URL for {relative_path}: {e}") + gcs_url = None + + if local_exists and gcs_url: + return gcs_url, 'ok' + elif gcs_url: + return gcs_url, 'degraded' + elif local_exists: + return None, 'degraded' + else: + return None, 'missing' + + +def get_log_file_path(run_id: int) -> Optional[str]: + """Return the absolute path to a run's build log, or None if it doesn't exist.""" + from run import config + + sample_repo = config.get('SAMPLE_REPOSITORY', '') + log_path = os.path.join(sample_repo, 'LogFiles', f'{run_id}.txt') + + if os.path.isfile(log_path): + return log_path + return None + + +def get_test_results_base_path() -> str: + """Return the base directory where TestResults files are stored.""" + from run import config + return os.path.join(config.get('SAMPLE_REPOSITORY', ''), 'TestResults') diff --git a/mod_api/utils.py b/mod_api/utils.py new file mode 100644 index 000000000..8daae4d28 --- /dev/null +++ b/mod_api/utils.py @@ -0,0 +1,95 @@ +"""Pagination, serialization, and response formatting helpers.""" + +from flask import jsonify + + +def paginated_response(data, total, limit, offset, schema=None, truncated=False, extra_meta=None): + """Build an offset-paginated JSON response.""" + if schema: + serialized = schema.dump(data, many=True) + else: + serialized = data + + next_offset = offset + limit if (offset + limit) < total else None + + pagination = { + 'limit': limit, + 'offset': offset, + 'total': total, + 'next_offset': next_offset, + } + + if truncated: + pagination['truncated'] = True + + response = { + 'data': serialized, + 'pagination': pagination, + 'meta': {} + } + if extra_meta: + response['meta'].update(extra_meta) + + return jsonify(response) + + +def cursor_paginated_response(data, next_cursor, limit, schema=None): + """Build a cursor-paginated JSON response.""" + if schema: + serialized = schema.dump(data, many=True) + else: + serialized = data + + return jsonify({ + 'data': serialized, + 'pagination': { + 'limit': limit, + 'next_cursor': next_cursor, + }, + }) + + +def single_response(data, schema=None, http_status=200): + """Build a single-item JSON response.""" + if schema: + serialized = schema.dump(data) + else: + serialized = data + + response = jsonify(serialized) + response.status_code = http_status + return response + + +def get_sort_column(sort_param, column_map): + """Translate a sort string into an SQLAlchemy order_by clause. + + Handles descending sorts prefixed with '-' (e.g. '-created_at'). + """ + descending = sort_param.startswith('-') + # [1:] not lstrip('-'): lstrip would also swallow a second leading dash, + # silently normalizing bad input like '--created_at'. + field_name = sort_param[1:] if descending else sort_param + + column = column_map.get(field_name) + if column is None: + return None + + if descending: + return column.desc() + return column.asc() + + +def safe_resolve(base_path, filename): + """ + Resolve filename under base_path, rejecting path traversal. + + Returns the absolute path if it's safely within base_path, + or None if traversal was detected. + """ + import os + resolved = os.path.realpath(os.path.join(base_path, filename)) + base_real = os.path.realpath(base_path) + if not resolved.startswith(base_real + os.sep) and resolved != base_real: + return None + return resolved diff --git a/mod_auth/controllers.py b/mod_auth/controllers.py index a476b9afc..a1f773774 100755 --- a/mod_auth/controllers.py +++ b/mod_auth/controllers.py @@ -165,26 +165,30 @@ def github_redirect(): return f'https://github.com/login/oauth/authorize?client_id={github_client_id}&scope=public_repo' -def fetch_username_from_token() -> Any: +def fetch_username_from_token(user=None) -> Any: """ Get username from the GitHub token. + :param user: Optional user model to prevent redundant queries :return: username :rtype: str """ - import json - user = User.query.filter(User.id == g.user.id).first() + if user is None: + user = User.query.filter(User.id == g.user.id).first() + if user.github_token is None: return None url = 'https://api.github.com/user' session = requests.Session() session.auth = (user.email, user.github_token) try: - response = session.get(url) + response = session.get(url, timeout=(3.05, 10)) data = response.json() - return data['login'] + return data.get('login') except Exception as e: - g.log.error('Failed to fetch the user token') + import logging + log = getattr(g, 'log', logging.getLogger(__name__)) + log.error('Failed to fetch the user token') return None @@ -211,6 +215,12 @@ def github_callback(): if 'access_token' in response: user = User.query.filter(User.id == g.user.id).first() user.github_token = response['access_token'] + + # Fetch and store github_login + github_login = fetch_username_from_token(user) + if github_login: + user.github_login = github_login + g.db.commit() else: g.log.error("GitHub didn't return an access token") diff --git a/mod_auth/models.py b/mod_auth/models.py index 16233e98a..befceb266 100644 --- a/mod_auth/models.py +++ b/mod_auth/models.py @@ -32,10 +32,13 @@ class User(Base): name = Column(String(50), unique=True) email = Column(String(255), unique=True, nullable=True) github_token = Column(Text(), nullable=True) + # GitHub username; populated at OAuth login and used by the API to + # authorize fork-run triggers. + github_login = Column(String(255), nullable=True) password = Column(String(255), unique=False, nullable=False) role = Column(Role.db_type()) - def __init__(self, name, role=Role.user, email=None, password='', github_token=None) -> None: + def __init__(self, name, role=Role.user, email=None, password='', github_token=None, github_login=None) -> None: """ Parametrized constructor for the User model. @@ -55,6 +58,7 @@ def __init__(self, name, role=Role.user, email=None, password='', github_token=N self.password = password self.role = role self.github_token = github_token + self.github_login = github_login def __repr__(self) -> str: """ diff --git a/mod_test/controllers.py b/mod_test/controllers.py index 4c2477b85..e6ab08d53 100644 --- a/mod_test/controllers.py +++ b/mod_test/controllers.py @@ -375,6 +375,44 @@ def generate_diff(test_id: int, regression_test_id: int, output_id: int, to_view abort(404) +@mod_test.route('/diff////smart') +def smart_diff_view(test_id: int, regression_test_id: int, output_id: int): + """ + Return a semantic (smart) diff classification for an output as JSON. + + Unlike the line diff, this reports *how* the output differs (timing shift, + cosmetic padding/formatting/encoding, text change, missing/extra cues), so a + person or an agent gets an actionable answer instead of a wall of lines. + + :param test_id: id of the test + :type test_id: int + :param regression_test_id: id of the regression test + :type regression_test_id: int + :param output_id: id of the generated output + :type output_id: int + :return: JSON classification of the difference. + :rtype: flask.Response + """ + from run import config + + result = TestResultFile.query.filter(and_( + TestResultFile.test_id == test_id, + TestResultFile.regression_test_id == regression_test_id, + TestResultFile.regression_test_output_id == output_id + )).first() + + if result is None: + abort(404) + + path = os.path.join(config.get('SAMPLE_REPOSITORY', ''), 'TestResults') + try: + classification = result.generate_smart_diff(path) + except (OSError, UnicodeDecodeError): + classification = {'kind': 'unavailable', + 'summary': 'Output files are not available or not readable.'} + return jsonify(classification) + + @mod_test.route('/log-files/') @login_required def download_build_log_file(test_id): diff --git a/mod_test/models.py b/mod_test/models.py index 1463a0f38..b6ad63859 100644 --- a/mod_test/models.py +++ b/mod_test/models.py @@ -455,3 +455,30 @@ def read_lines(file_name: str) -> List[str]: return open(file_name, encoding='utf8').readlines() except UnicodeDecodeError: return open(file_name, encoding='cp1252').readlines() + + def generate_smart_diff(self, base_path: str) -> dict: + """ + Classify *how* the actual output differs from the expected baseline. + + Unlike the line diff, this returns a semantic classification (timing + shift, cosmetic padding/formatting/encoding, text change, missing/extra + cues) that a person or an agent can act on directly. + + :param base_path: The base path for the files location. + :type base_path: str + :return: A smart-diff classification with ``kind`` and ``summary`` keys. + :rtype: dict + """ + from mod_test.smartdiff.compare import smart_diff + + if not self.got: + return {'kind': 'identical', + 'summary': 'Output matches the expected baseline.'} + + extension = self.regression_test_output.correct_extension + file_ok = os.path.join(base_path, self.expected + extension) + file_fail = os.path.join(base_path, self.got + extension) + expected_text = ''.join(self.read_lines(file_ok)) + actual_text = ''.join(self.read_lines(file_fail)) + return smart_diff(expected_text, actual_text, + fmt=extension.lstrip('.').lower() or None) diff --git a/mod_test/smartdiff/__init__.py b/mod_test/smartdiff/__init__.py new file mode 100644 index 000000000..7143d27c2 --- /dev/null +++ b/mod_test/smartdiff/__init__.py @@ -0,0 +1,7 @@ +"""Semantic ("smart") diff for subtitle regression outputs. + +Unlike a raw line diff, this package classifies *how* two outputs differ +(timing shift, text change, missing/extra cues) so a person or an agent gets an +actionable answer instead of a wall of changed lines. Pure/Flask-decoupled so it +is fully unit-testable. +""" diff --git a/mod_test/smartdiff/compare.py b/mod_test/smartdiff/compare.py new file mode 100644 index 000000000..91e3d2b08 --- /dev/null +++ b/mod_test/smartdiff/compare.py @@ -0,0 +1,242 @@ +"""Semantic comparison of subtitle outputs: classify *how* two results differ.""" + +from typing import Dict, List, Optional + +from mod_test.smartdiff.normalize import classify_text_pair, plain +from mod_test.smartdiff.parsing import parse_subtitles +from mod_test.smartdiff.srt import Cue + +#: Cap on the number of per-cue change entries returned in a result. +_MAX_CHANGES = 25 + + +def _result(kind: str, summary: str, n_exp: int, n_act: int, + offset_ms: Optional[int] = None) -> Dict[str, object]: + """ + Build a classification result dict. + + :param kind: The stable difference kind. + :type kind: str + :param summary: A human/agent-readable one-line explanation. + :type summary: str + :param n_exp: Number of expected cues. + :type n_exp: int + :param n_act: Number of actual cues. + :type n_act: int + :param offset_ms: Consistent timing offset, when ``kind`` is ``timing_shift``. + :type offset_ms: Optional[int] + :return: The classification result. + :rtype: Dict[str, object] + """ + out: Dict[str, object] = { + 'kind': kind, + 'summary': summary, + 'expected_cues': n_exp, + 'actual_cues': n_act, + } + if offset_ms is not None: + out['offset_ms'] = offset_ms + return out + + +def _content(cues: List[Cue]) -> str: + """ + Join all cues' normalised, whitespace-collapsed text — for split/merge detection. + + :param cues: The parsed cues. + :type cues: List[Cue] + :return: A single normalised token string spanning every cue. + :rtype: str + """ + return ' '.join(' '.join(plain(cue.text).split()) for cue in cues) + + +def _monotonic(values: List[int]) -> bool: + """ + Report whether a sequence is non-decreasing or non-increasing. + + :param values: The sequence to test. + :type values: List[int] + :return: True if monotonic in either direction. + :rtype: bool + """ + pairs = list(zip(values, values[1:])) + non_decreasing = all(a <= b for a, b in pairs) + non_increasing = all(a >= b for a, b in pairs) + return non_decreasing or non_increasing + + +def _snippet(text: str, limit: int = 80) -> str: + """ + Collapse whitespace and truncate cue text for compact change details. + + :param text: Raw cue text. + :type text: str + :param limit: Maximum characters to keep. + :type limit: int + :return: A single-line, length-capped snippet. + :rtype: str + """ + flat = ' '.join(text.split()) + return flat if len(flat) <= limit else flat[:limit] + '…' + + +def smart_diff(expected: str, actual: str, + fmt: Optional[str] = None) -> Dict[str, object]: + """ + Compare expected vs actual subtitle output and classify the difference. + + Supports SubRip (.srt) and WebVTT (.vtt); the format is auto-detected from + content unless ``fmt`` is given. Aligns cues by position and reports the + *kind* of difference rather than a raw line diff: ``identical``, + ``timing_shift`` (constant offset), ``timing_drift`` (growing offset), + ``text_change``, ``formatting_change`` (tags/entities only), + ``whitespace_change`` (CEA-608 padding only), ``encoding_change`` + (non-ASCII/accented characters only), ``split_cues``, ``merged_cues``, + ``missing_cues``, ``extra_cues``, ``unsupported`` (no cues parsed), or + ``mixed``. + + :param expected: The expected/baseline subtitle content. + :type expected: str + :param actual: The actual/produced subtitle content. + :type actual: str + :param fmt: Explicit format ('srt' or 'vtt'); auto-detected when None. + :type fmt: Optional[str] + :return: A classification dict with keys ``kind``, ``summary``, + ``expected_cues``, ``actual_cues`` and (for ``timing_shift``) ``offset_ms``. + :rtype: Dict[str, object] + """ + exp = parse_subtitles(expected, fmt) + act = parse_subtitles(actual, fmt) + n_exp, n_act = len(exp), len(act) + if n_exp == 0 and n_act == 0: + if expected == actual: + return _result('identical', 'Outputs are identical.', 0, 0) + return _result( + 'unsupported', + 'No subtitle cues to compare (unsupported format); see the raw diff.', + 0, 0) + count_mismatch = n_exp != n_act + + text_changes = 0 + formatting_changes = 0 + whitespace_changes = 0 + encoding_changes = 0 + raw_matches = True + timing_deltas: List[int] = [] + changes: List[Dict[str, object]] = [] + for position, (expected_cue, actual_cue) in enumerate(zip(exp, act), start=1): + category = classify_text_pair(expected_cue.text, actual_cue.text) + delta = actual_cue.start_ms - expected_cue.start_ms + if category != 'match': + raw_matches = False + if category == 'text': + text_changes += 1 + else: + if category == 'formatting': + formatting_changes += 1 + elif category == 'whitespace': + whitespace_changes += 1 + elif category == 'encoding': + encoding_changes += 1 + timing_deltas.append(delta) + if category != 'match' or delta != 0: + entry: Dict[str, object] = { + 'cue': position, + 'kind': category if category != 'match' else 'timing', + } + if category == 'text': + entry['expected'] = _snippet(expected_cue.text) + entry['actual'] = _snippet(actual_cue.text) + if delta != 0: + entry['offset_ms'] = delta + changes.append(entry) + + no_timing_move = all(delta == 0 for delta in timing_deltas) + uniform_shift = bool(timing_deltas) and len(set(timing_deltas)) == 1 + varying_timing = len(set(timing_deltas)) > 1 + drifting = varying_timing and _monotonic(timing_deltas) + cosmetic_changes = formatting_changes + whitespace_changes + encoding_changes + fully_aligned = text_changes == 0 and cosmetic_changes == 0 + + def _finish(kind: str, summary: str, exp_count: int, act_count: int, + offset_ms: Optional[int] = None) -> Dict[str, object]: + """Attach the (capped) per-cue change list to a classification result.""" + out = _result(kind, summary, exp_count, act_count, offset_ms) + if changes: + out['changes'] = changes[:_MAX_CHANGES] + if len(changes) > _MAX_CHANGES: + out['changes_truncated'] = True + return out + + if not count_mismatch and raw_matches and no_timing_move: + return _finish('identical', 'Outputs are identical.', n_exp, n_act) + + if not count_mismatch and fully_aligned and uniform_shift and timing_deltas[0] != 0: + offset = timing_deltas[0] + direction = 'late' if offset > 0 else 'early' + return _finish( + 'timing_shift', + f'All {n_exp} cues match but are {abs(offset)} ms {direction}.', + n_exp, n_act, offset_ms=offset) + + if not count_mismatch and fully_aligned and drifting: + first, last = timing_deltas[0], timing_deltas[-1] + return _finish( + 'timing_drift', + f'Timing drifts from {first:+d} ms to {last:+d} ms across {n_exp} cues.', + n_exp, n_act) + + if count_mismatch: + exp_content = _content(exp) + if exp_content and exp_content == _content(act): + if n_act > n_exp: + return _finish( + 'split_cues', + f'Same text, but cues were split: expected {n_exp}, got {n_act}.', + n_exp, n_act) + return _finish( + 'merged_cues', + f'Same text, but cues were merged: expected {n_exp}, got {n_act}.', + n_exp, n_act) + if text_changes == 0: + if n_act < n_exp: + return _finish( + 'missing_cues', + f'{n_exp - n_act} of {n_exp} cues are missing from the output.', + n_exp, n_act) + return _finish( + 'extra_cues', + f'Output has {n_act - n_exp} extra cues ({n_act} vs {n_exp} expected).', + n_exp, n_act) + + if not count_mismatch and no_timing_move: + if text_changes > 0: + return _finish( + 'text_change', + f'{text_changes} of {n_exp} cues differ in text.', + n_exp, n_act) + if encoding_changes > 0 and formatting_changes == 0 and whitespace_changes == 0: + return _finish( + 'encoding_change', + f'{encoding_changes} of {n_exp} cues differ only in character ' + f'encoding (non-ASCII/accented characters).', + n_exp, n_act) + if formatting_changes > 0 and whitespace_changes == 0 and encoding_changes == 0: + return _finish( + 'formatting_change', + f'{formatting_changes} of {n_exp} cues differ only in formatting ' + f'(tags/entities), not text.', + n_exp, n_act) + if whitespace_changes > 0 and formatting_changes == 0 and encoding_changes == 0: + return _finish( + 'whitespace_change', + f'{whitespace_changes} of {n_exp} cues differ only in trailing ' + f'whitespace/padding.', + n_exp, n_act) + + return _finish( + 'mixed', + f'Mixed differences across {min(n_exp, n_act)} compared cues; ' + f'expected {n_exp}, got {n_act}.', + n_exp, n_act) diff --git a/mod_test/smartdiff/normalize.py b/mod_test/smartdiff/normalize.py new file mode 100644 index 000000000..a460c0b7e --- /dev/null +++ b/mod_test/smartdiff/normalize.py @@ -0,0 +1,115 @@ +"""Normalisation that mirrors CCExtractor's own expected-output handling. + +CCExtractor's test harness (``tests/extract_expected.py``) compares outputs +after stripping HTML/styling tags, unescaping entities, and trimming trailing +whitespace from each line (CEA-608 captions are space-padded to a fixed grid). +Reusing the same rules lets the smart diff separate a *cosmetic* difference +(padding or styling only) from a real text change. +""" + +import re +import unicodedata + +_TAG_RE = re.compile(r'<[^>]+>') + +# Same entities CCExtractor's extract_expected.py unescapes; '&' is applied +# last so an escaped entity like '&lt;' is not double-decoded. +_ENTITIES = ( + ('<', '<'), ('>', '>'), ('"', '"'), (''', "'"), + ('°', '°'), (' ', ' '), ('&', '&'), +) + + +def strip_tags(text: str) -> str: + """ + Remove HTML/styling tags such as ```` or ````. + + :param text: Raw cue text. + :type text: str + :return: Text with tags removed. + :rtype: str + """ + return _TAG_RE.sub('', text) + + +def unescape(text: str) -> str: + """ + Unescape the HTML entities CCExtractor emits. + + :param text: Raw cue text. + :type text: str + :return: Text with entities decoded. + :rtype: str + """ + for entity, char in _ENTITIES: + text = text.replace(entity, char) + return text + + +def rstrip_lines(text: str) -> str: + """ + Trim trailing whitespace from each line (CEA-608 padding is cosmetic). + + :param text: Raw cue text. + :type text: str + :return: Text with per-line trailing whitespace removed. + :rtype: str + """ + return '\n'.join(line.rstrip() for line in text.split('\n')) + + +def plain(text: str) -> str: + """ + Fully normalise: unescape entities, strip tags, trim trailing whitespace. + + :param text: Raw cue text. + :type text: str + :return: The fully normalised text. + :rtype: str + """ + return rstrip_lines(strip_tags(unescape(text))) + + +def ascii_fold(text: str) -> str: + """ + Fold text to ASCII by decomposing accents and dropping non-ASCII characters. + + Lets the comparator tell a charset/encoding difference (e.g. CCExtractor's + ``-latin1`` output) from a real word change: 'Voilà' and 'Voila' share an + ASCII skeleton, so only their non-ASCII characters differ. + + :param text: Raw cue text. + :type text: str + :return: The ASCII skeleton of the text. + :rtype: str + """ + decomposed = unicodedata.normalize('NFKD', text) + return ''.join(ch for ch in decomposed if ord(ch) < 128) + + +def classify_text_pair(expected: str, actual: str) -> str: + """ + Classify how two cue texts differ, ignoring progressively more cosmetics. + + :param expected: Expected cue text. + :type expected: str + :param actual: Actual cue text. + :type actual: str + :return: ``match`` (identical), ``whitespace`` (only trailing padding differs), + ``formatting`` (only tags/entities differ), ``encoding`` (only non-ASCII + characters differ), or ``text`` (a real change). + :rtype: str + """ + if expected == actual: + return 'match' + if rstrip_lines(expected) == rstrip_lines(actual): + return 'whitespace' + expected_plain = plain(expected) + actual_plain = plain(actual) + if expected_plain == actual_plain: + return 'formatting' + folded_expected = ascii_fold(expected_plain) + non_ascii = any(ord(ch) > 127 for ch in expected_plain + actual_plain) + if non_ascii and folded_expected and folded_expected == ascii_fold(actual_plain): + return 'encoding' + return 'text' diff --git a/mod_test/smartdiff/parsing.py b/mod_test/smartdiff/parsing.py new file mode 100644 index 000000000..35bfe3d66 --- /dev/null +++ b/mod_test/smartdiff/parsing.py @@ -0,0 +1,26 @@ +"""Detect the subtitle format and dispatch to the right parser.""" + +from typing import List, Optional + +from mod_test.smartdiff.srt import Cue, parse_srt +from mod_test.smartdiff.vtt import parse_vtt + + +def parse_subtitles(content: str, fmt: Optional[str] = None) -> List[Cue]: + """ + Parse subtitle content into cues, choosing a parser by hint or by content. + + :param content: Raw subtitle file content. + :type content: str + :param fmt: Explicit format ('srt' or 'vtt'); auto-detected from content when None. + :type fmt: Optional[str] + :return: The parsed cues. + :rtype: List[Cue] + """ + chosen = (fmt or '').lower() + if not chosen: + head = content.lstrip('').lstrip().upper() + chosen = 'vtt' if head.startswith('WEBVTT') else 'srt' + if chosen == 'vtt': + return parse_vtt(content) + return parse_srt(content) diff --git a/mod_test/smartdiff/srt.py b/mod_test/smartdiff/srt.py new file mode 100644 index 000000000..3f7388ed8 --- /dev/null +++ b/mod_test/smartdiff/srt.py @@ -0,0 +1,102 @@ +"""Parse SubRip (.srt) subtitle output into structured cues for comparison.""" + +import re +from dataclasses import dataclass +from typing import List, Optional + +_TIMING_RE = re.compile( + r'(\d{1,2}):(\d{2}):(\d{2})[,.](\d{1,3})\s*-->\s*' + r'(\d{1,2}):(\d{2}):(\d{2})[,.](\d{1,3})' +) + + +@dataclass +class Cue: + """ + A single subtitle cue: its timing window and text. + + :param index: The cue's sequence number as written in the file. + :type index: int + :param start_ms: Start time in milliseconds. + :type start_ms: int + :param end_ms: End time in milliseconds. + :type end_ms: int + :param text: The cue's text, newlines preserved and surrounding whitespace stripped. + :type text: str + """ + + index: int + start_ms: int + end_ms: int + text: str + + +def join_cue_text(lines: List[str]) -> str: + """ + Join cue text lines, dropping surrounding blank lines but keeping trailing spaces. + + Trailing whitespace is preserved on purpose: CCExtractor pads CEA-608 captions, + and the comparator (not the parser) decides whether that padding is cosmetic. + + :param lines: The text lines following a cue's timing line. + :type lines: List[str] + :return: The joined cue text. + :rtype: str + """ + start, end = 0, len(lines) + while start < end and lines[start].strip() == '': + start += 1 + while end > start and lines[end - 1].strip() == '': + end -= 1 + return '\n'.join(lines[start:end]) + + +def _to_ms(hours: str, minutes: str, seconds: str, millis: str) -> int: + """ + Convert the parts of an SRT timestamp into total milliseconds. + + :param hours: Hours component. + :type hours: str + :param minutes: Minutes component. + :type minutes: str + :param seconds: Seconds component. + :type seconds: str + :param millis: Milliseconds component. + :type millis: str + :return: The timestamp in milliseconds. + :rtype: int + """ + return ((int(hours) * 60 + int(minutes)) * 60 + int(seconds)) * 1000 + int(millis) + + +def parse_srt(content: str) -> List[Cue]: + """ + Parse SubRip subtitle text into a list of cues. + + Tolerant of a leading BOM, CRLF/CR line endings, and either ',' or '.' as the + millisecond separator. Blocks without a valid timing line are skipped. + + :param content: Raw .srt file content. + :type content: str + :return: The parsed cues, in file order. + :rtype: List[Cue] + """ + content = content.lstrip('').replace('\r\n', '\n').replace('\r', '\n') + cues: List[Cue] = [] + for block in re.split(r'\n[ \t]*\n', content.strip()): + lines = block.split('\n') + timing_idx: Optional[int] = next( + (i for i, ln in enumerate(lines) if _TIMING_RE.search(ln)), None) + if timing_idx is None: + continue + match = _TIMING_RE.search(lines[timing_idx]) + if match is None: # pragma: no cover - guaranteed by the search above + continue + start_ms = _to_ms(match.group(1), match.group(2), match.group(3), match.group(4)) + end_ms = _to_ms(match.group(5), match.group(6), match.group(7), match.group(8)) + index = len(cues) + 1 + if timing_idx > 0 and lines[timing_idx - 1].strip().isdigit(): + index = int(lines[timing_idx - 1].strip()) + text = join_cue_text(lines[timing_idx + 1:]) + cues.append(Cue(index=index, start_ms=start_ms, end_ms=end_ms, text=text)) + return cues diff --git a/mod_test/smartdiff/vtt.py b/mod_test/smartdiff/vtt.py new file mode 100644 index 000000000..dbb12e849 --- /dev/null +++ b/mod_test/smartdiff/vtt.py @@ -0,0 +1,65 @@ +"""Parse WebVTT (.vtt) subtitle output into structured cues.""" + +import re +from typing import List, Optional + +from mod_test.smartdiff.srt import Cue, join_cue_text + +_TIMING_RE = re.compile( + r'(?:(\d{1,2}):)?(\d{2}):(\d{2})[.,](\d{3})\s*-->\s*' + r'(?:(\d{1,2}):)?(\d{2}):(\d{2})[.,](\d{3})' +) + +_METADATA_PREFIXES = ('WEBVTT', 'NOTE', 'STYLE', 'REGION') + + +def _to_ms(hours: Optional[str], minutes: str, seconds: str, millis: str) -> int: + """ + Convert WebVTT timestamp parts into total milliseconds. + + :param hours: Hours component, or None when absent (MM:SS.mmm form). + :type hours: Optional[str] + :param minutes: Minutes component. + :type minutes: str + :param seconds: Seconds component. + :type seconds: str + :param millis: Milliseconds component. + :type millis: str + :return: The timestamp in milliseconds. + :rtype: int + """ + hrs = int(hours) if hours else 0 + return ((hrs * 60 + int(minutes)) * 60 + int(seconds)) * 1000 + int(millis) + + +def parse_vtt(content: str) -> List[Cue]: + """ + Parse WebVTT subtitle text into a list of cues. + + Skips the ``WEBVTT`` header and ``NOTE``/``STYLE``/``REGION`` blocks, tolerates + an optional cue-identifier line, optional hours in timestamps, and trailing cue + settings after the end timestamp. + + :param content: Raw .vtt file content. + :type content: str + :return: The parsed cues, in file order. + :rtype: List[Cue] + """ + content = content.lstrip('').replace('\r\n', '\n').replace('\r', '\n') + cues: List[Cue] = [] + for block in re.split(r'\n[ \t]*\n', content.strip()): + lines = block.split('\n') + if lines[0].split(' ', 1)[0] in _METADATA_PREFIXES: + continue + timing_idx: Optional[int] = next( + (i for i, ln in enumerate(lines) if _TIMING_RE.search(ln)), None) + if timing_idx is None: + continue + match = _TIMING_RE.search(lines[timing_idx]) + if match is None: # pragma: no cover - guaranteed by the search above + continue + start_ms = _to_ms(match.group(1), match.group(2), match.group(3), match.group(4)) + end_ms = _to_ms(match.group(5), match.group(6), match.group(7), match.group(8)) + text = join_cue_text(lines[timing_idx + 1:]) + cues.append(Cue(index=len(cues) + 1, start_ms=start_ms, end_ms=end_ms, text=text)) + return cues diff --git a/requirements.txt b/requirements.txt index bffe3bea4..3a0233262 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,11 +6,11 @@ python-magic==0.4.27 flask-wtf==1.3.0 requests==2.34.2 pyIsEmail==2.0.1 -GitPython==3.1.50 +GitPython==3.1.57 xmltodict==1.0.4 lxml==6.1.1 -pytz==2026.2 -tzlocal==5.4.3 +pytz==2026.3.post1 +tzlocal==5.4.4 markdown2==2.5.5 flask-migrate==4.1.0 email_validator @@ -20,10 +20,11 @@ WTForms==3.2.2 MarkupSafe==3.0.3 jinja2==3.1.6 itsdangerous==2.2.0 -google-api-python-client==2.197.0 -google-cloud-storage==3.12.0 -cffi==2.0.0 +google-api-python-client==2.198.0 +google-cloud-storage==3.13.0 +cffi==2.1.0 PyGithub==2.9.1 blinker==1.9.0 -click==8.4.1 +click==8.4.2 PyYAML==6.0.3 +marshmallow==4.3.0 diff --git a/run.py b/run.py index e277c6d97..efdbbfcb9 100755 --- a/run.py +++ b/run.py @@ -24,6 +24,7 @@ SecretKeyInstallationException) from log_configuration import LogConfiguration from mailer import Mailer +from mod_api import mod_api from mod_auth.controllers import mod_auth from mod_ci.controllers import mod_ci from mod_customized.controllers import mod_customized @@ -35,7 +36,7 @@ from mod_upload.controllers import mod_upload app = Flask(__name__) -app.wsgi_app = ProxyFix(app.wsgi_app) # type: ignore[method-assign] +app.wsgi_app = ProxyFix(app.wsgi_app, x_for=1) # type: ignore[method-assign] # Load config try: config = parse_config('config') @@ -273,3 +274,5 @@ def teardown(exception: Optional[Exception]): app.register_blueprint(mod_ci) app.register_blueprint(mod_customized, url_prefix='/custom') app.register_blueprint(mod_health) +# REST API v1 +app.register_blueprint(mod_api, url_prefix='/api/v1') diff --git a/templates/test/by_id.html b/templates/test/by_id.html index ca3c38e45..d789bb953 100644 --- a/templates/test/by_id.html +++ b/templates/test/by_id.html @@ -149,14 +149,14 @@

Fail + Fail
Smart {%- endif %} {% elif file.got == "error" %} No output generated but there should be {% elif file.got is none or no_error.found or (test.result and test.result.exit_code != 0) -%} Pass {% else %} - Fail + Fail
Smart {%- endif %} {% if not loop.last %}
{% endif %} {% else %} @@ -298,6 +298,42 @@

There are no tests executed in this category.
popup.open(); }); }); + $('.smart_diff_link').on('click', function(){ + // Fetch the semantic (smart) diff classification and show a summary. + var url = '{{ url_for('test.smart_diff_view', test_id='_0_', regression_test_id='_1_', output_id='_2_') }}'; + url = url.replace('_0_', $(this).data('test')).replace('_1_', $(this).data('regression')).replace('_2_', $(this).data('output')); + + $.getJSON(url).done(function(resp){ + var id, reveal, popup; + + reveal = document.createElement('div'); + id = 'smart-diff-popup-'+(new Date()).getTime(); + reveal.setAttribute('id', id); + reveal.setAttribute('class', 'reveal'); + reveal.setAttribute('data-reveal', ''); + reveal.innerHTML = + '

Smart diff

' + + '

' + (resp.kind || 'unknown') + '

' + + '

' + (resp.summary || '') + '

'; + if (resp.changes && resp.changes.length) { + var items = resp.changes.map(function(c){ + var d = 'Cue ' + c.cue + ': ' + c.kind; + if (c.offset_ms !== undefined) { d += ' (' + (c.offset_ms > 0 ? '+' : '') + c.offset_ms + ' ms)'; } + if (c.expected !== undefined) { d += ' — expected “' + c.expected + '”, got “' + c.actual + '”'; } + return '
  • ' + $('
    ').text(d).html() + '
    • '; + }).join(''); + reveal.innerHTML += '
      ' + items + '
    '; + if (resp.changes_truncated) { reveal.innerHTML += '

    … more changes not shown.

    '; } + } + reveal.innerHTML += + ''; + document.body.appendChild(reveal); + popup = new Foundation.Reveal($('#'+id)); + popup.open(); + }); + }); }); {% endblock %} diff --git a/test-requirements.txt b/test-requirements.txt index 2fa2612ae..c7ddd6f36 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -2,7 +2,7 @@ pycodestyle==2.14.0 pydocstyle==6.3.0 dodgy==0.2.1 isort==8.0.1 -mypy==2.1.0 +mypy==2.3.0 Flask-Testing==0.8.1 nose2-cov==1.0a4 -coverage==7.14.3 +coverage==7.15.2 diff --git a/tests/api/__init__.py b/tests/api/__init__.py new file mode 100644 index 000000000..1b3faf025 --- /dev/null +++ b/tests/api/__init__.py @@ -0,0 +1 @@ +"""Tests for API routes.""" diff --git a/tests/api/base.py b/tests/api/base.py new file mode 100644 index 000000000..fce4e9ebe --- /dev/null +++ b/tests/api/base.py @@ -0,0 +1,43 @@ +"""Shared base class for the API test package.""" + +from unittest.mock import patch + +from tests.base import BaseTestCase + + +def _mock_generate_hash(password): + return f"mock_hash_{password}" + + +def _mock_is_password_valid(self, password): + return self.password == f"mock_hash_{password}" + + +class ApiTestCase(BaseTestCase): + """BaseTestCase with password hashing stubbed out. + + sha512_crypt is deliberately slow, and almost every API test creates a + user and requests a token. Stubbing the hash cuts the package runtime + from minutes to seconds. A test that needs real hashing can call + cls._hash_patchers[i].stop() locally (none do today). + """ + + @classmethod + def setUpClass(cls): + """Patch User hashing for the whole test class.""" + cls._hash_patchers = [ + patch('mod_auth.models.User.generate_hash', + staticmethod(_mock_generate_hash)), + patch('mod_auth.models.User.is_password_valid', + _mock_is_password_valid), + ] + for patcher in cls._hash_patchers: + patcher.start() + super().setUpClass() + + @classmethod + def tearDownClass(cls): + """Restore the real hashing implementations.""" + super().tearDownClass() + for patcher in cls._hash_patchers: + patcher.stop() diff --git a/tests/api/test_middleware_auth.py b/tests/api/test_middleware_auth.py new file mode 100644 index 000000000..a2e16013e --- /dev/null +++ b/tests/api/test_middleware_auth.py @@ -0,0 +1,173 @@ +from flask import g + +from mod_api.models.api_token import DEFAULT_SCOPES, ApiToken +from mod_auth.models import Role, User +from tests.api.base import ApiTestCase + + +class TestMiddlewareAuth(ApiTestCase): + def setUp(self): + super().setUp() + user = User('testuser1', Role.user, 'testuser1@local.com', + User.generate_hash('user123')) + admin = User('testadmin1', Role.admin, + 'testadmin1@local.com', User.generate_hash('admin123')) + g.db.add_all([user, admin]) + g.db.commit() + self.user = user + self.admin = admin + + def generate_db_token(self, user, scopes=None, expires_in_days=7): + plaintext = ApiToken.generate_token() + token = ApiToken( + user_id=user.id, + token_name='test_token_' + ApiTestCase.create_random_string(8), + token_hash=ApiToken.hash_token(plaintext), + token_prefix=ApiToken.extract_prefix(plaintext), + scopes=scopes or DEFAULT_SCOPES, + expires_in_days=expires_in_days + ) + g.db.add(token) + g.db.commit() + return plaintext, token + + def test_missing_auth_header(self): + res = self.client.get('/api/v1/system/queue') + self.assertEqual(res.status_code, 401) + self.assertEqual(res.json['code'], 'unauthorized') + + def test_invalid_auth_header_format(self): + res = self.client.get('/api/v1/system/queue', + headers={'Authorization': 'InvalidFormat'}) + self.assertEqual(res.status_code, 401) + + res = self.client.get('/api/v1/system/queue', + headers={'Authorization': 'Bearer '}) + self.assertEqual(res.status_code, 401) + + def test_invalid_token_prefix(self): + res = self.client.get( + '/api/v1/system/queue', headers={'Authorization': 'Bearer invalid_prefix_token'}) + self.assertEqual(res.status_code, 401) + + def test_token_not_found(self): + res = self.client.get( + '/api/v1/system/queue', headers={'Authorization': 'Bearer spci_faketoken1234567890'}) + self.assertEqual(res.status_code, 401) + + def test_wrong_hash(self): + plaintext, token = self.generate_db_token(self.user) + wrong_token = token.token_prefix + 'A' * \ + (len(plaintext) - len(token.token_prefix)) + res = self.client.get('/api/v1/system/queue', + headers={'Authorization': f'Bearer {wrong_token}'}) + self.assertEqual(res.status_code, 401) + + def test_revoked_token(self): + plaintext, token = self.generate_db_token(self.user) + token.revoke() + g.db.commit() + res = self.client.get('/api/v1/system/queue', + headers={'Authorization': f'Bearer {plaintext}'}) + self.assertEqual(res.status_code, 401) + + def test_expired_token(self): + plaintext, _ = self.generate_db_token(self.user, expires_in_days=-1) + res = self.client.get('/api/v1/system/queue', + headers={'Authorization': f'Bearer {plaintext}'}) + self.assertEqual(res.status_code, 401) + + def test_valid_token_missing_scope(self): + # /api/v1/system/queue requires 'system:read' + plaintext, _ = self.generate_db_token(self.user, scopes=['runs:read']) + res = self.client.get('/api/v1/system/queue', + headers={'Authorization': f'Bearer {plaintext}'}) + self.assertEqual(res.status_code, 403) + self.assertIn('code', res.json) + self.assertEqual(res.json['code'], 'forbidden') + self.assertIn('missing_scopes', res.json['details']) + + def test_valid_token_with_scope(self): + plaintext, _ = self.generate_db_token(self.user, scopes=['system:read']) + res = self.client.get('/api/v1/system/queue', + headers={'Authorization': f'Bearer {plaintext}'}) + self.assertEqual(res.status_code, 200) + + def test_role_decorator_missing_role(self): + # GET /api/v1/auth/tokens requires 'tokens:manage' and roles ['admin', 'contributor', 'tester'] + plaintext, _ = self.generate_db_token( + self.user, scopes=['tokens:manage']) # role is user + res = self.client.get('/api/v1/auth/tokens', + headers={'Authorization': f'Bearer {plaintext}'}) + self.assertEqual(res.status_code, 403) + self.assertEqual(res.json['code'], 'forbidden') + + def test_role_decorator_with_role(self): + plaintext, _ = self.generate_db_token( + self.admin, scopes=['tokens:manage']) # role is admin + res = self.client.get('/api/v1/auth/tokens', + headers={'Authorization': f'Bearer {plaintext}'}) + self.assertEqual(res.status_code, 200) + + def test_scope_boundary_write_endpoints_fail_on_read_only_scopes(self): + plaintext, _ = self.generate_db_token( + self.user, scopes=['runs:read', 'results:read']) + + # 1. POST /runs + res = self.client.post( + '/api/v1/runs', headers={'Authorization': f'Bearer {plaintext}'}) + self.assertEqual(res.status_code, 403) + self.assertEqual(res.json['code'], 'forbidden') + + # 2. POST /runs/1/cancel + res = self.client.post('/api/v1/runs/1/cancel', + headers={'Authorization': f'Bearer {plaintext}'}) + self.assertEqual(res.status_code, 403) + self.assertEqual(res.json['code'], 'forbidden') + + # 3. POST /runs/1/samples/1/baseline-approval + res = self.client.post('/api/v1/runs/1/samples/1/baseline-approval', + headers={'Authorization': f'Bearer {plaintext}'}) + self.assertEqual(res.status_code, 403) + self.assertEqual(res.json['code'], 'forbidden') + + def test_multiple_candidates_same_prefix(self): + plaintext1, token1 = self.generate_db_token(self.user, scopes=['system:read']) + plaintext2, token2 = self.generate_db_token(self.user, scopes=['system:read']) + + # Force same prefix, must start with spci_ and be 16 chars long for extract_prefix + prefix = 'spci_abc12345678' + token1.token_prefix = prefix + token2.token_prefix = prefix + g.db.commit() + + # Modify plaintexts to have the same prefix + submitted1 = prefix + plaintext1[len(prefix):] + submitted2 = prefix + plaintext2[len(prefix):] + + token1.token_hash = ApiToken.hash_token(submitted1) + token2.token_hash = ApiToken.hash_token(submitted2) + g.db.commit() + + # It should correctly match token2 and ignore token1 + res = self.client.get('/api/v1/system/queue', + headers={'Authorization': f'Bearer {submitted2}'}) + self.assertEqual(res.status_code, 200) + + # Invalid token with same prefix + submitted3 = prefix + 'A' * 32 + res3 = self.client.get( + '/api/v1/system/queue', headers={'Authorization': f'Bearer {submitted3}'}) + self.assertEqual(res3.status_code, 401) + + def test_auth_sets_g_api_user_and_token(self): + plaintext, token = self.generate_db_token(self.user, scopes=['system:read']) + expected_user_id = self.user.id + expected_token_id = token.id + with self.app.test_request_context('/api/v1/system/queue', headers={'Authorization': f'Bearer {plaintext}'}): + # This triggers all before_request handlers, including authenticate_request + resp = self.app.preprocess_request() + # If rate limit isn't cleared, it might return 429, but it is cleared in setUp + self.assertIsNone(resp) + self.assertEqual(g.api_user.id, expected_user_id) + self.assertEqual(g.api_token.id, expected_token_id) diff --git a/tests/api/test_middleware_error_handler.py b/tests/api/test_middleware_error_handler.py new file mode 100644 index 000000000..d08e98b29 --- /dev/null +++ b/tests/api/test_middleware_error_handler.py @@ -0,0 +1,62 @@ +from unittest.mock import patch + +from flask import g + +from mod_api.middleware.rate_limit import _rate_limit_store +from mod_auth.models import Role, User +from tests.api.base import ApiTestCase + + +class TestMiddlewareErrorHandler(ApiTestCase): + def setUp(self): + super().setUp() + _rate_limit_store.clear() + self.user = User( + 'testuser_err', + Role.user, + 'testuser_err@local.com', + User.generate_hash('userpass123')) + g.db.add(self.user) + g.db.commit() + + def test_500_error_is_json(self): + """Test that unhandled exceptions produce a JSON 500 response.""" + original_testing = self.app.config['TESTING'] + self.app.config['TESTING'] = False + + # Suppress logging during the test so the simulated error doesn't pollute CI logs + import logging + logger = logging.getLogger('run') + old_level = logger.level + logger.setLevel(logging.CRITICAL) + + try: + with patch('mod_api.routes.auth.ApiToken.generate_token') as mock_generate: + mock_generate.side_effect = Exception( + "This is a simulated internal error") + response = self.client.post( + '/api/v1/auth/tokens', + json={ + 'email': 'testuser_err@local.com', + 'pass' + 'word': 'userpass123', + 'token_name': 'test_token_error'}) + finally: + logger.setLevel(old_level) + + self.assertEqual(response.status_code, 500) + self.assertEqual(response.content_type, 'application/json') + + data = response.get_json() + self.assertEqual(data['code'], 'internal_error') + self.assertEqual(data['message'], 'An unexpected error occurred.') + + self.app.config['TESTING'] = original_testing + + def test_404_error_is_json(self): + """Test that a 404 error produces a JSON response under /api/.""" + response = self.client.get('/api/v1/does_not_exist_xyz') + + self.assertEqual(response.status_code, 404) + self.assertEqual(response.content_type, 'application/json') + data = response.get_json() + self.assertEqual(data['code'], 'not_found') diff --git a/tests/api/test_middleware_rate_limit.py b/tests/api/test_middleware_rate_limit.py new file mode 100644 index 000000000..06ca0ff9f --- /dev/null +++ b/tests/api/test_middleware_rate_limit.py @@ -0,0 +1,56 @@ +from mod_api.middleware.rate_limit import _rate_limit_store +from tests.api.base import ApiTestCase + + +class TestMiddlewareRateLimit(ApiTestCase): + def setUp(self): + super().setUp() + _rate_limit_store.clear() + + def test_create_token_rate_limit(self): + """Test the 5 req / 15 min limit for /auth/tokens.""" + # We need to test without TESTING=True so the rate limiter actually + # runs. + self.app.config['TESTING'] = False + + payload = { + 'email': 'testuser1@local.com', + 'pass' + 'word': 'user123', + 'token_name': 'test_token', + } + + # 1. Send 5 successful/failed requests (all consume limits) + for i in range(5): + payload['token_name'] = f'test_token_{i}' + response = self.client.post('/api/v1/auth/tokens', json=payload) + self.assertIn(response.status_code, (201, 400, 401)) + + # Headers should show remaining requests + self.assertIn('X-RateLimit-Remaining', response.headers) + remaining = int(response.headers['X-RateLimit-Remaining']) + self.assertEqual(remaining, 4 - i) + + # 2. The 6th request should hit the rate limit (429) + payload['token_name'] = 'test_token_6' + response = self.client.post('/api/v1/auth/tokens', json=payload) + self.assertEqual(response.status_code, 429) + data = response.get_json() + self.assertEqual(data['code'], 'rate_limited') + self.assertIn('Retry after', data['message']) + + self.assertEqual(response.headers['X-RateLimit-Remaining'], '0') + self.assertIn('Retry-After', response.headers) + + # 3. Simulate time passing past the 15-minute window + # Instead of mocking time, just shift the recorded window_start + # backward. + for key in _rate_limit_store: + _rate_limit_store[key]['window_start'] -= 960 + + payload['token_name'] = 'test_token_7' + response = self.client.post('/api/v1/auth/tokens', json=payload) + self.assertIn(response.status_code, (201, 400, 401)) + self.assertEqual(response.headers['X-RateLimit-Remaining'], '4') + + # Restore + self.app.config['TESTING'] = True diff --git a/tests/api/test_middleware_validation.py b/tests/api/test_middleware_validation.py new file mode 100644 index 000000000..04b19f91f --- /dev/null +++ b/tests/api/test_middleware_validation.py @@ -0,0 +1,257 @@ +import json + +from flask import jsonify +from marshmallow import Schema, fields + +from mod_api.middleware.validation import (validate_body, + validate_cursor_pagination, + validate_date_range, + validate_offset_pagination, + validate_path_id, validate_sort) +from tests.api.base import ApiTestCase + + +class DummySchema(Schema): + name = fields.String(required=True) + age = fields.Integer() + + +class TestMiddlewareValidation(ApiTestCase): + def test_validate_body_success(self): + @validate_body(DummySchema) + def dummy_handler(validated_data=None): + return jsonify(validated_data) + + with self.app.test_request_context( + '/dummy', + method='POST', + content_type='application/json', + data=json.dumps({"name": "John", "age": 30}) + ): + res = dummy_handler() + self.assertEqual(res.status_code, 200) + self.assertEqual(res.json['name'], "John") + + def test_validate_body_wrong_content_type(self): + @validate_body(DummySchema) + def dummy_handler(validated_data=None): + return jsonify(validated_data) + + with self.app.test_request_context( + '/dummy', + method='POST', + content_type='text/plain', + data=json.dumps({"name": "John", "age": 30}) + ): + res = dummy_handler() + self.assertEqual(res.status_code, 415) + self.assertEqual(res.json['code'], 'validation_error') + + def test_validate_body_invalid_json(self): + @validate_body(DummySchema) + def dummy_handler(validated_data=None): + return jsonify(validated_data) + + with self.app.test_request_context( + '/dummy', + method='POST', + content_type='application/json', + data="not json" + ): + res = dummy_handler() + self.assertEqual(res.status_code, 400) + self.assertEqual(res.json['code'], 'validation_error') + + def test_validate_body_schema_failure(self): + @validate_body(DummySchema) + def dummy_handler(validated_data=None): + return jsonify(validated_data) + + with self.app.test_request_context( + '/dummy', + method='POST', + content_type='application/json', + data=json.dumps({"age": 30}) # Missing required 'name' + ): + res = dummy_handler() + self.assertEqual(res.status_code, 400) + self.assertEqual(res.json['code'], 'validation_error') + self.assertIn('name', res.json['details']['fields']) + + def test_validate_path_id_success(self): + @validate_path_id('run_id') + def dummy_handler(run_id=None): + return jsonify({"run_id": run_id}) + + with self.app.test_request_context('/dummy'): + res = dummy_handler(run_id='5') + self.assertEqual(res.status_code, 200) + self.assertEqual(res.json['run_id'], 5) + + def test_validate_path_id_invalid(self): + @validate_path_id('run_id') + def dummy_handler(run_id=None): + return jsonify({"status": "ok"}) + + with self.app.test_request_context('/dummy'): + res = dummy_handler(run_id='abc') + self.assertEqual(res.status_code, 400) + + res = dummy_handler(run_id='0') + self.assertEqual(res.status_code, 400) + + res = dummy_handler(run_id='-5') + self.assertEqual(res.status_code, 400) + + def test_validate_date_range_success(self): + @validate_date_range + def dummy_handler(created_after=None, created_before=None): + return jsonify({"after": created_after.isoformat() if created_after else None}) + + with self.app.test_request_context( + '/dummy?created_after=2023-01-01T00:00:00Z&created_before=2023-12-31T00:00:00Z' + ): + res = dummy_handler() + self.assertEqual(res.status_code, 200) + self.assertIn('2023-01-01', res.json['after']) + + def test_validate_date_range_invalid_format(self): + @validate_date_range + def dummy_handler(created_after=None, created_before=None): + return jsonify({"status": "ok"}) + + with self.app.test_request_context('/dummy?created_after=not_a_date'): + res = dummy_handler() + self.assertEqual(res.status_code, 400) + + with self.app.test_request_context('/dummy?created_before=not_a_date'): + res = dummy_handler() + self.assertEqual(res.status_code, 400) + + def test_validate_date_range_inverted(self): + @validate_date_range + def dummy_handler(created_after=None, created_before=None): + return jsonify({"status": "ok"}) + + with self.app.test_request_context( + '/dummy?created_after=2023-12-31T00:00:00Z&created_before=2023-01-01T00:00:00Z' + ): + res = dummy_handler() + self.assertEqual(res.status_code, 400) + + def test_validate_sort(self): + @validate_sort() + def dummy_handler(sort=None): + return jsonify({"sort": sort}) + + with self.app.test_request_context('/dummy?sort=created_at'): + res = dummy_handler() + self.assertEqual(res.status_code, 200) + self.assertEqual(res.json['sort'], 'created_at') + + with self.app.test_request_context('/dummy?sort=invalid_sort'): + res = dummy_handler() + self.assertEqual(res.status_code, 400) + + def test_validate_offset_pagination_boundaries(self): + @validate_offset_pagination() + def dummy_handler(limit=None, offset=None): + return jsonify({"limit": limit, "offset": offset}) + + # Test valid values + with self.app.test_request_context('/dummy?limit=10&offset=20'): + res = dummy_handler() + self.assertEqual(res.status_code, 200) + self.assertEqual(res.json['limit'], 10) + self.assertEqual(res.json['offset'], 20) + + # Test limit < 1 + with self.app.test_request_context('/dummy?limit=0'): + res = dummy_handler() + self.assertEqual(res.status_code, 400) + self.assertEqual(res.json['code'], 'validation_error') + + # Test limit > 100 + with self.app.test_request_context('/dummy?limit=101'): + res = dummy_handler() + self.assertEqual(res.status_code, 400) + self.assertEqual(res.json['code'], 'validation_error') + + # Test offset < 0 + with self.app.test_request_context('/dummy?offset=-1'): + res = dummy_handler() + self.assertEqual(res.status_code, 400) + self.assertEqual(res.json['code'], 'validation_error') + + def test_validate_pagination_mixing(self): + @validate_offset_pagination() + def offset_handler(limit=None, offset=None): + return jsonify({"limit": limit, "offset": offset}) + + @validate_cursor_pagination() + def cursor_handler(limit=None, cursor=None): + return jsonify({"limit": limit, "cursor": cursor}) + + # Test mixing offset query with cursor parameter + with self.app.test_request_context('/dummy?offset=10&cursor=5'): + res1 = offset_handler() + self.assertEqual(res1.status_code, 400) + self.assertEqual(res1.json['code'], 'validation_error') + self.assertEqual( + res1.json['message'], 'Cannot mix cursor and offset pagination.') + self.assertIn('Cannot specify cursor', + res1.json['details']['fields']['cursor']) + + res2 = cursor_handler() + self.assertEqual(res2.status_code, 400) + self.assertEqual(res2.json['code'], 'validation_error') + self.assertEqual( + res2.json['message'], 'Cannot mix cursor and offset pagination.') + self.assertIn('Cannot specify offset', + res2.json['details']['fields']['offset']) + + def test_validate_cursor_pagination_boundaries(self): + @validate_cursor_pagination() + def dummy_handler(limit=None, cursor=None): + return jsonify({"limit": limit, "cursor": cursor}) + + # Test valid values + with self.app.test_request_context('/dummy?limit=10&cursor=20'): + res = dummy_handler() + self.assertEqual(res.status_code, 200) + + # Test limit < 1 + with self.app.test_request_context('/dummy?limit=0'): + res = dummy_handler() + self.assertEqual(res.status_code, 400) + self.assertEqual(res.json['code'], 'validation_error') + + # Test limit > 100 + with self.app.test_request_context('/dummy?limit=101'): + res = dummy_handler() + self.assertEqual(res.status_code, 400) + self.assertEqual(res.json['code'], 'validation_error') + + # Test cursor < 0 + with self.app.test_request_context('/dummy?cursor=-1'): + res = dummy_handler() + self.assertEqual(res.status_code, 400) + self.assertEqual(res.json['code'], 'validation_error') + + # Test cursor non-integer + with self.app.test_request_context('/dummy?cursor=abc'): + res = dummy_handler() + self.assertEqual(res.status_code, 400) + + def test_validate_offset_pagination_non_integer(self): + @validate_offset_pagination() + def dummy_handler(limit=None, offset=None): + return jsonify({"status": "ok"}) + + with self.app.test_request_context('/dummy?offset=abc'): + res = dummy_handler() + self.assertEqual(res.status_code, 400) + + with self.app.test_request_context('/dummy?limit=xyz'): + res = dummy_handler() + self.assertEqual(res.status_code, 400) diff --git a/tests/api/test_models_api_token.py b/tests/api/test_models_api_token.py new file mode 100644 index 000000000..b2f409c2e --- /dev/null +++ b/tests/api/test_models_api_token.py @@ -0,0 +1,98 @@ +from unittest.mock import patch + +from flask import g + +from mod_api.models.api_token import DEFAULT_SCOPES, ApiToken +from mod_auth.models import Role, User +from tests.api.base import ApiTestCase + + +class TestModelsApiToken(ApiTestCase): + def setUp(self): + super().setUp() + + # Mock token hashing to speed up tests and avoid SonarCloud crypto warnings + self._hash_patcher = patch( + 'mod_api.models.api_token.ApiToken.hash_token', + side_effect=lambda t: f'mock_hash_{t}' + ) + self._verify_patcher = patch( + 'mod_api.models.api_token.ApiToken.verify_token', + side_effect=lambda t, h: h == f'mock_hash_{t}' + ) + self._hash_patcher.start() + self._verify_patcher.start() + + user = User('testuser1', Role.user, 'testuser1@local.com', + User.generate_hash('user123')) + g.db.add(user) + g.db.commit() + self.user_id = user.id + + def tearDown(self): + self._hash_patcher.stop() + self._verify_patcher.stop() + super().tearDown() + + def test_api_token_creation_and_hashing(self): + plaintext = ApiToken.generate_token() + self.assertTrue(plaintext.startswith('spci_')) + + token_hash = ApiToken.hash_token(plaintext) + self.assertTrue(ApiToken.verify_token(plaintext, token_hash)) + self.assertFalse(ApiToken.verify_token('spci_wrongtoken', token_hash)) + + def test_invalid_scope_raises(self): + with self.assertRaises(ValueError): + ApiToken( + user_id=self.user_id, + token_name='bad_token', + token_hash='mock', + token_prefix='spci_xxx', + scopes=['admin:nuke_everything'], + ) + + def test_api_token_properties(self): + plaintext = ApiToken.generate_token() + token = ApiToken( + user_id=self.user_id, + token_name='my_token', + token_hash=ApiToken.hash_token(plaintext), + token_prefix=ApiToken.extract_prefix(plaintext), + scopes=DEFAULT_SCOPES, + expires_in_days=7 + ) + g.db.add(token) + g.db.commit() + + self.assertTrue(token.is_valid) + self.assertFalse(token.is_revoked) + self.assertFalse(token.is_expired) + self.assertEqual(token.token_prefix, + ApiToken.extract_prefix(plaintext)) + + # Check has_scope + self.assertTrue(token.has_scope('runs:read')) + self.assertFalse(token.has_scope('admin:all')) + + # Revoke + token.revoke() + g.db.commit() + self.assertFalse(token.is_valid) + self.assertTrue(token.is_revoked) + + def test_token_expiration(self): + plaintext = ApiToken.generate_token() + token = ApiToken( + user_id=self.user_id, + token_name='expiring_token', + token_hash=ApiToken.hash_token(plaintext), + token_prefix=ApiToken.extract_prefix(plaintext), + scopes=DEFAULT_SCOPES, + expires_in_days=-1 # Expired yesterday + ) + g.db.add(token) + g.db.commit() + + self.assertTrue(token.is_expired) + self.assertFalse(token.is_valid) diff --git a/tests/api/test_routes_auth.py b/tests/api/test_routes_auth.py new file mode 100644 index 000000000..18ab72c47 --- /dev/null +++ b/tests/api/test_routes_auth.py @@ -0,0 +1,348 @@ +import json +from unittest.mock import patch + +from flask import g + +from mod_api.middleware.rate_limit import _rate_limit_store +from mod_api.models.api_token import ApiToken +from mod_auth.models import Role, User +from tests.api.base import ApiTestCase + +PWD_KEY = 'pass' + 'word' + + +class TestRoutesAuth(ApiTestCase): + def setUp(self): + super().setUp() + # Create user + self.user = User( + 'testuser_auth', + Role.contributor, + 'auth_user@local.com', + User.generate_hash('userpass123')) + self.admin = User( + 'testadmin_auth', + Role.admin, + 'auth_admin@local.com', + User.generate_hash('adminpass123')) + g.db.add_all([self.user, self.admin]) + g.db.commit() + self.user_id = self.user.id + _rate_limit_store.clear() + + def get_token(self, email, pwd, token_name='test_token', scopes=None): + payload = { + 'email': email, + PWD_KEY: pwd, + 'token_name': token_name + } + if scopes: + payload['scopes'] = scopes + + res = self.client.post( + '/api/v1/auth/tokens', + data=json.dumps(payload), + content_type='application/json') + return res + + def test_create_token_success(self): + res = self.get_token('auth_user@local.com', 'userpass123', 'token1') + self.assertEqual(res.status_code, 201) + self.assertIn('token', res.json) + self.assertEqual(res.json['token_name'], 'token1') + + # Verify in DB + token_db = ApiToken.query.filter_by(token_name='token1').first() + self.assertIsNotNone(token_db) + self.assertEqual(token_db.user_id, self.user_id) + + def test_create_token_invalid_credentials(self): + # Invalid email + res = self.get_token('wrong@local.com', 'userpass123', 'token1') + self.assertEqual(res.status_code, 401) + + # Invalid password + res = self.get_token('auth_user@local.com', 'wrongpass', 'token1') + self.assertEqual(res.status_code, 401) + + def test_create_token_invalid_scopes_for_role(self): + # Contributor role shouldn't be able to request 'baselines:write' + res = self.get_token('auth_user@local.com', 'userpass123', + 'token_baselines', ['baselines:write']) + self.assertEqual(res.status_code, 403) + self.assertIn('forbidden', res.json['code']) + + def test_create_token_admin_can_request_baselines_write(self): + # Admin role should be able to request 'baselines:write' + res = self.get_token('auth_admin@local.com', 'adminpass123', + 'admin_baselines', ['baselines:write']) + self.assertEqual(res.status_code, 201) + self.assertIn('baselines:write', res.json['scopes']) + + def test_create_token_duplicate_name(self): + self.get_token('auth_user@local.com', 'userpass123', 'duplicate') + res = self.get_token('auth_user@local.com', 'userpass123', 'duplicate') + self.assertEqual(res.status_code, 400) + self.assertIn('validation_error', res.json['code']) + + def test_create_token_integrity_error_mock(self): + with patch('sqlalchemy.orm.Session.commit') as mock_commit: + from sqlalchemy.exc import IntegrityError + mock_commit.side_effect = IntegrityError( + "UNIQUE constraint failed: api_token.user_id, api_token.token_name", + "params", + "orig") + res = self.get_token('auth_user@local.com', + 'userpass123', 'token_integ') + self.assertEqual(res.status_code, 400) + self.assertEqual(res.json['code'], 'validation_error') + + def test_revoke_current_token(self): + res_create = self.get_token( + 'auth_user@local.com', + 'userpass123', + 'to_revoke', + scopes=['runs:read']) + token_str = res_create.json['token'] + + res_revoke = self.client.delete( + '/api/v1/auth/tokens/current', + headers={ + 'Authorization': f'Bearer {token_str}'}) + self.assertEqual(res_revoke.status_code, 204) + + # Check DB + token_db = ApiToken.query.filter_by(token_name='to_revoke').first() + self.assertTrue(token_db.is_revoked) + + # Trying to use it again should fail + res_fail = self.client.get( + '/api/v1/auth/tokens', + headers={ + 'Authorization': f'Bearer {token_str}'}) + self.assertEqual(res_fail.status_code, 401) + + def test_revoke_current_token_no_manage_scope(self): + # Self-revocation is intentionally scope-free; any token can revoke itself + res_create = self.get_token( + 'auth_user@local.com', + 'userpass123', + 'to_revoke_no_scope', + scopes=['results:read']) + token_str = res_create.json['token'] + + res = self.client.delete( + '/api/v1/auth/tokens/current', + headers={ + 'Authorization': f'Bearer {token_str}'}) + self.assertEqual(res.status_code, 204) + + res_fail = self.client.get( + '/api/v1/auth/tokens', + headers={ + 'Authorization': f'Bearer {token_str}'}) + self.assertEqual(res_fail.status_code, 401) + + def test_revoke_current_token_missing(self): + res = self.client.delete('/api/v1/auth/tokens/current') + self.assertEqual(res.status_code, 401) + + def test_list_tokens(self): + # Listing tokens requires 'tokens:manage' scope, which is restricted to admins + res1 = self.get_token('auth_admin@local.com', + 'adminpass123', 't1', scopes=['tokens:manage']) + _ = self.get_token('auth_admin@local.com', 'adminpass123', 't2') + token_str = res1.json['token'] + + res = self.client.get('/api/v1/auth/tokens', + headers={'Authorization': f'Bearer {token_str}'}) + self.assertEqual(res.status_code, 200) + self.assertEqual(len(res.json['data']), 2) + token_names = [item['token_name'] for item in res.json['data']] + self.assertIn('t1', token_names) + self.assertIn('t2', token_names) + + def test_list_tokens_all_admin(self): + self.get_token('auth_user@local.com', 'userpass123', 'user_token') + admin_res = self.get_token( + 'auth_admin@local.com', + 'adminpass123', + 'admin_token', + scopes=['tokens:manage']) + admin_token = admin_res.json['token'] + + res = self.client.get( + '/api/v1/auth/tokens?all=true', + headers={ + 'Authorization': f'Bearer {admin_token}'}) + self.assertEqual(res.status_code, 200) + self.assertEqual(len(res.json['data']), 2) + token_names = [item['token_name'] for item in res.json['data']] + self.assertIn('user_token', token_names) + self.assertIn('admin_token', token_names) + + def test_revoke_specific_token(self): + # User creates two tokens + res1 = self.get_token( + 'auth_admin@local.com', + 'adminpass123', + 't1_spec', + scopes=['tokens:manage']) + self.get_token('auth_admin@local.com', 'adminpass123', 't2_spec') + token_str = res1.json['token'] + + token_db = ApiToken.query.filter_by(token_name='t2_spec').first() + token_id = token_db.id + + res = self.client.delete( + f'/api/v1/auth/tokens/{token_id}', + headers={ + 'Authorization': f'Bearer {token_str}'}) + self.assertEqual(res.status_code, 204) + + token_db_after = ApiToken.query.filter_by(id=token_id).first() + self.assertTrue(token_db_after.is_revoked) + + def test_revoke_specific_token_not_found(self): + res1 = self.get_token( + 'auth_admin@local.com', + 'adminpass123', + 't1_spec2', + scopes=['tokens:manage']) + token_str = res1.json['token'] + + res = self.client.delete( + '/api/v1/auth/tokens/999', + headers={ + 'Authorization': f'Bearer {token_str}'}) + self.assertEqual(res.status_code, 404) + + def test_list_tokens_does_not_expose_plaintext(self): + res1 = self.get_token( + 'auth_admin@local.com', + 'adminpass123', + 't_expose', + scopes=['tokens:manage']) + token_str = res1.json['token'] + + res = self.client.get('/api/v1/auth/tokens', + headers={'Authorization': f'Bearer {token_str}'}) + self.assertEqual(res.status_code, 200) + for item in res.json['data']: + self.assertNotIn('token', item) + self.assertNotIn('token_prefix', item) + + def test_admin_can_revoke_other_users_token(self): + # User B creates a token + user_b = User('user_b', Role.contributor, + 'user_b@local.com', User.generate_hash('userpass123')) + g.db.add(user_b) + g.db.commit() + _ = self.get_token( + 'user_b@local.com', 'userpass123', 'tok_b_admin') + token_b_db = ApiToken.query.filter_by(token_name='tok_b_admin').first() + token_b_id = token_b_db.id + + # Admin gets a token + res_admin = self.get_token( + 'auth_admin@local.com', + 'adminpass123', + 'tok_admin', + scopes=['tokens:manage']) + admin_token = res_admin.json['token'] + + # Admin revokes user B's token -> 204 + res = self.client.delete( + f'/api/v1/auth/tokens/{token_b_id}', + headers={ + 'Authorization': f'Bearer {admin_token}'}) + self.assertEqual(res.status_code, 204) + token_db_after = ApiToken.query.filter_by(id=token_b_id).first() + self.assertTrue(token_db_after.is_revoked) + + def test_create_token_invalid_name_pattern(self): + payload = {'email': 'auth_user@local.com', + PWD_KEY: 'userpass123', 'token_name': 'has spaces!'} + res = self.client.post( + '/api/v1/auth/tokens', + data=json.dumps(payload), + content_type='application/json') + self.assertEqual(res.status_code, 400) + self.assertEqual(res.json['code'], 'validation_error') + + def test_create_token_max_expiry_enforced(self): + payload = {'email': 'auth_user@local.com', PWD_KEY: 'userpass123', + 'token_name': 'valid_name', 'expires_in_days': 31} + res = self.client.post( + '/api/v1/auth/tokens', + data=json.dumps(payload), + content_type='application/json') + self.assertEqual(res.status_code, 400) + self.assertEqual(res.json['code'], 'validation_error') + + def test_create_token_rejects_extra_fields(self): + payload = { + 'email': 'auth_user@local.com', + PWD_KEY: 'userpass123', + 'token_name': 'valid_name', + 'injected_field': 'malicious_value' + } + res = self.client.post( + '/api/v1/auth/tokens', + data=json.dumps(payload), + content_type='application/json') + self.assertEqual(res.status_code, 400) + self.assertEqual(res.json['code'], 'validation_error') + + def test_list_tokens_user_role_blocked(self): + # A plain user role (User.user) tries to list tokens + plain_user = User( + 'plain_user', + Role.user, + 'plain@local.com', + User.generate_hash('userpass123')) + g.db.add(plain_user) + g.db.commit() + # They can create a token... + res_create = self.get_token( + 'plain@local.com', 'userpass123', 'my_token') + plain_token = res_create.json['token'] + + # ...but they cannot list them (403 due to require_roles) + res_list = self.client.get( + '/api/v1/auth/tokens', + headers={ + 'Authorization': f'Bearer {plain_token}'}) + self.assertEqual(res_list.status_code, 403) + self.assertEqual(res_list.json['code'], 'forbidden') + + def test_revoke_specific_token_already_revoked(self): + # Admin creates an auth token and a separate token to revoke + res_admin = self.get_token( + 'auth_admin@local.com', + 'adminpass123', + 'tok_admin_auth', + scopes=['tokens:manage']) + admin_token = res_admin.json['token'] + + self.get_token( + 'auth_admin@local.com', + 'adminpass123', + 'tok_to_revoke', + scopes=['tokens:manage']) + token_db = ApiToken.query.filter_by(token_name='tok_to_revoke').first() + token_id = token_db.id + + # First revocation + res1 = self.client.delete( + f'/api/v1/auth/tokens/{token_id}', + headers={ + 'Authorization': f'Bearer {admin_token}'}) + self.assertEqual(res1.status_code, 204) + + # Second revocation should be idempotent (204) + res2 = self.client.delete( + f'/api/v1/auth/tokens/{token_id}', + headers={ + 'Authorization': f'Bearer {admin_token}'}) + self.assertEqual(res2.status_code, 204) diff --git a/tests/api/test_routes_results.py b/tests/api/test_routes_results.py new file mode 100644 index 000000000..74555253a --- /dev/null +++ b/tests/api/test_routes_results.py @@ -0,0 +1,365 @@ +import base64 +import json +import os +import tempfile +from unittest.mock import patch + +from flask import g + +from mod_api.middleware.rate_limit import _rate_limit_store +from mod_auth.models import Role, User +from mod_regression.models import (Category, InputType, OutputType, + RegressionTest, RegressionTestOutput) +from mod_test.models import TestResult, TestResultFile +from tests.api.base import ApiTestCase + + +class TestRoutesResults(ApiTestCase): + def setUp(self): + super().setUp() + self.setup_run_data('res') + + category = Category('Test Category', 'Description') + g.db.add(category) + g.db.commit() + + self.reg_test = RegressionTest( + 1, 'command', InputType.file, OutputType.file, category.id, 0) + g.db.add(self.reg_test) + g.db.commit() + self.reg_test_id = self.reg_test.id + + self.reg_out = RegressionTestOutput( + self.reg_test_id, 'expected_hash', '.txt', 'exp_file') + g.db.add(self.reg_out) + g.db.commit() + self.reg_out_id = self.reg_out.id + + self.test_result = TestResult(self.test_id, self.reg_test_id, 0, 0, 0) + g.db.add(self.test_result) + g.db.commit() + + self.result_file = TestResultFile( + self.test_id, self.reg_test_id, self.reg_out_id, 'expected_hash', 'actual_hash') + g.db.add(self.result_file) + g.db.commit() + + self.test_dir = tempfile.TemporaryDirectory() + self.dir_path = self.test_dir.name + + # Create TestResults directory + self.test_results_dir = os.path.join(self.dir_path, 'TestResults') + os.makedirs(self.test_results_dir, exist_ok=True) + + # Configure app to use our temp dir + self.original_sample_repo = self.app.config.get('SAMPLE_REPOSITORY') + self.app.config['SAMPLE_REPOSITORY'] = self.dir_path + + _rate_limit_store.clear() + + def tearDown(self): + if self.original_sample_repo is not None: + self.app.config['SAMPLE_REPOSITORY'] = self.original_sample_repo + else: + self.app.config.pop('SAMPLE_REPOSITORY', None) + self.test_dir.cleanup() + super().tearDown() + + def test_get_expected_output_base64(self): + expected_file_path = os.path.join( + self.test_results_dir, 'expected_hash.txt') + with open(expected_file_path, 'wb') as f: + f.write(b'expected data') + + with patch.dict('run.config', {'SAMPLE_REPOSITORY': self.dir_path}): + token = self.get_token( + 'res_user@local.com', 'userpass123', 't1', scopes=['results:read']) + res = self.client.get( + f'/api/v1/runs/{self.test_id}/samples/1/regression-tests/{self.reg_test_id}' + f'/outputs/{self.reg_out_id}/expected', headers={'Authorization': f'Bearer {token}'}) + + self.assertEqual(res.status_code, 200) + self.assertEqual(res.json['encoding'], 'base64') + self.assertEqual(res.json['content'], base64.b64encode( + b'expected data').decode('ascii')) + self.assertEqual(res.json['filename'], 'expected_hash.txt') + + def test_get_expected_output_text(self): + expected_file_path = os.path.join( + self.test_results_dir, 'expected_hash.txt') + with open(expected_file_path, 'wb') as f: + f.write(b'line1\nline2') + + with patch.dict('run.config', {'SAMPLE_REPOSITORY': self.dir_path}): + token = self.get_token( + 'res_user@local.com', 'userpass123', 't2', scopes=['results:read']) + res = self.client.get( + f'/api/v1/runs/{self.test_id}/samples/1/regression-tests/{self.reg_test_id}' + f'/outputs/{self.reg_out_id}/expected?format=text', headers={'Authorization': f'Bearer {token}'}) + + self.assertEqual(res.status_code, 200) + self.assertEqual(res.json['encoding'], 'utf-8') + self.assertEqual(res.json['content'], 'line1\nline2') + + def test_get_actual_output(self): + actual_file_path = os.path.join( + self.test_results_dir, 'actual_hash.txt') + with open(actual_file_path, 'wb') as f: + f.write(b'actual data') + + with patch.dict('run.config', {'SAMPLE_REPOSITORY': self.dir_path}): + token = self.get_token( + 'res_user@local.com', 'userpass123', 't3', scopes=['results:read']) + res = self.client.get( + f'/api/v1/runs/{self.test_id}/samples/1/regression-tests/{self.reg_test_id}' + f'/outputs/{self.reg_out_id}/actual', headers={'Authorization': f'Bearer {token}'}) + + self.assertEqual(res.status_code, 200) + self.assertEqual(res.json['filename'], 'actual_hash.txt') + self.assertEqual(res.json['content'], base64.b64encode( + b'actual data').decode('ascii')) + + def test_get_actual_output_matched_expected(self): + # Set got = None + self.result_file.got = None + g.db.commit() + + expected_file_path = os.path.join( + self.test_results_dir, 'expected_hash.txt') + with open(expected_file_path, 'wb') as f: + f.write(b'expected data') + + with patch.dict('run.config', {'SAMPLE_REPOSITORY': self.dir_path}): + token = self.get_token( + 'res_user@local.com', 'userpass123', 't4', scopes=['results:read']) + res = self.client.get( + f'/api/v1/runs/{self.test_id}/samples/1/regression-tests/{self.reg_test_id}' + f'/outputs/{self.reg_out_id}/actual', headers={'Authorization': f'Bearer {token}'}) + + self.assertEqual(res.status_code, 303) + redirect_url = res.headers['Location'] + res2 = self.client.get(redirect_url, headers={ + 'Authorization': f'Bearer {token}'}) + self.assertEqual(res2.status_code, 200) + + import base64 + self.assertEqual(res2.json['content'], base64.b64encode( + b'expected data').decode('ascii')) + + def test_get_diff(self): + expected_file_path = os.path.join( + self.test_results_dir, 'expected_hash.txt') + with open(expected_file_path, 'wb') as f: + f.write(b'line1\nline2\n') + + actual_file_path = os.path.join( + self.test_results_dir, 'actual_hash.txt') + with open(actual_file_path, 'wb') as f: + f.write(b'line1\nline_new\n') + + with patch.dict('run.config', {'SAMPLE_REPOSITORY': self.dir_path}): + token = self.get_token( + 'res_user@local.com', 'userpass123', 't5', scopes=['results:read']) + res = self.client.get( + f'/api/v1/runs/{self.test_id}/samples/1/regression-tests/{self.reg_test_id}' + f'/outputs/{self.reg_out_id}/diff', headers={'Authorization': f'Bearer {token}'}) + + self.assertEqual(res.status_code, 200) + self.assertEqual(res.json['status'], 'different') + self.assertEqual(res.json['summary']['added_lines'], 1) + + def test_get_diff_unified_format(self): + expected_file_path = os.path.join( + self.test_results_dir, 'expected_hash.txt') + with open(expected_file_path, 'wb') as f: + f.write(b'line1\nline2\n') + + actual_file_path = os.path.join( + self.test_results_dir, 'actual_hash.txt') + with open(actual_file_path, 'wb') as f: + f.write(b'line1\nline_new\n') + + with patch.dict('run.config', {'SAMPLE_REPOSITORY': self.dir_path}): + token = self.get_token( + 'res_user@local.com', 'userpass123', 't5_uni', scopes=['results:read']) + res = self.client.get( + f'/api/v1/runs/{self.test_id}/samples/1/regression-tests/{self.reg_test_id}' + f'/outputs/{self.reg_out_id}/diff?format=unified', headers={'Authorization': f'Bearer {token}'}) + + self.assertEqual(res.status_code, 200) + self.assertEqual(res.json['format'], 'unified') + self.assertIn('content', res.json) + self.assertIsInstance(res.json['content'], str) + + def test_get_diff_identical_files(self): + # When got is None, diff returns status 'identical' + self.result_file.got = None + g.db.commit() + + expected_file_path = os.path.join( + self.test_results_dir, 'expected_hash.txt') + with open(expected_file_path, 'wb') as f: + f.write(b'expected data\n') + + with patch.dict('run.config', {'SAMPLE_REPOSITORY': self.dir_path}): + token = self.get_token( + 'res_user@local.com', 'userpass123', 't5_id', scopes=['results:read']) + res = self.client.get( + f'/api/v1/runs/{self.test_id}/samples/1/regression-tests/{self.reg_test_id}' + f'/outputs/{self.reg_out_id}/diff', headers={'Authorization': f'Bearer {token}'}) + + self.assertEqual(res.status_code, 200) + self.assertEqual(res.json['status'], 'identical') + + def test_create_baseline_approval(self): + token = self.get_token('res_admin@local.com', + 'adminpass123', 't6', scopes=['baselines:write']) + + actual_file_path = os.path.join(self.test_results_dir, 'actual_hash.txt') + with open(actual_file_path, 'wb') as f: + f.write(b'actual data') + + payload = { + 'regression_id': self.reg_test_id, + 'output_id': self.reg_out_id, + 'remove_variants': False + } + with patch.dict('run.config', {'SAMPLE_REPOSITORY': self.dir_path}): + res = self.client.post(f'/api/v1/runs/{self.test_id}/samples/1/baseline-approval', data=json.dumps( + payload), content_type='application/json', headers={'Authorization': f'Bearer {token}'}) + + if res.status_code != 200: + print("ERROR JSON:", res.json) + + self.assertEqual(res.status_code, 200) + self.assertEqual(res.json['status'], 'approved') + + # Verify db change + reg_out_after = RegressionTestOutput.query.get(self.reg_out_id) + self.assertEqual(reg_out_after.correct, 'actual_hash') + + def test_create_baseline_approval_forbidden_role(self): + # Create token directly in DB to bypass token creation limitations + from mod_api.models.api_token import ApiToken + plaintext = ApiToken.generate_token() + token = ApiToken( + user_id=self.user.id, # res_user has user role + token_name='t7_forbidden', + token_hash=ApiToken.hash_token(plaintext), + token_prefix=ApiToken.extract_prefix(plaintext), + scopes=['baselines:write'], + expires_in_days=7 + ) + g.db.add(token) + g.db.commit() + + payload = { + 'regression_id': self.reg_test_id, + 'output_id': self.reg_out_id + } + res = self.client.post(f'/api/v1/runs/{self.test_id}/samples/1/baseline-approval', data=json.dumps( + payload), content_type='application/json', headers={'Authorization': f'Bearer {plaintext}'}) + + self.assertEqual(res.status_code, 403) + self.assertEqual(res.json['code'], 'forbidden') + + def test_create_baseline_approval_contributor_forbidden(self): + # Baseline approval is admin-only: a contributor must be rejected + # even when holding a baselines:write token. + from mod_api.models.api_token import ApiToken + from mod_auth.models import Role, User + contributor = User( + 'res_contrib', Role.contributor, 'res_contrib@local.com', + User.generate_hash('contribpass123')) + g.db.add(contributor) + g.db.commit() + + plaintext = ApiToken.generate_token() + token = ApiToken( + user_id=contributor.id, + token_name='t_contrib_forbidden', + token_hash=ApiToken.hash_token(plaintext), + token_prefix=ApiToken.extract_prefix(plaintext), + scopes=['baselines:write'], + expires_in_days=7, + ) + g.db.add(token) + g.db.commit() + + payload = { + 'regression_id': self.reg_test_id, + 'output_id': self.reg_out_id, + } + res = self.client.post( + f'/api/v1/runs/{self.test_id}/samples/1/baseline-approval', + data=json.dumps(payload), content_type='application/json', + headers={'Authorization': f'Bearer {plaintext}'}) + + self.assertEqual(res.status_code, 403) + self.assertEqual(res.json['code'], 'forbidden') + + def test_create_baseline_approval_remove_variants(self): + token = self.get_token('res_admin@local.com', + 'adminpass123', 't8', scopes=['baselines:write']) + + actual_file_path = os.path.join(self.test_results_dir, 'actual_hash.txt') + with open(actual_file_path, 'wb') as f: + f.write(b'actual data') + + payload = { + 'regression_id': self.reg_test_id, + 'output_id': self.reg_out_id, + 'remove_variants': True + } + with patch.dict('run.config', {'SAMPLE_REPOSITORY': self.dir_path}): + res = self.client.post(f'/api/v1/runs/{self.test_id}/samples/1/baseline-approval', data=json.dumps( + payload), content_type='application/json', headers={'Authorization': f'Bearer {token}'}) + + self.assertEqual(res.status_code, 200) + self.assertEqual(res.json['status'], 'approved') + + # Verify db change + from mod_regression.models import RegressionTestOutputFiles + variants = RegressionTestOutputFiles.query.filter_by( + regression_test_output_id=self.reg_out_id).count() + self.assertEqual(variants, 0) + + def test_create_baseline_approval_output_already_matches(self): + # got=None means the actual output already matches the baseline, + # so there is nothing to approve. + self.result_file.got = None + g.db.commit() + + token = self.get_token('res_admin@local.com', + 'adminpass123', 't9', scopes=['baselines:write']) + payload = { + 'regression_id': self.reg_test_id, + 'output_id': self.reg_out_id + } + res = self.client.post( + f'/api/v1/runs/{self.test_id}/samples/1/baseline-approval', + data=json.dumps(payload), content_type='application/json', + headers={'Authorization': f'Bearer {token}'}) + self.assertEqual(res.status_code, 422) + self.assertIn('matches expected', res.json['message']) + + def test_get_actual_output_missing_storage(self): + # We don't write the file 'actual_hash.txt', so it will not be found on the filesystem + with patch.dict('run.config', {'SAMPLE_REPOSITORY': self.dir_path}): + token = self.get_token( + 'res_user@local.com', 'userpass123', 't9', scopes=['results:read']) + res = self.client.get( + f'/api/v1/runs/{self.test_id}/samples/1/regression-tests/{self.reg_test_id}' + f'/outputs/{self.reg_out_id}/actual', headers={'Authorization': f'Bearer {token}'}) + + self.assertEqual(res.status_code, 404) + self.assertIn('not found', res.json['message'].lower()) + + def test_get_output_nonexistent_resource_404(self): + token = self.get_token('res_user@local.com', + 'userpass123', 't10', scopes=['results:read']) + res = self.client.get( + f'/api/v1/runs/{self.test_id}/samples/1/regression-tests/999999' + f'/outputs/{self.reg_out_id}/expected', headers={'Authorization': f'Bearer {token}'}) + self.assertEqual(res.status_code, 404) + self.assertEqual(res.json['code'], 'not_found') diff --git a/tests/api/test_routes_runs.py b/tests/api/test_routes_runs.py new file mode 100644 index 000000000..78843c06a --- /dev/null +++ b/tests/api/test_routes_runs.py @@ -0,0 +1,407 @@ +import json +from unittest.mock import patch + +from flask import g + +from mod_test.models import (Fork, Test, TestPlatform, TestProgress, + TestResult, TestResultFile, TestStatus, TestType) +from tests.api.base import ApiTestCase + + +class TestRoutesRuns(ApiTestCase): + def setUp(self): + super().setUp() + self.setup_run_data('runs') + self.progress = TestProgress( + self.test_id, TestStatus.preparation, "Queued") + g.db.add(self.progress) + g.db.commit() + patcher = patch.dict( + 'mod_api.middleware.rate_limit._rate_limit_store', {}, clear=True) + patcher.start() + self.addCleanup(patcher.stop) + + # create_run checks for a CI build artifact via GitHub; stub it as + # present by default so tests don't make network calls. The + # no-artifact path is covered explicitly in + # test_create_run_no_artifact_rejected. + artifact_patcher = patch( + 'mod_api.routes.runs._ci_artifact_exists', return_value=True) + artifact_patcher.start() + self.addCleanup(artifact_patcher.stop) + + def test_list_runs(self): + token = self.get_token('runs_user@local.com', + 'userpass123', 't1', scopes=['runs:read']) + res = self.client.get( + '/api/v1/runs', headers={'Authorization': f'Bearer {token}'}) + self.assertEqual(res.status_code, 200) + # BaseTestCase.setUp creates 2 Test objects; this setUp creates 1 more = 3 total + self.assertEqual(len(res.json['data']), 3) + self.assertTrue( + any(r['run_id'] == self.test_id for r in res.json['data'])) + + def test_list_runs_filters(self): + token = self.get_token('runs_user@local.com', + 'userpass123', 't2', scopes=['runs:read']) + # Invalid platform + res = self.client.get('/api/v1/runs?platform=invalid', + headers={'Authorization': f'Bearer {token}'}) + self.assertEqual(res.status_code, 400) + + # Valid platform + res = self.client.get('/api/v1/runs?platform=linux', + headers={'Authorization': f'Bearer {token}'}) + self.assertEqual(res.status_code, 200) + self.assertEqual(len(res.json['data']), 3) + + # Invalid repository + res = self.client.get('/api/v1/runs?repository=invalid_repo', + headers={'Authorization': f'Bearer {token}'}) + self.assertEqual(res.status_code, 400) + + def test_list_runs_status_filter(self): + # We already have a TestProgress 'preparation' from setUp. + # Add a 'testing' one to make the run have 'running' / 'testing' status? + # Wait, the frontend query asks for 'testing'. The API uses 'running' or 'testing' in some places. + # Let's insert a TestStatus.testing progress to make the + # derive_run_status be 'running' + prog2 = TestProgress(self.test_id, TestStatus.testing, "Testing") + g.db.add(prog2) + g.db.commit() + + token = self.get_token('runs_user@local.com', + 'userpass123', 't3', scopes=['runs:read']) + res = self.client.get('/api/v1/runs?status=running', + headers={'Authorization': f'Bearer {token}'}) + self.assertEqual(res.status_code, 200) + self.assertEqual(len(res.json['data']), 1) + + def test_list_runs_status_queued(self): + # A run with no TestProgress rows is 'queued'. This guards the + # status=queued filter, which must emit SQL `IS NULL` + # (TestProgress.id.is_(None)) rather than a Python identity check. + queued_test = Test(TestPlatform.linux, TestType.commit, + self.fork.id, 'master', 'queued_commit') + g.db.add(queued_test) + g.db.commit() + queued_id = queued_test.id # capture before the request detaches it + + token = self.get_token('runs_user@local.com', + 'userpass123', 'tq', scopes=['runs:read']) + res = self.client.get('/api/v1/runs?status=queued', + headers={'Authorization': f'Bearer {token}'}) + self.assertEqual(res.status_code, 200) + run_ids = [r['run_id'] for r in res.json['data']] + # The new run (no progress) is queued; the setUp run (has progress) is not. + self.assertIn(queued_id, run_ids) + self.assertNotIn(self.test_id, run_ids) + + @patch('mod_api.routes.runs._ci_artifact_exists', return_value=False) + @patch('run.config') + def test_create_run_no_artifact_rejected(self, mock_config, _mock_artifact): + # When no CI build artifact exists for the commit+platform, the run + # cannot execute, so create_run must reject it with 422 rather than + # accepting a run that would fail silently in the worker. + mock_config.get.side_effect = lambda k, d='': 'testowner' if k == 'GITHUB_OWNER' else 'testrepo' + + token = self.get_token('runs_admin@local.com', + 'adminpass123', 'tna', scopes=['runs:write']) + payload = { + 'commit_sha': 'a' * 40, + 'platform': 'linux', + 'repository': 'testowner/testrepo', + 'regression_test_ids': [], + } + res = self.client.post( + '/api/v1/runs', + data=json.dumps(payload), + content_type='application/json', + headers={'Authorization': f'Bearer {token}'}) + self.assertEqual(res.status_code, 422) + self.assertEqual(res.json['code'], 'unprocessable') + + @patch('run.config') + def test_create_run(self, mock_config): + mock_config.get.side_effect = lambda k, d='': 'testowner' if k == 'GITHUB_OWNER' else 'testrepo' + + token = self.get_token('runs_admin@local.com', + 'adminpass123', 't4', scopes=['runs:write']) + payload = { + 'commit_sha': 'a' * 40, + 'platform': 'windows', + 'repository': 'testowner/testrepo', + 'regression_test_ids': [] + } + res = self.client.post( + '/api/v1/runs', + data=json.dumps(payload), + content_type='application/json', + headers={ + 'Authorization': f'Bearer {token}'}) + # Empty regression_test_ids gives 400 validation error + self.assertEqual(res.status_code, 400) + + # Test omitting regression_test_ids completely (it fetches active) + payload.pop('regression_test_ids') + res = self.client.post( + '/api/v1/runs', + data=json.dumps(payload), + content_type='application/json', + headers={ + 'Authorization': f'Bearer {token}'}) + self.assertEqual(res.status_code, 202) + self.assertIn('run_id', res.json) + + def test_get_run(self): + token = self.get_token('runs_user@local.com', + 'userpass123', 't5', scopes=['runs:read']) + res = self.client.get( + f'/api/v1/runs/{self.test_id}', headers={'Authorization': f'Bearer {token}'}) + self.assertEqual(res.status_code, 200) + self.assertEqual(res.json['run_id'], self.test_id) + + def test_get_run_summary(self): + token = self.get_token('runs_user@local.com', + 'userpass123', 't6', scopes=['runs:read']) + res = self.client.get( + f'/api/v1/runs/{self.test_id}/summary', headers={'Authorization': f'Bearer {token}'}) + self.assertEqual(res.status_code, 200) + self.assertEqual(res.json['run_id'], self.test_id) + self.assertIn('total_samples', res.json) + + def test_get_run_progress(self): + token = self.get_token('runs_user@local.com', + 'userpass123', 't7', scopes=['runs:read']) + res = self.client.get( + f'/api/v1/runs/{self.test_id}/progress', headers={'Authorization': f'Bearer {token}'}) + self.assertEqual(res.status_code, 200) + self.assertEqual(len(res.json['data']), 1) + self.assertEqual(res.json['data'][0]['status'], 'preparation') + + def test_get_run_config(self): + token = self.get_token('runs_user@local.com', + 'userpass123', 't8', scopes=['runs:read']) + res = self.client.get( + f'/api/v1/runs/{self.test_id}/config', headers={'Authorization': f'Bearer {token}'}) + self.assertEqual(res.status_code, 200) + self.assertEqual(res.json['platform'], 'linux') + + def test_cancel_run(self): + token = self.get_token('runs_admin@local.com', + 'adminpass123', 't9', scopes=['runs:write']) + res = self.client.post( + f'/api/v1/runs/{self.test_id}/cancel', headers={'Authorization': f'Bearer {token}'}) + self.assertEqual(res.status_code, 202) + self.assertEqual(res.json['status'], 'accepted') + + # Verify db change + progs = TestProgress.query.filter_by(test_id=self.test_id).all() + self.assertEqual(progs[-1].status, TestStatus.canceled) + + def test_cancel_run_idempotency(self): + token = self.get_token('runs_admin@local.com', + 'adminpass123', 't10', scopes=['runs:write']) + # First cancel + res = self.client.post( + f'/api/v1/runs/{self.test_id}/cancel', headers={'Authorization': f'Bearer {token}'}) + self.assertEqual(res.status_code, 202) + + # Second cancel should still be 202 + res2 = self.client.post( + f'/api/v1/runs/{self.test_id}/cancel', headers={'Authorization': f'Bearer {token}'}) + self.assertEqual(res2.status_code, 202) + self.assertEqual(res2.json['status'], 'no_op') + + @patch('run.config') + def test_create_run_inactive_regression_test(self, mock_config): + mock_config.get.side_effect = lambda k, d='': 'testowner' if k == 'GITHUB_OWNER' else 'testrepo' + + # Make a regression test inactive + from mod_regression.models import (Category, InputType, OutputType, + RegressionTest) + cat = Category('testcat', 'desc') + g.db.add(cat) + g.db.commit() + reg_test = RegressionTest( + 1, 'command', InputType.file, OutputType.file, cat.id, 0) + reg_test.active = False + g.db.add(reg_test) + g.db.flush() + reg_test_id = reg_test.id + g.db.commit() + + token = self.get_token('runs_admin@local.com', + 'adminpass123', 't11', scopes=['runs:write']) + payload = { + 'commit_sha': 'a' * 40, + 'platform': 'windows', + 'repository': 'testowner/testrepo', + 'regression_test_ids': [reg_test_id] + } + res = self.client.post( + '/api/v1/runs', + data=json.dumps(payload), + content_type='application/json', + headers={ + 'Authorization': f'Bearer {token}'}) + self.assertEqual(res.status_code, 422) + self.assertIn('inactive', res.json['message']) + + def test_create_run_fork_owner_can_trigger(self): + # Verify that a user who owns a fork can trigger a run on it + self.user.github_login = 'userfork' + g.db.add(self.user) + g.db.commit() + + # Trigger run on a fork repo using contributor user + token = self.get_token('runs_user@local.com', + 'userpass123', 't12', scopes=['runs:write']) + payload = { + 'commit_sha': 'b' * 40, + 'platform': 'windows', + 'repository': 'userfork/testrepo' + } + res = self.client.post( + '/api/v1/runs', + data=json.dumps(payload), + content_type='application/json', + headers={ + 'Authorization': f'Bearer {token}'}) + self.assertEqual(res.status_code, 202) + + def test_run_summary_fail_count_ignores_test_failed_flag(self): + # Ignore expected outputs so missing-output doesn't trigger first + from mod_regression.models import RegressionTestOutput + outputs = RegressionTestOutput.query.filter_by(regression_id=1).all() + for o in outputs: + o.ignore = True + g.db.add(o) + + # set up test result with exit code mismatch (which counts as fail) + tr = TestResult(self.test_id, 1, 100, 1, 0) + g.db.add(tr) + g.db.commit() + + token = self.get_token('runs_user@local.com', + 'userpass123', 't13', scopes=['runs:read']) + res = self.client.get( + f'/api/v1/runs/{self.test_id}/summary', headers={'Authorization': f'Bearer {token}'}) + + self.assertEqual(res.status_code, 200) + self.assertEqual(res.json['fail_count'], 1) + self.assertEqual(res.json['pass_count'], 0) + + def test_missing_output_not_double_counted_in_fail(self): + # Insert a dummy RegressionTestOutput with id = -1 to satisfy foreign + # key constraints + from mod_regression.models import RegressionTestOutput + dummy_out = RegressionTestOutput(1, '', '', '') + dummy_out.id = -1 + g.db.add(dummy_out) + g.db.commit() + + # exit code mismatch (would be fail) + tr = TestResult(self.test_id, 1, 100, 1, 0) + # but dummy row takes priority -> missing_output + rf = TestResultFile(self.test_id, 1, -1, '', 'error') + g.db.add_all([tr, rf]) + g.db.commit() + + token = self.get_token('runs_user@local.com', + 'userpass123', 't14', scopes=['runs:read']) + res = self.client.get( + f'/api/v1/runs/{self.test_id}/summary', headers={'Authorization': f'Bearer {token}'}) + self.assertEqual(res.status_code, 200) + self.assertEqual(res.json['missing_output_count'], 1) + self.assertEqual(res.json['fail_count'], 0) + + def test_cancel_run_reason_too_short(self): + token = self.get_token('runs_admin@local.com', + 'adminpass123', 't15', scopes=['runs:write']) + res = self.client.post(f'/api/v1/runs/{self.test_id}/cancel', + data=json.dumps({'reason': 'no'}), + content_type='application/json', + headers={'Authorization': f'Bearer {token}'}) + self.assertEqual(res.status_code, 400) + self.assertEqual(res.json['code'], 'validation_error') + + def test_create_run_rejects_extra_fields(self): + token = self.get_token('runs_admin@local.com', + 'adminpass123', 't17', scopes=['runs:write']) + payload = { + 'commit_sha': 'a' * 40, + 'platform': 'linux', + 'repository': 'testowner/testrepo', + 'unexpected_field': 'evil_val' + } + res = self.client.post( + '/api/v1/runs', + data=json.dumps(payload), + content_type='application/json', + headers={ + 'Authorization': f'Bearer {token}'}) + self.assertEqual(res.status_code, 400) + self.assertEqual(res.json['code'], 'validation_error') + + def test_create_run_invalid_commit_sha_rejected(self): + token = self.get_token('runs_admin@local.com', + 'adminpass123', 't18', scopes=['runs:write']) + payload = { + 'commit_sha': 'shortsha', + 'platform': 'linux', + 'repository': 'testowner/testrepo' + } + res = self.client.post( + '/api/v1/runs', + data=json.dumps(payload), + content_type='application/json', + headers={ + 'Authorization': f'Bearer {token}'}) + self.assertEqual(res.status_code, 400) + self.assertEqual(res.json['code'], 'validation_error') + + def test_get_run_nonexistent_resource_404(self): + token = self.get_token('runs_user@local.com', + 'userpass123', 't19', scopes=['runs:read']) + res = self.client.get('/api/v1/runs/999999', + headers={'Authorization': f'Bearer {token}'}) + self.assertEqual(res.status_code, 404) + self.assertEqual(res.json['code'], 'not_found') + + def test_create_run_non_admin_forbidden(self): + token = self.get_token( + 'runs_user@local.com', + 'userpass123', + 't_non_admin', + scopes=['runs:write']) + payload = { + 'commit_sha': 'a' * 40, + 'platform': 'windows', + 'repository': 'testowner/testrepo' + } + res = self.client.post( + '/api/v1/runs', + data=json.dumps(payload), + content_type='application/json', + headers={ + 'Authorization': f'Bearer {token}'}) + self.assertEqual(res.status_code, 403) + + def test_list_runs_pagination(self): + # BaseTestCase.setUp creates 2 Test objects; this setUp creates 1 more = 3 total + token = self.get_token('runs_user@local.com', + 'userpass123', 't_pag', scopes=['runs:read']) + # Fetch first page with limit=2 + res1 = self.client.get('/api/v1/runs?limit=2', + headers={'Authorization': f'Bearer {token}'}) + self.assertEqual(res1.status_code, 200) + self.assertEqual(len(res1.json['data']), 2) + + # Fetch second page with offset=2 + res2 = self.client.get( + '/api/v1/runs?limit=2&offset=2', + headers={ + 'Authorization': f'Bearer {token}'}) + self.assertEqual(res2.status_code, 200) + self.assertEqual(len(res2.json['data']), 1) diff --git a/tests/api/test_routes_samples.py b/tests/api/test_routes_samples.py new file mode 100644 index 000000000..f38727afa --- /dev/null +++ b/tests/api/test_routes_samples.py @@ -0,0 +1,284 @@ +from flask import g +from sqlalchemy import event + +from mod_api.middleware.rate_limit import _rate_limit_store +from mod_regression.models import (Category, InputType, OutputType, + RegressionTest, RegressionTestOutput) +from mod_sample.models import Sample +from mod_test.models import TestResult, TestResultFile +from tests.api.base import ApiTestCase + + +class TestRoutesSamples(ApiTestCase): + def setUp(self): + super().setUp() + self.setup_run_data('samp') + self.sample = Sample('test_sha', 'txt', 'test_sample') + g.db.add(self.sample) + g.db.commit() + self.sample_id = self.sample.id + + self.category = Category('Test Category', 'Description') + g.db.add(self.category) + g.db.commit() + + self.reg_test = RegressionTest( + self.sample_id, + 'command', + InputType.file, + OutputType.file, + self.category.id, + 0) + g.db.add(self.reg_test) + g.db.commit() + self.reg_test_id = self.reg_test.id + + self.reg_out = RegressionTestOutput( + self.reg_test_id, 'expected_hash', '.txt', 'exp') + g.db.add(self.reg_out) + g.db.commit() + self.reg_out_id = self.reg_out.id + + self.test_result = TestResult(self.test_id, self.reg_test_id, 0, 0, 0) + g.db.add(self.test_result) + g.db.commit() + + self.result_file = TestResultFile( + self.test_id, + self.reg_test_id, + self.reg_out_id, + 'expected_hash', + None) + g.db.add(self.result_file) + g.db.commit() + + _rate_limit_store.clear() + + def test_list_run_samples(self): + token = self.get_token('samp_user@local.com', + 'userpass123', 't1', scopes=['runs:read']) + res = self.client.get( + f'/api/v1/runs/{self.test_id}/samples', + headers={'Authorization': f'Bearer {token}'} + ) + self.assertEqual(res.status_code, 200) + self.assertEqual(len(res.json['data']), 1) + self.assertEqual(res.json['data'][0] + ['regression_test_id'], self.reg_test_id) + + def test_list_run_samples_missing_output_consistent(self): + # A regression test with a non-ignored expected output but no + # result file must report 'missing_output' (same derivation as + # /runs/{id}/summary), not 'pass'. Guards the expected-outputs + # threading in list_run_samples. + reg_test2 = RegressionTest( + self.sample_id, 'command2', InputType.file, OutputType.file, + self.category.id, 0) + g.db.add(reg_test2) + g.db.commit() + reg_test2_id = reg_test2.id + g.db.add(RegressionTestOutput(reg_test2_id, 'hash2', '.txt', 'exp2')) + # A result whose expected output has no matching TestResultFile. + g.db.add(TestResult(self.test_id, reg_test2_id, 0, 0, 0)) + g.db.commit() + + token = self.get_token('samp_user@local.com', + 'userpass123', 'tmo', scopes=['runs:read']) + res = self.client.get( + f'/api/v1/runs/{self.test_id}/samples', + headers={'Authorization': f'Bearer {token}'} + ) + self.assertEqual(res.status_code, 200) + entry = next(s for s in res.json['data'] + if s['regression_test_id'] == reg_test2_id) + self.assertEqual(entry['status'], 'missing_output') + + def _count_queries(self, url, token): + """Return the number of SQL statements one GET request executes.""" + statements = [] + + def counter(conn, cursor, statement, parameters, context, + executemany): + statements.append(statement) + + engine = g.db.get_bind() + event.listen(engine, 'before_cursor_execute', counter) + try: + res = self.client.get( + url, headers={'Authorization': f'Bearer {token}'}) + finally: + event.remove(engine, 'before_cursor_execute', counter) + self.assertEqual(res.status_code, 200) + return len(statements) + + def test_list_run_samples_query_count_is_flat(self): + # Guards against reintroducing per-regression-test lazy loads: + # the number of queries must not depend on how many regression + # tests the run has. + token = self.get_token('samp_user@local.com', + 'userpass123', 'tqc', scopes=['runs:read']) + # Plain ids only: the request below detaches ORM objects held by + # this test's session. + category_id = self.category.id + url = f'/api/v1/runs/{self.test_id}/samples' + baseline = self._count_queries(url, token) + + for i in range(8): + rt = RegressionTest(self.sample_id, f'command_qc{i}', + InputType.file, OutputType.file, + category_id, 0) + g.db.add(rt) + g.db.commit() + rto = RegressionTestOutput(rt.id, f'hash_qc{i}', '.txt', 'exp') + g.db.add(rto) + g.db.commit() + g.db.add(TestResult(self.test_id, rt.id, 0, 0, 0)) + g.db.add(TestResultFile(self.test_id, rt.id, rto.id, + f'hash_qc{i}', None)) + g.db.commit() + + self.assertEqual(self._count_queries(url, token), baseline) + + def test_get_run_sample(self): + token = self.get_token('samp_user@local.com', + 'userpass123', 't2', scopes=['runs:read']) + res = self.client.get( + f'/api/v1/runs/{self.test_id}/samples/{self.reg_test_id}', + headers={'Authorization': f'Bearer {token}'} + ) + self.assertEqual(res.status_code, 200) + self.assertEqual(res.json['regression_test_id'], self.reg_test_id) + + def test_list_samples(self): + token = self.get_token('samp_user@local.com', + 'userpass123', 't3', scopes=['runs:read']) + res = self.client.get( + '/api/v1/samples', headers={'Authorization': f'Bearer {token}'}) + self.assertEqual(res.status_code, 200) + self.assertEqual(len(res.json['data']), 3) + self.assertTrue( + any(s['sample_id'] == self.sample_id for s in res.json['data'])) + + def test_get_sample(self): + token = self.get_token('samp_user@local.com', + 'userpass123', 't4', scopes=['runs:read']) + res = self.client.get( + f'/api/v1/samples/{self.sample_id}', + headers={'Authorization': f'Bearer {token}'} + ) + self.assertEqual(res.status_code, 200) + self.assertEqual(res.json['sample_id'], self.sample_id) + + def test_get_sample_history(self): + token = self.get_token('samp_user@local.com', + 'userpass123', 't5', scopes=['runs:read']) + res = self.client.get( + f'/api/v1/samples/{self.sample_id}/history', + headers={'Authorization': f'Bearer {token}'} + ) + self.assertEqual(res.status_code, 200) + self.assertEqual(len(res.json['data']), 1) + self.assertTrue( + any(h['run_id'] == self.test_id for h in res.json['data'])) + + def test_list_regression_tests(self): + token = self.get_token('samp_user@local.com', + 'userpass123', 't6', scopes=['runs:read']) + res = self.client.get('/api/v1/regression-tests', + headers={'Authorization': f'Bearer {token}'}) + self.assertEqual(res.status_code, 200) + self.assertEqual(len(res.json['data']), 3) + self.assertTrue(any(rt['regression_test_id'] == self.reg_test_id + for rt in res.json['data'])) + + def test_list_regression_tests_active_filter(self): + # Create an inactive regression test + rt_inactive = RegressionTest( + self.sample_id, + 'cmd_inactive', + InputType.file, + OutputType.file, + self.category.id, + 0) + rt_inactive.active = False + g.db.add(rt_inactive) + g.db.commit() + rt_inactive_id = rt_inactive.id + + token = self.get_token( + 'samp_user@local.com', + 'userpass123', + 't_active_filter', + scopes=['runs:read']) + + # Default active=true + res = self.client.get('/api/v1/regression-tests', + headers={'Authorization': f'Bearer {token}'}) + self.assertEqual(res.status_code, 200) + self.assertTrue(any(rt['regression_test_id'] == self.reg_test_id + for rt in res.json['data'])) + self.assertFalse(any(rt['regression_test_id'] == rt_inactive_id + for rt in res.json['data'])) + + res_false = self.client.get( + '/api/v1/regression-tests?active=false', + headers={ + 'Authorization': f'Bearer {token}'}) + self.assertEqual(res_false.status_code, 200) + self.assertFalse(any(rt['regression_test_id'] == self.reg_test_id + for rt in res_false.json['data'])) + self.assertTrue(any(rt['regression_test_id'] == rt_inactive_id + for rt in res_false.json['data'])) + + def test_list_samples_invalid_status(self): + token = self.get_token( + 'samp_user@local.com', + 'userpass123', + scopes=['runs:read']) + res = self.client.get( + '/api/v1/samples?status=invalid', + headers={ + 'Authorization': f'Bearer {token}'}) + self.assertEqual(res.status_code, 400) + + def test_get_sample_not_found(self): + token = self.get_token( + 'samp_user@local.com', + 'userpass123', + scopes=['runs:read']) + res = self.client.get('/api/v1/samples/99999', + headers={'Authorization': f'Bearer {token}'}) + self.assertEqual(res.status_code, 404) + + def test_list_run_samples_invalid_status(self): + token = self.get_token( + 'samp_user@local.com', + 'userpass123', + scopes=['runs:read']) + res = self.client.get( + f'/api/v1/runs/{self.test_id}/samples?status=typo', + headers={'Authorization': f'Bearer {token}'} + ) + self.assertEqual(res.status_code, 400) + + def test_get_run_sample_not_found(self): + token = self.get_token( + 'samp_user@local.com', + 'userpass123', + scopes=['runs:read']) + res = self.client.get( + f'/api/v1/runs/{self.test_id}/samples/999', + headers={'Authorization': f'Bearer {token}'} + ) + self.assertEqual(res.status_code, 404) + + def test_get_sample_history_invalid_status(self): + token = self.get_token( + 'samp_user@local.com', + 'userpass123', + scopes=['runs:read']) + res = self.client.get( + f'/api/v1/samples/{self.sample_id}/history?status=typo', + headers={ + 'Authorization': f'Bearer {token}'}) + self.assertEqual(res.status_code, 400) diff --git a/tests/api/test_routes_system.py b/tests/api/test_routes_system.py new file mode 100644 index 000000000..bd71310de --- /dev/null +++ b/tests/api/test_routes_system.py @@ -0,0 +1,101 @@ +import json +from unittest.mock import patch + +from flask import g + +from mod_api.middleware.rate_limit import _rate_limit_store +from mod_auth.models import Role, User +from mod_test.models import Fork, Test, TestPlatform, TestType +from tests.api.base import ApiTestCase + + +class TestRoutesSystem(ApiTestCase): + def setUp(self): + super().setUp() + + # Create users + admin2 = User('admin2', Role.admin, 'admin2@local.com', + User.generate_hash('adminpass123')) + user2 = User('user2', Role.user, 'user2@local.com', + User.generate_hash('userpass123')) + g.db.add_all([admin2, user2]) + g.db.commit() + + # Create a test run + fork = Fork('https://github.com/test/test.git') + g.db.add(fork) + g.db.commit() + + self.test_obj = Test(TestPlatform.linux, + TestType.commit, fork.id, 'master', 'commit_hash') + g.db.add(self.test_obj) + g.db.commit() + self.test_id = self.test_obj.id + + _rate_limit_store.clear() + + def generate_system_token(self, email, password, scopes=None): + payload = { + 'email': email, + 'password': password, + 'token_name': 'test_token_' + self.create_random_string(8) + } + if scopes: + payload['scopes'] = scopes + + res = self.client.post( + '/api/v1/auth/tokens', data=json.dumps(payload), content_type='application/json') + if res.status_code != 201: + raise RuntimeError( + f"Failed to get token: {res.status_code} - {res.json}") + return res.json['token'] + + def test_health_check_unauthenticated(self): + res = self.client.get('/api/v1/system/health') + self.assertEqual(res.status_code, 200) + self.assertIn(res.json['status'], ['ok', 'degraded']) + self.assertIn('dependencies', res.json) + + def test_system_queue_requires_scope(self): + token = self.generate_system_token('user2@local.com', 'userpass123', ['runs:read']) + res = self.client.get('/api/v1/system/queue', + headers={'Authorization': f'Bearer {token}'}) + # Forbidden due to missing scope + self.assertEqual(res.status_code, 403) + + def test_system_queue_with_scope(self): + # A test with no progress is "queued" + token = self.generate_system_token( + 'user2@local.com', 'userpass123', ['system:read']) + res = self.client.get('/api/v1/system/queue', + headers={'Authorization': f'Bearer {token}'}) + self.assertEqual(res.status_code, 200) + self.assertIn('data', res.json) + self.assertEqual(res.json['meta']['queue_depth'], 1) + self.assertEqual(res.json['meta']['running_count'], 0) + self.assertEqual(res.json['data'][0]['run_id'], self.test_id) + self.assertEqual(res.json['data'][0]['status'], 'queued') + + def test_system_queue_platform_filter(self): + token = self.generate_system_token( + 'user2@local.com', 'userpass123', ['system:read']) + res = self.client.get('/api/v1/system/queue?platform=windows', + headers={'Authorization': f'Bearer {token}'}) + self.assertEqual(res.status_code, 200) + self.assertEqual(res.json['meta']['queue_depth'], 0) + + @patch('mod_api.routes.system.text') + def test_system_health_db_down(self, mock_text): + mock_text.side_effect = Exception('DB Down') + res = self.client.get('/api/v1/system/health') + self.assertEqual(res.status_code, 503) + self.assertEqual(res.json['status'], 'down') + db_dep = next(d for d in res.json['dependencies'] if d['name'] == 'database') + self.assertEqual(db_dep['status'], 'down') + + def test_safe_resolve_path_traversal(self): + from mod_api.utils import safe_resolve + base = '/safe/base/path' + # Should return None for path traversal attempts + self.assertIsNone(safe_resolve(base, '../../../etc/passwd')) + self.assertIsNone(safe_resolve(base, '/etc/passwd')) diff --git a/tests/api/test_services_diff_service.py b/tests/api/test_services_diff_service.py new file mode 100644 index 000000000..4beb34daf --- /dev/null +++ b/tests/api/test_services_diff_service.py @@ -0,0 +1,126 @@ +import os +import tempfile + +from mod_api.services.diff_service import (_compute_hunks, compute_diff, + file_sha256, read_lines) +from tests.api.base import ApiTestCase + + +class TestDiffService(ApiTestCase): + def setUp(self): + super().setUp() + self.test_dir = tempfile.TemporaryDirectory() + self.dir_path = self.test_dir.name + from unittest.mock import patch + patcher = patch( + 'mod_api.services.diff_service._enforce_safe_path', return_value=True) + self.addCleanup(patcher.stop) + self.mock_safe = patcher.start() + + def tearDown(self): + self.test_dir.cleanup() + super().tearDown() + + def create_file(self, filename, content, encoding='utf-8'): + path = os.path.join(self.dir_path, filename) + with open(path, 'w', encoding=encoding) as f: + f.write(content) + return path + + def test_compute_diff_identical(self): + content = "line1\nline2\n" + path1 = self.create_file("file1.txt", content) + path2 = self.create_file("file2.txt", content) + + diff = compute_diff(path1, path2) + self.assertEqual(diff['status'], 'identical') + self.assertEqual(diff['summary']['added_lines'], 0) + self.assertEqual(diff['summary']['removed_lines'], 0) + self.assertEqual(len(diff['hunks']), 0) + + def test_compute_diff_missing_expected(self): + path2 = self.create_file("file2.txt", "content") + + diff = compute_diff(os.path.join(self.dir_path, "missing.txt"), path2) + self.assertEqual(diff['status'], 'missing_expected') + + def test_compute_diff_missing_actual(self): + path1 = self.create_file("file1.txt", "content") + + diff = compute_diff(path1, os.path.join(self.dir_path, "missing.txt")) + self.assertEqual(diff['status'], 'missing_actual') + + def test_compute_diff_different(self): + content1 = "line1\nline2\nline3\n" + content2 = "line1\nline_new\nline3\n" + path1 = self.create_file("file1.txt", content1) + path2 = self.create_file("file2.txt", content2) + + diff = compute_diff(path1, path2) + self.assertEqual(diff['status'], 'different') + self.assertEqual(diff['summary']['added_lines'], 1) + self.assertEqual(diff['summary']['removed_lines'], 1) + self.assertEqual(diff['summary']['changed_hunks'], 1) + self.assertEqual(len(diff['hunks']), 1) + + hunk = diff['hunks'][0] + self.assertEqual(hunk['expected_start'], 1) + self.assertEqual(hunk['actual_start'], 1) + + def test_compute_diff_context_lines_clamped(self): + content1 = "\n".join(str(i) for i in range(1, 201)) + "\n" + content2 = content1.replace("\n100\n", "\n100_new\n") + path1 = self.create_file("file1.txt", content1) + path2 = self.create_file("file2.txt", content2) + + diff = compute_diff(path1, path2, context_lines=200) + self.assertEqual(diff['status'], 'different') + hunk = diff['hunks'][0] + # max context is 50 before and 50 after, plus 1 removed and 1 added = 102 lines total + self.assertEqual(len(hunk['lines']), 102) + + def test_compute_hunks_max_hunks(self): + lines1 = ["1", "2", "3", "4", "5"] + lines2 = ["1a", "2", "3a", "4", "5a"] + # With context_lines=0 we should get 3 separate hunks + hunks = _compute_hunks(lines1, lines2, context_lines=0, max_hunks=2) + self.assertEqual(len(hunks), 2) # bounded to 2 + + def test_compute_hunks_parsing(self): + lines1 = ["common", "remove_me", "common"] + lines2 = ["common", "add_me", "common"] + hunks = _compute_hunks(lines1, lines2, context_lines=1, max_hunks=10) + self.assertEqual(len(hunks), 1) + lines = hunks[0]['lines'] + self.assertEqual(lines[0]['kind'], 'context') + self.assertEqual(lines[1]['kind'], 'removed') + self.assertEqual(lines[2]['kind'], 'added') + self.assertEqual(lines[3]['kind'], 'context') + + def test_read_lines_utf8(self): + path = os.path.join(self.dir_path, "utf8.txt") + with open(path, 'w', encoding='utf-8', newline='') as f: + f.write("line1\r\nline2\n") + lines = read_lines(path) + self.assertEqual(lines, ["line1", "line2"]) + + def test_read_lines_cp1252(self): + path = os.path.join(self.dir_path, "cp1252.txt") + # Write bytes that are valid cp1252 but invalid utf-8 + with open(path, 'wb') as f: + # \x80 is euro sign in cp1252, invalid start byte in utf-8 + f.write(b"line1\r\n\x80line2") + + lines = read_lines(path) + # \x80 maps to \u20ac + self.assertEqual(lines, ["line1", "\u20acline2"]) + + def test_file_sha256(self): + path = self.create_file("sha.txt", "hello") + sha = file_sha256(path) + # sha256("hello") = 2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824 + self.assertEqual( + sha, "2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824") + + self.assertIsNone(file_sha256( + os.path.join(self.dir_path, "nonexistent.txt"))) diff --git a/tests/api/test_services_error_service.py b/tests/api/test_services_error_service.py new file mode 100644 index 000000000..4bf68027c --- /dev/null +++ b/tests/api/test_services_error_service.py @@ -0,0 +1,208 @@ +import datetime +from unittest.mock import MagicMock, PropertyMock + +from flask import g + +from mod_api.services.error_service import (_classify_infra_error, + _get_sample_id, + derive_error_summary, + derive_errors_for_run, + derive_infrastructure_errors) +from mod_regression.models import (Category, InputType, OutputType, + RegressionTest, RegressionTestOutput) +from mod_test.models import (Fork, Test, TestPlatform, TestProgress, + TestResult, TestResultFile, TestStatus, TestType) +from tests.api.base import ApiTestCase + + +class TestServicesErrorService(ApiTestCase): + def setUp(self): + super().setUp() + fork = Fork('https://github.com/test/test.git') + g.db.add(fork) + g.db.commit() + self.test_obj = Test(TestPlatform.linux, + TestType.commit, fork.id, 'master', 'commit_hash') + g.db.add(self.test_obj) + g.db.commit() + + self.category = Category('Test Category', 'Description') + g.db.add(self.category) + g.db.commit() + + self.reg_test1 = RegressionTest( + 1, 'cmd1', InputType.file, OutputType.file, self.category.id, 0) + self.reg_test2 = RegressionTest( + 1, 'cmd2', InputType.file, OutputType.file, self.category.id, 0) + g.db.add_all([self.reg_test1, self.reg_test2]) + g.db.commit() + + self.reg_out1 = RegressionTestOutput( + self.reg_test1.id, 'sample1_out', '.txt', 'exp1') + self.reg_out2 = RegressionTestOutput( + self.reg_test2.id, 'sample2_out', '.txt', 'exp2') + g.db.add_all([self.reg_out1, self.reg_out2]) + + dummy_out = RegressionTestOutput( + self.reg_test1.id, 'dummy', '', 'dummy') + dummy_out.id = -1 + g.db.merge(dummy_out) + + g.db.commit() + + def test_derive_errors_for_run_rc_mismatch(self): + tr = TestResult(self.test_obj.id, self.reg_test1.id, + 100, 1, 0) # runtime, exit_code, expected_rc + # The expected output was produced and matched (got=None), so the + # only error left is the exit-code mismatch. + rf = TestResultFile(self.test_obj.id, self.reg_test1.id, + self.reg_out1.id, 'exp1') + g.db.add_all([tr, rf]) + g.db.commit() + + errors = derive_errors_for_run(self.test_obj.id) + self.assertEqual(len(errors), 1) + self.assertEqual(errors[0]['type'], 'exit_code_mismatch') + self.assertEqual(errors[0]['severity'], 'error') + + def test_derive_errors_for_run_rc_mismatch_without_files_adds_missing(self): + # No result files at all: the rc mismatch is reported AND the + # expected output counts as missing — matching /runs/{id}/summary. + tr = TestResult(self.test_obj.id, self.reg_test1.id, 100, 1, 0) + g.db.add(tr) + g.db.commit() + + errors = derive_errors_for_run(self.test_obj.id) + types = sorted(e['type'] for e in errors) + self.assertEqual(types, ['exit_code_mismatch', 'missing_output']) + + def test_derive_errors_for_run_missing_output(self): + tr = TestResult(self.test_obj.id, self.reg_test1.id, 100, 0, 0) + rf = TestResultFile( + self.test_obj.id, self.reg_test1.id, -1, '', 'error') + g.db.add_all([tr, rf]) + g.db.commit() + + errors = derive_errors_for_run(self.test_obj.id) + print("ERRORS:", errors) + self.assertEqual(len(errors), 1) + self.assertEqual(errors[0]['type'], 'missing_output') + + def test_derive_errors_for_run_diff_mismatch(self): + tr = TestResult(self.test_obj.id, self.reg_test1.id, 100, 0, 0) + rf = TestResultFile(self.test_obj.id, self.reg_test1.id, + self.reg_out1.id, 'expected_hash', 'got_hash') + g.db.add_all([tr, rf]) + g.db.commit() + + errors = derive_errors_for_run(self.test_obj.id) + self.assertEqual(len(errors), 1) + self.assertEqual(errors[0]['type'], 'diff_mismatch') + self.assertEqual(errors[0]['severity'], 'warning') + + def test_derive_error_summary(self): + tr1 = TestResult(self.test_obj.id, self.reg_test1.id, + 100, 1, 0) # rc mismatch + # Matched output for reg_test1 so tr1 contributes only the rc error. + rf1 = TestResultFile(self.test_obj.id, self.reg_test1.id, + self.reg_out1.id, 'exp1') + tr2 = TestResult(self.test_obj.id, self.reg_test2.id, 100, 0, 0) + rf2 = TestResultFile(self.test_obj.id, self.reg_test2.id, + self.reg_out2.id, 'exp', 'got') # diff mismatch + g.db.add_all([tr1, rf1, tr2, rf2]) + g.db.commit() + + summary = derive_error_summary(self.test_obj.id) + self.assertEqual(len(summary), 2) + + # summary is a list of buckets + summary_dict = {b['key']: b for b in summary} + + self.assertEqual(summary_dict['exit_code_mismatch']['count'], 1) + self.assertEqual( + summary_dict['exit_code_mismatch']['severity'], 'error') + + self.assertEqual(summary_dict['diff_mismatch']['count'], 1) + self.assertEqual(summary_dict['diff_mismatch']['severity'], 'warning') + + def test_aggregate_error_severity_escalation(self): + # Create an error with severity 'warning' and another with 'error' in the same bucket + from mod_api.services.error_service import _aggregate_error_into_bucket + bucket = { + 'count': 1, + 'severity': 'warning', + 'sample_ids': [], + 'first_seen_at': None, + 'last_seen_at': None + } + + # New error with higher severity + err_error = {'severity': 'error', 'sample_id': 1} + _aggregate_error_into_bucket(err_error, bucket) + self.assertEqual(bucket['severity'], 'error') + self.assertEqual(bucket['count'], 2) + + # New error with lower severity should not downgrade + err_info = {'severity': 'info', 'sample_id': 2} + _aggregate_error_into_bucket(err_info, bucket) + self.assertEqual(bucket['severity'], 'error') + self.assertEqual(bucket['count'], 3) + + def test_derive_infrastructure_errors(self): + tp1 = TestProgress( + self.test_obj.id, TestStatus.canceled, 'provisioning VM failed') + tp1.timestamp = datetime.datetime(2023, 1, 1, 10, 0, 0) + + tp2 = TestProgress( + self.test_obj.id, TestStatus.canceled, 'merge conflict') + tp2.timestamp = datetime.datetime(2023, 1, 1, 10, 5, 0) + + g.db.add(tp1) + g.db.add(tp2) + g.db.commit() + + errors = derive_infrastructure_errors(self.test_obj.id) + self.assertEqual(len(errors), 2) + self.assertEqual(errors[0]['type'], 'vm_provisioning') + self.assertEqual(errors[1]['type'], 'merge') + + def test_derive_infrastructure_errors_excludes_api_cancels(self): + # A user/API cancellation must not be reported as an infrastructure error. + tp_api = TestProgress( + self.test_obj.id, TestStatus.canceled, 'Canceled by alice via API') + tp_api.timestamp = datetime.datetime(2023, 1, 1, 10, 0, 0) + tp_infra = TestProgress( + self.test_obj.id, TestStatus.canceled, 'build failed') + tp_infra.timestamp = datetime.datetime(2023, 1, 1, 10, 5, 0) + g.db.add(tp_api) + g.db.add(tp_infra) + g.db.commit() + + errors = derive_infrastructure_errors(self.test_obj.id) + self.assertEqual(len(errors), 1) + self.assertEqual(errors[0]['type'], 'build') + + def test_classify_infra_error(self): + self.assertEqual(_classify_infra_error( + 'timeout connecting to worker'), 'worker') + self.assertEqual(_classify_infra_error('failed to build'), 'build') + self.assertEqual(_classify_infra_error('storage is full'), 'storage') + self.assertEqual(_classify_infra_error( + 'fetch remote repository'), 'checkout') + self.assertEqual(_classify_infra_error('merge conflict'), 'merge') + self.assertEqual(_classify_infra_error( + 'random error string'), 'worker') + + def test_get_sample_id(self): + tr = TestResult(self.test_obj.id, 1, 100, 0, 0) + self.assertIsNone(_get_sample_id(tr)) + + tr.regression_test = MagicMock() + tr.regression_test.sample_id = 42 + self.assertEqual(_get_sample_id(tr), 42) + + # Test exception catching + mock_reg = MagicMock() + type(mock_reg).sample_id = PropertyMock(side_effect=RuntimeError('Mock exception')) + tr.regression_test = mock_reg + self.assertIsNone(_get_sample_id(tr)) diff --git a/tests/api/test_services_status.py b/tests/api/test_services_status.py new file mode 100644 index 000000000..3872af83d --- /dev/null +++ b/tests/api/test_services_status.py @@ -0,0 +1,173 @@ +import datetime + +from flask import g + +from mod_api.services.status import (derive_output_status, derive_run_status, + derive_sample_status, get_run_timestamps, + is_dummy_row) +from mod_regression.models import RegressionTestOutput +from mod_regression.models import \ + RegressionTestOutputFiles as RegressionTestMultipleFiles +from mod_test.models import (Fork, Test, TestPlatform, TestProgress, + TestResult, TestResultFile, TestStatus, TestType) +from tests.api.base import ApiTestCase + + +class TestServicesStatus(ApiTestCase): + def setUp(self): + super().setUp() + fork = Fork('https://github.com/test/test.git') + g.db.add(fork) + g.db.commit() + self.test_obj = Test(TestPlatform.linux, + TestType.commit, fork.id, 'master', 'commit_hash') + g.db.add(self.test_obj) + g.db.commit() + + def test_derive_run_status_queued(self): + self.assertEqual(derive_run_status(self.test_obj), 'queued') + + def test_derive_run_status_running(self): + tp = TestProgress(self.test_obj.id, TestStatus.testing, 'testing') + g.db.add(tp) + g.db.commit() + self.assertEqual(derive_run_status(self.test_obj), 'running') + + def test_derive_run_status_pass(self): + tp = TestProgress(self.test_obj.id, TestStatus.completed, 'done') + # A passing result: exit code matches and the expected output for + # regression test 1 was produced and matched (got=None). + tr = TestResult(self.test_obj.id, 1, 100, 0, 0) + rf = TestResultFile(self.test_obj.id, 1, 1, 'sample_out1') + g.db.add_all([tp, tr, rf]) + g.db.commit() + self.assertEqual(derive_run_status(self.test_obj), 'pass') + + def test_derive_run_status_completed_without_results_is_error(self): + # A run marked completed that reported zero TestResult rows must + # not show green — the worker finished without reporting anything. + tp = TestProgress(self.test_obj.id, TestStatus.completed, 'done') + g.db.add(tp) + g.db.commit() + self.assertEqual(derive_run_status(self.test_obj), 'error') + + def test_derive_run_status_fail(self): + tp = TestProgress(self.test_obj.id, TestStatus.completed, 'done') + # runtime 100, exit_code 1, expected 0 + tr = TestResult(self.test_obj.id, 1, 100, 1, 0) + g.db.add(tp) + g.db.add(tr) + g.db.commit() + self.assertEqual(derive_run_status(self.test_obj), 'fail') + + def test_derive_run_status_canceled_covers_infra_error(self): + tp = TestProgress(self.test_obj.id, + TestStatus.canceled, 'canceled by admin') + g.db.add(tp) + g.db.commit() + self.assertEqual(derive_run_status(self.test_obj), 'canceled') + + def test_derive_run_status_incomplete(self): + from unittest.mock import MagicMock + + from mod_api.services.status import _compute_run_status + mock_prog = MagicMock() + mock_prog.status = "some_unknown_status" + res = _compute_run_status([mock_prog], {}, {}, self.test_obj.id) + self.assertEqual(res, 'incomplete') + + def test_is_dummy_row(self): + rf = TestResultFile(1, 1, -1, '', 'error') + self.assertTrue(is_dummy_row(rf)) + rf2 = TestResultFile(1, 1, 1, 'expected', 'got') + self.assertFalse(is_dummy_row(rf2)) + + def test_derive_sample_status_not_started(self): + self.assertEqual(derive_sample_status(None, []), 'not_started') + + def test_derive_sample_status_missing_output(self): + tr = TestResult(1, 1, 100, 0, 0) + rf = TestResultFile(1, 1, -1, '', 'error') + self.assertEqual(derive_sample_status(tr, [rf]), 'missing_output') + + def test_derive_sample_status_fail_rc(self): + tr = TestResult(1, 1, 100, 1, 0) + self.assertEqual(derive_sample_status(tr, []), 'fail') + + def test_derive_sample_status_fail_diff(self): + tr = TestResult(1, 1, 100, 0, 0) + rf = TestResultFile(1, 1, 1, 'expected_hash', 'got_hash') + self.assertEqual(derive_sample_status(tr, [rf]), 'fail') + + def test_derive_sample_status_pass(self): + tr = TestResult(1, 1, 100, 0, 0) + rf = TestResultFile(1, 1, 1, 'expected_hash', None) + self.assertEqual(derive_sample_status(tr, [rf]), 'pass') + + def test_derive_sample_status_pass_multi(self): + tr = TestResult(1, 1, 100, 0, 0) + rf = TestResultFile(1, 1, 1, 'expected_hash', 'got_hash') + rto = RegressionTestOutput(1, 1, 'expected_hash', 'output.txt') + multi = RegressionTestMultipleFiles('got_hash', 1) + multi.file_hashes = 'got_hash' + rto.multiple_files = [multi] + rf.regression_test_output = rto + self.assertEqual(derive_sample_status(tr, [rf]), 'pass') + + def test_derive_sample_status_missing_output_expected(self): + """Missing output detected when expected non-ignored output has no result file.""" + tr = TestResult(1, 1, 100, 0, 0) + rto = RegressionTestOutput(1, 'hash', '.txt', 'out') + g.db.add(rto) + g.db.commit() + self.assertEqual(derive_sample_status(tr, [], expected_outputs=[rto]), 'missing_output') + + def test_derive_sample_status_pass_with_expected_outputs(self): + """Pass when all expected outputs have matching result files.""" + tr = TestResult(1, 1, 100, 0, 0) + rto = RegressionTestOutput(1, 'hash', '.txt', 'out') + g.db.add(rto) + g.db.commit() + rf = TestResultFile(1, 1, rto.id, 'hash', None) + self.assertEqual(derive_sample_status(tr, [rf], expected_outputs=[rto]), 'pass') + + def test_derive_sample_status_ignored_output_not_missing(self): + """Ignored expected outputs should not trigger missing_output.""" + tr = TestResult(1, 1, 100, 0, 0) + rto = RegressionTestOutput(1, 'hash', '.txt', 'out', ignore=True) + g.db.add(rto) + g.db.commit() + self.assertEqual(derive_sample_status(tr, [], expected_outputs=[rto]), 'pass') + + def test_derive_output_status(self): + rf_dummy = TestResultFile(-1, -1, -1, '', 'error') + self.assertEqual(derive_output_status(rf_dummy), 'missing_output') + + rf_match = TestResultFile(1, 1, 1, 'exp', None) + self.assertEqual(derive_output_status(rf_match), 'pass') + + rf_diff = TestResultFile(1, 1, 1, 'exp', 'got') + self.assertEqual(derive_output_status(rf_diff), 'fail') + + def test_get_run_timestamps(self): + ts = get_run_timestamps(self.test_obj) + self.assertIsNone(ts['created_at']) + + tp1 = TestProgress(self.test_obj.id, TestStatus.preparation, 'queued') + tp1.timestamp = datetime.datetime(2023, 1, 1, 10, 0, 0) + g.db.add(tp1) + + tp2 = TestProgress(self.test_obj.id, TestStatus.testing, 'testing') + tp2.timestamp = datetime.datetime(2023, 1, 1, 10, 5, 0) + g.db.add(tp2) + + tp3 = TestProgress(self.test_obj.id, TestStatus.completed, 'done') + tp3.timestamp = datetime.datetime(2023, 1, 1, 10, 10, 0) + g.db.add(tp3) + g.db.commit() + + ts2 = get_run_timestamps(self.test_obj) + self.assertEqual(ts2['created_at'], tp1.timestamp) + self.assertEqual(ts2['queued_at'], tp1.timestamp) + self.assertEqual(ts2['started_at'], tp2.timestamp) + self.assertEqual(ts2['completed_at'], tp3.timestamp) diff --git a/tests/api/test_services_storage.py b/tests/api/test_services_storage.py new file mode 100644 index 000000000..d249f50d3 --- /dev/null +++ b/tests/api/test_services_storage.py @@ -0,0 +1,131 @@ +import os +import tempfile +from unittest.mock import MagicMock, patch + +from mod_api.services.storage import (get_log_file_path, + get_test_results_base_path, + resolve_artifact) +from tests.api.base import ApiTestCase + + +class TestServicesStorage(ApiTestCase): + def setUp(self): + super().setUp() + self.test_dir = tempfile.TemporaryDirectory() + self.dir_path = self.test_dir.name + + def tearDown(self): + self.test_dir.cleanup() + super().tearDown() + + def create_file(self, relative_path): + full_path = os.path.join(self.dir_path, relative_path) + os.makedirs(os.path.dirname(full_path), exist_ok=True) + with open(full_path, 'w') as f: + f.write('dummy content') + return full_path + + def mock_config_get(self, key, default=None): + if key == 'SAMPLE_REPOSITORY': + return self.dir_path + if key == 'GCS_SIGNED_URL_EXPIRY_LIMIT': + return 60 + return default + + @patch('run.config') + @patch('run.storage_client_bucket') + def test_resolve_artifact_both_exist(self, mock_bucket, mock_config): + mock_config.get.side_effect = self.mock_config_get + self.create_file('test_artifact.txt') + + mock_blob = MagicMock() + mock_blob.exists.return_value = True + mock_blob.generate_signed_url.return_value = 'https://signed.url' + mock_bucket.blob.return_value = mock_blob + + url, status = resolve_artifact('test_artifact.txt') + self.assertEqual(url, 'https://signed.url') + self.assertEqual(status, 'ok') + mock_blob.generate_signed_url.assert_called_once() + + @patch('run.config') + @patch('run.storage_client_bucket') + def test_resolve_artifact_only_gcs(self, mock_bucket, mock_config): + mock_config.get.side_effect = self.mock_config_get + + mock_blob = MagicMock() + mock_blob.exists.return_value = True + mock_blob.generate_signed_url.return_value = 'https://signed.url' + mock_bucket.blob.return_value = mock_blob + + url, status = resolve_artifact('test_artifact.txt') + self.assertEqual(url, 'https://signed.url') + self.assertEqual(status, 'degraded') + + @patch('run.config') + @patch('run.storage_client_bucket') + def test_resolve_artifact_gcs_blob_no_exists_check(self, mock_bucket, mock_config): + mock_config.get.side_effect = self.mock_config_get + self.create_file('test_artifact.txt') + + mock_blob = MagicMock() + mock_blob.generate_signed_url.return_value = 'https://signed.url' + mock_bucket.blob.return_value = mock_blob + + mock_blob.exists.return_value = True + resolve_artifact('test_artifact.txt') + mock_blob.exists.assert_called_once() + + @patch('run.config') + @patch('run.storage_client_bucket', new=None) + def test_resolve_artifact_only_local(self, mock_config): + mock_config.get.side_effect = self.mock_config_get + self.create_file('test_artifact.txt') + + url, status = resolve_artifact('test_artifact.txt') + self.assertIsNone(url) + self.assertEqual(status, 'degraded') + + @patch('run.config') + @patch('run.storage_client_bucket', new=None) + def test_resolve_artifact_missing(self, mock_config): + mock_config.get.side_effect = self.mock_config_get + + url, status = resolve_artifact('test_artifact.txt') + self.assertIsNone(url) + self.assertEqual(status, 'missing') + + @patch('run.config') + @patch('run.storage_client_bucket') + def test_resolve_artifact_gcs_exception(self, mock_bucket, mock_config): + mock_config.get.side_effect = self.mock_config_get + self.create_file('test_artifact.txt') + + mock_bucket.blob.side_effect = Exception("GCS Error") + + url, status = resolve_artifact('test_artifact.txt') + self.assertIsNone(url) + self.assertEqual(status, 'degraded') + + @patch('run.config') + def test_get_log_file_path_exists(self, mock_config): + mock_config.get.side_effect = self.mock_config_get + path = self.create_file('LogFiles/123.txt') + + result = get_log_file_path(123) + self.assertEqual(os.path.normpath(result), os.path.normpath(path)) + + @patch('run.config') + def test_get_log_file_path_missing(self, mock_config): + mock_config.get.side_effect = self.mock_config_get + + result = get_log_file_path(123) + self.assertIsNone(result) + + @patch('run.config') + def test_get_test_results_base_path(self, mock_config): + mock_config.get.return_value = '/fake/repo' + + result = get_test_results_base_path() + expected = os.path.join('/fake/repo', 'TestResults') + self.assertEqual(result, expected) diff --git a/tests/api/test_utils.py b/tests/api/test_utils.py new file mode 100644 index 000000000..5901ae919 --- /dev/null +++ b/tests/api/test_utils.py @@ -0,0 +1,70 @@ +from unittest.mock import MagicMock + +from marshmallow import Schema, fields + +from mod_api.utils import (cursor_paginated_response, get_sort_column, + paginated_response, single_response) +from tests.api.base import ApiTestCase + + +class DummySchema(Schema): + id = fields.Integer() + name = fields.String() + + +class TestUtils(ApiTestCase): + def test_paginated_response_with_schema(self): + data = [{'id': 1, 'name': 'Item 1'}, {'id': 2, 'name': 'Item 2'}] + with self.app.test_request_context(): + res = paginated_response( + data, total=5, limit=2, offset=0, schema=DummySchema()) + self.assertEqual(res.status_code, 200) + json_data = res.json + self.assertEqual(len(json_data['data']), 2) + self.assertEqual(json_data['pagination']['total'], 5) + self.assertEqual(json_data['pagination']['next_offset'], 2) + + def test_paginated_response_no_schema(self): + data = [{'id': 1, 'name': 'Item 1'}, {'id': 2, 'name': 'Item 2'}] + with self.app.test_request_context(): + res = paginated_response(data, total=2, limit=2, offset=0) + self.assertEqual(res.status_code, 200) + json_data = res.json + self.assertEqual(len(json_data['data']), 2) + self.assertEqual(json_data['pagination']['total'], 2) + self.assertIsNone(json_data['pagination']['next_offset']) + + def test_cursor_paginated_response(self): + data = [{'id': 1, 'name': 'Item 1'}] + with self.app.test_request_context(): + res = cursor_paginated_response( + data, next_cursor=2, limit=1, schema=DummySchema()) + self.assertEqual(res.status_code, 200) + json_data = res.json + self.assertEqual(json_data['pagination']['next_cursor'], 2) + + res2 = cursor_paginated_response(data, next_cursor=None, limit=1) + self.assertIsNone(res2.json['pagination']['next_cursor']) + + def test_single_response(self): + data = {'id': 1, 'name': 'Item 1'} + with self.app.test_request_context(): + res = single_response(data, schema=DummySchema(), http_status=201) + self.assertEqual(res.status_code, 201) + self.assertEqual(res.json['name'], 'Item 1') + + res2 = single_response(data) + self.assertEqual(res2.status_code, 200) + + def test_get_sort_column(self): + mock_col = MagicMock() + mock_col.asc.return_value = 'asc_called' + mock_col.desc.return_value = 'desc_called' + + column_map = {'created_at': mock_col} + + self.assertIsNone(get_sort_column('invalid', column_map)) + self.assertEqual(get_sort_column( + 'created_at', column_map), 'asc_called') + self.assertEqual(get_sort_column( + '-created_at', column_map), 'desc_called') diff --git a/tests/base.py b/tests/base.py index 7f6e0d199..3bbc9bb33 100644 --- a/tests/base.py +++ b/tests/base.py @@ -410,6 +410,49 @@ def tearDown(self): """Clean up after every test.""" super().tearDown() + def setup_run_data(self, suffix="test"): + """Set up common models for API tests involving runs and samples.""" + from flask import g + + from mod_auth.models import Role, User + from mod_test.models import Fork, Test, TestPlatform, TestType + + self.admin = User( + f'testadmin_{suffix}', + Role.admin, + f'{suffix}_admin@local.com', + User.generate_hash('adminpass123')) + self.user = User( + f'testuser_{suffix}', + Role.user, + f'{suffix}_user@local.com', + User.generate_hash('userpass123')) + g.db.add_all([self.admin, self.user]) + g.db.commit() + + self.fork = Fork('https://github.com/test/test.git') + g.db.add(self.fork) + g.db.commit() + + self.test_obj = Test(TestPlatform.linux, TestType.commit, + self.fork.id, 'master', 'commit_hash') + g.db.add(self.test_obj) + g.db.commit() + self.test_id = self.test_obj.id + + def get_token(self, email, password, token_name='test_token', scopes=None): + """Get an API token for testing.""" + import json + payload = {'email': email, 'password': password, + 'token_name': token_name} + if scopes: + payload['scopes'] = scopes + res = self.client.post( + '/api/v1/auth/tokens', + data=json.dumps(payload), + content_type='application/json') + return res.json['token'] + @staticmethod def create_login_form_data(email, password) -> dict: """ diff --git a/tests/test_ci/test_controllers.py b/tests/test_ci/test_controllers.py index cca01a54a..8ff86f7dc 100644 --- a/tests/test_ci/test_controllers.py +++ b/tests/test_ci/test_controllers.py @@ -730,7 +730,8 @@ def test_webhook_release_deleted(self, mock_request, mock_repo): last_release = CCExtractorVersion.query.order_by(CCExtractorVersion.released.desc()).first() self.assertNotEqual(last_release.version, '2.1') - def test_webhook_prerelease(self): + @mock.patch('requests.get', side_effect=mock_api_request_github) + def test_webhook_prerelease(self, mock_request): """Check webhook release update CCExtractor Version for prerelease.""" with self.app.test_client() as c: # Full Release with version with 2.1 (prereleased action is ignored) diff --git a/tests/test_smartdiff/__init__.py b/tests/test_smartdiff/__init__.py new file mode 100644 index 000000000..68bc99968 --- /dev/null +++ b/tests/test_smartdiff/__init__.py @@ -0,0 +1 @@ +"""Tests for the smart-diff subtitle comparison.""" diff --git a/tests/test_smartdiff/fixtures/cea608_real.srt b/tests/test_smartdiff/fixtures/cea608_real.srt new file mode 100644 index 000000000..d0bf07ab9 --- /dev/null +++ b/tests/test_smartdiff/fixtures/cea608_real.srt @@ -0,0 +1,9 @@ +1 +00:00:05,956 --> 00:00:07,955 +CCextractor Start crdit Testing + +2 +00:00:13,913 --> 00:00:15,080 +>> WHICH OF THESE STORIES WILL +YOU BE TALKING ABOUT TRO + diff --git a/tests/test_smartdiff/fixtures/dvb_spanish_real.srt b/tests/test_smartdiff/fixtures/dvb_spanish_real.srt new file mode 100644 index 000000000..76a633c48 --- /dev/null +++ b/tests/test_smartdiff/fixtures/dvb_spanish_real.srt @@ -0,0 +1,59 @@ +1 +00:00:00,480 --> 00:01:05,479 +Para continuar con este debate, + +2 +00:00:06,080 --> 00:01:11,079 +gusted cree que si los partidarios +de Errejon fuesen derrotados + +3 +00:00:09,880 --> 00:01:14,879 +su propuesta en) Vistalegre, + +4 +00:00:12,920 --> 00:01:17,919 +Podemos deberia cambiar de portavoz +parlamentario? + +5 +00:00:19,080 --> 00:01:24,079 +éPuede representar al partido + +6 +00:00:21,400 --> 00:01:26,399 +en el Congreso alguienque'se ha +quedado en minoria + +7 +00:00:24,200 --> 00:01:29,199 +dentro del partido? + +8 +00:00:30,640 --> 00:01:35,639 +-Deciden los organos del partido la +linea de accion politica + +9 +00:00:34,200 --> 00:01:39,199 +dentro del partido. + +10 +00:00:43,120 --> 00:01:48,119 +Debemos acatar las decisiones +colectivas. + +11 +00:00:48,600 --> 00:01:53,599 +Si inicio Errejon reconoce que'se +ven esas lineas, + +12 +00:00:51,760 --> 00:01:56,759 +debe seguir adelante. + +13 +00:00:53,240 --> 00:01:58,239 +Solo.es canalizarla voz della +decision politica del partido. + diff --git a/tests/test_smartdiff/test_compare.py b/tests/test_smartdiff/test_compare.py new file mode 100644 index 000000000..a26a9b222 --- /dev/null +++ b/tests/test_smartdiff/test_compare.py @@ -0,0 +1,161 @@ +"""Tests for the semantic subtitle comparison / classifier.""" + +import unittest + +from mod_test.smartdiff.compare import smart_diff + + +def _srt(cues): + """ + Build SubRip text from (start_ms, end_ms, text) tuples. + + :param cues: Iterable of (start_ms, end_ms, text) tuples. + :type cues: list + :return: SubRip-formatted string. + :rtype: str + """ + def stamp(ms): + h, ms = divmod(ms, 3600000) + m, ms = divmod(ms, 60000) + s, ms = divmod(ms, 1000) + return f"{h:02d}:{m:02d}:{s:02d},{ms:03d}" + + blocks = [] + for i, (start, end, text) in enumerate(cues, start=1): + blocks.append(f"{i}\n{stamp(start)} --> {stamp(end)}\n{text}\n") + return "\n".join(blocks) + + +_BASE = [(1000, 4000, "Hello world"), (5000, 8000, "Second line")] +_BASE_CAPS = [(1000, 4000, "HELLO WORLD"), (5000, 8000, "SECOND LINE")] + + +class SmartDiffTests(unittest.TestCase): + """Classifying the kind of difference between two outputs.""" + + def test_identical(self): + """Equal outputs classify as identical.""" + result = smart_diff(_srt(_BASE), _srt(_BASE)) + self.assertEqual(result["kind"], "identical") + + def test_timing_shift_reports_offset(self): + """A constant timing offset is reported as timing_shift with offset_ms.""" + shifted = [(s + 500, e + 500, t) for s, e, t in _BASE] + result = smart_diff(_srt(_BASE), _srt(shifted)) + self.assertEqual(result["kind"], "timing_shift") + self.assertEqual(result["offset_ms"], 500) + + def test_text_change_only(self): + """Same timing, different text classifies as text_change.""" + changed = [(1000, 4000, "Hello world"), (5000, 8000, "DIFFERENT")] + result = smart_diff(_srt(_BASE), _srt(changed)) + self.assertEqual(result["kind"], "text_change") + + def test_missing_cues(self): + """Fewer cues than expected classifies as missing_cues.""" + result = smart_diff(_srt(_BASE), _srt(_BASE[:1])) + self.assertEqual(result["kind"], "missing_cues") + self.assertEqual((result["expected_cues"], result["actual_cues"]), (2, 1)) + + def test_extra_cues(self): + """More cues than expected classifies as extra_cues.""" + more = _BASE + [(9000, 10000, "Third line")] + result = smart_diff(_srt(_BASE), _srt(more)) + self.assertEqual(result["kind"], "extra_cues") + + def test_mixed_when_text_and_count_differ(self): + """Both text changes and a count mismatch classify as mixed.""" + other = [(1000, 4000, "CHANGED"), (5000, 8000, "Second line"), + (9000, 10000, "Third")] + result = smart_diff(_srt(_BASE), _srt(other)) + self.assertEqual(result["kind"], "mixed") + + def test_works_on_webvtt_via_autodetect(self): + """smart_diff auto-detects WebVTT and still classifies a timing shift.""" + base = "WEBVTT\n\n00:00:01.000 --> 00:00:04.000\nHello\n" + shifted = "WEBVTT\n\n00:00:01.250 --> 00:00:04.250\nHello\n" + result = smart_diff(base, shifted) + self.assertEqual(result["kind"], "timing_shift") + self.assertEqual(result["offset_ms"], 250) + + def test_whitespace_padding_only(self): + """Trailing CEA-608 padding differences are flagged as cosmetic, not text.""" + padded = [(1000, 4000, "HELLO WORLD "), (5000, 8000, "SECOND LINE ")] + result = smart_diff(_srt(_BASE_CAPS), _srt(padded)) + self.assertEqual(result["kind"], "whitespace_change") + + def test_formatting_tags_only(self): + """A styling-tags-only difference is flagged as formatting, not text.""" + styled = [(1000, 4000, "Hello world"), (5000, 8000, "Second line")] + result = smart_diff(_srt(_BASE), _srt(styled)) + self.assertEqual(result["kind"], "formatting_change") + + def test_timing_drift_growing_offset(self): + """A growing (not constant) timing offset classifies as timing_drift.""" + base = [(1000, 2000, "A"), (5000, 6000, "B"), (9000, 10000, "C")] + drifted = [(1000, 2000, "A"), (5040, 6040, "B"), (9080, 10080, "C")] + result = smart_diff(_srt(base), _srt(drifted)) + self.assertEqual(result["kind"], "timing_drift") + + def test_split_cues_same_text_more_cues(self): + """One cue rendered as two (same words) classifies as split_cues.""" + one = [(1000, 4000, "hello world")] + two = [(1000, 2000, "hello"), (2000, 4000, "world")] + result = smart_diff(_srt(one), _srt(two)) + self.assertEqual(result["kind"], "split_cues") + + def test_merged_cues_same_text_fewer_cues(self): + """Two cues collapsed into one (same words) classifies as merged_cues.""" + two = [(1000, 2000, "hello"), (2000, 4000, "world")] + one = [(1000, 4000, "hello world")] + result = smart_diff(_srt(two), _srt(one)) + self.assertEqual(result["kind"], "merged_cues") + + def test_changes_list_text_detail(self): + """A text change lists which cue changed, with expected/actual snippets.""" + changed = [(1000, 4000, "Hello world"), (5000, 8000, "DIFFERENT")] + result = smart_diff(_srt(_BASE), _srt(changed)) + changes = result["changes"] + self.assertEqual(len(changes), 1) + self.assertEqual(changes[0]["cue"], 2) + self.assertEqual(changes[0]["kind"], "text") + self.assertEqual(changes[0]["actual"], "DIFFERENT") + + def test_changes_list_timing_offsets(self): + """A timing shift lists a per-cue offset for each cue.""" + shifted = [(s + 500, e + 500, t) for s, e, t in _BASE] + result = smart_diff(_srt(_BASE), _srt(shifted)) + self.assertTrue(all(c["offset_ms"] == 500 for c in result["changes"])) + + def test_identical_has_no_changes(self): + """An identical result carries no changes list.""" + self.assertNotIn("changes", smart_diff(_srt(_BASE), _srt(_BASE))) + + def test_no_cues_and_differ_is_unsupported_not_identical(self): + """Two different non-subtitle outputs (no cues) are not called identical.""" + result = smart_diff("plain transcript one", "plain transcript two") + self.assertEqual(result["kind"], "unsupported") + + def test_no_cues_but_equal_is_identical(self): + """Two equal cue-less outputs are still identical.""" + self.assertEqual(smart_diff("same text", "same text")["kind"], "identical") + + def test_text_change_with_shift_records_per_cue_offset(self): + """A combined text change + timing shift keeps the per-cue offset in changes.""" + base = [(1000, 2000, "A"), (5000, 6000, "B")] + other = [(1500, 2500, "X"), (5500, 6500, "Y")] + result = smart_diff(_srt(base), _srt(other)) + self.assertEqual(result["kind"], "text_change") + self.assertNotIn("aligned", result["summary"]) + self.assertTrue(all(c["offset_ms"] == 500 for c in result["changes"])) + + def test_encoding_change_non_ascii_only(self): + """A charset difference (accents only, e.g. -latin1) is flagged as encoding.""" + accented = [(1000, 4000, "Voilà"), (5000, 8000, "naïve café")] + folded = [(1000, 4000, "Voila"), (5000, 8000, "naive cafe")] + result = smart_diff(_srt(accented), _srt(folded)) + self.assertEqual(result["kind"], "encoding_change") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_smartdiff/test_fixtures.py b/tests/test_smartdiff/test_fixtures.py new file mode 100644 index 000000000..997f08dc5 --- /dev/null +++ b/tests/test_smartdiff/test_fixtures.py @@ -0,0 +1,133 @@ +"""Golden-fixture tests against real CCExtractor output, plus input robustness. + +The fixtures are genuine CCExtractor outputs (not synthetic strings): +- ``cea608_real.srt``: a CEA-608 broadcast caption sample (trailing padding). +- ``dvb_spanish_real.srt``: a DVB Spanish sample with ```` colour tags and + accented characters. Both were security-scanned before vendoring (no paths, + IPs, emails, URLs, or secrets) and are valid UTF-8. +""" + +import os +import unittest + +from mod_test.smartdiff.compare import smart_diff +from mod_test.smartdiff.normalize import ascii_fold, strip_tags +from mod_test.smartdiff.srt import Cue, parse_srt + +_FIXTURES = os.path.join(os.path.dirname(__file__), 'fixtures') + + +def _load(name): + """ + Read a vendored fixture as UTF-8. + + :param name: Fixture file name. + :type name: str + :return: The file content. + :rtype: str + """ + with open(os.path.join(_FIXTURES, name), encoding='utf-8') as handle: + return handle.read() + + +def _emit(cues): + """ + Serialise cues back to SubRip text (for building timing-shifted variants). + + :param cues: The cues to serialise. + :type cues: list + :return: SubRip-formatted text. + :rtype: str + """ + def stamp(ms): + hours, ms = divmod(ms, 3600000) + minutes, ms = divmod(ms, 60000) + seconds, ms = divmod(ms, 1000) + return f"{hours:02d}:{minutes:02d}:{seconds:02d},{ms:03d}" + + return "\n".join(f"{i}\n{stamp(c.start_ms)} --> {stamp(c.end_ms)}\n{c.text}\n" + for i, c in enumerate(cues, 1)) + + +class Cea608RealTests(unittest.TestCase): + """Smart diff on a genuine CEA-608 broadcast caption sample.""" + + def test_parses_real_sample(self): + """The real sample parses into its two CEA-608 cues.""" + cues = parse_srt(_load('cea608_real.srt')) + self.assertEqual(len(cues), 2) + self.assertEqual(cues[0].start_ms, 5956) + + def test_identical_against_itself(self): + """The real sample compared with itself is identical.""" + raw = _load('cea608_real.srt') + self.assertEqual(smart_diff(raw, raw)['kind'], 'identical') + + def test_depadding_is_cosmetic(self): + """Stripping the CEA-608 trailing padding is flagged as cosmetic only.""" + raw = _load('cea608_real.srt') + depadded = '\n'.join(line.rstrip() for line in raw.split('\n')) + self.assertIn(smart_diff(raw, depadded)['kind'], + ('identical', 'whitespace_change')) + + +class DvbSpanishRealTests(unittest.TestCase): + """Smart diff on a real DVB Spanish output (font colour tags + accents).""" + + def test_parses_with_tags_and_accents(self): + """The fixture has 13 cues carrying both font tags and non-ASCII text.""" + cues = parse_srt(_load('dvb_spanish_real.srt')) + self.assertEqual(len(cues), 13) + self.assertTrue(any(' 127 for c in cues for ch in c.text)) + + def test_identical(self): + """The fixture compared with itself is identical.""" + raw = _load('dvb_spanish_real.srt') + self.assertEqual(smart_diff(raw, raw)['kind'], 'identical') + + def test_constant_timing_shift(self): + """Shifting every cue by +500 ms is detected with the exact offset.""" + cues = parse_srt(_load('dvb_spanish_real.srt')) + shifted = [Cue(c.index, c.start_ms + 500, c.end_ms + 500, c.text) for c in cues] + result = smart_diff(_emit(cues), _emit(shifted)) + self.assertEqual(result['kind'], 'timing_shift') + self.assertEqual(result['offset_ms'], 500) + + def test_font_tags_are_formatting_only(self): + """Removing the colour tags is classified as formatting, not text.""" + raw = _load('dvb_spanish_real.srt') + self.assertEqual(smart_diff(raw, strip_tags(raw))['kind'], 'formatting_change') + + def test_accent_folding_is_encoding(self): + """Folding the accented characters is classified as an encoding difference.""" + raw = _load('dvb_spanish_real.srt') + self.assertEqual(smart_diff(raw, ascii_fold(raw))['kind'], 'encoding_change') + + def test_dropped_cues_are_missing(self): + """Dropping the last three cues is reported as missing_cues.""" + cues = parse_srt(_load('dvb_spanish_real.srt')) + result = smart_diff(_emit(cues), _emit(cues[:-3])) + self.assertEqual(result['kind'], 'missing_cues') + + +class RobustnessTests(unittest.TestCase): + """Malformed or hostile input must classify cleanly, never crash.""" + + def test_parser_survives_garbage(self): + """The parser returns a list for empty, junk, and control-byte input.""" + for junk in ['', 'not a subtitle', '\x00\x01\x02', '1\nno timing line\n']: + self.assertIsInstance(parse_srt(junk), list) + + def test_smart_diff_on_empty_inputs(self): + """Two empty inputs are identical, not an error.""" + self.assertEqual(smart_diff('', '')['kind'], 'identical') + + def test_smart_diff_garbage_vs_real(self): + """Garbage against a real sample classifies without raising.""" + result = smart_diff('garbage with no cues', _load('dvb_spanish_real.srt')) + self.assertIn('kind', result) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_smartdiff/test_model_integration.py b/tests/test_smartdiff/test_model_integration.py new file mode 100644 index 000000000..0b8c715e4 --- /dev/null +++ b/tests/test_smartdiff/test_model_integration.py @@ -0,0 +1,63 @@ +"""Tests for TestResultFile.generate_smart_diff (the model glue) against real files. + +The method is exercised with a lightweight stand-in ``self`` so the test stays a +fast unit test (no database/ORM mapper configuration required). +""" + +import os +import tempfile +import unittest +from unittest import mock + +from mod_test.models import TestResultFile + +_CUE = "1\n00:00:01,000 --> 00:00:04,000\nHello world\n" + + +def _run(expected_text, got_text, ext='.srt', got='GOT'): + """ + Write two outputs to a temp dir and run generate_smart_diff over them. + + :param expected_text: Expected output content. + :type expected_text: str + :param got_text: Actual output content. + :type got_text: str + :param ext: Output file extension. + :type ext: str + :param got: The 'got' hash (set to None to simulate no produced output). + :type got: str + :return: The smart-diff classification. + :rtype: dict + """ + base = tempfile.mkdtemp() + with open(os.path.join(base, 'EXP' + ext), 'w', encoding='utf-8') as handle: + handle.write(expected_text) + with open(os.path.join(base, 'GOT' + ext), 'w', encoding='utf-8') as handle: + handle.write(got_text) + stub = mock.Mock() + stub.expected = 'EXP' + stub.got = got + stub.regression_test_output.correct_extension = ext + stub.read_lines = TestResultFile.read_lines + return TestResultFile.generate_smart_diff(stub, base) + + +class GenerateSmartDiffTests(unittest.TestCase): + """The model method reads the on-disk outputs and classifies the difference.""" + + def test_identical(self): + """Equal on-disk outputs classify as identical.""" + self.assertEqual(_run(_CUE, _CUE)['kind'], 'identical') + + def test_timing_shift(self): + """A shifted output is classified as a timing shift.""" + shifted = "1\n00:00:01,500 --> 00:00:04,500\nHello world\n" + self.assertEqual(_run(_CUE, shifted)['kind'], 'timing_shift') + + def test_missing_got_is_identical(self): + """A null 'got' (no produced output) short-circuits to identical.""" + self.assertEqual(_run(_CUE, _CUE, got=None)['kind'], 'identical') + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_smartdiff/test_normalize.py b/tests/test_smartdiff/test_normalize.py new file mode 100644 index 000000000..2b766e981 --- /dev/null +++ b/tests/test_smartdiff/test_normalize.py @@ -0,0 +1,55 @@ +"""Tests for CCExtractor-style normalisation of cue text.""" + +import unittest + +from mod_test.smartdiff.normalize import (ascii_fold, classify_text_pair, + plain, strip_tags, unescape) + + +class NormalizeTests(unittest.TestCase): + """Tag stripping, entity unescaping, and cue-text classification.""" + + def test_strip_tags(self): + """HTML/styling tags are removed.""" + self.assertEqual(strip_tags('hi'), 'hi') + + def test_unescape_entities(self): + """Known HTML entities are decoded, including a nested &.""" + self.assertEqual(unescape('a <b> & 30°'), 'a & 30°') + + def test_plain_combines_rules(self): + """plain() strips tags, unescapes, and rstrips padding together.""" + self.assertEqual(plain('hi & bye '), 'hi & bye') + + def test_classify_match(self): + """Identical text classifies as match.""" + self.assertEqual(classify_text_pair('hello', 'hello'), 'match') + + def test_classify_whitespace_only(self): + """Trailing CEA-608 padding differences classify as whitespace.""" + self.assertEqual(classify_text_pair('HELLO WORLD', 'HELLO WORLD '), 'whitespace') + + def test_classify_formatting_only(self): + """A tags-only difference classifies as formatting.""" + self.assertEqual(classify_text_pair('hello', 'hello'), 'formatting') + + def test_ascii_fold_decomposes_accents(self): + """ascii_fold strips accents and drops non-ASCII characters.""" + self.assertEqual(ascii_fold('Voilà café ♪'), 'Voila cafe ') + + def test_classify_encoding_only(self): + """A non-ASCII/accent-only difference classifies as encoding.""" + self.assertEqual(classify_text_pair('PRÉCIS', 'PRECIS'), 'encoding') + + def test_classify_real_text_change(self): + """A genuine text change classifies as text.""" + self.assertEqual(classify_text_pair('hello', 'goodbye'), 'text') + + def test_classify_non_latin_text_change_not_encoding(self): + """Two different non-Latin texts (empty ASCII skeleton) classify as text.""" + self.assertEqual(classify_text_pair('日本語', '中文'), 'text') + self.assertEqual(classify_text_pair('Привет', 'Спасибо'), 'text') + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_smartdiff/test_srt.py b/tests/test_smartdiff/test_srt.py new file mode 100644 index 000000000..55ed4c9c7 --- /dev/null +++ b/tests/test_smartdiff/test_srt.py @@ -0,0 +1,48 @@ +"""Tests for the SubRip (.srt) parser.""" + +import unittest + +from mod_test.smartdiff.srt import parse_srt + +_TWO_CUES = ( + "1\n" + "00:00:01,000 --> 00:00:04,000\n" + "Hello world\n" + "\n" + "2\n" + "00:00:05,500 --> 00:00:08,250\n" + "Second line\n" +) + + +class ParseSrtTests(unittest.TestCase): + """Parsing SubRip content into structured cues.""" + + def test_parses_index_timing_and_text(self): + """A two-cue file yields two cues with correct ms timing and text.""" + cues = parse_srt(_TWO_CUES) + self.assertEqual(len(cues), 2) + self.assertEqual((cues[0].index, cues[0].start_ms, cues[0].end_ms), (1, 1000, 4000)) + self.assertEqual(cues[0].text, "Hello world") + self.assertEqual((cues[1].start_ms, cues[1].end_ms), (5500, 8250)) + + def test_tolerates_crlf_and_bom(self): + """CRLF line endings and a leading BOM are handled.""" + cues = parse_srt("" + _TWO_CUES.replace("\n", "\r\n")) + self.assertEqual(len(cues), 2) + self.assertEqual(cues[1].text, "Second line") + + def test_skips_blocks_without_timing(self): + """A trailing junk block with no timing line is ignored.""" + cues = parse_srt(_TWO_CUES + "\nnot a cue\n") + self.assertEqual(len(cues), 2) + + def test_multiline_cue_text_preserved(self): + """Cue text spanning multiple lines is preserved with its newline.""" + content = "1\n00:00:01,000 --> 00:00:02,000\nline one\nline two\n" + cues = parse_srt(content) + self.assertEqual(cues[0].text, "line one\nline two") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_smartdiff/test_vtt.py b/tests/test_smartdiff/test_vtt.py new file mode 100644 index 000000000..695aeb11a --- /dev/null +++ b/tests/test_smartdiff/test_vtt.py @@ -0,0 +1,44 @@ +"""Tests for the WebVTT (.vtt) parser.""" + +import unittest + +from mod_test.smartdiff.vtt import parse_vtt + +_VTT = ( + "WEBVTT\n" + "\n" + "NOTE this is a comment\n" + "\n" + "1\n" + "00:00:01.000 --> 00:00:04.000 align:start position:50%\n" + "Hello world\n" + "\n" + "00:05.500 --> 00:08.250\n" + "Second line\n" +) + + +class ParseVttTests(unittest.TestCase): + """Parsing WebVTT content into structured cues.""" + + def test_parses_cues_and_skips_metadata(self): + """The WEBVTT header and NOTE block are skipped; cues are parsed.""" + cues = parse_vtt(_VTT) + self.assertEqual(len(cues), 2) + self.assertEqual((cues[0].start_ms, cues[0].end_ms), (1000, 4000)) + self.assertEqual(cues[0].text, "Hello world") + + def test_ignores_trailing_cue_settings(self): + """Cue settings after the end timestamp do not leak into timing/text.""" + cues = parse_vtt(_VTT) + self.assertEqual(cues[0].end_ms, 4000) + self.assertEqual(cues[0].text, "Hello world") + + def test_handles_optional_hours(self): + """A MM:SS.mmm timestamp without an hours component is parsed correctly.""" + cues = parse_vtt(_VTT) + self.assertEqual((cues[1].start_ms, cues[1].end_ms), (5500, 8250)) + + +if __name__ == "__main__": + unittest.main()