From 3a042a39d20ceac43adac6883a8c94971a50e6fe Mon Sep 17 00:00:00 2001 From: unnawut Date: Fri, 30 Jan 2026 16:35:39 +0700 Subject: [PATCH 1/6] add API testing --- README.md | 5 + packages/testing/pyproject.toml | 1 + packages/testing/src/framework/cli/apitest.py | 71 +++++++++ .../cli/pytest_ini_files/pytest-apitest.ini | 21 +++ .../pytest_plugins/api_conformance.py | 26 ++++ src/lean_spec/subspecs/api/server.py | 3 +- tests/api_conformance/__init__.py | 1 + tests/api_conformance/conftest.py | 145 ++++++++++++++++++ tests/api_conformance/test_finalized_state.py | 78 ++++++++++ tests/api_conformance/test_health.py | 28 ++++ .../test_justified_checkpoint.py | 42 +++++ tests/api_conformance/test_metrics.py | 33 ++++ tests/api_conformance/test_unknown_routes.py | 15 ++ tests/lean_spec/subspecs/api/test_server.py | 138 ++--------------- 14 files changed, 483 insertions(+), 124 deletions(-) create mode 100644 packages/testing/src/framework/cli/apitest.py create mode 100644 packages/testing/src/framework/cli/pytest_ini_files/pytest-apitest.ini create mode 100644 packages/testing/src/framework/pytest_plugins/api_conformance.py create mode 100644 tests/api_conformance/__init__.py create mode 100644 tests/api_conformance/conftest.py create mode 100644 tests/api_conformance/test_finalized_state.py create mode 100644 tests/api_conformance/test_health.py create mode 100644 tests/api_conformance/test_justified_checkpoint.py create mode 100644 tests/api_conformance/test_metrics.py create mode 100644 tests/api_conformance/test_unknown_routes.py diff --git a/README.md b/README.md index 90bbccac..15ca0c97 100644 --- a/README.md +++ b/README.md @@ -115,6 +115,10 @@ uv run pytest -m "not slow" # --fork Target fork (default: devnet) # --output Optional directory for filled fixtures uv run fill --clean --fork=devnet + +# Run API conformance tests against an external client implementation +# Usage: uv run apitest [pytest-args] +uv run apitest http://localhost:5052 ``` ### Code Quality @@ -221,6 +225,7 @@ def test_withdrawal_amount_above_uint64_max(): | Serve docs | `uv run mkdocs serve` | | Run everything (checks + tests + docs) | `uvx tox` | | Run all quality checks (no tests/docs) | `uvx tox -e all-checks` | +| Test external client API conformance | `uv run apitest http://localhost:5052` | | Run consensus node | `uv run python -m lean_spec --genesis config.yaml` | | Build Docker test image | `docker build -t lean-spec:test .` | | Build Docker node image | `docker build --target node -t lean-spec:node .` | diff --git a/packages/testing/pyproject.toml b/packages/testing/pyproject.toml index 821b2b24..9b2dda5e 100644 --- a/packages/testing/pyproject.toml +++ b/packages/testing/pyproject.toml @@ -35,6 +35,7 @@ Issues = "https://github.com/leanEthereum/lean-spec/issues" [project.scripts] fill = "framework.cli.fill:fill" +apitest = "framework.cli.apitest:apitest" [tool.setuptools.packages.find] where = ["src"] diff --git a/packages/testing/src/framework/cli/apitest.py b/packages/testing/src/framework/cli/apitest.py new file mode 100644 index 00000000..014db68b --- /dev/null +++ b/packages/testing/src/framework/cli/apitest.py @@ -0,0 +1,71 @@ +"""CLI command for running API conformance tests against an external server.""" + +import sys +from pathlib import Path +from typing import Sequence + +import click +import pytest + + +@click.command( + context_settings={ + "ignore_unknown_options": True, + "allow_extra_args": True, + } +) +@click.argument("server_url") +@click.argument("pytest_args", nargs=-1, type=click.UNPROCESSED) +@click.pass_context +def apitest( + ctx: click.Context, + server_url: str, + pytest_args: Sequence[str], +) -> None: + """ + Run API conformance tests against an external server, i.e. a client implementation. + + SERVER_URL is the base URL of the API server (e.g., http://localhost:5052). + + For testing the local leanSpec implementation, use `uv run pytest tests/api_conformance` + which automatically starts a local server. + + Examples: + # Run against external server + apitest http://localhost:5052 + + # Run with verbose output + apitest http://localhost:5052 -v + + # Run specific test + apitest http://localhost:5052 -k test_health + """ + config_path = Path(__file__).parent / "pytest_ini_files" / "pytest-apitest.ini" + + # Find project root + project_root = Path.cwd() + while project_root != project_root.parent: + if (project_root / "pyproject.toml").exists(): + pyproject = project_root / "pyproject.toml" + if "[tool.uv.workspace]" in pyproject.read_text(): + break + project_root = project_root.parent + + # Build pytest arguments + args = [ + "-c", + str(config_path), + f"--rootdir={project_root}", + f"--server-url={server_url}", + "tests/api_conformance", + ] + + args.extend(pytest_args) + args.extend(ctx.args) + + exit_code = pytest.main(args) + sys.exit(exit_code) + + +if __name__ == "__main__": + apitest() diff --git a/packages/testing/src/framework/cli/pytest_ini_files/pytest-apitest.ini b/packages/testing/src/framework/cli/pytest_ini_files/pytest-apitest.ini new file mode 100644 index 00000000..5e17f587 --- /dev/null +++ b/packages/testing/src/framework/cli/pytest_ini_files/pytest-apitest.ini @@ -0,0 +1,21 @@ +[pytest] +# Configuration for apitest command + +# Search for API conformance tests +testpaths = tests/api_conformance + +# Load pytest plugins +addopts = + -p framework.pytest_plugins.api_conformance + # Show shorter tracebacks + --tb=short + # Disable coverage for API testing + --no-cov + +# Test discovery +python_files = test_*.py +python_classes = Test* +python_functions = test_* + +# Minimal output +console_output_style = classic diff --git a/packages/testing/src/framework/pytest_plugins/api_conformance.py b/packages/testing/src/framework/pytest_plugins/api_conformance.py new file mode 100644 index 00000000..9fd28a34 --- /dev/null +++ b/packages/testing/src/framework/pytest_plugins/api_conformance.py @@ -0,0 +1,26 @@ +"""Pytest plugin for API conformance testing via the apitest CLI. + +This plugin is used by the apitest CLI command to pass the --server-url +option to pytest. The option is registered here so pytest can parse it. + +When running via `uv run pytest`, the conftest.py in tests/api_conformance +handles server startup and provides the server_url fixture directly. +""" + +import pytest + + +def pytest_addoption(parser: pytest.Parser) -> None: + """ + Register the --server-url option with pytest. + + The apitest CLI passes this option to pytest. + When running via regular pytest, the conftest.py handles this. + """ + group = parser.getgroup("apitest", "leanSpec API conformance testing") + group.addoption( + "--server-url", + action="store", + default=None, + help="Base URL of the API server to test", + ) diff --git a/src/lean_spec/subspecs/api/server.py b/src/lean_spec/subspecs/api/server.py index 70996f62..a486f17d 100644 --- a/src/lean_spec/subspecs/api/server.py +++ b/src/lean_spec/subspecs/api/server.py @@ -42,7 +42,8 @@ async def _handle_metrics(_request: web.Request) -> web.Response: """Handle Prometheus metrics endpoint.""" return web.Response( body=generate_metrics(), - content_type="text/plain; version=0.0.4; charset=utf-8", + content_type="text/plain; version=0.0.4", + charset="utf-8", ) diff --git a/tests/api_conformance/__init__.py b/tests/api_conformance/__init__.py new file mode 100644 index 00000000..895b275a --- /dev/null +++ b/tests/api_conformance/__init__.py @@ -0,0 +1 @@ +"""API conformance test suite for validating leanSpec client implementations.""" diff --git a/tests/api_conformance/conftest.py b/tests/api_conformance/conftest.py new file mode 100644 index 00000000..b5903884 --- /dev/null +++ b/tests/api_conformance/conftest.py @@ -0,0 +1,145 @@ +"""Pytest configuration for API conformance tests.""" + +import asyncio +import threading +import time +from typing import TYPE_CHECKING, Generator + +import httpx +import pytest + +if TYPE_CHECKING: + from lean_spec.subspecs.api import ApiServer + +# Default port for auto-started local server +DEFAULT_PORT = 15099 + + +class _ServerThread(threading.Thread): + """Thread that runs the API server in its own event loop.""" + + def __init__(self, port: int): + super().__init__(daemon=True) + self.port = port + self.server: ApiServer | None = None + self.loop: asyncio.AbstractEventLoop | None = None + self.ready = threading.Event() + self.error: Exception | None = None + + def run(self) -> None: + """Run the server in a new event loop.""" + try: + self.loop = asyncio.new_event_loop() + asyncio.set_event_loop(self.loop) + + self.server = self._create_server() + self.loop.run_until_complete(self.server.start()) + self.ready.set() + + self.loop.run_forever() + + except Exception as e: + self.error = e + self.ready.set() + finally: + if self.loop: + self.loop.close() + + def _create_server(self) -> "ApiServer": + """Create the API server with a test store.""" + from lean_spec.subspecs.api import ApiServer, ApiServerConfig + from lean_spec.subspecs.containers import Block, BlockBody, State, Validator + from lean_spec.subspecs.containers.block.types import AggregatedAttestations + from lean_spec.subspecs.containers.slot import Slot + from lean_spec.subspecs.containers.state import Validators + from lean_spec.subspecs.containers.validator import ValidatorIndex + from lean_spec.subspecs.forkchoice import Store + from lean_spec.subspecs.ssz.hash import hash_tree_root + from lean_spec.types import Bytes32, Bytes52, Uint64 + + validators = Validators( + data=[ + Validator(pubkey=Bytes52(b"\x00" * 52), index=ValidatorIndex(i)) for i in range(3) + ] + ) + + genesis_state = State.generate_genesis( + genesis_time=Uint64(int(time.time())), + validators=validators, + ) + + genesis_block = Block( + slot=Slot(0), + proposer_index=ValidatorIndex(0), + parent_root=Bytes32.zero(), + state_root=hash_tree_root(genesis_state), + body=BlockBody(attestations=AggregatedAttestations(data=[])), + ) + + store = Store.get_forkchoice_store(genesis_state, genesis_block) + + config = ApiServerConfig(host="127.0.0.1", port=self.port) + return ApiServer(config=config, store_getter=lambda: store) + + def stop(self) -> None: + """Stop the server and event loop.""" + if self.server and self.loop: + self.loop.call_soon_threadsafe(self.server.stop) + self.loop.call_soon_threadsafe(self.loop.stop) + + +def _wait_for_server(url: str, timeout: float = 5.0) -> bool: + """Wait for server to be ready by polling the health endpoint.""" + start = time.time() + while time.time() - start < timeout: + try: + response = httpx.get(f"{url}/lean/v0/health", timeout=1.0) + if response.status_code == 200: + return True + except (httpx.ConnectError, httpx.ReadTimeout): + pass + time.sleep(0.1) + return False + + +def pytest_addoption(parser: pytest.Parser) -> None: + """Add --server-url option for testing against external servers.""" + parser.addoption( + "--server-url", + action="store", + default=None, + help="External server URL. If not provided, starts a local server.", + ) + + +@pytest.fixture(scope="session") +def server_url(request: pytest.FixtureRequest) -> Generator[str, None, None]: + """ + Provide the server URL for API tests. + + If --server-url is provided, uses that external server. + Otherwise, starts a local leanSpec server for the test session. + """ + external_url = request.config.getoption("--server-url") + + if external_url: + # Use external server + yield external_url + else: + # Start local server + server_thread = _ServerThread(DEFAULT_PORT) + server_thread.start() + server_thread.ready.wait(timeout=10.0) + + if server_thread.error: + pytest.fail(f"Failed to start local server: {server_thread.error}") + + url = f"http://127.0.0.1:{DEFAULT_PORT}" + + if not _wait_for_server(url): + server_thread.stop() + pytest.fail("Local server failed to become ready") + + yield url + + server_thread.stop() diff --git a/tests/api_conformance/test_finalized_state.py b/tests/api_conformance/test_finalized_state.py new file mode 100644 index 00000000..c0e68648 --- /dev/null +++ b/tests/api_conformance/test_finalized_state.py @@ -0,0 +1,78 @@ +"""Tests for the finalized state endpoint with deep SSZ validation.""" + +import httpx + +from lean_spec.subspecs.containers import State + + +def test_finalized_state_returns_200(server_url: str) -> None: + """Finalized state endpoint returns 200 status code.""" + response = httpx.get(f"{server_url}/lean/v0/states/finalized") + assert response.status_code == 200 + + +def test_finalized_state_content_type_is_octet_stream(server_url: str) -> None: + """Finalized state endpoint returns octet-stream content type.""" + response = httpx.get(f"{server_url}/lean/v0/states/finalized") + content_type = response.headers.get("content-type", "") + assert "application/octet-stream" in content_type + + +def test_finalized_state_ssz_deserializes(server_url: str) -> None: + """Finalized state SSZ bytes deserialize to a valid State object.""" + response = httpx.get(f"{server_url}/lean/v0/states/finalized") + ssz_bytes = response.content + + # This will raise if the bytes are not valid SSZ for State + state = State.decode_bytes(ssz_bytes) + + # Basic structural validation + assert state is not None + + +def test_finalized_state_has_valid_slot(server_url: str) -> None: + """Finalized state has a non-negative slot.""" + response = httpx.get(f"{server_url}/lean/v0/states/finalized") + state = State.decode_bytes(response.content) + + assert int(state.slot) >= 0 + + +def test_finalized_state_has_validators(server_url: str) -> None: + """Finalized state has at least one validator.""" + response = httpx.get(f"{server_url}/lean/v0/states/finalized") + state = State.decode_bytes(response.content) + + assert len(state.validators) > 0 + + +def test_finalized_state_has_valid_config(server_url: str) -> None: + """Finalized state has a valid config with genesis time.""" + response = httpx.get(f"{server_url}/lean/v0/states/finalized") + state = State.decode_bytes(response.content) + + # Genesis time should be a positive timestamp + assert int(state.config.genesis_time) >= 0 + + +def test_finalized_state_has_valid_checkpoints(server_url: str) -> None: + """Finalized state has valid justified and finalized checkpoints.""" + response = httpx.get(f"{server_url}/lean/v0/states/finalized") + state = State.decode_bytes(response.content) + + # Checkpoints should have valid slots + assert int(state.latest_justified.slot) >= 0 + assert int(state.latest_finalized.slot) >= 0 + + # Finalized slot should not exceed justified slot + assert state.latest_finalized.slot <= state.latest_justified.slot + + +def test_finalized_state_has_valid_block_header(server_url: str) -> None: + """Finalized state has a valid latest block header.""" + response = httpx.get(f"{server_url}/lean/v0/states/finalized") + state = State.decode_bytes(response.content) + + # Block header should have valid slot and proposer index + assert int(state.latest_block_header.slot) >= 0 + assert int(state.latest_block_header.proposer_index) >= 0 diff --git a/tests/api_conformance/test_health.py b/tests/api_conformance/test_health.py new file mode 100644 index 00000000..23912a48 --- /dev/null +++ b/tests/api_conformance/test_health.py @@ -0,0 +1,28 @@ +"""Tests for the health endpoint.""" + +import httpx + + +def test_health_returns_200(server_url: str) -> None: + """Health endpoint returns 200 status code.""" + response = httpx.get(f"{server_url}/lean/v0/health") + assert response.status_code == 200 + + +def test_health_content_type_is_json(server_url: str) -> None: + """Health endpoint returns JSON content type.""" + response = httpx.get(f"{server_url}/lean/v0/health") + content_type = response.headers.get("content-type", "") + assert "application/json" in content_type + + +def test_health_response_structure(server_url: str) -> None: + """Health endpoint returns expected JSON structure.""" + response = httpx.get(f"{server_url}/lean/v0/health") + data = response.json() + + assert "status" in data + assert data["status"] == "healthy" + + assert "service" in data + assert data["service"] == "lean-spec-api" diff --git a/tests/api_conformance/test_justified_checkpoint.py b/tests/api_conformance/test_justified_checkpoint.py new file mode 100644 index 00000000..b8d34c09 --- /dev/null +++ b/tests/api_conformance/test_justified_checkpoint.py @@ -0,0 +1,42 @@ +"""Tests for the justified checkpoint endpoint.""" + +import httpx + + +def test_justified_checkpoint_returns_200(server_url: str) -> None: + """Justified checkpoint endpoint returns 200 status code.""" + response = httpx.get(f"{server_url}/lean/v0/checkpoints/justified") + assert response.status_code == 200 + + +def test_justified_checkpoint_content_type_is_json(server_url: str) -> None: + """Justified checkpoint endpoint returns JSON content type.""" + response = httpx.get(f"{server_url}/lean/v0/checkpoints/justified") + content_type = response.headers.get("content-type", "") + assert "application/json" in content_type + + +def test_justified_checkpoint_has_slot(server_url: str) -> None: + """Justified checkpoint response has a slot field.""" + response = httpx.get(f"{server_url}/lean/v0/checkpoints/justified") + data = response.json() + + assert "slot" in data + assert isinstance(data["slot"], int) + assert data["slot"] >= 0 + + +def test_justified_checkpoint_has_root(server_url: str) -> None: + """Justified checkpoint response has a valid root field.""" + response = httpx.get(f"{server_url}/lean/v0/checkpoints/justified") + data = response.json() + + assert "root" in data + root = data["root"] + + # Root should be a 64-character hex string (32 bytes) + assert isinstance(root, str) + assert len(root) == 64 + + # Should be valid hex + int(root, 16) diff --git a/tests/api_conformance/test_metrics.py b/tests/api_conformance/test_metrics.py new file mode 100644 index 00000000..f8912fa6 --- /dev/null +++ b/tests/api_conformance/test_metrics.py @@ -0,0 +1,33 @@ +"""Tests for the metrics endpoint.""" + +import httpx + + +def test_metrics_returns_200(server_url: str) -> None: + """Metrics endpoint returns 200 status code.""" + response = httpx.get(f"{server_url}/metrics") + assert response.status_code == 200 + + +def test_metrics_content_type_is_text(server_url: str) -> None: + """Metrics endpoint returns text/plain content type.""" + response = httpx.get(f"{server_url}/metrics") + content_type = response.headers.get("content-type", "") + assert content_type.startswith("text/plain") + + +def test_metrics_contains_prometheus_format(server_url: str) -> None: + """Metrics endpoint returns Prometheus-format metrics.""" + response = httpx.get(f"{server_url}/metrics") + body = response.text + + # Prometheus format has lines like: + # - metric_name value + # - # HELP metric_name description + # - # TYPE metric_name type + + # Check for at least one metric line (non-comment, non-empty) + lines = body.strip().split("\n") + metric_lines = [line for line in lines if line and not line.startswith("#")] + + assert len(metric_lines) > 0, "Expected at least one metric line" diff --git a/tests/api_conformance/test_unknown_routes.py b/tests/api_conformance/test_unknown_routes.py new file mode 100644 index 00000000..7f07f880 --- /dev/null +++ b/tests/api_conformance/test_unknown_routes.py @@ -0,0 +1,15 @@ +"""Tests for unknown route handling.""" + +import httpx + + +def test_unknown_root_route_returns_404(server_url: str) -> None: + """Unknown route at root level returns 404.""" + response = httpx.get(f"{server_url}/unknown") + assert response.status_code == 404 + + +def test_unknown_api_route_returns_404(server_url: str) -> None: + """Unknown route under API namespace returns 404.""" + response = httpx.get(f"{server_url}/lean/v0/unknown") + assert response.status_code == 404 diff --git a/tests/lean_spec/subspecs/api/test_server.py b/tests/lean_spec/subspecs/api/test_server.py index 55c007d4..6099dc30 100644 --- a/tests/lean_spec/subspecs/api/test_server.py +++ b/tests/lean_spec/subspecs/api/test_server.py @@ -1,11 +1,19 @@ -"""Tests for the API server checkpoint sync functionality.""" +"""Tests for the API server implementation details. + +API contract tests (status codes, content types, response structure) are in +tests/api_conformance/ and run automatically with `uv run pytest`. + +These tests cover leanSpec-specific implementation details: +- Configuration behavior +- Store integration +- Error handling when store not initialized +- Client helper functions +""" from __future__ import annotations import asyncio -import httpx - from lean_spec.subspecs.api import ( ApiServer, ApiServerConfig, @@ -61,39 +69,12 @@ def test_store_getter_provides_access_to_store(self, base_store: Store) -> None: assert server.store is base_store -class TestHealthEndpoint: - """Tests for the /lean/v0/health endpoint behavior.""" - - def test_returns_healthy_status_json(self) -> None: - """Health endpoint returns JSON with healthy status.""" - - async def run_test() -> None: - config = ApiServerConfig(port=15052) - server = ApiServer(config=config) - - await server.start() - - try: - async with httpx.AsyncClient() as client: - response = await client.get("http://127.0.0.1:15052/lean/v0/health") - - assert response.status_code == 200 - data = response.json() - assert data["status"] == "healthy" - assert data["service"] == "lean-spec-api" - - finally: - server.stop() - await asyncio.sleep(0.1) - - asyncio.run(run_test()) - - class TestFinalizedStateEndpoint: - """Tests for the /lean/v0/states/finalized endpoint behavior.""" + """Tests for the /lean/v0/states/finalized endpoint error handling.""" def test_returns_503_when_store_not_initialized(self) -> None: """Endpoint returns 503 Service Unavailable when store is not set.""" + import httpx async def run_test() -> None: config = ApiServerConfig(port=15054) @@ -113,40 +94,13 @@ async def run_test() -> None: asyncio.run(run_test()) - def test_returns_ssz_state_when_store_available(self, base_store: Store) -> None: - """Endpoint returns SSZ-encoded state as octet-stream.""" - - async def run_test() -> None: - config = ApiServerConfig(port=15056) - server = ApiServer(config=config, store_getter=lambda: base_store) - - await server.start() - - try: - async with httpx.AsyncClient() as client: - response = await client.get("http://127.0.0.1:15056/lean/v0/states/finalized") - - assert response.status_code == 200 - assert response.headers["content-type"] == "application/octet-stream" - - # Verify SSZ data can be deserialized - ssz_data = response.content - assert len(ssz_data) > 0 - state = State.decode_bytes(ssz_data) - assert state.slot == Slot(0) - - finally: - server.stop() - await asyncio.sleep(0.1) - - asyncio.run(run_test()) - class TestJustifiedCheckpointEndpoint: - """Tests for the /lean/v0/checkpoints/justified endpoint behavior.""" + """Tests for the /lean/v0/checkpoints/justified endpoint error handling.""" def test_returns_503_when_store_not_initialized(self) -> None: """Endpoint returns 503 Service Unavailable when store is not set.""" + import httpx async def run_test() -> None: config = ApiServerConfig(port=15057) @@ -168,68 +122,6 @@ async def run_test() -> None: asyncio.run(run_test()) - def test_returns_json_with_justified_checkpoint(self, base_store: Store) -> None: - """Endpoint returns JSON with latest justified checkpoint information.""" - - async def run_test() -> None: - config = ApiServerConfig(port=15058) - server = ApiServer(config=config, store_getter=lambda: base_store) - - await server.start() - - try: - async with httpx.AsyncClient() as client: - response = await client.get( - "http://127.0.0.1:15058/lean/v0/checkpoints/justified" - ) - - assert response.status_code == 200 - assert "application/json" in response.headers["content-type"] - - # Verify JSON structure and types - data = response.json() - assert "slot" in data - assert "root" in data - assert isinstance(data["slot"], int) - assert isinstance(data["root"], str) - # Root should be a hex string - assert len(data["root"]) == 64 # 32 bytes * 2 hex chars - - # Verify actual values match the store's latest justified checkpoint - assert data["slot"] == int(base_store.latest_justified.slot) - assert data["root"] == base_store.latest_justified.root.hex() - - finally: - server.stop() - await asyncio.sleep(0.1) - - asyncio.run(run_test()) - - -class TestUnknownEndpoints: - """Tests for unknown endpoint handling.""" - - def test_returns_404_for_unknown_path(self) -> None: - """Unknown paths return 404 Not Found.""" - - async def run_test() -> None: - config = ApiServerConfig(port=15053) - server = ApiServer(config=config) - - await server.start() - - try: - async with httpx.AsyncClient() as client: - response = await client.get("http://127.0.0.1:15053/unknown") - - assert response.status_code == 404 - - finally: - server.stop() - await asyncio.sleep(0.1) - - asyncio.run(run_test()) - class TestStateVerification: """Tests for checkpoint state verification logic.""" From 7691b0deed770372fa893f331e9741ec823717a9 Mon Sep 17 00:00:00 2001 From: unnawut Date: Fri, 30 Jan 2026 17:04:14 +0700 Subject: [PATCH 2/6] fix: remove duplicate --server-url option causing apitest CLI conflict --- .../cli/pytest_ini_files/pytest-apitest.ini | 3 +-- .../pytest_plugins/api_conformance.py | 26 ------------------- 2 files changed, 1 insertion(+), 28 deletions(-) delete mode 100644 packages/testing/src/framework/pytest_plugins/api_conformance.py diff --git a/packages/testing/src/framework/cli/pytest_ini_files/pytest-apitest.ini b/packages/testing/src/framework/cli/pytest_ini_files/pytest-apitest.ini index 5e17f587..eb6a66f2 100644 --- a/packages/testing/src/framework/cli/pytest_ini_files/pytest-apitest.ini +++ b/packages/testing/src/framework/cli/pytest_ini_files/pytest-apitest.ini @@ -4,9 +4,8 @@ # Search for API conformance tests testpaths = tests/api_conformance -# Load pytest plugins +# Options addopts = - -p framework.pytest_plugins.api_conformance # Show shorter tracebacks --tb=short # Disable coverage for API testing diff --git a/packages/testing/src/framework/pytest_plugins/api_conformance.py b/packages/testing/src/framework/pytest_plugins/api_conformance.py deleted file mode 100644 index 9fd28a34..00000000 --- a/packages/testing/src/framework/pytest_plugins/api_conformance.py +++ /dev/null @@ -1,26 +0,0 @@ -"""Pytest plugin for API conformance testing via the apitest CLI. - -This plugin is used by the apitest CLI command to pass the --server-url -option to pytest. The option is registered here so pytest can parse it. - -When running via `uv run pytest`, the conftest.py in tests/api_conformance -handles server startup and provides the server_url fixture directly. -""" - -import pytest - - -def pytest_addoption(parser: pytest.Parser) -> None: - """ - Register the --server-url option with pytest. - - The apitest CLI passes this option to pytest. - When running via regular pytest, the conftest.py handles this. - """ - group = parser.getgroup("apitest", "leanSpec API conformance testing") - group.addoption( - "--server-url", - action="store", - default=None, - help="Base URL of the API server to test", - ) From 07a11b74e887490994d9da6c5cd11edc9d962b4a Mon Sep 17 00:00:00 2001 From: unnawut Date: Fri, 30 Jan 2026 17:20:27 +0700 Subject: [PATCH 3/6] align with ream impl --- src/lean_spec/subspecs/api/server.py | 4 +- tests/api_conformance/test_finalized_state.py | 68 ++++++------------- tests/api_conformance/test_health.py | 2 +- .../test_justified_checkpoint.py | 5 +- tests/api_conformance/test_metrics.py | 33 --------- 5 files changed, 28 insertions(+), 84 deletions(-) delete mode 100644 tests/api_conformance/test_metrics.py diff --git a/src/lean_spec/subspecs/api/server.py b/src/lean_spec/subspecs/api/server.py index a486f17d..6e8d80e5 100644 --- a/src/lean_spec/subspecs/api/server.py +++ b/src/lean_spec/subspecs/api/server.py @@ -35,7 +35,7 @@ def _no_store() -> Store | None: async def _handle_health(_request: web.Request) -> web.Response: """Handle health check endpoint.""" - return web.json_response({"status": "healthy", "service": "lean-spec-api"}) + return web.json_response({"status": "healthy", "service": "lean-rpc-api"}) async def _handle_metrics(_request: web.Request) -> web.Response: @@ -189,6 +189,6 @@ async def _handle_justified_checkpoint(self, _request: web.Request) -> web.Respo return web.json_response( { "slot": justified.slot, - "root": justified.root.hex(), + "root": "0x" + justified.root.hex(), } ) diff --git a/tests/api_conformance/test_finalized_state.py b/tests/api_conformance/test_finalized_state.py index c0e68648..1ada4d47 100644 --- a/tests/api_conformance/test_finalized_state.py +++ b/tests/api_conformance/test_finalized_state.py @@ -1,4 +1,4 @@ -"""Tests for the finalized state endpoint with deep SSZ validation.""" +"""Tests for the finalized state endpoint with SSZ validation.""" import httpx @@ -7,72 +7,48 @@ def test_finalized_state_returns_200(server_url: str) -> None: """Finalized state endpoint returns 200 status code.""" - response = httpx.get(f"{server_url}/lean/v0/states/finalized") + response = httpx.get( + f"{server_url}/lean/v0/states/finalized", + headers={"Accept": "application/octet-stream"}, + ) assert response.status_code == 200 def test_finalized_state_content_type_is_octet_stream(server_url: str) -> None: """Finalized state endpoint returns octet-stream content type.""" - response = httpx.get(f"{server_url}/lean/v0/states/finalized") + response = httpx.get( + f"{server_url}/lean/v0/states/finalized", + headers={"Accept": "application/octet-stream"}, + ) content_type = response.headers.get("content-type", "") assert "application/octet-stream" in content_type def test_finalized_state_ssz_deserializes(server_url: str) -> None: """Finalized state SSZ bytes deserialize to a valid State object.""" - response = httpx.get(f"{server_url}/lean/v0/states/finalized") - ssz_bytes = response.content - - # This will raise if the bytes are not valid SSZ for State - state = State.decode_bytes(ssz_bytes) - - # Basic structural validation + response = httpx.get( + f"{server_url}/lean/v0/states/finalized", + headers={"Accept": "application/octet-stream"}, + ) + state = State.decode_bytes(response.content) assert state is not None def test_finalized_state_has_valid_slot(server_url: str) -> None: """Finalized state has a non-negative slot.""" - response = httpx.get(f"{server_url}/lean/v0/states/finalized") + response = httpx.get( + f"{server_url}/lean/v0/states/finalized", + headers={"Accept": "application/octet-stream"}, + ) state = State.decode_bytes(response.content) - assert int(state.slot) >= 0 def test_finalized_state_has_validators(server_url: str) -> None: """Finalized state has at least one validator.""" - response = httpx.get(f"{server_url}/lean/v0/states/finalized") + response = httpx.get( + f"{server_url}/lean/v0/states/finalized", + headers={"Accept": "application/octet-stream"}, + ) state = State.decode_bytes(response.content) - assert len(state.validators) > 0 - - -def test_finalized_state_has_valid_config(server_url: str) -> None: - """Finalized state has a valid config with genesis time.""" - response = httpx.get(f"{server_url}/lean/v0/states/finalized") - state = State.decode_bytes(response.content) - - # Genesis time should be a positive timestamp - assert int(state.config.genesis_time) >= 0 - - -def test_finalized_state_has_valid_checkpoints(server_url: str) -> None: - """Finalized state has valid justified and finalized checkpoints.""" - response = httpx.get(f"{server_url}/lean/v0/states/finalized") - state = State.decode_bytes(response.content) - - # Checkpoints should have valid slots - assert int(state.latest_justified.slot) >= 0 - assert int(state.latest_finalized.slot) >= 0 - - # Finalized slot should not exceed justified slot - assert state.latest_finalized.slot <= state.latest_justified.slot - - -def test_finalized_state_has_valid_block_header(server_url: str) -> None: - """Finalized state has a valid latest block header.""" - response = httpx.get(f"{server_url}/lean/v0/states/finalized") - state = State.decode_bytes(response.content) - - # Block header should have valid slot and proposer index - assert int(state.latest_block_header.slot) >= 0 - assert int(state.latest_block_header.proposer_index) >= 0 diff --git a/tests/api_conformance/test_health.py b/tests/api_conformance/test_health.py index 23912a48..a524fb2f 100644 --- a/tests/api_conformance/test_health.py +++ b/tests/api_conformance/test_health.py @@ -25,4 +25,4 @@ def test_health_response_structure(server_url: str) -> None: assert data["status"] == "healthy" assert "service" in data - assert data["service"] == "lean-spec-api" + assert data["service"] == "lean-rpc-api" diff --git a/tests/api_conformance/test_justified_checkpoint.py b/tests/api_conformance/test_justified_checkpoint.py index b8d34c09..578e64b1 100644 --- a/tests/api_conformance/test_justified_checkpoint.py +++ b/tests/api_conformance/test_justified_checkpoint.py @@ -34,9 +34,10 @@ def test_justified_checkpoint_has_root(server_url: str) -> None: assert "root" in data root = data["root"] - # Root should be a 64-character hex string (32 bytes) + # Root should be a 0x-prefixed hex string (32 bytes = 66 chars with prefix) assert isinstance(root, str) - assert len(root) == 64 + assert root.startswith("0x"), "Root must have 0x prefix" + assert len(root) == 66 # Should be valid hex int(root, 16) diff --git a/tests/api_conformance/test_metrics.py b/tests/api_conformance/test_metrics.py deleted file mode 100644 index f8912fa6..00000000 --- a/tests/api_conformance/test_metrics.py +++ /dev/null @@ -1,33 +0,0 @@ -"""Tests for the metrics endpoint.""" - -import httpx - - -def test_metrics_returns_200(server_url: str) -> None: - """Metrics endpoint returns 200 status code.""" - response = httpx.get(f"{server_url}/metrics") - assert response.status_code == 200 - - -def test_metrics_content_type_is_text(server_url: str) -> None: - """Metrics endpoint returns text/plain content type.""" - response = httpx.get(f"{server_url}/metrics") - content_type = response.headers.get("content-type", "") - assert content_type.startswith("text/plain") - - -def test_metrics_contains_prometheus_format(server_url: str) -> None: - """Metrics endpoint returns Prometheus-format metrics.""" - response = httpx.get(f"{server_url}/metrics") - body = response.text - - # Prometheus format has lines like: - # - metric_name value - # - # HELP metric_name description - # - # TYPE metric_name type - - # Check for at least one metric line (non-comment, non-empty) - lines = body.strip().split("\n") - metric_lines = [line for line in lines if line and not line.startswith("#")] - - assert len(metric_lines) > 0, "Expected at least one metric line" From 78cae04e785bc6e9134a8493e2a6c7e73b8f82bf Mon Sep 17 00:00:00 2001 From: unnawut Date: Fri, 30 Jan 2026 17:27:38 +0700 Subject: [PATCH 4/6] inline unnecessary _no_store() --- src/lean_spec/subspecs/api/server.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/src/lean_spec/subspecs/api/server.py b/src/lean_spec/subspecs/api/server.py index 6e8d80e5..dbfca759 100644 --- a/src/lean_spec/subspecs/api/server.py +++ b/src/lean_spec/subspecs/api/server.py @@ -28,11 +28,6 @@ logger = logging.getLogger(__name__) -def _no_store() -> Store | None: - """Default store getter that returns None.""" - return None - - async def _handle_health(_request: web.Request) -> web.Response: """Handle health check endpoint.""" return web.json_response({"status": "healthy", "service": "lean-rpc-api"}) @@ -76,7 +71,7 @@ class ApiServer: config: ApiServerConfig """Server configuration.""" - store_getter: Callable[[], Store | None] = _no_store + store_getter: Callable[[], Store | None] | None = None """Callable that returns the current Store instance.""" _runner: web.AppRunner | None = field(default=None, init=False) @@ -88,7 +83,7 @@ class ApiServer: @property def store(self) -> Store | None: """Get the current Store instance.""" - return self.store_getter() + return self.store_getter() if self.store_getter else None async def start(self) -> None: """Start the API server in the background.""" From 2b72f5181b89ac364831e87aa7cfdc1ea6a26426 Mon Sep 17 00:00:00 2001 From: unnawut Date: Fri, 30 Jan 2026 17:32:41 +0700 Subject: [PATCH 5/6] add API docs --- src/lean_spec/subspecs/api/server.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/lean_spec/subspecs/api/server.py b/src/lean_spec/subspecs/api/server.py index dbfca759..c5e051e5 100644 --- a/src/lean_spec/subspecs/api/server.py +++ b/src/lean_spec/subspecs/api/server.py @@ -29,7 +29,13 @@ async def _handle_health(_request: web.Request) -> web.Response: - """Handle health check endpoint.""" + """ + Handle health check endpoint. + + Response format: + - status: The status of the API server. Always return "healthy" when the API endpoint is served. + - service: The API service name. Fixed to "lean-rpc-api". + """ return web.json_response({"status": "healthy", "service": "lean-rpc-api"}) From 267cd3c86e11ccb3d89f60d8bf00cd6bc9b0d8f4 Mon Sep 17 00:00:00 2001 From: unnawut Date: Fri, 30 Jan 2026 18:04:49 +0700 Subject: [PATCH 6/6] rename api_conformance to api for conciseness --- packages/testing/src/framework/cli/apitest.py | 4 ++-- .../src/framework/cli/pytest_ini_files/pytest-apitest.ini | 2 +- tests/{api_conformance => api}/__init__.py | 0 tests/{api_conformance => api}/conftest.py | 0 tests/{api_conformance => api}/test_finalized_state.py | 0 tests/{api_conformance => api}/test_health.py | 0 tests/{api_conformance => api}/test_justified_checkpoint.py | 0 tests/{api_conformance => api}/test_unknown_routes.py | 0 tests/lean_spec/subspecs/api/test_server.py | 2 +- 9 files changed, 4 insertions(+), 4 deletions(-) rename tests/{api_conformance => api}/__init__.py (100%) rename tests/{api_conformance => api}/conftest.py (100%) rename tests/{api_conformance => api}/test_finalized_state.py (100%) rename tests/{api_conformance => api}/test_health.py (100%) rename tests/{api_conformance => api}/test_justified_checkpoint.py (100%) rename tests/{api_conformance => api}/test_unknown_routes.py (100%) diff --git a/packages/testing/src/framework/cli/apitest.py b/packages/testing/src/framework/cli/apitest.py index 014db68b..c43bbcff 100644 --- a/packages/testing/src/framework/cli/apitest.py +++ b/packages/testing/src/framework/cli/apitest.py @@ -27,7 +27,7 @@ def apitest( SERVER_URL is the base URL of the API server (e.g., http://localhost:5052). - For testing the local leanSpec implementation, use `uv run pytest tests/api_conformance` + For testing the local leanSpec implementation, use `uv run pytest tests/api` which automatically starts a local server. Examples: @@ -57,7 +57,7 @@ def apitest( str(config_path), f"--rootdir={project_root}", f"--server-url={server_url}", - "tests/api_conformance", + "tests/api", ] args.extend(pytest_args) diff --git a/packages/testing/src/framework/cli/pytest_ini_files/pytest-apitest.ini b/packages/testing/src/framework/cli/pytest_ini_files/pytest-apitest.ini index eb6a66f2..27afb374 100644 --- a/packages/testing/src/framework/cli/pytest_ini_files/pytest-apitest.ini +++ b/packages/testing/src/framework/cli/pytest_ini_files/pytest-apitest.ini @@ -2,7 +2,7 @@ # Configuration for apitest command # Search for API conformance tests -testpaths = tests/api_conformance +testpaths = tests/api # Options addopts = diff --git a/tests/api_conformance/__init__.py b/tests/api/__init__.py similarity index 100% rename from tests/api_conformance/__init__.py rename to tests/api/__init__.py diff --git a/tests/api_conformance/conftest.py b/tests/api/conftest.py similarity index 100% rename from tests/api_conformance/conftest.py rename to tests/api/conftest.py diff --git a/tests/api_conformance/test_finalized_state.py b/tests/api/test_finalized_state.py similarity index 100% rename from tests/api_conformance/test_finalized_state.py rename to tests/api/test_finalized_state.py diff --git a/tests/api_conformance/test_health.py b/tests/api/test_health.py similarity index 100% rename from tests/api_conformance/test_health.py rename to tests/api/test_health.py diff --git a/tests/api_conformance/test_justified_checkpoint.py b/tests/api/test_justified_checkpoint.py similarity index 100% rename from tests/api_conformance/test_justified_checkpoint.py rename to tests/api/test_justified_checkpoint.py diff --git a/tests/api_conformance/test_unknown_routes.py b/tests/api/test_unknown_routes.py similarity index 100% rename from tests/api_conformance/test_unknown_routes.py rename to tests/api/test_unknown_routes.py diff --git a/tests/lean_spec/subspecs/api/test_server.py b/tests/lean_spec/subspecs/api/test_server.py index 6099dc30..bdbc42a9 100644 --- a/tests/lean_spec/subspecs/api/test_server.py +++ b/tests/lean_spec/subspecs/api/test_server.py @@ -1,7 +1,7 @@ """Tests for the API server implementation details. API contract tests (status codes, content types, response structure) are in -tests/api_conformance/ and run automatically with `uv run pytest`. +tests/api/ and also run automatically with `uv run pytest`. These tests cover leanSpec-specific implementation details: - Configuration behavior