diff --git a/README.md b/README.md index 90bbccac..15ca0c97 100644 --- a/README.md +++ b/README.md @@ -115,6 +115,10 @@ uv run pytest -m "not slow" # --fork Target fork (default: devnet) # --output Optional directory for filled fixtures uv run fill --clean --fork=devnet + +# Run API conformance tests against an external client implementation +# Usage: uv run apitest [pytest-args] +uv run apitest http://localhost:5052 ``` ### Code Quality @@ -221,6 +225,7 @@ def test_withdrawal_amount_above_uint64_max(): | Serve docs | `uv run mkdocs serve` | | Run everything (checks + tests + docs) | `uvx tox` | | Run all quality checks (no tests/docs) | `uvx tox -e all-checks` | +| Test external client API conformance | `uv run apitest http://localhost:5052` | | Run consensus node | `uv run python -m lean_spec --genesis config.yaml` | | Build Docker test image | `docker build -t lean-spec:test .` | | Build Docker node image | `docker build --target node -t lean-spec:node .` | diff --git a/packages/testing/pyproject.toml b/packages/testing/pyproject.toml index 821b2b24..9b2dda5e 100644 --- a/packages/testing/pyproject.toml +++ b/packages/testing/pyproject.toml @@ -35,6 +35,7 @@ Issues = "https://github.com/leanEthereum/lean-spec/issues" [project.scripts] fill = "framework.cli.fill:fill" +apitest = "framework.cli.apitest:apitest" [tool.setuptools.packages.find] where = ["src"] diff --git a/packages/testing/src/framework/cli/apitest.py b/packages/testing/src/framework/cli/apitest.py new file mode 100644 index 00000000..c43bbcff --- /dev/null +++ b/packages/testing/src/framework/cli/apitest.py @@ -0,0 +1,71 @@ +"""CLI command for running API conformance tests against an external server.""" + +import sys +from pathlib import Path +from typing import Sequence + +import click +import pytest + + +@click.command( + context_settings={ + "ignore_unknown_options": True, + "allow_extra_args": True, + } +) +@click.argument("server_url") +@click.argument("pytest_args", nargs=-1, type=click.UNPROCESSED) +@click.pass_context +def apitest( + ctx: click.Context, + server_url: str, + pytest_args: Sequence[str], +) -> None: + """ + Run API conformance tests against an external server, i.e. a client implementation. + + SERVER_URL is the base URL of the API server (e.g., http://localhost:5052). + + For testing the local leanSpec implementation, use `uv run pytest tests/api` + which automatically starts a local server. + + Examples: + # Run against external server + apitest http://localhost:5052 + + # Run with verbose output + apitest http://localhost:5052 -v + + # Run specific test + apitest http://localhost:5052 -k test_health + """ + config_path = Path(__file__).parent / "pytest_ini_files" / "pytest-apitest.ini" + + # Find project root + project_root = Path.cwd() + while project_root != project_root.parent: + if (project_root / "pyproject.toml").exists(): + pyproject = project_root / "pyproject.toml" + if "[tool.uv.workspace]" in pyproject.read_text(): + break + project_root = project_root.parent + + # Build pytest arguments + args = [ + "-c", + str(config_path), + f"--rootdir={project_root}", + f"--server-url={server_url}", + "tests/api", + ] + + args.extend(pytest_args) + args.extend(ctx.args) + + exit_code = pytest.main(args) + sys.exit(exit_code) + + +if __name__ == "__main__": + apitest() diff --git a/packages/testing/src/framework/cli/pytest_ini_files/pytest-apitest.ini b/packages/testing/src/framework/cli/pytest_ini_files/pytest-apitest.ini new file mode 100644 index 00000000..27afb374 --- /dev/null +++ b/packages/testing/src/framework/cli/pytest_ini_files/pytest-apitest.ini @@ -0,0 +1,20 @@ +[pytest] +# Configuration for apitest command + +# Search for API conformance tests +testpaths = tests/api + +# Options +addopts = + # Show shorter tracebacks + --tb=short + # Disable coverage for API testing + --no-cov + +# Test discovery +python_files = test_*.py +python_classes = Test* +python_functions = test_* + +# Minimal output +console_output_style = classic diff --git a/src/lean_spec/subspecs/api/server.py b/src/lean_spec/subspecs/api/server.py index 70996f62..c5e051e5 100644 --- a/src/lean_spec/subspecs/api/server.py +++ b/src/lean_spec/subspecs/api/server.py @@ -28,21 +28,23 @@ logger = logging.getLogger(__name__) -def _no_store() -> Store | None: - """Default store getter that returns None.""" - return None - - async def _handle_health(_request: web.Request) -> web.Response: - """Handle health check endpoint.""" - return web.json_response({"status": "healthy", "service": "lean-spec-api"}) + """ + Handle health check endpoint. + + Response format: + - status: The status of the API server. Always return "healthy" when the API endpoint is served. + - service: The API service name. Fixed to "lean-rpc-api". + """ + return web.json_response({"status": "healthy", "service": "lean-rpc-api"}) async def _handle_metrics(_request: web.Request) -> web.Response: """Handle Prometheus metrics endpoint.""" return web.Response( body=generate_metrics(), - content_type="text/plain; version=0.0.4; charset=utf-8", + content_type="text/plain; version=0.0.4", + charset="utf-8", ) @@ -75,7 +77,7 @@ class ApiServer: config: ApiServerConfig """Server configuration.""" - store_getter: Callable[[], Store | None] = _no_store + store_getter: Callable[[], Store | None] | None = None """Callable that returns the current Store instance.""" _runner: web.AppRunner | None = field(default=None, init=False) @@ -87,7 +89,7 @@ class ApiServer: @property def store(self) -> Store | None: """Get the current Store instance.""" - return self.store_getter() + return self.store_getter() if self.store_getter else None async def start(self) -> None: """Start the API server in the background.""" @@ -188,6 +190,6 @@ async def _handle_justified_checkpoint(self, _request: web.Request) -> web.Respo return web.json_response( { "slot": justified.slot, - "root": justified.root.hex(), + "root": "0x" + justified.root.hex(), } ) diff --git a/tests/api/__init__.py b/tests/api/__init__.py new file mode 100644 index 00000000..895b275a --- /dev/null +++ b/tests/api/__init__.py @@ -0,0 +1 @@ +"""API conformance test suite for validating leanSpec client implementations.""" diff --git a/tests/api/conftest.py b/tests/api/conftest.py new file mode 100644 index 00000000..b5903884 --- /dev/null +++ b/tests/api/conftest.py @@ -0,0 +1,145 @@ +"""Pytest configuration for API conformance tests.""" + +import asyncio +import threading +import time +from typing import TYPE_CHECKING, Generator + +import httpx +import pytest + +if TYPE_CHECKING: + from lean_spec.subspecs.api import ApiServer + +# Default port for auto-started local server +DEFAULT_PORT = 15099 + + +class _ServerThread(threading.Thread): + """Thread that runs the API server in its own event loop.""" + + def __init__(self, port: int): + super().__init__(daemon=True) + self.port = port + self.server: ApiServer | None = None + self.loop: asyncio.AbstractEventLoop | None = None + self.ready = threading.Event() + self.error: Exception | None = None + + def run(self) -> None: + """Run the server in a new event loop.""" + try: + self.loop = asyncio.new_event_loop() + asyncio.set_event_loop(self.loop) + + self.server = self._create_server() + self.loop.run_until_complete(self.server.start()) + self.ready.set() + + self.loop.run_forever() + + except Exception as e: + self.error = e + self.ready.set() + finally: + if self.loop: + self.loop.close() + + def _create_server(self) -> "ApiServer": + """Create the API server with a test store.""" + from lean_spec.subspecs.api import ApiServer, ApiServerConfig + from lean_spec.subspecs.containers import Block, BlockBody, State, Validator + from lean_spec.subspecs.containers.block.types import AggregatedAttestations + from lean_spec.subspecs.containers.slot import Slot + from lean_spec.subspecs.containers.state import Validators + from lean_spec.subspecs.containers.validator import ValidatorIndex + from lean_spec.subspecs.forkchoice import Store + from lean_spec.subspecs.ssz.hash import hash_tree_root + from lean_spec.types import Bytes32, Bytes52, Uint64 + + validators = Validators( + data=[ + Validator(pubkey=Bytes52(b"\x00" * 52), index=ValidatorIndex(i)) for i in range(3) + ] + ) + + genesis_state = State.generate_genesis( + genesis_time=Uint64(int(time.time())), + validators=validators, + ) + + genesis_block = Block( + slot=Slot(0), + proposer_index=ValidatorIndex(0), + parent_root=Bytes32.zero(), + state_root=hash_tree_root(genesis_state), + body=BlockBody(attestations=AggregatedAttestations(data=[])), + ) + + store = Store.get_forkchoice_store(genesis_state, genesis_block) + + config = ApiServerConfig(host="127.0.0.1", port=self.port) + return ApiServer(config=config, store_getter=lambda: store) + + def stop(self) -> None: + """Stop the server and event loop.""" + if self.server and self.loop: + self.loop.call_soon_threadsafe(self.server.stop) + self.loop.call_soon_threadsafe(self.loop.stop) + + +def _wait_for_server(url: str, timeout: float = 5.0) -> bool: + """Wait for server to be ready by polling the health endpoint.""" + start = time.time() + while time.time() - start < timeout: + try: + response = httpx.get(f"{url}/lean/v0/health", timeout=1.0) + if response.status_code == 200: + return True + except (httpx.ConnectError, httpx.ReadTimeout): + pass + time.sleep(0.1) + return False + + +def pytest_addoption(parser: pytest.Parser) -> None: + """Add --server-url option for testing against external servers.""" + parser.addoption( + "--server-url", + action="store", + default=None, + help="External server URL. If not provided, starts a local server.", + ) + + +@pytest.fixture(scope="session") +def server_url(request: pytest.FixtureRequest) -> Generator[str, None, None]: + """ + Provide the server URL for API tests. + + If --server-url is provided, uses that external server. + Otherwise, starts a local leanSpec server for the test session. + """ + external_url = request.config.getoption("--server-url") + + if external_url: + # Use external server + yield external_url + else: + # Start local server + server_thread = _ServerThread(DEFAULT_PORT) + server_thread.start() + server_thread.ready.wait(timeout=10.0) + + if server_thread.error: + pytest.fail(f"Failed to start local server: {server_thread.error}") + + url = f"http://127.0.0.1:{DEFAULT_PORT}" + + if not _wait_for_server(url): + server_thread.stop() + pytest.fail("Local server failed to become ready") + + yield url + + server_thread.stop() diff --git a/tests/api/test_finalized_state.py b/tests/api/test_finalized_state.py new file mode 100644 index 00000000..1ada4d47 --- /dev/null +++ b/tests/api/test_finalized_state.py @@ -0,0 +1,54 @@ +"""Tests for the finalized state endpoint with SSZ validation.""" + +import httpx + +from lean_spec.subspecs.containers import State + + +def test_finalized_state_returns_200(server_url: str) -> None: + """Finalized state endpoint returns 200 status code.""" + response = httpx.get( + f"{server_url}/lean/v0/states/finalized", + headers={"Accept": "application/octet-stream"}, + ) + assert response.status_code == 200 + + +def test_finalized_state_content_type_is_octet_stream(server_url: str) -> None: + """Finalized state endpoint returns octet-stream content type.""" + response = httpx.get( + f"{server_url}/lean/v0/states/finalized", + headers={"Accept": "application/octet-stream"}, + ) + content_type = response.headers.get("content-type", "") + assert "application/octet-stream" in content_type + + +def test_finalized_state_ssz_deserializes(server_url: str) -> None: + """Finalized state SSZ bytes deserialize to a valid State object.""" + response = httpx.get( + f"{server_url}/lean/v0/states/finalized", + headers={"Accept": "application/octet-stream"}, + ) + state = State.decode_bytes(response.content) + assert state is not None + + +def test_finalized_state_has_valid_slot(server_url: str) -> None: + """Finalized state has a non-negative slot.""" + response = httpx.get( + f"{server_url}/lean/v0/states/finalized", + headers={"Accept": "application/octet-stream"}, + ) + state = State.decode_bytes(response.content) + assert int(state.slot) >= 0 + + +def test_finalized_state_has_validators(server_url: str) -> None: + """Finalized state has at least one validator.""" + response = httpx.get( + f"{server_url}/lean/v0/states/finalized", + headers={"Accept": "application/octet-stream"}, + ) + state = State.decode_bytes(response.content) + assert len(state.validators) > 0 diff --git a/tests/api/test_health.py b/tests/api/test_health.py new file mode 100644 index 00000000..a524fb2f --- /dev/null +++ b/tests/api/test_health.py @@ -0,0 +1,28 @@ +"""Tests for the health endpoint.""" + +import httpx + + +def test_health_returns_200(server_url: str) -> None: + """Health endpoint returns 200 status code.""" + response = httpx.get(f"{server_url}/lean/v0/health") + assert response.status_code == 200 + + +def test_health_content_type_is_json(server_url: str) -> None: + """Health endpoint returns JSON content type.""" + response = httpx.get(f"{server_url}/lean/v0/health") + content_type = response.headers.get("content-type", "") + assert "application/json" in content_type + + +def test_health_response_structure(server_url: str) -> None: + """Health endpoint returns expected JSON structure.""" + response = httpx.get(f"{server_url}/lean/v0/health") + data = response.json() + + assert "status" in data + assert data["status"] == "healthy" + + assert "service" in data + assert data["service"] == "lean-rpc-api" diff --git a/tests/api/test_justified_checkpoint.py b/tests/api/test_justified_checkpoint.py new file mode 100644 index 00000000..578e64b1 --- /dev/null +++ b/tests/api/test_justified_checkpoint.py @@ -0,0 +1,43 @@ +"""Tests for the justified checkpoint endpoint.""" + +import httpx + + +def test_justified_checkpoint_returns_200(server_url: str) -> None: + """Justified checkpoint endpoint returns 200 status code.""" + response = httpx.get(f"{server_url}/lean/v0/checkpoints/justified") + assert response.status_code == 200 + + +def test_justified_checkpoint_content_type_is_json(server_url: str) -> None: + """Justified checkpoint endpoint returns JSON content type.""" + response = httpx.get(f"{server_url}/lean/v0/checkpoints/justified") + content_type = response.headers.get("content-type", "") + assert "application/json" in content_type + + +def test_justified_checkpoint_has_slot(server_url: str) -> None: + """Justified checkpoint response has a slot field.""" + response = httpx.get(f"{server_url}/lean/v0/checkpoints/justified") + data = response.json() + + assert "slot" in data + assert isinstance(data["slot"], int) + assert data["slot"] >= 0 + + +def test_justified_checkpoint_has_root(server_url: str) -> None: + """Justified checkpoint response has a valid root field.""" + response = httpx.get(f"{server_url}/lean/v0/checkpoints/justified") + data = response.json() + + assert "root" in data + root = data["root"] + + # Root should be a 0x-prefixed hex string (32 bytes = 66 chars with prefix) + assert isinstance(root, str) + assert root.startswith("0x"), "Root must have 0x prefix" + assert len(root) == 66 + + # Should be valid hex + int(root, 16) diff --git a/tests/api/test_unknown_routes.py b/tests/api/test_unknown_routes.py new file mode 100644 index 00000000..7f07f880 --- /dev/null +++ b/tests/api/test_unknown_routes.py @@ -0,0 +1,15 @@ +"""Tests for unknown route handling.""" + +import httpx + + +def test_unknown_root_route_returns_404(server_url: str) -> None: + """Unknown route at root level returns 404.""" + response = httpx.get(f"{server_url}/unknown") + assert response.status_code == 404 + + +def test_unknown_api_route_returns_404(server_url: str) -> None: + """Unknown route under API namespace returns 404.""" + response = httpx.get(f"{server_url}/lean/v0/unknown") + assert response.status_code == 404 diff --git a/tests/lean_spec/subspecs/api/test_server.py b/tests/lean_spec/subspecs/api/test_server.py index 55c007d4..bdbc42a9 100644 --- a/tests/lean_spec/subspecs/api/test_server.py +++ b/tests/lean_spec/subspecs/api/test_server.py @@ -1,11 +1,19 @@ -"""Tests for the API server checkpoint sync functionality.""" +"""Tests for the API server implementation details. + +API contract tests (status codes, content types, response structure) are in +tests/api/ and also run automatically with `uv run pytest`. + +These tests cover leanSpec-specific implementation details: +- Configuration behavior +- Store integration +- Error handling when store not initialized +- Client helper functions +""" from __future__ import annotations import asyncio -import httpx - from lean_spec.subspecs.api import ( ApiServer, ApiServerConfig, @@ -61,39 +69,12 @@ def test_store_getter_provides_access_to_store(self, base_store: Store) -> None: assert server.store is base_store -class TestHealthEndpoint: - """Tests for the /lean/v0/health endpoint behavior.""" - - def test_returns_healthy_status_json(self) -> None: - """Health endpoint returns JSON with healthy status.""" - - async def run_test() -> None: - config = ApiServerConfig(port=15052) - server = ApiServer(config=config) - - await server.start() - - try: - async with httpx.AsyncClient() as client: - response = await client.get("http://127.0.0.1:15052/lean/v0/health") - - assert response.status_code == 200 - data = response.json() - assert data["status"] == "healthy" - assert data["service"] == "lean-spec-api" - - finally: - server.stop() - await asyncio.sleep(0.1) - - asyncio.run(run_test()) - - class TestFinalizedStateEndpoint: - """Tests for the /lean/v0/states/finalized endpoint behavior.""" + """Tests for the /lean/v0/states/finalized endpoint error handling.""" def test_returns_503_when_store_not_initialized(self) -> None: """Endpoint returns 503 Service Unavailable when store is not set.""" + import httpx async def run_test() -> None: config = ApiServerConfig(port=15054) @@ -113,40 +94,13 @@ async def run_test() -> None: asyncio.run(run_test()) - def test_returns_ssz_state_when_store_available(self, base_store: Store) -> None: - """Endpoint returns SSZ-encoded state as octet-stream.""" - - async def run_test() -> None: - config = ApiServerConfig(port=15056) - server = ApiServer(config=config, store_getter=lambda: base_store) - - await server.start() - - try: - async with httpx.AsyncClient() as client: - response = await client.get("http://127.0.0.1:15056/lean/v0/states/finalized") - - assert response.status_code == 200 - assert response.headers["content-type"] == "application/octet-stream" - - # Verify SSZ data can be deserialized - ssz_data = response.content - assert len(ssz_data) > 0 - state = State.decode_bytes(ssz_data) - assert state.slot == Slot(0) - - finally: - server.stop() - await asyncio.sleep(0.1) - - asyncio.run(run_test()) - class TestJustifiedCheckpointEndpoint: - """Tests for the /lean/v0/checkpoints/justified endpoint behavior.""" + """Tests for the /lean/v0/checkpoints/justified endpoint error handling.""" def test_returns_503_when_store_not_initialized(self) -> None: """Endpoint returns 503 Service Unavailable when store is not set.""" + import httpx async def run_test() -> None: config = ApiServerConfig(port=15057) @@ -168,68 +122,6 @@ async def run_test() -> None: asyncio.run(run_test()) - def test_returns_json_with_justified_checkpoint(self, base_store: Store) -> None: - """Endpoint returns JSON with latest justified checkpoint information.""" - - async def run_test() -> None: - config = ApiServerConfig(port=15058) - server = ApiServer(config=config, store_getter=lambda: base_store) - - await server.start() - - try: - async with httpx.AsyncClient() as client: - response = await client.get( - "http://127.0.0.1:15058/lean/v0/checkpoints/justified" - ) - - assert response.status_code == 200 - assert "application/json" in response.headers["content-type"] - - # Verify JSON structure and types - data = response.json() - assert "slot" in data - assert "root" in data - assert isinstance(data["slot"], int) - assert isinstance(data["root"], str) - # Root should be a hex string - assert len(data["root"]) == 64 # 32 bytes * 2 hex chars - - # Verify actual values match the store's latest justified checkpoint - assert data["slot"] == int(base_store.latest_justified.slot) - assert data["root"] == base_store.latest_justified.root.hex() - - finally: - server.stop() - await asyncio.sleep(0.1) - - asyncio.run(run_test()) - - -class TestUnknownEndpoints: - """Tests for unknown endpoint handling.""" - - def test_returns_404_for_unknown_path(self) -> None: - """Unknown paths return 404 Not Found.""" - - async def run_test() -> None: - config = ApiServerConfig(port=15053) - server = ApiServer(config=config) - - await server.start() - - try: - async with httpx.AsyncClient() as client: - response = await client.get("http://127.0.0.1:15053/unknown") - - assert response.status_code == 404 - - finally: - server.stop() - await asyncio.sleep(0.1) - - asyncio.run(run_test()) - class TestStateVerification: """Tests for checkpoint state verification logic."""