fix(Monitoring): Serve /metrics on a dedicated port

src/common/gunicorn/conf.py

@@ -4,6 +4,7 @@
 It is used to correctly support Prometheus metrics in a multi-process environment.
 """
 
+import os
 import typing
 
 from prometheus_client.multiprocess import mark_process_dead
@@ -13,6 +14,15 @@
     from gunicorn.workers.base import Worker  # type: ignore[import-untyped]
 
 
-def worker_exit(server: "Arbiter", worker: "Worker") -> None:
+def when_ready(server: "Arbiter") -> None:
+    """Start the standalone Prometheus metrics server after Gunicorn is ready."""
+    prometheus_enabled = os.getenv("PROMETHEUS_ENABLED", "")
+    if prometheus_enabled.lower() == "true":  # Django settings are not available
+        from common.gunicorn.metrics_server import start_metrics_server
+
+        start_metrics_server()
+
+
+def child_exit(server: "Arbiter", worker: "Worker") -> None:
     """Detach the process Prometheus metrics collector when a worker exits."""
     mark_process_dead(worker.pid)  # type: ignore[no-untyped-call]

src/common/gunicorn/metrics_server.py

@@ -0,0 +1,65 @@
+"""
+Standalone Prometheus metrics HTTP server.
+
+This module provides a separate HTTP server for Prometheus metrics,
+independent of the main Gunicorn application server. This improves
+metrics reliability under high API load.
+
+The server runs in a daemon thread and serves metrics from the shared
+PROMETHEUS_MULTIPROC_DIR directory.
+"""
+
+import logging
+import os
+import threading
+
+from prometheus_client import CollectorRegistry, start_http_server
+from prometheus_client.multiprocess import MultiProcessCollector
+
+logger = logging.getLogger(__name__)
+
+METRICS_SERVER_PORT = 9100
+
+_server_started = False
+_server_lock = threading.Lock()
+
+
+def get_multiprocess_registry() -> CollectorRegistry:
+    """Create a registry configured for multiprocess metric collection."""
+    registry = CollectorRegistry()
+    MultiProcessCollector(registry)  # type: ignore[no-untyped-call]
+    return registry
+
+
+def start_metrics_server(
+    port: int = METRICS_SERVER_PORT,
+) -> None:
+    """
+    Start the standalone Prometheus metrics HTTP server.
+
+    This function is idempotent - calling it multiple times will only
+    start one server. The server runs in a daemon thread.
+
+    Args:
+        port: The port to serve metrics on. Defaults to 9100.
+    """
+    global _server_started
+
+    with _server_lock:
+        if _server_started:
+            logger.debug("Metrics server already started")
+            return
+
+        prometheus_multiproc_dir = os.environ.get("PROMETHEUS_MULTIPROC_DIR")
+        if not prometheus_multiproc_dir:
+            logger.warning("PROMETHEUS_MULTIPROC_DIR not set, skipping metrics server")
+            return
+
+        registry = get_multiprocess_registry()
+
+        try:
+            start_http_server(port=port, registry=registry)
+            _server_started = True
+            logger.info("Prometheus metrics server started on port %d", port)
+        except OSError as e:
+            logger.error("Failed to start metrics server on port %d: %s", port, e)

tests/integration/gunicorn/conftest.py

@@ -0,0 +1,15 @@
+from typing import Generator
+
+import pytest
+
+
+@pytest.fixture(autouse=True)
+def reset_metrics_server_state() -> Generator[None, None, None]:
+    """Reset the metrics server global state between tests."""
+    from common.gunicorn import metrics_server
+
+    metrics_server._server_started = False
+
+    yield
+
+    metrics_server._server_started = False

tests/integration/gunicorn/test_metrics_server.py

@@ -0,0 +1,119 @@
+import socket
+import urllib.request
+
+import prometheus_client
+import pytest
+from rest_framework.test import APIClient
+
+from common.gunicorn.metrics_server import start_metrics_server
+from tests import GetLogsFixture
+
+
+@pytest.mark.prometheus_multiprocess_mode
+def test_start_metrics_server__multiprocess_mode__serves_metrics(
+    unused_tcp_port: int,
+    test_metric: prometheus_client.Counter,
+) -> None:
+    # Given
+    test_metric.labels(test_name="standalone_server_test").inc()
+
+    # When
+    start_metrics_server(port=unused_tcp_port)
+
+    # Then
+    with urllib.request.urlopen(
+        f"http://localhost:{unused_tcp_port}/metrics"
+    ) as response:
+        content = response.read().decode()
+
+    assert response.status == 200
+    assert "pytest_tests_run_total" in content
+    assert 'test_name="standalone_server_test"' in content
+
+
+# NOTE: This test is temporary. Remove once the Django /metrics endpoint is
+# deprecated in favour of the standalone metrics server.
+@pytest.mark.prometheus_multiprocess_mode
+def test_start_metrics_server__multiprocess_mode__output_matches_django_view(
+    unused_tcp_port: int,
+    test_metric: prometheus_client.Counter,
+    client: APIClient,
+) -> None:
+    # Given
+    test_metric.labels(test_name="equivalence_test").inc()
+    start_metrics_server(port=unused_tcp_port)
+
+    # When
+    with urllib.request.urlopen(
+        f"http://localhost:{unused_tcp_port}/metrics"
+    ) as response:
+        standalone_content = response.read().decode()
+
+    django_response = client.get("/metrics", follow=True)
+    django_content = django_response.content.decode()
+
+    # Then
+    assert "pytest_tests_run_total" in standalone_content
+    assert "pytest_tests_run_total" in django_content
+    assert 'test_name="equivalence_test"' in standalone_content
+    assert 'test_name="equivalence_test"' in django_content
+
+
+def test_start_metrics_server__multiproc_dir_unset__logs_warning_and_skips(
+    get_logs: GetLogsFixture,
+) -> None:
+    # Given
+    # PROMETHEUS_MULTIPROC_DIR is not set (default state)
+
+    # When
+    start_metrics_server()
+
+    # Then
+    logs = get_logs("common.gunicorn.metrics_server")
+    assert (
+        "WARNING",
+        "PROMETHEUS_MULTIPROC_DIR not set, skipping metrics server",
+    ) in logs
+
+
+@pytest.mark.prometheus_multiprocess_mode
+def test_start_metrics_server__called_multiple_times__remains_idempotent(
+    unused_tcp_port: int,
+) -> None:
+    # Given
+    start_metrics_server(port=unused_tcp_port)
+
+    # When
+    start_metrics_server(port=unused_tcp_port)
+    start_metrics_server(port=unused_tcp_port)
+
+    # Then
+    with urllib.request.urlopen(
+        f"http://localhost:{unused_tcp_port}/metrics"
+    ) as response:
+        assert response.status == 200
+
+
+@pytest.mark.prometheus_multiprocess_mode
+def test_start_metrics_server__port_unavailable__logs_error(
+    unused_tcp_port: int,
+    get_logs: GetLogsFixture,
+) -> None:
+    # Given
+    # Bind to 0.0.0.0 to match prometheus_client's default address
+    blocker = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+    blocker.bind(("0.0.0.0", unused_tcp_port))
+    blocker.listen(1)
+
+    try:
+        # When
+        start_metrics_server(port=unused_tcp_port)
+
+        # Then
+        logs = get_logs("common.gunicorn.metrics_server")
+        assert any(
+            level == "ERROR" and "Failed to start metrics server" in msg
+            for level, msg in logs
+        )
+    finally:
+        blocker.close()

tests/unit/common/gunicorn/test_conf.py

@@ -0,0 +1,21 @@
+from unittest.mock import Mock
+
+from pytest_mock import MockerFixture
+
+from common.gunicorn.conf import child_exit
+
+
+def test_child_exit__calls_mark_process_dead_with_worker_pid(
+    mocker: MockerFixture,
+) -> None:
+    # Given
+    mark_process_dead_mock = mocker.patch("common.gunicorn.conf.mark_process_dead")
+    server = Mock()
+    worker = Mock()
+    worker.pid = 12345
+
+    # When
+    child_exit(server, worker)
+
+    # Then
+    mark_process_dead_mock.assert_called_once_with(12345)

tests/unit/common/gunicorn/test_utils.py

@@ -44,7 +44,6 @@ def test_run_server__default_config_file__runs_expected(
     # Given
     # prevent real forking from Gunicorn
     mocker.patch("os.fork").return_value = 0
-    mark_process_dead_mock = mocker.patch("common.gunicorn.conf.mark_process_dead")
 
     pid = os.getpid()
 
@@ -58,9 +57,6 @@ def delay_kill(pid: int = pid) -> None:
     with pytest.raises(SystemExit):
         run_server({"bind": f"0.0.0.0:{unused_tcp_port}"})
 
-    # Then
-    mark_process_dead_mock.assert_called_once_with(pid)
-
 
 def test_get_route_template__returns_expected__caches_expected(
     mocker: MockerFixture,