From 65ab4b9666eb4fc56b096898e2935710db08662b Mon Sep 17 00:00:00 2001
From: Liang Yan <lyan@coreweave.com>
Date: Wed, 29 Jul 2026 15:27:06 -0400
Subject: [PATCH 01/14] fix(lcb-service): use the fork start method explicitly

Python 3.14 changed the default multiprocessing start method on Linux
from fork to forkserver. The grading pipeline (pool workers forking a
per-problem mp.Process + mp.Manager) only works with fork: under
forkserver the grading children die at startup, every result comes back
as an error, and execute_code_single_suppressed_errors turns that into
all-failed tests, so the service sits at 0/N forever.

Pin the fork context for the executor, the per-problem Process and its
Manager. Also raise if every subprocess reported an execution error --
that means the judge is broken, not that all samples failed -- and log
those errors at error level instead of warning.

Seen on a python 3.14 lcb-service image: 0/349 after 3.5h, one defunct
child per pool worker. Same inputs with fork forced: done in 6 min.
The repo pins 3.12 so CI won't hit this, but shipped images have.
---
 .../evaluation/livecodebench/lcb_serve.py     | 27 ++++++++++++++-----
 1 file changed, 21 insertions(+), 6 deletions(-)

diff --git a/src/inference_endpoint/evaluation/livecodebench/lcb_serve.py b/src/inference_endpoint/evaluation/livecodebench/lcb_serve.py
index 71fc6bd2f..9e4eff074 100644
--- a/src/inference_endpoint/evaluation/livecodebench/lcb_serve.py
+++ b/src/inference_endpoint/evaluation/livecodebench/lcb_serve.py
@@ -52,6 +52,10 @@
 
 logger = logging.getLogger(__name__)
 
+# Grading assumes fork semantics; under Python 3.14's forkserver default the
+# grading children die at startup and the service reports 0/N forever.
+_MP_CTX = mp.get_context("fork")
+
 
 def execute_code_single(test_suite_json: str, code: str, timeout_sec: int = 60):
     # Run code with lcb_runner. Note that the lcb_runner has a very rudimentary sandbox
@@ -117,9 +121,9 @@ def run_code_subprocess(
         suite["inputs"]
     ) + flat_timeout_extension
 
-    manager = mp.Manager()
+    manager = _MP_CTX.Manager()
     resp_buffer = manager.list()
-    p = mp.Process(
+    p = _MP_CTX.Process(
         target=execute_code_single_suppressed_errors,
         args=(
             test_suite_json,
@@ -270,8 +274,11 @@ def __call__(
         for qid, test_codes in zip(question_ids, codes, strict=False):
             results[qid] = [False] * len(test_codes)
         futures = {}
+        execution_errors = 0
 
-        with ProcessPoolExecutor(max_workers=self.n_lcb_workers) as executor:
+        with ProcessPoolExecutor(
+            max_workers=self.n_lcb_workers, mp_context=_MP_CTX
+        ) as executor:
             for qid, test_codes in zip(question_ids, codes, strict=False):
                 test_suite_json = self.test_loader[qid]
                 for i, code in enumerate(test_codes):
@@ -290,9 +297,8 @@ def __call__(
                 qid, code_idx = futures[future]
                 res, metadata = future.result()
                 if "error" in metadata:
-                    logger.warning(
-                        f"Test execution error for question {qid}: {metadata}"
-                    )
+                    execution_errors += 1
+                    logger.error(f"Test execution error for question {qid}: {metadata}")
 
                 # LCB uses any result > 0 as a 'pass' since:
                 # Negative numbers indicate error codes
@@ -317,6 +323,15 @@ def __call__(
                             exc_info=True,
                         )
 
+        # All subprocesses erroring means the judge itself is broken, not that
+        # every code sample failed its tests.
+        if futures and execution_errors == len(futures):
+            raise RuntimeError(
+                f"All {len(futures)} grading subprocesses reported execution "
+                "errors - the LCB judging infrastructure is broken; refusing "
+                "to report a 0 score. See the logged error metadata above."
+            )
+
         return results
 
 

From 21fef9d624756b5e9d0577ea1193cd55e1b59a1c Mon Sep 17 00:00:00 2001
From: Liang Yan <lyan@coreweave.com>
Date: Mon, 3 Aug 2026 21:08:31 -0400
Subject: [PATCH 02/14] fix(lcb-service): only count judge errors toward the
 all-errors guard

Timeouts were counted as execution errors, so a small batch where every
submission loops forever would trip the guard and raise instead of
scoring 0. Split the empty-buffer case in run_code_subprocess: child
still alive at the deadline -> timeout (-1, submission's fault), child
exited without reporting -> new GradingChildDied (-6, judge's fault).
The guard now only counts -5/-6, so the forkserver startup deaths still
raise and all-timeout batches score normally.

Also log the multiprocessing start method at service init; that would
have made the original 0/N a one-line diagnosis.
---
 .../evaluation/livecodebench/lcb_serve.py     | 53 +++++++++++++------
 1 file changed, 38 insertions(+), 15 deletions(-)

diff --git a/src/inference_endpoint/evaluation/livecodebench/lcb_serve.py b/src/inference_endpoint/evaluation/livecodebench/lcb_serve.py
index 9e4eff074..88f5cf0f6 100644
--- a/src/inference_endpoint/evaluation/livecodebench/lcb_serve.py
+++ b/src/inference_endpoint/evaluation/livecodebench/lcb_serve.py
@@ -54,8 +54,15 @@
 
 # Grading assumes fork semantics; under Python 3.14's forkserver default the
 # grading children die at startup and the service reports 0/N forever.
+# This module only runs inside the Linux lcb-service container, so "fork"
+# is always available.
 _MP_CTX = mp.get_context("fork")
 
+# Error codes that mean the judge is broken, as opposed to the submitted code
+# failing its tests: -5 TestRunnerError, -6 GradingChildDied. Timeouts (-1)
+# are the submitted code's fault and do not count.
+_LCB_INFRA_ERROR_CODES = {-5, -6}
+
 
 def execute_code_single(test_suite_json: str, code: str, timeout_sec: int = 60):
     # Run code with lcb_runner. Note that the lcb_runner has a very rudimentary sandbox
@@ -137,17 +144,29 @@ def run_code_subprocess(
     p.start()
     p.join(timeout=global_timeout)
 
-    if p.is_alive():
+    timed_out = p.is_alive()
+    if timed_out:
         p.kill()
 
     if len(resp_buffer) == 0:
-        # Assume timeout
-        res = [-1] * len(suite["inputs"])
-        metadata = {
-            "error": "Test suite timeout",
-            "error_code": -1,
-            "error_message": f"Subprocess did not complete in time ({global_timeout}s)",
-        }
+        if timed_out:
+            # Still running at the deadline: the submitted code took too long.
+            res = [-1] * len(suite["inputs"])
+            metadata = {
+                "error": "Test suite timeout",
+                "error_code": -1,
+                "error_message": f"Subprocess did not complete in time ({global_timeout}s)",
+            }
+        else:
+            # Exited before the deadline without reporting a result: the
+            # grading child died (e.g. bad start method, OOM kill), not the
+            # submitted code.
+            res = [-1] * len(suite["inputs"])
+            metadata = {
+                "error": "Grading subprocess died before reporting a result",
+                "error_code": -6,
+                "error_message": f"GradingChildDied (exitcode={p.exitcode})",
+            }
         return res, metadata
     else:
         res, metadata = resp_buffer[0]
@@ -274,7 +293,7 @@ def __call__(
         for qid, test_codes in zip(question_ids, codes, strict=False):
             results[qid] = [False] * len(test_codes)
         futures = {}
-        execution_errors = 0
+        infra_errors = 0
 
         with ProcessPoolExecutor(
             max_workers=self.n_lcb_workers, mp_context=_MP_CTX
@@ -297,7 +316,8 @@ def __call__(
                 qid, code_idx = futures[future]
                 res, metadata = future.result()
                 if "error" in metadata:
-                    execution_errors += 1
+                    if metadata.get("error_code") in _LCB_INFRA_ERROR_CODES:
+                        infra_errors += 1
                     logger.error(f"Test execution error for question {qid}: {metadata}")
 
                 # LCB uses any result > 0 as a 'pass' since:
@@ -323,12 +343,14 @@ def __call__(
                             exc_info=True,
                         )
 
-        # All subprocesses erroring means the judge itself is broken, not that
-        # every code sample failed its tests.
-        if futures and execution_errors == len(futures):
+        # Every subprocess hitting an infra error means the judge itself is
+        # broken, not that every code sample failed its tests. Timeouts are
+        # excluded: a batch where every submission loops forever is a valid
+        # 0 score, not a broken judge.
+        if futures and infra_errors == len(futures):
             raise RuntimeError(
-                f"All {len(futures)} grading subprocesses reported execution "
-                "errors - the LCB judging infrastructure is broken; refusing "
+                f"All {len(futures)} grading subprocesses reported "
+                "infrastructure errors - the LCB judge is broken; refusing "
                 "to report a 0 score. See the logged error metadata above."
             )
 
@@ -363,6 +385,7 @@ def __init__(
         if n_workers is None:
             n_workers = mp.cpu_count() // 2
         logger.info("Using %d workers for LCB eval", n_workers)
+        logger.info("Multiprocessing start method: %s", _MP_CTX.get_start_method())
         self.n_workers = n_workers
 
         self.path_to_dataset = (

From 4968773b93ecbabb45e93d19dd4dd65b83b65b76 Mon Sep 17 00:00:00 2001
From: Liang Yan <lyan@coreweave.com>
Date: Wed, 5 Aug 2026 17:56:29 -0400
Subject: [PATCH 03/14] fix(lcb-service): catch sys.exit() in submitted code as
 a submission failure

sys.exit() is a BaseException, not caught by the existing `except
Exception`. grade_call_based's method invocation has no SystemExit
guard (unlike the stdio path's call_method, which already does),
so a call-based submission calling sys.exit() killed the grading
child before it filled resp_buffer and got misclassified as -6
GradingChildDied - an infra error that can trip the all-errors guard
even for a single-sample batch. Give it its own code (-7) instead,
kept out of _LCB_INFRA_ERROR_CODES.
---
 .../evaluation/livecodebench/lcb_serve.py              | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/src/inference_endpoint/evaluation/livecodebench/lcb_serve.py b/src/inference_endpoint/evaluation/livecodebench/lcb_serve.py
index 88f5cf0f6..dc0ff70ec 100644
--- a/src/inference_endpoint/evaluation/livecodebench/lcb_serve.py
+++ b/src/inference_endpoint/evaluation/livecodebench/lcb_serve.py
@@ -99,6 +99,16 @@ def execute_code_single_suppressed_errors(
 
         if not isinstance(metadata, dict):
             raise ValueError(f"Expected metadata to be a dict, got {type(metadata)}")
+    except SystemExit as e:
+        # sys.exit() in submitted code is a BaseException, not caught below.
+        # It's the submission's fault, not the judge's, so give it its own
+        # error code and keep it out of _LCB_INFRA_ERROR_CODES.
+        res = [-2]
+        metadata = {
+            "error": f"Submission called sys.exit({e.code!r})",
+            "error_code": -7,
+            "error_message": "SubmissionExit",
+        }
     except Exception:
         # Magic number (see https://github.com/LiveCodeBench/LiveCodeBench/blob/28fef95ea8c9f7a547c8329f2cd3d32b92c1fa24/lcb_runner/evaluation/compute_code_generation_metrics.py#L65)
         res = [-2]  # LCB internal error code for test runner failed test cases

From c9a1428a46cc03e4a788045ec19d2db0a30ffd47 Mon Sep 17 00:00:00 2001
From: Liang Yan <lyan@coreweave.com>
Date: Wed, 5 Aug 2026 18:30:27 -0400
Subject: [PATCH 04/14] fix(lcb-service): use spawn for the grading pool
 workers

evaluate() runs on an executor thread (the server dispatches it via
run_in_executor), so the per-request ProcessPoolExecutor was forking an
already-multithreaded process - a known deadlock risk: only the forking
thread survives in the child, locks held by other threads stay locked
forever. Switch the pool to spawn: fork+exec inherits no locks, so it is
safe to start from a thread, and everything submitted to the pool is
picklable, so it is a drop-in.

Tried forkserver first, but its helper hangs at pool shutdown in the
lcb-service container (Python 3.14.5) and leaks semaphores. Probed all
three start methods in the deployment image: fork and spawn tear down
cleanly, forkserver hangs indefinitely.

The inner grading child keeps fork: grading relies on fork semantics, and
forking from a freshly exec'd single-threaded pool worker is fine. The
startup log now prints both start methods.
---
 .../evaluation/livecodebench/lcb_serve.py     | 21 +++++++++++++------
 1 file changed, 15 insertions(+), 6 deletions(-)

diff --git a/src/inference_endpoint/evaluation/livecodebench/lcb_serve.py b/src/inference_endpoint/evaluation/livecodebench/lcb_serve.py
index dc0ff70ec..6f3898f1e 100644
--- a/src/inference_endpoint/evaluation/livecodebench/lcb_serve.py
+++ b/src/inference_endpoint/evaluation/livecodebench/lcb_serve.py
@@ -52,10 +52,15 @@
 
 logger = logging.getLogger(__name__)
 
-# Grading assumes fork semantics; under Python 3.14's forkserver default the
-# grading children die at startup and the service reports 0/N forever.
-# This module only runs inside the Linux lcb-service container, so "fork"
-# is always available.
+# The pool is created from a worker thread of the event loop's default
+# executor (the server calls evaluate via run_in_executor), so it must not
+# fork the multithreaded parent; spawn (fork+exec) is safe from threads.
+# forkserver would also work in theory but hangs at pool shutdown in this
+# container.
+_MP_POOL_CTX = mp.get_context("spawn")
+
+# Grading assumes fork semantics; forking here is safe because the
+# spawn-started pool worker is freshly exec'd and single-threaded.
 _MP_CTX = mp.get_context("fork")
 
 # Error codes that mean the judge is broken, as opposed to the submitted code
@@ -306,7 +311,7 @@ def __call__(
         infra_errors = 0
 
         with ProcessPoolExecutor(
-            max_workers=self.n_lcb_workers, mp_context=_MP_CTX
+            max_workers=self.n_lcb_workers, mp_context=_MP_POOL_CTX
         ) as executor:
             for qid, test_codes in zip(question_ids, codes, strict=False):
                 test_suite_json = self.test_loader[qid]
@@ -395,7 +400,11 @@ def __init__(
         if n_workers is None:
             n_workers = mp.cpu_count() // 2
         logger.info("Using %d workers for LCB eval", n_workers)
-        logger.info("Multiprocessing start method: %s", _MP_CTX.get_start_method())
+        logger.info(
+            "Multiprocessing start methods: pool=%s, grading child=%s",
+            _MP_POOL_CTX.get_start_method(),
+            _MP_CTX.get_start_method(),
+        )
         self.n_workers = n_workers
 
         self.path_to_dataset = (

From 186251fd1131ad97bae1666ab382c32271c52aa9 Mon Sep 17 00:00:00 2001
From: Liang Yan <lyan@coreweave.com>
Date: Wed, 5 Aug 2026 19:42:07 -0400
Subject: [PATCH 05/14] fix(lcb-service): only treat pre-grading child deaths
 as judge errors

A submission can kill its own grading child in ways no except block sees
(os._exit(), a native segfault, an OOM kill). That landed in the -6
GradingChildDied bucket and counted as an infrastructure error, so a batch
where every submission crashed its interpreter tripped the all-infra-errors
guard and aborted instead of reporting a legitimate 0 score.

The child now sets a shared started flag right before grading begins, so an
empty resp_buffer can be attributed: died before the flag means a judge
startup failure - still -6, still counted by the guard; died after means
the submission killed the interpreter - new -8 SubmissionKilledChild,
scored as a normal failed sample. Exit codes cannot make this distinction
because os._exit() lets the submission pick any code.
---
 .../evaluation/livecodebench/lcb_serve.py     | 40 +++++++++++++++----
 1 file changed, 33 insertions(+), 7 deletions(-)

diff --git a/src/inference_endpoint/evaluation/livecodebench/lcb_serve.py b/src/inference_endpoint/evaluation/livecodebench/lcb_serve.py
index 6f3898f1e..bfcefad2f 100644
--- a/src/inference_endpoint/evaluation/livecodebench/lcb_serve.py
+++ b/src/inference_endpoint/evaluation/livecodebench/lcb_serve.py
@@ -35,6 +35,7 @@
 """
 
 import argparse
+import ctypes
 import json
 import logging
 import multiprocessing as mp
@@ -43,7 +44,9 @@
 from collections.abc import Callable
 from concurrent.futures import ProcessPoolExecutor, as_completed
 from functools import lru_cache
+from multiprocessing.sharedctypes import Synchronized
 from pathlib import Path
+from typing import cast
 
 import pandas as pd
 from tqdm import tqdm
@@ -64,8 +67,10 @@
 _MP_CTX = mp.get_context("fork")
 
 # Error codes that mean the judge is broken, as opposed to the submitted code
-# failing its tests: -5 TestRunnerError, -6 GradingChildDied. Timeouts (-1)
-# are the submitted code's fault and do not count.
+# failing its tests: -5 TestRunnerError, -6 GradingChildDied (child died
+# before grading started, e.g. a bad start method). Submission-attributed
+# codes stay out: timeouts (-1), sys.exit() (-7 SubmissionExit), and
+# submissions that kill their own interpreter (-8 SubmissionKilledChild).
 _LCB_INFRA_ERROR_CODES = {-5, -6}
 
 
@@ -94,9 +99,18 @@ def execute_code_single(test_suite_json: str, code: str, timeout_sec: int = 60):
 
 
 def execute_code_single_suppressed_errors(
-    *args, resp_buffer: list | None = None, **kwargs
+    *args,
+    resp_buffer: list | None = None,
+    started_flag: Synchronized | None = None,
+    **kwargs,
 ):
     """Wrapper around execute code so that all errors are resurfaced as failed tests"""
+    if started_flag is not None:
+        # From here on the wrapper is running and grading (i.e. the
+        # submission) is about to execute; if the interpreter dies now it is
+        # the submission's doing. Deaths before this point are judge startup
+        # failures.
+        started_flag.value = True
     try:
         res, metadata = execute_code_single(*args, **kwargs)
         if not isinstance(res, list):
@@ -145,6 +159,8 @@ def run_code_subprocess(
 
     manager = _MP_CTX.Manager()
     resp_buffer = manager.list()
+    # typeshed types ctx.Value() as SynchronizedBase, which lacks .value
+    started_flag = cast(Synchronized, _MP_CTX.Value(ctypes.c_bool, False))
     p = _MP_CTX.Process(
         target=execute_code_single_suppressed_errors,
         args=(
@@ -154,6 +170,7 @@ def run_code_subprocess(
         kwargs={
             "resp_buffer": resp_buffer,
             "timeout_sec": timeout_sec,
+            "started_flag": started_flag,
         },
     )
     p.start()
@@ -172,13 +189,22 @@ def run_code_subprocess(
                 "error_code": -1,
                 "error_message": f"Subprocess did not complete in time ({global_timeout}s)",
             }
+        elif started_flag.value:
+            # The interpreter died while grading was running (os._exit(),
+            # segfault, OOM, ...). Grading executes the untrusted submission,
+            # so this is the submission's fault, not the judge's.
+            res = [-1] * len(suite["inputs"])
+            metadata = {
+                "error": "Grading child killed while executing the submission",
+                "error_code": -8,
+                "error_message": f"SubmissionKilledChild (exitcode={p.exitcode})",
+            }
         else:
-            # Exited before the deadline without reporting a result: the
-            # grading child died (e.g. bad start method, OOM kill), not the
-            # submitted code.
+            # Died before grading started (e.g. bad start method): the judge
+            # is broken.
             res = [-1] * len(suite["inputs"])
             metadata = {
-                "error": "Grading subprocess died before reporting a result",
+                "error": "Grading subprocess died before grading started",
                 "error_code": -6,
                 "error_message": f"GradingChildDied (exitcode={p.exitcode})",
             }

From 6a66afcaf5447ee6f6bc4a3263233732b6e4daf6 Mon Sep 17 00:00:00 2001
From: Liang Yan <lyan@coreweave.com>
Date: Wed, 5 Aug 2026 20:01:52 -0400
Subject: [PATCH 06/14] refactor(lcb-service): flatten the no-result handling
 in run_code_subprocess

Return the reported result early, hoist the shared all-failed res out of
the attribution branches, and keep only the metadata construction per
branch. No behavior change.
---
 .../evaluation/livecodebench/lcb_serve.py     | 61 +++++++++----------
 1 file changed, 30 insertions(+), 31 deletions(-)

diff --git a/src/inference_endpoint/evaluation/livecodebench/lcb_serve.py b/src/inference_endpoint/evaluation/livecodebench/lcb_serve.py
index bfcefad2f..d2232a21f 100644
--- a/src/inference_endpoint/evaluation/livecodebench/lcb_serve.py
+++ b/src/inference_endpoint/evaluation/livecodebench/lcb_serve.py
@@ -180,38 +180,37 @@ def run_code_subprocess(
     if timed_out:
         p.kill()
 
-    if len(resp_buffer) == 0:
-        if timed_out:
-            # Still running at the deadline: the submitted code took too long.
-            res = [-1] * len(suite["inputs"])
-            metadata = {
-                "error": "Test suite timeout",
-                "error_code": -1,
-                "error_message": f"Subprocess did not complete in time ({global_timeout}s)",
-            }
-        elif started_flag.value:
-            # The interpreter died while grading was running (os._exit(),
-            # segfault, OOM, ...). Grading executes the untrusted submission,
-            # so this is the submission's fault, not the judge's.
-            res = [-1] * len(suite["inputs"])
-            metadata = {
-                "error": "Grading child killed while executing the submission",
-                "error_code": -8,
-                "error_message": f"SubmissionKilledChild (exitcode={p.exitcode})",
-            }
-        else:
-            # Died before grading started (e.g. bad start method): the judge
-            # is broken.
-            res = [-1] * len(suite["inputs"])
-            metadata = {
-                "error": "Grading subprocess died before grading started",
-                "error_code": -6,
-                "error_message": f"GradingChildDied (exitcode={p.exitcode})",
-            }
-        return res, metadata
+    if len(resp_buffer) > 0:
+        return resp_buffer[0]
+
+    # No result was reported: every test case counts as failed, only the
+    # attribution differs.
+    res = [-1] * len(suite["inputs"])
+    if timed_out:
+        # Still running at the deadline: the submitted code took too long.
+        metadata = {
+            "error": "Test suite timeout",
+            "error_code": -1,
+            "error_message": f"Subprocess did not complete in time ({global_timeout}s)",
+        }
+    elif started_flag.value:
+        # The interpreter died while grading was running (os._exit(),
+        # segfault, OOM, ...). Grading executes the untrusted submission, so
+        # this is the submission's fault, not the judge's.
+        metadata = {
+            "error": "Grading child killed while executing the submission",
+            "error_code": -8,
+            "error_message": f"SubmissionKilledChild (exitcode={p.exitcode})",
+        }
     else:
-        res, metadata = resp_buffer[0]
-        return res, metadata
+        # Died before grading started (e.g. bad start method): the judge is
+        # broken.
+        metadata = {
+            "error": "Grading subprocess died before grading started",
+            "error_code": -6,
+            "error_message": f"GradingChildDied (exitcode={p.exitcode})",
+        }
+    return res, metadata
 
 
 class LCBTestLoader:

From 00efd863683f680cd914446ba1913f7533d5cc2b Mon Sep 17 00:00:00 2001
From: Liang Yan <lyan@coreweave.com>
Date: Wed, 5 Aug 2026 20:24:48 -0400
Subject: [PATCH 07/14] refactor(lcb-service): scope the per-sample Manager to
 a with-block

Each graded sample created a Manager (its own server process) and relied on
the GC finalizer to shut it down. Make the lifecycle explicit with a
with-block so the process count under load is bounded deterministically,
capture the child's started/exitcode state before the scope closes, and
reap a killed grading child with join() instead of leaving a zombie in the
pool worker.
---
 .../evaluation/livecodebench/lcb_serve.py     | 56 ++++++++++---------
 1 file changed, 30 insertions(+), 26 deletions(-)

diff --git a/src/inference_endpoint/evaluation/livecodebench/lcb_serve.py b/src/inference_endpoint/evaluation/livecodebench/lcb_serve.py
index d2232a21f..6c9c7fe2b 100644
--- a/src/inference_endpoint/evaluation/livecodebench/lcb_serve.py
+++ b/src/inference_endpoint/evaluation/livecodebench/lcb_serve.py
@@ -157,31 +157,35 @@ def run_code_subprocess(
         suite["inputs"]
     ) + flat_timeout_extension
 
-    manager = _MP_CTX.Manager()
-    resp_buffer = manager.list()
-    # typeshed types ctx.Value() as SynchronizedBase, which lacks .value
-    started_flag = cast(Synchronized, _MP_CTX.Value(ctypes.c_bool, False))
-    p = _MP_CTX.Process(
-        target=execute_code_single_suppressed_errors,
-        args=(
-            test_suite_json,
-            code,
-        ),
-        kwargs={
-            "resp_buffer": resp_buffer,
-            "timeout_sec": timeout_sec,
-            "started_flag": started_flag,
-        },
-    )
-    p.start()
-    p.join(timeout=global_timeout)
+    with _MP_CTX.Manager() as manager:
+        resp_buffer = manager.list()
+        # typeshed types ctx.Value() as SynchronizedBase, which lacks .value
+        started_flag = cast(Synchronized, _MP_CTX.Value(ctypes.c_bool, False))
+        p = _MP_CTX.Process(
+            target=execute_code_single_suppressed_errors,
+            args=(
+                test_suite_json,
+                code,
+            ),
+            kwargs={
+                "resp_buffer": resp_buffer,
+                "timeout_sec": timeout_sec,
+                "started_flag": started_flag,
+            },
+        )
+        p.start()
+        p.join(timeout=global_timeout)
 
-    timed_out = p.is_alive()
-    if timed_out:
-        p.kill()
+        timed_out = p.is_alive()
+        if timed_out:
+            p.kill()
+            p.join()
+
+        if len(resp_buffer) > 0:
+            return resp_buffer[0]
 
-    if len(resp_buffer) > 0:
-        return resp_buffer[0]
+        started = bool(started_flag.value)
+        exitcode = p.exitcode
 
     # No result was reported: every test case counts as failed, only the
     # attribution differs.
@@ -193,14 +197,14 @@ def run_code_subprocess(
             "error_code": -1,
             "error_message": f"Subprocess did not complete in time ({global_timeout}s)",
         }
-    elif started_flag.value:
+    elif started:
         # The interpreter died while grading was running (os._exit(),
         # segfault, OOM, ...). Grading executes the untrusted submission, so
         # this is the submission's fault, not the judge's.
         metadata = {
             "error": "Grading child killed while executing the submission",
             "error_code": -8,
-            "error_message": f"SubmissionKilledChild (exitcode={p.exitcode})",
+            "error_message": f"SubmissionKilledChild (exitcode={exitcode})",
         }
     else:
         # Died before grading started (e.g. bad start method): the judge is
@@ -208,7 +212,7 @@ def run_code_subprocess(
         metadata = {
             "error": "Grading subprocess died before grading started",
             "error_code": -6,
-            "error_message": f"GradingChildDied (exitcode={p.exitcode})",
+            "error_message": f"GradingChildDied (exitcode={exitcode})",
         }
     return res, metadata
 

From 81098e12b72548dca8d87c88f9d47f83db740e14 Mon Sep 17 00:00:00 2001
From: Liang Yan <lyan@coreweave.com>
Date: Wed, 5 Aug 2026 20:39:58 -0400
Subject: [PATCH 08/14] refactor(lcb-service): concrete signatures for the
 grading helpers

execute_code_single_suppressed_errors is the fork target, but took fully
untyped variadics, so a miswired argument only failed at runtime inside the
grading child (surfacing as a spurious child-death error). Give it named
parameters, and declare the tuple[list, dict] return type on all three
grading helpers so the res/metadata unpacking is checked.
---
 .../evaluation/livecodebench/lcb_serve.py      | 18 ++++++++++++------
 1 file changed, 12 insertions(+), 6 deletions(-)

diff --git a/src/inference_endpoint/evaluation/livecodebench/lcb_serve.py b/src/inference_endpoint/evaluation/livecodebench/lcb_serve.py
index 6c9c7fe2b..9851fcf0c 100644
--- a/src/inference_endpoint/evaluation/livecodebench/lcb_serve.py
+++ b/src/inference_endpoint/evaluation/livecodebench/lcb_serve.py
@@ -74,7 +74,9 @@
 _LCB_INFRA_ERROR_CODES = {-5, -6}
 
 
-def execute_code_single(test_suite_json: str, code: str, timeout_sec: int = 60):
+def execute_code_single(
+    test_suite_json: str, code: str, timeout_sec: int = 60
+) -> tuple[list, dict]:
     # Run code with lcb_runner. Note that the lcb_runner has a very rudimentary sandbox
     # which is extremely easy to bypass, and as such it is recommended to run this both
     # in an unprivileged container and in a separate process.
@@ -99,11 +101,13 @@ def execute_code_single(test_suite_json: str, code: str, timeout_sec: int = 60):
 
 
 def execute_code_single_suppressed_errors(
-    *args,
+    test_suite_json: str,
+    code: str,
+    timeout_sec: int = 60,
+    *,
     resp_buffer: list | None = None,
     started_flag: Synchronized | None = None,
-    **kwargs,
-):
+) -> tuple[list, dict]:
     """Wrapper around execute code so that all errors are resurfaced as failed tests"""
     if started_flag is not None:
         # From here on the wrapper is running and grading (i.e. the
@@ -112,7 +116,9 @@ def execute_code_single_suppressed_errors(
         # failures.
         started_flag.value = True
     try:
-        res, metadata = execute_code_single(*args, **kwargs)
+        res, metadata = execute_code_single(
+            test_suite_json, code, timeout_sec=timeout_sec
+        )
         if not isinstance(res, list):
             raise ValueError(f"Expected boolean result, got {type(res)}")
 
@@ -146,7 +152,7 @@ def run_code_subprocess(
     test_suite_json: str,
     code: str,
     timeout_sec: int = 60,
-):
+) -> tuple[list, dict]:
     # Compute global timeout -
     # https://github.com/LiveCodeBench/LiveCodeBench/blob/28fef95ea8c9f7a547c8329f2cd3d32b92c1fa24/lcb_runner/evaluation/compute_code_generation_metrics.py#L43
 

From bda7c0f6d622e7a08c8ddb51ac6c1c4a7707f511 Mon Sep 17 00:00:00 2001
From: Liang Yan <lyan@coreweave.com>
Date: Wed, 12 Aug 2026 14:47:28 -0400
Subject: [PATCH 09/14] test(lcb-service): add lcb_serve error-attribution
 tests

Should've gone out with the earlier attribution-fix commits -- had this
written already, just missed staging it at the time. Covers timeout (-1),
sys.exit (-7), os._exit (-8), and judge-side deaths (-6, both before
run_test and during its own setup), plus the all-infra guard: an
os._exit()-only batch scores 0 without tripping it, a judge-startup-death
batch does.
---
 tests/unit/evaluation/test_lcb_serve.py | 178 ++++++++++++++++++++++++
 1 file changed, 178 insertions(+)
 create mode 100644 tests/unit/evaluation/test_lcb_serve.py

diff --git a/tests/unit/evaluation/test_lcb_serve.py b/tests/unit/evaluation/test_lcb_serve.py
new file mode 100644
index 000000000..b8de395bd
--- /dev/null
+++ b/tests/unit/evaluation/test_lcb_serve.py
@@ -0,0 +1,178 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 CoreWeave, Inc. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Tests for lcb_serve.py's grading-child error attribution.
+
+run_code_subprocess forks a grading child per sample and must classify a
+child that reports no result into exactly one of:
+  -1 timeout            (submission ran past the deadline)
+  -6 GradingChildDied   (judge-side: died before or while doing its own
+                         setup -- reliability_guard, suite parse -- i.e.
+                         before the submission's code ever ran)
+  -8 SubmissionKilledChild (submission-side: died while its own code, or
+                         run_test's dispatch into it, was executing)
+
+These tests call run_code_subprocess directly (not through _LCBWorker's
+spawn pool) so that monkeypatches applied in the test process are inherited
+by the forked grading child; the batch-level guard tests below need the
+real pool and monkeypatch _MP_POOL_CTX to fork for the same reason.
+"""
+
+import os
+
+import pytest
+from inference_endpoint.evaluation.livecodebench import lcb_serve, run_lcb_tests
+
+CALL_BASED_SUITE = '{"fn_name": "solve", "inputs": ["1"], "outputs": ["1"]}'
+PASSING_CALL_BASED = "def solve(x):\n    return x\n"
+CALL_SYSEXIT = "def solve(x):\n    import sys\n    sys.exit(3)\n"
+CALL_OS_EXIT = "import os\nos._exit(3)\ndef solve(x):\n    return x\n"
+
+# Keep timeouts short; the grading itself is instant, this only bounds how
+# long a genuinely hung test would take to fail.
+TIMEOUT_SEC = 5
+
+
+def test_passing_submission_scores_true():
+    res, metadata = lcb_serve.run_code_subprocess(
+        CALL_BASED_SUITE, PASSING_CALL_BASED, timeout_sec=TIMEOUT_SEC
+    )
+    assert res == [True]
+    assert metadata.get("error_code") is None
+
+
+def test_sys_exit_is_submission_exit_not_infra():
+    res, metadata = lcb_serve.run_code_subprocess(
+        CALL_BASED_SUITE, CALL_SYSEXIT, timeout_sec=TIMEOUT_SEC
+    )
+    assert metadata["error_code"] == -7
+    assert metadata["error_code"] not in lcb_serve._LCB_INFRA_ERROR_CODES
+
+
+def test_os_exit_in_submission_is_submission_killed_child_not_infra():
+    res, metadata = lcb_serve.run_code_subprocess(
+        CALL_BASED_SUITE, CALL_OS_EXIT, timeout_sec=TIMEOUT_SEC
+    )
+    assert metadata["error_code"] == -8
+    assert metadata["error_code"] not in lcb_serve._LCB_INFRA_ERROR_CODES
+
+
+def test_death_before_run_test_is_infra_error():
+    """A judge process that dies before run_test is even reached (e.g. a
+    bad start method, or a crash while importing) is -6, not -8."""
+    orig = lcb_serve.execute_code_single_suppressed_errors
+
+    def _die_immediately(*args, **kwargs):
+        os._exit(1)
+
+    lcb_serve.execute_code_single_suppressed_errors = _die_immediately
+    try:
+        res, metadata = lcb_serve.run_code_subprocess(
+            CALL_BASED_SUITE, PASSING_CALL_BASED, timeout_sec=TIMEOUT_SEC
+        )
+    finally:
+        lcb_serve.execute_code_single_suppressed_errors = orig
+
+    assert metadata["error_code"] == -6
+    assert metadata["error_code"] in lcb_serve._LCB_INFRA_ERROR_CODES
+
+
+def test_death_during_judge_setup_is_infra_error_not_submission():
+    """A death inside run_test's own setup (reliability_guard, suite parse)
+    -- after run_test has started but before started_flag is set -- must
+    still be -6. This is the exact boundary the started_flag fix moved: it
+    used to flip True on wrapper entry, before this setup ran, which would
+    have misattributed this case as -8."""
+    orig_guard = run_lcb_tests.reliability_guard
+
+    def _guard_then_die(*args, **kwargs):
+        orig_guard(*args, **kwargs)
+        os._exit(1)
+
+    run_lcb_tests.reliability_guard = _guard_then_die
+    try:
+        res, metadata = lcb_serve.run_code_subprocess(
+            CALL_BASED_SUITE, PASSING_CALL_BASED, timeout_sec=TIMEOUT_SEC
+        )
+    finally:
+        run_lcb_tests.reliability_guard = orig_guard
+
+    assert metadata["error_code"] == -6
+    assert metadata["error_code"] in lcb_serve._LCB_INFRA_ERROR_CODES
+
+
+def test_timeout_is_not_infra_error():
+    # grade_call_based enforces its own per-test-case timeout via
+    # signal.alarm and returns a normal -3 result well within it, so
+    # exercising the *outer* process-level timeout (run_code_subprocess's
+    # p.join(timeout=...)) needs a submission that defeats the alarm too.
+    ignore_alarm_and_hang = (
+        "import signal\n"
+        "signal.signal(signal.SIGALRM, signal.SIG_IGN)\n"
+        "def solve(x):\n"
+        "    while True: pass\n"
+    )
+    res, metadata = lcb_serve.run_code_subprocess(
+        CALL_BASED_SUITE, ignore_alarm_and_hang, timeout_sec=1
+    )
+    assert metadata["error_code"] == -1
+    assert metadata["error_code"] not in lcb_serve._LCB_INFRA_ERROR_CODES
+
+
+class _DictTestLoader(dict):
+    """Minimal stand-in for LCBTestLoader: _LCBWorker only does
+    self.test_loader[qid], which dict already supports."""
+
+
+def test_all_os_exit_batch_scores_zero_without_raising():
+    """A batch where every submission kills its own interpreter is a
+    legitimate 0, not an infra failure -- the guard must not fire."""
+    worker = lcb_serve._LCBWorker(
+        _DictTestLoader(q1=CALL_BASED_SUITE, q2=CALL_BASED_SUITE),
+        n_lcb_workers=2,
+        worker_timeout_sec=TIMEOUT_SEC,
+    )
+    results = worker(["q1", "q2"], [[CALL_OS_EXIT], [CALL_OS_EXIT]])
+    assert results == {"q1": [False], "q2": [False]}
+
+
+def test_all_judge_startup_death_batch_raises(monkeypatch):
+    """A batch where every grading child dies before run_test is reached
+    means the judge itself is broken, not that every sample failed -- the
+    all-infra guard must raise instead of silently reporting a 0.
+
+    Uses a fork pool so the monkeypatch below (applied in this process)
+    reaches the pool workers; spawn workers re-import the module fresh and
+    would not see it. The guard logic under test lives in
+    _LCBWorker.__call__ and is identical regardless of pool context.
+    """
+    import multiprocessing as mp
+
+    monkeypatch.setattr(lcb_serve, "_MP_POOL_CTX", mp.get_context("fork"))
+
+    def _die_immediately(*args, **kwargs):
+        os._exit(1)
+
+    monkeypatch.setattr(
+        lcb_serve, "execute_code_single_suppressed_errors", _die_immediately
+    )
+
+    worker = lcb_serve._LCBWorker(
+        _DictTestLoader(q1=CALL_BASED_SUITE, q2=CALL_BASED_SUITE),
+        n_lcb_workers=2,
+        worker_timeout_sec=TIMEOUT_SEC,
+    )
+    with pytest.raises(RuntimeError, match="infrastructure errors"):
+        worker(["q1", "q2"], [[PASSING_CALL_BASED], [PASSING_CALL_BASED]])

From f3a0e5b86eac6febcac94020eff68e876a0f17ed Mon Sep 17 00:00:00 2001
From: Liang Yan <lyan@coreweave.com>
Date: Wed, 12 Aug 2026 15:13:57 -0400
Subject: [PATCH 10/14] fix(lcb-service): move started_flag past judge-side
 setup

started_flag flipped True at wrapper entry, before run_test's own setup
(reliability_guard, suite parse) ran -- a death in that window got
misattributed as -8 SubmissionKilledChild instead of -6 GradingChildDied,
so a real judge bug could sneak past the all-infra guard as a silent 0.

Moved the flag into run_test itself, set right before dispatch to
grade_call_based/grade_stdio -- the actual first line of the submission's
own code. Hoisted the numpy/run_lcb_tests imports to module scope while
in there too (same window, and it'd been flagged as a lazy import anyway);
costs the grading child nothing since it forks from an already-warm pool
worker.
---
 .../evaluation/livecodebench/lcb_serve.py     | 28 ++++++++++---------
 .../evaluation/livecodebench/run_lcb_tests.py | 12 +++++++-
 2 files changed, 26 insertions(+), 14 deletions(-)

diff --git a/src/inference_endpoint/evaluation/livecodebench/lcb_serve.py b/src/inference_endpoint/evaluation/livecodebench/lcb_serve.py
index 9851fcf0c..5a1e44ffb 100644
--- a/src/inference_endpoint/evaluation/livecodebench/lcb_serve.py
+++ b/src/inference_endpoint/evaluation/livecodebench/lcb_serve.py
@@ -48,10 +48,12 @@
 from pathlib import Path
 from typing import cast
 
+import numpy as np
 import pandas as pd
 from tqdm import tqdm
 
 from .generate import generate_dataset
+from .run_lcb_tests import run_test
 
 logger = logging.getLogger(__name__)
 
@@ -75,17 +77,20 @@
 
 
 def execute_code_single(
-    test_suite_json: str, code: str, timeout_sec: int = 60
+    test_suite_json: str,
+    code: str,
+    timeout_sec: int = 60,
+    *,
+    started_flag: Synchronized | None = None,
 ) -> tuple[list, dict]:
     # Run code with lcb_runner. Note that the lcb_runner has a very rudimentary sandbox
     # which is extremely easy to bypass, and as such it is recommended to run this both
     # in an unprivileged container and in a separate process.
-    import numpy as np
-
-    from .run_lcb_tests import run_test
-
     res, metadata = run_test(
-        {"input_output": test_suite_json}, test=code, timeout=timeout_sec
+        {"input_output": test_suite_json},
+        test=code,
+        timeout=timeout_sec,
+        started_flag=started_flag,
     )
 
     # LCB results are expected to be plain booleans or error codes.
@@ -109,15 +114,12 @@ def execute_code_single_suppressed_errors(
     started_flag: Synchronized | None = None,
 ) -> tuple[list, dict]:
     """Wrapper around execute code so that all errors are resurfaced as failed tests"""
-    if started_flag is not None:
-        # From here on the wrapper is running and grading (i.e. the
-        # submission) is about to execute; if the interpreter dies now it is
-        # the submission's doing. Deaths before this point are judge startup
-        # failures.
-        started_flag.value = True
     try:
+        # started_flag is flipped inside run_test, once judge-side setup
+        # (reliability_guard, suite parse) is done and the submission's own
+        # code is next; see run_lcb_tests.run_test.
         res, metadata = execute_code_single(
-            test_suite_json, code, timeout_sec=timeout_sec
+            test_suite_json, code, timeout_sec=timeout_sec, started_flag=started_flag
         )
         if not isinstance(res, list):
             raise ValueError(f"Expected boolean result, got {type(res)}")
diff --git a/src/inference_endpoint/evaluation/livecodebench/run_lcb_tests.py b/src/inference_endpoint/evaluation/livecodebench/run_lcb_tests.py
index 16798f6ff..f2263d1df 100644
--- a/src/inference_endpoint/evaluation/livecodebench/run_lcb_tests.py
+++ b/src/inference_endpoint/evaluation/livecodebench/run_lcb_tests.py
@@ -30,6 +30,7 @@
 from decimal import Decimal
 from enum import Enum
 from io import StringIO
+from multiprocessing.sharedctypes import Synchronized
 
 # from pyext import RuntimeModule
 from types import ModuleType
@@ -475,7 +476,7 @@ def grade_stdio(
     return all_results, {"execution time": total_execution_time}
 
 
-def run_test(sample, test=None, timeout=6):
+def run_test(sample, test=None, timeout=6, started_flag: Synchronized | None = None):
     """
     if test(generated_code) is not None it'll try to run the code.
     otherwise it'll just return an input and output pair.
@@ -506,7 +507,16 @@ def run_test(sample, test=None, timeout=6):
     elif test is not None:
         results = []
 
+        if started_flag is not None:
+            # Judge-side setup (reliability_guard, suite parse) is done; the
+            # next thing that runs is the submission itself (grade_call_based
+            # / grade_stdio compile and exec `test`), so a death from here on
+            # is the submission's doing, not the judge's.
+            started_flag.value = True
+
         if which_type == CODE_TYPE.call_based:
+            # method_name is only None for CODE_TYPE.standard_input (above).
+            assert method_name is not None
             signal.alarm(timeout)
             try:
                 results, metadata = grade_call_based(

From 3feb5ffbb5885a6dfdebac2592571df373190158 Mon Sep 17 00:00:00 2001
From: Liang Yan <lyan@coreweave.com>
Date: Wed, 12 Aug 2026 16:11:17 -0400
Subject: [PATCH 11/14] fix(lcb-service): tag submission compile errors and
 quiet down routine logging

Two gaps in the all-infra guard/logging: the outer except in
grade_call_based/grade_stdio's callers returns -4 with no "error" key, so a
bad submission that fails to compile or define the expected function never
gets logged (the classification gate keys on "error" in metadata) -- give it
one. Left -4 out of _LCB_INFRA_ERROR_CODES on purpose: it's reached whenever
the submission's own code fails to compile, which is the common case and is
plainly not the judge's fault, not some rare harness bug worth aborting a
whole run over.

Also split logger.error into error (infra codes) vs warning (everything
else) -- it was firing for every timeout/sys.exit/os._exit/bad-code sample,
which drowns the real -5/-6 signal in ops on any batch with a few slow or
broken submissions.
---
 .../evaluation/livecodebench/lcb_serve.py     | 20 +++++-
 .../evaluation/livecodebench/run_lcb_tests.py |  7 +++
 tests/unit/evaluation/test_lcb_serve.py       | 62 ++++++++++++++++++-
 3 files changed, 84 insertions(+), 5 deletions(-)

diff --git a/src/inference_endpoint/evaluation/livecodebench/lcb_serve.py b/src/inference_endpoint/evaluation/livecodebench/lcb_serve.py
index 5a1e44ffb..aef992a09 100644
--- a/src/inference_endpoint/evaluation/livecodebench/lcb_serve.py
+++ b/src/inference_endpoint/evaluation/livecodebench/lcb_serve.py
@@ -71,8 +71,12 @@
 # Error codes that mean the judge is broken, as opposed to the submitted code
 # failing its tests: -5 TestRunnerError, -6 GradingChildDied (child died
 # before grading started, e.g. a bad start method). Submission-attributed
-# codes stay out: timeouts (-1), sys.exit() (-7 SubmissionExit), and
-# submissions that kill their own interpreter (-8 SubmissionKilledChild).
+# codes stay out: timeouts (-1), sys.exit() (-7 SubmissionExit), submissions
+# that kill their own interpreter (-8 SubmissionKilledChild), and -4 (the
+# submission's code failed to compile or define the expected function --
+# see the outer except in run_lcb_tests.grade_call_based/grade_stdio callers
+# -- which is the submission's fault, not the judge's, even though it's
+# labeled "Error during testing").
 _LCB_INFRA_ERROR_CODES = {-5, -6}
 
 
@@ -370,7 +374,17 @@ def __call__(
                 if "error" in metadata:
                     if metadata.get("error_code") in _LCB_INFRA_ERROR_CODES:
                         infra_errors += 1
-                    logger.error(f"Test execution error for question {qid}: {metadata}")
+                        logger.error(
+                            f"Test execution error for question {qid}: {metadata}"
+                        )
+                    else:
+                        # Routine submission-attributed outcomes (timeout,
+                        # sys.exit, os._exit, bad code) -- expected at scale,
+                        # would otherwise flood ERROR and drown out the
+                        # infra signal above.
+                        logger.warning(
+                            f"Test execution error for question {qid}: {metadata}"
+                        )
 
                 # LCB uses any result > 0 as a 'pass' since:
                 # Negative numbers indicate error codes
diff --git a/src/inference_endpoint/evaluation/livecodebench/run_lcb_tests.py b/src/inference_endpoint/evaluation/livecodebench/run_lcb_tests.py
index f2263d1df..9ac56b30a 100644
--- a/src/inference_endpoint/evaluation/livecodebench/run_lcb_tests.py
+++ b/src/inference_endpoint/evaluation/livecodebench/run_lcb_tests.py
@@ -528,7 +528,11 @@ def run_test(sample, test=None, timeout=6, started_flag: Synchronized | None = N
                 )
                 return results, metadata
             except Exception as e:
+                # Reached only if the submission's code fails to compile or
+                # doesn't define the expected function -- grade_call_based's
+                # own per-test-case loop already handles runtime errors.
                 return [-4], {
+                    "error": repr(e),
                     "error_code": -4,
                     "error_message": f"Error during testing: {e}",
                 }
@@ -548,7 +552,10 @@ def run_test(sample, test=None, timeout=6, started_flag: Synchronized | None = N
                 )
                 return results, metadata
             except Exception as e:
+                # Same as the call_based branch above: a compile/definition
+                # failure in the submission's own code, not a judge bug.
                 return [-4], {
+                    "error": repr(e),
                     "error_code": -4,
                     "error_message": f"Error during testing: {e}",
                 }
diff --git a/tests/unit/evaluation/test_lcb_serve.py b/tests/unit/evaluation/test_lcb_serve.py
index b8de395bd..5e580353e 100644
--- a/tests/unit/evaluation/test_lcb_serve.py
+++ b/tests/unit/evaluation/test_lcb_serve.py
@@ -30,6 +30,7 @@
 real pool and monkeypatch _MP_POOL_CTX to fork for the same reason.
 """
 
+import multiprocessing as mp
 import os
 
 import pytest
@@ -113,6 +114,19 @@ def _guard_then_die(*args, **kwargs):
     assert metadata["error_code"] in lcb_serve._LCB_INFRA_ERROR_CODES
 
 
+def test_compile_error_in_submission_is_not_infra_but_is_logged():
+    """A submission that doesn't even compile hits the outer except in
+    grade_call_based's caller, gets -4, and is the submission's fault, not
+    the judge's -- but must still carry an "error" key so it's visible."""
+    bad_syntax = "def solve(x\n    return x\n"  # missing closing paren
+    res, metadata = lcb_serve.run_code_subprocess(
+        CALL_BASED_SUITE, bad_syntax, timeout_sec=TIMEOUT_SEC
+    )
+    assert metadata["error_code"] == -4
+    assert "error" in metadata
+    assert metadata["error_code"] not in lcb_serve._LCB_INFRA_ERROR_CODES
+
+
 def test_timeout_is_not_infra_error():
     # grade_call_based enforces its own per-test-case timeout via
     # signal.alarm and returns a normal -3 result well within it, so
@@ -148,6 +162,52 @@ def test_all_os_exit_batch_scores_zero_without_raising():
     assert results == {"q1": [False], "q2": [False]}
 
 
+def test_submission_error_logs_at_warning_not_error(caplog):
+    """Routine submission-attributed outcomes (-8 here) must not log at
+    ERROR -- that level is reserved for judge-side infra failures, so ops
+    can alert on it without being flooded by ordinary bad submissions."""
+    worker = lcb_serve._LCBWorker(
+        _DictTestLoader(q1=CALL_BASED_SUITE),
+        n_lcb_workers=1,
+        worker_timeout_sec=TIMEOUT_SEC,
+    )
+    with caplog.at_level("WARNING", logger=lcb_serve.logger.name):
+        results = worker(["q1"], [[CALL_OS_EXIT]])
+
+    assert results == {"q1": [False]}
+    levels = [r.levelname for r in caplog.records]
+    assert "WARNING" in levels
+    assert "ERROR" not in levels
+
+
+def test_infra_error_logs_at_error_level(monkeypatch, caplog):
+    """A judge-side death (-6) must log at ERROR so it's distinguishable
+    from the routine submission-attributed WARNINGs above."""
+    monkeypatch.setattr(lcb_serve, "_MP_POOL_CTX", mp.get_context("fork"))
+
+    def _die_immediately(*args, **kwargs):
+        os._exit(1)
+
+    monkeypatch.setattr(
+        lcb_serve, "execute_code_single_suppressed_errors", _die_immediately
+    )
+
+    worker = lcb_serve._LCBWorker(
+        _DictTestLoader(q1=CALL_BASED_SUITE),
+        n_lcb_workers=1,
+        worker_timeout_sec=TIMEOUT_SEC,
+    )
+    with caplog.at_level("WARNING", logger=lcb_serve.logger.name):
+        # Single-sample batch is also all-infra, so the guard raises; the
+        # log call happens before that, inside the executor's with-block.
+        with pytest.raises(RuntimeError, match="infrastructure errors"):
+            worker(["q1"], [[PASSING_CALL_BASED]])
+
+    levels = [r.levelname for r in caplog.records]
+    assert "ERROR" in levels
+    assert "WARNING" not in levels
+
+
 def test_all_judge_startup_death_batch_raises(monkeypatch):
     """A batch where every grading child dies before run_test is reached
     means the judge itself is broken, not that every sample failed -- the
@@ -158,8 +218,6 @@ def test_all_judge_startup_death_batch_raises(monkeypatch):
     would not see it. The guard logic under test lives in
     _LCBWorker.__call__ and is identical regardless of pool context.
     """
-    import multiprocessing as mp
-
     monkeypatch.setattr(lcb_serve, "_MP_POOL_CTX", mp.get_context("fork"))
 
     def _die_immediately(*args, **kwargs):

From 86a69864206873dc46eb4182c6117e4d94e5c4ac Mon Sep 17 00:00:00 2001
From: Liang Yan <lyan@coreweave.com>
Date: Thu, 13 Aug 2026 09:16:11 -0400
Subject: [PATCH 12/14] test(lcb-service): add pytestmark = pytest.mark.unit to
 test_lcb_serve.py

---
 tests/unit/evaluation/test_lcb_serve.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tests/unit/evaluation/test_lcb_serve.py b/tests/unit/evaluation/test_lcb_serve.py
index 5e580353e..3d8aed31a 100644
--- a/tests/unit/evaluation/test_lcb_serve.py
+++ b/tests/unit/evaluation/test_lcb_serve.py
@@ -36,6 +36,8 @@
 import pytest
 from inference_endpoint.evaluation.livecodebench import lcb_serve, run_lcb_tests
 
+pytestmark = pytest.mark.unit
+
 CALL_BASED_SUITE = '{"fn_name": "solve", "inputs": ["1"], "outputs": ["1"]}'
 PASSING_CALL_BASED = "def solve(x):\n    return x\n"
 CALL_SYSEXIT = "def solve(x):\n    import sys\n    sys.exit(3)\n"

From c14c3c2c5ac3476539e438ad6af92dc2da0ba980 Mon Sep 17 00:00:00 2001
From: Liang Yan <lyan@coreweave.com>
Date: Wed, 19 Aug 2026 14:35:45 -0400
Subject: [PATCH 13/14] revert(lcb-service): drop explicit spawn/fork pinning,
 defer to a follow-up PR
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Reviewer flagged a perf concern with pinning the pool to spawn; dropping the
explicit context here and following up with a separate PR + perf numbers.
Everything else (SystemExit, started_flag attribution, Manager with-block,
type annotations, logging levels) stays as-is.

Verified in a python:3.11-slim container (< 3.14, defaults to fork) — all
11 regression tests pass.
---
 .../evaluation/livecodebench/lcb_serve.py     | 27 ++++---------------
 tests/unit/evaluation/test_lcb_serve.py       | 23 +++++++++-------
 2 files changed, 18 insertions(+), 32 deletions(-)

diff --git a/src/inference_endpoint/evaluation/livecodebench/lcb_serve.py b/src/inference_endpoint/evaluation/livecodebench/lcb_serve.py
index aef992a09..cb0db9d73 100644
--- a/src/inference_endpoint/evaluation/livecodebench/lcb_serve.py
+++ b/src/inference_endpoint/evaluation/livecodebench/lcb_serve.py
@@ -57,17 +57,6 @@
 
 logger = logging.getLogger(__name__)
 
-# The pool is created from a worker thread of the event loop's default
-# executor (the server calls evaluate via run_in_executor), so it must not
-# fork the multithreaded parent; spawn (fork+exec) is safe from threads.
-# forkserver would also work in theory but hangs at pool shutdown in this
-# container.
-_MP_POOL_CTX = mp.get_context("spawn")
-
-# Grading assumes fork semantics; forking here is safe because the
-# spawn-started pool worker is freshly exec'd and single-threaded.
-_MP_CTX = mp.get_context("fork")
-
 # Error codes that mean the judge is broken, as opposed to the submitted code
 # failing its tests: -5 TestRunnerError, -6 GradingChildDied (child died
 # before grading started, e.g. a bad start method). Submission-attributed
@@ -169,11 +158,11 @@ def run_code_subprocess(
         suite["inputs"]
     ) + flat_timeout_extension
 
-    with _MP_CTX.Manager() as manager:
+    with mp.Manager() as manager:
         resp_buffer = manager.list()
         # typeshed types ctx.Value() as SynchronizedBase, which lacks .value
-        started_flag = cast(Synchronized, _MP_CTX.Value(ctypes.c_bool, False))
-        p = _MP_CTX.Process(
+        started_flag = cast(Synchronized, mp.Value(ctypes.c_bool, False))
+        p = mp.Process(
             target=execute_code_single_suppressed_errors,
             args=(
                 test_suite_json,
@@ -351,9 +340,7 @@ def __call__(
         futures = {}
         infra_errors = 0
 
-        with ProcessPoolExecutor(
-            max_workers=self.n_lcb_workers, mp_context=_MP_POOL_CTX
-        ) as executor:
+        with ProcessPoolExecutor(max_workers=self.n_lcb_workers) as executor:
             for qid, test_codes in zip(question_ids, codes, strict=False):
                 test_suite_json = self.test_loader[qid]
                 for i, code in enumerate(test_codes):
@@ -451,11 +438,7 @@ def __init__(
         if n_workers is None:
             n_workers = mp.cpu_count() // 2
         logger.info("Using %d workers for LCB eval", n_workers)
-        logger.info(
-            "Multiprocessing start methods: pool=%s, grading child=%s",
-            _MP_POOL_CTX.get_start_method(),
-            _MP_CTX.get_start_method(),
-        )
+        logger.info("Multiprocessing start method: %s", mp.get_start_method())
         self.n_workers = n_workers
 
         self.path_to_dataset = (
diff --git a/tests/unit/evaluation/test_lcb_serve.py b/tests/unit/evaluation/test_lcb_serve.py
index 3d8aed31a..2739a0ee8 100644
--- a/tests/unit/evaluation/test_lcb_serve.py
+++ b/tests/unit/evaluation/test_lcb_serve.py
@@ -25,12 +25,12 @@
                          run_test's dispatch into it, was executing)
 
 These tests call run_code_subprocess directly (not through _LCBWorker's
-spawn pool) so that monkeypatches applied in the test process are inherited
-by the forked grading child; the batch-level guard tests below need the
-real pool and monkeypatch _MP_POOL_CTX to fork for the same reason.
+pool) so that monkeypatches applied in the test process are inherited by
+the forked grading child; the batch-level guard tests below go through the
+real pool instead, and rely on the interpreter's default start method
+being "fork" (true on 3.11/3.12) for the same reason.
 """
 
-import multiprocessing as mp
 import os
 
 import pytest
@@ -184,8 +184,11 @@ def test_submission_error_logs_at_warning_not_error(caplog):
 
 def test_infra_error_logs_at_error_level(monkeypatch, caplog):
     """A judge-side death (-6) must log at ERROR so it's distinguishable
-    from the routine submission-attributed WARNINGs above."""
-    monkeypatch.setattr(lcb_serve, "_MP_POOL_CTX", mp.get_context("fork"))
+    from the routine submission-attributed WARNINGs above.
+
+    Relies on the pool using the default "fork" start method (true on
+    3.11/3.12) so the monkeypatch below reaches the pool workers.
+    """
 
     def _die_immediately(*args, **kwargs):
         os._exit(1)
@@ -215,12 +218,12 @@ def test_all_judge_startup_death_batch_raises(monkeypatch):
     means the judge itself is broken, not that every sample failed -- the
     all-infra guard must raise instead of silently reporting a 0.
 
-    Uses a fork pool so the monkeypatch below (applied in this process)
-    reaches the pool workers; spawn workers re-import the module fresh and
-    would not see it. The guard logic under test lives in
+    Relies on the pool using the default "fork" start method (true on
+    3.11/3.12) so the monkeypatch below (applied in this process) reaches
+    the pool workers; a spawn/forkserver worker would re-import the module
+    fresh and not see it. The guard logic under test lives in
     _LCBWorker.__call__ and is identical regardless of pool context.
     """
-    monkeypatch.setattr(lcb_serve, "_MP_POOL_CTX", mp.get_context("fork"))
 
     def _die_immediately(*args, **kwargs):
         os._exit(1)

From 7038582ec125e7c755353d64a448cf4ca17bbbca Mon Sep 17 00:00:00 2001
From: Liang Yan <lyan@coreweave.com>
Date: Wed, 19 Aug 2026 15:40:38 -0400
Subject: [PATCH 14/14] test(lcb-service): pin fault-injection tests to fork
 explicitly

CI's full test run already forces the process-wide multiprocessing default
to spawn (endpoint_client/worker.py sets it at import time), breaking these
tests' fork-only monkeypatch assumptions. Force fork locally in the tests
that need it instead of relying on the ambient default.
---
 tests/unit/evaluation/test_lcb_serve.py | 51 +++++++++++++++++++------
 1 file changed, 40 insertions(+), 11 deletions(-)

diff --git a/tests/unit/evaluation/test_lcb_serve.py b/tests/unit/evaluation/test_lcb_serve.py
index 2739a0ee8..154cc0c80 100644
--- a/tests/unit/evaluation/test_lcb_serve.py
+++ b/tests/unit/evaluation/test_lcb_serve.py
@@ -27,17 +27,42 @@
 These tests call run_code_subprocess directly (not through _LCBWorker's
 pool) so that monkeypatches applied in the test process are inherited by
 the forked grading child; the batch-level guard tests below go through the
-real pool instead, and rely on the interpreter's default start method
-being "fork" (true on 3.11/3.12) for the same reason.
+real pool instead, for the same reason.
+
+The fault-injection tests below need "fork" specifically -- not just as the
+interpreter's default, but pinned explicitly -- because pytest runs this
+whole suite in one process, and some other module (see
+endpoint_client/worker.py's multiprocessing.set_start_method("spawn"),
+called at import time) may have already claimed the process-wide default
+before this file's tests run. Pinning fork here, locally, keeps the fault
+injection deterministic regardless of import order without deciding
+lcb_serve's own production start-method policy, which is being tracked in
+a follow-up PR.
 """
 
+import multiprocessing
 import os
+from concurrent.futures import ProcessPoolExecutor
+from functools import partial
 
 import pytest
 from inference_endpoint.evaluation.livecodebench import lcb_serve, run_lcb_tests
 
 pytestmark = pytest.mark.unit
 
+
+def _force_fork_context(monkeypatch: pytest.MonkeyPatch) -> None:
+    """Pin lcb_serve's pool and inner Process/Manager/Value to "fork" for a
+    single test, regardless of the process-wide multiprocessing default."""
+    fork_ctx = multiprocessing.get_context("fork")
+    monkeypatch.setattr(lcb_serve, "mp", fork_ctx)
+    monkeypatch.setattr(
+        lcb_serve,
+        "ProcessPoolExecutor",
+        partial(ProcessPoolExecutor, mp_context=fork_ctx),
+    )
+
+
 CALL_BASED_SUITE = '{"fn_name": "solve", "inputs": ["1"], "outputs": ["1"]}'
 PASSING_CALL_BASED = "def solve(x):\n    return x\n"
 CALL_SYSEXIT = "def solve(x):\n    import sys\n    sys.exit(3)\n"
@@ -72,9 +97,10 @@ def test_os_exit_in_submission_is_submission_killed_child_not_infra():
     assert metadata["error_code"] not in lcb_serve._LCB_INFRA_ERROR_CODES
 
 
-def test_death_before_run_test_is_infra_error():
+def test_death_before_run_test_is_infra_error(monkeypatch):
     """A judge process that dies before run_test is even reached (e.g. a
     bad start method, or a crash while importing) is -6, not -8."""
+    _force_fork_context(monkeypatch)
     orig = lcb_serve.execute_code_single_suppressed_errors
 
     def _die_immediately(*args, **kwargs):
@@ -92,12 +118,13 @@ def _die_immediately(*args, **kwargs):
     assert metadata["error_code"] in lcb_serve._LCB_INFRA_ERROR_CODES
 
 
-def test_death_during_judge_setup_is_infra_error_not_submission():
+def test_death_during_judge_setup_is_infra_error_not_submission(monkeypatch):
     """A death inside run_test's own setup (reliability_guard, suite parse)
     -- after run_test has started but before started_flag is set -- must
     still be -6. This is the exact boundary the started_flag fix moved: it
     used to flip True on wrapper entry, before this setup ran, which would
     have misattributed this case as -8."""
+    _force_fork_context(monkeypatch)
     orig_guard = run_lcb_tests.reliability_guard
 
     def _guard_then_die(*args, **kwargs):
@@ -186,9 +213,10 @@ def test_infra_error_logs_at_error_level(monkeypatch, caplog):
     """A judge-side death (-6) must log at ERROR so it's distinguishable
     from the routine submission-attributed WARNINGs above.
 
-    Relies on the pool using the default "fork" start method (true on
-    3.11/3.12) so the monkeypatch below reaches the pool workers.
+    Forces the pool onto "fork" so the monkeypatch below reaches the pool
+    workers -- see _force_fork_context.
     """
+    _force_fork_context(monkeypatch)
 
     def _die_immediately(*args, **kwargs):
         os._exit(1)
@@ -218,12 +246,13 @@ def test_all_judge_startup_death_batch_raises(monkeypatch):
     means the judge itself is broken, not that every sample failed -- the
     all-infra guard must raise instead of silently reporting a 0.
 
-    Relies on the pool using the default "fork" start method (true on
-    3.11/3.12) so the monkeypatch below (applied in this process) reaches
-    the pool workers; a spawn/forkserver worker would re-import the module
-    fresh and not see it. The guard logic under test lives in
-    _LCBWorker.__call__ and is identical regardless of pool context.
+    Forces the pool onto "fork" so the monkeypatch below (applied in this
+    process) reaches the pool workers -- see _force_fork_context. A
+    spawn/forkserver worker would re-import the module fresh and not see
+    it. The guard logic under test lives in _LCBWorker.__call__ and is
+    identical regardless of pool context.
     """
+    _force_fork_context(monkeypatch)
 
     def _die_immediately(*args, **kwargs):
         os._exit(1)