From 7e948eb12f17c49067f31e7f41b3eda159047034 Mon Sep 17 00:00:00 2001 From: Prann Bansal Date: Fri, 7 Aug 2026 19:15:55 +0530 Subject: [PATCH] parse: prefer top-level class over nested def in extract_function_name `extract_function_name` searches for `def ...(` first and only falls back to `class ...(`. A function_header for a class-based step carries the class line *and* its first method: class Maxwell: """ The base class for evolution of Maxwell's equations. """ def __init__(self, n_grid, x_out): so the `def` pattern matches first and the name comes back as `__init__`. `get_function_from_code` then returns only that method, and the class -- plus every other method on it -- is dropped from the code accumulated into later sub-steps' prompts and test scripts. Any later sub-step that instantiates the class fails with NameError regardless of what the model generated. Two related problems in the same function: * the class pattern `\bclass\s+(\w+)\s*\(` requires a base-class paren, so `class Maxwell:` would not match even if it were tried first; * both patterns are unanchored, so they can match inside the header's docstring -- several headers document a parameter as `env: class Block`, which yields the "function name" `env`. Anchoring to a top-level definition line fixes all three: the first line at column 0 opening a `class` or `def` is by construction the declared signature. Measured over the 341 published sub-step headers: 11 change, all of them genuine class headers that previously resolved to `__init__` (13.6, 30.1, 30.2, 30.3, 46.1, 46.2, 62.1, 68.1, 68.2, 68.3, 68.4). The other 330 are identical. Fraction of each skip-step's reference file that survives extraction: 13.6 1855/2098 (88%, a bare __init__) -> 2016/2098 (96%, class Maxwell) 62.1 151/978 (15%, a bare __init__) -> 493/978 (50%, class EnlargedBlock) 76.3 987/1020 (97%) -> 987/1020 (97%, unchanged) NOTE this changes benchmark scores: sub-steps downstream of a class-based step were previously unwinnable. SciCode numbers produced before and after this commit are not comparable. Refs #49, #59. Co-Authored-By: Claude Opus 5 --- src/scicode/parse/parse.py | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/src/scicode/parse/parse.py b/src/scicode/parse/parse.py index 3403780..ac50b25 100644 --- a/src/scicode/parse/parse.py +++ b/src/scicode/parse/parse.py @@ -14,18 +14,26 @@ H5PY_FILE = "eval/data/test_data.h5" +# The first line at column 0 that opens a `class` or a `def`. Anchoring to the +# start of a line keeps docstring prose (several headers document a parameter as +# `env: class Block`) from being read as the definition, and taking whichever +# comes first keeps a class header from resolving to the `def` of its own first +# method. `class Foo:` is matched as well as `class Foo(Base):`. +_TOP_LEVEL_DEF = re.compile( + r'^(?:class\s+(\w+)\s*[(:]|def\s+(\w+)\s*\()', re.MULTILINE) + + def extract_function_name(function_header): - pattern = r'\bdef\s+(\w+)\s*\(' - match = re.search(pattern, function_header) + """Return the name of the class or function a step's header declares. + + A class-based header carries its first method too, so the class must win; + otherwise the caller extracts only that method and silently drops the rest + of the class from the code accumulated into later sub-steps. + """ + match = _TOP_LEVEL_DEF.search(function_header) if match: - return match.group(1) - else: - pattern = r'\bclass\s+(\w+)\s*\(' - match = re.search(pattern, function_header) - if match: - return match.group(1) - else: - raise ValueError('Function name or class name not found.') + return match.group(1) or match.group(2) + raise ValueError('Function name or class name not found.') def get_function_from_code(code_string, function_name): """