From 08d83ad77e12b45f572db267312fd4e83b98abc9 Mon Sep 17 00:00:00 2001 From: Darreck Date: Mon, 27 Jul 2026 05:05:16 +0300 Subject: [PATCH 1/2] add bounded parallel self-hosting controls --- openevolve/api.py | 11 +- openevolve/config.py | 85 +++- openevolve/controller.py | 50 +- openevolve/evaluation_result.py | 3 + openevolve/evaluator.py | 17 +- openevolve/llm/ensemble.py | 5 +- openevolve/llm/openai.py | 38 +- openevolve/process_parallel.py | 509 +++++++++++++++++++-- openevolve/utils/code_utils.py | 142 ++++++ tests/test_api.py | 67 ++- tests/test_code_utils.py | 90 ++++ tests/test_controller_scheduler.py | 91 ++++ tests/test_evaluator_timeout.py | 2 + tests/test_initial_program_artifacts.py | 14 + tests/test_llm_budget_config.py | 66 +++ tests/test_process_parallel.py | 578 ++++++++++++++++++++++++ tests/test_reasoning_effort_config.py | 18 +- 17 files changed, 1727 insertions(+), 59 deletions(-) create mode 100644 tests/test_controller_scheduler.py create mode 100644 tests/test_llm_budget_config.py diff --git a/openevolve/api.py b/openevolve/api.py index 9452391763..3f77aff3b5 100644 --- a/openevolve/api.py +++ b/openevolve/api.py @@ -8,7 +8,7 @@ import uuid import inspect from typing import Union, Callable, Optional, List, Dict, Any, Tuple -from dataclasses import dataclass +from dataclasses import dataclass, field from pathlib import Path from openevolve.controller import OpenEvolve @@ -25,6 +25,10 @@ class EvolutionResult: best_code: str metrics: Dict[str, Any] output_dir: Optional[str] + completion_reason: str = "unknown" + last_completed_iteration: Optional[int] = None + completed_iteration_count: int = 0 + llm_usage: Dict[str, Any] = field(default_factory=dict) def __repr__(self): return f"EvolutionResult(best_score={self.best_score:.4f})" @@ -178,12 +182,17 @@ async def _run_evolution_async( if numeric_metrics: best_score = sum(numeric_metrics) / len(numeric_metrics) + llm_usage = getattr(controller, "llm_usage", {}) return EvolutionResult( best_program=best_program, best_score=best_score, best_code=best_code, metrics=metrics, output_dir=actual_output_dir if not cleanup else None, + completion_reason=controller.completion_reason, + last_completed_iteration=controller.last_completed_iteration, + completed_iteration_count=controller.completed_iteration_count, + llm_usage=dict(llm_usage) if isinstance(llm_usage, dict) else {}, ) finally: diff --git a/openevolve/config.py b/openevolve/config.py index c19ab4ca1d..6358b7d71a 100644 --- a/openevolve/config.py +++ b/openevolve/config.py @@ -4,6 +4,7 @@ import os import re +import math from dataclasses import asdict, dataclass, field from pathlib import Path from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Union @@ -306,6 +307,20 @@ class PromptConfig: ) +@dataclass +class ControllerSchedulerConfig: + """Optional observed-result island allocator for bounded live comparisons.""" + + enabled: bool = False + exploitation_weight: float = 0.0 + underexplored_weight: float = 0.0 + validity_weight: float = 0.0 + diversity_weight: float = 0.0 + token_efficiency_weight: float = 0.0 + rejection_penalty: float = 0.0 + minimum_calls: int = 1 + + @dataclass class DatabaseConfig: """Configuration for the program database""" @@ -326,6 +341,9 @@ class DatabaseConfig: elite_selection_ratio: float = 0.1 exploration_ratio: float = 0.2 exploitation_ratio: float = 0.7 + controller_scheduler: ControllerSchedulerConfig = field( + default_factory=ControllerSchedulerConfig + ) # Note: diversity_metric fixed to "edit_distance" diversity_metric: str = "edit_distance" # Options: "edit_distance", "feature_based" @@ -418,6 +436,10 @@ class Config: # General settings max_iterations: int = 10000 + # Optional per-run proposal-model budgets. Calls are reserved before + # submission; provider-token totals come from completed unique receipts. + max_llm_calls: Optional[int] = None + max_total_provider_tokens: Optional[int] = None checkpoint_interval: int = 100 log_level: str = "INFO" log_dir: Optional[str] = None @@ -436,6 +458,9 @@ class Config: diff_based_evolution: bool = True max_code_length: int = 10000 diff_pattern: str = r"<<<<<<< SEARCH\n(.*?)=======\n(.*?)>>>>>>> REPLACE" + strict_diff_application: bool = False + enforce_evolve_blocks: bool = False + max_diff_blocks: int = 32 # Early stopping settings early_stopping_patience: Optional[int] = None @@ -489,13 +514,67 @@ def from_dict(cls, config_dict: Dict[str, Any]) -> "Config": if config.database.random_seed is None and config.random_seed is not None: config.database.random_seed = config.random_seed - if config.prompt.programs_as_changes_description and not config.diff_based_evolution: + config.validate() + return config + + def validate(self) -> None: + """Validate combinations used by both YAML and programmatic callers.""" + for field_name in ("max_llm_calls", "max_total_provider_tokens"): + value = getattr(self, field_name) + if value is not None and ( + isinstance(value, bool) or not isinstance(value, int) or value < 1 + ): + raise ValueError(f"{field_name} must be a positive integer or None") + try: + re.compile(self.diff_pattern) + except re.error as error: + raise ValueError(f"Invalid regex pattern in diff_pattern: {error}") from error + if self.prompt.programs_as_changes_description and not self.diff_based_evolution: raise ValueError( "prompt.programs_as_changes_description=true requires diff_based_evolution=true " "(full rewrites cannot reliably update code and changes_description together)" ) - - return config + if self.enforce_evolve_blocks and not self.strict_diff_application: + raise ValueError( + "enforce_evolve_blocks=true requires strict_diff_application=true" + ) + if self.enforce_evolve_blocks and not self.diff_based_evolution: + raise ValueError( + "enforce_evolve_blocks=true requires diff_based_evolution=true" + ) + if self.max_diff_blocks < 1: + raise ValueError("max_diff_blocks must be at least 1") + scheduler = self.database.controller_scheduler + if not isinstance(scheduler.enabled, bool): + raise ValueError("database.controller_scheduler.enabled must be boolean") + for field_name in ( + "exploitation_weight", + "underexplored_weight", + "validity_weight", + "diversity_weight", + "token_efficiency_weight", + "rejection_penalty", + ): + value = getattr(scheduler, field_name) + if ( + isinstance(value, bool) + or not isinstance(value, (int, float)) + or not math.isfinite(float(value)) + or value < 0 + ): + raise ValueError( + f"database.controller_scheduler.{field_name} " + "must be finite and nonnegative" + ) + if ( + isinstance(scheduler.minimum_calls, bool) + or not isinstance(scheduler.minimum_calls, int) + or scheduler.minimum_calls < 1 + ): + raise ValueError( + "database.controller_scheduler.minimum_calls " + "must be a positive integer" + ) def to_dict(self) -> Dict[str, Any]: return asdict(self) diff --git a/openevolve/controller.py b/openevolve/controller.py index a3f096bf8b..c83be5bf1d 100644 --- a/openevolve/controller.py +++ b/openevolve/controller.py @@ -48,6 +48,7 @@ def __init__( ): # Load configuration (loaded in main_async) self.config = config + self.config.validate() # Set up output directory self.output_dir = output_dir or os.path.join( @@ -163,6 +164,12 @@ def __init__( # Initialize improved parallel processing components self.parallel_controller = None + # Preserve authoritative run completion metadata after the process + # controller is stopped and released. + self.completion_reason = "not_started" + self.last_completed_iteration: Optional[int] = None + self.completed_iteration_count = 0 + self.llm_usage: Dict[str, Any] = {} def _setup_logging(self) -> None: """Set up logging""" @@ -314,6 +321,7 @@ async def run( self.database, self.evolution_tracer, file_suffix=self.config.file_suffix, + usage_output_path=os.path.join(self.output_dir, "llm_usage.jsonl"), ) # Set up signal handlers for graceful shutdown @@ -354,6 +362,15 @@ def force_exit_handler(signum, frame): finally: # Clean up parallel processing resources if self.parallel_controller: + self.completion_reason = self.parallel_controller.completion_reason + self.last_completed_iteration = ( + self.parallel_controller.last_completed_iteration + ) + self.completed_iteration_count = ( + self.parallel_controller.completed_iteration_count + ) + llm_usage = getattr(self.parallel_controller, "llm_usage", {}) + self.llm_usage = dict(llm_usage) if isinstance(llm_usage, dict) else {} self.parallel_controller.stop() self.parallel_controller = None @@ -501,15 +518,36 @@ async def _run_evolution_with_checkpoints( if self.parallel_controller.shutdown_event.is_set(): logger.info("Evolution stopped due to shutdown request") return - elif self.parallel_controller.early_stopping_triggered: + elif getattr(self.parallel_controller, "early_stopping_triggered", False) is True: logger.info("Evolution stopped due to early stopping - saving final checkpoint") # Continue to save final checkpoint for early stopping - # Save final checkpoint if needed - # Note: start_iteration here is the evolution start (1 for fresh start, not 0) - # max_iterations is the number of evolution iterations to run - final_iteration = start_iteration + max_iterations - 1 - if final_iteration > 0 and final_iteration % self.config.checkpoint_interval == 0: + # Bind the final checkpoint to work that actually completed. A target + # reached by one worker can leave higher-numbered work in flight, so + # the requested budget is not a valid completion cursor. + final_iteration = getattr( + self.parallel_controller, "last_completed_iteration", None + ) + target_score_reached = ( + getattr(self.parallel_controller, "target_score_reached", False) is True + ) + early_stopping_triggered = ( + getattr(self.parallel_controller, "early_stopping_triggered", False) is True + ) + budget_triggered = bool( + getattr(self.parallel_controller, "budget_completion_reason", None) + ) + should_save_final = ( + isinstance(final_iteration, int) + and final_iteration > 0 + and ( + target_score_reached + or early_stopping_triggered + or budget_triggered + or final_iteration % self.config.checkpoint_interval == 0 + ) + ) + if should_save_final: self._save_checkpoint(final_iteration) def _save_best_program(self, program: Optional[Program] = None) -> None: diff --git a/openevolve/evaluation_result.py b/openevolve/evaluation_result.py index cdc355539e..608a8f548a 100644 --- a/openevolve/evaluation_result.py +++ b/openevolve/evaluation_result.py @@ -7,6 +7,9 @@ from typing import Dict, Union +EVALUATION_FAILED_METRIC = "__evaluation_failed__" + + @dataclass class EvaluationResult: """ diff --git a/openevolve/evaluator.py b/openevolve/evaluator.py index b1142ece50..90705053ed 100644 --- a/openevolve/evaluator.py +++ b/openevolve/evaluator.py @@ -19,7 +19,10 @@ from openevolve.config import EvaluatorConfig from openevolve.database import ProgramDatabase -from openevolve.evaluation_result import EvaluationResult +from openevolve.evaluation_result import ( + EVALUATION_FAILED_METRIC, + EvaluationResult, +) from openevolve.database import ProgramDatabase from openevolve.llm.ensemble import LLMEnsemble from openevolve.utils.async_utils import TaskPool, run_in_executor @@ -262,7 +265,11 @@ async def evaluate_program( "error_type": "timeout", } - return {"error": 0.0, "timeout": True} + return { + EVALUATION_FAILED_METRIC: 1.0, + "error": 0.0, + "timeout": True, + } except Exception as e: last_exception = e @@ -293,7 +300,7 @@ async def evaluate_program( logger.error( f"All evaluation attempts failed for program{program_id_str}. Last error: {str(last_exception)}" ) - return {"error": 0.0} + return {EVALUATION_FAILED_METRIC: 1.0, "error": 0.0} def _process_evaluation_result(self, result: Any) -> EvaluationResult: """ @@ -314,7 +321,9 @@ def _process_evaluation_result(self, result: Any) -> EvaluationResult: else: # Error case - return error metrics logger.warning(f"Unexpected evaluation result type: {type(result)}") - return EvaluationResult(metrics={"error": 0.0}) + return EvaluationResult( + metrics={EVALUATION_FAILED_METRIC: 1.0, "error": 0.0} + ) def get_pending_artifacts(self, program_id: str) -> Optional[Dict[str, Union[str, bytes]]]: """ diff --git a/openevolve/llm/ensemble.py b/openevolve/llm/ensemble.py index b9161382a1..0963ee5752 100644 --- a/openevolve/llm/ensemble.py +++ b/openevolve/llm/ensemble.py @@ -38,6 +38,7 @@ class LLMEnsemble: def __init__(self, models_cfg: List[LLMModelConfig]): self.models_cfg = models_cfg + self.last_call_metadata: Dict[str, object] = {} # Initialize models from the configuration self.models = [_create_model(model_cfg) for model_cfg in models_cfg] @@ -81,7 +82,9 @@ async def generate_with_context( ) -> str: """Generate text using a system message and conversational context""" model = self._sample_model() - return await model.generate_with_context(system_message, messages, **kwargs) + response = await model.generate_with_context(system_message, messages, **kwargs) + self.last_call_metadata = dict(getattr(model, "last_call_metadata", {}) or {}) + return response def _sample_model(self) -> LLMInterface: """Sample a model from the ensemble based on weights""" diff --git a/openevolve/llm/openai.py b/openevolve/llm/openai.py index f7d0648de2..129b05a12b 100644 --- a/openevolve/llm/openai.py +++ b/openevolve/llm/openai.py @@ -21,6 +21,16 @@ logger = logging.getLogger(__name__) +def _uses_provider_managed_sampling(api_base: str | None, model: str | None) -> bool: + """Whether this provider/model pair requires sampling knobs to be omitted.""" + base = str(api_base or "").rstrip("/") + name = str(model or "").lower() + return ( + base.startswith("https://generativelanguage.googleapis.com/") + and name.startswith(("gemini-3.5-", "gemini-3.6-")) + ) + + def _iso_now() -> str: return datetime.now(tz=timezone.utc).isoformat() @@ -63,6 +73,7 @@ def __init__( self.api_key = model_cfg.api_key self.random_seed = getattr(model_cfg, "random_seed", None) self.reasoning_effort = getattr(model_cfg, "reasoning_effort", None) + self.last_call_metadata: Dict[str, Any] = {} # Manual mode: enabled via llm.manual_mode in config.yaml self.manual_mode = (getattr(model_cfg, "manual_mode", False) is True) @@ -154,12 +165,15 @@ async def generate_with_context( params = { "model": self.model, "messages": formatted_messages, - "temperature": kwargs.get("temperature", self.temperature), "max_tokens": kwargs.get("max_tokens", self.max_tokens), } - top_p = kwargs.get("top_p", self.top_p) - if top_p is not None: - params["top_p"] = top_p + if not _uses_provider_managed_sampling(self.api_base, self.model): + temperature = kwargs.get("temperature", self.temperature) + if temperature is not None: + params["temperature"] = temperature + top_p = kwargs.get("top_p", self.top_p) + if top_p is not None: + params["top_p"] = top_p # Handle reasoning_effort for open source reasoning models. reasoning_effort = kwargs.get("reasoning_effort", self.reasoning_effort) @@ -221,6 +235,22 @@ async def _call_api(self, params: Dict[str, Any]) -> str: response = await loop.run_in_executor( None, lambda: self.client.chat.completions.create(**params) ) + usage = getattr(response, "usage", None) + usage_details = ( + usage.model_dump(mode="json") + if usage is not None and hasattr(usage, "model_dump") + else {} + ) + self.last_call_metadata = { + "provider_response_id": getattr(response, "id", None), + "model": getattr(response, "model", None) or self.model, + "usage": { + "prompt_tokens": getattr(usage, "prompt_tokens", None), + "completion_tokens": getattr(usage, "completion_tokens", None), + "total_tokens": getattr(usage, "total_tokens", None), + "details": usage_details, + }, + } # Logging of system prompt, user message and response content logger = logging.getLogger(__name__) logger.debug(f"API parameters: {params}") diff --git a/openevolve/process_parallel.py b/openevolve/process_parallel.py index b2cfeab788..2bc7233d17 100644 --- a/openevolve/process_parallel.py +++ b/openevolve/process_parallel.py @@ -3,6 +3,8 @@ """ import asyncio +import hashlib +import json import logging import multiprocessing as mp import pickle @@ -16,6 +18,7 @@ from openevolve.config import Config from openevolve.database import Program, ProgramDatabase +from openevolve.evaluation_result import EVALUATION_FAILED_METRIC from openevolve.utils.metrics_utils import safe_numeric_average logger = logging.getLogger(__name__) @@ -30,6 +33,7 @@ class SerializableResult: iteration_time: float = 0.0 prompt: Optional[Dict[str, str]] = None llm_response: Optional[str] = None + llm_metadata: Optional[Dict[str, Any]] = None artifacts: Optional[Dict[str, Any]] = None iteration: int = 0 error: Optional[str] = None @@ -54,6 +58,7 @@ def _worker_init(config_dict: dict, evaluation_file: str, parent_env: dict = Non # Reconstruct Config object from nested dictionaries from openevolve.config import ( Config, + ControllerSchedulerConfig, DatabaseConfig, EvaluatorConfig, LLMConfig, @@ -73,7 +78,11 @@ def _worker_init(config_dict: dict, evaluation_file: str, parent_env: dict = Non # Create other configs prompt_config = PromptConfig(**config_dict["prompt"]) - database_config = DatabaseConfig(**config_dict["database"]) + database_dict = config_dict["database"].copy() + scheduler = database_dict.get("controller_scheduler") + if isinstance(scheduler, dict): + database_dict["controller_scheduler"] = ControllerSchedulerConfig(**scheduler) + database_config = DatabaseConfig(**database_dict) evaluator_config = EvaluatorConfig(**config_dict["evaluator"]) _worker_config = Config( @@ -135,6 +144,9 @@ def _run_iteration_worker( iteration: int, db_snapshot: Dict[str, Any], parent_id: str, inspiration_ids: List[str] ) -> SerializableResult: """Run a single iteration in a worker process""" + prompt: Optional[Dict[str, str]] = None + llm_response: Optional[str] = None + llm_metadata: Optional[Dict[str, Any]] = None try: # Lazy initialization _lazy_init_worker_components() @@ -205,16 +217,42 @@ def _run_iteration_worker( ) except Exception as e: logger.error(f"LLM generation failed: {e}") - return SerializableResult(error=f"LLM generation failed: {str(e)}", iteration=iteration) + llm_metadata = dict( + getattr(_worker_llm_ensemble, "last_call_metadata", {}) or {} + ) or None + return SerializableResult( + error=f"LLM generation failed: {str(e)}", + iteration=iteration, + prompt=prompt, + llm_metadata=llm_metadata, + ) + llm_metadata = dict( + getattr(_worker_llm_ensemble, "last_call_metadata", {}) or {} + ) or None # Check for None response if llm_response is None: - return SerializableResult(error="LLM returned None response", iteration=iteration) + return SerializableResult( + error="LLM returned None response", + iteration=iteration, + prompt=prompt, + llm_metadata=llm_metadata, + ) + + def reject_proposal(message: str) -> SerializableResult: + return SerializableResult( + error=message, + iteration=iteration, + llm_response=llm_response, + llm_metadata=llm_metadata, + ) # Parse response based on evolution mode if _worker_config.diff_based_evolution: from openevolve.utils.code_utils import ( apply_diff, + apply_diff_blocks_strict, + apply_diff_strict, apply_diff_blocks, extract_diffs, format_diff_summary, @@ -223,24 +261,43 @@ def _run_iteration_worker( diff_blocks = extract_diffs(llm_response, _worker_config.diff_pattern) if not diff_blocks: - return SerializableResult( - error="No valid diffs found in response", iteration=iteration - ) + return reject_proposal("No valid diffs found in response") if _worker_config.prompt.programs_as_changes_description: try: - code_blocks, desc_blocks, _unmatched = split_diffs_by_target( + code_blocks, desc_blocks, unmatched = split_diffs_by_target( diff_blocks, code_text=parent.code, changes_description_text=parent_changes_desc, ) except Exception as e: - return SerializableResult(error=str(e), iteration=iteration) + return reject_proposal(str(e)) - child_code, _ = apply_diff_blocks(parent.code, code_blocks) - child_changes_desc, desc_applied = apply_diff_blocks( - parent_changes_desc, desc_blocks - ) + if unmatched: + return reject_proposal( + f"{len(unmatched)} SEARCH/REPLACE blocks match no declared target" + ) + if _worker_config.strict_diff_application: + try: + child_code = apply_diff_blocks_strict( + parent.code, + code_blocks, + enforce_evolve_blocks=_worker_config.enforce_evolve_blocks, + max_diff_blocks=_worker_config.max_diff_blocks, + ) + child_changes_desc = apply_diff_blocks_strict( + parent_changes_desc, + desc_blocks, + max_diff_blocks=_worker_config.max_diff_blocks, + ) + desc_applied = len(desc_blocks) + except Exception as e: + return reject_proposal(str(e)) + else: + child_code, _ = apply_diff_blocks(parent.code, code_blocks) + child_changes_desc, desc_applied = apply_diff_blocks( + parent_changes_desc, desc_blocks + ) # Must update the previous changes description if ( @@ -248,9 +305,8 @@ def _run_iteration_worker( or not child_changes_desc.strip() or child_changes_desc.strip() == parent_changes_desc.strip() ): - return SerializableResult( - error="changes_description was not updated or empty, program is discarded", - iteration=iteration, + return reject_proposal( + "changes_description was not updated or empty, program is discarded" ) changes_summary = format_diff_summary( @@ -260,7 +316,21 @@ def _run_iteration_worker( ) else: # All diffs applied only to code - child_code = apply_diff(parent.code, llm_response, _worker_config.diff_pattern) + if _worker_config.strict_diff_application: + try: + child_code = apply_diff_strict( + parent.code, + llm_response, + _worker_config.diff_pattern, + enforce_evolve_blocks=_worker_config.enforce_evolve_blocks, + max_diff_blocks=_worker_config.max_diff_blocks, + ) + except Exception as e: + return reject_proposal(str(e)) + else: + child_code = apply_diff( + parent.code, llm_response, _worker_config.diff_pattern + ) changes_summary = format_diff_summary( diff_blocks, max_line_len=_worker_config.prompt.diff_summary_max_line_len, @@ -271,18 +341,15 @@ def _run_iteration_worker( new_code = parse_full_rewrite(llm_response, _worker_config.language) if not new_code: - return SerializableResult( - error=f"No valid code found in response", iteration=iteration - ) + return reject_proposal("No valid code found in response") child_code = new_code changes_summary = "Full rewrite" # Check code length if len(child_code) > _worker_config.max_code_length: - return SerializableResult( - error=f"Generated code exceeds maximum length ({len(child_code)} > {_worker_config.max_code_length})", - iteration=iteration, + return reject_proposal( + f"Generated code exceeds maximum length ({len(child_code)} > {_worker_config.max_code_length})" ) # Evaluate the child program @@ -293,6 +360,21 @@ def _run_iteration_worker( # Get artifacts artifacts = _worker_evaluator.get_pending_artifacts(child_id) + if bool(child_metrics.pop(EVALUATION_FAILED_METRIC, 0.0)): + detail = "candidate evaluation failed" + if isinstance(artifacts, dict): + failure = artifacts.get("stderr") or artifacts.get("error_type") + if failure: + detail = f"{detail}: {failure}" + return SerializableResult( + error=detail, + iteration=iteration, + prompt=prompt, + llm_response=llm_response, + llm_metadata=llm_metadata, + artifacts=artifacts, + target_island=db_snapshot.get("sampling_island"), + ) # Create child program child_program = Program( @@ -308,6 +390,7 @@ def _run_iteration_worker( "changes": changes_summary, "parent_metrics": parent.metrics, "island": parent_island, + "llm": llm_metadata, }, ) @@ -322,6 +405,7 @@ def _run_iteration_worker( iteration_time=iteration_time, prompt=prompt, llm_response=llm_response, + llm_metadata=llm_metadata, artifacts=artifacts, iteration=iteration, target_island=target_island, @@ -329,7 +413,13 @@ def _run_iteration_worker( except Exception as e: logger.exception(f"Error in worker iteration {iteration}") - return SerializableResult(error=str(e), iteration=iteration) + return SerializableResult( + error=str(e), + iteration=iteration, + prompt=prompt, + llm_response=llm_response, + llm_metadata=llm_metadata, + ) def _wait_for_processes(processes: tuple[mp.Process, ...], timeout: float) -> list[mp.Process]: @@ -397,16 +487,45 @@ def __init__( database: ProgramDatabase, evolution_tracer=None, file_suffix: str = ".py", + usage_output_path: Optional[str] = None, ): self.config = config self.evaluation_file = evaluation_file self.database = database self.evolution_tracer = evolution_tracer self.file_suffix = file_suffix + self.usage_output_path = Path(usage_output_path) if usage_output_path else None self.executor: Optional[ProcessPoolExecutor] = None self.shutdown_event = mp.Event() self.early_stopping_triggered = False + self.target_score_reached = False + self.completion_reason = "not_started" + self.last_completed_iteration: Optional[int] = None + self.completed_iteration_count = 0 + self.submitted_proposal_count = 0 + self.llm_call_count = 0 + self.prompt_token_count = 0 + self.completion_token_count = 0 + self.total_provider_tokens = 0 + self.unreported_provider_token_calls = 0 + self.llm_call_budget_overshoot = 0 + self.provider_token_budget_overshoot = 0 + self.budget_limits_reached: List[str] = [] + self.budget_completion_reason: Optional[str] = None + self.inflight_proposals_at_budget_stop = 0 + self._seen_provider_response_ids: set[str] = set() + self._controller_island_state: Dict[int, Dict[str, Any]] = { + island_id: { + "accepted": 0, + "best_score": 0.0, + "calls": 0, + "rejected": 0, + "tokens": 0, + "unique": set(), + } + for island_id in range(config.database.num_islands) + } # Number of worker processes self.num_workers = config.evaluator.parallel_evaluations @@ -414,6 +533,233 @@ def __init__( logger.info(f"Initialized process parallel controller with {self.num_workers} workers") + @staticmethod + def _nonnegative_int(value: Any) -> Optional[int]: + """Return provider counters only when they are exact non-negative integers.""" + if isinstance(value, bool) or not isinstance(value, int) or value < 0: + return None + return value + + @staticmethod + def _budget_reason(limits_reached: List[str]) -> Optional[str]: + """Return a stable public completion reason for the first reached budget.""" + call_limit = "max_llm_calls" in limits_reached + token_limit = "max_total_provider_tokens" in limits_reached + if call_limit and token_limit: + return "max_llm_calls_and_max_total_provider_tokens_reached" + if call_limit: + return "max_llm_calls_reached" + if token_limit: + return "max_total_provider_tokens_reached" + return None + + def _refresh_budget_state(self, inflight_proposals: int) -> Optional[str]: + """Refresh budget state after a reservation or verified usage receipt.""" + limits_reached = [] + if ( + self.config.max_llm_calls is not None + and self.submitted_proposal_count >= self.config.max_llm_calls + ): + limits_reached.append("max_llm_calls") + self.llm_call_budget_overshoot = ( + self.submitted_proposal_count - self.config.max_llm_calls + ) + if ( + self.config.max_total_provider_tokens is not None + and self.total_provider_tokens >= self.config.max_total_provider_tokens + ): + limits_reached.append("max_total_provider_tokens") + self.provider_token_budget_overshoot = ( + self.total_provider_tokens - self.config.max_total_provider_tokens + ) + self.budget_limits_reached = limits_reached + + reached_reason = self._budget_reason(limits_reached) + if reached_reason and self.budget_completion_reason is None: + self.budget_completion_reason = reached_reason + self.inflight_proposals_at_budget_stop = max(0, inflight_proposals) + return self.budget_completion_reason + + def _record_controller_result( + self, + island_id: Optional[int], + result: SerializableResult, + ) -> None: + """Update only observed per-island counters used by the live allocator.""" + + if ( + not self.config.database.controller_scheduler.enabled + or not isinstance(island_id, int) + or island_id not in self._controller_island_state + ): + return + state = self._controller_island_state[island_id] + state["calls"] += 1 + metadata = result.llm_metadata if isinstance(result.llm_metadata, dict) else {} + usage = metadata.get("usage") + if isinstance(usage, dict): + total_tokens = self._nonnegative_int(usage.get("total_tokens")) + if total_tokens is not None: + state["tokens"] += total_tokens + if result.error is not None or not result.child_program_dict: + state["rejected"] += 1 + return + state["accepted"] += 1 + child = result.child_program_dict + code = child.get("code") + if isinstance(code, str): + state["unique"].add(hashlib.sha256(code.encode("utf-8")).hexdigest()) + metrics = child.get("metrics") + if isinstance(metrics, dict): + score = metrics.get("combined_score") + if not isinstance(score, (int, float)) or isinstance(score, bool): + score = safe_numeric_average(metrics) + if isinstance(score, (int, float)) and not isinstance(score, bool): + state["best_score"] = max(float(state["best_score"]), float(score)) + + def _select_controller_island( + self, + island_pending: Dict[int, List[int]], + batch_size: int, + ) -> int: + """Choose an island from prior observed results with deterministic ties.""" + + scheduler = self.config.database.controller_scheduler + available = [ + island_id + for island_id in range(self.num_islands) + if len(island_pending[island_id]) < batch_size + ] + if not available: + raise RuntimeError("controller scheduler has no available island") + total_calls = sum( + int(state["calls"]) for state in self._controller_island_state.values() + ) + if total_calls < scheduler.minimum_calls: + return min( + available, + key=lambda island_id: ( + int(self._controller_island_state[island_id]["calls"]), + len(island_pending[island_id]), + island_id, + ), + ) + + def priority(island_id: int) -> tuple[float, int]: + state = self._controller_island_state[island_id] + calls = int(state["calls"]) + accepted = int(state["accepted"]) + rejected = int(state["rejected"]) + tokens = int(state["tokens"]) + unique = len(state["unique"]) + best_score = float(state["best_score"]) + validity = accepted / calls if calls else 1.0 + diversity = unique / accepted if accepted else 1.0 + token_efficiency = best_score / max(tokens / 1000.0, 1.0) + value = ( + scheduler.exploitation_weight * best_score + + scheduler.underexplored_weight / (1.0 + calls) + + scheduler.validity_weight * validity + + scheduler.diversity_weight * diversity + + scheduler.token_efficiency_weight * token_efficiency + - scheduler.rejection_penalty + * (rejected / calls if calls else 0.0) + ) + return (-value, island_id) + + return min(available, key=priority) + + def _register_submitted_proposal(self, inflight_proposals: int) -> Optional[str]: + """Reserve one logical proposal-model call against the exact call cap.""" + self.submitted_proposal_count += 1 + return self._refresh_budget_state(inflight_proposals) + + @property + def llm_usage(self) -> Dict[str, Any]: + """Public, JSON-serializable proposal-model usage summary for this run.""" + return { + "call_budget_counting_basis": "submitted_proposals", + "provider_usage_counting_basis": "unique_provider_receipts", + "llm_calls_submitted": self.submitted_proposal_count, + "llm_calls": self.llm_call_count, + "prompt_tokens": self.prompt_token_count, + "completion_tokens": self.completion_token_count, + "total_provider_tokens": self.total_provider_tokens, + "unreported_provider_token_calls": self.unreported_provider_token_calls, + "max_llm_calls": self.config.max_llm_calls, + "max_total_provider_tokens": self.config.max_total_provider_tokens, + "llm_call_budget_overshoot": self.llm_call_budget_overshoot, + "provider_token_budget_overshoot": self.provider_token_budget_overshoot, + "limits_reached": list(self.budget_limits_reached), + "inflight_proposals_at_budget_stop": self.inflight_proposals_at_budget_stop, + } + + def _record_llm_usage( + self, + iteration: int, + result: SerializableResult, + inflight_proposals: int = 0, + ) -> Optional[str]: + """Aggregate and persist one uniquely identified provider receipt. + + The call cap is reserved before submission. Provider response identity + is required for token accounting; missing counters are never guessed. + """ + if not isinstance(result.llm_metadata, dict): + return self.budget_completion_reason + + metadata = result.llm_metadata + response_id = metadata.get("provider_response_id") + verified_receipt = isinstance(response_id, str) and bool(response_id.strip()) + duplicate_receipt = bool( + verified_receipt and response_id in self._seen_provider_response_ids + ) + usage = metadata.get("usage") + usage = usage if isinstance(usage, dict) else {} + + if verified_receipt and not duplicate_receipt: + self._seen_provider_response_ids.add(response_id) + self.llm_call_count += 1 + + prompt_tokens = self._nonnegative_int(usage.get("prompt_tokens")) + completion_tokens = self._nonnegative_int(usage.get("completion_tokens")) + total_tokens = self._nonnegative_int(usage.get("total_tokens")) + + if prompt_tokens is not None: + self.prompt_token_count += prompt_tokens + if completion_tokens is not None: + self.completion_token_count += completion_tokens + if ( + total_tokens is None + and prompt_tokens is not None + and completion_tokens is not None + ): + total_tokens = prompt_tokens + completion_tokens + + if total_tokens is None: + self.unreported_provider_token_calls += 1 + else: + self.total_provider_tokens += total_tokens + + self._refresh_budget_state(inflight_proposals) + + payload = { + **metadata, + "iteration": iteration, + "accepted_for_evaluation": result.error is None, + "verified_provider_receipt": verified_receipt, + "duplicate_provider_receipt": duplicate_receipt, + "cumulative_usage": self.llm_usage, + } + if self.usage_output_path: + self.usage_output_path.parent.mkdir(parents=True, exist_ok=True) + with self.usage_output_path.open("a", encoding="utf-8") as handle: + handle.write( + json.dumps(payload, sort_keys=True, separators=(",", ":")) + "\n" + ) + + return self.budget_completion_reason + def _serialize_config(self, config: Config) -> dict: """Serialize config object to a dictionary that can be pickled""" # Manual serialization to handle nested objects properly @@ -438,12 +784,18 @@ def _serialize_config(self, config: Config) -> dict: "database": asdict(config.database), "evaluator": asdict(config.evaluator), "max_iterations": config.max_iterations, + "max_llm_calls": config.max_llm_calls, + "max_total_provider_tokens": config.max_total_provider_tokens, "checkpoint_interval": config.checkpoint_interval, "log_level": config.log_level, "log_dir": config.log_dir, "random_seed": config.random_seed, "diff_based_evolution": config.diff_based_evolution, "max_code_length": config.max_code_length, + "diff_pattern": config.diff_pattern, + "strict_diff_application": config.strict_diff_application, + "enforce_evolve_blocks": config.enforce_evolve_blocks, + "max_diff_blocks": config.max_diff_blocks, "language": config.language, "file_suffix": self.file_suffix, } @@ -551,15 +903,23 @@ async def run_evolution( # Submit initial batch - distribute across islands batch_per_island = max(1, batch_size // self.num_islands) if batch_size > 0 else 0 current_iteration = start_iteration + stop_scheduling = False + self.completion_reason = "running" # Round-robin distribution across islands for island_id in range(self.num_islands): for _ in range(batch_per_island): - if current_iteration < total_iterations: + if current_iteration < total_iterations and not stop_scheduling: future = self._submit_iteration(current_iteration, island_id) if future: pending_futures[current_iteration] = future island_pending[island_id].append(current_iteration) + budget_reason = self._register_submitted_proposal( + inflight_proposals=len(pending_futures) + ) + if budget_reason: + stop_scheduling = True + self.completion_reason = budget_reason current_iteration += 1 next_iteration = current_iteration @@ -608,12 +968,47 @@ async def run_evolution( # Use evaluator timeout + buffer to gracefully handle stuck processes timeout_seconds = self.config.evaluator.timeout + 30 result = future.result(timeout=timeout_seconds) + budget_reason = self._record_llm_usage( + completed_iteration, + result, + inflight_proposals=len(pending_futures), + ) + completed_island = ( + result.target_island + if isinstance(result.target_island, int) + else next( + ( + island_id + for island_id, iterations in island_pending.items() + if completed_iteration in iterations + ), + None, + ) + ) + self._record_controller_result(completed_island, result) + if budget_reason: + if not stop_scheduling: + logger.info( + "LLM budget reached at iteration %s; draining %s in-flight " + "proposal(s) without scheduling new work", + completed_iteration, + len(pending_futures), + ) + stop_scheduling = True + self.completion_reason = budget_reason if result.error: logger.warning(f"Iteration {completed_iteration} error: {result.error}") elif result.child_program_dict: # Reconstruct program from dict child_program = Program(**result.child_program_dict) + # Capture lineage before insertion. MAP-Elites replacement + # may remove the parent while adding the child. + trace_parent_program = ( + self.database.get(result.parent_id) + if result.parent_id + else None + ) # Add to database with explicit target_island to ensure proper island placement # This fixes issue #391: children should go to the target island, not inherit @@ -630,11 +1025,7 @@ async def run_evolution( # Log evolution trace if self.evolution_tracer: - # Retrieve parent program for trace logging - parent_program = ( - self.database.get(result.parent_id) if result.parent_id else None - ) - if parent_program: + if trace_parent_program: # Determine island ID island_id = child_program.metadata.get( "island", self.database.current_island @@ -642,7 +1033,7 @@ async def run_evolution( self.evolution_tracer.log_trace( iteration=completed_iteration, - parent_program=parent_program, + parent_program=trace_parent_program, child_program=child_program, prompt=result.prompt, llm_response=result.llm_response, @@ -651,6 +1042,7 @@ async def run_evolution( metadata={ "iteration_time": result.iteration_time, "changes": child_program.metadata.get("changes", ""), + "llm": result.llm_metadata, }, ) @@ -742,7 +1134,9 @@ async def run_evolution( logger.info( f"Target score {target_score} reached at iteration {completed_iteration}" ) - break + self.target_score_reached = True + self.completion_reason = "target_score_reached" + stop_scheduling = True # Check early stopping if early_stopping_enabled and child_program.metrics: @@ -780,14 +1174,20 @@ async def run_evolution( if ( iterations_without_improvement >= self.config.early_stopping_patience + and ( + not self.config.database.controller_scheduler.enabled + or completed_iterations + 1 + >= self.config.database.controller_scheduler.minimum_calls + ) ): self.early_stopping_triggered = True + self.completion_reason = "early_stopping" + stop_scheduling = True logger.info( f"🛑 Early stopping triggered at iteration {completed_iteration}: " f"No improvement for {iterations_without_improvement} iterations " f"(best score: {best_score:.4f})" ) - break else: # Event-based early stopping @@ -798,7 +1198,8 @@ async def run_evolution( f"Task successfully solved with score {best_score:.4f}." ) self.early_stopping_triggered = True - break + self.completion_reason = "early_stopping" + stop_scheduling = True except FutureTimeoutError: logger.error( @@ -812,6 +1213,15 @@ async def run_evolution( logger.error(f"Error processing result from iteration {completed_iteration}: {e}") completed_iterations += 1 + self.completed_iteration_count = completed_iterations + self.last_completed_iteration = ( + completed_iteration + if self.last_completed_iteration is None + else max(self.last_completed_iteration, completed_iteration) + ) + self.database.last_iteration = max( + self.database.last_iteration, completed_iteration + ) # Remove completed iteration from island tracking for island_id, iteration_list in island_pending.items(): @@ -819,17 +1229,33 @@ async def run_evolution( iteration_list.remove(completed_iteration) break - # Submit next iterations maintaining island balance - for island_id in range(self.num_islands): + # Submit the next iteration either with the original balanced + # allocator or the explicitly enabled observed-result controller. + if self.config.database.controller_scheduler.enabled: + island_order = [ + self._select_controller_island(island_pending, batch_size) + ] + per_island_limit = batch_size + else: + island_order = range(self.num_islands) + per_island_limit = batch_per_island + for island_id in island_order: if ( - len(island_pending[island_id]) < batch_per_island + len(island_pending[island_id]) < per_island_limit and next_iteration < total_iterations and not self.shutdown_event.is_set() + and not stop_scheduling ): future = self._submit_iteration(next_iteration, island_id) if future: pending_futures[next_iteration] = future island_pending[island_id].append(next_iteration) + budget_reason = self._register_submitted_proposal( + inflight_proposals=len(pending_futures) + ) + if budget_reason: + stop_scheduling = True + self.completion_reason = budget_reason next_iteration += 1 break # Only submit one iteration per completion to maintain balance @@ -842,9 +1268,20 @@ async def run_evolution( # Log completion reason if self.early_stopping_triggered: logger.info("✅ Evolution completed - Early stopping triggered due to convergence") + elif self.target_score_reached: + logger.info("✅ Evolution completed - Target reached; in-flight work recorded") + elif self.budget_completion_reason: + self.completion_reason = self.budget_completion_reason + logger.info( + "✅ Evolution completed - %s; usage=%s", + self.budget_completion_reason, + self.llm_usage, + ) elif self.shutdown_event.is_set(): + self.completion_reason = "shutdown_requested" logger.info("✅ Evolution completed - Shutdown requested") else: + self.completion_reason = "maximum_iterations" logger.info("✅ Evolution completed - Maximum iterations reached") return self.database.get_best_program() diff --git a/openevolve/utils/code_utils.py b/openevolve/utils/code_utils.py index cde6a971af..62d2973e6a 100644 --- a/openevolve/utils/code_utils.py +++ b/openevolve/utils/code_utils.py @@ -6,6 +6,49 @@ from typing import Dict, List, Optional, Tuple, Union +class DiffApplicationError(ValueError): + """Raised when a strict SEARCH/REPLACE proposal cannot be applied exactly.""" + + +def validate_evolve_blocks(code: str) -> List[Tuple[int, int, str]]: + """Parse a balanced, non-nested set of EVOLVE blocks or raise.""" + lines = code.split("\n") + blocks: List[Tuple[int, int, str]] = [] + start_line: Optional[int] = None + content: List[str] = [] + for index, line in enumerate(lines): + has_start = "# EVOLVE-BLOCK-START" in line + has_end = "# EVOLVE-BLOCK-END" in line + if has_start and has_end: + raise DiffApplicationError( + f"EVOLVE markers share a line at line {index + 1}" + ) + if has_start: + if start_line is not None: + raise DiffApplicationError( + f"nested EVOLVE block starts at line {index + 1}" + ) + start_line = index + content = [] + continue + if has_end: + if start_line is None: + raise DiffApplicationError( + f"unmatched EVOLVE block end at line {index + 1}" + ) + blocks.append((start_line, index, "\n".join(content))) + start_line = None + content = [] + continue + if start_line is not None: + content.append(line) + if start_line is not None: + raise DiffApplicationError( + f"unclosed EVOLVE block starting at line {start_line + 1}" + ) + return blocks + + def parse_evolve_blocks(code: str) -> List[Tuple[int, int, str]]: """ Parse evolve blocks from code @@ -37,6 +80,105 @@ def parse_evolve_blocks(code: str) -> List[Tuple[int, int, str]]: return blocks +def _matching_line_offsets(haystack: List[str], needle: List[str]) -> List[int]: + """Return every exact line-wise occurrence of ``needle`` in ``haystack``.""" + if not needle: + return [] + return [ + index + for index in range(len(haystack) - len(needle) + 1) + if haystack[index : index + len(needle)] == needle + ] + + +def apply_diff_strict( + original_code: str, + diff_text: str, + diff_pattern: str = r"<<<<<<< SEARCH\n(.*?)=======\n(.*?)>>>>>>> REPLACE", + *, + enforce_evolve_blocks: bool = False, + max_diff_blocks: int = 32, +) -> str: + """Apply an atomic proposal with exact-one-match semantics. + + Every SEARCH block must be nonempty and match exactly once in the current + in-memory revision. Blocks are applied sequentially. When + ``enforce_evolve_blocks`` is enabled, each match must lie wholly inside an + existing EVOLVE block and the marker lines themselves cannot be replaced. + Any violation rejects the complete proposal. + """ + diff_blocks = extract_diffs(diff_text, diff_pattern) + return apply_diff_blocks_strict( + original_code, + diff_blocks, + enforce_evolve_blocks=enforce_evolve_blocks, + max_diff_blocks=max_diff_blocks, + ) + + +def apply_diff_blocks_strict( + original_code: str, + diff_blocks: List[Tuple[str, str]], + *, + enforce_evolve_blocks: bool = False, + max_diff_blocks: int = 32, +) -> str: + """Strictly apply an already parsed list of SEARCH/REPLACE blocks.""" + if not diff_blocks: + raise DiffApplicationError("proposal contains no SEARCH/REPLACE blocks") + if len(diff_blocks) > max_diff_blocks: + raise DiffApplicationError( + f"proposal contains {len(diff_blocks)} blocks; maximum is {max_diff_blocks}" + ) + original_ranges = ( + validate_evolve_blocks(original_code) if enforce_evolve_blocks else [] + ) + if enforce_evolve_blocks and not original_ranges: + raise DiffApplicationError("proposal requires at least one EVOLVE block") + + lines = original_code.split("\n") + for block_index, (search_text, replace_text) in enumerate(diff_blocks, start=1): + if not search_text: + raise DiffApplicationError(f"block {block_index} has an empty SEARCH section") + if search_text == replace_text: + raise DiffApplicationError(f"block {block_index} makes no change") + if "EVOLVE-BLOCK-START" in replace_text or "EVOLVE-BLOCK-END" in replace_text: + raise DiffApplicationError(f"block {block_index} attempts to modify scope markers") + + search_lines = search_text.split("\n") + replacement_lines = replace_text.split("\n") + matches = _matching_line_offsets(lines, search_lines) + if len(matches) != 1: + raise DiffApplicationError( + f"block {block_index} SEARCH matched {len(matches)} times; expected exactly one" + ) + start = matches[0] + stop = start + len(search_lines) + + if enforce_evolve_blocks: + current_code = "\n".join(lines) + ranges = validate_evolve_blocks(current_code) + inside_scope = any( + start > block_start and stop <= block_end + for block_start, block_end, _ in ranges + ) + if not inside_scope: + raise DiffApplicationError( + f"block {block_index} changes text outside an EVOLVE block" + ) + + lines[start:stop] = replacement_lines + + result = "\n".join(lines) + if result == original_code: + raise DiffApplicationError("proposal leaves the program unchanged") + if enforce_evolve_blocks and len(validate_evolve_blocks(result)) != len( + original_ranges + ): + raise DiffApplicationError("proposal changed the EVOLVE block structure") + return result + + def apply_diff( original_code: str, diff_text: str, diff --git a/tests/test_api.py b/tests/test_api.py index 9316efc1d1..0764e5a511 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -1,6 +1,7 @@ """ Test the library API functionality """ +import asyncio import unittest import unittest.mock import tempfile @@ -13,10 +14,12 @@ evolve_algorithm, evolve_code, EvolutionResult, + _run_evolution_async, _prepare_program, _prepare_evaluator ) -from openevolve.config import Config +from openevolve.config import Config, LLMModelConfig +from openevolve.database import Program class TestAPIFunctions(unittest.TestCase): @@ -40,7 +43,67 @@ def test_evolution_result_class(self): self.assertEqual(result.best_score, 0.85) self.assertEqual(result.best_code, "def test(): pass") + self.assertEqual(result.completion_reason, "unknown") + self.assertIsNone(result.last_completed_iteration) + self.assertEqual(result.llm_usage, {}) self.assertIn("0.8500", str(result)) + + def test_async_result_carries_authoritative_controller_completion_metadata(self): + """The public result reports the controller cursor, not an inferred score state.""" + program_file = os.path.join(self.temp_dir, "program.py") + evaluator_file = os.path.join(self.temp_dir, "evaluator.py") + with open(program_file, "w") as handle: + handle.write("def solve(): return 1\n") + with open(evaluator_file, "w") as handle: + handle.write( + "def evaluate(program_path): return {'combined_score': 1.0}\n" + ) + + config = Config() + config.llm.models = [LLMModelConfig(name="fake-model", api_key="test")] + config.evaluator.cascade_evaluation = False + best_program = Program( + id="best", + code="def solve(): return 2\n", + language="python", + metrics={"combined_score": 1.0}, + iteration_found=1, + ) + controller = unittest.mock.MagicMock() + controller.run = unittest.mock.AsyncMock(return_value=best_program) + controller.completion_reason = "target_score_reached" + controller.last_completed_iteration = 3 + controller.completed_iteration_count = 3 + controller.llm_usage = { + "llm_calls_submitted": 3, + "llm_calls": 3, + "total_provider_tokens": 1234, + "llm_call_budget_overshoot": 0, + "provider_token_budget_overshoot": 34, + } + + with unittest.mock.patch( + "openevolve.api.OpenEvolve", return_value=controller + ): + result = asyncio.run( + _run_evolution_async( + initial_program=program_file, + evaluator=evaluator_file, + config=config, + iterations=12, + output_dir=os.path.join(self.temp_dir, "output"), + cleanup=False, + target_score=1.0, + ) + ) + + self.assertEqual(result.completion_reason, "target_score_reached") + self.assertEqual(result.last_completed_iteration, 3) + self.assertEqual(result.completed_iteration_count, 3) + self.assertEqual(result.llm_usage["llm_calls_submitted"], 3) + self.assertEqual(result.llm_usage["llm_calls"], 3) + self.assertEqual(result.llm_usage["total_provider_tokens"], 1234) + self.assertEqual(result.llm_usage["provider_token_budget_overshoot"], 34) def test_prepare_program_from_file(self): """Test _prepare_program with existing file""" @@ -383,4 +446,4 @@ def test_run_evolution_cleanup_false(self): if __name__ == '__main__': - unittest.main() \ No newline at end of file + unittest.main() diff --git a/tests/test_code_utils.py b/tests/test_code_utils.py index 20f269b7a9..6adcf4d95b 100644 --- a/tests/test_code_utils.py +++ b/tests/test_code_utils.py @@ -5,10 +5,13 @@ import unittest from openevolve.utils.code_utils import ( + DiffApplicationError, _format_block_lines, apply_diff, + apply_diff_strict, extract_diffs, format_diff_summary, + validate_evolve_blocks, ) @@ -94,6 +97,93 @@ def hello(): expected_code, ) + def test_strict_diff_requires_exactly_one_match(self): + original = "x = 1\nx = 1\n" + diff = "\n".join( + ["<" * 7 + " SEARCH", "x = 1", "=" * 7, "x = 2", ">" * 7 + " REPLACE"] + ) + with self.assertRaises(DiffApplicationError): + apply_diff_strict(original, diff) + + def test_strict_diff_is_atomic(self): + original = "x = 1\ny = 2\n" + diff = "\n".join( + [ + "<" * 7 + " SEARCH", + "x = 1", + "=" * 7, + "x = 3", + ">" * 7 + " REPLACE", + "<" * 7 + " SEARCH", + "missing = 0", + "=" * 7, + "missing = 1", + ">" * 7 + " REPLACE", + ] + ) + with self.assertRaises(DiffApplicationError): + apply_diff_strict(original, diff) + self.assertEqual(original, "x = 1\ny = 2\n") + + def test_strict_diff_can_enforce_evolve_blocks(self): + original = """header = 1 +# EVOLVE-BLOCK-START +x = 1 +# EVOLVE-BLOCK-END +footer = 2""" + inside = "\n".join( + ["<" * 7 + " SEARCH", "x = 1", "=" * 7, "x = 2", ">" * 7 + " REPLACE"] + ) + self.assertIn( + "x = 2", + apply_diff_strict(original, inside, enforce_evolve_blocks=True), + ) + outside = "\n".join( + [ + "<" * 7 + " SEARCH", + "header = 1", + "=" * 7, + "header = 2", + ">" * 7 + " REPLACE", + ] + ) + with self.assertRaises(DiffApplicationError): + apply_diff_strict(original, outside, enforce_evolve_blocks=True) + + def test_strict_diff_applies_sequential_dependent_blocks(self): + original = """# EVOLVE-BLOCK-START +x = 1 +# EVOLVE-BLOCK-END""" + diff = "\n".join( + [ + "<" * 7 + " SEARCH", + "x = 1", + "=" * 7, + "x = 2", + ">" * 7 + " REPLACE", + "<" * 7 + " SEARCH", + "x = 2", + "=" * 7, + "x = 3", + ">" * 7 + " REPLACE", + ] + ) + result = apply_diff_strict(original, diff, enforce_evolve_blocks=True) + self.assertIn("x = 3", result) + + def test_evolve_blocks_must_be_balanced_and_non_nested(self): + with self.assertRaises(DiffApplicationError): + validate_evolve_blocks("# EVOLVE-BLOCK-START\nx = 1") + with self.assertRaises(DiffApplicationError): + validate_evolve_blocks("# EVOLVE-BLOCK-END") + nested = """# EVOLVE-BLOCK-START +# EVOLVE-BLOCK-START +x = 1 +# EVOLVE-BLOCK-END +# EVOLVE-BLOCK-END""" + with self.assertRaises(DiffApplicationError): + validate_evolve_blocks(nested) + class TestFormatDiffSummary(unittest.TestCase): """Tests for format_diff_summary showing actual diff content""" diff --git a/tests/test_controller_scheduler.py b/tests/test_controller_scheduler.py new file mode 100644 index 0000000000..ea9c7bbb13 --- /dev/null +++ b/tests/test_controller_scheduler.py @@ -0,0 +1,91 @@ +from __future__ import annotations + +import pytest + +from openevolve.config import Config, ControllerSchedulerConfig +from openevolve.database import ProgramDatabase +from openevolve.process_parallel import ( + ProcessParallelController, + SerializableResult, +) + + +def _controller(config: Config) -> ProcessParallelController: + return ProcessParallelController( + config, + "unused-evaluator.py", + ProgramDatabase(config.database), + ) + + +def test_controller_scheduler_loads_from_nested_config(): + config = Config.from_dict( + { + "database": { + "controller_scheduler": { + "enabled": True, + "exploitation_weight": 1.0, + "underexplored_weight": 0.2, + "validity_weight": 0.5, + "diversity_weight": 0.5, + "token_efficiency_weight": 0.5, + "rejection_penalty": 0.2, + "minimum_calls": 3, + } + } + } + ) + + assert isinstance(config.database.controller_scheduler, ControllerSchedulerConfig) + assert config.database.controller_scheduler.enabled is True + assert config.database.controller_scheduler.minimum_calls == 3 + + +def test_controller_scheduler_rejects_negative_weight(): + config = Config() + config.database.controller_scheduler.exploitation_weight = -0.1 + + with pytest.raises(ValueError, match="finite and nonnegative"): + config.validate() + + +def test_observed_result_allocator_prefers_productive_island(): + config = Config() + config.database.num_islands = 3 + config.database.controller_scheduler = ControllerSchedulerConfig( + enabled=True, + exploitation_weight=1.0, + minimum_calls=1, + ) + controller = _controller(config) + result = SerializableResult( + child_program_dict={ + "code": "def candidate(): return 1", + "metrics": {"combined_score": 0.9}, + }, + llm_metadata={ + "usage": { + "prompt_tokens": 10, + "completion_tokens": 5, + "total_tokens": 20, + } + }, + target_island=2, + ) + + controller._record_controller_result(2, result) + selected = controller._select_controller_island( + {0: [], 1: [], 2: []}, + batch_size=3, + ) + + assert selected == 2 + assert controller._controller_island_state[2]["calls"] == 1 + assert controller._controller_island_state[2]["tokens"] == 20 + + +def test_disabled_scheduler_preserves_default_configuration(): + config = Config() + + assert config.database.controller_scheduler.enabled is False + config.validate() diff --git a/tests/test_evaluator_timeout.py b/tests/test_evaluator_timeout.py index d9053e4a07..e2c16a17cc 100644 --- a/tests/test_evaluator_timeout.py +++ b/tests/test_evaluator_timeout.py @@ -305,6 +305,7 @@ async def run_test(): # Should return error result after all retries fail self.assertIn("error", result) self.assertEqual(result["error"], 0.0) + self.assertEqual(result["__evaluation_failed__"], 1.0) asyncio.run(run_test()) @@ -341,6 +342,7 @@ async def run_test(): # Should return timeout result self.assertIn("timeout", result) self.assertTrue(result["timeout"]) + self.assertEqual(result["__evaluation_failed__"], 1.0) asyncio.run(run_test()) diff --git a/tests/test_initial_program_artifacts.py b/tests/test_initial_program_artifacts.py index 079ff9eea1..b6f10e9653 100644 --- a/tests/test_initial_program_artifacts.py +++ b/tests/test_initial_program_artifacts.py @@ -84,9 +84,23 @@ def test_initial_program_artifacts_are_stored(self): instance = mock_ppc.return_value instance.start.return_value = None instance.stop.return_value = None + instance.completion_reason = "target_score_reached" + instance.last_completed_iteration = 3 + instance.completed_iteration_count = 3 + instance.llm_usage = { + "llm_calls_submitted": 3, + "llm_calls": 3, + "total_provider_tokens": 120, + } asyncio.run(controller.run(iterations=1)) + self.assertEqual(controller.completion_reason, "target_score_reached") + self.assertEqual(controller.last_completed_iteration, 3) + self.assertEqual(controller.completed_iteration_count, 3) + self.assertEqual(controller.llm_usage["llm_calls_submitted"], 3) + self.assertEqual(controller.llm_usage["total_provider_tokens"], 120) + # Exactly one (initial) program should have been added self.assertEqual(len(controller.database.programs), 1) initial_id = next(iter(controller.database.programs)) diff --git a/tests/test_llm_budget_config.py b/tests/test_llm_budget_config.py new file mode 100644 index 0000000000..03b522c26b --- /dev/null +++ b/tests/test_llm_budget_config.py @@ -0,0 +1,66 @@ +"""Tests for per-run proposal-model budget configuration.""" + +import tempfile +import unittest +from pathlib import Path + +from openevolve.config import Config, load_config + + +class TestLLMBudgetConfig(unittest.TestCase): + def test_defaults_preserve_unlimited_legacy_behavior(self): + config = Config() + + self.assertIsNone(config.max_llm_calls) + self.assertIsNone(config.max_total_provider_tokens) + config.validate() + + def test_dict_yaml_and_serialization_round_trip(self): + config = Config.from_dict( + { + "max_llm_calls": 12, + "max_total_provider_tokens": 240_000, + } + ) + + self.assertEqual(config.max_llm_calls, 12) + self.assertEqual(config.max_total_provider_tokens, 240_000) + self.assertEqual(config.to_dict()["max_llm_calls"], 12) + self.assertEqual( + config.to_dict()["max_total_provider_tokens"], + 240_000, + ) + + with tempfile.TemporaryDirectory() as temp_dir: + config_path = Path(temp_dir) / "config.yaml" + config_path.write_text( + "max_llm_calls: 7\nmax_total_provider_tokens: 9000\n", + encoding="utf-8", + ) + loaded = load_config(config_path) + + self.assertEqual(loaded.max_llm_calls, 7) + self.assertEqual(loaded.max_total_provider_tokens, 9000) + + def test_programmatic_validation_accepts_positive_integers(self): + config = Config() + config.max_llm_calls = 1 + config.max_total_provider_tokens = 1 + + config.validate() + + def test_programmatic_validation_rejects_invalid_limits(self): + for field_name in ("max_llm_calls", "max_total_provider_tokens"): + for invalid_value in (0, -1, True, 1.5, "10"): + with self.subTest(field_name=field_name, invalid_value=invalid_value): + config = Config() + setattr(config, field_name, invalid_value) + with self.assertRaisesRegex( + ValueError, + rf"^{field_name} must be a positive integer or None$", + ): + config.validate() + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_process_parallel.py b/tests/test_process_parallel.py index 23c6f92197..c93bbae2f0 100644 --- a/tests/test_process_parallel.py +++ b/tests/test_process_parallel.py @@ -3,6 +3,7 @@ """ import asyncio +import json import os from pathlib import Path import tempfile @@ -81,6 +82,20 @@ def test_controller_initialization(self): self.assertEqual(controller.num_workers, 2) self.assertIsNone(controller.executor) self.assertIsNotNone(controller.shutdown_event) + self.assertEqual(controller.llm_usage["llm_calls"], 0) + self.assertEqual(controller.llm_usage["total_provider_tokens"], 0) + + def test_worker_config_serialization_preserves_llm_budgets(self): + self.config.max_llm_calls = 9 + self.config.max_total_provider_tokens = 12_345 + controller = ProcessParallelController( + self.config, self.eval_file, self.database + ) + + serialized = controller._serialize_config(self.config) + + self.assertEqual(serialized["max_llm_calls"], 9) + self.assertEqual(serialized["max_total_provider_tokens"], 12_345) def test_controller_start_stop(self): """Test starting and stopping the controller""" @@ -213,6 +228,569 @@ async def run_test(): # Run the async test asyncio.run(run_test()) + def test_target_score_drains_submitted_batch_without_new_work(self): + """A reached target records the bounded in-flight batch and its true cursor.""" + + async def run_test(): + controller = ProcessParallelController( + self.config, self.eval_file, self.database + ) + controller.executor = Mock() + futures = {} + for iteration in (1, 2, 3): + future = MagicMock() + future.done.return_value = True + future.result.return_value = SerializableResult( + child_program_dict={ + "id": f"target_child_{iteration}", + "code": f"def target_{iteration}(): return {iteration}", + "language": "python", + "parent_id": "test_0", + "generation": 1, + "metrics": {"combined_score": 1.0}, + "iteration_found": iteration, + "metadata": {"changes": "target", "island": iteration - 1}, + }, + parent_id="test_0", + iteration=iteration, + target_island=iteration - 1, + ) + futures[iteration] = future + + with patch.object( + controller, + "_submit_iteration", + side_effect=lambda iteration, island_id: futures[iteration], + ) as submit: + await controller.run_evolution( + start_iteration=1, + max_iterations=6, + target_score=1.0, + ) + + self.assertEqual(submit.call_count, 3) + self.assertTrue(controller.target_score_reached) + self.assertEqual(controller.completion_reason, "target_score_reached") + self.assertEqual(controller.completed_iteration_count, 3) + self.assertEqual(controller.last_completed_iteration, 3) + self.assertEqual(self.database.last_iteration, 3) + for iteration in (1, 2, 3): + self.assertIn(f"target_child_{iteration}", self.database.programs) + + asyncio.run(run_test()) + + def test_llm_call_budget_reserves_exactly_and_stops_refill(self): + """The logical call reservation cap never submits excess proposals.""" + + async def run_test(): + self.config.max_llm_calls = 1 + controller = ProcessParallelController( + self.config, self.eval_file, self.database + ) + controller.executor = Mock() + futures = {} + for iteration in (1, 2, 3): + future = MagicMock() + future.done.return_value = True + future.result.return_value = SerializableResult( + error="proposal rejected after generation", + iteration=iteration, + llm_metadata={ + "provider_response_id": f"call-{iteration}", + "model": "test-model", + "usage": { + "prompt_tokens": 10, + "completion_tokens": 5, + "total_tokens": 15, + }, + }, + ) + futures[iteration] = future + + with patch.object( + controller, + "_submit_iteration", + side_effect=lambda iteration, island_id: futures[iteration], + ) as submit: + await controller.run_evolution( + start_iteration=1, + max_iterations=6, + ) + + self.assertEqual(submit.call_count, 1) + self.assertEqual(controller.completion_reason, "max_llm_calls_reached") + self.assertEqual(controller.completed_iteration_count, 1) + self.assertEqual( + controller.llm_usage, + { + "call_budget_counting_basis": "submitted_proposals", + "provider_usage_counting_basis": "unique_provider_receipts", + "llm_calls_submitted": 1, + "llm_calls": 1, + "prompt_tokens": 10, + "completion_tokens": 5, + "total_provider_tokens": 15, + "unreported_provider_token_calls": 0, + "max_llm_calls": 1, + "max_total_provider_tokens": None, + "llm_call_budget_overshoot": 0, + "provider_token_budget_overshoot": 0, + "limits_reached": ["max_llm_calls"], + "inflight_proposals_at_budget_stop": 1, + }, + ) + + asyncio.run(run_test()) + + def test_provider_token_budget_stops_refill_then_counts_drained_receipts(self): + """Token usage can cross the cap once and includes every in-flight receipt.""" + + async def run_test(): + self.config.max_total_provider_tokens = 20 + controller = ProcessParallelController( + self.config, self.eval_file, self.database + ) + controller.executor = Mock() + futures = {} + for iteration in (1, 2, 3, 4): + future = MagicMock() + future.done.return_value = True + future.result.return_value = SerializableResult( + error="proposal rejected after generation", + iteration=iteration, + llm_metadata={ + "provider_response_id": f"token-call-{iteration}", + "model": "test-model", + "usage": { + "prompt_tokens": 10, + "completion_tokens": 5, + "total_tokens": 15, + }, + }, + ) + futures[iteration] = future + + with patch.object( + controller, + "_submit_iteration", + side_effect=lambda iteration, island_id: futures[iteration], + ) as submit: + await controller.run_evolution( + start_iteration=1, + max_iterations=8, + ) + + self.assertEqual(submit.call_count, 4) + self.assertEqual( + controller.completion_reason, + "max_total_provider_tokens_reached", + ) + self.assertEqual(controller.llm_usage["llm_calls"], 4) + self.assertEqual(controller.llm_usage["llm_calls_submitted"], 4) + self.assertEqual(controller.llm_usage["total_provider_tokens"], 60) + self.assertEqual(controller.llm_usage["provider_token_budget_overshoot"], 40) + self.assertEqual( + controller.llm_usage["inflight_proposals_at_budget_stop"], + 2, + ) + + asyncio.run(run_test()) + + def test_llm_call_budget_stops_exactly_when_refill_reserves_last_call(self): + async def run_test(): + self.config.max_llm_calls = 4 + controller = ProcessParallelController( + self.config, self.eval_file, self.database + ) + controller.executor = Mock() + futures = {} + for iteration in (1, 2, 3, 4): + future = MagicMock() + future.done.return_value = True + future.result.return_value = SerializableResult( + error="proposal rejected after generation", + iteration=iteration, + llm_metadata={ + "provider_response_id": f"refill-call-{iteration}", + "usage": {"total_tokens": 10}, + }, + ) + futures[iteration] = future + + with patch.object( + controller, + "_submit_iteration", + side_effect=lambda iteration, island_id: futures[iteration], + ) as submit: + await controller.run_evolution( + start_iteration=1, + max_iterations=8, + ) + + self.assertEqual(submit.call_count, 4) + self.assertEqual(controller.completion_reason, "max_llm_calls_reached") + self.assertEqual(controller.llm_usage["llm_calls_submitted"], 4) + self.assertEqual(controller.llm_usage["llm_calls"], 4) + self.assertEqual(controller.llm_usage["llm_call_budget_overshoot"], 0) + self.assertEqual( + controller.llm_usage["inflight_proposals_at_budget_stop"], + 3, + ) + + asyncio.run(run_test()) + + def test_usage_aggregation_requires_unique_provider_receipts(self): + """Duplicate or unidentified metadata cannot consume a budget twice.""" + controller = ProcessParallelController( + self.config, self.eval_file, self.database + ) + + controller._record_llm_usage( + 1, + SerializableResult( + llm_metadata={ + "provider_response_id": "receipt-1", + "usage": { + "prompt_tokens": 6, + "completion_tokens": 4, + "total_tokens": 10, + }, + } + ), + ) + controller._record_llm_usage( + 2, + SerializableResult( + llm_metadata={ + "provider_response_id": "receipt-1", + "usage": {"total_tokens": 10}, + } + ), + ) + controller._record_llm_usage( + 3, + SerializableResult( + llm_metadata={ + "usage": { + "prompt_tokens": 100, + "completion_tokens": 100, + "total_tokens": 200, + }, + } + ), + ) + controller._record_llm_usage( + 4, + SerializableResult( + llm_metadata={ + "provider_response_id": "receipt-2", + "usage": { + "prompt_tokens": 3, + "completion_tokens": 4, + "total_tokens": None, + }, + } + ), + ) + controller._record_llm_usage( + 5, + SerializableResult( + llm_metadata={ + "provider_response_id": "receipt-3", + "usage": {}, + } + ), + ) + + self.assertEqual(controller.llm_usage["llm_calls"], 3) + self.assertEqual(controller.llm_usage["prompt_tokens"], 9) + self.assertEqual(controller.llm_usage["completion_tokens"], 8) + self.assertEqual(controller.llm_usage["total_provider_tokens"], 17) + self.assertEqual(controller.llm_usage["unreported_provider_token_calls"], 1) + + def test_same_receipt_can_reach_both_budgets(self): + self.config.max_llm_calls = 1 + self.config.max_total_provider_tokens = 10 + controller = ProcessParallelController( + self.config, self.eval_file, self.database + ) + + reason = controller._register_submitted_proposal(inflight_proposals=1) + self.assertEqual(reason, "max_llm_calls_reached") + + reason = controller._record_llm_usage( + 1, + SerializableResult( + llm_metadata={ + "provider_response_id": "both-limits", + "usage": {"total_tokens": 12}, + } + ), + inflight_proposals=0, + ) + + self.assertEqual( + reason, + "max_llm_calls_reached", + ) + self.assertEqual( + controller.llm_usage["limits_reached"], + ["max_llm_calls", "max_total_provider_tokens"], + ) + self.assertEqual(controller.llm_usage["llm_call_budget_overshoot"], 0) + self.assertEqual(controller.llm_usage["provider_token_budget_overshoot"], 2) + self.assertEqual(controller.llm_usage["inflight_proposals_at_budget_stop"], 1) + + def test_target_score_takes_precedence_when_same_receipt_reaches_budget(self): + """A solved target remains the primary run outcome while usage stays visible.""" + + async def run_test(): + self.config.max_llm_calls = 1 + controller = ProcessParallelController( + self.config, self.eval_file, self.database + ) + controller.executor = Mock() + futures = {} + for iteration in (1, 2, 3): + future = MagicMock() + future.done.return_value = True + future.result.return_value = SerializableResult( + child_program_dict={ + "id": f"budget_target_child_{iteration}", + "code": f"def target_{iteration}(): return {iteration}", + "language": "python", + "parent_id": "test_0", + "generation": 1, + "metrics": {"combined_score": 1.0}, + "iteration_found": iteration, + "metadata": {"changes": "target", "island": iteration - 1}, + }, + parent_id="test_0", + iteration=iteration, + target_island=iteration - 1, + llm_metadata={ + "provider_response_id": f"target-call-{iteration}", + "usage": {"total_tokens": 10}, + }, + ) + futures[iteration] = future + + with patch.object( + controller, + "_submit_iteration", + side_effect=lambda iteration, island_id: futures[iteration], + ): + await controller.run_evolution( + start_iteration=1, + max_iterations=6, + target_score=1.0, + ) + + self.assertEqual(controller.completion_reason, "target_score_reached") + self.assertEqual(controller.llm_usage["limits_reached"], ["max_llm_calls"]) + self.assertEqual(controller.llm_usage["llm_calls_submitted"], 1) + self.assertEqual(controller.llm_usage["llm_call_budget_overshoot"], 0) + + asyncio.run(run_test()) + + def test_evaluator_error_preserves_completed_model_call_usage(self): + """A post-generation evaluator error still produces a provider receipt.""" + + class FakeLLM: + last_call_metadata = { + "provider_response_id": "response-after-evaluator-error", + "model": "fake-model", + "usage": { + "prompt_tokens": 11, + "completion_tokens": 7, + "total_tokens": 18, + }, + } + + async def generate_with_context(self, **_kwargs): + return "\n".join( + [ + "<" * 7 + " SEARCH", + " return 1", + "=" * 7, + " return 2", + ">" * 7 + " REPLACE", + ] + ) + + class FailingEvaluator: + async def evaluate_program(self, _code, _child_id): + raise RuntimeError("measurement failed") + + class FakePromptSampler: + def build_prompt(self, **_kwargs): + return {"system": "system", "user": "user"} + + parent = Program( + id="parent", + code="def solve():\n return 1\n", + language="python", + metrics={"combined_score": 0.5}, + metadata={"island": 0}, + ) + snapshot = { + "programs": {"parent": parent.to_dict()}, + "artifacts": {}, + "current_island": 0, + "islands": [["parent"]], + "feature_dimensions": [], + "sampling_island": 0, + } + + with ( + patch.object( + process_parallel_module, + "_lazy_init_worker_components", + return_value=None, + ), + patch.object( + process_parallel_module, + "_worker_config", + self.config, + create=True, + ), + patch.object( + process_parallel_module, + "_worker_llm_ensemble", + FakeLLM(), + create=True, + ), + patch.object( + process_parallel_module, + "_worker_evaluator", + FailingEvaluator(), + create=True, + ), + patch.object( + process_parallel_module, + "_worker_prompt_sampler", + FakePromptSampler(), + create=True, + ), + ): + result = process_parallel_module._run_iteration_worker( + 7, snapshot, "parent", [] + ) + + self.assertIn("measurement failed", result.error) + self.assertEqual( + result.llm_metadata["provider_response_id"], + "response-after-evaluator-error", + ) + usage_path = Path(self.test_dir) / "usage.jsonl" + controller = ProcessParallelController( + self.config, + self.eval_file, + self.database, + usage_output_path=str(usage_path), + ) + controller._record_llm_usage(7, result) + receipt = json.loads(usage_path.read_text(encoding="utf-8")) + self.assertEqual(receipt["iteration"], 7) + self.assertFalse(receipt["accepted_for_evaluation"]) + self.assertEqual( + receipt["provider_response_id"], "response-after-evaluator-error" + ) + + def test_structured_evaluator_failure_is_not_added_to_an_island(self): + """A measured failure remains a rejected proposal with recorded usage.""" + + class FakeLLM: + last_call_metadata = { + "provider_response_id": "response-with-structured-failure", + "model": "fake-model", + "usage": { + "prompt_tokens": 5, + "completion_tokens": 3, + "total_tokens": 8, + }, + } + + async def generate_with_context(self, **_kwargs): + return "\n".join( + [ + "<" * 7 + " SEARCH", + " return 1", + "=" * 7, + " return 2", + ">" * 7 + " REPLACE", + ] + ) + + class FailedEvaluator: + async def evaluate_program(self, _code, _child_id): + return {"__evaluation_failed__": 1.0, "error": 0.0} + + def get_pending_artifacts(self, _child_id): + return {"stderr": "invalid candidate"} + + class FakePromptSampler: + def build_prompt(self, **_kwargs): + return {"system": "system", "user": "user"} + + parent = Program( + id="parent", + code="def solve():\n return 1\n", + language="python", + metrics={"combined_score": 0.5}, + metadata={"island": 0}, + ) + snapshot = { + "programs": {"parent": parent.to_dict()}, + "artifacts": {}, + "current_island": 0, + "islands": [["parent"]], + "feature_dimensions": ["combined_score"], + "sampling_island": 0, + } + + with ( + patch.object( + process_parallel_module, + "_lazy_init_worker_components", + return_value=None, + ), + patch.object( + process_parallel_module, + "_worker_config", + self.config, + create=True, + ), + patch.object( + process_parallel_module, + "_worker_llm_ensemble", + FakeLLM(), + create=True, + ), + patch.object( + process_parallel_module, + "_worker_evaluator", + FailedEvaluator(), + create=True, + ), + patch.object( + process_parallel_module, + "_worker_prompt_sampler", + FakePromptSampler(), + create=True, + ), + ): + result = process_parallel_module._run_iteration_worker( + 8, snapshot, "parent", [] + ) + + self.assertIsNone(result.child_program_dict) + self.assertIn("invalid candidate", result.error) + self.assertEqual( + result.llm_metadata["provider_response_id"], + "response-with-structured-failure", + ) + def test_request_shutdown(self): """Test graceful shutdown request""" controller = ProcessParallelController(self.config, self.eval_file, self.database) diff --git a/tests/test_reasoning_effort_config.py b/tests/test_reasoning_effort_config.py index 584c7ddfdf..5d7e1f25b2 100644 --- a/tests/test_reasoning_effort_config.py +++ b/tests/test_reasoning_effort_config.py @@ -10,7 +10,7 @@ import os from openevolve.config import Config, LLMConfig, LLMModelConfig -from openevolve.llm.openai import OpenAILLM +from openevolve.llm.openai import OpenAILLM, _uses_provider_managed_sampling class TestReasoningEffortConfig(unittest.TestCase): @@ -140,6 +140,20 @@ def test_openai_llm_uses_reasoning_effort(self): # Verify the reasoning_effort is stored self.assertEqual(llm.reasoning_effort, "high") + def test_gemini_36_uses_provider_managed_sampling(self): + self.assertTrue( + _uses_provider_managed_sampling( + "https://generativelanguage.googleapis.com/v1beta/openai/", + "gemini-3.6-flash", + ) + ) + self.assertFalse( + _uses_provider_managed_sampling( + "https://api.openai.com/v1", + "gemini-3.6-flash", + ) + ) + def test_reasoning_effort_passed_to_api_params(self): """Test that reasoning_effort is included in API call parameters""" model_cfg = Mock() @@ -208,4 +222,4 @@ def test_yaml_file_loading_with_reasoning_effort(self): if __name__ == "__main__": - unittest.main() \ No newline at end of file + unittest.main() From 2e6345ec03b47e5e3adaecdbde4e7e94d56a348e Mon Sep 17 00:00:00 2001 From: Darreck Date: Mon, 3 Aug 2026 16:47:28 +0300 Subject: [PATCH 2/2] add history-aware self-hosting search --- README.md | 42 +- openevolve/api.py | 4 +- openevolve/cli.py | 36 ++ openevolve/config.py | 166 ++++++ openevolve/controller.py | 54 ++ openevolve/database.py | 173 +++++- openevolve/evaluator.py | 12 +- openevolve/process_parallel.py | 661 +++++++++++++++++++-- openevolve/prompt/sampler.py | 12 +- openevolve/utils/metrics_utils.py | 22 +- tests/test_checkpoint_resume.py | 45 ++ tests/test_controller_scheduler.py | 263 +++++++- tests/test_island_isolation.py | 5 + tests/test_island_migration.py | 9 + tests/test_process_parallel.py | 483 ++++++++++++++- tests/test_program_identity_config.py | 129 ++++ tests/test_prompt_sampler_comprehensive.py | 21 + tests/test_selection_score.py | 94 +++ 18 files changed, 2144 insertions(+), 87 deletions(-) create mode 100644 tests/test_program_identity_config.py create mode 100644 tests/test_selection_score.py diff --git a/README.md b/README.md index 785a1d5804..50b76bbd5e 100644 --- a/README.md +++ b/README.md @@ -215,6 +215,40 @@ OpenEvolve implements a sophisticated **evolutionary coding pipeline** that goes - **Adaptive Feature Dimensions**: Custom quality-diversity metrics - **Migration Patterns**: Ring topology with controlled gene flow - **Multi-Strategy Sampling**: Elite, diverse, and exploratory selection +- **Evaluator-Defined Identity**: Set `program_identity_artifact` to reject + structurally duplicate measured children across the complete run history +- **Measured-Phenotype Identity**: Set `phenotype_identity_artifact` to reject + structurally different children whose complete measured behavior is already + present +- **Observed-Result Allocation**: Optional `database.controller_scheduler` + routes new calls using a domain-selected `score_metric`, validity, diversity, + rejection rate, exploration, and token efficiency. Set `diversity_artifact` + to count evaluator-defined niches, and reserve at least one island so live + parallel scheduling retains a real allocation choice. Optionally set + `adaptive_parallelism` below the worker count to use fast balanced warmup + followed by a fresher-context quality phase. Set it to `1` when complete + archive freshness matters more than late-stage throughput. The allocator + reads retained seed and migrated-program scores as well as child results. + Set `leader_score_band` to keep post-warmup calls within an absolute score + distance of the best retained population. Set `parent_score_band` to keep + post-warmup parents within a score distance of the selected population's + retained leader +- **Compact Measured-History Context**: Set + `prompt.archive_context_artifact` to show the proposal model a deterministic, + bounded inventory of one evaluator-defined artifact from measured programs. + The append-only values survive MAP-Elites displacement and checkpoint resume +- **Filtered Proposal Neighborhoods**: Set + `prompt.proposal_neighborhood_artifact` when an evaluator can enumerate + valid parent-local options. The controller removes identities measured + anywhere in the complete persisted history, including behaviorally rejected + or displaced programs, and renders the bounded remainder as + `proposal-options.json` without embedding domain rules in OpenEvolve +- **Prompt Artifact Selection**: Set ordered + `prompt.artifact_include_names` to keep complete evaluator receipts in the + archive while rendering only task-relevant artifacts to the proposal model +- **Auditable Rejections**: Usage receipts distinguish measured candidates + from archive admission and record the selected parent plus a provider-response + digest without duplicating full prompt content @@ -859,7 +893,7 @@ Just set the `api_base` in your config to point to your endpoint. **Multiple success metrics:** -1. **Primary Metric**: Your evaluator's `combined_score` or metric average +1. **Primary Metric**: `selection_score` when supplied, then `combined_score` 2. **Convergence**: Best score improvement over time 3. **Diversity**: MAP-Elites grid coverage 4. **Efficiency**: Iterations to reach target performance @@ -867,6 +901,12 @@ Just set the `api_base` in your config to point to your endpoint. **Use the visualizer** to track all metrics in real-time and identify when evolution has converged. +For exact scientific searches, an evaluator can additionally return +`selection_eligible`. A value at or below zero retains the attempt in lineage +records but excludes it from parent selection, MAP-Elites cells, and the elite +archive. This separates non-negotiable admission checks from the +multi-objective quality score while preserving older evaluator behavior. + ### **Contributors** diff --git a/openevolve/api.py b/openevolve/api.py index 3f77aff3b5..f81622f673 100644 --- a/openevolve/api.py +++ b/openevolve/api.py @@ -175,7 +175,9 @@ async def _run_evolution_async( best_code = best_program.code metrics = best_program.metrics or {} - if "combined_score" in metrics: + if "selection_score" in metrics: + best_score = metrics["selection_score"] + elif "combined_score" in metrics: best_score = metrics["combined_score"] elif metrics: numeric_metrics = [v for v in metrics.values() if isinstance(v, (int, float))] diff --git a/openevolve/cli.py b/openevolve/cli.py index d1da28137c..0b45992cea 100644 --- a/openevolve/cli.py +++ b/openevolve/cli.py @@ -4,9 +4,11 @@ import argparse import asyncio +import json import logging import os import sys +from pathlib import Path from typing import Dict, List, Optional from openevolve import OpenEvolve @@ -25,6 +27,16 @@ def parse_args() -> argparse.Namespace: "evaluation_file", help="Path to the evaluation file containing an 'evaluate' function" ) + parser.add_argument( + "--seed-program", + action="append", + default=[], + help=( + "Additional evaluated starting parent. Repeat the option to seed " + "multiple islands while retaining initial_program as the incumbent." + ), + ) + parser.add_argument("--config", "-c", help="Path to configuration file (YAML)", default=None) parser.add_argument("--output", "-o", help="Output directory for results", default=None) @@ -77,6 +89,10 @@ async def main_async() -> int: if not os.path.exists(args.evaluation_file): print(f"Error: Evaluation file '{args.evaluation_file}' not found") return 1 + missing_seeds = [path for path in args.seed_program if not os.path.exists(path)] + if missing_seeds: + print(f"Error: Seed program file not found: '{missing_seeds[0]}'") + return 1 # Load base config from file or defaults config = load_config(args.config) @@ -110,6 +126,7 @@ async def main_async() -> int: evaluation_file=args.evaluation_file, config=config, output_dir=args.output, + seed_program_paths=args.seed_program, ) # Load from checkpoint if specified @@ -161,6 +178,25 @@ async def main_async() -> int: print(f"\nLatest checkpoint saved at: {latest_checkpoint}") print(f"To resume, use: --checkpoint {latest_checkpoint}") + summary = { + "schema_version": 1, + "record_type": "openevolve_run_summary", + "completion_reason": openevolve.completion_reason, + "last_completed_iteration": openevolve.last_completed_iteration, + "completed_iteration_count": openevolve.completed_iteration_count, + "latest_checkpoint": latest_checkpoint, + "best_program_id": best_program.id, + "best_program_metrics": best_program.metrics, + "llm_usage": openevolve.llm_usage, + } + summary_path = Path(openevolve.output_dir) / "run-summary.json" + temporary = summary_path.with_name(f".{summary_path.name}.tmp") + temporary.write_text( + json.dumps(summary, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + temporary.replace(summary_path) + return 0 except Exception as e: diff --git a/openevolve/config.py b/openevolve/config.py index 6358b7d71a..2a90446a19 100644 --- a/openevolve/config.py +++ b/openevolve/config.py @@ -282,6 +282,19 @@ class PromptConfig: include_artifacts: bool = True max_artifact_bytes: int = 20 * 1024 # 20KB in prompt artifact_security_filter: bool = True + # Optional ordered subset of evaluator artifacts rendered in prompts. + # Stored artifacts remain complete. + artifact_include_names: Optional[List[str]] = None + # Optional evaluator artifact collected across the complete live archive + # and rendered as one compact deterministic prompt artifact. + archive_context_artifact: Optional[str] = None + archive_context_max_items: int = 64 + # Optional evaluator-produced neighborhood. The artifact must be schema + # version 1 JSON with an identity_artifact name and ordered options carrying + # identity fields. The controller removes identities already retained in the + # live archive and renders the remainder as proposal-options.json. + proposal_neighborhood_artifact: Optional[str] = None + proposal_options_max_items: int = 32 # Feature extraction and program labeling suggest_simplification_after_chars: Optional[int] = ( @@ -312,6 +325,10 @@ class ControllerSchedulerConfig: """Optional observed-result island allocator for bounded live comparisons.""" enabled: bool = False + score_metric: str = "combined_score" + # Optional evaluator artifact used to count distinct measured phenotypes. + # When absent, source-code hashes preserve the legacy definition. + diversity_artifact: Optional[str] = None exploitation_weight: float = 0.0 underexplored_weight: float = 0.0 validity_weight: float = 0.0 @@ -319,6 +336,22 @@ class ControllerSchedulerConfig: token_efficiency_weight: float = 0.0 rejection_penalty: float = 0.0 minimum_calls: int = 1 + # Optional post-warmup quality band. When set, allocate only among islands + # whose retained best score is within this absolute distance of the global + # retained leader. None preserves unrestricted legacy allocation. + leader_score_band: Optional[float] = None + # Optional post-warmup parent band within the selected island. When set, + # parent sampling is limited to retained programs whose score is within + # this absolute distance of that island's retained leader. + parent_score_band: Optional[float] = None + # Keep this many islands outside the active worker frontier so a completed + # call leaves the allocator a real choice. This does not reduce concurrency + # when num_islands > parallel_evaluations. + reserve_islands: int = 1 + # Optional lower in-flight limit after the balanced warmup proposal count + # has been submitted. This trades some throughput for fresher archive + # context during quality-focused adaptive search. + adaptive_parallelism: Optional[int] = None @dataclass @@ -461,6 +494,14 @@ class Config: strict_diff_application: bool = False enforce_evolve_blocks: bool = False max_diff_blocks: int = 32 + # Optional evaluator artifact whose exact value identifies the candidate's + # behaviorally meaningful contents. When configured, a child duplicating + # any program already known to the live archive is rejected. + program_identity_artifact: Optional[str] = None + # Optional second evaluator identity for exact measured behavior. This is + # checked only after evaluation, so structurally different programs that + # produce an already archived phenotype do not bloat parent selection. + phenotype_identity_artifact: Optional[str] = None # Early stopping settings early_stopping_patience: Optional[int] = None @@ -544,9 +585,92 @@ def validate(self) -> None: ) if self.max_diff_blocks < 1: raise ValueError("max_diff_blocks must be at least 1") + if self.prompt.archive_context_artifact is not None and ( + not isinstance(self.prompt.archive_context_artifact, str) + or not self.prompt.archive_context_artifact.strip() + ): + raise ValueError( + "prompt.archive_context_artifact must be a non-empty string or None" + ) + if self.prompt.proposal_neighborhood_artifact is not None and ( + not isinstance(self.prompt.proposal_neighborhood_artifact, str) + or not self.prompt.proposal_neighborhood_artifact.strip() + ): + raise ValueError( + "prompt.proposal_neighborhood_artifact must be a non-empty " + "string or None" + ) + artifact_names = self.prompt.artifact_include_names + if artifact_names is not None: + if not isinstance(artifact_names, list) or any( + not isinstance(name, str) or not name.strip() + for name in artifact_names + ): + raise ValueError( + "prompt.artifact_include_names must be a list of non-empty strings or None" + ) + if len(set(artifact_names)) != len(artifact_names): + raise ValueError( + "prompt.artifact_include_names must not contain duplicates" + ) + if ( + self.prompt.archive_context_artifact is not None + and "archive-context.json" not in artifact_names + ): + raise ValueError( + "prompt.artifact_include_names must include archive-context.json " + "when archive_context_artifact is configured" + ) + if ( + self.prompt.proposal_neighborhood_artifact is not None + and "proposal-options.json" not in artifact_names + ): + raise ValueError( + "prompt.artifact_include_names must include " + "proposal-options.json when proposal_neighborhood_artifact " + "is configured" + ) + if ( + isinstance(self.prompt.archive_context_max_items, bool) + or not isinstance(self.prompt.archive_context_max_items, int) + or self.prompt.archive_context_max_items < 1 + ): + raise ValueError( + "prompt.archive_context_max_items must be a positive integer" + ) + if ( + isinstance(self.prompt.proposal_options_max_items, bool) + or not isinstance(self.prompt.proposal_options_max_items, int) + or self.prompt.proposal_options_max_items < 1 + ): + raise ValueError( + "prompt.proposal_options_max_items must be a positive integer" + ) + for field_name in ( + "program_identity_artifact", + "phenotype_identity_artifact", + ): + value = getattr(self, field_name) + if value is not None and ( + not isinstance(value, str) or not value.strip() + ): + raise ValueError(f"{field_name} must be a non-empty string or None") scheduler = self.database.controller_scheduler if not isinstance(scheduler.enabled, bool): raise ValueError("database.controller_scheduler.enabled must be boolean") + if not isinstance(scheduler.score_metric, str) or not scheduler.score_metric.strip(): + raise ValueError( + "database.controller_scheduler.score_metric " + "must be a non-empty string" + ) + if scheduler.diversity_artifact is not None and ( + not isinstance(scheduler.diversity_artifact, str) + or not scheduler.diversity_artifact.strip() + ): + raise ValueError( + "database.controller_scheduler.diversity_artifact " + "must be a non-empty string or None" + ) for field_name in ( "exploitation_weight", "underexplored_weight", @@ -566,6 +690,18 @@ def validate(self) -> None: f"database.controller_scheduler.{field_name} " "must be finite and nonnegative" ) + for field_name in ("leader_score_band", "parent_score_band"): + value = getattr(scheduler, field_name) + if value is not None and ( + isinstance(value, bool) + or not isinstance(value, (int, float)) + or not math.isfinite(float(value)) + or float(value) < 0.0 + ): + raise ValueError( + f"database.controller_scheduler.{field_name} " + "must be finite and nonnegative when provided" + ) if ( isinstance(scheduler.minimum_calls, bool) or not isinstance(scheduler.minimum_calls, int) @@ -575,6 +711,36 @@ def validate(self) -> None: "database.controller_scheduler.minimum_calls " "must be a positive integer" ) + if ( + isinstance(scheduler.reserve_islands, bool) + or not isinstance(scheduler.reserve_islands, int) + or scheduler.reserve_islands < 0 + or ( + scheduler.enabled + and scheduler.reserve_islands >= self.database.num_islands + ) + ): + raise ValueError( + "database.controller_scheduler.reserve_islands must be an integer " + "in [0, database.num_islands) when the scheduler is enabled" + ) + if scheduler.adaptive_parallelism is not None and ( + isinstance(scheduler.adaptive_parallelism, bool) + or not isinstance(scheduler.adaptive_parallelism, int) + or scheduler.adaptive_parallelism < 1 + or scheduler.adaptive_parallelism + > self.evaluator.parallel_evaluations + or ( + scheduler.enabled + and scheduler.adaptive_parallelism + > self.database.num_islands - scheduler.reserve_islands + ) + ): + raise ValueError( + "database.controller_scheduler.adaptive_parallelism must be a " + "positive integer no greater than the evaluator worker count or " + "selectable island count" + ) def to_dict(self) -> Dict[str, Any]: return asdict(self) diff --git a/openevolve/controller.py b/openevolve/controller.py index c83be5bf1d..0ca751dd78 100644 --- a/openevolve/controller.py +++ b/openevolve/controller.py @@ -45,6 +45,7 @@ def __init__( evaluation_file: str, config: Config, output_dir: Optional[str] = None, + seed_program_paths: Optional[List[str]] = None, ): # Load configuration (loaded in main_async) self.config = config @@ -96,6 +97,11 @@ def __init__( # Load initial program self.initial_program_path = initial_program_path self.initial_program_code = self._load_initial_program() + self.seed_program_paths = [ + str(Path(path).expanduser().resolve()) + for path in (seed_program_paths or []) + ] + self.seed_program_codes = self._load_seed_programs() if not self.config.language: self.config.language = extract_code_language(self.initial_program_code) @@ -229,6 +235,22 @@ def _load_initial_program(self) -> str: with open(self.initial_program_path, "r") as f: return f.read() + def _load_seed_programs(self) -> List[tuple[str, str]]: + """Load distinct optional starting parents without replacing the incumbent.""" + + loaded: List[tuple[str, str]] = [] + seen = {self.initial_program_code} + for path in self.seed_program_paths: + if not Path(path).is_file(): + raise ValueError(f"Seed program does not exist: {path}") + code = Path(path).read_text(encoding="utf-8") + if code in seen: + logger.info("Skipping duplicate seed program %s", path) + continue + seen.add(code) + loaded.append((path, code)) + return loaded + async def run( self, iterations: Optional[int] = None, @@ -307,6 +329,38 @@ async def run( f"For better evolution results, please modify your evaluator to return a 'combined_score' " f"metric that properly weights different aspects of program performance." ) + + for index, (seed_path, seed_code) in enumerate(self.seed_program_codes): + seed_program_id = str(uuid.uuid4()) + seed_metrics = await self.evaluator.evaluate_program( + seed_code, + seed_program_id, + ) + seed_program = Program( + id=seed_program_id, + code=seed_code, + changes_description=f"Starting parent from {Path(seed_path).name}", + language=self.config.language, + metrics=seed_metrics, + iteration_found=start_iteration, + metadata={ + "seed_program": True, + "seed_program_path": seed_path, + }, + ) + target_island = (index + 1) % self.config.database.num_islands + self.database.add( + seed_program, + target_island=target_island, + ) + seed_artifacts = self.evaluator.get_pending_artifacts(seed_program_id) + if seed_artifacts: + self.database.store_artifacts(seed_program_id, seed_artifacts) + logger.info( + "Added seed program %s to island %s", + Path(seed_path).name, + target_island, + ) else: logger.info( f"Skipping initial program addition (resuming from iteration {start_iteration} " diff --git a/openevolve/database.py b/openevolve/database.py index 8abe2bdc0a..67ad781a91 100644 --- a/openevolve/database.py +++ b/openevolve/database.py @@ -162,6 +162,12 @@ def __init__(self, config: DatabaseConfig): # Track the last iteration number (for resuming) self.last_iteration: int = 0 + # Complete text-artifact history used by controller-side identity and + # prompt filters. Unlike the live MAP-Elites population, this ledger is + # append-only so displaced or rejected measured candidates are not + # proposed again after a checkpoint resume. + self.historical_artifact_values: Dict[str, Set[str]] = {} + # Load database from disk if path is provided if config.db_path and os.path.exists(config.db_path): self.load(config.db_path) @@ -233,6 +239,22 @@ def add( self.programs[program.id] = program + # Evaluators with an explicit admission stage may retain declined + # attempts for lineage and feedback without allowing them to become + # parents, MAP-Elites occupants, or archive members. + if "selection_eligible" in program.metrics: + try: + selection_eligible = float(program.metrics["selection_eligible"]) + except (TypeError, ValueError, OverflowError): + selection_eligible = 0.0 + if selection_eligible <= 0.0: + program.metadata["selection_ineligible"] = True + logger.info( + "Retained ineligible program %s as a lineage record only", + program.id, + ) + return program.id + # Calculate feature coordinates for MAP-Elites feature_coords = self._calculate_feature_coords(program) @@ -481,6 +503,65 @@ def sample_from_island( ) return parent, inspirations + def sample_from_island_score_band( + self, + island_id: int, + *, + score_metric: str, + score_band: float, + num_inspirations: Optional[int] = None, + ) -> Tuple[Program, List[Program]]: + """Sample a parent only from one island's retained quality frontier.""" + + island_id = island_id % len(self.islands) + programs = [ + self.programs[program_id] + for program_id in self.islands[island_id] + if program_id in self.programs + ] + scored: list[tuple[Program, float]] = [] + for program in programs: + value = program.metrics.get(score_metric) + if ( + score_metric == "combined_score" + and ( + not isinstance(value, (int, float)) + or isinstance(value, bool) + ) + ): + value = safe_numeric_average(program.metrics) + if isinstance(value, (int, float)) and not isinstance(value, bool): + scored.append((program, float(value))) + if not scored: + return self.sample_from_island(island_id, num_inspirations) + best_score = max(score for _, score in scored) + frontier = sorted( + ( + program + for program, score in scored + if best_score - score <= score_band + ), + key=lambda program: program.id, + ) + parent = random.choice(frontier) + if num_inspirations is None: + num_inspirations = 5 + inspirations = self._sample_inspirations( + parent, + n=num_inspirations, + island_id=island_id, + ) + logger.debug( + "Sampled parent %s from island %d score band %.6f " + "(metric=%s, frontier=%d)", + parent.id, + island_id, + score_band, + score_metric, + len(frontier), + ) + return parent, inspirations + def get_best_program(self, metric: Optional[str] = None) -> Optional[Program]: """ Get the best program based on a metric @@ -641,6 +722,12 @@ def save(self, path: Optional[str] = None, iteration: int = 0) -> None: "island_generations": self.island_generations, "last_migration_generation": self.last_migration_generation, "feature_stats": self._serialize_feature_stats(), + "historical_artifact_values": { + name: sorted(values) + for name, values in sorted( + self.historical_artifact_values.items() + ) + }, } with open(os.path.join(save_path, "metadata.json"), "w") as f: @@ -679,6 +766,15 @@ def load(self, path: str) -> None: self.current_island = metadata.get("current_island", 0) self.island_generations = metadata.get("island_generations", [0] * len(saved_islands)) self.last_migration_generation = metadata.get("last_migration_generation", 0) + raw_history = metadata.get("historical_artifact_values", {}) + if isinstance(raw_history, dict): + self.historical_artifact_values = { + str(name): { + value for value in values if isinstance(value, str) + } + for name, values in raw_history.items() + if isinstance(name, str) and isinstance(values, list) + } # Load feature_stats for MAP-Elites grid stability self.feature_stats = self._deserialize_feature_stats(metadata.get("feature_stats", {})) @@ -1214,16 +1310,27 @@ def _update_best_program(self, program: Program) -> None: old_id = self.best_program_id self.best_program_id = program.id - # Log the change - if "combined_score" in program.metrics and "combined_score" in current_best.metrics: - old_score = current_best.metrics["combined_score"] - new_score = program.metrics["combined_score"] - score_diff = new_score - old_score - logger.info( - f"New best program {program.id} replaces {old_id} (combined_score: {old_score:.4f} → {new_score:.4f}, +{score_diff:.4f})" - ) - else: - logger.info(f"New best program {program.id} replaces {old_id}") + # Report the same fitness that made the replacement decision. + old_score = get_fitness_score( + current_best.metrics, self.config.feature_dimensions + ) + new_score = get_fitness_score( + program.metrics, self.config.feature_dimensions + ) + metric_name = ( + "selection_score" + if "selection_score" in program.metrics + and "selection_score" in current_best.metrics + else "combined_score" + if "combined_score" in program.metrics + and "combined_score" in current_best.metrics + else "fitness_score" + ) + logger.info( + f"New best program {program.id} replaces {old_id} " + f"({metric_name}: {old_score:.4f} → {new_score:.4f}, " + f"{new_score - old_score:+.4f})" + ) def _update_island_best_program(self, program: Program, island_idx: int) -> None: """ @@ -1260,22 +1367,26 @@ def _update_island_best_program(self, program: Program, island_idx: int) -> None old_id = current_island_best_id self.island_best_programs[island_idx] = program.id - # Log the change - if ( - "combined_score" in program.metrics + old_score = get_fitness_score( + current_island_best.metrics, self.config.feature_dimensions + ) + new_score = get_fitness_score( + program.metrics, self.config.feature_dimensions + ) + metric_name = ( + "selection_score" + if "selection_score" in program.metrics + and "selection_score" in current_island_best.metrics + else "combined_score" + if "combined_score" in program.metrics and "combined_score" in current_island_best.metrics - ): - old_score = current_island_best.metrics["combined_score"] - new_score = program.metrics["combined_score"] - score_diff = new_score - old_score - logger.debug( - f"Island {island_idx}: New best program {program.id} replaces {old_id} " - f"(combined_score: {old_score:.4f} → {new_score:.4f}, +{score_diff:.4f})" - ) - else: - logger.debug( - f"Island {island_idx}: New best program {program.id} replaces {old_id}" - ) + else "fitness_score" + ) + logger.debug( + f"Island {island_idx}: New best program {program.id} replaces {old_id} " + f"({metric_name}: {old_score:.4f} → {new_score:.4f}, " + f"{new_score - old_score:+.4f})" + ) def _sample_parent(self) -> Program: """ @@ -1910,8 +2021,20 @@ def migrate_programs(self) -> None: language=migrant.language, parent_id=migrant.id, generation=migrant.generation, + timestamp=migrant.timestamp, + iteration_found=migrant.iteration_found, metrics=migrant.metrics.copy(), + complexity=migrant.complexity, + diversity=migrant.diversity, metadata={**migrant.metadata, "island": target_island, "migrant": True}, + prompts=migrant.prompts, + artifacts_json=migrant.artifacts_json, + artifact_dir=migrant.artifact_dir, + embedding=( + list(migrant.embedding) + if migrant.embedding is not None + else None + ), ) # Use add() method to properly handle MAP-Elites deduplication, diff --git a/openevolve/evaluator.py b/openevolve/evaluator.py index 90705053ed..f6c86a8fd1 100644 --- a/openevolve/evaluator.py +++ b/openevolve/evaluator.py @@ -678,7 +678,7 @@ def _passes_threshold(self, metrics: Dict[str, float], threshold: float) -> bool """ Check if metrics pass a threshold - Uses 'combined_score' if available (for consistency with evolution), + Uses 'selection_score' when supplied, then 'combined_score', otherwise falls back to averaging all numeric metrics except 'error' Args: @@ -691,6 +691,16 @@ def _passes_threshold(self, metrics: Dict[str, float], threshold: float) -> bool if not metrics: return False + if "selection_eligible" in metrics: + eligible = metrics.get("selection_eligible") + if not isinstance(eligible, (int, float)) or float(eligible) <= 0.0: + return False + + if "selection_score" in metrics: + score = metrics.get("selection_score") + if isinstance(score, (int, float)): + return float(score) >= threshold + # Use combined_score if available - this is what evolution uses if "combined_score" in metrics: score = metrics.get("combined_score") diff --git a/openevolve/process_parallel.py b/openevolve/process_parallel.py index 2bc7233d17..11cee8be8e 100644 --- a/openevolve/process_parallel.py +++ b/openevolve/process_parallel.py @@ -140,6 +140,182 @@ def _lazy_init_worker_components(): ) +def _with_archive_context( + parent_artifacts: Optional[Dict[str, Any]], + db_snapshot: Dict[str, Any], +) -> Optional[Dict[str, Any]]: + """Add evaluator-defined complete search history to one prompt.""" + + archive_artifact = _worker_config.prompt.archive_context_artifact + neighborhood_artifact = ( + _worker_config.prompt.proposal_neighborhood_artifact + ) + if archive_artifact is None and neighborhood_artifact is None: + return parent_artifacts + augmented = dict(parent_artifacts or {}) + snapshot_artifacts = db_snapshot.get("artifacts") + historical_artifacts = db_snapshot.get("historical_artifact_values") + if archive_artifact is not None: + live_values = _snapshot_artifact_values( + snapshot_artifacts, + archive_artifact, + ) + historical_values = _historical_artifact_values( + historical_artifacts, + archive_artifact, + ) + values = live_values | historical_values + ordered = sorted(values) + limit = _worker_config.prompt.archive_context_max_items + selected = ordered[:limit] + context = { + "artifact": archive_artifact, + "archive_item_count": len(ordered), + "live_archive_item_count": len(live_values), + "historical_item_count": len(historical_values), + "included_item_count": len(selected), + "items": selected, + "truncated": len(selected) != len(ordered), + } + augmented["archive-context.json"] = json.dumps( + context, + ensure_ascii=False, + separators=(",", ":"), + sort_keys=True, + ) + if neighborhood_artifact is not None: + raw_neighborhood = augmented.get(neighborhood_artifact) + if isinstance(raw_neighborhood, bytes): + raw_neighborhood = raw_neighborhood.decode( + "utf-8", errors="strict" + ) + if not isinstance(raw_neighborhood, str): + raise ValueError( + "parent is missing configured proposal neighborhood artifact: " + f"{neighborhood_artifact}" + ) + try: + neighborhood = json.loads(raw_neighborhood) + except json.JSONDecodeError as error: + raise ValueError( + f"proposal neighborhood artifact is not valid JSON: {error}" + ) from error + if ( + not isinstance(neighborhood, dict) + or neighborhood.get("schema_version") != 1 + ): + raise ValueError( + "proposal neighborhood artifact must use schema_version 1" + ) + identity_artifact = neighborhood.get("identity_artifact") + options = neighborhood.get("options") + if ( + not isinstance(identity_artifact, str) + or not identity_artifact.strip() + or not isinstance(options, list) + ): + raise ValueError( + "proposal neighborhood must declare identity_artifact and options" + ) + retained = _snapshot_artifact_values( + snapshot_artifacts, + identity_artifact, + ) + historical = _historical_artifact_values( + historical_artifacts, + identity_artifact, + ) + known = retained | historical + available: list[dict[str, Any]] = [] + seen: set[str] = set() + excluded_retained = 0 + excluded_historical = 0 + excluded_known = 0 + for option in options: + if not isinstance(option, dict): + raise ValueError( + "proposal neighborhood options must be JSON objects" + ) + identity = option.get("identity") + option_id = option.get("id") + if ( + not isinstance(identity, str) + or not identity + or not isinstance(option_id, str) + or not option_id + ): + raise ValueError( + "proposal neighborhood options require non-empty id and identity" + ) + if identity in seen: + continue + seen.add(identity) + if identity in known: + excluded_known += 1 + if identity in retained: + excluded_retained += 1 + if identity in historical: + excluded_historical += 1 + continue + available.append(option) + limit = _worker_config.prompt.proposal_options_max_items + selected_options = available[:limit] + proposal_context = { + "source_artifact": neighborhood_artifact, + "identity_artifact": identity_artifact, + "declared_option_count": len(options), + "excluded_retained_count": excluded_retained, + "excluded_historical_count": excluded_historical, + "excluded_known_count": excluded_known, + "available_option_count": len(available), + "included_option_count": len(selected_options), + "options": selected_options, + "truncated": len(selected_options) != len(available), + } + augmented["proposal-options.json"] = json.dumps( + proposal_context, + ensure_ascii=False, + separators=(",", ":"), + sort_keys=True, + ) + return augmented + + +def _snapshot_artifact_values( + snapshot_artifacts: Any, + artifact_name: str, +) -> set[str]: + """Return complete text values for one artifact across a worker snapshot.""" + + values: set[str] = set() + if not isinstance(snapshot_artifacts, dict): + return values + for program_id in sorted(snapshot_artifacts): + artifacts = snapshot_artifacts.get(program_id) + if not isinstance(artifacts, dict): + continue + value = artifacts.get(artifact_name) + if isinstance(value, bytes): + value = value.decode("utf-8", errors="replace") + if isinstance(value, str): + values.add(value) + return values + + +def _historical_artifact_values( + historical_artifacts: Any, + artifact_name: str, +) -> set[str]: + """Return persisted text values for one append-only artifact ledger.""" + + if not isinstance(historical_artifacts, dict): + return set() + values = historical_artifacts.get(artifact_name) + if not isinstance(values, list): + return set() + return {value for value in values if isinstance(value, str)} + + def _run_iteration_worker( iteration: int, db_snapshot: Dict[str, Any], parent_id: str, inspiration_ids: List[str] ) -> SerializableResult: @@ -158,7 +334,10 @@ def _run_iteration_worker( inspirations = [programs[pid] for pid in inspiration_ids if pid in programs] # Get parent artifacts if available - parent_artifacts = db_snapshot["artifacts"].get(parent_id) + parent_artifacts = _with_archive_context( + db_snapshot["artifacts"].get(parent_id), + db_snapshot, + ) # Get island-specific programs for context parent_island = parent.metadata.get("island", db_snapshot["current_island"]) @@ -243,8 +422,11 @@ def reject_proposal(message: str) -> SerializableResult: return SerializableResult( error=message, iteration=iteration, + parent_id=parent.id, + prompt=prompt, llm_response=llm_response, llm_metadata=llm_metadata, + target_island=db_snapshot.get("sampling_island"), ) # Parse response based on evolution mode @@ -376,6 +558,21 @@ def reject_proposal(message: str) -> SerializableResult: target_island=db_snapshot.get("sampling_island"), ) + identity_artifact = _worker_config.program_identity_artifact + if identity_artifact is not None: + if not isinstance(parent_artifacts, dict) or identity_artifact not in parent_artifacts: + return reject_proposal( + f"Parent is missing configured program identity artifact: {identity_artifact}" + ) + if not isinstance(artifacts, dict) or identity_artifact not in artifacts: + return reject_proposal( + f"Candidate is missing configured program identity artifact: {identity_artifact}" + ) + if artifacts[identity_artifact] == parent_artifacts[identity_artifact]: + return reject_proposal( + "Candidate has the same program identity as its parent" + ) + # Create child program child_program = Program( id=child_id, @@ -514,7 +711,50 @@ def __init__( self.budget_limits_reached: List[str] = [] self.budget_completion_reason: Optional[str] = None self.inflight_proposals_at_budget_stop = 0 + self.accepted_proposal_count = 0 + self.rejected_proposal_count = 0 self._seen_provider_response_ids: set[str] = set() + self._known_program_identities: set[tuple[str, Any]] = set() + self._known_phenotype_identities: set[tuple[str, Any]] = set() + self._historical_artifact_values = ( + self.database.historical_artifact_values + ) + self._tracked_history_artifact_names: set[str] = { + artifact_name + for artifact_name in ( + self.config.prompt.archive_context_artifact, + self.config.program_identity_artifact, + self.config.phenotype_identity_artifact, + ) + if isinstance(artifact_name, str) and artifact_name + } + self._tracked_history_artifact_names.update( + self._historical_artifact_values + ) + for program_id in self.database.programs: + self._discover_neighborhood_identity_artifact( + self.database.get_artifacts(program_id) + ) + for program_id in self.database.programs: + self._record_historical_artifacts( + self.database.get_artifacts(program_id) + ) + identity_artifact = self.config.program_identity_artifact + if identity_artifact is not None: + for value in self._historical_artifact_values.get( + identity_artifact, set() + ): + identity = self._identity_key(value) + if identity is not None: + self._known_program_identities.add(identity) + phenotype_artifact = self.config.phenotype_identity_artifact + if phenotype_artifact is not None: + for value in self._historical_artifact_values.get( + phenotype_artifact, set() + ): + identity = self._identity_key(value) + if identity is not None: + self._known_phenotype_identities.add(identity) self._controller_island_state: Dict[int, Dict[str, Any]] = { island_id: { "accepted": 0, @@ -533,6 +773,27 @@ def __init__( logger.info(f"Initialized process parallel controller with {self.num_workers} workers") + def _early_stopping_score(self, metrics: Dict[str, Any]) -> Optional[float]: + """Return the configured convergence score for one evaluated program.""" + metric = self.config.early_stopping_metric + if metric in metrics: + value = metrics[metric] + elif metric == "combined_score": + value = safe_numeric_average(metrics) + else: + return None + if isinstance(value, (int, float)) and not isinstance(value, bool): + return float(value) + return None + + def _initial_early_stopping_score(self) -> float: + """Seed convergence tracking from programs already in the archive.""" + scores = ( + self._early_stopping_score(program.metrics) + for program in self.database.programs.values() + ) + return max((score for score in scores if score is not None), default=float("-inf")) + @staticmethod def _nonnegative_int(value: Any) -> Optional[int]: """Return provider counters only when they are exact non-negative integers.""" @@ -540,6 +801,103 @@ def _nonnegative_int(value: Any) -> Optional[int]: return None return value + @staticmethod + def _identity_key(value: Any) -> Optional[tuple[str, Any]]: + """Normalize supported identity artifact values into stable keys.""" + if isinstance(value, str): + return ("text", value) + if isinstance(value, bytes): + return ("bytes", bytes(value)) + return None + + @staticmethod + def _artifact_text(value: Any) -> Optional[str]: + """Normalize a persistable evaluator artifact to text.""" + + if isinstance(value, bytes): + return value.decode("utf-8", errors="replace") + if isinstance(value, str): + return value + return None + + def _discover_neighborhood_identity_artifact( + self, + artifacts: Any, + ) -> None: + """Track the identity named by a valid configured neighborhood.""" + + neighborhood_name = ( + self.config.prompt.proposal_neighborhood_artifact + ) + if not isinstance(neighborhood_name, str) or not isinstance( + artifacts, dict + ): + return + raw = self._artifact_text(artifacts.get(neighborhood_name)) + if raw is None: + return + try: + neighborhood = json.loads(raw) + except json.JSONDecodeError: + return + if ( + not isinstance(neighborhood, dict) + or neighborhood.get("schema_version") != 1 + ): + return + identity_artifact = neighborhood.get("identity_artifact") + if isinstance(identity_artifact, str) and identity_artifact: + self._tracked_history_artifact_names.add(identity_artifact) + + def _record_historical_artifacts(self, artifacts: Any) -> None: + """Remember relevant measured artifacts even if MAP-Elites drops them.""" + + if not isinstance(artifacts, dict): + return + self._discover_neighborhood_identity_artifact(artifacts) + for artifact_name in self._tracked_history_artifact_names: + value = self._artifact_text(artifacts.get(artifact_name)) + if value is not None: + self._historical_artifact_values.setdefault( + artifact_name, set() + ).add(value) + + def _apply_program_identity_gate(self, result: SerializableResult) -> None: + """Reject a measured child whose structural or phenotype identity exists.""" + artifact_name = self.config.program_identity_artifact + if result.error is not None or not result.child_program_dict: + return + artifacts = result.artifacts if isinstance(result.artifacts, dict) else {} + if artifact_name is not None: + identity = self._identity_key(artifacts.get(artifact_name)) + if identity is None: + result.error = ( + "Candidate is missing configured program identity artifact: " + f"{artifact_name}" + ) + return + if identity in self._known_program_identities: + result.error = "Candidate program identity already exists in the archive" + return + # Reserve at result processing time so simultaneously generated + # duplicates cannot both enter before the database snapshot refreshes. + self._known_program_identities.add(identity) + + phenotype_artifact = self.config.phenotype_identity_artifact + if phenotype_artifact is None: + return + phenotype = self._identity_key(artifacts.get(phenotype_artifact)) + if phenotype is None: + result.error = ( + "Candidate is missing configured phenotype identity artifact: " + f"{phenotype_artifact}" + ) + return + if phenotype in self._known_phenotype_identities: + result.error = "Candidate phenotype identity already exists in the archive" + return + self._known_phenotype_identities.add(phenotype) + @staticmethod def _budget_reason(limits_reached: List[str]) -> Optional[str]: """Return a stable public completion reason for the first reached budget.""" @@ -587,6 +945,11 @@ def _record_controller_result( ) -> None: """Update only observed per-island counters used by the live allocator.""" + if result.error is not None or not result.child_program_dict: + self.rejected_proposal_count += 1 + else: + self.accepted_proposal_count += 1 + if ( not self.config.database.controller_scheduler.enabled or not isinstance(island_id, int) @@ -606,13 +969,33 @@ def _record_controller_result( return state["accepted"] += 1 child = result.child_program_dict - code = child.get("code") - if isinstance(code, str): - state["unique"].add(hashlib.sha256(code.encode("utf-8")).hexdigest()) + diversity_artifact = ( + self.config.database.controller_scheduler.diversity_artifact + ) + if diversity_artifact is not None: + artifacts = result.artifacts if isinstance(result.artifacts, dict) else {} + diversity_identity = self._identity_key( + artifacts.get(diversity_artifact) + ) + if diversity_identity is not None: + state["unique"].add(diversity_identity) + else: + code = child.get("code") + if isinstance(code, str): + state["unique"].add( + ("code", hashlib.sha256(code.encode("utf-8")).hexdigest()) + ) metrics = child.get("metrics") if isinstance(metrics, dict): - score = metrics.get("combined_score") - if not isinstance(score, (int, float)) or isinstance(score, bool): + score_metric = self.config.database.controller_scheduler.score_metric + score = metrics.get(score_metric) + if ( + score_metric == "combined_score" + and ( + not isinstance(score, (int, float)) + or isinstance(score, bool) + ) + ): score = safe_numeric_average(metrics) if isinstance(score, (int, float)) and not isinstance(score, bool): state["best_score"] = max(float(state["best_score"]), float(score)) @@ -632,10 +1015,7 @@ def _select_controller_island( ] if not available: raise RuntimeError("controller scheduler has no available island") - total_calls = sum( - int(state["calls"]) for state in self._controller_island_state.values() - ) - if total_calls < scheduler.minimum_calls: + if self.submitted_proposal_count < scheduler.minimum_calls: return min( available, key=lambda island_id: ( @@ -644,6 +1024,21 @@ def _select_controller_island( island_id, ), ) + if scheduler.leader_score_band is not None: + global_best = max( + self._controller_island_best_score(island_id) + for island_id in range(self.num_islands) + ) + quality_frontier = [ + island_id + for island_id in available + if ( + global_best - self._controller_island_best_score(island_id) + <= float(scheduler.leader_score_band) + ) + ] + if quality_frontier: + available = quality_frontier def priority(island_id: int) -> tuple[float, int]: state = self._controller_island_state[island_id] @@ -652,7 +1047,7 @@ def priority(island_id: int) -> tuple[float, int]: rejected = int(state["rejected"]) tokens = int(state["tokens"]) unique = len(state["unique"]) - best_score = float(state["best_score"]) + best_score = self._controller_island_best_score(island_id) validity = accepted / calls if calls else 1.0 diversity = unique / accepted if accepted else 1.0 token_efficiency = best_score / max(tokens / 1000.0, 1.0) @@ -669,6 +1064,103 @@ def priority(island_id: int) -> tuple[float, int]: return min(available, key=priority) + def _controller_island_best_score(self, island_id: int) -> float: + """Return the best observed or currently retained score for an island.""" + + state = self._controller_island_state[island_id] + best_score = float(state["best_score"]) + score_metric = self.config.database.controller_scheduler.score_metric + if not 0 <= island_id < len(self.database.islands): + return best_score + for program_id in self.database.islands[island_id]: + program = self.database.programs.get(program_id) + if program is None or not isinstance(program.metrics, dict): + continue + score = program.metrics.get(score_metric) + if ( + score_metric == "combined_score" + and ( + not isinstance(score, (int, float)) + or isinstance(score, bool) + ) + ): + score = safe_numeric_average(program.metrics) + if isinstance(score, (int, float)) and not isinstance(score, bool): + best_score = max(best_score, float(score)) + return best_score + + def _controller_parallelism(self, max_iterations: int) -> int: + """Return active scheduler slots while preserving a selectable island.""" + + if max_iterations < 1: + return 0 + scheduler = self.config.database.controller_scheduler + if not scheduler.enabled: + return min(self.num_islands, max_iterations) + selectable_islands = max(1, self.num_islands - scheduler.reserve_islands) + return min( + self.num_workers, + selectable_islands, + max_iterations, + ) + + def _adaptive_controller_parallelism(self, max_iterations: int) -> int: + """Return the lower quality-phase frontier when one is configured.""" + + warmup = self._controller_parallelism(max_iterations) + configured = ( + self.config.database.controller_scheduler.adaptive_parallelism + ) + if configured is None: + return warmup + return min(warmup, configured) + + def _target_controller_parallelism(self, max_iterations: int) -> int: + """Return warmup or adaptive frontier from submitted proposal count.""" + + scheduler = self.config.database.controller_scheduler + if ( + scheduler.enabled + and self.submitted_proposal_count >= scheduler.minimum_calls + ): + return self._adaptive_controller_parallelism(max_iterations) + return self._controller_parallelism(max_iterations) + + @property + def controller_allocation(self) -> Dict[str, Any]: + """Return a serializable receipt for observed-result island allocation.""" + + scheduler = self.config.database.controller_scheduler + islands = [] + for island_id in range(self.num_islands): + state = self._controller_island_state[island_id] + islands.append( + { + "island": island_id, + "calls": int(state["calls"]), + "accepted": int(state["accepted"]), + "rejected": int(state["rejected"]), + "distinct_behaviors": len(state["unique"]), + "best_score": self._controller_island_best_score(island_id), + "tokens": int(state["tokens"]), + } + ) + return { + "enabled": bool(scheduler.enabled), + "score_metric": scheduler.score_metric, + "diversity_artifact": scheduler.diversity_artifact, + "leader_score_band": scheduler.leader_score_band, + "parent_score_band": scheduler.parent_score_band, + "reserve_islands": scheduler.reserve_islands, + "active_parallelism": self._controller_parallelism( + max(1, self.config.max_iterations) + ), + "adaptive_parallelism": self._adaptive_controller_parallelism( + max(1, self.config.max_iterations) + ), + "islands": islands, + } + def _register_submitted_proposal(self, inflight_proposals: int) -> Optional[str]: """Reserve one logical proposal-model call against the exact call cap.""" self.submitted_proposal_count += 1 @@ -682,6 +1174,8 @@ def llm_usage(self) -> Dict[str, Any]: "provider_usage_counting_basis": "unique_provider_receipts", "llm_calls_submitted": self.submitted_proposal_count, "llm_calls": self.llm_call_count, + "accepted_proposals": self.accepted_proposal_count, + "rejected_proposals": self.rejected_proposal_count, "prompt_tokens": self.prompt_token_count, "completion_tokens": self.completion_token_count, "total_provider_tokens": self.total_provider_tokens, @@ -692,6 +1186,13 @@ def llm_usage(self) -> Dict[str, Any]: "provider_token_budget_overshoot": self.provider_token_budget_overshoot, "limits_reached": list(self.budget_limits_reached), "inflight_proposals_at_budget_stop": self.inflight_proposals_at_budget_stop, + "measured_artifact_history": { + name: len(values) + for name, values in sorted( + self._historical_artifact_values.items() + ) + }, + "controller_allocation": self.controller_allocation, } def _record_llm_usage( @@ -746,7 +1247,21 @@ def _record_llm_usage( payload = { **metadata, "iteration": iteration, + "parent_id": result.parent_id, + "target_island": result.target_island, + "candidate_measured": bool(result.child_program_dict) + or bool(result.artifacts), + "accepted_for_archive": ( + result.error is None and bool(result.child_program_dict) + ), + # Backward-compatible field retained for existing receipt readers. "accepted_for_evaluation": result.error is None, + "rejection_reason": result.error, + "llm_response_sha256": ( + hashlib.sha256(result.llm_response.encode("utf-8")).hexdigest() + if isinstance(result.llm_response, str) + else None + ), "verified_provider_receipt": verified_receipt, "duplicate_provider_receipt": duplicate_receipt, "cumulative_usage": self.llm_usage, @@ -796,6 +1311,8 @@ def _serialize_config(self, config: Config) -> dict: "strict_diff_application": config.strict_diff_application, "enforce_evolve_blocks": config.enforce_evolve_blocks, "max_diff_blocks": config.max_diff_blocks, + "program_identity_artifact": config.program_identity_artifact, + "phenotype_identity_artifact": config.phenotype_identity_artifact, "language": config.language, "file_suffix": self.file_suffix, } @@ -856,6 +1373,12 @@ def _create_database_snapshot(self) -> Dict[str, Any]: "current_island": self.database.current_island, "feature_dimensions": self.database.config.feature_dimensions, "artifacts": {}, # Will be populated selectively + "historical_artifact_values": { + name: sorted(values) + for name, values in sorted( + self._historical_artifact_values.items() + ) + }, } # Include artifacts for programs that might be selected @@ -898,16 +1421,36 @@ async def run_evolution( # Track pending futures by island to maintain distribution pending_futures: Dict[int, Future] = {} island_pending: Dict[int, List[int]] = {i: [] for i in range(self.num_islands)} - batch_size = min(self.num_workers * 2, max_iterations) - - # Submit initial batch - distribute across islands - batch_per_island = max(1, batch_size // self.num_islands) if batch_size > 0 else 0 + # Keep at most one proposal in flight per island. Multiple simultaneous + # calls from the same unchanged parent can carry identical prompts and + # waste the proposal budget on duplicate children. + batch_per_island = 1 if max_iterations > 0 else 0 current_iteration = start_iteration stop_scheduling = False self.completion_reason = "running" - # Round-robin distribution across islands - for island_id in range(self.num_islands): + # The live controller deliberately leaves configured reserve islands + # outside the active frontier. With at least one free island, every + # completion creates a real allocation choice instead of mechanically + # refilling the island that just finished. + initial_parallelism = self._controller_parallelism(max_iterations) + initial_island_pending: Dict[int, List[int]] = { + i: [] for i in range(self.num_islands) + } + initial_islands: List[int] = [] + for slot in range(initial_parallelism): + if self.config.database.controller_scheduler.enabled: + island_id = self._select_controller_island( + initial_island_pending, + batch_per_island, + ) + else: + island_id = slot + initial_islands.append(island_id) + initial_island_pending[island_id].append(start_iteration + slot) + + # Round-robin distribution across the active island frontier. + for island_id in initial_islands: for _ in range(batch_per_island): if current_iteration < total_iterations and not stop_scheduling: future = self._submit_iteration(current_iteration, island_id) @@ -928,7 +1471,7 @@ async def run_evolution( # Early stopping tracking early_stopping_enabled = self.config.early_stopping_patience is not None if early_stopping_enabled: - best_score = float("-inf") + best_score = self._initial_early_stopping_score() iterations_without_improvement = 0 if self.config.early_stopping_patience < 0: logger.info( @@ -939,7 +1482,8 @@ async def run_evolution( logger.info( f"Early stopping enabled: patience={self.config.early_stopping_patience}, " f"threshold={self.config.convergence_threshold}, " - f"metric={self.config.early_stopping_metric}" + f"metric={self.config.early_stopping_metric}, " + f"initial_best={best_score:.4f}" ) else: logger.info("Early stopping disabled") @@ -968,11 +1512,6 @@ async def run_evolution( # Use evaluator timeout + buffer to gracefully handle stuck processes timeout_seconds = self.config.evaluator.timeout + 30 result = future.result(timeout=timeout_seconds) - budget_reason = self._record_llm_usage( - completed_iteration, - result, - inflight_proposals=len(pending_futures), - ) completed_island = ( result.target_island if isinstance(result.target_island, int) @@ -985,7 +1524,14 @@ async def run_evolution( None, ) ) + self._record_historical_artifacts(result.artifacts) + self._apply_program_identity_gate(result) self._record_controller_result(completed_island, result) + budget_reason = self._record_llm_usage( + completed_iteration, + result, + inflight_proposals=len(pending_futures), + ) if budget_reason: if not stop_scheduling: logger.info( @@ -1140,21 +1686,14 @@ async def run_evolution( # Check early stopping if early_stopping_enabled and child_program.metrics: - # Get the metric to track for early stopping - current_score = None - if self.config.early_stopping_metric in child_program.metrics: - current_score = child_program.metrics[self.config.early_stopping_metric] - elif self.config.early_stopping_metric == "combined_score": - # Default metric not found, use safe average (standard pattern) - current_score = safe_numeric_average(child_program.metrics) - else: - # User specified a custom metric that doesn't exist + current_score = self._early_stopping_score(child_program.metrics) + if current_score is None: logger.warning( - f"Early stopping metric '{self.config.early_stopping_metric}' not found, using safe numeric average" + f"Early stopping metric '{self.config.early_stopping_metric}' " + "not found or non-numeric; ignoring this result" ) - current_score = safe_numeric_average(child_program.metrics) - if current_score is not None and isinstance(current_score, (int, float)): + if current_score is not None: # Check for improvement if self.config.early_stopping_patience > 0: improvement = current_score - best_score @@ -1232,10 +1771,20 @@ async def run_evolution( # Submit the next iteration either with the original balanced # allocator or the explicitly enabled observed-result controller. if self.config.database.controller_scheduler.enabled: - island_order = [ - self._select_controller_island(island_pending, batch_size) - ] - per_island_limit = batch_size + target_parallelism = self._target_controller_parallelism( + max_iterations + ) + island_order = ( + [ + self._select_controller_island( + island_pending, + batch_per_island, + ) + ] + if len(pending_futures) < target_parallelism + else [] + ) + per_island_limit = batch_per_island else: island_order = range(self.num_islands) per_island_limit = batch_per_island @@ -1299,14 +1848,38 @@ def _submit_iteration( # Inspirations are the diverse/creative examples; size them by # num_diverse_programs (not num_top_programs) so the config parameter # actually controls the inspiration count (GitHub issue #452). - parent, inspirations = self.database.sample_from_island( - island_id=target_island, - num_inspirations=self.config.prompt.num_diverse_programs, - ) + scheduler = self.config.database.controller_scheduler + if ( + scheduler.enabled + and self.submitted_proposal_count >= scheduler.minimum_calls + and scheduler.parent_score_band is not None + ): + parent, inspirations = ( + self.database.sample_from_island_score_band( + target_island, + score_metric=scheduler.score_metric, + score_band=float(scheduler.parent_score_band), + num_inspirations=( + self.config.prompt.num_diverse_programs + ), + ) + ) + else: + parent, inspirations = self.database.sample_from_island( + island_id=target_island, + num_inspirations=( + self.config.prompt.num_diverse_programs + ), + ) # Create database snapshot db_snapshot = self._create_database_snapshot() db_snapshot["sampling_island"] = target_island # Mark which island this is for + # The selected parent must always carry its evaluator context even + # when snapshot artifact limits omit older archive entries. + parent_artifacts = self.database.get_artifacts(parent.id) + if parent_artifacts: + db_snapshot["artifacts"][parent.id] = parent_artifacts # Submit to process pool future = self.executor.submit( diff --git a/openevolve/prompt/sampler.py b/openevolve/prompt/sampler.py index 0febbe5fd4..fe4519361f 100644 --- a/openevolve/prompt/sampler.py +++ b/openevolve/prompt/sampler.py @@ -663,8 +663,16 @@ def _render_artifacts(self, artifacts: Dict[str, Union[str, bytes]]) -> str: sections = [] - # Process all artifacts using .items() - for key, value in artifacts.items(): + if self.config.artifact_include_names is None: + artifact_items = artifacts.items() + else: + artifact_items = ( + (key, artifacts[key]) + for key in self.config.artifact_include_names + if key in artifacts + ) + + for key, value in artifact_items: content = self._safe_decode_artifact(value) # Truncate if too long if len(content) > self.config.max_artifact_bytes: diff --git a/openevolve/utils/metrics_utils.py b/openevolve/utils/metrics_utils.py index 5c176e0e2e..49bf82c0dc 100644 --- a/openevolve/utils/metrics_utils.py +++ b/openevolve/utils/metrics_utils.py @@ -82,12 +82,32 @@ def get_fitness_score( metrics: All metrics from evaluation feature_dimensions: List of MAP-Elites dimensions to exclude from fitness + Evaluators may expose ``selection_eligible`` and ``selection_score`` when + admission is separate from scientific quality. An ineligible result always + ranks below an eligible one; otherwise the explicit selection score wins. + Older evaluators retain their original combined-score behavior. + Returns: - Fitness score (combined_score if available, otherwise average of non-feature metrics) + Explicit selection score, combined score, or an average of non-feature metrics """ if not metrics: return 0.0 + if "selection_eligible" in metrics: + try: + if float(metrics["selection_eligible"]) <= 0.0: + return float("-inf") + except (ValueError, TypeError, OverflowError): + return float("-inf") + + if "selection_score" in metrics: + try: + score = float(metrics["selection_score"]) + if not (score != score): + return score + except (ValueError, TypeError, OverflowError): + pass + # Always prefer combined_score if available if "combined_score" in metrics: try: diff --git a/tests/test_checkpoint_resume.py b/tests/test_checkpoint_resume.py index 791d5d5d84..895d53d4b2 100644 --- a/tests/test_checkpoint_resume.py +++ b/tests/test_checkpoint_resume.py @@ -134,6 +134,51 @@ async def run_test(): # Run the async test asyncio.run(run_test()) + def test_fresh_start_distributes_additional_seed_programs(self): + """Additional starting parents are evaluated and placed on other islands.""" + + async def run_test(): + seed_paths = [] + for index in range(3): + path = os.path.join(self.test_dir, f"seed_{index}.py") + with open(path, "w") as stream: + stream.write(f"def seed_{index}():\\n return {index}\\n") + seed_paths.append(path) + + with patch("openevolve.controller.Evaluator") as mock_evaluator_class: + mock_evaluator = MockEvaluator() + mock_evaluator_class.return_value = mock_evaluator + controller = OpenEvolve( + initial_program_path=self.test_program_path, + evaluation_file=self.evaluator_path, + config=self.config, + output_dir=self.test_dir, + seed_program_paths=seed_paths, + ) + with patch( + "openevolve.controller.ProcessParallelController" + ) as mock_controller_class: + mock_controller = Mock() + mock_controller.run_evolution = AsyncMock(return_value=None) + mock_controller.start = Mock(return_value=None) + mock_controller.stop = Mock(return_value=None) + mock_controller.shutdown_event = Mock() + mock_controller.shutdown_event.is_set.return_value = False + mock_controller_class.return_value = mock_controller + await controller.run(iterations=0) + + self.assertEqual(4, len(controller.database.programs)) + self.assertEqual(4, mock_evaluator.call_count) + seeds = [ + program + for program in controller.database.programs.values() + if program.metadata.get("seed_program") + ] + self.assertEqual(3, len(seeds)) + self.assertEqual({1, 2, 3}, {program.metadata["island"] for program in seeds}) + + asyncio.run(run_test()) + def test_duplicate_content_prevention(self): """Test that programs with identical content are not added multiple times""" diff --git a/tests/test_controller_scheduler.py b/tests/test_controller_scheduler.py index ea9c7bbb13..af5a754acb 100644 --- a/tests/test_controller_scheduler.py +++ b/tests/test_controller_scheduler.py @@ -3,7 +3,7 @@ import pytest from openevolve.config import Config, ControllerSchedulerConfig -from openevolve.database import ProgramDatabase +from openevolve.database import Program, ProgramDatabase from openevolve.process_parallel import ( ProcessParallelController, SerializableResult, @@ -21,24 +21,37 @@ def _controller(config: Config) -> ProcessParallelController: def test_controller_scheduler_loads_from_nested_config(): config = Config.from_dict( { + "evaluator": {"parallel_evaluations": 3}, "database": { - "controller_scheduler": { - "enabled": True, - "exploitation_weight": 1.0, - "underexplored_weight": 0.2, - "validity_weight": 0.5, - "diversity_weight": 0.5, - "token_efficiency_weight": 0.5, - "rejection_penalty": 0.2, - "minimum_calls": 3, + "controller_scheduler": { + "enabled": True, + "score_metric": "selection_score", + "diversity_artifact": "phenotype.txt", + "exploitation_weight": 1.0, + "underexplored_weight": 0.2, + "validity_weight": 0.5, + "diversity_weight": 0.5, + "token_efficiency_weight": 0.5, + "rejection_penalty": 0.2, + "minimum_calls": 3, + "leader_score_band": 0.05, + "parent_score_band": 0.02, + "reserve_islands": 2, + "adaptive_parallelism": 2, + } } } - } ) assert isinstance(config.database.controller_scheduler, ControllerSchedulerConfig) assert config.database.controller_scheduler.enabled is True + assert config.database.controller_scheduler.score_metric == "selection_score" + assert config.database.controller_scheduler.diversity_artifact == "phenotype.txt" assert config.database.controller_scheduler.minimum_calls == 3 + assert config.database.controller_scheduler.leader_score_band == 0.05 + assert config.database.controller_scheduler.parent_score_band == 0.02 + assert config.database.controller_scheduler.reserve_islands == 2 + assert config.database.controller_scheduler.adaptive_parallelism == 2 def test_controller_scheduler_rejects_negative_weight(): @@ -49,6 +62,60 @@ def test_controller_scheduler_rejects_negative_weight(): config.validate() +def test_controller_scheduler_rejects_negative_leader_score_band(): + config = Config() + config.database.controller_scheduler.leader_score_band = -0.01 + + with pytest.raises(ValueError, match="leader_score_band"): + config.validate() + + +def test_controller_scheduler_rejects_negative_parent_score_band(): + config = Config() + config.database.controller_scheduler.parent_score_band = -0.01 + + with pytest.raises(ValueError, match="parent_score_band"): + config.validate() + + +def test_controller_scheduler_rejects_invalid_reserve(): + config = Config() + config.database.num_islands = 3 + config.database.controller_scheduler.enabled = True + config.database.controller_scheduler.reserve_islands = 3 + + with pytest.raises(ValueError, match="reserve_islands"): + config.validate() + + +def test_score_band_parent_sampling_excludes_weak_programs(): + config = Config() + config.database.num_islands = 1 + database = ProgramDatabase(config.database) + for name, score in (("leader", 0.9), ("near", 0.895), ("weak", 0.7)): + program = Program( + id=name, + code=f"def candidate(): return {score}", + metrics={"selection_score": score}, + metadata={"island": 0}, + ) + database.programs[name] = program + database.islands[0].add(name) + + selected = { + database.sample_from_island_score_band( + 0, + score_metric="selection_score", + score_band=0.01, + num_inspirations=1, + )[0].id + for _ in range(30) + } + + assert selected <= {"leader", "near"} + assert "weak" not in selected + + def test_observed_result_allocator_prefers_productive_island(): config = Config() config.database.num_islands = 3 @@ -74,6 +141,7 @@ def test_observed_result_allocator_prefers_productive_island(): ) controller._record_controller_result(2, result) + controller.submitted_proposal_count = 1 selected = controller._select_controller_island( {0: [], 1: [], 2: []}, batch_size=3, @@ -84,6 +152,179 @@ def test_observed_result_allocator_prefers_productive_island(): assert controller._controller_island_state[2]["tokens"] == 20 +def test_controller_scheduler_uses_configured_score_metric(): + config = Config() + config.database.num_islands = 2 + config.database.controller_scheduler = ControllerSchedulerConfig( + enabled=True, + score_metric="selection_score", + exploitation_weight=1.0, + minimum_calls=1, + ) + controller = _controller(config) + controller._record_controller_result( + 1, + SerializableResult( + child_program_dict={ + "code": "def candidate(): return 1", + "metrics": { + "combined_score": 0.9, + "selection_score": 0.4, + }, + }, + target_island=1, + ), + ) + + assert controller._controller_island_state[1]["best_score"] == 0.4 + + +def test_controller_scheduler_reads_seed_and_migrated_archive_leaders(): + config = Config() + config.database.num_islands = 2 + config.database.controller_scheduler = ControllerSchedulerConfig( + enabled=True, + score_metric="selection_score", + exploitation_weight=1.0, + minimum_calls=1, + ) + database = ProgramDatabase(config.database) + database.add( + Program( + id="initial-low", + code="def candidate(): return 0", + metrics={"selection_score": 0.2}, + ), + target_island=0, + ) + controller = ProcessParallelController( + config, + "unused-evaluator.py", + database, + ) + database.add( + Program( + id="migrated-high", + code="def candidate(): return 1", + metrics={"selection_score": 0.9}, + ), + target_island=1, + ) + controller.submitted_proposal_count = 1 + + selected = controller._select_controller_island( + {0: [], 1: []}, + batch_size=1, + ) + + assert selected == 1 + assert controller.controller_allocation["islands"][1]["best_score"] == 0.9 + + +def test_controller_scheduler_keeps_post_warmup_calls_on_quality_frontier(): + config = Config() + config.database.num_islands = 3 + config.database.controller_scheduler = ControllerSchedulerConfig( + enabled=True, + score_metric="selection_score", + exploitation_weight=0.1, + underexplored_weight=1.0, + minimum_calls=3, + leader_score_band=0.01, + ) + database = ProgramDatabase(config.database) + for island, score in enumerate((0.8, 0.795, 0.6)): + database.add( + Program( + id=f"seed-{island}", + code=f"def candidate(): return {island}", + metrics={"selection_score": score}, + ), + target_island=island, + ) + controller = ProcessParallelController( + config, + "unused-evaluator.py", + database, + ) + controller.submitted_proposal_count = 3 + controller._controller_island_state[0]["calls"] = 10 + controller._controller_island_state[1]["calls"] = 9 + + selected = controller._select_controller_island( + {0: [], 1: [], 2: []}, + batch_size=1, + ) + + assert selected == 1 + assert controller.controller_allocation["leader_score_band"] == 0.01 + + +def test_controller_scheduler_counts_evaluator_defined_behavior(): + config = Config() + config.database.num_islands = 2 + config.database.controller_scheduler = ControllerSchedulerConfig( + enabled=True, + diversity_artifact="phenotype.txt", + diversity_weight=1.0, + minimum_calls=1, + ) + controller = _controller(config) + for code in ("return 1", "return 2"): + controller._record_controller_result( + 1, + SerializableResult( + child_program_dict={ + "code": code, + "metrics": {"combined_score": 0.5}, + }, + artifacts={"phenotype.txt": "same-measured-behavior"}, + target_island=1, + ), + ) + + assert controller._controller_island_state[1]["accepted"] == 2 + assert len(controller._controller_island_state[1]["unique"]) == 1 + assert controller.controller_allocation["islands"][1]["distinct_behaviors"] == 1 + + +def test_controller_scheduler_reserves_a_real_allocation_choice(): + config = Config() + config.database.num_islands = 5 + config.evaluator.parallel_evaluations = 4 + config.database.controller_scheduler = ControllerSchedulerConfig( + enabled=True, + reserve_islands=1, + ) + controller = _controller(config) + + assert controller._controller_parallelism(20) == 4 + assert controller._adaptive_controller_parallelism(20) == 4 + assert controller._controller_parallelism(2) == 2 + + config.database.num_islands = 4 + config.database.controller_scheduler.adaptive_parallelism = 2 + controller = _controller(config) + assert controller._controller_parallelism(20) == 3 + assert controller._adaptive_controller_parallelism(20) == 2 + assert controller._target_controller_parallelism(20) == 3 + controller.submitted_proposal_count = 1 + assert controller._target_controller_parallelism(20) == 2 + + +def test_controller_scheduler_rejects_excess_adaptive_parallelism(): + config = Config() + config.database.num_islands = 5 + config.evaluator.parallel_evaluations = 3 + config.database.controller_scheduler = ControllerSchedulerConfig( + enabled=True, + adaptive_parallelism=4, + ) + + with pytest.raises(ValueError, match="adaptive_parallelism"): + config.validate() + + def test_disabled_scheduler_preserves_default_configuration(): config = Config() diff --git a/tests/test_island_isolation.py b/tests/test_island_isolation.py index 5de584878c..5a64b38b10 100644 --- a/tests/test_island_isolation.py +++ b/tests/test_island_isolation.py @@ -152,6 +152,11 @@ def mock_submit_iteration(iteration, island_id=None): # Each island should have received iterations for count in island_counts.values(): self.assertGreater(count, 0) + self.assertEqual( + [0, 1, 2], + submitted_islands[:3], + "the initial parallel batch must contain one call per island", + ) finally: controller.stop() diff --git a/tests/test_island_migration.py b/tests/test_island_migration.py index 5765bd33d4..8466db7d79 100644 --- a/tests/test_island_migration.py +++ b/tests/test_island_migration.py @@ -215,6 +215,11 @@ def test_migration_creates_proper_copies(self): """Test that migration creates proper program copies""" program = self._create_test_program("original", 0.7, 0) self.db.add(program, target_island=0) + artifacts = { + "program-identity.txt": "stable-program-identity", + "search-map.txt": "compact-search-map", + } + self.db.store_artifacts(program.id, artifacts) # Set up for migration self.db.island_generations = [6, 6, 6] @@ -249,6 +254,10 @@ def test_migration_creates_proper_copies(self): # Should be marked as migrant self.assertTrue(migrant.metadata.get("migrant", False)) + # A migrated program can be sampled as a parent. Preserve the + # evaluator context required by identity and archive gates. + self.assertEqual(artifacts, self.db.get_artifacts(migrant.id)) + # Should be in correct target island target_island = migrant.metadata["island"] self.assertIn(migrant.id, self.db.islands[target_island]) diff --git a/tests/test_process_parallel.py b/tests/test_process_parallel.py index c93bbae2f0..dabbd42772 100644 --- a/tests/test_process_parallel.py +++ b/tests/test_process_parallel.py @@ -25,7 +25,11 @@ def _slow_test_worker(marker_path: str) -> str: from openevolve.config import Config, DatabaseConfig, EvaluatorConfig, LLMConfig, PromptConfig from openevolve.database import Program, ProgramDatabase from openevolve import process_parallel as process_parallel_module -from openevolve.process_parallel import ProcessParallelController, SerializableResult +from openevolve.process_parallel import ( + ProcessParallelController, + SerializableResult, + _with_archive_context, +) class TestProcessParallel(unittest.TestCase): @@ -85,9 +89,29 @@ def test_controller_initialization(self): self.assertEqual(controller.llm_usage["llm_calls"], 0) self.assertEqual(controller.llm_usage["total_provider_tokens"], 0) + def test_early_stopping_starts_from_best_archived_program(self): + self.config.early_stopping_metric = "score" + controller = ProcessParallelController( + self.config, self.eval_file, self.database + ) + + self.assertEqual(controller._initial_early_stopping_score(), 0.7) + + def test_missing_custom_early_stopping_metric_is_not_averaged(self): + self.config.early_stopping_metric = "missing" + controller = ProcessParallelController( + self.config, self.eval_file, self.database + ) + + self.assertEqual( + controller._initial_early_stopping_score(), float("-inf") + ) + def test_worker_config_serialization_preserves_llm_budgets(self): self.config.max_llm_calls = 9 self.config.max_total_provider_tokens = 12_345 + self.config.program_identity_artifact = "program-identity.txt" + self.config.phenotype_identity_artifact = "phenotype-identity.txt" controller = ProcessParallelController( self.config, self.eval_file, self.database ) @@ -96,6 +120,406 @@ def test_worker_config_serialization_preserves_llm_budgets(self): self.assertEqual(serialized["max_llm_calls"], 9) self.assertEqual(serialized["max_total_provider_tokens"], 12_345) + self.assertEqual( + serialized["program_identity_artifact"], "program-identity.txt" + ) + self.assertEqual( + serialized["phenotype_identity_artifact"], "phenotype-identity.txt" + ) + + def test_worker_rejects_unchanged_program_identity(self): + self.config.program_identity_artifact = "program-identity.txt" + + class FakeLLM: + last_call_metadata = { + "provider_response_id": "response-semantic-noop", + "model": "fake-model", + "usage": { + "prompt_tokens": 5, + "completion_tokens": 3, + "total_tokens": 8, + }, + } + + async def generate_with_context(self, **_kwargs): + return "\n".join( + [ + "<" * 7 + " SEARCH", + " return 1", + "=" * 7, + " return 2", + ">" * 7 + " REPLACE", + ] + ) + + class IdentityEvaluator: + async def evaluate_program(self, _code, _child_id): + return {"combined_score": 0.8} + + def get_pending_artifacts(self, _child_id): + return {"program-identity.txt": "same-identity"} + + class FakePromptSampler: + def build_prompt(self, **_kwargs): + return {"system": "system", "user": "user"} + + parent = Program( + id="parent", + code="def solve():\n return 1\n", + language="python", + metrics={"combined_score": 0.5}, + metadata={"island": 0}, + ) + snapshot = { + "programs": {"parent": parent.to_dict()}, + "artifacts": { + "parent": {"program-identity.txt": "same-identity"} + }, + "current_island": 0, + "islands": [["parent"]], + "feature_dimensions": ["combined_score"], + "sampling_island": 0, + } + + with ( + patch.object( + process_parallel_module, + "_lazy_init_worker_components", + return_value=None, + ), + patch.object( + process_parallel_module, + "_worker_config", + self.config, + create=True, + ), + patch.object( + process_parallel_module, + "_worker_llm_ensemble", + FakeLLM(), + create=True, + ), + patch.object( + process_parallel_module, + "_worker_evaluator", + IdentityEvaluator(), + create=True, + ), + patch.object( + process_parallel_module, + "_worker_prompt_sampler", + FakePromptSampler(), + create=True, + ), + ): + result = process_parallel_module._run_iteration_worker( + 9, snapshot, "parent", [] + ) + + self.assertIsNone(result.child_program_dict) + self.assertIn("same program identity", result.error) + + def test_worker_builds_deterministic_archive_context(self): + self.config.prompt.archive_context_artifact = "program-structure.json" + self.config.prompt.archive_context_max_items = 2 + parent_artifacts = { + "program-structure.json": '{"schedule":[1]}', + "guidance": "keep this", + } + snapshot = { + "artifacts": { + "z": {"program-structure.json": '{"schedule":[2]}'}, + "a": {"program-structure.json": '{"schedule":[1]}'}, + "m": {"program-structure.json": '{"schedule":[3]}'}, + } + } + + with patch.object( + process_parallel_module, + "_worker_config", + self.config, + create=True, + ): + augmented = _with_archive_context(parent_artifacts, snapshot) + + self.assertIsNot(augmented, parent_artifacts) + self.assertNotIn("archive-context.json", parent_artifacts) + context = json.loads(augmented["archive-context.json"]) + self.assertEqual(context["archive_item_count"], 3) + self.assertEqual(context["included_item_count"], 2) + self.assertEqual( + context["items"], + ['{"schedule":[1]}', '{"schedule":[2]}'], + ) + self.assertTrue(context["truncated"]) + + def test_worker_filters_retained_proposal_options(self): + self.config.prompt.proposal_neighborhood_artifact = ( + "proposal-neighborhood.json" + ) + self.config.prompt.proposal_options_max_items = 1 + parent_artifacts = { + "program-structure.txt": "parent", + "proposal-neighborhood.json": json.dumps( + { + "schema_version": 1, + "identity_artifact": "program-structure.txt", + "options": [ + {"id": "occupied", "identity": "held", "edit": "skip"}, + {"id": "first", "identity": "new-a", "edit": "use a"}, + {"id": "second", "identity": "new-b", "edit": "use b"}, + ], + } + ), + } + snapshot = { + "artifacts": { + "parent": {"program-structure.txt": "parent"}, + "other": {"program-structure.txt": "held"}, + } + } + + with patch.object( + process_parallel_module, + "_worker_config", + self.config, + create=True, + ): + augmented = _with_archive_context(parent_artifacts, snapshot) + + context = json.loads(augmented["proposal-options.json"]) + self.assertEqual(context["declared_option_count"], 3) + self.assertEqual(context["excluded_retained_count"], 1) + self.assertEqual(context["available_option_count"], 2) + self.assertEqual(context["included_option_count"], 1) + self.assertEqual(context["options"][0]["id"], "first") + self.assertTrue(context["truncated"]) + + def test_worker_filters_options_from_complete_measured_history(self): + self.config.prompt.proposal_neighborhood_artifact = ( + "proposal-neighborhood.json" + ) + parent_artifacts = { + "program-structure.txt": "parent", + "proposal-neighborhood.json": json.dumps( + { + "schema_version": 1, + "identity_artifact": "program-structure.txt", + "options": [ + {"id": "live", "identity": "held", "edit": "skip"}, + { + "id": "displaced", + "identity": "measured-before", + "edit": "skip", + }, + {"id": "fresh", "identity": "new", "edit": "use"}, + ], + } + ), + } + snapshot = { + "artifacts": { + "parent": {"program-structure.txt": "parent"}, + "other": {"program-structure.txt": "held"}, + }, + "historical_artifact_values": { + "program-structure.txt": [ + "parent", + "held", + "measured-before", + ] + }, + } + + with patch.object( + process_parallel_module, + "_worker_config", + self.config, + create=True, + ): + augmented = _with_archive_context(parent_artifacts, snapshot) + + context = json.loads(augmented["proposal-options.json"]) + self.assertEqual(context["excluded_retained_count"], 1) + self.assertEqual(context["excluded_historical_count"], 2) + self.assertEqual(context["excluded_known_count"], 2) + self.assertEqual( + [option["id"] for option in context["options"]], + ["fresh"], + ) + + def test_worker_rejects_malformed_proposal_neighborhood(self): + self.config.prompt.proposal_neighborhood_artifact = ( + "proposal-neighborhood.json" + ) + with patch.object( + process_parallel_module, + "_worker_config", + self.config, + create=True, + ): + with self.assertRaisesRegex(ValueError, "schema_version 1"): + _with_archive_context( + {"proposal-neighborhood.json": "{}"}, + {"artifacts": {}}, + ) + + def test_proposal_totals_include_rejections_without_live_scheduler(self): + controller = ProcessParallelController( + self.config, self.eval_file, self.database + ) + + controller._record_controller_result( + 0, SerializableResult(error="invalid diff") + ) + controller._record_controller_result( + 0, + SerializableResult( + child_program_dict={"id": "child", "code": "pass"} + ), + ) + + self.assertEqual(controller.llm_usage["rejected_proposals"], 1) + self.assertEqual(controller.llm_usage["accepted_proposals"], 1) + + def test_live_archive_rejects_duplicate_program_identity(self): + self.config.program_identity_artifact = "program-identity.txt" + self.database.store_artifacts( + "test_0", + {"program-identity.txt": "existing-program"}, + ) + usage_path = Path(self.test_dir) / "identity-usage.jsonl" + controller = ProcessParallelController( + self.config, + self.eval_file, + self.database, + usage_output_path=str(usage_path), + ) + duplicate = SerializableResult( + child_program_dict={"id": "duplicate", "code": "pass"}, + parent_id="test_0", + llm_response="duplicate proposal", + artifacts={"program-identity.txt": "existing-program"}, + target_island=2, + llm_metadata={ + "provider_response_id": "identity-duplicate", + "usage": { + "prompt_tokens": 3, + "completion_tokens": 2, + "total_tokens": 5, + }, + }, + ) + first_new = SerializableResult( + child_program_dict={"id": "new", "code": "pass"}, + artifacts={"program-identity.txt": "new-program"}, + ) + concurrent_duplicate = SerializableResult( + child_program_dict={"id": "duplicate-new", "code": "pass"}, + artifacts={"program-identity.txt": "new-program"}, + ) + + controller._apply_program_identity_gate(duplicate) + controller._record_controller_result(2, duplicate) + controller._record_llm_usage(4, duplicate) + controller._apply_program_identity_gate(first_new) + controller._apply_program_identity_gate(concurrent_duplicate) + + self.assertIn("already exists", duplicate.error) + self.assertIsNone(first_new.error) + self.assertIn("already exists", concurrent_duplicate.error) + receipt = json.loads(usage_path.read_text(encoding="utf-8")) + self.assertFalse(receipt["accepted_for_evaluation"]) + self.assertFalse(receipt["accepted_for_archive"]) + self.assertTrue(receipt["candidate_measured"]) + self.assertIn("already exists", receipt["rejection_reason"]) + self.assertEqual(receipt["parent_id"], "test_0") + self.assertEqual( + receipt["llm_response_sha256"], + "e7eaec4f27fa10c6425a346a7697c42320588cfc59bc6b" + "ac458a5ddfb268c7a1", + ) + self.assertEqual(receipt["target_island"], 2) + self.assertEqual(receipt["cumulative_usage"]["rejected_proposals"], 1) + self.assertEqual( + receipt["cumulative_usage"]["measured_artifact_history"], + {"program-identity.txt": 1}, + ) + + def test_live_archive_rejects_duplicate_phenotype_identity(self): + self.config.phenotype_identity_artifact = "phenotype-identity.txt" + self.database.store_artifacts( + "test_0", + {"phenotype-identity.txt": "existing-phenotype"}, + ) + controller = ProcessParallelController( + self.config, + self.eval_file, + self.database, + ) + duplicate = SerializableResult( + child_program_dict={"id": "duplicate", "code": "different source"}, + artifacts={"phenotype-identity.txt": "existing-phenotype"}, + ) + novel = SerializableResult( + child_program_dict={"id": "novel", "code": "different source"}, + artifacts={"phenotype-identity.txt": "novel-phenotype"}, + ) + + controller._apply_program_identity_gate(duplicate) + controller._apply_program_identity_gate(novel) + + self.assertIn("phenotype identity already exists", duplicate.error) + self.assertIsNone(novel.error) + + def test_measured_history_survives_behavior_rejection_and_snapshot(self): + self.config.program_identity_artifact = "program-identity.txt" + self.config.phenotype_identity_artifact = "phenotype-identity.txt" + self.config.prompt.proposal_neighborhood_artifact = ( + "proposal-neighborhood.json" + ) + neighborhood = json.dumps( + { + "schema_version": 1, + "identity_artifact": "program-identity.txt", + "options": [], + } + ) + self.database.store_artifacts( + "test_0", + { + "program-identity.txt": "baseline-program", + "phenotype-identity.txt": "baseline-behavior", + "proposal-neighborhood.json": neighborhood, + }, + ) + controller = ProcessParallelController( + self.config, self.eval_file, self.database + ) + measured = SerializableResult( + child_program_dict={"id": "measured", "code": "pass"}, + artifacts={ + "program-identity.txt": "new-program", + "phenotype-identity.txt": "baseline-behavior", + "proposal-neighborhood.json": neighborhood, + }, + ) + + controller._record_historical_artifacts(measured.artifacts) + controller._apply_program_identity_gate(measured) + snapshot = controller._create_database_snapshot() + + self.assertIn("phenotype identity already exists", measured.error) + self.assertIn( + "new-program", + snapshot["historical_artifact_values"]["program-identity.txt"], + ) + self.assertIn( + "new-program", + self.database.historical_artifact_values[ + "program-identity.txt" + ], + ) def test_controller_start_stop(self): """Test starting and stopping the controller""" @@ -178,6 +602,21 @@ def test_database_snapshot_creation(self): self.assertIn("id", prog_dict) self.assertIn("code", prog_dict) + def test_database_checkpoint_preserves_measured_artifact_history(self): + checkpoint = Path(self.test_dir) / "history-checkpoint" + self.database.historical_artifact_values = { + "program-identity.txt": {"first", "displaced"} + } + + self.database.save(str(checkpoint), iteration=7) + restored = ProgramDatabase(self.config.database) + restored.load(str(checkpoint)) + + self.assertEqual( + restored.historical_artifact_values, + {"program-identity.txt": {"first", "displaced"}}, + ) + def test_run_evolution_basic(self): """Test basic evolution run""" @@ -327,6 +766,8 @@ async def run_test(): "provider_usage_counting_basis": "unique_provider_receipts", "llm_calls_submitted": 1, "llm_calls": 1, + "accepted_proposals": 0, + "rejected_proposals": 1, "prompt_tokens": 10, "completion_tokens": 5, "total_provider_tokens": 15, @@ -337,6 +778,46 @@ async def run_test(): "provider_token_budget_overshoot": 0, "limits_reached": ["max_llm_calls"], "inflight_proposals_at_budget_stop": 1, + "measured_artifact_history": {}, + "controller_allocation": { + "enabled": False, + "score_metric": "combined_score", + "diversity_artifact": None, + "leader_score_band": None, + "parent_score_band": None, + "reserve_islands": 1, + "active_parallelism": 3, + "adaptive_parallelism": 3, + "islands": [ + { + "island": 0, + "calls": 0, + "accepted": 0, + "rejected": 0, + "distinct_behaviors": 0, + "best_score": 0.45, + "tokens": 0, + }, + { + "island": 1, + "calls": 0, + "accepted": 0, + "rejected": 0, + "distinct_behaviors": 0, + "best_score": 0.55, + "tokens": 0, + }, + { + "island": 2, + "calls": 0, + "accepted": 0, + "rejected": 0, + "distinct_behaviors": 0, + "best_score": 0.65, + "tokens": 0, + }, + ], + }, }, ) diff --git a/tests/test_program_identity_config.py b/tests/test_program_identity_config.py new file mode 100644 index 0000000000..9df43702c4 --- /dev/null +++ b/tests/test_program_identity_config.py @@ -0,0 +1,129 @@ +"""Configuration checks for evaluator-defined program identities.""" + +import unittest + +from openevolve.config import Config + + +class ProgramIdentityConfigTests(unittest.TestCase): + def test_identity_artifact_round_trips_from_mapping(self): + config = Config.from_dict( + {"program_identity_artifact": "program-identity.txt"} + ) + + self.assertEqual( + config.program_identity_artifact, + "program-identity.txt", + ) + self.assertEqual( + config.to_dict()["program_identity_artifact"], + "program-identity.txt", + ) + + def test_empty_identity_artifact_is_rejected(self): + with self.assertRaisesRegex( + ValueError, + "program_identity_artifact must be a non-empty string", + ): + Config.from_dict({"program_identity_artifact": " "}) + + def test_archive_context_configuration_round_trips(self): + config = Config.from_dict( + { + "prompt": { + "archive_context_artifact": "program-structure.json", + "archive_context_max_items": 48, + "artifact_include_names": [ + "archive-context.json", + "guidance.txt", + ], + } + } + ) + + self.assertEqual( + config.prompt.archive_context_artifact, + "program-structure.json", + ) + self.assertEqual(config.prompt.archive_context_max_items, 48) + self.assertEqual( + config.prompt.artifact_include_names, + ["archive-context.json", "guidance.txt"], + ) + + def test_empty_archive_context_artifact_is_rejected(self): + with self.assertRaisesRegex( + ValueError, + "archive_context_artifact must be a non-empty string", + ): + Config.from_dict( + {"prompt": {"archive_context_artifact": " "}} + ) + + def test_artifact_include_names_require_archive_context(self): + with self.assertRaisesRegex( + ValueError, + "must include archive-context.json", + ): + Config.from_dict( + { + "prompt": { + "archive_context_artifact": "search-map.txt", + "artifact_include_names": ["guidance"], + } + } + ) + + def test_duplicate_artifact_include_names_are_rejected(self): + with self.assertRaisesRegex(ValueError, "must not contain duplicates"): + Config.from_dict( + { + "prompt": { + "artifact_include_names": ["guidance", "guidance"], + } + } + ) + + def test_proposal_neighborhood_configuration_round_trips(self): + config = Config.from_dict( + { + "prompt": { + "proposal_neighborhood_artifact": "neighborhood.json", + "proposal_options_max_items": 17, + "artifact_include_names": ["proposal-options.json"], + } + } + ) + + self.assertEqual( + config.prompt.proposal_neighborhood_artifact, + "neighborhood.json", + ) + self.assertEqual(config.prompt.proposal_options_max_items, 17) + + def test_proposal_neighborhood_requires_rendered_options(self): + with self.assertRaisesRegex( + ValueError, + "must include proposal-options.json", + ): + Config.from_dict( + { + "prompt": { + "proposal_neighborhood_artifact": "neighborhood.json", + "artifact_include_names": ["guidance"], + } + } + ) + + def test_proposal_option_limit_must_be_positive(self): + with self.assertRaisesRegex( + ValueError, + "proposal_options_max_items must be a positive integer", + ): + Config.from_dict( + {"prompt": {"proposal_options_max_items": 0}} + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_prompt_sampler_comprehensive.py b/tests/test_prompt_sampler_comprehensive.py index b001c5e670..9ef74d43e1 100644 --- a/tests/test_prompt_sampler_comprehensive.py +++ b/tests/test_prompt_sampler_comprehensive.py @@ -184,6 +184,27 @@ def test_build_prompt_with_all_optional_parameters(self): self.assertIn("best", prompt["user"]) self.assertIn("creative", prompt["user"]) + def test_artifact_include_names_limit_and_order_prompt_context(self): + config = Config() + config.prompt.artifact_include_names = ["search-map.txt", "guidance"] + sampler = PromptSampler(config.prompt) + + prompt = sampler.build_prompt( + current_program="def main(): pass", + program_artifacts={ + "large-receipt.json": "omit me", + "guidance": "short direction", + "search-map.txt": "compact map", + }, + ) + + self.assertNotIn("large-receipt.json", prompt["user"]) + self.assertNotIn("omit me", prompt["user"]) + self.assertLess( + prompt["user"].index("search-map.txt"), + prompt["user"].index("guidance"), + ) + def test_fitness_calculation_consistency(self): """Test that fitness calculation is consistent across all methods""" metrics = { diff --git a/tests/test_selection_score.py b/tests/test_selection_score.py new file mode 100644 index 0000000000..2796135544 --- /dev/null +++ b/tests/test_selection_score.py @@ -0,0 +1,94 @@ +"""Selection-score conventions for exact scientific evaluators.""" + +import logging + +from openevolve.utils.metrics_utils import get_fitness_score +from openevolve.config import DatabaseConfig +from openevolve.database import Program, ProgramDatabase + + +def test_selection_score_precedes_combined_score() -> None: + metrics = { + "selection_eligible": 1.0, + "selection_score": 0.42, + "combined_score": 0.99, + } + assert get_fitness_score(metrics) == 0.42 + + +def test_ineligible_measurement_ranks_below_valid_zero() -> None: + declined = get_fitness_score( + { + "selection_eligible": 0.0, + "selection_score": 100.0, + "combined_score": 100.0, + } + ) + valid_zero = get_fitness_score( + { + "selection_eligible": 1.0, + "selection_score": 0.0, + "combined_score": 0.0, + } + ) + assert declined < valid_zero + + +def test_legacy_combined_score_is_unchanged() -> None: + assert get_fitness_score({"combined_score": 0.73, "other": 1.0}) == 0.73 + + +def test_ineligible_program_is_lineage_only() -> None: + database = ProgramDatabase( + DatabaseConfig( + in_memory=True, + population_size=8, + archive_size=4, + num_islands=2, + ) + ) + program = Program( + id="declined", + code="return 0", + metrics={"selection_eligible": 0.0, "selection_score": 100.0}, + ) + database.add(program) + + assert "declined" in database.programs + assert "declined" not in database.archive + assert all("declined" not in island for island in database.islands) + assert program.metadata["selection_ineligible"] is True + + +def test_best_program_log_reports_the_deciding_metric(caplog) -> None: + database = ProgramDatabase( + DatabaseConfig( + in_memory=True, + population_size=8, + archive_size=4, + num_islands=2, + ) + ) + database.add( + Program( + id="old", + code="return 0", + metrics={"selection_score": 0.70, "combined_score": 0.90}, + ) + ) + with caplog.at_level(logging.INFO, logger="openevolve.database"): + database.add( + Program( + id="new", + code="return 1", + metrics={"selection_score": 0.75, "combined_score": 0.60}, + ) + ) + + message = next( + record.message + for record in caplog.records + if record.message.startswith("New best program new") + ) + assert "selection_score: 0.7000 → 0.7500, +0.0500" in message + assert "combined_score" not in message