Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 19 additions & 5 deletions packages/dash_evals/src/dash_evals/runner/tasks/task_helpers.py
Original file line number Diff line number Diff line change
Expand Up @@ -90,9 +90,17 @@ def append_context_injection(solver_chain: list, config: dict) -> None:

Args:
solver_chain: The solver chain list to append to.
config: Task manifest entry with 'variant' key.
config: Task manifest entry with 'variant' key or 'metadata' key.
"""
variant = config.get("variant", {})
metadata = config.get("metadata", {})
variant = metadata.get("variant_config")
if variant is None:
variant = config.get("variant", {})

# If variant is still just a name string, we can't extract files from it.
if isinstance(variant, str):
return

# Support both old "context_files" and new "files" key
context_files = variant.get("files") or variant.get("context_files", [])
if context_files:
Expand All @@ -109,12 +117,18 @@ def append_model_interaction(

Args:
solver_chain: The solver chain list to append to.
config: Task manifest entry with 'variant' key.
config: Task manifest entry with 'variant' key or 'metadata' key.
extra_tools: Additional tools to include alongside MCP (optional).
"""
tools: list[Tool | MCPServer] = []
variant = config.get("variant", {})
mcp_servers_config = variant.get("mcp_servers", [])
metadata = config.get("metadata", {})
variant = metadata.get("variant_config")
if variant is None:
variant = config.get("variant", {})

mcp_servers_config = []
if not isinstance(variant, str):
mcp_servers_config = variant.get("mcp_servers", [])

if mcp_servers_config:
sandbox_type = config.get("sandbox_type", "local")
Expand Down
25 changes: 25 additions & 0 deletions packages/dataset_config_dart/lib/src/models/task.dart
Original file line number Diff line number Diff line change
Expand Up @@ -121,7 +121,32 @@ sealed class Task with _$Task {
Map<String, dynamic>? metadata,
}) = _Task;

const Task._();

factory Task.fromJson(Map<String, dynamic> json) => _$TaskFromJson(json);

/// Get a job-level task argument.
Object? getArg(String key, [Object? defaultValue]) {
final args = metadata?['args'] as Map<String, dynamic>?;
return args?[key] ?? defaultValue;
}

/// Hydrate and return MCP servers defined in the variant.
///
/// This is a convenience method that returns the raw config maps.
/// In Dart, hydration happens in the Python runner, so we return
/// the config maps that the runner will use.
List<Map<String, dynamic>> getMcp() {
final vcfg = metadata?['variant_config'] as Map<String, dynamic>?;
return (vcfg?['mcp_servers'] as List?)?.cast<Map<String, dynamic>>() ??
const [];
}

/// Return the skill paths defined in the variant.
List<String> getSkills() {
final vcfg = metadata?['variant_config'] as Map<String, dynamic>?;
return (vcfg?['skills'] as List?)?.cast<String>() ?? const [];
}
}

class TaskMetadata {
Expand Down
4 changes: 2 additions & 2 deletions packages/dataset_config_dart/lib/src/models/task.freezed.dart

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

6 changes: 6 additions & 0 deletions packages/dataset_config_dart/lib/src/models/variant.dart
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,12 @@ sealed class Variant with _$Variant {
@JsonKey(name: 'task_parameters')
@Default({})
Map<String, dynamic> taskParameters,

/// Optional metadata for the variant.
@JsonKey(name: 'metadata') @Default({}) Map<String, dynamic> metadata,

/// Optional tags for the variant.
@JsonKey(name: 'tags') @Default([]) List<String> tags,
}) = _Variant;

const Variant._();
Expand Down
Loading
Loading