-
Notifications
You must be signed in to change notification settings - Fork 691
FEAT: Adding PyRITInitializer parameters #1456
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: main
Are you sure you want to change the base?
Changes from all commits
e89f98e
76acc5f
45b38fb
2409ee3
ae7f1fc
0242b32
76c3c23
af609dc
ceac2f1
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -30,14 +30,19 @@ memory_db_type: sqlite | |
| # | ||
| # Each initializer can be specified as: | ||
| # - A simple string (name only) | ||
| # - A dictionary with 'name' and optional 'args' for constructor arguments | ||
| # - A dictionary with 'name' and optional 'args' for parameters | ||
| # | ||
| # Parameters are lists of strings. Use the CLI command | ||
| # `pyrit_scan --list-initializers` to see available parameters. | ||
| # | ||
| # Example: | ||
| # initializers: | ||
| # - simple | ||
| # - name: airt | ||
|
Comment on lines
40
to
-38
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. should this be Also, why is |
||
| # - name: target | ||
| # args: | ||
| # some_param: value | ||
| # tags: | ||
| # - default | ||
| # - scorer | ||
| initializers: | ||
| - simple | ||
|
|
||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -44,7 +44,7 @@ A list of built-in initializers to run during PyRIT initialization. Initializers | |
| Each entry can be: | ||
|
|
||
| - **A simple string** — just the initializer name | ||
| - **A dictionary** — with `name` and optional `args` for constructor arguments | ||
| - **A dictionary** — with `name` and optional `args` (each arg is a list of strings passed to `initialize_async`) | ||
|
|
||
| Example: | ||
|
|
||
|
|
@@ -53,7 +53,9 @@ initializers: | |
| - simple | ||
| - name: airt | ||
| args: | ||
| some_param: value | ||
| tags: | ||
| - default | ||
| - scorer | ||
|
Comment on lines
+56
to
+58
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. these should be for target, not airt right? |
||
| ``` | ||
|
|
||
| Use `pyrit list initializers` in the CLI to see all registered initializers. See the [initializer documentation notebook](../code/setup/pyrit_initializer.ipynb) for reference. | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -77,7 +77,7 @@ def __init__( | |
| config_file: Optional[Path] = None, | ||
| database: Optional[str] = None, | ||
| initialization_scripts: Optional[list[Path]] = None, | ||
| initializer_names: Optional[list[str]] = None, | ||
| initializer_names: Optional[list[Any]] = None, | ||
| env_files: Optional[list[Path]] = None, | ||
| log_level: Optional[int] = None, | ||
| ): | ||
|
|
@@ -94,7 +94,9 @@ def __init__( | |
| The file uses .pyrit_conf extension but is YAML format. | ||
| database: Database type (InMemory, SQLite, or AzureSQL). | ||
| initialization_scripts: Optional list of initialization script paths. | ||
| initializer_names: Optional list of built-in initializer names to run. | ||
| initializer_names: Optional list of initializer entries. Each entry can be | ||
| a string name (e.g., "simple") or a dict with 'name' and optional 'args' | ||
| (e.g., {"name": "target", "args": {"tags": "default,scorer"}}). | ||
| env_files: Optional list of environment file paths to load in order. | ||
| log_level: Logging level constant (e.g., logging.WARNING). Defaults to logging.WARNING. | ||
|
|
||
|
|
@@ -130,9 +132,7 @@ def __init__( | |
| # Use canonical mapping from configuration_loader | ||
| self._database = _MEMORY_DB_TYPE_MAP[config.memory_db_type] | ||
| self._initialization_scripts = config._resolve_initialization_scripts() | ||
| self._initializer_names = ( | ||
| [ic.name for ic in config._initializer_configs] if config._initializer_configs else None | ||
| ) | ||
| self._initializer_configs = config._initializer_configs if config._initializer_configs else None | ||
| self._env_files = config._resolve_env_files() | ||
| self._operator = config.operator | ||
| self._operation = config.operation | ||
|
|
@@ -289,15 +289,20 @@ async def run_scenario_async( | |
|
|
||
| # Run initializers before scenario | ||
| initializer_instances = None | ||
| if context._initializer_names: | ||
| print(f"Running {len(context._initializer_names)} initializer(s)...") | ||
| if context._initializer_configs: | ||
| print(f"Running {len(context._initializer_configs)} initializer(s)...") | ||
| sys.stdout.flush() | ||
|
|
||
| initializer_instances = [] | ||
|
|
||
| for name in context._initializer_names: | ||
| initializer_class = context.initializer_registry.get_class(name) | ||
| initializer_instances.append(initializer_class()) | ||
| for config in context._initializer_configs: | ||
| initializer_class = context.initializer_registry.get_class(config.name) | ||
| instance = initializer_class() | ||
| if config.args: | ||
| instance.params = { | ||
| k: [str(i) for i in v] if isinstance(v, list) else [str(v)] for k, v in config.args.items() | ||
| } | ||
| initializer_instances.append(instance) | ||
|
|
||
| # Re-initialize PyRIT with the scenario-specific initializers | ||
| # This resets memory and applies initializer defaults | ||
|
|
@@ -479,6 +484,13 @@ def format_initializer_metadata(*, initializer_metadata: InitializerMetadata) -> | |
| else: | ||
| print(" Required Environment Variables: None") | ||
|
|
||
| if initializer_metadata.supported_parameters: | ||
| print(" Supported Parameters:") | ||
| for param_name, param_desc, param_required, param_default in initializer_metadata.supported_parameters: | ||
| req_str = " (required)" if param_required else "" | ||
| default_str = f" [default: {param_default}]" if param_default else "" | ||
| print(f" - {param_name}{req_str}{default_str}: {param_desc}") | ||
|
|
||
| if initializer_metadata.class_description: | ||
| print(" Description:") | ||
| print(_format_wrapped_text(text=initializer_metadata.class_description, indent=" ")) | ||
|
|
@@ -775,7 +787,10 @@ async def print_initializers_list_async(*, context: FrontendCore, discovery_path | |
| "initialization scripts, and env files. CLI arguments override config file values. " | ||
| "If not specified, ~/.pyrit/.pyrit_conf is loaded if it exists." | ||
| ), | ||
| "initializers": "Built-in initializer names to run before the scenario (e.g., openai_objective_target)", | ||
| "initializers": ( | ||
| "Built-in initializer names to run before the scenario. " | ||
| "Supports optional params with name:key=val syntax (e.g., target:tags=default,scorer)" | ||
| ), | ||
| "initialization_scripts": "Paths to custom Python initialization scripts to run before the scenario", | ||
| "env_files": "Paths to environment files to load in order (e.g., .env.production .env.local). Later files " | ||
| "override earlier ones.", | ||
|
|
@@ -792,6 +807,50 @@ async def print_initializers_list_async(*, context: FrontendCore, discovery_path | |
| } | ||
|
|
||
|
|
||
| def _parse_initializer_arg(arg: str) -> dict[str, Any]: | ||
| """ | ||
| Parse an initializer CLI argument into a dict for ConfigurationLoader. | ||
|
|
||
| Supports two formats: | ||
| - Simple name: "simple" → {"name": "simple"} | ||
| - Name with params: "target:tags=default,scorer" → {"name": "target", "args": {"tags": "default,scorer"}} | ||
|
|
||
| For multiple params, separate with semicolons: "name:key1=val1;key2=val2" | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. maybe this is not common and I don't have a solution for this but this has the potential to get long esp if a user has many initializers with many tags
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Potentially! But I hope people use the config, and also I could see this shortening things in some cases (e.g. something that would otherwise take several initializers can use an argument). |
||
|
|
||
| Args: | ||
| arg: The CLI argument string. | ||
|
|
||
| Returns: | ||
| dict: A dict with 'name' and optionally 'args' keys. | ||
|
|
||
| Raises: | ||
| ValueError: If the argument format is invalid. | ||
| """ | ||
| if ":" not in arg: | ||
| return arg # type: ignore[return-value] | ||
|
|
||
| name, params_str = arg.split(":", 1) | ||
| if not name: | ||
| raise ValueError(f"Invalid initializer argument '{arg}': missing name before ':'") | ||
|
|
||
| args: dict[str, list[str]] = {} | ||
| for pair in params_str.split(";"): | ||
| pair = pair.strip() | ||
| if not pair: | ||
| continue | ||
| if "=" not in pair: | ||
| raise ValueError(f"Invalid initializer parameter '{pair}' in '{arg}': expected key=value format") | ||
| key, value = pair.split("=", 1) | ||
| key = key.strip() | ||
| if not key: | ||
| raise ValueError(f"Invalid initializer parameter in '{arg}': empty key") | ||
| args[key] = [v.strip() for v in value.split(",")] | ||
|
|
||
| if args: | ||
| return {"name": name, "args": args} | ||
| return name # type: ignore[return-value] | ||
|
|
||
|
|
||
| def parse_run_arguments(*, args_string: str) -> dict[str, Any]: | ||
| """ | ||
| Parse run command arguments from a string (for shell mode). | ||
|
|
@@ -839,11 +898,11 @@ def parse_run_arguments(*, args_string: str) -> dict[str, Any]: | |
| i = 1 | ||
| while i < len(parts): | ||
| if parts[i] == "--initializers": | ||
| # Collect initializers until next flag | ||
| # Collect initializers until next flag, parsing name:key=val syntax | ||
| result["initializers"] = [] | ||
| i += 1 | ||
| while i < len(parts) and not parts[i].startswith("--"): | ||
| result["initializers"].append(parts[i]) | ||
| result["initializers"].append(_parse_initializer_arg(parts[i])) | ||
| i += 1 | ||
| elif parts[i] == "--initialization-scripts": | ||
| # Collect script paths until next flag | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -44,7 +44,7 @@ class PyRITShell(cmd.Cmd): | |
| --no-animation Disable the animated startup banner | ||
|
|
||
| Run Command Options: | ||
| --initializers <name> ... Built-in initializers to run before the scenario | ||
| --initializers <name> ... Built-in initializers (supports name:key=val1,val2 syntax) | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I'm confused by that syntax. Do you have an example?
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Found |
||
| --initialization-scripts <...> Custom Python scripts to run before the scenario | ||
| --env-files <path> ... Environment files to load in order (overrides startup default) | ||
| --strategies, -s <s1> ... Strategy names to use | ||
|
|
@@ -135,7 +135,7 @@ def do_run(self, line: str) -> None: | |
| run <scenario_name> [options] | ||
|
|
||
| Options: | ||
| --initializers <name> ... Built-in initializers to run before the scenario | ||
| --initializers <name> ... Built-in initializers (supports name:key=val1,val2 syntax) | ||
| --initialization-scripts <...> Custom Python scripts to run before the scenario | ||
| --env-files <path> ... Environment files to load in order | ||
| --strategies, -s <s1> <s2> ... Strategy names to use | ||
|
|
@@ -360,6 +360,7 @@ def do_help(self, arg: str) -> None: | |
| print(f" {frontend_core.ARG_HELP['initializers']}") | ||
| print(" Every scenario requires at least one initializer") | ||
| print(" Example: run foundry --initializers openai_objective_target load_default_datasets") | ||
| print(" With params: run foundry --initializers target:tags=default,scorer") | ||
| print() | ||
| print(" --initialization-scripts <path> [<path> ...] (Alternative to --initializers)") | ||
| print(f" {frontend_core.ARG_HELP['initialization_scripts']}") | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -325,8 +325,12 @@ def _resolve_initializers(self) -> Sequence["PyRITInitializer"]: | |
| f"Initializer '{config.name}' not found in registry.\nAvailable initializers: {available}" | ||
| ) | ||
|
|
||
| # Instantiate with args if provided | ||
| instance = initializer_class(**config.args) if config.args else initializer_class() | ||
| # Instantiate and set params if provided | ||
| instance = initializer_class() | ||
| if config.args: | ||
| instance.params = { | ||
| k: [str(i) for i in v] if isinstance(v, list) else [str(v)] for k, v in config.args.items() | ||
| } | ||
|
Comment on lines
-328
to
+333
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Nit: might be wrong, but the old implementation would implicitly validate the args passed, since by instantiating Also, neither |
||
|
|
||
| resolved.append(instance) | ||
|
|
||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Should we implement something like
pyrit_scan --list-initializers --helporpyrit_scan --list-parameters simpleinstead to see available parameters? I imagine that users would want a dedicated interface for figuring out parameters once they choose one or more initializers and don't need to list them all