diff --git a/docs/examples/README.md b/docs/examples/README.md index 66e380ccb..691dcd12a 100644 --- a/docs/examples/README.md +++ b/docs/examples/README.md @@ -112,6 +112,9 @@ Implementing and composing requirements for output validation. - Custom requirement definitions - Requirement composition - Validation with requirements +- Python code generation and execution with `bash_executor` +- CLI data processing with shell commands +- Safe subprocess execution with `python_tool` ### Integration & Deployment diff --git a/docs/examples/requirements/README.md b/docs/examples/requirements/README.md index 109b9d5ca..a7739a485 100644 --- a/docs/examples/requirements/README.md +++ b/docs/examples/requirements/README.md @@ -6,6 +6,117 @@ and code generation tasks like plotting. ## Files +### code_generation_and_execution_bash.py + +Demonstrates Python code generation with `bash_executor` for CLI data processing tasks. +Uses `python_tool` for safe code execution and shows how to orchestrate multiple bash +commands via Python control flow. + +**Key Features:** + +- Generate Python code that orchestrates multiple bash_executor calls +- Avoid shell operators (pipes, redirects, semicolons) by using Python control flow +- Use `PythonSyntaxValid` requirement to validate generated code +- Execute via `python_tool` with configurable execution tiers +- Process bash command output in Python variables for aggregation/transformation +- Import allowlisting to restrict which modules can be imported + +**Prerequisites:** + +Before running this example, ensure you have: + +1. Ollama running with the granite4.1:3b model: + ```bash + ollama serve # in one terminal + ollama pull granite4.1:3b # in another terminal (if not already downloaded) + ``` + +2. Required dependencies installed: + ```bash + uv sync # installs all mellea dependencies + ``` + +**Usage Examples:** + +```bash +# Run with default sample workspace and predefined requests +uv run python docs/examples/requirements/code_generation_and_execution_bash.py + +# Use a custom workspace directory +uv run python docs/examples/requirements/code_generation_and_execution_bash.py \ + --workspace /path/to/workspace + +# Accept user input interactively +uv run python docs/examples/requirements/code_generation_and_execution_bash.py \ + --interactive +``` + +**Pipeline Steps:** + +1. Create sample workspace with logs, data, and config directories +2. Accept natural language request for file/CLI operations +3. Generate Python code with sequential `bash_executor()` calls +4. Validate syntax using `PythonSyntaxValid` requirement +5. Execute generated code using `python_tool` tier "local_unsafe" +6. Display results from command outputs + +**Example Requests:** + +```text +Find all ERROR entries in the log files and count them +List all log files and count the total number of lines across all of them +Show the contents of all data files in the data directory +Find all WARN entries in the log files and display the files containing them +Count how many log files contain ERROR entries +``` + +**Generated Code Pattern:** + +The LLM generates Python code that looks like: + +```python +from mellea.stdlib.tools.shell import bash_executor + +# Step 1: Find files +result = bash_executor("find logs -name '*.log'", working_dir=workspace_dir) +if not result.success: + print(f"Failed: {result.skip_message}") +else: + files = result.stdout.strip().split('\n') + + # Step 2: Process each file + for file in files: + if file: + result = bash_executor(f"grep -c ERROR {file}", working_dir=workspace_dir) + if result.success: + print(f"{file}: {result.stdout} errors") +``` + +**Key Design Principles:** + +1. **No Shell Operators** - Avoid pipes (`|`), redirects (`>`, `>>`), and semicolons (`;`) +2. **Python Control Flow** - Use loops, conditionals, and variables instead +3. **ExecutionResult API** - Check `result.success`, `result.stdout`, `result.skip_message` +4. **Safe Execution** - Use `python_tool` tier to run code in subprocess +5. **Output Parsing** - Extract and transform command output in Python + +**Output Examples:** + +Example 1 output: +``` +logs/app_1.log: 2 ERROR entries +logs/app_2.log: 2 ERROR entries +logs/app_3.log: 2 ERROR entries +Total ERROR entries found in all logs: 6 +``` + +Example 4 output: +``` +logs/app_1.log: contains 2 WARN entries +logs/app_2.log: contains 2 WARN entries +logs/app_3.log: contains 2 WARN entries +``` + ### code_generation_and_execution.py Demonstrates the complete pipeline of code generation, data extraction, and graph diff --git a/docs/examples/requirements/code_generation_and_execution_bash.py b/docs/examples/requirements/code_generation_and_execution_bash.py new file mode 100644 index 000000000..ad4d89d59 --- /dev/null +++ b/docs/examples/requirements/code_generation_and_execution_bash.py @@ -0,0 +1,429 @@ +# pytest: e2e, ollama, qualitative, slow +"""Example demonstrating Python code generation for CLI data processing tasks. + +This example shows how to use Mellea to: +1. Accept user input specifying file/directory management and CLI tasks +2. Generate Python code with bash_executor() calls for data processing workflows +3. Execute the generated Python code to orchestrate multiple bash_executor calls + +The pipeline implements a 4-step process: +1. User Input - Accept natural language request for file/CLI operations +2. Context Loading - Load file listings and sample data +3. Code Generation - Generate Python code with sequential bash_executor calls +4. Code Execution - Execute the generated Python code to process data + +Example tasks: +- Find and process files matching patterns +- Transform and reorganize directory structures +- Extract and aggregate data from multiple files +- Generate reports from log files or data files +- Search and filter data across multiple files + +This approach avoids shell operators (pipes, redirects) by generating Python control flow instead. +""" + +import argparse +import os +import sys +import tempfile +from pathlib import Path + +import mellea +from mellea.stdlib.context import ChatContext +from mellea.stdlib.requirements.python_tools import PythonSyntaxValid +from mellea.stdlib.sampling import ModelFriendlyRepairStrategy +from mellea.stdlib.tools.interpreter import python_tool +from mellea.stdlib.tools.shell import bash_executor + + +def _extract_python_code_from_output(generated: str) -> str | None: + """Extract Python code block from model output. + + Uses Mellea's internal _has_python_code_listing() utility to ensure consistent + code extraction behavior across the framework. This utility: + - Handles both markdown and RST code block delimiters + - Scores multiple code blocks and selects the best one + - Falls back to tool_calls if text blocks are not found + - Is the same utility used by PythonCodeExtraction and other validators + + We import this internal function (not public API) because examples should align + with framework patterns. Other validators like matplotlib_repair.py already use + this same utility. Local import keeps the dependency explicit and avoids + suggesting it's a public API. + + Args: + generated: Raw model output string. + + Returns: + Extracted Python code string, or None if extraction failed. + """ + # Import _has_python_code_listing locally to avoid importing private functions + # at module level. Local import makes the dependency on internals explicit. + from mellea.core import ModelOutputThunk + from mellea.stdlib.requirements.python_reqs import _has_python_code_listing + + # Wrap the text output in a ChatContext using ModelOutputThunk, which is what + # Mellea uses internally to represent model output for code extraction. + ctx = ChatContext().add(ModelOutputThunk(value=generated)) + + # Use Mellea's internal code extraction (same utility as PythonCodeExtraction) + extraction_result = _has_python_code_listing(ctx) + + if extraction_result.as_bool(): + # extraction_result.reason contains the extracted code string + code = extraction_result.reason + return code if code else None + + return None + + +def create_sample_workspace(workspace_dir: str) -> None: + """Create sample files for demonstration.""" + Path(workspace_dir).mkdir(exist_ok=True) + + # Create logs directory with sample log files + logs_dir = Path(workspace_dir) / "logs" + logs_dir.mkdir(exist_ok=True) + + log_content = """2024-01-15 10:23:45 INFO Starting application +2024-01-15 10:24:01 DEBUG Connecting to database +2024-01-15 10:24:05 ERROR Database connection failed +2024-01-15 10:24:06 WARN Retrying connection +2024-01-15 10:24:10 INFO Connected successfully +2024-01-15 10:25:30 DEBUG Processing user request +2024-01-15 10:25:45 INFO Request completed +2024-01-15 10:26:00 ERROR Out of memory exception +2024-01-15 10:26:01 WARN Application degraded""" + + for i in range(1, 4): + (logs_dir / f"app_{i}.log").write_text(log_content) + + # Create data directory with sample data files + data_dir = Path(workspace_dir) / "data" + data_dir.mkdir(exist_ok=True) + + data_files = { + "sales_2024_q1.txt": """Product,Revenue,Units +Widget A,15000,300 +Widget B,22000,400 +Gadget X,18000,250 +Gadget Y,25000,350""", + "sales_2024_q2.txt": """Product,Revenue,Units +Widget A,16500,320 +Widget B,24000,420 +Gadget X,19500,270 +Gadget Y,27500,380""", + "sales_2024_q3.txt": """Product,Revenue,Units +Widget A,17000,340 +Widget B,26000,450 +Gadget X,21000,290 +Gadget Y,29000,400""", + } + + for filename, content in data_files.items(): + (data_dir / filename).write_text(content) + + # Create config directory + config_dir = Path(workspace_dir) / "config" + config_dir.mkdir(exist_ok=True) + + (config_dir / "app.conf").write_text("APP_NAME=DataProcessor\nVERSION=1.0.0") + (config_dir / "db.conf").write_text("HOST=localhost\nPORT=5432\nDB=myapp") + + +def get_workspace_context(workspace_dir: str) -> str: + """Get file listing and structure of workspace.""" + context = f"Workspace directory: {workspace_dir}\n\n" + context += "Directory structure:\n" + + for root, dirs, files in os.walk(workspace_dir): + level = root.replace(workspace_dir, "").count(os.sep) + indent = " " * 2 * level + context += f"{indent}{os.path.basename(root)}/\n" + + subindent = " " * 2 * (level + 1) + for file in sorted(files): + file_path = Path(root) / file + size = file_path.stat().st_size + context += f"{subindent}{file} ({size} bytes)\n" + + return context + + +def process_user_request( + user_request: str, workspace_dir: str, request_number: int = 1 +) -> None: + """Process a user request to generate and execute Bash code. + + Args: + user_request: Natural language request for file/CLI operations + workspace_dir: Working directory with sample files + request_number: Number for naming purposes + """ + print(f"\nUser request: {user_request}") + print("-" * 70) + + m = mellea.start_session() + + workspace_context = get_workspace_context(workspace_dir) + sanitized_request = repr(user_request) + + prompt = f"""You are a Python code generator that uses bash_executor for CLI operations. + +User request: +{sanitized_request} + +Current workspace context: +{workspace_context} + +Generate Python code that: +1. Imports bash_executor from mellea.stdlib.tools.shell +2. Uses sequential bash_executor() calls to process files and data +3. Breaks complex tasks into multiple simple commands (NO pipes, redirects, or shell chaining) +4. Uses Python loops, conditionals, and string operations to orchestrate commands +5. Processes intermediate results in Python variables +6. Prints the final results + +IMPORTANT CONSTRAINTS: +- NO shell operators (pipes |, redirects >, >>, semicolons ;) +- Each bash_executor call executes ONE simple command +- Use Python control flow (for loops, if statements) to sequence commands +- Extract and process bash_executor output in Python +- Working directory is: {workspace_dir} + +IMPORTANT API DETAILS: +- bash_executor() returns an ExecutionResult object +- Check result.success (bool) to verify command executed successfully +- Check result.stdout (str) for command output (or None if failed) +- Check result.skipped (bool) to see if command was blocked by guardrails +- Check result.skip_message (str) for why a command was skipped + +IMPORTANT PARSING NOTES: +- wc -l FILE outputs: "N FILE" (number + filename), extract the first word only +- grep -c PATTERN FILE outputs just the count (easier to parse) +- grep PATTERN FILE outputs matching lines (use for displaying content) +- cat FILE outputs entire file contents (use for showing all data) +- Always strip() output and check if parsing will work +- When in doubt, use commands that output just data without filenames + +DIRECTORY STRUCTURE: +The workspace contains: +- logs/ — log files (*.log) +- data/ — data files (*.txt, *.csv) +- config/ — configuration files + +COMMON FILE OPERATIONS: +- Show file contents: cat data/sales_2024_q1.txt +- Find files: find data -name '*.txt' or find data -name '*.csv' +- Search in files: grep PATTERN filename (finds lines containing PATTERN anywhere) +- Count matching lines: grep -c PATTERN filename (outputs just the number) +- Count all lines: wc -l filename (output: "N filename", extract first word) + +GREP PATTERN TIPS: +- To find lines containing a word: grep -c ERROR logs/app.log (matches ERROR anywhere in line) +- Do NOT use anchors like ^ERROR or ^WARN unless you know the line starts with that text +- For log files with timestamps: use grep -c ERROR (without ^) to match anywhere in the line +- Example log format: "2024-01-15 10:24:05 ERROR Database failed" → grep ERROR will match +- grep -c outputs just the count, perfect for parsing with int() + +BANNED OPERATIONS (will be blocked): +- Input/output redirection: >, >>, <, 2>&1, >&2 +- Pipes: |, |& +- Command chaining: ;, &&, || +- Variable expansion: $(...), `...`, ${{...}} +- Code execution: python -c, bash -c, etc. + +Example pattern for showing file contents: +```python +from mellea.stdlib.tools.shell import bash_executor + +# Step 1: Find all data files in data directory +result = bash_executor("find data -name '*.txt'", working_dir="{workspace_dir}") +if not result.success: + print(f"Failed: {{result.skip_message}}") +else: + files = result.stdout.strip().split('\\n') + + # Step 2: Display each file's contents + for file in files: + if file: + result = bash_executor(f"cat {{file}}", working_dir="{workspace_dir}") + if result.success: + print(f"\\nContents of {{file}}:") + print(result.stdout) +``` + +Example pattern for counting with grep (CORRECT - no anchors): +```python +# Count ERROR entries - use grep -c ERROR (NOT ^ERROR) +result = bash_executor(f"grep -c ERROR {{file}}", working_dir="{workspace_dir}") +if result.success: + count = int(result.stdout.strip()) # grep -c outputs just the number + print(f"{{file}}: {{count}} errors") + +# WRONG: Do NOT use this - anchors won't match timestamped logs: +# result = bash_executor(f"grep -c ^ERROR {{file}}", ...) # WRONG! +``` + +ALWAYS use working_dir="{workspace_dir}" (the main workspace), NOT subdirectories like "logs/" or "data/". + +Generate the Python code now:""" + + print("Generating Python code for data processing...") + requirements: list = [PythonSyntaxValid()] + strategy = ModelFriendlyRepairStrategy(loop_budget=3, requirements=requirements) + generated = m.instruct(prompt, strategy=strategy) + + if generated is None: + print(" ✗ Model failed to generate output (requirements loop exhausted)") + return + + generated_str = str(generated) + if not generated_str.strip(): + print(" ✗ Model failed to generate output") + return + + code = _extract_python_code_from_output(generated_str) + if code is None: + print(" ✗ Failed to extract Python code from model output") + print(f"\nModel output:\n{generated_str}") + return + + print("\nGenerated Python code:") + print(code) + + # Execute the generated Python code using python_tool + print("\nExecuting generated code...") + + # Create a python execution tool that allows bash_executor imports + tool = python_tool( + tier="local_unsafe", allowed_imports=["mellea"], name="bash_executor_runner" + ) + + # Prepend the bash_executor import if not already present + if "from mellea.stdlib.tools.shell import bash_executor" not in code: + code = "from mellea.stdlib.tools.shell import bash_executor\n" + code + + # Execute the code + result = tool.run(code=code) + + if result.skipped: + print(f" ✗ Code execution was skipped: {result.skip_message}") + return + + if not result.success: + print(" ✗ Code execution failed") + if result.stderr: + print(f" Error: {result.stderr}") + if result.exit_code is not None: + print(f" Exit code: {result.exit_code}") + return + + # Print any output from the code + if result.stdout: + print(result.stdout) + + print("\n ✓ Code executed successfully") + + +def main(): + """Demonstrate complete pipeline for CLI data processing tasks.""" + parser = argparse.ArgumentParser( + description="Bash Code Generation: CLI Data Processing and File Operations", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=""" +Examples: + # Run with default sample workspace and predefined requests + uv run python code_generation_and_execution_bash.py + + # Use a custom workspace directory + uv run python code_generation_and_execution_bash.py --workspace /path/to/workspace + + # Accept user input interactively + uv run python code_generation_and_execution_bash.py --interactive + """, + ) + parser.add_argument( + "--workspace", + type=str, + default=None, + help="Workspace directory (uses temp directory if not specified)", + ) + parser.add_argument( + "--interactive", + action="store_true", + help="Accept user input interactively instead of using predefined requests", + ) + + args = parser.parse_args() + + print("=" * 70) + print("Python Code Generation: CLI Data Processing and File Operations") + print("=" * 70) + + # Use provided workspace or create temporary one + if args.workspace: + workspace_dir = args.workspace + Path(workspace_dir).mkdir(parents=True, exist_ok=True) + else: + workspace_dir = tempfile.mkdtemp(prefix="mellea_bash_") + + print(f"\nWorkspace directory: {workspace_dir}") + + # Create sample files + create_sample_workspace(workspace_dir) + print("Created sample workspace with files and directories") + + if args.interactive: + print("\n" + "=" * 70) + print("Interactive Mode") + print("=" * 70) + print("\nExamples of requests:") + print(" - Find all ERROR entries in log files and count them") + print(" - Count the total number of lines in all log files") + print(" - Find all WARN entries and show which files contain them") + print(" - Count how many log files contain ERROR entries") + + request_number = 1 + while True: + print("\nEnter a request (or 'quit' to exit):") + user_input = input("\n> ").strip() + + if user_input.lower() in ["quit", "exit", "q"]: + break + + if not user_input: + print("Please enter a valid request.") + continue + + process_user_request( + user_input, workspace_dir, request_number=request_number + ) + request_number += 1 + + else: + # Use predefined requests that benefit from Python orchestration + user_requests = [ + "Find all ERROR entries in the log files and count them", + "List all log files and count the total number of lines across all of them", + "Show the contents of all data files in the data directory", + "Find all WARN entries in the log files and display the files containing them", + "Count how many log files contain ERROR entries", + ] + + # Process each user request + for i, request in enumerate(user_requests, 1): + print("\n" + "=" * 70) + print(f"Example {i}") + print("=" * 70) + + process_user_request(request, workspace_dir, request_number=i) + + print("\n" + "=" * 70) + print("Pipeline completed!") + print(f"Workspace preserved at: {workspace_dir}") + print("=" * 70) + + +if __name__ == "__main__": + main() diff --git a/mellea/stdlib/tools/_bash_patterns.py b/mellea/stdlib/tools/_bash_patterns.py index f092ff06b..09583cdbc 100644 --- a/mellea/stdlib/tools/_bash_patterns.py +++ b/mellea/stdlib/tools/_bash_patterns.py @@ -97,7 +97,11 @@ def check(self, argv: list[str]) -> tuple[bool, str]: for arg in argv: # Exact match: standalone operators like "&&", "|" + # Exception: standalone ";" is safe (used by find -exec, etc.) if arg in shell_operators: + if arg == ";": + # Standalone semicolon is safe when passed via argv (not shell interpretation) + continue rule = SHELL_OPERATOR_RULES.get(arg) reason = rule.rationale if rule else "Shell operator is not allowed" return True, reason @@ -105,12 +109,17 @@ def check(self, argv: list[str]) -> tuple[bool, str]: # Prefix match: operators with content like ">&2", ">file" for op in shell_operators: if arg.startswith(op) and len(arg) > len(op): + # Skip if this is the semicolon substring check (handled below) + if op == ";": + continue rule = SHELL_OPERATOR_RULES.get(op) reason = rule.rationale if rule else "Shell operator is not allowed" return True, reason - # Semicolon: substring check (dangerous even in some quote contexts) - if ";" in arg: + # Semicolon: reject any arg containing semicolon (except standalone `;` from find -exec) + # After shlex.split(), escaped semicolons become bare `;` tokens, making them safe + # because they're passed to subprocess as argv elements, not shell strings. + if ";" in arg and arg != ";": return True, "Command chaining (;) is not allowed" return False, "" diff --git a/mellea/stdlib/tools/shell.py b/mellea/stdlib/tools/shell.py index 578840d67..75027308e 100644 --- a/mellea/stdlib/tools/shell.py +++ b/mellea/stdlib/tools/shell.py @@ -162,20 +162,27 @@ def _is_dangerous_command(argv: list[str]) -> tuple[bool, str]: # for legitimate patterns like regex or AWK/sed code. # Exact match first (standalone operators like "&&", "|", etc.) + # Exception: standalone ";" is safe (used by find -exec, etc.) if arg in shell_operators: + if arg == ";": + # Standalone semicolon is safe when passed via argv (not shell interpretation) + continue return True, f"Shell operator '{arg}' is not allowed" # Check if argument starts with a shell operator (e.g., ">&2", ">file", "2>&1") for op in shell_operators: if arg.startswith(op) and len(arg) > len(op): + # Skip semicolon here (handled below) + if op == ";": + continue # Token starts with operator and has additional content (e.g., ">&2", ">file") # This is a shell redirection/operator usage return True, f"Shell operator '{op}' is not allowed" - # Note: semicolon is already in shell_operators, but we check for it separately - # as a substring because semicolon can be dangerous even in patterns. - # Unlike && or ||, semicolon rarely appears legitimately in arguments. - if ";" in arg: + # Semicolon: reject any arg containing semicolon (except standalone `;` from find -exec) + # After shlex.split(), escaped semicolons become bare `;` tokens, making them safe + # because they're passed to subprocess as argv elements, not shell strings. + if ";" in arg and arg != ";": return True, "Command chaining (;) is not allowed" # Check for command substitution (backticks, $(...)) diff --git a/test/stdlib/tools/test_bash_guardrails.py b/test/stdlib/tools/test_bash_guardrails.py index 9a849b53a..7db61aaec 100644 --- a/test/stdlib/tools/test_bash_guardrails.py +++ b/test/stdlib/tools/test_bash_guardrails.py @@ -175,10 +175,20 @@ def test_and_operator_rejected(self) -> None: is_dangerous, _ = pattern.check(["cmd1", "&&", "cmd2"]) assert is_dangerous is True - def test_semicolon_rejected(self) -> None: - """Semicolon chaining should be rejected.""" + def test_semicolon_standalone_allowed(self) -> None: + """Standalone semicolon should be allowed (used by find -exec).""" pattern = ShellOperatorPattern() - is_dangerous, _ = pattern.check(["cmd1", ";", "cmd2"]) + # After shlex.split("find -exec cat {} \;"), we get [..., ";"] + # This is safe because subprocess.run(..., shell=False) doesn't interpret it + is_dangerous, _ = pattern.check(["find", "-exec", "cat", "{}", ";"]) + assert is_dangerous is False + + def test_semicolon_embedded_rejected(self) -> None: + """Embedded semicolon in argument should be rejected.""" + pattern = ShellOperatorPattern() + # After shlex.split("echo 'hello;rm'"), we get ["echo", "hello;rm"] + # Embedded semicolon could indicate LLM trying to hide a command + is_dangerous, _ = pattern.check(["echo", "hello;rm"]) assert is_dangerous is True def test_quoted_pipe_in_string_allowed(self) -> None: