Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions AGENTS.md
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,8 @@ Every compiled pipeline runs as three sequential jobs:
│ │ ├── common.rs # Shared helpers across targets
│ │ ├── standalone.rs # Standalone pipeline compiler
│ │ ├── onees.rs # 1ES Pipeline Template compiler
│ │ ├── job.rs # Job-level ADO template compiler (target: job)
│ │ ├── stage.rs # Stage-level ADO template compiler (target: stage)
│ │ ├── gitattributes.rs # .gitattributes management for compiled pipelines
│ │ ├── filter_ir.rs # Filter expression IR: Fact/Predicate types, lowering, validation, codegen
│ │ ├── pr_filters.rs # PR trigger filter generation (native ADO + gate steps)
Expand Down Expand Up @@ -122,6 +124,8 @@ Every compiled pipeline runs as three sequential jobs:
│ ├── data/
│ │ ├── base.yml # Base pipeline template for standalone
│ │ ├── 1es-base.yml # Base pipeline template for 1ES target
│ │ ├── job-base.yml # Job-level ADO template for target: job
│ │ ├── stage-base.yml # Stage-level ADO template for target: stage
│ │ ├── ecosystem_domains.json # Network allowlists per ecosystem
│ │ ├── init-agent.md # Dispatcher agent template for `init` command
│ │ └── threat-analysis.md # Threat detection analysis prompt template
Expand Down
1 change: 1 addition & 0 deletions docs/cli.md
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ Global flags (apply to all subcommands): `--verbose, -v` (enable info-level logg
- `--output, -o <path>` - Optional output path for the generated YAML (only valid when a path is provided). If the path is an existing directory, the compiled YAML is written inside that directory using the default filename derived from the markdown source (e.g. `foo.md` → `<dir>/foo.lock.yml`).
- `--skip-integrity` - *(debug builds only)* Omit the "Verify pipeline integrity" step from the generated pipeline. Useful during local development when the compiled output won't match a released compiler version. This flag is not available in release builds.
- `--debug-pipeline` - *(debug builds only)* Include MCPG debug diagnostics in the generated pipeline: `DEBUG=*` environment variable for verbose MCPG logging, stderr streaming to log files, and a "Verify MCP backends" step that probes each backend with MCP initialize + tools/list before the agent runs. This flag is not available in release builds.
- For `target: job` and `target: stage`, the output is an ADO YAML template (not a complete pipeline). Job names are prefixed with the agent name for uniqueness. Triggers configured via `on:` are ignored with a warning.
- `check <pipeline>` - Verify that a compiled pipeline matches its source markdown
- `<pipeline>` - Path to the pipeline YAML file to verify
- The source markdown path is auto-detected from the `@ado-aw` header in the pipeline file
Expand Down
2 changes: 1 addition & 1 deletion docs/front-matter.md
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ The compiler expects markdown files with YAML front matter similar to gh-aw:
---
name: "name for this agent"
description: "One line description for this agent"
target: standalone # Optional: "standalone" (default) or "1es". See docs/targets.md.
target: standalone # Optional: "standalone" (default), "1es", "job", or "stage". See docs/targets.md.
engine: copilot # Engine identifier. Defaults to copilot. Currently only 'copilot' (GitHub Copilot CLI) is supported.
# engine: # Alternative object format (with additional options)
# id: copilot
Expand Down
76 changes: 76 additions & 0 deletions docs/targets.md
Original file line number Diff line number Diff line change
Expand Up @@ -31,3 +31,79 @@ target: 1es
```

When using `target: 1es`, the pipeline will extend `1es/1ES.Unofficial.PipelineTemplate.yml@1ESPipelinesTemplates`.

### `job`

Generates a **job-level ADO YAML template** with `jobs:` at root. This is a
reusable template that can be included in an existing pipeline — it does not
generate a complete pipeline.

The output contains the same 3-job chain (Agent → Detection → Execution) as
`standalone`, with:
- Job names prefixed with the agent name for uniqueness (e.g., `DailyReview_Agent`)
- No triggers, pipeline name, or resource declarations (the parent pipeline owns those)
- Pool baked in from the front matter `pool:` field

Example front matter:
```yaml
target: job
```

#### Usage in a flat pipeline

```yaml
jobs:
- job: Build
steps: ...
- template: agents/review.lock.yml
```

#### Usage inside a user-defined stage

```yaml
stages:
- stage: Build
jobs: ...
- stage: AgenticReview
dependsOn: Build
jobs:
- template: agents/review.lock.yml
```

#### Notes

- Triggers (`on:`) are ignored with a warning (the parent pipeline controls triggers)
- If the agent declares additional repositories via `repos:`, add them to the
parent pipeline's `resources:` block (documented in the generated file header)

### `stage`

Generates a **stage-level ADO YAML template** with `stages:` at root. This
wraps the 3-job chain inside a stage block for direct inclusion in multi-stage
pipelines.

Example front matter:
```yaml
target: stage
```

#### Usage

```yaml
stages:
- stage: Build
jobs: ...
- template: agents/review.lock.yml
dependsOn: Build
condition: succeeded()
```

ADO natively supports `dependsOn` and `condition` at the template call site —
no template parameters are needed for stage ordering.

#### Notes

- Same 3-job chain, job-name prefixing, and pool handling as `target: job`
- Triggers (`on:`) are ignored with a warning
- If the agent declares additional repositories via `repos:`, add them to the
parent pipeline's `resources:` block
53 changes: 53 additions & 0 deletions docs/template-markers.md
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,8 @@ The compiler transforms the input into valid Azure DevOps pipeline YAML based on

- **Standalone**: Uses `src/data/base.yml`
- **1ES**: Uses `src/data/1es-base.yml`
- **Job template**: Uses `src/data/job-base.yml`
- **Stage template**: Uses `src/data/stage-base.yml`

Explicit markings are embedded in these templates that the compiler is allowed to replace e.g. `{{ engine_run }}` denotes the full engine invocation command. The compiler should not replace sections denoted by `${{ some content }}`. What follows is a mapping of markings to responsibilities (primarily for the standalone template).

Expand Down Expand Up @@ -495,3 +497,54 @@ Should be replaced with the domain the AWF-sandboxed agent uses to reach MCPG on
The 1ES target uses the same template markers as standalone, plus the 1ES-specific `extends:` / `stages:` / `templateContext` wrapping. The 1ES template includes `templateContext.type: buildJob` for all jobs, and the pool is specified at the top-level `parameters.pool` rather than per-job.

Both targets share the same execution model (Copilot CLI + AWF + MCPG) and the same set of template markers.

## Job/Stage Template Markers

The `target: job` and `target: stage` targets use `job-base.yml` and `stage-base.yml`
respectively. Both include all the standard AWF/MCPG markers above, plus the two
template-specific markers below.

### {{ stage_prefix }}

Replaced with a PascalCase ADO-safe identifier derived from the agent `name:` front
matter field. Used to prefix the three job names so that including multiple templates
in the same pipeline produces unique job identifiers.

Derivation rules:

- Non-ASCII-alphanumeric characters are treated as word separators (they are not
included in the output).
- Each word is capitalised and the words are concatenated: `"daily code review"` →
`"DailyCodeReview"`.
- An empty result (all characters stripped) falls back to `"Agent"`.
- A result starting with a digit is prefixed with `_`: `"123start"` → `"_123start"`.
- Names containing non-ASCII alphanumeric characters (e.g. `"über-agent"`) produce a
compiler warning because those characters are silently dropped.

Example job names produced for `name: Daily Code Review`:

```yaml
jobs:
- job: DailyCodeReview_Agent
- job: DailyCodeReview_Detection
dependsOn: DailyCodeReview_Agent
- job: DailyCodeReview_Execution
dependsOn: [DailyCodeReview_Agent, DailyCodeReview_Detection]
```

### {{ template_parameters }}

Replaced with the `parameters:` block that callers pass when including the template.
Contains `clearMemory` (auto-injected when `tools.cache-memory` is configured) and any
user-defined `parameters:` from front matter. Replaced with an empty string when no
parameters are needed.

Example output when `tools.cache-memory` is configured:

```yaml
parameters:
- name: clearMemory
displayName: Clear agent memory
type: boolean
default: false
```
173 changes: 170 additions & 3 deletions src/compile/common.rs
Original file line number Diff line number Diff line change
Expand Up @@ -999,6 +999,73 @@ pub fn sanitize_filename(name: &str) -> String {
/// Default pool name
pub const DEFAULT_POOL: &str = "AZS-1ES-L-MMS-ubuntu-22.04";

/// Derive a valid ADO identifier from the agent name for use as a job-name
/// prefix and stage name. Converts to PascalCase, stripping non-alphanumeric
/// characters.
///
/// Examples:
/// - `"Daily Code Review"` → `"DailyCodeReview"`
/// - `"my-agent-123"` → `"MyAgent123"`
/// - `""` → `"Agent"` (fallback)
/// - `"123start"` → `"_123start"` (prefix underscore for leading digit)
/// - `"über-agent"` → `"BerAgent"` (non-ASCII stripped; ADO requires `[A-Za-z0-9_]`)
pub fn generate_stage_prefix(name: &str) -> String {
// Warn if any Unicode alphanumeric characters are present — they will be
// treated as word-separator boundaries and stripped from the output, which
// may surprise users whose agent name starts with a non-ASCII letter.
if name.chars().any(|c| c.is_alphanumeric() && !c.is_ascii_alphanumeric()) {
log::warn!(
"Agent name '{}' contains non-ASCII alphanumeric characters; \
these are dropped from the ADO job-name prefix because ADO identifiers \
require [A-Za-z0-9_]. Rename the agent to use ASCII characters only \
if the prefix is important.",
name
);
}

let pascal: String = name
.split(|c: char| !c.is_ascii_alphanumeric())
.filter(|s| !s.is_empty())
.map(|word| {
let mut chars = word.chars();
match chars.next() {
None => String::new(),
Some(first) => {
let upper = first.to_uppercase().to_string();
upper + chars.as_str()
}
}
})
.collect();

if pascal.is_empty() {
"Agent".to_string()
} else if pascal.starts_with(|c: char| c.is_ascii_digit()) {
format!("_{}", pascal)
} else {
pascal
}
}

/// Generate the template-level `parameters:` YAML block for job/stage
/// template targets.
///
/// Includes clearMemory (if cache-memory enabled) and user-defined
/// parameters from front matter. Returns empty string if no parameters
/// are needed.
pub fn generate_template_parameters(front_matter: &FrontMatter) -> Result<String> {
let has_memory = front_matter
.tools
.as_ref()
.and_then(|t| t.cache_memory.as_ref())
.is_some_and(|cm| cm.is_enabled());
let params = build_parameters(&front_matter.parameters, has_memory);
if params.is_empty() {
return Ok(String::new());
}
generate_parameters(&params)
}

/// Version of the AWF (Agentic Workflow Firewall) binary to download from GitHub Releases.
/// Update this when upgrading to a new AWF release.
/// See: https://github.com/github/gh-aw-firewall/releases
Expand Down Expand Up @@ -2423,6 +2490,23 @@ pub struct CompileConfig {
/// to append `GITHUB_PATH: $(GITHUB_PATH)` to the engine env block without
/// re-collecting path prepends from extensions.
pub has_awf_paths: bool,
/// When true, `compile_shared` omits the standard `# @ado-aw` header.
/// Template-producing compilers (Job, Stage) set this to prepend their
/// own custom header with usage instructions.
pub skip_header: bool,
}

/// Input configuration for [`compile_template_target`].
///
/// Groups the template-specific settings so that the function stays within
/// the seven-argument limit while remaining easy to extend.
pub struct TemplateTargetConfig<'a> {
/// Raw YAML template string (e.g. `job-base.yml` or `stage-base.yml`).
pub template: &'a str,
/// When true, the "Verify pipeline integrity" step is omitted.
pub skip_integrity: bool,
/// When true, MCPG debug diagnostics are included in the generated pipeline.
pub debug_pipeline: bool,
}

/// Shared compilation flow used by both standalone and 1ES compilers.
Expand Down Expand Up @@ -2704,9 +2788,92 @@ pub async fn compile_shared(
replace_with_indent(&yaml, placeholder, replacement)
});

// 15. Prepend header
let header = generate_header_comment(input_path);
Ok(format!("{}{}", header, pipeline_yaml))
// 15. Prepend header (unless the caller will prepend its own)
if config.skip_header {
Ok(pipeline_yaml)
} else {
let header = generate_header_comment(input_path);
Ok(format!("{}{}", header, pipeline_yaml))
}
}

/// Shared compilation flow for template-producing compilers (`target: job` and
/// `target: stage`).
///
/// Handles the full setup — collecting extensions, building the compile context,
/// generating the stage prefix and template parameters, computing AWF/MCPG
/// values — and delegates to [`compile_shared`]. The caller supplies:
///
/// - `cfg`: target-specific settings (template string, integrity / debug flags).
/// - `header_fn`: a function that generates the leading comment block prepended
/// to the compiled YAML. The two template compilers use different header
/// layouts, so this lets each compiler keep its own generator while sharing
/// all of the boilerplate setup.
///
/// Returns the final YAML string with the header prepended.
pub async fn compile_template_target(
input_path: &Path,
output_path: &Path,
front_matter: &FrontMatter,
markdown_body: &str,
cfg: TemplateTargetConfig<'_>,
header_fn: impl FnOnce(&Path, &FrontMatter) -> String,
) -> Result<String> {
// Collect extensions (needed before compile_shared for MCPG config)
let extensions = super::extensions::collect_extensions(front_matter);

// Build compile context for MCPG config generation
let input_dir = input_path.parent().unwrap_or(Path::new("."));
let ctx = CompileContext::new(front_matter, input_dir).await?;

// Generate stage prefix for job-name uniqueness and template parameters
let stage_prefix = generate_stage_prefix(&front_matter.name);
let template_params = generate_template_parameters(front_matter)?;

// AWF / MCPG values (same as standalone)
let allowed_domains = generate_allowed_domains(front_matter, &extensions)?;
let awf_mounts = generate_awf_mounts(&extensions);
let awf_paths = collect_awf_path_prepends(&extensions);
let awf_path_step = generate_awf_path_step(&awf_paths);
let enabled_tools_args = generate_enabled_tools_args(front_matter);

let config_obj = generate_mcpg_config(front_matter, &ctx, &extensions)?;
let mcpg_config_json =
serde_json::to_string_pretty(&config_obj).context("Failed to serialize MCPG config")?;
let mcpg_docker_env = generate_mcpg_docker_env(front_matter, &extensions);
let mcpg_step_env = generate_mcpg_step_env(&extensions);

let config = CompileConfig {
template: cfg.template.to_string(),
extra_replacements: vec![
("{{ stage_prefix }}".into(), stage_prefix),
("{{ template_parameters }}".into(), template_params),
("{{ firewall_version }}".into(), AWF_VERSION.into()),
("{{ mcpg_version }}".into(), MCPG_VERSION.into()),
("{{ mcpg_image }}".into(), MCPG_IMAGE.into()),
("{{ mcpg_port }}".into(), MCPG_PORT.to_string()),
("{{ mcpg_domain }}".into(), MCPG_DOMAIN.into()),
("{{ allowed_domains }}".into(), allowed_domains),
("{{ awf_mounts }}".into(), awf_mounts),
("{{ awf_path_step }}".into(), awf_path_step),
("{{ enabled_tools_args }}".into(), enabled_tools_args),
("{{ mcpg_config }}".into(), mcpg_config_json),
("{{ mcpg_docker_env }}".into(), mcpg_docker_env),
("{{ mcpg_step_env }}".into(), mcpg_step_env),
],
skip_integrity: cfg.skip_integrity,
debug_pipeline: cfg.debug_pipeline,
has_awf_paths: !awf_paths.is_empty(),
skip_header: true,
};

let yaml = compile_shared(
input_path, output_path, front_matter, markdown_body,
&extensions, &ctx, config,
).await?;

let header = header_fn(input_path, front_matter);
Ok(format!("{}{}", header, yaml))
}

#[cfg(test)]
Expand Down
Loading
Loading