From c3555afd1d51b5df57e244430b4970ac75467c96 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 5 Aug 2026 20:49:52 +0000 Subject: [PATCH 1/5] Initial plan From 92c41bf4f040e9b85cb7ea4739301ebdca86850e Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 5 Aug 2026 21:05:58 +0000 Subject: [PATCH 2/5] test(audit): cover artifact layouts across public entry points Co-authored-by: jamesadevine <4742697+jamesadevine@users.noreply.github.com> --- .../src/compiler-smoke-e2e/index.ts | 17 +- .../src/compiler-smoke-e2e/signals.ts | 78 +++ src/ado/mod.rs | 33 +- src/audit/analyzers/custom_jobs.rs | 5 +- src/audit/cli.rs | 84 ++-- tests/audit_it.rs | 450 +++++++++++++++++- 6 files changed, 621 insertions(+), 46 deletions(-) diff --git a/scripts/ado-script/src/compiler-smoke-e2e/index.ts b/scripts/ado-script/src/compiler-smoke-e2e/index.ts index bc854c55..903c59cb 100644 --- a/scripts/ado-script/src/compiler-smoke-e2e/index.ts +++ b/scripts/ado-script/src/compiler-smoke-e2e/index.ts @@ -58,7 +58,7 @@ import { import { prepareCaseSource } from "./source.js"; import { renderResultsTable } from "./report.js"; import { runFixtures, type FixtureBuildRequest, type FixtureBuildResult } from "./runner.js"; -import { verifyCaseSignals } from "./signals.js"; +import { verifyCandidateAudit, verifyCaseSignals } from "./signals.js"; import { scanStaleRefs } from "./stale.js"; function log(msg: string): void { @@ -379,8 +379,19 @@ export async function main(): Promise { log, }); const signalOutcome = await verifyCaseSignals(rest, resolved.cases, outcome.results); - results = signalOutcome.results; - overallOk = outcome.ok && signalOutcome.ok; + const auditOutcome = + config.compilerSource === "candidate" + ? await verifyCandidateAudit(signalOutcome.results, { + adoAwBin: config.adoAwBin, + cwd: config.sourcesDirectory, + orgUrl: config.orgUrl, + project: config.project, + token: config.token, + timeoutMs: config.childTimeoutMs, + }) + : signalOutcome; + results = auditOutcome.results; + overallOk = outcome.ok && signalOutcome.ok && auditOutcome.ok; allTerminal = outcome.allTerminal; if (!overallOk) failureMessage = "one or more smoke cases did not succeed"; if (!allTerminal) { diff --git a/scripts/ado-script/src/compiler-smoke-e2e/signals.ts b/scripts/ado-script/src/compiler-smoke-e2e/signals.ts index 69216876..b02bbcb6 100644 --- a/scripts/ado-script/src/compiler-smoke-e2e/signals.ts +++ b/scripts/ado-script/src/compiler-smoke-e2e/signals.ts @@ -11,6 +11,10 @@ */ import { expandBuildTag, type ResolvedCase } from "./cases.js"; import type { FixtureBuildResult } from "./runner.js"; +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { redact, safeSpawn } from "./process.js"; export interface BuildTagClient { getBuildTags( @@ -72,3 +76,77 @@ export async function verifyCaseSignals( results: verified, }; } + +/** Audit one completed candidate child through the released CLI contract. */ +export async function verifyCandidateAudit( + results: readonly FixtureBuildResult[], + options: { + adoAwBin: string; + cwd: string; + orgUrl: string; + project: string; + token: string; + timeoutMs: number; + }, +): Promise { + const target = results.find( + (result) => result.caseId === "canary" && result.status === "succeeded" && result.buildId !== undefined, + ); + if (!target?.buildId) return { ok: false, results: results.map((result) => ({ ...result })) }; + + const outputDir = await mkdtemp(join(tmpdir(), "ado-aw-smoke-audit-")); + try { + const outcome = await safeSpawn({ + cmd: options.adoAwBin, + args: [ + "audit", + String(target.buildId), + "--json", + "--no-cache", + "--output", + outputDir, + "--org", + options.orgUrl, + "--project", + options.project, + ], + cwd: options.cwd, + env: { AZURE_DEVOPS_EXT_PAT: options.token }, + timeoutMs: options.timeoutMs, + }); + let error: string | undefined; + if (outcome.timedOut || outcome.status !== 0) { + error = `exit=${outcome.status ?? "signal"} timedOut=${outcome.timedOut}; stderr=${redact(outcome.stderr, [options.token])}`; + } else { + try { + const audit = JSON.parse(outcome.stdout) as { + overview?: { build_id?: number }; + downloaded_files?: unknown[]; + }; + if (audit.overview?.build_id !== target.buildId || !audit.downloaded_files?.length) { + error = "JSON report did not contain the child build id and published artifact files"; + } + } catch (parseError) { + error = `invalid JSON report: ${parseError instanceof Error ? parseError.message : String(parseError)}`; + } + } + if (!error) return { ok: true, results: results.map((result) => ({ ...result })) }; + + return { + ok: false, + results: results.map((result) => + result.caseId === target.caseId + ? { + ...result, + status: "failed", + message: + `candidate audit contract failed for build #${target.buildId} (${target.url ?? "URL unavailable"}); ` + + `expected artifacts agent_outputs_${target.buildId}, analyzed_outputs_${target.buildId}, and safe_outputs: ${error}`, + } + : { ...result }, + ), + }; + } finally { + await rm(outputDir, { recursive: true, force: true }); + } +} diff --git a/src/ado/mod.rs b/src/ado/mod.rs index 6d1ee607..9aaea683 100644 --- a/src/ado/mod.rs +++ b/src/ado/mod.rs @@ -2116,6 +2116,30 @@ pub async fn download_build_artifact( ) })?; + let repeated_root = (0..archive.len()).try_fold(true, |all_match, index| { + let entry = archive.by_index(index).with_context(|| { + format!( + "Failed to read zip entry {} from build artifact '{}'", + index, artifact.name + ) + })?; + let entry_name = entry.name().to_string(); + let relative_path = entry.enclosed_name().ok_or_else(|| { + anyhow::anyhow!( + "Refusing to extract unsafe path '{}' from build artifact '{}'", + entry_name, + artifact.name + ) + })?; + Ok::<_, anyhow::Error>( + all_match + && relative_path + .components() + .next() + .is_some_and(|component| component.as_os_str() == artifact.name.as_str()), + ) + })?; + for index in 0..archive.len() { let mut entry = archive.by_index(index).with_context(|| { format!( @@ -2134,7 +2158,14 @@ pub async fn download_build_artifact( artifact.name ) })?; - let output_path = artifact_dir.join(&relative_path); + let relative_path = if repeated_root { + relative_path + .strip_prefix(&artifact.name) + .expect("repeated artifact root was validated") + } else { + relative_path.as_path() + }; + let output_path = artifact_dir.join(relative_path); if entry.is_dir() { std::fs::create_dir_all(&output_path).with_context(|| { diff --git a/src/audit/analyzers/custom_jobs.rs b/src/audit/analyzers/custom_jobs.rs index 1edcb3ec..ad8e60ad 100644 --- a/src/audit/analyzers/custom_jobs.rs +++ b/src/audit/analyzers/custom_jobs.rs @@ -64,7 +64,6 @@ pub(crate) async fn load_custom_tool_catalog( supplied_aw_info: Option<&AwInfo>, ) -> anyhow::Result { let mut catalog = CustomToolCatalog::default(); - let mut primary_catalog_loaded = false; if let Some(path) = find_metadata_file(download_root, CUSTOM_TOOLS_FILENAME).await? { let contents = tokio::fs::read_to_string(&path) @@ -87,7 +86,6 @@ pub(crate) async fn load_custom_tool_catalog( ) })? }; - primary_catalog_loaded = true; for config in resolved.custom_tools { let tool = config.name.trim(); if tool.is_empty() { @@ -117,14 +115,13 @@ pub(crate) async fn load_custom_tool_catalog( let disk_aw_info = if supplied_aw_info.is_none() { match load_aw_info(download_root).await { Ok(value) => value, - Err(error) if primary_catalog_loaded => { + Err(error) => { warn!("Failed to read optional aw_info.json metadata: {error:#}"); catalog .warnings .push(crate::audit::malformed_aw_info_warning()); None } - Err(error) => return Err(error), } } else { None diff --git a/src/audit/cli.rs b/src/audit/cli.rs index 0733b3b1..695fddfc 100644 --- a/src/audit/cli.rs +++ b/src/audit/cli.rs @@ -406,45 +406,51 @@ async fn run_analyzers( |a, files| a.downloaded_files = files, ); - if let Some(agent_outputs_dir) = find_artifact_dir(run_dir, "agent_outputs").await { - run_agent_output_analyzers(&agent_outputs_dir, audit).await; + if artifact_family_selected(artifact_filters, "agent") { + if let Some(agent_outputs_dir) = find_artifact_dir(run_dir, "agent_outputs").await { + run_agent_output_analyzers(&agent_outputs_dir, audit).await; + } + run_analyzer( + audit, + "audit::missing_tools", + "missing-tool extraction failed", + missing::extract_missing_tools(run_dir).await, + |a, result| a.missing_tools = result, + ); + run_analyzer( + audit, + "audit::missing_data", + "missing-data extraction failed", + missing::extract_missing_data(run_dir).await, + |a, result| a.missing_data = result, + ); + run_analyzer( + audit, + "audit::noops", + "noop extraction failed", + missing::extract_noops(run_dir).await, + |a, result| a.noops = result, + ); } - run_analyzer( - audit, - "audit::safe_outputs", - "safe-output analysis failed", - safe_outputs::analyze_safe_outputs(run_dir).await, - apply_safe_output_analysis, - ); - run_analyzer( - audit, - "audit::detection", - "detection analysis failed", - detection::analyze_detection(run_dir).await, - |a, result| a.detection_analysis = result, - ); - run_analyzer( - audit, - "audit::missing_tools", - "missing-tool extraction failed", - missing::extract_missing_tools(run_dir).await, - |a, result| a.missing_tools = result, - ); - run_analyzer( - audit, - "audit::missing_data", - "missing-data extraction failed", - missing::extract_missing_data(run_dir).await, - |a, result| a.missing_data = result, - ); - run_analyzer( - audit, - "audit::noops", - "noop extraction failed", - missing::extract_noops(run_dir).await, - |a, result| a.noops = result, - ); + if artifact_family_selected(artifact_filters, "safe-outputs") { + run_analyzer( + audit, + "audit::safe_outputs", + "safe-output analysis failed", + safe_outputs::analyze_safe_outputs(run_dir).await, + apply_safe_output_analysis, + ); + } + if artifact_family_selected(artifact_filters, "detection") { + run_analyzer( + audit, + "audit::detection", + "detection analysis failed", + detection::analyze_detection(run_dir).await, + |a, result| a.detection_analysis = result, + ); + } run_analyzer( audit, "audit::jobs", @@ -454,6 +460,10 @@ async fn run_analyzers( ); } +fn artifact_family_selected(filters: Option<&[String]>, family: &str) -> bool { + filters.is_none_or(|filters| filters.iter().any(|filter| filter == family)) +} + fn apply_safe_output_analysis(audit: &mut AuditData, result: safe_outputs::SafeOutputAnalysis) { audit.safe_output_summary = result.summary; audit.safe_output_execution = result.execution; diff --git a/tests/audit_it.rs b/tests/audit_it.rs index c1f3e1e1..54822a89 100644 --- a/tests/audit_it.rs +++ b/tests/audit_it.rs @@ -1,11 +1,14 @@ //! End-to-end integration tests for `ado-aw audit` against a fake ADO server. +use std::io::{Cursor, Write}; use std::path::{Path, PathBuf}; +use std::process::Stdio; use serde::Deserialize; -use serde_json::json; +use serde_json::{Value, json}; use tempfile::TempDir; use tokio::fs; +use tokio::io::AsyncWriteExt; use tokio::process::Command; use wiremock::matchers::{method, path, query_param}; use wiremock::{Mock, MockServer, ResponseTemplate}; @@ -82,6 +85,225 @@ async fn run_audit( command.output().await.expect("run ado-aw audit") } +fn artifact_zip(name: &str, repeated_root: bool, files: &[(&str, &[u8])]) -> Vec { + let mut writer = zip::ZipWriter::new(Cursor::new(Vec::new())); + for (path, contents) in files { + let path = if repeated_root { + format!("{name}/{path}") + } else { + (*path).to_string() + }; + writer + .start_file(path, zip::write::SimpleFileOptions::default()) + .expect("start fixture zip entry"); + writer.write_all(contents).expect("write fixture zip entry"); + } + writer + .finish() + .expect("finish fixture zip") + .into_inner() +} + +async fn mount_complete_build( + server: &MockServer, + repeated_root: bool, + malformed_aw_info: bool, +) { + const BUILD_ID: u64 = 630125; + let artifact_names = [ + format!("agent_outputs_{BUILD_ID}"), + format!("analyzed_outputs_{BUILD_ID}"), + String::from("safe_outputs"), + ]; + let otel = include_bytes!("fixtures/copilot-otel.jsonl"); + let aw_info: &[u8] = if malformed_aw_info { + b"{\"schema\":\"ado-aw/aw_info/999\",\"token\":\"do-not-leak\"" + } else { + br#"{"schema":"ado-aw/aw_info/1","engine":"copilot","model":"claude-sonnet-4.5","threat_detection_enabled":true,"detection_engine":"copilot","detection_model":"gpt-5-mini","agent_name":"audit-fixture","target":"standalone","source":"agent.md","compiler_version":"0.48.0"}"# + }; + let agent_files: &[(&str, &[u8])] = &[ + ( + "staging/aw_info.json", + aw_info, + ), + ("staging/otel.jsonl", otel), + ( + "staging/safe_outputs.ndjson", + b"{\"name\":\"noop\",\"context\":\"noop-1\",\"reason\":\"already complete\"}\n{\"name\":\"missing-tool\",\"tool\":\"bash\",\"reason\":\"not configured\"}\n{\"name\":\"missing_data\",\"reason\":\"title unavailable\"}\n", + ), + ( + "logs/firewall/policy-manifest.json", + br#"{"version":1,"rules":[{"pattern":"api.github.com","verdict":"allow"},{"pattern":"blocked.example","verdict":"deny"}]}"#, + ), + ( + "logs/firewall/audit.jsonl", + b"{\"timestamp\":\"2026-05-21T12:01:00Z\",\"host\":\"api.github.com\",\"rule\":\"api.github.com\",\"verdict\":\"allowed\"}\n{\"timestamp\":\"2026-05-21T12:01:01Z\",\"host\":\"blocked.example\",\"rule\":\"blocked.example\",\"verdict\":\"denied\"}\nmalformed source line with https://secret.example/?token=do-not-leak\n", + ), + ( + "logs/mcpg/gateway.jsonl", + b"{\"ts\":\"2026-05-21T12:02:00Z\",\"server\":\"github\",\"event\":\"server_start\"}\n{\"ts\":\"2026-05-21T12:02:01Z\",\"server\":\"github\",\"tool\":\"search_code\",\"event\":\"tool_call\",\"input_size\":10,\"output_size\":20}\n{\"ts\":\"2026-05-21T12:02:02Z\",\"server\":\"github\",\"tool\":\"create_issue\",\"event\":\"tool_error\",\"error\":\"sanitized failure\"}\n", + ), + ]; + let detection_files: &[(&str, &[u8])] = &[( + "threat-analysis.json", + br#"{"prompt_injection":true,"secret_leak":false,"malicious_patch":false,"reasons":["synthetic prompt injection"]}"#, + )]; + let safe_output_files: &[(&str, &[u8])] = &[( + "executed-safe-outputs.ndjson", + b"{\"name\":\"noop\",\"status\":\"succeeded\",\"context\":\"noop-1\",\"result\":{\"status\":\"ok\"}}\n", + )]; + + let artifacts = artifact_names + .iter() + .enumerate() + .map(|(index, name)| { + json!({ + "id": index + 1, + "name": name, + "source": BUILD_ID.to_string(), + "resource": { + "type": "PipelineArtifact", + "downloadUrl": format!("{}/download/{}", server.uri(), name) + } + }) + }) + .collect::>(); + + Mock::given(method("GET")) + .and(path(format!( + "/test-project/_apis/build/builds/{BUILD_ID}" + ))) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "id": BUILD_ID, + "status": "completed", + "result": "succeeded", + "definition": { "name": "audit-discovery-630125" }, + "sourceBranch": "refs/heads/main", + "sourceVersion": "deadbeef1234", + "queueTime": "2026-05-21T12:00:00Z", + "startTime": "2026-05-21T12:00:30Z", + "finishTime": "2026-05-21T12:05:30Z" + }))) + .mount(server) + .await; + Mock::given(method("GET")) + .and(path(format!( + "/test-project/_apis/build/builds/{BUILD_ID}/artifacts" + ))) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ "value": artifacts }))) + .mount(server) + .await; + Mock::given(method("GET")) + .and(path(format!( + "/test-project/_apis/build/builds/{BUILD_ID}/timeline" + ))) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "records": [ + {"id":"agent","type":"Job","name":"Agent","state":"completed","result":"succeeded","startTime":"2026-05-21T12:00:30Z","finishTime":"2026-05-21T12:03:00Z"}, + {"id":"detection","type":"Job","name":"Detection","state":"completed","result":"succeeded","startTime":"2026-05-21T12:03:01Z","finishTime":"2026-05-21T12:04:00Z"}, + {"id":"safe","type":"Job","name":"SafeOutputs","state":"completed","result":"succeeded","startTime":"2026-05-21T12:04:01Z","finishTime":"2026-05-21T12:05:00Z"} + ] + }))) + .mount(server) + .await; + + for (name, files) in [ + (&artifact_names[0], agent_files), + (&artifact_names[1], detection_files), + (&artifact_names[2], safe_output_files), + ] { + Mock::given(method("GET")) + .and(path(format!("/download/{name}"))) + .respond_with( + ResponseTemplate::new(200) + .set_body_bytes(artifact_zip(name, repeated_root, files)), + ) + .mount(server) + .await; + } +} + +async fn run_audit_json( + workspace: &Path, + output_dir: &Path, + server: &MockServer, + extra_args: &[&str], +) -> Value { + let mut command = Command::new(binary()); + command + .current_dir(workspace) + .env("CI", "1") + .env("ADO_AW_TEST_ORG_URL", server.uri()) + .args(["audit", "630125", "--json", "--output"]) + .arg(output_dir) + .args([ + "--org", + "test-org", + "--project", + "test-project", + "--pat", + "test-pat", + ]) + .args(extra_args); + let output = command.output().await.expect("run JSON audit"); + assert!( + output.status.success(), + "audit should succeed: stdout={} stderr={}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + serde_json::from_slice(&output.stdout).expect("audit stdout should be JSON") +} + +fn normalize_server_url(value: &mut Value) { + value["overview"]["url"] = Value::Null; + value["overview"]["logs_path"] = Value::Null; +} + +async fn run_mcp_author( + workspace: &Path, + cache_root: &Path, + server: &MockServer, +) -> Vec { + let mut child = Command::new(binary()) + .arg("mcp-author") + .current_dir(workspace) + .env("CI", "1") + .env("TMPDIR", cache_root) + .env("ADO_AW_TEST_ORG_URL", server.uri()) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .expect("spawn mcp-author"); + let requests = [ + json!({"jsonrpc":"2.0","id":1,"method":"initialize","params":{"protocolVersion":"2025-03-26","capabilities":{},"clientInfo":{"name":"audit-it","version":"1"}}}), + json!({"jsonrpc":"2.0","method":"notifications/initialized","params":{}}), + json!({"jsonrpc":"2.0","id":2,"method":"tools/call","params":{"name":"audit_build","arguments":{"build_id_or_url":"630125","org":"test-org","project":"test-project","pat":"test-pat","no_cache":true}}}), + json!({"jsonrpc":"2.0","id":3,"method":"tools/call","params":{"name":"trace_failure","arguments":{"build_id_or_url":"630125","org":"test-org","project":"test-project","pat":"test-pat"}}}), + ]; + let mut stdin = child.stdin.take().expect("mcp stdin"); + for request in requests { + stdin + .write_all(format!("{request}\n").as_bytes()) + .await + .expect("write MCP request"); + } + drop(stdin); + let output = child.wait_with_output().await.expect("wait for mcp-author"); + assert!( + output.status.success(), + "mcp-author should succeed: stdout={} stderr={}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + String::from_utf8(output.stdout) + .expect("MCP stdout is UTF-8") + .lines() + .filter_map(|line| serde_json::from_str(line).ok()) + .collect() +} + #[tokio::test] async fn audit_happy_path_against_fake_ado() { let server = MockServer::start().await; @@ -456,3 +678,229 @@ async fn cached_audit_correlates_custom_safe_output_at_job_level() { "provenance mismatches should produce a finding: {finding_titles:?}" ); } + +#[tokio::test] +async fn audit_pipeline_artifact_layouts_are_equivalent_end_to_end() { + let flat_server = MockServer::start().await; + let repeated_server = MockServer::start().await; + mount_complete_build(&flat_server, false, false).await; + mount_complete_build(&repeated_server, true, false).await; + + let workspace = TempDir::new().expect("create workspace"); + fs::write( + workspace.path().join("agent.md"), + "---\nname: audit fixture\ndescription: audit fixture\nsafe-outputs:\n noop:\n---\n\nAudit fixture.\n", + ) + .await + .expect("write workflow source"); + let flat_output = TempDir::new().expect("create flat output"); + let repeated_output = TempDir::new().expect("create repeated output"); + + let mut flat = + run_audit_json(workspace.path(), flat_output.path(), &flat_server, &[]).await; + let mut repeated = + run_audit_json(workspace.path(), repeated_output.path(), &repeated_server, &[]).await; + normalize_server_url(&mut flat); + normalize_server_url(&mut repeated); + assert_eq!( + flat, repeated, + "flat and repeated artifact-name roots must produce identical AuditData" + ); + + for section in [ + "firewall_analysis", + "policy_analysis", + "mcp_server_health", + "mcp_tool_usage", + "engine_config", + "detection_analysis", + "safe_output_summary", + "safe_output_execution", + "pipeline_graph", + ] { + assert!( + !flat[section].is_null(), + "expected analyzer section '{section}' to be populated" + ); + } + assert!(flat["metrics"]["token_usage"].as_u64().unwrap_or(0) > 0); + assert!(!flat["mcp_failures"].as_array().expect("MCP failures").is_empty()); + assert!(!flat["missing_tools"].as_array().expect("missing tools").is_empty()); + assert!(!flat["missing_data"].as_array().expect("missing data").is_empty()); + assert!(!flat["noops"].as_array().expect("noops").is_empty()); + assert_eq!(flat["jobs"].as_array().expect("jobs").len(), 3); + + let rendered = serde_json::to_string(&flat).expect("serialize normalized audit"); + for secret in [ + "do-not-leak", + "secret.example", + "test-pat", + "malformed source line", + ] { + assert!( + !rendered.contains(secret), + "audit output leaked fixture secret marker '{secret}'" + ); + } + + let cached = + run_audit_json(workspace.path(), flat_output.path(), &flat_server, &[]).await; + let refreshed = run_audit_json( + workspace.path(), + flat_output.path(), + &flat_server, + &["--no-cache"], + ) + .await; + assert_eq!(cached, refreshed, "cached and --no-cache audits must agree"); + + for (filter, included, excluded) in [ + ( + "agent", + vec!["firewall_analysis", "mcp_tool_usage", "engine_config"], + vec!["detection_analysis", "safe_output_execution"], + ), + ( + "detection", + vec!["detection_analysis"], + vec!["firewall_analysis", "engine_config", "safe_output_execution"], + ), + ( + "safe-outputs", + vec![], + vec!["firewall_analysis", "engine_config", "detection_analysis"], + ), + ] { + let output = TempDir::new().expect("create filtered output"); + let audit = run_audit_json( + workspace.path(), + output.path(), + &flat_server, + &["--artifacts", filter], + ) + .await; + for section in included { + assert!( + !audit[section].is_null(), + "{filter} audit should populate {section}" + ); + } + for section in excluded { + assert!( + audit[section].is_null(), + "{filter} audit must not populate excluded section {section}" + ); + } + let files = audit["downloaded_files"] + .as_array() + .expect("downloaded files"); + assert!( + files.iter().all(|file| file["path"] + .as_str() + .is_some_and(|path| match filter { + "agent" => path.starts_with("agent_outputs_"), + "detection" => path.starts_with("analyzed_outputs_"), + "safe-outputs" => path.starts_with("safe_outputs/"), + _ => false, + })), + "{filter} audit included an excluded artifact family: {files:?}" + ); + } + + let console = + run_audit(workspace.path(), flat_output.path(), "630125", Some(&flat_server)).await; + assert!(console.status.success(), "console audit should succeed"); + let console = String::from_utf8_lossy(&console.stdout); + for heading in [ + "## Overview", + "## Safe Output Summary", + "## MCP Server Health", + "## Firewall Analysis", + "## Detection Analysis", + "## MCP Failures", + "## MCP Tool Usage", + ] { + assert!( + console.contains(heading), + "console audit omitted '{heading}'" + ); + } + + let trace_cache = TempDir::new().expect("create trace cache"); + let trace = Command::new(binary()) + .current_dir(workspace.path()) + .env("CI", "1") + .env("TMPDIR", trace_cache.path()) + .env("ADO_AW_TEST_ORG_URL", flat_server.uri()) + .args([ + "trace", + "630125", + "--json", + "--org", + "test-org", + "--project", + "test-project", + "--pat", + "test-pat", + ]) + .output() + .await + .expect("run trace"); + assert!( + trace.status.success(), + "trace should succeed: stdout={} stderr={}", + String::from_utf8_lossy(&trace.stdout), + String::from_utf8_lossy(&trace.stderr) + ); + let trace: Value = serde_json::from_slice(&trace.stdout).expect("trace JSON"); + assert_eq!(trace["build_id"], 630125); + + let mcp_cache = TempDir::new().expect("create MCP cache"); + let responses = + run_mcp_author(workspace.path(), mcp_cache.path(), &flat_server).await; + let audit_build = responses + .iter() + .find(|response| response["id"] == 2) + .expect("audit_build MCP response"); + let trace_failure = responses + .iter() + .find(|response| response["id"] == 3) + .expect("trace_failure MCP response"); + assert_eq!( + audit_build["result"]["structuredContent"]["overview"]["build_id"], + 630125 + ); + assert_eq!( + trace_failure["result"]["structuredContent"]["build_id"], + trace["build_id"] + ); + + let malformed_server = MockServer::start().await; + mount_complete_build(&malformed_server, true, true).await; + let malformed_output = TempDir::new().expect("create malformed output"); + let malformed = run_audit_json( + workspace.path(), + malformed_output.path(), + &malformed_server, + &[], + ) + .await; + assert!(malformed["engine_config"].is_null()); + assert!(malformed["pipeline_graph"].is_null()); + assert!(malformed["metrics"]["token_usage"].as_u64().unwrap_or(0) > 0); + assert!(!malformed["firewall_analysis"].is_null()); + assert!(!malformed["detection_analysis"].is_null()); + assert!( + !malformed["safe_output_summary"].is_null(), + "safe-output evidence should survive malformed aw_info: {malformed}" + ); + let warnings = malformed["warnings"].as_array().expect("warnings"); + assert!( + warnings + .iter() + .any(|warning| warning["source"] == "audit::aw_info"), + "malformed optional metadata should emit one bounded warning: {warnings:?}" + ); + let rendered = serde_json::to_string(&malformed).expect("serialize malformed audit"); + assert!(!rendered.contains("do-not-leak")); +} From 9fcc8f20353017e5118cf57ee75d5b8668b7b10a Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 5 Aug 2026 21:08:22 +0000 Subject: [PATCH 3/5] fix(audit): isolate filtered artifact caches Co-authored-by: jamesadevine <4742697+jamesadevine@users.noreply.github.com> --- src/audit/cli.rs | 44 ++++++++++++++++++- tests/audit_it.rs | 105 +++++++++++++++++++++++++++++----------------- 2 files changed, 109 insertions(+), 40 deletions(-) diff --git a/src/audit/cli.rs b/src/audit/cli.rs index 695fddfc..8edc8ea8 100644 --- a/src/audit/cli.rs +++ b/src/audit/cli.rs @@ -105,9 +105,15 @@ async fn resolve_run_setup(opts: &AuditOptions<'_>) -> Result { async fn fetch_audit_data_inner(opts: AuditOptions<'_>) -> Result { let setup = resolve_run_setup(&opts).await?; - if let Some(cached) = try_serve_from_cache(opts.no_cache, opts.json, &setup.run_dir).await? { + if setup.artifact_filters.is_none() + && let Some(cached) = + try_serve_from_cache(opts.no_cache, opts.json, &setup.run_dir).await? + { return Ok(cached); } + if let Some(filters) = setup.artifact_filters.as_deref() { + prune_unselected_artifacts(&setup.run_dir, filters).await?; + } let client = reqwest::Client::builder() .timeout(Duration::from_secs(60)) @@ -139,6 +145,42 @@ async fn fetch_audit_data_inner(opts: AuditOptions<'_>) -> Result Result<()> { + let mut entries = tokio::fs::read_dir(run_dir) + .await + .with_context(|| format!("read audit output directory {}", run_dir.display()))?; + while let Some(entry) = entries + .next_entry() + .await + .with_context(|| format!("iterate audit output directory {}", run_dir.display()))? + { + if !entry + .file_type() + .await + .with_context(|| format!("inspect audit output path {}", entry.path().display()))? + .is_dir() + { + continue; + } + let name = entry.file_name(); + let Some(prefix) = name.to_str().and_then(artifact_name_to_prefix) else { + continue; + }; + let family = match prefix { + "agent_outputs" => "agent", + "analyzed_outputs" => "detection", + "safe_outputs" => "safe-outputs", + _ => continue, + }; + if !artifact_family_selected(Some(filters), family) { + tokio::fs::remove_dir_all(entry.path()) + .await + .with_context(|| format!("remove excluded artifact {}", entry.path().display()))?; + } + } + Ok(()) + } + run_analyzers( &client, &setup.ctx, diff --git a/tests/audit_it.rs b/tests/audit_it.rs index 54822a89..4a780570 100644 --- a/tests/audit_it.rs +++ b/tests/audit_it.rs @@ -98,17 +98,10 @@ fn artifact_zip(name: &str, repeated_root: bool, files: &[(&str, &[u8])]) -> Vec .expect("start fixture zip entry"); writer.write_all(contents).expect("write fixture zip entry"); } - writer - .finish() - .expect("finish fixture zip") - .into_inner() + writer.finish().expect("finish fixture zip").into_inner() } -async fn mount_complete_build( - server: &MockServer, - repeated_root: bool, - malformed_aw_info: bool, -) { +async fn mount_complete_build(server: &MockServer, repeated_root: bool, malformed_aw_info: bool) { const BUILD_ID: u64 = 630125; let artifact_names = [ format!("agent_outputs_{BUILD_ID}"), @@ -170,9 +163,7 @@ async fn mount_complete_build( .collect::>(); Mock::given(method("GET")) - .and(path(format!( - "/test-project/_apis/build/builds/{BUILD_ID}" - ))) + .and(path(format!("/test-project/_apis/build/builds/{BUILD_ID}"))) .respond_with(ResponseTemplate::new(200).set_body_json(json!({ "id": BUILD_ID, "status": "completed", @@ -214,10 +205,11 @@ async fn mount_complete_build( ] { Mock::given(method("GET")) .and(path(format!("/download/{name}"))) - .respond_with( - ResponseTemplate::new(200) - .set_body_bytes(artifact_zip(name, repeated_root, files)), - ) + .respond_with(ResponseTemplate::new(200).set_body_bytes(artifact_zip( + name, + repeated_root, + files, + ))) .mount(server) .await; } @@ -260,11 +252,7 @@ fn normalize_server_url(value: &mut Value) { value["overview"]["logs_path"] = Value::Null; } -async fn run_mcp_author( - workspace: &Path, - cache_root: &Path, - server: &MockServer, -) -> Vec { +async fn run_mcp_author(workspace: &Path, cache_root: &Path, server: &MockServer) -> Vec { let mut child = Command::new(binary()) .arg("mcp-author") .current_dir(workspace) @@ -696,10 +684,14 @@ async fn audit_pipeline_artifact_layouts_are_equivalent_end_to_end() { let flat_output = TempDir::new().expect("create flat output"); let repeated_output = TempDir::new().expect("create repeated output"); - let mut flat = - run_audit_json(workspace.path(), flat_output.path(), &flat_server, &[]).await; - let mut repeated = - run_audit_json(workspace.path(), repeated_output.path(), &repeated_server, &[]).await; + let mut flat = run_audit_json(workspace.path(), flat_output.path(), &flat_server, &[]).await; + let mut repeated = run_audit_json( + workspace.path(), + repeated_output.path(), + &repeated_server, + &[], + ) + .await; normalize_server_url(&mut flat); normalize_server_url(&mut repeated); assert_eq!( @@ -724,9 +716,24 @@ async fn audit_pipeline_artifact_layouts_are_equivalent_end_to_end() { ); } assert!(flat["metrics"]["token_usage"].as_u64().unwrap_or(0) > 0); - assert!(!flat["mcp_failures"].as_array().expect("MCP failures").is_empty()); - assert!(!flat["missing_tools"].as_array().expect("missing tools").is_empty()); - assert!(!flat["missing_data"].as_array().expect("missing data").is_empty()); + assert!( + !flat["mcp_failures"] + .as_array() + .expect("MCP failures") + .is_empty() + ); + assert!( + !flat["missing_tools"] + .as_array() + .expect("missing tools") + .is_empty() + ); + assert!( + !flat["missing_data"] + .as_array() + .expect("missing data") + .is_empty() + ); assert!(!flat["noops"].as_array().expect("noops").is_empty()); assert_eq!(flat["jobs"].as_array().expect("jobs").len(), 3); @@ -743,8 +750,7 @@ async fn audit_pipeline_artifact_layouts_are_equivalent_end_to_end() { ); } - let cached = - run_audit_json(workspace.path(), flat_output.path(), &flat_server, &[]).await; + let cached = run_audit_json(workspace.path(), flat_output.path(), &flat_server, &[]).await; let refreshed = run_audit_json( workspace.path(), flat_output.path(), @@ -763,7 +769,11 @@ async fn audit_pipeline_artifact_layouts_are_equivalent_end_to_end() { ( "detection", vec!["detection_analysis"], - vec!["firewall_analysis", "engine_config", "safe_output_execution"], + vec![ + "firewall_analysis", + "engine_config", + "safe_output_execution", + ], ), ( "safe-outputs", @@ -795,20 +805,38 @@ async fn audit_pipeline_artifact_layouts_are_equivalent_end_to_end() { .as_array() .expect("downloaded files"); assert!( - files.iter().all(|file| file["path"] - .as_str() - .is_some_and(|path| match filter { + files.iter().all( + |file| file["path"].as_str().is_some_and(|path| match filter { "agent" => path.starts_with("agent_outputs_"), "detection" => path.starts_with("analyzed_outputs_"), "safe-outputs" => path.starts_with("safe_outputs/"), _ => false, - })), + }) + ), "{filter} audit included an excluded artifact family: {files:?}" ); } + let filter_cache = TempDir::new().expect("create filter cache"); + let _ = run_audit_json(workspace.path(), filter_cache.path(), &flat_server, &[]).await; + let filtered_after_full = run_audit_json( + workspace.path(), + filter_cache.path(), + &flat_server, + &["--artifacts", "agent"], + ) + .await; + assert!( + filtered_after_full["detection_analysis"].is_null(), + "an artifact filter must not reuse an incompatible full-audit cache" + ); - let console = - run_audit(workspace.path(), flat_output.path(), "630125", Some(&flat_server)).await; + let console = run_audit( + workspace.path(), + flat_output.path(), + "630125", + Some(&flat_server), + ) + .await; assert!(console.status.success(), "console audit should succeed"); let console = String::from_utf8_lossy(&console.stdout); for heading in [ @@ -856,8 +884,7 @@ async fn audit_pipeline_artifact_layouts_are_equivalent_end_to_end() { assert_eq!(trace["build_id"], 630125); let mcp_cache = TempDir::new().expect("create MCP cache"); - let responses = - run_mcp_author(workspace.path(), mcp_cache.path(), &flat_server).await; + let responses = run_mcp_author(workspace.path(), mcp_cache.path(), &flat_server).await; let audit_build = responses .iter() .find(|response| response["id"] == 2) From f5d6d70a27d80dec880b548417ffcd5134b613b6 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 5 Aug 2026 21:17:43 +0000 Subject: [PATCH 4/5] fix(audit): preserve full cache after filtered runs Co-authored-by: jamesadevine <4742697+jamesadevine@users.noreply.github.com> --- .../src/compiler-smoke-e2e/signals.ts | 17 +++++++-- src/audit/cli.rs | 35 +++++++++++++------ tests/audit_it.rs | 6 ++++ 3 files changed, 44 insertions(+), 14 deletions(-) diff --git a/scripts/ado-script/src/compiler-smoke-e2e/signals.ts b/scripts/ado-script/src/compiler-smoke-e2e/signals.ts index b02bbcb6..a4b72792 100644 --- a/scripts/ado-script/src/compiler-smoke-e2e/signals.ts +++ b/scripts/ado-script/src/compiler-smoke-e2e/signals.ts @@ -121,10 +121,21 @@ export async function verifyCandidateAudit( try { const audit = JSON.parse(outcome.stdout) as { overview?: { build_id?: number }; - downloaded_files?: unknown[]; + downloaded_files?: { path?: string }[]; }; - if (audit.overview?.build_id !== target.buildId || !audit.downloaded_files?.length) { - error = "JSON report did not contain the child build id and published artifact files"; + const paths = audit.downloaded_files?.flatMap((file) => file.path ?? []) ?? []; + const expectedRoots = [ + `agent_outputs_${target.buildId}/`, + `analyzed_outputs_${target.buildId}/`, + "safe_outputs/", + ]; + const missingRoots = expectedRoots.filter( + (root) => !paths.some((path) => path.startsWith(root)), + ); + if (audit.overview?.build_id !== target.buildId || missingRoots.length > 0) { + error = + `JSON report did not contain the child build id and every published artifact family; ` + + `missing roots: ${missingRoots.join(", ") || ""}`; } } catch (parseError) { error = `invalid JSON report: ${parseError instanceof Error ? parseError.message : String(parseError)}`; diff --git a/src/audit/cli.rs b/src/audit/cli.rs index 8edc8ea8..2fea7474 100644 --- a/src/audit/cli.rs +++ b/src/audit/cli.rs @@ -12,7 +12,9 @@ use crate::ado::{ use crate::audit::analyzers::{ custom_jobs, detection, firewall, jobs, mcp, missing, otel, policy, safe_outputs, }; -use crate::audit::cache::{RunSummary, load_run_summary, save_run_summary}; +use crate::audit::cache::{ + RUN_SUMMARY_FILENAME, RunSummary, load_run_summary, save_run_summary, +}; use crate::audit::findings; use crate::audit::model::{AuditData, ErrorInfo, FileInfo, OverviewData}; use crate::audit::pipeline_graph; @@ -113,6 +115,7 @@ async fn fetch_audit_data_inner(opts: AuditOptions<'_>) -> Result) -> Result) -> Result Result<()> { + match tokio::fs::remove_file(run_dir.join(RUN_SUMMARY_FILENAME)).await { + Ok(()) => Ok(()), + Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(()), + Err(error) => Err(error).context("remove incompatible unfiltered audit cache"), + } +} + /// Try to serve an audit result from the local on-disk cache. /// /// Returns `Some(result)` when a cache hit is found (version matches, JSON diff --git a/tests/audit_it.rs b/tests/audit_it.rs index 4a780570..6efdbdf7 100644 --- a/tests/audit_it.rs +++ b/tests/audit_it.rs @@ -829,6 +829,12 @@ async fn audit_pipeline_artifact_layouts_are_equivalent_end_to_end() { filtered_after_full["detection_analysis"].is_null(), "an artifact filter must not reuse an incompatible full-audit cache" ); + let full_after_filtered = + run_audit_json(workspace.path(), filter_cache.path(), &flat_server, &[]).await; + assert!( + !full_after_filtered["detection_analysis"].is_null(), + "a filtered audit must not poison a later unfiltered cache" + ); let console = run_audit( workspace.path(), From 543affb425069126c0a577806ea0e02658e67433 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 5 Aug 2026 21:20:59 +0000 Subject: [PATCH 5/5] fix(audit): isolate filtered artifact storage Co-authored-by: jamesadevine <4742697+jamesadevine@users.noreply.github.com> --- src/audit/cli.rs | 59 +++++------------------------------------------- 1 file changed, 6 insertions(+), 53 deletions(-) diff --git a/src/audit/cli.rs b/src/audit/cli.rs index 2fea7474..f0abfd89 100644 --- a/src/audit/cli.rs +++ b/src/audit/cli.rs @@ -12,9 +12,7 @@ use crate::ado::{ use crate::audit::analyzers::{ custom_jobs, detection, firewall, jobs, mcp, missing, otel, policy, safe_outputs, }; -use crate::audit::cache::{ - RUN_SUMMARY_FILENAME, RunSummary, load_run_summary, save_run_summary, -}; +use crate::audit::cache::{RunSummary, load_run_summary, save_run_summary}; use crate::audit::findings; use crate::audit::model::{AuditData, ErrorInfo, FileInfo, OverviewData}; use crate::audit::pipeline_graph; @@ -91,7 +89,10 @@ async fn resolve_run_setup(opts: &AuditOptions<'_>) -> Result { .context("Could not resolve current directory")?; let ctx = resolve_audit_context(&cwd, opts.org, opts.project, &parsed).await?; let auth = resolve_auth(opts.pat).await?; - let run_dir = opts.output.join(format!("build-{}", parsed.build_id)); + let mut run_dir = opts.output.join(format!("build-{}", parsed.build_id)); + if let Some(filters) = artifact_filters.as_deref() { + run_dir = run_dir.join(format!("artifacts-{}", filters.join("-"))); + } tokio::fs::create_dir_all(&run_dir) .await .with_context(|| format!("create audit output directory {}", run_dir.display()))?; @@ -113,11 +114,6 @@ async fn fetch_audit_data_inner(opts: AuditOptions<'_>) -> Result) -> Result Result<()> { - let mut entries = tokio::fs::read_dir(run_dir) - .await - .with_context(|| format!("read audit output directory {}", run_dir.display()))?; - while let Some(entry) = entries - .next_entry() - .await - .with_context(|| format!("iterate audit output directory {}", run_dir.display()))? - { - if !entry - .file_type() - .await - .with_context(|| format!("inspect audit output path {}", entry.path().display()))? - .is_dir() - { - continue; - } - let name = entry.file_name(); - let Some(prefix) = name.to_str().and_then(artifact_name_to_prefix) else { - continue; - }; - let family = match prefix { - "agent_outputs" => "agent", - "analyzed_outputs" => "detection", - "safe_outputs" => "safe-outputs", - _ => continue, - }; - if !artifact_family_selected(Some(filters), family) { - tokio::fs::remove_dir_all(entry.path()) - .await - .with_context(|| format!("remove excluded artifact {}", entry.path().display()))?; - } - } - Ok(()) - } - run_analyzers( &client, &setup.ctx, @@ -220,14 +180,6 @@ async fn fetch_audit_data_inner(opts: AuditOptions<'_>) -> Result Result<()> { - match tokio::fs::remove_file(run_dir.join(RUN_SUMMARY_FILENAME)).await { - Ok(()) => Ok(()), - Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(()), - Err(error) => Err(error).context("remove incompatible unfiltered audit cache"), - } -} - /// Try to serve an audit result from the local on-disk cache. /// /// Returns `Some(result)` when a cache hit is found (version matches, JSON @@ -845,6 +797,7 @@ fn normalize_artifact_filters(filters: Option<&[String]>) -> Result