test(audit): add end-to-end audit integration coverage

scripts/ado-script/src/compiler-smoke-e2e/index.ts

@@ -58,7 +58,7 @@ import {
 import { prepareCaseSource } from "./source.js";
 import { renderResultsTable } from "./report.js";
 import { runFixtures, type FixtureBuildRequest, type FixtureBuildResult } from "./runner.js";
-import { verifyCaseSignals } from "./signals.js";
+import { verifyCandidateAudit, verifyCaseSignals } from "./signals.js";
 import { scanStaleRefs } from "./stale.js";
 
 function log(msg: string): void {
@@ -379,8 +379,19 @@ export async function main(): Promise<number> {
       log,
     });
     const signalOutcome = await verifyCaseSignals(rest, resolved.cases, outcome.results);
-    results = signalOutcome.results;
-    overallOk = outcome.ok && signalOutcome.ok;
+    const auditOutcome =
+      config.compilerSource === "candidate"
+        ? await verifyCandidateAudit(signalOutcome.results, {
+            adoAwBin: config.adoAwBin,
+            cwd: config.sourcesDirectory,
+            orgUrl: config.orgUrl,
+            project: config.project,
+            token: config.token,
+            timeoutMs: config.childTimeoutMs,
+          })
+        : signalOutcome;
+    results = auditOutcome.results;
+    overallOk = outcome.ok && signalOutcome.ok && auditOutcome.ok;
     allTerminal = outcome.allTerminal;
     if (!overallOk) failureMessage = "one or more smoke cases did not succeed";
     if (!allTerminal) {

scripts/ado-script/src/compiler-smoke-e2e/signals.ts

@@ -11,6 +11,10 @@
  */
 import { expandBuildTag, type ResolvedCase } from "./cases.js";
 import type { FixtureBuildResult } from "./runner.js";
+import { mkdtemp, rm } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { redact, safeSpawn } from "./process.js";
 
 export interface BuildTagClient {
   getBuildTags(
@@ -72,3 +76,88 @@ export async function verifyCaseSignals(
     results: verified,
   };
 }
+
+/** Audit one completed candidate child through the released CLI contract. */
+export async function verifyCandidateAudit(
+  results: readonly FixtureBuildResult[],
+  options: {
+    adoAwBin: string;
+    cwd: string;
+    orgUrl: string;
+    project: string;
+    token: string;
+    timeoutMs: number;
+  },
+): Promise<SignalVerificationOutcome> {
+  const target = results.find(
+    (result) => result.caseId === "canary" && result.status === "succeeded" && result.buildId !== undefined,
+  );
+  if (!target?.buildId) return { ok: false, results: results.map((result) => ({ ...result })) };
+
+  const outputDir = await mkdtemp(join(tmpdir(), "ado-aw-smoke-audit-"));
+  try {
+    const outcome = await safeSpawn({
+      cmd: options.adoAwBin,
+      args: [
+        "audit",
+        String(target.buildId),
+        "--json",
+        "--no-cache",
+        "--output",
+        outputDir,
+        "--org",
+        options.orgUrl,
+        "--project",
+        options.project,
+      ],
+      cwd: options.cwd,
+      env: { AZURE_DEVOPS_EXT_PAT: options.token },
+      timeoutMs: options.timeoutMs,
+    });
+    let error: string | undefined;
+    if (outcome.timedOut || outcome.status !== 0) {
+      error = `exit=${outcome.status ?? "signal"} timedOut=${outcome.timedOut}; stderr=${redact(outcome.stderr, [options.token])}`;
+    } else {
+      try {
+        const audit = JSON.parse(outcome.stdout) as {
+          overview?: { build_id?: number };
+          downloaded_files?: { path?: string }[];
+        };
+        const paths = audit.downloaded_files?.flatMap((file) => file.path ?? []) ?? [];
+        const expectedRoots = [
+          `agent_outputs_${target.buildId}/`,
+          `analyzed_outputs_${target.buildId}/`,
+          "safe_outputs/",
+        ];
+        const missingRoots = expectedRoots.filter(
+          (root) => !paths.some((path) => path.startsWith(root)),
+        );
+        if (audit.overview?.build_id !== target.buildId || missingRoots.length > 0) {
+          error =
+            `JSON report did not contain the child build id and every published artifact family; ` +
+            `missing roots: ${missingRoots.join(", ") || "<none>"}`;
+        }
+      } catch (parseError) {
+        error = `invalid JSON report: ${parseError instanceof Error ? parseError.message : String(parseError)}`;
+      }
+    }
+    if (!error) return { ok: true, results: results.map((result) => ({ ...result })) };
+
+    return {
+      ok: false,
+      results: results.map((result) =>
+        result.caseId === target.caseId
+          ? {
+              ...result,
+              status: "failed",
+              message:
+                `candidate audit contract failed for build #${target.buildId} (${target.url ?? "URL unavailable"}); ` +
+                `expected artifacts agent_outputs_${target.buildId}, analyzed_outputs_${target.buildId}, and safe_outputs: ${error}`,
+            }
+          : { ...result },
+      ),
+    };
+  } finally {
+    await rm(outputDir, { recursive: true, force: true });
+  }
+}

src/ado/mod.rs

@@ -2116,6 +2116,30 @@ pub async fn download_build_artifact(
         )
     })?;
 
+    let repeated_root = (0..archive.len()).try_fold(true, |all_match, index| {
+        let entry = archive.by_index(index).with_context(|| {
+            format!(
+                "Failed to read zip entry {} from build artifact '{}'",
+                index, artifact.name
+            )
+        })?;
+        let entry_name = entry.name().to_string();
+        let relative_path = entry.enclosed_name().ok_or_else(|| {
+            anyhow::anyhow!(
+                "Refusing to extract unsafe path '{}' from build artifact '{}'",
+                entry_name,
+                artifact.name
+            )
+        })?;
+        Ok::<_, anyhow::Error>(
+            all_match
+                && relative_path
+                    .components()
+                    .next()
+                    .is_some_and(|component| component.as_os_str() == artifact.name.as_str()),
+        )
+    })?;
+
     for index in 0..archive.len() {
         let mut entry = archive.by_index(index).with_context(|| {
             format!(
@@ -2134,7 +2158,14 @@ pub async fn download_build_artifact(
                     artifact.name
                 )
             })?;
-        let output_path = artifact_dir.join(&relative_path);
+        let relative_path = if repeated_root {
+            relative_path
+                .strip_prefix(&artifact.name)
+                .expect("repeated artifact root was validated")
+        } else {
+            relative_path.as_path()
+        };
+        let output_path = artifact_dir.join(relative_path);
 
         if entry.is_dir() {
             std::fs::create_dir_all(&output_path).with_context(|| {

src/audit/analyzers/custom_jobs.rs

@@ -64,7 +64,6 @@ pub(crate) async fn load_custom_tool_catalog(
     supplied_aw_info: Option<&AwInfo>,
 ) -> anyhow::Result<CustomToolCatalog> {
     let mut catalog = CustomToolCatalog::default();
-    let mut primary_catalog_loaded = false;
 
     if let Some(path) = find_metadata_file(download_root, CUSTOM_TOOLS_FILENAME).await? {
         let contents = tokio::fs::read_to_string(&path)
@@ -87,7 +86,6 @@ pub(crate) async fn load_custom_tool_catalog(
                 )
             })?
         };
-        primary_catalog_loaded = true;
         for config in resolved.custom_tools {
             let tool = config.name.trim();
             if tool.is_empty() {
@@ -117,14 +115,13 @@ pub(crate) async fn load_custom_tool_catalog(
     let disk_aw_info = if supplied_aw_info.is_none() {
         match load_aw_info(download_root).await {
             Ok(value) => value,
-            Err(error) if primary_catalog_loaded => {
+            Err(error) => {
                 warn!("Failed to read optional aw_info.json metadata: {error:#}");
                 catalog
                     .warnings
                     .push(crate::audit::malformed_aw_info_warning());
                 None
             }
-            Err(error) => return Err(error),
         }
     } else {
         None

src/audit/cli.rs

@@ -89,7 +89,10 @@ async fn resolve_run_setup(opts: &AuditOptions<'_>) -> Result<AuditRunSetup> {
         .context("Could not resolve current directory")?;
     let ctx = resolve_audit_context(&cwd, opts.org, opts.project, &parsed).await?;
     let auth = resolve_auth(opts.pat).await?;
-    let run_dir = opts.output.join(format!("build-{}", parsed.build_id));
+    let mut run_dir = opts.output.join(format!("build-{}", parsed.build_id));
+    if let Some(filters) = artifact_filters.as_deref() {
+        run_dir = run_dir.join(format!("artifacts-{}", filters.join("-")));
+    }
     tokio::fs::create_dir_all(&run_dir)
         .await
         .with_context(|| format!("create audit output directory {}", run_dir.display()))?;
@@ -105,10 +108,12 @@ async fn resolve_run_setup(opts: &AuditOptions<'_>) -> Result<AuditRunSetup> {
 async fn fetch_audit_data_inner(opts: AuditOptions<'_>) -> Result<FetchAuditDataResult> {
     let setup = resolve_run_setup(&opts).await?;
 
-    if let Some(cached) = try_serve_from_cache(opts.no_cache, opts.json, &setup.run_dir).await? {
+    if setup.artifact_filters.is_none()
+        && let Some(cached) =
+            try_serve_from_cache(opts.no_cache, opts.json, &setup.run_dir).await?
+    {
         return Ok(cached);
     }
-
     let client = reqwest::Client::builder()
         .timeout(Duration::from_secs(60))
         .build()
@@ -154,16 +159,18 @@ async fn fetch_audit_data_inner(opts: AuditOptions<'_>) -> Result<FetchAuditData
     audit.metrics.error_count = audit.errors.len() as u64;
     derive_post_processing(&mut audit, &setup.run_dir).await;
 
-    save_run_summary(
-        &setup.run_dir,
-        &RunSummary {
-            ado_aw_version: env!("CARGO_PKG_VERSION").to_string(),
-            build_id: setup.parsed.build_id,
-            processed_at: Utc::now(),
-            audit_data: audit.clone(),
-        },
-    )
-    .await?;
+    if setup.artifact_filters.is_none() {
+        save_run_summary(
+            &setup.run_dir,
+            &RunSummary {
+                ado_aw_version: env!("CARGO_PKG_VERSION").to_string(),
+                build_id: setup.parsed.build_id,
+                processed_at: Utc::now(),
+                audit_data: audit.clone(),
+            },
+        )
+        .await?;
+    }
 
     Ok(FetchAuditDataResult {
         audit,
@@ -406,45 +413,51 @@ async fn run_analyzers(
         |a, files| a.downloaded_files = files,
     );
 
-    if let Some(agent_outputs_dir) = find_artifact_dir(run_dir, "agent_outputs").await {
-        run_agent_output_analyzers(&agent_outputs_dir, audit).await;
+    if artifact_family_selected(artifact_filters, "agent") {
+        if let Some(agent_outputs_dir) = find_artifact_dir(run_dir, "agent_outputs").await {
+            run_agent_output_analyzers(&agent_outputs_dir, audit).await;
+        }
+        run_analyzer(
+            audit,
+            "audit::missing_tools",
+            "missing-tool extraction failed",
+            missing::extract_missing_tools(run_dir).await,
+            |a, result| a.missing_tools = result,
+        );
+        run_analyzer(
+            audit,
+            "audit::missing_data",
+            "missing-data extraction failed",
+            missing::extract_missing_data(run_dir).await,
+            |a, result| a.missing_data = result,
+        );
+        run_analyzer(
+            audit,
+            "audit::noops",
+            "noop extraction failed",
+            missing::extract_noops(run_dir).await,
+            |a, result| a.noops = result,
+        );
     }
 
-    run_analyzer(
-        audit,
-        "audit::safe_outputs",
-        "safe-output analysis failed",
-        safe_outputs::analyze_safe_outputs(run_dir).await,
-        apply_safe_output_analysis,
-    );
-    run_analyzer(
-        audit,
-        "audit::detection",
-        "detection analysis failed",
-        detection::analyze_detection(run_dir).await,
-        |a, result| a.detection_analysis = result,
-    );
-    run_analyzer(
-        audit,
-        "audit::missing_tools",
-        "missing-tool extraction failed",
-        missing::extract_missing_tools(run_dir).await,
-        |a, result| a.missing_tools = result,
-    );
-    run_analyzer(
-        audit,
-        "audit::missing_data",
-        "missing-data extraction failed",
-        missing::extract_missing_data(run_dir).await,
-        |a, result| a.missing_data = result,
-    );
-    run_analyzer(
-        audit,
-        "audit::noops",
-        "noop extraction failed",
-        missing::extract_noops(run_dir).await,
-        |a, result| a.noops = result,
-    );
+    if artifact_family_selected(artifact_filters, "safe-outputs") {
+        run_analyzer(
+            audit,
+            "audit::safe_outputs",
+            "safe-output analysis failed",
+            safe_outputs::analyze_safe_outputs(run_dir).await,
+            apply_safe_output_analysis,
+        );
+    }
+    if artifact_family_selected(artifact_filters, "detection") {
+        run_analyzer(
+            audit,
+            "audit::detection",
+            "detection analysis failed",
+            detection::analyze_detection(run_dir).await,
+            |a, result| a.detection_analysis = result,
+        );
+    }
     run_analyzer(
         audit,
         "audit::jobs",
@@ -454,6 +467,10 @@ async fn run_analyzers(
     );
 }
 
+fn artifact_family_selected(filters: Option<&[String]>, family: &str) -> bool {
+    filters.is_none_or(|filters| filters.iter().any(|filter| filter == family))
+}
+
 fn apply_safe_output_analysis(audit: &mut AuditData, result: safe_outputs::SafeOutputAnalysis) {
     audit.safe_output_summary = result.summary;
     audit.safe_output_execution = result.execution;
@@ -780,6 +797,7 @@ fn normalize_artifact_filters(filters: Option<&[String]>) -> Result<Option<Vec<S
             normalized.push(canonical.to_string());
         }
     }
+    normalized.sort();
 
     Ok(Some(normalized))
 }