diff --git a/xcresult/build.rs b/xcresult/build.rs index 1b2867b1..ee1b3522 100644 --- a/xcresult/build.rs +++ b/xcresult/build.rs @@ -16,7 +16,11 @@ fn main() { ]; for (schema_name, top_level_type_refs) in schema_names_and_top_level_types { - let content = fs::read_to_string(format!("./{schema_name}.json")).unwrap(); + let schema_path = format!("{schema_name}.json"); + // Without this the generated types are not rebuilt when a schema changes, so + // an edit appears to have no effect until something else forces a rerun. + println!("cargo:rerun-if-changed={schema_path}"); + let content = fs::read_to_string(format!("./{schema_path}")).unwrap(); let schema = serde_json::from_str::(&content).unwrap(); let mut type_space = TypeSpace::new(TypeSpaceSettings::default().with_struct_builder(true)); diff --git a/xcresult/create-xcrun-xcresulttool-formatDescription-get---format-json---legacy-json-schema.py b/xcresult/create-xcrun-xcresulttool-formatDescription-get---format-json---legacy-json-schema.py index 332bd688..98d1273c 100644 --- a/xcresult/create-xcrun-xcresulttool-formatDescription-get---format-json---legacy-json-schema.py +++ b/xcresult/create-xcrun-xcresulttool-formatDescription-get---format-json---legacy-json-schema.py @@ -87,6 +87,9 @@ def convert_fd_object_to_json_schema_format( "_type": {"type": "object"}, } json_schema_object_required_properties: Set[str] = set() + # Whether a property was dropped because its type cannot be modelled. Such an + # object must not also be declared exhaustive; see `additionalProperties` below. + has_unmodellable_property = False if "supertype" in fd_type["type"]: fd_type_supertype_name = fd_type["type"]["supertype"] @@ -96,14 +99,18 @@ def convert_fd_object_to_json_schema_format( supertype_json_schema_object = convert_fd_object_to_json_schema_format( supertype_fd_type, fd_types, fd_types_inheritance_hierarchy ) - if "properties" in supertype_json_schema_object: - json_schema_object_properties |= supertype_json_schema_object["properties"] - else: - json_schema_object_properties |= supertype_json_schema_object["oneOf"][-1][ - "properties" - ] + supertype_json_schema_def = ( + supertype_json_schema_object + if "properties" in supertype_json_schema_object + else supertype_json_schema_object["oneOf"][-1] + ) + json_schema_object_properties |= supertype_json_schema_def["properties"] for required_property in supertype_json_schema_object.get("required", []): json_schema_object_required_properties.add(required_property) + # A supertype that had to drop a property passes that on: the sub-type's + # payload contains it too. + if supertype_json_schema_def.get("additionalProperties"): + has_unmodellable_property = True for fd_property in fd_type["properties"]: fd_property_name = fd_property["name"] @@ -122,6 +129,7 @@ def convert_fd_object_to_json_schema_format( fd_property_type = fd_property["wrappedType"] # NOTE: This check must be after updating the `fd_property_type` variable. if fd_property_type in BAD_FD_TYPES: + has_unmodellable_property = True continue if fd_property_type in FdValue: @@ -156,7 +164,16 @@ def convert_fd_object_to_json_schema_format( json_schema_def: Dict[str, Any] = { "type": "object", "properties": json_schema_object_properties, - "additionalProperties": fd_type_name == "ActionTestPlanRunSummaries", + # NOTE: `additionalProperties: false` becomes `deny_unknown_fields`, which is + # what keeps the untagged `oneOf` sub-type unions apart — so it has to stay on + # by default. But a property skipped above is still present in the data Apple + # emits, and rejecting the whole object over it loses far more than the one + # field: `SortedKeyValueArrayPair.value` is `SchemaSerializable`, which is not + # defined anywhere in the format description, and dropping it while staying + # exhaustive made every test summary carrying attachment metadata unparseable. + "additionalProperties": ( + fd_type_name == "ActionTestPlanRunSummaries" or has_unmodellable_property + ), } if len(json_schema_object_required_properties) > 0: json_schema_def["required"] = list(json_schema_object_required_properties) diff --git a/xcresult/src/file_attribution.rs b/xcresult/src/file_attribution.rs new file mode 100644 index 00000000..6c8cc44e --- /dev/null +++ b/xcresult/src/file_attribution.rs @@ -0,0 +1,468 @@ +//! Deciding which source file a failed test is reported against. +//! +//! An `.xcresult` records where a *failure was raised*. It does not record where a +//! *test is declared* — there is no per-test source location anywhere in the bundle. +//! So the file we report is inferred from the failure, and every source below is +//! answering a slightly different question than the one we are asking. +//! +//! That distinction matters because the reported file is what codeowners are +//! resolved from: a failure raised inside a helper is reported against the helper's +//! file, and the helper may belong to someone else entirely. +//! +//! [`FileSource`] names each place we look so a candidate can be traced back to +//! where it came from, rather than arriving as an anonymous `String`. + +use crate::types::legacy_schema; + +/// A file path in the form it is reported in the JUnit output. +/// +/// Normalization happens once, here, so it cannot be forgotten at a call site or +/// applied inconsistently between two paths that are then compared. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct ReportedPath(String); + +impl ReportedPath { + pub fn new(path: &str) -> Self { + Self(path.replace(' ', "%20")) + } + + pub fn as_str(&self) -> &str { + &self.0 + } + + pub fn into_string(self) -> String { + self.0 + } + + /// Whether this path is vendored dependency source rather than the repo's own + /// code: SPM's build dir (Tuist vendors into `/Tuist/.build/checkouts`), + /// SPM checkouts under Xcode's `DerivedData/SourcePackages/checkouts`, and + /// anything else Xcode generates under DerivedData. + /// + /// Reporting one of these hands the test to whoever owns the vendored + /// directory, because that is where codeowners are resolved from. + pub fn is_vendored_dependency(&self) -> bool { + DEPENDENCY_PATH_SEGMENTS + .iter() + .any(|segment| self.0.contains(segment)) + } +} + +const DEPENDENCY_PATH_SEGMENTS: [&str; 3] = ["/.build/", "/checkouts/", "/DerivedData/"]; + +/// Where a candidate file came from. +/// +/// Every variant here answers "where did this failure surface", which is only ever +/// a proxy for "which file owns this test". Keeping the provenance lets a reader — +/// and later a caller that wants to vet a candidate — tell the sources apart. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum FileSource { + /// The call-stack frame whose symbol names the test itself. The only source + /// that identifies the test rather than the failure, and so the only one that + /// is right by construction. + TestFrame, + /// `failureSummary.fileName`. The site the failure was raised from, which for an + /// assertion helper is the helper's file rather than the caller's. + RaisedFrom, + /// `sourceCodeContext.location.filePath`. The same site as [`Self::RaisedFrom`] + /// reported through a different field; the two usually agree. + SourceCodeLocation, + /// The last Swift or Objective-C frame of the failure's call stack. Frames run + /// innermost first, so this is the outermost frame with source — typically the + /// framework or trait that invoked the test rather than the test itself. + LastStackFrame, + /// `documentLocationInCreatingWorkspace` on an action-level issue summary. The + /// only source available without fetching the per-test failure summary. + DocumentLocation, +} + +impl FileSource { + /// Whether this source identifies the test itself, or merely where a failure + /// surfaced. Only the latter is a guess, and only a guess needs vetting before + /// it is reported. + pub fn is_positive_identification(&self) -> bool { + matches!(self, Self::TestFrame) + } +} + +/// A test as the bundle names it, and the symbols that name it in a call stack. +/// +/// Swift symbolizes a test method as `Suite.testCase()` and Objective-C as +/// `-[Suite testCase]`; a closure declared inside the test is prefixed +/// (`closure #1 in Suite.testCase()`) but is still defined in the test's file. A +/// swift-testing test declared at the top level has no suite and symbolizes as the +/// bare function. +#[derive(Debug, Clone, Copy)] +pub struct TestIdentity<'a> { + pub suite: Option<&'a str>, + pub case: &'a str, +} + +impl TestIdentity<'_> { + /// Whether `symbol` is this test's own frame, rather than a helper it called or + /// the framework that invoked it. + pub fn is_named_by(&self, symbol: &str) -> bool { + let expected = match self.suite { + Some(suite) => vec![ + format!("{}.{}", suite, self.case), + format!("-[{} {}]", suite, self.case.trim_end_matches("()")), + ], + None => vec![self.case.to_string()], + }; + expected + .iter() + .any(|expected| symbol == expected || symbol.ends_with(&format!(" in {}", expected))) + } + + /// How the action-level issue summaries key this test when Xcode records no + /// producing target. + pub fn fallback_key(&self) -> String { + match self.suite { + Some(suite) => format!("{}.{}", suite, self.case), + None => self.case.to_string(), + } + } +} + +/// A file we could report for a test, and the place we found it. +#[derive(Debug, Clone)] +pub struct FileCandidate { + pub path: ReportedPath, + pub source: FileSource, +} + +impl FileCandidate { + fn new(path: &str, source: FileSource) -> Self { + Self { + path: ReportedPath::new(path), + source, + } + } + + /// Every file a failure summary offers, in the order we prefer them. + /// + /// Ordered rather than merged because the sources disagree: the raised-from + /// fields point at where the assertion fired, the call-stack fallback at + /// whatever frame happened to be outermost. + pub fn from_failure_summary( + failure_summary: &legacy_schema::ActionTestFailureSummary, + identity: &TestIdentity, + ) -> Vec { + [ + test_frame(failure_summary, identity), + raised_from(failure_summary), + source_code_location(failure_summary), + ] + .into_iter() + .flatten() + .chain(stack_frames(failure_summary)) + .collect() + } + + /// The file an action-level issue summary points at, with the `file://` scheme + /// and line-number fragment stripped. + pub fn from_issue_summary( + failure_summary: &legacy_schema::TestFailureIssueSummary, + ) -> Option { + let url = failure_summary + .document_location_in_creating_workspace + .as_ref()? + .url + .as_ref()?; + let path = url + .value + .replace("file://", "") + .split('#') + .next() + .unwrap_or_default() + .to_string(); + Some(Self::new(&path, FileSource::DocumentLocation)) + } +} + +/// The frame that names the test, and so the test's own file. +/// +/// Frames run innermost first, so this sits in the middle of the stack — helpers it +/// called below it, the framework that invoked it above — which is why taking the +/// last frame lands on a dependency. +/// +/// `imageName` looks like the natural discriminator here and is not: SPM +/// dependencies are statically linked into the test bundle, so every frame reports +/// the test bundle's name. +fn test_frame( + failure_summary: &legacy_schema::ActionTestFailureSummary, + identity: &TestIdentity, +) -> Option { + let call_stack = failure_summary + .source_code_context + .as_ref()? + .call_stack + .as_ref()?; + call_stack.values.iter().find_map(|frame| { + let symbol_info = frame.symbol_info.as_ref()?; + if !identity.is_named_by(&symbol_info.symbol_name.as_ref()?.value) { + return None; + } + let file_path = symbol_info.location.as_ref()?.file_path.as_ref()?; + Some(FileCandidate::new(&file_path.value, FileSource::TestFrame)) + }) +} + +fn raised_from(failure_summary: &legacy_schema::ActionTestFailureSummary) -> Option { + let file_name = failure_summary.file_name.as_ref()?; + Some(FileCandidate::new(&file_name.value, FileSource::RaisedFrom)) +} + +fn source_code_location( + failure_summary: &legacy_schema::ActionTestFailureSummary, +) -> Option { + let file_path = failure_summary + .source_code_context + .as_ref()? + .location + .as_ref()? + .file_path + .as_ref()?; + Some(FileCandidate::new( + &file_path.value, + FileSource::SourceCodeLocation, + )) +} + +/// The failure's Swift and Objective-C frames, outermost first. +/// +/// Emitted as a sequence rather than a single "last frame" so that a caller +/// rejecting unusable paths lands on the outermost frame it can actually report, +/// rather than giving up because the outermost one happened to be a dependency. +fn stack_frames(failure_summary: &legacy_schema::ActionTestFailureSummary) -> Vec { + let Some(call_stack) = failure_summary + .source_code_context + .as_ref() + .and_then(|context| context.call_stack.as_ref()) + else { + return Vec::new(); + }; + call_stack + .values + .iter() + .filter_map(|frame| { + let file_path = frame + .symbol_info + .as_ref()? + .location + .as_ref()? + .file_path + .as_ref()?; + Some(FileCandidate::new( + &file_path.value, + FileSource::LastStackFrame, + )) + }) + // Frames from other languages and from generated code are not files we can + // report a Swift or Objective-C test against. + .filter(|candidate| { + std::path::Path::new(candidate.path.as_str()) + .extension() + .map(|extension| extension == "swift" || extension == "m") + .unwrap_or(false) + }) + .rev() + .collect() +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + use serde_json::{Value, json}; + + use super::*; + + const SUITE: &str = "SnapshotReproTests"; + const CASE: &str = "failingSnapshot()"; + + fn xc_string(value: &str) -> Value { + json!({ "_value": value }) + } + + fn failure_summary( + file_name: Option<&str>, + location: Option<&str>, + stack: &[(&str, &str)], + ) -> legacy_schema::ActionTestFailureSummary { + serde_json::from_value(json!({ + "fileName": file_name.map(xc_string), + "sourceCodeContext": { + "location": { "filePath": location.map(xc_string) }, + "callStack": { "_values": stack.iter().map(|(symbol, path)| json!({ + "symbolInfo": { + "symbolName": xc_string(symbol), + "location": { "filePath": xc_string(path) } + } + })).collect::>() } + } + })) + .unwrap() + } + + fn identity() -> TestIdentity<'static> { + TestIdentity { + suite: Some(SUITE), + case: CASE, + } + } + + #[rstest] + #[case::spaces_are_encoded("/repo/Tests/My Test.swift", "/repo/Tests/My%20Test.swift")] + #[case::already_safe("/repo/Tests/Test.swift", "/repo/Tests/Test.swift")] + fn reported_path_normalizes_once(#[case] path: &str, #[case] expected: &str) { + assert_eq!(ReportedPath::new(path).as_str(), expected); + } + + #[rstest] + #[case::tuist_checkout("/repo/Tuist/.build/checkouts/Dep/Dep.swift", true)] + #[case::derived_data("/repo/DerivedData/SourcePackages/checkouts/Dep/Dep.swift", true)] + #[case::the_repos_own_code("/repo/Tests/SnapshotReproTests.swift", false)] + fn reported_path_recognizes_vendored_sources(#[case] path: &str, #[case] expected: bool) { + assert_eq!(ReportedPath::new(path).is_vendored_dependency(), expected); + } + + #[rstest] + #[case::swift_symbol("SnapshotReproTests.failingSnapshot()", true)] + #[case::objc_symbol("-[SnapshotReproTests failingSnapshot]", true)] + #[case::closure_inside_test("closure #1 in SnapshotReproTests.failingSnapshot()", true)] + #[case::helper_the_test_called("assertSnapshot(of:as:)", false)] + #[case::same_case_name_in_another_suite("OtherTests.failingSnapshot()", false)] + #[case::trait_that_invoked_the_test( + "closure #1 in _SnapshotsTestTrait.provideScope(for:testCase:performing:)", + false + )] + fn identity_recognizes_only_the_tests_own_frame(#[case] symbol: &str, #[case] expected: bool) { + assert_eq!(identity().is_named_by(symbol), expected); + } + + #[rstest] + #[case::top_level_swift_testing_function("failingSnapshot()", true)] + #[case::closure_inside_it("closure #1 in failingSnapshot()", true)] + #[case::suite_scoped_symbol("SnapshotReproTests.failingSnapshot()", false)] + fn a_suiteless_test_is_matched_by_its_bare_function( + #[case] symbol: &str, + #[case] expected: bool, + ) { + let identity = TestIdentity { + suite: None, + case: CASE, + }; + assert_eq!(identity.is_named_by(symbol), expected); + } + + #[test] + fn candidates_are_offered_in_preference_order_and_keep_their_provenance() { + let summary = failure_summary( + Some("/repo/Tests/Raised.swift"), + Some("/repo/Tests/Location.swift"), + &[ + ("helper()", "/repo/Tests/Inner.swift"), + ( + "SnapshotReproTests.failingSnapshot()", + "/repo/Tests/Own.swift", + ), + ("framework()", "/repo/Tests/Outer.swift"), + ], + ); + assert_eq!( + FileCandidate::from_failure_summary(&summary, &identity()) + .iter() + .map(|candidate| (candidate.path.as_str(), candidate.source)) + .collect::>(), + vec![ + ("/repo/Tests/Own.swift", FileSource::TestFrame), + ("/repo/Tests/Raised.swift", FileSource::RaisedFrom), + ("/repo/Tests/Location.swift", FileSource::SourceCodeLocation), + // Frames run innermost first, so they are offered outermost first. + ("/repo/Tests/Outer.swift", FileSource::LastStackFrame), + ("/repo/Tests/Own.swift", FileSource::LastStackFrame), + ("/repo/Tests/Inner.swift", FileSource::LastStackFrame), + ] + ); + } + + #[test] + fn a_summary_offering_nothing_yields_no_candidates() { + let summary = failure_summary(None, None, &[]); + assert!(FileCandidate::from_failure_summary(&summary, &identity()).is_empty()); + } + + #[rstest] + #[case::other_languages_skipped( + &[("a", "/repo/Tests/Real.swift"), ("b", "/repo/Tests/Generated.cc"), ("c", "/repo/Readme.md")], + vec!["/repo/Tests/Real.swift"] + )] + #[case::nothing_usable(&[("a", "/repo/Tests/Generated.cc")], vec![])] + fn only_swift_and_objc_frames_are_offered( + #[case] stack: &[(&str, &str)], + #[case] expected: Vec<&str>, + ) { + let summary = failure_summary(None, None, stack); + assert_eq!( + stack_frames(&summary) + .iter() + .map(|candidate| candidate.path.as_str()) + .collect::>(), + expected + ); + } + + // Apple declares `SortedKeyValueArrayPair.value` as `SchemaSerializable`, a type + // the format description never defines, so the generator cannot model it and drops + // the property. The data still carries it, and an object that is both missing the + // property and declared exhaustive fails to deserialize — which silently disabled + // the whole experimental path for any bundle with test attachments. + #[test] + fn a_summary_parses_despite_properties_the_schema_cannot_model() { + let summary: legacy_schema::ActionTestPlanRunSummaries = serde_json::from_value(json!({ + "failureSummaries": { "_values": [{ + "fileName": xc_string("/repo/Tests/SnapshotReproTests.swift"), + "attachments": { "_values": [{ + "userInfo": { "storage": { "_values": [{ + "_type": { "_name": "SortedKeyValueArrayPair" }, + "key": xc_string("Encoding"), + "value": xc_string("{ XCTImageEncodingCompressionQualityKey = 0.7; }") + }] } } + }] } + }] } + })) + .expect("a summary carrying attachment metadata must still deserialize"); + let failure_summary = &summary.failure_summaries.unwrap().values[0]; + assert_eq!( + FileCandidate::from_failure_summary(failure_summary, &identity()) + .first() + .map(|candidate| candidate.path.as_str().to_string()), + Some(String::from("/repo/Tests/SnapshotReproTests.swift")) + ); + } + + #[rstest] + #[case::scheme_and_fragment_stripped( + Some("file:///repo/Tests/Test.swift#EndingLineNumber=8"), + Some("/repo/Tests/Test.swift") + )] + #[case::spaces_encoded( + Some("file:///repo/Tests/My Test.swift"), + Some("/repo/Tests/My%20Test.swift") + )] + #[case::no_document_location(None, None)] + fn an_issue_summary_yields_a_cleaned_document_location( + #[case] url: Option<&str>, + #[case] expected: Option<&str>, + ) { + let summary = serde_json::from_value(json!({ + "documentLocationInCreatingWorkspace": { "url": url.map(xc_string) } + })) + .unwrap(); + let candidate = FileCandidate::from_issue_summary(&summary); + assert_eq!(candidate.as_ref().map(|c| c.path.as_str()), expected); + if let Some(candidate) = candidate { + assert_eq!(candidate.source, FileSource::DocumentLocation); + } + } +} diff --git a/xcresult/src/lib.rs b/xcresult/src/lib.rs index 72902442..5b5a30aa 100644 --- a/xcresult/src/lib.rs +++ b/xcresult/src/lib.rs @@ -1,3 +1,4 @@ +pub mod file_attribution; pub mod types; pub mod xcresult; pub mod xcresult_legacy; diff --git a/xcresult/src/xcresult_legacy.rs b/xcresult/src/xcresult_legacy.rs index 7fdcea4b..b833124d 100644 --- a/xcresult/src/xcresult_legacy.rs +++ b/xcresult/src/xcresult_legacy.rs @@ -9,6 +9,7 @@ use petgraph::{ graph::{DiGraph, NodeIndex}, }; +use crate::file_attribution::{FileCandidate, TestIdentity}; use crate::types::{SWIFT_DEFAULT_TEST_SUITE_NAME, legacy_schema}; use crate::xcrun::{xcresulttool_get_object, xcresulttool_get_object_id}; @@ -24,7 +25,11 @@ pub struct XCResultTestLegacy { } impl XCResultTestLegacy { - fn find_file_in_test_summary(failure_summary_id: &str, path: &OsStr) -> Option { + fn find_file_in_test_summary( + failure_summary_id: &str, + path: &OsStr, + identity: &TestIdentity, + ) -> Option { let summary = xcresulttool_get_object_id(path, failure_summary_id); summary.ok().and_then(|summary| { summary @@ -34,99 +39,56 @@ impl XCResultTestLegacy { // grab the first failure summary if there are multiple failure_summaries.values.first() }) - .and_then(Self::find_file_in_failure_summary) + .and_then(|failure_summary| { + Self::find_file_in_failure_summary(failure_summary, identity) + }) }) } + /// The file to report for a failure: the first candidate we are willing to + /// stand behind. + /// + /// Only the test's own frame identifies the test; every other source says where + /// the failure surfaced, which for a snapshot trait, a mocking framework or a + /// page object is inside the dependency. Those are vetted here — in one place, + /// so a source added later cannot quietly skip the check — and a test that + /// crashes or fails to launch never reaches its own frame, leaving nothing + /// reportable. We then report no file at all rather than one that would hand the + /// test to whoever owns the vendored directory. Consumers treat a missing file + /// as "unchanged" rather than "cleared", so it keeps the path and owners it had. fn find_file_in_failure_summary( failure_summary: &legacy_schema::ActionTestFailureSummary, + identity: &TestIdentity, ) -> Option { - Self::normalize_file_path(failure_summary.file_name.as_ref().map(|file| &file.value)) - .or_else(|| { - Self::normalize_file_path( - failure_summary - .source_code_context - .as_ref() - .and_then(|source_code_context| source_code_context.location.as_ref()) - .and_then(|location| location.file_path.as_ref()) - .map(|file_path| &file_path.value), - ) - }) - .or_else(|| { - failure_summary - .source_code_context - .as_ref() - .and_then(Self::find_file_in_source_code_context_call_stack) - }) - } - - fn find_file_in_source_code_context_call_stack( - source_code_context: &legacy_schema::SourceCodeContext, - ) -> Option { - source_code_context - .call_stack - .as_ref() - .and_then(|call_stack| { - call_stack - .values - .iter() - .filter_map(|call_stack| { - call_stack - .symbol_info - .as_ref() - .and_then(|symbol_info| { - symbol_info - .location - .as_ref() - .and_then(|location| location.file_path.as_ref()) - }) - .and_then(|file_path| Self::normalize_file_path(Some(&file_path.value))) - }) - .filter(|file_path| { - std::path::Path::new(&file_path) - .extension() - .map(|ext| ext == "swift" || ext == "m") - .unwrap_or(false) - }) - // use the last valid swift / obj-c file-path in the stack - .last() - }) + FileCandidate::from_failure_summary(failure_summary, identity) + .into_iter() + .find(Self::is_reportable) + .map(|candidate| candidate.path.into_string()) } - fn normalize_file_path(file_path: Option<&String>) -> Option { - file_path.map(|file_path| file_path.replace(' ', "%20")) + fn is_reportable(candidate: &FileCandidate) -> bool { + candidate.source.is_positive_identification() || !candidate.path.is_vendored_dependency() } + /// The action-level issue summaries keyed by whatever names the test they + /// belong to, which is the producing target when Xcode records one and the test + /// case name otherwise. fn fallback_file_from_failure_issue_summary( failure_summary: &legacy_schema::TestFailureIssueSummary, ) -> Option<(Option<&str>, String)> { - failure_summary - .document_location_in_creating_workspace + let candidate = + FileCandidate::from_issue_summary(failure_summary).filter(Self::is_reportable)?; + let producing_target = failure_summary + .producing_target .as_ref() - .and_then(|document_location_in_creating_workspace| { - document_location_in_creating_workspace.url.as_ref() - }) - .map(|file| { - let file = file - .value - .replace("file://", "") - .split('#') - .next() - .unwrap_or_default() - .into(); - let producing_target = failure_summary - .producing_target - .as_ref() - .map(|x| x.value.as_ref()); - if producing_target.is_some() { - return (producing_target, file); - } - let test_case_name = failure_summary - .test_case_name - .as_ref() - .map(|x| x.value.as_ref()); - (test_case_name, file) - }) + .map(|x| x.value.as_ref()); + let key = producing_target.or_else(|| { + failure_summary + .test_case_name + .as_ref() + .map(|x| x.value.as_ref()) + }); + Some((key, candidate.path.into_string())) } fn find_fallback_file<'a>( @@ -330,12 +292,11 @@ impl XCResultTestLegacy { let test_suite_name = parent_node.map(|node| node.weight.name); let test_case_name = node.weight.name; - let formatted_test_case_name = - if let Some(test_suite_name) = test_suite_name { - format!("{}.{}", test_suite_name, test_case_name) - } else { - test_case_name.to_string() - }; + let identity = TestIdentity { + suite: test_suite_name, + case: test_case_name, + }; + let formatted_test_case_name = identity.fallback_key(); let failure_summary_id = node.weight.failure_summary_id; let mut file = if use_experimental_failure_summary && failure_summary_id.is_some() @@ -343,6 +304,7 @@ impl XCResultTestLegacy { Self::find_file_in_test_summary( failure_summary_id.unwrap_or_default(), path.as_ref(), + &identity, ) } else { None @@ -556,8 +518,29 @@ mod tests { json!({ "_value": value }) } + const TEST_SUITE: &str = "SnapshotReproTests"; + const TEST_CASE: &str = "failingSnapshot()"; + #[rstest] - #[case::file_name_wins( + // The test's own frame beats the file the failure was raised from, which here is + // the assertion helper inside the dependency. + #[case::test_frame_wins_over_raised_from_file( + Some("/repo/Tests/Assertion.swift"), + Some("/repo/Tests/Assertion.swift"), + &[ + ("assertSnapshot(of:as:)", "/repo/Tuist/.build/checkouts/swift-snapshot-testing/Assert.swift"), + ("SnapshotReproTests.failingSnapshot()", "/repo/Tests/SnapshotReproTests.swift"), + ("closure #1 in _SnapshotsTestTrait.provideScope(for:)", "/repo/Tuist/.build/checkouts/swift-snapshot-testing/Trait.swift"), + ], + Some("/repo/Tests/SnapshotReproTests.swift") + )] + #[case::objc_symbol_and_closure_frames_name_the_test( + None, + None, + &[("closure #1 in -[SnapshotReproTests failingSnapshot]", "/repo/Tests/SnapshotReproTests.m")], + Some("/repo/Tests/SnapshotReproTests.m") + )] + #[case::file_name_wins_when_no_frame_names_the_test( Some("/repo/Tests/My Test.swift"), Some("/repo/Tests/Other.swift"), &[], @@ -566,51 +549,78 @@ mod tests { #[case::location_before_call_stack( None, Some("/repo/Tests/Assertion.swift"), - &["/repo/Packages/SnapshotTesting/SnapshotsTestTrait.swift"], + &[("provideScope(for:)", "/repo/Packages/SnapshotTesting/SnapshotsTestTrait.swift")], + Some("/repo/Tests/Assertion.swift") + )] + #[case::dependency_file_name_falls_through_to_location( + Some("/repo/Tuist/.build/checkouts/UITestSupport/PageObject.swift"), + Some("/repo/Tests/Assertion.swift"), + &[], Some("/repo/Tests/Assertion.swift") )] #[case::last_swift_or_objc_stack_frame( None, None, &[ - "/repo/Tests/Generated.cc", - "/repo/Tests/First.swift", - "/repo/Tests/Second.m", - "/repo/Tests/Readme.md", + ("first", "/repo/Tests/Generated.cc"), + ("second", "/repo/Tests/First.swift"), + ("third", "/repo/Tests/Second.m"), + ("fourth", "/repo/Tests/Readme.md"), ], Some("/repo/Tests/Second.m") )] + // The outermost frame is a dependency, so the next one out is taken instead. + #[case::dependency_frames_skipped_in_stack_fallback( + None, + None, + &[ + ("first", "/repo/Tests/First.swift"), + ("second", "/repo/Tuist/.build/checkouts/UITestSupport/Launching.swift"), + ], + Some("/repo/Tests/First.swift") + )] + // A launch failure or crash never reaches the test's own frame, so every + // remaining source points into the dependency: report no file rather than one + // that would re-own the test. + #[case::only_dependency_sources_yields_nothing( + None, + Some("/repo/DerivedData/SourcePackages/checkouts/UITestSupport/Launching.swift"), + &[("launch", "/repo/Tuist/.build/checkouts/UITestSupport/Launching.swift")], + None + )] #[case::no_usable_file( None, None, - &["/repo/Tests/Generated.cc", "/repo/Tests/Readme.md"], + &[("first", "/repo/Tests/Generated.cc"), ("second", "/repo/Tests/Readme.md")], None )] fn failure_summary_file_sources( #[case] file_name: Option<&str>, #[case] location: Option<&str>, - #[case] stack: &[&str], + #[case] stack: &[(&str, &str)], #[case] expected: Option<&str>, ) { let summary = serde_json::from_value(json!({ "fileName": file_name.map(xc_string), "sourceCodeContext": { "location": { "filePath": location.map(xc_string) }, - "callStack": { "_values": stack.iter().map(|path| { - let stack_frame = json!({ - "symbolInfo": { - "location": { - "filePath": xc_string(path) - } - } - }); - stack_frame - }).collect::>() } + "callStack": { "_values": stack.iter().map(|(symbol, path)| json!({ + "symbolInfo": { + "symbolName": xc_string(symbol), + "location": { "filePath": xc_string(path) } + } + })).collect::>() } } })) .unwrap(); - let file = XCResultTestLegacy::find_file_in_failure_summary(&summary); - assert_eq!(file, expected.map(String::from)); + let identity = TestIdentity { + suite: Some(TEST_SUITE), + case: TEST_CASE, + }; + assert_eq!( + XCResultTestLegacy::find_file_in_failure_summary(&summary, &identity), + expected.map(String::from) + ); } #[rstest] @@ -626,6 +636,14 @@ mod tests { Some("SnapshotReproTests.failingSnapshot()"), Some((Some("SnapshotReproTests.failingSnapshot()"), "/repo/Tests/Test.swift")) )] + #[case::dependency_document_location( + Some( + "file:///repo/Tuist/.build/checkouts/UITestSupport/PageObject.swift#EndingLineNumber=377" + ), + Some("SnapshotReproTests"), + Some("SnapshotReproTests.failingSnapshot()"), + None + )] #[case::missing_document_location( None, None, diff --git a/xcresult/tests/data/test-crash-in-dependency.junit.xml b/xcresult/tests/data/test-crash-in-dependency.junit.xml new file mode 100644 index 00000000..97db712b --- /dev/null +++ b/xcresult/tests/data/test-crash-in-dependency.junit.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + diff --git a/xcresult/tests/data/test-crash-in-dependency.xcresult.tar.gz b/xcresult/tests/data/test-crash-in-dependency.xcresult.tar.gz new file mode 100644 index 00000000..1acc63c6 Binary files /dev/null and b/xcresult/tests/data/test-crash-in-dependency.xcresult.tar.gz differ diff --git a/xcresult/tests/data/test-dependency-raises-failure.junit.xml b/xcresult/tests/data/test-dependency-raises-failure.junit.xml new file mode 100644 index 00000000..367a44c8 --- /dev/null +++ b/xcresult/tests/data/test-dependency-raises-failure.junit.xml @@ -0,0 +1,8 @@ + + + + + + + + diff --git a/xcresult/tests/data/test-dependency-raises-failure.legacy.junit.xml b/xcresult/tests/data/test-dependency-raises-failure.legacy.junit.xml new file mode 100644 index 00000000..76c97bc8 --- /dev/null +++ b/xcresult/tests/data/test-dependency-raises-failure.legacy.junit.xml @@ -0,0 +1,8 @@ + + + + + + + + diff --git a/xcresult/tests/data/test-dependency-raises-failure.xcresult.tar.gz b/xcresult/tests/data/test-dependency-raises-failure.xcresult.tar.gz new file mode 100644 index 00000000..eeb7fc9e Binary files /dev/null and b/xcresult/tests/data/test-dependency-raises-failure.xcresult.tar.gz differ diff --git a/xcresult/tests/data/test-in-repo-helper-raises-failure.junit.xml b/xcresult/tests/data/test-in-repo-helper-raises-failure.junit.xml new file mode 100644 index 00000000..49b60781 --- /dev/null +++ b/xcresult/tests/data/test-in-repo-helper-raises-failure.junit.xml @@ -0,0 +1,8 @@ + + + + + + + + diff --git a/xcresult/tests/data/test-in-repo-helper-raises-failure.legacy.junit.xml b/xcresult/tests/data/test-in-repo-helper-raises-failure.legacy.junit.xml new file mode 100644 index 00000000..2f0d31a2 --- /dev/null +++ b/xcresult/tests/data/test-in-repo-helper-raises-failure.legacy.junit.xml @@ -0,0 +1,8 @@ + + + + + + + + diff --git a/xcresult/tests/data/test-in-repo-helper-raises-failure.xcresult.tar.gz b/xcresult/tests/data/test-in-repo-helper-raises-failure.xcresult.tar.gz new file mode 100644 index 00000000..e45beab9 Binary files /dev/null and b/xcresult/tests/data/test-in-repo-helper-raises-failure.xcresult.tar.gz differ diff --git a/xcresult/tests/data/test-objc-xctest.junit.xml b/xcresult/tests/data/test-objc-xctest.junit.xml new file mode 100644 index 00000000..ca8fe9fa --- /dev/null +++ b/xcresult/tests/data/test-objc-xctest.junit.xml @@ -0,0 +1,8 @@ + + + + + + + + diff --git a/xcresult/tests/data/test-objc-xctest.legacy.junit.xml b/xcresult/tests/data/test-objc-xctest.legacy.junit.xml new file mode 100644 index 00000000..849a0bc7 --- /dev/null +++ b/xcresult/tests/data/test-objc-xctest.legacy.junit.xml @@ -0,0 +1,8 @@ + + + + + + + + diff --git a/xcresult/tests/data/test-objc-xctest.xcresult.tar.gz b/xcresult/tests/data/test-objc-xctest.xcresult.tar.gz new file mode 100644 index 00000000..2288cef7 Binary files /dev/null and b/xcresult/tests/data/test-objc-xctest.xcresult.tar.gz differ diff --git a/xcresult/tests/data/test-toplevel-swift-testing.junit.xml b/xcresult/tests/data/test-toplevel-swift-testing.junit.xml new file mode 100644 index 00000000..ced884f2 --- /dev/null +++ b/xcresult/tests/data/test-toplevel-swift-testing.junit.xml @@ -0,0 +1,8 @@ + + + + + + + + diff --git a/xcresult/tests/data/test-toplevel-swift-testing.legacy.junit.xml b/xcresult/tests/data/test-toplevel-swift-testing.legacy.junit.xml new file mode 100644 index 00000000..89b64233 --- /dev/null +++ b/xcresult/tests/data/test-toplevel-swift-testing.legacy.junit.xml @@ -0,0 +1,8 @@ + + + + + + + + diff --git a/xcresult/tests/data/test-toplevel-swift-testing.xcresult.tar.gz b/xcresult/tests/data/test-toplevel-swift-testing.xcresult.tar.gz new file mode 100644 index 00000000..51f3e092 Binary files /dev/null and b/xcresult/tests/data/test-toplevel-swift-testing.xcresult.tar.gz differ diff --git a/xcresult/tests/fixture-src/.gitignore b/xcresult/tests/fixture-src/.gitignore new file mode 100644 index 00000000..a8be5a09 --- /dev/null +++ b/xcresult/tests/fixture-src/.gitignore @@ -0,0 +1,7 @@ +# SwiftPM and SourceKit build in place if the packages are opened in an editor; +# regeneration always builds in a copy under /tmp, so nothing here is ever needed. +.build/ +Package.resolved +DerivedData/ +*.xcresult/ +__pycache__/ diff --git a/xcresult/tests/fixture-src/README.md b/xcresult/tests/fixture-src/README.md new file mode 100644 index 00000000..8838bd97 --- /dev/null +++ b/xcresult/tests/fixture-src/README.md @@ -0,0 +1,94 @@ +# xcresult fixture sources + +The `.xcresult` bundles in `../data/` are captured from the SwiftPM packages here. +Each package exists to reproduce one shape in which the file we report for a failed +test used to be a vendored dependency's rather than the test's own — see the commit +"fix(xcresult): attribute a failure to the test's own frame, not a dependency's". + +Nothing in this directory is compiled by `cargo`; it is source for `regenerate.sh`, +checked in so the bundles can be rebuilt rather than being opaque binaries. + +## Regenerating + +Requires **macOS with Xcode** (`xcodebuild` and `xcrun xcresulttool`). + +```sh +./regenerate.sh # every scenario +./regenerate.sh objc-xctest # just one +``` + +For each scenario the script copies the package to `/tmp/xcresult-fixtures/`, turns +its `Dependency` directory into a git repository if it has one, runs + +```sh +xcodebuild test -scheme -Package -destination 'platform=macOS' \ + -derivedDataPath DerivedData -resultBundlePath .xcresult +``` + +(nonzero exit is the expected outcome — these tests are meant to fail), strips the +bundle with `prune-bundle.py`, dumps the failure summaries with +`dump-failure-summaries.py`, checks them with `verify-failure-summaries.py`, and +only then packages the bundle into `../data/test-.xcresult.tar.gz`. + +Pruning is not an optimization; without it these are unshippable. Xcode 26 writes +about 95MB of dyld shared-cache symbolication data into every result bundle, none +of it referenced from the invocation record — a scenario whose actual test data is +under 100KB produces a 96MB bundle. `prune-bundle.py` walks the object graph from +the root (whose own id lives in `Info.plist`, not in any object) and deletes the +`Data/` entries nothing reached; for these scenarios that is 9-11 objects kept out +of ~1570 files. `regenerate.sh` captures both `xcresulttool` outputs the crate +reads — `get object --legacy` and `get test-results tests` — before and after, and +fails if pruning changed either, so the saving is verified rather than assumed. + +Three details that are easy to get wrong: + +- **The dependency has to be a git repository.** A `.package(path:)` dependency is + built in place; only a git URL is checked out into + `DerivedData/SourcePackages/checkouts/`, and that path _is_ the shape being + reproduced. `regenerate.sh` creates the repository in the working copy so the + checked-in sources stay a plain directory. +- **Absolute paths are baked into a bundle at capture time** and end up in the + expected JUnit XML, which is why capture happens in a fixed directory. Set + `FIXTURE_WORK_DIR` to move it, and expect every `file` attribute to change. +- **Recapturing changes the timestamps** in the expected JUnit XML even when + nothing else moves. Diff with timestamps normalized to confirm that is all that + changed before saving. + +The expected JUnit XML is _not_ regenerated automatically. After recapturing, run +`cargo test -p xcresult`, read the diff, check each `file` attribute against the +table below, and only then save the new output over `../data/test-*.junit.xml`. A +`file` that lands on a dependency path is a bug to report, not output to snapshot. + +## What each scenario must exhibit + +`verify-failure-summaries.py` enforces the "captured shape" column and fails the +regeneration if a bundle stops reproducing it. + +| Scenario | Captured shape | Expected `file` (experimental) | Expected `file` (legacy) | +| ------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------ | ------------------------ | +| `dependency-raises-failure` | A swift-testing test calls a dependency helper that records the issue at its own `#filePath`. `fileName`, the source code context's location and the innermost frame are all under `SourcePackages/checkouts/`; only the test's own frame names the test file. | the test's own file | _(none)_ | +| `in-repo-helper-raises-failure` | Same, with the helper in the test target. Nothing rejects the helper's path, so the test's own frame has to win on its own merits. | the test's own file | the helper's file | +| `crash-in-dependency` | Two tests that never reach their own frame: one `fatalError`s inside the dependency (Xcode records a summary with no file at all), and one is failed by the dependency's `TestScoping` trait after its body returned (every file source is a checkout path). | _(none)_ | _(none)_ | +| `objc-xctest` | An Objective-C `XCTestCase` whose failure is raised by `XCTFail` in a shared category in another file, so the frame is symbolicated as `-[ObjcXCTestTests testFailsInsideSharedHelper]`. | the test's own file | _(none)_ | +| `toplevel-swift-testing` | A top-level `@Test func` with no suite, failed by a helper in another file, so the frame is the bare function name. | the test's own file | the helper's file | + +The two columns differ because only the experimental path reads the failure +summary's call stack, which is the only source that can identify the test's own +file. The legacy path sees just the workspace document location — the file the +failure was _raised_ from — so where that is a dependency it now reports no file, +and where it is an in-repo helper it still reports the helper. `objc-xctest` has no +legacy file for an unrelated reason: the fallback is keyed by test-case name and +Xcode spells the Objective-C one `-[Suite testCase]`, which never matches the +`Suite.testCase` key the lookup builds. + +To read what a captured bundle actually contains, run `dump-failure-summaries.py` +against it — the dump is derived from the bundle, so it is not checked in. + +## Why not the older SnapshotTesting fixture + +`../data/test-swift-snapshot-testing.xcresult.tar.gz` looks like it covers the +first scenario and does not: `assertSnapshot` takes `filePath: StaticString = +#filePath`, which is evaluated at the _call site_, so its `fileName` already points +at the test's own source. It passes with or without the fix. A helper that wants to +report its own location — which is what a trait or a page object does — has to build +the `SourceLocation` inside its body, which is what these fixtures do. diff --git a/xcresult/tests/fixture-src/crash-in-dependency/Dependency/Package.swift b/xcresult/tests/fixture-src/crash-in-dependency/Dependency/Package.swift new file mode 100644 index 00000000..4b45832a --- /dev/null +++ b/xcresult/tests/fixture-src/crash-in-dependency/Dependency/Package.swift @@ -0,0 +1,13 @@ +// swift-tools-version: 6.0 +import PackageDescription + +let package = Package( + name: "FixtureSupport", + platforms: [.macOS(.v13)], + products: [ + .library(name: "FixtureSupport", targets: ["FixtureSupport"]) + ], + targets: [ + .target(name: "FixtureSupport") + ] +) diff --git a/xcresult/tests/fixture-src/crash-in-dependency/Dependency/Sources/FixtureSupport/Crash.swift b/xcresult/tests/fixture-src/crash-in-dependency/Dependency/Sources/FixtureSupport/Crash.swift new file mode 100644 index 00000000..62be6b71 --- /dev/null +++ b/xcresult/tests/fixture-src/crash-in-dependency/Dependency/Sources/FixtureSupport/Crash.swift @@ -0,0 +1,4 @@ +/// Kills the test process from inside the dependency. +public func crashInsideDependency() -> Never { + fatalError("the dependency crashed the test process") +} diff --git a/xcresult/tests/fixture-src/crash-in-dependency/Dependency/Sources/FixtureSupport/TeardownFailureTrait.swift b/xcresult/tests/fixture-src/crash-in-dependency/Dependency/Sources/FixtureSupport/TeardownFailureTrait.swift new file mode 100644 index 00000000..ed60e3b8 --- /dev/null +++ b/xcresult/tests/fixture-src/crash-in-dependency/Dependency/Sources/FixtureSupport/TeardownFailureTrait.swift @@ -0,0 +1,28 @@ +import Testing + +/// A trait that fails the test *after* its body has returned, so the test's own +/// frame is already off the stack when the issue is recorded — the shape of a +/// snapshot-verification or screenshot-diffing trait that checks its work in +/// teardown. Every file the failure summary offers is then the dependency's. +public struct TeardownFailureTrait: TestTrait, TestScoping { + public func provideScope( + for test: Test, + testCase: Test.Case?, + performing function: () async throws -> Void + ) async throws { + try await function() + Issue.record( + Comment(rawValue: "the dependency's trait failed the test after its body returned"), + sourceLocation: SourceLocation( + fileID: #fileID, + filePath: #filePath, + line: #line, + column: #column + ) + ) + } +} + +extension Trait where Self == TeardownFailureTrait { + public static var teardownFailure: Self { .init() } +} diff --git a/xcresult/tests/fixture-src/crash-in-dependency/Package.swift b/xcresult/tests/fixture-src/crash-in-dependency/Package.swift new file mode 100644 index 00000000..8ab2898b --- /dev/null +++ b/xcresult/tests/fixture-src/crash-in-dependency/Package.swift @@ -0,0 +1,18 @@ +// swift-tools-version: 6.0 +import PackageDescription + +// See `dependency-raises-failure/Package.swift` for why the dependency is a git +// URL rather than a path. +let package = Package( + name: "CrashInDependency", + platforms: [.macOS(.v13)], + dependencies: [ + .package(url: "./Dependency", branch: "main") + ], + targets: [ + .testTarget( + name: "CrashInDependencyTests", + dependencies: [.product(name: "FixtureSupport", package: "Dependency")] + ) + ] +) diff --git a/xcresult/tests/fixture-src/crash-in-dependency/Tests/CrashInDependencyTests/CrashInDependencyTests.swift b/xcresult/tests/fixture-src/crash-in-dependency/Tests/CrashInDependencyTests/CrashInDependencyTests.swift new file mode 100644 index 00000000..287fb0d1 --- /dev/null +++ b/xcresult/tests/fixture-src/crash-in-dependency/Tests/CrashInDependencyTests/CrashInDependencyTests.swift @@ -0,0 +1,11 @@ +import XCTest + +import FixtureSupport + +final class CrashInDependencyTests: XCTestCase { + /// The process dies inside the dependency, so the failure summary Xcode + /// records has no file at all — not even a wrong one. + func testCrashesInsideDependency() { + crashInsideDependency() + } +} diff --git a/xcresult/tests/fixture-src/crash-in-dependency/Tests/CrashInDependencyTests/TeardownFailureTests.swift b/xcresult/tests/fixture-src/crash-in-dependency/Tests/CrashInDependencyTests/TeardownFailureTests.swift new file mode 100644 index 00000000..fded6ec8 --- /dev/null +++ b/xcresult/tests/fixture-src/crash-in-dependency/Tests/CrashInDependencyTests/TeardownFailureTests.swift @@ -0,0 +1,12 @@ +import Testing + +import FixtureSupport + +@Suite +struct TeardownFailureTests { + /// The test body succeeds and the dependency's trait fails it afterwards, so + /// the test's own frame is gone by the time the failure is recorded: every + /// file source is a dependency path and none of them may be reported. + @Test(.teardownFailure) + func failsAfterItsOwnFrameIsGone() {} +} diff --git a/xcresult/tests/fixture-src/dependency-raises-failure/Dependency/Package.swift b/xcresult/tests/fixture-src/dependency-raises-failure/Dependency/Package.swift new file mode 100644 index 00000000..4b45832a --- /dev/null +++ b/xcresult/tests/fixture-src/dependency-raises-failure/Dependency/Package.swift @@ -0,0 +1,13 @@ +// swift-tools-version: 6.0 +import PackageDescription + +let package = Package( + name: "FixtureSupport", + platforms: [.macOS(.v13)], + products: [ + .library(name: "FixtureSupport", targets: ["FixtureSupport"]) + ], + targets: [ + .target(name: "FixtureSupport") + ] +) diff --git a/xcresult/tests/fixture-src/dependency-raises-failure/Dependency/Sources/FixtureSupport/FixtureSupport.swift b/xcresult/tests/fixture-src/dependency-raises-failure/Dependency/Sources/FixtureSupport/FixtureSupport.swift new file mode 100644 index 00000000..e822ef7a --- /dev/null +++ b/xcresult/tests/fixture-src/dependency-raises-failure/Dependency/Sources/FixtureSupport/FixtureSupport.swift @@ -0,0 +1,19 @@ +import Testing + +/// Records a failure against whatever test is running, attributed to *this* file. +/// +/// `#filePath` in a function body is the file the function is defined in, so the +/// issue is raised from the dependency's source rather than from the caller's — +/// the same shape as a snapshot-testing assertion helper, a mocking framework, or +/// a page object that reports the failure at its own location. +public func recordIssueFromDependency(_ message: String) { + Issue.record( + Comment(rawValue: message), + sourceLocation: SourceLocation( + fileID: #fileID, + filePath: #filePath, + line: #line, + column: #column + ) + ) +} diff --git a/xcresult/tests/fixture-src/dependency-raises-failure/Package.swift b/xcresult/tests/fixture-src/dependency-raises-failure/Package.swift new file mode 100644 index 00000000..d5ea00fa --- /dev/null +++ b/xcresult/tests/fixture-src/dependency-raises-failure/Package.swift @@ -0,0 +1,20 @@ +// swift-tools-version: 6.0 +import PackageDescription + +// `Dependency` is referenced by git URL rather than by path so SPM checks it out +// into `DerivedData/SourcePackages/checkouts/Dependency` — the vendored location +// whose paths the fixture exists to reproduce. `regenerate.sh` turns the checked-in +// `Dependency` directory into a git repository before building. +let package = Package( + name: "DependencyRaisesFailure", + platforms: [.macOS(.v13)], + dependencies: [ + .package(url: "./Dependency", branch: "main") + ], + targets: [ + .testTarget( + name: "DependencyRaisesFailureTests", + dependencies: [.product(name: "FixtureSupport", package: "Dependency")] + ) + ] +) diff --git a/xcresult/tests/fixture-src/dependency-raises-failure/Tests/DependencyRaisesFailureTests/DependencyRaisesFailureTests.swift b/xcresult/tests/fixture-src/dependency-raises-failure/Tests/DependencyRaisesFailureTests/DependencyRaisesFailureTests.swift new file mode 100644 index 00000000..1966d563 --- /dev/null +++ b/xcresult/tests/fixture-src/dependency-raises-failure/Tests/DependencyRaisesFailureTests/DependencyRaisesFailureTests.swift @@ -0,0 +1,14 @@ +import Testing + +import FixtureSupport + +@Suite +struct DependencyRaisesFailureTests { + /// Every file the failure summary offers — `fileName`, the source code + /// context's location, and the innermost call-stack frame — points into the + /// dependency's checkout. Only the test's own call-stack frame names this file. + @Test + func failsInsideDependency() { + recordIssueFromDependency("recorded from the dependency's own source") + } +} diff --git a/xcresult/tests/fixture-src/dump-failure-summaries.py b/xcresult/tests/fixture-src/dump-failure-summaries.py new file mode 100755 index 00000000..64542400 --- /dev/null +++ b/xcresult/tests/fixture-src/dump-failure-summaries.py @@ -0,0 +1,148 @@ +#!/usr/bin/env python3 +"""Dump the per-test failure summaries out of an .xcresult bundle. + +`xcrun xcresulttool get object --legacy` returns one object at a time and the +failure summaries hang several references deep, so walking to them takes a +handful of calls. This prints a JSON array — one entry per test that has +failure summaries — with the pieces the Rust attribution logic reads: +`fileName`, `sourceCodeContext.location.filePath`, and the call stack's +symbol names and file paths. + + ./dump-failure-summaries.py .xcresult > .failure-summaries.json +""" + +import json +import subprocess +import sys + + +def get_object(path, object_id=None): + cmd = [ + "xcrun", + "xcresulttool", + "get", + "object", + "--path", + path, + "--format", + "json", + "--legacy", + ] + if object_id: + cmd += ["--id", object_id] + return json.loads(subprocess.run(cmd, check=True, capture_output=True).stdout) + + +def value(node): + """Unwrap the legacy schema's `{"_value": ...}` boxes.""" + if isinstance(node, dict): + return node.get("_value") + return None + + +def values(node): + if isinstance(node, dict): + return node.get("_values", []) + return [] + + +def walk_tests(node, ancestors): + """Yield every (name-path, summaryRef id, status) leaf under a test node.""" + name = value(node.get("name")) + subtests = node.get("subtests") + if subtests is not None: + for subtest in values(subtests): + yield from walk_tests(subtest, ancestors + [name]) + return + summary_ref = node.get("summaryRef") + if summary_ref is not None: + yield ( + ancestors + [name], + value(summary_ref.get("id")), + value(node.get("testStatus")), + value(node.get("identifier")), + ) + + +def source_code_context(context): + if context is None: + return None + location = context.get("location") or {} + return { + "location.filePath": value(location.get("filePath")), + "location.lineNumber": value(location.get("lineNumber")), + "callStack": [ + { + "symbolName": value((frame.get("symbolInfo") or {}).get("symbolName")), + "filePath": value( + (((frame.get("symbolInfo") or {}).get("location")) or {}).get( + "filePath" + ) + ), + "imageName": value((frame.get("symbolInfo") or {}).get("imageName")), + } + for frame in values(context.get("callStack")) + ], + } + + +def main(): + path = sys.argv[1] + root = get_object(path) + + out = [] + for action in values(root.get("actions")): + action_result = action.get("actionResult") or {} + + for issue in values( + (action_result.get("issues") or {}).get("testFailureSummaries") + ): + document_location = issue.get("documentLocationInCreatingWorkspace") or {} + out.append( + { + "kind": "actionResult.issues.testFailureSummaries", + "testCaseName": value(issue.get("testCaseName")), + "producingTarget": value(issue.get("producingTarget")), + "documentLocationInCreatingWorkspace.url": value( + document_location.get("url") + ), + } + ) + + tests_ref = action_result.get("testsRef") + if tests_ref is None: + continue + plan = get_object(path, value(tests_ref.get("id"))) + for plan_summary in values(plan.get("summaries")): + for testable in values(plan_summary.get("testableSummaries")): + for test in values(testable.get("tests")): + for names, summary_id, status, identifier in walk_tests(test, []): + if status == "Success": + continue + summary = get_object(path, summary_id) + for failure in values(summary.get("failureSummaries")): + out.append( + { + "kind": "test.failureSummaries", + "testBundle": value(testable.get("name")), + "namePath": names, + "identifier": identifier, + "testStatus": status, + "message": value(failure.get("message")), + "fileName": value(failure.get("fileName")), + "lineNumber": value(failure.get("lineNumber")), + "isPerformanceFailure": value( + failure.get("isPerformanceFailure") + ), + "sourceCodeContext": source_code_context( + failure.get("sourceCodeContext") + ), + } + ) + + json.dump(out, sys.stdout, indent=2) + sys.stdout.write("\n") + + +if __name__ == "__main__": + main() diff --git a/xcresult/tests/fixture-src/in-repo-helper-raises-failure/Package.swift b/xcresult/tests/fixture-src/in-repo-helper-raises-failure/Package.swift new file mode 100644 index 00000000..4ab83ad3 --- /dev/null +++ b/xcresult/tests/fixture-src/in-repo-helper-raises-failure/Package.swift @@ -0,0 +1,10 @@ +// swift-tools-version: 6.0 +import PackageDescription + +let package = Package( + name: "InRepoHelperRaisesFailure", + platforms: [.macOS(.v13)], + targets: [ + .testTarget(name: "InRepoHelperRaisesFailureTests") + ] +) diff --git a/xcresult/tests/fixture-src/in-repo-helper-raises-failure/Tests/InRepoHelperRaisesFailureTests/FailureHelper.swift b/xcresult/tests/fixture-src/in-repo-helper-raises-failure/Tests/InRepoHelperRaisesFailureTests/FailureHelper.swift new file mode 100644 index 00000000..7dbd11b4 --- /dev/null +++ b/xcresult/tests/fixture-src/in-repo-helper-raises-failure/Tests/InRepoHelperRaisesFailureTests/FailureHelper.swift @@ -0,0 +1,15 @@ +import Testing + +/// The same shape as the dependency helper, but living in the test target itself: +/// the issue is raised at this file, not at the caller's. +func recordIssueFromHelper(_ message: String) { + Issue.record( + Comment(rawValue: message), + sourceLocation: SourceLocation( + fileID: #fileID, + filePath: #filePath, + line: #line, + column: #column + ) + ) +} diff --git a/xcresult/tests/fixture-src/in-repo-helper-raises-failure/Tests/InRepoHelperRaisesFailureTests/InRepoHelperRaisesFailureTests.swift b/xcresult/tests/fixture-src/in-repo-helper-raises-failure/Tests/InRepoHelperRaisesFailureTests/InRepoHelperRaisesFailureTests.swift new file mode 100644 index 00000000..b0371891 --- /dev/null +++ b/xcresult/tests/fixture-src/in-repo-helper-raises-failure/Tests/InRepoHelperRaisesFailureTests/InRepoHelperRaisesFailureTests.swift @@ -0,0 +1,11 @@ +import Testing + +@Suite +struct InRepoHelperRaisesFailureTests { + /// The helper's file is in the repo, so nothing rejects it — the test's own + /// call-stack frame has to win on its own merits for this file to be reported. + @Test + func failsInsideHelper() { + recordIssueFromHelper("recorded from a helper in the test target") + } +} diff --git a/xcresult/tests/fixture-src/objc-xctest/Package.swift b/xcresult/tests/fixture-src/objc-xctest/Package.swift new file mode 100644 index 00000000..c41d7dff --- /dev/null +++ b/xcresult/tests/fixture-src/objc-xctest/Package.swift @@ -0,0 +1,10 @@ +// swift-tools-version: 6.0 +import PackageDescription + +let package = Package( + name: "ObjcXCTest", + platforms: [.macOS(.v13)], + targets: [ + .testTarget(name: "ObjcXCTestTests") + ] +) diff --git a/xcresult/tests/fixture-src/objc-xctest/Tests/ObjcXCTestTests/FailureHelper.m b/xcresult/tests/fixture-src/objc-xctest/Tests/ObjcXCTestTests/FailureHelper.m new file mode 100644 index 00000000..c53e5ab4 --- /dev/null +++ b/xcresult/tests/fixture-src/objc-xctest/Tests/ObjcXCTestTests/FailureHelper.m @@ -0,0 +1,11 @@ +#import "FailureHelper.h" + +@implementation XCTestCase (FixtureFailureHelper) + +// `XCTFail` expands with this file's `__FILE__`, so the failure is raised here +// rather than at the call site. +- (void)fixtureFailWithMessage:(NSString *)message { + XCTFail(@"%@", message); +} + +@end diff --git a/xcresult/tests/fixture-src/objc-xctest/Tests/ObjcXCTestTests/ObjcXCTestTests.m b/xcresult/tests/fixture-src/objc-xctest/Tests/ObjcXCTestTests/ObjcXCTestTests.m new file mode 100644 index 00000000..67f39841 --- /dev/null +++ b/xcresult/tests/fixture-src/objc-xctest/Tests/ObjcXCTestTests/ObjcXCTestTests.m @@ -0,0 +1,14 @@ +#import "FailureHelper.h" + +@interface ObjcXCTestTests : XCTestCase +@end + +@implementation ObjcXCTestTests + +// Symbolicates as `-[ObjcXCTestTests testFailsInsideSharedHelper]`, which is the +// Objective-C spelling the frame matching has to recognize. +- (void)testFailsInsideSharedHelper { + [self fixtureFailWithMessage:@"raised from the shared Objective-C helper"]; +} + +@end diff --git a/xcresult/tests/fixture-src/objc-xctest/Tests/ObjcXCTestTests/include/FailureHelper.h b/xcresult/tests/fixture-src/objc-xctest/Tests/ObjcXCTestTests/include/FailureHelper.h new file mode 100644 index 00000000..04ec2c47 --- /dev/null +++ b/xcresult/tests/fixture-src/objc-xctest/Tests/ObjcXCTestTests/include/FailureHelper.h @@ -0,0 +1,5 @@ +#import + +@interface XCTestCase (FixtureFailureHelper) +- (void)fixtureFailWithMessage:(NSString *)message; +@end diff --git a/xcresult/tests/fixture-src/prune-bundle.py b/xcresult/tests/fixture-src/prune-bundle.py new file mode 100755 index 00000000..d7c2d197 --- /dev/null +++ b/xcresult/tests/fixture-src/prune-bundle.py @@ -0,0 +1,136 @@ +#!/usr/bin/env python3 +"""Drop the objects in an .xcresult bundle that nothing in it references. + +Xcode 26 writes ~140MB of dyld shared-cache symbolication data plus a payload per +loaded image into every result bundle, none of it reachable from the invocation +record. That is fine for a bundle you throw away and not fine for one checked into +git — it is ~95MB of the ~96MB a scenario produces. + +This walks the object graph from the root, following every reference id, and +deletes the `Data/` entries nothing reached. `regenerate.sh` re-dumps the failure +summaries afterwards and fails if pruning changed them. + + ./prune-bundle.py .xcresult +""" + +import json +import os +import re +import subprocess +import sys + +# Object ids are content-addressed and are also the `Data/` file names, e.g. +# `data.0~aBc.../refs.0~aBc...`. +OBJECT_ID = re.compile(r"^0~[A-Za-z0-9_\-+/=]+$") + + +def get_object(path, object_id=None): + cmd = [ + "xcrun", + "xcresulttool", + "get", + "object", + "--path", + path, + "--format", + "json", + "--legacy", + ] + if object_id: + cmd += ["--id", object_id] + result = subprocess.run(cmd, capture_output=True) + if result.returncode != 0: + return None + try: + return json.loads(result.stdout) + except json.JSONDecodeError: + return None + + +def object_ids_in(node): + """Every value anywhere in an object that is shaped like an object id.""" + if isinstance(node, dict): + for key, child in node.items(): + if key == "_value" and isinstance(child, str) and OBJECT_ID.match(child): + yield child + else: + yield from object_ids_in(child) + elif isinstance(node, list): + for child in node: + yield from object_ids_in(child) + + +def root_id(path): + """The invocation record's own id, which lives in Info.plist rather than in + any object, so nothing in the graph points at it.""" + # Info.plist holds a date, which `plutil -convert json` refuses to write, so + # the one field is extracted rather than the whole file converted. + info = subprocess.run( + [ + "plutil", + "-extract", + "rootId.hash", + "raw", + "-o", + "-", + os.path.join(path, "Info.plist"), + ], + check=True, + capture_output=True, + ) + return info.stdout.decode().strip() + + +def reachable_ids(path): + root = get_object(path) + if root is None: + raise SystemExit(f"{path}: could not read the invocation record") + + seen = {root_id(path)} + queue = list(object_ids_in(root)) + while queue: + object_id = queue.pop() + if object_id in seen: + continue + seen.add(object_id) + child = get_object(path, object_id) + if child is not None: + queue.extend(object_ids_in(child)) + return seen + + +def main(): + path = sys.argv[1] + data_dir = os.path.join(path, "Data") + + keep = reachable_ids(path) + before = sum( + os.path.getsize(os.path.join(data_dir, name)) for name in os.listdir(data_dir) + ) + + removed = 0 + for name in os.listdir(data_dir): + prefix, _, object_id = name.partition(".") + if prefix not in ("data", "refs"): + continue + if object_id in keep: + continue + file_path = os.path.join(data_dir, name) + removed += os.path.getsize(file_path) + os.remove(file_path) + + # Built lazily by xcresulttool and rebuilt on demand, so it is noise in a + # checked-in fixture. + index = os.path.join(path, "database.sqlite3") + if os.path.exists(index): + os.remove(index) + + print( + f" pruned {removed / 1e6:.0f}MB of unreferenced objects " + f"({before / 1e6:.0f}MB -> {(before - removed) / 1e6:.0f}MB), " + f"kept {len(keep)}" + ) + + +if __name__ == "__main__": + main() diff --git a/xcresult/tests/fixture-src/regenerate.sh b/xcresult/tests/fixture-src/regenerate.sh new file mode 100755 index 00000000..b7a24583 --- /dev/null +++ b/xcresult/tests/fixture-src/regenerate.sh @@ -0,0 +1,141 @@ +#!/usr/bin/env bash +# +# Capture the .xcresult fixtures in `xcresult/tests/data/` from the scenario +# packages next to this script. Requires macOS and Xcode. +# +# ./regenerate.sh # every scenario +# ./regenerate.sh objc-xctest # just one +# FIXTURE_WORK_DIR=/tmp/elsewhere ./regenerate.sh +# +# Absolute paths are baked into a bundle at capture time and end up in the +# expected JUnit XML, so the scenarios are copied to a fixed working directory +# (`/tmp/xcresult-fixtures` by default) and built there. Regenerating from a +# different directory rewrites every path in the expected output — see README.md. + +set -euo pipefail + +FIXTURE_SRC_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +DATA_DIR="$(cd "${FIXTURE_SRC_DIR}/../data" && pwd)" +WORK_DIR="${FIXTURE_WORK_DIR:-/tmp/xcresult-fixtures}" + +ALL_SCENARIOS=( + dependency-raises-failure + in-repo-helper-raises-failure + crash-in-dependency + objc-xctest + toplevel-swift-testing +) + +# scenario -> the package name, which is both the xcodebuild scheme prefix and the +# name of the captured bundle. +package_name() { + case "$1" in + dependency-raises-failure) echo DependencyRaisesFailure ;; + in-repo-helper-raises-failure) echo InRepoHelperRaisesFailure ;; + crash-in-dependency) echo CrashInDependency ;; + objc-xctest) echo ObjcXCTest ;; + toplevel-swift-testing) echo ToplevelSwiftTesting ;; + *) + echo "unknown scenario: $1" >&2 + exit 1 + ;; + esac +} + +regenerate() { + local scenario="$1" + local package + package="$(package_name "${scenario}")" + local scenario_work_dir="${WORK_DIR}/${scenario}" + local bundle="${scenario_work_dir}/${package}.xcresult" + + echo "==> ${scenario}" + + rm -rf "${scenario_work_dir}" + mkdir -p "${WORK_DIR}" + cp -R "${FIXTURE_SRC_DIR}/${scenario}" "${scenario_work_dir}" + + # A `.package(url: "./Dependency", ...)` is only checked out into + # `SourcePackages/checkouts` if it is a git repository, and the fixture is + # pointless unless it is: that checkout path is the whole shape being + # reproduced. The repository is created in the working copy so the checked-in + # sources stay a plain directory. + if [[ -d "${scenario_work_dir}/Dependency" ]]; then + git -C "${scenario_work_dir}/Dependency" init -q + git -C "${scenario_work_dir}/Dependency" add -A + git -C "${scenario_work_dir}/Dependency" \ + -c user.email=fixtures@trunk.io -c user.name=fixtures \ + commit -qm "xcresult fixture dependency" + git -C "${scenario_work_dir}/Dependency" branch -qM main + fi + + # Tests are meant to fail here, so xcodebuild exits nonzero on a good run. + ( + cd "${scenario_work_dir}" + xcodebuild test \ + -scheme "${package}-Package" \ + -destination 'platform=macOS' \ + -derivedDataPath DerivedData \ + -resultBundlePath "${package}.xcresult" \ + >xcodebuild.log 2>&1 + ) || true + + if [[ ! -d ${bundle} ]]; then + echo " no bundle at ${bundle}; see ${scenario_work_dir}/xcodebuild.log" >&2 + return 1 + fi + + # Xcode 26 writes ~95MB of unreferenced symbolication data into every bundle, + # so pruning is what makes these checkable into git at all. Both xcresulttool + # APIs the crate calls are compared before and after to prove the prune is + # invisible to consumers. + local before_legacy="${scenario_work_dir}/before-legacy.json" + local before_modern="${scenario_work_dir}/before-modern.json" + "${FIXTURE_SRC_DIR}/dump-failure-summaries.py" "${bundle}" >"${before_legacy}" + xcrun xcresulttool get test-results tests --path "${bundle}" --format json \ + >"${before_modern}" + + "${FIXTURE_SRC_DIR}/prune-bundle.py" "${bundle}" + + # Derived from the bundle, so it is written to the working directory rather + # than checked in; `dump-failure-summaries.py` regenerates it on demand. + local dump="${scenario_work_dir}/${scenario}.failure-summaries.json" + "${FIXTURE_SRC_DIR}/dump-failure-summaries.py" "${bundle}" >"${dump}" + diff -q "${before_legacy}" "${dump}" >/dev/null || + { + echo " pruning changed the legacy failure summaries" >&2 + return 1 + } + xcrun xcresulttool get test-results tests --path "${bundle}" --format json | + diff -q "${before_modern}" - >/dev/null || + { + echo " pruning changed the test-results output" >&2 + return 1 + } + + "${FIXTURE_SRC_DIR}/verify-failure-summaries.py" "${scenario}" "${dump}" + + tar -czf "${DATA_DIR}/test-${scenario}.xcresult.tar.gz" \ + -C "${scenario_work_dir}" "${package}.xcresult" + echo " wrote ${DATA_DIR}/test-${scenario}.xcresult.tar.gz" +} + +scenarios=("${@-}") +if [[ -z ${scenarios[0]} ]]; then + scenarios=("${ALL_SCENARIOS[@]}") +fi + +for scenario in "${scenarios[@]}"; do + regenerate "${scenario}" +done + +cat <<'EOF' + +Bundles regenerated. The expected JUnit XML in `xcresult/tests/data/` still has +the old absolute paths baked in; update it with + + cargo test -p xcresult + +and check every `file` attribute by hand before saving the new output — see +README.md for what each scenario must report. +EOF diff --git a/xcresult/tests/fixture-src/toplevel-swift-testing/Package.swift b/xcresult/tests/fixture-src/toplevel-swift-testing/Package.swift new file mode 100644 index 00000000..d5eca214 --- /dev/null +++ b/xcresult/tests/fixture-src/toplevel-swift-testing/Package.swift @@ -0,0 +1,10 @@ +// swift-tools-version: 6.0 +import PackageDescription + +let package = Package( + name: "ToplevelSwiftTesting", + platforms: [.macOS(.v13)], + targets: [ + .testTarget(name: "ToplevelSwiftTestingTests") + ] +) diff --git a/xcresult/tests/fixture-src/toplevel-swift-testing/Tests/ToplevelSwiftTestingTests/FailureHelper.swift b/xcresult/tests/fixture-src/toplevel-swift-testing/Tests/ToplevelSwiftTestingTests/FailureHelper.swift new file mode 100644 index 00000000..8b85ffca --- /dev/null +++ b/xcresult/tests/fixture-src/toplevel-swift-testing/Tests/ToplevelSwiftTestingTests/FailureHelper.swift @@ -0,0 +1,13 @@ +import Testing + +func recordIssueFromHelper(_ message: String) { + Issue.record( + Comment(rawValue: message), + sourceLocation: SourceLocation( + fileID: #fileID, + filePath: #filePath, + line: #line, + column: #column + ) + ) +} diff --git a/xcresult/tests/fixture-src/toplevel-swift-testing/Tests/ToplevelSwiftTestingTests/ToplevelSwiftTestingTests.swift b/xcresult/tests/fixture-src/toplevel-swift-testing/Tests/ToplevelSwiftTestingTests/ToplevelSwiftTestingTests.swift new file mode 100644 index 00000000..8bf2224c --- /dev/null +++ b/xcresult/tests/fixture-src/toplevel-swift-testing/Tests/ToplevelSwiftTestingTests/ToplevelSwiftTestingTests.swift @@ -0,0 +1,8 @@ +import Testing + +/// Declared at the top level rather than in a `@Suite`, so it symbolicates as the +/// bare function `failsInsideHelperWithoutASuite()` with no suite to qualify it. +@Test +func failsInsideHelperWithoutASuite() { + recordIssueFromHelper("raised from the in-repo helper") +} diff --git a/xcresult/tests/fixture-src/verify-failure-summaries.py b/xcresult/tests/fixture-src/verify-failure-summaries.py new file mode 100755 index 00000000..45ade9d0 --- /dev/null +++ b/xcresult/tests/fixture-src/verify-failure-summaries.py @@ -0,0 +1,195 @@ +#!/usr/bin/env python3 +"""Assert that a captured .xcresult actually exhibits the shape it was captured for. + +A fixture that no longer reproduces its shape is worse than no fixture — it keeps +passing while guarding nothing (which is what happened to the older +`test-swift-snapshot-testing` bundle, whose `fileName` points at the test's own +source). `regenerate.sh` runs this before packaging anything, and refuses to +package a bundle that fails. + + ./verify-failure-summaries.py .failure-summaries.json +""" + +import json +import sys + +# Directory segments that mark vendored dependency sources. Kept in sync with +# `DEPENDENCY_PATH_SEGMENTS` in `xcresult/src/xcresult_legacy.rs`. +DEPENDENCY_PATH_SEGMENTS = ["/.build/", "/checkouts/", "/DerivedData/"] + +# One entry per failure the scenario must produce, keyed by the test's identifier. +# +# raised_from: where `fileName` / the source code context's location must point +# - "dependency": a vendored path, which is what makes the fixture a regression +# test — every non-call-stack source is unusable +# - "in_repo_helper": a real path that is *not* the test's own file +# - "absent": the failure carries no file at all +# test_frame_symbol: the call-stack symbol that names the test itself, which is +# the only thing that can identify the test's file. `None` means the stack must +# *not* reach the test — the crash and teardown shapes, where no file may be +# reported at all. +# other_frames: whether the stack must also contain frames that are not the +# test's, i.e. whether picking the last frame instead of the test's would land +# somewhere else. +SCENARIOS = { + "dependency-raises-failure": [ + { + "identifier": "DependencyRaisesFailureTests/failsInsideDependency()", + "raised_from": "dependency", + "test_frame_symbol": "DependencyRaisesFailureTests.failsInsideDependency()", + "other_frames": True, + }, + ], + "in-repo-helper-raises-failure": [ + { + "identifier": "InRepoHelperRaisesFailureTests/failsInsideHelper()", + "raised_from": "in_repo_helper", + "test_frame_symbol": "InRepoHelperRaisesFailureTests.failsInsideHelper()", + "other_frames": True, + }, + ], + "crash-in-dependency": [ + { + "identifier": "CrashInDependencyTests/testCrashesInsideDependency()", + "raised_from": "absent", + "test_frame_symbol": None, + "other_frames": False, + }, + { + "identifier": "TeardownFailureTests/failsAfterItsOwnFrameIsGone()", + "raised_from": "dependency", + "test_frame_symbol": None, + "other_frames": True, + }, + ], + "objc-xctest": [ + { + "identifier": "ObjcXCTestTests/testFailsInsideSharedHelper", + "raised_from": "in_repo_helper", + "test_frame_symbol": "-[ObjcXCTestTests testFailsInsideSharedHelper]", + "other_frames": True, + }, + ], + "toplevel-swift-testing": [ + { + "identifier": "failsInsideHelperWithoutASuite()", + "raised_from": "in_repo_helper", + "test_frame_symbol": "failsInsideHelperWithoutASuite()", + "other_frames": True, + }, + ], +} + + +def is_dependency_path(path): + return path is not None and any(seg in path for seg in DEPENDENCY_PATH_SEGMENTS) + + +class Failures(list): + def check(self, condition, message): + if not condition: + self.append(message) + + +def check_summary(failures, expectation, summary): + identifier = expectation["identifier"] + context = summary["sourceCodeContext"] or {} + frames = context.get("callStack", []) + raised_from = [summary["fileName"], context.get("location.filePath")] + named = [ + frame + for frame in frames + if expectation["test_frame_symbol"] is not None + and frame["symbolName"] == expectation["test_frame_symbol"] + ] + + if expectation["raised_from"] == "dependency": + for source in raised_from: + failures.check( + is_dependency_path(source), + f"{identifier}: expected the failure to be raised from a dependency " + f"path, got {source!r}", + ) + elif expectation["raised_from"] == "absent": + for source in raised_from: + failures.check( + source is None, + f"{identifier}: expected no file source at all, got {source!r}", + ) + else: + for source in raised_from: + failures.check( + source is not None and not is_dependency_path(source), + f"{identifier}: expected the failure to be raised from an in-repo " + f"helper, got {source!r}", + ) + + if expectation["test_frame_symbol"] is None: + failures.check( + not any( + frame["filePath"] and not is_dependency_path(frame["filePath"]) + for frame in frames + ), + f"{identifier}: expected the stack never to reach the test, but it has " + f"a frame outside the dependency", + ) + else: + failures.check( + len(named) == 1, + f"{identifier}: expected exactly one frame named " + f"{expectation['test_frame_symbol']!r}, found {len(named)}", + ) + for frame in named: + failures.check( + frame["filePath"] is not None + and not is_dependency_path(frame["filePath"]), + f"{identifier}: the test's own frame must carry a non-dependency " + f"file path, got {frame['filePath']!r}", + ) + failures.check( + frame["filePath"] not in raised_from, + f"{identifier}: the test's own frame points at the same file the " + f"failure was raised from ({frame['filePath']!r}), so the fixture " + f"would pass without the fix", + ) + + if expectation["other_frames"]: + symbolicated = [frame for frame in frames if frame["symbolName"]] + failures.check( + len(symbolicated) > len(named), + f"{identifier}: expected at least one frame besides the test's own", + ) + + +def main(): + scenario, dump_path = sys.argv[1], sys.argv[2] + expectations = SCENARIOS[scenario] + dump = json.load(open(dump_path)) + summaries = { + entry["identifier"]: entry + for entry in dump + if entry["kind"] == "test.failureSummaries" + } + + failures = Failures() + for expectation in expectations: + summary = summaries.get(expectation["identifier"]) + if summary is None: + failures.append( + f"{expectation['identifier']}: no failure summary in the bundle " + f"(found {sorted(summaries)})" + ) + continue + check_summary(failures, expectation, summary) + + if failures: + print(f"{scenario}: bundle does not exhibit its shape", file=sys.stderr) + for failure in failures: + print(f" - {failure}", file=sys.stderr) + sys.exit(1) + + print(f"{scenario}: verified {len(expectations)} failure summary shape(s)") + + +if __name__ == "__main__": + main() diff --git a/xcresult/tests/xcresult.rs b/xcresult/tests/xcresult.rs index 7514ed83..35397b5d 100644 --- a/xcresult/tests/xcresult.rs +++ b/xcresult/tests/xcresult.rs @@ -34,6 +34,16 @@ lazy_static! { unpack_archive_to_temp_dir("tests/data/test-swift-mix.xcresult.tar.gz"); static ref TEMP_DIR_TEST_SWIFT_SNAPSHOT_TESTING: TempDir = unpack_archive_to_temp_dir("tests/data/test-swift-snapshot-testing.xcresult.tar.gz"); + static ref TEMP_DIR_TEST_DEPENDENCY_RAISES_FAILURE: TempDir = + unpack_archive_to_temp_dir("tests/data/test-dependency-raises-failure.xcresult.tar.gz"); + static ref TEMP_DIR_TEST_IN_REPO_HELPER_RAISES_FAILURE: TempDir = + unpack_archive_to_temp_dir("tests/data/test-in-repo-helper-raises-failure.xcresult.tar.gz"); + static ref TEMP_DIR_TEST_CRASH_IN_DEPENDENCY: TempDir = + unpack_archive_to_temp_dir("tests/data/test-crash-in-dependency.xcresult.tar.gz"); + static ref TEMP_DIR_TEST_OBJC_XCTEST: TempDir = + unpack_archive_to_temp_dir("tests/data/test-objc-xctest.xcresult.tar.gz"); + static ref TEMP_DIR_TEST_TOPLEVEL_SWIFT_TESTING: TempDir = + unpack_archive_to_temp_dir("tests/data/test-toplevel-swift-testing.xcresult.tar.gz"); static ref TEMP_DIR_TEST_TIMESTAMP: TempDir = unpack_archive_to_temp_dir("tests/data/test-timestamp.xcresult.tar.gz"); static ref TEMP_DIR_TEST_VARIANT: TempDir = @@ -245,6 +255,162 @@ fn test_swift_snapshot_testing_trait_failure_uses_assertion_file( ); } +// Real bundles for the file sources in `xcresult::file_attribution`, one per shape +// that used to attribute a failed test to a vendored dependency. Each test below +// names the source it exercises; `tests/fixture-src/README.md` covers what each +// bundle must exhibit and how to regenerate it. +// +// The two cases expect different JUnit because they have different sources +// available: only the experimental path reads the per-test failure summary, and so +// the call stack, so it is the only one that can produce a `FileSource::TestFrame`. +// The legacy path sees `FileSource::DocumentLocation` alone. +#[cfg(target_os = "macos")] +fn assert_junit>( + bundle_path: T, + use_experimental_failure_summary: bool, + expected_junit_xml: &str, +) { + let path_str = bundle_path.as_ref().to_str().unwrap(); + let xcresult = XCResult::new( + path_str, + ORG_URL_SLUG.clone(), + REPO_FULL_NAME.clone(), + use_experimental_failure_summary, + ); + assert!(xcresult.is_ok()); + + let mut junits = xcresult.unwrap().generate_junits(); + assert_eq!(junits.len(), 1); + let junit = junits.pop().unwrap(); + let mut junit_writer: Vec = Vec::new(); + junit.serialize(&mut junit_writer).unwrap(); + pretty_assertions::assert_eq!(String::from_utf8(junit_writer).unwrap(), expected_junit_xml); +} + +// `FileSource::TestFrame` is the only usable source. The failure is recorded inside +// the dependency, so `RaisedFrom`, `SourceCodeLocation` and the innermost +// `LastStackFrame` all point into `DerivedData/SourcePackages/checkouts/` and are +// rejected as vendored. The legacy path has only `DocumentLocation`, which points +// there too, so it reports nothing at all. +#[cfg(target_os = "macos")] +#[rstest] +#[case::experimental_failure_summary( + true, + include_str!("data/test-dependency-raises-failure.junit.xml") +)] +#[case::legacy_fallback( + false, + include_str!("data/test-dependency-raises-failure.legacy.junit.xml") +)] +fn test_dependency_raised_failure_uses_the_tests_own_file( + #[case] use_experimental_failure_summary: bool, + #[case] expected_junit_xml: &str, +) { + assert_junit( + TEMP_DIR_TEST_DEPENDENCY_RAISES_FAILURE + .as_ref() + .join("DependencyRaisesFailure.xcresult"), + use_experimental_failure_summary, + expected_junit_xml, + ); +} + +// `FileSource::TestFrame` versus `RaisedFrom` with nothing to separate them by path. +// The helper is in the test target, so `is_vendored_dependency` says nothing useful +// and the ordering of the sources is what decides. The legacy path, having only +// `DocumentLocation`, still lands on the helper. +#[cfg(target_os = "macos")] +#[rstest] +#[case::experimental_failure_summary( + true, + include_str!("data/test-in-repo-helper-raises-failure.junit.xml") +)] +#[case::legacy_fallback( + false, + include_str!("data/test-in-repo-helper-raises-failure.legacy.junit.xml") +)] +fn test_in_repo_helper_raised_failure_uses_the_tests_own_file( + #[case] use_experimental_failure_summary: bool, + #[case] expected_junit_xml: &str, +) { + assert_junit( + TEMP_DIR_TEST_IN_REPO_HELPER_RAISES_FAILURE + .as_ref() + .join("InRepoHelperRaisesFailure.xcresult"), + use_experimental_failure_summary, + expected_junit_xml, + ); +} + +// No source survives vetting. Neither test reaches its own frame — one crashes +// inside the dependency, the other is failed by the dependency's trait after its +// body returned — so there is no `TestFrame`, and every remaining candidate is +// either absent or vendored. Both cases must come out with no `file` attribute at +// all rather than the dependency's. +#[cfg(target_os = "macos")] +#[rstest] +#[case::experimental_failure_summary(true, include_str!("data/test-crash-in-dependency.junit.xml"))] +#[case::legacy_fallback(false, include_str!("data/test-crash-in-dependency.junit.xml"))] +fn test_crash_in_dependency_reports_no_file( + #[case] use_experimental_failure_summary: bool, + #[case] expected_junit_xml: &str, +) { + assert_junit( + TEMP_DIR_TEST_CRASH_IN_DEPENDENCY + .as_ref() + .join("CrashInDependency.xcresult"), + use_experimental_failure_summary, + expected_junit_xml, + ); +} + +// `TestIdentity::is_named_by` against real symbolication: Xcode spells the frame +// `-[ObjcXCTestTests testFailsInsideSharedHelper]`, not `Suite.testCase()`. The +// legacy path reports nothing here for an unrelated reason — the `DocumentLocation` +// candidates are keyed by test case name, and Xcode's Objective-C spelling never +// matches the `Suite.testCase` key the lookup builds. +#[cfg(target_os = "macos")] +#[rstest] +#[case::experimental_failure_summary(true, include_str!("data/test-objc-xctest.junit.xml"))] +#[case::legacy_fallback(false, include_str!("data/test-objc-xctest.legacy.junit.xml"))] +fn test_objc_xctest_helper_failure_uses_the_tests_own_file( + #[case] use_experimental_failure_summary: bool, + #[case] expected_junit_xml: &str, +) { + assert_junit( + TEMP_DIR_TEST_OBJC_XCTEST + .as_ref() + .join("ObjcXCTest.xcresult"), + use_experimental_failure_summary, + expected_junit_xml, + ); +} + +// `TestIdentity` with no suite: a top-level swift-testing `@Test func` symbolicates +// as the bare function, so there is nothing to qualify the match with. +#[cfg(target_os = "macos")] +#[rstest] +#[case::experimental_failure_summary( + true, + include_str!("data/test-toplevel-swift-testing.junit.xml") +)] +#[case::legacy_fallback( + false, + include_str!("data/test-toplevel-swift-testing.legacy.junit.xml") +)] +fn test_toplevel_swift_testing_helper_failure_uses_the_tests_own_file( + #[case] use_experimental_failure_summary: bool, + #[case] expected_junit_xml: &str, +) { + assert_junit( + TEMP_DIR_TEST_TOPLEVEL_SWIFT_TESTING + .as_ref() + .join("ToplevelSwiftTesting.xcresult"), + use_experimental_failure_summary, + expected_junit_xml, + ); +} + #[cfg(target_os = "macos")] #[test] fn test_expected_failures_xcresult_with_valid_path() { diff --git a/xcresult/xcrun-xcresulttool-formatDescription-get---format-json---legacy-json-schema.json b/xcresult/xcrun-xcresulttool-formatDescription-get---format-json---legacy-json-schema.json index e5c7d35c..27f32a76 100644 --- a/xcresult/xcrun-xcresulttool-formatDescription-get---format-json---legacy-json-schema.json +++ b/xcresult/xcrun-xcresulttool-formatDescription-get---format-json---legacy-json-schema.json @@ -1346,7 +1346,7 @@ "$ref": "#/$defs/Int" } }, - "additionalProperties": false + "additionalProperties": true }, "ActivityLogMajorSection": { "type": "object", @@ -1584,7 +1584,7 @@ "$ref": "#/$defs/String" } }, - "additionalProperties": false + "additionalProperties": true }, "ActivityLogUnitTestSection": { "type": "object", @@ -1668,7 +1668,7 @@ "$ref": "#/$defs/String" } }, - "additionalProperties": false + "additionalProperties": true }, "ArchiveInfo": { "type": "object", @@ -2532,7 +2532,7 @@ "$ref": "#/$defs/String" } }, - "additionalProperties": false + "additionalProperties": true }, "SourceCodeContext": { "type": "object",