diff --git a/xcresult/tests/data/test-crash-in-dependency.junit.xml b/xcresult/tests/data/test-crash-in-dependency.junit.xml new file mode 100644 index 00000000..97db712b --- /dev/null +++ b/xcresult/tests/data/test-crash-in-dependency.junit.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + diff --git a/xcresult/tests/data/test-crash-in-dependency.xcresult.tar.gz b/xcresult/tests/data/test-crash-in-dependency.xcresult.tar.gz new file mode 100644 index 00000000..1acc63c6 Binary files /dev/null and b/xcresult/tests/data/test-crash-in-dependency.xcresult.tar.gz differ diff --git a/xcresult/tests/data/test-dependency-raises-failure.junit.xml b/xcresult/tests/data/test-dependency-raises-failure.junit.xml new file mode 100644 index 00000000..367a44c8 --- /dev/null +++ b/xcresult/tests/data/test-dependency-raises-failure.junit.xml @@ -0,0 +1,8 @@ + + + + + + + + diff --git a/xcresult/tests/data/test-dependency-raises-failure.legacy.junit.xml b/xcresult/tests/data/test-dependency-raises-failure.legacy.junit.xml new file mode 100644 index 00000000..76c97bc8 --- /dev/null +++ b/xcresult/tests/data/test-dependency-raises-failure.legacy.junit.xml @@ -0,0 +1,8 @@ + + + + + + + + diff --git a/xcresult/tests/data/test-dependency-raises-failure.xcresult.tar.gz b/xcresult/tests/data/test-dependency-raises-failure.xcresult.tar.gz new file mode 100644 index 00000000..eeb7fc9e Binary files /dev/null and b/xcresult/tests/data/test-dependency-raises-failure.xcresult.tar.gz differ diff --git a/xcresult/tests/data/test-in-repo-helper-raises-failure.junit.xml b/xcresult/tests/data/test-in-repo-helper-raises-failure.junit.xml new file mode 100644 index 00000000..49b60781 --- /dev/null +++ b/xcresult/tests/data/test-in-repo-helper-raises-failure.junit.xml @@ -0,0 +1,8 @@ + + + + + + + + diff --git a/xcresult/tests/data/test-in-repo-helper-raises-failure.legacy.junit.xml b/xcresult/tests/data/test-in-repo-helper-raises-failure.legacy.junit.xml new file mode 100644 index 00000000..2f0d31a2 --- /dev/null +++ b/xcresult/tests/data/test-in-repo-helper-raises-failure.legacy.junit.xml @@ -0,0 +1,8 @@ + + + + + + + + diff --git a/xcresult/tests/data/test-in-repo-helper-raises-failure.xcresult.tar.gz b/xcresult/tests/data/test-in-repo-helper-raises-failure.xcresult.tar.gz new file mode 100644 index 00000000..e45beab9 Binary files /dev/null and b/xcresult/tests/data/test-in-repo-helper-raises-failure.xcresult.tar.gz differ diff --git a/xcresult/tests/data/test-objc-xctest.junit.xml b/xcresult/tests/data/test-objc-xctest.junit.xml new file mode 100644 index 00000000..ca8fe9fa --- /dev/null +++ b/xcresult/tests/data/test-objc-xctest.junit.xml @@ -0,0 +1,8 @@ + + + + + + + + diff --git a/xcresult/tests/data/test-objc-xctest.legacy.junit.xml b/xcresult/tests/data/test-objc-xctest.legacy.junit.xml new file mode 100644 index 00000000..849a0bc7 --- /dev/null +++ b/xcresult/tests/data/test-objc-xctest.legacy.junit.xml @@ -0,0 +1,8 @@ + + + + + + + + diff --git a/xcresult/tests/data/test-objc-xctest.xcresult.tar.gz b/xcresult/tests/data/test-objc-xctest.xcresult.tar.gz new file mode 100644 index 00000000..2288cef7 Binary files /dev/null and b/xcresult/tests/data/test-objc-xctest.xcresult.tar.gz differ diff --git a/xcresult/tests/data/test-toplevel-swift-testing.junit.xml b/xcresult/tests/data/test-toplevel-swift-testing.junit.xml new file mode 100644 index 00000000..ced884f2 --- /dev/null +++ b/xcresult/tests/data/test-toplevel-swift-testing.junit.xml @@ -0,0 +1,8 @@ + + + + + + + + diff --git a/xcresult/tests/data/test-toplevel-swift-testing.legacy.junit.xml b/xcresult/tests/data/test-toplevel-swift-testing.legacy.junit.xml new file mode 100644 index 00000000..89b64233 --- /dev/null +++ b/xcresult/tests/data/test-toplevel-swift-testing.legacy.junit.xml @@ -0,0 +1,8 @@ + + + + + + + + diff --git a/xcresult/tests/data/test-toplevel-swift-testing.xcresult.tar.gz b/xcresult/tests/data/test-toplevel-swift-testing.xcresult.tar.gz new file mode 100644 index 00000000..51f3e092 Binary files /dev/null and b/xcresult/tests/data/test-toplevel-swift-testing.xcresult.tar.gz differ diff --git a/xcresult/tests/fixture-src/.gitignore b/xcresult/tests/fixture-src/.gitignore new file mode 100644 index 00000000..a8be5a09 --- /dev/null +++ b/xcresult/tests/fixture-src/.gitignore @@ -0,0 +1,7 @@ +# SwiftPM and SourceKit build in place if the packages are opened in an editor; +# regeneration always builds in a copy under /tmp, so nothing here is ever needed. +.build/ +Package.resolved +DerivedData/ +*.xcresult/ +__pycache__/ diff --git a/xcresult/tests/fixture-src/README.md b/xcresult/tests/fixture-src/README.md new file mode 100644 index 00000000..8838bd97 --- /dev/null +++ b/xcresult/tests/fixture-src/README.md @@ -0,0 +1,94 @@ +# xcresult fixture sources + +The `.xcresult` bundles in `../data/` are captured from the SwiftPM packages here. +Each package exists to reproduce one shape in which the file we report for a failed +test used to be a vendored dependency's rather than the test's own — see the commit +"fix(xcresult): attribute a failure to the test's own frame, not a dependency's". + +Nothing in this directory is compiled by `cargo`; it is source for `regenerate.sh`, +checked in so the bundles can be rebuilt rather than being opaque binaries. + +## Regenerating + +Requires **macOS with Xcode** (`xcodebuild` and `xcrun xcresulttool`). + +```sh +./regenerate.sh # every scenario +./regenerate.sh objc-xctest # just one +``` + +For each scenario the script copies the package to `/tmp/xcresult-fixtures/`, turns +its `Dependency` directory into a git repository if it has one, runs + +```sh +xcodebuild test -scheme -Package -destination 'platform=macOS' \ + -derivedDataPath DerivedData -resultBundlePath .xcresult +``` + +(nonzero exit is the expected outcome — these tests are meant to fail), strips the +bundle with `prune-bundle.py`, dumps the failure summaries with +`dump-failure-summaries.py`, checks them with `verify-failure-summaries.py`, and +only then packages the bundle into `../data/test-.xcresult.tar.gz`. + +Pruning is not an optimization; without it these are unshippable. Xcode 26 writes +about 95MB of dyld shared-cache symbolication data into every result bundle, none +of it referenced from the invocation record — a scenario whose actual test data is +under 100KB produces a 96MB bundle. `prune-bundle.py` walks the object graph from +the root (whose own id lives in `Info.plist`, not in any object) and deletes the +`Data/` entries nothing reached; for these scenarios that is 9-11 objects kept out +of ~1570 files. `regenerate.sh` captures both `xcresulttool` outputs the crate +reads — `get object --legacy` and `get test-results tests` — before and after, and +fails if pruning changed either, so the saving is verified rather than assumed. + +Three details that are easy to get wrong: + +- **The dependency has to be a git repository.** A `.package(path:)` dependency is + built in place; only a git URL is checked out into + `DerivedData/SourcePackages/checkouts/`, and that path _is_ the shape being + reproduced. `regenerate.sh` creates the repository in the working copy so the + checked-in sources stay a plain directory. +- **Absolute paths are baked into a bundle at capture time** and end up in the + expected JUnit XML, which is why capture happens in a fixed directory. Set + `FIXTURE_WORK_DIR` to move it, and expect every `file` attribute to change. +- **Recapturing changes the timestamps** in the expected JUnit XML even when + nothing else moves. Diff with timestamps normalized to confirm that is all that + changed before saving. + +The expected JUnit XML is _not_ regenerated automatically. After recapturing, run +`cargo test -p xcresult`, read the diff, check each `file` attribute against the +table below, and only then save the new output over `../data/test-*.junit.xml`. A +`file` that lands on a dependency path is a bug to report, not output to snapshot. + +## What each scenario must exhibit + +`verify-failure-summaries.py` enforces the "captured shape" column and fails the +regeneration if a bundle stops reproducing it. + +| Scenario | Captured shape | Expected `file` (experimental) | Expected `file` (legacy) | +| ------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------ | ------------------------ | +| `dependency-raises-failure` | A swift-testing test calls a dependency helper that records the issue at its own `#filePath`. `fileName`, the source code context's location and the innermost frame are all under `SourcePackages/checkouts/`; only the test's own frame names the test file. | the test's own file | _(none)_ | +| `in-repo-helper-raises-failure` | Same, with the helper in the test target. Nothing rejects the helper's path, so the test's own frame has to win on its own merits. | the test's own file | the helper's file | +| `crash-in-dependency` | Two tests that never reach their own frame: one `fatalError`s inside the dependency (Xcode records a summary with no file at all), and one is failed by the dependency's `TestScoping` trait after its body returned (every file source is a checkout path). | _(none)_ | _(none)_ | +| `objc-xctest` | An Objective-C `XCTestCase` whose failure is raised by `XCTFail` in a shared category in another file, so the frame is symbolicated as `-[ObjcXCTestTests testFailsInsideSharedHelper]`. | the test's own file | _(none)_ | +| `toplevel-swift-testing` | A top-level `@Test func` with no suite, failed by a helper in another file, so the frame is the bare function name. | the test's own file | the helper's file | + +The two columns differ because only the experimental path reads the failure +summary's call stack, which is the only source that can identify the test's own +file. The legacy path sees just the workspace document location — the file the +failure was _raised_ from — so where that is a dependency it now reports no file, +and where it is an in-repo helper it still reports the helper. `objc-xctest` has no +legacy file for an unrelated reason: the fallback is keyed by test-case name and +Xcode spells the Objective-C one `-[Suite testCase]`, which never matches the +`Suite.testCase` key the lookup builds. + +To read what a captured bundle actually contains, run `dump-failure-summaries.py` +against it — the dump is derived from the bundle, so it is not checked in. + +## Why not the older SnapshotTesting fixture + +`../data/test-swift-snapshot-testing.xcresult.tar.gz` looks like it covers the +first scenario and does not: `assertSnapshot` takes `filePath: StaticString = +#filePath`, which is evaluated at the _call site_, so its `fileName` already points +at the test's own source. It passes with or without the fix. A helper that wants to +report its own location — which is what a trait or a page object does — has to build +the `SourceLocation` inside its body, which is what these fixtures do. diff --git a/xcresult/tests/fixture-src/crash-in-dependency/Dependency/Package.swift b/xcresult/tests/fixture-src/crash-in-dependency/Dependency/Package.swift new file mode 100644 index 00000000..4b45832a --- /dev/null +++ b/xcresult/tests/fixture-src/crash-in-dependency/Dependency/Package.swift @@ -0,0 +1,13 @@ +// swift-tools-version: 6.0 +import PackageDescription + +let package = Package( + name: "FixtureSupport", + platforms: [.macOS(.v13)], + products: [ + .library(name: "FixtureSupport", targets: ["FixtureSupport"]) + ], + targets: [ + .target(name: "FixtureSupport") + ] +) diff --git a/xcresult/tests/fixture-src/crash-in-dependency/Dependency/Sources/FixtureSupport/Crash.swift b/xcresult/tests/fixture-src/crash-in-dependency/Dependency/Sources/FixtureSupport/Crash.swift new file mode 100644 index 00000000..62be6b71 --- /dev/null +++ b/xcresult/tests/fixture-src/crash-in-dependency/Dependency/Sources/FixtureSupport/Crash.swift @@ -0,0 +1,4 @@ +/// Kills the test process from inside the dependency. +public func crashInsideDependency() -> Never { + fatalError("the dependency crashed the test process") +} diff --git a/xcresult/tests/fixture-src/crash-in-dependency/Dependency/Sources/FixtureSupport/TeardownFailureTrait.swift b/xcresult/tests/fixture-src/crash-in-dependency/Dependency/Sources/FixtureSupport/TeardownFailureTrait.swift new file mode 100644 index 00000000..ed60e3b8 --- /dev/null +++ b/xcresult/tests/fixture-src/crash-in-dependency/Dependency/Sources/FixtureSupport/TeardownFailureTrait.swift @@ -0,0 +1,28 @@ +import Testing + +/// A trait that fails the test *after* its body has returned, so the test's own +/// frame is already off the stack when the issue is recorded — the shape of a +/// snapshot-verification or screenshot-diffing trait that checks its work in +/// teardown. Every file the failure summary offers is then the dependency's. +public struct TeardownFailureTrait: TestTrait, TestScoping { + public func provideScope( + for test: Test, + testCase: Test.Case?, + performing function: () async throws -> Void + ) async throws { + try await function() + Issue.record( + Comment(rawValue: "the dependency's trait failed the test after its body returned"), + sourceLocation: SourceLocation( + fileID: #fileID, + filePath: #filePath, + line: #line, + column: #column + ) + ) + } +} + +extension Trait where Self == TeardownFailureTrait { + public static var teardownFailure: Self { .init() } +} diff --git a/xcresult/tests/fixture-src/crash-in-dependency/Package.swift b/xcresult/tests/fixture-src/crash-in-dependency/Package.swift new file mode 100644 index 00000000..8ab2898b --- /dev/null +++ b/xcresult/tests/fixture-src/crash-in-dependency/Package.swift @@ -0,0 +1,18 @@ +// swift-tools-version: 6.0 +import PackageDescription + +// See `dependency-raises-failure/Package.swift` for why the dependency is a git +// URL rather than a path. +let package = Package( + name: "CrashInDependency", + platforms: [.macOS(.v13)], + dependencies: [ + .package(url: "./Dependency", branch: "main") + ], + targets: [ + .testTarget( + name: "CrashInDependencyTests", + dependencies: [.product(name: "FixtureSupport", package: "Dependency")] + ) + ] +) diff --git a/xcresult/tests/fixture-src/crash-in-dependency/Tests/CrashInDependencyTests/CrashInDependencyTests.swift b/xcresult/tests/fixture-src/crash-in-dependency/Tests/CrashInDependencyTests/CrashInDependencyTests.swift new file mode 100644 index 00000000..287fb0d1 --- /dev/null +++ b/xcresult/tests/fixture-src/crash-in-dependency/Tests/CrashInDependencyTests/CrashInDependencyTests.swift @@ -0,0 +1,11 @@ +import XCTest + +import FixtureSupport + +final class CrashInDependencyTests: XCTestCase { + /// The process dies inside the dependency, so the failure summary Xcode + /// records has no file at all — not even a wrong one. + func testCrashesInsideDependency() { + crashInsideDependency() + } +} diff --git a/xcresult/tests/fixture-src/crash-in-dependency/Tests/CrashInDependencyTests/TeardownFailureTests.swift b/xcresult/tests/fixture-src/crash-in-dependency/Tests/CrashInDependencyTests/TeardownFailureTests.swift new file mode 100644 index 00000000..fded6ec8 --- /dev/null +++ b/xcresult/tests/fixture-src/crash-in-dependency/Tests/CrashInDependencyTests/TeardownFailureTests.swift @@ -0,0 +1,12 @@ +import Testing + +import FixtureSupport + +@Suite +struct TeardownFailureTests { + /// The test body succeeds and the dependency's trait fails it afterwards, so + /// the test's own frame is gone by the time the failure is recorded: every + /// file source is a dependency path and none of them may be reported. + @Test(.teardownFailure) + func failsAfterItsOwnFrameIsGone() {} +} diff --git a/xcresult/tests/fixture-src/dependency-raises-failure/Dependency/Package.swift b/xcresult/tests/fixture-src/dependency-raises-failure/Dependency/Package.swift new file mode 100644 index 00000000..4b45832a --- /dev/null +++ b/xcresult/tests/fixture-src/dependency-raises-failure/Dependency/Package.swift @@ -0,0 +1,13 @@ +// swift-tools-version: 6.0 +import PackageDescription + +let package = Package( + name: "FixtureSupport", + platforms: [.macOS(.v13)], + products: [ + .library(name: "FixtureSupport", targets: ["FixtureSupport"]) + ], + targets: [ + .target(name: "FixtureSupport") + ] +) diff --git a/xcresult/tests/fixture-src/dependency-raises-failure/Dependency/Sources/FixtureSupport/FixtureSupport.swift b/xcresult/tests/fixture-src/dependency-raises-failure/Dependency/Sources/FixtureSupport/FixtureSupport.swift new file mode 100644 index 00000000..e822ef7a --- /dev/null +++ b/xcresult/tests/fixture-src/dependency-raises-failure/Dependency/Sources/FixtureSupport/FixtureSupport.swift @@ -0,0 +1,19 @@ +import Testing + +/// Records a failure against whatever test is running, attributed to *this* file. +/// +/// `#filePath` in a function body is the file the function is defined in, so the +/// issue is raised from the dependency's source rather than from the caller's — +/// the same shape as a snapshot-testing assertion helper, a mocking framework, or +/// a page object that reports the failure at its own location. +public func recordIssueFromDependency(_ message: String) { + Issue.record( + Comment(rawValue: message), + sourceLocation: SourceLocation( + fileID: #fileID, + filePath: #filePath, + line: #line, + column: #column + ) + ) +} diff --git a/xcresult/tests/fixture-src/dependency-raises-failure/Package.swift b/xcresult/tests/fixture-src/dependency-raises-failure/Package.swift new file mode 100644 index 00000000..d5ea00fa --- /dev/null +++ b/xcresult/tests/fixture-src/dependency-raises-failure/Package.swift @@ -0,0 +1,20 @@ +// swift-tools-version: 6.0 +import PackageDescription + +// `Dependency` is referenced by git URL rather than by path so SPM checks it out +// into `DerivedData/SourcePackages/checkouts/Dependency` — the vendored location +// whose paths the fixture exists to reproduce. `regenerate.sh` turns the checked-in +// `Dependency` directory into a git repository before building. +let package = Package( + name: "DependencyRaisesFailure", + platforms: [.macOS(.v13)], + dependencies: [ + .package(url: "./Dependency", branch: "main") + ], + targets: [ + .testTarget( + name: "DependencyRaisesFailureTests", + dependencies: [.product(name: "FixtureSupport", package: "Dependency")] + ) + ] +) diff --git a/xcresult/tests/fixture-src/dependency-raises-failure/Tests/DependencyRaisesFailureTests/DependencyRaisesFailureTests.swift b/xcresult/tests/fixture-src/dependency-raises-failure/Tests/DependencyRaisesFailureTests/DependencyRaisesFailureTests.swift new file mode 100644 index 00000000..1966d563 --- /dev/null +++ b/xcresult/tests/fixture-src/dependency-raises-failure/Tests/DependencyRaisesFailureTests/DependencyRaisesFailureTests.swift @@ -0,0 +1,14 @@ +import Testing + +import FixtureSupport + +@Suite +struct DependencyRaisesFailureTests { + /// Every file the failure summary offers — `fileName`, the source code + /// context's location, and the innermost call-stack frame — points into the + /// dependency's checkout. Only the test's own call-stack frame names this file. + @Test + func failsInsideDependency() { + recordIssueFromDependency("recorded from the dependency's own source") + } +} diff --git a/xcresult/tests/fixture-src/dump-failure-summaries.py b/xcresult/tests/fixture-src/dump-failure-summaries.py new file mode 100755 index 00000000..64542400 --- /dev/null +++ b/xcresult/tests/fixture-src/dump-failure-summaries.py @@ -0,0 +1,148 @@ +#!/usr/bin/env python3 +"""Dump the per-test failure summaries out of an .xcresult bundle. + +`xcrun xcresulttool get object --legacy` returns one object at a time and the +failure summaries hang several references deep, so walking to them takes a +handful of calls. This prints a JSON array — one entry per test that has +failure summaries — with the pieces the Rust attribution logic reads: +`fileName`, `sourceCodeContext.location.filePath`, and the call stack's +symbol names and file paths. + + ./dump-failure-summaries.py .xcresult > .failure-summaries.json +""" + +import json +import subprocess +import sys + + +def get_object(path, object_id=None): + cmd = [ + "xcrun", + "xcresulttool", + "get", + "object", + "--path", + path, + "--format", + "json", + "--legacy", + ] + if object_id: + cmd += ["--id", object_id] + return json.loads(subprocess.run(cmd, check=True, capture_output=True).stdout) + + +def value(node): + """Unwrap the legacy schema's `{"_value": ...}` boxes.""" + if isinstance(node, dict): + return node.get("_value") + return None + + +def values(node): + if isinstance(node, dict): + return node.get("_values", []) + return [] + + +def walk_tests(node, ancestors): + """Yield every (name-path, summaryRef id, status) leaf under a test node.""" + name = value(node.get("name")) + subtests = node.get("subtests") + if subtests is not None: + for subtest in values(subtests): + yield from walk_tests(subtest, ancestors + [name]) + return + summary_ref = node.get("summaryRef") + if summary_ref is not None: + yield ( + ancestors + [name], + value(summary_ref.get("id")), + value(node.get("testStatus")), + value(node.get("identifier")), + ) + + +def source_code_context(context): + if context is None: + return None + location = context.get("location") or {} + return { + "location.filePath": value(location.get("filePath")), + "location.lineNumber": value(location.get("lineNumber")), + "callStack": [ + { + "symbolName": value((frame.get("symbolInfo") or {}).get("symbolName")), + "filePath": value( + (((frame.get("symbolInfo") or {}).get("location")) or {}).get( + "filePath" + ) + ), + "imageName": value((frame.get("symbolInfo") or {}).get("imageName")), + } + for frame in values(context.get("callStack")) + ], + } + + +def main(): + path = sys.argv[1] + root = get_object(path) + + out = [] + for action in values(root.get("actions")): + action_result = action.get("actionResult") or {} + + for issue in values( + (action_result.get("issues") or {}).get("testFailureSummaries") + ): + document_location = issue.get("documentLocationInCreatingWorkspace") or {} + out.append( + { + "kind": "actionResult.issues.testFailureSummaries", + "testCaseName": value(issue.get("testCaseName")), + "producingTarget": value(issue.get("producingTarget")), + "documentLocationInCreatingWorkspace.url": value( + document_location.get("url") + ), + } + ) + + tests_ref = action_result.get("testsRef") + if tests_ref is None: + continue + plan = get_object(path, value(tests_ref.get("id"))) + for plan_summary in values(plan.get("summaries")): + for testable in values(plan_summary.get("testableSummaries")): + for test in values(testable.get("tests")): + for names, summary_id, status, identifier in walk_tests(test, []): + if status == "Success": + continue + summary = get_object(path, summary_id) + for failure in values(summary.get("failureSummaries")): + out.append( + { + "kind": "test.failureSummaries", + "testBundle": value(testable.get("name")), + "namePath": names, + "identifier": identifier, + "testStatus": status, + "message": value(failure.get("message")), + "fileName": value(failure.get("fileName")), + "lineNumber": value(failure.get("lineNumber")), + "isPerformanceFailure": value( + failure.get("isPerformanceFailure") + ), + "sourceCodeContext": source_code_context( + failure.get("sourceCodeContext") + ), + } + ) + + json.dump(out, sys.stdout, indent=2) + sys.stdout.write("\n") + + +if __name__ == "__main__": + main() diff --git a/xcresult/tests/fixture-src/in-repo-helper-raises-failure/Package.swift b/xcresult/tests/fixture-src/in-repo-helper-raises-failure/Package.swift new file mode 100644 index 00000000..4ab83ad3 --- /dev/null +++ b/xcresult/tests/fixture-src/in-repo-helper-raises-failure/Package.swift @@ -0,0 +1,10 @@ +// swift-tools-version: 6.0 +import PackageDescription + +let package = Package( + name: "InRepoHelperRaisesFailure", + platforms: [.macOS(.v13)], + targets: [ + .testTarget(name: "InRepoHelperRaisesFailureTests") + ] +) diff --git a/xcresult/tests/fixture-src/in-repo-helper-raises-failure/Tests/InRepoHelperRaisesFailureTests/FailureHelper.swift b/xcresult/tests/fixture-src/in-repo-helper-raises-failure/Tests/InRepoHelperRaisesFailureTests/FailureHelper.swift new file mode 100644 index 00000000..7dbd11b4 --- /dev/null +++ b/xcresult/tests/fixture-src/in-repo-helper-raises-failure/Tests/InRepoHelperRaisesFailureTests/FailureHelper.swift @@ -0,0 +1,15 @@ +import Testing + +/// The same shape as the dependency helper, but living in the test target itself: +/// the issue is raised at this file, not at the caller's. +func recordIssueFromHelper(_ message: String) { + Issue.record( + Comment(rawValue: message), + sourceLocation: SourceLocation( + fileID: #fileID, + filePath: #filePath, + line: #line, + column: #column + ) + ) +} diff --git a/xcresult/tests/fixture-src/in-repo-helper-raises-failure/Tests/InRepoHelperRaisesFailureTests/InRepoHelperRaisesFailureTests.swift b/xcresult/tests/fixture-src/in-repo-helper-raises-failure/Tests/InRepoHelperRaisesFailureTests/InRepoHelperRaisesFailureTests.swift new file mode 100644 index 00000000..b0371891 --- /dev/null +++ b/xcresult/tests/fixture-src/in-repo-helper-raises-failure/Tests/InRepoHelperRaisesFailureTests/InRepoHelperRaisesFailureTests.swift @@ -0,0 +1,11 @@ +import Testing + +@Suite +struct InRepoHelperRaisesFailureTests { + /// The helper's file is in the repo, so nothing rejects it — the test's own + /// call-stack frame has to win on its own merits for this file to be reported. + @Test + func failsInsideHelper() { + recordIssueFromHelper("recorded from a helper in the test target") + } +} diff --git a/xcresult/tests/fixture-src/objc-xctest/Package.swift b/xcresult/tests/fixture-src/objc-xctest/Package.swift new file mode 100644 index 00000000..c41d7dff --- /dev/null +++ b/xcresult/tests/fixture-src/objc-xctest/Package.swift @@ -0,0 +1,10 @@ +// swift-tools-version: 6.0 +import PackageDescription + +let package = Package( + name: "ObjcXCTest", + platforms: [.macOS(.v13)], + targets: [ + .testTarget(name: "ObjcXCTestTests") + ] +) diff --git a/xcresult/tests/fixture-src/objc-xctest/Tests/ObjcXCTestTests/FailureHelper.m b/xcresult/tests/fixture-src/objc-xctest/Tests/ObjcXCTestTests/FailureHelper.m new file mode 100644 index 00000000..c53e5ab4 --- /dev/null +++ b/xcresult/tests/fixture-src/objc-xctest/Tests/ObjcXCTestTests/FailureHelper.m @@ -0,0 +1,11 @@ +#import "FailureHelper.h" + +@implementation XCTestCase (FixtureFailureHelper) + +// `XCTFail` expands with this file's `__FILE__`, so the failure is raised here +// rather than at the call site. +- (void)fixtureFailWithMessage:(NSString *)message { + XCTFail(@"%@", message); +} + +@end diff --git a/xcresult/tests/fixture-src/objc-xctest/Tests/ObjcXCTestTests/ObjcXCTestTests.m b/xcresult/tests/fixture-src/objc-xctest/Tests/ObjcXCTestTests/ObjcXCTestTests.m new file mode 100644 index 00000000..67f39841 --- /dev/null +++ b/xcresult/tests/fixture-src/objc-xctest/Tests/ObjcXCTestTests/ObjcXCTestTests.m @@ -0,0 +1,14 @@ +#import "FailureHelper.h" + +@interface ObjcXCTestTests : XCTestCase +@end + +@implementation ObjcXCTestTests + +// Symbolicates as `-[ObjcXCTestTests testFailsInsideSharedHelper]`, which is the +// Objective-C spelling the frame matching has to recognize. +- (void)testFailsInsideSharedHelper { + [self fixtureFailWithMessage:@"raised from the shared Objective-C helper"]; +} + +@end diff --git a/xcresult/tests/fixture-src/objc-xctest/Tests/ObjcXCTestTests/include/FailureHelper.h b/xcresult/tests/fixture-src/objc-xctest/Tests/ObjcXCTestTests/include/FailureHelper.h new file mode 100644 index 00000000..04ec2c47 --- /dev/null +++ b/xcresult/tests/fixture-src/objc-xctest/Tests/ObjcXCTestTests/include/FailureHelper.h @@ -0,0 +1,5 @@ +#import + +@interface XCTestCase (FixtureFailureHelper) +- (void)fixtureFailWithMessage:(NSString *)message; +@end diff --git a/xcresult/tests/fixture-src/prune-bundle.py b/xcresult/tests/fixture-src/prune-bundle.py new file mode 100755 index 00000000..d7c2d197 --- /dev/null +++ b/xcresult/tests/fixture-src/prune-bundle.py @@ -0,0 +1,136 @@ +#!/usr/bin/env python3 +"""Drop the objects in an .xcresult bundle that nothing in it references. + +Xcode 26 writes ~140MB of dyld shared-cache symbolication data plus a payload per +loaded image into every result bundle, none of it reachable from the invocation +record. That is fine for a bundle you throw away and not fine for one checked into +git — it is ~95MB of the ~96MB a scenario produces. + +This walks the object graph from the root, following every reference id, and +deletes the `Data/` entries nothing reached. `regenerate.sh` re-dumps the failure +summaries afterwards and fails if pruning changed them. + + ./prune-bundle.py .xcresult +""" + +import json +import os +import re +import subprocess +import sys + +# Object ids are content-addressed and are also the `Data/` file names, e.g. +# `data.0~aBc.../refs.0~aBc...`. +OBJECT_ID = re.compile(r"^0~[A-Za-z0-9_\-+/=]+$") + + +def get_object(path, object_id=None): + cmd = [ + "xcrun", + "xcresulttool", + "get", + "object", + "--path", + path, + "--format", + "json", + "--legacy", + ] + if object_id: + cmd += ["--id", object_id] + result = subprocess.run(cmd, capture_output=True) + if result.returncode != 0: + return None + try: + return json.loads(result.stdout) + except json.JSONDecodeError: + return None + + +def object_ids_in(node): + """Every value anywhere in an object that is shaped like an object id.""" + if isinstance(node, dict): + for key, child in node.items(): + if key == "_value" and isinstance(child, str) and OBJECT_ID.match(child): + yield child + else: + yield from object_ids_in(child) + elif isinstance(node, list): + for child in node: + yield from object_ids_in(child) + + +def root_id(path): + """The invocation record's own id, which lives in Info.plist rather than in + any object, so nothing in the graph points at it.""" + # Info.plist holds a date, which `plutil -convert json` refuses to write, so + # the one field is extracted rather than the whole file converted. + info = subprocess.run( + [ + "plutil", + "-extract", + "rootId.hash", + "raw", + "-o", + "-", + os.path.join(path, "Info.plist"), + ], + check=True, + capture_output=True, + ) + return info.stdout.decode().strip() + + +def reachable_ids(path): + root = get_object(path) + if root is None: + raise SystemExit(f"{path}: could not read the invocation record") + + seen = {root_id(path)} + queue = list(object_ids_in(root)) + while queue: + object_id = queue.pop() + if object_id in seen: + continue + seen.add(object_id) + child = get_object(path, object_id) + if child is not None: + queue.extend(object_ids_in(child)) + return seen + + +def main(): + path = sys.argv[1] + data_dir = os.path.join(path, "Data") + + keep = reachable_ids(path) + before = sum( + os.path.getsize(os.path.join(data_dir, name)) for name in os.listdir(data_dir) + ) + + removed = 0 + for name in os.listdir(data_dir): + prefix, _, object_id = name.partition(".") + if prefix not in ("data", "refs"): + continue + if object_id in keep: + continue + file_path = os.path.join(data_dir, name) + removed += os.path.getsize(file_path) + os.remove(file_path) + + # Built lazily by xcresulttool and rebuilt on demand, so it is noise in a + # checked-in fixture. + index = os.path.join(path, "database.sqlite3") + if os.path.exists(index): + os.remove(index) + + print( + f" pruned {removed / 1e6:.0f}MB of unreferenced objects " + f"({before / 1e6:.0f}MB -> {(before - removed) / 1e6:.0f}MB), " + f"kept {len(keep)}" + ) + + +if __name__ == "__main__": + main() diff --git a/xcresult/tests/fixture-src/regenerate.sh b/xcresult/tests/fixture-src/regenerate.sh new file mode 100755 index 00000000..b7a24583 --- /dev/null +++ b/xcresult/tests/fixture-src/regenerate.sh @@ -0,0 +1,141 @@ +#!/usr/bin/env bash +# +# Capture the .xcresult fixtures in `xcresult/tests/data/` from the scenario +# packages next to this script. Requires macOS and Xcode. +# +# ./regenerate.sh # every scenario +# ./regenerate.sh objc-xctest # just one +# FIXTURE_WORK_DIR=/tmp/elsewhere ./regenerate.sh +# +# Absolute paths are baked into a bundle at capture time and end up in the +# expected JUnit XML, so the scenarios are copied to a fixed working directory +# (`/tmp/xcresult-fixtures` by default) and built there. Regenerating from a +# different directory rewrites every path in the expected output — see README.md. + +set -euo pipefail + +FIXTURE_SRC_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +DATA_DIR="$(cd "${FIXTURE_SRC_DIR}/../data" && pwd)" +WORK_DIR="${FIXTURE_WORK_DIR:-/tmp/xcresult-fixtures}" + +ALL_SCENARIOS=( + dependency-raises-failure + in-repo-helper-raises-failure + crash-in-dependency + objc-xctest + toplevel-swift-testing +) + +# scenario -> the package name, which is both the xcodebuild scheme prefix and the +# name of the captured bundle. +package_name() { + case "$1" in + dependency-raises-failure) echo DependencyRaisesFailure ;; + in-repo-helper-raises-failure) echo InRepoHelperRaisesFailure ;; + crash-in-dependency) echo CrashInDependency ;; + objc-xctest) echo ObjcXCTest ;; + toplevel-swift-testing) echo ToplevelSwiftTesting ;; + *) + echo "unknown scenario: $1" >&2 + exit 1 + ;; + esac +} + +regenerate() { + local scenario="$1" + local package + package="$(package_name "${scenario}")" + local scenario_work_dir="${WORK_DIR}/${scenario}" + local bundle="${scenario_work_dir}/${package}.xcresult" + + echo "==> ${scenario}" + + rm -rf "${scenario_work_dir}" + mkdir -p "${WORK_DIR}" + cp -R "${FIXTURE_SRC_DIR}/${scenario}" "${scenario_work_dir}" + + # A `.package(url: "./Dependency", ...)` is only checked out into + # `SourcePackages/checkouts` if it is a git repository, and the fixture is + # pointless unless it is: that checkout path is the whole shape being + # reproduced. The repository is created in the working copy so the checked-in + # sources stay a plain directory. + if [[ -d "${scenario_work_dir}/Dependency" ]]; then + git -C "${scenario_work_dir}/Dependency" init -q + git -C "${scenario_work_dir}/Dependency" add -A + git -C "${scenario_work_dir}/Dependency" \ + -c user.email=fixtures@trunk.io -c user.name=fixtures \ + commit -qm "xcresult fixture dependency" + git -C "${scenario_work_dir}/Dependency" branch -qM main + fi + + # Tests are meant to fail here, so xcodebuild exits nonzero on a good run. + ( + cd "${scenario_work_dir}" + xcodebuild test \ + -scheme "${package}-Package" \ + -destination 'platform=macOS' \ + -derivedDataPath DerivedData \ + -resultBundlePath "${package}.xcresult" \ + >xcodebuild.log 2>&1 + ) || true + + if [[ ! -d ${bundle} ]]; then + echo " no bundle at ${bundle}; see ${scenario_work_dir}/xcodebuild.log" >&2 + return 1 + fi + + # Xcode 26 writes ~95MB of unreferenced symbolication data into every bundle, + # so pruning is what makes these checkable into git at all. Both xcresulttool + # APIs the crate calls are compared before and after to prove the prune is + # invisible to consumers. + local before_legacy="${scenario_work_dir}/before-legacy.json" + local before_modern="${scenario_work_dir}/before-modern.json" + "${FIXTURE_SRC_DIR}/dump-failure-summaries.py" "${bundle}" >"${before_legacy}" + xcrun xcresulttool get test-results tests --path "${bundle}" --format json \ + >"${before_modern}" + + "${FIXTURE_SRC_DIR}/prune-bundle.py" "${bundle}" + + # Derived from the bundle, so it is written to the working directory rather + # than checked in; `dump-failure-summaries.py` regenerates it on demand. + local dump="${scenario_work_dir}/${scenario}.failure-summaries.json" + "${FIXTURE_SRC_DIR}/dump-failure-summaries.py" "${bundle}" >"${dump}" + diff -q "${before_legacy}" "${dump}" >/dev/null || + { + echo " pruning changed the legacy failure summaries" >&2 + return 1 + } + xcrun xcresulttool get test-results tests --path "${bundle}" --format json | + diff -q "${before_modern}" - >/dev/null || + { + echo " pruning changed the test-results output" >&2 + return 1 + } + + "${FIXTURE_SRC_DIR}/verify-failure-summaries.py" "${scenario}" "${dump}" + + tar -czf "${DATA_DIR}/test-${scenario}.xcresult.tar.gz" \ + -C "${scenario_work_dir}" "${package}.xcresult" + echo " wrote ${DATA_DIR}/test-${scenario}.xcresult.tar.gz" +} + +scenarios=("${@-}") +if [[ -z ${scenarios[0]} ]]; then + scenarios=("${ALL_SCENARIOS[@]}") +fi + +for scenario in "${scenarios[@]}"; do + regenerate "${scenario}" +done + +cat <<'EOF' + +Bundles regenerated. The expected JUnit XML in `xcresult/tests/data/` still has +the old absolute paths baked in; update it with + + cargo test -p xcresult + +and check every `file` attribute by hand before saving the new output — see +README.md for what each scenario must report. +EOF diff --git a/xcresult/tests/fixture-src/toplevel-swift-testing/Package.swift b/xcresult/tests/fixture-src/toplevel-swift-testing/Package.swift new file mode 100644 index 00000000..d5eca214 --- /dev/null +++ b/xcresult/tests/fixture-src/toplevel-swift-testing/Package.swift @@ -0,0 +1,10 @@ +// swift-tools-version: 6.0 +import PackageDescription + +let package = Package( + name: "ToplevelSwiftTesting", + platforms: [.macOS(.v13)], + targets: [ + .testTarget(name: "ToplevelSwiftTestingTests") + ] +) diff --git a/xcresult/tests/fixture-src/toplevel-swift-testing/Tests/ToplevelSwiftTestingTests/FailureHelper.swift b/xcresult/tests/fixture-src/toplevel-swift-testing/Tests/ToplevelSwiftTestingTests/FailureHelper.swift new file mode 100644 index 00000000..8b85ffca --- /dev/null +++ b/xcresult/tests/fixture-src/toplevel-swift-testing/Tests/ToplevelSwiftTestingTests/FailureHelper.swift @@ -0,0 +1,13 @@ +import Testing + +func recordIssueFromHelper(_ message: String) { + Issue.record( + Comment(rawValue: message), + sourceLocation: SourceLocation( + fileID: #fileID, + filePath: #filePath, + line: #line, + column: #column + ) + ) +} diff --git a/xcresult/tests/fixture-src/toplevel-swift-testing/Tests/ToplevelSwiftTestingTests/ToplevelSwiftTestingTests.swift b/xcresult/tests/fixture-src/toplevel-swift-testing/Tests/ToplevelSwiftTestingTests/ToplevelSwiftTestingTests.swift new file mode 100644 index 00000000..8bf2224c --- /dev/null +++ b/xcresult/tests/fixture-src/toplevel-swift-testing/Tests/ToplevelSwiftTestingTests/ToplevelSwiftTestingTests.swift @@ -0,0 +1,8 @@ +import Testing + +/// Declared at the top level rather than in a `@Suite`, so it symbolicates as the +/// bare function `failsInsideHelperWithoutASuite()` with no suite to qualify it. +@Test +func failsInsideHelperWithoutASuite() { + recordIssueFromHelper("raised from the in-repo helper") +} diff --git a/xcresult/tests/fixture-src/verify-failure-summaries.py b/xcresult/tests/fixture-src/verify-failure-summaries.py new file mode 100755 index 00000000..45ade9d0 --- /dev/null +++ b/xcresult/tests/fixture-src/verify-failure-summaries.py @@ -0,0 +1,195 @@ +#!/usr/bin/env python3 +"""Assert that a captured .xcresult actually exhibits the shape it was captured for. + +A fixture that no longer reproduces its shape is worse than no fixture — it keeps +passing while guarding nothing (which is what happened to the older +`test-swift-snapshot-testing` bundle, whose `fileName` points at the test's own +source). `regenerate.sh` runs this before packaging anything, and refuses to +package a bundle that fails. + + ./verify-failure-summaries.py .failure-summaries.json +""" + +import json +import sys + +# Directory segments that mark vendored dependency sources. Kept in sync with +# `DEPENDENCY_PATH_SEGMENTS` in `xcresult/src/xcresult_legacy.rs`. +DEPENDENCY_PATH_SEGMENTS = ["/.build/", "/checkouts/", "/DerivedData/"] + +# One entry per failure the scenario must produce, keyed by the test's identifier. +# +# raised_from: where `fileName` / the source code context's location must point +# - "dependency": a vendored path, which is what makes the fixture a regression +# test — every non-call-stack source is unusable +# - "in_repo_helper": a real path that is *not* the test's own file +# - "absent": the failure carries no file at all +# test_frame_symbol: the call-stack symbol that names the test itself, which is +# the only thing that can identify the test's file. `None` means the stack must +# *not* reach the test — the crash and teardown shapes, where no file may be +# reported at all. +# other_frames: whether the stack must also contain frames that are not the +# test's, i.e. whether picking the last frame instead of the test's would land +# somewhere else. +SCENARIOS = { + "dependency-raises-failure": [ + { + "identifier": "DependencyRaisesFailureTests/failsInsideDependency()", + "raised_from": "dependency", + "test_frame_symbol": "DependencyRaisesFailureTests.failsInsideDependency()", + "other_frames": True, + }, + ], + "in-repo-helper-raises-failure": [ + { + "identifier": "InRepoHelperRaisesFailureTests/failsInsideHelper()", + "raised_from": "in_repo_helper", + "test_frame_symbol": "InRepoHelperRaisesFailureTests.failsInsideHelper()", + "other_frames": True, + }, + ], + "crash-in-dependency": [ + { + "identifier": "CrashInDependencyTests/testCrashesInsideDependency()", + "raised_from": "absent", + "test_frame_symbol": None, + "other_frames": False, + }, + { + "identifier": "TeardownFailureTests/failsAfterItsOwnFrameIsGone()", + "raised_from": "dependency", + "test_frame_symbol": None, + "other_frames": True, + }, + ], + "objc-xctest": [ + { + "identifier": "ObjcXCTestTests/testFailsInsideSharedHelper", + "raised_from": "in_repo_helper", + "test_frame_symbol": "-[ObjcXCTestTests testFailsInsideSharedHelper]", + "other_frames": True, + }, + ], + "toplevel-swift-testing": [ + { + "identifier": "failsInsideHelperWithoutASuite()", + "raised_from": "in_repo_helper", + "test_frame_symbol": "failsInsideHelperWithoutASuite()", + "other_frames": True, + }, + ], +} + + +def is_dependency_path(path): + return path is not None and any(seg in path for seg in DEPENDENCY_PATH_SEGMENTS) + + +class Failures(list): + def check(self, condition, message): + if not condition: + self.append(message) + + +def check_summary(failures, expectation, summary): + identifier = expectation["identifier"] + context = summary["sourceCodeContext"] or {} + frames = context.get("callStack", []) + raised_from = [summary["fileName"], context.get("location.filePath")] + named = [ + frame + for frame in frames + if expectation["test_frame_symbol"] is not None + and frame["symbolName"] == expectation["test_frame_symbol"] + ] + + if expectation["raised_from"] == "dependency": + for source in raised_from: + failures.check( + is_dependency_path(source), + f"{identifier}: expected the failure to be raised from a dependency " + f"path, got {source!r}", + ) + elif expectation["raised_from"] == "absent": + for source in raised_from: + failures.check( + source is None, + f"{identifier}: expected no file source at all, got {source!r}", + ) + else: + for source in raised_from: + failures.check( + source is not None and not is_dependency_path(source), + f"{identifier}: expected the failure to be raised from an in-repo " + f"helper, got {source!r}", + ) + + if expectation["test_frame_symbol"] is None: + failures.check( + not any( + frame["filePath"] and not is_dependency_path(frame["filePath"]) + for frame in frames + ), + f"{identifier}: expected the stack never to reach the test, but it has " + f"a frame outside the dependency", + ) + else: + failures.check( + len(named) == 1, + f"{identifier}: expected exactly one frame named " + f"{expectation['test_frame_symbol']!r}, found {len(named)}", + ) + for frame in named: + failures.check( + frame["filePath"] is not None + and not is_dependency_path(frame["filePath"]), + f"{identifier}: the test's own frame must carry a non-dependency " + f"file path, got {frame['filePath']!r}", + ) + failures.check( + frame["filePath"] not in raised_from, + f"{identifier}: the test's own frame points at the same file the " + f"failure was raised from ({frame['filePath']!r}), so the fixture " + f"would pass without the fix", + ) + + if expectation["other_frames"]: + symbolicated = [frame for frame in frames if frame["symbolName"]] + failures.check( + len(symbolicated) > len(named), + f"{identifier}: expected at least one frame besides the test's own", + ) + + +def main(): + scenario, dump_path = sys.argv[1], sys.argv[2] + expectations = SCENARIOS[scenario] + dump = json.load(open(dump_path)) + summaries = { + entry["identifier"]: entry + for entry in dump + if entry["kind"] == "test.failureSummaries" + } + + failures = Failures() + for expectation in expectations: + summary = summaries.get(expectation["identifier"]) + if summary is None: + failures.append( + f"{expectation['identifier']}: no failure summary in the bundle " + f"(found {sorted(summaries)})" + ) + continue + check_summary(failures, expectation, summary) + + if failures: + print(f"{scenario}: bundle does not exhibit its shape", file=sys.stderr) + for failure in failures: + print(f" - {failure}", file=sys.stderr) + sys.exit(1) + + print(f"{scenario}: verified {len(expectations)} failure summary shape(s)") + + +if __name__ == "__main__": + main() diff --git a/xcresult/tests/xcresult.rs b/xcresult/tests/xcresult.rs index 7514ed83..35397b5d 100644 --- a/xcresult/tests/xcresult.rs +++ b/xcresult/tests/xcresult.rs @@ -34,6 +34,16 @@ lazy_static! { unpack_archive_to_temp_dir("tests/data/test-swift-mix.xcresult.tar.gz"); static ref TEMP_DIR_TEST_SWIFT_SNAPSHOT_TESTING: TempDir = unpack_archive_to_temp_dir("tests/data/test-swift-snapshot-testing.xcresult.tar.gz"); + static ref TEMP_DIR_TEST_DEPENDENCY_RAISES_FAILURE: TempDir = + unpack_archive_to_temp_dir("tests/data/test-dependency-raises-failure.xcresult.tar.gz"); + static ref TEMP_DIR_TEST_IN_REPO_HELPER_RAISES_FAILURE: TempDir = + unpack_archive_to_temp_dir("tests/data/test-in-repo-helper-raises-failure.xcresult.tar.gz"); + static ref TEMP_DIR_TEST_CRASH_IN_DEPENDENCY: TempDir = + unpack_archive_to_temp_dir("tests/data/test-crash-in-dependency.xcresult.tar.gz"); + static ref TEMP_DIR_TEST_OBJC_XCTEST: TempDir = + unpack_archive_to_temp_dir("tests/data/test-objc-xctest.xcresult.tar.gz"); + static ref TEMP_DIR_TEST_TOPLEVEL_SWIFT_TESTING: TempDir = + unpack_archive_to_temp_dir("tests/data/test-toplevel-swift-testing.xcresult.tar.gz"); static ref TEMP_DIR_TEST_TIMESTAMP: TempDir = unpack_archive_to_temp_dir("tests/data/test-timestamp.xcresult.tar.gz"); static ref TEMP_DIR_TEST_VARIANT: TempDir = @@ -245,6 +255,162 @@ fn test_swift_snapshot_testing_trait_failure_uses_assertion_file( ); } +// Real bundles for the file sources in `xcresult::file_attribution`, one per shape +// that used to attribute a failed test to a vendored dependency. Each test below +// names the source it exercises; `tests/fixture-src/README.md` covers what each +// bundle must exhibit and how to regenerate it. +// +// The two cases expect different JUnit because they have different sources +// available: only the experimental path reads the per-test failure summary, and so +// the call stack, so it is the only one that can produce a `FileSource::TestFrame`. +// The legacy path sees `FileSource::DocumentLocation` alone. +#[cfg(target_os = "macos")] +fn assert_junit>( + bundle_path: T, + use_experimental_failure_summary: bool, + expected_junit_xml: &str, +) { + let path_str = bundle_path.as_ref().to_str().unwrap(); + let xcresult = XCResult::new( + path_str, + ORG_URL_SLUG.clone(), + REPO_FULL_NAME.clone(), + use_experimental_failure_summary, + ); + assert!(xcresult.is_ok()); + + let mut junits = xcresult.unwrap().generate_junits(); + assert_eq!(junits.len(), 1); + let junit = junits.pop().unwrap(); + let mut junit_writer: Vec = Vec::new(); + junit.serialize(&mut junit_writer).unwrap(); + pretty_assertions::assert_eq!(String::from_utf8(junit_writer).unwrap(), expected_junit_xml); +} + +// `FileSource::TestFrame` is the only usable source. The failure is recorded inside +// the dependency, so `RaisedFrom`, `SourceCodeLocation` and the innermost +// `LastStackFrame` all point into `DerivedData/SourcePackages/checkouts/` and are +// rejected as vendored. The legacy path has only `DocumentLocation`, which points +// there too, so it reports nothing at all. +#[cfg(target_os = "macos")] +#[rstest] +#[case::experimental_failure_summary( + true, + include_str!("data/test-dependency-raises-failure.junit.xml") +)] +#[case::legacy_fallback( + false, + include_str!("data/test-dependency-raises-failure.legacy.junit.xml") +)] +fn test_dependency_raised_failure_uses_the_tests_own_file( + #[case] use_experimental_failure_summary: bool, + #[case] expected_junit_xml: &str, +) { + assert_junit( + TEMP_DIR_TEST_DEPENDENCY_RAISES_FAILURE + .as_ref() + .join("DependencyRaisesFailure.xcresult"), + use_experimental_failure_summary, + expected_junit_xml, + ); +} + +// `FileSource::TestFrame` versus `RaisedFrom` with nothing to separate them by path. +// The helper is in the test target, so `is_vendored_dependency` says nothing useful +// and the ordering of the sources is what decides. The legacy path, having only +// `DocumentLocation`, still lands on the helper. +#[cfg(target_os = "macos")] +#[rstest] +#[case::experimental_failure_summary( + true, + include_str!("data/test-in-repo-helper-raises-failure.junit.xml") +)] +#[case::legacy_fallback( + false, + include_str!("data/test-in-repo-helper-raises-failure.legacy.junit.xml") +)] +fn test_in_repo_helper_raised_failure_uses_the_tests_own_file( + #[case] use_experimental_failure_summary: bool, + #[case] expected_junit_xml: &str, +) { + assert_junit( + TEMP_DIR_TEST_IN_REPO_HELPER_RAISES_FAILURE + .as_ref() + .join("InRepoHelperRaisesFailure.xcresult"), + use_experimental_failure_summary, + expected_junit_xml, + ); +} + +// No source survives vetting. Neither test reaches its own frame — one crashes +// inside the dependency, the other is failed by the dependency's trait after its +// body returned — so there is no `TestFrame`, and every remaining candidate is +// either absent or vendored. Both cases must come out with no `file` attribute at +// all rather than the dependency's. +#[cfg(target_os = "macos")] +#[rstest] +#[case::experimental_failure_summary(true, include_str!("data/test-crash-in-dependency.junit.xml"))] +#[case::legacy_fallback(false, include_str!("data/test-crash-in-dependency.junit.xml"))] +fn test_crash_in_dependency_reports_no_file( + #[case] use_experimental_failure_summary: bool, + #[case] expected_junit_xml: &str, +) { + assert_junit( + TEMP_DIR_TEST_CRASH_IN_DEPENDENCY + .as_ref() + .join("CrashInDependency.xcresult"), + use_experimental_failure_summary, + expected_junit_xml, + ); +} + +// `TestIdentity::is_named_by` against real symbolication: Xcode spells the frame +// `-[ObjcXCTestTests testFailsInsideSharedHelper]`, not `Suite.testCase()`. The +// legacy path reports nothing here for an unrelated reason — the `DocumentLocation` +// candidates are keyed by test case name, and Xcode's Objective-C spelling never +// matches the `Suite.testCase` key the lookup builds. +#[cfg(target_os = "macos")] +#[rstest] +#[case::experimental_failure_summary(true, include_str!("data/test-objc-xctest.junit.xml"))] +#[case::legacy_fallback(false, include_str!("data/test-objc-xctest.legacy.junit.xml"))] +fn test_objc_xctest_helper_failure_uses_the_tests_own_file( + #[case] use_experimental_failure_summary: bool, + #[case] expected_junit_xml: &str, +) { + assert_junit( + TEMP_DIR_TEST_OBJC_XCTEST + .as_ref() + .join("ObjcXCTest.xcresult"), + use_experimental_failure_summary, + expected_junit_xml, + ); +} + +// `TestIdentity` with no suite: a top-level swift-testing `@Test func` symbolicates +// as the bare function, so there is nothing to qualify the match with. +#[cfg(target_os = "macos")] +#[rstest] +#[case::experimental_failure_summary( + true, + include_str!("data/test-toplevel-swift-testing.junit.xml") +)] +#[case::legacy_fallback( + false, + include_str!("data/test-toplevel-swift-testing.legacy.junit.xml") +)] +fn test_toplevel_swift_testing_helper_failure_uses_the_tests_own_file( + #[case] use_experimental_failure_summary: bool, + #[case] expected_junit_xml: &str, +) { + assert_junit( + TEMP_DIR_TEST_TOPLEVEL_SWIFT_TESTING + .as_ref() + .join("ToplevelSwiftTesting.xcresult"), + use_experimental_failure_summary, + expected_junit_xml, + ); +} + #[cfg(target_os = "macos")] #[test] fn test_expected_failures_xcresult_with_valid_path() {