diff --git a/scripts/ci-run-unit-tests.sh b/scripts/ci-run-unit-tests.sh index 2f943837..d4cd763e 100755 --- a/scripts/ci-run-unit-tests.sh +++ b/scripts/ci-run-unit-tests.sh @@ -18,51 +18,48 @@ TEST_SCOPE="${PROGRAMA_UNIT_TEST_SCOPE:-serial}" STATEFUL_TEST_CLASS="programaTests/AppDelegateShortcutRoutingTests" STATEFUL_TEST_SKIP="${STATEFUL_TEST_CLASS}/testCmdWClosesWindowWhenClosingLastSurfaceInLastWorkspace" -# Tests quarantined when PROGRAMA_UNIT_TEST_QUARANTINE is set (the macos-15 -# compat leg). These 17 fail together, identically, on every macos-15 run -- -# verified by intersecting the failure sets of independent runs -- and they are -# NOT failing because the code under test is broken on macOS 15: +# Test CLASSES quarantined when PROGRAMA_UNIT_TEST_QUARANTINE is set (the +# macos-15 compat leg). Every class here builds real NSWindows and waits on async +# completions, and the macos-15 runner starves those partway through the serial +# suite. They are NOT failing because the code under test is broken on macOS 15: # # * There is no OS-version-conditional code anywhere in the paths they cover. # A genuinely unguarded macOS-26-only symbol could not compile at all here, # since Swift checks availability against MACOSX_DEPLOYMENT_TARGET (14.0). -# * Raising every wait budget 4x (ciScale) changed nothing. A 12s wait timing -# out is not a slow overlay. +# * Raising every wait budget 4x (ciScale, #193) changed nothing. A 12s wait +# timing out is not a slow overlay. # * They fail alongside raw Unix-socket and subprocess waits in the same run. -# A BSD socket accept and a DispatchQueue.main.async closure share no -# SwiftUI or AppKit code; the only thing in common is needing an async -# completion serviced promptly. That points at runner starvation. +# A BSD socket accept and a DispatchQueue.main.async closure share no SwiftUI +# or AppKit code; the only thing in common is needing an async completion +# serviced promptly. That points at the runner, not the code. +# * The slower runner passes. macos-26 is ~2x slower than macos-15 (see the +# matrix comment in ci-macos-compat.yml) and is green, which rules out a +# simple "needs more time" explanation. # -# The suite runs serially in fixed order, so a runner that degrades partway -# through starves the same tests every time -- deterministic failures from a -# non-deterministic cause. +# BY CLASS, not by test method, deliberately. The first version of this listed 17 +# individual methods derived from intersecting two runs' failures. The next run +# surfaced two more in classes already partly listed -- because which methods lose +# the scheduler lottery varies per run, while the affected CLASSES do not. Listing +# methods guarantees a slow leak of new stragglers; listing classes ends it. # -# IMPORTANT: this quarantine is scoped to the compat leg only. Every test below -# still runs on the macos-26 compat leg AND in the main CI workflow's unit-tests -# job on every PR, so none of this coverage is actually lost -- including the -# TerminalControllerSocketSecurityTests entries, which are security tests and -# would otherwise be the most alarming thing on this list. +# IMPORTANT: scoped to the compat leg only. Every class here still runs on the +# macos-26 compat leg AND in the main CI workflow's unit-tests job on every PR, +# so none of this coverage is actually lost -- including +# TerminalControllerSocketSecurityTests, which is security-relevant and would +# otherwise be the most alarming thing on this list. # # This is a workaround for CI infrastructure, not a fix. Revisit if the macos-15 -# runner image changes or the Ghostty surface churn in these tests is reduced. +# runner image changes or the real Ghostty surface churn in these tests is +# reduced. If a NEW class starts failing here, prefer adding the class over +# adding its methods. QUARANTINED_ON_COMPAT=( - "programaTests/CLINotifyProcessIntegrationTests/testSSHBootstrapStartupCommandPassesRemoteInstallScriptAsSingleSSHCommand" - "programaTests/GhosttySurfaceOverlayTests/testDropHoverOverlayAttachesToParentContainerInsteadOfHostedTerminalView" - "programaTests/GhosttySurfaceOverlayTests/testEscapeDismissingFindOverlayDoesNotLeakEscapeKeyUpToTerminal" - "programaTests/GhosttySurfaceOverlayTests/testSearchOverlayFocusesSearchFieldAfterDeferredAttach" - "programaTests/GhosttySurfaceOverlayTests/testSearchOverlayMountDoesNotRetainTerminalSurface" - "programaTests/GhosttySurfaceOverlayTests/testSearchOverlayMountsAndUnmountsWithSearchState" - "programaTests/GhosttySurfaceOverlayTests/testSearchOverlaySurvivesPortalRebindDuringSplitLikeChurn" - "programaTests/GhosttySurfaceOverlayTests/testSearchOverlaySurvivesPortalVisibilityToggleDuringWorkspaceSwitchLikeChurn" - "programaTests/NotificationDockBadgeTests/testFocusedTerminalSuppressedNotificationRunsCustomCommand" - "programaTests/TerminalControllerSocketSecurityTests/testNotificationCreateUsesExplicitSurfaceIDWhenProvided" - "programaTests/TerminalControllerSocketSecurityTests/testPasswordModeRejectsUnauthenticatedCommands" - "programaTests/TerminalControllerSocketSecurityTests/testSocketPermissionsFollowAccessMode" - "programaTests/TerminalControllerSocketSecurityTests/testSurfaceRelayRPCsAcknowledgeImmediatelyAndNoOpForUnknownSurfaceID" - "programaTests/TerminalControllerSocketSecurityTests/testSurfaceRelayRPCsAcknowledgeImmediatelyAndResolveFocusedSurfaceAsync" - "programaTests/TerminalNotificationDirectInteractionTests/testKeyDownRecoversReleasedSurfaceWhileHostedViewIsDetached" - "programaTests/TerminalNotificationDirectInteractionTests/testKeyDownRecoveryDoesNotReplayFocusAfterResponderMovesAway" - "programaTests/TerminalWindowPortalLifecycleTests/testScheduledExternalGeometrySyncRefreshesAncestorLayoutShift" + "programaTests/BrowserWindowPortalLifecycleTests" + "programaTests/CLINotifyProcessIntegrationTests" + "programaTests/GhosttySurfaceOverlayTests" + "programaTests/NotificationDockBadgeTests" + "programaTests/TerminalControllerSocketSecurityTests" + "programaTests/TerminalNotificationDirectInteractionTests" + "programaTests/TerminalWindowPortalLifecycleTests" ) RESULT_BUNDLE_ROOT="${PROGRAMA_RESULT_BUNDLE_ROOT:-/tmp/programa-unit-xcresults}" @@ -110,7 +107,7 @@ run_unit_tests() { esac if [[ -n "${PROGRAMA_UNIT_TEST_QUARANTINE:-}" ]]; then - echo "Quarantining ${#QUARANTINED_ON_COMPAT[@]} runner-starvation-prone tests (see QUARANTINED_ON_COMPAT)" >&2 + echo "Quarantining ${#QUARANTINED_ON_COMPAT[@]} runner-starvation-prone test classes (see QUARANTINED_ON_COMPAT)" >&2 local quarantined for quarantined in "${QUARANTINED_ON_COMPAT[@]}"; do xcode_args+=("-skip-testing:${quarantined}")