From b500704bcfd2e1b1b1f68c5d4834450bab53ef01 Mon Sep 17 00:00:00 2001 From: John Lambert Date: Fri, 20 Mar 2026 13:01:27 -0400 Subject: [PATCH 1/4] test: eliminate popup-based test failures --- Build/scripts/Invoke-CppTest.ps1 | 2 + Src/AppForTests.config | 1 + Src/AssemblyInfoForTests.cs | 6 + Src/AssemblyInfoForUiIndependentTests.cs | 3 + ...ndleApplicationThreadExceptionAttribute.cs | 4 + .../LogUnhandledExceptionsAttribute.cs | 88 +++++++++++ .../SuppressAssertDialogsAttribute.cs | 143 ++++++++++++++++++ Src/DebugProcs/DebugProcs.cpp | 35 +++-- Src/Generic/Test/TestGeneric.vcxproj | 1 + Src/Generic/Test/testGeneric.cpp | 2 + .../InstallValidatorTests.csproj | 3 + .../InstallValidatorTests/TestAssemblyInfo.cs | 7 + .../InstallerArtifactsTests.csproj | 6 + .../TestAssemblyInfo.cs | 7 + Test.runsettings | 3 + test.ps1 | 1 + 16 files changed, 302 insertions(+), 10 deletions(-) create mode 100644 Src/Common/FwUtils/FwUtilsTests/Attributes/LogUnhandledExceptionsAttribute.cs create mode 100644 Src/Common/FwUtils/FwUtilsTests/Attributes/SuppressAssertDialogsAttribute.cs create mode 100644 Src/InstallValidator/InstallValidatorTests/TestAssemblyInfo.cs create mode 100644 Src/InstallValidator/InstallerArtifactsTests/TestAssemblyInfo.cs diff --git a/Build/scripts/Invoke-CppTest.ps1 b/Build/scripts/Invoke-CppTest.ps1 index 2c349085e2..9ec28e7c59 100644 --- a/Build/scripts/Invoke-CppTest.ps1 +++ b/Build/scripts/Invoke-CppTest.ps1 @@ -84,6 +84,8 @@ Initialize-VsDevEnvironment # Suppress assertion dialog boxes (DebugProcs.dll checks this env var) # This prevents tests from blocking on MessageBox popups $env:AssertUiEnabled = 'false' +# Unconditional test-mode override: bypasses registry AssertMessageBox key in DebugProcs.dll +$env:FW_TEST_MODE = '1' # Suppress Windows Error Reporting and crash dialogs # SEM_FAILCRITICALERRORS = 0x0001 diff --git a/Src/AppForTests.config b/Src/AppForTests.config index 71ef27fed2..42a8ee6d0c 100644 --- a/Src/AppForTests.config +++ b/Src/AppForTests.config @@ -4,6 +4,7 @@ + diff --git a/Src/AssemblyInfoForTests.cs b/Src/AssemblyInfoForTests.cs index 5cd47f26dd..944ef0ef60 100644 --- a/Src/AssemblyInfoForTests.cs +++ b/Src/AssemblyInfoForTests.cs @@ -16,6 +16,12 @@ // Set stub for messagebox so that we don't pop up a message box when running tests. [assembly: SetMessageBoxAdapter] +// Log last-chance managed exceptions to console output before process termination. +[assembly: LogUnhandledExceptions] + +// Suppress all assertion dialog boxes (native + managed) regardless of config file coverage +[assembly: SuppressAssertDialogs] + // Cleanup all singletons after running tests [assembly: CleanupSingletons] diff --git a/Src/AssemblyInfoForUiIndependentTests.cs b/Src/AssemblyInfoForUiIndependentTests.cs index e827eb58a9..e44755f916 100644 --- a/Src/AssemblyInfoForUiIndependentTests.cs +++ b/Src/AssemblyInfoForUiIndependentTests.cs @@ -10,6 +10,9 @@ // This file is for test fixtures for UI independent projects, i.e. projects that don't // reference System.Windows.Forms et al. +// Log last-chance managed exceptions to console output before process termination. +[assembly: LogUnhandledExceptions] + // Cleanup all singletons after running tests [assembly: CleanupSingletons] diff --git a/Src/Common/FwUtils/FwUtilsTests/Attributes/HandleApplicationThreadExceptionAttribute.cs b/Src/Common/FwUtils/FwUtilsTests/Attributes/HandleApplicationThreadExceptionAttribute.cs index 8169541043..8e28c5237a 100644 --- a/Src/Common/FwUtils/FwUtilsTests/Attributes/HandleApplicationThreadExceptionAttribute.cs +++ b/Src/Common/FwUtils/FwUtilsTests/Attributes/HandleApplicationThreadExceptionAttribute.cs @@ -42,6 +42,10 @@ public override void AfterTest(ITest test) private void OnThreadException(object sender, ThreadExceptionEventArgs e) { + Console.Error.WriteLine("Unhandled Windows Forms thread exception during test run:"); + Console.Error.WriteLine(e.Exception.ToString()); + Console.Error.Flush(); + throw new ApplicationException(e.Exception.Message, e.Exception); } } diff --git a/Src/Common/FwUtils/FwUtilsTests/Attributes/LogUnhandledExceptionsAttribute.cs b/Src/Common/FwUtils/FwUtilsTests/Attributes/LogUnhandledExceptionsAttribute.cs new file mode 100644 index 0000000000..77cc166b99 --- /dev/null +++ b/Src/Common/FwUtils/FwUtilsTests/Attributes/LogUnhandledExceptionsAttribute.cs @@ -0,0 +1,88 @@ +// Copyright (c) 2026 SIL International +// This software is licensed under the LGPL, version 2.1 or later +// (http://www.gnu.org/licenses/lgpl-2.1.html) + +using System; +using System.Threading.Tasks; +using NUnit.Framework; +using NUnit.Framework.Interfaces; + +namespace SIL.FieldWorks.Common.FwUtils.Attributes +{ + /// + /// Assembly-level test bootstrap that logs last-chance managed exceptions to the + /// console so unattended test runs always have readable failure details. + /// + /// This is intentionally a logging-only hook. It does not try to recover or keep + /// the process alive after an unhandled exception. + /// + [AttributeUsage(AttributeTargets.Assembly)] + public class LogUnhandledExceptionsAttribute : TestActionAttribute + { + private UnhandledExceptionEventHandler m_unhandledExceptionHandler; + private EventHandler m_unobservedTaskExceptionHandler; + + /// + public override ActionTargets Targets => ActionTargets.Suite; + + /// + public override void BeforeTest(ITest test) + { + base.BeforeTest(test); + + m_unhandledExceptionHandler = OnUnhandledException; + AppDomain.CurrentDomain.UnhandledException += m_unhandledExceptionHandler; + + m_unobservedTaskExceptionHandler = OnUnobservedTaskException; + TaskScheduler.UnobservedTaskException += m_unobservedTaskExceptionHandler; + } + + /// + public override void AfterTest(ITest test) + { + if (m_unhandledExceptionHandler != null) + { + AppDomain.CurrentDomain.UnhandledException -= m_unhandledExceptionHandler; + m_unhandledExceptionHandler = null; + } + + if (m_unobservedTaskExceptionHandler != null) + { + TaskScheduler.UnobservedTaskException -= m_unobservedTaskExceptionHandler; + m_unobservedTaskExceptionHandler = null; + } + + base.AfterTest(test); + } + + private static void OnUnhandledException(object sender, UnhandledExceptionEventArgs e) + { + Console.Error.WriteLine("Unhandled managed exception during test run:"); + Console.Error.WriteLine($"IsTerminating: {e.IsTerminating}"); + Console.Error.WriteLine(e.ExceptionObject?.ToString() ?? ""); + Console.Error.Flush(); + } + + private static void OnUnobservedTaskException(object sender, UnobservedTaskExceptionEventArgs e) + { + Console.Error.WriteLine("Unobserved task exception during test run:"); + Console.Error.WriteLine(e.Exception.ToString()); + Console.Error.Flush(); + + // Escalate instead of allowing the exception to be quietly ignored at finalization time. + throw new UnobservedTaskExceptionLoggedException(e.Exception); + } + } + + /// + /// Exception used to surface unobserved task failures in test output after the original + /// AggregateException has been written to the console. + /// + public class UnobservedTaskExceptionLoggedException : Exception + { + public UnobservedTaskExceptionLoggedException(AggregateException innerException) + : base("Unobserved task exception during test run.", innerException) + { + } + } +} \ No newline at end of file diff --git a/Src/Common/FwUtils/FwUtilsTests/Attributes/SuppressAssertDialogsAttribute.cs b/Src/Common/FwUtils/FwUtilsTests/Attributes/SuppressAssertDialogsAttribute.cs new file mode 100644 index 0000000000..dc07d87e56 --- /dev/null +++ b/Src/Common/FwUtils/FwUtilsTests/Attributes/SuppressAssertDialogsAttribute.cs @@ -0,0 +1,143 @@ +// Copyright (c) 2026 SIL International +// This software is licensed under the LGPL, version 2.1 or later +// (http://www.gnu.org/licenses/lgpl-2.1.html) + +using System; +using System.Diagnostics; +using NUnit.Framework; +using NUnit.Framework.Interfaces; + +namespace SIL.FieldWorks.Common.FwUtils.Attributes +{ + /// + /// NUnit assembly-level attribute that suppresses all assertion dialog boxes during tests. + /// Sets environment variables that DebugProcs.dll (native) and EnvVarTraceListener (managed) + /// honor, and ensures debug trace/assert output is mirrored to the console. + /// + /// For test projects that link AppForTests.config, EnvVarTraceListener already handles + /// assert-to-exception conversion and file logging. For the remaining projects, this + /// attribute installs a listener that converts Debug.Fail into test failures. + /// + [AttributeUsage(AttributeTargets.Assembly)] + public class SuppressAssertDialogsAttribute : TestActionAttribute + { + private ConsoleErrorTraceListener m_listener; + + /// + public override ActionTargets Targets => ActionTargets.Suite; + + /// + public override void BeforeTest(ITest test) + { + base.BeforeTest(test); + + // Force environment variables that control native (DebugProcs.dll) and + // managed (EnvVarTraceListener) assertion behavior. + Environment.SetEnvironmentVariable("AssertUiEnabled", "false"); + Environment.SetEnvironmentVariable("AssertExceptionEnabled", "true"); + Environment.SetEnvironmentVariable("FW_TEST_MODE", "1"); + + // If EnvVarTraceListener is already installed (via AppForTests.config), keep + // its assert-to-exception and file logging behavior, but still mirror output + // to the console. Otherwise, our listener is responsible for failing tests. + bool hasEnvVarListener = false; + foreach (TraceListener listener in Trace.Listeners) + { + // Check by type name to avoid a hard dependency on SIL.LCModel.Utils. + if (listener.GetType().Name == "EnvVarTraceListener") + { + hasEnvVarListener = true; + break; + } + } + + // Suppress the DefaultTraceListener dialog even when another listener is + // present so assertions never degrade back to modal UI. + foreach (TraceListener listener in Trace.Listeners) + { + if (listener is DefaultTraceListener dtl) + { + dtl.AssertUiEnabled = false; + break; + } + } + + m_listener = new ConsoleErrorTraceListener(throwOnFail: !hasEnvVarListener); + Trace.Listeners.Insert(0, m_listener); + } + + /// + public override void AfterTest(ITest test) + { + if (m_listener != null) + { + Trace.Listeners.Remove(m_listener); + m_listener = null; + } + + base.AfterTest(test); + } + } + + /// + /// Mirrors debug trace output to Console.Error and optionally converts + /// Debug.Fail/failed Debug.Assert calls into exceptions. + /// + internal class ConsoleErrorTraceListener : TraceListener + { + private readonly bool m_throwOnFail; + + public ConsoleErrorTraceListener(bool throwOnFail) + { + m_throwOnFail = throwOnFail; + } + + public override void Fail(string message) + { + WriteFailure(message, null); + } + + public override void Fail(string message, string detailMessage) + { + WriteFailure(message, detailMessage); + } + + public override void Write(string message) + { + Console.Error.Write(message); + Console.Error.Flush(); + } + + public override void WriteLine(string message) + { + Console.Error.WriteLine(message); + Console.Error.Flush(); + } + + private void WriteFailure(string message, string detailMessage) + { + var full = string.IsNullOrEmpty(detailMessage) + ? message + : $"{message}{Environment.NewLine}{detailMessage}"; + + Console.Error.WriteLine("Debug.Fail/Assert fired during test:"); + Console.Error.WriteLine(full); + Console.Error.Flush(); + + if (m_throwOnFail) + throw new AssertionDialogException(full); + } + } + + /// + /// Exception thrown when a Debug.Fail or Debug.Assert fires during a test, + /// replacing the modal Abort/Retry/Ignore dialog with a clear test failure. + /// + public class AssertionDialogException : Exception + { + public AssertionDialogException(string message) + : base($"Debug.Fail/Assert fired during test (would have shown a modal dialog):\n{message}") + { + } + } +} diff --git a/Src/DebugProcs/DebugProcs.cpp b/Src/DebugProcs/DebugProcs.cpp index 06d7f8ecea..6d65754ea6 100644 --- a/Src/DebugProcs/DebugProcs.cpp +++ b/Src/DebugProcs/DebugProcs.cpp @@ -286,10 +286,29 @@ extern "C" __declspec(dllexport) int APIENTRY DebugProcsExit(void) /*---------------------------------------------------------------------------------------------- Returns the AssertMessageBox value from the registry; if not set return the value of the - environment variable AssertUiEnabled; if not set return true + environment variable AssertUiEnabled; if not set return true. + FW_TEST_MODE=1 takes absolute priority and always returns false (no dialog). ----------------------------------------------------------------------------------------------*/ bool GetShowAssertMessageBox() { +// getenv is deprecated on Windows +#if defined(WIN32) || defined(WIN64) +#pragma warning(push) +#pragma warning(disable: 4996) + +// Windows doesn't know strcasecmp, it calls it stricmp instead... +#ifndef strcasecmp +#define strcasecmp stricmp +#endif +#endif // WIN32 + + // FW_TEST_MODE is an unconditional override set by test runners. + // It takes priority over both the registry and AssertUiEnabled so that + // developer machines with the registry key set never pop dialogs during tests. + const char* pTestMode = getenv("FW_TEST_MODE"); + if (pTestMode && strcasecmp(pTestMode, "1") == 0) + return false; + #if defined(WIN32) || defined(WIN64) HKEY hk; if (::RegOpenKeyEx(HKEY_LOCAL_MACHINE, "Software\\SIL\\FieldWorks", 0, @@ -304,15 +323,6 @@ bool GetShowAssertMessageBox() if (ret == ERROR_SUCCESS) return fShowAssertMessageBox ? true : false; // otherwise we get a performance warning } -// getenv is deprecated on Windows -#pragma warning(push) -#pragma warning(disable: 4996) - -// Windows doesn't know strcasecmp, it calls it stricmp instead... -#ifndef strcasecmp -#define strcasecmp stricmp -#endif - #endif // WIN32 const char* pEnvVar = getenv("AssertUiEnabled"); return !pEnvVar || @@ -581,6 +591,11 @@ void __cdecl SilAssert ( else OutputDebugString(assertbuf); + // Always mirror assertion text to the process error stream so test runners, + // humans, and automation can see the failure details without a debugger. + fprintf(stderr, "%s\n", assertbuf); + fflush(stderr); + // NOTE: this method is intented to be used by unmanaged apps only; // managed apps should use DebugProcs.AssertProc in DebugProcs.cs diff --git a/Src/Generic/Test/TestGeneric.vcxproj b/Src/Generic/Test/TestGeneric.vcxproj index 895df81cbe..b5b7ea1784 100644 --- a/Src/Generic/Test/TestGeneric.vcxproj +++ b/Src/Generic/Test/TestGeneric.vcxproj @@ -86,6 +86,7 @@ $(FwRoot)\Include; $(FwRoot)\Src\Generic; $(FwRoot)\Src\Generic\Test; + $(FwRoot)\Src\DebugProcs; $(FwRoot)\Output\$(Configuration); $(FwRoot)\Output\$(Configuration)\Common; $(IntDir); diff --git a/Src/Generic/Test/testGeneric.cpp b/Src/Generic/Test/testGeneric.cpp index aeeb288376..d20ccbe13f 100644 --- a/Src/Generic/Test/testGeneric.cpp +++ b/Src/Generic/Test/testGeneric.cpp @@ -11,11 +11,13 @@ Last reviewed: -------------------------------------------------------------------------------*//*:End Ignore*/ #include "testGenericLib.h" #include "RedirectHKCU.h" +#include "DebugProcs.h" namespace unitpp { void GlobalSetup(bool verbose) { + ShowAssertMessageBox(0); // Disable assertion dialogs #if defined(WIN32) || defined(WIN64) ModuleEntry::DllMain(0, DLL_PROCESS_ATTACH); #endif diff --git a/Src/InstallValidator/InstallValidatorTests/InstallValidatorTests.csproj b/Src/InstallValidator/InstallValidatorTests/InstallValidatorTests.csproj index b5f34fede6..5a6544524e 100644 --- a/Src/InstallValidator/InstallValidatorTests/InstallValidatorTests.csproj +++ b/Src/InstallValidator/InstallValidatorTests/InstallValidatorTests.csproj @@ -50,5 +50,8 @@ Properties\CommonAssemblyInfo.cs + + Attributes\LogUnhandledExceptionsAttribute.cs + \ No newline at end of file diff --git a/Src/InstallValidator/InstallValidatorTests/TestAssemblyInfo.cs b/Src/InstallValidator/InstallValidatorTests/TestAssemblyInfo.cs new file mode 100644 index 0000000000..e917e2ee41 --- /dev/null +++ b/Src/InstallValidator/InstallValidatorTests/TestAssemblyInfo.cs @@ -0,0 +1,7 @@ +// Copyright (c) 2026 SIL International +// This software is licensed under the LGPL, version 2.1 or later +// (http://www.gnu.org/licenses/lgpl-2.1.html) + +using SIL.FieldWorks.Common.FwUtils.Attributes; + +[assembly: LogUnhandledExceptions] \ No newline at end of file diff --git a/Src/InstallValidator/InstallerArtifactsTests/InstallerArtifactsTests.csproj b/Src/InstallValidator/InstallerArtifactsTests/InstallerArtifactsTests.csproj index c50355ad5a..2e7f2c832d 100644 --- a/Src/InstallValidator/InstallerArtifactsTests/InstallerArtifactsTests.csproj +++ b/Src/InstallValidator/InstallerArtifactsTests/InstallerArtifactsTests.csproj @@ -28,9 +28,15 @@ + + + Properties\CommonAssemblyInfo.cs + + Attributes\LogUnhandledExceptionsAttribute.cs + diff --git a/Src/InstallValidator/InstallerArtifactsTests/TestAssemblyInfo.cs b/Src/InstallValidator/InstallerArtifactsTests/TestAssemblyInfo.cs new file mode 100644 index 0000000000..e917e2ee41 --- /dev/null +++ b/Src/InstallValidator/InstallerArtifactsTests/TestAssemblyInfo.cs @@ -0,0 +1,7 @@ +// Copyright (c) 2026 SIL International +// This software is licensed under the LGPL, version 2.1 or later +// (http://www.gnu.org/licenses/lgpl-2.1.html) + +using SIL.FieldWorks.Common.FwUtils.Attributes; + +[assembly: LogUnhandledExceptions] \ No newline at end of file diff --git a/Test.runsettings b/Test.runsettings index 0aac51d02d..cc33ad56c0 100644 --- a/Test.runsettings +++ b/Test.runsettings @@ -50,6 +50,9 @@ true + + + 1 diff --git a/test.ps1 b/test.ps1 index b5f5c81238..9bc759e921 100644 --- a/test.ps1 +++ b/test.ps1 @@ -224,6 +224,7 @@ try { # invoked outside the .runsettings flow. $env:AssertUiEnabled = 'false' $env:AssertExceptionEnabled = 'true' + $env:FW_TEST_MODE = '1' $outputDir = Join-Path $PSScriptRoot "Output/$Configuration" From 4564c757deaf06b2e12057536f2fb28dd218f2ba Mon Sep 17 00:00:00 2001 From: John Lambert Date: Fri, 20 Mar 2026 13:01:27 -0400 Subject: [PATCH 2/4] test: fail unobserved task exceptions deterministically --- .../LogUnhandledExceptionsAttribute.cs | 86 ++++++++++++++++--- .../LogUnhandledExceptionsAttributeTests.cs | 61 +++++++++++++ 2 files changed, 134 insertions(+), 13 deletions(-) create mode 100644 Src/Common/FwUtils/FwUtilsTests/LogUnhandledExceptionsAttributeTests.cs diff --git a/Src/Common/FwUtils/FwUtilsTests/Attributes/LogUnhandledExceptionsAttribute.cs b/Src/Common/FwUtils/FwUtilsTests/Attributes/LogUnhandledExceptionsAttribute.cs index 77cc166b99..c0e133f59b 100644 --- a/Src/Common/FwUtils/FwUtilsTests/Attributes/LogUnhandledExceptionsAttribute.cs +++ b/Src/Common/FwUtils/FwUtilsTests/Attributes/LogUnhandledExceptionsAttribute.cs @@ -3,6 +3,8 @@ // (http://www.gnu.org/licenses/lgpl-2.1.html) using System; +using System.Collections.Concurrent; +using System.Text; using System.Threading.Tasks; using NUnit.Framework; using NUnit.Framework.Interfaces; @@ -19,6 +21,9 @@ namespace SIL.FieldWorks.Common.FwUtils.Attributes [AttributeUsage(AttributeTargets.Assembly)] public class LogUnhandledExceptionsAttribute : TestActionAttribute { + private static readonly ConcurrentQueue s_unobservedTaskExceptions = + new ConcurrentQueue(); + private UnhandledExceptionEventHandler m_unhandledExceptionHandler; private EventHandler m_unobservedTaskExceptionHandler; @@ -29,6 +34,7 @@ public class LogUnhandledExceptionsAttribute : TestActionAttribute public override void BeforeTest(ITest test) { base.BeforeTest(test); + ResetCapturedUnobservedTaskExceptionsForTesting(); m_unhandledExceptionHandler = OnUnhandledException; AppDomain.CurrentDomain.UnhandledException += m_unhandledExceptionHandler; @@ -40,6 +46,8 @@ public override void BeforeTest(ITest test) /// public override void AfterTest(ITest test) { + var unobservedTaskExceptions = FlushAndDrainCapturedUnobservedTaskExceptions(); + if (m_unhandledExceptionHandler != null) { AppDomain.CurrentDomain.UnhandledException -= m_unhandledExceptionHandler; @@ -53,6 +61,8 @@ public override void AfterTest(ITest test) } base.AfterTest(test); + + ThrowIfCapturedUnobservedTaskExceptions(unobservedTaskExceptions); } private static void OnUnhandledException(object sender, UnhandledExceptionEventArgs e) @@ -63,26 +73,76 @@ private static void OnUnhandledException(object sender, UnhandledExceptionEventA Console.Error.Flush(); } - private static void OnUnobservedTaskException(object sender, UnobservedTaskExceptionEventArgs e) + internal static void OnUnobservedTaskException( + object sender, + UnobservedTaskExceptionEventArgs e + ) { Console.Error.WriteLine("Unobserved task exception during test run:"); Console.Error.WriteLine(e.Exception.ToString()); Console.Error.Flush(); - // Escalate instead of allowing the exception to be quietly ignored at finalization time. - throw new UnobservedTaskExceptionLoggedException(e.Exception); + s_unobservedTaskExceptions.Enqueue(e.Exception); + e.SetObserved(); } - } - /// - /// Exception used to surface unobserved task failures in test output after the original - /// AggregateException has been written to the console. - /// - public class UnobservedTaskExceptionLoggedException : Exception - { - public UnobservedTaskExceptionLoggedException(AggregateException innerException) - : base("Unobserved task exception during test run.", innerException) + internal static void ResetCapturedUnobservedTaskExceptionsForTesting() + { + AggregateException ignored; + while (s_unobservedTaskExceptions.TryDequeue(out ignored)) { } + } + + internal static AggregateException[] FlushAndDrainCapturedUnobservedTaskExceptions() + { + for (int i = 0; i < 3; i++) + { + GC.Collect(); + GC.WaitForPendingFinalizers(); + } + + return DrainCapturedUnobservedTaskExceptions(); + } + + internal static AggregateException[] DrainCapturedUnobservedTaskExceptions() + { + var exceptions = new ConcurrentQueue(); + AggregateException capturedException; + while (s_unobservedTaskExceptions.TryDequeue(out capturedException)) + exceptions.Enqueue(capturedException); + + return exceptions.ToArray(); + } + + internal static void ThrowIfCapturedUnobservedTaskExceptions( + AggregateException[] exceptions + ) + { + if (exceptions == null || exceptions.Length == 0) + return; + + throw new AssertionException(BuildFailureMessage(exceptions)); + } + + internal static string BuildFailureMessage(AggregateException[] exceptions) { + var builder = new StringBuilder(); + builder.AppendLine( + string.Format( + "{0} unobserved task exception(s) were detected during the test run.", + exceptions.Length + ) + ); + builder.AppendLine( + "These exceptions were captured from TaskScheduler.UnobservedTaskException and surfaced at suite teardown so the test host fails deterministically without crashing on the finalizer thread." + ); + + for (int i = 0; i < exceptions.Length; i++) + { + builder.AppendLine(); + builder.AppendLine(string.Format("[{0}] {1}", i + 1, exceptions[i])); + } + + return builder.ToString(); } } -} \ No newline at end of file +} diff --git a/Src/Common/FwUtils/FwUtilsTests/LogUnhandledExceptionsAttributeTests.cs b/Src/Common/FwUtils/FwUtilsTests/LogUnhandledExceptionsAttributeTests.cs new file mode 100644 index 0000000000..cb5434313c --- /dev/null +++ b/Src/Common/FwUtils/FwUtilsTests/LogUnhandledExceptionsAttributeTests.cs @@ -0,0 +1,61 @@ +// Copyright (c) 2026 SIL International +// This software is licensed under the LGPL, version 2.1 or later +// (http://www.gnu.org/licenses/lgpl-2.1.html) + +using System; +using System.Threading.Tasks; +using NUnit.Framework; +using SIL.FieldWorks.Common.FwUtils.Attributes; + +namespace SIL.FieldWorks.Common.FwUtils +{ + [TestFixture] + public class LogUnhandledExceptionsAttributeTests + { + [SetUp] + public void SetUp() + { + LogUnhandledExceptionsAttribute.ResetCapturedUnobservedTaskExceptionsForTesting(); + } + + [TearDown] + public void TearDown() + { + LogUnhandledExceptionsAttribute.ResetCapturedUnobservedTaskExceptionsForTesting(); + } + + [Test] + public void OnUnobservedTaskException_RecordsExceptionWithoutThrowing() + { + var aggregateException = new AggregateException(new InvalidOperationException("boom")); + var eventArgs = new UnobservedTaskExceptionEventArgs(aggregateException); + + Assert.DoesNotThrow(() => LogUnhandledExceptionsAttribute.OnUnobservedTaskException(this, eventArgs)); + CollectionAssert.AreEqual( + new[] { aggregateException }, + LogUnhandledExceptionsAttribute.DrainCapturedUnobservedTaskExceptions()); + } + + [Test] + public void ThrowIfCapturedUnobservedTaskExceptions_DoesNothingWhenNoExceptionsWereCaptured() + { + Assert.DoesNotThrow(() => LogUnhandledExceptionsAttribute.ThrowIfCapturedUnobservedTaskExceptions( + LogUnhandledExceptionsAttribute.DrainCapturedUnobservedTaskExceptions())); + } + + [Test] + public void ThrowIfCapturedUnobservedTaskExceptions_ThrowsAssertionExceptionWithCapturedDetails() + { + var first = new AggregateException(new InvalidOperationException("first failure")); + var second = new AggregateException(new ApplicationException("second failure")); + + var exception = Assert.Throws(() => + LogUnhandledExceptionsAttribute.ThrowIfCapturedUnobservedTaskExceptions(new[] { first, second })); + + Assert.That(exception.Message, Does.Contain("2 unobserved task exception(s)")); + Assert.That(exception.Message, Does.Contain("first failure")); + Assert.That(exception.Message, Does.Contain("second failure")); + Assert.That(exception.Message, Does.Contain("fails deterministically")); + } + } +} \ No newline at end of file From 8157ee33227fb155cc56e46d064747d9199c9306 Mon Sep 17 00:00:00 2001 From: John Lambert Date: Fri, 20 Mar 2026 13:01:28 -0400 Subject: [PATCH 3/4] Finish test exception cleanup follow-ups --- Src/AppForTests.config | 39 ++++---- Src/AssemblyInfoForUiIndependentTests.cs | 3 + .../SuppressAssertDialogsAttribute.cs | 58 +++++++----- Src/DebugProcs/DebugProcs.cpp | 89 +++++++++++++------ 4 files changed, 124 insertions(+), 65 deletions(-) diff --git a/Src/AppForTests.config b/Src/AppForTests.config index 42a8ee6d0c..eac76783eb 100644 --- a/Src/AppForTests.config +++ b/Src/AppForTests.config @@ -1,21 +1,22 @@ - + - - - - - - - - - - - - - - - + + + + + + + + + + + + + + diff --git a/Src/AssemblyInfoForUiIndependentTests.cs b/Src/AssemblyInfoForUiIndependentTests.cs index e44755f916..4b079e4cf8 100644 --- a/Src/AssemblyInfoForUiIndependentTests.cs +++ b/Src/AssemblyInfoForUiIndependentTests.cs @@ -13,6 +13,9 @@ // Log last-chance managed exceptions to console output before process termination. [assembly: LogUnhandledExceptions] +// Suppress all assertion dialog boxes (native + managed) regardless of config file coverage +[assembly: SuppressAssertDialogs] + // Cleanup all singletons after running tests [assembly: CleanupSingletons] diff --git a/Src/Common/FwUtils/FwUtilsTests/Attributes/SuppressAssertDialogsAttribute.cs b/Src/Common/FwUtils/FwUtilsTests/Attributes/SuppressAssertDialogsAttribute.cs index dc07d87e56..e5802ab810 100644 --- a/Src/Common/FwUtils/FwUtilsTests/Attributes/SuppressAssertDialogsAttribute.cs +++ b/Src/Common/FwUtils/FwUtilsTests/Attributes/SuppressAssertDialogsAttribute.cs @@ -22,6 +22,9 @@ namespace SIL.FieldWorks.Common.FwUtils.Attributes public class SuppressAssertDialogsAttribute : TestActionAttribute { private ConsoleErrorTraceListener m_listener; + private string m_previousAssertUiEnabled; + private string m_previousAssertExceptionEnabled; + private string m_previousTestMode; /// public override ActionTargets Targets => ActionTargets.Suite; @@ -33,22 +36,29 @@ public override void BeforeTest(ITest test) // Force environment variables that control native (DebugProcs.dll) and // managed (EnvVarTraceListener) assertion behavior. + m_previousAssertUiEnabled = Environment.GetEnvironmentVariable("AssertUiEnabled"); + m_previousAssertExceptionEnabled = Environment.GetEnvironmentVariable( + "AssertExceptionEnabled" + ); + m_previousTestMode = Environment.GetEnvironmentVariable("FW_TEST_MODE"); + Environment.SetEnvironmentVariable("AssertUiEnabled", "false"); Environment.SetEnvironmentVariable("AssertExceptionEnabled", "true"); Environment.SetEnvironmentVariable("FW_TEST_MODE", "1"); // If EnvVarTraceListener is already installed (via AppForTests.config), keep - // its assert-to-exception and file logging behavior, but still mirror output - // to the console. Otherwise, our listener is responsible for failing tests. + // its assert-to-exception and file logging behavior. Otherwise, our listener + // is responsible for converting Debug.Fail into a test failure. bool hasEnvVarListener = false; + bool hasConsoleErrorListener = false; foreach (TraceListener listener in Trace.Listeners) { // Check by type name to avoid a hard dependency on SIL.LCModel.Utils. if (listener.GetType().Name == "EnvVarTraceListener") - { hasEnvVarListener = true; - break; - } + + if (listener is ConsoleErrorTraceListener) + hasConsoleErrorListener = true; } // Suppress the DefaultTraceListener dialog even when another listener is @@ -62,8 +72,11 @@ public override void BeforeTest(ITest test) } } - m_listener = new ConsoleErrorTraceListener(throwOnFail: !hasEnvVarListener); - Trace.Listeners.Insert(0, m_listener); + if (!hasConsoleErrorListener) + { + m_listener = new ConsoleErrorTraceListener(throwOnFail: !hasEnvVarListener); + Trace.Listeners.Insert(0, m_listener); + } } /// @@ -75,12 +88,23 @@ public override void AfterTest(ITest test) m_listener = null; } + Environment.SetEnvironmentVariable("AssertUiEnabled", m_previousAssertUiEnabled); + Environment.SetEnvironmentVariable( + "AssertExceptionEnabled", + m_previousAssertExceptionEnabled + ); + Environment.SetEnvironmentVariable("FW_TEST_MODE", m_previousTestMode); + + m_previousAssertUiEnabled = null; + m_previousAssertExceptionEnabled = null; + m_previousTestMode = null; + base.AfterTest(test); } } /// - /// Mirrors debug trace output to Console.Error and optionally converts + /// Writes debug failure details to Console.Error and optionally converts /// Debug.Fail/failed Debug.Assert calls into exceptions. /// internal class ConsoleErrorTraceListener : TraceListener @@ -102,17 +126,9 @@ public override void Fail(string message, string detailMessage) WriteFailure(message, detailMessage); } - public override void Write(string message) - { - Console.Error.Write(message); - Console.Error.Flush(); - } + public override void Write(string message) { } - public override void WriteLine(string message) - { - Console.Error.WriteLine(message); - Console.Error.Flush(); - } + public override void WriteLine(string message) { } private void WriteFailure(string message, string detailMessage) { @@ -136,8 +152,8 @@ private void WriteFailure(string message, string detailMessage) public class AssertionDialogException : Exception { public AssertionDialogException(string message) - : base($"Debug.Fail/Assert fired during test (would have shown a modal dialog):\n{message}") - { - } + : base( + $"Debug.Fail/Assert fired during test (would have shown a modal dialog):\n{message}" + ) { } } } diff --git a/Src/DebugProcs/DebugProcs.cpp b/Src/DebugProcs/DebugProcs.cpp index 6d65754ea6..75452de3f2 100644 --- a/Src/DebugProcs/DebugProcs.cpp +++ b/Src/DebugProcs/DebugProcs.cpp @@ -19,6 +19,12 @@ Last reviewed: #include #include #endif +#include +#if defined(WIN32) || defined(WIN64) +#include +#else +#include +#endif #include #include #if defined(WIN32) || defined(WIN64) @@ -38,6 +44,46 @@ void APIENTRY DefWarnProc(const char * pszExp, const char * pszFile, int nLine, void APIENTRY DefAssertProc(const char * pszExp, const char * pszFile, int nLine, HMODULE hmod); bool GetShowAssertMessageBox(); +static bool TryGetEnvironmentVariableValue(const char * pszName, char ** ppszValue) +{ +#if defined(WIN32) || defined(WIN64) + size_t cchValue = 0; + return _dupenv_s(ppszValue, &cchValue, pszName) == 0 && *ppszValue != NULL; +#else + *ppszValue = getenv(pszName); + return *ppszValue != NULL; +#endif +} + +static void FreeEnvironmentVariableValue(char * pszValue) +{ +#if defined(WIN32) || defined(WIN64) + free(pszValue); +#else + (void)pszValue; +#endif +} + +static bool EqualsIgnoreCase(const char * pszLeft, const char * pszRight) +{ +#if defined(WIN32) || defined(WIN64) + return _stricmp(pszLeft, pszRight) == 0; +#else + return strcasecmp(pszLeft, pszRight) == 0; +#endif +} + +static bool IsTestModeEnabled() +{ + char * pTestMode = NULL; + if (!TryGetEnvironmentVariableValue("FW_TEST_MODE", &pTestMode)) + return false; + + const bool fIsTestMode = strcmp(pTestMode, "1") == 0; + FreeEnvironmentVariableValue(pTestMode); + return fIsTestMode; +} + typedef void (__stdcall * _DBG_REPORT_HOOK)(int, char *); typedef int (__stdcall * _DBG_DISPLAYMSGBOX_HOOK)(char *); @@ -291,22 +337,10 @@ extern "C" __declspec(dllexport) int APIENTRY DebugProcsExit(void) ----------------------------------------------------------------------------------------------*/ bool GetShowAssertMessageBox() { -// getenv is deprecated on Windows -#if defined(WIN32) || defined(WIN64) -#pragma warning(push) -#pragma warning(disable: 4996) - -// Windows doesn't know strcasecmp, it calls it stricmp instead... -#ifndef strcasecmp -#define strcasecmp stricmp -#endif -#endif // WIN32 - // FW_TEST_MODE is an unconditional override set by test runners. // It takes priority over both the registry and AssertUiEnabled so that // developer machines with the registry key set never pop dialogs during tests. - const char* pTestMode = getenv("FW_TEST_MODE"); - if (pTestMode && strcasecmp(pTestMode, "1") == 0) + if (IsTestModeEnabled()) return false; #if defined(WIN32) || defined(WIN64) @@ -324,14 +358,16 @@ bool GetShowAssertMessageBox() return fShowAssertMessageBox ? true : false; // otherwise we get a performance warning } #endif // WIN32 - const char* pEnvVar = getenv("AssertUiEnabled"); - return !pEnvVar || - strcasecmp(pEnvVar, "0") != 0 && - strcasecmp(pEnvVar, "false") != 0 && - strcasecmp(pEnvVar, "no") != 0; -#if defined(WIN32) || defined(WIN64) -#pragma warning(pop) -#endif // WIN32 + char * pEnvVar = NULL; + if (!TryGetEnvironmentVariableValue("AssertUiEnabled", &pEnvVar)) + return true; + + const bool fShowAssertMessageBox = + !EqualsIgnoreCase(pEnvVar, "0") && + !EqualsIgnoreCase(pEnvVar, "false") && + !EqualsIgnoreCase(pEnvVar, "no"); + FreeEnvironmentVariableValue(pEnvVar); + return fShowAssertMessageBox; } /*---------------------------------------------------------------------------------------------- @@ -591,10 +627,13 @@ void __cdecl SilAssert ( else OutputDebugString(assertbuf); - // Always mirror assertion text to the process error stream so test runners, - // humans, and automation can see the failure details without a debugger. - fprintf(stderr, "%s\n", assertbuf); - fflush(stderr); + if (IsTestModeEnabled()) + { + // In test mode, mirror assertion text to stderr so unattended runs capture + // the failure details without depending on a debugger or modal UI. + fprintf(stderr, "%s\n", assertbuf); + fflush(stderr); + } // NOTE: this method is intented to be used by unmanaged apps only; // managed apps should use DebugProcs.AssertProc in DebugProcs.cs From 3374c5d4d4ad8b7c7fb144e88a746bdf26c5205c Mon Sep 17 00:00:00 2001 From: John Lambert Date: Fri, 20 Mar 2026 13:01:28 -0400 Subject: [PATCH 4/4] Final updates for no-popups --- Build/scripts/Invoke-CppTest.ps1 | 34 +++- Lib/src/unit++/main.cc | 118 ++++++++++++- .../LogUnhandledExceptionsAttributeTests.cs | 11 ++ .../SuppressAssertDialogsAttributeTests.cs | 158 ++++++++++++++++++ Src/DebugProcs/DebugProcs.cpp | 7 +- Src/Generic/Test/TestErrorHandling.h | 27 +++ Src/Generic/Test/testGeneric.cpp | 70 ++++++++ .../InstallValidatorTests.csproj | 3 + .../InstallValidatorTests/TestAssemblyInfo.cs | 3 +- .../InstallerArtifactsTests.csproj | 3 + .../TestAssemblyInfo.cs | 3 +- Src/views/Test/testViews.cpp | 70 ++++++++ test.ps1 | 20 +++ 13 files changed, 513 insertions(+), 14 deletions(-) create mode 100644 Src/Common/FwUtils/FwUtilsTests/SuppressAssertDialogsAttributeTests.cs diff --git a/Build/scripts/Invoke-CppTest.ps1 b/Build/scripts/Invoke-CppTest.ps1 index 9ec28e7c59..f6f2746f84 100644 --- a/Build/scripts/Invoke-CppTest.ps1 +++ b/Build/scripts/Invoke-CppTest.ps1 @@ -551,6 +551,13 @@ function Invoke-Run { } $process.WaitForExit() + $nativeExitCode = $null + try { + $nativeExitCode = $process.ExitCode + } + catch { + $nativeExitCode = $null + } $logTail = @() if (Test-Path $LogPath) { @@ -564,15 +571,34 @@ function Invoke-Run { Write-Host "--- end output ---" -ForegroundColor Yellow } - # Determine exit code: parse the Unit++ summary line from the log as the authoritative - # source. Start-Process -RedirectStandardOutput in PowerShell 5.1 can return a null - # ExitCode even after WaitForExit(), so the process exit code is not reliable here. + # Determine exit code using both the real process exit code and the Unit++ summary. + # The process exit code is authoritative for crashes/teardown failures that occur after + # the Unit++ summary has already been written. $exitCode = -1 + $summaryExitCode = $null if (-not $timedOut) { $summaryLine = $logTail | Where-Object { $_ -match 'Tests \[Ok-Fail-Error\]: \[\d+-\d+-\d+\]' } | Select-Object -Last 1 if ($summaryLine) { $m = [regex]::Match($summaryLine, 'Tests \[Ok-Fail-Error\]: \[(\d+)-(\d+)-(\d+)\]') - $exitCode = [int]$m.Groups[2].Value + [int]$m.Groups[3].Value + $summaryExitCode = [int]$m.Groups[2].Value + [int]$m.Groups[3].Value + } + + if ($terminatedAfterCompletion) { + if ($null -ne $nativeExitCode -and $nativeExitCode -ne 0) { + $exitCode = $nativeExitCode + } + else { + $exitCode = 1 + } + } + elseif ($null -ne $nativeExitCode -and $nativeExitCode -ne 0) { + $exitCode = $nativeExitCode + } + elseif ($null -ne $summaryExitCode) { + $exitCode = $summaryExitCode + } + elseif ($null -ne $nativeExitCode) { + $exitCode = $nativeExitCode } } diff --git a/Lib/src/unit++/main.cc b/Lib/src/unit++/main.cc index e38858b6d9..1e26187847 100644 --- a/Lib/src/unit++/main.cc +++ b/Lib/src/unit++/main.cc @@ -2,10 +2,86 @@ // Terms of use are in the file COPYING #include "main.h" #include +#include #include +#include +#if defined(WIN32) || defined(WIN64) +#define WINDOWS_LEAN_AND_MEAN +#include +#include +#endif using namespace std; using namespace unitpp; +#if defined(WIN32) || defined(WIN64) +namespace +{ + void TerminateOnSigAbrt(int) + { + _exit(3); + } + + typedef HRESULT (WINAPI * PfnWerGetFlags)(HANDLE, PDWORD); + typedef HRESULT (WINAPI * PfnWerSetFlags)(DWORD); + + const DWORD kWerFaultReportingNoUi = 0x00000004; + const DWORD kWerFaultReportingAlwaysShowUi = 0x00000010; + + void ConfigureWindowsErrorReportingUi() + { + DWORD errorMode = GetErrorMode(); + errorMode |= SEM_FAILCRITICALERRORS; + errorMode |= SEM_NOGPFAULTERRORBOX; + errorMode |= SEM_NOOPENFILEERRORBOX; + SetErrorMode(errorMode); + + HMODULE hWer = LoadLibraryA("wer.dll"); + if (!hWer) + return; + + PfnWerGetFlags pfnWerGetFlags = reinterpret_cast( + GetProcAddress(hWer, "WerGetFlags") + ); + PfnWerSetFlags pfnWerSetFlags = reinterpret_cast( + GetProcAddress(hWer, "WerSetFlags") + ); + + if (pfnWerSetFlags) + { + DWORD flags = 0; + if (pfnWerGetFlags) + pfnWerGetFlags(GetCurrentProcess(), &flags); + + flags |= kWerFaultReportingNoUi; + flags &= ~kWerFaultReportingAlwaysShowUi; + pfnWerSetFlags(flags); + } + + FreeLibrary(hWer); + } + + void ConfigureCrtReportUi() + { + _set_error_mode(_OUT_TO_STDERR); + + _CrtSetReportMode(_CRT_WARN, _CRTDBG_MODE_FILE); + _CrtSetReportFile(_CRT_WARN, _CRTDBG_FILE_STDERR); + _CrtSetReportMode(_CRT_ERROR, _CRTDBG_MODE_FILE); + _CrtSetReportFile(_CRT_ERROR, _CRTDBG_FILE_STDERR); + _CrtSetReportMode(_CRT_ASSERT, _CRTDBG_MODE_FILE); + _CrtSetReportFile(_CRT_ASSERT, _CRTDBG_FILE_STDERR); + + _set_abort_behavior(0, _WRITE_ABORT_MSG | _CALL_REPORTFAULT); + } + + void SuppressInteractiveCrashUi() + { + ConfigureWindowsErrorReportingUi(); + ConfigureCrtReportUi(); + } +} +#endif + bool unitpp::verbose = false; int unitpp::verbose_lvl = 0; bool unitpp::line_fmt = false; @@ -25,6 +101,9 @@ void unitpp::set_tester(test_runner* tr) int main(int argc, const char* argv[]) { +#if defined(WIN32) || defined(WIN64) + SuppressInteractiveCrashUi(); +#endif printf("DEBUG: unit++ main start\n"); fflush(stdout); options().add("v", new options_utils::opt_flag(verbose)); options().alias("verbose", "v"); @@ -42,14 +121,41 @@ int main(int argc, const char* argv[]) if (!runner) runner = &plain; - printf("DEBUG: Calling GlobalSetup\n"); fflush(stdout); - GlobalSetup(verbose); - printf("DEBUG: Returned from GlobalSetup\n"); fflush(stdout); + int retval = 0; - int retval = runner->run_tests(argc, argv) ? 0 : 1; + try { + printf("DEBUG: Calling GlobalSetup\n"); fflush(stdout); + GlobalSetup(verbose); + printf("DEBUG: Returned from GlobalSetup\n"); fflush(stdout); + } + catch (const std::exception& e) { + fprintf(stderr, "GlobalSetup threw std::exception: %s\n", e.what()); + fflush(stderr); + return 1; + } + catch (...) { + fprintf(stderr, "GlobalSetup threw an unknown exception\n"); + fflush(stderr); + return 1; + } + + retval = runner->run_tests(argc, argv) ? 0 : 1; + signal(SIGABRT, TerminateOnSigAbrt); - printf("DEBUG: Calling GlobalTeardown\n"); fflush(stdout); - GlobalTeardown(); + try { + printf("DEBUG: Calling GlobalTeardown\n"); fflush(stdout); + GlobalTeardown(); + } + catch (const std::exception& e) { + fprintf(stderr, "GlobalTeardown threw std::exception: %s\n", e.what()); + fflush(stderr); + retval = 1; + } + catch (...) { + fprintf(stderr, "GlobalTeardown threw an unknown exception\n"); + fflush(stderr); + retval = 1; + } printf("DEBUG: unit++ main end (retval=%d)\n", retval); fflush(stdout); return retval; } diff --git a/Src/Common/FwUtils/FwUtilsTests/LogUnhandledExceptionsAttributeTests.cs b/Src/Common/FwUtils/FwUtilsTests/LogUnhandledExceptionsAttributeTests.cs index cb5434313c..94c7372714 100644 --- a/Src/Common/FwUtils/FwUtilsTests/LogUnhandledExceptionsAttributeTests.cs +++ b/Src/Common/FwUtils/FwUtilsTests/LogUnhandledExceptionsAttributeTests.cs @@ -57,5 +57,16 @@ public void ThrowIfCapturedUnobservedTaskExceptions_ThrowsAssertionExceptionWith Assert.That(exception.Message, Does.Contain("second failure")); Assert.That(exception.Message, Does.Contain("fails deterministically")); } + + [Test] + public void OnUnobservedTaskException_MarksExceptionObserved() + { + var aggregateException = new AggregateException(new InvalidOperationException("boom")); + var eventArgs = new UnobservedTaskExceptionEventArgs(aggregateException); + + LogUnhandledExceptionsAttribute.OnUnobservedTaskException(this, eventArgs); + + Assert.That(eventArgs.Observed, Is.True); + } } } \ No newline at end of file diff --git a/Src/Common/FwUtils/FwUtilsTests/SuppressAssertDialogsAttributeTests.cs b/Src/Common/FwUtils/FwUtilsTests/SuppressAssertDialogsAttributeTests.cs new file mode 100644 index 0000000000..e187809572 --- /dev/null +++ b/Src/Common/FwUtils/FwUtilsTests/SuppressAssertDialogsAttributeTests.cs @@ -0,0 +1,158 @@ +// Copyright (c) 2026 SIL International +// This software is licensed under the LGPL, version 2.1 or later +// (http://www.gnu.org/licenses/lgpl-2.1.html) + +using System; +using System.Diagnostics; +using System.IO; +using NUnit.Framework; +using SIL.FieldWorks.Common.FwUtils.Attributes; + +namespace SIL.FieldWorks.Common.FwUtils +{ + [TestFixture] + public class SuppressAssertDialogsAttributeTests + { + private TraceListener[] m_originalListeners; + private TextWriter m_originalError; + + [SetUp] + public void SetUp() + { + m_originalListeners = new TraceListener[Trace.Listeners.Count]; + Trace.Listeners.CopyTo(m_originalListeners, 0); + Trace.Listeners.Clear(); + m_originalError = Console.Error; + + Environment.SetEnvironmentVariable("AssertUiEnabled", null); + Environment.SetEnvironmentVariable("AssertExceptionEnabled", null); + Environment.SetEnvironmentVariable("FW_TEST_MODE", null); + } + + [TearDown] + public void TearDown() + { + Trace.Listeners.Clear(); + Trace.Listeners.AddRange(m_originalListeners); + Console.SetError(m_originalError); + + Environment.SetEnvironmentVariable("AssertUiEnabled", null); + Environment.SetEnvironmentVariable("AssertExceptionEnabled", null); + Environment.SetEnvironmentVariable("FW_TEST_MODE", null); + } + + [Test] + public void ConsoleErrorTraceListener_Fail_ThrowsAssertionDialogExceptionAndWritesToStderr() + { + var stderr = new StringWriter(); + Console.SetError(stderr); + + var listener = new ConsoleErrorTraceListener(throwOnFail: true); + + var exception = Assert.Throws( + () => listener.Fail("boom", "detail") + ); + + Assert.That(exception.Message, Does.Contain("boom")); + Assert.That(exception.Message, Does.Contain("detail")); + Assert.That(stderr.ToString(), Does.Contain("Debug.Fail/Assert fired during test:")); + } + + [Test] + public void SuppressAssertDialogsAttribute_BeforeAndAfterTest_RestoresEnvironmentVariables() + { + Environment.SetEnvironmentVariable("AssertUiEnabled", "true"); + Environment.SetEnvironmentVariable("AssertExceptionEnabled", "false"); + Environment.SetEnvironmentVariable("FW_TEST_MODE", "0"); + + var defaultListener = new DefaultTraceListener(); + Trace.Listeners.Add(defaultListener); + + var attribute = new SuppressAssertDialogsAttribute(); + + attribute.BeforeTest(null); + + Assert.That(Environment.GetEnvironmentVariable("AssertUiEnabled"), Is.EqualTo("false")); + Assert.That( + Environment.GetEnvironmentVariable("AssertExceptionEnabled"), + Is.EqualTo("true") + ); + Assert.That(Environment.GetEnvironmentVariable("FW_TEST_MODE"), Is.EqualTo("1")); + Assert.That(defaultListener.AssertUiEnabled, Is.False); + Assert.That(CountListeners(), Is.EqualTo(1)); + + attribute.AfterTest(null); + + Assert.That(Environment.GetEnvironmentVariable("AssertUiEnabled"), Is.EqualTo("true")); + Assert.That( + Environment.GetEnvironmentVariable("AssertExceptionEnabled"), + Is.EqualTo("false") + ); + Assert.That(Environment.GetEnvironmentVariable("FW_TEST_MODE"), Is.EqualTo("0")); + Assert.That(CountListeners(), Is.EqualTo(0)); + } + + [Test] + public void SuppressAssertDialogsAttribute_DoesNotAddDuplicateConsoleListener() + { + Trace.Listeners.Add(new ConsoleErrorTraceListener(throwOnFail: false)); + var attribute = new SuppressAssertDialogsAttribute(); + + attribute.BeforeTest(null); + + Assert.That(CountListeners(), Is.EqualTo(1)); + + attribute.AfterTest(null); + } + + [Test] + public void SuppressAssertDialogsAttribute_WithEnvVarTraceListener_LeavesFailAsLoggingOnly() + { + Trace.Listeners.Add(new EnvVarTraceListener()); + var stderr = new StringWriter(); + Console.SetError(stderr); + + var attribute = new SuppressAssertDialogsAttribute(); + attribute.BeforeTest(null); + + var listener = FindConsoleErrorTraceListener(); + Assert.That(listener, Is.Not.Null); + + Assert.DoesNotThrow(() => listener.Fail("boom", null)); + Assert.That(stderr.ToString(), Does.Contain("Debug.Fail/Assert fired during test:")); + + attribute.AfterTest(null); + } + + private static int CountListeners() + where T : TraceListener + { + int count = 0; + foreach (TraceListener listener in Trace.Listeners) + { + if (listener is T) + count++; + } + + return count; + } + + private static ConsoleErrorTraceListener FindConsoleErrorTraceListener() + { + foreach (TraceListener listener in Trace.Listeners) + { + if (listener is ConsoleErrorTraceListener consoleListener) + return consoleListener; + } + + return null; + } + + private sealed class EnvVarTraceListener : TraceListener + { + public override void Write(string message) { } + + public override void WriteLine(string message) { } + } + } +} \ No newline at end of file diff --git a/Src/DebugProcs/DebugProcs.cpp b/Src/DebugProcs/DebugProcs.cpp index 75452de3f2..7961d396a9 100644 --- a/Src/DebugProcs/DebugProcs.cpp +++ b/Src/DebugProcs/DebugProcs.cpp @@ -677,11 +677,14 @@ void __cdecl SilAssert ( else // !g_fShowMessageBox { // if we don't show a message box we should at least abort (and output the assertion - // text if we haven't done that already). Note that we don't call _exit(3) as above - // so that we can trap the signal and ignore it in unit tests + // text if we haven't done that already). In FW_TEST_MODE we must not continue after + // a post-summary assert even if SIGABRT is ignored or intercepted by the native test + // runner, so force process termination if raise(SIGABRT) returns. if (g_ReportHook) OutputDebugString(assertbuf); raise(SIGABRT); + if (IsTestModeEnabled()) + _exit(3); } /* Ignore: continue execution */ diff --git a/Src/Generic/Test/TestErrorHandling.h b/Src/Generic/Test/TestErrorHandling.h index 39a2959316..9d31b0cdf9 100644 --- a/Src/Generic/Test/TestErrorHandling.h +++ b/Src/Generic/Test/TestErrorHandling.h @@ -15,6 +15,8 @@ Last reviewed: #pragma once #include "testGenericLib.h" +#include +#include namespace TestGenericLib { @@ -156,6 +158,31 @@ namespace TestGenericLib #endif } + void testNativeAssertThrowsExceptionWithoutContinuing() + { +#if defined(WIN32) || defined(_M_X64) + bool fReachedAfterAssert = false; + try + { + AssertMsg(false, "Test-induced native assert"); + fReachedAfterAssert = true; + unitpp::assert_fail("Assert did not throw an exception in native test mode"); + } + catch (const std::runtime_error & e) + { + std::string message(e.what()); + unitpp::assert_true( + "Native assert produced wrong exception text", + message.find("Test-induced native assert") != std::string::npos + ); + } + + unitpp::assert_true("Execution continued after native assert", !fReachedAfterAssert); +#else + // TODO-Linux: port +#endif + } + public: TestErrorHandling(); }; diff --git a/Src/Generic/Test/testGeneric.cpp b/Src/Generic/Test/testGeneric.cpp index d20ccbe13f..d90f72ff64 100644 --- a/Src/Generic/Test/testGeneric.cpp +++ b/Src/Generic/Test/testGeneric.cpp @@ -12,11 +12,62 @@ Last reviewed: #include "testGenericLib.h" #include "RedirectHKCU.h" #include "DebugProcs.h" +#include +#include +#include +#include + +namespace +{ + Pfn_Assert g_previousAssertProc = NULL; + + void RestorePreviousAssertProc() + { + if (g_previousAssertProc != NULL) + { + SetAssertProc(g_previousAssertProc); + g_previousAssertProc = NULL; + } + } + + void TerminateOnSigAbrt(int) + { + TerminateProcess(GetCurrentProcess(), 3); + } + + bool IsFailureInjectionEnabled(const char * pszName) + { + size_t cchValue = 0; + char * pszValue = NULL; + const bool fEnabled = + _dupenv_s(&pszValue, &cchValue, pszName) == 0 && + pszValue != NULL && + strcmp(pszValue, "1") == 0; + free(pszValue); + return fEnabled; + } + + void WINAPI ThrowingAssertProc(const char * pszExp, const char * pszFile, int nLine, HMODULE) + { + char szMessage[1024]; + sprintf_s( + szMessage, + "Native assert fired during test: %s (%s:%d)", + pszExp, + pszFile, + nLine + ); + fprintf(stderr, "%s\n", szMessage); + fflush(stderr); + throw std::runtime_error(szMessage); + } +} namespace unitpp { void GlobalSetup(bool verbose) { + g_previousAssertProc = SetAssertProc(ThrowingAssertProc); ShowAssertMessageBox(0); // Disable assertion dialogs #if defined(WIN32) || defined(WIN64) ModuleEntry::DllMain(0, DLL_PROCESS_ATTACH); @@ -27,10 +78,29 @@ namespace unitpp } void GlobalTeardown() { + signal(SIGABRT, TerminateOnSigAbrt); + + const bool fInjectTeardownAssert = IsFailureInjectionEnabled("FW_TEST_INDUCE_TEARDOWN_ASSERT"); + const bool fInjectTeardownAbort = IsFailureInjectionEnabled("FW_TEST_INDUCE_TEARDOWN_ABORT"); + if (fInjectTeardownAssert || fInjectTeardownAbort) + RestorePreviousAssertProc(); + + if (fInjectTeardownAssert) + AssertMsg(false, "Injected teardown assert for native test infrastructure validation"); + + if (fInjectTeardownAbort) + { + _set_error_mode(_OUT_TO_STDERR); + _set_abort_behavior(0, _WRITE_ABORT_MSG | _CALL_REPORTFAULT); + abort(); + } + #if defined(WIN32) || defined(WIN64) ModuleEntry::DllMain(0, DLL_PROCESS_DETACH); #endif ::OleUninitialize(); + + RestorePreviousAssertProc(); } } diff --git a/Src/InstallValidator/InstallValidatorTests/InstallValidatorTests.csproj b/Src/InstallValidator/InstallValidatorTests/InstallValidatorTests.csproj index 5a6544524e..cdea09d4ad 100644 --- a/Src/InstallValidator/InstallValidatorTests/InstallValidatorTests.csproj +++ b/Src/InstallValidator/InstallValidatorTests/InstallValidatorTests.csproj @@ -53,5 +53,8 @@ Attributes\LogUnhandledExceptionsAttribute.cs + + Attributes\SuppressAssertDialogsAttribute.cs + \ No newline at end of file diff --git a/Src/InstallValidator/InstallValidatorTests/TestAssemblyInfo.cs b/Src/InstallValidator/InstallValidatorTests/TestAssemblyInfo.cs index e917e2ee41..e0f708d6ac 100644 --- a/Src/InstallValidator/InstallValidatorTests/TestAssemblyInfo.cs +++ b/Src/InstallValidator/InstallValidatorTests/TestAssemblyInfo.cs @@ -4,4 +4,5 @@ using SIL.FieldWorks.Common.FwUtils.Attributes; -[assembly: LogUnhandledExceptions] \ No newline at end of file +[assembly: LogUnhandledExceptions] +[assembly: SuppressAssertDialogs] \ No newline at end of file diff --git a/Src/InstallValidator/InstallerArtifactsTests/InstallerArtifactsTests.csproj b/Src/InstallValidator/InstallerArtifactsTests/InstallerArtifactsTests.csproj index 2e7f2c832d..8fe0d6ec34 100644 --- a/Src/InstallValidator/InstallerArtifactsTests/InstallerArtifactsTests.csproj +++ b/Src/InstallValidator/InstallerArtifactsTests/InstallerArtifactsTests.csproj @@ -38,5 +38,8 @@ Attributes\LogUnhandledExceptionsAttribute.cs + + Attributes\SuppressAssertDialogsAttribute.cs + diff --git a/Src/InstallValidator/InstallerArtifactsTests/TestAssemblyInfo.cs b/Src/InstallValidator/InstallerArtifactsTests/TestAssemblyInfo.cs index e917e2ee41..e0f708d6ac 100644 --- a/Src/InstallValidator/InstallerArtifactsTests/TestAssemblyInfo.cs +++ b/Src/InstallValidator/InstallerArtifactsTests/TestAssemblyInfo.cs @@ -4,4 +4,5 @@ using SIL.FieldWorks.Common.FwUtils.Attributes; -[assembly: LogUnhandledExceptions] \ No newline at end of file +[assembly: LogUnhandledExceptions] +[assembly: SuppressAssertDialogs] \ No newline at end of file diff --git a/Src/views/Test/testViews.cpp b/Src/views/Test/testViews.cpp index 2938ef6284..636041b4bb 100644 --- a/Src/views/Test/testViews.cpp +++ b/Src/views/Test/testViews.cpp @@ -15,18 +15,69 @@ Last reviewed: #include "testViews.h" #include "RedirectHKCU.h" #include "DebugProcs.h" +#include +#include +#include +#include #if !defined(WIN32) && !defined(_M_X64) // These define GUIDs that we need to define globally somewhere #include "TestVwTxtSrc.h" #include "TestLayoutPage.h" #endif +namespace +{ + Pfn_Assert g_previousAssertProc = NULL; + + void RestorePreviousAssertProc() + { + if (g_previousAssertProc != NULL) + { + SetAssertProc(g_previousAssertProc); + g_previousAssertProc = NULL; + } + } + + void TerminateOnSigAbrt(int) + { + TerminateProcess(GetCurrentProcess(), 3); + } + + bool IsFailureInjectionEnabled(const char * pszName) + { + size_t cchValue = 0; + char * pszValue = NULL; + const bool fEnabled = + _dupenv_s(&pszValue, &cchValue, pszName) == 0 && + pszValue != NULL && + strcmp(pszValue, "1") == 0; + free(pszValue); + return fEnabled; + } + + void WINAPI ThrowingAssertProc(const char * pszExp, const char * pszFile, int nLine, HMODULE) + { + char szMessage[1024]; + sprintf_s( + szMessage, + "Native assert fired during test: %s (%s:%d)", + pszExp, + pszFile, + nLine + ); + fprintf(stderr, "%s\n", szMessage); + fflush(stderr); + throw std::runtime_error(szMessage); + } +} + namespace unitpp { void GlobalSetup(bool verbose) { printf("DEBUG: Entering GlobalSetup\n"); fflush(stdout); + g_previousAssertProc = SetAssertProc(ThrowingAssertProc); ShowAssertMessageBox(0); // Disable assertion dialogs printf("DEBUG: After ShowAssertMessageBox\n"); fflush(stdout); @@ -48,10 +99,29 @@ namespace unitpp } void GlobalTeardown() { + signal(SIGABRT, TerminateOnSigAbrt); + + const bool fInjectTeardownAssert = IsFailureInjectionEnabled("FW_TEST_INDUCE_TEARDOWN_ASSERT"); + const bool fInjectTeardownAbort = IsFailureInjectionEnabled("FW_TEST_INDUCE_TEARDOWN_ABORT"); + if (fInjectTeardownAssert || fInjectTeardownAbort) + RestorePreviousAssertProc(); + + if (fInjectTeardownAssert) + AssertMsg(false, "Injected teardown assert for native test infrastructure validation"); + + if (fInjectTeardownAbort) + { + _set_error_mode(_OUT_TO_STDERR); + _set_abort_behavior(0, _WRITE_ABORT_MSG | _CALL_REPORTFAULT); + abort(); + } + ::OleUninitialize(); #if defined(WIN32) || defined(_M_X64) ModuleEntry::DllMain(0, DLL_PROCESS_DETACH); #endif + + RestorePreviousAssertProc(); } } diff --git a/test.ps1 b/test.ps1 index 9bc759e921..3b3352b289 100644 --- a/test.ps1 +++ b/test.ps1 @@ -199,6 +199,23 @@ try { Write-Host "" } + elseif ($TestProject -and ($normalizedTestProjectForBuild -match '(^|/)Src/InstallValidator/InstallValidatorTests($|/InstallValidatorTests\.csproj$)')) { + Write-Host "Building InstallValidatorTests before running tests..." -ForegroundColor Cyan + + Invoke-MSBuild ` + -Arguments @( + 'Src/InstallValidator/InstallValidatorTests/InstallValidatorTests.csproj', + '/t:Restore;Build', + "/p:Configuration=$Configuration", + '/p:Platform=x64', + '/nr:false', + '/v:minimal', + '/nologo' + ) ` + -Description 'InstallValidatorTests' + + Write-Host "" + } else { Write-Host "Building before running tests..." -ForegroundColor Cyan & "$PSScriptRoot\build.ps1" -Configuration $Configuration -BuildTests @@ -243,6 +260,9 @@ try { else { # Assume it's a project path, find the DLL $projectName = Split-Path $TestProject -Leaf + if ($projectName -match '\.csproj$') { + $projectName = [System.IO.Path]::GetFileNameWithoutExtension($projectName) + } if ($projectName -notmatch 'Tests?$') { $projectName = "${projectName}Tests" }