Skip to content

Commit 7222b1a

Browse files
gh-132581: Report why the execution environment was altered
The list of tests which altered the execution environment now includes the reasons -- an unraisable exception, a modified sys.path, leaked temporary files, etc -- one per line. The final result no longer repeats the same state twice, like "ENV CHANGED then ENV CHANGED".
1 parent c3aefdb commit 7222b1a

10 files changed

Lines changed: 67 additions & 17 deletions

File tree

Lib/test/libregrtest/main.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -448,7 +448,7 @@ def run_tests_sequentially(self, runtests: RunTests) -> None:
448448

449449
def get_state(self) -> str:
450450
state = self.results.get_state(self.fail_env_changed)
451-
if self.first_state:
451+
if self.first_state and self.first_state != state:
452452
state = f'{self.first_state} then {state}'
453453
return state
454454

Lib/test/libregrtest/result.py

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -100,6 +100,9 @@ class TestResult:
100100
# partial coverage in a worker run; not used by sequential in-process runs
101101
covered_lines: list[Location] | None = None
102102

103+
# short descriptions of how the test altered the execution environment
104+
env_changed_reasons: list[str] | None = None
105+
103106
def is_failed(self, fail_env_changed: bool) -> bool:
104107
if self.state == State.ENV_CHANGED:
105108
return fail_env_changed
@@ -175,9 +178,15 @@ def __str__(self) -> str:
175178
def has_meaningful_duration(self):
176179
return State.has_meaningful_duration(self.state)
177180

178-
def set_env_changed(self):
181+
def set_env_changed(self, *reasons):
179182
if self.state is None or self.state == State.PASSED:
180183
self.state = State.ENV_CHANGED
184+
if reasons:
185+
if self.env_changed_reasons is None:
186+
self.env_changed_reasons = []
187+
for reason in reasons:
188+
if reason not in self.env_changed_reasons:
189+
self.env_changed_reasons.append(reason)
181190

182191
def must_stop(self, fail_fast: bool, fail_env_changed: bool) -> bool:
183192
if State.must_stop(self.state):

Lib/test/libregrtest/results.py

Lines changed: 17 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,8 @@ def __init__(self) -> None:
3030
self.skipped: TestList = []
3131
self.resource_denied: TestList = []
3232
self.env_changed: TestList = []
33+
# test name => how the test altered the execution environment
34+
self.env_changed_reasons: dict[TestName, list[str]] = {}
3335
self.run_no_tests: TestList = []
3436
self.rerun: TestList = []
3537
self.rerun_results: list[TestResult] = []
@@ -107,6 +109,9 @@ def accumulate_result(self, result: TestResult, runtests: RunTests) -> None:
107109
self.good.append(test_name)
108110
case State.ENV_CHANGED:
109111
self.env_changed.append(test_name)
112+
if result.env_changed_reasons:
113+
self.env_changed_reasons[test_name] = \
114+
result.env_changed_reasons
110115
self.rerun_results.append(result)
111116
case State.SKIPPED:
112117
self.skipped.append(test_name)
@@ -254,7 +259,18 @@ def display_result(self, tests: TestTuple, quiet: bool, print_slowest: bool) ->
254259
print()
255260
count_text = count(len(tests_list), count_text)
256261
print(title_format.format(count_text))
257-
printlist(tests_list)
262+
if tests_list is self.env_changed:
263+
# List every test and every reason on a separate line.
264+
for test_name in sorted(tests_list):
265+
reasons = self.env_changed_reasons.get(test_name)
266+
if reasons:
267+
print(f" {test_name}:")
268+
for reason in reasons:
269+
print(f" {reason}")
270+
else:
271+
print(f" {test_name}")
272+
else:
273+
printlist(tests_list)
258274

259275
if self.good and not quiet:
260276
print()

Lib/test/libregrtest/run_workers.py

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -386,7 +386,8 @@ def _runtest(self, test_name: TestName) -> MultiprocessResult:
386386
f'Warning -- {test_name} leaked temporary files '
387387
f'({len(tmp_files)}): {", ".join(sorted(tmp_files))}')
388388
stdout += msg
389-
result.set_env_changed()
389+
result.set_env_changed(
390+
f"leaked temporary files: {', '.join(sorted(tmp_files))}")
390391

391392
return MultiprocessResult(result, stdout)
392393

Lib/test/libregrtest/save_env.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -347,7 +347,7 @@ def __exit__(self, exc_type, exc_val, exc_tb):
347347
current = get()
348348
# Check for changes to the resource's value
349349
if current != original:
350-
support.environment_altered = True
350+
support.set_environment_altered(f"{name} was modified")
351351
restore(original)
352352
if not self.quiet and not self.pgo:
353353
print_warning(

Lib/test/libregrtest/single.py

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -173,7 +173,8 @@ def test_func():
173173
remove_testfn(test_name, runtests.verbose)
174174

175175
if gc.garbage:
176-
support.environment_altered = True
176+
support.set_environment_altered(
177+
f"{len(gc.garbage)} uncollectable object(s)")
177178
print_warning(f"{test_name} created {len(gc.garbage)} "
178179
f"uncollectable object(s)")
179180

@@ -194,6 +195,7 @@ def _runtest_env_changed_exc(result: TestResult, runtests: RunTests,
194195
# Reset the environment_altered flag to detect if a test altered
195196
# the environment
196197
support.environment_altered = False
198+
support.environment_altered_reasons.clear()
197199

198200
pgo = runtests.pgo
199201
if pgo:
@@ -261,7 +263,7 @@ def _runtest_env_changed_exc(result: TestResult, runtests: RunTests,
261263
return
262264

263265
if support.environment_altered:
264-
result.set_env_changed()
266+
result.set_env_changed(*support.environment_altered_reasons)
265267
# Don't override the state if it was already set (REFLEAK or ENV_CHANGED)
266268
if result.state is None:
267269
result.state = State.PASSED

Lib/test/libregrtest/utils.py

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -141,7 +141,8 @@ def print_warning(msg: str) -> None:
141141

142142
def regrtest_unraisable_hook(unraisable) -> None:
143143
global orig_unraisablehook
144-
support.environment_altered = True
144+
support.set_environment_altered(
145+
f"unraisable exception ({unraisable.exc_type.__name__})")
145146
support.print_warning("Unraisable exception")
146147
old_stderr = sys.stderr
147148
try:
@@ -165,7 +166,8 @@ def setup_unraisable_hook() -> None:
165166

166167
def regrtest_threading_excepthook(args) -> None:
167168
global orig_threading_excepthook
168-
support.environment_altered = True
169+
support.set_environment_altered(
170+
f"uncaught thread exception ({args.exc_type.__name__})")
169171
support.print_warning(f"Uncaught thread exception: {args.exc_type.__name__}")
170172
old_stderr = sys.stderr
171173
try:
@@ -524,7 +526,7 @@ def remove_testfn(test_name: TestName, verbose: int) -> None:
524526

525527
if verbose:
526528
print_warning(f"{test_name} left behind {kind} {name!r}")
527-
support.environment_altered = True
529+
support.set_environment_altered(f"left behind {kind} {name!r}")
528530

529531
try:
530532
import stat

Lib/test/support/__init__.py

Lines changed: 14 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1550,14 +1550,25 @@ def print_warning(msg):
15501550
# to cleanup threads.
15511551
environment_altered = False
15521552

1553+
# Short descriptions of what was altered, e.g. "unraisable exception".
1554+
# They are reported by regrtest together with the name of the test.
1555+
environment_altered_reasons = []
1556+
1557+
1558+
def set_environment_altered(reason):
1559+
"""Set the environment_altered flag and record why it was set."""
1560+
global environment_altered
1561+
environment_altered = True
1562+
if reason not in environment_altered_reasons:
1563+
environment_altered_reasons.append(reason)
1564+
1565+
15531566
def reap_children():
15541567
"""Use this function at the end of test_main() whenever sub-processes
15551568
are started. This will help ensure that no extra children (zombies)
15561569
stick around to hog resources and create problems when looking
15571570
for refleaks.
15581571
"""
1559-
global environment_altered
1560-
15611572
# Need os.waitpid(-1, os.WNOHANG): Windows is not supported
15621573
if not (hasattr(os, 'waitpid') and hasattr(os, 'WNOHANG')):
15631574
return
@@ -1577,7 +1588,7 @@ def reap_children():
15771588
break
15781589

15791590
print_warning(f"reap_children() reaped child process {pid}")
1580-
environment_altered = True
1591+
set_environment_altered("reaped child process")
15811592

15821593

15831594
@contextlib.contextmanager

Lib/test/test_regrtest.py

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -716,9 +716,13 @@ def list_regex(line_format, tests):
716716
self.check_line(output, regex)
717717

718718
if env_changed:
719-
regex = list_regex(r'%s test%s altered the execution environment '
720-
r'\(env changed\)',
721-
env_changed)
719+
# Every test is listed on a separate line, followed by the
720+
# reasons why the environment was altered, one per line.
721+
count = len(env_changed)
722+
regex = (r'%s test%s altered the execution environment '
723+
r'\(env changed\):\n' % (count, plural(count)))
724+
regex += ''.join(r' %s:?\n(?: .*\n)*' % re.escape(name)
725+
for name in sorted(env_changed))
722726
self.check_line(output, regex)
723727

724728
if omitted:
@@ -807,7 +811,8 @@ def list_regex(line_format, tests):
807811
state = ', '.join(state)
808812
if rerun is not None:
809813
new_state = 'SUCCESS' if rerun.success else 'FAILURE'
810-
state = f'{state} then {new_state}'
814+
if new_state != state:
815+
state = f'{state} then {new_state}'
811816
self.check_line(output, f'Result: {state}', full=True)
812817

813818
def parse_random_seed(self, output: str) -> str:
Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
The list of tests which altered the execution environment now includes the
2+
reasons why the environment was considered altered, for example an unraisable
3+
exception or a modified :data:`sys.path`. The final result no longer repeats
4+
the same state twice (like ``ENV CHANGED then ENV CHANGED``).

0 commit comments

Comments
 (0)