Skip to content

Commit 669b34a

Browse files
gh-86427: Fix the stdio encoding in the legacy Windows stdio mode
It was the ANSI code page instead of the encoding of the device the stream is connected to, as in 3.7. The stdio encoding is now left undefined in this mode and determined for every standard stream.
1 parent 998b890 commit 669b34a

5 files changed

Lines changed: 67 additions & 5 deletions

File tree

Lib/test/test_cmd_line.py

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1067,6 +1067,42 @@ def test_python_legacy_windows_stdio(self):
10671067
support.skip_on_low_desktop_heap_memory_subprocess(p.returncode)
10681068
self.assertEqual(p.returncode, 0)
10691069

1070+
@unittest.skipUnless(support.MS_WINDOWS, 'Test only applicable on Windows')
1071+
def test_python_legacy_windows_stdio_encoding(self):
1072+
# gh-86427: In the legacy mode the encoding of a standard stream is
1073+
# the encoding of the console it is connected to, which can differ
1074+
# for input and output.
1075+
import ctypes
1076+
kernel32 = ctypes.WinDLL('kernel32', use_last_error=True)
1077+
try:
1078+
fin = open('CONIN$')
1079+
except OSError:
1080+
self.skipTest('no console')
1081+
# We cannot use PIPE, because the standard streams should be
1082+
# connected to the console. So we use the exit code.
1083+
code = ("import sys; sys.exit(sys.stdin.encoding != 'cp850' or "
1084+
"sys.stdout.encoding != 'cp437')")
1085+
env = os.environ.copy()
1086+
env['PYTHONLEGACYWINDOWSSTDIO'] = '1'
1087+
env['PYTHONUTF8'] = '0'
1088+
env.pop('PYTHONIOENCODING', None)
1089+
old_cp = kernel32.GetConsoleCP()
1090+
old_output_cp = kernel32.GetConsoleOutputCP()
1091+
with fin, open('CONOUT$', 'w') as fout:
1092+
try:
1093+
if not kernel32.SetConsoleCP(850):
1094+
self.skipTest('cannot set the console input code page')
1095+
if not kernel32.SetConsoleOutputCP(437):
1096+
self.skipTest('cannot set the console output code page')
1097+
proc = subprocess.run([sys.executable, '-c', code], env=env,
1098+
stdin=fin, stdout=fout,
1099+
stderr=subprocess.DEVNULL)
1100+
finally:
1101+
kernel32.SetConsoleCP(old_cp)
1102+
kernel32.SetConsoleOutputCP(old_output_cp)
1103+
support.skip_on_low_desktop_heap_memory_subprocess(proc.returncode)
1104+
self.assertEqual(proc.returncode, 0)
1105+
10701106
@unittest.skipIf("-fsanitize" in sysconfig.get_config_vars().get('PY_CFLAGS', ()),
10711107
"PYTHONMALLOCSTATS doesn't work with ASAN")
10721108
def test_python_malloc_stats(self):
Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
Fix the encoding of the standard streams in the legacy Windows stdio mode
2+
(:envvar:`PYTHONLEGACYWINDOWSSTDIO`). It is now the encoding of the device
3+
the stream is connected to, as in Python 3.7, not the ANSI code page.

Objects/unicodeobject.c

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15209,6 +15209,10 @@ init_stdio_encoding(PyInterpreterState *interp)
1520915209
{
1521015210
/* Update the stdio encoding to the normalized Python codec name. */
1521115211
PyConfig *config = (PyConfig*)_PyInterpreterState_GetConfig(interp);
15212+
if (config->stdio_encoding == NULL) {
15213+
/* gh-86427: The encoding is determined for every stream. */
15214+
return _PyStatus_OK();
15215+
}
1521215216
if (config_get_codec_name(&config->stdio_encoding) < 0) {
1521315217
return _PyStatus_ERR("failed to get the Python codec name "
1521415218
"of the stdio encoding");

Python/initconfig.c

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -196,7 +196,7 @@ static const PyConfigSpec PYCONFIG_SPEC[] = {
196196
SPEC(show_ref_count, BOOL, READ_ONLY, NO_SYS, NO_GLOBAL),
197197
SPEC(site_import, BOOL, READ_ONLY, NO_SYS, GLOBAL(&Py_NoSiteFlag, 1)), // sys.flags.no_site
198198
SPEC(skip_source_first_line, BOOL, READ_ONLY, NO_SYS, NO_GLOBAL),
199-
SPEC(stdio_encoding, WSTR, READ_ONLY, NO_SYS, NO_GLOBAL),
199+
SPEC(stdio_encoding, WSTR_OPT, READ_ONLY, NO_SYS, NO_GLOBAL),
200200
SPEC(stdio_errors, WSTR, READ_ONLY, NO_SYS, NO_GLOBAL),
201201
SPEC(tracemalloc, UINT, READ_ONLY, NO_SYS, NO_GLOBAL),
202202
SPEC(use_frozen_modules, BOOL, READ_ONLY, NO_SYS, NO_GLOBAL),
@@ -1074,11 +1074,14 @@ config_check_consistency(const PyConfig *config)
10741074
assert(config->module_search_paths_set >= 0);
10751075
assert(config->filesystem_encoding != NULL);
10761076
assert(config->filesystem_errors != NULL);
1077-
assert(config->stdio_encoding != NULL);
1078-
assert(config->stdio_errors != NULL);
10791077
#ifdef MS_WINDOWS
1078+
/* stdio_encoding can be NULL in the legacy Windows stdio mode. */
1079+
assert(config->stdio_encoding != NULL || config->legacy_windows_stdio);
10801080
assert(config->legacy_windows_stdio >= 0);
1081+
#else
1082+
assert(config->stdio_encoding != NULL);
10811083
#endif
1084+
assert(config->stdio_errors != NULL);
10821085
/* -c and -m options are exclusive */
10831086
assert(!(config->run_command != NULL && config->run_module != NULL));
10841087
assert(config->check_hash_pycs_mode != NULL);
@@ -2709,7 +2712,12 @@ config_init_stdio_encoding(PyConfig *config,
27092712
}
27102713

27112714
/* Choose the default error handler based on the current locale. */
2712-
if (config->stdio_encoding == NULL) {
2715+
if (config->stdio_encoding == NULL
2716+
#ifdef MS_WINDOWS
2717+
/* gh-86427: it is determined for each stream. */
2718+
&& !config->legacy_windows_stdio
2719+
#endif
2720+
) {
27132721
status = config_get_locale_encoding(config, preconfig,
27142722
&config->stdio_encoding);
27152723
if (_PyStatus_EXCEPTION(status)) {

Python/pylifecycle.c

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3070,7 +3070,18 @@ create_stdio(const PyConfig *config, PyObject* io,
30703070
newline = "\n";
30713071
#endif
30723072

3073-
PyObject *encoding_str = PyUnicode_FromWideChar(encoding, -1);
3073+
PyObject *encoding_str;
3074+
if (encoding != NULL) {
3075+
encoding_str = PyUnicode_FromWideChar(encoding, -1);
3076+
}
3077+
else {
3078+
/* gh-86427: use the encoding of the device. */
3079+
encoding_str = _Py_device_encoding(fd);
3080+
if (encoding_str == Py_None) {
3081+
Py_DECREF(encoding_str);
3082+
encoding_str = _Py_GetLocaleEncodingObject();
3083+
}
3084+
}
30743085
if (encoding_str == NULL) {
30753086
Py_CLEAR(buf);
30763087
goto error;

0 commit comments

Comments
 (0)