[Python-checkins] cpython: Issue #1602: Windows console doesn't input or print Unicode (PEP 528)
steve.dower
python-checkins at python.org
Thu Sep 8 17:15:20 EDT 2016
https://hg.python.org/cpython/rev/6142d2d3c471
changeset: 103357:6142d2d3c471
user: Steve Dower <steve.dower at microsoft.com>
date: Tue Aug 30 21:22:36 2016 -0700
summary:
Issue #1602: Windows console doesn't input or print Unicode (PEP 528)
Closes #17602: Adds a readline implementation for the Windows console
files:
Doc/library/functions.rst | 40 +-
Doc/using/cmdline.rst | 17 +
Doc/whatsnew/3.6.rst | 19 +
Include/pydebug.h | 4 +
Lib/io.py | 7 +
Lib/test/test_os.py | 2 +-
Lib/test/test_winconsoleio.py | 72 +
Misc/NEWS | 3 +-
Modules/_io/_iomodule.c | 24 +-
Modules/_io/_iomodule.h | 10 +
Modules/_io/clinic/winconsoleio.c.h | 331 ++++
Modules/_io/winconsoleio.c | 1096 +++++++++++++++
PCbuild/pythoncore.vcxproj | 1 +
PCbuild/pythoncore.vcxproj.filters | 3 +
Parser/myreadline.c | 113 +
Python/pylifecycle.c | 18 +
16 files changed, 1739 insertions(+), 21 deletions(-)
diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst
--- a/Doc/library/functions.rst
+++ b/Doc/library/functions.rst
@@ -1055,30 +1055,38 @@
(where :func:`open` is declared), :mod:`os`, :mod:`os.path`, :mod:`tempfile`,
and :mod:`shutil`.
- .. versionchanged:: 3.3
- The *opener* parameter was added.
- The ``'x'`` mode was added.
- :exc:`IOError` used to be raised, it is now an alias of :exc:`OSError`.
- :exc:`FileExistsError` is now raised if the file opened in exclusive
- creation mode (``'x'``) already exists.
+ .. versionchanged::
+ 3.3
- .. versionchanged:: 3.4
- The file is now non-inheritable.
+ * The *opener* parameter was added.
+ * The ``'x'`` mode was added.
+ * :exc:`IOError` used to be raised, it is now an alias of :exc:`OSError`.
+ * :exc:`FileExistsError` is now raised if the file opened in exclusive
+ * creation mode (``'x'``) already exists.
+
+ .. versionchanged::
+ 3.4
+
+ * The file is now non-inheritable.
.. deprecated-removed:: 3.4 4.0
The ``'U'`` mode.
- .. versionchanged:: 3.5
- If the system call is interrupted and the signal handler does not raise an
- exception, the function now retries the system call instead of raising an
- :exc:`InterruptedError` exception (see :pep:`475` for the rationale).
+ .. versionchanged::
+ 3.5
- .. versionchanged:: 3.5
- The ``'namereplace'`` error handler was added.
+ * If the system call is interrupted and the signal handler does not raise an
+ exception, the function now retries the system call instead of raising an
+ :exc:`InterruptedError` exception (see :pep:`475` for the rationale).
+ * The ``'namereplace'`` error handler was added.
- .. versionchanged:: 3.6
- Support added to accept objects implementing :class:`os.PathLike`.
+ .. versionchanged::
+ 3.6
+
+ * Support added to accept objects implementing :class:`os.PathLike`.
+ * On Windows, opening a console buffer may return a subclass of
+ :class:`io.RawIOBase` other than :class:`io.FileIO`.
.. function:: ord(c)
diff --git a/Doc/using/cmdline.rst b/Doc/using/cmdline.rst
--- a/Doc/using/cmdline.rst
+++ b/Doc/using/cmdline.rst
@@ -559,6 +559,10 @@
.. versionchanged:: 3.4
The ``encodingname`` part is now optional.
+ .. versionchanged:: 3.6
+ On Windows, the encoding specified by this variable is ignored for interactive
+ console buffers unless :envvar:`PYTHONLEGACYWINDOWSIOENCODING` is also specified.
+ Files and pipes redirected through the standard streams are not affected.
.. envvar:: PYTHONNOUSERSITE
@@ -686,6 +690,19 @@
.. versionadded:: 3.6
See :pep:`529` for more details.
+.. envvar:: PYTHONLEGACYWINDOWSIOENCODING
+
+ If set to a non-empty string, does not use the new console reader and
+ writer. This means that Unicode characters will be encoded according to
+ the active console code page, rather than using utf-8.
+
+ This variable is ignored if the standard streams are redirected (to files
+ or pipes) rather than referring to console buffers.
+
+ Availability: Windows
+
+ .. versionadded:: 3.6
+
Debug-mode variables
~~~~~~~~~~~~~~~~~~~~
diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst
--- a/Doc/whatsnew/3.6.rst
+++ b/Doc/whatsnew/3.6.rst
@@ -78,6 +78,8 @@
* PEP 529: :ref:`Change Windows filesystem encoding to UTF-8 <pep-529>`
+* PEP 528: :ref:`Change Windows console encoding to UTF-8 <pep-528>`
+
* The ``py.exe`` launcher, when used interactively, no longer prefers
Python 2 over Python 3 when the user doesn't specify a version (via
command line arguments or a config file). Handling of shebang lines
@@ -267,6 +269,23 @@
(Contributed by Martin Teichmann in :issue:`27366`)
+.. _pep-528:
+
+PEP 528: Change Windows console encoding to UTF-8
+-------------------------------------------------
+
+The default console on Windows will now accept all Unicode characters and
+provide correctly read str objects to Python code. ``sys.stdin``,
+``sys.stdout`` and ``sys.stderr`` now default to utf-8 encoding.
+
+This change only applies when using an interactive console, and not when
+redirecting files or pipes. To revert to the previous behaviour for interactive
+console use, set :envvar:`PYTHONLEGACYWINDOWSIOENCODING`.
+
+.. seealso::
+
+ :pep:`528` -- Change Windows console encoding to UTF-8
+ PEP written and implemented by Steve Dower.
PYTHONMALLOC environment variable
=================================
diff --git a/Include/pydebug.h b/Include/pydebug.h
--- a/Include/pydebug.h
+++ b/Include/pydebug.h
@@ -24,6 +24,10 @@
PyAPI_DATA(int) Py_HashRandomizationFlag;
PyAPI_DATA(int) Py_IsolatedFlag;
+#ifdef MS_WINDOWS
+PyAPI_DATA(int) Py_LegacyWindowsStdioFlag;
+#endif
+
/* this is a wrapper around getenv() that pays attention to
Py_IgnoreEnvironmentFlag. It should be used for getting variables like
PYTHONPATH and PYTHONHOME from the environment */
diff --git a/Lib/io.py b/Lib/io.py
--- a/Lib/io.py
+++ b/Lib/io.py
@@ -90,3 +90,10 @@
for klass in (StringIO, TextIOWrapper):
TextIOBase.register(klass)
del klass
+
+try:
+ from _io import _WindowsConsoleIO
+except ImportError:
+ pass
+else:
+ RawIOBase.register(_WindowsConsoleIO)
diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py
--- a/Lib/test/test_os.py
+++ b/Lib/test/test_os.py
@@ -1518,7 +1518,7 @@
singles = ["fchdir", "dup", "fdopen", "fdatasync", "fstat",
"fstatvfs", "fsync", "tcgetpgrp", "ttyname"]
#singles.append("close")
- #We omit close because it doesn'r raise an exception on some platforms
+ #We omit close because it doesn't raise an exception on some platforms
def get_single(f):
def helper(self):
if hasattr(os, f):
diff --git a/Lib/test/test_winconsoleio.py b/Lib/test/test_winconsoleio.py
new file mode 100644
--- /dev/null
+++ b/Lib/test/test_winconsoleio.py
@@ -0,0 +1,72 @@
+'''Tests for WindowsConsoleIO
+
+Unfortunately, most testing requires interactive use, since we have no
+API to read back from a real console, and this class is only for use
+with real consoles.
+
+Instead, we validate that basic functionality such as opening, closing
+and in particular fileno() work, but are forced to leave real testing
+to real people with real keyborads.
+'''
+
+import io
+import unittest
+import sys
+
+if sys.platform != 'win32':
+ raise unittest.SkipTest("test only relevant on win32")
+
+ConIO = io._WindowsConsoleIO
+
+class WindowsConsoleIOTests(unittest.TestCase):
+ def test_abc(self):
+ self.assertTrue(issubclass(ConIO, io.RawIOBase))
+ self.assertFalse(issubclass(ConIO, io.BufferedIOBase))
+ self.assertFalse(issubclass(ConIO, io.TextIOBase))
+
+ def test_open_fd(self):
+ f = ConIO(0)
+ self.assertTrue(f.readable())
+ self.assertFalse(f.writable())
+ self.assertEqual(0, f.fileno())
+ f.close() # multiple close should not crash
+ f.close()
+
+ f = ConIO(1, 'w')
+ self.assertFalse(f.readable())
+ self.assertTrue(f.writable())
+ self.assertEqual(1, f.fileno())
+ f.close()
+ f.close()
+
+ f = ConIO(2, 'w')
+ self.assertFalse(f.readable())
+ self.assertTrue(f.writable())
+ self.assertEqual(2, f.fileno())
+ f.close()
+ f.close()
+
+ def test_open_name(self):
+ f = ConIO("CON")
+ self.assertTrue(f.readable())
+ self.assertFalse(f.writable())
+ self.assertIsNotNone(f.fileno())
+ f.close() # multiple close should not crash
+ f.close()
+
+ f = ConIO('CONIN$')
+ self.assertTrue(f.readable())
+ self.assertFalse(f.writable())
+ self.assertIsNotNone(f.fileno())
+ f.close()
+ f.close()
+
+ f = ConIO('CONOUT$', 'w')
+ self.assertFalse(f.readable())
+ self.assertTrue(f.writable())
+ self.assertIsNotNone(f.fileno())
+ f.close()
+ f.close()
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/Misc/NEWS b/Misc/NEWS
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -300,6 +300,8 @@
Windows
-------
+- Issue #1602: Windows console doesn't input or print Unicode (PEP 528)
+
- Issue #27781: Change file system encoding on Windows to UTF-8 (PEP 529)
- Issue #27731: Opt-out of MAX_PATH on Windows 10
@@ -556,7 +558,6 @@
- Issue #10910: Avoid C++ compilation errors on FreeBSD and OS X.
Also update FreedBSD version checks for the original ctype UTF-8 workaround.
-
What's New in Python 3.6.0 alpha 3
==================================
diff --git a/Modules/_io/_iomodule.c b/Modules/_io/_iomodule.c
--- a/Modules/_io/_iomodule.c
+++ b/Modules/_io/_iomodule.c
@@ -20,6 +20,9 @@
#include <sys/stat.h>
#endif /* HAVE_SYS_STAT_H */
+#ifdef MS_WINDOWS
+#include <consoleapi.h>
+#endif
/* Various interned strings */
@@ -52,7 +55,6 @@
PyObject *_PyIO_empty_bytes;
PyObject *_PyIO_zero;
-
PyDoc_STRVAR(module_doc,
"The io module provides the Python interfaces to stream handling. The\n"
"builtin open function is defined in this module.\n"
@@ -362,8 +364,18 @@
}
/* Create the Raw file stream */
- raw = PyObject_CallFunction((PyObject *)&PyFileIO_Type,
- "OsiO", path_or_fd, rawmode, closefd, opener);
+ {
+ PyObject *RawIO_class = (PyObject *)&PyFileIO_Type;
+#ifdef MS_WINDOWS
+ if (!Py_LegacyWindowsStdioFlag && _PyIO_get_console_type(path_or_fd) != '\0') {
+ RawIO_class = (PyObject *)&PyWindowsConsoleIO_Type;
+ encoding = "utf-8";
+ }
+#endif
+ raw = PyObject_CallFunction(RawIO_class,
+ "OsiO", path_or_fd, rawmode, closefd, opener);
+ }
+
if (raw == NULL)
goto error;
result = raw;
@@ -708,6 +720,12 @@
PyStringIO_Type.tp_base = &PyTextIOBase_Type;
ADD_TYPE(&PyStringIO_Type, "StringIO");
+#ifdef MS_WINDOWS
+ /* WindowsConsoleIO */
+ PyWindowsConsoleIO_Type.tp_base = &PyRawIOBase_Type;
+ ADD_TYPE(&PyWindowsConsoleIO_Type, "_WindowsConsoleIO");
+#endif
+
/* BufferedReader */
PyBufferedReader_Type.tp_base = &PyBufferedIOBase_Type;
ADD_TYPE(&PyBufferedReader_Type, "BufferedReader");
diff --git a/Modules/_io/_iomodule.h b/Modules/_io/_iomodule.h
--- a/Modules/_io/_iomodule.h
+++ b/Modules/_io/_iomodule.h
@@ -19,6 +19,12 @@
extern PyTypeObject PyTextIOWrapper_Type;
extern PyTypeObject PyIncrementalNewlineDecoder_Type;
+#ifndef Py_LIMITED_API
+#ifdef MS_WINDOWS
+extern PyTypeObject PyWindowsConsoleIO_Type;
+#define PyWindowsConsoleIO_Check(op) (PyObject_TypeCheck((op), &PyWindowsConsoleIO_Type))
+#endif /* MS_WINDOWS */
+#endif /* Py_LIMITED_API */
extern int _PyIO_ConvertSsize_t(PyObject *, void *);
@@ -145,6 +151,10 @@
extern _PyIO_State *_PyIO_get_module_state(void);
extern PyObject *_PyIO_get_locale_module(_PyIO_State *);
+#ifdef MS_WINDOWS
+extern char _PyIO_get_console_type(PyObject *);
+#endif
+
extern PyObject *_PyIO_str_close;
extern PyObject *_PyIO_str_closed;
extern PyObject *_PyIO_str_decode;
diff --git a/Modules/_io/clinic/winconsoleio.c.h b/Modules/_io/clinic/winconsoleio.c.h
new file mode 100644
--- /dev/null
+++ b/Modules/_io/clinic/winconsoleio.c.h
@@ -0,0 +1,331 @@
+/*[clinic input]
+preserve
+[clinic start generated code]*/
+
+#if defined(MS_WINDOWS)
+
+PyDoc_STRVAR(_io__WindowsConsoleIO_close__doc__,
+"close($self, /)\n"
+"--\n"
+"\n"
+"Close the handle.\n"
+"\n"
+"A closed handle cannot be used for further I/O operations. close() may be\n"
+"called more than once without error.");
+
+#define _IO__WINDOWSCONSOLEIO_CLOSE_METHODDEF \
+ {"close", (PyCFunction)_io__WindowsConsoleIO_close, METH_NOARGS, _io__WindowsConsoleIO_close__doc__},
+
+static PyObject *
+_io__WindowsConsoleIO_close_impl(winconsoleio *self);
+
+static PyObject *
+_io__WindowsConsoleIO_close(winconsoleio *self, PyObject *Py_UNUSED(ignored))
+{
+ return _io__WindowsConsoleIO_close_impl(self);
+}
+
+#endif /* defined(MS_WINDOWS) */
+
+#if defined(MS_WINDOWS)
+
+PyDoc_STRVAR(_io__WindowsConsoleIO___init____doc__,
+"_WindowsConsoleIO(file, mode=\'r\', closefd=True, opener=None)\n"
+"--\n"
+"\n"
+"Open a console buffer by file descriptor.\n"
+"\n"
+"The mode can be \'rb\' (default), or \'wb\' for reading or writing bytes. All\n"
+"other mode characters will be ignored. Mode \'b\' will be assumed if it is\n"
+"omitted. The *opener* parameter is always ignored.");
+
+static int
+_io__WindowsConsoleIO___init___impl(winconsoleio *self, PyObject *nameobj,
+ const char *mode, int closefd,
+ PyObject *opener);
+
+static int
+_io__WindowsConsoleIO___init__(PyObject *self, PyObject *args, PyObject *kwargs)
+{
+ int return_value = -1;
+ static const char * const _keywords[] = {"file", "mode", "closefd", "opener", NULL};
+ static _PyArg_Parser _parser = {"O|siO:_WindowsConsoleIO", _keywords, 0};
+ PyObject *nameobj;
+ const char *mode = "r";
+ int closefd = 1;
+ PyObject *opener = Py_None;
+
+ if (!_PyArg_ParseTupleAndKeywordsFast(args, kwargs, &_parser,
+ &nameobj, &mode, &closefd, &opener)) {
+ goto exit;
+ }
+ return_value = _io__WindowsConsoleIO___init___impl((winconsoleio *)self, nameobj, mode, closefd, opener);
+
+exit:
+ return return_value;
+}
+
+#endif /* defined(MS_WINDOWS) */
+
+#if defined(MS_WINDOWS)
+
+PyDoc_STRVAR(_io__WindowsConsoleIO_fileno__doc__,
+"fileno($self, /)\n"
+"--\n"
+"\n"
+"Return the underlying file descriptor (an integer).\n"
+"\n"
+"fileno is only set when a file descriptor is used to open\n"
+"one of the standard streams.");
+
+#define _IO__WINDOWSCONSOLEIO_FILENO_METHODDEF \
+ {"fileno", (PyCFunction)_io__WindowsConsoleIO_fileno, METH_NOARGS, _io__WindowsConsoleIO_fileno__doc__},
+
+static PyObject *
+_io__WindowsConsoleIO_fileno_impl(winconsoleio *self);
+
+static PyObject *
+_io__WindowsConsoleIO_fileno(winconsoleio *self, PyObject *Py_UNUSED(ignored))
+{
+ return _io__WindowsConsoleIO_fileno_impl(self);
+}
+
+#endif /* defined(MS_WINDOWS) */
+
+#if defined(MS_WINDOWS)
+
+PyDoc_STRVAR(_io__WindowsConsoleIO_readable__doc__,
+"readable($self, /)\n"
+"--\n"
+"\n"
+"True if console is an input buffer.");
+
+#define _IO__WINDOWSCONSOLEIO_READABLE_METHODDEF \
+ {"readable", (PyCFunction)_io__WindowsConsoleIO_readable, METH_NOARGS, _io__WindowsConsoleIO_readable__doc__},
+
+static PyObject *
+_io__WindowsConsoleIO_readable_impl(winconsoleio *self);
+
+static PyObject *
+_io__WindowsConsoleIO_readable(winconsoleio *self, PyObject *Py_UNUSED(ignored))
+{
+ return _io__WindowsConsoleIO_readable_impl(self);
+}
+
+#endif /* defined(MS_WINDOWS) */
+
+#if defined(MS_WINDOWS)
+
+PyDoc_STRVAR(_io__WindowsConsoleIO_writable__doc__,
+"writable($self, /)\n"
+"--\n"
+"\n"
+"True if console is an output buffer.");
+
+#define _IO__WINDOWSCONSOLEIO_WRITABLE_METHODDEF \
+ {"writable", (PyCFunction)_io__WindowsConsoleIO_writable, METH_NOARGS, _io__WindowsConsoleIO_writable__doc__},
+
+static PyObject *
+_io__WindowsConsoleIO_writable_impl(winconsoleio *self);
+
+static PyObject *
+_io__WindowsConsoleIO_writable(winconsoleio *self, PyObject *Py_UNUSED(ignored))
+{
+ return _io__WindowsConsoleIO_writable_impl(self);
+}
+
+#endif /* defined(MS_WINDOWS) */
+
+#if defined(MS_WINDOWS)
+
+PyDoc_STRVAR(_io__WindowsConsoleIO_readinto__doc__,
+"readinto($self, buffer, /)\n"
+"--\n"
+"\n"
+"Same as RawIOBase.readinto().");
+
+#define _IO__WINDOWSCONSOLEIO_READINTO_METHODDEF \
+ {"readinto", (PyCFunction)_io__WindowsConsoleIO_readinto, METH_O, _io__WindowsConsoleIO_readinto__doc__},
+
+static PyObject *
+_io__WindowsConsoleIO_readinto_impl(winconsoleio *self, Py_buffer *buffer);
+
+static PyObject *
+_io__WindowsConsoleIO_readinto(winconsoleio *self, PyObject *arg)
+{
+ PyObject *return_value = NULL;
+ Py_buffer buffer = {NULL, NULL};
+
+ if (!PyArg_Parse(arg, "w*:readinto", &buffer)) {
+ goto exit;
+ }
+ return_value = _io__WindowsConsoleIO_readinto_impl(self, &buffer);
+
+exit:
+ /* Cleanup for buffer */
+ if (buffer.obj) {
+ PyBuffer_Release(&buffer);
+ }
+
+ return return_value;
+}
+
+#endif /* defined(MS_WINDOWS) */
+
+#if defined(MS_WINDOWS)
+
+PyDoc_STRVAR(_io__WindowsConsoleIO_readall__doc__,
+"readall($self, /)\n"
+"--\n"
+"\n"
+"Read all data from the console, returned as bytes.\n"
+"\n"
+"Return an empty bytes object at EOF.");
+
+#define _IO__WINDOWSCONSOLEIO_READALL_METHODDEF \
+ {"readall", (PyCFunction)_io__WindowsConsoleIO_readall, METH_NOARGS, _io__WindowsConsoleIO_readall__doc__},
+
+static PyObject *
+_io__WindowsConsoleIO_readall_impl(winconsoleio *self);
+
+static PyObject *
+_io__WindowsConsoleIO_readall(winconsoleio *self, PyObject *Py_UNUSED(ignored))
+{
+ return _io__WindowsConsoleIO_readall_impl(self);
+}
+
+#endif /* defined(MS_WINDOWS) */
+
+#if defined(MS_WINDOWS)
+
+PyDoc_STRVAR(_io__WindowsConsoleIO_read__doc__,
+"read($self, size=-1, /)\n"
+"--\n"
+"\n"
+"Read at most size bytes, returned as bytes.\n"
+"\n"
+"Only makes one system call when size is a positive integer,\n"
+"so less data may be returned than requested.\n"
+"Return an empty bytes object at EOF.");
+
+#define _IO__WINDOWSCONSOLEIO_READ_METHODDEF \
+ {"read", (PyCFunction)_io__WindowsConsoleIO_read, METH_VARARGS, _io__WindowsConsoleIO_read__doc__},
+
+static PyObject *
+_io__WindowsConsoleIO_read_impl(winconsoleio *self, Py_ssize_t size);
+
+static PyObject *
+_io__WindowsConsoleIO_read(winconsoleio *self, PyObject *args)
+{
+ PyObject *return_value = NULL;
+ Py_ssize_t size = -1;
+
+ if (!PyArg_ParseTuple(args, "|O&:read",
+ _PyIO_ConvertSsize_t, &size)) {
+ goto exit;
+ }
+ return_value = _io__WindowsConsoleIO_read_impl(self, size);
+
+exit:
+ return return_value;
+}
+
+#endif /* defined(MS_WINDOWS) */
+
+#if defined(MS_WINDOWS)
+
+PyDoc_STRVAR(_io__WindowsConsoleIO_write__doc__,
+"write($self, b, /)\n"
+"--\n"
+"\n"
+"Write buffer b to file, return number of bytes written.\n"
+"\n"
+"Only makes one system call, so not all of the data may be written.\n"
+"The number of bytes actually written is returned.");
+
+#define _IO__WINDOWSCONSOLEIO_WRITE_METHODDEF \
+ {"write", (PyCFunction)_io__WindowsConsoleIO_write, METH_O, _io__WindowsConsoleIO_write__doc__},
+
+static PyObject *
+_io__WindowsConsoleIO_write_impl(winconsoleio *self, Py_buffer *b);
+
+static PyObject *
+_io__WindowsConsoleIO_write(winconsoleio *self, PyObject *arg)
+{
+ PyObject *return_value = NULL;
+ Py_buffer b = {NULL, NULL};
+
+ if (!PyArg_Parse(arg, "y*:write", &b)) {
+ goto exit;
+ }
+ return_value = _io__WindowsConsoleIO_write_impl(self, &b);
+
+exit:
+ /* Cleanup for b */
+ if (b.obj) {
+ PyBuffer_Release(&b);
+ }
+
+ return return_value;
+}
+
+#endif /* defined(MS_WINDOWS) */
+
+#if defined(MS_WINDOWS)
+
+PyDoc_STRVAR(_io__WindowsConsoleIO_isatty__doc__,
+"isatty($self, /)\n"
+"--\n"
+"\n"
+"Always True.");
+
+#define _IO__WINDOWSCONSOLEIO_ISATTY_METHODDEF \
+ {"isatty", (PyCFunction)_io__WindowsConsoleIO_isatty, METH_NOARGS, _io__WindowsConsoleIO_isatty__doc__},
+
+static PyObject *
+_io__WindowsConsoleIO_isatty_impl(winconsoleio *self);
+
+static PyObject *
+_io__WindowsConsoleIO_isatty(winconsoleio *self, PyObject *Py_UNUSED(ignored))
+{
+ return _io__WindowsConsoleIO_isatty_impl(self);
+}
+
+#endif /* defined(MS_WINDOWS) */
+
+#ifndef _IO__WINDOWSCONSOLEIO_CLOSE_METHODDEF
+ #define _IO__WINDOWSCONSOLEIO_CLOSE_METHODDEF
+#endif /* !defined(_IO__WINDOWSCONSOLEIO_CLOSE_METHODDEF) */
+
+#ifndef _IO__WINDOWSCONSOLEIO_FILENO_METHODDEF
+ #define _IO__WINDOWSCONSOLEIO_FILENO_METHODDEF
+#endif /* !defined(_IO__WINDOWSCONSOLEIO_FILENO_METHODDEF) */
+
+#ifndef _IO__WINDOWSCONSOLEIO_READABLE_METHODDEF
+ #define _IO__WINDOWSCONSOLEIO_READABLE_METHODDEF
+#endif /* !defined(_IO__WINDOWSCONSOLEIO_READABLE_METHODDEF) */
+
+#ifndef _IO__WINDOWSCONSOLEIO_WRITABLE_METHODDEF
+ #define _IO__WINDOWSCONSOLEIO_WRITABLE_METHODDEF
+#endif /* !defined(_IO__WINDOWSCONSOLEIO_WRITABLE_METHODDEF) */
+
+#ifndef _IO__WINDOWSCONSOLEIO_READINTO_METHODDEF
+ #define _IO__WINDOWSCONSOLEIO_READINTO_METHODDEF
+#endif /* !defined(_IO__WINDOWSCONSOLEIO_READINTO_METHODDEF) */
+
+#ifndef _IO__WINDOWSCONSOLEIO_READALL_METHODDEF
+ #define _IO__WINDOWSCONSOLEIO_READALL_METHODDEF
+#endif /* !defined(_IO__WINDOWSCONSOLEIO_READALL_METHODDEF) */
+
+#ifndef _IO__WINDOWSCONSOLEIO_READ_METHODDEF
+ #define _IO__WINDOWSCONSOLEIO_READ_METHODDEF
+#endif /* !defined(_IO__WINDOWSCONSOLEIO_READ_METHODDEF) */
+
+#ifndef _IO__WINDOWSCONSOLEIO_WRITE_METHODDEF
+ #define _IO__WINDOWSCONSOLEIO_WRITE_METHODDEF
+#endif /* !defined(_IO__WINDOWSCONSOLEIO_WRITE_METHODDEF) */
+
+#ifndef _IO__WINDOWSCONSOLEIO_ISATTY_METHODDEF
+ #define _IO__WINDOWSCONSOLEIO_ISATTY_METHODDEF
+#endif /* !defined(_IO__WINDOWSCONSOLEIO_ISATTY_METHODDEF) */
+/*[clinic end generated code: output=9eba916f8537fff7 input=a9049054013a1b77]*/
diff --git a/Modules/_io/winconsoleio.c b/Modules/_io/winconsoleio.c
new file mode 100644
--- /dev/null
+++ b/Modules/_io/winconsoleio.c
@@ -0,0 +1,1096 @@
+/*
+ An implementation of Windows console I/O
+
+ Classes defined here: _WindowsConsoleIO
+
+ Written by Steve Dower
+*/
+
+#define PY_SSIZE_T_CLEAN
+#include "Python.h"
+
+#ifdef MS_WINDOWS
+
+#include "structmember.h"
+#ifdef HAVE_SYS_TYPES_H
+#include <sys/types.h>
+#endif
+#ifdef HAVE_SYS_STAT_H
+#include <sys/stat.h>
+#endif
+#include <stddef.h> /* For offsetof */
+
+#define WIN32_LEAN_AND_MEAN
+#include <windows.h>
+
+#include "_iomodule.h"
+
+/* BUFSIZ determines how many characters can be typed at the console
+ before it starts blocking. */
+#if BUFSIZ < (16*1024)
+#define SMALLCHUNK (2*1024)
+#elif (BUFSIZ >= (2 << 25))
+#error "unreasonable BUFSIZ > 64MB defined"
+#else
+#define SMALLCHUNK BUFSIZ
+#endif
+
+/* BUFMAX determines how many bytes can be read in one go. */
+#define BUFMAX (32*1024*1024)
+
+char _get_console_type(HANDLE handle) {
+ DWORD mode, peek_count;
+
+ if (handle == INVALID_HANDLE_VALUE)
+ return '\0';
+
+ if (!GetConsoleMode(handle, &mode))
+ return '\0';
+
+ /* Peek at the handle to see whether it is an input or output handle */
+ if (GetNumberOfConsoleInputEvents(handle, &peek_count))
+ return 'r';
+ return 'w';
+}
+
+char _PyIO_get_console_type(PyObject *path_or_fd) {
+ int fd;
+
+ fd = PyLong_AsLong(path_or_fd);
+ PyErr_Clear();
+ if (fd >= 0) {
+ HANDLE handle;
+ _Py_BEGIN_SUPPRESS_IPH
+ handle = (HANDLE)_get_osfhandle(fd);
+ _Py_END_SUPPRESS_IPH
+ if (!handle)
+ return '\0';
+ return _get_console_type(handle);
+ }
+
+ PyObject *decoded = Py_None;
+ Py_INCREF(decoded);
+
+ int d = PyUnicode_FSDecoder(path_or_fd, &decoded);
+ if (!d) {
+ PyErr_Clear();
+ Py_CLEAR(decoded);
+ return '\0';
+ }
+
+ char m = '\0';
+ if (!PyUnicode_Check(decoded)) {
+ return '\0';
+ } else if (PyUnicode_CompareWithASCIIString(decoded, "CONIN$") == 0) {
+ m = 'r';
+ } else if (PyUnicode_CompareWithASCIIString(decoded, "CONOUT$") == 0 ||
+ PyUnicode_CompareWithASCIIString(decoded, "CON") == 0) {
+ m = 'w';
+ }
+
+ Py_CLEAR(decoded);
+ return m;
+}
+
+/*[clinic input]
+module _io
+class _io._WindowsConsoleIO "winconsoleio *" "&PyWindowsConsoleIO_Type"
+[clinic start generated code]*/
+/*[clinic end generated code: output=da39a3ee5e6b4b0d input=e897fdc1fba4e131]*/
+
+/*[python input]
+class io_ssize_t_converter(CConverter):
+ type = 'Py_ssize_t'
+ converter = '_PyIO_ConvertSsize_t'
+[python start generated code]*/
+/*[python end generated code: output=da39a3ee5e6b4b0d input=d0a811d3cbfd1b33]*/
+
+typedef struct {
+ PyObject_HEAD
+ HANDLE handle;
+ int fd;
+ unsigned int created : 1;
+ unsigned int readable : 1;
+ unsigned int writable : 1;
+ unsigned int closehandle : 1;
+ char finalizing;
+ unsigned int blksize;
+ PyObject *weakreflist;
+ PyObject *dict;
+ char buf[4];
+} winconsoleio;
+
+PyTypeObject PyWindowsConsoleIO_Type;
+
+_Py_IDENTIFIER(name);
+
+int
+_PyWindowsConsoleIO_closed(PyObject *self)
+{
+ return ((winconsoleio *)self)->handle == INVALID_HANDLE_VALUE;
+}
+
+
+/* Returns 0 on success, -1 with exception set on failure. */
+static int
+internal_close(winconsoleio *self)
+{
+ if (self->handle != INVALID_HANDLE_VALUE) {
+ if (self->closehandle) {
+ if (self->fd >= 0) {
+ _Py_BEGIN_SUPPRESS_IPH
+ close(self->fd);
+ _Py_END_SUPPRESS_IPH
+ }
+ CloseHandle(self->handle);
+ }
+ self->handle = INVALID_HANDLE_VALUE;
+ self->fd = -1;
+ }
+ return 0;
+}
+
+/*[clinic input]
+_io._WindowsConsoleIO.close
+
+Close the handle.
+
+A closed handle cannot be used for further I/O operations. close() may be
+called more than once without error.
+[clinic start generated code]*/
+
+static PyObject *
+_io__WindowsConsoleIO_close_impl(winconsoleio *self)
+/*[clinic end generated code: output=27ef95b66c29057b input=185617e349ae4c7b]*/
+{
+ PyObject *res;
+ PyObject *exc, *val, *tb;
+ int rc;
+ _Py_IDENTIFIER(close);
+ res = _PyObject_CallMethodId((PyObject*)&PyRawIOBase_Type,
+ &PyId_close, "O", self);
+ if (!self->closehandle) {
+ self->handle = INVALID_HANDLE_VALUE;
+ return res;
+ }
+ if (res == NULL)
+ PyErr_Fetch(&exc, &val, &tb);
+ rc = internal_close(self);
+ if (res == NULL)
+ _PyErr_ChainExceptions(exc, val, tb);
+ if (rc < 0)
+ Py_CLEAR(res);
+ return res;
+}
+
+static PyObject *
+winconsoleio_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
+{
+ winconsoleio *self;
+
+ assert(type != NULL && type->tp_alloc != NULL);
+
+ self = (winconsoleio *) type->tp_alloc(type, 0);
+ if (self != NULL) {
+ self->handle = INVALID_HANDLE_VALUE;
+ self->fd = -1;
+ self->created = 0;
+ self->readable = 0;
+ self->writable = 0;
+ self->closehandle = 0;
+ self->blksize = 0;
+ self->weakreflist = NULL;
+ }
+
+ return (PyObject *) self;
+}
+
+/*[clinic input]
+_io._WindowsConsoleIO.__init__
+ file as nameobj: object
+ mode: str = "r"
+ closefd: int(c_default="1") = True
+ opener: object = None
+
+Open a console buffer by file descriptor.
+
+The mode can be 'rb' (default), or 'wb' for reading or writing bytes. All
+other mode characters will be ignored. Mode 'b' will be assumed if it is
+omitted. The *opener* parameter is always ignored.
+[clinic start generated code]*/
+
+static int
+_io__WindowsConsoleIO___init___impl(winconsoleio *self, PyObject *nameobj,
+ const char *mode, int closefd,
+ PyObject *opener)
+/*[clinic end generated code: output=3fd9cbcdd8d95429 input=61be39633a86f5d7]*/
+{
+ const char *s;
+ wchar_t *name = NULL;
+ int ret = 0;
+ int rwa = 0;
+ int fd = -1;
+ int fd_is_own = 0;
+
+ assert(PyWindowsConsoleIO_Check(self));
+ if (self->handle >= 0) {
+ if (self->closehandle) {
+ /* Have to close the existing file first. */
+ if (internal_close(self) < 0)
+ return -1;
+ }
+ else
+ self->handle = INVALID_HANDLE_VALUE;
+ }
+
+ if (PyFloat_Check(nameobj)) {
+ PyErr_SetString(PyExc_TypeError,
+ "integer argument expected, got float");
+ return -1;
+ }
+
+ fd = _PyLong_AsInt(nameobj);
+ if (fd < 0) {
+ if (!PyErr_Occurred()) {
+ PyErr_SetString(PyExc_ValueError,
+ "negative file descriptor");
+ return -1;
+ }
+ PyErr_Clear();
+ }
+ self->fd = fd;
+
+ if (fd < 0) {
+ PyObject *decodedname = Py_None;
+ Py_INCREF(decodedname);
+
+ int d = PyUnicode_FSDecoder(nameobj, (void*)&decodedname);
+ if (!d)
+ return -1;
+
+ Py_ssize_t length;
+ name = PyUnicode_AsWideCharString(decodedname, &length);
+ Py_CLEAR(decodedname);
+ if (name == NULL)
+ return -1;
+
+ if (wcslen(name) != length) {
+ PyMem_Free(name);
+ PyErr_SetString(PyExc_ValueError, "embedded null character");
+ return -1;
+ }
+ }
+
+ s = mode;
+ while (*s) {
+ switch (*s++) {
+ case '+':
+ case 'a':
+ case 'b':
+ case 'x':
+ break;
+ case 'r':
+ if (rwa)
+ goto bad_mode;
+ rwa = 1;
+ self->readable = 1;
+ break;
+ case 'w':
+ if (rwa)
+ goto bad_mode;
+ rwa = 1;
+ self->writable = 1;
+ break;
+ default:
+ PyErr_Format(PyExc_ValueError,
+ "invalid mode: %.200s", mode);
+ goto error;
+ }
+ }
+
+ if (!rwa)
+ goto bad_mode;
+
+ if (fd >= 0) {
+ _Py_BEGIN_SUPPRESS_IPH
+ self->handle = (HANDLE)_get_osfhandle(fd);
+ _Py_END_SUPPRESS_IPH
+ self->closehandle = 0;
+ } else {
+ DWORD access = GENERIC_READ;
+
+ self->closehandle = 1;
+ if (!closefd) {
+ PyErr_SetString(PyExc_ValueError,
+ "Cannot use closefd=False with file name");
+ goto error;
+ }
+
+ if (self->writable)
+ access |= GENERIC_WRITE;
+
+ Py_BEGIN_ALLOW_THREADS
+ /* Attempt to open for read/write initially, then fall back
+ on the specific access. This is required for modern names
+ CONIN$ and CONOUT$, which allow reading/writing state as
+ well as reading/writing content. */
+ self->handle = CreateFileW(name, GENERIC_READ | GENERIC_WRITE,
+ FILE_SHARE_READ | FILE_SHARE_WRITE, NULL, OPEN_EXISTING, 0, NULL);
+ if (self->handle == INVALID_HANDLE_VALUE)
+ self->handle = CreateFileW(name, access,
+ FILE_SHARE_READ | FILE_SHARE_WRITE, NULL, OPEN_EXISTING, 0, NULL);
+ Py_END_ALLOW_THREADS
+
+ if (self->handle == INVALID_HANDLE_VALUE) {
+ PyErr_SetExcFromWindowsErrWithFilenameObject(PyExc_OSError, GetLastError(), nameobj);
+ goto error;
+ }
+ }
+
+ if (self->writable && _get_console_type(self->handle) != 'w') {
+ PyErr_SetString(PyExc_ValueError,
+ "Cannot open console input buffer for writing");
+ goto error;
+ }
+ if (self->readable && _get_console_type(self->handle) != 'r') {
+ PyErr_SetString(PyExc_ValueError,
+ "Cannot open console output buffer for reading");
+ goto error;
+ }
+
+ self->blksize = DEFAULT_BUFFER_SIZE;
+ memset(self->buf, 0, 4);
+
+ if (_PyObject_SetAttrId((PyObject *)self, &PyId_name, nameobj) < 0)
+ goto error;
+
+ goto done;
+
+bad_mode:
+ PyErr_SetString(PyExc_ValueError,
+ "Must have exactly one of read or write mode");
+error:
+ ret = -1;
+ internal_close(self);
+
+done:
+ if (name)
+ PyMem_Free(name);
+ return ret;
+}
+
+static int
+winconsoleio_traverse(winconsoleio *self, visitproc visit, void *arg)
+{
+ Py_VISIT(self->dict);
+ return 0;
+}
+
+static int
+winconsoleio_clear(winconsoleio *self)
+{
+ Py_CLEAR(self->dict);
+ return 0;
+}
+
+static void
+winconsoleio_dealloc(winconsoleio *self)
+{
+ self->finalizing = 1;
+ if (_PyIOBase_finalize((PyObject *) self) < 0)
+ return;
+ _PyObject_GC_UNTRACK(self);
+ if (self->weakreflist != NULL)
+ PyObject_ClearWeakRefs((PyObject *) self);
+ Py_CLEAR(self->dict);
+ Py_TYPE(self)->tp_free((PyObject *)self);
+}
+
+static PyObject *
+err_closed(void)
+{
+ PyErr_SetString(PyExc_ValueError, "I/O operation on closed file");
+ return NULL;
+}
+
+static PyObject *
+err_mode(const char *action)
+{
+ _PyIO_State *state = IO_STATE();
+ if (state != NULL)
+ PyErr_Format(state->unsupported_operation,
+ "Console buffer does not support %s", action);
+ return NULL;
+}
+
+/*[clinic input]
+_io._WindowsConsoleIO.fileno
+
+Return the underlying file descriptor (an integer).
+
+fileno is only set when a file descriptor is used to open
+one of the standard streams.
+
+[clinic start generated code]*/
+
+static PyObject *
+_io__WindowsConsoleIO_fileno_impl(winconsoleio *self)
+/*[clinic end generated code: output=006fa74ce3b5cfbf input=079adc330ddaabe6]*/
+{
+ if (self->fd < 0 && self->handle != INVALID_HANDLE_VALUE) {
+ _Py_BEGIN_SUPPRESS_IPH
+ if (self->writable)
+ self->fd = _open_osfhandle((intptr_t)self->handle, 'wb');
+ else
+ self->fd = _open_osfhandle((intptr_t)self->handle, 'rb');
+ _Py_END_SUPPRESS_IPH
+ }
+ if (self->fd < 0)
+ return err_mode("fileno");
+ return PyLong_FromLong(self->fd);
+}
+
+/*[clinic input]
+_io._WindowsConsoleIO.readable
+
+True if console is an input buffer.
+[clinic start generated code]*/
+
+static PyObject *
+_io__WindowsConsoleIO_readable_impl(winconsoleio *self)
+/*[clinic end generated code: output=daf9cef2743becf0 input=6be9defb5302daae]*/
+{
+ if (self->handle == INVALID_HANDLE_VALUE)
+ return err_closed();
+ return PyBool_FromLong((long) self->readable);
+}
+
+/*[clinic input]
+_io._WindowsConsoleIO.writable
+
+True if console is an output buffer.
+[clinic start generated code]*/
+
+static PyObject *
+_io__WindowsConsoleIO_writable_impl(winconsoleio *self)
+/*[clinic end generated code: output=e0a2ad7eae5abf67 input=cefbd8abc24df6a0]*/
+{
+ if (self->handle == INVALID_HANDLE_VALUE)
+ return err_closed();
+ return PyBool_FromLong((long) self->writable);
+}
+
+static DWORD
+_buflen(winconsoleio *self)
+{
+ for (DWORD i = 0; i < 4; ++i) {
+ if (!self->buf[i])
+ return i;
+ }
+ return 4;
+}
+
+static DWORD
+_copyfrombuf(winconsoleio *self, char *buf, DWORD len)
+{
+ DWORD n = 0;
+
+ while (self->buf[0] && len--) {
+ n += 1;
+ buf[0] = self->buf[0];
+ self->buf[0] = self->buf[1];
+ self->buf[1] = self->buf[2];
+ self->buf[2] = self->buf[3];
+ self->buf[3] = 0;
+ }
+
+ return n;
+}
+
+static wchar_t *
+read_console_w(HANDLE handle, DWORD maxlen, DWORD *readlen) {
+ int err = 0, sig = 0;
+
+ wchar_t *buf = (wchar_t*)PyMem_Malloc(maxlen * sizeof(wchar_t));
+ if (!buf)
+ goto error;
+ *readlen = 0;
+
+ Py_BEGIN_ALLOW_THREADS
+ for (DWORD off = 0; off < maxlen; off += BUFSIZ) {
+ DWORD n, len = min(maxlen - off, BUFSIZ);
+ SetLastError(0);
+ BOOL res = ReadConsoleW(handle, &buf[off], len, &n, NULL);
+
+ if (!res) {
+ err = GetLastError();
+ break;
+ }
+ if (n == 0) {
+ err = GetLastError();
+ if (err != ERROR_OPERATION_ABORTED)
+ break;
+ err = 0;
+ HANDLE hInterruptEvent = _PyOS_SigintEvent();
+ if (WaitForSingleObjectEx(hInterruptEvent, 100, FALSE)
+ == WAIT_OBJECT_0) {
+ ResetEvent(hInterruptEvent);
+ Py_BLOCK_THREADS
+ sig = PyErr_CheckSignals();
+ Py_UNBLOCK_THREADS
+ if (sig < 0)
+ break;
+ }
+ }
+ *readlen += n;
+
+ /* If we didn't read a full buffer that time, don't try
+ again or we will block a second time. */
+ if (n < len)
+ break;
+ /* If the buffer ended with a newline, break out */
+ if (buf[*readlen - 1] == '\n')
+ break;
+ }
+ Py_END_ALLOW_THREADS
+
+ if (sig)
+ goto error;
+ if (err) {
+ PyErr_SetFromWindowsErr(err);
+ goto error;
+ }
+
+ if (*readlen > 0 && buf[0] == L'\x1a') {
+ PyMem_Free(buf);
+ buf = (wchar_t *)PyMem_Malloc(sizeof(wchar_t));
+ if (!buf)
+ goto error;
+ buf[0] = L'\0';
+ *readlen = 0;
+ }
+
+ return buf;
+
+error:
+ if (buf)
+ PyMem_Free(buf);
+ return NULL;
+}
+
+
+static Py_ssize_t
+readinto(winconsoleio *self, char *buf, Py_ssize_t len)
+{
+ if (self->handle == INVALID_HANDLE_VALUE) {
+ err_closed();
+ return -1;
+ }
+ if (!self->readable) {
+ err_mode("reading");
+ return -1;
+ }
+ if (len == 0)
+ return 0;
+ if (len > BUFMAX) {
+ PyErr_Format(PyExc_ValueError, "cannot read more than %d bytes", BUFMAX);
+ return -1;
+ }
+
+ /* Each character may take up to 4 bytes in the final buffer.
+ This is highly conservative, but necessary to avoid
+ failure for any given Unicode input (e.g. \U0010ffff).
+ If the caller requests fewer than 4 bytes, we buffer one
+ character.
+ */
+ DWORD wlen = (DWORD)(len / 4);
+ if (wlen == 0) {
+ wlen = 1;
+ }
+
+ DWORD read_len = _copyfrombuf(self, buf, (DWORD)len);
+ if (read_len) {
+ buf = &buf[read_len];
+ len -= read_len;
+ wlen -= 1;
+ }
+ if (len == read_len || wlen == 0)
+ return read_len;
+
+ DWORD n;
+ wchar_t *wbuf = read_console_w(self->handle, wlen, &n);
+ if (wbuf == NULL)
+ return -1;
+ if (n == 0) {
+ PyMem_Free(wbuf);
+ return read_len;
+ }
+
+ int err = 0;
+ DWORD u8n = 0;
+
+ Py_BEGIN_ALLOW_THREADS
+ if (len < 4) {
+ if (WideCharToMultiByte(CP_UTF8, 0, wbuf, n,
+ self->buf, sizeof(self->buf) / sizeof(self->buf[0]),
+ NULL, NULL))
+ u8n = _copyfrombuf(self, buf, (DWORD)len);
+ } else {
+ u8n = WideCharToMultiByte(CP_UTF8, 0, wbuf, n,
+ buf, (DWORD)len, NULL, NULL);
+ }
+
+ if (u8n) {
+ read_len += u8n;
+ u8n = 0;
+ } else {
+ err = GetLastError();
+ if (err == ERROR_INSUFFICIENT_BUFFER) {
+ /* Calculate the needed buffer for a more useful error, as this
+ means our "/ 4" logic above is insufficient for some input.
+ */
+ u8n = WideCharToMultiByte(CP_UTF8, 0, wbuf, n,
+ NULL, 0, NULL, NULL);
+ }
+ }
+ Py_END_ALLOW_THREADS
+
+ PyMem_Free(wbuf);
+
+ if (u8n) {
+ PyErr_Format(PyExc_SystemError,
+ "Buffer had room for %d bytes but %d bytes required",
+ len, u8n);
+ return -1;
+ }
+ if (err) {
+ PyErr_SetFromWindowsErr(err);
+ return -1;
+ }
+
+ return read_len;
+}
+
+/*[clinic input]
+_io._WindowsConsoleIO.readinto
+ buffer: Py_buffer(accept={rwbuffer})
+ /
+
+Same as RawIOBase.readinto().
+[clinic start generated code]*/
+
+static PyObject *
+_io__WindowsConsoleIO_readinto_impl(winconsoleio *self, Py_buffer *buffer)
+/*[clinic end generated code: output=66d1bdfa3f20af39 input=4ed68da48a6baffe]*/
+{
+ Py_ssize_t len = readinto(self, buffer->buf, buffer->len);
+ if (len < 0)
+ return NULL;
+
+ return PyLong_FromSsize_t(len);
+}
+
+static DWORD
+new_buffersize(winconsoleio *self, DWORD currentsize)
+{
+ DWORD addend;
+
+ /* Expand the buffer by an amount proportional to the current size,
+ giving us amortized linear-time behavior. For bigger sizes, use a
+ less-than-double growth factor to avoid excessive allocation. */
+ if (currentsize > 65536)
+ addend = currentsize >> 3;
+ else
+ addend = 256 + currentsize;
+ if (addend < SMALLCHUNK)
+ /* Avoid tiny read() calls. */
+ addend = SMALLCHUNK;
+ return addend + currentsize;
+}
+
+/*[clinic input]
+_io._WindowsConsoleIO.readall
+
+Read all data from the console, returned as bytes.
+
+Return an empty bytes object at EOF.
+[clinic start generated code]*/
+
+static PyObject *
+_io__WindowsConsoleIO_readall_impl(winconsoleio *self)
+/*[clinic end generated code: output=e6d312c684f6e23b input=4024d649a1006e69]*/
+{
+ wchar_t *buf;
+ DWORD bufsize, n, len = 0;
+ PyObject *bytes;
+ DWORD bytes_size, rn;
+
+ if (self->handle == INVALID_HANDLE_VALUE)
+ return err_closed();
+
+ bufsize = BUFSIZ;
+
+ buf = (wchar_t*)PyMem_Malloc((bufsize + 1) * sizeof(wchar_t));
+ if (buf == NULL)
+ return NULL;
+
+ while (1) {
+ wchar_t *subbuf;
+
+ if (len >= (Py_ssize_t)bufsize) {
+ DWORD newsize = new_buffersize(self, len);
+ if (newsize > BUFMAX)
+ break;
+ if (newsize < bufsize) {
+ PyErr_SetString(PyExc_OverflowError,
+ "unbounded read returned more bytes "
+ "than a Python bytes object can hold");
+ PyMem_Free(buf);
+ return NULL;
+ }
+ bufsize = newsize;
+
+ buf = PyMem_Realloc(buf, (bufsize + 1) * sizeof(wchar_t));
+ if (!buf) {
+ PyMem_Free(buf);
+ return NULL;
+ }
+ }
+
+ subbuf = read_console_w(self->handle, bufsize - len, &n);
+
+ if (subbuf == NULL) {
+ PyMem_Free(buf);
+ return NULL;
+ }
+
+ if (n > 0)
+ wcsncpy_s(&buf[len], bufsize - len + 1, subbuf, n);
+
+ PyMem_Free(subbuf);
+
+ /* when the read starts with ^Z or is empty we break */
+ if (n == 0 || buf[len] == '\x1a')
+ break;
+
+ len += n;
+ }
+
+ if (len > 0 && buf[0] == '\x1a' && _buflen(self) == 0) {
+ /* when the result starts with ^Z we return an empty buffer */
+ PyMem_Free(buf);
+ return PyBytes_FromStringAndSize(NULL, 0);
+ }
+
+ Py_BEGIN_ALLOW_THREADS
+ bytes_size = WideCharToMultiByte(CP_UTF8, 0, buf, len,
+ NULL, 0, NULL, NULL);
+ Py_END_ALLOW_THREADS
+
+ if (!bytes_size) {
+ DWORD err = GetLastError();
+ PyMem_Free(buf);
+ return PyErr_SetFromWindowsErr(err);
+ }
+
+ bytes_size += _buflen(self);
+ bytes = PyBytes_FromStringAndSize(NULL, bytes_size);
+ rn = _copyfrombuf(self, PyBytes_AS_STRING(bytes), bytes_size);
+
+ Py_BEGIN_ALLOW_THREADS
+ bytes_size = WideCharToMultiByte(CP_UTF8, 0, buf, len,
+ &PyBytes_AS_STRING(bytes)[rn], bytes_size - rn, NULL, NULL);
+ Py_END_ALLOW_THREADS
+
+ if (!bytes_size) {
+ DWORD err = GetLastError();
+ PyMem_Free(buf);
+ Py_CLEAR(bytes);
+ return PyErr_SetFromWindowsErr(err);
+ }
+
+ PyMem_Free(buf);
+ if (bytes_size < (size_t)PyBytes_GET_SIZE(bytes)) {
+ if (_PyBytes_Resize(&bytes, n * sizeof(wchar_t)) < 0) {
+ Py_CLEAR(bytes);
+ return NULL;
+ }
+ }
+ return bytes;
+}
+
+/*[clinic input]
+_io._WindowsConsoleIO.read
+ size: io_ssize_t = -1
+ /
+
+Read at most size bytes, returned as bytes.
+
+Only makes one system call when size is a positive integer,
+so less data may be returned than requested.
+Return an empty bytes object at EOF.
+[clinic start generated code]*/
+
+static PyObject *
+_io__WindowsConsoleIO_read_impl(winconsoleio *self, Py_ssize_t size)
+/*[clinic end generated code: output=57df68af9f4b22d0 input=6c56fceec460f1dd]*/
+{
+ PyObject *bytes;
+ Py_ssize_t bytes_size;
+
+ if (self->handle == INVALID_HANDLE_VALUE)
+ return err_closed();
+ if (!self->readable)
+ return err_mode("reading");
+
+ if (size < 0)
+ return _io__WindowsConsoleIO_readall_impl(self);
+ if (size > BUFMAX) {
+ PyErr_Format(PyExc_ValueError, "cannot read more than %d bytes", BUFMAX);
+ return NULL;
+ }
+
+ bytes = PyBytes_FromStringAndSize(NULL, size);
+ if (bytes == NULL)
+ return NULL;
+
+ bytes_size = readinto(self, PyBytes_AS_STRING(bytes), PyBytes_GET_SIZE(bytes));
+ if (bytes_size < 0) {
+ Py_CLEAR(bytes);
+ return NULL;
+ }
+
+ if (bytes_size < PyBytes_GET_SIZE(bytes)) {
+ if (_PyBytes_Resize(&bytes, bytes_size) < 0) {
+ Py_CLEAR(bytes);
+ return NULL;
+ }
+ }
+
+ return bytes;
+}
+
+/*[clinic input]
+_io._WindowsConsoleIO.write
+ b: Py_buffer
+ /
+
+Write buffer b to file, return number of bytes written.
+
+Only makes one system call, so not all of the data may be written.
+The number of bytes actually written is returned.
+[clinic start generated code]*/
+
+static PyObject *
+_io__WindowsConsoleIO_write_impl(winconsoleio *self, Py_buffer *b)
+/*[clinic end generated code: output=775bdb16fbf9137b input=be35fb624f97c941]*/
+{
+ BOOL res = TRUE;
+ wchar_t *wbuf;
+ DWORD len, wlen, n = 0;
+
+ if (self->handle == INVALID_HANDLE_VALUE)
+ return err_closed();
+ if (!self->writable)
+ return err_mode("writing");
+
+ if (b->len > BUFMAX)
+ len = BUFMAX;
+ else
+ len = (DWORD)b->len;
+
+ Py_BEGIN_ALLOW_THREADS
+ wlen = MultiByteToWideChar(CP_UTF8, 0, b->buf, len, NULL, 0);
+
+ /* issue11395 there is an unspecified upper bound on how many bytes
+ can be written at once. We cap at 32k - the caller will have to
+ handle partial writes.
+ Since we don't know how many input bytes are being ignored, we
+ have to reduce and recalculate. */
+ while (wlen > 32766 / sizeof(wchar_t)) {
+ len /= 2;
+ wlen = MultiByteToWideChar(CP_UTF8, 0, b->buf, len, NULL, 0);
+ }
+ Py_END_ALLOW_THREADS
+
+ if (!wlen)
+ return PyErr_SetFromWindowsErr(0);
+
+ wbuf = (wchar_t*)PyMem_Malloc(wlen * sizeof(wchar_t));
+
+ Py_BEGIN_ALLOW_THREADS
+ wlen = MultiByteToWideChar(CP_UTF8, 0, b->buf, len, wbuf, wlen);
+ if (wlen) {
+ res = WriteConsoleW(self->handle, wbuf, wlen, &n, NULL);
+ if (n < wlen) {
+ /* Wrote fewer characters than expected, which means our
+ * len value may be wrong. So recalculate it from the
+ * characters that were written. As this could potentially
+ * result in a different value, we also validate that value.
+ */
+ len = WideCharToMultiByte(CP_UTF8, 0, wbuf, n,
+ NULL, 0, NULL, NULL);
+ if (len) {
+ wlen = MultiByteToWideChar(CP_UTF8, 0, b->buf, len,
+ NULL, 0);
+ assert(wlen == len);
+ }
+ }
+ } else
+ res = 0;
+ Py_END_ALLOW_THREADS
+
+ if (!res) {
+ DWORD err = GetLastError();
+ PyMem_Free(wbuf);
+ return PyErr_SetFromWindowsErr(err);
+ }
+
+ PyMem_Free(wbuf);
+ return PyLong_FromSsize_t(len);
+}
+
+static PyObject *
+winconsoleio_repr(winconsoleio *self)
+{
+ if (self->handle == INVALID_HANDLE_VALUE)
+ return PyUnicode_FromFormat("<_io._WindowsConsoleIO [closed]>");
+
+ if (self->readable)
+ return PyUnicode_FromFormat("<_io._WindowsConsoleIO mode='rb' closefd=%s>",
+ self->closehandle ? "True" : "False");
+ if (self->writable)
+ return PyUnicode_FromFormat("<_io._WindowsConsoleIO mode='wb' closefd=%s>",
+ self->closehandle ? "True" : "False");
+
+ PyErr_SetString(PyExc_SystemError, "_WindowsConsoleIO has invalid mode");
+ return NULL;
+}
+
+/*[clinic input]
+_io._WindowsConsoleIO.isatty
+
+Always True.
+[clinic start generated code]*/
+
+static PyObject *
+_io__WindowsConsoleIO_isatty_impl(winconsoleio *self)
+/*[clinic end generated code: output=9eac09d287c11bd7 input=9b91591dbe356f86]*/
+{
+ if (self->handle == INVALID_HANDLE_VALUE)
+ return err_closed();
+
+ Py_RETURN_TRUE;
+}
+
+static PyObject *
+winconsoleio_getstate(winconsoleio *self)
+{
+ PyErr_Format(PyExc_TypeError,
+ "cannot serialize '%s' object", Py_TYPE(self)->tp_name);
+ return NULL;
+}
+
+#include "clinic/winconsoleio.c.h"
+
+static PyMethodDef winconsoleio_methods[] = {
+ _IO__WINDOWSCONSOLEIO_READ_METHODDEF
+ _IO__WINDOWSCONSOLEIO_READALL_METHODDEF
+ _IO__WINDOWSCONSOLEIO_READINTO_METHODDEF
+ _IO__WINDOWSCONSOLEIO_WRITE_METHODDEF
+ _IO__WINDOWSCONSOLEIO_CLOSE_METHODDEF
+ _IO__WINDOWSCONSOLEIO_READABLE_METHODDEF
+ _IO__WINDOWSCONSOLEIO_WRITABLE_METHODDEF
+ _IO__WINDOWSCONSOLEIO_FILENO_METHODDEF
+ _IO__WINDOWSCONSOLEIO_ISATTY_METHODDEF
+ {"__getstate__", (PyCFunction)winconsoleio_getstate, METH_NOARGS, NULL},
+ {NULL, NULL} /* sentinel */
+};
+
+/* 'closed' and 'mode' are attributes for compatibility with FileIO. */
+
+static PyObject *
+get_closed(winconsoleio *self, void *closure)
+{
+ return PyBool_FromLong((long)(self->handle == INVALID_HANDLE_VALUE));
+}
+
+static PyObject *
+get_closefd(winconsoleio *self, void *closure)
+{
+ return PyBool_FromLong((long)(self->closehandle));
+}
+
+static PyObject *
+get_mode(winconsoleio *self, void *closure)
+{
+ return PyUnicode_FromString(self->readable ? "rb" : "wb");
+}
+
+static PyGetSetDef winconsoleio_getsetlist[] = {
+ {"closed", (getter)get_closed, NULL, "True if the file is closed"},
+ {"closefd", (getter)get_closefd, NULL,
+ "True if the file descriptor will be closed by close()."},
+ {"mode", (getter)get_mode, NULL, "String giving the file mode"},
+ {NULL},
+};
+
+static PyMemberDef winconsoleio_members[] = {
+ {"_blksize", T_UINT, offsetof(winconsoleio, blksize), 0},
+ {"_finalizing", T_BOOL, offsetof(winconsoleio, finalizing), 0},
+ {NULL}
+};
+
+PyTypeObject PyWindowsConsoleIO_Type = {
+ PyVarObject_HEAD_INIT(NULL, 0)
+ "_io._WindowsConsoleIO",
+ sizeof(winconsoleio),
+ 0,
+ (destructor)winconsoleio_dealloc, /* tp_dealloc */
+ 0, /* tp_print */
+ 0, /* tp_getattr */
+ 0, /* tp_setattr */
+ 0, /* tp_reserved */
+ (reprfunc)winconsoleio_repr, /* tp_repr */
+ 0, /* tp_as_number */
+ 0, /* tp_as_sequence */
+ 0, /* tp_as_mapping */
+ 0, /* tp_hash */
+ 0, /* tp_call */
+ 0, /* tp_str */
+ PyObject_GenericGetAttr, /* tp_getattro */
+ 0, /* tp_setattro */
+ 0, /* tp_as_buffer */
+ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE
+ | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_HAVE_FINALIZE, /* tp_flags */
+ _io__WindowsConsoleIO___init____doc__, /* tp_doc */
+ (traverseproc)winconsoleio_traverse, /* tp_traverse */
+ (inquiry)winconsoleio_clear, /* tp_clear */
+ 0, /* tp_richcompare */
+ offsetof(winconsoleio, weakreflist), /* tp_weaklistoffset */
+ 0, /* tp_iter */
+ 0, /* tp_iternext */
+ winconsoleio_methods, /* tp_methods */
+ winconsoleio_members, /* tp_members */
+ winconsoleio_getsetlist, /* tp_getset */
+ 0, /* tp_base */
+ 0, /* tp_dict */
+ 0, /* tp_descr_get */
+ 0, /* tp_descr_set */
+ offsetof(winconsoleio, dict), /* tp_dictoffset */
+ _io__WindowsConsoleIO___init__, /* tp_init */
+ PyType_GenericAlloc, /* tp_alloc */
+ winconsoleio_new, /* tp_new */
+ PyObject_GC_Del, /* tp_free */
+ 0, /* tp_is_gc */
+ 0, /* tp_bases */
+ 0, /* tp_mro */
+ 0, /* tp_cache */
+ 0, /* tp_subclasses */
+ 0, /* tp_weaklist */
+ 0, /* tp_del */
+ 0, /* tp_version_tag */
+ 0, /* tp_finalize */
+};
+
+#endif /* MS_WINDOWS */
diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj
--- a/PCbuild/pythoncore.vcxproj
+++ b/PCbuild/pythoncore.vcxproj
@@ -270,6 +270,7 @@
<ClCompile Include="..\Modules\_io\bufferedio.c" />
<ClCompile Include="..\Modules\_io\iobase.c" />
<ClCompile Include="..\Modules\_io\textio.c" />
+ <ClCompile Include="..\Modules\_io\winconsoleio.c" />
<ClCompile Include="..\Modules\_io\_iomodule.c" />
<ClCompile Include="..\Modules\zlib\adler32.c" />
<ClCompile Include="..\Modules\zlib\compress.c" />
diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters
--- a/PCbuild/pythoncore.vcxproj.filters
+++ b/PCbuild/pythoncore.vcxproj.filters
@@ -605,6 +605,9 @@
<ClCompile Include="..\Modules\_io\textio.c">
<Filter>Modules\_io</Filter>
</ClCompile>
+ <ClCompile Include="..\Modules\_io\winconsoleio.c">
+ <Filter>Modules\_io</Filter>
+ </ClCompile>
<ClCompile Include="..\Modules\_io\_iomodule.c">
<Filter>Modules\_io</Filter>
</ClCompile>
diff --git a/Parser/myreadline.c b/Parser/myreadline.c
--- a/Parser/myreadline.c
+++ b/Parser/myreadline.c
@@ -98,6 +98,100 @@
/* NOTREACHED */
}
+#ifdef MS_WINDOWS
+/* Readline implementation using ReadConsoleW */
+
+extern char _get_console_type(HANDLE handle);
+
+char *
+_PyOS_WindowsConsoleReadline(HANDLE hStdIn)
+{
+ static wchar_t wbuf_local[1024 * 16];
+ const DWORD chunk_size = 1024;
+
+ DWORD n_read, total_read, wbuflen, u8len;
+ wchar_t *wbuf;
+ char *buf = NULL;
+ int err = 0;
+
+ n_read = 0;
+ total_read = 0;
+ wbuf = wbuf_local;
+ wbuflen = sizeof(wbuf_local) / sizeof(wbuf_local[0]) - 1;
+ while (1) {
+ if (!ReadConsoleW(hStdIn, &wbuf[total_read], wbuflen - total_read, &n_read, NULL)) {
+ err = GetLastError();
+ goto exit;
+ }
+ if (n_read == 0) {
+ int s;
+ err = GetLastError();
+ if (err != ERROR_OPERATION_ABORTED)
+ goto exit;
+ err = 0;
+ HANDLE hInterruptEvent = _PyOS_SigintEvent();
+ if (WaitForSingleObjectEx(hInterruptEvent, 100, FALSE)
+ == WAIT_OBJECT_0) {
+ ResetEvent(hInterruptEvent);
+#ifdef WITH_THREAD
+ PyEval_RestoreThread(_PyOS_ReadlineTState);
+#endif
+ s = PyErr_CheckSignals();
+#ifdef WITH_THREAD
+ PyEval_SaveThread();
+#endif
+ if (s < 0)
+ goto exit;
+ }
+ break;
+ }
+
+ total_read += n_read;
+ if (total_read == 0 || wbuf[total_read - 1] == L'\n') {
+ break;
+ }
+ wbuflen += chunk_size;
+ if (wbuf == wbuf_local) {
+ wbuf[total_read] = '\0';
+ wbuf = (wchar_t*)PyMem_RawMalloc(wbuflen * sizeof(wchar_t));
+ if (wbuf)
+ wcscpy_s(wbuf, wbuflen, wbuf_local);
+ }
+ else
+ wbuf = (wchar_t*)PyMem_RawRealloc(wbuf, wbuflen * sizeof(wchar_t));
+ }
+
+ if (wbuf[0] == '\x1a') {
+ buf = PyMem_RawMalloc(1);
+ if (buf)
+ buf[0] = '\0';
+ goto exit;
+ }
+
+ u8len = WideCharToMultiByte(CP_UTF8, 0, wbuf, total_read, NULL, 0, NULL, NULL);
+ buf = PyMem_RawMalloc(u8len + 1);
+ u8len = WideCharToMultiByte(CP_UTF8, 0, wbuf, total_read, buf, u8len, NULL, NULL);
+ buf[u8len] = '\0';
+
+exit:
+ if (wbuf != wbuf_local)
+ PyMem_RawFree(wbuf);
+
+ if (err) {
+#ifdef WITH_THREAD
+ PyEval_RestoreThread(_PyOS_ReadlineTState);
+#endif
+ PyErr_SetFromWindowsErr(err);
+#ifdef WITH_THREAD
+ PyEval_SaveThread();
+#endif
+ }
+
+ return buf;
+}
+
+#endif
+
/* Readline implementation using fgets() */
@@ -107,6 +201,25 @@
size_t n;
char *p, *pr;
+#ifdef MS_WINDOWS
+ if (!Py_LegacyWindowsStdioFlag && sys_stdin == stdin) {
+ HANDLE hStdIn;
+
+ _Py_BEGIN_SUPPRESS_IPH
+ hStdIn = (HANDLE)_get_osfhandle(fileno(sys_stdin));
+ _Py_END_SUPPRESS_IPH
+
+ if (_get_console_type(hStdIn) == 'r') {
+ fflush(sys_stdout);
+ if (prompt)
+ fprintf(stderr, "%s", prompt);
+ fflush(stderr);
+ clearerr(sys_stdin);
+ return _PyOS_WindowsConsoleReadline(hStdIn);
+ }
+ }
+#endif
+
n = 100;
p = (char *)PyMem_RawMalloc(n);
if (p == NULL)
diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c
--- a/Python/pylifecycle.c
+++ b/Python/pylifecycle.c
@@ -31,6 +31,9 @@
#ifdef MS_WINDOWS
#undef BYTE
#include "windows.h"
+
+extern PyTypeObject PyWindowsConsoleIO_Type;
+#define PyWindowsConsoleIO_Check(op) (PyObject_TypeCheck((op), &PyWindowsConsoleIO_Type))
#endif
_Py_IDENTIFIER(flush);
@@ -92,6 +95,7 @@
int Py_IsolatedFlag = 0; /* for -I, isolate from user's env */
#ifdef MS_WINDOWS
int Py_LegacyWindowsFSEncodingFlag = 0; /* Uses mbcs instead of utf-8 */
+int Py_LegacyWindowsStdioFlag = 0; /* Uses FileIO instead of WindowsConsoleIO */
#endif
PyThreadState *_Py_Finalizing = NULL;
@@ -154,6 +158,12 @@
return -3;
}
}
+#ifdef MS_WINDOWS
+ if (_Py_StandardStreamEncoding) {
+ /* Overriding the stream encoding implies legacy streams */
+ Py_LegacyWindowsStdioFlag = 1;
+ }
+#endif
return 0;
}
@@ -327,6 +337,8 @@
#ifdef MS_WINDOWS
if ((p = Py_GETENV("PYTHONLEGACYWINDOWSFSENCODING")) && *p != '\0')
Py_LegacyWindowsFSEncodingFlag = add_flag(Py_LegacyWindowsFSEncodingFlag, p);
+ if ((p = Py_GETENV("PYTHONLEGACYWINDOWSSTDIO")) && *p != '\0')
+ Py_LegacyWindowsStdioFlag = add_flag(Py_LegacyWindowsStdioFlag, p);
#endif
_PyRandom_Init();
@@ -1089,6 +1101,12 @@
Py_INCREF(raw);
}
+#ifdef MS_WINDOWS
+ /* Windows console IO is always UTF-8 encoded */
+ if (PyWindowsConsoleIO_Check(raw))
+ encoding = "utf-8";
+#endif
+
text = PyUnicode_FromString(name);
if (text == NULL || _PyObject_SetAttrId(raw, &PyId_name, text) < 0)
goto error;
--
Repository URL: https://hg.python.org/cpython
More information about the Python-checkins
mailing list