Issue #1602: Windows console doesn't input or print Unicode (PEP 528)

Closes #17602: Adds a readline implementation for the Windows console

Doc/library/functions.rst

@@ -1055,30 +1055,38 @@ are always available.  They are listed here in alphabetical order.
    (where :func:`open` is declared), :mod:`os`, :mod:`os.path`, :mod:`tempfile`,
    and :mod:`shutil`.
 
-   .. versionchanged:: 3.3
-      The *opener* parameter was added.
-      The ``'x'`` mode was added.
-      :exc:`IOError` used to be raised, it is now an alias of :exc:`OSError`.
-      :exc:`FileExistsError` is now raised if the file opened in exclusive
-      creation mode (``'x'``) already exists.
+   .. versionchanged::
+      3.3
 
-   .. versionchanged:: 3.4
-      The file is now non-inheritable.
+         * The *opener* parameter was added.
+         * The ``'x'`` mode was added.
+         * :exc:`IOError` used to be raised, it is now an alias of :exc:`OSError`.
+         * :exc:`FileExistsError` is now raised if the file opened in exclusive
+         * creation mode (``'x'``) already exists.
+
+   .. versionchanged::
+      3.4
+
+         * The file is now non-inheritable.
 
    .. deprecated-removed:: 3.4 4.0
 
       The ``'U'`` mode.
 
-   .. versionchanged:: 3.5
-      If the system call is interrupted and the signal handler does not raise an
+   .. versionchanged::
+      3.5
+
+         * If the system call is interrupted and the signal handler does not raise an
            exception, the function now retries the system call instead of raising an
            :exc:`InterruptedError` exception (see :pep:`475` for the rationale).
+         * The ``'namereplace'`` error handler was added.
 
-   .. versionchanged:: 3.5
-      The ``'namereplace'`` error handler was added.
+   .. versionchanged::
+      3.6
 
-   .. versionchanged:: 3.6
-      Support added to accept objects implementing :class:`os.PathLike`.
+         * Support added to accept objects implementing :class:`os.PathLike`.
+         * On Windows, opening a console buffer may return a subclass of
+           :class:`io.RawIOBase` other than :class:`io.FileIO`.
 
 .. function:: ord(c)
 

Doc/using/cmdline.rst

@@ -559,6 +559,10 @@ conflict.
    .. versionchanged:: 3.4
       The ``encodingname`` part is now optional.
 
+   .. versionchanged:: 3.6
+      On Windows, the encoding specified by this variable is ignored for interactive
+      console buffers unless :envvar:`PYTHONLEGACYWINDOWSIOENCODING` is also specified.
+      Files and pipes redirected through the standard streams are not affected.
 
 .. envvar:: PYTHONNOUSERSITE
 
@@ -686,6 +690,19 @@ conflict.
    .. versionadded:: 3.6
       See :pep:`529` for more details.
 
+.. envvar:: PYTHONLEGACYWINDOWSIOENCODING
+
+   If set to a non-empty string, does not use the new console reader and
+   writer. This means that Unicode characters will be encoded according to
+   the active console code page, rather than using utf-8.
+
+   This variable is ignored if the standard streams are redirected (to files
+   or pipes) rather than referring to console buffers.
+
+   Availability: Windows
+
+   .. versionadded:: 3.6
+
 Debug-mode variables
 ~~~~~~~~~~~~~~~~~~~~
 

Doc/whatsnew/3.6.rst

@@ -78,6 +78,8 @@ Windows improvements:
 
 * PEP 529: :ref:`Change Windows filesystem encoding to UTF-8 <pep-529>`
 
+* PEP 528: :ref:`Change Windows console encoding to UTF-8 <pep-528>`
+
 * The ``py.exe`` launcher, when used interactively, no longer prefers
   Python 2 over Python 3 when the user doesn't specify a version (via
   command line arguments or a config file).  Handling of shebang lines
@@ -267,6 +269,23 @@ Also see :pep:`487` and the updated class customization documentation at
 
 (Contributed by Martin Teichmann in :issue:`27366`)
 
+.. _pep-528:
+
+PEP 528: Change Windows console encoding to UTF-8
+-------------------------------------------------
+
+The default console on Windows will now accept all Unicode characters and
+provide correctly read str objects to Python code. ``sys.stdin``,
+``sys.stdout`` and ``sys.stderr`` now default to utf-8 encoding.
+
+This change only applies when using an interactive console, and not when
+redirecting files or pipes. To revert to the previous behaviour for interactive
+console use, set :envvar:`PYTHONLEGACYWINDOWSIOENCODING`.
+
+.. seealso::
+
+   :pep:`528` -- Change Windows console encoding to UTF-8
+      PEP written and implemented by Steve Dower.
 
 PYTHONMALLOC environment variable
 =================================

Include/pydebug.h

@@ -24,6 +24,10 @@ PyAPI_DATA(int) Py_UnbufferedStdioFlag;
 PyAPI_DATA(int) Py_HashRandomizationFlag;
 PyAPI_DATA(int) Py_IsolatedFlag;
 
+#ifdef MS_WINDOWS
+PyAPI_DATA(int) Py_LegacyWindowsStdioFlag;
+#endif
+
 /* this is a wrapper around getenv() that pays attention to
    Py_IgnoreEnvironmentFlag.  It should be used for getting variables like
    PYTHONPATH and PYTHONHOME from the environment */

Lib/io.py

@@ -90,3 +90,10 @@ for klass in (BytesIO, BufferedReader, BufferedWriter, BufferedRandom,
 for klass in (StringIO, TextIOWrapper):
     TextIOBase.register(klass)
 del klass
+
+try:
+    from _io import _WindowsConsoleIO
+except ImportError:
+    pass
+else:
+    RawIOBase.register(_WindowsConsoleIO)

Lib/test/test_os.py

@@ -1518,7 +1518,7 @@ class TestInvalidFD(unittest.TestCase):
     singles = ["fchdir", "dup", "fdopen", "fdatasync", "fstat",
                "fstatvfs", "fsync", "tcgetpgrp", "ttyname"]
     #singles.append("close")
-    #We omit close because it doesn'r raise an exception on some platforms
+    #We omit close because it doesn't raise an exception on some platforms
     def get_single(f):
         def helper(self):
             if  hasattr(os, f):

Lib/test/test_winconsoleio.py

@@ -0,0 +1,72 @@
+'''Tests for WindowsConsoleIO
+
+Unfortunately, most testing requires interactive use, since we have no
+API to read back from a real console, and this class is only for use
+with real consoles.
+
+Instead, we validate that basic functionality such as opening, closing
+and in particular fileno() work, but are forced to leave real testing
+to real people with real keyborads.
+'''
+
+import io
+import unittest
+import sys
+
+if sys.platform != 'win32':
+    raise unittest.SkipTest("test only relevant on win32")
+
+ConIO = io._WindowsConsoleIO
+
+class WindowsConsoleIOTests(unittest.TestCase):
+    def test_abc(self):
+        self.assertTrue(issubclass(ConIO, io.RawIOBase))
+        self.assertFalse(issubclass(ConIO, io.BufferedIOBase))
+        self.assertFalse(issubclass(ConIO, io.TextIOBase))
+
+    def test_open_fd(self):
+        f = ConIO(0)
+        self.assertTrue(f.readable())
+        self.assertFalse(f.writable())
+        self.assertEqual(0, f.fileno())
+        f.close()   # multiple close should not crash
+        f.close()
+
+        f = ConIO(1, 'w')
+        self.assertFalse(f.readable())
+        self.assertTrue(f.writable())
+        self.assertEqual(1, f.fileno())
+        f.close()
+        f.close()
+
+        f = ConIO(2, 'w')
+        self.assertFalse(f.readable())
+        self.assertTrue(f.writable())
+        self.assertEqual(2, f.fileno())
+        f.close()
+        f.close()
+
+    def test_open_name(self):
+        f = ConIO("CON")
+        self.assertTrue(f.readable())
+        self.assertFalse(f.writable())
+        self.assertIsNotNone(f.fileno())
+        f.close()   # multiple close should not crash
+        f.close()
+
+        f = ConIO('CONIN$')
+        self.assertTrue(f.readable())
+        self.assertFalse(f.writable())
+        self.assertIsNotNone(f.fileno())
+        f.close()
+        f.close()
+
+        f = ConIO('CONOUT$', 'w')
+        self.assertFalse(f.readable())
+        self.assertTrue(f.writable())
+        self.assertIsNotNone(f.fileno())
+        f.close()
+        f.close()
+
+if __name__ == "__main__":
+    unittest.main()

Misc/NEWS

@@ -300,6 +300,8 @@ Build
 Windows
 -------
 
+- Issue #1602: Windows console doesn't input or print Unicode (PEP 528)
+
 - Issue #27781: Change file system encoding on Windows to UTF-8 (PEP 529)
 
 - Issue #27731: Opt-out of MAX_PATH on Windows 10
@@ -556,7 +558,6 @@ Build
 - Issue #10910: Avoid C++ compilation errors on FreeBSD and OS X.
   Also update FreedBSD version checks for the original ctype UTF-8 workaround.
 
-
 What's New in Python 3.6.0 alpha 3
 ==================================
 

Modules/_io/_iomodule.c

@@ -20,6 +20,9 @@
 #include <sys/stat.h>
 #endif /* HAVE_SYS_STAT_H */
 
+#ifdef MS_WINDOWS
+#include <consoleapi.h>
+#endif
 
 /* Various interned strings */
 
@@ -52,7 +55,6 @@ PyObject *_PyIO_empty_str;
 PyObject *_PyIO_empty_bytes;
 PyObject *_PyIO_zero;
 
-
 PyDoc_STRVAR(module_doc,
 "The io module provides the Python interfaces to stream handling. The\n"
 "builtin open function is defined in this module.\n"
@@ -362,8 +364,18 @@ _io_open_impl(PyObject *module, PyObject *file, const char *mode,
     }
 
     /* Create the Raw file stream */
-    raw = PyObject_CallFunction((PyObject *)&PyFileIO_Type,
+    {
+        PyObject *RawIO_class = (PyObject *)&PyFileIO_Type;
+#ifdef MS_WINDOWS
+        if (!Py_LegacyWindowsStdioFlag && _PyIO_get_console_type(path_or_fd) != '\0') {
+            RawIO_class = (PyObject *)&PyWindowsConsoleIO_Type;
+            encoding = "utf-8";
+        }
+#endif
+        raw = PyObject_CallFunction(RawIO_class,
                                     "OsiO", path_or_fd, rawmode, closefd, opener);
+    }
+
     if (raw == NULL)
         goto error;
     result = raw;
@@ -708,6 +720,12 @@ PyInit__io(void)
     PyStringIO_Type.tp_base = &PyTextIOBase_Type;
     ADD_TYPE(&PyStringIO_Type, "StringIO");
 
+#ifdef MS_WINDOWS
+    /* WindowsConsoleIO */
+    PyWindowsConsoleIO_Type.tp_base = &PyRawIOBase_Type;
+    ADD_TYPE(&PyWindowsConsoleIO_Type, "_WindowsConsoleIO");
+#endif
+
     /* BufferedReader */
     PyBufferedReader_Type.tp_base = &PyBufferedIOBase_Type;
     ADD_TYPE(&PyBufferedReader_Type, "BufferedReader");

Modules/_io/_iomodule.h

@@ -19,6 +19,12 @@ extern PyTypeObject PyBufferedRandom_Type;
 extern PyTypeObject PyTextIOWrapper_Type;
 extern PyTypeObject PyIncrementalNewlineDecoder_Type;
 
+#ifndef Py_LIMITED_API
+#ifdef MS_WINDOWS
+extern PyTypeObject PyWindowsConsoleIO_Type;
+#define PyWindowsConsoleIO_Check(op) (PyObject_TypeCheck((op), &PyWindowsConsoleIO_Type))
+#endif /* MS_WINDOWS */
+#endif /* Py_LIMITED_API */
 
 extern int _PyIO_ConvertSsize_t(PyObject *, void *);
 
@@ -145,6 +151,10 @@ typedef struct {
 extern _PyIO_State *_PyIO_get_module_state(void);
 extern PyObject *_PyIO_get_locale_module(_PyIO_State *);
 
+#ifdef MS_WINDOWS
+extern char _PyIO_get_console_type(PyObject *);
+#endif
+
 extern PyObject *_PyIO_str_close;
 extern PyObject *_PyIO_str_closed;
 extern PyObject *_PyIO_str_decode;

Modules/_io/clinic/winconsoleio.c.h

@@ -0,0 +1,331 @@
+/*[clinic input]
+preserve
+[clinic start generated code]*/
+
+#if defined(MS_WINDOWS)
+
+PyDoc_STRVAR(_io__WindowsConsoleIO_close__doc__,
+"close($self, /)\n"
+"--\n"
+"\n"
+"Close the handle.\n"
+"\n"
+"A closed handle cannot be used for further I/O operations.  close() may be\n"
+"called more than once without error.");
+
+#define _IO__WINDOWSCONSOLEIO_CLOSE_METHODDEF    \
+    {"close", (PyCFunction)_io__WindowsConsoleIO_close, METH_NOARGS, _io__WindowsConsoleIO_close__doc__},
+
+static PyObject *
+_io__WindowsConsoleIO_close_impl(winconsoleio *self);
+
+static PyObject *
+_io__WindowsConsoleIO_close(winconsoleio *self, PyObject *Py_UNUSED(ignored))
+{
+    return _io__WindowsConsoleIO_close_impl(self);
+}
+
+#endif /* defined(MS_WINDOWS) */
+
+#if defined(MS_WINDOWS)
+
+PyDoc_STRVAR(_io__WindowsConsoleIO___init____doc__,
+"_WindowsConsoleIO(file, mode=\'r\', closefd=True, opener=None)\n"
+"--\n"
+"\n"
+"Open a console buffer by file descriptor.\n"
+"\n"
+"The mode can be \'rb\' (default), or \'wb\' for reading or writing bytes. All\n"
+"other mode characters will be ignored. Mode \'b\' will be assumed if it is\n"
+"omitted. The *opener* parameter is always ignored.");
+
+static int
+_io__WindowsConsoleIO___init___impl(winconsoleio *self, PyObject *nameobj,
+                                    const char *mode, int closefd,
+                                    PyObject *opener);
+
+static int
+_io__WindowsConsoleIO___init__(PyObject *self, PyObject *args, PyObject *kwargs)
+{
+    int return_value = -1;
+    static const char * const _keywords[] = {"file", "mode", "closefd", "opener", NULL};
+    static _PyArg_Parser _parser = {"O|siO:_WindowsConsoleIO", _keywords, 0};
+    PyObject *nameobj;
+    const char *mode = "r";
+    int closefd = 1;
+    PyObject *opener = Py_None;
+
+    if (!_PyArg_ParseTupleAndKeywordsFast(args, kwargs, &_parser,
+        &nameobj, &mode, &closefd, &opener)) {
+        goto exit;
+    }
+    return_value = _io__WindowsConsoleIO___init___impl((winconsoleio *)self, nameobj, mode, closefd, opener);
+
+exit:
+    return return_value;
+}
+
+#endif /* defined(MS_WINDOWS) */
+
+#if defined(MS_WINDOWS)
+
+PyDoc_STRVAR(_io__WindowsConsoleIO_fileno__doc__,
+"fileno($self, /)\n"
+"--\n"
+"\n"
+"Return the underlying file descriptor (an integer).\n"
+"\n"
+"fileno is only set when a file descriptor is used to open\n"
+"one of the standard streams.");
+
+#define _IO__WINDOWSCONSOLEIO_FILENO_METHODDEF    \
+    {"fileno", (PyCFunction)_io__WindowsConsoleIO_fileno, METH_NOARGS, _io__WindowsConsoleIO_fileno__doc__},
+
+static PyObject *
+_io__WindowsConsoleIO_fileno_impl(winconsoleio *self);
+
+static PyObject *
+_io__WindowsConsoleIO_fileno(winconsoleio *self, PyObject *Py_UNUSED(ignored))
+{
+    return _io__WindowsConsoleIO_fileno_impl(self);
+}
+
+#endif /* defined(MS_WINDOWS) */
+
+#if defined(MS_WINDOWS)
+
+PyDoc_STRVAR(_io__WindowsConsoleIO_readable__doc__,
+"readable($self, /)\n"
+"--\n"
+"\n"
+"True if console is an input buffer.");
+
+#define _IO__WINDOWSCONSOLEIO_READABLE_METHODDEF    \
+    {"readable", (PyCFunction)_io__WindowsConsoleIO_readable, METH_NOARGS, _io__WindowsConsoleIO_readable__doc__},
+
+static PyObject *
+_io__WindowsConsoleIO_readable_impl(winconsoleio *self);
+
+static PyObject *
+_io__WindowsConsoleIO_readable(winconsoleio *self, PyObject *Py_UNUSED(ignored))
+{
+    return _io__WindowsConsoleIO_readable_impl(self);
+}
+
+#endif /* defined(MS_WINDOWS) */
+
+#if defined(MS_WINDOWS)
+
+PyDoc_STRVAR(_io__WindowsConsoleIO_writable__doc__,
+"writable($self, /)\n"
+"--\n"
+"\n"
+"True if console is an output buffer.");
+
+#define _IO__WINDOWSCONSOLEIO_WRITABLE_METHODDEF    \
+    {"writable", (PyCFunction)_io__WindowsConsoleIO_writable, METH_NOARGS, _io__WindowsConsoleIO_writable__doc__},
+
+static PyObject *
+_io__WindowsConsoleIO_writable_impl(winconsoleio *self);
+
+static PyObject *
+_io__WindowsConsoleIO_writable(winconsoleio *self, PyObject *Py_UNUSED(ignored))
+{
+    return _io__WindowsConsoleIO_writable_impl(self);
+}
+
+#endif /* defined(MS_WINDOWS) */
+
+#if defined(MS_WINDOWS)
+
+PyDoc_STRVAR(_io__WindowsConsoleIO_readinto__doc__,
+"readinto($self, buffer, /)\n"
+"--\n"
+"\n"
+"Same as RawIOBase.readinto().");
+
+#define _IO__WINDOWSCONSOLEIO_READINTO_METHODDEF    \
+    {"readinto", (PyCFunction)_io__WindowsConsoleIO_readinto, METH_O, _io__WindowsConsoleIO_readinto__doc__},
+
+static PyObject *
+_io__WindowsConsoleIO_readinto_impl(winconsoleio *self, Py_buffer *buffer);
+
+static PyObject *
+_io__WindowsConsoleIO_readinto(winconsoleio *self, PyObject *arg)
+{
+    PyObject *return_value = NULL;
+    Py_buffer buffer = {NULL, NULL};
+
+    if (!PyArg_Parse(arg, "w*:readinto", &buffer)) {
+        goto exit;
+    }
+    return_value = _io__WindowsConsoleIO_readinto_impl(self, &buffer);
+
+exit:
+    /* Cleanup for buffer */
+    if (buffer.obj) {
+       PyBuffer_Release(&buffer);
+    }
+
+    return return_value;
+}
+
+#endif /* defined(MS_WINDOWS) */
+
+#if defined(MS_WINDOWS)
+
+PyDoc_STRVAR(_io__WindowsConsoleIO_readall__doc__,
+"readall($self, /)\n"
+"--\n"
+"\n"
+"Read all data from the console, returned as bytes.\n"
+"\n"
+"Return an empty bytes object at EOF.");
+
+#define _IO__WINDOWSCONSOLEIO_READALL_METHODDEF    \
+    {"readall", (PyCFunction)_io__WindowsConsoleIO_readall, METH_NOARGS, _io__WindowsConsoleIO_readall__doc__},
+
+static PyObject *
+_io__WindowsConsoleIO_readall_impl(winconsoleio *self);
+
+static PyObject *
+_io__WindowsConsoleIO_readall(winconsoleio *self, PyObject *Py_UNUSED(ignored))
+{
+    return _io__WindowsConsoleIO_readall_impl(self);
+}
+
+#endif /* defined(MS_WINDOWS) */
+
+#if defined(MS_WINDOWS)
+
+PyDoc_STRVAR(_io__WindowsConsoleIO_read__doc__,
+"read($self, size=-1, /)\n"
+"--\n"
+"\n"
+"Read at most size bytes, returned as bytes.\n"
+"\n"
+"Only makes one system call when size is a positive integer,\n"
+"so less data may be returned than requested.\n"
+"Return an empty bytes object at EOF.");
+
+#define _IO__WINDOWSCONSOLEIO_READ_METHODDEF    \
+    {"read", (PyCFunction)_io__WindowsConsoleIO_read, METH_VARARGS, _io__WindowsConsoleIO_read__doc__},
+
+static PyObject *
+_io__WindowsConsoleIO_read_impl(winconsoleio *self, Py_ssize_t size);
+
+static PyObject *
+_io__WindowsConsoleIO_read(winconsoleio *self, PyObject *args)
+{
+    PyObject *return_value = NULL;
+    Py_ssize_t size = -1;
+
+    if (!PyArg_ParseTuple(args, "|O&:read",
+        _PyIO_ConvertSsize_t, &size)) {
+        goto exit;
+    }
+    return_value = _io__WindowsConsoleIO_read_impl(self, size);
+
+exit:
+    return return_value;
+}
+
+#endif /* defined(MS_WINDOWS) */
+
+#if defined(MS_WINDOWS)
+
+PyDoc_STRVAR(_io__WindowsConsoleIO_write__doc__,
+"write($self, b, /)\n"
+"--\n"
+"\n"
+"Write buffer b to file, return number of bytes written.\n"
+"\n"
+"Only makes one system call, so not all of the data may be written.\n"
+"The number of bytes actually written is returned.");
+
+#define _IO__WINDOWSCONSOLEIO_WRITE_METHODDEF    \
+    {"write", (PyCFunction)_io__WindowsConsoleIO_write, METH_O, _io__WindowsConsoleIO_write__doc__},
+
+static PyObject *
+_io__WindowsConsoleIO_write_impl(winconsoleio *self, Py_buffer *b);
+
+static PyObject *
+_io__WindowsConsoleIO_write(winconsoleio *self, PyObject *arg)
+{
+    PyObject *return_value = NULL;
+    Py_buffer b = {NULL, NULL};
+
+    if (!PyArg_Parse(arg, "y*:write", &b)) {
+        goto exit;
+    }
+    return_value = _io__WindowsConsoleIO_write_impl(self, &b);
+
+exit:
+    /* Cleanup for b */
+    if (b.obj) {
+       PyBuffer_Release(&b);
+    }
+
+    return return_value;
+}
+
+#endif /* defined(MS_WINDOWS) */
+
+#if defined(MS_WINDOWS)
+
+PyDoc_STRVAR(_io__WindowsConsoleIO_isatty__doc__,
+"isatty($self, /)\n"
+"--\n"
+"\n"
+"Always True.");
+
+#define _IO__WINDOWSCONSOLEIO_ISATTY_METHODDEF    \
+    {"isatty", (PyCFunction)_io__WindowsConsoleIO_isatty, METH_NOARGS, _io__WindowsConsoleIO_isatty__doc__},
+
+static PyObject *
+_io__WindowsConsoleIO_isatty_impl(winconsoleio *self);
+
+static PyObject *
+_io__WindowsConsoleIO_isatty(winconsoleio *self, PyObject *Py_UNUSED(ignored))
+{
+    return _io__WindowsConsoleIO_isatty_impl(self);
+}
+
+#endif /* defined(MS_WINDOWS) */
+
+#ifndef _IO__WINDOWSCONSOLEIO_CLOSE_METHODDEF
+    #define _IO__WINDOWSCONSOLEIO_CLOSE_METHODDEF
+#endif /* !defined(_IO__WINDOWSCONSOLEIO_CLOSE_METHODDEF) */
+
+#ifndef _IO__WINDOWSCONSOLEIO_FILENO_METHODDEF
+    #define _IO__WINDOWSCONSOLEIO_FILENO_METHODDEF
+#endif /* !defined(_IO__WINDOWSCONSOLEIO_FILENO_METHODDEF) */
+
+#ifndef _IO__WINDOWSCONSOLEIO_READABLE_METHODDEF
+    #define _IO__WINDOWSCONSOLEIO_READABLE_METHODDEF
+#endif /* !defined(_IO__WINDOWSCONSOLEIO_READABLE_METHODDEF) */
+
+#ifndef _IO__WINDOWSCONSOLEIO_WRITABLE_METHODDEF
+    #define _IO__WINDOWSCONSOLEIO_WRITABLE_METHODDEF
+#endif /* !defined(_IO__WINDOWSCONSOLEIO_WRITABLE_METHODDEF) */
+
+#ifndef _IO__WINDOWSCONSOLEIO_READINTO_METHODDEF
+    #define _IO__WINDOWSCONSOLEIO_READINTO_METHODDEF
+#endif /* !defined(_IO__WINDOWSCONSOLEIO_READINTO_METHODDEF) */
+
+#ifndef _IO__WINDOWSCONSOLEIO_READALL_METHODDEF
+    #define _IO__WINDOWSCONSOLEIO_READALL_METHODDEF
+#endif /* !defined(_IO__WINDOWSCONSOLEIO_READALL_METHODDEF) */
+
+#ifndef _IO__WINDOWSCONSOLEIO_READ_METHODDEF
+    #define _IO__WINDOWSCONSOLEIO_READ_METHODDEF
+#endif /* !defined(_IO__WINDOWSCONSOLEIO_READ_METHODDEF) */
+
+#ifndef _IO__WINDOWSCONSOLEIO_WRITE_METHODDEF
+    #define _IO__WINDOWSCONSOLEIO_WRITE_METHODDEF
+#endif /* !defined(_IO__WINDOWSCONSOLEIO_WRITE_METHODDEF) */
+
+#ifndef _IO__WINDOWSCONSOLEIO_ISATTY_METHODDEF
+    #define _IO__WINDOWSCONSOLEIO_ISATTY_METHODDEF
+#endif /* !defined(_IO__WINDOWSCONSOLEIO_ISATTY_METHODDEF) */
+/*[clinic end generated code: output=9eba916f8537fff7 input=a9049054013a1b77]*/

PCbuild/pythoncore.vcxproj

@@ -270,6 +270,7 @@
     <ClCompile Include="..\Modules\_io\bufferedio.c" />
     <ClCompile Include="..\Modules\_io\iobase.c" />
     <ClCompile Include="..\Modules\_io\textio.c" />
+    <ClCompile Include="..\Modules\_io\winconsoleio.c" />
     <ClCompile Include="..\Modules\_io\_iomodule.c" />
     <ClCompile Include="..\Modules\zlib\adler32.c" />
     <ClCompile Include="..\Modules\zlib\compress.c" />

PCbuild/pythoncore.vcxproj.filters

@@ -605,6 +605,9 @@
     <ClCompile Include="..\Modules\_io\textio.c">
       <Filter>Modules\_io</Filter>
     </ClCompile>
+    <ClCompile Include="..\Modules\_io\winconsoleio.c">
+      <Filter>Modules\_io</Filter>
+    </ClCompile>
     <ClCompile Include="..\Modules\_io\_iomodule.c">
       <Filter>Modules\_io</Filter>
     </ClCompile>

Parser/myreadline.c

@@ -98,6 +98,100 @@ my_fgets(char *buf, int len, FILE *fp)
     /* NOTREACHED */
 }
 
+#ifdef MS_WINDOWS
+/* Readline implementation using ReadConsoleW */
+
+extern char _get_console_type(HANDLE handle);
+
+char *
+_PyOS_WindowsConsoleReadline(HANDLE hStdIn)
+{
+    static wchar_t wbuf_local[1024 * 16];
+    const DWORD chunk_size = 1024;
+
+    DWORD n_read, total_read, wbuflen, u8len;
+    wchar_t *wbuf;
+    char *buf = NULL;
+    int err = 0;
+
+    n_read = 0;
+    total_read = 0;
+    wbuf = wbuf_local;
+    wbuflen = sizeof(wbuf_local) / sizeof(wbuf_local[0]) - 1;
+    while (1) {
+        if (!ReadConsoleW(hStdIn, &wbuf[total_read], wbuflen - total_read, &n_read, NULL)) {
+            err = GetLastError();
+            goto exit;
+        }
+        if (n_read == 0) {
+            int s;
+            err = GetLastError();
+            if (err != ERROR_OPERATION_ABORTED)
+                goto exit;
+            err = 0;
+            HANDLE hInterruptEvent = _PyOS_SigintEvent();
+            if (WaitForSingleObjectEx(hInterruptEvent, 100, FALSE)
+                    == WAIT_OBJECT_0) {
+                ResetEvent(hInterruptEvent);
+#ifdef WITH_THREAD
+                PyEval_RestoreThread(_PyOS_ReadlineTState);
+#endif
+                s = PyErr_CheckSignals();
+#ifdef WITH_THREAD
+                PyEval_SaveThread();
+#endif
+                if (s < 0)
+                    goto exit;
+            }
+            break;
+        }
+
+        total_read += n_read;
+        if (total_read == 0 || wbuf[total_read - 1] == L'\n') {
+            break;
+        }
+        wbuflen += chunk_size;
+        if (wbuf == wbuf_local) {
+            wbuf[total_read] = '\0';
+            wbuf = (wchar_t*)PyMem_RawMalloc(wbuflen * sizeof(wchar_t));
+            if (wbuf)
+                wcscpy_s(wbuf, wbuflen, wbuf_local);
+        }
+        else
+            wbuf = (wchar_t*)PyMem_RawRealloc(wbuf, wbuflen * sizeof(wchar_t));
+    }
+
+    if (wbuf[0] == '\x1a') {
+        buf = PyMem_RawMalloc(1);
+        if (buf)
+            buf[0] = '\0';
+        goto exit;
+    }
+
+    u8len = WideCharToMultiByte(CP_UTF8, 0, wbuf, total_read, NULL, 0, NULL, NULL);
+    buf = PyMem_RawMalloc(u8len + 1);
+    u8len = WideCharToMultiByte(CP_UTF8, 0, wbuf, total_read, buf, u8len, NULL, NULL);
+    buf[u8len] = '\0';
+    
+exit:
+    if (wbuf != wbuf_local)
+        PyMem_RawFree(wbuf);
+
+    if (err) {
+#ifdef WITH_THREAD
+        PyEval_RestoreThread(_PyOS_ReadlineTState);
+#endif
+        PyErr_SetFromWindowsErr(err);
+#ifdef WITH_THREAD
+        PyEval_SaveThread();
+#endif
+    }
+
+    return buf;
+}
+
+#endif
+
 
 /* Readline implementation using fgets() */
 
@@ -107,6 +201,25 @@ PyOS_StdioReadline(FILE *sys_stdin, FILE *sys_stdout, const char *prompt)
     size_t n;
     char *p, *pr;
 
+#ifdef MS_WINDOWS
+    if (!Py_LegacyWindowsStdioFlag && sys_stdin == stdin) {
+        HANDLE hStdIn;
+        
+        _Py_BEGIN_SUPPRESS_IPH
+        hStdIn = (HANDLE)_get_osfhandle(fileno(sys_stdin));
+        _Py_END_SUPPRESS_IPH
+        
+        if (_get_console_type(hStdIn) == 'r') {
+            fflush(sys_stdout);
+            if (prompt)
+                fprintf(stderr, "%s", prompt);
+            fflush(stderr);
+            clearerr(sys_stdin);
+            return _PyOS_WindowsConsoleReadline(hStdIn);
+        }
+    }
+#endif
+
     n = 100;
     p = (char *)PyMem_RawMalloc(n);
     if (p == NULL)

Python/pylifecycle.c

@@ -31,6 +31,9 @@
 #ifdef MS_WINDOWS
 #undef BYTE
 #include "windows.h"
+
+extern PyTypeObject PyWindowsConsoleIO_Type;
+#define PyWindowsConsoleIO_Check(op) (PyObject_TypeCheck((op), &PyWindowsConsoleIO_Type))
 #endif
 
 _Py_IDENTIFIER(flush);
@@ -92,6 +95,7 @@ int Py_HashRandomizationFlag = 0; /* for -R and PYTHONHASHSEED */
 int Py_IsolatedFlag = 0; /* for -I, isolate from user's env */
 #ifdef MS_WINDOWS
 int Py_LegacyWindowsFSEncodingFlag = 0; /* Uses mbcs instead of utf-8 */
+int Py_LegacyWindowsStdioFlag = 0; /* Uses FileIO instead of WindowsConsoleIO */
 #endif
 
 PyThreadState *_Py_Finalizing = NULL;
@@ -154,6 +158,12 @@ Py_SetStandardStreamEncoding(const char *encoding, const char *errors)
             return -3;
         }
     }
+#ifdef MS_WINDOWS
+    if (_Py_StandardStreamEncoding) {
+        /* Overriding the stream encoding implies legacy streams */
+        Py_LegacyWindowsStdioFlag = 1;
+    }
+#endif
     return 0;
 }
 
@@ -327,6 +337,8 @@ _Py_InitializeEx_Private(int install_sigs, int install_importlib)
 #ifdef MS_WINDOWS
     if ((p = Py_GETENV("PYTHONLEGACYWINDOWSFSENCODING")) && *p != '\0')
         Py_LegacyWindowsFSEncodingFlag = add_flag(Py_LegacyWindowsFSEncodingFlag, p);
+    if ((p = Py_GETENV("PYTHONLEGACYWINDOWSSTDIO")) && *p != '\0')
+        Py_LegacyWindowsStdioFlag = add_flag(Py_LegacyWindowsStdioFlag, p);
 #endif
 
     _PyRandom_Init();
@@ -1089,6 +1101,12 @@ create_stdio(PyObject* io,
         Py_INCREF(raw);
     }
 
+#ifdef MS_WINDOWS
+    /* Windows console IO is always UTF-8 encoded */
+    if (PyWindowsConsoleIO_Check(raw))
+        encoding = "utf-8";
+#endif
+
     text = PyUnicode_FromString(name);
     if (text == NULL || _PyObject_SetAttrId(raw, &PyId_name, text) < 0)
         goto error;