From 3929499914d47365ae744df312e16da8955c90ac Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Tue, 30 Aug 2016 21:22:36 -0700 Subject: [PATCH] Issue #1602: Windows console doesn't input or print Unicode (PEP 528) Closes #17602: Adds a readline implementation for the Windows console --- Doc/library/functions.rst | 40 +- Doc/using/cmdline.rst | 17 + Doc/whatsnew/3.6.rst | 19 + Include/pydebug.h | 4 + Lib/io.py | 7 + Lib/test/test_os.py | 2 +- Lib/test/test_winconsoleio.py | 72 ++ Misc/NEWS | 3 +- Modules/_io/_iomodule.c | 24 +- Modules/_io/_iomodule.h | 10 + Modules/_io/clinic/winconsoleio.c.h | 331 ++++++++ Modules/_io/winconsoleio.c | 1096 +++++++++++++++++++++++++++ PCbuild/pythoncore.vcxproj | 1 + PCbuild/pythoncore.vcxproj.filters | 3 + Parser/myreadline.c | 113 +++ Python/pylifecycle.c | 18 + 16 files changed, 1739 insertions(+), 21 deletions(-) create mode 100644 Lib/test/test_winconsoleio.py create mode 100644 Modules/_io/clinic/winconsoleio.c.h create mode 100644 Modules/_io/winconsoleio.c diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index 3e2fb72841a..db04b105c86 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -1055,30 +1055,38 @@ are always available. They are listed here in alphabetical order. (where :func:`open` is declared), :mod:`os`, :mod:`os.path`, :mod:`tempfile`, and :mod:`shutil`. - .. versionchanged:: 3.3 - The *opener* parameter was added. - The ``'x'`` mode was added. - :exc:`IOError` used to be raised, it is now an alias of :exc:`OSError`. - :exc:`FileExistsError` is now raised if the file opened in exclusive - creation mode (``'x'``) already exists. + .. versionchanged:: + 3.3 - .. versionchanged:: 3.4 - The file is now non-inheritable. + * The *opener* parameter was added. + * The ``'x'`` mode was added. + * :exc:`IOError` used to be raised, it is now an alias of :exc:`OSError`. + * :exc:`FileExistsError` is now raised if the file opened in exclusive + * creation mode (``'x'``) already exists. + + .. versionchanged:: + 3.4 + + * The file is now non-inheritable. .. deprecated-removed:: 3.4 4.0 The ``'U'`` mode. - .. versionchanged:: 3.5 - If the system call is interrupted and the signal handler does not raise an - exception, the function now retries the system call instead of raising an - :exc:`InterruptedError` exception (see :pep:`475` for the rationale). + .. versionchanged:: + 3.5 - .. versionchanged:: 3.5 - The ``'namereplace'`` error handler was added. + * If the system call is interrupted and the signal handler does not raise an + exception, the function now retries the system call instead of raising an + :exc:`InterruptedError` exception (see :pep:`475` for the rationale). + * The ``'namereplace'`` error handler was added. - .. versionchanged:: 3.6 - Support added to accept objects implementing :class:`os.PathLike`. + .. versionchanged:: + 3.6 + + * Support added to accept objects implementing :class:`os.PathLike`. + * On Windows, opening a console buffer may return a subclass of + :class:`io.RawIOBase` other than :class:`io.FileIO`. .. function:: ord(c) diff --git a/Doc/using/cmdline.rst b/Doc/using/cmdline.rst index 2a83bd1c189..75cb8ea24be 100644 --- a/Doc/using/cmdline.rst +++ b/Doc/using/cmdline.rst @@ -559,6 +559,10 @@ conflict. .. versionchanged:: 3.4 The ``encodingname`` part is now optional. + .. versionchanged:: 3.6 + On Windows, the encoding specified by this variable is ignored for interactive + console buffers unless :envvar:`PYTHONLEGACYWINDOWSIOENCODING` is also specified. + Files and pipes redirected through the standard streams are not affected. .. envvar:: PYTHONNOUSERSITE @@ -686,6 +690,19 @@ conflict. .. versionadded:: 3.6 See :pep:`529` for more details. +.. envvar:: PYTHONLEGACYWINDOWSIOENCODING + + If set to a non-empty string, does not use the new console reader and + writer. This means that Unicode characters will be encoded according to + the active console code page, rather than using utf-8. + + This variable is ignored if the standard streams are redirected (to files + or pipes) rather than referring to console buffers. + + Availability: Windows + + .. versionadded:: 3.6 + Debug-mode variables ~~~~~~~~~~~~~~~~~~~~ diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index 14d05797afe..fa3886cffef 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -78,6 +78,8 @@ Windows improvements: * PEP 529: :ref:`Change Windows filesystem encoding to UTF-8 ` +* PEP 528: :ref:`Change Windows console encoding to UTF-8 ` + * The ``py.exe`` launcher, when used interactively, no longer prefers Python 2 over Python 3 when the user doesn't specify a version (via command line arguments or a config file). Handling of shebang lines @@ -267,6 +269,23 @@ Also see :pep:`487` and the updated class customization documentation at (Contributed by Martin Teichmann in :issue:`27366`) +.. _pep-528: + +PEP 528: Change Windows console encoding to UTF-8 +------------------------------------------------- + +The default console on Windows will now accept all Unicode characters and +provide correctly read str objects to Python code. ``sys.stdin``, +``sys.stdout`` and ``sys.stderr`` now default to utf-8 encoding. + +This change only applies when using an interactive console, and not when +redirecting files or pipes. To revert to the previous behaviour for interactive +console use, set :envvar:`PYTHONLEGACYWINDOWSIOENCODING`. + +.. seealso:: + + :pep:`528` -- Change Windows console encoding to UTF-8 + PEP written and implemented by Steve Dower. PYTHONMALLOC environment variable ================================= diff --git a/Include/pydebug.h b/Include/pydebug.h index 19bec2bd81b..6e23a896c3d 100644 --- a/Include/pydebug.h +++ b/Include/pydebug.h @@ -24,6 +24,10 @@ PyAPI_DATA(int) Py_UnbufferedStdioFlag; PyAPI_DATA(int) Py_HashRandomizationFlag; PyAPI_DATA(int) Py_IsolatedFlag; +#ifdef MS_WINDOWS +PyAPI_DATA(int) Py_LegacyWindowsStdioFlag; +#endif + /* this is a wrapper around getenv() that pays attention to Py_IgnoreEnvironmentFlag. It should be used for getting variables like PYTHONPATH and PYTHONHOME from the environment */ diff --git a/Lib/io.py b/Lib/io.py index e03db97e4de..968ee5073df 100644 --- a/Lib/io.py +++ b/Lib/io.py @@ -90,3 +90,10 @@ for klass in (BytesIO, BufferedReader, BufferedWriter, BufferedRandom, for klass in (StringIO, TextIOWrapper): TextIOBase.register(klass) del klass + +try: + from _io import _WindowsConsoleIO +except ImportError: + pass +else: + RawIOBase.register(_WindowsConsoleIO) diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index b504cf7976e..95c74824d7d 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -1518,7 +1518,7 @@ class TestInvalidFD(unittest.TestCase): singles = ["fchdir", "dup", "fdopen", "fdatasync", "fstat", "fstatvfs", "fsync", "tcgetpgrp", "ttyname"] #singles.append("close") - #We omit close because it doesn'r raise an exception on some platforms + #We omit close because it doesn't raise an exception on some platforms def get_single(f): def helper(self): if hasattr(os, f): diff --git a/Lib/test/test_winconsoleio.py b/Lib/test/test_winconsoleio.py new file mode 100644 index 00000000000..9e932fef1f9 --- /dev/null +++ b/Lib/test/test_winconsoleio.py @@ -0,0 +1,72 @@ +'''Tests for WindowsConsoleIO + +Unfortunately, most testing requires interactive use, since we have no +API to read back from a real console, and this class is only for use +with real consoles. + +Instead, we validate that basic functionality such as opening, closing +and in particular fileno() work, but are forced to leave real testing +to real people with real keyborads. +''' + +import io +import unittest +import sys + +if sys.platform != 'win32': + raise unittest.SkipTest("test only relevant on win32") + +ConIO = io._WindowsConsoleIO + +class WindowsConsoleIOTests(unittest.TestCase): + def test_abc(self): + self.assertTrue(issubclass(ConIO, io.RawIOBase)) + self.assertFalse(issubclass(ConIO, io.BufferedIOBase)) + self.assertFalse(issubclass(ConIO, io.TextIOBase)) + + def test_open_fd(self): + f = ConIO(0) + self.assertTrue(f.readable()) + self.assertFalse(f.writable()) + self.assertEqual(0, f.fileno()) + f.close() # multiple close should not crash + f.close() + + f = ConIO(1, 'w') + self.assertFalse(f.readable()) + self.assertTrue(f.writable()) + self.assertEqual(1, f.fileno()) + f.close() + f.close() + + f = ConIO(2, 'w') + self.assertFalse(f.readable()) + self.assertTrue(f.writable()) + self.assertEqual(2, f.fileno()) + f.close() + f.close() + + def test_open_name(self): + f = ConIO("CON") + self.assertTrue(f.readable()) + self.assertFalse(f.writable()) + self.assertIsNotNone(f.fileno()) + f.close() # multiple close should not crash + f.close() + + f = ConIO('CONIN$') + self.assertTrue(f.readable()) + self.assertFalse(f.writable()) + self.assertIsNotNone(f.fileno()) + f.close() + f.close() + + f = ConIO('CONOUT$', 'w') + self.assertFalse(f.readable()) + self.assertTrue(f.writable()) + self.assertIsNotNone(f.fileno()) + f.close() + f.close() + +if __name__ == "__main__": + unittest.main() diff --git a/Misc/NEWS b/Misc/NEWS index 36db3955fb2..c9ba8eca47e 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -300,6 +300,8 @@ Build Windows ------- +- Issue #1602: Windows console doesn't input or print Unicode (PEP 528) + - Issue #27781: Change file system encoding on Windows to UTF-8 (PEP 529) - Issue #27731: Opt-out of MAX_PATH on Windows 10 @@ -556,7 +558,6 @@ Build - Issue #10910: Avoid C++ compilation errors on FreeBSD and OS X. Also update FreedBSD version checks for the original ctype UTF-8 workaround. - What's New in Python 3.6.0 alpha 3 ================================== diff --git a/Modules/_io/_iomodule.c b/Modules/_io/_iomodule.c index ec700f64c8b..60f8b549a82 100644 --- a/Modules/_io/_iomodule.c +++ b/Modules/_io/_iomodule.c @@ -20,6 +20,9 @@ #include #endif /* HAVE_SYS_STAT_H */ +#ifdef MS_WINDOWS +#include +#endif /* Various interned strings */ @@ -52,7 +55,6 @@ PyObject *_PyIO_empty_str; PyObject *_PyIO_empty_bytes; PyObject *_PyIO_zero; - PyDoc_STRVAR(module_doc, "The io module provides the Python interfaces to stream handling. The\n" "builtin open function is defined in this module.\n" @@ -362,8 +364,18 @@ _io_open_impl(PyObject *module, PyObject *file, const char *mode, } /* Create the Raw file stream */ - raw = PyObject_CallFunction((PyObject *)&PyFileIO_Type, - "OsiO", path_or_fd, rawmode, closefd, opener); + { + PyObject *RawIO_class = (PyObject *)&PyFileIO_Type; +#ifdef MS_WINDOWS + if (!Py_LegacyWindowsStdioFlag && _PyIO_get_console_type(path_or_fd) != '\0') { + RawIO_class = (PyObject *)&PyWindowsConsoleIO_Type; + encoding = "utf-8"; + } +#endif + raw = PyObject_CallFunction(RawIO_class, + "OsiO", path_or_fd, rawmode, closefd, opener); + } + if (raw == NULL) goto error; result = raw; @@ -708,6 +720,12 @@ PyInit__io(void) PyStringIO_Type.tp_base = &PyTextIOBase_Type; ADD_TYPE(&PyStringIO_Type, "StringIO"); +#ifdef MS_WINDOWS + /* WindowsConsoleIO */ + PyWindowsConsoleIO_Type.tp_base = &PyRawIOBase_Type; + ADD_TYPE(&PyWindowsConsoleIO_Type, "_WindowsConsoleIO"); +#endif + /* BufferedReader */ PyBufferedReader_Type.tp_base = &PyBufferedIOBase_Type; ADD_TYPE(&PyBufferedReader_Type, "BufferedReader"); diff --git a/Modules/_io/_iomodule.h b/Modules/_io/_iomodule.h index 8b5ec88948e..5d3da29ce1f 100644 --- a/Modules/_io/_iomodule.h +++ b/Modules/_io/_iomodule.h @@ -19,6 +19,12 @@ extern PyTypeObject PyBufferedRandom_Type; extern PyTypeObject PyTextIOWrapper_Type; extern PyTypeObject PyIncrementalNewlineDecoder_Type; +#ifndef Py_LIMITED_API +#ifdef MS_WINDOWS +extern PyTypeObject PyWindowsConsoleIO_Type; +#define PyWindowsConsoleIO_Check(op) (PyObject_TypeCheck((op), &PyWindowsConsoleIO_Type)) +#endif /* MS_WINDOWS */ +#endif /* Py_LIMITED_API */ extern int _PyIO_ConvertSsize_t(PyObject *, void *); @@ -145,6 +151,10 @@ typedef struct { extern _PyIO_State *_PyIO_get_module_state(void); extern PyObject *_PyIO_get_locale_module(_PyIO_State *); +#ifdef MS_WINDOWS +extern char _PyIO_get_console_type(PyObject *); +#endif + extern PyObject *_PyIO_str_close; extern PyObject *_PyIO_str_closed; extern PyObject *_PyIO_str_decode; diff --git a/Modules/_io/clinic/winconsoleio.c.h b/Modules/_io/clinic/winconsoleio.c.h new file mode 100644 index 00000000000..e44fcbb7a16 --- /dev/null +++ b/Modules/_io/clinic/winconsoleio.c.h @@ -0,0 +1,331 @@ +/*[clinic input] +preserve +[clinic start generated code]*/ + +#if defined(MS_WINDOWS) + +PyDoc_STRVAR(_io__WindowsConsoleIO_close__doc__, +"close($self, /)\n" +"--\n" +"\n" +"Close the handle.\n" +"\n" +"A closed handle cannot be used for further I/O operations. close() may be\n" +"called more than once without error."); + +#define _IO__WINDOWSCONSOLEIO_CLOSE_METHODDEF \ + {"close", (PyCFunction)_io__WindowsConsoleIO_close, METH_NOARGS, _io__WindowsConsoleIO_close__doc__}, + +static PyObject * +_io__WindowsConsoleIO_close_impl(winconsoleio *self); + +static PyObject * +_io__WindowsConsoleIO_close(winconsoleio *self, PyObject *Py_UNUSED(ignored)) +{ + return _io__WindowsConsoleIO_close_impl(self); +} + +#endif /* defined(MS_WINDOWS) */ + +#if defined(MS_WINDOWS) + +PyDoc_STRVAR(_io__WindowsConsoleIO___init____doc__, +"_WindowsConsoleIO(file, mode=\'r\', closefd=True, opener=None)\n" +"--\n" +"\n" +"Open a console buffer by file descriptor.\n" +"\n" +"The mode can be \'rb\' (default), or \'wb\' for reading or writing bytes. All\n" +"other mode characters will be ignored. Mode \'b\' will be assumed if it is\n" +"omitted. The *opener* parameter is always ignored."); + +static int +_io__WindowsConsoleIO___init___impl(winconsoleio *self, PyObject *nameobj, + const char *mode, int closefd, + PyObject *opener); + +static int +_io__WindowsConsoleIO___init__(PyObject *self, PyObject *args, PyObject *kwargs) +{ + int return_value = -1; + static const char * const _keywords[] = {"file", "mode", "closefd", "opener", NULL}; + static _PyArg_Parser _parser = {"O|siO:_WindowsConsoleIO", _keywords, 0}; + PyObject *nameobj; + const char *mode = "r"; + int closefd = 1; + PyObject *opener = Py_None; + + if (!_PyArg_ParseTupleAndKeywordsFast(args, kwargs, &_parser, + &nameobj, &mode, &closefd, &opener)) { + goto exit; + } + return_value = _io__WindowsConsoleIO___init___impl((winconsoleio *)self, nameobj, mode, closefd, opener); + +exit: + return return_value; +} + +#endif /* defined(MS_WINDOWS) */ + +#if defined(MS_WINDOWS) + +PyDoc_STRVAR(_io__WindowsConsoleIO_fileno__doc__, +"fileno($self, /)\n" +"--\n" +"\n" +"Return the underlying file descriptor (an integer).\n" +"\n" +"fileno is only set when a file descriptor is used to open\n" +"one of the standard streams."); + +#define _IO__WINDOWSCONSOLEIO_FILENO_METHODDEF \ + {"fileno", (PyCFunction)_io__WindowsConsoleIO_fileno, METH_NOARGS, _io__WindowsConsoleIO_fileno__doc__}, + +static PyObject * +_io__WindowsConsoleIO_fileno_impl(winconsoleio *self); + +static PyObject * +_io__WindowsConsoleIO_fileno(winconsoleio *self, PyObject *Py_UNUSED(ignored)) +{ + return _io__WindowsConsoleIO_fileno_impl(self); +} + +#endif /* defined(MS_WINDOWS) */ + +#if defined(MS_WINDOWS) + +PyDoc_STRVAR(_io__WindowsConsoleIO_readable__doc__, +"readable($self, /)\n" +"--\n" +"\n" +"True if console is an input buffer."); + +#define _IO__WINDOWSCONSOLEIO_READABLE_METHODDEF \ + {"readable", (PyCFunction)_io__WindowsConsoleIO_readable, METH_NOARGS, _io__WindowsConsoleIO_readable__doc__}, + +static PyObject * +_io__WindowsConsoleIO_readable_impl(winconsoleio *self); + +static PyObject * +_io__WindowsConsoleIO_readable(winconsoleio *self, PyObject *Py_UNUSED(ignored)) +{ + return _io__WindowsConsoleIO_readable_impl(self); +} + +#endif /* defined(MS_WINDOWS) */ + +#if defined(MS_WINDOWS) + +PyDoc_STRVAR(_io__WindowsConsoleIO_writable__doc__, +"writable($self, /)\n" +"--\n" +"\n" +"True if console is an output buffer."); + +#define _IO__WINDOWSCONSOLEIO_WRITABLE_METHODDEF \ + {"writable", (PyCFunction)_io__WindowsConsoleIO_writable, METH_NOARGS, _io__WindowsConsoleIO_writable__doc__}, + +static PyObject * +_io__WindowsConsoleIO_writable_impl(winconsoleio *self); + +static PyObject * +_io__WindowsConsoleIO_writable(winconsoleio *self, PyObject *Py_UNUSED(ignored)) +{ + return _io__WindowsConsoleIO_writable_impl(self); +} + +#endif /* defined(MS_WINDOWS) */ + +#if defined(MS_WINDOWS) + +PyDoc_STRVAR(_io__WindowsConsoleIO_readinto__doc__, +"readinto($self, buffer, /)\n" +"--\n" +"\n" +"Same as RawIOBase.readinto()."); + +#define _IO__WINDOWSCONSOLEIO_READINTO_METHODDEF \ + {"readinto", (PyCFunction)_io__WindowsConsoleIO_readinto, METH_O, _io__WindowsConsoleIO_readinto__doc__}, + +static PyObject * +_io__WindowsConsoleIO_readinto_impl(winconsoleio *self, Py_buffer *buffer); + +static PyObject * +_io__WindowsConsoleIO_readinto(winconsoleio *self, PyObject *arg) +{ + PyObject *return_value = NULL; + Py_buffer buffer = {NULL, NULL}; + + if (!PyArg_Parse(arg, "w*:readinto", &buffer)) { + goto exit; + } + return_value = _io__WindowsConsoleIO_readinto_impl(self, &buffer); + +exit: + /* Cleanup for buffer */ + if (buffer.obj) { + PyBuffer_Release(&buffer); + } + + return return_value; +} + +#endif /* defined(MS_WINDOWS) */ + +#if defined(MS_WINDOWS) + +PyDoc_STRVAR(_io__WindowsConsoleIO_readall__doc__, +"readall($self, /)\n" +"--\n" +"\n" +"Read all data from the console, returned as bytes.\n" +"\n" +"Return an empty bytes object at EOF."); + +#define _IO__WINDOWSCONSOLEIO_READALL_METHODDEF \ + {"readall", (PyCFunction)_io__WindowsConsoleIO_readall, METH_NOARGS, _io__WindowsConsoleIO_readall__doc__}, + +static PyObject * +_io__WindowsConsoleIO_readall_impl(winconsoleio *self); + +static PyObject * +_io__WindowsConsoleIO_readall(winconsoleio *self, PyObject *Py_UNUSED(ignored)) +{ + return _io__WindowsConsoleIO_readall_impl(self); +} + +#endif /* defined(MS_WINDOWS) */ + +#if defined(MS_WINDOWS) + +PyDoc_STRVAR(_io__WindowsConsoleIO_read__doc__, +"read($self, size=-1, /)\n" +"--\n" +"\n" +"Read at most size bytes, returned as bytes.\n" +"\n" +"Only makes one system call when size is a positive integer,\n" +"so less data may be returned than requested.\n" +"Return an empty bytes object at EOF."); + +#define _IO__WINDOWSCONSOLEIO_READ_METHODDEF \ + {"read", (PyCFunction)_io__WindowsConsoleIO_read, METH_VARARGS, _io__WindowsConsoleIO_read__doc__}, + +static PyObject * +_io__WindowsConsoleIO_read_impl(winconsoleio *self, Py_ssize_t size); + +static PyObject * +_io__WindowsConsoleIO_read(winconsoleio *self, PyObject *args) +{ + PyObject *return_value = NULL; + Py_ssize_t size = -1; + + if (!PyArg_ParseTuple(args, "|O&:read", + _PyIO_ConvertSsize_t, &size)) { + goto exit; + } + return_value = _io__WindowsConsoleIO_read_impl(self, size); + +exit: + return return_value; +} + +#endif /* defined(MS_WINDOWS) */ + +#if defined(MS_WINDOWS) + +PyDoc_STRVAR(_io__WindowsConsoleIO_write__doc__, +"write($self, b, /)\n" +"--\n" +"\n" +"Write buffer b to file, return number of bytes written.\n" +"\n" +"Only makes one system call, so not all of the data may be written.\n" +"The number of bytes actually written is returned."); + +#define _IO__WINDOWSCONSOLEIO_WRITE_METHODDEF \ + {"write", (PyCFunction)_io__WindowsConsoleIO_write, METH_O, _io__WindowsConsoleIO_write__doc__}, + +static PyObject * +_io__WindowsConsoleIO_write_impl(winconsoleio *self, Py_buffer *b); + +static PyObject * +_io__WindowsConsoleIO_write(winconsoleio *self, PyObject *arg) +{ + PyObject *return_value = NULL; + Py_buffer b = {NULL, NULL}; + + if (!PyArg_Parse(arg, "y*:write", &b)) { + goto exit; + } + return_value = _io__WindowsConsoleIO_write_impl(self, &b); + +exit: + /* Cleanup for b */ + if (b.obj) { + PyBuffer_Release(&b); + } + + return return_value; +} + +#endif /* defined(MS_WINDOWS) */ + +#if defined(MS_WINDOWS) + +PyDoc_STRVAR(_io__WindowsConsoleIO_isatty__doc__, +"isatty($self, /)\n" +"--\n" +"\n" +"Always True."); + +#define _IO__WINDOWSCONSOLEIO_ISATTY_METHODDEF \ + {"isatty", (PyCFunction)_io__WindowsConsoleIO_isatty, METH_NOARGS, _io__WindowsConsoleIO_isatty__doc__}, + +static PyObject * +_io__WindowsConsoleIO_isatty_impl(winconsoleio *self); + +static PyObject * +_io__WindowsConsoleIO_isatty(winconsoleio *self, PyObject *Py_UNUSED(ignored)) +{ + return _io__WindowsConsoleIO_isatty_impl(self); +} + +#endif /* defined(MS_WINDOWS) */ + +#ifndef _IO__WINDOWSCONSOLEIO_CLOSE_METHODDEF + #define _IO__WINDOWSCONSOLEIO_CLOSE_METHODDEF +#endif /* !defined(_IO__WINDOWSCONSOLEIO_CLOSE_METHODDEF) */ + +#ifndef _IO__WINDOWSCONSOLEIO_FILENO_METHODDEF + #define _IO__WINDOWSCONSOLEIO_FILENO_METHODDEF +#endif /* !defined(_IO__WINDOWSCONSOLEIO_FILENO_METHODDEF) */ + +#ifndef _IO__WINDOWSCONSOLEIO_READABLE_METHODDEF + #define _IO__WINDOWSCONSOLEIO_READABLE_METHODDEF +#endif /* !defined(_IO__WINDOWSCONSOLEIO_READABLE_METHODDEF) */ + +#ifndef _IO__WINDOWSCONSOLEIO_WRITABLE_METHODDEF + #define _IO__WINDOWSCONSOLEIO_WRITABLE_METHODDEF +#endif /* !defined(_IO__WINDOWSCONSOLEIO_WRITABLE_METHODDEF) */ + +#ifndef _IO__WINDOWSCONSOLEIO_READINTO_METHODDEF + #define _IO__WINDOWSCONSOLEIO_READINTO_METHODDEF +#endif /* !defined(_IO__WINDOWSCONSOLEIO_READINTO_METHODDEF) */ + +#ifndef _IO__WINDOWSCONSOLEIO_READALL_METHODDEF + #define _IO__WINDOWSCONSOLEIO_READALL_METHODDEF +#endif /* !defined(_IO__WINDOWSCONSOLEIO_READALL_METHODDEF) */ + +#ifndef _IO__WINDOWSCONSOLEIO_READ_METHODDEF + #define _IO__WINDOWSCONSOLEIO_READ_METHODDEF +#endif /* !defined(_IO__WINDOWSCONSOLEIO_READ_METHODDEF) */ + +#ifndef _IO__WINDOWSCONSOLEIO_WRITE_METHODDEF + #define _IO__WINDOWSCONSOLEIO_WRITE_METHODDEF +#endif /* !defined(_IO__WINDOWSCONSOLEIO_WRITE_METHODDEF) */ + +#ifndef _IO__WINDOWSCONSOLEIO_ISATTY_METHODDEF + #define _IO__WINDOWSCONSOLEIO_ISATTY_METHODDEF +#endif /* !defined(_IO__WINDOWSCONSOLEIO_ISATTY_METHODDEF) */ +/*[clinic end generated code: output=9eba916f8537fff7 input=a9049054013a1b77]*/ diff --git a/Modules/_io/winconsoleio.c b/Modules/_io/winconsoleio.c new file mode 100644 index 00000000000..2527ff13cae --- /dev/null +++ b/Modules/_io/winconsoleio.c @@ -0,0 +1,1096 @@ +/* + An implementation of Windows console I/O + + Classes defined here: _WindowsConsoleIO + + Written by Steve Dower +*/ + +#define PY_SSIZE_T_CLEAN +#include "Python.h" + +#ifdef MS_WINDOWS + +#include "structmember.h" +#ifdef HAVE_SYS_TYPES_H +#include +#endif +#ifdef HAVE_SYS_STAT_H +#include +#endif +#include /* For offsetof */ + +#define WIN32_LEAN_AND_MEAN +#include + +#include "_iomodule.h" + +/* BUFSIZ determines how many characters can be typed at the console + before it starts blocking. */ +#if BUFSIZ < (16*1024) +#define SMALLCHUNK (2*1024) +#elif (BUFSIZ >= (2 << 25)) +#error "unreasonable BUFSIZ > 64MB defined" +#else +#define SMALLCHUNK BUFSIZ +#endif + +/* BUFMAX determines how many bytes can be read in one go. */ +#define BUFMAX (32*1024*1024) + +char _get_console_type(HANDLE handle) { + DWORD mode, peek_count; + + if (handle == INVALID_HANDLE_VALUE) + return '\0'; + + if (!GetConsoleMode(handle, &mode)) + return '\0'; + + /* Peek at the handle to see whether it is an input or output handle */ + if (GetNumberOfConsoleInputEvents(handle, &peek_count)) + return 'r'; + return 'w'; +} + +char _PyIO_get_console_type(PyObject *path_or_fd) { + int fd; + + fd = PyLong_AsLong(path_or_fd); + PyErr_Clear(); + if (fd >= 0) { + HANDLE handle; + _Py_BEGIN_SUPPRESS_IPH + handle = (HANDLE)_get_osfhandle(fd); + _Py_END_SUPPRESS_IPH + if (!handle) + return '\0'; + return _get_console_type(handle); + } + + PyObject *decoded = Py_None; + Py_INCREF(decoded); + + int d = PyUnicode_FSDecoder(path_or_fd, &decoded); + if (!d) { + PyErr_Clear(); + Py_CLEAR(decoded); + return '\0'; + } + + char m = '\0'; + if (!PyUnicode_Check(decoded)) { + return '\0'; + } else if (PyUnicode_CompareWithASCIIString(decoded, "CONIN$") == 0) { + m = 'r'; + } else if (PyUnicode_CompareWithASCIIString(decoded, "CONOUT$") == 0 || + PyUnicode_CompareWithASCIIString(decoded, "CON") == 0) { + m = 'w'; + } + + Py_CLEAR(decoded); + return m; +} + +/*[clinic input] +module _io +class _io._WindowsConsoleIO "winconsoleio *" "&PyWindowsConsoleIO_Type" +[clinic start generated code]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=e897fdc1fba4e131]*/ + +/*[python input] +class io_ssize_t_converter(CConverter): + type = 'Py_ssize_t' + converter = '_PyIO_ConvertSsize_t' +[python start generated code]*/ +/*[python end generated code: output=da39a3ee5e6b4b0d input=d0a811d3cbfd1b33]*/ + +typedef struct { + PyObject_HEAD + HANDLE handle; + int fd; + unsigned int created : 1; + unsigned int readable : 1; + unsigned int writable : 1; + unsigned int closehandle : 1; + char finalizing; + unsigned int blksize; + PyObject *weakreflist; + PyObject *dict; + char buf[4]; +} winconsoleio; + +PyTypeObject PyWindowsConsoleIO_Type; + +_Py_IDENTIFIER(name); + +int +_PyWindowsConsoleIO_closed(PyObject *self) +{ + return ((winconsoleio *)self)->handle == INVALID_HANDLE_VALUE; +} + + +/* Returns 0 on success, -1 with exception set on failure. */ +static int +internal_close(winconsoleio *self) +{ + if (self->handle != INVALID_HANDLE_VALUE) { + if (self->closehandle) { + if (self->fd >= 0) { + _Py_BEGIN_SUPPRESS_IPH + close(self->fd); + _Py_END_SUPPRESS_IPH + } + CloseHandle(self->handle); + } + self->handle = INVALID_HANDLE_VALUE; + self->fd = -1; + } + return 0; +} + +/*[clinic input] +_io._WindowsConsoleIO.close + +Close the handle. + +A closed handle cannot be used for further I/O operations. close() may be +called more than once without error. +[clinic start generated code]*/ + +static PyObject * +_io__WindowsConsoleIO_close_impl(winconsoleio *self) +/*[clinic end generated code: output=27ef95b66c29057b input=185617e349ae4c7b]*/ +{ + PyObject *res; + PyObject *exc, *val, *tb; + int rc; + _Py_IDENTIFIER(close); + res = _PyObject_CallMethodId((PyObject*)&PyRawIOBase_Type, + &PyId_close, "O", self); + if (!self->closehandle) { + self->handle = INVALID_HANDLE_VALUE; + return res; + } + if (res == NULL) + PyErr_Fetch(&exc, &val, &tb); + rc = internal_close(self); + if (res == NULL) + _PyErr_ChainExceptions(exc, val, tb); + if (rc < 0) + Py_CLEAR(res); + return res; +} + +static PyObject * +winconsoleio_new(PyTypeObject *type, PyObject *args, PyObject *kwds) +{ + winconsoleio *self; + + assert(type != NULL && type->tp_alloc != NULL); + + self = (winconsoleio *) type->tp_alloc(type, 0); + if (self != NULL) { + self->handle = INVALID_HANDLE_VALUE; + self->fd = -1; + self->created = 0; + self->readable = 0; + self->writable = 0; + self->closehandle = 0; + self->blksize = 0; + self->weakreflist = NULL; + } + + return (PyObject *) self; +} + +/*[clinic input] +_io._WindowsConsoleIO.__init__ + file as nameobj: object + mode: str = "r" + closefd: int(c_default="1") = True + opener: object = None + +Open a console buffer by file descriptor. + +The mode can be 'rb' (default), or 'wb' for reading or writing bytes. All +other mode characters will be ignored. Mode 'b' will be assumed if it is +omitted. The *opener* parameter is always ignored. +[clinic start generated code]*/ + +static int +_io__WindowsConsoleIO___init___impl(winconsoleio *self, PyObject *nameobj, + const char *mode, int closefd, + PyObject *opener) +/*[clinic end generated code: output=3fd9cbcdd8d95429 input=61be39633a86f5d7]*/ +{ + const char *s; + wchar_t *name = NULL; + int ret = 0; + int rwa = 0; + int fd = -1; + int fd_is_own = 0; + + assert(PyWindowsConsoleIO_Check(self)); + if (self->handle >= 0) { + if (self->closehandle) { + /* Have to close the existing file first. */ + if (internal_close(self) < 0) + return -1; + } + else + self->handle = INVALID_HANDLE_VALUE; + } + + if (PyFloat_Check(nameobj)) { + PyErr_SetString(PyExc_TypeError, + "integer argument expected, got float"); + return -1; + } + + fd = _PyLong_AsInt(nameobj); + if (fd < 0) { + if (!PyErr_Occurred()) { + PyErr_SetString(PyExc_ValueError, + "negative file descriptor"); + return -1; + } + PyErr_Clear(); + } + self->fd = fd; + + if (fd < 0) { + PyObject *decodedname = Py_None; + Py_INCREF(decodedname); + + int d = PyUnicode_FSDecoder(nameobj, (void*)&decodedname); + if (!d) + return -1; + + Py_ssize_t length; + name = PyUnicode_AsWideCharString(decodedname, &length); + Py_CLEAR(decodedname); + if (name == NULL) + return -1; + + if (wcslen(name) != length) { + PyMem_Free(name); + PyErr_SetString(PyExc_ValueError, "embedded null character"); + return -1; + } + } + + s = mode; + while (*s) { + switch (*s++) { + case '+': + case 'a': + case 'b': + case 'x': + break; + case 'r': + if (rwa) + goto bad_mode; + rwa = 1; + self->readable = 1; + break; + case 'w': + if (rwa) + goto bad_mode; + rwa = 1; + self->writable = 1; + break; + default: + PyErr_Format(PyExc_ValueError, + "invalid mode: %.200s", mode); + goto error; + } + } + + if (!rwa) + goto bad_mode; + + if (fd >= 0) { + _Py_BEGIN_SUPPRESS_IPH + self->handle = (HANDLE)_get_osfhandle(fd); + _Py_END_SUPPRESS_IPH + self->closehandle = 0; + } else { + DWORD access = GENERIC_READ; + + self->closehandle = 1; + if (!closefd) { + PyErr_SetString(PyExc_ValueError, + "Cannot use closefd=False with file name"); + goto error; + } + + if (self->writable) + access |= GENERIC_WRITE; + + Py_BEGIN_ALLOW_THREADS + /* Attempt to open for read/write initially, then fall back + on the specific access. This is required for modern names + CONIN$ and CONOUT$, which allow reading/writing state as + well as reading/writing content. */ + self->handle = CreateFileW(name, GENERIC_READ | GENERIC_WRITE, + FILE_SHARE_READ | FILE_SHARE_WRITE, NULL, OPEN_EXISTING, 0, NULL); + if (self->handle == INVALID_HANDLE_VALUE) + self->handle = CreateFileW(name, access, + FILE_SHARE_READ | FILE_SHARE_WRITE, NULL, OPEN_EXISTING, 0, NULL); + Py_END_ALLOW_THREADS + + if (self->handle == INVALID_HANDLE_VALUE) { + PyErr_SetExcFromWindowsErrWithFilenameObject(PyExc_OSError, GetLastError(), nameobj); + goto error; + } + } + + if (self->writable && _get_console_type(self->handle) != 'w') { + PyErr_SetString(PyExc_ValueError, + "Cannot open console input buffer for writing"); + goto error; + } + if (self->readable && _get_console_type(self->handle) != 'r') { + PyErr_SetString(PyExc_ValueError, + "Cannot open console output buffer for reading"); + goto error; + } + + self->blksize = DEFAULT_BUFFER_SIZE; + memset(self->buf, 0, 4); + + if (_PyObject_SetAttrId((PyObject *)self, &PyId_name, nameobj) < 0) + goto error; + + goto done; + +bad_mode: + PyErr_SetString(PyExc_ValueError, + "Must have exactly one of read or write mode"); +error: + ret = -1; + internal_close(self); + +done: + if (name) + PyMem_Free(name); + return ret; +} + +static int +winconsoleio_traverse(winconsoleio *self, visitproc visit, void *arg) +{ + Py_VISIT(self->dict); + return 0; +} + +static int +winconsoleio_clear(winconsoleio *self) +{ + Py_CLEAR(self->dict); + return 0; +} + +static void +winconsoleio_dealloc(winconsoleio *self) +{ + self->finalizing = 1; + if (_PyIOBase_finalize((PyObject *) self) < 0) + return; + _PyObject_GC_UNTRACK(self); + if (self->weakreflist != NULL) + PyObject_ClearWeakRefs((PyObject *) self); + Py_CLEAR(self->dict); + Py_TYPE(self)->tp_free((PyObject *)self); +} + +static PyObject * +err_closed(void) +{ + PyErr_SetString(PyExc_ValueError, "I/O operation on closed file"); + return NULL; +} + +static PyObject * +err_mode(const char *action) +{ + _PyIO_State *state = IO_STATE(); + if (state != NULL) + PyErr_Format(state->unsupported_operation, + "Console buffer does not support %s", action); + return NULL; +} + +/*[clinic input] +_io._WindowsConsoleIO.fileno + +Return the underlying file descriptor (an integer). + +fileno is only set when a file descriptor is used to open +one of the standard streams. + +[clinic start generated code]*/ + +static PyObject * +_io__WindowsConsoleIO_fileno_impl(winconsoleio *self) +/*[clinic end generated code: output=006fa74ce3b5cfbf input=079adc330ddaabe6]*/ +{ + if (self->fd < 0 && self->handle != INVALID_HANDLE_VALUE) { + _Py_BEGIN_SUPPRESS_IPH + if (self->writable) + self->fd = _open_osfhandle((intptr_t)self->handle, 'wb'); + else + self->fd = _open_osfhandle((intptr_t)self->handle, 'rb'); + _Py_END_SUPPRESS_IPH + } + if (self->fd < 0) + return err_mode("fileno"); + return PyLong_FromLong(self->fd); +} + +/*[clinic input] +_io._WindowsConsoleIO.readable + +True if console is an input buffer. +[clinic start generated code]*/ + +static PyObject * +_io__WindowsConsoleIO_readable_impl(winconsoleio *self) +/*[clinic end generated code: output=daf9cef2743becf0 input=6be9defb5302daae]*/ +{ + if (self->handle == INVALID_HANDLE_VALUE) + return err_closed(); + return PyBool_FromLong((long) self->readable); +} + +/*[clinic input] +_io._WindowsConsoleIO.writable + +True if console is an output buffer. +[clinic start generated code]*/ + +static PyObject * +_io__WindowsConsoleIO_writable_impl(winconsoleio *self) +/*[clinic end generated code: output=e0a2ad7eae5abf67 input=cefbd8abc24df6a0]*/ +{ + if (self->handle == INVALID_HANDLE_VALUE) + return err_closed(); + return PyBool_FromLong((long) self->writable); +} + +static DWORD +_buflen(winconsoleio *self) +{ + for (DWORD i = 0; i < 4; ++i) { + if (!self->buf[i]) + return i; + } + return 4; +} + +static DWORD +_copyfrombuf(winconsoleio *self, char *buf, DWORD len) +{ + DWORD n = 0; + + while (self->buf[0] && len--) { + n += 1; + buf[0] = self->buf[0]; + self->buf[0] = self->buf[1]; + self->buf[1] = self->buf[2]; + self->buf[2] = self->buf[3]; + self->buf[3] = 0; + } + + return n; +} + +static wchar_t * +read_console_w(HANDLE handle, DWORD maxlen, DWORD *readlen) { + int err = 0, sig = 0; + + wchar_t *buf = (wchar_t*)PyMem_Malloc(maxlen * sizeof(wchar_t)); + if (!buf) + goto error; + *readlen = 0; + + Py_BEGIN_ALLOW_THREADS + for (DWORD off = 0; off < maxlen; off += BUFSIZ) { + DWORD n, len = min(maxlen - off, BUFSIZ); + SetLastError(0); + BOOL res = ReadConsoleW(handle, &buf[off], len, &n, NULL); + + if (!res) { + err = GetLastError(); + break; + } + if (n == 0) { + err = GetLastError(); + if (err != ERROR_OPERATION_ABORTED) + break; + err = 0; + HANDLE hInterruptEvent = _PyOS_SigintEvent(); + if (WaitForSingleObjectEx(hInterruptEvent, 100, FALSE) + == WAIT_OBJECT_0) { + ResetEvent(hInterruptEvent); + Py_BLOCK_THREADS + sig = PyErr_CheckSignals(); + Py_UNBLOCK_THREADS + if (sig < 0) + break; + } + } + *readlen += n; + + /* If we didn't read a full buffer that time, don't try + again or we will block a second time. */ + if (n < len) + break; + /* If the buffer ended with a newline, break out */ + if (buf[*readlen - 1] == '\n') + break; + } + Py_END_ALLOW_THREADS + + if (sig) + goto error; + if (err) { + PyErr_SetFromWindowsErr(err); + goto error; + } + + if (*readlen > 0 && buf[0] == L'\x1a') { + PyMem_Free(buf); + buf = (wchar_t *)PyMem_Malloc(sizeof(wchar_t)); + if (!buf) + goto error; + buf[0] = L'\0'; + *readlen = 0; + } + + return buf; + +error: + if (buf) + PyMem_Free(buf); + return NULL; +} + + +static Py_ssize_t +readinto(winconsoleio *self, char *buf, Py_ssize_t len) +{ + if (self->handle == INVALID_HANDLE_VALUE) { + err_closed(); + return -1; + } + if (!self->readable) { + err_mode("reading"); + return -1; + } + if (len == 0) + return 0; + if (len > BUFMAX) { + PyErr_Format(PyExc_ValueError, "cannot read more than %d bytes", BUFMAX); + return -1; + } + + /* Each character may take up to 4 bytes in the final buffer. + This is highly conservative, but necessary to avoid + failure for any given Unicode input (e.g. \U0010ffff). + If the caller requests fewer than 4 bytes, we buffer one + character. + */ + DWORD wlen = (DWORD)(len / 4); + if (wlen == 0) { + wlen = 1; + } + + DWORD read_len = _copyfrombuf(self, buf, (DWORD)len); + if (read_len) { + buf = &buf[read_len]; + len -= read_len; + wlen -= 1; + } + if (len == read_len || wlen == 0) + return read_len; + + DWORD n; + wchar_t *wbuf = read_console_w(self->handle, wlen, &n); + if (wbuf == NULL) + return -1; + if (n == 0) { + PyMem_Free(wbuf); + return read_len; + } + + int err = 0; + DWORD u8n = 0; + + Py_BEGIN_ALLOW_THREADS + if (len < 4) { + if (WideCharToMultiByte(CP_UTF8, 0, wbuf, n, + self->buf, sizeof(self->buf) / sizeof(self->buf[0]), + NULL, NULL)) + u8n = _copyfrombuf(self, buf, (DWORD)len); + } else { + u8n = WideCharToMultiByte(CP_UTF8, 0, wbuf, n, + buf, (DWORD)len, NULL, NULL); + } + + if (u8n) { + read_len += u8n; + u8n = 0; + } else { + err = GetLastError(); + if (err == ERROR_INSUFFICIENT_BUFFER) { + /* Calculate the needed buffer for a more useful error, as this + means our "/ 4" logic above is insufficient for some input. + */ + u8n = WideCharToMultiByte(CP_UTF8, 0, wbuf, n, + NULL, 0, NULL, NULL); + } + } + Py_END_ALLOW_THREADS + + PyMem_Free(wbuf); + + if (u8n) { + PyErr_Format(PyExc_SystemError, + "Buffer had room for %d bytes but %d bytes required", + len, u8n); + return -1; + } + if (err) { + PyErr_SetFromWindowsErr(err); + return -1; + } + + return read_len; +} + +/*[clinic input] +_io._WindowsConsoleIO.readinto + buffer: Py_buffer(accept={rwbuffer}) + / + +Same as RawIOBase.readinto(). +[clinic start generated code]*/ + +static PyObject * +_io__WindowsConsoleIO_readinto_impl(winconsoleio *self, Py_buffer *buffer) +/*[clinic end generated code: output=66d1bdfa3f20af39 input=4ed68da48a6baffe]*/ +{ + Py_ssize_t len = readinto(self, buffer->buf, buffer->len); + if (len < 0) + return NULL; + + return PyLong_FromSsize_t(len); +} + +static DWORD +new_buffersize(winconsoleio *self, DWORD currentsize) +{ + DWORD addend; + + /* Expand the buffer by an amount proportional to the current size, + giving us amortized linear-time behavior. For bigger sizes, use a + less-than-double growth factor to avoid excessive allocation. */ + if (currentsize > 65536) + addend = currentsize >> 3; + else + addend = 256 + currentsize; + if (addend < SMALLCHUNK) + /* Avoid tiny read() calls. */ + addend = SMALLCHUNK; + return addend + currentsize; +} + +/*[clinic input] +_io._WindowsConsoleIO.readall + +Read all data from the console, returned as bytes. + +Return an empty bytes object at EOF. +[clinic start generated code]*/ + +static PyObject * +_io__WindowsConsoleIO_readall_impl(winconsoleio *self) +/*[clinic end generated code: output=e6d312c684f6e23b input=4024d649a1006e69]*/ +{ + wchar_t *buf; + DWORD bufsize, n, len = 0; + PyObject *bytes; + DWORD bytes_size, rn; + + if (self->handle == INVALID_HANDLE_VALUE) + return err_closed(); + + bufsize = BUFSIZ; + + buf = (wchar_t*)PyMem_Malloc((bufsize + 1) * sizeof(wchar_t)); + if (buf == NULL) + return NULL; + + while (1) { + wchar_t *subbuf; + + if (len >= (Py_ssize_t)bufsize) { + DWORD newsize = new_buffersize(self, len); + if (newsize > BUFMAX) + break; + if (newsize < bufsize) { + PyErr_SetString(PyExc_OverflowError, + "unbounded read returned more bytes " + "than a Python bytes object can hold"); + PyMem_Free(buf); + return NULL; + } + bufsize = newsize; + + buf = PyMem_Realloc(buf, (bufsize + 1) * sizeof(wchar_t)); + if (!buf) { + PyMem_Free(buf); + return NULL; + } + } + + subbuf = read_console_w(self->handle, bufsize - len, &n); + + if (subbuf == NULL) { + PyMem_Free(buf); + return NULL; + } + + if (n > 0) + wcsncpy_s(&buf[len], bufsize - len + 1, subbuf, n); + + PyMem_Free(subbuf); + + /* when the read starts with ^Z or is empty we break */ + if (n == 0 || buf[len] == '\x1a') + break; + + len += n; + } + + if (len > 0 && buf[0] == '\x1a' && _buflen(self) == 0) { + /* when the result starts with ^Z we return an empty buffer */ + PyMem_Free(buf); + return PyBytes_FromStringAndSize(NULL, 0); + } + + Py_BEGIN_ALLOW_THREADS + bytes_size = WideCharToMultiByte(CP_UTF8, 0, buf, len, + NULL, 0, NULL, NULL); + Py_END_ALLOW_THREADS + + if (!bytes_size) { + DWORD err = GetLastError(); + PyMem_Free(buf); + return PyErr_SetFromWindowsErr(err); + } + + bytes_size += _buflen(self); + bytes = PyBytes_FromStringAndSize(NULL, bytes_size); + rn = _copyfrombuf(self, PyBytes_AS_STRING(bytes), bytes_size); + + Py_BEGIN_ALLOW_THREADS + bytes_size = WideCharToMultiByte(CP_UTF8, 0, buf, len, + &PyBytes_AS_STRING(bytes)[rn], bytes_size - rn, NULL, NULL); + Py_END_ALLOW_THREADS + + if (!bytes_size) { + DWORD err = GetLastError(); + PyMem_Free(buf); + Py_CLEAR(bytes); + return PyErr_SetFromWindowsErr(err); + } + + PyMem_Free(buf); + if (bytes_size < (size_t)PyBytes_GET_SIZE(bytes)) { + if (_PyBytes_Resize(&bytes, n * sizeof(wchar_t)) < 0) { + Py_CLEAR(bytes); + return NULL; + } + } + return bytes; +} + +/*[clinic input] +_io._WindowsConsoleIO.read + size: io_ssize_t = -1 + / + +Read at most size bytes, returned as bytes. + +Only makes one system call when size is a positive integer, +so less data may be returned than requested. +Return an empty bytes object at EOF. +[clinic start generated code]*/ + +static PyObject * +_io__WindowsConsoleIO_read_impl(winconsoleio *self, Py_ssize_t size) +/*[clinic end generated code: output=57df68af9f4b22d0 input=6c56fceec460f1dd]*/ +{ + PyObject *bytes; + Py_ssize_t bytes_size; + + if (self->handle == INVALID_HANDLE_VALUE) + return err_closed(); + if (!self->readable) + return err_mode("reading"); + + if (size < 0) + return _io__WindowsConsoleIO_readall_impl(self); + if (size > BUFMAX) { + PyErr_Format(PyExc_ValueError, "cannot read more than %d bytes", BUFMAX); + return NULL; + } + + bytes = PyBytes_FromStringAndSize(NULL, size); + if (bytes == NULL) + return NULL; + + bytes_size = readinto(self, PyBytes_AS_STRING(bytes), PyBytes_GET_SIZE(bytes)); + if (bytes_size < 0) { + Py_CLEAR(bytes); + return NULL; + } + + if (bytes_size < PyBytes_GET_SIZE(bytes)) { + if (_PyBytes_Resize(&bytes, bytes_size) < 0) { + Py_CLEAR(bytes); + return NULL; + } + } + + return bytes; +} + +/*[clinic input] +_io._WindowsConsoleIO.write + b: Py_buffer + / + +Write buffer b to file, return number of bytes written. + +Only makes one system call, so not all of the data may be written. +The number of bytes actually written is returned. +[clinic start generated code]*/ + +static PyObject * +_io__WindowsConsoleIO_write_impl(winconsoleio *self, Py_buffer *b) +/*[clinic end generated code: output=775bdb16fbf9137b input=be35fb624f97c941]*/ +{ + BOOL res = TRUE; + wchar_t *wbuf; + DWORD len, wlen, n = 0; + + if (self->handle == INVALID_HANDLE_VALUE) + return err_closed(); + if (!self->writable) + return err_mode("writing"); + + if (b->len > BUFMAX) + len = BUFMAX; + else + len = (DWORD)b->len; + + Py_BEGIN_ALLOW_THREADS + wlen = MultiByteToWideChar(CP_UTF8, 0, b->buf, len, NULL, 0); + + /* issue11395 there is an unspecified upper bound on how many bytes + can be written at once. We cap at 32k - the caller will have to + handle partial writes. + Since we don't know how many input bytes are being ignored, we + have to reduce and recalculate. */ + while (wlen > 32766 / sizeof(wchar_t)) { + len /= 2; + wlen = MultiByteToWideChar(CP_UTF8, 0, b->buf, len, NULL, 0); + } + Py_END_ALLOW_THREADS + + if (!wlen) + return PyErr_SetFromWindowsErr(0); + + wbuf = (wchar_t*)PyMem_Malloc(wlen * sizeof(wchar_t)); + + Py_BEGIN_ALLOW_THREADS + wlen = MultiByteToWideChar(CP_UTF8, 0, b->buf, len, wbuf, wlen); + if (wlen) { + res = WriteConsoleW(self->handle, wbuf, wlen, &n, NULL); + if (n < wlen) { + /* Wrote fewer characters than expected, which means our + * len value may be wrong. So recalculate it from the + * characters that were written. As this could potentially + * result in a different value, we also validate that value. + */ + len = WideCharToMultiByte(CP_UTF8, 0, wbuf, n, + NULL, 0, NULL, NULL); + if (len) { + wlen = MultiByteToWideChar(CP_UTF8, 0, b->buf, len, + NULL, 0); + assert(wlen == len); + } + } + } else + res = 0; + Py_END_ALLOW_THREADS + + if (!res) { + DWORD err = GetLastError(); + PyMem_Free(wbuf); + return PyErr_SetFromWindowsErr(err); + } + + PyMem_Free(wbuf); + return PyLong_FromSsize_t(len); +} + +static PyObject * +winconsoleio_repr(winconsoleio *self) +{ + if (self->handle == INVALID_HANDLE_VALUE) + return PyUnicode_FromFormat("<_io._WindowsConsoleIO [closed]>"); + + if (self->readable) + return PyUnicode_FromFormat("<_io._WindowsConsoleIO mode='rb' closefd=%s>", + self->closehandle ? "True" : "False"); + if (self->writable) + return PyUnicode_FromFormat("<_io._WindowsConsoleIO mode='wb' closefd=%s>", + self->closehandle ? "True" : "False"); + + PyErr_SetString(PyExc_SystemError, "_WindowsConsoleIO has invalid mode"); + return NULL; +} + +/*[clinic input] +_io._WindowsConsoleIO.isatty + +Always True. +[clinic start generated code]*/ + +static PyObject * +_io__WindowsConsoleIO_isatty_impl(winconsoleio *self) +/*[clinic end generated code: output=9eac09d287c11bd7 input=9b91591dbe356f86]*/ +{ + if (self->handle == INVALID_HANDLE_VALUE) + return err_closed(); + + Py_RETURN_TRUE; +} + +static PyObject * +winconsoleio_getstate(winconsoleio *self) +{ + PyErr_Format(PyExc_TypeError, + "cannot serialize '%s' object", Py_TYPE(self)->tp_name); + return NULL; +} + +#include "clinic/winconsoleio.c.h" + +static PyMethodDef winconsoleio_methods[] = { + _IO__WINDOWSCONSOLEIO_READ_METHODDEF + _IO__WINDOWSCONSOLEIO_READALL_METHODDEF + _IO__WINDOWSCONSOLEIO_READINTO_METHODDEF + _IO__WINDOWSCONSOLEIO_WRITE_METHODDEF + _IO__WINDOWSCONSOLEIO_CLOSE_METHODDEF + _IO__WINDOWSCONSOLEIO_READABLE_METHODDEF + _IO__WINDOWSCONSOLEIO_WRITABLE_METHODDEF + _IO__WINDOWSCONSOLEIO_FILENO_METHODDEF + _IO__WINDOWSCONSOLEIO_ISATTY_METHODDEF + {"__getstate__", (PyCFunction)winconsoleio_getstate, METH_NOARGS, NULL}, + {NULL, NULL} /* sentinel */ +}; + +/* 'closed' and 'mode' are attributes for compatibility with FileIO. */ + +static PyObject * +get_closed(winconsoleio *self, void *closure) +{ + return PyBool_FromLong((long)(self->handle == INVALID_HANDLE_VALUE)); +} + +static PyObject * +get_closefd(winconsoleio *self, void *closure) +{ + return PyBool_FromLong((long)(self->closehandle)); +} + +static PyObject * +get_mode(winconsoleio *self, void *closure) +{ + return PyUnicode_FromString(self->readable ? "rb" : "wb"); +} + +static PyGetSetDef winconsoleio_getsetlist[] = { + {"closed", (getter)get_closed, NULL, "True if the file is closed"}, + {"closefd", (getter)get_closefd, NULL, + "True if the file descriptor will be closed by close()."}, + {"mode", (getter)get_mode, NULL, "String giving the file mode"}, + {NULL}, +}; + +static PyMemberDef winconsoleio_members[] = { + {"_blksize", T_UINT, offsetof(winconsoleio, blksize), 0}, + {"_finalizing", T_BOOL, offsetof(winconsoleio, finalizing), 0}, + {NULL} +}; + +PyTypeObject PyWindowsConsoleIO_Type = { + PyVarObject_HEAD_INIT(NULL, 0) + "_io._WindowsConsoleIO", + sizeof(winconsoleio), + 0, + (destructor)winconsoleio_dealloc, /* tp_dealloc */ + 0, /* tp_print */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + 0, /* tp_reserved */ + (reprfunc)winconsoleio_repr, /* tp_repr */ + 0, /* tp_as_number */ + 0, /* tp_as_sequence */ + 0, /* tp_as_mapping */ + 0, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ + PyObject_GenericGetAttr, /* tp_getattro */ + 0, /* tp_setattro */ + 0, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE + | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_HAVE_FINALIZE, /* tp_flags */ + _io__WindowsConsoleIO___init____doc__, /* tp_doc */ + (traverseproc)winconsoleio_traverse, /* tp_traverse */ + (inquiry)winconsoleio_clear, /* tp_clear */ + 0, /* tp_richcompare */ + offsetof(winconsoleio, weakreflist), /* tp_weaklistoffset */ + 0, /* tp_iter */ + 0, /* tp_iternext */ + winconsoleio_methods, /* tp_methods */ + winconsoleio_members, /* tp_members */ + winconsoleio_getsetlist, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + offsetof(winconsoleio, dict), /* tp_dictoffset */ + _io__WindowsConsoleIO___init__, /* tp_init */ + PyType_GenericAlloc, /* tp_alloc */ + winconsoleio_new, /* tp_new */ + PyObject_GC_Del, /* tp_free */ + 0, /* tp_is_gc */ + 0, /* tp_bases */ + 0, /* tp_mro */ + 0, /* tp_cache */ + 0, /* tp_subclasses */ + 0, /* tp_weaklist */ + 0, /* tp_del */ + 0, /* tp_version_tag */ + 0, /* tp_finalize */ +}; + +#endif /* MS_WINDOWS */ diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index cab517e9767..69bd5196d0e 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -270,6 +270,7 @@ + diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters index 405974dce47..e4f98a0664b 100644 --- a/PCbuild/pythoncore.vcxproj.filters +++ b/PCbuild/pythoncore.vcxproj.filters @@ -605,6 +605,9 @@ Modules\_io + + Modules\_io + Modules\_io diff --git a/Parser/myreadline.c b/Parser/myreadline.c index 9522c794e87..c8b92da18aa 100644 --- a/Parser/myreadline.c +++ b/Parser/myreadline.c @@ -98,6 +98,100 @@ my_fgets(char *buf, int len, FILE *fp) /* NOTREACHED */ } +#ifdef MS_WINDOWS +/* Readline implementation using ReadConsoleW */ + +extern char _get_console_type(HANDLE handle); + +char * +_PyOS_WindowsConsoleReadline(HANDLE hStdIn) +{ + static wchar_t wbuf_local[1024 * 16]; + const DWORD chunk_size = 1024; + + DWORD n_read, total_read, wbuflen, u8len; + wchar_t *wbuf; + char *buf = NULL; + int err = 0; + + n_read = 0; + total_read = 0; + wbuf = wbuf_local; + wbuflen = sizeof(wbuf_local) / sizeof(wbuf_local[0]) - 1; + while (1) { + if (!ReadConsoleW(hStdIn, &wbuf[total_read], wbuflen - total_read, &n_read, NULL)) { + err = GetLastError(); + goto exit; + } + if (n_read == 0) { + int s; + err = GetLastError(); + if (err != ERROR_OPERATION_ABORTED) + goto exit; + err = 0; + HANDLE hInterruptEvent = _PyOS_SigintEvent(); + if (WaitForSingleObjectEx(hInterruptEvent, 100, FALSE) + == WAIT_OBJECT_0) { + ResetEvent(hInterruptEvent); +#ifdef WITH_THREAD + PyEval_RestoreThread(_PyOS_ReadlineTState); +#endif + s = PyErr_CheckSignals(); +#ifdef WITH_THREAD + PyEval_SaveThread(); +#endif + if (s < 0) + goto exit; + } + break; + } + + total_read += n_read; + if (total_read == 0 || wbuf[total_read - 1] == L'\n') { + break; + } + wbuflen += chunk_size; + if (wbuf == wbuf_local) { + wbuf[total_read] = '\0'; + wbuf = (wchar_t*)PyMem_RawMalloc(wbuflen * sizeof(wchar_t)); + if (wbuf) + wcscpy_s(wbuf, wbuflen, wbuf_local); + } + else + wbuf = (wchar_t*)PyMem_RawRealloc(wbuf, wbuflen * sizeof(wchar_t)); + } + + if (wbuf[0] == '\x1a') { + buf = PyMem_RawMalloc(1); + if (buf) + buf[0] = '\0'; + goto exit; + } + + u8len = WideCharToMultiByte(CP_UTF8, 0, wbuf, total_read, NULL, 0, NULL, NULL); + buf = PyMem_RawMalloc(u8len + 1); + u8len = WideCharToMultiByte(CP_UTF8, 0, wbuf, total_read, buf, u8len, NULL, NULL); + buf[u8len] = '\0'; + +exit: + if (wbuf != wbuf_local) + PyMem_RawFree(wbuf); + + if (err) { +#ifdef WITH_THREAD + PyEval_RestoreThread(_PyOS_ReadlineTState); +#endif + PyErr_SetFromWindowsErr(err); +#ifdef WITH_THREAD + PyEval_SaveThread(); +#endif + } + + return buf; +} + +#endif + /* Readline implementation using fgets() */ @@ -107,6 +201,25 @@ PyOS_StdioReadline(FILE *sys_stdin, FILE *sys_stdout, const char *prompt) size_t n; char *p, *pr; +#ifdef MS_WINDOWS + if (!Py_LegacyWindowsStdioFlag && sys_stdin == stdin) { + HANDLE hStdIn; + + _Py_BEGIN_SUPPRESS_IPH + hStdIn = (HANDLE)_get_osfhandle(fileno(sys_stdin)); + _Py_END_SUPPRESS_IPH + + if (_get_console_type(hStdIn) == 'r') { + fflush(sys_stdout); + if (prompt) + fprintf(stderr, "%s", prompt); + fflush(stderr); + clearerr(sys_stdin); + return _PyOS_WindowsConsoleReadline(hStdIn); + } + } +#endif + n = 100; p = (char *)PyMem_RawMalloc(n); if (p == NULL) diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index dab5c3c1ab6..1a68255c93f 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -31,6 +31,9 @@ #ifdef MS_WINDOWS #undef BYTE #include "windows.h" + +extern PyTypeObject PyWindowsConsoleIO_Type; +#define PyWindowsConsoleIO_Check(op) (PyObject_TypeCheck((op), &PyWindowsConsoleIO_Type)) #endif _Py_IDENTIFIER(flush); @@ -92,6 +95,7 @@ int Py_HashRandomizationFlag = 0; /* for -R and PYTHONHASHSEED */ int Py_IsolatedFlag = 0; /* for -I, isolate from user's env */ #ifdef MS_WINDOWS int Py_LegacyWindowsFSEncodingFlag = 0; /* Uses mbcs instead of utf-8 */ +int Py_LegacyWindowsStdioFlag = 0; /* Uses FileIO instead of WindowsConsoleIO */ #endif PyThreadState *_Py_Finalizing = NULL; @@ -154,6 +158,12 @@ Py_SetStandardStreamEncoding(const char *encoding, const char *errors) return -3; } } +#ifdef MS_WINDOWS + if (_Py_StandardStreamEncoding) { + /* Overriding the stream encoding implies legacy streams */ + Py_LegacyWindowsStdioFlag = 1; + } +#endif return 0; } @@ -327,6 +337,8 @@ _Py_InitializeEx_Private(int install_sigs, int install_importlib) #ifdef MS_WINDOWS if ((p = Py_GETENV("PYTHONLEGACYWINDOWSFSENCODING")) && *p != '\0') Py_LegacyWindowsFSEncodingFlag = add_flag(Py_LegacyWindowsFSEncodingFlag, p); + if ((p = Py_GETENV("PYTHONLEGACYWINDOWSSTDIO")) && *p != '\0') + Py_LegacyWindowsStdioFlag = add_flag(Py_LegacyWindowsStdioFlag, p); #endif _PyRandom_Init(); @@ -1089,6 +1101,12 @@ create_stdio(PyObject* io, Py_INCREF(raw); } +#ifdef MS_WINDOWS + /* Windows console IO is always UTF-8 encoded */ + if (PyWindowsConsoleIO_Check(raw)) + encoding = "utf-8"; +#endif + text = PyUnicode_FromString(name); if (text == NULL || _PyObject_SetAttrId(raw, &PyId_name, text) < 0) goto error;