Merge issue #28164 and issue #29409

Lib/test/test_fileio.py

@@ -9,7 +9,8 @@ from array import array
 from weakref import proxy
 from functools import wraps
 
-from test.support import TESTFN, check_warnings, run_unittest, make_bad_fd, cpython_only
+from test.support import (TESTFN, TESTFN_UNICODE, check_warnings, run_unittest,
+                          make_bad_fd, cpython_only)
 from collections import UserList
 
 import _io  # C implementation of io
@@ -432,6 +433,23 @@ class OtherFileTests:
         finally:
             os.unlink(TESTFN)
 
+    @unittest.skipIf(sys.getfilesystemencoding() != 'utf-8',
+                     "test only works for utf-8 filesystems")
+    def testUtf8BytesOpen(self):
+        # Opening a UTF-8 bytes filename
+        try:
+            fn = TESTFN_UNICODE.encode("utf-8")
+        except UnicodeEncodeError:
+            self.skipTest('could not encode %r to utf-8' % TESTFN_UNICODE)
+        f = self.FileIO(fn, "w")
+        try:
+            f.write(b"abc")
+            f.close()
+            with open(TESTFN_UNICODE, "rb") as f:
+                self.assertEqual(f.read(), b"abc")
+        finally:
+            os.unlink(TESTFN_UNICODE)
+
     def testConstructorHandlesNULChars(self):
         fn_with_NUL = 'foo\0bar'
         self.assertRaises(ValueError, self.FileIO, fn_with_NUL, 'w')

Lib/test/test_winconsoleio.py

@@ -1,9 +1,11 @@
 '''Tests for WindowsConsoleIO
 '''
 
+import os
 import io
-import unittest
 import sys
+import unittest
+import tempfile
 
 if sys.platform != 'win32':
     raise unittest.SkipTest("test only relevant on win32")
@@ -19,6 +21,16 @@ class WindowsConsoleIOTests(unittest.TestCase):
         self.assertFalse(issubclass(ConIO, io.TextIOBase))
 
     def test_open_fd(self):
+        self.assertRaisesRegex(ValueError,
+            "negative file descriptor", ConIO, -1)
+
+        fd, _ = tempfile.mkstemp()
+        try:
+            self.assertRaisesRegex(ValueError,
+                "Cannot open non-console file", ConIO, fd)
+        finally:
+            os.close(fd)
+
         try:
             f = ConIO(0)
         except ValueError:
@@ -56,6 +68,20 @@ class WindowsConsoleIOTests(unittest.TestCase):
             f.close()
 
     def test_open_name(self):
+        self.assertRaises(ValueError, ConIO, sys.executable)
+
+        f = open('C:/con', 'rb', buffering=0)
+        self.assertIsInstance(f, ConIO)
+        f.close()
+
+        f = open(r'\\.\conin$', 'rb', buffering=0)
+        self.assertIsInstance(f, ConIO)
+        f.close()
+
+        f = open('//?/conout$', 'wb', buffering=0)
+        self.assertIsInstance(f, ConIO)
+        f.close()
+
         f = ConIO("CON")
         self.assertTrue(f.readable())
         self.assertFalse(f.writable())

Misc/NEWS

@@ -871,6 +871,10 @@ Library
 Windows
 -------
 
+- Issue #28164: Correctly handle special console filenames (patch by Eryk Sun)
+
+- Issue #29409: Implement PEP 529 for io.FileIO (Patch by Eryk Sun)
+
 - Issue #29392: Prevent crash when passing invalid arguments into msvcrt module.
 
 - Issue #25778: winreg does not truncate string correctly (Patch by Eryk Sun)

Modules/_io/fileio.c

@@ -230,12 +230,13 @@ _io_FileIO___init___impl(fileio *self, PyObject *nameobj, const char *mode,
                          int closefd, PyObject *opener)
 /*[clinic end generated code: output=23413f68e6484bbd input=193164e293d6c097]*/
 {
-    const char *name = NULL;
-    PyObject *stringobj = NULL;
-    const char *s;
 #ifdef MS_WINDOWS
     Py_UNICODE *widename = NULL;
+#else
+    const char *name = NULL;
 #endif
+    PyObject *stringobj = NULL;
+    const char *s;
     int ret = 0;
     int rwa = 0, plus = 0;
     int flags = 0;
@@ -277,24 +278,21 @@ _io_FileIO___init___impl(fileio *self, PyObject *nameobj, const char *mode,
         PyErr_Clear();
     }
 
+    if (fd < 0) {
 #ifdef MS_WINDOWS
-    if (PyUnicode_Check(nameobj)) {
         Py_ssize_t length;
-        widename = PyUnicode_AsUnicodeAndSize(nameobj, &length);
-        if (widename == NULL)
-            return -1;
-        if (wcslen(widename) != length) {
-            PyErr_SetString(PyExc_ValueError, "embedded null character");
+        if (!PyUnicode_FSDecoder(nameobj, &stringobj)) {
             return -1;
         }
-    } else
-#endif
-    if (fd < 0)
-    {
+        widename = PyUnicode_AsUnicodeAndSize(stringobj, &length);
+        if (widename == NULL)
+            return -1;
+#else
         if (!PyUnicode_FSConverter(nameobj, &stringobj)) {
             return -1;
         }
         name = PyBytes_AS_STRING(stringobj);
+#endif
     }
 
     s = mode;
@@ -386,11 +384,10 @@ _io_FileIO___init___impl(fileio *self, PyObject *nameobj, const char *mode,
             do {
                 Py_BEGIN_ALLOW_THREADS
 #ifdef MS_WINDOWS
-                if (widename != NULL)
-                    self->fd = _wopen(widename, flags, 0666);
-                else
+                self->fd = _wopen(widename, flags, 0666);
+#else
+                self->fd = open(name, flags, 0666);
 #endif
-                    self->fd = open(name, flags, 0666);
                 Py_END_ALLOW_THREADS
             } while (self->fd < 0 && errno == EINTR &&
                      !(async_err = PyErr_CheckSignals()));

Modules/_io/winconsoleio.c

@@ -60,51 +60,68 @@ char _get_console_type(HANDLE handle) {
 }
 
 char _PyIO_get_console_type(PyObject *path_or_fd) {
-    int fd;
-
-    fd = PyLong_AsLong(path_or_fd);
+    int fd = PyLong_AsLong(path_or_fd);
     PyErr_Clear();
     if (fd >= 0) {
         HANDLE handle;
         _Py_BEGIN_SUPPRESS_IPH
         handle = (HANDLE)_get_osfhandle(fd);
         _Py_END_SUPPRESS_IPH
-        if (!handle)
+        if (handle == INVALID_HANDLE_VALUE)
             return '\0';
         return _get_console_type(handle);
     }
 
-    PyObject *decoded, *decoded_upper;
+    PyObject *decoded;
+    wchar_t *decoded_wstr;
 
-    int d = PyUnicode_FSDecoder(path_or_fd, &decoded);
-    if (!d) {
+    if (!PyUnicode_FSDecoder(path_or_fd, &decoded)) {
         PyErr_Clear();
         return '\0';
     }
-    if (!PyUnicode_Check(decoded)) {
-        Py_CLEAR(decoded);
-        return '\0';
-    }
-    decoded_upper = PyObject_CallMethod(decoded, "upper", NULL);
+    decoded_wstr = PyUnicode_AsWideCharString(decoded, NULL);
     Py_CLEAR(decoded);
-    if (!decoded_upper) {
+    if (!decoded_wstr) {
         PyErr_Clear();
         return '\0';
     }
 
+    DWORD length;
+    wchar_t name_buf[MAX_PATH], *pname_buf = name_buf;
+    
+    length = GetFullPathNameW(decoded_wstr, MAX_PATH, pname_buf, NULL);
+    if (length > MAX_PATH) {
+        pname_buf = PyMem_New(wchar_t, length);
+        if (pname_buf)
+            length = GetFullPathNameW(decoded_wstr, length, pname_buf, NULL);
+        else
+            length = 0;
+    }
+    PyMem_Free(decoded_wstr);
+
     char m = '\0';
-    if (_PyUnicode_EqualToASCIIString(decoded_upper, "CONIN$")) {
-        m = 'r';
-    } else if (_PyUnicode_EqualToASCIIString(decoded_upper, "CONOUT$")) {
-        m = 'w';
-    } else if (_PyUnicode_EqualToASCIIString(decoded_upper, "CON")) {
-        m = 'x';
+    if (length) {
+        wchar_t *name = pname_buf;
+        if (length >= 4 && name[3] == L'\\' &&
+            (name[2] == L'.' || name[2] == L'?') &&
+            name[1] == L'\\' && name[0] == L'\\') {
+            name += 4;
+        }
+        if (!_wcsicmp(name, L"CONIN$")) {
+            m = 'r';
+        } else if (!_wcsicmp(name, L"CONOUT$")) {
+            m = 'w';
+        } else if (!_wcsicmp(name, L"CON")) {
+            m = 'x';
+        }
     }
 
-    Py_CLEAR(decoded_upper);
+    if (pname_buf != name_buf)
+        PyMem_Free(pname_buf);
     return m;
 }
 
+
 /*[clinic input]
 module _io
 class _io._WindowsConsoleIO "winconsoleio *" "&PyWindowsConsoleIO_Type"