Merge Python 3.5.0a4 release engineering commits.

Include/unicodeobject.h

@@ -2060,12 +2060,6 @@ PyAPI_FUNC(int) PyUnicode_Contains(
     PyObject *element           /* Element string */
     );
 
-/* Checks whether the string contains any NUL characters. */
-
-#ifndef Py_LIMITED_API
-PyAPI_FUNC(int) _PyUnicode_HasNULChars(PyObject *);
-#endif
-
 /* Checks whether argument is a valid identifier. */
 
 PyAPI_FUNC(int) PyUnicode_IsIdentifier(PyObject *s);

Lib/test/test_binascii.py

@@ -135,6 +135,18 @@ class BinASCIITest(unittest.TestCase):
         # Issue #7701 (crash on a pydebug build)
         self.assertEqual(binascii.b2a_uu(b'x'), b'!>   \n')
 
+    def test_crc_hqx(self):
+        crc = binascii.crc_hqx(self.type2test(b"Test the CRC-32 of"), 0)
+        crc = binascii.crc_hqx(self.type2test(b" this string."), crc)
+        self.assertEqual(crc, 14290)
+
+        self.assertRaises(TypeError, binascii.crc_hqx)
+        self.assertRaises(TypeError, binascii.crc_hqx, self.type2test(b''))
+
+        for crc in 0, 1, 0x1234, 0x12345, 0x12345678, -1:
+            self.assertEqual(binascii.crc_hqx(self.type2test(b''), crc),
+                             crc & 0xffff)
+
     def test_crc32(self):
         crc = binascii.crc32(self.type2test(b"Test the CRC-32 of"))
         crc = binascii.crc32(self.type2test(b" this string."), crc)

Lib/test/test_posix.py

@@ -442,6 +442,36 @@ class PosixTester(unittest.TestCase):
         else:
             self.assertTrue(stat.S_ISFIFO(posix.stat(support.TESTFN).st_mode))
 
+    @unittest.skipUnless(hasattr(posix, 'stat'), 'test needs posix.stat()')
+    @unittest.skipUnless(hasattr(posix, 'makedev'), 'test needs posix.makedev()')
+    def test_makedev(self):
+        st = posix.stat(support.TESTFN)
+        dev = st.st_dev
+        self.assertIsInstance(dev, int)
+        self.assertGreaterEqual(dev, 0)
+
+        major = posix.major(dev)
+        self.assertIsInstance(major, int)
+        self.assertGreaterEqual(major, 0)
+        self.assertEqual(posix.major(dev), major)
+        self.assertRaises(TypeError, posix.major, float(dev))
+        self.assertRaises(TypeError, posix.major)
+        self.assertRaises((ValueError, OverflowError), posix.major, -1)
+
+        minor = posix.minor(dev)
+        self.assertIsInstance(minor, int)
+        self.assertGreaterEqual(minor, 0)
+        self.assertEqual(posix.minor(dev), minor)
+        self.assertRaises(TypeError, posix.minor, float(dev))
+        self.assertRaises(TypeError, posix.minor)
+        self.assertRaises((ValueError, OverflowError), posix.minor, -1)
+
+        self.assertEqual(posix.makedev(major, minor), dev)
+        self.assertRaises(TypeError, posix.makedev, float(major), minor)
+        self.assertRaises(TypeError, posix.makedev, major, float(minor))
+        self.assertRaises(TypeError, posix.makedev, major)
+        self.assertRaises(TypeError, posix.makedev)
+
     def _test_all_chown_common(self, chown_func, first_param, stat_func):
         """Common code for chown, fchown and lchown tests."""
         def check_stat(uid, gid):
@@ -1138,6 +1168,42 @@ class PosixTester(unittest.TestCase):
             else:
                 self.fail("No valid path_error2() test for os." + name)
 
+    def test_path_with_null_character(self):
+        fn = support.TESTFN
+        fn_with_NUL = fn + '\0'
+        self.addCleanup(support.unlink, fn)
+        support.unlink(fn)
+        fd = None
+        try:
+            with self.assertRaises(ValueError):
+                fd = os.open(fn_with_NUL, os.O_WRONLY | os.O_CREAT) # raises
+        finally:
+            if fd is not None:
+                os.close(fd)
+        self.assertFalse(os.path.exists(fn))
+        self.assertRaises(ValueError, os.mkdir, fn_with_NUL)
+        self.assertFalse(os.path.exists(fn))
+        open(fn, 'wb').close()
+        self.assertRaises(ValueError, os.stat, fn_with_NUL)
+
+    def test_path_with_null_byte(self):
+        fn = os.fsencode(support.TESTFN)
+        fn_with_NUL = fn + b'\0'
+        self.addCleanup(support.unlink, fn)
+        support.unlink(fn)
+        fd = None
+        try:
+            with self.assertRaises(ValueError):
+                fd = os.open(fn_with_NUL, os.O_WRONLY | os.O_CREAT) # raises
+        finally:
+            if fd is not None:
+                os.close(fd)
+        self.assertFalse(os.path.exists(fn))
+        self.assertRaises(ValueError, os.mkdir, fn_with_NUL)
+        self.assertFalse(os.path.exists(fn))
+        open(fn, 'wb').close()
+        self.assertRaises(ValueError, os.stat, fn_with_NUL)
+
 class PosixGroupsTester(unittest.TestCase):
 
     def setUp(self):

Misc/NEWS

@@ -49,6 +49,12 @@ Core and Builtins
 Library
 -------
 
+- Issue #23908: os functions now reject paths with embedded null character
+  on Windows instead of silently truncate them.
+
+- Issue #23728: binascii.crc_hqx() could return an integer outside of the range
+  0-0xffff for empty data.
+
 - Issue #16914: new debuglevel 2 in smtplib adds timestamps to debug output.
 
 - Issue #7159: urllib.request now supports sending auth credentials

Modules/_io/fileio.c

@@ -281,15 +281,14 @@ _io_FileIO___init___impl(fileio *self, PyObject *nameobj, const char *mode,
 
 #ifdef MS_WINDOWS
     if (PyUnicode_Check(nameobj)) {
-        int rv = _PyUnicode_HasNULChars(nameobj);
-        if (rv) {
-            if (rv != -1)
+        Py_ssize_t length;
+        widename = PyUnicode_AsUnicodeAndSize(nameobj, &length);
+        if (widename == NULL)
+            return -1;
+        if (wcslen(widename) != length) {
             PyErr_SetString(PyExc_ValueError, "embedded null character");
             return -1;
         }
-        widename = PyUnicode_AsUnicode(nameobj);
-        if (widename == NULL)
-            return -1;
     } else
 #endif
     if (fd < 0)

Modules/binascii.c

@@ -908,31 +908,31 @@ binascii_rledecode_hqx_impl(PyModuleDef *module, Py_buffer *data)
 
 
 /*[clinic input]
-binascii.crc_hqx -> int
+binascii.crc_hqx -> unsigned_int
 
     data: Py_buffer
-    crc: int
+    crc: unsigned_int(bitwise=True)
     /
 
 Compute hqx CRC incrementally.
 [clinic start generated code]*/
 
-static int
-binascii_crc_hqx_impl(PyModuleDef *module, Py_buffer *data, int crc)
-/*[clinic end generated code: output=634dac18dfa863d7 input=68060931b2f51c8a]*/
+static unsigned int
+binascii_crc_hqx_impl(PyModuleDef *module, Py_buffer *data, unsigned int crc)
+/*[clinic end generated code: output=167c2dac62625717 input=add8c53712ccceda]*/
 {
     unsigned char *bin_data;
-    unsigned int ucrc = (unsigned int)crc;
     Py_ssize_t len;
 
+    crc &= 0xffff;
     bin_data = data->buf;
     len = data->len;
 
     while(len-- > 0) {
-        ucrc=((ucrc<<8)&0xff00)^crctab_hqx[((ucrc>>8)&0xff)^*bin_data++];
+        crc = ((crc<<8)&0xff00) ^ crctab_hqx[(crc>>8)^*bin_data++];
     }
 
-    return (int)ucrc;
+    return crc;
 }
 
 #ifndef USE_ZLIB_CRC32

Modules/clinic/binascii.c.h

@@ -267,25 +267,25 @@ PyDoc_STRVAR(binascii_crc_hqx__doc__,
 #define BINASCII_CRC_HQX_METHODDEF    \
     {"crc_hqx", (PyCFunction)binascii_crc_hqx, METH_VARARGS, binascii_crc_hqx__doc__},
 
-static int
-binascii_crc_hqx_impl(PyModuleDef *module, Py_buffer *data, int crc);
+static unsigned int
+binascii_crc_hqx_impl(PyModuleDef *module, Py_buffer *data, unsigned int crc);
 
 static PyObject *
 binascii_crc_hqx(PyModuleDef *module, PyObject *args)
 {
     PyObject *return_value = NULL;
     Py_buffer data = {NULL, NULL};
-    int crc;
-    int _return_value;
+    unsigned int crc;
+    unsigned int _return_value;
 
     if (!PyArg_ParseTuple(args,
-        "y*i:crc_hqx",
+        "y*I:crc_hqx",
         &data, &crc))
         goto exit;
     _return_value = binascii_crc_hqx_impl(module, &data, crc);
-    if ((_return_value == -1) && PyErr_Occurred())
+    if ((_return_value == (unsigned int)-1) && PyErr_Occurred())
         goto exit;
-    return_value = PyLong_FromLong((long)_return_value);
+    return_value = PyLong_FromUnsignedLong((unsigned long)_return_value);
 
 exit:
     /* Cleanup for data */
@@ -544,4 +544,4 @@ exit:
 
     return return_value;
 }
-/*[clinic end generated code: output=175025a8a94fbdd1 input=a9049054013a1b77]*/
+/*[clinic end generated code: output=5f8d3578618b3432 input=a9049054013a1b77]*/

Modules/posixmodule.c

@@ -866,6 +866,11 @@ path_converter(PyObject *o, void *p) {
             Py_DECREF(unicode);
             return 0;
         }
+        if (wcslen(wide) != length) {
+            FORMAT_EXCEPTION(PyExc_ValueError, "embedded null character");
+            Py_DECREF(unicode);
+            return 0;
+        }
 
         path->wide = wide;
         path->narrow = NULL;

Objects/unicodeobject.c

@@ -3606,21 +3606,6 @@ PyUnicode_DecodeFSDefaultAndSize(const char *s, Py_ssize_t size)
 }
 
 
-int
-_PyUnicode_HasNULChars(PyObject* str)
-{
-    Py_ssize_t pos;
-
-    if (PyUnicode_READY(str) == -1)
-        return -1;
-    pos = findchar(PyUnicode_DATA(str), PyUnicode_KIND(str),
-                   PyUnicode_GET_LENGTH(str), '\0', 1);
-    if (pos == -1)
-        return 0;
-    else
-        return 1;
-}
-
 int
 PyUnicode_FSConverter(PyObject* arg, void* addr)
 {

README

@@ -15,8 +15,6 @@ Build Instructions
 
 On Unix, Linux, BSD, OSX, and Cygwin:
 
-New text
-
     ./configure
     make
     make test

Tools/msi/make_zip.py

@@ -30,9 +30,16 @@ def include_in_lib(p):
             return False
         return True
 
-    if name in {'_ctypes_test.pyd', '_testbuffer.pyd', '_testcapi.pyd', '_testimportmultiple.pyd', 'xxlimited.pyd'}:
-        return False
-    return p.suffix.lower() not in {'.pyc', '.pyo'}
+    suffix = p.suffix.lower()
+    if suffix == '.pyd':
+        return name not in {
+            '_ctypes_test.pyd',
+            '_testbuffer.pyd',
+            '_testcapi.pyd',
+            '_testimportmultiple.pyd',
+            'xxlimited.pyd',
+        }
+    return suffix not in {'.pyc', '.pyo'}
 
 def include_in_tools(p):
     if p.is_dir() and p.name.lower() in {'scripts', 'i18n', 'pynche', 'demo', 'parser'}:

Tools/msi/testrelease.bat

@@ -1,4 +1,4 @@
-@setlocal
+@setlocal enableextensions
 @echo off
 
 set D=%~dp0
@@ -51,6 +51,9 @@ exit /B 0
 
 @if not exist "%~1" exit /B 1
 
+@set EXE=%~1
+@if not "%EXE:embed=%"=="%EXE%" exit /B 0
+
 @set EXITCODE=0
 @echo Installing %1 into %2
 "%~1" /passive /log "%~2\install\log.txt" TargetDir="%~2\Python" Include_debug=1 Include_symbols=1 %~3