From d6b727e2c947240804b8e434b305ba2890122550 Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Tue, 12 May 2020 23:32:32 +0100 Subject: [PATCH] bpo-40501: Replace ctypes code in uuid with native module (GH-19948) --- Lib/test/test_uuid.py | 27 +-- Lib/uuid.py | 180 +++--------------- .../2020-05-06-00-41-11.bpo-40501._61wv_.rst | 2 + Modules/_uuidmodule.c | 54 +++++- PCbuild/_uuid.vcxproj | 115 +++++++++++ PCbuild/_uuid.vcxproj.filters | 14 ++ PCbuild/pcbuild.proj | 2 +- PCbuild/pcbuild.sln | 38 +++- Tools/msi/lib/lib_files.wxs | 2 +- 9 files changed, 248 insertions(+), 186 deletions(-) create mode 100644 Misc/NEWS.d/next/Security/2020-05-06-00-41-11.bpo-40501._61wv_.rst create mode 100644 PCbuild/_uuid.vcxproj create mode 100644 PCbuild/_uuid.vcxproj.filters diff --git a/Lib/test/test_uuid.py b/Lib/test/test_uuid.py index ac166ced38a..b1c92427dd2 100644 --- a/Lib/test/test_uuid.py +++ b/Lib/test/test_uuid.py @@ -852,17 +852,6 @@ eth0 Link encap:Ethernet HWaddr 12:34:56:78:90:ab node = self.uuid._netstat_getnode() self.check_node(node, 'netstat') - @unittest.skipUnless(os.name == 'nt', 'requires Windows') - def test_ipconfig_getnode(self): - node = self.uuid._ipconfig_getnode() - self.check_node(node, 'ipconfig') - - @unittest.skipUnless(importable('win32wnet'), 'requires win32wnet') - @unittest.skipUnless(importable('netbios'), 'requires netbios') - def test_netbios_getnode(self): - node = self.uuid._netbios_getnode() - self.check_node(node) - def test_random_getnode(self): node = self.uuid._random_getnode() # The multicast bit, i.e. the least significant bit of first octet, @@ -874,6 +863,13 @@ eth0 Link encap:Ethernet HWaddr 12:34:56:78:90:ab node2 = self.uuid._random_getnode() self.assertNotEqual(node2, node, '%012x' % node) +class TestInternalsWithoutExtModule(BaseTestInternals, unittest.TestCase): + uuid = py_uuid + +@unittest.skipUnless(c_uuid, 'requires the C _uuid module') +class TestInternalsWithExtModule(BaseTestInternals, unittest.TestCase): + uuid = c_uuid + @unittest.skipUnless(os.name == 'posix', 'requires Posix') def test_unix_getnode(self): if not importable('_uuid') and not importable('ctypes'): @@ -885,19 +881,10 @@ eth0 Link encap:Ethernet HWaddr 12:34:56:78:90:ab self.check_node(node, 'unix') @unittest.skipUnless(os.name == 'nt', 'requires Windows') - @unittest.skipUnless(importable('ctypes'), 'requires ctypes') def test_windll_getnode(self): node = self.uuid._windll_getnode() self.check_node(node) -class TestInternalsWithoutExtModule(BaseTestInternals, unittest.TestCase): - uuid = py_uuid - -@unittest.skipUnless(c_uuid, 'requires the C _uuid module') -class TestInternalsWithExtModule(BaseTestInternals, unittest.TestCase): - uuid = c_uuid - - if __name__ == '__main__': unittest.main() diff --git a/Lib/uuid.py b/Lib/uuid.py index 2799c75ba6a..9ddce813fc4 100644 --- a/Lib/uuid.py +++ b/Lib/uuid.py @@ -555,178 +555,44 @@ def _netstat_getnode(): return _find_mac_under_heading('netstat', '-ian', b'Address') def _ipconfig_getnode(): - """Get the hardware address on Windows by running ipconfig.exe.""" - import os, re, subprocess - first_local_mac = None - dirs = ['', r'c:\windows\system32', r'c:\winnt\system32'] - try: - import ctypes - buffer = ctypes.create_string_buffer(300) - ctypes.windll.kernel32.GetSystemDirectoryA(buffer, 300) - dirs.insert(0, buffer.value.decode('mbcs')) - except: - pass - for dir in dirs: - try: - proc = subprocess.Popen([os.path.join(dir, 'ipconfig'), '/all'], - stdout=subprocess.PIPE, - encoding="oem") - except OSError: - continue - with proc: - for line in proc.stdout: - value = line.split(':')[-1].strip().lower() - if re.fullmatch('(?:[0-9a-f][0-9a-f]-){5}[0-9a-f][0-9a-f]', value): - mac = int(value.replace('-', ''), 16) - if _is_universal(mac): - return mac - first_local_mac = first_local_mac or mac - return first_local_mac or None + """[DEPRECATED] Get the hardware address on Windows.""" + # bpo-40501: UuidCreateSequential() is now the only supported approach + return _windll_getnode() def _netbios_getnode(): - """Get the hardware address on Windows using NetBIOS calls. - See http://support.microsoft.com/kb/118623 for details.""" - import win32wnet, netbios - first_local_mac = None - ncb = netbios.NCB() - ncb.Command = netbios.NCBENUM - ncb.Buffer = adapters = netbios.LANA_ENUM() - adapters._pack() - if win32wnet.Netbios(ncb) != 0: - return None - adapters._unpack() - for i in range(adapters.length): - ncb.Reset() - ncb.Command = netbios.NCBRESET - ncb.Lana_num = ord(adapters.lana[i]) - if win32wnet.Netbios(ncb) != 0: - continue - ncb.Reset() - ncb.Command = netbios.NCBASTAT - ncb.Lana_num = ord(adapters.lana[i]) - ncb.Callname = '*'.ljust(16) - ncb.Buffer = status = netbios.ADAPTER_STATUS() - if win32wnet.Netbios(ncb) != 0: - continue - status._unpack() - bytes = status.adapter_address[:6] - if len(bytes) != 6: - continue - mac = int.from_bytes(bytes, 'big') - if _is_universal(mac): - return mac - first_local_mac = first_local_mac or mac - return first_local_mac or None + """[DEPRECATED] Get the hardware address on Windows.""" + # bpo-40501: UuidCreateSequential() is now the only supported approach + return _windll_getnode() -_generate_time_safe = _UuidCreate = None -_has_uuid_generate_time_safe = None - # Import optional C extension at toplevel, to help disabling it when testing try: import _uuid + _generate_time_safe = getattr(_uuid, "generate_time_safe", None) + _UuidCreate = getattr(_uuid, "UuidCreate", None) + _has_uuid_generate_time_safe = _uuid.has_uuid_generate_time_safe except ImportError: _uuid = None + _generate_time_safe = None + _UuidCreate = None + _has_uuid_generate_time_safe = None def _load_system_functions(): - """ - Try to load platform-specific functions for generating uuids. - """ - global _generate_time_safe, _UuidCreate, _has_uuid_generate_time_safe - - if _has_uuid_generate_time_safe is not None: - return - - _has_uuid_generate_time_safe = False - - if sys.platform == "darwin" and int(os.uname().release.split('.')[0]) < 9: - # The uuid_generate_* functions are broken on MacOS X 10.5, as noted - # in issue #8621 the function generates the same sequence of values - # in the parent process and all children created using fork (unless - # those children use exec as well). - # - # Assume that the uuid_generate functions are broken from 10.5 onward, - # the test can be adjusted when a later version is fixed. - pass - elif _uuid is not None: - _generate_time_safe = _uuid.generate_time_safe - _has_uuid_generate_time_safe = _uuid.has_uuid_generate_time_safe - return - - try: - # If we couldn't find an extension module, try ctypes to find - # system routines for UUID generation. - # Thanks to Thomas Heller for ctypes and for his help with its use here. - import ctypes - import ctypes.util - - # The uuid_generate_* routines are provided by libuuid on at least - # Linux and FreeBSD, and provided by libc on Mac OS X. - _libnames = ['uuid'] - if not sys.platform.startswith('win'): - _libnames.append('c') - for libname in _libnames: - try: - lib = ctypes.CDLL(ctypes.util.find_library(libname)) - except Exception: # pragma: nocover - continue - # Try to find the safe variety first. - if hasattr(lib, 'uuid_generate_time_safe'): - _uuid_generate_time_safe = lib.uuid_generate_time_safe - # int uuid_generate_time_safe(uuid_t out); - def _generate_time_safe(): - _buffer = ctypes.create_string_buffer(16) - res = _uuid_generate_time_safe(_buffer) - return bytes(_buffer.raw), res - _has_uuid_generate_time_safe = True - break - - elif hasattr(lib, 'uuid_generate_time'): # pragma: nocover - _uuid_generate_time = lib.uuid_generate_time - # void uuid_generate_time(uuid_t out); - _uuid_generate_time.restype = None - def _generate_time_safe(): - _buffer = ctypes.create_string_buffer(16) - _uuid_generate_time(_buffer) - return bytes(_buffer.raw), None - break - - # On Windows prior to 2000, UuidCreate gives a UUID containing the - # hardware address. On Windows 2000 and later, UuidCreate makes a - # random UUID and UuidCreateSequential gives a UUID containing the - # hardware address. These routines are provided by the RPC runtime. - # NOTE: at least on Tim's WinXP Pro SP2 desktop box, while the last - # 6 bytes returned by UuidCreateSequential are fixed, they don't appear - # to bear any relationship to the MAC address of any network device - # on the box. - try: - lib = ctypes.windll.rpcrt4 - except: - lib = None - _UuidCreate = getattr(lib, 'UuidCreateSequential', - getattr(lib, 'UuidCreate', None)) - - except Exception as exc: - import warnings - warnings.warn(f"Could not find fallback ctypes uuid functions: {exc}", - ImportWarning) + """[DEPRECATED] Platform-specific functions loaded at import time""" def _unix_getnode(): - """Get the hardware address on Unix using the _uuid extension module - or ctypes.""" - _load_system_functions() - uuid_time, _ = _generate_time_safe() - return UUID(bytes=uuid_time).node + """Get the hardware address on Unix using the _uuid extension module.""" + if _generate_time_safe: + uuid_time, _ = _generate_time_safe() + return UUID(bytes=uuid_time).node def _windll_getnode(): - """Get the hardware address on Windows using ctypes.""" - import ctypes - _load_system_functions() - _buffer = ctypes.create_string_buffer(16) - if _UuidCreate(_buffer) == 0: - return UUID(bytes=bytes_(_buffer.raw)).node + """Get the hardware address on Windows using the _uuid extension module.""" + if _UuidCreate: + uuid_bytes = _UuidCreate() + return UUID(bytes_le=uuid_bytes).node def _random_getnode(): """Get a random node ID.""" @@ -755,7 +621,8 @@ if _LINUX: elif _DARWIN: _OS_GETTERS = [_ifconfig_getnode, _arp_getnode, _netstat_getnode] elif _WINDOWS: - _OS_GETTERS = [_netbios_getnode, _ipconfig_getnode] + # bpo-40201: _windll_getnode will always succeed, so these are not needed + _OS_GETTERS = [] elif _AIX: _OS_GETTERS = [_netstat_getnode] else: @@ -802,7 +669,6 @@ def uuid1(node=None, clock_seq=None): # When the system provides a version-1 UUID generator, use it (but don't # use UuidCreate here because its UUIDs don't conform to RFC 4122). - _load_system_functions() if _generate_time_safe is not None and node is clock_seq is None: uuid_time, safely_generated = _generate_time_safe() try: diff --git a/Misc/NEWS.d/next/Security/2020-05-06-00-41-11.bpo-40501._61wv_.rst b/Misc/NEWS.d/next/Security/2020-05-06-00-41-11.bpo-40501._61wv_.rst new file mode 100644 index 00000000000..5ce22eb8a92 --- /dev/null +++ b/Misc/NEWS.d/next/Security/2020-05-06-00-41-11.bpo-40501._61wv_.rst @@ -0,0 +1,2 @@ +:mod:`uuid` no longer uses :mod:`ctypes` to load :file:`libuuid` or +:file:`rpcrt4.dll` at runtime. diff --git a/Modules/_uuidmodule.c b/Modules/_uuidmodule.c index 3be6c848ad6..3f33e22a055 100644 --- a/Modules/_uuidmodule.c +++ b/Modules/_uuidmodule.c @@ -1,5 +1,5 @@ /* - * Python UUID module that wraps libuuid - + * Python UUID module that wraps libuuid or Windows rpcrt4.dll. * DCE compatible Universally Unique Identifier library. */ @@ -12,6 +12,12 @@ #include #endif +#ifdef MS_WINDOWS +#include +#endif + +#ifndef MS_WINDOWS + static PyObject * py_uuid_generate_time_safe(PyObject *Py_UNUSED(context), PyObject *Py_UNUSED(ignored)) @@ -31,17 +37,50 @@ py_uuid_generate_time_safe(PyObject *Py_UNUSED(context), return Py_BuildValue("y#i", buf, sizeof(uuid), (int) status); # else return Py_BuildValue("y#i", (const char *) &uuid, sizeof(uuid), (int) status); -# endif -#else +# endif /* HAVE_UUID_CREATE */ +#else /* HAVE_UUID_GENERATE_TIME_SAFE */ uuid_generate_time(uuid); return Py_BuildValue("y#O", (const char *) uuid, sizeof(uuid), Py_None); -#endif +#endif /* HAVE_UUID_GENERATE_TIME_SAFE */ } +#else /* MS_WINDOWS */ + +static PyObject * +py_UuidCreate(PyObject *Py_UNUSED(context), + PyObject *Py_UNUSED(ignored)) +{ + UUID uuid; + RPC_STATUS res; + + Py_BEGIN_ALLOW_THREADS + res = UuidCreateSequential(&uuid); + Py_END_ALLOW_THREADS + + switch (res) { + case RPC_S_OK: + case RPC_S_UUID_LOCAL_ONLY: + case RPC_S_UUID_NO_ADDRESS: + /* + All success codes, but the latter two indicate that the UUID is random + rather than based on the MAC address. If the OS can't figure this out, + neither can we, so we'll take it anyway. + */ + return Py_BuildValue("y#", (const char *)&uuid, sizeof(uuid)); + } + PyErr_SetFromWindowsErr(res); + return NULL; +} + +#endif /* MS_WINDOWS */ + + static int uuid_exec(PyObject *module) { assert(sizeof(uuid_t) == 16); -#ifdef HAVE_UUID_GENERATE_TIME_SAFE +#if defined(MS_WINDOWS) + int has_uuid_generate_time_safe = 0; +#elif defined(HAVE_UUID_GENERATE_TIME_SAFE) int has_uuid_generate_time_safe = 1; #else int has_uuid_generate_time_safe = 0; @@ -54,7 +93,12 @@ uuid_exec(PyObject *module) { } static PyMethodDef uuid_methods[] = { +#if defined(HAVE_UUID_UUID_H) || defined(HAVE_UUID_H) {"generate_time_safe", py_uuid_generate_time_safe, METH_NOARGS, NULL}, +#endif +#if defined(MS_WINDOWS) + {"UuidCreate", py_UuidCreate, METH_NOARGS, NULL}, +#endif {NULL, NULL, 0, NULL} /* sentinel */ }; diff --git a/PCbuild/_uuid.vcxproj b/PCbuild/_uuid.vcxproj new file mode 100644 index 00000000000..2437b7eb2d9 --- /dev/null +++ b/PCbuild/_uuid.vcxproj @@ -0,0 +1,115 @@ + + + + + Debug + ARM + + + Debug + ARM64 + + + Debug + Win32 + + + Debug + x64 + + + PGInstrument + ARM + + + PGInstrument + ARM64 + + + PGInstrument + Win32 + + + PGInstrument + x64 + + + PGUpdate + ARM + + + PGUpdate + ARM64 + + + PGUpdate + Win32 + + + PGUpdate + x64 + + + Release + ARM + + + Release + ARM64 + + + Release + Win32 + + + Release + x64 + + + + {CB435430-EBB1-478B-8F4E-C256F6838F55} + _uuid + Win32Proj + false + + + + + DynamicLibrary + NotSet + + + + .pyd + + + + + + + + + + <_ProjectFileVersion>10.0.30319.1 + + + + rpcrt4.lib;%(AdditionalDependencies) + + + + + + + + + + + {cf7ac3d1-e2df-41d2-bea6-1e2556cdea26} + false + + + + + + \ No newline at end of file diff --git a/PCbuild/_uuid.vcxproj.filters b/PCbuild/_uuid.vcxproj.filters new file mode 100644 index 00000000000..17949292314 --- /dev/null +++ b/PCbuild/_uuid.vcxproj.filters @@ -0,0 +1,14 @@ + + + + + {4FC737F1-C7A5-4376-A066-2A32D752A2FF} + cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx + + + + + Source Files + + + \ No newline at end of file diff --git a/PCbuild/pcbuild.proj b/PCbuild/pcbuild.proj index 22a9eed18d4..9c4d352b434 100644 --- a/PCbuild/pcbuild.proj +++ b/PCbuild/pcbuild.proj @@ -51,7 +51,7 @@ - + diff --git a/PCbuild/pcbuild.sln b/PCbuild/pcbuild.sln index 6dc0139bc42..6d4c9506e5e 100644 --- a/PCbuild/pcbuild.sln +++ b/PCbuild/pcbuild.sln @@ -1,6 +1,6 @@ Microsoft Visual Studio Solution File, Format Version 12.00 -# Visual Studio 15 -VisualStudioVersion = 15.0.27130.2024 +# Visual Studio Version 16 +VisualStudioVersion = 16.0.30028.174 MinimumVisualStudioVersion = 10.0.40219.1 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution Items", "{553EC33E-9816-4996-A660-5D6186A0B0B3}" ProjectSection(SolutionItems) = preProject @@ -103,6 +103,8 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "venvwlauncher", "venvwlaunc EndProject Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "pythonw_uwp", "pythonw_uwp.vcxproj", "{AB603547-1E2A-45B3-9E09-B04596006393}" EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_uuid", "_uuid.vcxproj", "{CB435430-EBB1-478B-8F4E-C256F6838F55}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|ARM = Debug|ARM @@ -1440,6 +1442,38 @@ Global {AB603547-1E2A-45B3-9E09-B04596006393}.Release|Win32.Build.0 = Release|Win32 {AB603547-1E2A-45B3-9E09-B04596006393}.Release|x64.ActiveCfg = Release|x64 {AB603547-1E2A-45B3-9E09-B04596006393}.Release|x64.Build.0 = Release|x64 + {CB435430-EBB1-478B-8F4E-C256F6838F55}.Debug|ARM.ActiveCfg = Debug|ARM + {CB435430-EBB1-478B-8F4E-C256F6838F55}.Debug|ARM.Build.0 = Debug|ARM + {CB435430-EBB1-478B-8F4E-C256F6838F55}.Debug|ARM64.ActiveCfg = Debug|ARM64 + {CB435430-EBB1-478B-8F4E-C256F6838F55}.Debug|ARM64.Build.0 = Debug|ARM64 + {CB435430-EBB1-478B-8F4E-C256F6838F55}.Debug|Win32.ActiveCfg = Debug|Win32 + {CB435430-EBB1-478B-8F4E-C256F6838F55}.Debug|Win32.Build.0 = Debug|Win32 + {CB435430-EBB1-478B-8F4E-C256F6838F55}.Debug|x64.ActiveCfg = Debug|x64 + {CB435430-EBB1-478B-8F4E-C256F6838F55}.Debug|x64.Build.0 = Debug|x64 + {CB435430-EBB1-478B-8F4E-C256F6838F55}.PGInstrument|ARM.ActiveCfg = PGInstrument|ARM + {CB435430-EBB1-478B-8F4E-C256F6838F55}.PGInstrument|ARM.Build.0 = PGInstrument|ARM + {CB435430-EBB1-478B-8F4E-C256F6838F55}.PGInstrument|ARM64.ActiveCfg = PGInstrument|ARM64 + {CB435430-EBB1-478B-8F4E-C256F6838F55}.PGInstrument|ARM64.Build.0 = PGInstrument|ARM64 + {CB435430-EBB1-478B-8F4E-C256F6838F55}.PGInstrument|Win32.ActiveCfg = PGInstrument|Win32 + {CB435430-EBB1-478B-8F4E-C256F6838F55}.PGInstrument|Win32.Build.0 = PGInstrument|Win32 + {CB435430-EBB1-478B-8F4E-C256F6838F55}.PGInstrument|x64.ActiveCfg = PGInstrument|x64 + {CB435430-EBB1-478B-8F4E-C256F6838F55}.PGInstrument|x64.Build.0 = PGInstrument|x64 + {CB435430-EBB1-478B-8F4E-C256F6838F55}.PGUpdate|ARM.ActiveCfg = PGUpdate|ARM + {CB435430-EBB1-478B-8F4E-C256F6838F55}.PGUpdate|ARM.Build.0 = PGUpdate|ARM + {CB435430-EBB1-478B-8F4E-C256F6838F55}.PGUpdate|ARM64.ActiveCfg = PGUpdate|ARM64 + {CB435430-EBB1-478B-8F4E-C256F6838F55}.PGUpdate|ARM64.Build.0 = PGUpdate|ARM64 + {CB435430-EBB1-478B-8F4E-C256F6838F55}.PGUpdate|Win32.ActiveCfg = PGUpdate|Win32 + {CB435430-EBB1-478B-8F4E-C256F6838F55}.PGUpdate|Win32.Build.0 = PGUpdate|Win32 + {CB435430-EBB1-478B-8F4E-C256F6838F55}.PGUpdate|x64.ActiveCfg = PGUpdate|x64 + {CB435430-EBB1-478B-8F4E-C256F6838F55}.PGUpdate|x64.Build.0 = PGUpdate|x64 + {CB435430-EBB1-478B-8F4E-C256F6838F55}.Release|ARM.ActiveCfg = Release|ARM + {CB435430-EBB1-478B-8F4E-C256F6838F55}.Release|ARM.Build.0 = Release|ARM + {CB435430-EBB1-478B-8F4E-C256F6838F55}.Release|ARM64.ActiveCfg = Release|ARM64 + {CB435430-EBB1-478B-8F4E-C256F6838F55}.Release|ARM64.Build.0 = Release|ARM64 + {CB435430-EBB1-478B-8F4E-C256F6838F55}.Release|Win32.ActiveCfg = Release|Win32 + {CB435430-EBB1-478B-8F4E-C256F6838F55}.Release|Win32.Build.0 = Release|Win32 + {CB435430-EBB1-478B-8F4E-C256F6838F55}.Release|x64.ActiveCfg = Release|x64 + {CB435430-EBB1-478B-8F4E-C256F6838F55}.Release|x64.Build.0 = Release|x64 EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/Tools/msi/lib/lib_files.wxs b/Tools/msi/lib/lib_files.wxs index b462372512f..95541599b9b 100644 --- a/Tools/msi/lib/lib_files.wxs +++ b/Tools/msi/lib/lib_files.wxs @@ -1,6 +1,6 @@  - +