diff --git a/Lib/urllib.py b/Lib/urllib.py index a56f16273ab..51e8df9c89a 100644 --- a/Lib/urllib.py +++ b/Lib/urllib.py @@ -1327,38 +1327,7 @@ def proxy_bypass_environment(host): if sys.platform == 'darwin': - - def _CFSetup(sc): - from ctypes import c_int32, c_void_p, c_char_p, c_int - sc.CFStringCreateWithCString.argtypes = [ c_void_p, c_char_p, c_int32 ] - sc.CFStringCreateWithCString.restype = c_void_p - sc.SCDynamicStoreCopyProxies.argtypes = [ c_void_p ] - sc.SCDynamicStoreCopyProxies.restype = c_void_p - sc.CFDictionaryGetValue.argtypes = [ c_void_p, c_void_p ] - sc.CFDictionaryGetValue.restype = c_void_p - sc.CFStringGetLength.argtypes = [ c_void_p ] - sc.CFStringGetLength.restype = c_int32 - sc.CFStringGetCString.argtypes = [ c_void_p, c_char_p, c_int32, c_int32 ] - sc.CFStringGetCString.restype = c_int32 - sc.CFNumberGetValue.argtypes = [ c_void_p, c_int, c_void_p ] - sc.CFNumberGetValue.restype = c_int32 - sc.CFRelease.argtypes = [ c_void_p ] - sc.CFRelease.restype = None - - def _CStringFromCFString(sc, value): - from ctypes import create_string_buffer - length = sc.CFStringGetLength(value) + 1 - buff = create_string_buffer(length) - sc.CFStringGetCString(value, buff, length, 0) - return buff.value - - def _CFNumberToInt32(sc, cfnum): - from ctypes import byref, c_int - val = c_int() - kCFNumberSInt32Type = 3 - sc.CFNumberGetValue(cfnum, kCFNumberSInt32Type, byref(val)) - return val.value - + from _scproxy import _get_proxy_settings, _get_proxies def proxy_bypass_macosx_sysconf(host): """ @@ -1367,8 +1336,6 @@ if sys.platform == 'darwin': This function uses the MacOSX framework SystemConfiguration to fetch the proxy information. """ - from ctypes import cdll - from ctypes.util import find_library import re import socket from fnmatch import fnmatch @@ -1380,63 +1347,35 @@ if sys.platform == 'darwin': parts = (parts + [0, 0, 0, 0])[:4] return (parts[0] << 24) | (parts[1] << 16) | (parts[2] << 8) | parts[3] - sc = cdll.LoadLibrary(find_library("SystemConfiguration")) - _CFSetup(sc) + proxy_settings = _get_proxy_settings() - hostIP = None + # Check for simple host names: + if '.' not in host: + if proxy_settings['exclude_simple']: + return True - if not sc: - return False + for value in proxy_settings.get('exceptions'): + # Items in the list are strings like these: *.local, 169.254/16 + value = sc.CFArrayGetValueAtIndex(exceptions, index) + if not value: continue - kSCPropNetProxiesExceptionsList = sc.CFStringCreateWithCString(0, "ExceptionsList", 0) - kSCPropNetProxiesExcludeSimpleHostnames = sc.CFStringCreateWithCString(0, - "ExcludeSimpleHostnames", 0) + m = re.match(r"(\d+(?:\.\d+)*)(/\d+)?", value) + if m is not None: + if hostIP is None: + hostIP = socket.gethostbyname(host) + hostIP = ip2num(hostIP) + base = ip2num(m.group(1)) + mask = int(m.group(2)[1:]) + mask = 32 - mask - proxyDict = sc.SCDynamicStoreCopyProxies(None) - if proxyDict is None: - return False - - try: - # Check for simple host names: - if '.' not in host: - exclude_simple = sc.CFDictionaryGetValue(proxyDict, - kSCPropNetProxiesExcludeSimpleHostnames) - if exclude_simple and _CFNumberToInt32(sc, exclude_simple): + if (hostIP >> mask) == (base >> mask): return True + elif fnmatch(host, value): + return True - # Check the exceptions list: - exceptions = sc.CFDictionaryGetValue(proxyDict, kSCPropNetProxiesExceptionsList) - if exceptions: - # Items in the list are strings like these: *.local, 169.254/16 - for index in xrange(sc.CFArrayGetCount(exceptions)): - value = sc.CFArrayGetValueAtIndex(exceptions, index) - if not value: continue - value = _CStringFromCFString(sc, value) - - m = re.match(r"(\d+(?:\.\d+)*)(/\d+)?", value) - if m is not None: - if hostIP is None: - hostIP = socket.gethostbyname(host) - hostIP = ip2num(hostIP) - - base = ip2num(m.group(1)) - mask = int(m.group(2)[1:]) - mask = 32 - mask - - if (hostIP >> mask) == (base >> mask): - return True - - elif fnmatch(host, value): - return True - - return False - - finally: - sc.CFRelease(kSCPropNetProxiesExceptionsList) - sc.CFRelease(kSCPropNetProxiesExcludeSimpleHostnames) - + return False def getproxies_macosx_sysconf(): @@ -1445,106 +1384,7 @@ if sys.platform == 'darwin': This function uses the MacOSX framework SystemConfiguration to fetch the proxy information. """ - from ctypes import cdll - from ctypes.util import find_library - - sc = cdll.LoadLibrary(find_library("SystemConfiguration")) - _CFSetup(sc) - - if not sc: - return {} - - kSCPropNetProxiesHTTPEnable = sc.CFStringCreateWithCString(0, "HTTPEnable", 0) - kSCPropNetProxiesHTTPProxy = sc.CFStringCreateWithCString(0, "HTTPProxy", 0) - kSCPropNetProxiesHTTPPort = sc.CFStringCreateWithCString(0, "HTTPPort", 0) - - kSCPropNetProxiesHTTPSEnable = sc.CFStringCreateWithCString(0, "HTTPSEnable", 0) - kSCPropNetProxiesHTTPSProxy = sc.CFStringCreateWithCString(0, "HTTPSProxy", 0) - kSCPropNetProxiesHTTPSPort = sc.CFStringCreateWithCString(0, "HTTPSPort", 0) - - kSCPropNetProxiesFTPEnable = sc.CFStringCreateWithCString(0, "FTPEnable", 0) - kSCPropNetProxiesFTPPassive = sc.CFStringCreateWithCString(0, "FTPPassive", 0) - kSCPropNetProxiesFTPPort = sc.CFStringCreateWithCString(0, "FTPPort", 0) - kSCPropNetProxiesFTPProxy = sc.CFStringCreateWithCString(0, "FTPProxy", 0) - - kSCPropNetProxiesGopherEnable = sc.CFStringCreateWithCString(0, "GopherEnable", 0) - kSCPropNetProxiesGopherPort = sc.CFStringCreateWithCString(0, "GopherPort", 0) - kSCPropNetProxiesGopherProxy = sc.CFStringCreateWithCString(0, "GopherProxy", 0) - - proxies = {} - proxyDict = sc.SCDynamicStoreCopyProxies(None) - - try: - # HTTP: - enabled = sc.CFDictionaryGetValue(proxyDict, kSCPropNetProxiesHTTPEnable) - if enabled and _CFNumberToInt32(sc, enabled): - proxy = sc.CFDictionaryGetValue(proxyDict, kSCPropNetProxiesHTTPProxy) - port = sc.CFDictionaryGetValue(proxyDict, kSCPropNetProxiesHTTPPort) - - if proxy: - proxy = _CStringFromCFString(sc, proxy) - if port: - port = _CFNumberToInt32(sc, port) - proxies["http"] = "http://%s:%i" % (proxy, port) - else: - proxies["http"] = "http://%s" % (proxy, ) - - # HTTPS: - enabled = sc.CFDictionaryGetValue(proxyDict, kSCPropNetProxiesHTTPSEnable) - if enabled and _CFNumberToInt32(sc, enabled): - proxy = sc.CFDictionaryGetValue(proxyDict, kSCPropNetProxiesHTTPSProxy) - port = sc.CFDictionaryGetValue(proxyDict, kSCPropNetProxiesHTTPSPort) - - if proxy: - proxy = _CStringFromCFString(sc, proxy) - if port: - port = _CFNumberToInt32(sc, port) - proxies["https"] = "http://%s:%i" % (proxy, port) - else: - proxies["https"] = "http://%s" % (proxy, ) - - # FTP: - enabled = sc.CFDictionaryGetValue(proxyDict, kSCPropNetProxiesFTPEnable) - if enabled and _CFNumberToInt32(sc, enabled): - proxy = sc.CFDictionaryGetValue(proxyDict, kSCPropNetProxiesFTPProxy) - port = sc.CFDictionaryGetValue(proxyDict, kSCPropNetProxiesFTPPort) - - if proxy: - proxy = _CStringFromCFString(sc, proxy) - if port: - port = _CFNumberToInt32(sc, port) - proxies["ftp"] = "http://%s:%i" % (proxy, port) - else: - proxies["ftp"] = "http://%s" % (proxy, ) - - # Gopher: - enabled = sc.CFDictionaryGetValue(proxyDict, kSCPropNetProxiesGopherEnable) - if enabled and _CFNumberToInt32(sc, enabled): - proxy = sc.CFDictionaryGetValue(proxyDict, kSCPropNetProxiesGopherProxy) - port = sc.CFDictionaryGetValue(proxyDict, kSCPropNetProxiesGopherPort) - - if proxy: - proxy = _CStringFromCFString(sc, proxy) - if port: - port = _CFNumberToInt32(sc, port) - proxies["gopher"] = "http://%s:%i" % (proxy, port) - else: - proxies["gopher"] = "http://%s" % (proxy, ) - finally: - sc.CFRelease(proxyDict) - - sc.CFRelease(kSCPropNetProxiesHTTPEnable) - sc.CFRelease(kSCPropNetProxiesHTTPProxy) - sc.CFRelease(kSCPropNetProxiesHTTPPort) - sc.CFRelease(kSCPropNetProxiesFTPEnable) - sc.CFRelease(kSCPropNetProxiesFTPPassive) - sc.CFRelease(kSCPropNetProxiesFTPPort) - sc.CFRelease(kSCPropNetProxiesFTPProxy) - sc.CFRelease(kSCPropNetProxiesGopherEnable) - sc.CFRelease(kSCPropNetProxiesGopherPort) - sc.CFRelease(kSCPropNetProxiesGopherProxy) - - return proxies + return _get_proxies() diff --git a/Mac/Modules/_scproxy.c b/Mac/Modules/_scproxy.c new file mode 100644 index 00000000000..003f6a483f0 --- /dev/null +++ b/Mac/Modules/_scproxy.c @@ -0,0 +1,226 @@ +/* + * Helper method for urllib to fetch the proxy configuration settings + * using the SystemConfiguration framework. + */ +#include +#include + +static int32_t +cfnum_to_int32(CFNumberRef num) +{ + int32_t result; + + CFNumberGetValue(num, kCFNumberSInt32Type, &result); + return result; +} + +static PyObject* +cfstring_to_pystring(CFStringRef ref) +{ + const char* s; + + s = CFStringGetCStringPtr(ref, kCFStringEncodingUTF8); + if (s) { + return PyString_FromString(s); + + } else { + CFIndex len = CFStringGetLength(ref); + Boolean ok; + PyObject* result; + result = PyString_FromStringAndSize(NULL, len*4); + + ok = CFStringGetCString(ref, + PyString_AS_STRING(result), + PyString_GET_SIZE(result), + kCFStringEncodingUTF8); + if (!ok) { + Py_DECREF(result); + return NULL; + } else { + _PyString_Resize(&result, + strlen(PyString_AS_STRING(result))); + } + return result; + } +} + + +static PyObject* +get_proxy_settings(PyObject* mod __attribute__((__unused__))) +{ + CFDictionaryRef proxyDict = NULL; + CFNumberRef aNum = NULL; + CFArrayRef anArray = NULL; + PyObject* result = NULL; + PyObject* v; + int r; + + proxyDict = SCDynamicStoreCopyProxies(NULL); + if (!proxyDict) { + Py_INCREF(Py_None); + return Py_None; + } + + result = PyDict_New(); + if (result == NULL) goto error; + + aNum = CFDictionaryGetValue(proxyDict, + kSCPropNetProxiesExcludeSimpleHostnames); + if (aNum == NULL) { + v = PyBool_FromLong(0); + } else { + v = PyBool_FromLong(cfnum_to_int32(aNum)); + } + if (v == NULL) goto error; + + r = PyDict_SetItemString(result, "exclude_simple", v); + Py_DECREF(v); v = NULL; + if (r == -1) goto error; + + anArray = CFDictionaryGetValue(proxyDict, + kSCPropNetProxiesExceptionsList); + if (anArray != NULL) { + CFIndex len = CFArrayGetCount(anArray); + CFIndex i; + v = PyTuple_New(len); + if (v == NULL) goto error; + + r = PyDict_SetItemString(result, "exceptions", v); + Py_DECREF(v); + if (r == -1) goto error; + + for (i = 0; i < len; i++) { + CFStringRef aString = NULL; + + aString = CFArrayGetValueAtIndex(anArray, i); + if (aString == NULL) { + PyTuple_SetItem(v, i, Py_None); + Py_INCREF(Py_None); + } else { + PyObject* t = cfstring_to_pystring(aString); + if (!t) { + PyTuple_SetItem(v, i, Py_None); + Py_INCREF(Py_None); + } else { + PyTuple_SetItem(v, i, t); + } + } + } + } + + CFRelease(proxyDict); + return result; + +error: + if (proxyDict) CFRelease(proxyDict); + Py_XDECREF(result); + return NULL; +} + +static int +set_proxy(PyObject* proxies, char* proto, CFDictionaryRef proxyDict, + CFStringRef enabledKey, + CFStringRef hostKey, CFStringRef portKey) +{ + CFNumberRef aNum; + + aNum = CFDictionaryGetValue(proxyDict, enabledKey); + if (aNum && cfnum_to_int32(aNum)) { + CFStringRef hostString; + + hostString = CFDictionaryGetValue(proxyDict, hostKey); + aNum = CFDictionaryGetValue(proxyDict, portKey); + + if (hostString) { + int r; + PyObject* h = cfstring_to_pystring(hostString); + PyObject* v; + if (h) { + if (aNum) { + int32_t port = cfnum_to_int32(aNum); + v = PyString_FromFormat("http://%s:%ld", + PyString_AS_STRING(h), + (long)port); + } else { + v = PyString_FromFormat("http://%s", + PyString_AS_STRING(h)); + } + Py_DECREF(h); + if (!v) return -1; + r = PyDict_SetItemString(proxies, proto, + v); + Py_DECREF(v); + return r; + } + } + + } + return 0; +} + + + +static PyObject* +get_proxies(PyObject* mod __attribute__((__unused__))) +{ + PyObject* result = NULL; + int r; + CFDictionaryRef proxyDict = NULL; + + proxyDict = SCDynamicStoreCopyProxies(NULL); + if (proxyDict == NULL) { + return PyDict_New(); + } + + result = PyDict_New(); + if (result == NULL) goto error; + + r = set_proxy(result, "http", proxyDict, + kSCPropNetProxiesHTTPEnable, + kSCPropNetProxiesHTTPProxy, + kSCPropNetProxiesHTTPPort); + if (r == -1) goto error; + r = set_proxy(result, "https", proxyDict, + kSCPropNetProxiesHTTPSEnable, + kSCPropNetProxiesHTTPSProxy, + kSCPropNetProxiesHTTPSPort); + if (r == -1) goto error; + r = set_proxy(result, "ftp", proxyDict, + kSCPropNetProxiesFTPEnable, + kSCPropNetProxiesFTPProxy, + kSCPropNetProxiesFTPPort); + if (r == -1) goto error; + r = set_proxy(result, "gopher", proxyDict, + kSCPropNetProxiesGopherEnable, + kSCPropNetProxiesGopherProxy, + kSCPropNetProxiesGopherPort); + if (r == -1) goto error; + + CFRelease(proxyDict); + return result; +error: + if (proxyDict) CFRelease(proxyDict); + Py_XDECREF(result); + return NULL; +} + +static PyMethodDef mod_methods[] = { + { + "_get_proxy_settings", + (PyCFunction)get_proxy_settings, + METH_NOARGS, + NULL, + }, + { + "_get_proxies", + (PyCFunction)get_proxies, + METH_NOARGS, + NULL, + }, + { 0, 0, 0, 0 } +}; + +void init_scproxy(void) +{ + (void)Py_InitModule4("_scproxy", mod_methods, NULL, NULL, PYTHON_API_VERSION); +} diff --git a/Misc/NEWS b/Misc/NEWS index cac78e1c947..bdd121cb8d3 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -379,6 +379,8 @@ Core and Builtins Library ------- +- Issue #6851: Fix urllib.urlopen crash on secondairy threads on OSX 10.6 + - Issue #4606: Passing 'None' if ctypes argtype is set to POINTER(...) does now always result in NULL. diff --git a/setup.py b/setup.py index e12869721d3..adc72c894ed 100644 --- a/setup.py +++ b/setup.py @@ -1400,6 +1400,17 @@ class PyBuildExt(build_ext): addMacExtension('_CF', core_kwds, ['cf/pycfbridge.c']) addMacExtension('autoGIL', core_kwds) + # _scproxy + sc_kwds = { + 'extra_compile_args': carbon_extra_compile_args, + 'extra_link_args': [ + '-framework', 'SystemConfiguration', + '-framework', 'CoreFoundation' + ], + } + addMacExtension("_scproxy", sc_kwds) + + # Carbon carbon_kwds = {'extra_compile_args': carbon_extra_compile_args, 'extra_link_args': ['-framework', 'Carbon'],