Fix for issue 6851: urllib.urlopen crashes in a thread on OSX 10.6

This commit is contained in:
Ronald Oussoren 2009-09-20 10:31:22 +00:00
parent 4b017bbac0
commit 51f0633efd
4 changed files with 262 additions and 183 deletions

View File

@ -1327,38 +1327,7 @@ def proxy_bypass_environment(host):
if sys.platform == 'darwin': if sys.platform == 'darwin':
from _scproxy import _get_proxy_settings, _get_proxies
def _CFSetup(sc):
from ctypes import c_int32, c_void_p, c_char_p, c_int
sc.CFStringCreateWithCString.argtypes = [ c_void_p, c_char_p, c_int32 ]
sc.CFStringCreateWithCString.restype = c_void_p
sc.SCDynamicStoreCopyProxies.argtypes = [ c_void_p ]
sc.SCDynamicStoreCopyProxies.restype = c_void_p
sc.CFDictionaryGetValue.argtypes = [ c_void_p, c_void_p ]
sc.CFDictionaryGetValue.restype = c_void_p
sc.CFStringGetLength.argtypes = [ c_void_p ]
sc.CFStringGetLength.restype = c_int32
sc.CFStringGetCString.argtypes = [ c_void_p, c_char_p, c_int32, c_int32 ]
sc.CFStringGetCString.restype = c_int32
sc.CFNumberGetValue.argtypes = [ c_void_p, c_int, c_void_p ]
sc.CFNumberGetValue.restype = c_int32
sc.CFRelease.argtypes = [ c_void_p ]
sc.CFRelease.restype = None
def _CStringFromCFString(sc, value):
from ctypes import create_string_buffer
length = sc.CFStringGetLength(value) + 1
buff = create_string_buffer(length)
sc.CFStringGetCString(value, buff, length, 0)
return buff.value
def _CFNumberToInt32(sc, cfnum):
from ctypes import byref, c_int
val = c_int()
kCFNumberSInt32Type = 3
sc.CFNumberGetValue(cfnum, kCFNumberSInt32Type, byref(val))
return val.value
def proxy_bypass_macosx_sysconf(host): def proxy_bypass_macosx_sysconf(host):
""" """
@ -1367,8 +1336,6 @@ if sys.platform == 'darwin':
This function uses the MacOSX framework SystemConfiguration This function uses the MacOSX framework SystemConfiguration
to fetch the proxy information. to fetch the proxy information.
""" """
from ctypes import cdll
from ctypes.util import find_library
import re import re
import socket import socket
from fnmatch import fnmatch from fnmatch import fnmatch
@ -1380,40 +1347,17 @@ if sys.platform == 'darwin':
parts = (parts + [0, 0, 0, 0])[:4] parts = (parts + [0, 0, 0, 0])[:4]
return (parts[0] << 24) | (parts[1] << 16) | (parts[2] << 8) | parts[3] return (parts[0] << 24) | (parts[1] << 16) | (parts[2] << 8) | parts[3]
sc = cdll.LoadLibrary(find_library("SystemConfiguration")) proxy_settings = _get_proxy_settings()
_CFSetup(sc)
hostIP = None
if not sc:
return False
kSCPropNetProxiesExceptionsList = sc.CFStringCreateWithCString(0, "ExceptionsList", 0)
kSCPropNetProxiesExcludeSimpleHostnames = sc.CFStringCreateWithCString(0,
"ExcludeSimpleHostnames", 0)
proxyDict = sc.SCDynamicStoreCopyProxies(None)
if proxyDict is None:
return False
try:
# Check for simple host names: # Check for simple host names:
if '.' not in host: if '.' not in host:
exclude_simple = sc.CFDictionaryGetValue(proxyDict, if proxy_settings['exclude_simple']:
kSCPropNetProxiesExcludeSimpleHostnames)
if exclude_simple and _CFNumberToInt32(sc, exclude_simple):
return True return True
for value in proxy_settings.get('exceptions'):
# Check the exceptions list:
exceptions = sc.CFDictionaryGetValue(proxyDict, kSCPropNetProxiesExceptionsList)
if exceptions:
# Items in the list are strings like these: *.local, 169.254/16 # Items in the list are strings like these: *.local, 169.254/16
for index in xrange(sc.CFArrayGetCount(exceptions)):
value = sc.CFArrayGetValueAtIndex(exceptions, index) value = sc.CFArrayGetValueAtIndex(exceptions, index)
if not value: continue if not value: continue
value = _CStringFromCFString(sc, value)
m = re.match(r"(\d+(?:\.\d+)*)(/\d+)?", value) m = re.match(r"(\d+(?:\.\d+)*)(/\d+)?", value)
if m is not None: if m is not None:
@ -1433,11 +1377,6 @@ if sys.platform == 'darwin':
return False return False
finally:
sc.CFRelease(kSCPropNetProxiesExceptionsList)
sc.CFRelease(kSCPropNetProxiesExcludeSimpleHostnames)
def getproxies_macosx_sysconf(): def getproxies_macosx_sysconf():
"""Return a dictionary of scheme -> proxy server URL mappings. """Return a dictionary of scheme -> proxy server URL mappings.
@ -1445,106 +1384,7 @@ if sys.platform == 'darwin':
This function uses the MacOSX framework SystemConfiguration This function uses the MacOSX framework SystemConfiguration
to fetch the proxy information. to fetch the proxy information.
""" """
from ctypes import cdll return _get_proxies()
from ctypes.util import find_library
sc = cdll.LoadLibrary(find_library("SystemConfiguration"))
_CFSetup(sc)
if not sc:
return {}
kSCPropNetProxiesHTTPEnable = sc.CFStringCreateWithCString(0, "HTTPEnable", 0)
kSCPropNetProxiesHTTPProxy = sc.CFStringCreateWithCString(0, "HTTPProxy", 0)
kSCPropNetProxiesHTTPPort = sc.CFStringCreateWithCString(0, "HTTPPort", 0)
kSCPropNetProxiesHTTPSEnable = sc.CFStringCreateWithCString(0, "HTTPSEnable", 0)
kSCPropNetProxiesHTTPSProxy = sc.CFStringCreateWithCString(0, "HTTPSProxy", 0)
kSCPropNetProxiesHTTPSPort = sc.CFStringCreateWithCString(0, "HTTPSPort", 0)
kSCPropNetProxiesFTPEnable = sc.CFStringCreateWithCString(0, "FTPEnable", 0)
kSCPropNetProxiesFTPPassive = sc.CFStringCreateWithCString(0, "FTPPassive", 0)
kSCPropNetProxiesFTPPort = sc.CFStringCreateWithCString(0, "FTPPort", 0)
kSCPropNetProxiesFTPProxy = sc.CFStringCreateWithCString(0, "FTPProxy", 0)
kSCPropNetProxiesGopherEnable = sc.CFStringCreateWithCString(0, "GopherEnable", 0)
kSCPropNetProxiesGopherPort = sc.CFStringCreateWithCString(0, "GopherPort", 0)
kSCPropNetProxiesGopherProxy = sc.CFStringCreateWithCString(0, "GopherProxy", 0)
proxies = {}
proxyDict = sc.SCDynamicStoreCopyProxies(None)
try:
# HTTP:
enabled = sc.CFDictionaryGetValue(proxyDict, kSCPropNetProxiesHTTPEnable)
if enabled and _CFNumberToInt32(sc, enabled):
proxy = sc.CFDictionaryGetValue(proxyDict, kSCPropNetProxiesHTTPProxy)
port = sc.CFDictionaryGetValue(proxyDict, kSCPropNetProxiesHTTPPort)
if proxy:
proxy = _CStringFromCFString(sc, proxy)
if port:
port = _CFNumberToInt32(sc, port)
proxies["http"] = "http://%s:%i" % (proxy, port)
else:
proxies["http"] = "http://%s" % (proxy, )
# HTTPS:
enabled = sc.CFDictionaryGetValue(proxyDict, kSCPropNetProxiesHTTPSEnable)
if enabled and _CFNumberToInt32(sc, enabled):
proxy = sc.CFDictionaryGetValue(proxyDict, kSCPropNetProxiesHTTPSProxy)
port = sc.CFDictionaryGetValue(proxyDict, kSCPropNetProxiesHTTPSPort)
if proxy:
proxy = _CStringFromCFString(sc, proxy)
if port:
port = _CFNumberToInt32(sc, port)
proxies["https"] = "http://%s:%i" % (proxy, port)
else:
proxies["https"] = "http://%s" % (proxy, )
# FTP:
enabled = sc.CFDictionaryGetValue(proxyDict, kSCPropNetProxiesFTPEnable)
if enabled and _CFNumberToInt32(sc, enabled):
proxy = sc.CFDictionaryGetValue(proxyDict, kSCPropNetProxiesFTPProxy)
port = sc.CFDictionaryGetValue(proxyDict, kSCPropNetProxiesFTPPort)
if proxy:
proxy = _CStringFromCFString(sc, proxy)
if port:
port = _CFNumberToInt32(sc, port)
proxies["ftp"] = "http://%s:%i" % (proxy, port)
else:
proxies["ftp"] = "http://%s" % (proxy, )
# Gopher:
enabled = sc.CFDictionaryGetValue(proxyDict, kSCPropNetProxiesGopherEnable)
if enabled and _CFNumberToInt32(sc, enabled):
proxy = sc.CFDictionaryGetValue(proxyDict, kSCPropNetProxiesGopherProxy)
port = sc.CFDictionaryGetValue(proxyDict, kSCPropNetProxiesGopherPort)
if proxy:
proxy = _CStringFromCFString(sc, proxy)
if port:
port = _CFNumberToInt32(sc, port)
proxies["gopher"] = "http://%s:%i" % (proxy, port)
else:
proxies["gopher"] = "http://%s" % (proxy, )
finally:
sc.CFRelease(proxyDict)
sc.CFRelease(kSCPropNetProxiesHTTPEnable)
sc.CFRelease(kSCPropNetProxiesHTTPProxy)
sc.CFRelease(kSCPropNetProxiesHTTPPort)
sc.CFRelease(kSCPropNetProxiesFTPEnable)
sc.CFRelease(kSCPropNetProxiesFTPPassive)
sc.CFRelease(kSCPropNetProxiesFTPPort)
sc.CFRelease(kSCPropNetProxiesFTPProxy)
sc.CFRelease(kSCPropNetProxiesGopherEnable)
sc.CFRelease(kSCPropNetProxiesGopherPort)
sc.CFRelease(kSCPropNetProxiesGopherProxy)
return proxies

226
Mac/Modules/_scproxy.c Normal file
View File

@ -0,0 +1,226 @@
/*
* Helper method for urllib to fetch the proxy configuration settings
* using the SystemConfiguration framework.
*/
#include <Python.h>
#include <SystemConfiguration/SystemConfiguration.h>
static int32_t
cfnum_to_int32(CFNumberRef num)
{
int32_t result;
CFNumberGetValue(num, kCFNumberSInt32Type, &result);
return result;
}
static PyObject*
cfstring_to_pystring(CFStringRef ref)
{
const char* s;
s = CFStringGetCStringPtr(ref, kCFStringEncodingUTF8);
if (s) {
return PyString_FromString(s);
} else {
CFIndex len = CFStringGetLength(ref);
Boolean ok;
PyObject* result;
result = PyString_FromStringAndSize(NULL, len*4);
ok = CFStringGetCString(ref,
PyString_AS_STRING(result),
PyString_GET_SIZE(result),
kCFStringEncodingUTF8);
if (!ok) {
Py_DECREF(result);
return NULL;
} else {
_PyString_Resize(&result,
strlen(PyString_AS_STRING(result)));
}
return result;
}
}
static PyObject*
get_proxy_settings(PyObject* mod __attribute__((__unused__)))
{
CFDictionaryRef proxyDict = NULL;
CFNumberRef aNum = NULL;
CFArrayRef anArray = NULL;
PyObject* result = NULL;
PyObject* v;
int r;
proxyDict = SCDynamicStoreCopyProxies(NULL);
if (!proxyDict) {
Py_INCREF(Py_None);
return Py_None;
}
result = PyDict_New();
if (result == NULL) goto error;
aNum = CFDictionaryGetValue(proxyDict,
kSCPropNetProxiesExcludeSimpleHostnames);
if (aNum == NULL) {
v = PyBool_FromLong(0);
} else {
v = PyBool_FromLong(cfnum_to_int32(aNum));
}
if (v == NULL) goto error;
r = PyDict_SetItemString(result, "exclude_simple", v);
Py_DECREF(v); v = NULL;
if (r == -1) goto error;
anArray = CFDictionaryGetValue(proxyDict,
kSCPropNetProxiesExceptionsList);
if (anArray != NULL) {
CFIndex len = CFArrayGetCount(anArray);
CFIndex i;
v = PyTuple_New(len);
if (v == NULL) goto error;
r = PyDict_SetItemString(result, "exceptions", v);
Py_DECREF(v);
if (r == -1) goto error;
for (i = 0; i < len; i++) {
CFStringRef aString = NULL;
aString = CFArrayGetValueAtIndex(anArray, i);
if (aString == NULL) {
PyTuple_SetItem(v, i, Py_None);
Py_INCREF(Py_None);
} else {
PyObject* t = cfstring_to_pystring(aString);
if (!t) {
PyTuple_SetItem(v, i, Py_None);
Py_INCREF(Py_None);
} else {
PyTuple_SetItem(v, i, t);
}
}
}
}
CFRelease(proxyDict);
return result;
error:
if (proxyDict) CFRelease(proxyDict);
Py_XDECREF(result);
return NULL;
}
static int
set_proxy(PyObject* proxies, char* proto, CFDictionaryRef proxyDict,
CFStringRef enabledKey,
CFStringRef hostKey, CFStringRef portKey)
{
CFNumberRef aNum;
aNum = CFDictionaryGetValue(proxyDict, enabledKey);
if (aNum && cfnum_to_int32(aNum)) {
CFStringRef hostString;
hostString = CFDictionaryGetValue(proxyDict, hostKey);
aNum = CFDictionaryGetValue(proxyDict, portKey);
if (hostString) {
int r;
PyObject* h = cfstring_to_pystring(hostString);
PyObject* v;
if (h) {
if (aNum) {
int32_t port = cfnum_to_int32(aNum);
v = PyString_FromFormat("http://%s:%ld",
PyString_AS_STRING(h),
(long)port);
} else {
v = PyString_FromFormat("http://%s",
PyString_AS_STRING(h));
}
Py_DECREF(h);
if (!v) return -1;
r = PyDict_SetItemString(proxies, proto,
v);
Py_DECREF(v);
return r;
}
}
}
return 0;
}
static PyObject*
get_proxies(PyObject* mod __attribute__((__unused__)))
{
PyObject* result = NULL;
int r;
CFDictionaryRef proxyDict = NULL;
proxyDict = SCDynamicStoreCopyProxies(NULL);
if (proxyDict == NULL) {
return PyDict_New();
}
result = PyDict_New();
if (result == NULL) goto error;
r = set_proxy(result, "http", proxyDict,
kSCPropNetProxiesHTTPEnable,
kSCPropNetProxiesHTTPProxy,
kSCPropNetProxiesHTTPPort);
if (r == -1) goto error;
r = set_proxy(result, "https", proxyDict,
kSCPropNetProxiesHTTPSEnable,
kSCPropNetProxiesHTTPSProxy,
kSCPropNetProxiesHTTPSPort);
if (r == -1) goto error;
r = set_proxy(result, "ftp", proxyDict,
kSCPropNetProxiesFTPEnable,
kSCPropNetProxiesFTPProxy,
kSCPropNetProxiesFTPPort);
if (r == -1) goto error;
r = set_proxy(result, "gopher", proxyDict,
kSCPropNetProxiesGopherEnable,
kSCPropNetProxiesGopherProxy,
kSCPropNetProxiesGopherPort);
if (r == -1) goto error;
CFRelease(proxyDict);
return result;
error:
if (proxyDict) CFRelease(proxyDict);
Py_XDECREF(result);
return NULL;
}
static PyMethodDef mod_methods[] = {
{
"_get_proxy_settings",
(PyCFunction)get_proxy_settings,
METH_NOARGS,
NULL,
},
{
"_get_proxies",
(PyCFunction)get_proxies,
METH_NOARGS,
NULL,
},
{ 0, 0, 0, 0 }
};
void init_scproxy(void)
{
(void)Py_InitModule4("_scproxy", mod_methods, NULL, NULL, PYTHON_API_VERSION);
}

View File

@ -379,6 +379,8 @@ Core and Builtins
Library Library
------- -------
- Issue #6851: Fix urllib.urlopen crash on secondairy threads on OSX 10.6
- Issue #4606: Passing 'None' if ctypes argtype is set to POINTER(...) - Issue #4606: Passing 'None' if ctypes argtype is set to POINTER(...)
does now always result in NULL. does now always result in NULL.

View File

@ -1400,6 +1400,17 @@ class PyBuildExt(build_ext):
addMacExtension('_CF', core_kwds, ['cf/pycfbridge.c']) addMacExtension('_CF', core_kwds, ['cf/pycfbridge.c'])
addMacExtension('autoGIL', core_kwds) addMacExtension('autoGIL', core_kwds)
# _scproxy
sc_kwds = {
'extra_compile_args': carbon_extra_compile_args,
'extra_link_args': [
'-framework', 'SystemConfiguration',
'-framework', 'CoreFoundation'
],
}
addMacExtension("_scproxy", sc_kwds)
# Carbon # Carbon
carbon_kwds = {'extra_compile_args': carbon_extra_compile_args, carbon_kwds = {'extra_compile_args': carbon_extra_compile_args,
'extra_link_args': ['-framework', 'Carbon'], 'extra_link_args': ['-framework', 'Carbon'],