Fix for issue 6851: urllib.urlopen crashes in a thread on OSX 10.6
This commit is contained in:
Ronald Oussoren
parent
4b017bbac0
commit
51f0633efd
206
Lib/urllib.py
206
Lib/urllib.py
|
|
@ -1327,38 +1327,7 @@ def proxy_bypass_environment(host):
|
|||
|
||||
|
||||
if sys.platform == 'darwin':
|
||||
|
||||
def _CFSetup(sc):
|
||||
from ctypes import c_int32, c_void_p, c_char_p, c_int
|
||||
sc.CFStringCreateWithCString.argtypes = [ c_void_p, c_char_p, c_int32 ]
|
||||
sc.CFStringCreateWithCString.restype = c_void_p
|
||||
sc.SCDynamicStoreCopyProxies.argtypes = [ c_void_p ]
|
||||
sc.SCDynamicStoreCopyProxies.restype = c_void_p
|
||||
sc.CFDictionaryGetValue.argtypes = [ c_void_p, c_void_p ]
|
||||
sc.CFDictionaryGetValue.restype = c_void_p
|
||||
sc.CFStringGetLength.argtypes = [ c_void_p ]
|
||||
sc.CFStringGetLength.restype = c_int32
|
||||
sc.CFStringGetCString.argtypes = [ c_void_p, c_char_p, c_int32, c_int32 ]
|
||||
sc.CFStringGetCString.restype = c_int32
|
||||
sc.CFNumberGetValue.argtypes = [ c_void_p, c_int, c_void_p ]
|
||||
sc.CFNumberGetValue.restype = c_int32
|
||||
sc.CFRelease.argtypes = [ c_void_p ]
|
||||
sc.CFRelease.restype = None
|
||||
|
||||
def _CStringFromCFString(sc, value):
|
||||
from ctypes import create_string_buffer
|
||||
length = sc.CFStringGetLength(value) + 1
|
||||
buff = create_string_buffer(length)
|
||||
sc.CFStringGetCString(value, buff, length, 0)
|
||||
return buff.value
|
||||
|
||||
def _CFNumberToInt32(sc, cfnum):
|
||||
from ctypes import byref, c_int
|
||||
val = c_int()
|
||||
kCFNumberSInt32Type = 3
|
||||
sc.CFNumberGetValue(cfnum, kCFNumberSInt32Type, byref(val))
|
||||
return val.value
|
||||
|
||||
from _scproxy import _get_proxy_settings, _get_proxies
|
||||
|
||||
def proxy_bypass_macosx_sysconf(host):
|
||||
"""
|
||||
|
|
@ -1367,8 +1336,6 @@ if sys.platform == 'darwin':
|
|||
This function uses the MacOSX framework SystemConfiguration
|
||||
to fetch the proxy information.
|
||||
"""
|
||||
from ctypes import cdll
|
||||
from ctypes.util import find_library
|
||||
import re
|
||||
import socket
|
||||
from fnmatch import fnmatch
|
||||
|
|
@ -1380,63 +1347,35 @@ if sys.platform == 'darwin':
|
|||
parts = (parts + [0, 0, 0, 0])[:4]
|
||||
return (parts[0] << 24) | (parts[1] << 16) | (parts[2] << 8) | parts[3]
|
||||
|
||||
sc = cdll.LoadLibrary(find_library("SystemConfiguration"))
|
||||
_CFSetup(sc)
|
||||
proxy_settings = _get_proxy_settings()
|
||||
|
||||
hostIP = None
|
||||
# Check for simple host names:
|
||||
if '.' not in host:
|
||||
if proxy_settings['exclude_simple']:
|
||||
return True
|
||||
|
||||
if not sc:
|
||||
return False
|
||||
for value in proxy_settings.get('exceptions'):
|
||||
# Items in the list are strings like these: *.local, 169.254/16
|
||||
value = sc.CFArrayGetValueAtIndex(exceptions, index)
|
||||
if not value: continue
|
||||
|
||||
kSCPropNetProxiesExceptionsList = sc.CFStringCreateWithCString(0, "ExceptionsList", 0)
|
||||
kSCPropNetProxiesExcludeSimpleHostnames = sc.CFStringCreateWithCString(0,
|
||||
"ExcludeSimpleHostnames", 0)
|
||||
m = re.match(r"(\d+(?:\.\d+)*)(/\d+)?", value)
|
||||
if m is not None:
|
||||
if hostIP is None:
|
||||
hostIP = socket.gethostbyname(host)
|
||||
hostIP = ip2num(hostIP)
|
||||
|
||||
base = ip2num(m.group(1))
|
||||
mask = int(m.group(2)[1:])
|
||||
mask = 32 - mask
|
||||
|
||||
proxyDict = sc.SCDynamicStoreCopyProxies(None)
|
||||
if proxyDict is None:
|
||||
return False
|
||||
|
||||
try:
|
||||
# Check for simple host names:
|
||||
if '.' not in host:
|
||||
exclude_simple = sc.CFDictionaryGetValue(proxyDict,
|
||||
kSCPropNetProxiesExcludeSimpleHostnames)
|
||||
if exclude_simple and _CFNumberToInt32(sc, exclude_simple):
|
||||
if (hostIP >> mask) == (base >> mask):
|
||||
return True
|
||||
|
||||
elif fnmatch(host, value):
|
||||
return True
|
||||
|
||||
# Check the exceptions list:
|
||||
exceptions = sc.CFDictionaryGetValue(proxyDict, kSCPropNetProxiesExceptionsList)
|
||||
if exceptions:
|
||||
# Items in the list are strings like these: *.local, 169.254/16
|
||||
for index in xrange(sc.CFArrayGetCount(exceptions)):
|
||||
value = sc.CFArrayGetValueAtIndex(exceptions, index)
|
||||
if not value: continue
|
||||
value = _CStringFromCFString(sc, value)
|
||||
|
||||
m = re.match(r"(\d+(?:\.\d+)*)(/\d+)?", value)
|
||||
if m is not None:
|
||||
if hostIP is None:
|
||||
hostIP = socket.gethostbyname(host)
|
||||
hostIP = ip2num(hostIP)
|
||||
|
||||
base = ip2num(m.group(1))
|
||||
mask = int(m.group(2)[1:])
|
||||
mask = 32 - mask
|
||||
|
||||
if (hostIP >> mask) == (base >> mask):
|
||||
return True
|
||||
|
||||
elif fnmatch(host, value):
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
finally:
|
||||
sc.CFRelease(kSCPropNetProxiesExceptionsList)
|
||||
sc.CFRelease(kSCPropNetProxiesExcludeSimpleHostnames)
|
||||
|
||||
return False
|
||||
|
||||
|
||||
def getproxies_macosx_sysconf():
|
||||
|
|
@ -1445,106 +1384,7 @@ if sys.platform == 'darwin':
|
|||
This function uses the MacOSX framework SystemConfiguration
|
||||
to fetch the proxy information.
|
||||
"""
|
||||
from ctypes import cdll
|
||||
from ctypes.util import find_library
|
||||
|
||||
sc = cdll.LoadLibrary(find_library("SystemConfiguration"))
|
||||
_CFSetup(sc)
|
||||
|
||||
if not sc:
|
||||
return {}
|
||||
|
||||
kSCPropNetProxiesHTTPEnable = sc.CFStringCreateWithCString(0, "HTTPEnable", 0)
|
||||
kSCPropNetProxiesHTTPProxy = sc.CFStringCreateWithCString(0, "HTTPProxy", 0)
|
||||
kSCPropNetProxiesHTTPPort = sc.CFStringCreateWithCString(0, "HTTPPort", 0)
|
||||
|
||||
kSCPropNetProxiesHTTPSEnable = sc.CFStringCreateWithCString(0, "HTTPSEnable", 0)
|
||||
kSCPropNetProxiesHTTPSProxy = sc.CFStringCreateWithCString(0, "HTTPSProxy", 0)
|
||||
kSCPropNetProxiesHTTPSPort = sc.CFStringCreateWithCString(0, "HTTPSPort", 0)
|
||||
|
||||
kSCPropNetProxiesFTPEnable = sc.CFStringCreateWithCString(0, "FTPEnable", 0)
|
||||
kSCPropNetProxiesFTPPassive = sc.CFStringCreateWithCString(0, "FTPPassive", 0)
|
||||
kSCPropNetProxiesFTPPort = sc.CFStringCreateWithCString(0, "FTPPort", 0)
|
||||
kSCPropNetProxiesFTPProxy = sc.CFStringCreateWithCString(0, "FTPProxy", 0)
|
||||
|
||||
kSCPropNetProxiesGopherEnable = sc.CFStringCreateWithCString(0, "GopherEnable", 0)
|
||||
kSCPropNetProxiesGopherPort = sc.CFStringCreateWithCString(0, "GopherPort", 0)
|
||||
kSCPropNetProxiesGopherProxy = sc.CFStringCreateWithCString(0, "GopherProxy", 0)
|
||||
|
||||
proxies = {}
|
||||
proxyDict = sc.SCDynamicStoreCopyProxies(None)
|
||||
|
||||
try:
|
||||
# HTTP:
|
||||
enabled = sc.CFDictionaryGetValue(proxyDict, kSCPropNetProxiesHTTPEnable)
|
||||
if enabled and _CFNumberToInt32(sc, enabled):
|
||||
proxy = sc.CFDictionaryGetValue(proxyDict, kSCPropNetProxiesHTTPProxy)
|
||||
port = sc.CFDictionaryGetValue(proxyDict, kSCPropNetProxiesHTTPPort)
|
||||
|
||||
if proxy:
|
||||
proxy = _CStringFromCFString(sc, proxy)
|
||||
if port:
|
||||
port = _CFNumberToInt32(sc, port)
|
||||
proxies["http"] = "http://%s:%i" % (proxy, port)
|
||||
else:
|
||||
proxies["http"] = "http://%s" % (proxy, )
|
||||
|
||||
# HTTPS:
|
||||
enabled = sc.CFDictionaryGetValue(proxyDict, kSCPropNetProxiesHTTPSEnable)
|
||||
if enabled and _CFNumberToInt32(sc, enabled):
|
||||
proxy = sc.CFDictionaryGetValue(proxyDict, kSCPropNetProxiesHTTPSProxy)
|
||||
port = sc.CFDictionaryGetValue(proxyDict, kSCPropNetProxiesHTTPSPort)
|
||||
|
||||
if proxy:
|
||||
proxy = _CStringFromCFString(sc, proxy)
|
||||
if port:
|
||||
port = _CFNumberToInt32(sc, port)
|
||||
proxies["https"] = "http://%s:%i" % (proxy, port)
|
||||
else:
|
||||
proxies["https"] = "http://%s" % (proxy, )
|
||||
|
||||
# FTP:
|
||||
enabled = sc.CFDictionaryGetValue(proxyDict, kSCPropNetProxiesFTPEnable)
|
||||
if enabled and _CFNumberToInt32(sc, enabled):
|
||||
proxy = sc.CFDictionaryGetValue(proxyDict, kSCPropNetProxiesFTPProxy)
|
||||
port = sc.CFDictionaryGetValue(proxyDict, kSCPropNetProxiesFTPPort)
|
||||
|
||||
if proxy:
|
||||
proxy = _CStringFromCFString(sc, proxy)
|
||||
if port:
|
||||
port = _CFNumberToInt32(sc, port)
|
||||
proxies["ftp"] = "http://%s:%i" % (proxy, port)
|
||||
else:
|
||||
proxies["ftp"] = "http://%s" % (proxy, )
|
||||
|
||||
# Gopher:
|
||||
enabled = sc.CFDictionaryGetValue(proxyDict, kSCPropNetProxiesGopherEnable)
|
||||
if enabled and _CFNumberToInt32(sc, enabled):
|
||||
proxy = sc.CFDictionaryGetValue(proxyDict, kSCPropNetProxiesGopherProxy)
|
||||
port = sc.CFDictionaryGetValue(proxyDict, kSCPropNetProxiesGopherPort)
|
||||
|
||||
if proxy:
|
||||
proxy = _CStringFromCFString(sc, proxy)
|
||||
if port:
|
||||
port = _CFNumberToInt32(sc, port)
|
||||
proxies["gopher"] = "http://%s:%i" % (proxy, port)
|
||||
else:
|
||||
proxies["gopher"] = "http://%s" % (proxy, )
|
||||
finally:
|
||||
sc.CFRelease(proxyDict)
|
||||
|
||||
sc.CFRelease(kSCPropNetProxiesHTTPEnable)
|
||||
sc.CFRelease(kSCPropNetProxiesHTTPProxy)
|
||||
sc.CFRelease(kSCPropNetProxiesHTTPPort)
|
||||
sc.CFRelease(kSCPropNetProxiesFTPEnable)
|
||||
sc.CFRelease(kSCPropNetProxiesFTPPassive)
|
||||
sc.CFRelease(kSCPropNetProxiesFTPPort)
|
||||
sc.CFRelease(kSCPropNetProxiesFTPProxy)
|
||||
sc.CFRelease(kSCPropNetProxiesGopherEnable)
|
||||
sc.CFRelease(kSCPropNetProxiesGopherPort)
|
||||
sc.CFRelease(kSCPropNetProxiesGopherProxy)
|
||||
|
||||
return proxies
|
||||
return _get_proxies()
|
||||
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,226 @@
|
|||
/*
|
||||
* Helper method for urllib to fetch the proxy configuration settings
|
||||
* using the SystemConfiguration framework.
|
||||
*/
|
||||
#include <Python.h>
|
||||
#include <SystemConfiguration/SystemConfiguration.h>
|
||||
|
||||
static int32_t
|
||||
cfnum_to_int32(CFNumberRef num)
|
||||
{
|
||||
int32_t result;
|
||||
|
||||
CFNumberGetValue(num, kCFNumberSInt32Type, &result);
|
||||
return result;
|
||||
}
|
||||
|
||||
static PyObject*
|
||||
cfstring_to_pystring(CFStringRef ref)
|
||||
{
|
||||
const char* s;
|
||||
|
||||
s = CFStringGetCStringPtr(ref, kCFStringEncodingUTF8);
|
||||
if (s) {
|
||||
return PyString_FromString(s);
|
||||
|
||||
} else {
|
||||
CFIndex len = CFStringGetLength(ref);
|
||||
Boolean ok;
|
||||
PyObject* result;
|
||||
result = PyString_FromStringAndSize(NULL, len*4);
|
||||
|
||||
ok = CFStringGetCString(ref,
|
||||
PyString_AS_STRING(result),
|
||||
PyString_GET_SIZE(result),
|
||||
kCFStringEncodingUTF8);
|
||||
if (!ok) {
|
||||
Py_DECREF(result);
|
||||
return NULL;
|
||||
} else {
|
||||
_PyString_Resize(&result,
|
||||
strlen(PyString_AS_STRING(result)));
|
||||
}
|
||||
return result;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
static PyObject*
|
||||
get_proxy_settings(PyObject* mod __attribute__((__unused__)))
|
||||
{
|
||||
CFDictionaryRef proxyDict = NULL;
|
||||
CFNumberRef aNum = NULL;
|
||||
CFArrayRef anArray = NULL;
|
||||
PyObject* result = NULL;
|
||||
PyObject* v;
|
||||
int r;
|
||||
|
||||
proxyDict = SCDynamicStoreCopyProxies(NULL);
|
||||
if (!proxyDict) {
|
||||
Py_INCREF(Py_None);
|
||||
return Py_None;
|
||||
}
|
||||
|
||||
result = PyDict_New();
|
||||
if (result == NULL) goto error;
|
||||
|
||||
aNum = CFDictionaryGetValue(proxyDict,
|
||||
kSCPropNetProxiesExcludeSimpleHostnames);
|
||||
if (aNum == NULL) {
|
||||
v = PyBool_FromLong(0);
|
||||
} else {
|
||||
v = PyBool_FromLong(cfnum_to_int32(aNum));
|
||||
}
|
||||
if (v == NULL) goto error;
|
||||
|
||||
r = PyDict_SetItemString(result, "exclude_simple", v);
|
||||
Py_DECREF(v); v = NULL;
|
||||
if (r == -1) goto error;
|
||||
|
||||
anArray = CFDictionaryGetValue(proxyDict,
|
||||
kSCPropNetProxiesExceptionsList);
|
||||
if (anArray != NULL) {
|
||||
CFIndex len = CFArrayGetCount(anArray);
|
||||
CFIndex i;
|
||||
v = PyTuple_New(len);
|
||||
if (v == NULL) goto error;
|
||||
|
||||
r = PyDict_SetItemString(result, "exceptions", v);
|
||||
Py_DECREF(v);
|
||||
if (r == -1) goto error;
|
||||
|
||||
for (i = 0; i < len; i++) {
|
||||
CFStringRef aString = NULL;
|
||||
|
||||
aString = CFArrayGetValueAtIndex(anArray, i);
|
||||
if (aString == NULL) {
|
||||
PyTuple_SetItem(v, i, Py_None);
|
||||
Py_INCREF(Py_None);
|
||||
} else {
|
||||
PyObject* t = cfstring_to_pystring(aString);
|
||||
if (!t) {
|
||||
PyTuple_SetItem(v, i, Py_None);
|
||||
Py_INCREF(Py_None);
|
||||
} else {
|
||||
PyTuple_SetItem(v, i, t);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
CFRelease(proxyDict);
|
||||
return result;
|
||||
|
||||
error:
|
||||
if (proxyDict) CFRelease(proxyDict);
|
||||
Py_XDECREF(result);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
static int
|
||||
set_proxy(PyObject* proxies, char* proto, CFDictionaryRef proxyDict,
|
||||
CFStringRef enabledKey,
|
||||
CFStringRef hostKey, CFStringRef portKey)
|
||||
{
|
||||
CFNumberRef aNum;
|
||||
|
||||
aNum = CFDictionaryGetValue(proxyDict, enabledKey);
|
||||
if (aNum && cfnum_to_int32(aNum)) {
|
||||
CFStringRef hostString;
|
||||
|
||||
hostString = CFDictionaryGetValue(proxyDict, hostKey);
|
||||
aNum = CFDictionaryGetValue(proxyDict, portKey);
|
||||
|
||||
if (hostString) {
|
||||
int r;
|
||||
PyObject* h = cfstring_to_pystring(hostString);
|
||||
PyObject* v;
|
||||
if (h) {
|
||||
if (aNum) {
|
||||
int32_t port = cfnum_to_int32(aNum);
|
||||
v = PyString_FromFormat("http://%s:%ld",
|
||||
PyString_AS_STRING(h),
|
||||
(long)port);
|
||||
} else {
|
||||
v = PyString_FromFormat("http://%s",
|
||||
PyString_AS_STRING(h));
|
||||
}
|
||||
Py_DECREF(h);
|
||||
if (!v) return -1;
|
||||
r = PyDict_SetItemString(proxies, proto,
|
||||
v);
|
||||
Py_DECREF(v);
|
||||
return r;
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
|
||||
static PyObject*
|
||||
get_proxies(PyObject* mod __attribute__((__unused__)))
|
||||
{
|
||||
PyObject* result = NULL;
|
||||
int r;
|
||||
CFDictionaryRef proxyDict = NULL;
|
||||
|
||||
proxyDict = SCDynamicStoreCopyProxies(NULL);
|
||||
if (proxyDict == NULL) {
|
||||
return PyDict_New();
|
||||
}
|
||||
|
||||
result = PyDict_New();
|
||||
if (result == NULL) goto error;
|
||||
|
||||
r = set_proxy(result, "http", proxyDict,
|
||||
kSCPropNetProxiesHTTPEnable,
|
||||
kSCPropNetProxiesHTTPProxy,
|
||||
kSCPropNetProxiesHTTPPort);
|
||||
if (r == -1) goto error;
|
||||
r = set_proxy(result, "https", proxyDict,
|
||||
kSCPropNetProxiesHTTPSEnable,
|
||||
kSCPropNetProxiesHTTPSProxy,
|
||||
kSCPropNetProxiesHTTPSPort);
|
||||
if (r == -1) goto error;
|
||||
r = set_proxy(result, "ftp", proxyDict,
|
||||
kSCPropNetProxiesFTPEnable,
|
||||
kSCPropNetProxiesFTPProxy,
|
||||
kSCPropNetProxiesFTPPort);
|
||||
if (r == -1) goto error;
|
||||
r = set_proxy(result, "gopher", proxyDict,
|
||||
kSCPropNetProxiesGopherEnable,
|
||||
kSCPropNetProxiesGopherProxy,
|
||||
kSCPropNetProxiesGopherPort);
|
||||
if (r == -1) goto error;
|
||||
|
||||
CFRelease(proxyDict);
|
||||
return result;
|
||||
error:
|
||||
if (proxyDict) CFRelease(proxyDict);
|
||||
Py_XDECREF(result);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
static PyMethodDef mod_methods[] = {
|
||||
{
|
||||
"_get_proxy_settings",
|
||||
(PyCFunction)get_proxy_settings,
|
||||
METH_NOARGS,
|
||||
NULL,
|
||||
},
|
||||
{
|
||||
"_get_proxies",
|
||||
(PyCFunction)get_proxies,
|
||||
METH_NOARGS,
|
||||
NULL,
|
||||
},
|
||||
{ 0, 0, 0, 0 }
|
||||
};
|
||||
|
||||
void init_scproxy(void)
|
||||
{
|
||||
(void)Py_InitModule4("_scproxy", mod_methods, NULL, NULL, PYTHON_API_VERSION);
|
||||
}
|
||||
|
|
@ -379,6 +379,8 @@ Core and Builtins
|
|||
Library
|
||||
-------
|
||||
|
||||
- Issue #6851: Fix urllib.urlopen crash on secondairy threads on OSX 10.6
|
||||
|
||||
- Issue #4606: Passing 'None' if ctypes argtype is set to POINTER(...)
|
||||
does now always result in NULL.
|
||||
|
||||
|
|
|
|||
11
setup.py
11
setup.py
|
|
@ -1400,6 +1400,17 @@ class PyBuildExt(build_ext):
|
|||
addMacExtension('_CF', core_kwds, ['cf/pycfbridge.c'])
|
||||
addMacExtension('autoGIL', core_kwds)
|
||||
|
||||
# _scproxy
|
||||
sc_kwds = {
|
||||
'extra_compile_args': carbon_extra_compile_args,
|
||||
'extra_link_args': [
|
||||
'-framework', 'SystemConfiguration',
|
||||
'-framework', 'CoreFoundation'
|
||||
],
|
||||
}
|
||||
addMacExtension("_scproxy", sc_kwds)
|
||||
|
||||
|
||||
# Carbon
|
||||
carbon_kwds = {'extra_compile_args': carbon_extra_compile_args,
|
||||
'extra_link_args': ['-framework', 'Carbon'],
|
||||
|
|
|
|||
Loading…
Reference in New Issue