[Python-checkins] r74962 - in python/trunk: Lib/urllib.py Mac/Modules/_scproxy.c Misc/NEWS setup.py
ronald.oussoren
python-checkins at python.org
Sun Sep 20 12:31:22 CEST 2009
Author: ronald.oussoren
Date: Sun Sep 20 12:31:22 2009
New Revision: 74962
Log:
Fix for issue 6851: urllib.urlopen crashes in a thread on OSX 10.6
Added:
python/trunk/Mac/Modules/_scproxy.c (contents, props changed)
Modified:
python/trunk/Lib/urllib.py
python/trunk/Misc/NEWS
python/trunk/setup.py
Modified: python/trunk/Lib/urllib.py
==============================================================================
--- python/trunk/Lib/urllib.py (original)
+++ python/trunk/Lib/urllib.py Sun Sep 20 12:31:22 2009
@@ -1327,38 +1327,7 @@
if sys.platform == 'darwin':
-
- def _CFSetup(sc):
- from ctypes import c_int32, c_void_p, c_char_p, c_int
- sc.CFStringCreateWithCString.argtypes = [ c_void_p, c_char_p, c_int32 ]
- sc.CFStringCreateWithCString.restype = c_void_p
- sc.SCDynamicStoreCopyProxies.argtypes = [ c_void_p ]
- sc.SCDynamicStoreCopyProxies.restype = c_void_p
- sc.CFDictionaryGetValue.argtypes = [ c_void_p, c_void_p ]
- sc.CFDictionaryGetValue.restype = c_void_p
- sc.CFStringGetLength.argtypes = [ c_void_p ]
- sc.CFStringGetLength.restype = c_int32
- sc.CFStringGetCString.argtypes = [ c_void_p, c_char_p, c_int32, c_int32 ]
- sc.CFStringGetCString.restype = c_int32
- sc.CFNumberGetValue.argtypes = [ c_void_p, c_int, c_void_p ]
- sc.CFNumberGetValue.restype = c_int32
- sc.CFRelease.argtypes = [ c_void_p ]
- sc.CFRelease.restype = None
-
- def _CStringFromCFString(sc, value):
- from ctypes import create_string_buffer
- length = sc.CFStringGetLength(value) + 1
- buff = create_string_buffer(length)
- sc.CFStringGetCString(value, buff, length, 0)
- return buff.value
-
- def _CFNumberToInt32(sc, cfnum):
- from ctypes import byref, c_int
- val = c_int()
- kCFNumberSInt32Type = 3
- sc.CFNumberGetValue(cfnum, kCFNumberSInt32Type, byref(val))
- return val.value
-
+ from _scproxy import _get_proxy_settings, _get_proxies
def proxy_bypass_macosx_sysconf(host):
"""
@@ -1367,8 +1336,6 @@
This function uses the MacOSX framework SystemConfiguration
to fetch the proxy information.
"""
- from ctypes import cdll
- from ctypes.util import find_library
import re
import socket
from fnmatch import fnmatch
@@ -1380,63 +1347,35 @@
parts = (parts + [0, 0, 0, 0])[:4]
return (parts[0] << 24) | (parts[1] << 16) | (parts[2] << 8) | parts[3]
- sc = cdll.LoadLibrary(find_library("SystemConfiguration"))
- _CFSetup(sc)
-
- hostIP = None
-
- if not sc:
- return False
-
- kSCPropNetProxiesExceptionsList = sc.CFStringCreateWithCString(0, "ExceptionsList", 0)
- kSCPropNetProxiesExcludeSimpleHostnames = sc.CFStringCreateWithCString(0,
- "ExcludeSimpleHostnames", 0)
-
+ proxy_settings = _get_proxy_settings()
- proxyDict = sc.SCDynamicStoreCopyProxies(None)
- if proxyDict is None:
- return False
+ # Check for simple host names:
+ if '.' not in host:
+ if proxy_settings['exclude_simple']:
+ return True
+
+ for value in proxy_settings.get('exceptions'):
+ # Items in the list are strings like these: *.local, 169.254/16
+ value = sc.CFArrayGetValueAtIndex(exceptions, index)
+ if not value: continue
+
+ m = re.match(r"(\d+(?:\.\d+)*)(/\d+)?", value)
+ if m is not None:
+ if hostIP is None:
+ hostIP = socket.gethostbyname(host)
+ hostIP = ip2num(hostIP)
+
+ base = ip2num(m.group(1))
+ mask = int(m.group(2)[1:])
+ mask = 32 - mask
- try:
- # Check for simple host names:
- if '.' not in host:
- exclude_simple = sc.CFDictionaryGetValue(proxyDict,
- kSCPropNetProxiesExcludeSimpleHostnames)
- if exclude_simple and _CFNumberToInt32(sc, exclude_simple):
+ if (hostIP >> mask) == (base >> mask):
return True
+ elif fnmatch(host, value):
+ return True
- # Check the exceptions list:
- exceptions = sc.CFDictionaryGetValue(proxyDict, kSCPropNetProxiesExceptionsList)
- if exceptions:
- # Items in the list are strings like these: *.local, 169.254/16
- for index in xrange(sc.CFArrayGetCount(exceptions)):
- value = sc.CFArrayGetValueAtIndex(exceptions, index)
- if not value: continue
- value = _CStringFromCFString(sc, value)
-
- m = re.match(r"(\d+(?:\.\d+)*)(/\d+)?", value)
- if m is not None:
- if hostIP is None:
- hostIP = socket.gethostbyname(host)
- hostIP = ip2num(hostIP)
-
- base = ip2num(m.group(1))
- mask = int(m.group(2)[1:])
- mask = 32 - mask
-
- if (hostIP >> mask) == (base >> mask):
- return True
-
- elif fnmatch(host, value):
- return True
-
- return False
-
- finally:
- sc.CFRelease(kSCPropNetProxiesExceptionsList)
- sc.CFRelease(kSCPropNetProxiesExcludeSimpleHostnames)
-
+ return False
def getproxies_macosx_sysconf():
@@ -1445,106 +1384,7 @@
This function uses the MacOSX framework SystemConfiguration
to fetch the proxy information.
"""
- from ctypes import cdll
- from ctypes.util import find_library
-
- sc = cdll.LoadLibrary(find_library("SystemConfiguration"))
- _CFSetup(sc)
-
- if not sc:
- return {}
-
- kSCPropNetProxiesHTTPEnable = sc.CFStringCreateWithCString(0, "HTTPEnable", 0)
- kSCPropNetProxiesHTTPProxy = sc.CFStringCreateWithCString(0, "HTTPProxy", 0)
- kSCPropNetProxiesHTTPPort = sc.CFStringCreateWithCString(0, "HTTPPort", 0)
-
- kSCPropNetProxiesHTTPSEnable = sc.CFStringCreateWithCString(0, "HTTPSEnable", 0)
- kSCPropNetProxiesHTTPSProxy = sc.CFStringCreateWithCString(0, "HTTPSProxy", 0)
- kSCPropNetProxiesHTTPSPort = sc.CFStringCreateWithCString(0, "HTTPSPort", 0)
-
- kSCPropNetProxiesFTPEnable = sc.CFStringCreateWithCString(0, "FTPEnable", 0)
- kSCPropNetProxiesFTPPassive = sc.CFStringCreateWithCString(0, "FTPPassive", 0)
- kSCPropNetProxiesFTPPort = sc.CFStringCreateWithCString(0, "FTPPort", 0)
- kSCPropNetProxiesFTPProxy = sc.CFStringCreateWithCString(0, "FTPProxy", 0)
-
- kSCPropNetProxiesGopherEnable = sc.CFStringCreateWithCString(0, "GopherEnable", 0)
- kSCPropNetProxiesGopherPort = sc.CFStringCreateWithCString(0, "GopherPort", 0)
- kSCPropNetProxiesGopherProxy = sc.CFStringCreateWithCString(0, "GopherProxy", 0)
-
- proxies = {}
- proxyDict = sc.SCDynamicStoreCopyProxies(None)
-
- try:
- # HTTP:
- enabled = sc.CFDictionaryGetValue(proxyDict, kSCPropNetProxiesHTTPEnable)
- if enabled and _CFNumberToInt32(sc, enabled):
- proxy = sc.CFDictionaryGetValue(proxyDict, kSCPropNetProxiesHTTPProxy)
- port = sc.CFDictionaryGetValue(proxyDict, kSCPropNetProxiesHTTPPort)
-
- if proxy:
- proxy = _CStringFromCFString(sc, proxy)
- if port:
- port = _CFNumberToInt32(sc, port)
- proxies["http"] = "http://%s:%i" % (proxy, port)
- else:
- proxies["http"] = "http://%s" % (proxy, )
-
- # HTTPS:
- enabled = sc.CFDictionaryGetValue(proxyDict, kSCPropNetProxiesHTTPSEnable)
- if enabled and _CFNumberToInt32(sc, enabled):
- proxy = sc.CFDictionaryGetValue(proxyDict, kSCPropNetProxiesHTTPSProxy)
- port = sc.CFDictionaryGetValue(proxyDict, kSCPropNetProxiesHTTPSPort)
-
- if proxy:
- proxy = _CStringFromCFString(sc, proxy)
- if port:
- port = _CFNumberToInt32(sc, port)
- proxies["https"] = "http://%s:%i" % (proxy, port)
- else:
- proxies["https"] = "http://%s" % (proxy, )
-
- # FTP:
- enabled = sc.CFDictionaryGetValue(proxyDict, kSCPropNetProxiesFTPEnable)
- if enabled and _CFNumberToInt32(sc, enabled):
- proxy = sc.CFDictionaryGetValue(proxyDict, kSCPropNetProxiesFTPProxy)
- port = sc.CFDictionaryGetValue(proxyDict, kSCPropNetProxiesFTPPort)
-
- if proxy:
- proxy = _CStringFromCFString(sc, proxy)
- if port:
- port = _CFNumberToInt32(sc, port)
- proxies["ftp"] = "http://%s:%i" % (proxy, port)
- else:
- proxies["ftp"] = "http://%s" % (proxy, )
-
- # Gopher:
- enabled = sc.CFDictionaryGetValue(proxyDict, kSCPropNetProxiesGopherEnable)
- if enabled and _CFNumberToInt32(sc, enabled):
- proxy = sc.CFDictionaryGetValue(proxyDict, kSCPropNetProxiesGopherProxy)
- port = sc.CFDictionaryGetValue(proxyDict, kSCPropNetProxiesGopherPort)
-
- if proxy:
- proxy = _CStringFromCFString(sc, proxy)
- if port:
- port = _CFNumberToInt32(sc, port)
- proxies["gopher"] = "http://%s:%i" % (proxy, port)
- else:
- proxies["gopher"] = "http://%s" % (proxy, )
- finally:
- sc.CFRelease(proxyDict)
-
- sc.CFRelease(kSCPropNetProxiesHTTPEnable)
- sc.CFRelease(kSCPropNetProxiesHTTPProxy)
- sc.CFRelease(kSCPropNetProxiesHTTPPort)
- sc.CFRelease(kSCPropNetProxiesFTPEnable)
- sc.CFRelease(kSCPropNetProxiesFTPPassive)
- sc.CFRelease(kSCPropNetProxiesFTPPort)
- sc.CFRelease(kSCPropNetProxiesFTPProxy)
- sc.CFRelease(kSCPropNetProxiesGopherEnable)
- sc.CFRelease(kSCPropNetProxiesGopherPort)
- sc.CFRelease(kSCPropNetProxiesGopherProxy)
-
- return proxies
+ return _get_proxies()
Added: python/trunk/Mac/Modules/_scproxy.c
==============================================================================
--- (empty file)
+++ python/trunk/Mac/Modules/_scproxy.c Sun Sep 20 12:31:22 2009
@@ -0,0 +1,226 @@
+/*
+ * Helper method for urllib to fetch the proxy configuration settings
+ * using the SystemConfiguration framework.
+ */
+#include <Python.h>
+#include <SystemConfiguration/SystemConfiguration.h>
+
+static int32_t
+cfnum_to_int32(CFNumberRef num)
+{
+ int32_t result;
+
+ CFNumberGetValue(num, kCFNumberSInt32Type, &result);
+ return result;
+}
+
+static PyObject*
+cfstring_to_pystring(CFStringRef ref)
+{
+ const char* s;
+
+ s = CFStringGetCStringPtr(ref, kCFStringEncodingUTF8);
+ if (s) {
+ return PyString_FromString(s);
+
+ } else {
+ CFIndex len = CFStringGetLength(ref);
+ Boolean ok;
+ PyObject* result;
+ result = PyString_FromStringAndSize(NULL, len*4);
+
+ ok = CFStringGetCString(ref,
+ PyString_AS_STRING(result),
+ PyString_GET_SIZE(result),
+ kCFStringEncodingUTF8);
+ if (!ok) {
+ Py_DECREF(result);
+ return NULL;
+ } else {
+ _PyString_Resize(&result,
+ strlen(PyString_AS_STRING(result)));
+ }
+ return result;
+ }
+}
+
+
+static PyObject*
+get_proxy_settings(PyObject* mod __attribute__((__unused__)))
+{
+ CFDictionaryRef proxyDict = NULL;
+ CFNumberRef aNum = NULL;
+ CFArrayRef anArray = NULL;
+ PyObject* result = NULL;
+ PyObject* v;
+ int r;
+
+ proxyDict = SCDynamicStoreCopyProxies(NULL);
+ if (!proxyDict) {
+ Py_INCREF(Py_None);
+ return Py_None;
+ }
+
+ result = PyDict_New();
+ if (result == NULL) goto error;
+
+ aNum = CFDictionaryGetValue(proxyDict,
+ kSCPropNetProxiesExcludeSimpleHostnames);
+ if (aNum == NULL) {
+ v = PyBool_FromLong(0);
+ } else {
+ v = PyBool_FromLong(cfnum_to_int32(aNum));
+ }
+ if (v == NULL) goto error;
+
+ r = PyDict_SetItemString(result, "exclude_simple", v);
+ Py_DECREF(v); v = NULL;
+ if (r == -1) goto error;
+
+ anArray = CFDictionaryGetValue(proxyDict,
+ kSCPropNetProxiesExceptionsList);
+ if (anArray != NULL) {
+ CFIndex len = CFArrayGetCount(anArray);
+ CFIndex i;
+ v = PyTuple_New(len);
+ if (v == NULL) goto error;
+
+ r = PyDict_SetItemString(result, "exceptions", v);
+ Py_DECREF(v);
+ if (r == -1) goto error;
+
+ for (i = 0; i < len; i++) {
+ CFStringRef aString = NULL;
+
+ aString = CFArrayGetValueAtIndex(anArray, i);
+ if (aString == NULL) {
+ PyTuple_SetItem(v, i, Py_None);
+ Py_INCREF(Py_None);
+ } else {
+ PyObject* t = cfstring_to_pystring(aString);
+ if (!t) {
+ PyTuple_SetItem(v, i, Py_None);
+ Py_INCREF(Py_None);
+ } else {
+ PyTuple_SetItem(v, i, t);
+ }
+ }
+ }
+ }
+
+ CFRelease(proxyDict);
+ return result;
+
+error:
+ if (proxyDict) CFRelease(proxyDict);
+ Py_XDECREF(result);
+ return NULL;
+}
+
+static int
+set_proxy(PyObject* proxies, char* proto, CFDictionaryRef proxyDict,
+ CFStringRef enabledKey,
+ CFStringRef hostKey, CFStringRef portKey)
+{
+ CFNumberRef aNum;
+
+ aNum = CFDictionaryGetValue(proxyDict, enabledKey);
+ if (aNum && cfnum_to_int32(aNum)) {
+ CFStringRef hostString;
+
+ hostString = CFDictionaryGetValue(proxyDict, hostKey);
+ aNum = CFDictionaryGetValue(proxyDict, portKey);
+
+ if (hostString) {
+ int r;
+ PyObject* h = cfstring_to_pystring(hostString);
+ PyObject* v;
+ if (h) {
+ if (aNum) {
+ int32_t port = cfnum_to_int32(aNum);
+ v = PyString_FromFormat("http://%s:%ld",
+ PyString_AS_STRING(h),
+ (long)port);
+ } else {
+ v = PyString_FromFormat("http://%s",
+ PyString_AS_STRING(h));
+ }
+ Py_DECREF(h);
+ if (!v) return -1;
+ r = PyDict_SetItemString(proxies, proto,
+ v);
+ Py_DECREF(v);
+ return r;
+ }
+ }
+
+ }
+ return 0;
+}
+
+
+
+static PyObject*
+get_proxies(PyObject* mod __attribute__((__unused__)))
+{
+ PyObject* result = NULL;
+ int r;
+ CFDictionaryRef proxyDict = NULL;
+
+ proxyDict = SCDynamicStoreCopyProxies(NULL);
+ if (proxyDict == NULL) {
+ return PyDict_New();
+ }
+
+ result = PyDict_New();
+ if (result == NULL) goto error;
+
+ r = set_proxy(result, "http", proxyDict,
+ kSCPropNetProxiesHTTPEnable,
+ kSCPropNetProxiesHTTPProxy,
+ kSCPropNetProxiesHTTPPort);
+ if (r == -1) goto error;
+ r = set_proxy(result, "https", proxyDict,
+ kSCPropNetProxiesHTTPSEnable,
+ kSCPropNetProxiesHTTPSProxy,
+ kSCPropNetProxiesHTTPSPort);
+ if (r == -1) goto error;
+ r = set_proxy(result, "ftp", proxyDict,
+ kSCPropNetProxiesFTPEnable,
+ kSCPropNetProxiesFTPProxy,
+ kSCPropNetProxiesFTPPort);
+ if (r == -1) goto error;
+ r = set_proxy(result, "gopher", proxyDict,
+ kSCPropNetProxiesGopherEnable,
+ kSCPropNetProxiesGopherProxy,
+ kSCPropNetProxiesGopherPort);
+ if (r == -1) goto error;
+
+ CFRelease(proxyDict);
+ return result;
+error:
+ if (proxyDict) CFRelease(proxyDict);
+ Py_XDECREF(result);
+ return NULL;
+}
+
+static PyMethodDef mod_methods[] = {
+ {
+ "_get_proxy_settings",
+ (PyCFunction)get_proxy_settings,
+ METH_NOARGS,
+ NULL,
+ },
+ {
+ "_get_proxies",
+ (PyCFunction)get_proxies,
+ METH_NOARGS,
+ NULL,
+ },
+ { 0, 0, 0, 0 }
+};
+
+void init_scproxy(void)
+{
+ (void)Py_InitModule4("_scproxy", mod_methods, NULL, NULL, PYTHON_API_VERSION);
+}
Modified: python/trunk/Misc/NEWS
==============================================================================
--- python/trunk/Misc/NEWS (original)
+++ python/trunk/Misc/NEWS Sun Sep 20 12:31:22 2009
@@ -379,6 +379,8 @@
Library
-------
+- Issue #6851: Fix urllib.urlopen crash on secondairy threads on OSX 10.6
+
- Issue #4606: Passing 'None' if ctypes argtype is set to POINTER(...)
does now always result in NULL.
Modified: python/trunk/setup.py
==============================================================================
--- python/trunk/setup.py (original)
+++ python/trunk/setup.py Sun Sep 20 12:31:22 2009
@@ -1400,6 +1400,17 @@
addMacExtension('_CF', core_kwds, ['cf/pycfbridge.c'])
addMacExtension('autoGIL', core_kwds)
+ # _scproxy
+ sc_kwds = {
+ 'extra_compile_args': carbon_extra_compile_args,
+ 'extra_link_args': [
+ '-framework', 'SystemConfiguration',
+ '-framework', 'CoreFoundation'
+ ],
+ }
+ addMacExtension("_scproxy", sc_kwds)
+
+
# Carbon
carbon_kwds = {'extra_compile_args': carbon_extra_compile_args,
'extra_link_args': ['-framework', 'Carbon'],
More information about the Python-checkins
mailing list