[pypy-commit] cffi thread-safe: Add locking. Not really tested, apart from the absence of double locking.
arigo
noreply at buildbot.pypy.org
Sat Nov 9 14:41:21 CET 2013
Author: Armin Rigo <arigo at tunes.org>
Branch: thread-safe
Changeset: r1402:0581b30c5944
Date: 2013-11-09 14:40 +0100
http://bitbucket.org/cffi/cffi/changeset/0581b30c5944/
Log: Add locking. Not really tested, apart from the absence of double
locking.
diff --git a/cffi/api.py b/cffi/api.py
--- a/cffi/api.py
+++ b/cffi/api.py
@@ -1,4 +1,5 @@
import types
+from .lock import allocate_lock
try:
callable
@@ -61,6 +62,7 @@
# rely on it! It's probably not going to work well.)
self._backend = backend
+ self._lock = allocate_lock()
self._parser = cparser.Parser()
self._cached_btypes = {}
self._parsed_types = types.ModuleType('parsed_types').__dict__
@@ -74,7 +76,8 @@
if name.startswith('RTLD_'):
setattr(self, name, getattr(backend, name))
#
- self.BVoidP = self._get_cached_btype(model.voidp_type)
+ with self._lock:
+ self.BVoidP = self._get_cached_btype(model.voidp_type)
if isinstance(backend, types.ModuleType):
# _cffi_backend: attach these constants to the class
if not hasattr(FFI, 'NULL'):
@@ -95,11 +98,12 @@
if not isinstance(csource, basestring):
raise TypeError("cdef() argument must be a string")
csource = csource.encode('ascii')
- self._parser.parse(csource, override=override)
- self._cdefsources.append(csource)
- if override:
- for cache in self._function_caches:
- cache.clear()
+ with self._lock:
+ self._parser.parse(csource, override=override)
+ self._cdefsources.append(csource)
+ if override:
+ for cache in self._function_caches:
+ cache.clear()
def dlopen(self, name, flags=0):
"""Load and return a dynamic library identified by 'name'.
@@ -109,28 +113,41 @@
library we only look for the actual (untyped) symbols.
"""
assert isinstance(name, basestring) or name is None
- lib, function_cache = _make_ffi_library(self, name, flags)
- self._function_caches.append(function_cache)
- self._libraries.append(lib)
+ with self._lock:
+ lib, function_cache = _make_ffi_library(self, name, flags)
+ self._function_caches.append(function_cache)
+ self._libraries.append(lib)
return lib
+ def _typeof_locked(self, cdecl):
+ # call me with the lock!
+ key = cdecl
+ if key in self._parsed_types:
+ return self._parsed_types[key]
+ #
+ if not isinstance(cdecl, str): # unicode, on Python 2
+ cdecl = cdecl.encode('ascii')
+ #
+ type = self._parser.parse_type(cdecl)
+ if hasattr(type, 'as_function_pointer'):
+ really_a_function_type = True
+ type = type.as_function_pointer()
+ else:
+ really_a_function_type = False
+ btype = self._get_cached_btype(type)
+ result = btype, really_a_function_type
+ self._parsed_types[key] = result
+ return result
+
def _typeof(self, cdecl, consider_function_as_funcptr=False):
# string -> ctype object
try:
- btype, really_a_function_type = self._parsed_types[cdecl]
+ result = self._parsed_types[cdecl]
except KeyError:
- key = cdecl
- if not isinstance(cdecl, str): # unicode, on Python 2
- cdecl = cdecl.encode('ascii')
- type = self._parser.parse_type(cdecl)
- if hasattr(type, 'as_function_pointer'):
- really_a_function_type = True
- type = type.as_function_pointer()
- else:
- really_a_function_type = False
- btype = self._get_cached_btype(type)
- self._parsed_types[key] = btype, really_a_function_type
+ with self._lock:
+ result = self._typeof_locked(cdecl)
#
+ btype, really_a_function_type = result
if really_a_function_type and not consider_function_as_funcptr:
raise CDefError("the type %r is a function type, not a "
"pointer-to-function type" % (cdecl,))
@@ -151,7 +168,8 @@
return res
if (isinstance(cdecl, types.FunctionType)
and hasattr(cdecl, '_cffi_base_type')):
- return self._get_cached_btype(cdecl._cffi_base_type)
+ with self._lock:
+ return self._get_cached_btype(cdecl._cffi_base_type)
raise TypeError(type(cdecl))
def sizeof(self, cdecl):
@@ -288,14 +306,17 @@
data. Later, when this new cdata object is garbage-collected,
'destructor(old_cdata_object)' will be called.
"""
- try:
- gc_weakrefs = self.gc_weakrefs
- except AttributeError:
- from .gc_weakref import GcWeakrefs
- gc_weakrefs = self.gc_weakrefs = GcWeakrefs(self)
- return gc_weakrefs.build(cdata, destructor)
+ with self._lock:
+ try:
+ gc_weakrefs = self.gc_weakrefs
+ except AttributeError:
+ from .gc_weakref import GcWeakrefs
+ gc_weakrefs = self.gc_weakrefs = GcWeakrefs(self)
+ return gc_weakrefs.build(cdata, destructor)
def _get_cached_btype(self, type):
+ assert self._lock.acquire(False) is False
+ # call me with the lock!
try:
BType = self._cached_btypes[type]
except KeyError:
@@ -330,7 +351,8 @@
def _pointer_to(self, ctype):
from . import model
- return model.pointer_cache(self, ctype)
+ with self._lock:
+ return model.pointer_cache(self, ctype)
def addressof(self, cdata, field=None):
"""Return the address of a <cdata 'struct-or-union'>.
@@ -350,10 +372,12 @@
variables, which must anyway be accessed directly from the
lib object returned by the original FFI instance.
"""
- self._parser.include(ffi_to_include._parser)
- self._cdefsources.append('[')
- self._cdefsources.extend(ffi_to_include._cdefsources)
- self._cdefsources.append(']')
+ with ffi_to_include._lock:
+ with self._lock:
+ self._parser.include(ffi_to_include._parser)
+ self._cdefsources.append('[')
+ self._cdefsources.extend(ffi_to_include._cdefsources)
+ self._cdefsources.append(']')
def new_handle(self, x):
return self._backend.newp_handle(self.BVoidP, x)
@@ -380,7 +404,7 @@
backendlib = backend.load_library(path, flags)
copied_enums = []
#
- def make_accessor(name):
+ def make_accessor_locked(name):
key = 'function ' + name
if key in ffi._parser._declarations:
tp = ffi._parser._declarations[key]
@@ -412,11 +436,17 @@
if enumname not in library.__dict__:
library.__dict__[enumname] = enumval
copied_enums.append(True)
+ if name in library.__dict__:
+ return
#
- if name in library.__dict__: # copied from an enum value just above,
- return # or multithread's race condition
raise AttributeError(name)
#
+ def make_accessor(name):
+ with ffi._lock:
+ if name in library.__dict__ or name in FFILibrary.__dict__:
+ return # added by another thread while waiting for the lock
+ make_accessor_locked(name)
+ #
class FFILibrary(object):
def __getattr__(self, name):
make_accessor(name)
@@ -452,4 +482,5 @@
except (KeyError, AttributeError, TypeError):
return None
else:
- return ffi._get_cached_btype(tp)
+ with ffi._lock:
+ return ffi._get_cached_btype(tp)
diff --git a/cffi/gc_weakref.py b/cffi/gc_weakref.py
--- a/cffi/gc_weakref.py
+++ b/cffi/gc_weakref.py
@@ -14,6 +14,6 @@
def build(self, cdata, destructor):
# make a new cdata of the same type as the original one
- new_cdata = self.ffi.cast(self.ffi.typeof(cdata), cdata)
+ new_cdata = self.ffi.cast(self.ffi._backend.typeof(cdata), cdata)
self.data[ref(new_cdata, self.remove)] = destructor, cdata
return new_cdata
diff --git a/cffi/lock.py b/cffi/lock.py
--- a/cffi/lock.py
+++ b/cffi/lock.py
@@ -10,3 +10,21 @@
from _thread import allocate_lock
except ImportError:
from _dummy_thread import allocate_lock
+
+
+##import sys
+##l1 = allocate_lock
+
+##class allocate_lock(object):
+## def __init__(self):
+## self._real = l1()
+## def __enter__(self):
+## for i in range(4, 0, -1):
+## print sys._getframe(i).f_code
+## print
+## return self._real.__enter__()
+## def __exit__(self, *args):
+## return self._real.__exit__(*args)
+## def acquire(self, f):
+## assert f is False
+## return self._real.acquire(f)
diff --git a/cffi/vengine_gen.py b/cffi/vengine_gen.py
--- a/cffi/vengine_gen.py
+++ b/cffi/vengine_gen.py
@@ -276,7 +276,7 @@
return # nothing to do with opaque structs
layoutfuncname = '_cffi_layout_%s_%s' % (prefix, name)
#
- BFunc = self.ffi.typeof("ssize_t(*)(ssize_t)")
+ BFunc = self.ffi._typeof_locked("ssize_t(*)(ssize_t)")[0]
function = module.load_function(BFunc, layoutfuncname)
layout = []
num = 0
@@ -386,15 +386,16 @@
def _load_constant(self, is_int, tp, name, module):
funcname = '_cffi_const_%s' % name
if is_int:
- BFunc = self.ffi.typeof("int(*)(long long*)")
+ BType = self.ffi._typeof_locked("long long*")[0]
+ BFunc = self.ffi._typeof_locked("int(*)(long long*)")[0]
function = module.load_function(BFunc, funcname)
- p = self.ffi.new("long long*")
+ p = self.ffi.new(BType)
negative = function(p)
value = int(p[0])
if value < 0 and not negative:
value += (1 << (8*self.ffi.sizeof("long long")))
else:
- BFunc = self.ffi.typeof(tp.get_c_name('(*)(void)', name))
+ BFunc = self.ffi._typeof_locked(tp.get_c_name('(*)(void)', name))[0]
function = module.load_function(BFunc, funcname)
value = function()
return value
@@ -438,10 +439,11 @@
tp.enumvalues = tuple(enumvalues)
tp.partial_resolved = True
else:
- BFunc = self.ffi.typeof("int(*)(char*)")
+ BType = self.ffi._typeof_locked("char[]")[0]
+ BFunc = self.ffi._typeof_locked("int(*)(char*)")[0]
funcname = '_cffi_e_%s_%s' % (prefix, name)
function = module.load_function(BFunc, funcname)
- p = self.ffi.new("char[]", 256)
+ p = self.ffi.new(BType, 256)
if function(p) < 0:
error = self.ffi.string(p)
if sys.version_info >= (3,):
@@ -493,7 +495,7 @@
# sense that "a=..." is forbidden
if tp.length == '...':
funcname = '_cffi_sizeof_%s' % (name,)
- BFunc = self.ffi.typeof('size_t(*)(void)')
+ BFunc = self.ffi._typeof_locked('size_t(*)(void)')[0]
function = module.load_function(BFunc, funcname)
size = function()
BItemType = self.ffi._get_cached_btype(tp.item)
@@ -516,7 +518,7 @@
# remove ptr=<cdata 'int *'> from the library instance, and replace
# it by a property on the class, which reads/writes into ptr[0].
funcname = '_cffi_var_%s' % name
- BFunc = self.ffi.typeof(tp.get_c_name('*(*)(void)', name))
+ BFunc = self.ffi._typeof_locked(tp.get_c_name('*(*)(void)', name))[0]
function = module.load_function(BFunc, funcname)
ptr = function()
def getter(library):
diff --git a/cffi/verifier.py b/cffi/verifier.py
--- a/cffi/verifier.py
+++ b/cffi/verifier.py
@@ -42,18 +42,21 @@
def write_source(self, file=None):
"""Write the C source code. It is produced in 'self.sourcefilename',
which can be tweaked beforehand."""
- if self._has_source and file is None:
- raise ffiplatform.VerificationError("source code already written")
- self._write_source(file)
+ with self.ffi._lock:
+ if self._has_source and file is None:
+ raise ffiplatform.VerificationError(
+ "source code already written")
+ self._write_source(file)
def compile_module(self):
"""Write the C source code (if not done already) and compile it.
This produces a dynamic link library in 'self.modulefilename'."""
- if self._has_module:
- raise ffiplatform.VerificationError("module already compiled")
- if not self._has_source:
- self._write_source()
- self._compile_module()
+ with self.ffi._lock:
+ if self._has_module:
+ raise ffiplatform.VerificationError("module already compiled")
+ if not self._has_source:
+ self._write_source()
+ self._compile_module()
def load_library(self):
"""Get a C module from this Verifier instance.
@@ -62,11 +65,14 @@
operations to the C module. If necessary, the C code is written
and compiled first.
"""
- if not self._has_module:
- self._locate_module()
+ with self.ffi._lock:
if not self._has_module:
- self.compile_module()
- return self._load_library()
+ self._locate_module()
+ if not self._has_module:
+ if not self._has_source:
+ self._write_source()
+ self._compile_module()
+ return self._load_library()
def get_module_name(self):
basename = os.path.basename(self.modulefilename)
@@ -81,7 +87,9 @@
def get_extension(self):
if not self._has_source:
- self._write_source()
+ with self.ffi._lock:
+ if not self._has_source:
+ self._write_source()
sourcename = ffiplatform.maybe_relative_path(self.sourcefilename)
modname = self.get_module_name()
return ffiplatform.get_extension(sourcename, modname, **self.kwds)
diff --git a/testing/test_parsing.py b/testing/test_parsing.py
--- a/testing/test_parsing.py
+++ b/testing/test_parsing.py
@@ -130,8 +130,9 @@
ffi.cdef("""
typedef int (*fn_t)(int[5]);
""")
- type = ffi._parser.parse_type("fn_t")
- BType = ffi._get_cached_btype(type)
+ with ffi._lock:
+ type = ffi._parser.parse_type("fn_t")
+ BType = ffi._get_cached_btype(type)
assert str(BType) == '<func (<pointer to <int>>), <int>, False>'
def test_remove_comments():
More information about the pypy-commit
mailing list