[pypy-commit] pypy py3k: merge default
pjenvey
noreply at buildbot.pypy.org
Thu Nov 14 05:38:30 CET 2013
Author: Philip Jenvey <pjenvey at underboss.org>
Branch: py3k
Changeset: r68013:f9c10dfe0d96
Date: 2013-11-13 19:47 -0800
http://bitbucket.org/pypy/pypy/changeset/f9c10dfe0d96/
Log: merge default
diff too long, truncating to 2000 out of 23611 lines
diff --git a/Makefile b/Makefile
--- a/Makefile
+++ b/Makefile
@@ -1,7 +1,38 @@
all: pypy-c
+PYPY_EXECUTABLE := $(shell which pypy)
+URAM := $(shell python -c "import sys; print 4.5 if sys.maxint>1<<32 else 2.5")
+
+ifeq ($(PYPY_EXECUTABLE),)
+RUNINTERP = python
+else
+RUNINTERP = $(PYPY_EXECUTABLE)
+endif
+
pypy-c:
- @echo "Building PyPy with JIT, it'll take about 40 minutes and 4G of RAM"
- @sleep 3
- rpython/bin/rpython -Ojit pypy/goal/targetpypystandalone.py
+ @echo
+ @echo "===================================================================="
+ifeq ($(PYPY_EXECUTABLE),)
+ @echo "Building a regular (jitting) version of PyPy, using CPython."
+ @echo "This takes around 2 hours and $(URAM) GB of RAM."
+ @echo "Note that pre-installing a PyPy binary would reduce this time"
+ @echo "and produce basically the same result."
+else
+ @echo "Building a regular (jitting) version of PyPy, using"
+ @echo "$(PYPY_EXECUTABLE) to run the translation itself."
+ @echo "This takes up to 1 hour and $(URAM) GB of RAM."
+endif
+ @echo
+ @echo "For more control (e.g. to use multiple CPU cores during part of"
+ @echo "the process) you need to run \`\`rpython/bin/rpython'' directly."
+ @echo "For more information see \`\`http://pypy.org/download.html''."
+ @echo "===================================================================="
+ @echo
+ @sleep 5
+ $(RUNINTERP) rpython/bin/rpython -Ojit pypy/goal/targetpypystandalone.py
+
+# Note: the -jN option, or MAKEFLAGS=-jN, are not usable. They are
+# replaced with an opaque --jobserver option by the time this Makefile
+# runs. We cannot get their original value either:
+# http://lists.gnu.org/archive/html/help-make/2010-08/msg00106.html
diff --git a/README.rst b/README.rst
--- a/README.rst
+++ b/README.rst
@@ -1,5 +1,5 @@
=====================================
-PyPy: Python in Python Implementation
+PyPy: Python in Python Implementation
=====================================
Welcome to PyPy!
@@ -26,9 +26,11 @@
Building
========
-build with::
+build with:
- rpython/bin/rpython -Ojit pypy/goal/targetpypystandalone.py
+.. code-block:: console
+
+ $ rpython/bin/rpython -Ojit pypy/goal/targetpypystandalone.py
This ends up with ``pypy-c`` binary in the main pypy directory. We suggest
to use virtualenv with the resulting pypy-c as the interpreter, you can
diff --git a/lib-python/2.7/json/encoder.py b/lib-python/2.7/json/encoder.py
--- a/lib-python/2.7/json/encoder.py
+++ b/lib-python/2.7/json/encoder.py
@@ -4,6 +4,21 @@
from __pypy__.builders import StringBuilder, UnicodeBuilder
+class StringOrUnicodeBuilder(object):
+ def __init__(self):
+ self._builder = StringBuilder()
+ def append(self, string):
+ try:
+ self._builder.append(string)
+ except UnicodeEncodeError:
+ ub = UnicodeBuilder()
+ ub.append(self._builder.build())
+ self._builder = ub
+ ub.append(string)
+ def build(self):
+ return self._builder.build()
+
+
ESCAPE = re.compile(r'[\x00-\x1f\\"\b\f\n\r\t]')
ESCAPE_ASCII = re.compile(r'([\\"]|[^\ -~])')
HAS_UTF8 = re.compile(r'[\x80-\xff]')
@@ -192,7 +207,7 @@
if self.ensure_ascii:
builder = StringBuilder()
else:
- builder = UnicodeBuilder()
+ builder = StringOrUnicodeBuilder()
self.__encode(o, markers, builder, 0)
return builder.build()
diff --git a/lib-python/2.7/string.py b/lib-python/2.7/string.py
--- a/lib-python/2.7/string.py
+++ b/lib-python/2.7/string.py
@@ -66,16 +66,17 @@
must be of the same length.
"""
- if len(fromstr) != len(tostr):
+ n = len(fromstr)
+ if n != len(tostr):
raise ValueError, "maketrans arguments must have same length"
- global _idmapL
- if not _idmapL:
- _idmapL = list(_idmap)
- L = _idmapL[:]
- fromstr = map(ord, fromstr)
- for i in range(len(fromstr)):
- L[fromstr[i]] = tostr[i]
- return ''.join(L)
+ # this function has been rewritten to suit PyPy better; it is
+ # almost 10x faster than the original.
+ buf = bytearray(256)
+ for i in range(256):
+ buf[i] = i
+ for i in range(n):
+ buf[ord(fromstr[i])] = tostr[i]
+ return str(buf)
diff --git a/lib-python/2.7/weakref.py b/lib-python/2.7/weakref.py
--- a/lib-python/2.7/weakref.py
+++ b/lib-python/2.7/weakref.py
@@ -48,7 +48,14 @@
def remove(wr, selfref=ref(self)):
self = selfref()
if self is not None:
- del self.data[wr.key]
+ # Changed this for PyPy: made more resistent. The
+ # issue is that in some corner cases, self.data
+ # might already be changed or removed by the time
+ # this weakref's callback is called. If that is
+ # the case, we don't want to randomly kill an
+ # unrelated entry.
+ if self.data.get(wr.key) is wr:
+ del self.data[wr.key]
self._remove = remove
UserDict.UserDict.__init__(self, *args, **kw)
@@ -160,22 +167,26 @@
try:
o = self.data.pop(key)()
except KeyError:
+ o = None
+ if o is None:
if args:
return args[0]
- raise
- if o is None:
raise KeyError, key
else:
return o
+ # The logic above was fixed in PyPy
def setdefault(self, key, default=None):
try:
- wr = self.data[key]
+ o = self.data[key]()
except KeyError:
+ o = None
+ if o is None:
self.data[key] = KeyedRef(default, self._remove, key)
return default
else:
- return wr()
+ return o
+ # The logic above was fixed in PyPy
def update(self, dict=None, **kwargs):
d = self.data
diff --git a/lib_pypy/cffi.egg-info b/lib_pypy/cffi.egg-info
--- a/lib_pypy/cffi.egg-info
+++ b/lib_pypy/cffi.egg-info
@@ -1,6 +1,6 @@
Metadata-Version: 1.0
Name: cffi
-Version: 0.7
+Version: 0.8
Summary: Foreign Function Interface for Python calling C code.
Home-page: http://cffi.readthedocs.org
Author: Armin Rigo, Maciej Fijalkowski
diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py
--- a/lib_pypy/cffi/__init__.py
+++ b/lib_pypy/cffi/__init__.py
@@ -4,5 +4,5 @@
from .api import FFI, CDefError, FFIError
from .ffiplatform import VerificationError, VerificationMissing
-__version__ = "0.7.2"
-__version_info__ = (0, 7, 2)
+__version__ = "0.8"
+__version_info__ = (0, 8)
diff --git a/lib_pypy/cffi/api.py b/lib_pypy/cffi/api.py
--- a/lib_pypy/cffi/api.py
+++ b/lib_pypy/cffi/api.py
@@ -1,4 +1,5 @@
import types
+from .lock import allocate_lock
try:
callable
@@ -61,6 +62,7 @@
# rely on it! It's probably not going to work well.)
self._backend = backend
+ self._lock = allocate_lock()
self._parser = cparser.Parser()
self._cached_btypes = {}
self._parsed_types = types.ModuleType('parsed_types').__dict__
@@ -74,7 +76,8 @@
if name.startswith('RTLD_'):
setattr(self, name, getattr(backend, name))
#
- self.BVoidP = self._get_cached_btype(model.voidp_type)
+ with self._lock:
+ self.BVoidP = self._get_cached_btype(model.voidp_type)
if isinstance(backend, types.ModuleType):
# _cffi_backend: attach these constants to the class
if not hasattr(FFI, 'NULL'):
@@ -95,11 +98,12 @@
if not isinstance(csource, basestring):
raise TypeError("cdef() argument must be a string")
csource = csource.encode('ascii')
- self._parser.parse(csource, override=override)
- self._cdefsources.append(csource)
- if override:
- for cache in self._function_caches:
- cache.clear()
+ with self._lock:
+ self._parser.parse(csource, override=override)
+ self._cdefsources.append(csource)
+ if override:
+ for cache in self._function_caches:
+ cache.clear()
def dlopen(self, name, flags=0):
"""Load and return a dynamic library identified by 'name'.
@@ -109,31 +113,47 @@
library we only look for the actual (untyped) symbols.
"""
assert isinstance(name, basestring) or name is None
- lib, function_cache = _make_ffi_library(self, name, flags)
- self._function_caches.append(function_cache)
- self._libraries.append(lib)
+ with self._lock:
+ lib, function_cache = _make_ffi_library(self, name, flags)
+ self._function_caches.append(function_cache)
+ self._libraries.append(lib)
return lib
+ def _typeof_locked(self, cdecl):
+ # call me with the lock!
+ key = cdecl
+ if key in self._parsed_types:
+ return self._parsed_types[key]
+ #
+ if not isinstance(cdecl, str): # unicode, on Python 2
+ cdecl = cdecl.encode('ascii')
+ #
+ type = self._parser.parse_type(cdecl)
+ really_a_function_type = type.is_raw_function
+ if really_a_function_type:
+ type = type.as_function_pointer()
+ btype = self._get_cached_btype(type)
+ result = btype, really_a_function_type
+ self._parsed_types[key] = result
+ return result
+
def _typeof(self, cdecl, consider_function_as_funcptr=False):
# string -> ctype object
try:
- btype, cfaf = self._parsed_types[cdecl]
- if consider_function_as_funcptr and not cfaf:
- raise KeyError
+ result = self._parsed_types[cdecl]
except KeyError:
- key = cdecl
- if not isinstance(cdecl, str): # unicode, on Python 2
- cdecl = cdecl.encode('ascii')
- cfaf = consider_function_as_funcptr
- type = self._parser.parse_type(cdecl,
- consider_function_as_funcptr=cfaf)
- btype = self._get_cached_btype(type)
- self._parsed_types[key] = btype, cfaf
+ with self._lock:
+ result = self._typeof_locked(cdecl)
+ #
+ btype, really_a_function_type = result
+ if really_a_function_type and not consider_function_as_funcptr:
+ raise CDefError("the type %r is a function type, not a "
+ "pointer-to-function type" % (cdecl,))
return btype
def typeof(self, cdecl):
"""Parse the C type given as a string and return the
- corresponding Python type: <class 'ffi.CData<...>'>.
+ corresponding <ctype> object.
It can also be used on 'cdata' instance to get its C type.
"""
if isinstance(cdecl, basestring):
@@ -144,6 +164,10 @@
res = _builtin_function_type(cdecl)
if res is not None:
return res
+ if (isinstance(cdecl, types.FunctionType)
+ and hasattr(cdecl, '_cffi_base_type')):
+ with self._lock:
+ return self._get_cached_btype(cdecl._cffi_base_type)
raise TypeError(type(cdecl))
def sizeof(self, cdecl):
@@ -280,14 +304,17 @@
data. Later, when this new cdata object is garbage-collected,
'destructor(old_cdata_object)' will be called.
"""
- try:
- gc_weakrefs = self.gc_weakrefs
- except AttributeError:
- from .gc_weakref import GcWeakrefs
- gc_weakrefs = self.gc_weakrefs = GcWeakrefs(self)
- return gc_weakrefs.build(cdata, destructor)
+ with self._lock:
+ try:
+ gc_weakrefs = self.gc_weakrefs
+ except AttributeError:
+ from .gc_weakref import GcWeakrefs
+ gc_weakrefs = self.gc_weakrefs = GcWeakrefs(self)
+ return gc_weakrefs.build(cdata, destructor)
def _get_cached_btype(self, type):
+ assert self._lock.acquire(False) is False
+ # call me with the lock!
try:
BType = self._cached_btypes[type]
except KeyError:
@@ -322,7 +349,8 @@
def _pointer_to(self, ctype):
from . import model
- return model.pointer_cache(self, ctype)
+ with self._lock:
+ return model.pointer_cache(self, ctype)
def addressof(self, cdata, field=None):
"""Return the address of a <cdata 'struct-or-union'>.
@@ -342,10 +370,12 @@
variables, which must anyway be accessed directly from the
lib object returned by the original FFI instance.
"""
- self._parser.include(ffi_to_include._parser)
- self._cdefsources.append('[')
- self._cdefsources.extend(ffi_to_include._cdefsources)
- self._cdefsources.append(']')
+ with ffi_to_include._lock:
+ with self._lock:
+ self._parser.include(ffi_to_include._parser)
+ self._cdefsources.append('[')
+ self._cdefsources.extend(ffi_to_include._cdefsources)
+ self._cdefsources.append(']')
def new_handle(self, x):
return self._backend.newp_handle(self.BVoidP, x)
@@ -372,7 +402,7 @@
backendlib = backend.load_library(path, flags)
copied_enums = []
#
- def make_accessor(name):
+ def make_accessor_locked(name):
key = 'function ' + name
if key in ffi._parser._declarations:
tp = ffi._parser._declarations[key]
@@ -404,11 +434,17 @@
if enumname not in library.__dict__:
library.__dict__[enumname] = enumval
copied_enums.append(True)
+ if name in library.__dict__:
+ return
#
- if name in library.__dict__: # copied from an enum value just above,
- return # or multithread's race condition
raise AttributeError(name)
#
+ def make_accessor(name):
+ with ffi._lock:
+ if name in library.__dict__ or name in FFILibrary.__dict__:
+ return # added by another thread while waiting for the lock
+ make_accessor_locked(name)
+ #
class FFILibrary(object):
def __getattr__(self, name):
make_accessor(name)
@@ -444,4 +480,5 @@
except (KeyError, AttributeError, TypeError):
return None
else:
- return ffi._get_cached_btype(tp)
+ with ffi._lock:
+ return ffi._get_cached_btype(tp)
diff --git a/lib_pypy/cffi/cparser.py b/lib_pypy/cffi/cparser.py
--- a/lib_pypy/cffi/cparser.py
+++ b/lib_pypy/cffi/cparser.py
@@ -142,7 +142,7 @@
if 1 <= linenum <= len(csourcelines):
line = csourcelines[linenum-1]
if line:
- msg = 'cannot parse "%s"\n%s' % (line, msg)
+ msg = 'cannot parse "%s"\n%s' % (line.strip(), msg)
else:
msg = 'parse error\n%s' % (msg,)
raise api.CDefError(msg)
@@ -217,19 +217,18 @@
#
if decl.name:
tp = self._get_type(node, partial_length_ok=True)
- if self._is_constant_declaration(node):
+ if self._is_constant_globalvar(node):
self._declare('constant ' + decl.name, tp)
else:
self._declare('variable ' + decl.name, tp)
- def parse_type(self, cdecl, consider_function_as_funcptr=False):
+ def parse_type(self, cdecl):
ast, macros = self._parse('void __dummy(\n%s\n);' % cdecl)
assert not macros
exprnode = ast.ext[-1].type.args.params[0]
if isinstance(exprnode, pycparser.c_ast.ID):
raise api.CDefError("unknown identifier '%s'" % (exprnode.name,))
- return self._get_type(exprnode.type,
- consider_function_as_funcptr=consider_function_as_funcptr)
+ return self._get_type(exprnode.type)
def _declare(self, name, obj):
if name in self._declarations:
@@ -249,28 +248,17 @@
return model.ConstPointerType(type)
return model.PointerType(type)
- def _get_type(self, typenode, convert_array_to_pointer=False,
- name=None, partial_length_ok=False,
- consider_function_as_funcptr=False):
+ def _get_type(self, typenode, name=None, partial_length_ok=False):
# first, dereference typedefs, if we have it already parsed, we're good
if (isinstance(typenode, pycparser.c_ast.TypeDecl) and
isinstance(typenode.type, pycparser.c_ast.IdentifierType) and
len(typenode.type.names) == 1 and
('typedef ' + typenode.type.names[0]) in self._declarations):
type = self._declarations['typedef ' + typenode.type.names[0]]
- if isinstance(type, model.ArrayType):
- if convert_array_to_pointer:
- return type.item
- else:
- if (consider_function_as_funcptr and
- isinstance(type, model.RawFunctionType)):
- return type.as_function_pointer()
return type
#
if isinstance(typenode, pycparser.c_ast.ArrayDecl):
# array type
- if convert_array_to_pointer:
- return self._get_type_pointer(self._get_type(typenode.type))
if typenode.dim is None:
length = None
else:
@@ -331,10 +319,7 @@
#
if isinstance(typenode, pycparser.c_ast.FuncDecl):
# a function type
- result = self._parse_function_type(typenode, name)
- if consider_function_as_funcptr:
- result = result.as_function_pointer()
- return result
+ return self._parse_function_type(typenode, name)
#
# nested anonymous structs or unions end up here
if isinstance(typenode, pycparser.c_ast.Struct):
@@ -365,21 +350,24 @@
isinstance(params[0].type.type, pycparser.c_ast.IdentifierType)
and list(params[0].type.type.names) == ['void']):
del params[0]
- args = [self._get_type(argdeclnode.type,
- convert_array_to_pointer=True,
- consider_function_as_funcptr=True)
+ args = [self._as_func_arg(self._get_type(argdeclnode.type))
for argdeclnode in params]
result = self._get_type(typenode.type)
return model.RawFunctionType(tuple(args), result, ellipsis)
- def _is_constant_declaration(self, typenode, const=False):
- if isinstance(typenode, pycparser.c_ast.ArrayDecl):
- return self._is_constant_declaration(typenode.type)
+ def _as_func_arg(self, type):
+ if isinstance(type, model.ArrayType):
+ return model.PointerType(type.item)
+ elif isinstance(type, model.RawFunctionType):
+ return type.as_function_pointer()
+ else:
+ return type
+
+ def _is_constant_globalvar(self, typenode):
if isinstance(typenode, pycparser.c_ast.PtrDecl):
- const = 'const' in typenode.quals
- return self._is_constant_declaration(typenode.type, const)
+ return 'const' in typenode.quals
if isinstance(typenode, pycparser.c_ast.TypeDecl):
- return const or 'const' in typenode.quals
+ return 'const' in typenode.quals
return False
def _get_struct_union_enum_type(self, kind, type, name=None, nested=False):
@@ -491,7 +479,7 @@
return tp
def _make_partial(self, tp, nested):
- if not isinstance(tp, model.StructType):
+ if not isinstance(tp, model.StructOrUnion):
raise api.CDefError("%s cannot be partial" % (tp,))
if not tp.has_c_name() and not nested:
raise NotImplementedError("%s is partial but has no C name" %(tp,))
@@ -511,7 +499,7 @@
if (isinstance(exprnode, pycparser.c_ast.ID) and
exprnode.name == '__dotdotdotarray__'):
self._partial_length = True
- return None
+ return '...'
#
raise api.FFIError("unsupported expression: expected a "
"simple numeric constant")
diff --git a/lib_pypy/cffi/gc_weakref.py b/lib_pypy/cffi/gc_weakref.py
--- a/lib_pypy/cffi/gc_weakref.py
+++ b/lib_pypy/cffi/gc_weakref.py
@@ -14,6 +14,6 @@
def build(self, cdata, destructor):
# make a new cdata of the same type as the original one
- new_cdata = self.ffi.cast(self.ffi.typeof(cdata), cdata)
+ new_cdata = self.ffi.cast(self.ffi._backend.typeof(cdata), cdata)
self.data[ref(new_cdata, self.remove)] = destructor, cdata
return new_cdata
diff --git a/lib_pypy/cffi/lock.py b/lib_pypy/cffi/lock.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/cffi/lock.py
@@ -0,0 +1,30 @@
+import sys
+
+if sys.version_info < (3,):
+ try:
+ from thread import allocate_lock
+ except ImportError:
+ from dummy_thread import allocate_lock
+else:
+ try:
+ from _thread import allocate_lock
+ except ImportError:
+ from _dummy_thread import allocate_lock
+
+
+##import sys
+##l1 = allocate_lock
+
+##class allocate_lock(object):
+## def __init__(self):
+## self._real = l1()
+## def __enter__(self):
+## for i in range(4, 0, -1):
+## print sys._getframe(i).f_code
+## print
+## return self._real.__enter__()
+## def __exit__(self, *args):
+## return self._real.__exit__(*args)
+## def acquire(self, f):
+## assert f is False
+## return self._real.acquire(f)
diff --git a/lib_pypy/cffi/model.py b/lib_pypy/cffi/model.py
--- a/lib_pypy/cffi/model.py
+++ b/lib_pypy/cffi/model.py
@@ -1,7 +1,10 @@
import weakref
+from .lock import allocate_lock
+
class BaseTypeByIdentity(object):
is_array_type = False
+ is_raw_function = False
def get_c_name(self, replace_with='', context='a C file'):
result = self.c_name_with_marker
@@ -146,6 +149,7 @@
# a function, but not a pointer-to-function. The backend has no
# notion of such a type; it's used temporarily by parsing.
_base_pattern = '(&)(%s)'
+ is_raw_function = True
def build_backend_type(self, ffi, finishlist):
from . import api
@@ -192,10 +196,6 @@
_base_pattern = " const *&"
_base_pattern_array = "(const *&)"
- def build_backend_type(self, ffi, finishlist):
- BPtr = PointerType(self.totype).get_cached_btype(ffi, finishlist)
- return BPtr
-
const_voidp_type = ConstPointerType(void_type)
@@ -216,10 +216,12 @@
self.item = item
self.length = length
#
- if self.length is None:
+ if length is None:
brackets = '&[]'
+ elif length == '...':
+ brackets = '&[/*...*/]'
else:
- brackets = '&[%d]' % self.length
+ brackets = '&[%d]' % length
self.c_name_with_marker = (
self.item.c_name_with_marker.replace('&', brackets))
@@ -227,6 +229,10 @@
return ArrayType(self.item, newlength)
def build_backend_type(self, ffi, finishlist):
+ if self.length == '...':
+ from . import api
+ raise api.CDefError("cannot render the type %r: unknown length" %
+ (self,))
self.item.get_cached_btype(ffi, finishlist) # force the item BType
BPtrItem = PointerType(self.item).get_cached_btype(ffi, finishlist)
return global_cache(self, ffi, 'new_array_type', BPtrItem, self.length)
@@ -252,6 +258,7 @@
class StructOrUnion(StructOrUnionOrEnum):
fixedlayout = None
completed = False
+ partial = False
def __init__(self, name, fldnames, fldtypes, fldbitsize):
self.name = name
@@ -303,20 +310,21 @@
return # not completing it: it's an opaque struct
#
self.completed = 1
- fldtypes = tuple(tp.get_cached_btype(ffi, finishlist)
- for tp in self.fldtypes)
#
if self.fixedlayout is None:
+ fldtypes = [tp.get_cached_btype(ffi, finishlist)
+ for tp in self.fldtypes]
lst = list(zip(self.fldnames, fldtypes, self.fldbitsize))
ffi._backend.complete_struct_or_union(BType, lst, self)
#
else:
+ fldtypes = []
fieldofs, fieldsize, totalsize, totalalignment = self.fixedlayout
for i in range(len(self.fldnames)):
fsize = fieldsize[i]
ftype = self.fldtypes[i]
#
- if isinstance(ftype, ArrayType) and ftype.length is None:
+ if isinstance(ftype, ArrayType) and ftype.length == '...':
# fix the length to match the total size
BItemType = ftype.item.get_cached_btype(ffi, finishlist)
nlen, nrest = divmod(fsize, ffi.sizeof(BItemType))
@@ -327,18 +335,20 @@
ftype = ftype.resolve_length(nlen)
self.fldtypes = (self.fldtypes[:i] + (ftype,) +
self.fldtypes[i+1:])
- BArrayType = ftype.get_cached_btype(ffi, finishlist)
- fldtypes = (fldtypes[:i] + (BArrayType,) +
- fldtypes[i+1:])
- continue
#
- bitemsize = ffi.sizeof(fldtypes[i])
- if bitemsize != fsize:
- self._verification_error(
- "field '%s.%s' is declared as %d bytes, but is "
- "really %d bytes" % (self.name,
- self.fldnames[i] or '{}',
- bitemsize, fsize))
+ BFieldType = ftype.get_cached_btype(ffi, finishlist)
+ if isinstance(ftype, ArrayType) and ftype.length is None:
+ assert fsize == 0
+ else:
+ bitemsize = ffi.sizeof(BFieldType)
+ if bitemsize != fsize:
+ self._verification_error(
+ "field '%s.%s' is declared as %d bytes, but is "
+ "really %d bytes" % (self.name,
+ self.fldnames[i] or '{}',
+ bitemsize, fsize))
+ fldtypes.append(BFieldType)
+ #
lst = list(zip(self.fldnames, fldtypes, self.fldbitsize, fieldofs))
ffi._backend.complete_struct_or_union(BType, lst, self,
totalsize, totalalignment)
@@ -348,11 +358,6 @@
from .ffiplatform import VerificationError
raise VerificationError(msg)
-
-class StructType(StructOrUnion):
- kind = 'struct'
- partial = False
-
def check_not_partial(self):
if self.partial and self.fixedlayout is None:
from . import ffiplatform
@@ -361,19 +366,18 @@
def build_backend_type(self, ffi, finishlist):
self.check_not_partial()
finishlist.append(self)
-
- return global_cache(self, ffi, 'new_struct_type',
+ #
+ return global_cache(self, ffi, 'new_%s_type' % self.kind,
self.get_official_name(), key=self)
+class StructType(StructOrUnion):
+ kind = 'struct'
+
+
class UnionType(StructOrUnion):
kind = 'union'
- def build_backend_type(self, ffi, finishlist):
- finishlist.append(self)
- return global_cache(self, ffi, 'new_union_type',
- self.get_official_name(), key=self)
-
class EnumType(StructOrUnionOrEnum):
kind = 'enum'
@@ -387,6 +391,12 @@
self.baseinttype = baseinttype
self.build_c_name_with_marker()
+ def force_the_name(self, forcename):
+ StructOrUnionOrEnum.force_the_name(self, forcename)
+ if self.forcename is None:
+ name = self.get_official_name()
+ self.forcename = '$' + name.replace(' ', '_')
+
def check_not_partial(self):
if self.partial and not self.partial_resolved:
from . import ffiplatform
@@ -444,6 +454,9 @@
tp = StructType(structname, None, None, None)
return NamedPointerType(tp, name)
+
+global_lock = allocate_lock()
+
def global_cache(srctype, ffi, funcname, *args, **kwds):
key = kwds.pop('key', (funcname, args))
assert not kwds
@@ -464,8 +477,17 @@
res = getattr(ffi._backend, funcname)(*args)
except NotImplementedError as e:
raise NotImplementedError("%r: %s" % (srctype, e))
- ffi._backend.__typecache[key] = res
- return res
+ # note that setdefault() on WeakValueDictionary is not atomic
+ # and contains a rare bug (http://bugs.python.org/issue19542);
+ # we have to use a lock and do it ourselves
+ cache = ffi._backend.__typecache
+ with global_lock:
+ res1 = cache.get(key)
+ if res1 is None:
+ cache[key] = res
+ return res
+ else:
+ return res1
def pointer_cache(ffi, BType):
return global_cache('?', ffi, 'new_pointer_type', BType)
diff --git a/lib_pypy/cffi/vengine_cpy.py b/lib_pypy/cffi/vengine_cpy.py
--- a/lib_pypy/cffi/vengine_cpy.py
+++ b/lib_pypy/cffi/vengine_cpy.py
@@ -15,7 +15,7 @@
def patch_extension_kwds(self, kwds):
pass
- def find_module(self, module_name, path, so_suffix):
+ def find_module(self, module_name, path, so_suffixes):
try:
f, filename, descr = imp.find_module(module_name, path)
except ImportError:
@@ -25,7 +25,7 @@
# Note that after a setuptools installation, there are both .py
# and .so files with the same basename. The code here relies on
# imp.find_module() locating the .so in priority.
- if descr[0] != so_suffix:
+ if descr[0] not in so_suffixes:
return None
return filename
@@ -160,7 +160,10 @@
def __dir__(self):
return FFILibrary._cffi_dir + list(self.__dict__)
library = FFILibrary()
- module._cffi_setup(lst, ffiplatform.VerificationError, library)
+ if module._cffi_setup(lst, ffiplatform.VerificationError, library):
+ import warnings
+ warnings.warn("reimporting %r might overwrite older definitions"
+ % (self.verifier.get_module_name()))
#
# finally, call the loaded_cpy_xxx() functions. This will perform
# the final adjustments, like copying the Python->C wrapper
@@ -280,8 +283,8 @@
return '_cffi_from_c_pointer((char *)%s, _cffi_type(%d))' % (
var, self._gettypenum(tp))
elif isinstance(tp, model.ArrayType):
- return '_cffi_from_c_deref((char *)%s, _cffi_type(%d))' % (
- var, self._gettypenum(tp))
+ return '_cffi_from_c_pointer((char *)%s, _cffi_type(%d))' % (
+ var, self._gettypenum(model.PointerType(tp.item)))
elif isinstance(tp, model.StructType):
if tp.fldnames is None:
raise TypeError("'%s' is used as %s, but is opaque" % (
@@ -464,11 +467,14 @@
prnt(' static Py_ssize_t nums[] = {')
prnt(' sizeof(%s),' % cname)
prnt(' offsetof(struct _cffi_aligncheck, y),')
- for fname, _, fbitsize in tp.enumfields():
+ for fname, ftype, fbitsize in tp.enumfields():
if fbitsize >= 0:
continue # xxx ignore fbitsize for now
prnt(' offsetof(%s, %s),' % (cname, fname))
- prnt(' sizeof(((%s *)0)->%s),' % (cname, fname))
+ if isinstance(ftype, model.ArrayType) and ftype.length is None:
+ prnt(' 0, /* %s */' % ftype._get_c_name())
+ else:
+ prnt(' sizeof(((%s *)0)->%s),' % (cname, fname))
prnt(' -1')
prnt(' };')
prnt(' return _cffi_get_struct_layout(nums);')
@@ -491,7 +497,7 @@
#
function = getattr(module, layoutfuncname)
layout = function()
- if isinstance(tp, model.StructType) and tp.partial:
+ if isinstance(tp, model.StructOrUnion) and tp.partial:
# use the function()'s sizes and offsets to guide the
# layout of the struct
totalsize = layout[0]
@@ -528,9 +534,10 @@
continue # xxx ignore fbitsize for now
check(layout[i], ffi.offsetof(BStruct, fname),
"wrong offset for field %r" % (fname,))
- BField = ffi._get_cached_btype(ftype)
- check(layout[i+1], ffi.sizeof(BField),
- "wrong size for field %r" % (fname,))
+ if layout[i+1] != 0:
+ BField = ffi._get_cached_btype(ftype)
+ check(layout[i+1], ffi.sizeof(BField),
+ "wrong size for field %r" % (fname,))
i += 2
assert i == len(layout)
@@ -566,7 +573,7 @@
# constants, likely declared with '#define'
def _generate_cpy_const(self, is_int, name, tp=None, category='const',
- vartp=None, delayed=True):
+ vartp=None, delayed=True, size_too=False):
prnt = self._prnt
funcname = '_cffi_%s_%s' % (category, name)
prnt('static int %s(PyObject *lib)' % funcname)
@@ -597,6 +604,15 @@
'(unsigned long long)(%s));' % (name,))
prnt(' if (o == NULL)')
prnt(' return -1;')
+ if size_too:
+ prnt(' {')
+ prnt(' PyObject *o1 = o;')
+ prnt(' o = Py_BuildValue("On", o1, (Py_ssize_t)sizeof(%s));'
+ % (name,))
+ prnt(' Py_DECREF(o1);')
+ prnt(' if (o == NULL)')
+ prnt(' return -1;')
+ prnt(' }')
prnt(' res = PyObject_SetAttrString(lib, "%s", o);' % name)
prnt(' Py_DECREF(o);')
prnt(' if (res < 0)')
@@ -633,12 +649,23 @@
prnt('static int %s(PyObject *lib)' % funcname)
prnt('{')
for enumerator, enumvalue in zip(tp.enumerators, tp.enumvalues):
- prnt(' if (%s != %d) {' % (enumerator, enumvalue))
+ if enumvalue < 0:
+ prnt(' if ((%s) >= 0 || (long)(%s) != %dL) {' % (
+ enumerator, enumerator, enumvalue))
+ else:
+ prnt(' if ((%s) < 0 || (unsigned long)(%s) != %dUL) {' % (
+ enumerator, enumerator, enumvalue))
+ prnt(' char buf[64];')
+ prnt(' if ((%s) < 0)' % enumerator)
+ prnt(' snprintf(buf, 63, "%%ld", (long)(%s));' % enumerator)
+ prnt(' else')
+ prnt(' snprintf(buf, 63, "%%lu", (unsigned long)(%s));' %
+ enumerator)
prnt(' PyErr_Format(_cffi_VerificationError,')
- prnt(' "enum %s: %s has the real value %d, '
- 'not %d",')
- prnt(' "%s", "%s", (int)%s, %d);' % (
- name, enumerator, enumerator, enumvalue))
+ prnt(' "enum %s: %s has the real value %s, '
+ 'not %s",')
+ prnt(' "%s", "%s", buf, "%d");' % (
+ name, enumerator, enumvalue))
prnt(' return -1;')
prnt(' }')
prnt(' return %s;' % self._chained_list_constants[True])
@@ -677,15 +704,16 @@
def _generate_cpy_variable_collecttype(self, tp, name):
if isinstance(tp, model.ArrayType):
- self._do_collect_type(tp)
+ tp_ptr = model.PointerType(tp.item)
else:
tp_ptr = model.PointerType(tp)
- self._do_collect_type(tp_ptr)
+ self._do_collect_type(tp_ptr)
def _generate_cpy_variable_decl(self, tp, name):
if isinstance(tp, model.ArrayType):
tp_ptr = model.PointerType(tp.item)
- self._generate_cpy_const(False, name, tp, vartp=tp_ptr)
+ self._generate_cpy_const(False, name, tp, vartp=tp_ptr,
+ size_too = (tp.length == '...'))
else:
tp_ptr = model.PointerType(tp)
self._generate_cpy_const(False, name, tp_ptr, category='var')
@@ -694,11 +722,29 @@
_loading_cpy_variable = _loaded_noop
def _loaded_cpy_variable(self, tp, name, module, library):
+ value = getattr(library, name)
if isinstance(tp, model.ArrayType): # int a[5] is "constant" in the
- return # sense that "a=..." is forbidden
+ # sense that "a=..." is forbidden
+ if tp.length == '...':
+ assert isinstance(value, tuple)
+ (value, size) = value
+ BItemType = self.ffi._get_cached_btype(tp.item)
+ length, rest = divmod(size, self.ffi.sizeof(BItemType))
+ if rest != 0:
+ raise ffiplatform.VerificationError(
+ "bad size: %r does not seem to be an array of %s" %
+ (name, tp.item))
+ tp = tp.resolve_length(length)
+ # 'value' is a <cdata 'type *'> which we have to replace with
+ # a <cdata 'type[N]'> if the N is actually known
+ if tp.length is not None:
+ BArray = self.ffi._get_cached_btype(tp)
+ value = self.ffi.cast(BArray, value)
+ setattr(library, name, value)
+ return
# remove ptr=<cdata 'int *'> from the library instance, and replace
# it by a property on the class, which reads/writes into ptr[0].
- ptr = getattr(library, name)
+ ptr = value
delattr(library, name)
def getter(library):
return ptr[0]
@@ -711,12 +757,9 @@
def _generate_setup_custom(self):
prnt = self._prnt
- prnt('static PyObject *_cffi_setup_custom(PyObject *lib)')
+ prnt('static int _cffi_setup_custom(PyObject *lib)')
prnt('{')
- prnt(' if (%s < 0)' % self._chained_list_constants[True])
- prnt(' return NULL;')
- prnt(' Py_INCREF(Py_None);')
- prnt(' return Py_None;')
+ prnt(' return %s;' % self._chained_list_constants[True])
prnt('}')
cffimod_header = r'''
@@ -834,17 +877,20 @@
static void *_cffi_exports[_CFFI_NUM_EXPORTS];
static PyObject *_cffi_types, *_cffi_VerificationError;
-static PyObject *_cffi_setup_custom(PyObject *lib); /* forward */
+static int _cffi_setup_custom(PyObject *lib); /* forward */
static PyObject *_cffi_setup(PyObject *self, PyObject *args)
{
PyObject *library;
+ int was_alive = (_cffi_types != NULL);
if (!PyArg_ParseTuple(args, "OOO", &_cffi_types, &_cffi_VerificationError,
&library))
return NULL;
Py_INCREF(_cffi_types);
Py_INCREF(_cffi_VerificationError);
- return _cffi_setup_custom(library);
+ if (_cffi_setup_custom(library) < 0)
+ return NULL;
+ return PyBool_FromLong(was_alive);
}
static void _cffi_init(void)
diff --git a/lib_pypy/cffi/vengine_gen.py b/lib_pypy/cffi/vengine_gen.py
--- a/lib_pypy/cffi/vengine_gen.py
+++ b/lib_pypy/cffi/vengine_gen.py
@@ -20,15 +20,15 @@
# up in kwds['export_symbols'].
kwds.setdefault('export_symbols', self.export_symbols)
- def find_module(self, module_name, path, so_suffix):
- basename = module_name + so_suffix
- if path is None:
- path = sys.path
- for dirname in path:
- filename = os.path.join(dirname, basename)
- if os.path.isfile(filename):
- return filename
- return None
+ def find_module(self, module_name, path, so_suffixes):
+ for so_suffix in so_suffixes:
+ basename = module_name + so_suffix
+ if path is None:
+ path = sys.path
+ for dirname in path:
+ filename = os.path.join(dirname, basename)
+ if os.path.isfile(filename):
+ return filename
def collect_types(self):
pass # not needed in the generic engine
@@ -173,6 +173,7 @@
newfunction = self._load_constant(False, tp, name, module)
else:
indirections = []
+ base_tp = tp
if any(isinstance(typ, model.StructOrUnion) for typ in tp.args):
indirect_args = []
for i, typ in enumerate(tp.args):
@@ -186,16 +187,18 @@
wrappername = '_cffi_f_%s' % name
newfunction = module.load_function(BFunc, wrappername)
for i, typ in indirections:
- newfunction = self._make_struct_wrapper(newfunction, i, typ)
+ newfunction = self._make_struct_wrapper(newfunction, i, typ,
+ base_tp)
setattr(library, name, newfunction)
type(library)._cffi_dir.append(name)
- def _make_struct_wrapper(self, oldfunc, i, tp):
+ def _make_struct_wrapper(self, oldfunc, i, tp, base_tp):
backend = self.ffi._backend
BType = self.ffi._get_cached_btype(tp)
def newfunc(*args):
args = args[:i] + (backend.newp(BType, args[i]),) + args[i+1:]
return oldfunc(*args)
+ newfunc._cffi_base_type = base_tp
return newfunc
# ----------
@@ -252,11 +255,14 @@
prnt(' static ssize_t nums[] = {')
prnt(' sizeof(%s),' % cname)
prnt(' offsetof(struct _cffi_aligncheck, y),')
- for fname, _, fbitsize in tp.enumfields():
+ for fname, ftype, fbitsize in tp.enumfields():
if fbitsize >= 0:
continue # xxx ignore fbitsize for now
prnt(' offsetof(%s, %s),' % (cname, fname))
- prnt(' sizeof(((%s *)0)->%s),' % (cname, fname))
+ if isinstance(ftype, model.ArrayType) and ftype.length is None:
+ prnt(' 0, /* %s */' % ftype._get_c_name())
+ else:
+ prnt(' sizeof(((%s *)0)->%s),' % (cname, fname))
prnt(' -1')
prnt(' };')
prnt(' return nums[i];')
@@ -270,7 +276,7 @@
return # nothing to do with opaque structs
layoutfuncname = '_cffi_layout_%s_%s' % (prefix, name)
#
- BFunc = self.ffi.typeof("ssize_t(*)(ssize_t)")
+ BFunc = self.ffi._typeof_locked("ssize_t(*)(ssize_t)")[0]
function = module.load_function(BFunc, layoutfuncname)
layout = []
num = 0
@@ -279,7 +285,7 @@
if x < 0: break
layout.append(x)
num += 1
- if isinstance(tp, model.StructType) and tp.partial:
+ if isinstance(tp, model.StructOrUnion) and tp.partial:
# use the function()'s sizes and offsets to guide the
# layout of the struct
totalsize = layout[0]
@@ -316,9 +322,10 @@
continue # xxx ignore fbitsize for now
check(layout[i], ffi.offsetof(BStruct, fname),
"wrong offset for field %r" % (fname,))
- BField = ffi._get_cached_btype(ftype)
- check(layout[i+1], ffi.sizeof(BField),
- "wrong size for field %r" % (fname,))
+ if layout[i+1] != 0:
+ BField = ffi._get_cached_btype(ftype)
+ check(layout[i+1], ffi.sizeof(BField),
+ "wrong size for field %r" % (fname,))
i += 2
assert i == len(layout)
@@ -379,15 +386,17 @@
def _load_constant(self, is_int, tp, name, module):
funcname = '_cffi_const_%s' % name
if is_int:
- BFunc = self.ffi.typeof("int(*)(long long*)")
+ BType = self.ffi._typeof_locked("long long*")[0]
+ BFunc = self.ffi._typeof_locked("int(*)(long long*)")[0]
function = module.load_function(BFunc, funcname)
- p = self.ffi.new("long long*")
+ p = self.ffi.new(BType)
negative = function(p)
value = int(p[0])
if value < 0 and not negative:
- value += (1 << (8*self.ffi.sizeof("long long")))
+ BLongLong = self.ffi._typeof_locked("long long")[0]
+ value += (1 << (8*self.ffi.sizeof(BLongLong)))
else:
- BFunc = self.ffi.typeof(tp.get_c_name('(*)(void)', name))
+ BFunc = self.ffi._typeof_locked(tp.get_c_name('(*)(void)', name))[0]
function = module.load_function(BFunc, funcname)
value = function()
return value
@@ -413,11 +422,22 @@
prnt('int %s(char *out_error)' % funcname)
prnt('{')
for enumerator, enumvalue in zip(tp.enumerators, tp.enumvalues):
- prnt(' if (%s != %d) {' % (enumerator, enumvalue))
+ if enumvalue < 0:
+ prnt(' if ((%s) >= 0 || (long)(%s) != %dL) {' % (
+ enumerator, enumerator, enumvalue))
+ else:
+ prnt(' if ((%s) < 0 || (unsigned long)(%s) != %dUL) {' % (
+ enumerator, enumerator, enumvalue))
+ prnt(' char buf[64];')
+ prnt(' if ((%s) < 0)' % enumerator)
+ prnt(' snprintf(buf, 63, "%%ld", (long)(%s));' % enumerator)
+ prnt(' else')
+ prnt(' snprintf(buf, 63, "%%lu", (unsigned long)(%s));' %
+ enumerator)
prnt(' snprintf(out_error, 255,'
- '"%s has the real value %d, not %d",')
- prnt(' "%s", (int)%s, %d);' % (
- enumerator, enumerator, enumvalue))
+ ' "%s has the real value %s, not %s",')
+ prnt(' "%s", buf, "%d");' % (
+ enumerator, enumvalue))
prnt(' return -1;')
prnt(' }')
prnt(' return 0;')
@@ -431,10 +451,11 @@
tp.enumvalues = tuple(enumvalues)
tp.partial_resolved = True
else:
- BFunc = self.ffi.typeof("int(*)(char*)")
+ BType = self.ffi._typeof_locked("char[]")[0]
+ BFunc = self.ffi._typeof_locked("int(*)(char*)")[0]
funcname = '_cffi_e_%s_%s' % (prefix, name)
function = module.load_function(BFunc, funcname)
- p = self.ffi.new("char[]", 256)
+ p = self.ffi.new(BType, 256)
if function(p) < 0:
error = self.ffi.string(p)
if sys.version_info >= (3,):
@@ -465,6 +486,14 @@
def _generate_gen_variable_decl(self, tp, name):
if isinstance(tp, model.ArrayType):
+ if tp.length == '...':
+ prnt = self._prnt
+ funcname = '_cffi_sizeof_%s' % (name,)
+ self.export_symbols.append(funcname)
+ prnt("size_t %s(void)" % funcname)
+ prnt("{")
+ prnt(" return sizeof(%s);" % (name,))
+ prnt("}")
tp_ptr = model.PointerType(tp.item)
self._generate_gen_const(False, name, tp_ptr)
else:
@@ -476,6 +505,18 @@
def _loaded_gen_variable(self, tp, name, module, library):
if isinstance(tp, model.ArrayType): # int a[5] is "constant" in the
# sense that "a=..." is forbidden
+ if tp.length == '...':
+ funcname = '_cffi_sizeof_%s' % (name,)
+ BFunc = self.ffi._typeof_locked('size_t(*)(void)')[0]
+ function = module.load_function(BFunc, funcname)
+ size = function()
+ BItemType = self.ffi._get_cached_btype(tp.item)
+ length, rest = divmod(size, self.ffi.sizeof(BItemType))
+ if rest != 0:
+ raise ffiplatform.VerificationError(
+ "bad size: %r does not seem to be an array of %s" %
+ (name, tp.item))
+ tp = tp.resolve_length(length)
tp_ptr = model.PointerType(tp.item)
value = self._load_constant(False, tp_ptr, name, module)
# 'value' is a <cdata 'type *'> which we have to replace with
@@ -489,7 +530,7 @@
# remove ptr=<cdata 'int *'> from the library instance, and replace
# it by a property on the class, which reads/writes into ptr[0].
funcname = '_cffi_var_%s' % name
- BFunc = self.ffi.typeof(tp.get_c_name('*(*)(void)', name))
+ BFunc = self.ffi._typeof_locked(tp.get_c_name('*(*)(void)', name))[0]
function = module.load_function(BFunc, funcname)
ptr = function()
def getter(library):
diff --git a/lib_pypy/cffi/verifier.py b/lib_pypy/cffi/verifier.py
--- a/lib_pypy/cffi/verifier.py
+++ b/lib_pypy/cffi/verifier.py
@@ -31,7 +31,7 @@
k2 = k2.lstrip('0').rstrip('L')
modulename = '_cffi_%s_%s%s%s' % (tag, self._vengine._class_key,
k1, k2)
- suffix = _get_so_suffix()
+ suffix = _get_so_suffixes()[0]
self.tmpdir = tmpdir or _caller_dir_pycache()
self.sourcefilename = os.path.join(self.tmpdir, modulename + '.c')
self.modulefilename = os.path.join(self.tmpdir, modulename + suffix)
@@ -42,18 +42,21 @@
def write_source(self, file=None):
"""Write the C source code. It is produced in 'self.sourcefilename',
which can be tweaked beforehand."""
- if self._has_source and file is None:
- raise ffiplatform.VerificationError("source code already written")
- self._write_source(file)
+ with self.ffi._lock:
+ if self._has_source and file is None:
+ raise ffiplatform.VerificationError(
+ "source code already written")
+ self._write_source(file)
def compile_module(self):
"""Write the C source code (if not done already) and compile it.
This produces a dynamic link library in 'self.modulefilename'."""
- if self._has_module:
- raise ffiplatform.VerificationError("module already compiled")
- if not self._has_source:
- self._write_source()
- self._compile_module()
+ with self.ffi._lock:
+ if self._has_module:
+ raise ffiplatform.VerificationError("module already compiled")
+ if not self._has_source:
+ self._write_source()
+ self._compile_module()
def load_library(self):
"""Get a C module from this Verifier instance.
@@ -62,11 +65,14 @@
operations to the C module. If necessary, the C code is written
and compiled first.
"""
- if not self._has_module:
- self._locate_module()
+ with self.ffi._lock:
if not self._has_module:
- self.compile_module()
- return self._load_library()
+ self._locate_module()
+ if not self._has_module:
+ if not self._has_source:
+ self._write_source()
+ self._compile_module()
+ return self._load_library()
def get_module_name(self):
basename = os.path.basename(self.modulefilename)
@@ -81,7 +87,9 @@
def get_extension(self):
if not self._has_source:
- self._write_source()
+ with self.ffi._lock:
+ if not self._has_source:
+ self._write_source()
sourcename = ffiplatform.maybe_relative_path(self.sourcefilename)
modname = self.get_module_name()
return ffiplatform.get_extension(sourcename, modname, **self.kwds)
@@ -103,7 +111,7 @@
else:
path = None
filename = self._vengine.find_module(self.get_module_name(), path,
- _get_so_suffix())
+ _get_so_suffixes())
if filename is None:
return
self.modulefilename = filename
@@ -193,7 +201,7 @@
if keep_so:
suffix = '.c' # only remove .c files
else:
- suffix = _get_so_suffix().lower()
+ suffix = _get_so_suffixes()[0].lower()
for fn in filelist:
if fn.lower().startswith('_cffi_') and (
fn.lower().endswith(suffix) or fn.lower().endswith('.c')):
@@ -213,15 +221,20 @@
except OSError:
pass
-def _get_so_suffix():
+def _get_so_suffixes():
+ suffixes = []
for suffix, mode, type in imp.get_suffixes():
if type == imp.C_EXTENSION:
- return suffix
- # bah, no C_EXTENSION available. Occurs on pypy without cpyext
- if sys.platform == 'win32':
- return ".pyd"
- else:
- return ".so"
+ suffixes.append(suffix)
+
+ if not suffixes:
+ # bah, no C_EXTENSION available. Occurs on pypy without cpyext
+ if sys.platform == 'win32':
+ suffixes = [".pyd"]
+ else:
+ suffixes = [".so"]
+
+ return suffixes
def _ensure_dir(filename):
try:
diff --git a/lib_pypy/numpypy/__init__.py b/lib_pypy/numpypy/__init__.py
deleted file mode 100644
--- a/lib_pypy/numpypy/__init__.py
+++ /dev/null
@@ -1,17 +0,0 @@
-from . import core
-from .core import *
-from . import lib
-from .lib import *
-
-from builtins import bool, int, int, float, complex, object, str, str
-
-from .core import round, abs, max, min
-
-__version__ = '1.7.0'
-
-__all__ = ['__version__']
-__all__ += core.__all__
-__all__ += lib.__all__
-
-#import sys
-#sys.modules.setdefault('numpy', sys.modules['numpypy'])
diff --git a/lib_pypy/numpypy/core/__init__.py b/lib_pypy/numpypy/core/__init__.py
deleted file mode 100644
--- a/lib_pypy/numpypy/core/__init__.py
+++ /dev/null
@@ -1,19 +0,0 @@
-
-
-from . import multiarray
-from . import umath
-from . import numeric
-from .numeric import *
-from . import fromnumeric
-from .fromnumeric import *
-from . import shape_base
-from .shape_base import *
-
-from .fromnumeric import amax as max, amin as min, \
- round_ as round
-from .numeric import absolute as abs
-
-__all__ = []
-__all__ += numeric.__all__
-__all__ += fromnumeric.__all__
-__all__ += shape_base.__all__
diff --git a/lib_pypy/numpypy/core/_methods.py b/lib_pypy/numpypy/core/_methods.py
deleted file mode 100644
--- a/lib_pypy/numpypy/core/_methods.py
+++ /dev/null
@@ -1,109 +0,0 @@
-# Array methods which are called by the both the C-code for the method
-# and the Python code for the NumPy-namespace function
-
-from . import multiarray as mu
-from . import umath as um
-from .numeric import asanyarray
-
-def _amax(a, axis=None, out=None, keepdims=False):
- return um.maximum.reduce(a, axis=axis,
- out=out, keepdims=keepdims)
-
-def _amin(a, axis=None, out=None, keepdims=False):
- return um.minimum.reduce(a, axis=axis,
- out=out, keepdims=keepdims)
-
-def _sum(a, axis=None, dtype=None, out=None, keepdims=False):
- return um.add.reduce(a, axis=axis, dtype=dtype,
- out=out, keepdims=keepdims)
-
-def _prod(a, axis=None, dtype=None, out=None, keepdims=False):
- return um.multiply.reduce(a, axis=axis, dtype=dtype,
- out=out, keepdims=keepdims)
-
-def _any(a, axis=None, dtype=None, out=None, keepdims=False):
- return um.logical_or.reduce(a, axis=axis, dtype=dtype, out=out,
- keepdims=keepdims)
-
-def _all(a, axis=None, dtype=None, out=None, keepdims=False):
- return um.logical_and.reduce(a, axis=axis, dtype=dtype, out=out,
- keepdims=keepdims)
-
-def _count_reduce_items(arr, axis):
- if axis is None:
- axis = tuple(range(arr.ndim))
- if not isinstance(axis, tuple):
- axis = (axis,)
- items = 1
- for ax in axis:
- items *= arr.shape[ax]
- return items
-
-def _mean(a, axis=None, dtype=None, out=None, keepdims=False):
- arr = asanyarray(a)
-
- # Upgrade bool, unsigned int, and int to float64
- if dtype is None and arr.dtype.kind in ['b','u','i']:
- ret = um.add.reduce(arr, axis=axis, dtype='f8',
- out=out, keepdims=keepdims)
- else:
- ret = um.add.reduce(arr, axis=axis, dtype=dtype,
- out=out, keepdims=keepdims)
- rcount = _count_reduce_items(arr, axis)
- if isinstance(ret, mu.ndarray):
- ret = um.true_divide(ret, rcount,
- out=ret, casting='unsafe', subok=False)
- else:
- ret = ret / float(rcount)
- return ret
-
-def _var(a, axis=None, dtype=None, out=None, ddof=0,
- keepdims=False):
- arr = asanyarray(a)
-
- # First compute the mean, saving 'rcount' for reuse later
- if dtype is None and arr.dtype.kind in ['b','u','i']:
- arrmean = um.add.reduce(arr, axis=axis, dtype='f8', keepdims=True)
- else:
- arrmean = um.add.reduce(arr, axis=axis, dtype=dtype, keepdims=True)
- rcount = _count_reduce_items(arr, axis)
- if isinstance(arrmean, mu.ndarray):
- arrmean = um.true_divide(arrmean, rcount,
- out=arrmean, casting='unsafe', subok=False)
- else:
- arrmean = arrmean / float(rcount)
-
- # arr - arrmean
- x = arr - arrmean
-
- # (arr - arrmean) ** 2
- if arr.dtype.kind == 'c':
- x = um.multiply(x, um.conjugate(x), out=x).real
- else:
- x = um.multiply(x, x, out=x)
-
- # add.reduce((arr - arrmean) ** 2, axis)
- ret = um.add.reduce(x, axis=axis, dtype=dtype, out=out, keepdims=keepdims)
-
- # add.reduce((arr - arrmean) ** 2, axis) / (n - ddof)
- if not keepdims and isinstance(rcount, mu.ndarray):
- rcount = rcount.squeeze(axis=axis)
- rcount -= ddof
- if isinstance(ret, mu.ndarray):
- ret = um.true_divide(ret, rcount,
- out=ret, casting='unsafe', subok=False)
- else:
- ret = ret / float(rcount)
-
- return ret
-
-def _std(a, axis=None, dtype=None, out=None, ddof=0, keepdims=False):
- ret = _var(a, axis=axis, dtype=dtype, out=out, ddof=ddof,
- keepdims=keepdims)
-
- if isinstance(ret, mu.ndarray):
- ret = um.sqrt(ret, out=ret)
- else:
- ret = um.sqrt(ret)
-
- return ret
diff --git a/lib_pypy/numpypy/core/arrayprint.py b/lib_pypy/numpypy/core/arrayprint.py
deleted file mode 100644
--- a/lib_pypy/numpypy/core/arrayprint.py
+++ /dev/null
@@ -1,751 +0,0 @@
-"""Array printing function
-
-$Id: arrayprint.py,v 1.9 2005/09/13 13:58:44 teoliphant Exp $
-"""
-__all__ = ["array2string", "set_printoptions", "get_printoptions"]
-__docformat__ = 'restructuredtext'
-
-#
-# Written by Konrad Hinsen <hinsenk at ere.umontreal.ca>
-# last revision: 1996-3-13
-# modified by Jim Hugunin 1997-3-3 for repr's and str's (and other details)
-# and by Perry Greenfield 2000-4-1 for numarray
-# and by Travis Oliphant 2005-8-22 for numpy
-
-import sys
-from . import numerictypes as _nt
-from .umath import maximum, minimum, absolute, not_equal, isnan, isinf
-#from multiarray import format_longfloat, datetime_as_string, datetime_data
-from .fromnumeric import ravel
-
-
-def product(x, y): return x*y
-
-_summaryEdgeItems = 3 # repr N leading and trailing items of each dimension
-_summaryThreshold = 1000 # total items > triggers array summarization
-
-_float_output_precision = 8
-_float_output_suppress_small = False
-_line_width = 75
-_nan_str = 'nan'
-_inf_str = 'inf'
-_formatter = None # formatting function for array elements
-
-if sys.version_info[0] >= 3:
- from functools import reduce
-
-def set_printoptions(precision=None, threshold=None, edgeitems=None,
- linewidth=None, suppress=None,
- nanstr=None, infstr=None,
- formatter=None):
- """
- Set printing options.
-
- These options determine the way floating point numbers, arrays and
- other NumPy objects are displayed.
-
- Parameters
- ----------
- precision : int, optional
- Number of digits of precision for floating point output (default 8).
- threshold : int, optional
- Total number of array elements which trigger summarization
- rather than full repr (default 1000).
- edgeitems : int, optional
- Number of array items in summary at beginning and end of
- each dimension (default 3).
- linewidth : int, optional
- The number of characters per line for the purpose of inserting
- line breaks (default 75).
- suppress : bool, optional
- Whether or not suppress printing of small floating point values
- using scientific notation (default False).
- nanstr : str, optional
- String representation of floating point not-a-number (default nan).
- infstr : str, optional
- String representation of floating point infinity (default inf).
- formatter : dict of callables, optional
- If not None, the keys should indicate the type(s) that the respective
- formatting function applies to. Callables should return a string.
- Types that are not specified (by their corresponding keys) are handled
- by the default formatters. Individual types for which a formatter
- can be set are::
-
- - 'bool'
- - 'int'
- - 'timedelta' : a `numpy.timedelta64`
- - 'datetime' : a `numpy.datetime64`
- - 'float'
- - 'longfloat' : 128-bit floats
- - 'complexfloat'
- - 'longcomplexfloat' : composed of two 128-bit floats
- - 'numpy_str' : types `numpy.string_` and `numpy.unicode_`
- - 'str' : all other strings
-
- Other keys that can be used to set a group of types at once are::
-
- - 'all' : sets all types
- - 'int_kind' : sets 'int'
- - 'float_kind' : sets 'float' and 'longfloat'
- - 'complex_kind' : sets 'complexfloat' and 'longcomplexfloat'
- - 'str_kind' : sets 'str' and 'numpystr'
-
- See Also
- --------
- get_printoptions, set_string_function, array2string
-
- Notes
- -----
- `formatter` is always reset with a call to `set_printoptions`.
-
- Examples
- --------
- Floating point precision can be set:
-
- >>> np.set_printoptions(precision=4)
- >>> print np.array([1.123456789])
- [ 1.1235]
-
- Long arrays can be summarised:
-
- >>> np.set_printoptions(threshold=5)
- >>> print np.arange(10)
- [0 1 2 ..., 7 8 9]
-
- Small results can be suppressed:
-
- >>> eps = np.finfo(float).eps
- >>> x = np.arange(4.)
- >>> x**2 - (x + eps)**2
- array([ -4.9304e-32, -4.4409e-16, 0.0000e+00, 0.0000e+00])
- >>> np.set_printoptions(suppress=True)
- >>> x**2 - (x + eps)**2
- array([-0., -0., 0., 0.])
-
- A custom formatter can be used to display array elements as desired:
-
- >>> np.set_printoptions(formatter={'all':lambda x: 'int: '+str(-x)})
- >>> x = np.arange(3)
- >>> x
- array([int: 0, int: -1, int: -2])
- >>> np.set_printoptions() # formatter gets reset
- >>> x
- array([0, 1, 2])
-
- To put back the default options, you can use:
-
- >>> np.set_printoptions(edgeitems=3,infstr='inf',
- ... linewidth=75, nanstr='nan', precision=8,
- ... suppress=False, threshold=1000, formatter=None)
- """
-
- global _summaryThreshold, _summaryEdgeItems, _float_output_precision, \
- _line_width, _float_output_suppress_small, _nan_str, _inf_str, \
- _formatter
- if linewidth is not None:
- _line_width = linewidth
- if threshold is not None:
- _summaryThreshold = threshold
- if edgeitems is not None:
- _summaryEdgeItems = edgeitems
- if precision is not None:
- _float_output_precision = precision
- if suppress is not None:
- _float_output_suppress_small = not not suppress
- if nanstr is not None:
- _nan_str = nanstr
- if infstr is not None:
- _inf_str = infstr
- _formatter = formatter
-
-def get_printoptions():
- """
- Return the current print options.
-
- Returns
- -------
- print_opts : dict
- Dictionary of current print options with keys
-
- - precision : int
- - threshold : int
- - edgeitems : int
- - linewidth : int
- - suppress : bool
- - nanstr : str
- - infstr : str
- - formatter : dict of callables
-
- For a full description of these options, see `set_printoptions`.
-
- See Also
- --------
- set_printoptions, set_string_function
-
- """
- d = dict(precision=_float_output_precision,
- threshold=_summaryThreshold,
- edgeitems=_summaryEdgeItems,
- linewidth=_line_width,
- suppress=_float_output_suppress_small,
- nanstr=_nan_str,
- infstr=_inf_str,
- formatter=_formatter)
- return d
-
-def _leading_trailing(a):
- from . import numeric as _nc
- if a.ndim == 1:
- if len(a) > 2*_summaryEdgeItems:
- b = _nc.concatenate((a[:_summaryEdgeItems],
- a[-_summaryEdgeItems:]))
- else:
- b = a
- else:
- if len(a) > 2*_summaryEdgeItems:
- l = [_leading_trailing(a[i]) for i in range(
- min(len(a), _summaryEdgeItems))]
- l.extend([_leading_trailing(a[-i]) for i in range(
- min(len(a), _summaryEdgeItems),0,-1)])
- else:
- l = [_leading_trailing(a[i]) for i in range(0, len(a))]
- b = _nc.concatenate(tuple(l))
- return b
-
-def _boolFormatter(x):
- if x:
- return ' True'
- else:
- return 'False'
-
-
-def repr_format(x):
- return repr(x)
-
-def _array2string(a, max_line_width, precision, suppress_small, separator=' ',
- prefix="", formatter=None):
-
- if max_line_width is None:
- max_line_width = _line_width
-
- if precision is None:
- precision = _float_output_precision
-
- if suppress_small is None:
- suppress_small = _float_output_suppress_small
-
- if formatter is None:
- formatter = _formatter
-
- if a.size > _summaryThreshold:
- summary_insert = "..., "
- data = _leading_trailing(a)
- else:
- summary_insert = ""
- data = ravel(a)
-
- formatdict = {'bool' : _boolFormatter,
- 'int' : IntegerFormat(data),
- 'float' : FloatFormat(data, precision, suppress_small),
- 'longfloat' : FloatFormat(data, precision, suppress_small),
- 'complexfloat' : ComplexFormat(data, precision,
- suppress_small),
- 'longcomplexfloat' : ComplexFormat(data, precision,
- suppress_small),
- 'datetime' : DatetimeFormat(data),
- 'timedelta' : TimedeltaFormat(data),
- 'numpystr' : repr_format,
- 'str' : str}
-
- if formatter is not None:
- fkeys = [k for k in list(formatter.keys()) if formatter[k] is not None]
- if 'all' in fkeys:
- for key in list(formatdict.keys()):
- formatdict[key] = formatter['all']
- if 'int_kind' in fkeys:
- for key in ['int']:
- formatdict[key] = formatter['int_kind']
- if 'float_kind' in fkeys:
- for key in ['float', 'longfloat']:
- formatdict[key] = formatter['float_kind']
- if 'complex_kind' in fkeys:
- for key in ['complexfloat', 'longcomplexfloat']:
- formatdict[key] = formatter['complex_kind']
- if 'str_kind' in fkeys:
- for key in ['numpystr', 'str']:
- formatdict[key] = formatter['str_kind']
- for key in list(formatdict.keys()):
- if key in fkeys:
- formatdict[key] = formatter[key]
-
- try:
- format_function = a._format
- msg = "The `_format` attribute is deprecated in Numpy 2.0 and " \
- "will be removed in 2.1. Use the `formatter` kw instead."
- import warnings
- warnings.warn(msg, DeprecationWarning)
- except AttributeError:
- # find the right formatting function for the array
- dtypeobj = a.dtype.type
- if issubclass(dtypeobj, _nt.bool_):
- format_function = formatdict['bool']
- elif issubclass(dtypeobj, _nt.integer):
- #if issubclass(dtypeobj, _nt.timedelta64):
- # format_function = formatdict['timedelta']
- #else:
- format_function = formatdict['int']
- elif issubclass(dtypeobj, _nt.floating):
- if hasattr(_nt, 'longfloat') and issubclass(dtypeobj, _nt.longfloat):
- format_function = formatdict['longfloat']
- else:
- format_function = formatdict['float']
- elif issubclass(dtypeobj, _nt.complexfloating):
- if hasattr(_nt, 'clongfloat') and issubclass(dtypeobj, _nt.clongfloat):
- format_function = formatdict['longcomplexfloat']
- else:
- format_function = formatdict['complexfloat']
- elif issubclass(dtypeobj, (_nt.unicode_, _nt.string_)):
- format_function = formatdict['numpystr']
- #elif issubclass(dtypeobj, _nt.datetime64):
- # format_function = formatdict['datetime']
- else:
- format_function = formatdict['str']
-
- # skip over "["
- next_line_prefix = " "
- # skip over array(
- next_line_prefix += " "*len(prefix)
-
- lst = _formatArray(a, format_function, len(a.shape), max_line_width,
- next_line_prefix, separator,
- _summaryEdgeItems, summary_insert)[:-1]
- return lst
-
-def _convert_arrays(obj):
- from . import numeric as _nc
- newtup = []
- for k in obj:
- if isinstance(k, _nc.ndarray):
- k = k.tolist()
- elif isinstance(k, tuple):
- k = _convert_arrays(k)
- newtup.append(k)
- return tuple(newtup)
-
-
-def array2string(a, max_line_width=None, precision=None,
- suppress_small=None, separator=' ', prefix="",
- style=repr, formatter=None):
- """
- Return a string representation of an array.
-
- Parameters
- ----------
- a : ndarray
- Input array.
- max_line_width : int, optional
- The maximum number of columns the string should span. Newline
- characters splits the string appropriately after array elements.
- precision : int, optional
- Floating point precision. Default is the current printing
- precision (usually 8), which can be altered using `set_printoptions`.
- suppress_small : bool, optional
- Represent very small numbers as zero. A number is "very small" if it
- is smaller than the current printing precision.
- separator : str, optional
- Inserted between elements.
- prefix : str, optional
- An array is typically printed as::
-
- 'prefix(' + array2string(a) + ')'
-
- The length of the prefix string is used to align the
- output correctly.
- style : function, optional
- A function that accepts an ndarray and returns a string. Used only
- when the shape of `a` is equal to ``()``, i.e. for 0-D arrays.
- formatter : dict of callables, optional
- If not None, the keys should indicate the type(s) that the respective
- formatting function applies to. Callables should return a string.
- Types that are not specified (by their corresponding keys) are handled
- by the default formatters. Individual types for which a formatter
- can be set are::
-
- - 'bool'
- - 'int'
- - 'timedelta' : a `numpy.timedelta64`
- - 'datetime' : a `numpy.datetime64`
- - 'float'
- - 'longfloat' : 128-bit floats
- - 'complexfloat'
- - 'longcomplexfloat' : composed of two 128-bit floats
- - 'numpy_str' : types `numpy.string_` and `numpy.unicode_`
- - 'str' : all other strings
-
- Other keys that can be used to set a group of types at once are::
-
- - 'all' : sets all types
- - 'int_kind' : sets 'int'
- - 'float_kind' : sets 'float' and 'longfloat'
- - 'complex_kind' : sets 'complexfloat' and 'longcomplexfloat'
- - 'str_kind' : sets 'str' and 'numpystr'
-
- Returns
- -------
- array_str : str
- String representation of the array.
-
- Raises
- ------
- TypeError : if a callable in `formatter` does not return a string.
-
- See Also
- --------
- array_str, array_repr, set_printoptions, get_printoptions
-
- Notes
- -----
- If a formatter is specified for a certain type, the `precision` keyword is
- ignored for that type.
-
- Examples
- --------
- >>> x = np.array([1e-16,1,2,3])
- >>> print np.array2string(x, precision=2, separator=',',
- ... suppress_small=True)
- [ 0., 1., 2., 3.]
-
- >>> x = np.arange(3.)
- >>> np.array2string(x, formatter={'float_kind':lambda x: "%.2f" % x})
- '[0.00 1.00 2.00]'
-
- >>> x = np.arange(3)
- >>> np.array2string(x, formatter={'int':lambda x: hex(x)})
- '[0x0L 0x1L 0x2L]'
-
- """
-
- if a.shape == ():
- x = a.item()
- try:
- lst = a._format(x)
- msg = "The `_format` attribute is deprecated in Numpy " \
- "2.0 and will be removed in 2.1. Use the " \
- "`formatter` kw instead."
- import warnings
- warnings.warn(msg, DeprecationWarning)
- except AttributeError:
- if isinstance(x, tuple):
- x = _convert_arrays(x)
- lst = style(x)
- elif reduce(product, a.shape) == 0:
- # treat as a null array if any of shape elements == 0
- lst = "[]"
- else:
- lst = _array2string(a, max_line_width, precision, suppress_small,
- separator, prefix, formatter=formatter)
- return lst
-
-def _extendLine(s, line, word, max_line_len, next_line_prefix):
- if len(line.rstrip()) + len(word.rstrip()) >= max_line_len:
- s += line.rstrip() + "\n"
- line = next_line_prefix
- line += word
- return s, line
-
-
-def _formatArray(a, format_function, rank, max_line_len,
- next_line_prefix, separator, edge_items, summary_insert):
- """formatArray is designed for two modes of operation:
-
- 1. Full output
-
- 2. Summarized output
-
- """
- if rank == 0:
- obj = a.item()
- if isinstance(obj, tuple):
- obj = _convert_arrays(obj)
- return str(obj)
-
- if summary_insert and 2*edge_items < len(a):
- leading_items, trailing_items, summary_insert1 = \
- edge_items, edge_items, summary_insert
- else:
- leading_items, trailing_items, summary_insert1 = 0, len(a), ""
-
- if rank == 1:
- s = ""
- line = next_line_prefix
- for i in range(leading_items):
- word = format_function(a[i]) + separator
- s, line = _extendLine(s, line, word, max_line_len, next_line_prefix)
-
- if summary_insert1:
- s, line = _extendLine(s, line, summary_insert1, max_line_len, next_line_prefix)
More information about the pypy-commit
mailing list