Filename	Size	Date	own tests	applevel tests
{{ d.text }}	{{ d.size }}	{{ d.type }}	{{ d.encoding }}
{{ f.text }}	{{ f.size }}	{{ f.date }}	{{ f.own_summary }}	{{ f.app_summary }}

Filename	Size
{{ d.text }}	{{ d.size}}

Home - - - Speed - Summary (trunk) - Summary - Nightly builds + - Speed + - Numpy compatability + - Summary (trunk) + - Summary + - Nightly builds - Waterfall + - Waterfall - Builders + - Builders From noreply at buildbot.pypy.org Mon Nov 11 21:59:13 2013 From: noreply at buildbot.pypy.org (ltratt) Date: Mon, 11 Nov 2013 21:59:13 +0100 (CET) Subject: [pypy-commit] pypy more_strategies: Remove pointless print statements. Message-ID: <20131111205913.719DE1C0144@cobra.cs.uni-duesseldorf.de> Author: Laurence Tratt Branch: more_strategies Changeset: r67959:3a4f3f694fe9 Date: 2013-11-11 15:29 +0000 http://bitbucket.org/pypy/pypy/changeset/3a4f3f694fe9/ Log: Remove pointless print statements. Presumably these are long forgotten debugging aids. diff --git a/pypy/objspace/std/test/test_listobject.py b/pypy/objspace/std/test/test_listobject.py --- a/pypy/objspace/std/test/test_listobject.py +++ b/pypy/objspace/std/test/test_listobject.py @@ -1187,7 +1187,6 @@ skip("not reliable on top of Boehm") class A(object): def __del__(self): - print 'del' del lst[:] for i in range(10): keepalive = [] @@ -1257,7 +1256,6 @@ (dict, []), (dict, [(5,6)]), (dict, [('x',7)]), (dict, [(X,8)]), (dict, [(u'x', 7)]), ]: - print base, arg class SubClass(base): def __iter__(self): return iter("foobar") From noreply at buildbot.pypy.org Mon Nov 11 21:59:16 2013 From: noreply at buildbot.pypy.org (ltratt) Date: Mon, 11 Nov 2013 21:59:16 +0100 (CET) Subject: [pypy-commit] pypy more_strategies: Merge default. Message-ID: <20131111205916.89DC71C0144@cobra.cs.uni-duesseldorf.de> Author: Laurence Tratt Branch: more_strategies Changeset: r67960:21ecaa0104be Date: 2013-11-11 20:47 +0000 http://bitbucket.org/pypy/pypy/changeset/21ecaa0104be/ Log: Merge default. diff too long, truncating to 2000 out of 4612 lines diff --git a/lib-python/2.7/json/encoder.py b/lib-python/2.7/json/encoder.py --- a/lib-python/2.7/json/encoder.py +++ b/lib-python/2.7/json/encoder.py @@ -4,6 +4,21 @@ from __pypy__.builders import StringBuilder, UnicodeBuilder +class StringOrUnicodeBuilder(object): + def __init__(self): + self._builder = StringBuilder() + def append(self, string): + try: + self._builder.append(string) + except UnicodeEncodeError: + ub = UnicodeBuilder() + ub.append(self._builder.build()) + self._builder = ub + ub.append(string) + def build(self): + return self._builder.build() + + ESCAPE = re.compile(r'[\x00-\x1f\\"\b\f\n\r\t]') ESCAPE_ASCII = re.compile(r'([\\"]|[^\ -~])') HAS_UTF8 = re.compile(r'[\x80-\xff]') @@ -192,7 +207,7 @@ if self.ensure_ascii: builder = StringBuilder() else: - builder = UnicodeBuilder() + builder = StringOrUnicodeBuilder() self.__encode(o, markers, builder, 0) return builder.build() diff --git a/lib-python/2.7/weakref.py b/lib-python/2.7/weakref.py --- a/lib-python/2.7/weakref.py +++ b/lib-python/2.7/weakref.py @@ -48,7 +48,14 @@ def remove(wr, selfref=ref(self)): self = selfref() if self is not None: - del self.data[wr.key] + # Changed this for PyPy: made more resistent. The + # issue is that in some corner cases, self.data + # might already be changed or removed by the time + # this weakref's callback is called. If that is + # the case, we don't want to randomly kill an + # unrelated entry. + if self.data.get(wr.key) is wr: + del self.data[wr.key] self._remove = remove UserDict.UserDict.__init__(self, *args, **kw) @@ -160,22 +167,26 @@ try: o = self.data.pop(key)() except KeyError: + o = None + if o is None: if args: return args[0] - raise - if o is None: raise KeyError, key else: return o + # The logic above was fixed in PyPy def setdefault(self, key, default=None): try: - wr = self.data[key] + o = self.data[key]() except KeyError: + o = None + if o is None: self.data[key] = KeyedRef(default, self._remove, key) return default else: - return wr() + return o + # The logic above was fixed in PyPy def update(self, dict=None, **kwargs): d = self.data diff --git a/lib_pypy/cffi.egg-info b/lib_pypy/cffi.egg-info --- a/lib_pypy/cffi.egg-info +++ b/lib_pypy/cffi.egg-info @@ -1,6 +1,6 @@ Metadata-Version: 1.0 Name: cffi -Version: 0.7 +Version: 0.8 Summary: Foreign Function Interface for Python calling C code. Home-page: http://cffi.readthedocs.org Author: Armin Rigo, Maciej Fijalkowski diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py --- a/lib_pypy/cffi/__init__.py +++ b/lib_pypy/cffi/__init__.py @@ -4,5 +4,5 @@ from .api import FFI, CDefError, FFIError from .ffiplatform import VerificationError, VerificationMissing -__version__ = "0.7.2" -__version_info__ = (0, 7, 2) +__version__ = "0.8" +__version_info__ = (0, 8) diff --git a/lib_pypy/cffi/api.py b/lib_pypy/cffi/api.py --- a/lib_pypy/cffi/api.py +++ b/lib_pypy/cffi/api.py @@ -1,4 +1,5 @@ import types +from .lock import allocate_lock try: callable @@ -61,6 +62,7 @@ # rely on it! It's probably not going to work well.) self._backend = backend + self._lock = allocate_lock() self._parser = cparser.Parser() self._cached_btypes = {} self._parsed_types = types.ModuleType('parsed_types').__dict__ @@ -74,7 +76,8 @@ if name.startswith('RTLD_'): setattr(self, name, getattr(backend, name)) # - self.BVoidP = self._get_cached_btype(model.voidp_type) + with self._lock: + self.BVoidP = self._get_cached_btype(model.voidp_type) if isinstance(backend, types.ModuleType): # _cffi_backend: attach these constants to the class if not hasattr(FFI, 'NULL'): @@ -95,11 +98,12 @@ if not isinstance(csource, basestring): raise TypeError("cdef() argument must be a string") csource = csource.encode('ascii') - self._parser.parse(csource, override=override) - self._cdefsources.append(csource) - if override: - for cache in self._function_caches: - cache.clear() + with self._lock: + self._parser.parse(csource, override=override) + self._cdefsources.append(csource) + if override: + for cache in self._function_caches: + cache.clear() def dlopen(self, name, flags=0): """Load and return a dynamic library identified by 'name'. @@ -109,31 +113,47 @@ library we only look for the actual (untyped) symbols. """ assert isinstance(name, basestring) or name is None - lib, function_cache = _make_ffi_library(self, name, flags) - self._function_caches.append(function_cache) - self._libraries.append(lib) + with self._lock: + lib, function_cache = _make_ffi_library(self, name, flags) + self._function_caches.append(function_cache) + self._libraries.append(lib) return lib + def _typeof_locked(self, cdecl): + # call me with the lock! + key = cdecl + if key in self._parsed_types: + return self._parsed_types[key] + # + if not isinstance(cdecl, str): # unicode, on Python 2 + cdecl = cdecl.encode('ascii') + # + type = self._parser.parse_type(cdecl) + really_a_function_type = type.is_raw_function + if really_a_function_type: + type = type.as_function_pointer() + btype = self._get_cached_btype(type) + result = btype, really_a_function_type + self._parsed_types[key] = result + return result + def _typeof(self, cdecl, consider_function_as_funcptr=False): # string -> ctype object try: - btype, cfaf = self._parsed_types[cdecl] - if consider_function_as_funcptr and not cfaf: - raise KeyError + result = self._parsed_types[cdecl] except KeyError: - key = cdecl - if not isinstance(cdecl, str): # unicode, on Python 2 - cdecl = cdecl.encode('ascii') - cfaf = consider_function_as_funcptr - type = self._parser.parse_type(cdecl, - consider_function_as_funcptr=cfaf) - btype = self._get_cached_btype(type) - self._parsed_types[key] = btype, cfaf + with self._lock: + result = self._typeof_locked(cdecl) + # + btype, really_a_function_type = result + if really_a_function_type and not consider_function_as_funcptr: + raise CDefError("the type %r is a function type, not a " + "pointer-to-function type" % (cdecl,)) return btype def typeof(self, cdecl): """Parse the C type given as a string and return the - corresponding Python type: '>. + corresponding object. It can also be used on 'cdata' instance to get its C type. """ if isinstance(cdecl, basestring): @@ -144,6 +164,10 @@ res = _builtin_function_type(cdecl) if res is not None: return res + if (isinstance(cdecl, types.FunctionType) + and hasattr(cdecl, '_cffi_base_type')): + with self._lock: + return self._get_cached_btype(cdecl._cffi_base_type) raise TypeError(type(cdecl)) def sizeof(self, cdecl): @@ -280,14 +304,17 @@ data. Later, when this new cdata object is garbage-collected, 'destructor(old_cdata_object)' will be called. """ - try: - gc_weakrefs = self.gc_weakrefs - except AttributeError: - from .gc_weakref import GcWeakrefs - gc_weakrefs = self.gc_weakrefs = GcWeakrefs(self) - return gc_weakrefs.build(cdata, destructor) + with self._lock: + try: + gc_weakrefs = self.gc_weakrefs + except AttributeError: + from .gc_weakref import GcWeakrefs + gc_weakrefs = self.gc_weakrefs = GcWeakrefs(self) + return gc_weakrefs.build(cdata, destructor) def _get_cached_btype(self, type): + assert self._lock.acquire(False) is False + # call me with the lock! try: BType = self._cached_btypes[type] except KeyError: @@ -322,7 +349,8 @@ def _pointer_to(self, ctype): from . import model - return model.pointer_cache(self, ctype) + with self._lock: + return model.pointer_cache(self, ctype) def addressof(self, cdata, field=None): """Return the address of a . @@ -342,10 +370,12 @@ variables, which must anyway be accessed directly from the lib object returned by the original FFI instance. """ - self._parser.include(ffi_to_include._parser) - self._cdefsources.append('[') - self._cdefsources.extend(ffi_to_include._cdefsources) - self._cdefsources.append(']') + with ffi_to_include._lock: + with self._lock: + self._parser.include(ffi_to_include._parser) + self._cdefsources.append('[') + self._cdefsources.extend(ffi_to_include._cdefsources) + self._cdefsources.append(']') def new_handle(self, x): return self._backend.newp_handle(self.BVoidP, x) @@ -372,7 +402,7 @@ backendlib = backend.load_library(path, flags) copied_enums = [] # - def make_accessor(name): + def make_accessor_locked(name): key = 'function ' + name if key in ffi._parser._declarations: tp = ffi._parser._declarations[key] @@ -404,11 +434,17 @@ if enumname not in library.__dict__: library.__dict__[enumname] = enumval copied_enums.append(True) + if name in library.__dict__: + return # - if name in library.__dict__: # copied from an enum value just above, - return # or multithread's race condition raise AttributeError(name) # + def make_accessor(name): + with ffi._lock: + if name in library.__dict__ or name in FFILibrary.__dict__: + return # added by another thread while waiting for the lock + make_accessor_locked(name) + # class FFILibrary(object): def __getattr__(self, name): make_accessor(name) @@ -444,4 +480,5 @@ except (KeyError, AttributeError, TypeError): return None else: - return ffi._get_cached_btype(tp) + with ffi._lock: + return ffi._get_cached_btype(tp) diff --git a/lib_pypy/cffi/cparser.py b/lib_pypy/cffi/cparser.py --- a/lib_pypy/cffi/cparser.py +++ b/lib_pypy/cffi/cparser.py @@ -142,7 +142,7 @@ if 1 <= linenum <= len(csourcelines): line = csourcelines[linenum-1] if line: - msg = 'cannot parse "%s"\n%s' % (line, msg) + msg = 'cannot parse "%s"\n%s' % (line.strip(), msg) else: msg = 'parse error\n%s' % (msg,) raise api.CDefError(msg) @@ -217,19 +217,18 @@ # if decl.name: tp = self._get_type(node, partial_length_ok=True) - if self._is_constant_declaration(node): + if self._is_constant_globalvar(node): self._declare('constant ' + decl.name, tp) else: self._declare('variable ' + decl.name, tp) - def parse_type(self, cdecl, consider_function_as_funcptr=False): + def parse_type(self, cdecl): ast, macros = self._parse('void __dummy(\n%s\n);' % cdecl) assert not macros exprnode = ast.ext[-1].type.args.params[0] if isinstance(exprnode, pycparser.c_ast.ID): raise api.CDefError("unknown identifier '%s'" % (exprnode.name,)) - return self._get_type(exprnode.type, - consider_function_as_funcptr=consider_function_as_funcptr) + return self._get_type(exprnode.type) def _declare(self, name, obj): if name in self._declarations: @@ -249,28 +248,17 @@ return model.ConstPointerType(type) return model.PointerType(type) - def _get_type(self, typenode, convert_array_to_pointer=False, - name=None, partial_length_ok=False, - consider_function_as_funcptr=False): + def _get_type(self, typenode, name=None, partial_length_ok=False): # first, dereference typedefs, if we have it already parsed, we're good if (isinstance(typenode, pycparser.c_ast.TypeDecl) and isinstance(typenode.type, pycparser.c_ast.IdentifierType) and len(typenode.type.names) == 1 and ('typedef ' + typenode.type.names[0]) in self._declarations): type = self._declarations['typedef ' + typenode.type.names[0]] - if isinstance(type, model.ArrayType): - if convert_array_to_pointer: - return type.item - else: - if (consider_function_as_funcptr and - isinstance(type, model.RawFunctionType)): - return type.as_function_pointer() return type # if isinstance(typenode, pycparser.c_ast.ArrayDecl): # array type - if convert_array_to_pointer: - return self._get_type_pointer(self._get_type(typenode.type)) if typenode.dim is None: length = None else: @@ -331,10 +319,7 @@ # if isinstance(typenode, pycparser.c_ast.FuncDecl): # a function type - result = self._parse_function_type(typenode, name) - if consider_function_as_funcptr: - result = result.as_function_pointer() - return result + return self._parse_function_type(typenode, name) # # nested anonymous structs or unions end up here if isinstance(typenode, pycparser.c_ast.Struct): @@ -365,21 +350,24 @@ isinstance(params[0].type.type, pycparser.c_ast.IdentifierType) and list(params[0].type.type.names) == ['void']): del params[0] - args = [self._get_type(argdeclnode.type, - convert_array_to_pointer=True, - consider_function_as_funcptr=True) + args = [self._as_func_arg(self._get_type(argdeclnode.type)) for argdeclnode in params] result = self._get_type(typenode.type) return model.RawFunctionType(tuple(args), result, ellipsis) - def _is_constant_declaration(self, typenode, const=False): - if isinstance(typenode, pycparser.c_ast.ArrayDecl): - return self._is_constant_declaration(typenode.type) + def _as_func_arg(self, type): + if isinstance(type, model.ArrayType): + return model.PointerType(type.item) + elif isinstance(type, model.RawFunctionType): + return type.as_function_pointer() + else: + return type + + def _is_constant_globalvar(self, typenode): if isinstance(typenode, pycparser.c_ast.PtrDecl): - const = 'const' in typenode.quals - return self._is_constant_declaration(typenode.type, const) + return 'const' in typenode.quals if isinstance(typenode, pycparser.c_ast.TypeDecl): - return const or 'const' in typenode.quals + return 'const' in typenode.quals return False def _get_struct_union_enum_type(self, kind, type, name=None, nested=False): @@ -491,7 +479,7 @@ return tp def _make_partial(self, tp, nested): - if not isinstance(tp, model.StructType): + if not isinstance(tp, model.StructOrUnion): raise api.CDefError("%s cannot be partial" % (tp,)) if not tp.has_c_name() and not nested: raise NotImplementedError("%s is partial but has no C name" %(tp,)) @@ -511,7 +499,7 @@ if (isinstance(exprnode, pycparser.c_ast.ID) and exprnode.name == '__dotdotdotarray__'): self._partial_length = True - return None + return '...' # raise api.FFIError("unsupported expression: expected a " "simple numeric constant") diff --git a/lib_pypy/cffi/gc_weakref.py b/lib_pypy/cffi/gc_weakref.py --- a/lib_pypy/cffi/gc_weakref.py +++ b/lib_pypy/cffi/gc_weakref.py @@ -14,6 +14,6 @@ def build(self, cdata, destructor): # make a new cdata of the same type as the original one - new_cdata = self.ffi.cast(self.ffi.typeof(cdata), cdata) + new_cdata = self.ffi.cast(self.ffi._backend.typeof(cdata), cdata) self.data[ref(new_cdata, self.remove)] = destructor, cdata return new_cdata diff --git a/lib_pypy/cffi/lock.py b/lib_pypy/cffi/lock.py new file mode 100644 --- /dev/null +++ b/lib_pypy/cffi/lock.py @@ -0,0 +1,30 @@ +import sys + +if sys.version_info < (3,): + try: + from thread import allocate_lock + except ImportError: + from dummy_thread import allocate_lock +else: + try: + from _thread import allocate_lock + except ImportError: + from _dummy_thread import allocate_lock + + +##import sys +##l1 = allocate_lock + +##class allocate_lock(object): +## def __init__(self): +## self._real = l1() +## def __enter__(self): +## for i in range(4, 0, -1): +## print sys._getframe(i).f_code +## print +## return self._real.__enter__() +## def __exit__(self, *args): +## return self._real.__exit__(*args) +## def acquire(self, f): +## assert f is False +## return self._real.acquire(f) diff --git a/lib_pypy/cffi/model.py b/lib_pypy/cffi/model.py --- a/lib_pypy/cffi/model.py +++ b/lib_pypy/cffi/model.py @@ -1,7 +1,10 @@ import weakref +from .lock import allocate_lock + class BaseTypeByIdentity(object): is_array_type = False + is_raw_function = False def get_c_name(self, replace_with='', context='a C file'): result = self.c_name_with_marker @@ -146,6 +149,7 @@ # a function, but not a pointer-to-function. The backend has no # notion of such a type; it's used temporarily by parsing. _base_pattern = '(&)(%s)' + is_raw_function = True def build_backend_type(self, ffi, finishlist): from . import api @@ -192,10 +196,6 @@ _base_pattern = " const *&" _base_pattern_array = "(const *&)" - def build_backend_type(self, ffi, finishlist): - BPtr = PointerType(self.totype).get_cached_btype(ffi, finishlist) - return BPtr - const_voidp_type = ConstPointerType(void_type) @@ -216,10 +216,12 @@ self.item = item self.length = length # - if self.length is None: + if length is None: brackets = '&[]' + elif length == '...': + brackets = '&[/*...*/]' else: - brackets = '&[%d]' % self.length + brackets = '&[%d]' % length self.c_name_with_marker = ( self.item.c_name_with_marker.replace('&', brackets)) @@ -227,6 +229,10 @@ return ArrayType(self.item, newlength) def build_backend_type(self, ffi, finishlist): + if self.length == '...': + from . import api + raise api.CDefError("cannot render the type %r: unknown length" % + (self,)) self.item.get_cached_btype(ffi, finishlist) # force the item BType BPtrItem = PointerType(self.item).get_cached_btype(ffi, finishlist) return global_cache(self, ffi, 'new_array_type', BPtrItem, self.length) @@ -252,6 +258,7 @@ class StructOrUnion(StructOrUnionOrEnum): fixedlayout = None completed = False + partial = False def __init__(self, name, fldnames, fldtypes, fldbitsize): self.name = name @@ -303,20 +310,21 @@ return # not completing it: it's an opaque struct # self.completed = 1 - fldtypes = tuple(tp.get_cached_btype(ffi, finishlist) - for tp in self.fldtypes) # if self.fixedlayout is None: + fldtypes = [tp.get_cached_btype(ffi, finishlist) + for tp in self.fldtypes] lst = list(zip(self.fldnames, fldtypes, self.fldbitsize)) ffi._backend.complete_struct_or_union(BType, lst, self) # else: + fldtypes = [] fieldofs, fieldsize, totalsize, totalalignment = self.fixedlayout for i in range(len(self.fldnames)): fsize = fieldsize[i] ftype = self.fldtypes[i] # - if isinstance(ftype, ArrayType) and ftype.length is None: + if isinstance(ftype, ArrayType) and ftype.length == '...': # fix the length to match the total size BItemType = ftype.item.get_cached_btype(ffi, finishlist) nlen, nrest = divmod(fsize, ffi.sizeof(BItemType)) @@ -327,18 +335,20 @@ ftype = ftype.resolve_length(nlen) self.fldtypes = (self.fldtypes[:i] + (ftype,) + self.fldtypes[i+1:]) - BArrayType = ftype.get_cached_btype(ffi, finishlist) - fldtypes = (fldtypes[:i] + (BArrayType,) + - fldtypes[i+1:]) - continue # - bitemsize = ffi.sizeof(fldtypes[i]) - if bitemsize != fsize: - self._verification_error( - "field '%s.%s' is declared as %d bytes, but is " - "really %d bytes" % (self.name, - self.fldnames[i] or '{}', - bitemsize, fsize)) + BFieldType = ftype.get_cached_btype(ffi, finishlist) + if isinstance(ftype, ArrayType) and ftype.length is None: + assert fsize == 0 + else: + bitemsize = ffi.sizeof(BFieldType) + if bitemsize != fsize: + self._verification_error( + "field '%s.%s' is declared as %d bytes, but is " + "really %d bytes" % (self.name, + self.fldnames[i] or '{}', + bitemsize, fsize)) + fldtypes.append(BFieldType) + # lst = list(zip(self.fldnames, fldtypes, self.fldbitsize, fieldofs)) ffi._backend.complete_struct_or_union(BType, lst, self, totalsize, totalalignment) @@ -348,11 +358,6 @@ from .ffiplatform import VerificationError raise VerificationError(msg) - -class StructType(StructOrUnion): - kind = 'struct' - partial = False - def check_not_partial(self): if self.partial and self.fixedlayout is None: from . import ffiplatform @@ -361,19 +366,18 @@ def build_backend_type(self, ffi, finishlist): self.check_not_partial() finishlist.append(self) - - return global_cache(self, ffi, 'new_struct_type', + # + return global_cache(self, ffi, 'new_%s_type' % self.kind, self.get_official_name(), key=self) +class StructType(StructOrUnion): + kind = 'struct' + + class UnionType(StructOrUnion): kind = 'union' - def build_backend_type(self, ffi, finishlist): - finishlist.append(self) - return global_cache(self, ffi, 'new_union_type', - self.get_official_name(), key=self) - class EnumType(StructOrUnionOrEnum): kind = 'enum' @@ -387,6 +391,12 @@ self.baseinttype = baseinttype self.build_c_name_with_marker() + def force_the_name(self, forcename): + StructOrUnionOrEnum.force_the_name(self, forcename) + if self.forcename is None: + name = self.get_official_name() + self.forcename = '$' + name.replace(' ', '_') + def check_not_partial(self): if self.partial and not self.partial_resolved: from . import ffiplatform @@ -444,6 +454,9 @@ tp = StructType(structname, None, None, None) return NamedPointerType(tp, name) + +global_lock = allocate_lock() + def global_cache(srctype, ffi, funcname, *args, **kwds): key = kwds.pop('key', (funcname, args)) assert not kwds @@ -464,8 +477,17 @@ res = getattr(ffi._backend, funcname)(*args) except NotImplementedError as e: raise NotImplementedError("%r: %s" % (srctype, e)) - ffi._backend.__typecache[key] = res - return res + # note that setdefault() on WeakValueDictionary is not atomic + # and contains a rare bug (http://bugs.python.org/issue19542); + # we have to use a lock and do it ourselves + cache = ffi._backend.__typecache + with global_lock: + res1 = cache.get(key) + if res1 is None: + cache[key] = res + return res + else: + return res1 def pointer_cache(ffi, BType): return global_cache('?', ffi, 'new_pointer_type', BType) diff --git a/lib_pypy/cffi/vengine_cpy.py b/lib_pypy/cffi/vengine_cpy.py --- a/lib_pypy/cffi/vengine_cpy.py +++ b/lib_pypy/cffi/vengine_cpy.py @@ -15,7 +15,7 @@ def patch_extension_kwds(self, kwds): pass - def find_module(self, module_name, path, so_suffix): + def find_module(self, module_name, path, so_suffixes): try: f, filename, descr = imp.find_module(module_name, path) except ImportError: @@ -25,7 +25,7 @@ # Note that after a setuptools installation, there are both .py # and .so files with the same basename. The code here relies on # imp.find_module() locating the .so in priority. - if descr[0] != so_suffix: + if descr[0] not in so_suffixes: return None return filename @@ -160,7 +160,10 @@ def __dir__(self): return FFILibrary._cffi_dir + list(self.__dict__) library = FFILibrary() - module._cffi_setup(lst, ffiplatform.VerificationError, library) + if module._cffi_setup(lst, ffiplatform.VerificationError, library): + import warnings + warnings.warn("reimporting %r might overwrite older definitions" + % (self.verifier.get_module_name())) # # finally, call the loaded_cpy_xxx() functions. This will perform # the final adjustments, like copying the Python->C wrapper @@ -280,8 +283,8 @@ return '_cffi_from_c_pointer((char *)%s, _cffi_type(%d))' % ( var, self._gettypenum(tp)) elif isinstance(tp, model.ArrayType): - return '_cffi_from_c_deref((char *)%s, _cffi_type(%d))' % ( - var, self._gettypenum(tp)) + return '_cffi_from_c_pointer((char *)%s, _cffi_type(%d))' % ( + var, self._gettypenum(model.PointerType(tp.item))) elif isinstance(tp, model.StructType): if tp.fldnames is None: raise TypeError("'%s' is used as %s, but is opaque" % ( @@ -464,11 +467,14 @@ prnt(' static Py_ssize_t nums[] = {') prnt(' sizeof(%s),' % cname) prnt(' offsetof(struct _cffi_aligncheck, y),') - for fname, _, fbitsize in tp.enumfields(): + for fname, ftype, fbitsize in tp.enumfields(): if fbitsize >= 0: continue # xxx ignore fbitsize for now prnt(' offsetof(%s, %s),' % (cname, fname)) - prnt(' sizeof(((%s *)0)->%s),' % (cname, fname)) + if isinstance(ftype, model.ArrayType) and ftype.length is None: + prnt(' 0, /* %s */' % ftype._get_c_name()) + else: + prnt(' sizeof(((%s *)0)->%s),' % (cname, fname)) prnt(' -1') prnt(' };') prnt(' return _cffi_get_struct_layout(nums);') @@ -491,7 +497,7 @@ # function = getattr(module, layoutfuncname) layout = function() - if isinstance(tp, model.StructType) and tp.partial: + if isinstance(tp, model.StructOrUnion) and tp.partial: # use the function()'s sizes and offsets to guide the # layout of the struct totalsize = layout[0] @@ -528,9 +534,10 @@ continue # xxx ignore fbitsize for now check(layout[i], ffi.offsetof(BStruct, fname), "wrong offset for field %r" % (fname,)) - BField = ffi._get_cached_btype(ftype) - check(layout[i+1], ffi.sizeof(BField), - "wrong size for field %r" % (fname,)) + if layout[i+1] != 0: + BField = ffi._get_cached_btype(ftype) + check(layout[i+1], ffi.sizeof(BField), + "wrong size for field %r" % (fname,)) i += 2 assert i == len(layout) @@ -566,7 +573,7 @@ # constants, likely declared with '#define' def _generate_cpy_const(self, is_int, name, tp=None, category='const', - vartp=None, delayed=True): + vartp=None, delayed=True, size_too=False): prnt = self._prnt funcname = '_cffi_%s_%s' % (category, name) prnt('static int %s(PyObject *lib)' % funcname) @@ -597,6 +604,15 @@ '(unsigned long long)(%s));' % (name,)) prnt(' if (o == NULL)') prnt(' return -1;') + if size_too: + prnt(' {') + prnt(' PyObject *o1 = o;') + prnt(' o = Py_BuildValue("On", o1, (Py_ssize_t)sizeof(%s));' + % (name,)) + prnt(' Py_DECREF(o1);') + prnt(' if (o == NULL)') + prnt(' return -1;') + prnt(' }') prnt(' res = PyObject_SetAttrString(lib, "%s", o);' % name) prnt(' Py_DECREF(o);') prnt(' if (res < 0)') @@ -633,12 +649,23 @@ prnt('static int %s(PyObject *lib)' % funcname) prnt('{') for enumerator, enumvalue in zip(tp.enumerators, tp.enumvalues): - prnt(' if (%s != %d) {' % (enumerator, enumvalue)) + if enumvalue < 0: + prnt(' if ((%s) >= 0 || (long)(%s) != %dL) {' % ( + enumerator, enumerator, enumvalue)) + else: + prnt(' if ((%s) < 0 || (unsigned long)(%s) != %dUL) {' % ( + enumerator, enumerator, enumvalue)) + prnt(' char buf[64];') + prnt(' if ((%s) < 0)' % enumerator) + prnt(' snprintf(buf, 63, "%%ld", (long)(%s));' % enumerator) + prnt(' else') + prnt(' snprintf(buf, 63, "%%lu", (unsigned long)(%s));' % + enumerator) prnt(' PyErr_Format(_cffi_VerificationError,') - prnt(' "enum %s: %s has the real value %d, ' - 'not %d",') - prnt(' "%s", "%s", (int)%s, %d);' % ( - name, enumerator, enumerator, enumvalue)) + prnt(' "enum %s: %s has the real value %s, ' + 'not %s",') + prnt(' "%s", "%s", buf, "%d");' % ( + name, enumerator, enumvalue)) prnt(' return -1;') prnt(' }') prnt(' return %s;' % self._chained_list_constants[True]) @@ -677,15 +704,16 @@ def _generate_cpy_variable_collecttype(self, tp, name): if isinstance(tp, model.ArrayType): - self._do_collect_type(tp) + tp_ptr = model.PointerType(tp.item) else: tp_ptr = model.PointerType(tp) - self._do_collect_type(tp_ptr) + self._do_collect_type(tp_ptr) def _generate_cpy_variable_decl(self, tp, name): if isinstance(tp, model.ArrayType): tp_ptr = model.PointerType(tp.item) - self._generate_cpy_const(False, name, tp, vartp=tp_ptr) + self._generate_cpy_const(False, name, tp, vartp=tp_ptr, + size_too = (tp.length == '...')) else: tp_ptr = model.PointerType(tp) self._generate_cpy_const(False, name, tp_ptr, category='var') @@ -694,11 +722,29 @@ _loading_cpy_variable = _loaded_noop def _loaded_cpy_variable(self, tp, name, module, library): + value = getattr(library, name) if isinstance(tp, model.ArrayType): # int a[5] is "constant" in the - return # sense that "a=..." is forbidden + # sense that "a=..." is forbidden + if tp.length == '...': + assert isinstance(value, tuple) + (value, size) = value + BItemType = self.ffi._get_cached_btype(tp.item) + length, rest = divmod(size, self.ffi.sizeof(BItemType)) + if rest != 0: + raise ffiplatform.VerificationError( + "bad size: %r does not seem to be an array of %s" % + (name, tp.item)) + tp = tp.resolve_length(length) + # 'value' is a which we have to replace with + # a if the N is actually known + if tp.length is not None: + BArray = self.ffi._get_cached_btype(tp) + value = self.ffi.cast(BArray, value) + setattr(library, name, value) + return # remove ptr= from the library instance, and replace # it by a property on the class, which reads/writes into ptr[0]. - ptr = getattr(library, name) + ptr = value delattr(library, name) def getter(library): return ptr[0] @@ -711,12 +757,9 @@ def _generate_setup_custom(self): prnt = self._prnt - prnt('static PyObject *_cffi_setup_custom(PyObject *lib)') + prnt('static int _cffi_setup_custom(PyObject *lib)') prnt('{') - prnt(' if (%s < 0)' % self._chained_list_constants[True]) - prnt(' return NULL;') - prnt(' Py_INCREF(Py_None);') - prnt(' return Py_None;') + prnt(' return %s;' % self._chained_list_constants[True]) prnt('}') cffimod_header = r''' @@ -834,17 +877,20 @@ static void *_cffi_exports[_CFFI_NUM_EXPORTS]; static PyObject *_cffi_types, *_cffi_VerificationError; -static PyObject *_cffi_setup_custom(PyObject *lib); /* forward */ +static int _cffi_setup_custom(PyObject *lib); /* forward */ static PyObject *_cffi_setup(PyObject *self, PyObject *args) { PyObject *library; + int was_alive = (_cffi_types != NULL); if (!PyArg_ParseTuple(args, "OOO", &_cffi_types, &_cffi_VerificationError, &library)) return NULL; Py_INCREF(_cffi_types); Py_INCREF(_cffi_VerificationError); - return _cffi_setup_custom(library); + if (_cffi_setup_custom(library) < 0) + return NULL; + return PyBool_FromLong(was_alive); } static void _cffi_init(void) diff --git a/lib_pypy/cffi/vengine_gen.py b/lib_pypy/cffi/vengine_gen.py --- a/lib_pypy/cffi/vengine_gen.py +++ b/lib_pypy/cffi/vengine_gen.py @@ -20,15 +20,15 @@ # up in kwds['export_symbols']. kwds.setdefault('export_symbols', self.export_symbols) - def find_module(self, module_name, path, so_suffix): - basename = module_name + so_suffix - if path is None: - path = sys.path - for dirname in path: - filename = os.path.join(dirname, basename) - if os.path.isfile(filename): - return filename - return None + def find_module(self, module_name, path, so_suffixes): + for so_suffix in so_suffixes: + basename = module_name + so_suffix + if path is None: + path = sys.path + for dirname in path: + filename = os.path.join(dirname, basename) + if os.path.isfile(filename): + return filename def collect_types(self): pass # not needed in the generic engine @@ -173,6 +173,7 @@ newfunction = self._load_constant(False, tp, name, module) else: indirections = [] + base_tp = tp if any(isinstance(typ, model.StructOrUnion) for typ in tp.args): indirect_args = [] for i, typ in enumerate(tp.args): @@ -186,16 +187,18 @@ wrappername = '_cffi_f_%s' % name newfunction = module.load_function(BFunc, wrappername) for i, typ in indirections: - newfunction = self._make_struct_wrapper(newfunction, i, typ) + newfunction = self._make_struct_wrapper(newfunction, i, typ, + base_tp) setattr(library, name, newfunction) type(library)._cffi_dir.append(name) - def _make_struct_wrapper(self, oldfunc, i, tp): + def _make_struct_wrapper(self, oldfunc, i, tp, base_tp): backend = self.ffi._backend BType = self.ffi._get_cached_btype(tp) def newfunc(*args): args = args[:i] + (backend.newp(BType, args[i]),) + args[i+1:] return oldfunc(*args) + newfunc._cffi_base_type = base_tp return newfunc # ---------- @@ -252,11 +255,14 @@ prnt(' static ssize_t nums[] = {') prnt(' sizeof(%s),' % cname) prnt(' offsetof(struct _cffi_aligncheck, y),') - for fname, _, fbitsize in tp.enumfields(): + for fname, ftype, fbitsize in tp.enumfields(): if fbitsize >= 0: continue # xxx ignore fbitsize for now prnt(' offsetof(%s, %s),' % (cname, fname)) - prnt(' sizeof(((%s *)0)->%s),' % (cname, fname)) + if isinstance(ftype, model.ArrayType) and ftype.length is None: + prnt(' 0, /* %s */' % ftype._get_c_name()) + else: + prnt(' sizeof(((%s *)0)->%s),' % (cname, fname)) prnt(' -1') prnt(' };') prnt(' return nums[i];') @@ -270,7 +276,7 @@ return # nothing to do with opaque structs layoutfuncname = '_cffi_layout_%s_%s' % (prefix, name) # - BFunc = self.ffi.typeof("ssize_t(*)(ssize_t)") + BFunc = self.ffi._typeof_locked("ssize_t(*)(ssize_t)")[0] function = module.load_function(BFunc, layoutfuncname) layout = [] num = 0 @@ -279,7 +285,7 @@ if x < 0: break layout.append(x) num += 1 - if isinstance(tp, model.StructType) and tp.partial: + if isinstance(tp, model.StructOrUnion) and tp.partial: # use the function()'s sizes and offsets to guide the # layout of the struct totalsize = layout[0] @@ -316,9 +322,10 @@ continue # xxx ignore fbitsize for now check(layout[i], ffi.offsetof(BStruct, fname), "wrong offset for field %r" % (fname,)) - BField = ffi._get_cached_btype(ftype) - check(layout[i+1], ffi.sizeof(BField), - "wrong size for field %r" % (fname,)) + if layout[i+1] != 0: + BField = ffi._get_cached_btype(ftype) + check(layout[i+1], ffi.sizeof(BField), + "wrong size for field %r" % (fname,)) i += 2 assert i == len(layout) @@ -379,15 +386,17 @@ def _load_constant(self, is_int, tp, name, module): funcname = '_cffi_const_%s' % name if is_int: - BFunc = self.ffi.typeof("int(*)(long long*)") + BType = self.ffi._typeof_locked("long long*")[0] + BFunc = self.ffi._typeof_locked("int(*)(long long*)")[0] function = module.load_function(BFunc, funcname) - p = self.ffi.new("long long*") + p = self.ffi.new(BType) negative = function(p) value = int(p[0]) if value < 0 and not negative: - value += (1 << (8*self.ffi.sizeof("long long"))) + BLongLong = self.ffi._typeof_locked("long long")[0] + value += (1 << (8*self.ffi.sizeof(BLongLong))) else: - BFunc = self.ffi.typeof(tp.get_c_name('(*)(void)', name)) + BFunc = self.ffi._typeof_locked(tp.get_c_name('(*)(void)', name))[0] function = module.load_function(BFunc, funcname) value = function() return value @@ -413,11 +422,22 @@ prnt('int %s(char *out_error)' % funcname) prnt('{') for enumerator, enumvalue in zip(tp.enumerators, tp.enumvalues): - prnt(' if (%s != %d) {' % (enumerator, enumvalue)) + if enumvalue < 0: + prnt(' if ((%s) >= 0 || (long)(%s) != %dL) {' % ( + enumerator, enumerator, enumvalue)) + else: + prnt(' if ((%s) < 0 || (unsigned long)(%s) != %dUL) {' % ( + enumerator, enumerator, enumvalue)) + prnt(' char buf[64];') + prnt(' if ((%s) < 0)' % enumerator) + prnt(' snprintf(buf, 63, "%%ld", (long)(%s));' % enumerator) + prnt(' else') + prnt(' snprintf(buf, 63, "%%lu", (unsigned long)(%s));' % + enumerator) prnt(' snprintf(out_error, 255,' - '"%s has the real value %d, not %d",') - prnt(' "%s", (int)%s, %d);' % ( - enumerator, enumerator, enumvalue)) + ' "%s has the real value %s, not %s",') + prnt(' "%s", buf, "%d");' % ( + enumerator, enumvalue)) prnt(' return -1;') prnt(' }') prnt(' return 0;') @@ -431,10 +451,11 @@ tp.enumvalues = tuple(enumvalues) tp.partial_resolved = True else: - BFunc = self.ffi.typeof("int(*)(char*)") + BType = self.ffi._typeof_locked("char[]")[0] + BFunc = self.ffi._typeof_locked("int(*)(char*)")[0] funcname = '_cffi_e_%s_%s' % (prefix, name) function = module.load_function(BFunc, funcname) - p = self.ffi.new("char[]", 256) + p = self.ffi.new(BType, 256) if function(p) < 0: error = self.ffi.string(p) if sys.version_info >= (3,): @@ -465,6 +486,14 @@ def _generate_gen_variable_decl(self, tp, name): if isinstance(tp, model.ArrayType): + if tp.length == '...': + prnt = self._prnt + funcname = '_cffi_sizeof_%s' % (name,) + self.export_symbols.append(funcname) + prnt("size_t %s(void)" % funcname) + prnt("{") + prnt(" return sizeof(%s);" % (name,)) + prnt("}") tp_ptr = model.PointerType(tp.item) self._generate_gen_const(False, name, tp_ptr) else: @@ -476,6 +505,18 @@ def _loaded_gen_variable(self, tp, name, module, library): if isinstance(tp, model.ArrayType): # int a[5] is "constant" in the # sense that "a=..." is forbidden + if tp.length == '...': + funcname = '_cffi_sizeof_%s' % (name,) + BFunc = self.ffi._typeof_locked('size_t(*)(void)')[0] + function = module.load_function(BFunc, funcname) + size = function() + BItemType = self.ffi._get_cached_btype(tp.item) + length, rest = divmod(size, self.ffi.sizeof(BItemType)) + if rest != 0: + raise ffiplatform.VerificationError( + "bad size: %r does not seem to be an array of %s" % + (name, tp.item)) + tp = tp.resolve_length(length) tp_ptr = model.PointerType(tp.item) value = self._load_constant(False, tp_ptr, name, module) # 'value' is a which we have to replace with @@ -489,7 +530,7 @@ # remove ptr= from the library instance, and replace # it by a property on the class, which reads/writes into ptr[0]. funcname = '_cffi_var_%s' % name - BFunc = self.ffi.typeof(tp.get_c_name('*(*)(void)', name)) + BFunc = self.ffi._typeof_locked(tp.get_c_name('*(*)(void)', name))[0] function = module.load_function(BFunc, funcname) ptr = function() def getter(library): diff --git a/lib_pypy/cffi/verifier.py b/lib_pypy/cffi/verifier.py --- a/lib_pypy/cffi/verifier.py +++ b/lib_pypy/cffi/verifier.py @@ -31,7 +31,7 @@ k2 = k2.lstrip('0').rstrip('L') modulename = '_cffi_%s_%s%s%s' % (tag, self._vengine._class_key, k1, k2) - suffix = _get_so_suffix() + suffix = _get_so_suffixes()[0] self.tmpdir = tmpdir or _caller_dir_pycache() self.sourcefilename = os.path.join(self.tmpdir, modulename + '.c') self.modulefilename = os.path.join(self.tmpdir, modulename + suffix) @@ -42,18 +42,21 @@ def write_source(self, file=None): """Write the C source code. It is produced in 'self.sourcefilename', which can be tweaked beforehand.""" - if self._has_source and file is None: - raise ffiplatform.VerificationError("source code already written") - self._write_source(file) + with self.ffi._lock: + if self._has_source and file is None: + raise ffiplatform.VerificationError( + "source code already written") + self._write_source(file) def compile_module(self): """Write the C source code (if not done already) and compile it. This produces a dynamic link library in 'self.modulefilename'.""" - if self._has_module: - raise ffiplatform.VerificationError("module already compiled") - if not self._has_source: - self._write_source() - self._compile_module() + with self.ffi._lock: + if self._has_module: + raise ffiplatform.VerificationError("module already compiled") + if not self._has_source: + self._write_source() + self._compile_module() def load_library(self): """Get a C module from this Verifier instance. @@ -62,11 +65,14 @@ operations to the C module. If necessary, the C code is written and compiled first. """ - if not self._has_module: - self._locate_module() + with self.ffi._lock: if not self._has_module: - self.compile_module() - return self._load_library() + self._locate_module() + if not self._has_module: + if not self._has_source: + self._write_source() + self._compile_module() + return self._load_library() def get_module_name(self): basename = os.path.basename(self.modulefilename) @@ -81,7 +87,9 @@ def get_extension(self): if not self._has_source: - self._write_source() + with self.ffi._lock: + if not self._has_source: + self._write_source() sourcename = ffiplatform.maybe_relative_path(self.sourcefilename) modname = self.get_module_name() return ffiplatform.get_extension(sourcename, modname, **self.kwds) @@ -103,7 +111,7 @@ else: path = None filename = self._vengine.find_module(self.get_module_name(), path, - _get_so_suffix()) + _get_so_suffixes()) if filename is None: return self.modulefilename = filename @@ -193,7 +201,7 @@ if keep_so: suffix = '.c' # only remove .c files else: - suffix = _get_so_suffix().lower() + suffix = _get_so_suffixes()[0].lower() for fn in filelist: if fn.lower().startswith('_cffi_') and ( fn.lower().endswith(suffix) or fn.lower().endswith('.c')): @@ -213,15 +221,20 @@ except OSError: pass -def _get_so_suffix(): +def _get_so_suffixes(): + suffixes = [] for suffix, mode, type in imp.get_suffixes(): if type == imp.C_EXTENSION: - return suffix - # bah, no C_EXTENSION available. Occurs on pypy without cpyext - if sys.platform == 'win32': - return ".pyd" - else: - return ".so" + suffixes.append(suffix) + + if not suffixes: + # bah, no C_EXTENSION available. Occurs on pypy without cpyext + if sys.platform == 'win32': + suffixes = [".pyd"] + else: + suffixes = [".so"] + + return suffixes def _ensure_dir(filename): try: diff --git a/lib_pypy/pyrepl/reader.py b/lib_pypy/pyrepl/reader.py --- a/lib_pypy/pyrepl/reader.py +++ b/lib_pypy/pyrepl/reader.py @@ -19,11 +19,13 @@ # CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN # CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -import types +import types, re from pyrepl import unicodedata_ from pyrepl import commands from pyrepl import input +_r_csi_seq = re.compile(r"\033\[[ -@]*[A-~]") + def _make_unctrl_map(): uc_map = {} for c in map(unichr, range(256)): @@ -309,6 +311,10 @@ excluded from the length calculation. So also a copy of the prompt is returned with these control characters removed. """ + # The logic below also ignores the length of common escape + # sequences if they were not explicitly within \x01...\x02. + # They are CSI (or ANSI) sequences ( ESC [ ... LETTER ) + out_prompt = '' l = len(prompt) pos = 0 @@ -321,9 +327,13 @@ break # Found start and end brackets, subtract from string length l = l - (e-s+1) - out_prompt += prompt[pos:s] + prompt[s+1:e] + keep = prompt[pos:s] + l -= sum(map(len, _r_csi_seq.findall(keep))) + out_prompt += keep + prompt[s+1:e] pos = e+1 - out_prompt += prompt[pos:] + keep = prompt[pos:] + l -= sum(map(len, _r_csi_seq.findall(keep))) + out_prompt += keep return out_prompt, l def bow(self, p=None): diff --git a/pypy/doc/contributor.rst b/pypy/doc/contributor.rst --- a/pypy/doc/contributor.rst +++ b/pypy/doc/contributor.rst @@ -17,24 +17,24 @@ David Schneider Holger Krekel Christian Tismer + Matti Picus Hakan Ardo Benjamin Peterson - Matti Picus Philip Jenvey Anders Chrigstrom Brian Kearns + Manuel Jacob Eric van Riet Paap + Wim Lavrijsen Richard Emslie Alexander Schremmer - Wim Lavrijsen Dan Villiom Podlaski Christiansen - Manuel Jacob + Ronan Lamy Lukas Diekmann Sven Hager Anders Lehmann Aurelien Campeas Niklaus Haldimann - Ronan Lamy Camillo Bruni Laura Creighton Toon Verwaest @@ -45,8 +45,10 @@ David Edelsohn Anders Hammarquist Jakub Gustak + Romain Guillebert Guido Wesdorp Lawrence Oluyede + Remi Meier Bartosz Skowron Daniel Roberts Niko Matsakis @@ -54,18 +56,17 @@ Ludovic Aubry Alexander Hesse Jacob Hallen - Romain Guillebert Jason Creighton Alex Martelli Michal Bendowski Jan de Mooij + stian Michael Foord Stephan Diehl Stefan Schwarzer Valentino Volonghi Tomek Meka Patrick Maupin - stian Bob Ippolito Bruno Gola Jean-Paul Calderone @@ -74,29 +75,33 @@ Simon Burton Marius Gedminas John Witulski + Konstantin Lopuhin Greg Price Dario Bertini Mark Pearse Simon Cross - Konstantin Lopuhin Andreas Stührk Jean-Philippe St. Pierre Guido van Rossum Pavel Vinogradov + Paweł Piotr Przeradowski Paul deGrandis Ilya Osadchiy Adrian Kuhn Boris Feigin + Stefano Rivera tav + Taavi Burns Georg Brandl Bert Freudenberg Stian Andreassen - Stefano Rivera Wanja Saatkamp Gerald Klix Mike Blume - Taavi Burns Oscar Nierstrasz + Stefan H. Muller + Laurence Tratt + Rami Chowdhury David Malcolm Eugene Oden Henry Mason @@ -105,14 +110,15 @@ David Ripton Dusty Phillips Lukas Renggli + Edd Barrett Guenter Jantzen Tobias Oberstein - Remi Meier Ned Batchelder Amit Regmi Ben Young Nicolas Chauvat Andrew Durdin + Andrew Chambers Michael Schneider Nicholas Riley Jason Chu @@ -128,10 +134,13 @@ Olivier Dormond Jared Grubb Karl Bartel + Tobias Pape Brian Dorsey Victor Stinner + Andrews Medina Stuart Williams Jasper Schulz + Christian Hudon Toby Watson Antoine Pitrou Aaron Iles @@ -141,7 +150,6 @@ Neil Shepperd Mikael Schönenberg Elmo Mäntynen - Tobias Pape Jonathan David Riehl Stanislaw Halik Anders Qvist @@ -153,19 +161,15 @@ Alexander Sedov Corbin Simpson Christopher Pope - Laurence Tratt - Guillebert Romain Christian Tismer Dan Stromberg Stefano Parmesan - Christian Hudon Alexis Daboville Jens-Uwe Mager Carl Meyer Karl Ramm Pieter Zieschang Gabriel - Paweł Piotr Przeradowski Andrew Dalke Sylvain Thenault Nathan Taylor @@ -189,13 +193,15 @@ Martin Blais Lene Wagner Tomo Cocoa - Andrews Medina roberto at goyle + Yury V. Zaytsev + Anna Katrina Dominguez William Leslie Bobby Impollonia timo at eistee.fritz.box Andrew Thompson Yusei Tahara + Ben Darnell Roberto De Ioris Juan Francisco Cantero Hurtado Godefroid Chappelle @@ -209,7 +215,7 @@ Akira Li Gustavo Niemeyer Stephan Busemann - Anna Katrina Dominguez + Rafał Gałczyński Christian Muirhead James Lan shoma hosaka @@ -219,6 +225,7 @@ Dinu Gherman Chris Lambacher coolbutuseless at gmail.com + w31rd0 Jim Baker Rodrigo Araújo Armin Ronacher @@ -234,12 +241,12 @@ Even Wiik Thomassen jbs soareschen + Mike Bayer Flavio Percoco Kristoffer Kleine yasirs Michael Chermside Anna Ravencroft - Andrew Chambers Julien Phalip Dan Loewenherz diff --git a/pypy/doc/interpreter.rst b/pypy/doc/interpreter.rst --- a/pypy/doc/interpreter.rst +++ b/pypy/doc/interpreter.rst @@ -137,7 +137,8 @@ control flow of a function (such as ``while`` and ``try`` constructs) - a value stack where bytecode interpretation pulls object - from and puts results on. + from and puts results on. (``locals_stack_w`` is actually a single + list containing both the local scope and the value stack.) - a reference to the *globals* dictionary, containing module-level name-value bindings @@ -151,10 +152,7 @@ - the class ``PyFrame`` is defined in `pypy/interpreter/pyframe.py`_. -- the file `pypy/interpreter/pyopcode.py`_ add support for all Python opcode. - -- nested scope support is added to the ``PyFrame`` class in - `pypy/interpreter/nestedscope.py`_. +- the file `pypy/interpreter/pyopcode.py`_ add support for all Python opcodes. .. _Code: @@ -184,12 +182,6 @@ * ``co_name`` name of the code object (often the function name) * ``co_lnotab`` a helper table to compute the line-numbers corresponding to bytecodes -In PyPy, code objects also have the responsibility of creating their Frame_ objects -via the `'create_frame()`` method. With proper parser and compiler support this would -allow to create custom Frame objects extending the execution of functions -in various ways. The several Frame_ classes already utilize this flexibility -in order to implement Generators and Nested Scopes. - .. _Function: Function and Method classes diff --git a/pypy/doc/tool/makecontributor.py b/pypy/doc/tool/makecontributor.py --- a/pypy/doc/tool/makecontributor.py +++ b/pypy/doc/tool/makecontributor.py @@ -1,3 +1,5 @@ +# NOTE: run this script with LANG=en_US.UTF-8 + import py import sys from collections import defaultdict @@ -132,7 +134,7 @@ if show_numbers: print '%5d %s' % (n, name) else: - print name + print ' ' + name if __name__ == '__main__': show_numbers = '-n' in sys.argv diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -91,6 +91,7 @@ .. branch: safe-win-mmap .. branch: boolean-indexing-cleanup .. branch: cpyext-best_base +.. branch: cpyext-int .. branch: fileops2 .. branch: nobold-backtrace diff --git a/pypy/module/__pypy__/test/test_signal.py b/pypy/module/__pypy__/test/test_signal.py --- a/pypy/module/__pypy__/test/test_signal.py +++ b/pypy/module/__pypy__/test/test_signal.py @@ -56,7 +56,7 @@ interrupted = [] print('--- start ---') thread.start_new_thread(subthread, ()) - for j in range(10): + for j in range(30): if len(done): break print('.') time.sleep(0.25) diff --git a/pypy/module/_cffi_backend/__init__.py b/pypy/module/_cffi_backend/__init__.py --- a/pypy/module/_cffi_backend/__init__.py +++ b/pypy/module/_cffi_backend/__init__.py @@ -7,7 +7,7 @@ appleveldefs = { } interpleveldefs = { - '__version__': 'space.wrap("0.7")', + '__version__': 'space.wrap("0.8")', 'load_library': 'libraryobj.load_library', diff --git a/pypy/module/_cffi_backend/cdataobj.py b/pypy/module/_cffi_backend/cdataobj.py --- a/pypy/module/_cffi_backend/cdataobj.py +++ b/pypy/module/_cffi_backend/cdataobj.py @@ -19,9 +19,9 @@ _cdata = lltype.nullptr(rffi.CCHARP.TO) def __init__(self, space, cdata, ctype): - from pypy.module._cffi_backend import ctypeprim + from pypy.module._cffi_backend import ctypeobj assert lltype.typeOf(cdata) == rffi.CCHARP - assert isinstance(ctype, ctypeprim.W_CType) + assert isinstance(ctype, ctypeobj.W_CType) self.space = space self._cdata = cdata # don't forget keepalive_until_here! self.ctype = ctype @@ -211,7 +211,21 @@ keepalive_until_here(w_value) return # + # A fast path for [0:N] = "somestring". + from pypy.module._cffi_backend import ctypeprim space = self.space + if (space.isinstance_w(w_value, space.w_str) and + isinstance(ctitem, ctypeprim.W_CTypePrimitiveChar)): + from rpython.rtyper.annlowlevel import llstr + from rpython.rtyper.lltypesystem.rstr import copy_string_to_raw + value = space.str_w(w_value) + if len(value) != length: + raise operationerrfmt(space.w_ValueError, + "need a string of length %d, got %d", + length, len(value)) + copy_string_to_raw(llstr(value), cdata, 0, length) + return + # w_iter = space.iter(w_value) for i in range(length): try: @@ -245,19 +259,22 @@ space = self.space if isinstance(w_other, W_CData): from pypy.module._cffi_backend import ctypeptr, ctypearray + from pypy.module._cffi_backend import ctypevoid ct = w_other.ctype if isinstance(ct, ctypearray.W_CTypeArray): ct = ct.ctptr # if (ct is not self.ctype or not isinstance(ct, ctypeptr.W_CTypePointer) or - ct.ctitem.size <= 0): + (ct.ctitem.size <= 0 and not ct.is_void_ptr)): raise operationerrfmt(space.w_TypeError, "cannot subtract cdata '%s' and cdata '%s'", self.ctype.name, ct.name) # + itemsize = ct.ctitem.size + if itemsize <= 0: itemsize = 1 diff = (rffi.cast(lltype.Signed, self._cdata) - - rffi.cast(lltype.Signed, w_other._cdata)) // ct.ctitem.size + rffi.cast(lltype.Signed, w_other._cdata)) // itemsize return space.wrap(diff) # return self._add_or_sub(w_other, -1) @@ -441,6 +458,7 @@ __getitem__ = interp2app(W_CData.getitem), __setitem__ = interp2app(W_CData.setitem), __add__ = interp2app(W_CData.add), + __radd__ = interp2app(W_CData.add), __sub__ = interp2app(W_CData.sub), __getattr__ = interp2app(W_CData.getattr), __setattr__ = interp2app(W_CData.setattr), diff --git a/pypy/module/_cffi_backend/ctypearray.py b/pypy/module/_cffi_backend/ctypearray.py --- a/pypy/module/_cffi_backend/ctypearray.py +++ b/pypy/module/_cffi_backend/ctypearray.py @@ -34,19 +34,8 @@ datasize = self.size # if datasize < 0: - if (space.isinstance_w(w_init, space.w_list) or - space.isinstance_w(w_init, space.w_tuple)): - length = space.int_w(space.len(w_init)) - elif space.isinstance_w(w_init, space.w_basestring): - # from a string, we add the null terminator - length = space.int_w(space.len(w_init)) + 1 - else: - length = space.getindex_w(w_init, space.w_OverflowError) - if length < 0: - raise OperationError(space.w_ValueError, - space.wrap("negative array length")) - w_init = space.w_None - # + from pypy.module._cffi_backend import misc + w_init, length = misc.get_new_array_length(space, w_init) try: datasize = ovfcheck(length * self.ctitem.size) except OverflowError: diff --git a/pypy/module/_cffi_backend/ctypeptr.py b/pypy/module/_cffi_backend/ctypeptr.py --- a/pypy/module/_cffi_backend/ctypeptr.py +++ b/pypy/module/_cffi_backend/ctypeptr.py @@ -2,27 +2,25 @@ Pointers. """ -from pypy.interpreter.error import OperationError, operationerrfmt, wrap_oserror - from rpython.rlib import rposix from rpython.rlib.objectmodel import keepalive_until_here from rpython.rlib.rarithmetic import ovfcheck +from rpython.rtyper.annlowlevel import llstr, llunicode from rpython.rtyper.lltypesystem import lltype, rffi +from rpython.rtyper.lltypesystem.rstr import copy_string_to_raw, copy_unicode_to_raw +from pypy.interpreter.error import OperationError, operationerrfmt, wrap_oserror from pypy.module._cffi_backend import cdataobj, misc, ctypeprim, ctypevoid from pypy.module._cffi_backend.ctypeobj import W_CType class W_CTypePtrOrArray(W_CType): - _attrs_ = ['ctitem', 'can_cast_anything', 'is_struct_ptr', - 'length'] - _immutable_fields_ = ['ctitem', 'can_cast_anything', 'is_struct_ptr', - 'length'] + _attrs_ = ['ctitem', 'can_cast_anything', 'length'] + _immutable_fields_ = ['ctitem', 'can_cast_anything', 'length'] length = -1 def __init__(self, space, size, extra, extra_position, ctitem, could_cast_anything=True): - from pypy.module._cffi_backend.ctypestruct import W_CTypeStructOrUnion name, name_position = ctitem.insert_name(extra, extra_position) W_CType.__init__(self, space, size, name, name_position) # this is the "underlying type": @@ -31,7 +29,6 @@ # - for functions, it is the return type self.ctitem = ctitem self.can_cast_anything = could_cast_anything and ctitem.cast_anything - self.is_struct_ptr = isinstance(ctitem, W_CTypeStructOrUnion) def is_char_ptr_or_array(self): return isinstance(self.ctitem, ctypeprim.W_CTypePrimitiveChar) @@ -90,8 +87,7 @@ "initializer string is too long for '%s'" " (got %d characters)", self.name, n) - for i in range(n): - cdata[i] = s[i] + copy_string_to_raw(llstr(s), cdata, 0, n) if n != self.length: cdata[n] = '\x00' elif isinstance(self.ctitem, ctypeprim.W_CTypePrimitiveUniChar): @@ -105,8 +101,7 @@ " (got %d characters)", self.name, n) unichardata = rffi.cast(rffi.CWCHARP, cdata) - for i in range(n): - unichardata[i] = s[i] + copy_unicode_to_raw(llunicode(s), unichardata, 0, n) if n != self.length: unichardata[n] = u'\x00' else: @@ -157,7 +152,6 @@ return cdataobj.W_CData(self.space, ptrdata, self) def convert_from_object(self, cdata, w_ob): - space = self.space if not isinstance(w_ob, cdataobj.W_CData): raise self._convert_error("cdata pointer", w_ob) other = w_ob.ctype @@ -197,6 +191,7 @@ W_CTypePtrBase.__init__(self, space, size, extra, 2, ctitem) def newp(self, w_init): + from pypy.module._cffi_backend.ctypestruct import W_CTypeStructOrUnion space = self.space ctitem = self.ctitem datasize = ctitem.size @@ -204,10 +199,15 @@ raise operationerrfmt(space.w_TypeError, "cannot instantiate ctype '%s' of unknown size", self.name) - if self.is_struct_ptr: + if isinstance(ctitem, W_CTypeStructOrUnion): # 'newp' on a struct-or-union pointer: in this case, we return # a W_CDataPtrToStruct object which has a strong reference # to a W_CDataNewOwning that really contains the structure. + # + if ctitem.with_var_array and not space.is_w(w_init, space.w_None): + datasize = ctitem.convert_struct_from_object( + lltype.nullptr(rffi.CCHARP.TO), w_init, datasize) + # cdatastruct = cdataobj.W_CDataNewOwning(space, datasize, ctitem) cdata = cdataobj.W_CDataPtrToStructOrUnion(space, cdatastruct._cdata, @@ -238,11 +238,15 @@ def add(self, cdata, i): space = self.space ctitem = self.ctitem + itemsize = ctitem.size if ctitem.size < 0: - raise operationerrfmt(space.w_TypeError, + if self.is_void_ptr: + itemsize = 1 + else: + raise operationerrfmt(space.w_TypeError, "ctype '%s' points to items of unknown size", self.name) - p = rffi.ptradd(cdata, i * self.ctitem.size) + p = rffi.ptradd(cdata, i * itemsize) return cdataobj.W_CData(space, p, self) def cast(self, w_ob): @@ -298,7 +302,6 @@ def convert_argument_from_object(self, cdata, w_ob): from pypy.module._cffi_backend.ctypefunc import set_mustfree_flag - space = self.space result = (not isinstance(w_ob, cdataobj.W_CData) and self._prepare_pointer_call_argument(w_ob, cdata)) if result == 0: @@ -320,7 +323,8 @@ space = self.space ctype2 = cdata.ctype if (isinstance(ctype2, W_CTypeStructOrUnion) or - (isinstance(ctype2, W_CTypePtrOrArray) and ctype2.is_struct_ptr)): + (isinstance(ctype2, W_CTypePtrOrArray) and + isinstance(ctype2.ctitem, W_CTypeStructOrUnion))): ptrdata = rffi.ptradd(cdata._cdata, offset) return cdataobj.W_CData(space, ptrdata, self) else: diff --git a/pypy/module/_cffi_backend/ctypestruct.py b/pypy/module/_cffi_backend/ctypestruct.py --- a/pypy/module/_cffi_backend/ctypestruct.py +++ b/pypy/module/_cffi_backend/ctypestruct.py @@ -9,7 +9,8 @@ from rpython.rlib import jit from rpython.rlib.objectmodel import keepalive_until_here from rpython.rlib.rarithmetic import r_uint, r_ulonglong, r_longlong, intmask -from rpython.rtyper.lltypesystem import rffi +from rpython.rlib.rarithmetic import ovfcheck +from rpython.rtyper.lltypesystem import lltype, rffi from pypy.module._cffi_backend import cdataobj, ctypeprim, misc from pypy.module._cffi_backend.ctypeobj import W_CType @@ -17,12 +18,13 @@ class W_CTypeStructOrUnion(W_CType): _immutable_fields_ = ['alignment?', 'fields_list?', 'fields_dict?', - 'custom_field_pos?'] + 'custom_field_pos?', 'with_var_array?'] # fields added by complete_struct_or_union(): alignment = -1 fields_list = None fields_dict = None custom_field_pos = False + with_var_array = False def __init__(self, space, name): W_CType.__init__(self, space, -1, name, len(name)) @@ -90,12 +92,16 @@ pass def convert_from_object(self, cdata, w_ob): - space = self.space - if self._copy_from_same(cdata, w_ob): - return + if not self._copy_from_same(cdata, w_ob): + self.convert_struct_from_object(cdata, w_ob, optvarsize=-1) + @jit.look_inside_iff( + lambda self, cdata, w_ob, optvarsize: jit.isvirtual(w_ob) + ) + def convert_struct_from_object(self, cdata, w_ob, optvarsize): self._check_only_one_argument_for_union(w_ob) + space = self.space if (space.isinstance_w(w_ob, space.w_list) or space.isinstance_w(w_ob, space.w_tuple)): lst_w = space.listview(w_ob) @@ -104,7 +110,9 @@ "too many initializers for '%s' (got %d)", self.name, len(lst_w)) for i in range(len(lst_w)): - self.fields_list[i].write(cdata, lst_w[i]) + optvarsize = self.fields_list[i].write_v(cdata, lst_w[i], + optvarsize) + return optvarsize elif space.isinstance_w(w_ob, space.w_dict): lst_w = space.fixedview(w_ob) @@ -116,11 +124,16 @@ except KeyError: space.raise_key_error(w_key) assert 0 - cf.write(cdata, space.getitem(w_ob, w_key)) + optvarsize = cf.write_v(cdata, space.getitem(w_ob, w_key), + optvarsize) + return optvarsize else: - raise self._convert_error("list or tuple or dict or struct-cdata", - w_ob) + if optvarsize == -1: + msg = "list or tuple or dict or struct-cdata" + else: + msg = "list or tuple or dict" + raise self._convert_error(msg, w_ob) @jit.elidable def _getcfield_const(self, attr): @@ -192,6 +205,37 @@ else: self.ctype.convert_from_object(cdata, w_ob) + def write_v(self, cdata, w_ob, optvarsize): + # a special case for var-sized C99 arrays + from pypy.module._cffi_backend import ctypearray + ct = self.ctype + if isinstance(ct, ctypearray.W_CTypeArray) and ct.length < 0: + space = ct.space + w_ob, varsizelength = misc.get_new_array_length(space, w_ob) + if optvarsize != -1: + # in this mode, the only purpose of this function is to compute + # the real size of the structure from a var-sized C99 array + assert cdata == lltype.nullptr(rffi.CCHARP.TO) + itemsize = ct.ctitem.size + try: + varsize = ovfcheck(itemsize * varsizelength) + size = ovfcheck(self.offset + varsize) + except OverflowError: + raise OperationError(space.w_OverflowError, + space.wrap("array size would overflow a ssize_t")) + assert size >= 0 + return max(size, optvarsize) + # if 'value' was only an integer, get_new_array_length() returns + # w_ob = space.w_None. Detect if this was the case, + # and if so, stop here, leaving the content uninitialized + # (it should be zero-initialized from somewhere else). + if space.is_w(w_ob, space.w_None): + return optvarsize + # + if optvarsize == -1: + self.write(cdata, w_ob) + return optvarsize + def convert_bitfield_to_object(self, cdata): ctype = self.ctype space = ctype.space diff --git a/pypy/module/_cffi_backend/misc.py b/pypy/module/_cffi_backend/misc.py --- a/pypy/module/_cffi_backend/misc.py +++ b/pypy/module/_cffi_backend/misc.py @@ -278,6 +278,22 @@ # ____________________________________________________________ +def get_new_array_length(space, w_value): + if (space.isinstance_w(w_value, space.w_list) or + space.isinstance_w(w_value, space.w_tuple)): + return (w_value, space.int_w(space.len(w_value))) + elif space.isinstance_w(w_value, space.w_basestring): + # from a string, we add the null terminator + return (w_value, space.int_w(space.len(w_value)) + 1) + else: + explicitlength = space.getindex_w(w_value, space.w_OverflowError) + if explicitlength < 0: + raise OperationError(space.w_ValueError, + space.wrap("negative array length")) + return (space.w_None, explicitlength) + +# ____________________________________________________________ + @specialize.arg(0) def _raw_memcopy_tp(TPP, source, dest): # in its own function: LONGLONG may make the whole function jit-opaque diff --git a/pypy/module/_cffi_backend/newtype.py b/pypy/module/_cffi_backend/newtype.py --- a/pypy/module/_cffi_backend/newtype.py +++ b/pypy/module/_cffi_backend/newtype.py @@ -158,8 +158,10 @@ fields_list = [] fields_dict = {} custom_field_pos = False + with_var_array = False - for w_field in fields_w: + for i in range(len(fields_w)): + w_field = fields_w[i] field_w = space.fixedview(w_field) if not (2 <= len(field_w) <= 4): raise OperationError(space.w_TypeError, @@ -176,7 +178,11 @@ "duplicate field name '%s'", fname) # if ftype.size < 0: - raise operationerrfmt(space.w_TypeError, + if (isinstance(ftype, ctypearray.W_CTypeArray) and fbitsize < 0 + and (i == len(fields_w) - 1 or foffset != -1)): + with_var_array = True + else: + raise operationerrfmt(space.w_TypeError, "field '%s.%s' has ctype '%s' of unknown size", w_ctype.name, fname, ftype.name) # @@ -235,7 +241,8 @@ fields_list.append(fld) fields_dict[fname] = fld - boffset += ftype.size * 8 + if ftype.size >= 0: + boffset += ftype.size * 8 prev_bitfield_size = 0 From noreply at buildbot.pypy.org Mon Nov 11 23:27:20 2013 From: noreply at buildbot.pypy.org (pjenvey) Date: Mon, 11 Nov 2013 23:27:20 +0100 (CET) Subject: [pypy-commit] pypy remove-intlong-smm: merge default Message-ID: <20131111222720.2B6CB1C02AE@cobra.cs.uni-duesseldorf.de> Author: Philip Jenvey Branch: remove-intlong-smm Changeset: r67961:9a43cf95947f Date: 2013-11-11 12:25 -0800 http://bitbucket.org/pypy/pypy/changeset/9a43cf95947f/ Log: merge default diff too long, truncating to 2000 out of 28759 lines diff --git a/Makefile b/Makefile --- a/Makefile +++ b/Makefile @@ -1,7 +1,38 @@ all: pypy-c +PYPY_EXECUTABLE := $(shell which pypy) +URAM := $(shell python -c "import sys; print 4.5 if sys.maxint>1<<32 else 2.5") + +ifeq ($(PYPY_EXECUTABLE),) +RUNINTERP = python +else +RUNINTERP = $(PYPY_EXECUTABLE) +endif + pypy-c: - @echo "Building PyPy with JIT, it'll take about 40 minutes and 4G of RAM" - @sleep 3 - rpython/bin/rpython -Ojit pypy/goal/targetpypystandalone.py + @echo + @echo "====================================================================" +ifeq ($(PYPY_EXECUTABLE),) + @echo "Building a regular (jitting) version of PyPy, using CPython." + @echo "This takes around 2 hours and $(URAM) GB of RAM." + @echo "Note that pre-installing a PyPy binary would reduce this time" + @echo "and produce basically the same result." +else + @echo "Building a regular (jitting) version of PyPy, using" + @echo "$(PYPY_EXECUTABLE) to run the translation itself." + @echo "This takes up to 1 hour and $(URAM) GB of RAM." +endif + @echo + @echo "For more control (e.g. to use multiple CPU cores during part of" + @echo "the process) you need to run \`\`rpython/bin/rpython'' directly." + @echo "For more information see \`\`http://pypy.org/download.html''." + @echo "====================================================================" + @echo + @sleep 5 + $(RUNINTERP) rpython/bin/rpython -Ojit pypy/goal/targetpypystandalone.py + +# Note: the -jN option, or MAKEFLAGS=-jN, are not usable. They are +# replaced with an opaque --jobserver option by the time this Makefile +# runs. We cannot get their original value either: +# http://lists.gnu.org/archive/html/help-make/2010-08/msg00106.html diff --git a/README.rst b/README.rst --- a/README.rst +++ b/README.rst @@ -1,5 +1,5 @@ ===================================== -PyPy: Python in Python Implementation +PyPy: Python in Python Implementation ===================================== Welcome to PyPy! @@ -26,9 +26,11 @@ Building ======== -build with:: +build with: - rpython/bin/rpython -Ojit pypy/goal/targetpypystandalone.py +.. code-block:: console + + $ rpython/bin/rpython -Ojit pypy/goal/targetpypystandalone.py This ends up with ``pypy-c`` binary in the main pypy directory. We suggest to use virtualenv with the resulting pypy-c as the interpreter, you can diff --git a/lib-python/2.7/json/encoder.py b/lib-python/2.7/json/encoder.py --- a/lib-python/2.7/json/encoder.py +++ b/lib-python/2.7/json/encoder.py @@ -4,6 +4,21 @@ from __pypy__.builders import StringBuilder, UnicodeBuilder +class StringOrUnicodeBuilder(object): + def __init__(self): + self._builder = StringBuilder() + def append(self, string): + try: + self._builder.append(string) + except UnicodeEncodeError: + ub = UnicodeBuilder() + ub.append(self._builder.build()) + self._builder = ub + ub.append(string) + def build(self): + return self._builder.build() + + ESCAPE = re.compile(r'[\x00-\x1f\\"\b\f\n\r\t]') ESCAPE_ASCII = re.compile(r'([\\"]|[^\ -~])') HAS_UTF8 = re.compile(r'[\x80-\xff]') @@ -192,7 +207,7 @@ if self.ensure_ascii: builder = StringBuilder() else: - builder = UnicodeBuilder() + builder = StringOrUnicodeBuilder() self.__encode(o, markers, builder, 0) return builder.build() diff --git a/lib-python/2.7/string.py b/lib-python/2.7/string.py --- a/lib-python/2.7/string.py +++ b/lib-python/2.7/string.py @@ -66,16 +66,17 @@ must be of the same length. """ - if len(fromstr) != len(tostr): + n = len(fromstr) + if n != len(tostr): raise ValueError, "maketrans arguments must have same length" - global _idmapL - if not _idmapL: - _idmapL = list(_idmap) - L = _idmapL[:] - fromstr = map(ord, fromstr) - for i in range(len(fromstr)): - L[fromstr[i]] = tostr[i] - return ''.join(L) + # this function has been rewritten to suit PyPy better; it is + # almost 10x faster than the original. + buf = bytearray(256) + for i in range(256): + buf[i] = i + for i in range(n): + buf[ord(fromstr[i])] = tostr[i] + return str(buf) diff --git a/lib-python/2.7/weakref.py b/lib-python/2.7/weakref.py --- a/lib-python/2.7/weakref.py +++ b/lib-python/2.7/weakref.py @@ -48,7 +48,14 @@ def remove(wr, selfref=ref(self)): self = selfref() if self is not None: - del self.data[wr.key] + # Changed this for PyPy: made more resistent. The + # issue is that in some corner cases, self.data + # might already be changed or removed by the time + # this weakref's callback is called. If that is + # the case, we don't want to randomly kill an + # unrelated entry. + if self.data.get(wr.key) is wr: + del self.data[wr.key] self._remove = remove UserDict.UserDict.__init__(self, *args, **kw) @@ -160,22 +167,26 @@ try: o = self.data.pop(key)() except KeyError: + o = None + if o is None: if args: return args[0] - raise - if o is None: raise KeyError, key else: return o + # The logic above was fixed in PyPy def setdefault(self, key, default=None): try: - wr = self.data[key] + o = self.data[key]() except KeyError: + o = None + if o is None: self.data[key] = KeyedRef(default, self._remove, key) return default else: - return wr() + return o + # The logic above was fixed in PyPy def update(self, dict=None, **kwargs): d = self.data diff --git a/lib_pypy/cffi.egg-info b/lib_pypy/cffi.egg-info --- a/lib_pypy/cffi.egg-info +++ b/lib_pypy/cffi.egg-info @@ -1,6 +1,6 @@ Metadata-Version: 1.0 Name: cffi -Version: 0.7 +Version: 0.8 Summary: Foreign Function Interface for Python calling C code. Home-page: http://cffi.readthedocs.org Author: Armin Rigo, Maciej Fijalkowski diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py --- a/lib_pypy/cffi/__init__.py +++ b/lib_pypy/cffi/__init__.py @@ -4,5 +4,5 @@ from .api import FFI, CDefError, FFIError from .ffiplatform import VerificationError, VerificationMissing -__version__ = "0.7.2" -__version_info__ = (0, 7, 2) +__version__ = "0.8" +__version_info__ = (0, 8) diff --git a/lib_pypy/cffi/api.py b/lib_pypy/cffi/api.py --- a/lib_pypy/cffi/api.py +++ b/lib_pypy/cffi/api.py @@ -1,4 +1,5 @@ import types +from .lock import allocate_lock try: callable @@ -61,6 +62,7 @@ # rely on it! It's probably not going to work well.) self._backend = backend + self._lock = allocate_lock() self._parser = cparser.Parser() self._cached_btypes = {} self._parsed_types = types.ModuleType('parsed_types').__dict__ @@ -74,7 +76,8 @@ if name.startswith('RTLD_'): setattr(self, name, getattr(backend, name)) # - self.BVoidP = self._get_cached_btype(model.voidp_type) + with self._lock: + self.BVoidP = self._get_cached_btype(model.voidp_type) if isinstance(backend, types.ModuleType): # _cffi_backend: attach these constants to the class if not hasattr(FFI, 'NULL'): @@ -95,11 +98,12 @@ if not isinstance(csource, basestring): raise TypeError("cdef() argument must be a string") csource = csource.encode('ascii') - self._parser.parse(csource, override=override) - self._cdefsources.append(csource) - if override: - for cache in self._function_caches: - cache.clear() + with self._lock: + self._parser.parse(csource, override=override) + self._cdefsources.append(csource) + if override: + for cache in self._function_caches: + cache.clear() def dlopen(self, name, flags=0): """Load and return a dynamic library identified by 'name'. @@ -109,31 +113,47 @@ library we only look for the actual (untyped) symbols. """ assert isinstance(name, basestring) or name is None - lib, function_cache = _make_ffi_library(self, name, flags) - self._function_caches.append(function_cache) - self._libraries.append(lib) + with self._lock: + lib, function_cache = _make_ffi_library(self, name, flags) + self._function_caches.append(function_cache) + self._libraries.append(lib) return lib + def _typeof_locked(self, cdecl): + # call me with the lock! + key = cdecl + if key in self._parsed_types: + return self._parsed_types[key] + # + if not isinstance(cdecl, str): # unicode, on Python 2 + cdecl = cdecl.encode('ascii') + # + type = self._parser.parse_type(cdecl) + really_a_function_type = type.is_raw_function + if really_a_function_type: + type = type.as_function_pointer() + btype = self._get_cached_btype(type) + result = btype, really_a_function_type + self._parsed_types[key] = result + return result + def _typeof(self, cdecl, consider_function_as_funcptr=False): # string -> ctype object try: - btype, cfaf = self._parsed_types[cdecl] - if consider_function_as_funcptr and not cfaf: - raise KeyError + result = self._parsed_types[cdecl] except KeyError: - key = cdecl - if not isinstance(cdecl, str): # unicode, on Python 2 - cdecl = cdecl.encode('ascii') - cfaf = consider_function_as_funcptr - type = self._parser.parse_type(cdecl, - consider_function_as_funcptr=cfaf) - btype = self._get_cached_btype(type) - self._parsed_types[key] = btype, cfaf + with self._lock: + result = self._typeof_locked(cdecl) + # + btype, really_a_function_type = result + if really_a_function_type and not consider_function_as_funcptr: + raise CDefError("the type %r is a function type, not a " + "pointer-to-function type" % (cdecl,)) return btype def typeof(self, cdecl): """Parse the C type given as a string and return the - corresponding Python type: '>. + corresponding object. It can also be used on 'cdata' instance to get its C type. """ if isinstance(cdecl, basestring): @@ -144,6 +164,10 @@ res = _builtin_function_type(cdecl) if res is not None: return res + if (isinstance(cdecl, types.FunctionType) + and hasattr(cdecl, '_cffi_base_type')): + with self._lock: + return self._get_cached_btype(cdecl._cffi_base_type) raise TypeError(type(cdecl)) def sizeof(self, cdecl): @@ -280,14 +304,17 @@ data. Later, when this new cdata object is garbage-collected, 'destructor(old_cdata_object)' will be called. """ - try: - gc_weakrefs = self.gc_weakrefs - except AttributeError: - from .gc_weakref import GcWeakrefs - gc_weakrefs = self.gc_weakrefs = GcWeakrefs(self) - return gc_weakrefs.build(cdata, destructor) + with self._lock: + try: + gc_weakrefs = self.gc_weakrefs + except AttributeError: + from .gc_weakref import GcWeakrefs + gc_weakrefs = self.gc_weakrefs = GcWeakrefs(self) + return gc_weakrefs.build(cdata, destructor) def _get_cached_btype(self, type): + assert self._lock.acquire(False) is False + # call me with the lock! try: BType = self._cached_btypes[type] except KeyError: @@ -322,7 +349,8 @@ def _pointer_to(self, ctype): from . import model - return model.pointer_cache(self, ctype) + with self._lock: + return model.pointer_cache(self, ctype) def addressof(self, cdata, field=None): """Return the address of a . @@ -342,10 +370,12 @@ variables, which must anyway be accessed directly from the lib object returned by the original FFI instance. """ - self._parser.include(ffi_to_include._parser) - self._cdefsources.append('[') - self._cdefsources.extend(ffi_to_include._cdefsources) - self._cdefsources.append(']') + with ffi_to_include._lock: + with self._lock: + self._parser.include(ffi_to_include._parser) + self._cdefsources.append('[') + self._cdefsources.extend(ffi_to_include._cdefsources) + self._cdefsources.append(']') def new_handle(self, x): return self._backend.newp_handle(self.BVoidP, x) @@ -372,7 +402,7 @@ backendlib = backend.load_library(path, flags) copied_enums = [] # - def make_accessor(name): + def make_accessor_locked(name): key = 'function ' + name if key in ffi._parser._declarations: tp = ffi._parser._declarations[key] @@ -404,11 +434,17 @@ if enumname not in library.__dict__: library.__dict__[enumname] = enumval copied_enums.append(True) + if name in library.__dict__: + return # - if name in library.__dict__: # copied from an enum value just above, - return # or multithread's race condition raise AttributeError(name) # + def make_accessor(name): + with ffi._lock: + if name in library.__dict__ or name in FFILibrary.__dict__: + return # added by another thread while waiting for the lock + make_accessor_locked(name) + # class FFILibrary(object): def __getattr__(self, name): make_accessor(name) @@ -444,4 +480,5 @@ except (KeyError, AttributeError, TypeError): return None else: - return ffi._get_cached_btype(tp) + with ffi._lock: + return ffi._get_cached_btype(tp) diff --git a/lib_pypy/cffi/cparser.py b/lib_pypy/cffi/cparser.py --- a/lib_pypy/cffi/cparser.py +++ b/lib_pypy/cffi/cparser.py @@ -142,7 +142,7 @@ if 1 <= linenum <= len(csourcelines): line = csourcelines[linenum-1] if line: - msg = 'cannot parse "%s"\n%s' % (line, msg) + msg = 'cannot parse "%s"\n%s' % (line.strip(), msg) else: msg = 'parse error\n%s' % (msg,) raise api.CDefError(msg) @@ -217,19 +217,18 @@ # if decl.name: tp = self._get_type(node, partial_length_ok=True) - if self._is_constant_declaration(node): + if self._is_constant_globalvar(node): self._declare('constant ' + decl.name, tp) else: self._declare('variable ' + decl.name, tp) - def parse_type(self, cdecl, consider_function_as_funcptr=False): + def parse_type(self, cdecl): ast, macros = self._parse('void __dummy(\n%s\n);' % cdecl) assert not macros exprnode = ast.ext[-1].type.args.params[0] if isinstance(exprnode, pycparser.c_ast.ID): raise api.CDefError("unknown identifier '%s'" % (exprnode.name,)) - return self._get_type(exprnode.type, - consider_function_as_funcptr=consider_function_as_funcptr) + return self._get_type(exprnode.type) def _declare(self, name, obj): if name in self._declarations: @@ -249,28 +248,17 @@ return model.ConstPointerType(type) return model.PointerType(type) - def _get_type(self, typenode, convert_array_to_pointer=False, - name=None, partial_length_ok=False, - consider_function_as_funcptr=False): + def _get_type(self, typenode, name=None, partial_length_ok=False): # first, dereference typedefs, if we have it already parsed, we're good if (isinstance(typenode, pycparser.c_ast.TypeDecl) and isinstance(typenode.type, pycparser.c_ast.IdentifierType) and len(typenode.type.names) == 1 and ('typedef ' + typenode.type.names[0]) in self._declarations): type = self._declarations['typedef ' + typenode.type.names[0]] - if isinstance(type, model.ArrayType): - if convert_array_to_pointer: - return type.item - else: - if (consider_function_as_funcptr and - isinstance(type, model.RawFunctionType)): - return type.as_function_pointer() return type # if isinstance(typenode, pycparser.c_ast.ArrayDecl): # array type - if convert_array_to_pointer: - return self._get_type_pointer(self._get_type(typenode.type)) if typenode.dim is None: length = None else: @@ -331,10 +319,7 @@ # if isinstance(typenode, pycparser.c_ast.FuncDecl): # a function type - result = self._parse_function_type(typenode, name) - if consider_function_as_funcptr: - result = result.as_function_pointer() - return result + return self._parse_function_type(typenode, name) # # nested anonymous structs or unions end up here if isinstance(typenode, pycparser.c_ast.Struct): @@ -365,21 +350,24 @@ isinstance(params[0].type.type, pycparser.c_ast.IdentifierType) and list(params[0].type.type.names) == ['void']): del params[0] - args = [self._get_type(argdeclnode.type, - convert_array_to_pointer=True, - consider_function_as_funcptr=True) + args = [self._as_func_arg(self._get_type(argdeclnode.type)) for argdeclnode in params] result = self._get_type(typenode.type) return model.RawFunctionType(tuple(args), result, ellipsis) - def _is_constant_declaration(self, typenode, const=False): - if isinstance(typenode, pycparser.c_ast.ArrayDecl): - return self._is_constant_declaration(typenode.type) + def _as_func_arg(self, type): + if isinstance(type, model.ArrayType): + return model.PointerType(type.item) + elif isinstance(type, model.RawFunctionType): + return type.as_function_pointer() + else: + return type + + def _is_constant_globalvar(self, typenode): if isinstance(typenode, pycparser.c_ast.PtrDecl): - const = 'const' in typenode.quals - return self._is_constant_declaration(typenode.type, const) + return 'const' in typenode.quals if isinstance(typenode, pycparser.c_ast.TypeDecl): - return const or 'const' in typenode.quals + return 'const' in typenode.quals return False def _get_struct_union_enum_type(self, kind, type, name=None, nested=False): @@ -491,7 +479,7 @@ return tp def _make_partial(self, tp, nested): - if not isinstance(tp, model.StructType): + if not isinstance(tp, model.StructOrUnion): raise api.CDefError("%s cannot be partial" % (tp,)) if not tp.has_c_name() and not nested: raise NotImplementedError("%s is partial but has no C name" %(tp,)) @@ -511,7 +499,7 @@ if (isinstance(exprnode, pycparser.c_ast.ID) and exprnode.name == '__dotdotdotarray__'): self._partial_length = True - return None + return '...' # raise api.FFIError("unsupported expression: expected a " "simple numeric constant") diff --git a/lib_pypy/cffi/gc_weakref.py b/lib_pypy/cffi/gc_weakref.py --- a/lib_pypy/cffi/gc_weakref.py +++ b/lib_pypy/cffi/gc_weakref.py @@ -14,6 +14,6 @@ def build(self, cdata, destructor): # make a new cdata of the same type as the original one - new_cdata = self.ffi.cast(self.ffi.typeof(cdata), cdata) + new_cdata = self.ffi.cast(self.ffi._backend.typeof(cdata), cdata) self.data[ref(new_cdata, self.remove)] = destructor, cdata return new_cdata diff --git a/lib_pypy/cffi/lock.py b/lib_pypy/cffi/lock.py new file mode 100644 --- /dev/null +++ b/lib_pypy/cffi/lock.py @@ -0,0 +1,30 @@ +import sys + +if sys.version_info < (3,): + try: + from thread import allocate_lock + except ImportError: + from dummy_thread import allocate_lock +else: + try: + from _thread import allocate_lock + except ImportError: + from _dummy_thread import allocate_lock + + +##import sys +##l1 = allocate_lock + +##class allocate_lock(object): +## def __init__(self): +## self._real = l1() +## def __enter__(self): +## for i in range(4, 0, -1): +## print sys._getframe(i).f_code +## print +## return self._real.__enter__() +## def __exit__(self, *args): +## return self._real.__exit__(*args) +## def acquire(self, f): +## assert f is False +## return self._real.acquire(f) diff --git a/lib_pypy/cffi/model.py b/lib_pypy/cffi/model.py --- a/lib_pypy/cffi/model.py +++ b/lib_pypy/cffi/model.py @@ -1,7 +1,10 @@ import weakref +from .lock import allocate_lock + class BaseTypeByIdentity(object): is_array_type = False + is_raw_function = False def get_c_name(self, replace_with='', context='a C file'): result = self.c_name_with_marker @@ -146,6 +149,7 @@ # a function, but not a pointer-to-function. The backend has no # notion of such a type; it's used temporarily by parsing. _base_pattern = '(&)(%s)' + is_raw_function = True def build_backend_type(self, ffi, finishlist): from . import api @@ -192,10 +196,6 @@ _base_pattern = " const *&" _base_pattern_array = "(const *&)" - def build_backend_type(self, ffi, finishlist): - BPtr = PointerType(self.totype).get_cached_btype(ffi, finishlist) - return BPtr - const_voidp_type = ConstPointerType(void_type) @@ -216,10 +216,12 @@ self.item = item self.length = length # - if self.length is None: + if length is None: brackets = '&[]' + elif length == '...': + brackets = '&[/*...*/]' else: - brackets = '&[%d]' % self.length + brackets = '&[%d]' % length self.c_name_with_marker = ( self.item.c_name_with_marker.replace('&', brackets)) @@ -227,6 +229,10 @@ return ArrayType(self.item, newlength) def build_backend_type(self, ffi, finishlist): + if self.length == '...': + from . import api + raise api.CDefError("cannot render the type %r: unknown length" % + (self,)) self.item.get_cached_btype(ffi, finishlist) # force the item BType BPtrItem = PointerType(self.item).get_cached_btype(ffi, finishlist) return global_cache(self, ffi, 'new_array_type', BPtrItem, self.length) @@ -252,6 +258,7 @@ class StructOrUnion(StructOrUnionOrEnum): fixedlayout = None completed = False + partial = False def __init__(self, name, fldnames, fldtypes, fldbitsize): self.name = name @@ -303,20 +310,21 @@ return # not completing it: it's an opaque struct # self.completed = 1 - fldtypes = tuple(tp.get_cached_btype(ffi, finishlist) - for tp in self.fldtypes) # if self.fixedlayout is None: + fldtypes = [tp.get_cached_btype(ffi, finishlist) + for tp in self.fldtypes] lst = list(zip(self.fldnames, fldtypes, self.fldbitsize)) ffi._backend.complete_struct_or_union(BType, lst, self) # else: + fldtypes = [] fieldofs, fieldsize, totalsize, totalalignment = self.fixedlayout for i in range(len(self.fldnames)): fsize = fieldsize[i] ftype = self.fldtypes[i] # - if isinstance(ftype, ArrayType) and ftype.length is None: + if isinstance(ftype, ArrayType) and ftype.length == '...': # fix the length to match the total size BItemType = ftype.item.get_cached_btype(ffi, finishlist) nlen, nrest = divmod(fsize, ffi.sizeof(BItemType)) @@ -327,18 +335,20 @@ ftype = ftype.resolve_length(nlen) self.fldtypes = (self.fldtypes[:i] + (ftype,) + self.fldtypes[i+1:]) - BArrayType = ftype.get_cached_btype(ffi, finishlist) - fldtypes = (fldtypes[:i] + (BArrayType,) + - fldtypes[i+1:]) - continue # - bitemsize = ffi.sizeof(fldtypes[i]) - if bitemsize != fsize: - self._verification_error( - "field '%s.%s' is declared as %d bytes, but is " - "really %d bytes" % (self.name, - self.fldnames[i] or '{}', - bitemsize, fsize)) + BFieldType = ftype.get_cached_btype(ffi, finishlist) + if isinstance(ftype, ArrayType) and ftype.length is None: + assert fsize == 0 + else: + bitemsize = ffi.sizeof(BFieldType) + if bitemsize != fsize: + self._verification_error( + "field '%s.%s' is declared as %d bytes, but is " + "really %d bytes" % (self.name, + self.fldnames[i] or '{}', + bitemsize, fsize)) + fldtypes.append(BFieldType) + # lst = list(zip(self.fldnames, fldtypes, self.fldbitsize, fieldofs)) ffi._backend.complete_struct_or_union(BType, lst, self, totalsize, totalalignment) @@ -348,11 +358,6 @@ from .ffiplatform import VerificationError raise VerificationError(msg) - -class StructType(StructOrUnion): - kind = 'struct' - partial = False - def check_not_partial(self): if self.partial and self.fixedlayout is None: from . import ffiplatform @@ -361,19 +366,18 @@ def build_backend_type(self, ffi, finishlist): self.check_not_partial() finishlist.append(self) - - return global_cache(self, ffi, 'new_struct_type', + # + return global_cache(self, ffi, 'new_%s_type' % self.kind, self.get_official_name(), key=self) +class StructType(StructOrUnion): + kind = 'struct' + + class UnionType(StructOrUnion): kind = 'union' - def build_backend_type(self, ffi, finishlist): - finishlist.append(self) - return global_cache(self, ffi, 'new_union_type', - self.get_official_name(), key=self) - class EnumType(StructOrUnionOrEnum): kind = 'enum' @@ -387,6 +391,12 @@ self.baseinttype = baseinttype self.build_c_name_with_marker() + def force_the_name(self, forcename): + StructOrUnionOrEnum.force_the_name(self, forcename) + if self.forcename is None: + name = self.get_official_name() + self.forcename = '$' + name.replace(' ', '_') + def check_not_partial(self): if self.partial and not self.partial_resolved: from . import ffiplatform @@ -444,6 +454,9 @@ tp = StructType(structname, None, None, None) return NamedPointerType(tp, name) + +global_lock = allocate_lock() + def global_cache(srctype, ffi, funcname, *args, **kwds): key = kwds.pop('key', (funcname, args)) assert not kwds @@ -464,8 +477,17 @@ res = getattr(ffi._backend, funcname)(*args) except NotImplementedError as e: raise NotImplementedError("%r: %s" % (srctype, e)) - ffi._backend.__typecache[key] = res - return res + # note that setdefault() on WeakValueDictionary is not atomic + # and contains a rare bug (http://bugs.python.org/issue19542); + # we have to use a lock and do it ourselves + cache = ffi._backend.__typecache + with global_lock: + res1 = cache.get(key) + if res1 is None: + cache[key] = res + return res + else: + return res1 def pointer_cache(ffi, BType): return global_cache('?', ffi, 'new_pointer_type', BType) diff --git a/lib_pypy/cffi/vengine_cpy.py b/lib_pypy/cffi/vengine_cpy.py --- a/lib_pypy/cffi/vengine_cpy.py +++ b/lib_pypy/cffi/vengine_cpy.py @@ -15,7 +15,7 @@ def patch_extension_kwds(self, kwds): pass - def find_module(self, module_name, path, so_suffix): + def find_module(self, module_name, path, so_suffixes): try: f, filename, descr = imp.find_module(module_name, path) except ImportError: @@ -25,7 +25,7 @@ # Note that after a setuptools installation, there are both .py # and .so files with the same basename. The code here relies on # imp.find_module() locating the .so in priority. - if descr[0] != so_suffix: + if descr[0] not in so_suffixes: return None return filename @@ -160,7 +160,10 @@ def __dir__(self): return FFILibrary._cffi_dir + list(self.__dict__) library = FFILibrary() - module._cffi_setup(lst, ffiplatform.VerificationError, library) + if module._cffi_setup(lst, ffiplatform.VerificationError, library): + import warnings + warnings.warn("reimporting %r might overwrite older definitions" + % (self.verifier.get_module_name())) # # finally, call the loaded_cpy_xxx() functions. This will perform # the final adjustments, like copying the Python->C wrapper @@ -280,8 +283,8 @@ return '_cffi_from_c_pointer((char *)%s, _cffi_type(%d))' % ( var, self._gettypenum(tp)) elif isinstance(tp, model.ArrayType): - return '_cffi_from_c_deref((char *)%s, _cffi_type(%d))' % ( - var, self._gettypenum(tp)) + return '_cffi_from_c_pointer((char *)%s, _cffi_type(%d))' % ( + var, self._gettypenum(model.PointerType(tp.item))) elif isinstance(tp, model.StructType): if tp.fldnames is None: raise TypeError("'%s' is used as %s, but is opaque" % ( @@ -464,11 +467,14 @@ prnt(' static Py_ssize_t nums[] = {') prnt(' sizeof(%s),' % cname) prnt(' offsetof(struct _cffi_aligncheck, y),') - for fname, _, fbitsize in tp.enumfields(): + for fname, ftype, fbitsize in tp.enumfields(): if fbitsize >= 0: continue # xxx ignore fbitsize for now prnt(' offsetof(%s, %s),' % (cname, fname)) - prnt(' sizeof(((%s *)0)->%s),' % (cname, fname)) + if isinstance(ftype, model.ArrayType) and ftype.length is None: + prnt(' 0, /* %s */' % ftype._get_c_name()) + else: + prnt(' sizeof(((%s *)0)->%s),' % (cname, fname)) prnt(' -1') prnt(' };') prnt(' return _cffi_get_struct_layout(nums);') @@ -491,7 +497,7 @@ # function = getattr(module, layoutfuncname) layout = function() - if isinstance(tp, model.StructType) and tp.partial: + if isinstance(tp, model.StructOrUnion) and tp.partial: # use the function()'s sizes and offsets to guide the # layout of the struct totalsize = layout[0] @@ -528,9 +534,10 @@ continue # xxx ignore fbitsize for now check(layout[i], ffi.offsetof(BStruct, fname), "wrong offset for field %r" % (fname,)) - BField = ffi._get_cached_btype(ftype) - check(layout[i+1], ffi.sizeof(BField), - "wrong size for field %r" % (fname,)) + if layout[i+1] != 0: + BField = ffi._get_cached_btype(ftype) + check(layout[i+1], ffi.sizeof(BField), + "wrong size for field %r" % (fname,)) i += 2 assert i == len(layout) @@ -566,7 +573,7 @@ # constants, likely declared with '#define' def _generate_cpy_const(self, is_int, name, tp=None, category='const', - vartp=None, delayed=True): + vartp=None, delayed=True, size_too=False): prnt = self._prnt funcname = '_cffi_%s_%s' % (category, name) prnt('static int %s(PyObject *lib)' % funcname) @@ -597,6 +604,15 @@ '(unsigned long long)(%s));' % (name,)) prnt(' if (o == NULL)') prnt(' return -1;') + if size_too: + prnt(' {') + prnt(' PyObject *o1 = o;') + prnt(' o = Py_BuildValue("On", o1, (Py_ssize_t)sizeof(%s));' + % (name,)) + prnt(' Py_DECREF(o1);') + prnt(' if (o == NULL)') + prnt(' return -1;') + prnt(' }') prnt(' res = PyObject_SetAttrString(lib, "%s", o);' % name) prnt(' Py_DECREF(o);') prnt(' if (res < 0)') @@ -633,12 +649,23 @@ prnt('static int %s(PyObject *lib)' % funcname) prnt('{') for enumerator, enumvalue in zip(tp.enumerators, tp.enumvalues): - prnt(' if (%s != %d) {' % (enumerator, enumvalue)) + if enumvalue < 0: + prnt(' if ((%s) >= 0 || (long)(%s) != %dL) {' % ( + enumerator, enumerator, enumvalue)) + else: + prnt(' if ((%s) < 0 || (unsigned long)(%s) != %dUL) {' % ( + enumerator, enumerator, enumvalue)) + prnt(' char buf[64];') + prnt(' if ((%s) < 0)' % enumerator) + prnt(' snprintf(buf, 63, "%%ld", (long)(%s));' % enumerator) + prnt(' else') + prnt(' snprintf(buf, 63, "%%lu", (unsigned long)(%s));' % + enumerator) prnt(' PyErr_Format(_cffi_VerificationError,') - prnt(' "enum %s: %s has the real value %d, ' - 'not %d",') - prnt(' "%s", "%s", (int)%s, %d);' % ( - name, enumerator, enumerator, enumvalue)) + prnt(' "enum %s: %s has the real value %s, ' + 'not %s",') + prnt(' "%s", "%s", buf, "%d");' % ( + name, enumerator, enumvalue)) prnt(' return -1;') prnt(' }') prnt(' return %s;' % self._chained_list_constants[True]) @@ -677,15 +704,16 @@ def _generate_cpy_variable_collecttype(self, tp, name): if isinstance(tp, model.ArrayType): - self._do_collect_type(tp) + tp_ptr = model.PointerType(tp.item) else: tp_ptr = model.PointerType(tp) - self._do_collect_type(tp_ptr) + self._do_collect_type(tp_ptr) def _generate_cpy_variable_decl(self, tp, name): if isinstance(tp, model.ArrayType): tp_ptr = model.PointerType(tp.item) - self._generate_cpy_const(False, name, tp, vartp=tp_ptr) + self._generate_cpy_const(False, name, tp, vartp=tp_ptr, + size_too = (tp.length == '...')) else: tp_ptr = model.PointerType(tp) self._generate_cpy_const(False, name, tp_ptr, category='var') @@ -694,11 +722,29 @@ _loading_cpy_variable = _loaded_noop def _loaded_cpy_variable(self, tp, name, module, library): + value = getattr(library, name) if isinstance(tp, model.ArrayType): # int a[5] is "constant" in the - return # sense that "a=..." is forbidden + # sense that "a=..." is forbidden + if tp.length == '...': + assert isinstance(value, tuple) + (value, size) = value + BItemType = self.ffi._get_cached_btype(tp.item) + length, rest = divmod(size, self.ffi.sizeof(BItemType)) + if rest != 0: + raise ffiplatform.VerificationError( + "bad size: %r does not seem to be an array of %s" % + (name, tp.item)) + tp = tp.resolve_length(length) + # 'value' is a which we have to replace with + # a if the N is actually known + if tp.length is not None: + BArray = self.ffi._get_cached_btype(tp) + value = self.ffi.cast(BArray, value) + setattr(library, name, value) + return # remove ptr= from the library instance, and replace # it by a property on the class, which reads/writes into ptr[0]. - ptr = getattr(library, name) + ptr = value delattr(library, name) def getter(library): return ptr[0] @@ -711,12 +757,9 @@ def _generate_setup_custom(self): prnt = self._prnt - prnt('static PyObject *_cffi_setup_custom(PyObject *lib)') + prnt('static int _cffi_setup_custom(PyObject *lib)') prnt('{') - prnt(' if (%s < 0)' % self._chained_list_constants[True]) - prnt(' return NULL;') - prnt(' Py_INCREF(Py_None);') - prnt(' return Py_None;') + prnt(' return %s;' % self._chained_list_constants[True]) prnt('}') cffimod_header = r''' @@ -834,17 +877,20 @@ static void *_cffi_exports[_CFFI_NUM_EXPORTS]; static PyObject *_cffi_types, *_cffi_VerificationError; -static PyObject *_cffi_setup_custom(PyObject *lib); /* forward */ +static int _cffi_setup_custom(PyObject *lib); /* forward */ static PyObject *_cffi_setup(PyObject *self, PyObject *args) { PyObject *library; + int was_alive = (_cffi_types != NULL); if (!PyArg_ParseTuple(args, "OOO", &_cffi_types, &_cffi_VerificationError, &library)) return NULL; Py_INCREF(_cffi_types); Py_INCREF(_cffi_VerificationError); - return _cffi_setup_custom(library); + if (_cffi_setup_custom(library) < 0) + return NULL; + return PyBool_FromLong(was_alive); } static void _cffi_init(void) diff --git a/lib_pypy/cffi/vengine_gen.py b/lib_pypy/cffi/vengine_gen.py --- a/lib_pypy/cffi/vengine_gen.py +++ b/lib_pypy/cffi/vengine_gen.py @@ -20,15 +20,15 @@ # up in kwds['export_symbols']. kwds.setdefault('export_symbols', self.export_symbols) - def find_module(self, module_name, path, so_suffix): - basename = module_name + so_suffix - if path is None: - path = sys.path - for dirname in path: - filename = os.path.join(dirname, basename) - if os.path.isfile(filename): - return filename - return None + def find_module(self, module_name, path, so_suffixes): + for so_suffix in so_suffixes: + basename = module_name + so_suffix + if path is None: + path = sys.path + for dirname in path: + filename = os.path.join(dirname, basename) + if os.path.isfile(filename): + return filename def collect_types(self): pass # not needed in the generic engine @@ -173,6 +173,7 @@ newfunction = self._load_constant(False, tp, name, module) else: indirections = [] + base_tp = tp if any(isinstance(typ, model.StructOrUnion) for typ in tp.args): indirect_args = [] for i, typ in enumerate(tp.args): @@ -186,16 +187,18 @@ wrappername = '_cffi_f_%s' % name newfunction = module.load_function(BFunc, wrappername) for i, typ in indirections: - newfunction = self._make_struct_wrapper(newfunction, i, typ) + newfunction = self._make_struct_wrapper(newfunction, i, typ, + base_tp) setattr(library, name, newfunction) type(library)._cffi_dir.append(name) - def _make_struct_wrapper(self, oldfunc, i, tp): + def _make_struct_wrapper(self, oldfunc, i, tp, base_tp): backend = self.ffi._backend BType = self.ffi._get_cached_btype(tp) def newfunc(*args): args = args[:i] + (backend.newp(BType, args[i]),) + args[i+1:] return oldfunc(*args) + newfunc._cffi_base_type = base_tp return newfunc # ---------- @@ -252,11 +255,14 @@ prnt(' static ssize_t nums[] = {') prnt(' sizeof(%s),' % cname) prnt(' offsetof(struct _cffi_aligncheck, y),') - for fname, _, fbitsize in tp.enumfields(): + for fname, ftype, fbitsize in tp.enumfields(): if fbitsize >= 0: continue # xxx ignore fbitsize for now prnt(' offsetof(%s, %s),' % (cname, fname)) - prnt(' sizeof(((%s *)0)->%s),' % (cname, fname)) + if isinstance(ftype, model.ArrayType) and ftype.length is None: + prnt(' 0, /* %s */' % ftype._get_c_name()) + else: + prnt(' sizeof(((%s *)0)->%s),' % (cname, fname)) prnt(' -1') prnt(' };') prnt(' return nums[i];') @@ -270,7 +276,7 @@ return # nothing to do with opaque structs layoutfuncname = '_cffi_layout_%s_%s' % (prefix, name) # - BFunc = self.ffi.typeof("ssize_t(*)(ssize_t)") + BFunc = self.ffi._typeof_locked("ssize_t(*)(ssize_t)")[0] function = module.load_function(BFunc, layoutfuncname) layout = [] num = 0 @@ -279,7 +285,7 @@ if x < 0: break layout.append(x) num += 1 - if isinstance(tp, model.StructType) and tp.partial: + if isinstance(tp, model.StructOrUnion) and tp.partial: # use the function()'s sizes and offsets to guide the # layout of the struct totalsize = layout[0] @@ -316,9 +322,10 @@ continue # xxx ignore fbitsize for now check(layout[i], ffi.offsetof(BStruct, fname), "wrong offset for field %r" % (fname,)) - BField = ffi._get_cached_btype(ftype) - check(layout[i+1], ffi.sizeof(BField), - "wrong size for field %r" % (fname,)) + if layout[i+1] != 0: + BField = ffi._get_cached_btype(ftype) + check(layout[i+1], ffi.sizeof(BField), + "wrong size for field %r" % (fname,)) i += 2 assert i == len(layout) @@ -379,15 +386,17 @@ def _load_constant(self, is_int, tp, name, module): funcname = '_cffi_const_%s' % name if is_int: - BFunc = self.ffi.typeof("int(*)(long long*)") + BType = self.ffi._typeof_locked("long long*")[0] + BFunc = self.ffi._typeof_locked("int(*)(long long*)")[0] function = module.load_function(BFunc, funcname) - p = self.ffi.new("long long*") + p = self.ffi.new(BType) negative = function(p) value = int(p[0]) if value < 0 and not negative: - value += (1 << (8*self.ffi.sizeof("long long"))) + BLongLong = self.ffi._typeof_locked("long long")[0] + value += (1 << (8*self.ffi.sizeof(BLongLong))) else: - BFunc = self.ffi.typeof(tp.get_c_name('(*)(void)', name)) + BFunc = self.ffi._typeof_locked(tp.get_c_name('(*)(void)', name))[0] function = module.load_function(BFunc, funcname) value = function() return value @@ -413,11 +422,22 @@ prnt('int %s(char *out_error)' % funcname) prnt('{') for enumerator, enumvalue in zip(tp.enumerators, tp.enumvalues): - prnt(' if (%s != %d) {' % (enumerator, enumvalue)) + if enumvalue < 0: + prnt(' if ((%s) >= 0 || (long)(%s) != %dL) {' % ( + enumerator, enumerator, enumvalue)) + else: + prnt(' if ((%s) < 0 || (unsigned long)(%s) != %dUL) {' % ( + enumerator, enumerator, enumvalue)) + prnt(' char buf[64];') + prnt(' if ((%s) < 0)' % enumerator) + prnt(' snprintf(buf, 63, "%%ld", (long)(%s));' % enumerator) + prnt(' else') + prnt(' snprintf(buf, 63, "%%lu", (unsigned long)(%s));' % + enumerator) prnt(' snprintf(out_error, 255,' - '"%s has the real value %d, not %d",') - prnt(' "%s", (int)%s, %d);' % ( - enumerator, enumerator, enumvalue)) + ' "%s has the real value %s, not %s",') + prnt(' "%s", buf, "%d");' % ( + enumerator, enumvalue)) prnt(' return -1;') prnt(' }') prnt(' return 0;') @@ -431,10 +451,11 @@ tp.enumvalues = tuple(enumvalues) tp.partial_resolved = True else: - BFunc = self.ffi.typeof("int(*)(char*)") + BType = self.ffi._typeof_locked("char[]")[0] + BFunc = self.ffi._typeof_locked("int(*)(char*)")[0] funcname = '_cffi_e_%s_%s' % (prefix, name) function = module.load_function(BFunc, funcname) - p = self.ffi.new("char[]", 256) + p = self.ffi.new(BType, 256) if function(p) < 0: error = self.ffi.string(p) if sys.version_info >= (3,): @@ -465,6 +486,14 @@ def _generate_gen_variable_decl(self, tp, name): if isinstance(tp, model.ArrayType): + if tp.length == '...': + prnt = self._prnt + funcname = '_cffi_sizeof_%s' % (name,) + self.export_symbols.append(funcname) + prnt("size_t %s(void)" % funcname) + prnt("{") + prnt(" return sizeof(%s);" % (name,)) + prnt("}") tp_ptr = model.PointerType(tp.item) self._generate_gen_const(False, name, tp_ptr) else: @@ -476,6 +505,18 @@ def _loaded_gen_variable(self, tp, name, module, library): if isinstance(tp, model.ArrayType): # int a[5] is "constant" in the # sense that "a=..." is forbidden + if tp.length == '...': + funcname = '_cffi_sizeof_%s' % (name,) + BFunc = self.ffi._typeof_locked('size_t(*)(void)')[0] + function = module.load_function(BFunc, funcname) + size = function() + BItemType = self.ffi._get_cached_btype(tp.item) + length, rest = divmod(size, self.ffi.sizeof(BItemType)) + if rest != 0: + raise ffiplatform.VerificationError( + "bad size: %r does not seem to be an array of %s" % + (name, tp.item)) + tp = tp.resolve_length(length) tp_ptr = model.PointerType(tp.item) value = self._load_constant(False, tp_ptr, name, module) # 'value' is a which we have to replace with @@ -489,7 +530,7 @@ # remove ptr= from the library instance, and replace # it by a property on the class, which reads/writes into ptr[0]. funcname = '_cffi_var_%s' % name - BFunc = self.ffi.typeof(tp.get_c_name('*(*)(void)', name)) + BFunc = self.ffi._typeof_locked(tp.get_c_name('*(*)(void)', name))[0] function = module.load_function(BFunc, funcname) ptr = function() def getter(library): diff --git a/lib_pypy/cffi/verifier.py b/lib_pypy/cffi/verifier.py --- a/lib_pypy/cffi/verifier.py +++ b/lib_pypy/cffi/verifier.py @@ -31,7 +31,7 @@ k2 = k2.lstrip('0').rstrip('L') modulename = '_cffi_%s_%s%s%s' % (tag, self._vengine._class_key, k1, k2) - suffix = _get_so_suffix() + suffix = _get_so_suffixes()[0] self.tmpdir = tmpdir or _caller_dir_pycache() self.sourcefilename = os.path.join(self.tmpdir, modulename + '.c') self.modulefilename = os.path.join(self.tmpdir, modulename + suffix) @@ -42,18 +42,21 @@ def write_source(self, file=None): """Write the C source code. It is produced in 'self.sourcefilename', which can be tweaked beforehand.""" - if self._has_source and file is None: - raise ffiplatform.VerificationError("source code already written") - self._write_source(file) + with self.ffi._lock: + if self._has_source and file is None: + raise ffiplatform.VerificationError( + "source code already written") + self._write_source(file) def compile_module(self): """Write the C source code (if not done already) and compile it. This produces a dynamic link library in 'self.modulefilename'.""" - if self._has_module: - raise ffiplatform.VerificationError("module already compiled") - if not self._has_source: - self._write_source() - self._compile_module() + with self.ffi._lock: + if self._has_module: + raise ffiplatform.VerificationError("module already compiled") + if not self._has_source: + self._write_source() + self._compile_module() def load_library(self): """Get a C module from this Verifier instance. @@ -62,11 +65,14 @@ operations to the C module. If necessary, the C code is written and compiled first. """ - if not self._has_module: - self._locate_module() + with self.ffi._lock: if not self._has_module: - self.compile_module() - return self._load_library() + self._locate_module() + if not self._has_module: + if not self._has_source: + self._write_source() + self._compile_module() + return self._load_library() def get_module_name(self): basename = os.path.basename(self.modulefilename) @@ -81,7 +87,9 @@ def get_extension(self): if not self._has_source: - self._write_source() + with self.ffi._lock: + if not self._has_source: + self._write_source() sourcename = ffiplatform.maybe_relative_path(self.sourcefilename) modname = self.get_module_name() return ffiplatform.get_extension(sourcename, modname, **self.kwds) @@ -103,7 +111,7 @@ else: path = None filename = self._vengine.find_module(self.get_module_name(), path, - _get_so_suffix()) + _get_so_suffixes()) if filename is None: return self.modulefilename = filename @@ -193,7 +201,7 @@ if keep_so: suffix = '.c' # only remove .c files else: - suffix = _get_so_suffix().lower() + suffix = _get_so_suffixes()[0].lower() for fn in filelist: if fn.lower().startswith('_cffi_') and ( fn.lower().endswith(suffix) or fn.lower().endswith('.c')): @@ -213,15 +221,20 @@ except OSError: pass -def _get_so_suffix(): +def _get_so_suffixes(): + suffixes = [] for suffix, mode, type in imp.get_suffixes(): if type == imp.C_EXTENSION: - return suffix - # bah, no C_EXTENSION available. Occurs on pypy without cpyext - if sys.platform == 'win32': - return ".pyd" - else: - return ".so" + suffixes.append(suffix) + + if not suffixes: + # bah, no C_EXTENSION available. Occurs on pypy without cpyext + if sys.platform == 'win32': + suffixes = [".pyd"] + else: + suffixes = [".so"] + + return suffixes def _ensure_dir(filename): try: diff --git a/lib_pypy/numpy.py b/lib_pypy/numpy.py deleted file mode 100644 --- a/lib_pypy/numpy.py +++ /dev/null @@ -1,12 +0,0 @@ -import warnings -import sys -if 'numpypy' not in sys.modules: - warnings.warn( - "The 'numpy' module of PyPy is in-development and not complete. " - "To avoid this warning, write 'import numpypy as numpy'. ", - UserWarning) # XXX is this the best warning type? - -from numpypy import * -import numpypy -__all__ = numpypy.__all__ -del numpypy diff --git a/lib_pypy/numpypy/__init__.py b/lib_pypy/numpypy/__init__.py deleted file mode 100644 --- a/lib_pypy/numpypy/__init__.py +++ /dev/null @@ -1,51 +0,0 @@ -import core -from core import * -import lib -from lib import * - -from __builtin__ import bool, int, long, float, complex, object, unicode, str -from core import abs, max, min - -__version__ = '1.7.0' - -import os -def get_include(): - """ - Return the directory that contains the NumPy \\*.h header files. - - Extension modules that need to compile against NumPy should use this - function to locate the appropriate include directory. - - Notes - ----- - When using ``distutils``, for example in ``setup.py``. - :: - - import numpy as np - ... - Extension('extension_name', ... - include_dirs=[np.get_include()]) - ... - - """ - import numpy - if getattr(numpy, 'show_config', None) is None: - # running from numpy source directory - head, tail = os.path.split(os.path.dirname(os.path.abspath(__file__))) - return os.path.join(head, '../include') - else: - # using installed numpy core headers - import numpy.core as core - d = os.path.join(os.path.dirname(core.__file__), 'include') - return d - - - -__all__ = ['__version__', 'get_include'] -__all__ += core.__all__ -__all__ += lib.__all__ - -#import sys -#sys.modules.setdefault('numpy', sys.modules['numpypy']) - - diff --git a/lib_pypy/numpypy/core/__init__.py b/lib_pypy/numpypy/core/__init__.py deleted file mode 100644 --- a/lib_pypy/numpypy/core/__init__.py +++ /dev/null @@ -1,14 +0,0 @@ -import numeric -from numeric import * -import fromnumeric -from fromnumeric import * -import shape_base -from shape_base import * - -from fromnumeric import amax as max, amin as min -from numeric import absolute as abs - -__all__ = [] -__all__ += numeric.__all__ -__all__ += fromnumeric.__all__ -__all__ += shape_base.__all__ diff --git a/lib_pypy/numpypy/core/_methods.py b/lib_pypy/numpypy/core/_methods.py deleted file mode 100644 --- a/lib_pypy/numpypy/core/_methods.py +++ /dev/null @@ -1,109 +0,0 @@ -# Array methods which are called by the both the C-code for the method -# and the Python code for the NumPy-namespace function - -import multiarray as mu -import umath as um -from numeric import asanyarray - -def _amax(a, axis=None, out=None, keepdims=False): - return um.maximum.reduce(a, axis=axis, - out=out, keepdims=keepdims) - -def _amin(a, axis=None, out=None, keepdims=False): - return um.minimum.reduce(a, axis=axis, - out=out, keepdims=keepdims) - -def _sum(a, axis=None, dtype=None, out=None, keepdims=False): - return um.add.reduce(a, axis=axis, dtype=dtype, - out=out, keepdims=keepdims) - -def _prod(a, axis=None, dtype=None, out=None, keepdims=False): - return um.multiply.reduce(a, axis=axis, dtype=dtype, - out=out, keepdims=keepdims) - -def _any(a, axis=None, dtype=None, out=None, keepdims=False): - return um.logical_or.reduce(a, axis=axis, dtype=dtype, out=out, - keepdims=keepdims) - -def _all(a, axis=None, dtype=None, out=None, keepdims=False): - return um.logical_and.reduce(a, axis=axis, dtype=dtype, out=out, - keepdims=keepdims) - -def _count_reduce_items(arr, axis): - if axis is None: - axis = tuple(xrange(arr.ndim)) - if not isinstance(axis, tuple): - axis = (axis,) - items = 1 - for ax in axis: - items *= arr.shape[ax] - return items - -def _mean(a, axis=None, dtype=None, out=None, keepdims=False): - arr = asanyarray(a) - - # Upgrade bool, unsigned int, and int to float64 - if dtype is None and arr.dtype.kind in ['b','u','i']: - ret = um.add.reduce(arr, axis=axis, dtype='f8', - out=out, keepdims=keepdims) - else: - ret = um.add.reduce(arr, axis=axis, dtype=dtype, - out=out, keepdims=keepdims) - rcount = _count_reduce_items(arr, axis) - if isinstance(ret, mu.ndarray): - ret = um.true_divide(ret, rcount, - out=ret, casting='unsafe', subok=False) - else: - ret = ret / float(rcount) - return ret - -def _var(a, axis=None, dtype=None, out=None, ddof=0, - keepdims=False): - arr = asanyarray(a) - - # First compute the mean, saving 'rcount' for reuse later - if dtype is None and arr.dtype.kind in ['b','u','i']: - arrmean = um.add.reduce(arr, axis=axis, dtype='f8', keepdims=True) - else: - arrmean = um.add.reduce(arr, axis=axis, dtype=dtype, keepdims=True) - rcount = _count_reduce_items(arr, axis) - if isinstance(arrmean, mu.ndarray): - arrmean = um.true_divide(arrmean, rcount, - out=arrmean, casting='unsafe', subok=False) - else: - arrmean = arrmean / float(rcount) - - # arr - arrmean - x = arr - arrmean - - # (arr - arrmean) ** 2 - if arr.dtype.kind == 'c': - x = um.multiply(x, um.conjugate(x), out=x).real - else: - x = um.multiply(x, x, out=x) - - # add.reduce((arr - arrmean) ** 2, axis) - ret = um.add.reduce(x, axis=axis, dtype=dtype, out=out, keepdims=keepdims) - - # add.reduce((arr - arrmean) ** 2, axis) / (n - ddof) - if not keepdims and isinstance(rcount, mu.ndarray): - rcount = rcount.squeeze(axis=axis) - rcount -= ddof - if isinstance(ret, mu.ndarray): - ret = um.true_divide(ret, rcount, - out=ret, casting='unsafe', subok=False) - else: - ret = ret / float(rcount) - - return ret - -def _std(a, axis=None, dtype=None, out=None, ddof=0, keepdims=False): - ret = _var(a, axis=axis, dtype=dtype, out=out, ddof=ddof, - keepdims=keepdims) - - if isinstance(ret, mu.ndarray): - ret = um.sqrt(ret, out=ret) - else: - ret = um.sqrt(ret) - - return ret diff --git a/lib_pypy/numpypy/core/arrayprint.py b/lib_pypy/numpypy/core/arrayprint.py deleted file mode 100644 --- a/lib_pypy/numpypy/core/arrayprint.py +++ /dev/null @@ -1,750 +0,0 @@ -"""Array printing function - -$Id: arrayprint.py,v 1.9 2005/09/13 13:58:44 teoliphant Exp $ -""" -__all__ = ["array2string", "set_printoptions", "get_printoptions"] -__docformat__ = 'restructuredtext' - -# -# Written by Konrad Hinsen -# last revision: 1996-3-13 -# modified by Jim Hugunin 1997-3-3 for repr's and str's (and other details) -# and by Perry Greenfield 2000-4-1 for numarray -# and by Travis Oliphant 2005-8-22 for numpy - -import sys -import numerictypes as _nt -from umath import maximum, minimum, absolute, not_equal, isnan, isinf -#from multiarray import format_longfloat, datetime_as_string, datetime_data -from fromnumeric import ravel - - -def product(x, y): return x*y - -_summaryEdgeItems = 3 # repr N leading and trailing items of each dimension -_summaryThreshold = 1000 # total items > triggers array summarization - -_float_output_precision = 8 -_float_output_suppress_small = False -_line_width = 75 -_nan_str = 'nan' -_inf_str = 'inf' -_formatter = None # formatting function for array elements - -if sys.version_info[0] >= 3: - from functools import reduce - -def set_printoptions(precision=None, threshold=None, edgeitems=None, - linewidth=None, suppress=None, - nanstr=None, infstr=None, - formatter=None): - """ - Set printing options. - - These options determine the way floating point numbers, arrays and - other NumPy objects are displayed. - - Parameters - ---------- - precision : int, optional - Number of digits of precision for floating point output (default 8). - threshold : int, optional - Total number of array elements which trigger summarization - rather than full repr (default 1000). - edgeitems : int, optional - Number of array items in summary at beginning and end of - each dimension (default 3). - linewidth : int, optional - The number of characters per line for the purpose of inserting - line breaks (default 75). - suppress : bool, optional - Whether or not suppress printing of small floating point values - using scientific notation (default False). - nanstr : str, optional - String representation of floating point not-a-number (default nan). - infstr : str, optional - String representation of floating point infinity (default inf). - formatter : dict of callables, optional - If not None, the keys should indicate the type(s) that the respective - formatting function applies to. Callables should return a string. - Types that are not specified (by their corresponding keys) are handled - by the default formatters. Individual types for which a formatter - can be set are:: - - - 'bool' - - 'int' - - 'timedelta' : a `numpy.timedelta64` - - 'datetime' : a `numpy.datetime64` - - 'float' - - 'longfloat' : 128-bit floats - - 'complexfloat' - - 'longcomplexfloat' : composed of two 128-bit floats - - 'numpy_str' : types `numpy.string_` and `numpy.unicode_` - - 'str' : all other strings - - Other keys that can be used to set a group of types at once are:: - - - 'all' : sets all types - - 'int_kind' : sets 'int' - - 'float_kind' : sets 'float' and 'longfloat' - - 'complex_kind' : sets 'complexfloat' and 'longcomplexfloat' - - 'str_kind' : sets 'str' and 'numpystr' - - See Also - -------- - get_printoptions, set_string_function, array2string - - Notes - ----- - `formatter` is always reset with a call to `set_printoptions`. - - Examples - -------- - Floating point precision can be set: - - >>> np.set_printoptions(precision=4) - >>> print np.array([1.123456789]) - [ 1.1235] - - Long arrays can be summarised: - - >>> np.set_printoptions(threshold=5) - >>> print np.arange(10) - [0 1 2 ..., 7 8 9] - - Small results can be suppressed: - - >>> eps = np.finfo(float).eps - >>> x = np.arange(4.) - >>> x**2 - (x + eps)**2 - array([ -4.9304e-32, -4.4409e-16, 0.0000e+00, 0.0000e+00]) - >>> np.set_printoptions(suppress=True) - >>> x**2 - (x + eps)**2 - array([-0., -0., 0., 0.]) - - A custom formatter can be used to display array elements as desired: - - >>> np.set_printoptions(formatter={'all':lambda x: 'int: '+str(-x)}) - >>> x = np.arange(3) - >>> x - array([int: 0, int: -1, int: -2]) - >>> np.set_printoptions() # formatter gets reset - >>> x - array([0, 1, 2]) - - To put back the default options, you can use: - - >>> np.set_printoptions(edgeitems=3,infstr='inf', - ... linewidth=75, nanstr='nan', precision=8, - ... suppress=False, threshold=1000, formatter=None) - """ - - global _summaryThreshold, _summaryEdgeItems, _float_output_precision, \ - _line_width, _float_output_suppress_small, _nan_str, _inf_str, \ - _formatter - if linewidth is not None: - _line_width = linewidth - if threshold is not None: - _summaryThreshold = threshold - if edgeitems is not None: - _summaryEdgeItems = edgeitems - if precision is not None: - _float_output_precision = precision - if suppress is not None: - _float_output_suppress_small = not not suppress - if nanstr is not None: - _nan_str = nanstr - if infstr is not None: - _inf_str = infstr - _formatter = formatter - -def get_printoptions(): - """ - Return the current print options. - - Returns - ------- - print_opts : dict - Dictionary of current print options with keys - - - precision : int - - threshold : int - - edgeitems : int - - linewidth : int - - suppress : bool - - nanstr : str - - infstr : str - - formatter : dict of callables - - For a full description of these options, see `set_printoptions`. - - See Also - -------- - set_printoptions, set_string_function - - """ - d = dict(precision=_float_output_precision, - threshold=_summaryThreshold, - edgeitems=_summaryEdgeItems, - linewidth=_line_width, - suppress=_float_output_suppress_small, - nanstr=_nan_str, - infstr=_inf_str, - formatter=_formatter) - return d - -def _leading_trailing(a): - import numeric as _nc - if a.ndim == 1: - if len(a) > 2*_summaryEdgeItems: - b = _nc.concatenate((a[:_summaryEdgeItems], - a[-_summaryEdgeItems:])) - else: - b = a - else: - if len(a) > 2*_summaryEdgeItems: - l = [_leading_trailing(a[i]) for i in range( - min(len(a), _summaryEdgeItems))] - l.extend([_leading_trailing(a[-i]) for i in range( - min(len(a), _summaryEdgeItems),0,-1)]) - else: - l = [_leading_trailing(a[i]) for i in range(0, len(a))] - b = _nc.concatenate(tuple(l)) - return b - -def _boolFormatter(x): - if x: - return ' True' - else: - return 'False' - - -def repr_format(x): - return repr(x) - -def _array2string(a, max_line_width, precision, suppress_small, separator=' ', - prefix="", formatter=None): - - if max_line_width is None: - max_line_width = _line_width - - if precision is None: - precision = _float_output_precision - - if suppress_small is None: - suppress_small = _float_output_suppress_small - - if formatter is None: - formatter = _formatter - - if a.size > _summaryThreshold: - summary_insert = "..., " - data = _leading_trailing(a) - else: - summary_insert = "" - data = ravel(a) - - formatdict = {'bool' : _boolFormatter, - 'int' : IntegerFormat(data), - 'float' : FloatFormat(data, precision, suppress_small), - 'longfloat' : LongFloatFormat(precision), - 'complexfloat' : ComplexFormat(data, precision, - suppress_small), - 'longcomplexfloat' : LongComplexFormat(precision), - 'datetime' : DatetimeFormat(data), - 'timedelta' : TimedeltaFormat(data), - 'numpystr' : repr_format, - 'str' : str} - - if formatter is not None: - fkeys = [k for k in formatter.keys() if formatter[k] is not None] - if 'all' in fkeys: - for key in formatdict.keys(): - formatdict[key] = formatter['all'] - if 'int_kind' in fkeys: - for key in ['int']: - formatdict[key] = formatter['int_kind'] - if 'float_kind' in fkeys: - for key in ['float', 'longfloat']: - formatdict[key] = formatter['float_kind'] - if 'complex_kind' in fkeys: - for key in ['complexfloat', 'longcomplexfloat']: - formatdict[key] = formatter['complex_kind'] - if 'str_kind' in fkeys: - for key in ['numpystr', 'str']: - formatdict[key] = formatter['str_kind'] - for key in formatdict.keys(): - if key in fkeys: - formatdict[key] = formatter[key] - - try: - format_function = a._format - msg = "The `_format` attribute is deprecated in Numpy 2.0 and " \ - "will be removed in 2.1. Use the `formatter` kw instead." - import warnings - warnings.warn(msg, DeprecationWarning) - except AttributeError: - # find the right formatting function for the array - dtypeobj = a.dtype.type - if issubclass(dtypeobj, _nt.bool_): - format_function = formatdict['bool'] - elif issubclass(dtypeobj, _nt.integer): - #if issubclass(dtypeobj, _nt.timedelta64): - # format_function = formatdict['timedelta'] - #else: - format_function = formatdict['int'] - elif issubclass(dtypeobj, _nt.floating): - if hasattr(_nt, 'longfloat') and issubclass(dtypeobj, _nt.longfloat): - format_function = formatdict['longfloat'] - else: - format_function = formatdict['float'] - elif issubclass(dtypeobj, _nt.complexfloating): - if hasattr(_nt, 'clongfloat') and issubclass(dtypeobj, _nt.clongfloat): - format_function = formatdict['longcomplexfloat'] - else: - format_function = formatdict['complexfloat'] - elif issubclass(dtypeobj, (_nt.unicode_, _nt.string_)): - format_function = formatdict['numpystr'] - #elif issubclass(dtypeobj, _nt.datetime64): - # format_function = formatdict['datetime'] - else: - format_function = formatdict['str'] - - # skip over "[" - next_line_prefix = " " - # skip over array( - next_line_prefix += " "*len(prefix) - - lst = _formatArray(a, format_function, len(a.shape), max_line_width, - next_line_prefix, separator, - _summaryEdgeItems, summary_insert)[:-1] - return lst - -def _convert_arrays(obj): - import numeric as _nc - newtup = [] - for k in obj: - if isinstance(k, _nc.ndarray): - k = k.tolist() - elif isinstance(k, tuple): - k = _convert_arrays(k) - newtup.append(k) - return tuple(newtup) - - -def array2string(a, max_line_width=None, precision=None, - suppress_small=None, separator=' ', prefix="", - style=repr, formatter=None): - """ - Return a string representation of an array. - - Parameters - ---------- - a : ndarray - Input array. - max_line_width : int, optional - The maximum number of columns the string should span. Newline - characters splits the string appropriately after array elements. - precision : int, optional - Floating point precision. Default is the current printing - precision (usually 8), which can be altered using `set_printoptions`. - suppress_small : bool, optional - Represent very small numbers as zero. A number is "very small" if it - is smaller than the current printing precision. - separator : str, optional - Inserted between elements. - prefix : str, optional - An array is typically printed as:: - - 'prefix(' + array2string(a) + ')' - - The length of the prefix string is used to align the - output correctly. - style : function, optional - A function that accepts an ndarray and returns a string. Used only - when the shape of `a` is equal to ``()``, i.e. for 0-D arrays. - formatter : dict of callables, optional - If not None, the keys should indicate the type(s) that the respective - formatting function applies to. Callables should return a string. - Types that are not specified (by their corresponding keys) are handled - by the default formatters. Individual types for which a formatter - can be set are:: - - - 'bool' - - 'int' - - 'timedelta' : a `numpy.timedelta64` - - 'datetime' : a `numpy.datetime64` - - 'float' - - 'longfloat' : 128-bit floats - - 'complexfloat' - - 'longcomplexfloat' : composed of two 128-bit floats - - 'numpy_str' : types `numpy.string_` and `numpy.unicode_` - - 'str' : all other strings - - Other keys that can be used to set a group of types at once are:: - - - 'all' : sets all types - - 'int_kind' : sets 'int' - - 'float_kind' : sets 'float' and 'longfloat' - - 'complex_kind' : sets 'complexfloat' and 'longcomplexfloat' - - 'str_kind' : sets 'str' and 'numpystr' - - Returns - ------- - array_str : str - String representation of the array. - - Raises - ------ - TypeError : if a callable in `formatter` does not return a string. - - See Also - -------- - array_str, array_repr, set_printoptions, get_printoptions - - Notes - ----- - If a formatter is specified for a certain type, the `precision` keyword is - ignored for that type. - - Examples - -------- - >>> x = np.array([1e-16,1,2,3]) - >>> print np.array2string(x, precision=2, separator=',', - ... suppress_small=True) - [ 0., 1., 2., 3.] - - >>> x = np.arange(3.) - >>> np.array2string(x, formatter={'float_kind':lambda x: "%.2f" % x}) - '[0.00 1.00 2.00]' - - >>> x = np.arange(3) - >>> np.array2string(x, formatter={'int':lambda x: hex(x)}) - '[0x0L 0x1L 0x2L]' - - """ - - if a.shape == (): - x = a.item() - try: - lst = a._format(x) - msg = "The `_format` attribute is deprecated in Numpy " \ - "2.0 and will be removed in 2.1. Use the " \ - "`formatter` kw instead." - import warnings - warnings.warn(msg, DeprecationWarning) - except AttributeError: - if isinstance(x, tuple): - x = _convert_arrays(x) - lst = style(x) - elif reduce(product, a.shape) == 0: From noreply at buildbot.pypy.org Mon Nov 11 23:27:21 2013 From: noreply at buildbot.pypy.org (pjenvey) Date: Mon, 11 Nov 2013 23:27:21 +0100 (CET) Subject: [pypy-commit] pypy default: some w_self -> self Message-ID: <20131111222721.6072B1C04FF@cobra.cs.uni-duesseldorf.de> Author: Philip Jenvey Branch: Changeset: r67962:261e5f91f5f0 Date: 2013-11-11 12:44 -0800 http://bitbucket.org/pypy/pypy/changeset/261e5f91f5f0/ Log: some w_self -> self diff --git a/pypy/objspace/std/complexobject.py b/pypy/objspace/std/complexobject.py --- a/pypy/objspace/std/complexobject.py +++ b/pypy/objspace/std/complexobject.py @@ -52,16 +52,16 @@ from pypy.objspace.std.complextype import complex_typedef as typedef _immutable_fields_ = ['realval', 'imagval'] - def __init__(w_self, realval=0.0, imgval=0.0): - w_self.realval = float(realval) - w_self.imagval = float(imgval) + def __init__(self, realval=0.0, imgval=0.0): + self.realval = float(realval) + self.imagval = float(imgval) - def unwrap(w_self, space): # for tests only - return complex(w_self.realval, w_self.imagval) + def unwrap(self, space): # for tests only + return complex(self.realval, self.imagval) - def __repr__(w_self): + def __repr__(self): """ representation for debugging purposes """ - return "" % (w_self.realval, w_self.imagval) + return "" % (self.realval, self.imagval) def as_tuple(self): return (self.realval, self.imagval) diff --git a/pypy/objspace/std/floatobject.py b/pypy/objspace/std/floatobject.py --- a/pypy/objspace/std/floatobject.py +++ b/pypy/objspace/std/floatobject.py @@ -27,11 +27,11 @@ typedef = float_typedef - def __init__(w_self, floatval): - w_self.floatval = floatval + def __init__(self, floatval): + self.floatval = floatval - def unwrap(w_self, space): - return w_self.floatval + def unwrap(self, space): + return self.floatval def float_w(self, space): return self.floatval diff --git a/pypy/objspace/std/smalllongobject.py b/pypy/objspace/std/smalllongobject.py --- a/pypy/objspace/std/smalllongobject.py +++ b/pypy/objspace/std/smalllongobject.py @@ -21,9 +21,9 @@ from pypy.objspace.std.longtype import long_typedef as typedef _immutable_fields_ = ['longlong'] - def __init__(w_self, value): + def __init__(self, value): assert isinstance(value, r_longlong) - w_self.longlong = value + self.longlong = value @staticmethod def fromint(value): @@ -33,17 +33,17 @@ def frombigint(bigint): return W_SmallLongObject(bigint.tolonglong()) - def asbigint(w_self): - return rbigint.fromrarith_int(w_self.longlong) + def asbigint(self): + return rbigint.fromrarith_int(self.longlong) def longval(self): return self.longlong - def __repr__(w_self): - return '' % w_self.longlong + def __repr__(self): + return '' % self.longlong - def int_w(w_self, space): - a = w_self.longlong + def int_w(self, space): + a = self.longlong b = intmask(a) if b == a: return b @@ -51,8 +51,8 @@ raise OperationError(space.w_OverflowError, space.wrap( "long int too large to convert to int")) - def uint_w(w_self, space): - a = w_self.longlong + def uint_w(self, space): + a = self.longlong if a < 0: raise OperationError(space.w_ValueError, space.wrap( "cannot convert negative integer to unsigned int")) @@ -63,8 +63,8 @@ raise OperationError(space.w_OverflowError, space.wrap( "long int too large to convert to unsigned int")) - def bigint_w(w_self, space): - return w_self.asbigint() + def bigint_w(self, space): + return self.asbigint() def float_w(self, space): return float(self.longlong) From noreply at buildbot.pypy.org Mon Nov 11 23:27:22 2013 From: noreply at buildbot.pypy.org (pjenvey) Date: Mon, 11 Nov 2013 23:27:22 +0100 (CET) Subject: [pypy-commit] pypy remove-intlong-smm: most of smalllong's SMM removal Message-ID: <20131111222722.D274D1C08A1@cobra.cs.uni-duesseldorf.de> Author: Philip Jenvey Branch: remove-intlong-smm Changeset: r67963:3ef7d154db05 Date: 2013-11-11 14:22 -0800 http://bitbucket.org/pypy/pypy/changeset/3ef7d154db05/ Log: most of smalllong's SMM removal diff --git a/pypy/objspace/std/floatobject.py b/pypy/objspace/std/floatobject.py --- a/pypy/objspace/std/floatobject.py +++ b/pypy/objspace/std/floatobject.py @@ -62,6 +62,7 @@ # long-to-float delegation def delegate_Long2Float(space, w_longobj): + # XXX: tofloat is not abstract (SmallLongs) return W_FloatObject(w_longobj.tofloat(space)) diff --git a/pypy/objspace/std/intobject.py b/pypy/objspace/std/intobject.py --- a/pypy/objspace/std/intobject.py +++ b/pypy/objspace/std/intobject.py @@ -5,6 +5,8 @@ for overflows, something CPython does not do anymore. """ +import operator + from rpython.rlib import jit from rpython.rlib.rarithmetic import ( LONG_BIT, is_valid_int, ovfcheck, string_to_int, r_uint) @@ -52,7 +54,6 @@ def _make_descr_binop(opname): # XXX: func_renamer or func_with_new_name? - import operator from rpython.tool.sourcetools import func_renamer op = getattr(operator, opname) @@ -99,9 +100,11 @@ descr_mul, descr_rmul = _make_descr_binop('mul') def _make_descr_cmp(opname): - import operator op = getattr(operator, opname) def f(self, space, w_other): + # XXX: this doesn't belong here, regardless of how we originally set this up. blargh + #if isinstance(w_other, W_SmallLongObject): + # return space.newbool(op(space.int_w(self), w_other.longlong)) if not space.isinstance_w(w_other, space.w_int): return space.w_NotImplemented @@ -416,9 +419,9 @@ b = b.lshift(3).or_(rbigint.fromint(tag)) return space.newlong_from_rbigint(b) - def unwrap(self, space): + def int_w(self, space): return int(self.intval) - int_w = unwrap + unwrap = int_w def uint_w(self, space): intval = self.intval @@ -447,7 +450,6 @@ def descr_repr(self, space): res = str(self.intval) return space.wrap(res) - descr_str = func_with_new_name(descr_repr, 'descr_str') def _delegate_Int2Long(space, w_intobj): diff --git a/pypy/objspace/std/longobject.py b/pypy/objspace/std/longobject.py --- a/pypy/objspace/std/longobject.py +++ b/pypy/objspace/std/longobject.py @@ -4,7 +4,7 @@ from rpython.rlib.rbigint import rbigint from rpython.rlib.rstring import ParseStringError -from rpython.tool.sourcetools import func_with_new_name +from rpython.tool.sourcetools import func_renamer, func_with_new_name from pypy.interpreter import typedef from pypy.interpreter.error import OperationError, operationerrfmt @@ -17,6 +17,23 @@ from pypy.objspace.std.stdtypedef import StdTypeDef +def delegate_other(func): + @func_renamer(func.__name__) + def delegated(self, space, w_other): + if space.isinstance_w(w_other, space.w_int): + w_other = _delegate_Int2Long(space, w_other) + elif not space.isinstance_w(w_other, space.w_long): + return space.w_NotImplemented + # XXX: if a smalllong, delegate to Long? + assert isinstance(w_other, W_AbstractLongObject) + return func(self, space, w_other) + return delegated + +def _delegate_Int2Long(space, w_intobj): + """int-to-long delegation""" + return W_LongObject.fromint(space, w_intobj.int_w(space)) + + class W_AbstractLongObject(W_Object): __slots__ = () @@ -41,6 +58,114 @@ def int(self, space): raise NotImplementedError + def asbigint(self): + raise NotImplementedError + + def descr_long(self, space): + raise NotImplementedError + descr_index = func_with_new_name(descr_long, 'descr_index') + descr_trunc = func_with_new_name(descr_long, 'descr_trunc') + descr_pos = func_with_new_name(descr_long, 'descr_pos') + + # XXX: + def descr_float(self, space): + raise NotImplementedError + descr_neg = func_with_new_name(descr_long, 'descr_neg') + descr_pos = func_with_new_name(descr_long, 'descr_pos') + descr_abs = func_with_new_name(descr_long, 'descr_abs') + descr_nonzero = func_with_new_name(descr_long, 'descr_nonzero') + descr_invert = func_with_new_name(descr_long, 'descr_invert') + + def descr_lt(self, space, w_other): + raise NotImplementedError + descr_le = func_with_new_name(descr_lt, 'descr_le') + descr_eq = func_with_new_name(descr_lt, 'descr_eq') + descr_ne = func_with_new_name(descr_lt, 'descr_ne') + descr_gt = func_with_new_name(descr_lt, 'descr_gt') + descr_ge = func_with_new_name(descr_lt, 'descr_ge') + + descr_add = func_with_new_name(descr_lt, 'descr_add') + descr_radd = func_with_new_name(descr_lt, 'descr_radd') + descr_sub = func_with_new_name(descr_lt, 'descr_sub') + descr_rsub = func_with_new_name(descr_lt, 'descr_rsub') + descr_mul = func_with_new_name(descr_lt, 'descr_mul') + descr_rmul = func_with_new_name(descr_lt, 'descr_rmul') + + descr_and = func_with_new_name(descr_lt, 'descr_and') + descr_rand = func_with_new_name(descr_lt, 'descr_rand') + descr_or = func_with_new_name(descr_lt, 'descr_or') + descr_ror = func_with_new_name(descr_lt, 'descr_ror') + descr_xor = func_with_new_name(descr_lt, 'descr_xor') + descr_rxor = func_with_new_name(descr_lt, 'descr_rxor') + + descr_lshift = func_with_new_name(descr_lt, 'descr_lshift') + descr_rshift = func_with_new_name(descr_lt, 'descr_rshift') + + descr_floordiv = func_with_new_name(descr_lt, 'descr_floordiv') + descr_rfloordiv = func_with_new_name(descr_lt, 'descr_rfloordiv') + descr_div = func_with_new_name(descr_lt, 'descr_div') + descr_rdiv = func_with_new_name(descr_lt, 'descr_rdiv') + descr_mod = func_with_new_name(descr_lt, 'descr_mod') + descr_rmod = func_with_new_name(descr_lt, 'descr_rmod') + descr_divmod = func_with_new_name(descr_lt, 'descr_divmod') + descr_rdivmod = func_with_new_name(descr_lt, 'descr_rdivmod') + + def descr_pow(self, space, w_exponent, w_modulus=None): + raise NotImplementedError + descr_rpow = func_with_new_name(descr_pow, 'descr_rpow') + + def descr_format(self, space, w_format_spec): + return newformat.run_formatter(space, w_format_spec, + "format_int_or_long", self, + newformat.LONG_KIND) + def descr_repr(self, space): + return space.wrap(self.asbigint().repr()) + + def descr_str(self, space): + return space.wrap(self.asbigint().str()) + + def descr_hash(self, space): + return space.wrap(self.asbigint().hash()) + + def descr_oct(self, space): + return space.wrap(self.asbigint().oct()) + + def descr_hex(self, space): + return space.wrap(self.asbigint().hex()) + + def descr_getnewargs(self, space): + return space.newtuple([W_LongObject(self.asbigint())]) + + def descr_conjugate(self, space): + return space.long(self) + + def descr_bit_length(self, space): + bigint = space.bigint_w(self) + try: + return space.wrap(bigint.bit_length()) + except OverflowError: + raise OperationError(space.w_OverflowError, + space.wrap("too many digits in integer")) + + # XXX: need rtruediv + @delegate_other + def descr_truediv(self, space, w_other): + try: + #f = self.num.truediv(w_other.num) + f = self.asbigint().truediv(w_other.asbigint()) + except ZeroDivisionError: + raise operationerrfmt(space.w_ZeroDivisionError, + "long division or modulo by zero") + except OverflowError: + raise operationerrfmt(space.w_OverflowError, + "long/long too large for a float") + return space.newfloat(f) + + @delegate_other + def descr_coerce(self, space, w_other): + # XXX: consider stian's branch where he optimizes long + ints + return space.newtuple([self, w_other]) + class W_LongObject(W_AbstractLongObject): """This is a wrapper of rbigint.""" @@ -111,12 +236,13 @@ except OverflowError: return self.descr_long(space) + def asbigint(self): + return self.num + def __repr__(self): return '' % self.num.tolong() - def descr_conjugate(self, space): - return space.long(self) - + # XXX: make these indirect def descr_get_numerator(self, space): return space.long(self) @@ -129,14 +255,6 @@ def descr_get_imag(self, space): return space.newlong(0) - def descr_get_bit_length(self, space): - bigint = space.bigint_w(self) - try: - return space.wrap(bigint.bit_length()) - except OverflowError: - raise OperationError(space.w_OverflowError, - space.wrap("too many digits in integer")) - def descr_long(self, space): # long__Long is supposed to do nothing, unless it has a derived # long object, where it should return an exact one. @@ -151,36 +269,23 @@ def descr_float(self, space): return space.newfloat(self.tofloat(space)) - def descr_repr(self, space): - return space.wrap(self.num.repr()) + def _make_descr_cmp(opname): + #from pypy.objspace.std.smalllongobject import W_SmallLongObject + op = getattr(rbigint, opname) + @delegate_other + def descr_impl(self, space, w_other): + ## XXX: these only need explicit SmallLong support whereas + ## everything else would delegate2Long. blah blah + #if isinstance(w_other, W_SmallLongObject): + # result = op(self.num, w_other.asbigint()) + #else: + # result = op(self.num, w_other.num) + #return space.newbool(result) - def descr_str(self, space): - return space.wrap(self.num.str()) - - def descr_format(self, space, w_format_spec): - return newformat.run_formatter(space, w_format_spec, - "format_int_or_long", self, - newformat.LONG_KIND) - - def descr_hash(self, space): - return space.wrap(self.num.hash()) - - def descr_coerce(self, space, w_other): - # XXX: consider stian's branch where he optimizes long + ints - if space.isinstance_w(w_other, space.w_int): - w_other = _delegate_Int2Long(space, w_other) - elif not space.isinstance_w(w_other, space.w_long): - return space.w_NotImplemented - return space.newtuple([self, w_other]) - - def _make_descr_cmp(opname): - op = getattr(rbigint, opname) - def descr_impl(self, space, w_other): - if space.isinstance_w(w_other, space.w_int): - w_other = _delegate_Int2Long(space, w_other) - elif not space.isinstance_w(w_other, space.w_long): - return space.w_NotImplemented - return space.newbool(op(self.num, w_other.num)) + # XXX: if we use self.asbigint then can this live on + # AbstractLong? eek not really, a '_cmp' (_lt) could live on + # it that just did this (without the checks..) + return space.newbool(op(self.num, w_other.asbigint())) return func_with_new_name(descr_impl, "descr_" + opname) descr_lt = _make_descr_cmp('lt') @@ -196,20 +301,14 @@ op = getattr(rbigint, methname) @func_renamer('descr_' + opname) + @delegate_other def descr_binop(self, space, w_other): - if space.isinstance_w(w_other, space.w_int): - w_other = _delegate_Int2Long(space, w_other) - elif not space.isinstance_w(w_other, space.w_long): - return space.w_NotImplemented - return W_LongObject(op(self.num, w_other.num)) + return W_LongObject(op(self.num, w_other.asbigint())) @func_renamer('descr_r' + opname) + @delegate_other def descr_rbinop(self, space, w_other): - if space.isinstance_w(w_other, space.w_int): - w_other = _delegate_Int2Long(space, w_other) - elif not space.isinstance_w(w_other, space.w_long): - return space.w_NotImplemented - return W_LongObject(op(w_other.num, self.num)) + return W_LongObject(op(w_other.asbigint(), self.num)) return descr_binop, descr_rbinop @@ -232,136 +331,103 @@ descr_abs = _make_descr_unaryop('abs') descr_invert = _make_descr_unaryop('invert') - def descr_oct(self, space): - return space.wrap(self.num.oct()) - - def descr_hex(self, space): - return space.wrap(self.num.hex()) - def descr_nonzero(self, space): return space.newbool(self.num.tobool()) + @delegate_other def descr_lshift(self, space, w_other): - if space.isinstance_w(w_other, space.w_int): - w_other = _delegate_Int2Long(space, w_other) - elif not space.isinstance_w(w_other, space.w_long): - return space.w_NotImplemented - # XXX need to replicate some of the logic, to get the errors right - if w_other.num.sign < 0: + if w_other.asbigint().sign < 0: raise operationerrfmt(space.w_ValueError, "negative shift count") try: - shift = w_other.num.toint() + shift = w_other.asbigint().toint() except OverflowError: # b too big raise operationerrfmt(space.w_OverflowError, "shift count too large") return W_LongObject(self.num.lshift(shift)) + @delegate_other def descr_rshift(self, space, w_other): - if space.isinstance_w(w_other, space.w_int): - w_other = _delegate_Int2Long(space, w_other) - elif not space.isinstance_w(w_other, space.w_long): - return space.w_NotImplemented - # XXX need to replicate some of the logic, to get the errors right - if w_other.num.sign < 0: + if w_other.asbigint().sign < 0: raise operationerrfmt(space.w_ValueError, "negative shift count") try: - shift = w_other.num.toint() + shift = w_other.asbigint().toint() except OverflowError: # b too big # XXX maybe just return 0L instead? raise operationerrfmt(space.w_OverflowError, "shift count too large") return newlong(space, self.num.rshift(shift)) - # XXX: need rtruediv etc - def descr_truediv(self, space, w_other): - if space.isinstance_w(w_other, space.w_int): - w_other = _delegate_Int2Long(space, w_other) - elif not space.isinstance_w(w_other, space.w_long): - return space.w_NotImplemented - + @delegate_other + def descr_floordiv(self, space, w_other): try: - f = self.num.truediv(w_other.num) + z = self.num.floordiv(w_other.asbigint()) except ZeroDivisionError: raise operationerrfmt(space.w_ZeroDivisionError, "long division or modulo by zero") - except OverflowError: - raise operationerrfmt(space.w_OverflowError, - "long/long too large for a float") - return space.newfloat(f) + return newlong(space, z) + descr_div = func_with_new_name(descr_floordiv, 'descr_div') - def descr_floordiv(self, space, w_other): - if space.isinstance_w(w_other, space.w_int): - w_other = _delegate_Int2Long(space, w_other) - elif not space.isinstance_w(w_other, space.w_long): - return space.w_NotImplemented - + @delegate_other + def descr_mod(self, space, w_other): try: - z = self.num.floordiv(w_other.num) + z = self.num.mod(w_other.asbigint()) except ZeroDivisionError: raise operationerrfmt(space.w_ZeroDivisionError, "long division or modulo by zero") return newlong(space, z) - def descr_div(self, space, w_other): - if space.isinstance_w(w_other, space.w_int): - w_other = _delegate_Int2Long(space, w_other) - elif not space.isinstance_w(w_other, space.w_long): - return space.w_NotImplemented - - return self.floordiv(space, w_other) - - def descr_mod(self, space, w_other): - if space.isinstance_w(w_other, space.w_int): - w_other = _delegate_Int2Long(space, w_other) - elif not space.isinstance_w(w_other, space.w_long): - return space.w_NotImplemented - + @delegate_other + def descr_divmod(self, space, w_other): try: - z = self.num.mod(w_other.num) - except ZeroDivisionError: - raise operationerrfmt(space.w_ZeroDivisionError, - "long division or modulo by zero") - return newlong(space, z) - - def descr_divmod(self, space, w_other): - if space.isinstance_w(w_other, space.w_int): - w_other = _delegate_Int2Long(space, w_other) - elif not space.isinstance_w(w_other, space.w_long): - return space.w_NotImplemented - - try: - div, mod = self.num.divmod(w_other.num) + div, mod = self.num.divmod(w_other.asbigint()) except ZeroDivisionError: raise operationerrfmt(space.w_ZeroDivisionError, "long division or modulo by zero") return space.newtuple([newlong(space, div), newlong(space, mod)]) + #@delegate_other # XXX: @unwrap_spec(w_modulus=WrappedDefault(None)) def descr_pow(self, space, w_exponent, w_modulus=None): if space.isinstance_w(w_exponent, space.w_int): w_exponent = _delegate_Int2Long(space, w_exponent) elif not space.isinstance_w(w_exponent, space.w_long): return space.w_NotImplemented - if space.isinstance_w(w_modulus, space.w_int): + assert isinstance(w_exponent, W_AbstractLongObject) + + #if space.is_none(w_modulus): + # from pypy.objspace.std.floatobject import delegate_Long2Float + # self = delegate_Long2Float(space, self) + # w_exponent = delegate_Long2Float(space, w_exponent) + # return space.pow(self, w_exponent, w_modulus) + #elif space.isinstance_w(w_modulus, space.w_int): + if space.is_none(w_modulus): + # XXX need to replicate some of the logic, to get the errors right + if w_exponent.asbigint().sign < 0: + from pypy.objspace.std.floatobject import delegate_Long2Float + w_exponent = delegate_Long2Float(space, w_exponent) + return space.pow(self.descr_float(space), w_exponent, space.w_None if w_modulus is None else w_modulus) + return W_LongObject(self.num.pow(w_exponent.asbigint(), None)) + elif space.isinstance_w(w_modulus, space.w_int): w_modulus = _delegate_Int2Long(space, w_modulus) - elif space.is_none(w_modulus): - # XXX need to replicate some of the logic, to get the errors right - if w_exponent.num.sign < 0: - return space.pow(self.descr_float(space), w_exponent, w_modulus) - return W_LongObject(self.num.pow(w_exponent.num, None)) + #elif space.is_none(w_modulus): + # # XXX need to replicate some of the logic, to get the errors right + # if w_exponent.num.sign < 0: + # return space.pow(self.descr_float(space), w_exponent, w_modulus) + # return W_LongObject(self.num.pow(w_exponent.num, None)) elif not space.isinstance_w(w_modulus, space.w_long): return space.w_NotImplemented + assert isinstance(w_modulus, W_AbstractLongObject) - # XXX need to replicate some of the logic, to get the errors right - if w_exponent.num.sign < 0: + if w_exponent.asbigint().sign < 0: raise OperationError( space.w_TypeError, space.wrap( "pow() 2nd argument " "cannot be negative when 3rd argument specified")) try: - return W_LongObject(self.num.pow(w_exponent.num, w_modulus.num)) + return W_LongObject(self.num.pow(w_exponent.asbigint(), + w_modulus.asbigint())) except ValueError: raise OperationError(space.w_ValueError, space.wrap("pow 3rd argument cannot be 0")) @@ -398,11 +464,6 @@ return W_LongObject(bigint) -def _delegate_Int2Long(space, w_intobj): - """int-to-long delegation""" - return W_LongObject.fromint(space, w_intobj.int_w(space)) - - # register implementations of ops that recover int op overflows def recover_with_smalllong(space): # True if there is a chance that a SmallLong would fit when an Int does not @@ -420,7 +481,8 @@ return %(opname)s_ovr(space, w_int1, w_int2) w_long1 = _delegate_Int2Long(space, w_int1) w_long2 = _delegate_Int2Long(space, w_int2) - return %(opname)s__Long_Long(space, w_long1, w_long2) + #return %(opname)s__Long_Long(space, w_long1, w_long2) + return w_long1.descr_%(opname)s(space, w_long2) """ % {'opname': opname}, '', 'exec') getattr(model.MM, opname).register(globals()['%s_ovr__Int_Int' % opname], @@ -434,7 +496,8 @@ from pypy.objspace.std.smalllongobject import %(opname)s_ovr return %(opname)s_ovr(space, w_int1) w_long1 = _delegate_Int2Long(space, w_int1) - return %(opname)s__Long(space, w_long1) + #return %(opname)s__Long(space, w_long1) + return w_long1.descr_%(opname)s(space) """ % {'opname': opname} getattr(model.MM, opname).register(globals()['%s_ovr__Int' % opname], @@ -447,12 +510,14 @@ return pow_ovr(space, w_int1, w_int2) w_long1 = _delegate_Int2Long(space, w_int1) w_long2 = _delegate_Int2Long(space, w_int2) - return pow__Long_Long_None(space, w_long1, w_long2, w_none3) + #return pow__Long_Long_None(space, w_long1, w_long2, w_none3) + return w_long1.descr_pow(space, w_long2, w_none3) def pow_ovr__Int_Int_Long(space, w_int1, w_int2, w_long3): w_long1 = _delegate_Int2Long(space, w_int1) w_long2 = _delegate_Int2Long(space, w_int2) - return pow__Long_Long_Long(space, w_long1, w_long2, w_long3) + #return pow__Long_Long_Long(space, w_long1, w_long2, w_long3) + return w_long1.descr_pow(space, w_long2, w_long3) model.MM.pow.register(pow_ovr__Int_Int_None, W_IntObject, W_IntObject, W_NoneObject, order=1) @@ -515,7 +580,7 @@ try: bigint = rbigint.fromstr(s, base) except ParseStringError as e: - raise operationerrfmt(space.w_ValueError, e.msg) + raise OperationError(space.w_ValueError, space.wrap(e.msg)) return newbigint(space, w_longtype, bigint) string_to_w_long._dont_inline_ = True @@ -540,7 +605,7 @@ return w_obj -W_LongObject.typedef = StdTypeDef("long", +W_AbstractLongObject.typedef = StdTypeDef("long", __doc__ = """long(x[, base]) -> integer Convert a string or number to a long integer, if possible. A floating @@ -549,66 +614,70 @@ string, use the optional base. It is an error to supply a base when converting a non-string.""", __new__ = interp2app(descr__new__), - conjugate = interp2app(W_LongObject.descr_conjugate), + conjugate = interp2app(W_AbstractLongObject.descr_conjugate), + # XXX: need indirect for these numerator = typedef.GetSetProperty(W_LongObject.descr_get_numerator), denominator = typedef.GetSetProperty(W_LongObject.descr_get_denominator), real = typedef.GetSetProperty(W_LongObject.descr_get_real), imag = typedef.GetSetProperty(W_LongObject.descr_get_imag), - bit_length = interp2app(W_LongObject.descr_get_bit_length), + bit_length = interp2app(W_AbstractLongObject.descr_bit_length), # XXX: likely need indirect everything for SmallLong __int__ = interpindirect2app(W_AbstractLongObject.int), - __long__ = interp2app(W_LongObject.descr_long), - __trunc__ = interp2app(W_LongObject.descr_trunc), - __index__ = interp2app(W_LongObject.descr_index), - __float__ = interp2app(W_LongObject.descr_float), - __repr__ = interp2app(W_LongObject.descr_repr), - __str__ = interp2app(W_LongObject.descr_str), - __format__ = interp2app(W_LongObject.descr_format), + __long__ = interpindirect2app(W_AbstractLongObject.descr_long), + __index__ = interpindirect2app(W_AbstractLongObject.descr_index), + __trunc__ = interpindirect2app(W_AbstractLongObject.descr_trunc), + __float__ = interpindirect2app(W_AbstractLongObject.descr_float), - __hash__ = interp2app(W_LongObject.descr_hash), - __coerce__ = interp2app(W_LongObject.descr_coerce), + __repr__ = interp2app(W_AbstractLongObject.descr_repr), + __str__ = interp2app(W_AbstractLongObject.descr_str), + __format__ = interp2app(W_AbstractLongObject.descr_format), - __lt__ = interp2app(W_LongObject.descr_lt), - __le__ = interp2app(W_LongObject.descr_le), - __eq__ = interp2app(W_LongObject.descr_eq), - __ne__ = interp2app(W_LongObject.descr_ne), - __gt__ = interp2app(W_LongObject.descr_gt), - __ge__ = interp2app(W_LongObject.descr_ge), + __hash__ = interp2app(W_AbstractLongObject.descr_hash), + __coerce__ = interp2app(W_AbstractLongObject.descr_coerce), - __add__ = interp2app(W_LongObject.descr_add), - __radd__ = interp2app(W_LongObject.descr_radd), - __sub__ = interp2app(W_LongObject.descr_sub), - __rsub__ = interp2app(W_LongObject.descr_rsub), - __mul__ = interp2app(W_LongObject.descr_mul), - __rmul__ = interp2app(W_LongObject.descr_rmul), + __lt__ = interpindirect2app(W_AbstractLongObject.descr_lt), + __le__ = interpindirect2app(W_AbstractLongObject.descr_le), + __eq__ = interpindirect2app(W_AbstractLongObject.descr_eq), + __ne__ = interpindirect2app(W_AbstractLongObject.descr_ne), + __gt__ = interpindirect2app(W_AbstractLongObject.descr_gt), + __ge__ = interpindirect2app(W_AbstractLongObject.descr_ge), - __and__ = interp2app(W_LongObject.descr_and), - __rand__ = interp2app(W_LongObject.descr_rand), - __or__ = interp2app(W_LongObject.descr_or), - __ror__ = interp2app(W_LongObject.descr_ror), - __xor__ = interp2app(W_LongObject.descr_xor), - __rxor__ = interp2app(W_LongObject.descr_rxor), + __add__ = interpindirect2app(W_AbstractLongObject.descr_add), + __radd__ = interpindirect2app(W_AbstractLongObject.descr_radd), + __sub__ = interpindirect2app(W_AbstractLongObject.descr_sub), + __rsub__ = interpindirect2app(W_AbstractLongObject.descr_rsub), + __mul__ = interpindirect2app(W_AbstractLongObject.descr_mul), + __rmul__ = interpindirect2app(W_AbstractLongObject.descr_rmul), - __neg__ = interp2app(W_LongObject.descr_neg), - __pos__ = interp2app(W_LongObject.descr_pos), - __abs__ = interp2app(W_LongObject.descr_abs), - __nonzero__ = interp2app(W_LongObject.descr_nonzero), - __invert__ = interp2app(W_LongObject.descr_invert), - __oct__ = interp2app(W_LongObject.descr_oct), - __hex__ = interp2app(W_LongObject.descr_hex), + __and__ = interpindirect2app(W_AbstractLongObject.descr_and), + __rand__ = interpindirect2app(W_AbstractLongObject.descr_rand), + __or__ = interpindirect2app(W_AbstractLongObject.descr_or), + __ror__ = interpindirect2app(W_AbstractLongObject.descr_ror), + __xor__ = interpindirect2app(W_AbstractLongObject.descr_xor), + __rxor__ = interpindirect2app(W_AbstractLongObject.descr_rxor), - __lshift__ = interp2app(W_LongObject.descr_lshift), - __rshift__ = interp2app(W_LongObject.descr_rshift), + __neg__ = interpindirect2app(W_AbstractLongObject.descr_neg), + __pos__ = interpindirect2app(W_AbstractLongObject.descr_pos), + __abs__ = interpindirect2app(W_AbstractLongObject.descr_abs), + __nonzero__ = interpindirect2app(W_AbstractLongObject.descr_nonzero), + __invert__ = interpindirect2app(W_AbstractLongObject.descr_invert), - __truediv__ = interp2app(W_LongObject.descr_truediv), - __floordiv__ = interp2app(W_LongObject.descr_floordiv), - __div__ = interp2app(W_LongObject.descr_div), - __mod__ = interp2app(W_LongObject.descr_mod), - __divmod__ = interp2app(W_LongObject.descr_divmod), + __oct__ = interp2app(W_AbstractLongObject.descr_oct), + __hex__ = interp2app(W_AbstractLongObject.descr_hex), - __pow__ = interp2app(W_LongObject.descr_pow), - __rpow__ = interp2app(W_LongObject.descr_rpow), + __lshift__ = interpindirect2app(W_AbstractLongObject.descr_lshift), + __rshift__ = interpindirect2app(W_AbstractLongObject.descr_rshift), - __getnewargs__ = interp2app(W_LongObject.descr_getnewargs), + # XXX: all these need r sides + __truediv__ = interp2app(W_AbstractLongObject.descr_truediv), + __floordiv__ = interpindirect2app(W_AbstractLongObject.descr_floordiv), + __div__ = interpindirect2app(W_AbstractLongObject.descr_div), + __mod__ = interpindirect2app(W_AbstractLongObject.descr_mod), + __divmod__ = interpindirect2app(W_AbstractLongObject.descr_divmod), + + __pow__ = interpindirect2app(W_AbstractLongObject.descr_pow), + __rpow__ = interpindirect2app(W_AbstractLongObject.descr_rpow), + + __getnewargs__ = interp2app(W_AbstractLongObject.descr_getnewargs), ) diff --git a/pypy/objspace/std/model.py b/pypy/objspace/std/model.py --- a/pypy/objspace/std/model.py +++ b/pypy/objspace/std/model.py @@ -143,7 +143,7 @@ ] self.typeorder[intobject.W_IntObject] += [ (floatobject.W_FloatObject, floatobject.delegate_Int2Float), - (longobject.W_LongObject, longobject.delegate_Int2Long), +# (longobject.W_LongObject, longobject.delegate_Int2Long), (complexobject.W_ComplexObject, complexobject.delegate_Int2Complex), ] if False and config.objspace.std.withsmalllong: diff --git a/pypy/objspace/std/smalllongobject.py b/pypy/objspace/std/smalllongobject.py --- a/pypy/objspace/std/smalllongobject.py +++ b/pypy/objspace/std/smalllongobject.py @@ -3,22 +3,23 @@ Useful for 32-bit applications manipulating values a bit larger than fits in an 'int'. """ -from pypy.objspace.std.model import registerimplementation, W_Object -from pypy.objspace.std.register_all import register_all +import operator + +from rpython.rlib.rarithmetic import LONGLONG_BIT, intmask, r_longlong, r_uint +from rpython.rlib.rbigint import rbigint +from rpython.tool.sourcetools import func_with_new_name + +from pypy.interpreter.error import OperationError +from pypy.interpreter.gateway import WrappedDefault, unwrap_spec from pypy.objspace.std.multimethod import FailedToImplementArgs -from rpython.rlib.rarithmetic import r_longlong, r_int, r_uint -from rpython.rlib.rarithmetic import intmask, LONGLONG_BIT -from rpython.rlib.rbigint import rbigint from pypy.objspace.std.longobject import W_AbstractLongObject, W_LongObject -from pypy.objspace.std.intobject import W_IntObject -from pypy.objspace.std.noneobject import W_NoneObject -from pypy.interpreter.error import OperationError +from pypy.objspace.std.intobject import _delegate_Int2Long LONGLONG_MIN = r_longlong((-1) << (LONGLONG_BIT-1)) class W_SmallLongObject(W_AbstractLongObject): - from pypy.objspace.std.longtype import long_typedef as typedef + _immutable_fields_ = ['longlong'] def __init__(w_self, value): @@ -77,8 +78,331 @@ else: return self + def descr_long(self, space): + # XXX: do subclasses never apply here? + return self + descr_index = func_with_new_name(descr_long, 'descr_index') + descr_trunc = func_with_new_name(descr_long, 'descr_trunc') + descr_pos = func_with_new_name(descr_long, 'descr_pos') + + def descr_index(self, space): + return self + + def descr_float(self, space): + return space.newfloat(float(self.longlong)) + + def _make_descr_cmp(opname): + op = getattr(operator, opname) + def descr_impl(self, space, w_other): + if space.isinstance_w(w_other, space.w_int): + result = op(self.longlong, w_other.int_w(space)) + elif not space.isinstance_w(w_other, space.w_long): + return space.w_NotImplemented + elif isinstance(w_other, W_SmallLongObject): + result = op(self.longlong, w_other.longlong) + else: + result = getattr(self.asbigint(), opname)(w_other.num) + return space.newbool(result) + return func_with_new_name(descr_impl, "descr_" + opname) + + descr_lt = _make_descr_cmp('lt') + descr_le = _make_descr_cmp('le') + descr_eq = _make_descr_cmp('eq') + descr_ne = _make_descr_cmp('ne') + descr_gt = _make_descr_cmp('gt') + descr_ge = _make_descr_cmp('ge') + + def _make_descr_binop(func): + # XXX: so if w_other is Long, what do we do? sigh + # how to handle delegation with descr_add on longobject? + opname = func.__name__[1:] + methname = opname + '_' if opname in ('and', 'or') else opname + + def descr_impl(self, space, w_other): + if space.isinstance_w(w_other, space.w_int): + w_other = delegate_Int2SmallLong(space, w_other) + elif not space.isinstance_w(w_other, space.w_long): + return space.w_NotImplemented + elif not isinstance(w_other, W_SmallLongObject): + self = delegate_SmallLong2Long(space, self) + return getattr(space, methname)(self, w_other) + + try: + return func(self, space, w_other) + except OverflowError: + self = delegate_SmallLong2Long(space, self) + w_other = delegate_SmallLong2Long(space, w_other) + return getattr(space, methname)(self, w_other) + + def descr_rimpl(self, space, w_other): + if space.isinstance_w(w_other, space.w_int): + w_other = delegate_Int2SmallLong(space, w_other) + elif not space.isinstance_w(w_other, space.w_long): + return space.w_NotImplemented + elif not isinstance(w_other, W_SmallLongObject): + self = delegate_SmallLong2Long(space, self) + return getattr(space, methname)(w_other, self) + + try: + return func(w_other, space, self) + except OverflowError: + self = delegate_SmallLong2Long(space, self) + w_other = delegate_SmallLong2Long(space, w_other) + return getattr(space, methname)(w_other, self) + + return descr_impl, descr_rimpl + + def _add(self, space, w_other): + x = self.longlong + y = w_other.longlong + z = x + y + if ((z^x)&(z^y)) < 0: + raise OverflowError + return W_SmallLongObject(z) + descr_add, descr_radd = _make_descr_binop(_add) + + def _sub(self, space, w_other): + x = self.longlong + y = w_other.longlong + z = x - y + if ((z^x)&(z^~y)) < 0: + raise OverflowError + return W_SmallLongObject(z) + descr_sub, descr_rsub = _make_descr_binop(_sub) + + def _mul(self, space, w_other): + x = self.longlong + y = w_other.longlong + z = llong_mul_ovf(x, y) + return W_SmallLongObject(z) + descr_mul, descr_rmul = _make_descr_binop(_mul) + + def _floordiv(self, space, w_other): + x = self.longlong + y = w_other.longlong + try: + if y == -1 and x == LONGLONG_MIN: + raise OverflowError + z = x // y + except ZeroDivisionError: + raise OperationError(space.w_ZeroDivisionError, + space.wrap("integer division by zero")) + #except OverflowError: + # raise FailedToImplementArgs(space.w_OverflowError, + # space.wrap("integer division")) + return W_SmallLongObject(z) + descr_floordiv, descr_rfloordiv = _make_descr_binop(_floordiv) + + _div = func_with_new_name(_floordiv, '_div') + descr_div, descr_rdiv = _make_descr_binop(_div) + + def _mod(self, space, w_other): + x = self.longlong + y = w_other.longlong + try: + if y == -1 and x == LONGLONG_MIN: + raise OverflowError + z = x % y + except ZeroDivisionError: + raise OperationError(space.w_ZeroDivisionError, + space.wrap("integer modulo by zero")) + #except OverflowError: + # raise FailedToImplementArgs(space.w_OverflowError, + # space.wrap("integer modulo")) + return W_SmallLongObject(z) + descr_mod, descr_rmod = _make_descr_binop(_mod) + + def _divmod(self, space, w_other): + x = self.longlong + y = w_other.longlong + try: + if y == -1 and x == LONGLONG_MIN: + raise OverflowError + z = x // y + except ZeroDivisionError: + raise OperationError(space.w_ZeroDivisionError, + space.wrap("integer divmod by zero")) + #except OverflowError: + # raise FailedToImplementArgs(space.w_OverflowError, + # space.wrap("integer modulo")) + # no overflow possible + m = x % y + return space.newtuple([W_SmallLongObject(z), W_SmallLongObject(m)]) + descr_divmod, descr_rdivmod = _make_descr_binop(_divmod) + + # XXX: + @unwrap_spec(w_modulus=WrappedDefault(None)) + #def descr_pow__SmallLong_Int_SmallLong(self, space, w_exponent, + def descr_pow(self, space, w_exponent, w_modulus=None): + if space.isinstance_w(w_exponent, space.w_long): + self = delegate_SmallLong2Long(space, self) + return space.pow(self, w_exponent, w_modulus) + elif not space.isinstance_w(w_exponent, space.w_int): + return space.w_NotImplemented -registerimplementation(W_SmallLongObject) + # XXX: this expects w_exponent as an int o_O + """ + if space.isinstance_w(w_exponent, space.w_int): + w_exponent = delegate_Int2SmallLong(space, w_exponent) + elif not space.isinstance_w(w_exponent, space.w_long): + return space.w_NotImplemented + elif not isinstance(w_exponent, W_SmallLongObject): + self = delegate_SmallLong2Long(space, self) + return space.pow(self, w_exponent, w_modulus) + """ + + if space.is_none(w_modulus): + #return _impl_pow(space, self.longlong, w_exponent) + try: + return _impl_pow(space, self.longlong, w_exponent) + except ValueError: + self = delegate_SmallLong2Float(space, self) + except OverflowError: + self = delegate_SmallLong2Long(space, self) + return space.pow(self, w_exponent, w_modulus) + elif space.isinstance_w(w_modulus, space.w_int): + w_modulus = delegate_Int2SmallLong(space, w_modulus) + elif not space.isinstance_w(w_modulus, space.w_long): + return space.w_NotImplemented + elif not isinstance(w_modulus, W_SmallLongObject): + self = delegate_SmallLong2Long(space, self) + #return space.pow(self, w_modulus, w_modulus) + return space.pow(self, w_exponent, w_modulus) + + z = w_modulus.longlong + if z == 0: + raise OperationError(space.w_ValueError, + space.wrap("pow() 3rd argument cannot be 0")) + try: + return _impl_pow(space, self.longlong, w_exponent, z) + except ValueError: + self = delegate_SmallLong2Float(space, self) + except OverflowError: + self = delegate_SmallLong2Long(space, self) + return space.pow(self, w_exponent, w_modulus) + + # XXX: + @unwrap_spec(w_modulus=WrappedDefault(None)) + def descr_rpow(self, space, w_exponent, w_modulus=None): + # XXX: blargh + if space.isinstance_w(w_exponent, space.w_int): + w_exponent = _delegate_Int2Long(space, w_exponent) + elif not space.isinstance_w(w_exponent, space.w_long): + return space.w_NotImplemented + return space.pow(w_exponent, self, w_modulus) + + #def descr_lshift__SmallLong_Int(space, w_small1, w_int2): + def descr_lshift(self, space, w_other): + if space.isinstance_w(w_other, space.w_long): + self = delegate_SmallLong2Long(space, self) + w_other = delegate_SmallLong2Long(space, w_other) + return space.lshift(self, w_other) + elif not space.isinstance_w(w_other, space.w_int): + return space.w_NotImplemented + + a = self.longlong + b = w_other.intval + if r_uint(b) < LONGLONG_BIT: # 0 <= b < LONGLONG_BIT + try: + c = a << b + if a != (c >> b): + raise OverflowError + except OverflowError: + #raise FailedToImplementArgs(space.w_OverflowError, + # space.wrap("integer left shift")) + self = delegate_SmallLong2Long(space, self) + w_other = _delegate_Int2Long(space, w_other) + return space.lshift(self, w_other) + return W_SmallLongObject(c) + if b < 0: + raise OperationError(space.w_ValueError, + space.wrap("negative shift count")) + else: #b >= LONGLONG_BIT + if a == 0: + return self + #raise FailedToImplementArgs(space.w_OverflowError, + # space.wrap("integer left shift")) + self = delegate_SmallLong2Long(space, self) + w_other = _delegate_Int2Long(space, w_other) + return space.lshift(self, w_other) + + def descr_rshift(self, space, w_other): + if space.isinstance_w(w_other, space.w_long): + self = delegate_SmallLong2Long(space, self) + w_other = delegate_SmallLong2Long(space, w_other) + return space.rshift(self, w_other) + elif not space.isinstance_w(w_other, space.w_int): + return space.w_NotImplemented + + a = self.longlong + b = w_other.intval + if r_uint(b) >= LONGLONG_BIT: # not (0 <= b < LONGLONG_BIT) + if b < 0: + raise OperationError(space.w_ValueError, + space.wrap("negative shift count")) + else: # b >= LONGLONG_BIT + if a == 0: + return self + if a < 0: + a = -1 + else: + a = 0 + else: + a = a >> b + return W_SmallLongObject(a) + + def _and(self, space, w_other): + a = self.longlong + b = w_other.longlong + res = a & b + return W_SmallLongObject(res) + descr_and, descr_rand = _make_descr_binop(_and) + + def _xor(self, space, w_other): + a = self.longlong + b = w_other.longlong + res = a ^ b + return W_SmallLongObject(res) + descr_xor, descr_rxor = _make_descr_binop(_xor) + + def _or(self, space, w_other): + a = self.longlong + b = w_other.longlong + res = a | b + return W_SmallLongObject(res) + descr_or, descr_ror = _make_descr_binop(_or) + + def descr_neg(self, space): + a = self.longlong + try: + if a == LONGLONG_MIN: + raise OverflowError + x = -a + except OverflowError: + return space.neg(delegate_SmallLong2Long(self)) + #raise FailedToImplementArgs(space.w_OverflowError, + # space.wrap("integer negation")) + return W_SmallLongObject(x) + #get_negint = neg__SmallLong + + #def descr_pos(self, space): + # return self + + def descr_abs(self, space): + if self.longlong >= 0: + return self + else: + #return get_negint(space, self) + return self.descr_neg(space) + + def descr_nonzero(self, space): + return space.newbool(bool(self.longlong)) + + def descr_invert(self, space): + x = self.longlong + a = ~x + return W_SmallLongObject(a) + # ____________________________________________________________ @@ -112,7 +436,8 @@ return W_SmallLongObject(r_longlong(space.is_true(w_bool))) def delegate_Int2SmallLong(space, w_int): - return W_SmallLongObject(r_longlong(w_int.intval)) + #return W_SmallLongObject(r_longlong(w_int.intval)) + return W_SmallLongObject(r_longlong(w_int.int_w(space))) def delegate_SmallLong2Long(space, w_small): return W_LongObject(w_small.asbigint()) @@ -123,194 +448,32 @@ def delegate_SmallLong2Complex(space, w_small): return space.newcomplex(float(w_small.longlong), 0.0) - -def long__SmallLong(space, w_value): - return w_value - -def index__SmallLong(space, w_value): - return w_value - -def float__SmallLong(space, w_value): - return space.newfloat(float(w_value.longlong)) - -def lt__SmallLong_SmallLong(space, w_small1, w_small2): - return space.newbool(w_small1.longlong < w_small2.longlong) -def le__SmallLong_SmallLong(space, w_small1, w_small2): - return space.newbool(w_small1.longlong <= w_small2.longlong) -def eq__SmallLong_SmallLong(space, w_small1, w_small2): - return space.newbool(w_small1.longlong == w_small2.longlong) -def ne__SmallLong_SmallLong(space, w_small1, w_small2): - return space.newbool(w_small1.longlong != w_small2.longlong) -def gt__SmallLong_SmallLong(space, w_small1, w_small2): - return space.newbool(w_small1.longlong > w_small2.longlong) -def ge__SmallLong_SmallLong(space, w_small1, w_small2): - return space.newbool(w_small1.longlong >= w_small2.longlong) - -def lt__SmallLong_Long(space, w_small1, w_long2): - return space.newbool(w_small1.asbigint().lt(w_long2.num)) -def le__SmallLong_Long(space, w_small1, w_long2): - return space.newbool(w_small1.asbigint().le(w_long2.num)) -def eq__SmallLong_Long(space, w_small1, w_long2): - return space.newbool(w_small1.asbigint().eq(w_long2.num)) -def ne__SmallLong_Long(space, w_small1, w_long2): - return space.newbool(w_small1.asbigint().ne(w_long2.num)) -def gt__SmallLong_Long(space, w_small1, w_long2): - return space.newbool(w_small1.asbigint().gt(w_long2.num)) -def ge__SmallLong_Long(space, w_small1, w_long2): - return space.newbool(w_small1.asbigint().ge(w_long2.num)) - -def lt__Long_SmallLong(space, w_long1, w_small2): - return space.newbool(w_long1.num.lt(w_small2.asbigint())) -def le__Long_SmallLong(space, w_long1, w_small2): - return space.newbool(w_long1.num.le(w_small2.asbigint())) -def eq__Long_SmallLong(space, w_long1, w_small2): - return space.newbool(w_long1.num.eq(w_small2.asbigint())) -def ne__Long_SmallLong(space, w_long1, w_small2): - return space.newbool(w_long1.num.ne(w_small2.asbigint())) -def gt__Long_SmallLong(space, w_long1, w_small2): - return space.newbool(w_long1.num.gt(w_small2.asbigint())) -def ge__Long_SmallLong(space, w_long1, w_small2): - return space.newbool(w_long1.num.ge(w_small2.asbigint())) - -def lt__SmallLong_Int(space, w_small1, w_int2): - return space.newbool(w_small1.longlong < w_int2.intval) -def le__SmallLong_Int(space, w_small1, w_int2): - return space.newbool(w_small1.longlong <= w_int2.intval) -def eq__SmallLong_Int(space, w_small1, w_int2): - return space.newbool(w_small1.longlong == w_int2.intval) -def ne__SmallLong_Int(space, w_small1, w_int2): - return space.newbool(w_small1.longlong != w_int2.intval) -def gt__SmallLong_Int(space, w_small1, w_int2): - return space.newbool(w_small1.longlong > w_int2.intval) -def ge__SmallLong_Int(space, w_small1, w_int2): - return space.newbool(w_small1.longlong >= w_int2.intval) - -def lt__Int_SmallLong(space, w_int1, w_small2): - return space.newbool(w_int1.intval < w_small2.longlong) -def le__Int_SmallLong(space, w_int1, w_small2): - return space.newbool(w_int1.intval <= w_small2.longlong) -def eq__Int_SmallLong(space, w_int1, w_small2): - return space.newbool(w_int1.intval == w_small2.longlong) -def ne__Int_SmallLong(space, w_int1, w_small2): - return space.newbool(w_int1.intval != w_small2.longlong) -def gt__Int_SmallLong(space, w_int1, w_small2): - return space.newbool(w_int1.intval > w_small2.longlong) -def ge__Int_SmallLong(space, w_int1, w_small2): - return space.newbool(w_int1.intval >= w_small2.longlong) - - -#hash: default implementation via Longs (a bit messy) - -def add__SmallLong_SmallLong(space, w_small1, w_small2): - x = w_small1.longlong - y = w_small2.longlong - try: - z = x + y - if ((z^x)&(z^y)) < 0: - raise OverflowError - except OverflowError: - raise FailedToImplementArgs(space.w_OverflowError, - space.wrap("integer addition")) - return W_SmallLongObject(z) - def add_ovr(space, w_int1, w_int2): x = r_longlong(w_int1.intval) y = r_longlong(w_int2.intval) return W_SmallLongObject(x + y) -def sub__SmallLong_SmallLong(space, w_small1, w_small2): - x = w_small1.longlong - y = w_small2.longlong - try: - z = x - y - if ((z^x)&(z^~y)) < 0: - raise OverflowError - except OverflowError: - raise FailedToImplementArgs(space.w_OverflowError, - space.wrap("integer subtraction")) - return W_SmallLongObject(z) - def sub_ovr(space, w_int1, w_int2): x = r_longlong(w_int1.intval) y = r_longlong(w_int2.intval) return W_SmallLongObject(x - y) -def mul__SmallLong_SmallLong(space, w_small1, w_small2): - x = w_small1.longlong - y = w_small2.longlong - try: - z = llong_mul_ovf(x, y) - except OverflowError: - raise FailedToImplementArgs(space.w_OverflowError, - space.wrap("integer multiplication")) - return W_SmallLongObject(z) - def mul_ovr(space, w_int1, w_int2): x = r_longlong(w_int1.intval) y = r_longlong(w_int2.intval) return W_SmallLongObject(x * y) -#truediv: default implementation via Longs - -def floordiv__SmallLong_SmallLong(space, w_small1, w_small2): - x = w_small1.longlong - y = w_small2.longlong - try: - if y == -1 and x == LONGLONG_MIN: - raise OverflowError - z = x // y - except ZeroDivisionError: - raise OperationError(space.w_ZeroDivisionError, - space.wrap("integer division by zero")) - except OverflowError: - raise FailedToImplementArgs(space.w_OverflowError, - space.wrap("integer division")) - return W_SmallLongObject(z) -div__SmallLong_SmallLong = floordiv__SmallLong_SmallLong - def floordiv_ovr(space, w_int1, w_int2): x = r_longlong(w_int1.intval) y = r_longlong(w_int2.intval) return W_SmallLongObject(x // y) div_ovr = floordiv_ovr -def mod__SmallLong_SmallLong(space, w_small1, w_small2): - x = w_small1.longlong - y = w_small2.longlong - try: - if y == -1 and x == LONGLONG_MIN: - raise OverflowError - z = x % y - except ZeroDivisionError: - raise OperationError(space.w_ZeroDivisionError, - space.wrap("integer modulo by zero")) - except OverflowError: - raise FailedToImplementArgs(space.w_OverflowError, - space.wrap("integer modulo")) - return W_SmallLongObject(z) - def mod_ovr(space, w_int1, w_int2): x = r_longlong(w_int1.intval) y = r_longlong(w_int2.intval) return W_SmallLongObject(x % y) -def divmod__SmallLong_SmallLong(space, w_small1, w_small2): - x = w_small1.longlong - y = w_small2.longlong - try: - if y == -1 and x == LONGLONG_MIN: - raise OverflowError - z = x // y - except ZeroDivisionError: - raise OperationError(space.w_ZeroDivisionError, - space.wrap("integer divmod by zero")) - except OverflowError: - raise FailedToImplementArgs(space.w_OverflowError, - space.wrap("integer modulo")) - # no overflow possible - m = x % y - return space.newtuple([W_SmallLongObject(z), W_SmallLongObject(m)]) - def divmod_ovr(space, w_int1, w_int2): return space.newtuple([div_ovr(space, w_int1, w_int2), mod_ovr(space, w_int1, w_int2)]) @@ -322,9 +485,7 @@ raise OperationError(space.w_TypeError, space.wrap("pow() 2nd argument " "cannot be negative when 3rd argument specified")) - ## bounce it, since it always returns float - raise FailedToImplementArgs(space.w_ValueError, - space.wrap("integer exponentiation")) + raise ValueError temp = iv ix = r_longlong(1) try: @@ -342,20 +503,10 @@ if iz: ix = ix % iz except OverflowError: - raise FailedToImplementArgs(space.w_OverflowError, - space.wrap("integer exponentiation")) + # XXX: + raise OverflowError return W_SmallLongObject(ix) -def pow__SmallLong_Int_SmallLong(space, w_small1, w_int2, w_small3): - z = w_small3.longlong - if z == 0: - raise OperationError(space.w_ValueError, - space.wrap("pow() 3rd argument cannot be 0")) - return _impl_pow(space, w_small1.longlong, w_int2, z) - -def pow__SmallLong_Int_None(space, w_small1, w_int2, _): - return _impl_pow(space, w_small1.longlong, w_int2) - def pow_ovr(space, w_int1, w_int2): try: return _impl_pow(space, r_longlong(w_int1.intval), w_int2) @@ -365,66 +516,15 @@ w_b = W_LongObject.fromint(space, w_int2.intval) return longobject.pow__Long_Long_None(space, w_a, w_b, space.w_None) -def neg__SmallLong(space, w_small): - a = w_small.longlong - try: - if a == LONGLONG_MIN: - raise OverflowError - x = -a - except OverflowError: - raise FailedToImplementArgs(space.w_OverflowError, - space.wrap("integer negation")) - return W_SmallLongObject(x) -get_negint = neg__SmallLong - def neg_ovr(space, w_int): a = r_longlong(w_int.intval) return W_SmallLongObject(-a) - -def pos__SmallLong(space, w_small): - return w_small - -def abs__SmallLong(space, w_small): - if w_small.longlong >= 0: - return w_small - else: - return get_negint(space, w_small) - def abs_ovr(space, w_int): a = r_longlong(w_int.intval) if a < 0: a = -a return W_SmallLongObject(a) -def nonzero__SmallLong(space, w_small): - return space.newbool(bool(w_small.longlong)) - -def invert__SmallLong(space, w_small): - x = w_small.longlong - a = ~x - return W_SmallLongObject(a) - -def lshift__SmallLong_Int(space, w_small1, w_int2): - a = w_small1.longlong - b = w_int2.intval - if r_uint(b) < LONGLONG_BIT: # 0 <= b < LONGLONG_BIT - try: - c = a << b - if a != (c >> b): - raise OverflowError - except OverflowError: - raise FailedToImplementArgs(space.w_OverflowError, - space.wrap("integer left shift")) - return W_SmallLongObject(c) - if b < 0: - raise OperationError(space.w_ValueError, - space.wrap("negative shift count")) - else: #b >= LONGLONG_BIT - if a == 0: - return w_small1 - raise FailedToImplementArgs(space.w_OverflowError, - space.wrap("integer left shift")) - def lshift_ovr(space, w_int1, w_int2): a = r_longlong(w_int1.intval) try: @@ -434,45 +534,3 @@ w_a = W_LongObject.fromint(space, w_int1.intval) w_b = W_LongObject.fromint(space, w_int2.intval) return longobject.lshift__Long_Long(space, w_a, w_b) - -def rshift__SmallLong_Int(space, w_small1, w_int2): - a = w_small1.longlong - b = w_int2.intval - if r_uint(b) >= LONGLONG_BIT: # not (0 <= b < LONGLONG_BIT) - if b < 0: - raise OperationError(space.w_ValueError, - space.wrap("negative shift count")) - else: # b >= LONGLONG_BIT - if a == 0: - return w_small1 - if a < 0: - a = -1 - else: - a = 0 - else: - a = a >> b - return W_SmallLongObject(a) - -def and__SmallLong_SmallLong(space, w_small1, w_small2): - a = w_small1.longlong - b = w_small2.longlong - res = a & b - return W_SmallLongObject(res) - -def xor__SmallLong_SmallLong(space, w_small1, w_small2): - a = w_small1.longlong - b = w_small2.longlong - res = a ^ b - return W_SmallLongObject(res) - -def or__SmallLong_SmallLong(space, w_small1, w_small2): - a = w_small1.longlong - b = w_small2.longlong - res = a | b - return W_SmallLongObject(res) - -#oct: default implementation via Longs -#hex: default implementation via Longs -#getnewargs: default implementation via Longs - -register_all(vars()) From noreply at buildbot.pypy.org Mon Nov 11 23:27:23 2013 From: noreply at buildbot.pypy.org (pjenvey) Date: Mon, 11 Nov 2013 23:27:23 +0100 (CET) Subject: [pypy-commit] pypy remove-intlong-smm: comment Message-ID: <20131111222723.F2E851C0E1C@cobra.cs.uni-duesseldorf.de> Author: Philip Jenvey Branch: remove-intlong-smm Changeset: r67964:5589ab3cae12 Date: 2013-11-11 14:26 -0800 http://bitbucket.org/pypy/pypy/changeset/5589ab3cae12/ Log: comment diff --git a/pypy/objspace/std/longobject.py b/pypy/objspace/std/longobject.py --- a/pypy/objspace/std/longobject.py +++ b/pypy/objspace/std/longobject.py @@ -406,6 +406,8 @@ if w_exponent.asbigint().sign < 0: from pypy.objspace.std.floatobject import delegate_Long2Float w_exponent = delegate_Long2Float(space, w_exponent) + # XXX: hack around multimethod annoyances for now (when + # w_modulus=None) return space.pow(self.descr_float(space), w_exponent, space.w_None if w_modulus is None else w_modulus) return W_LongObject(self.num.pow(w_exponent.asbigint(), None)) elif space.isinstance_w(w_modulus, space.w_int): From noreply at buildbot.pypy.org Mon Nov 11 23:27:25 2013 From: noreply at buildbot.pypy.org (pjenvey) Date: Mon, 11 Nov 2013 23:27:25 +0100 (CET) Subject: [pypy-commit] pypy remove-intlong-smm: merge default Message-ID: <20131111222725.1B1801C1041@cobra.cs.uni-duesseldorf.de> Author: Philip Jenvey Branch: remove-intlong-smm Changeset: r67965:ab62397eb222 Date: 2013-11-11 14:26 -0800 http://bitbucket.org/pypy/pypy/changeset/ab62397eb222/ Log: merge default diff --git a/pypy/objspace/std/complexobject.py b/pypy/objspace/std/complexobject.py --- a/pypy/objspace/std/complexobject.py +++ b/pypy/objspace/std/complexobject.py @@ -52,16 +52,16 @@ from pypy.objspace.std.complextype import complex_typedef as typedef _immutable_fields_ = ['realval', 'imagval'] - def __init__(w_self, realval=0.0, imgval=0.0): - w_self.realval = float(realval) - w_self.imagval = float(imgval) + def __init__(self, realval=0.0, imgval=0.0): + self.realval = float(realval) + self.imagval = float(imgval) - def unwrap(w_self, space): # for tests only - return complex(w_self.realval, w_self.imagval) + def unwrap(self, space): # for tests only + return complex(self.realval, self.imagval) - def __repr__(w_self): + def __repr__(self): """ representation for debugging purposes """ - return "" % (w_self.realval, w_self.imagval) + return "" % (self.realval, self.imagval) def as_tuple(self): return (self.realval, self.imagval) diff --git a/pypy/objspace/std/floatobject.py b/pypy/objspace/std/floatobject.py --- a/pypy/objspace/std/floatobject.py +++ b/pypy/objspace/std/floatobject.py @@ -27,11 +27,11 @@ typedef = float_typedef - def __init__(w_self, floatval): - w_self.floatval = floatval + def __init__(self, floatval): + self.floatval = floatval - def unwrap(w_self, space): - return w_self.floatval + def unwrap(self, space): + return self.floatval def float_w(self, space): return self.floatval diff --git a/pypy/objspace/std/smalllongobject.py b/pypy/objspace/std/smalllongobject.py --- a/pypy/objspace/std/smalllongobject.py +++ b/pypy/objspace/std/smalllongobject.py @@ -22,9 +22,9 @@ _immutable_fields_ = ['longlong'] - def __init__(w_self, value): + def __init__(self, value): assert isinstance(value, r_longlong) - w_self.longlong = value + self.longlong = value @staticmethod def fromint(value): @@ -34,17 +34,17 @@ def frombigint(bigint): return W_SmallLongObject(bigint.tolonglong()) - def asbigint(w_self): - return rbigint.fromrarith_int(w_self.longlong) + def asbigint(self): + return rbigint.fromrarith_int(self.longlong) def longval(self): return self.longlong - def __repr__(w_self): - return '' % w_self.longlong + def __repr__(self): + return '' % self.longlong - def int_w(w_self, space): - a = w_self.longlong + def int_w(self, space): + a = self.longlong b = intmask(a) if b == a: return b @@ -52,8 +52,8 @@ raise OperationError(space.w_OverflowError, space.wrap( "long int too large to convert to int")) - def uint_w(w_self, space): - a = w_self.longlong + def uint_w(self, space): + a = self.longlong if a < 0: raise OperationError(space.w_ValueError, space.wrap( "cannot convert negative integer to unsigned int")) @@ -64,8 +64,8 @@ raise OperationError(space.w_OverflowError, space.wrap( "long int too large to convert to unsigned int")) - def bigint_w(w_self, space): - return w_self.asbigint() + def bigint_w(self, space): + return self.asbigint() def float_w(self, space): return float(self.longlong) From noreply at buildbot.pypy.org Tue Nov 12 02:45:16 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Tue, 12 Nov 2013 02:45:16 +0100 (CET) Subject: [pypy-commit] pypy default: test/fix some numpy dtype creation from tuple cases Message-ID: <20131112014516.0C9F81C00B3@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r67966:5dc341236760 Date: 2013-11-11 20:37 -0500 http://bitbucket.org/pypy/pypy/changeset/5dc341236760/ Log: test/fix some numpy dtype creation from tuple cases diff --git a/pypy/module/micronumpy/interp_dtype.py b/pypy/module/micronumpy/interp_dtype.py --- a/pypy/module/micronumpy/interp_dtype.py +++ b/pypy/module/micronumpy/interp_dtype.py @@ -382,7 +382,14 @@ elif space.isinstance_w(w_dtype, space.w_list): return dtype_from_list(space, w_dtype) elif space.isinstance_w(w_dtype, space.w_tuple): - return descr__new__(space, w_subtype, space.getitem(w_dtype, space.wrap(0)), w_align, w_copy, w_shape=space.getitem(w_dtype, space.wrap(1))) + w_dtype0 = space.getitem(w_dtype, space.wrap(0)) + w_dtype1 = space.getitem(w_dtype, space.wrap(1)) + subdtype = descr__new__(space, w_subtype, w_dtype0, w_align, w_copy) + assert isinstance(subdtype, W_Dtype) + if subdtype.get_size() == 0: + name = "%s%d" % (subdtype.kind, space.int_w(w_dtype1)) + return descr__new__(space, w_subtype, space.wrap(name), w_align, w_copy) + return descr__new__(space, w_subtype, w_dtype0, w_align, w_copy, w_shape=w_dtype1) elif space.isinstance_w(w_dtype, space.w_dict): return dtype_from_dict(space, w_dtype) for dtype in cache.builtin_dtypes: diff --git a/pypy/module/micronumpy/test/dummy_module.py b/pypy/module/micronumpy/test/dummy_module.py --- a/pypy/module/micronumpy/test/dummy_module.py +++ b/pypy/module/micronumpy/test/dummy_module.py @@ -20,7 +20,7 @@ for t in types: globals()[t] = dtype(t).type -types = ['bool', 'int', 'float', 'complex', 'str', 'unicode'] +types = ['bool', 'int', 'float', 'complex', 'str', 'string', 'unicode'] for t in types: globals()[t + '_'] = dtype(t).type del types diff --git a/pypy/module/micronumpy/test/test_dtypes.py b/pypy/module/micronumpy/test/test_dtypes.py --- a/pypy/module/micronumpy/test/test_dtypes.py +++ b/pypy/module/micronumpy/test/test_dtypes.py @@ -46,6 +46,18 @@ assert 'data type not understood' in str(exc.value) raises(KeyError, 'dtype(int)["asdasd"]') + def test_dtype_from_tuple(self): + import numpy as np + d = np.dtype((np.int64, 4)) + assert d == np.dtype(('i8', (4,))) + assert d.shape == (4,) + d = np.dtype((np.string_, 4)) + assert d == np.dtype('S4') + assert d.shape == () + d = np.dtype(('S', 4)) + assert d == np.dtype('S4') + assert d.shape == () + def test_dtype_eq(self): from numpypy import dtype From noreply at buildbot.pypy.org Tue Nov 12 08:30:50 2013 From: noreply at buildbot.pypy.org (fijal) Date: Tue, 12 Nov 2013 08:30:50 +0100 (CET) Subject: [pypy-commit] pypy default: update project-ideas Message-ID: <20131112073050.ACD621C0206@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: Changeset: r67967:c8cb82052fbf Date: 2013-11-12 09:30 +0200 http://bitbucket.org/pypy/pypy/changeset/c8cb82052fbf/ Log: update project-ideas diff --git a/pypy/doc/project-ideas.rst b/pypy/doc/project-ideas.rst --- a/pypy/doc/project-ideas.rst +++ b/pypy/doc/project-ideas.rst @@ -158,6 +158,8 @@ Embedding PyPy ---------------------------------------- +Note: there is a basic proof-of-concept for that as a `uwsgi pypy plugin`_ + Being able to embed PyPy, say with its own limited C API, would be useful. But here is the most interesting variant, straight from EuroPython live discussion :-) We can have a generic "libpypy.so" that @@ -166,6 +168,8 @@ exported. This would give us a one-size-fits-all generic .so file to be imported by any application that wants to load .so files :-) +.. _`uwsgi pypy plugin`: http://uwsgi-docs.readthedocs.org/en/latest/PyPy.html + Optimising cpyext (CPython C-API compatibility layer) ----------------------------------------------------- From noreply at buildbot.pypy.org Tue Nov 12 08:46:33 2013 From: noreply at buildbot.pypy.org (fijal) Date: Tue, 12 Nov 2013 08:46:33 +0100 (CET) Subject: [pypy-commit] pypy default: this has been done Message-ID: <20131112074633.9BFF31C019D@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: Changeset: r67968:83ebf63bc891 Date: 2013-11-12 09:45 +0200 http://bitbucket.org/pypy/pypy/changeset/83ebf63bc891/ Log: this has been done diff --git a/pypy/doc/project-ideas.rst b/pypy/doc/project-ideas.rst --- a/pypy/doc/project-ideas.rst +++ b/pypy/doc/project-ideas.rst @@ -90,9 +90,6 @@ collectors can be written for specialized purposes, or even various experiments can be done for the general purpose. Examples: -* An incremental garbage collector that has specified maximal pause times, - crucial for games - * A garbage collector that compact memory better for mobile devices * A concurrent garbage collector (a lot of work) From noreply at buildbot.pypy.org Tue Nov 12 08:47:41 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Tue, 12 Nov 2013 08:47:41 +0100 (CET) Subject: [pypy-commit] pypy release-2.2.x: test/fix some numpy dtype creation from tuple cases Message-ID: <20131112074741.600761C019D@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: release-2.2.x Changeset: r67969:7b5eb1c5a0d1 Date: 2013-11-11 20:37 -0500 http://bitbucket.org/pypy/pypy/changeset/7b5eb1c5a0d1/ Log: test/fix some numpy dtype creation from tuple cases (transplanted from 5dc3412367605bff81ba8a4bfa7372670257bc77) diff --git a/pypy/module/micronumpy/interp_dtype.py b/pypy/module/micronumpy/interp_dtype.py --- a/pypy/module/micronumpy/interp_dtype.py +++ b/pypy/module/micronumpy/interp_dtype.py @@ -382,7 +382,14 @@ elif space.isinstance_w(w_dtype, space.w_list): return dtype_from_list(space, w_dtype) elif space.isinstance_w(w_dtype, space.w_tuple): - return descr__new__(space, w_subtype, space.getitem(w_dtype, space.wrap(0)), w_align, w_copy, w_shape=space.getitem(w_dtype, space.wrap(1))) + w_dtype0 = space.getitem(w_dtype, space.wrap(0)) + w_dtype1 = space.getitem(w_dtype, space.wrap(1)) + subdtype = descr__new__(space, w_subtype, w_dtype0, w_align, w_copy) + assert isinstance(subdtype, W_Dtype) + if subdtype.get_size() == 0: + name = "%s%d" % (subdtype.kind, space.int_w(w_dtype1)) + return descr__new__(space, w_subtype, space.wrap(name), w_align, w_copy) + return descr__new__(space, w_subtype, w_dtype0, w_align, w_copy, w_shape=w_dtype1) elif space.isinstance_w(w_dtype, space.w_dict): return dtype_from_dict(space, w_dtype) for dtype in cache.builtin_dtypes: diff --git a/pypy/module/micronumpy/test/dummy_module.py b/pypy/module/micronumpy/test/dummy_module.py --- a/pypy/module/micronumpy/test/dummy_module.py +++ b/pypy/module/micronumpy/test/dummy_module.py @@ -20,7 +20,7 @@ for t in types: globals()[t] = dtype(t).type -types = ['bool', 'int', 'float', 'complex', 'str', 'unicode'] +types = ['bool', 'int', 'float', 'complex', 'str', 'string', 'unicode'] for t in types: globals()[t + '_'] = dtype(t).type del types diff --git a/pypy/module/micronumpy/test/test_dtypes.py b/pypy/module/micronumpy/test/test_dtypes.py --- a/pypy/module/micronumpy/test/test_dtypes.py +++ b/pypy/module/micronumpy/test/test_dtypes.py @@ -46,6 +46,18 @@ assert 'data type not understood' in str(exc.value) raises(KeyError, 'dtype(int)["asdasd"]') + def test_dtype_from_tuple(self): + import numpy as np + d = np.dtype((np.int64, 4)) + assert d == np.dtype(('i8', (4,))) + assert d.shape == (4,) + d = np.dtype((np.string_, 4)) + assert d == np.dtype('S4') + assert d.shape == () + d = np.dtype(('S', 4)) + assert d == np.dtype('S4') + assert d.shape == () + def test_dtype_eq(self): from numpypy import dtype From noreply at buildbot.pypy.org Tue Nov 12 12:42:58 2013 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 12 Nov 2013 12:42:58 +0100 (CET) Subject: [pypy-commit] pypy release-2.2.x: Add os.startfile() using cffi. Message-ID: <20131112114258.E92D71C019D@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: release-2.2.x Changeset: r67970:4a3ba24a561a Date: 2013-11-12 12:42 +0100 http://bitbucket.org/pypy/pypy/changeset/4a3ba24a561a/ Log: Add os.startfile() using cffi. diff --git a/pypy/module/posix/__init__.py b/pypy/module/posix/__init__.py --- a/pypy/module/posix/__init__.py +++ b/pypy/module/posix/__init__.py @@ -27,6 +27,7 @@ 'popen2': 'app_posix.popen2', 'popen3': 'app_posix.popen3', 'popen4': 'app_posix.popen4', + 'startfile': 'app_startfile.startfile', }) if hasattr(os, 'wait'): diff --git a/pypy/module/posix/app_startfile.py b/pypy/module/posix/app_startfile.py new file mode 100644 --- /dev/null +++ b/pypy/module/posix/app_startfile.py @@ -0,0 +1,44 @@ + +class CFFIWrapper(object): + def __init__(self): + import cffi + ffi = cffi.FFI() + ffi.cdef(""" + HINSTANCE ShellExecuteA(HWND, LPCSTR, LPCSTR, LPCSTR, LPCSTR, INT); + HINSTANCE ShellExecuteW(HWND, LPCWSTR, LPCWSTR, LPCWSTR, LPCWSTR, INT); + DWORD GetLastError(void); + """) + self.NULL = ffi.NULL + self.cast = ffi.cast + self.libK = ffi.dlopen("Kernel32.dll") + self.libS = ffi.dlopen("Shell32.dll") + self.SW_SHOWNORMAL = 1 + +_cffi_wrapper = None + + +def startfile(filepath, operation=None): + global _cffi_wrapper + if _cffi_wrapper is None: + _cffi_wrapper = CFFIWrapper() + w = _cffi_wrapper + # + if operation is None: + operation = w.NULL + if isinstance(filepath, str): + if isinstance(operation, unicode): + operation = operation.encode("ascii") + rc = w.libS.ShellExecuteA(w.NULL, operation, filepath, + w.NULL, w.NULL, w.SW_SHOWNORMAL) + elif isinstance(filepath, unicode): + if isinstance(operation, str): + operation = operation.decode("ascii") + rc = w.libS.ShellExecuteW(w.NULL, operation, filepath, + w.NULL, w.NULL, w.SW_SHOWNORMAL) + else: + raise TypeError("argument 1 must be str or unicode") + rc = int(w.cast("uintptr_t", rc)) + if rc <= 32: + # sorry, no way to get the error message in less than one page of code + code = w.libK.GetLastError() + raise WindowsError(code, "Error %s" % code, filepath) From noreply at buildbot.pypy.org Tue Nov 12 12:44:02 2013 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 12 Nov 2013 12:44:02 +0100 (CET) Subject: [pypy-commit] pypy release-2.2.x: Add the NOT_RPYTHON comment at the top of the file, although that might Message-ID: <20131112114402.48C7C1C019D@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: release-2.2.x Changeset: r67971:7adf80f17ffb Date: 2013-11-12 12:43 +0100 http://bitbucket.org/pypy/pypy/changeset/7adf80f17ffb/ Log: Add the NOT_RPYTHON comment at the top of the file, although that might not have any effect any more. diff --git a/pypy/module/posix/app_startfile.py b/pypy/module/posix/app_startfile.py --- a/pypy/module/posix/app_startfile.py +++ b/pypy/module/posix/app_startfile.py @@ -1,3 +1,4 @@ +# NOT_RPYTHON class CFFIWrapper(object): def __init__(self): From noreply at buildbot.pypy.org Tue Nov 12 12:51:49 2013 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 12 Nov 2013 12:51:49 +0100 (CET) Subject: [pypy-commit] pypy release-2.2.x: Found out that FormatError() is exposed via _rawffi. Would you believe. Message-ID: <20131112115149.80A191C0206@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: release-2.2.x Changeset: r67972:f262c7a6776c Date: 2013-11-12 12:51 +0100 http://bitbucket.org/pypy/pypy/changeset/f262c7a6776c/ Log: Found out that FormatError() is exposed via _rawffi. Would you believe. diff --git a/pypy/module/posix/app_startfile.py b/pypy/module/posix/app_startfile.py --- a/pypy/module/posix/app_startfile.py +++ b/pypy/module/posix/app_startfile.py @@ -40,6 +40,10 @@ raise TypeError("argument 1 must be str or unicode") rc = int(w.cast("uintptr_t", rc)) if rc <= 32: - # sorry, no way to get the error message in less than one page of code code = w.libK.GetLastError() - raise WindowsError(code, "Error %s" % code, filepath) + try: + import _rawffi + msg = _rawffi.FormatError(code) + except ImportError: + msg = 'Error %s' % code + raise WindowsError(code, msg, filepath) From noreply at buildbot.pypy.org Tue Nov 12 13:48:33 2013 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 12 Nov 2013 13:48:33 +0100 (CET) Subject: [pypy-commit] cffi default: Add ffi.getwinerror(). Message-ID: <20131112124833.46C001C07D2@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r1412:8a16eff7850c Date: 2013-11-12 13:48 +0100 http://bitbucket.org/cffi/cffi/changeset/8a16eff7850c/ Log: Add ffi.getwinerror(). diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c --- a/c/_cffi_backend.c +++ b/c/_cffi_backend.c @@ -5227,6 +5227,9 @@ {"set_errno", b_set_errno, METH_VARARGS}, {"newp_handle", b_newp_handle, METH_VARARGS}, {"from_handle", b_from_handle, METH_O}, +#ifdef MS_WIN32 + {"getwinerror", b_getwinerror, METH_VARARGS}, +#endif {"_get_types", b__get_types, METH_NOARGS}, {"_testfunc", b__testfunc, METH_VARARGS}, {NULL, NULL} /* Sentinel */ diff --git a/c/misc_win32.h b/c/misc_win32.h --- a/c/misc_win32.h +++ b/c/misc_win32.h @@ -80,6 +80,54 @@ /* else: cannot report the error */ } +static PyObject *b_getwinerror(PyObject *self, PyObject *args) +{ + int err = -1; + int len; + char *s; + char *s_buf = NULL; /* Free via LocalFree */ + char s_small_buf[28]; /* Room for "Windows Error 0xFFFFFFFF" */ + PyObject *v; + + if (!PyArg_ParseTuple(args, "|i", &err)) + return NULL; + + if (err == -1) { + struct cffi_errno_s *p; + p = _geterrno_object(); + if (p == NULL) + return PyErr_NoMemory(); + err = p->saved_lasterror; + } + + len = FormatMessage( + /* Error API error */ + FORMAT_MESSAGE_ALLOCATE_BUFFER | + FORMAT_MESSAGE_FROM_SYSTEM | + FORMAT_MESSAGE_IGNORE_INSERTS, + NULL, /* no message source */ + err, + MAKELANGID(LANG_NEUTRAL, + SUBLANG_DEFAULT), /* Default language */ + (LPTSTR) &s_buf, + 0, /* size not used */ + NULL); /* no args */ + if (len==0) { + /* Only seen this in out of mem situations */ + sprintf(s_small_buf, "Windows Error 0x%X", err); + s = s_small_buf; + s_buf = NULL; + } else { + s = s_buf; + /* remove trailing cr/lf and dots */ + while (len > 0 && (s[len-1] <= ' ' || s[len-1] == '.')) + s[--len] = '\0'; + } + v = Py_BuildValue("(is)", err, s); + LocalFree(s_buf); + return v; +} + /************************************************************/ /* Emulate dlopen()&co. from the Windows API */ diff --git a/c/test_c.py b/c/test_c.py --- a/c/test_c.py +++ b/c/test_c.py @@ -2698,6 +2698,16 @@ # res = GetLastError() assert res == 42 + # + SetLastError(2) + code, message = getwinerror() + assert code == 2 + assert message == "The system cannot find the file specified" + # + code, message = getwinerror(1155) + assert code == 1155 + assert message == ("No application is associated with the " + "specified file for this operation") def test_nonstandard_integer_types(): for typename in ['int8_t', 'uint8_t', 'int16_t', 'uint16_t', 'int32_t', diff --git a/cffi/api.py b/cffi/api.py --- a/cffi/api.py +++ b/cffi/api.py @@ -347,6 +347,9 @@ errno = property(_get_errno, _set_errno, None, "the value of 'errno' from/to the C calls") + def getwinerror(self, code=-1): + return self._backend.getwinerror(code) + def _pointer_to(self, ctype): from . import model with self._lock: diff --git a/doc/source/index.rst b/doc/source/index.rst --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -1122,9 +1122,18 @@ ``ffi.errno``: the value of ``errno`` received from the most recent C call in this thread, and passed to the following C call, is available via -reads and writes of the property ``ffi.errno``. On Windows we also save -and restore the ``GetLastError()`` value, but to access it you need to -declare and call the ``GetLastError()`` function as usual. +reads and writes of the property ``ffi.errno``. + +``ffi.getwinerror(code=-1)``: on Windows, in addition to ``errno`` we +also save and restore the ``GetLastError()`` value across function +calls. This function returns this error code as a tuple ``(code, +message)``, adding a readable message like Python does when raising +WindowsError. If the argument ``code`` is given, format that code into +a message instead of using ``GetLastError()``. *New in version 0.8.* +(Note that it is also possible to declare and call the ``GetLastError()`` +function as usual.) + +.. "versionadded:: 0.8" --- inlined in the previous paragraph ``ffi.string(cdata, [maxlen])``: return a Python string (or unicode string) from the 'cdata'. *New in version 0.3.* diff --git a/testing/test_ffi_backend.py b/testing/test_ffi_backend.py --- a/testing/test_ffi_backend.py +++ b/testing/test_ffi_backend.py @@ -194,3 +194,20 @@ assert p.a[0] == 200 assert p.a[1] == 300 assert p.a[2] == 400 + + @pytest.mark.skipif("sys.platform != 'win32'") + def test_getwinerror(self): + ffi = FFI() + code, message = ffi.getwinerror(1155) + assert code == 1155 + assert message == ("No application is associated with the " + "specified file for this operation") + ffi.cdef("void SetLastError(int);") + lib = ffi.dlopen("Kernel32.dll") + lib.SetLastError(2) + code, message = ffi.getwinerror() + assert code == 2 + assert message == "The system cannot find the file specified" + code, message = ffi.getwinerror(-1) + assert code == 2 + assert message == "The system cannot find the file specified" From noreply at buildbot.pypy.org Tue Nov 12 13:56:17 2013 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 12 Nov 2013 13:56:17 +0100 (CET) Subject: [pypy-commit] pypy release-2.2.x: Like CPython, strip the message from trailing dots. Message-ID: <20131112125617.2DFD71C0F88@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: release-2.2.x Changeset: r67973:7ce0a6e10446 Date: 2013-11-12 13:54 +0100 http://bitbucket.org/pypy/pypy/changeset/7ce0a6e10446/ Log: Like CPython, strip the message from trailing dots. diff --git a/rpython/rlib/rwin32.py b/rpython/rlib/rwin32.py --- a/rpython/rlib/rwin32.py +++ b/rpython/rlib/rwin32.py @@ -216,7 +216,7 @@ def llimpl_FormatError(code): "Return a message corresponding to the given Windows error code." buf = lltype.malloc(rffi.CCHARPP.TO, 1, flavor='raw') - + buf[0] = lltype.nullptr(rffi.CCHARP.TO) try: msglen = FormatMessage(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM, @@ -225,17 +225,20 @@ DEFAULT_LANGUAGE, rffi.cast(rffi.CCHARP, buf), 0, None) + buflen = intmask(msglen) - if msglen <= 2: # includes the case msglen < 0 - return fake_FormatError(code) + # remove trailing cr/lf and dots + s_buf = buf[0] + while buflen > 0 and (s_buf[buflen - 1] <= ' ' or + s_buf[buflen - 1] == '.'): + buflen -= 1 - # FormatMessage always appends \r\n. - buflen = intmask(msglen - 2) - assert buflen > 0 - - result = rffi.charpsize2str(buf[0], buflen) + if buflen <= 0: + result = fake_FormatError(code) + else: + result = rffi.charpsize2str(s_buf, buflen) + finally: LocalFree(rffi.cast(rffi.VOIDP, buf[0])) - finally: lltype.free(buf, flavor='raw') return result From noreply at buildbot.pypy.org Tue Nov 12 13:56:18 2013 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 12 Nov 2013 13:56:18 +0100 (CET) Subject: [pypy-commit] pypy release-2.2.x: Update to cffi/8a16eff7850c. Message-ID: <20131112125618.785B01C0F88@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: release-2.2.x Changeset: r67974:662e4114634e Date: 2013-11-12 13:55 +0100 http://bitbucket.org/pypy/pypy/changeset/662e4114634e/ Log: Update to cffi/8a16eff7850c. diff --git a/pypy/module/_cffi_backend/__init__.py b/pypy/module/_cffi_backend/__init__.py --- a/pypy/module/_cffi_backend/__init__.py +++ b/pypy/module/_cffi_backend/__init__.py @@ -1,3 +1,4 @@ +import sys from pypy.interpreter.mixedmodule import MixedModule from rpython.rlib import rdynload @@ -43,6 +44,8 @@ 'FFI_DEFAULT_ABI': 'ctypefunc._get_abi(space, "FFI_DEFAULT_ABI")', 'FFI_CDECL': 'ctypefunc._get_abi(space,"FFI_DEFAULT_ABI")',#win32 name } + if sys.platform == 'win32': + interpleveldefs['getwinerror'] = 'cerrno.getwinerror' for _name in ["RTLD_LAZY", "RTLD_NOW", "RTLD_GLOBAL", "RTLD_LOCAL", "RTLD_NODELETE", "RTLD_NOLOAD", "RTLD_DEEPBIND"]: diff --git a/pypy/module/_cffi_backend/cerrno.py b/pypy/module/_cffi_backend/cerrno.py --- a/pypy/module/_cffi_backend/cerrno.py +++ b/pypy/module/_cffi_backend/cerrno.py @@ -39,3 +39,14 @@ def set_errno(space, errno): ec = get_errno_container(space) ec._cffi_saved_errno = errno + +# ____________________________________________________________ + + at unwrap_spec(code=int) +def getwinerror(space, code=-1): + from rpython.rlib.rwin32 import FormatError + if code == -1: + ec = get_errno_container(space) + code = ec._cffi_saved_LastError + message = FormatError(code) + return space.newtuple([space.wrap(code), space.wrap(message)]) diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py --- a/pypy/module/_cffi_backend/test/_backend_test_c.py +++ b/pypy/module/_cffi_backend/test/_backend_test_c.py @@ -2687,6 +2687,16 @@ # res = GetLastError() assert res == 42 + # + SetLastError(2) + code, message = getwinerror() + assert code == 2 + assert message == "The system cannot find the file specified" + # + code, message = getwinerror(1155) + assert code == 1155 + assert message == ("No application is associated with the " + "specified file for this operation") def test_nonstandard_integer_types(): for typename in ['int8_t', 'uint8_t', 'int16_t', 'uint16_t', 'int32_t', From noreply at buildbot.pypy.org Tue Nov 12 13:57:11 2013 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 12 Nov 2013 13:57:11 +0100 (CET) Subject: [pypy-commit] pypy release-2.2.x: Import cffi/8a16eff7850c here too Message-ID: <20131112125711.C24411C0F88@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: release-2.2.x Changeset: r67975:edcc6d30feff Date: 2013-11-12 13:56 +0100 http://bitbucket.org/pypy/pypy/changeset/edcc6d30feff/ Log: Import cffi/8a16eff7850c here too diff --git a/lib_pypy/cffi/api.py b/lib_pypy/cffi/api.py --- a/lib_pypy/cffi/api.py +++ b/lib_pypy/cffi/api.py @@ -347,6 +347,9 @@ errno = property(_get_errno, _set_errno, None, "the value of 'errno' from/to the C calls") + def getwinerror(self, code=-1): + return self._backend.getwinerror(code) + def _pointer_to(self, ctype): from . import model with self._lock: diff --git a/pypy/module/test_lib_pypy/cffi_tests/test_ffi_backend.py b/pypy/module/test_lib_pypy/cffi_tests/test_ffi_backend.py --- a/pypy/module/test_lib_pypy/cffi_tests/test_ffi_backend.py +++ b/pypy/module/test_lib_pypy/cffi_tests/test_ffi_backend.py @@ -195,3 +195,20 @@ assert p.a[0] == 200 assert p.a[1] == 300 assert p.a[2] == 400 + + @pytest.mark.skipif("sys.platform != 'win32'") + def test_getwinerror(self): + ffi = FFI() + code, message = ffi.getwinerror(1155) + assert code == 1155 + assert message == ("No application is associated with the " + "specified file for this operation") + ffi.cdef("void SetLastError(int);") + lib = ffi.dlopen("Kernel32.dll") + lib.SetLastError(2) + code, message = ffi.getwinerror() + assert code == 2 + assert message == "The system cannot find the file specified" + code, message = ffi.getwinerror(-1) + assert code == 2 + assert message == "The system cannot find the file specified" From noreply at buildbot.pypy.org Tue Nov 12 13:59:34 2013 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 12 Nov 2013 13:59:34 +0100 (CET) Subject: [pypy-commit] pypy release-2.2.x: Use getwinerror() here Message-ID: <20131112125934.57BD71C140E@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: release-2.2.x Changeset: r67976:7341987b8055 Date: 2013-11-12 13:58 +0100 http://bitbucket.org/pypy/pypy/changeset/7341987b8055/ Log: Use getwinerror() here diff --git a/pypy/module/posix/app_startfile.py b/pypy/module/posix/app_startfile.py --- a/pypy/module/posix/app_startfile.py +++ b/pypy/module/posix/app_startfile.py @@ -11,9 +11,9 @@ """) self.NULL = ffi.NULL self.cast = ffi.cast - self.libK = ffi.dlopen("Kernel32.dll") - self.libS = ffi.dlopen("Shell32.dll") + self.lib = ffi.dlopen("Shell32.dll") self.SW_SHOWNORMAL = 1 + self.getwinerror = ffi.getwinerror _cffi_wrapper = None @@ -29,21 +29,16 @@ if isinstance(filepath, str): if isinstance(operation, unicode): operation = operation.encode("ascii") - rc = w.libS.ShellExecuteA(w.NULL, operation, filepath, - w.NULL, w.NULL, w.SW_SHOWNORMAL) + rc = w.lib.ShellExecuteA(w.NULL, operation, filepath, + w.NULL, w.NULL, w.SW_SHOWNORMAL) elif isinstance(filepath, unicode): if isinstance(operation, str): operation = operation.decode("ascii") - rc = w.libS.ShellExecuteW(w.NULL, operation, filepath, - w.NULL, w.NULL, w.SW_SHOWNORMAL) + rc = w.lib.ShellExecuteW(w.NULL, operation, filepath, + w.NULL, w.NULL, w.SW_SHOWNORMAL) else: raise TypeError("argument 1 must be str or unicode") rc = int(w.cast("uintptr_t", rc)) if rc <= 32: - code = w.libK.GetLastError() - try: - import _rawffi - msg = _rawffi.FormatError(code) - except ImportError: - msg = 'Error %s' % code + code, msg = w.getwinerror() raise WindowsError(code, msg, filepath) From noreply at buildbot.pypy.org Tue Nov 12 14:18:05 2013 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 12 Nov 2013 14:18:05 +0100 (CET) Subject: [pypy-commit] pypy release-2.2.x: Python 2.6 compatibility Message-ID: <20131112131805.DE28A1C00F8@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: release-2.2.x Changeset: r67977:3e1a49627e13 Date: 2013-11-12 14:17 +0100 http://bitbucket.org/pypy/pypy/changeset/3e1a49627e13/ Log: Python 2.6 compatibility diff --git a/rpython/annotator/bookkeeper.py b/rpython/annotator/bookkeeper.py --- a/rpython/annotator/bookkeeper.py +++ b/rpython/annotator/bookkeeper.py @@ -5,7 +5,6 @@ from __future__ import absolute_import import sys, types, inspect, weakref -from collections import OrderedDict from rpython.flowspace.model import Constant from rpython.annotator.model import (SomeOrderedDict, @@ -371,7 +370,7 @@ for e in x: listdef.generalize(self.immutablevalue(e, False)) result = SomeList(listdef) - elif tp is dict or tp is r_dict or tp is OrderedDict: + elif tp is dict or tp is r_dict or tp is SomeOrderedDict.knowntype: if need_const: key = Constant(x) try: @@ -413,7 +412,7 @@ dictdef.generalize_key(self.immutablevalue(ek, False)) dictdef.generalize_value(self.immutablevalue(ev, False)) dictdef.seen_prebuilt_key(ek) - if tp is OrderedDict: + if tp is SomeOrderedDict.knowntype: result = SomeOrderedDict(dictdef) else: result = SomeDict(dictdef) diff --git a/rpython/annotator/builtin.py b/rpython/annotator/builtin.py --- a/rpython/annotator/builtin.py +++ b/rpython/annotator/builtin.py @@ -2,7 +2,6 @@ Built-in functions. """ import sys -from collections import OrderedDict from rpython.annotator.model import ( SomeInteger, SomeObject, SomeChar, SomeBool, SomeString, SomeTuple, s_Bool, @@ -364,7 +363,7 @@ BUILTIN_ANALYZERS[rpython.rlib.objectmodel.instantiate] = robjmodel_instantiate BUILTIN_ANALYZERS[rpython.rlib.objectmodel.r_dict] = robjmodel_r_dict BUILTIN_ANALYZERS[rpython.rlib.objectmodel.r_ordereddict] = robjmodel_r_ordereddict -BUILTIN_ANALYZERS[OrderedDict] = lambda : SomeOrderedDict(getbookkeeper().getdictdef()) +BUILTIN_ANALYZERS[SomeOrderedDict.knowntype] = lambda : SomeOrderedDict(getbookkeeper().getdictdef()) BUILTIN_ANALYZERS[rpython.rlib.objectmodel.hlinvoke] = robjmodel_hlinvoke BUILTIN_ANALYZERS[rpython.rlib.objectmodel.keepalive_until_here] = robjmodel_keepalive_until_here BUILTIN_ANALYZERS[rpython.rtyper.lltypesystem.llmemory.cast_ptr_to_adr] = llmemory_cast_ptr_to_adr diff --git a/rpython/annotator/model.py b/rpython/annotator/model.py --- a/rpython/annotator/model.py +++ b/rpython/annotator/model.py @@ -32,7 +32,6 @@ import inspect import weakref from types import BuiltinFunctionType, MethodType -from collections import OrderedDict import rpython from rpython.tool import descriptor @@ -357,7 +356,11 @@ return '{...%s...}' % (len(const),) class SomeOrderedDict(SomeDict): - knowntype = OrderedDict + try: + from collections import OrderedDict as knowntype + except ImportError: # Python 2.6 + class PseudoOrderedDict(dict): pass + knowntype = PseudoOrderedDict def method_copy(dct): return SomeOrderedDict(dct.dictdef) diff --git a/rpython/rtyper/rbuiltin.py b/rpython/rtyper/rbuiltin.py --- a/rpython/rtyper/rbuiltin.py +++ b/rpython/rtyper/rbuiltin.py @@ -1,4 +1,3 @@ -from collections import OrderedDict from rpython.annotator import model as annmodel from rpython.flowspace.model import Constant @@ -750,7 +749,7 @@ BUILTIN_TYPER[isinstance] = rtype_builtin_isinstance BUILTIN_TYPER[hasattr] = rtype_builtin_hasattr BUILTIN_TYPER[objectmodel.r_dict] = rtype_r_dict -BUILTIN_TYPER[OrderedDict] = rtype_ordered_dict +BUILTIN_TYPER[annmodel.SomeOrderedDict.knowntype] = rtype_ordered_dict BUILTIN_TYPER[objectmodel.r_ordereddict] = rtype_ordered_dict # _________________________________________________________________ diff --git a/rpython/rtyper/test/test_rordereddict.py b/rpython/rtyper/test/test_rordereddict.py --- a/rpython/rtyper/test/test_rordereddict.py +++ b/rpython/rtyper/test/test_rordereddict.py @@ -1,6 +1,9 @@ import py -from collections import OrderedDict +try: + from collections import OrderedDict +except ImportError: # Python 2.6 + py.test.skip("requires collections.OrderedDict") from rpython.rtyper.lltypesystem import lltype, rffi from rpython.rtyper.lltypesystem import rordereddict, rstr from rpython.rlib.rarithmetic import intmask From noreply at buildbot.pypy.org Tue Nov 12 14:24:30 2013 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 12 Nov 2013 14:24:30 +0100 (CET) Subject: [pypy-commit] pypy release-2.2.x: Add an app-level-only test for os.startfile() Message-ID: <20131112132430.2EBED1C0206@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: release-2.2.x Changeset: r67978:2cec7296d7fb Date: 2013-11-12 14:23 +0100 http://bitbucket.org/pypy/pypy/changeset/2cec7296d7fb/ Log: Add an app-level-only test for os.startfile() diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py --- a/pypy/module/posix/test/test_posix2.py +++ b/pypy/module/posix/test/test_posix2.py @@ -52,6 +52,7 @@ def setup_class(cls): cls.space = space + cls.w_runappdirect = space.wrap(cls.runappdirect) cls.w_posix = space.appexec([], GET_POSIX) cls.w_path = space.wrap(str(path)) cls.w_path2 = space.wrap(str(path2)) @@ -1108,6 +1109,28 @@ assert False, "urandom() always returns the same string" # Or very unlucky + if hasattr(os, 'startfile'): + def test_startfile(self): + if not self.runappdirect: + skip("should not try to import cffi at app-level") + startfile = self.posix.startfile + for t1 in [str, unicode]: + for t2 in [str, unicode]: + e = raises(WindowsError, startfile, t1("\\"), t2("close")) + assert e.value.args[0] == 1155 + assert e.value.args[1] == ( + "No application is associated with the " + "specified file for this operation") + if len(e.value.args) > 2: + assert e.value.args[2] == t1("\\") + # + e = raises(WindowsError, startfile, "\\foo\\bar\\baz") + assert e.value.args[0] == 2 + assert e.value.args[1] == ( + "The system cannot find the file specified") + if len(e.value.args) > 2: + assert e.value.args[2] == "\\foo\\bar\\baz" + class AppTestEnvironment(object): def setup_class(cls): From noreply at buildbot.pypy.org Tue Nov 12 14:26:34 2013 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 12 Nov 2013 14:26:34 +0100 (CET) Subject: [pypy-commit] pypy release-2.2.x: Progress Message-ID: <20131112132634.47BB61C00F8@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: release-2.2.x Changeset: r67979:49bfc678ad16 Date: 2013-11-12 14:25 +0100 http://bitbucket.org/pypy/pypy/changeset/49bfc678ad16/ Log: Progress diff --git a/pypy/doc/release-2.2.0.rst b/pypy/doc/release-2.2.0.rst --- a/pypy/doc/release-2.2.0.rst +++ b/pypy/doc/release-2.2.0.rst @@ -1,9 +1,87 @@ ======================================= -PyPy 2.2 - xxx +PyPy 2.2 - Incrementalism ======================================= -GC! +We're pleased to announce PyPy 2.2, which targets version 2.7.3 of the Python +language. This release main highlight is the introduction of the incremental +garbage collector, sponsored by the `Raspberry Pi Foundation`_. +This release also contains several bugfixes and performance improvements. -numpypy module was removed in favor of an external numpy fork +You can download the PyPy 2.2 release here: + + http://pypy.org/download.html + +We would like to thank our donors for the continued support of the PyPy +project. We showed quite a bit of progress on all three projects (see below) +and we're slowly running out of funds. +Please consider donating more so we can finish those projects! The three +projects are: + +* Py3k (supporting Python 3.x): the release PyPy3 2.2 is imminent. + +* STM (software transactional memory): a preview will be released very soon, + as soon as we fix a few bugs + +* NumPy: the work done is included in the PyPy 2.2 release. More details below. + +.. _`Raspberry Pi Foundation`: http://www.raspberrypi.org + +What is PyPy? +============= + +PyPy is a very compliant Python interpreter, almost a drop-in replacement for +CPython 2.7. It's fast (`pypy 2.2 and cpython 2.7.2`_ performance comparison) +due to its integrated tracing JIT compiler. + +This release supports x86 machines running Linux 32/64, Mac OS X 64, Windows +32, or ARM (ARMv6 or ARMv7, with VFPv3). + +Work on the native Windows 64 is still stalling, we would welcome a volunteer +to handle that. + +.. _`pypy 2.2 and cpython 2.7.2`: http://speed.pypy.org + +Highlights +========== + +* Our Garbage Collector is now "incremental". It should avoid almost all pauses due + to a major collection taking place. Previously, it would pause the program (rarely) + to walk all live objects, which could take arbitrarily long if your process is using + a whole lot of RAM. Now the same work is done in steps. This should make PyPy + more responsive, e.g. in games. There are still other pauses, from the GC and the JIT, + but they should be on the order of 5 milliseconds each. + +* The JIT counters for hot code were never reset, which meant that a process running + for long enough would eventually JIT-compile more and more rarely executed code. + Not only is it useless to compile such code, but as more compiled code means more + memory used, this gives the impression of a memory leak. This has been tentatively + fixed by decreasing the counters from time to time. + +* NumPy has been split: now PyPy only contains the core module, called ``_numpypy``. + The ``numpy`` module itself has been moved to ``https://bitbucket.org/pypy/numpy``. + You need to install it separately in a virtualenv with + ``pip install git+https://bitbucket.org/pypy/numpy.git``. + +* improvements to non-inlined calls + +* sys.set_trace is now JITted (think coverage) + +* faster json + +* improvements in buffer copying + +* tk is supported (XXX was already in pypy 2.1 it seems?? maybe not correctly packaged?) + +* We finally wrote all the missing ``os.xxx()`` functions. There are a lot of strange + ones that nobody ever heard about, except those who really need them. + +* numpy C API + + + the core module is included in PyPy 2.2, but you must now install + an external fork of numpy from https://bitbucket.org/pypy/numpy + + +removed in favor of an external numpy fork at https://bitbucket.org/pypy/numpy From noreply at buildbot.pypy.org Tue Nov 12 14:30:27 2013 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 12 Nov 2013 14:30:27 +0100 (CET) Subject: [pypy-commit] cffi default: Ah bah Message-ID: <20131112133027.568891C00F8@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r1413:71a521ed3573 Date: 2013-11-12 14:30 +0100 http://bitbucket.org/cffi/cffi/changeset/71a521ed3573/ Log: Ah bah diff --git a/testing/test_verify.py b/testing/test_verify.py --- a/testing/test_verify.py +++ b/testing/test_verify.py @@ -711,7 +711,7 @@ "#define BAZ ...\n") lib = ffi.verify("#define FOO 42\n" "#define BAR (-44)\n" - "#define BAZ 0xffffffffffffffffLL\n") + "#define BAZ 0xffffffffffffffffULL\n") assert lib.FOO == 42 assert lib.BAR == -44 assert lib.BAZ == 0xffffffffffffffff From noreply at buildbot.pypy.org Tue Nov 12 14:31:45 2013 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 12 Nov 2013 14:31:45 +0100 (CET) Subject: [pypy-commit] pypy release-2.2.x: Import cffi/71a521ed3573 Message-ID: <20131112133145.310891C0206@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: release-2.2.x Changeset: r67980:729af32ca599 Date: 2013-11-12 14:31 +0100 http://bitbucket.org/pypy/pypy/changeset/729af32ca599/ Log: Import cffi/71a521ed3573 diff --git a/pypy/module/test_lib_pypy/cffi_tests/test_verify.py b/pypy/module/test_lib_pypy/cffi_tests/test_verify.py --- a/pypy/module/test_lib_pypy/cffi_tests/test_verify.py +++ b/pypy/module/test_lib_pypy/cffi_tests/test_verify.py @@ -712,7 +712,7 @@ "#define BAZ ...\n") lib = ffi.verify("#define FOO 42\n" "#define BAR (-44)\n" - "#define BAZ 0xffffffffffffffffLL\n") + "#define BAZ 0xffffffffffffffffULL\n") assert lib.FOO == 42 assert lib.BAR == -44 assert lib.BAZ == 0xffffffffffffffff From noreply at buildbot.pypy.org Tue Nov 12 14:34:24 2013 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 12 Nov 2013 14:34:24 +0100 (CET) Subject: [pypy-commit] cffi default: Skip half the test with MSVC Message-ID: <20131112133424.1797C1C0206@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r1414:8c9468c7c428 Date: 2013-11-12 14:33 +0100 http://bitbucket.org/cffi/cffi/changeset/8c9468c7c428/ Log: Skip half the test with MSVC diff --git a/testing/test_verify.py b/testing/test_verify.py --- a/testing/test_verify.py +++ b/testing/test_verify.py @@ -1601,6 +1601,8 @@ (maxulong, -1, ''), (-1, 0xffffffff, 'U'), (-1, maxulong, 'UL')]: + if c2c and sys.platform == 'win32': + continue # enums may always be signed with MSVC ffi = FFI() ffi.cdef("enum foo_e { AA=%s };" % c1) e = py.test.raises(VerificationError, ffi.verify, From noreply at buildbot.pypy.org Tue Nov 12 14:35:49 2013 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 12 Nov 2013 14:35:49 +0100 (CET) Subject: [pypy-commit] pypy release-2.2.x: Import cffi/8c9468c7c428 Message-ID: <20131112133549.8E4481C0206@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: release-2.2.x Changeset: r67981:f3423a62cd09 Date: 2013-11-12 14:33 +0100 http://bitbucket.org/pypy/pypy/changeset/f3423a62cd09/ Log: Import cffi/8c9468c7c428 diff --git a/pypy/module/test_lib_pypy/cffi_tests/test_verify.py b/pypy/module/test_lib_pypy/cffi_tests/test_verify.py --- a/pypy/module/test_lib_pypy/cffi_tests/test_verify.py +++ b/pypy/module/test_lib_pypy/cffi_tests/test_verify.py @@ -1602,6 +1602,8 @@ (maxulong, -1, ''), (-1, 0xffffffff, 'U'), (-1, maxulong, 'UL')]: + if c2c and sys.platform == 'win32': + continue # enums may always be signed with MSVC ffi = FFI() ffi.cdef("enum foo_e { AA=%s };" % c1) e = py.test.raises(VerificationError, ffi.verify, From noreply at buildbot.pypy.org Tue Nov 12 15:01:01 2013 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 12 Nov 2013 15:01:01 +0100 (CET) Subject: [pypy-commit] cffi default: Carefully write the Python 3 version of getwinerror(). I have no way to Message-ID: <20131112140101.D66051C12CD@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r1415:3f83df0a79e3 Date: 2013-11-12 15:00 +0100 http://bitbucket.org/cffi/cffi/changeset/3f83df0a79e3/ Log: Carefully write the Python 3 version of getwinerror(). I have no way to know if I did a typo there. diff --git a/c/misc_win32.h b/c/misc_win32.h --- a/c/misc_win32.h +++ b/c/misc_win32.h @@ -80,6 +80,54 @@ /* else: cannot report the error */ } +#if PY_MAJOR_VERSION >= 3 +static PyObject *b_getwinerror(PyObject *self, PyObject *args) +{ + int err = -1; + int len; + WCHAR *s_buf = NULL; /* Free via LocalFree */ + PyObject *v, *message; + + if (!PyArg_ParseTuple(args, "|i", &err)) + return NULL; + + if (err == -1) { + struct cffi_errno_s *p; + p = _geterrno_object(); + if (p == NULL) + return PyErr_NoMemory(); + err = p->saved_lasterror; + } + + len = FormatMessageW( + /* Error API error */ + FORMAT_MESSAGE_ALLOCATE_BUFFER | + FORMAT_MESSAGE_FROM_SYSTEM | + FORMAT_MESSAGE_IGNORE_INSERTS, + NULL, /* no message source */ + err, + MAKELANGID(LANG_NEUTRAL, + SUBLANG_DEFAULT), /* Default language */ + (LPWSTR) &s_buf, + 0, /* size not used */ + NULL); /* no args */ + if (len==0) { + /* Only seen this in out of mem situations */ + message = PyUnicode_FromFormat("Windows Error 0x%X", err); + } else { + /* remove trailing cr/lf and dots */ + while (len > 0 && (s_buf[len-1] <= L' ' || s_buf[len-1] == L'.')) + s_buf[--len] = L'\0'; + message = PyUnicode_FromWideChar(s_buf, len); + } + if (message != NULL) + v = Py_BuildValue("(iO)", err, message); + else + v = NULL; + LocalFree(s_buf); + return v; +} +#else static PyObject *b_getwinerror(PyObject *self, PyObject *args) { int err = -1; @@ -127,6 +175,7 @@ LocalFree(s_buf); return v; } +#endif /************************************************************/ /* Emulate dlopen()&co. from the Windows API */ From noreply at buildbot.pypy.org Tue Nov 12 15:19:17 2013 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 12 Nov 2013 15:19:17 +0100 (CET) Subject: [pypy-commit] pypy default: Don't crash if no user group is called 'root'. Message-ID: <20131112141917.629EC1C147B@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r67982:c151e8359d3a Date: 2013-11-12 15:18 +0100 http://bitbucket.org/pypy/pypy/changeset/c151e8359d3a/ Log: Don't crash if no user group is called 'root'. diff --git a/pypy/module/test_lib_pypy/test_grp_extra.py b/pypy/module/test_lib_pypy/test_grp_extra.py --- a/pypy/module/test_lib_pypy/test_grp_extra.py +++ b/pypy/module/test_lib_pypy/test_grp_extra.py @@ -10,7 +10,11 @@ "No grp module on this platform") def test_basic(self): - g = self.grp.getgrnam("root") + raises(KeyError, self.grp.getgrnam, "dEkLofcG") + try: + g = self.grp.getgrnam("root") + except KeyError: + return # no 'root' group on OS/X? assert g.gr_gid == 0 assert g.gr_mem == ['root'] or g.gr_mem == [] assert g.gr_name == 'root' From noreply at buildbot.pypy.org Tue Nov 12 16:02:34 2013 From: noreply at buildbot.pypy.org (Raemi) Date: Tue, 12 Nov 2013 16:02:34 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c4: fix for threadlocal access. It was missing barriers and push_roots. Message-ID: <20131112150234.A67E21C036B@cobra.cs.uni-duesseldorf.de> Author: Remi Meier Branch: stmgc-c4 Changeset: r67983:73cb960f51c7 Date: 2013-11-12 16:01 +0100 http://bitbucket.org/pypy/pypy/changeset/73cb960f51c7/ Log: fix for threadlocal access. It was missing barriers and push_roots. diff --git a/TODO b/TODO --- a/TODO +++ b/TODO @@ -1,3 +1,9 @@ +------------------------------------------------------------ + +should stm_thread_local_obj always be read & writeable? would +a write-barrier in begin_transaction be too much for small +transactions? should we handle it specially (undolog?) + ------------------------------------------------------------ looking at trace of fibo.tlc (targettlc.py), there are a lot diff --git a/rpython/memory/gctransform/stmframework.py b/rpython/memory/gctransform/stmframework.py --- a/rpython/memory/gctransform/stmframework.py +++ b/rpython/memory/gctransform/stmframework.py @@ -104,6 +104,7 @@ gct_stm_perform_transaction = _gct_with_roots_pushed gct_stm_allocate_nonmovable_int_adr = _gct_with_roots_pushed gct_stm_inspect_abort_info = _gct_with_roots_pushed + gct_stm_threadlocalref_set = _gct_with_roots_pushed class StmRootWalker(BaseRootWalker): diff --git a/rpython/rtyper/lltypesystem/lloperation.py b/rpython/rtyper/lltypesystem/lloperation.py --- a/rpython/rtyper/lltypesystem/lloperation.py +++ b/rpython/rtyper/lltypesystem/lloperation.py @@ -440,7 +440,8 @@ 'stm_weakref_allocate': LLOp(sideeffects=False, canmallocgc=True), 'stm_threadlocalref_get': LLOp(sideeffects=False), - 'stm_threadlocalref_set': LLOp(), + 'stm_threadlocalref_set': LLOp(canmallocgc=True), # may allocate new array, + # see threadlocalref.py 'stm_threadlocal_get': LLOp(sideeffects=False), 'stm_threadlocal_set': LLOp(), diff --git a/rpython/translator/stm/funcgen.py b/rpython/translator/stm/funcgen.py --- a/rpython/translator/stm/funcgen.py +++ b/rpython/translator/stm/funcgen.py @@ -53,7 +53,14 @@ def stm_barrier(funcgen, op): category_change = op.args[0].value - frm, middle, to = category_change + # XXX: how to unify the stm_barrier llop generation in + # writebarrier.py and threadlocalref.py? + if isinstance(category_change, str): + frm, middle, to = category_change + else: # rstr + frm, middle, to = (category_change.chars[0], + category_change.chars[1], + category_change.chars[2]) assert middle == '2' assert frm < to if to == 'W': diff --git a/rpython/translator/stm/threadlocalref.py b/rpython/translator/stm/threadlocalref.py --- a/rpython/translator/stm/threadlocalref.py +++ b/rpython/translator/stm/threadlocalref.py @@ -28,13 +28,19 @@ if not array: return lltype.nullptr(rclass.OBJECTPTR.TO) else: + array = llop.stm_barrier(lltype.Ptr(ARRAY), 'A2R', array) return array[index] # def ll_threadlocalref_set(index, newvalue): array = llop.stm_threadlocal_get(lltype.Ptr(ARRAY)) if not array: - array = lltype.malloc(ARRAY, total) + array = lltype.malloc(ARRAY, total) # llop may allocate! llop.stm_threadlocal_set(lltype.Void, array) + else: + array = llop.stm_barrier(lltype.Ptr(ARRAY), 'A2W', array) + # invalidating other barriers after an llop.threadlocalref_set + # is not necessary since no other variable should contain + # a reference to stm_threadlocal_obj array[index] = newvalue # annhelper = annlowlevel.MixLevelHelperAnnotator(t.rtyper) From noreply at buildbot.pypy.org Tue Nov 12 16:13:27 2013 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 12 Nov 2013 16:13:27 +0100 (CET) Subject: [pypy-commit] cffi default: Test various combinations of calls. Mostly a libffi stress-test. Message-ID: <20131112151327.4A48D1C00F8@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r1416:be9745bbc509 Date: 2013-11-12 16:12 +0100 http://bitbucket.org/cffi/cffi/changeset/be9745bbc509/ Log: Test various combinations of calls. Mostly a libffi stress-test. diff --git a/testing/test_verify.py b/testing/test_verify.py --- a/testing/test_verify.py +++ b/testing/test_verify.py @@ -1767,3 +1767,87 @@ ffi.cdef("""const int a[];""") lib = ffi.verify("""const int a[5];""") assert repr(ffi.typeof(lib.a)) == "" + +def _test_various_calls(force_libffi): + cdef_source = """ + int xvalue; + long long ivalue, rvalue; + float fvalue; + double dvalue; + long double Dvalue; + signed char tf_bb(signed char x, signed char c); + unsigned char tf_bB(signed char x, unsigned char c); + short tf_bh(signed char x, short c); + unsigned short tf_bH(signed char x, unsigned short c); + int tf_bi(signed char x, int c); + unsigned int tf_bI(signed char x, unsigned int c); + long tf_bl(signed char x, long c); + unsigned long tf_bL(signed char x, unsigned long c); + long long tf_bq(signed char x, long long c); + float tf_bf(signed char x, float c); + double tf_bd(signed char x, double c); + long double tf_bD(signed char x, long double c); + """ + if force_libffi: + cdef_source = (cdef_source + .replace('tf_', '(*const tf_') + .replace('(signed char x', ')(signed char x')) + ffi = FFI() + ffi.cdef(cdef_source) + lib = ffi.verify(""" + int xvalue; + long long ivalue, rvalue; + float fvalue; + double dvalue; + long double Dvalue; + + #define S(letter) xvalue = x; letter##value = c; return rvalue; + + signed char tf_bb(signed char x, signed char c) { S(i) } + unsigned char tf_bB(signed char x, unsigned char c) { S(i) } + short tf_bh(signed char x, short c) { S(i) } + unsigned short tf_bH(signed char x, unsigned short c) { S(i) } + int tf_bi(signed char x, int c) { S(i) } + unsigned int tf_bI(signed char x, unsigned int c) { S(i) } + long tf_bl(signed char x, long c) { S(i) } + unsigned long tf_bL(signed char x, unsigned long c) { S(i) } + long long tf_bq(signed char x, long long c) { S(i) } + float tf_bf(signed char x, float c) { S(f) } + double tf_bd(signed char x, double c) { S(d) } + long double tf_bD(signed char x, long double c) { S(D) } + """) + lib.rvalue = 0x7182838485868788 + for kind, cname in [('b', 'signed char'), + ('B', 'unsigned char'), + ('h', 'short'), + ('H', 'unsigned short'), + ('i', 'int'), + ('I', 'unsigned int'), + ('l', 'long'), + ('L', 'unsigned long'), + ('q', 'long long'), + ('f', 'float'), + ('d', 'double'), + ('D', 'long double')]: + sign = +1 if 'unsigned' in cname else -1 + lib.xvalue = 0 + lib.ivalue = 0 + lib.fvalue = 0 + lib.dvalue = 0 + lib.Dvalue = 0 + fun = getattr(lib, 'tf_b' + kind) + res = fun(-42, sign * 99) + if kind == 'D': + res = float(res) + assert res == int(ffi.cast(cname, 0x7182838485868788)) + assert lib.xvalue == -42 + if kind in 'fdD': + assert float(getattr(lib, kind + 'value')) == -99.0 + else: + assert lib.ivalue == sign * 99 + +def test_various_calls_direct(): + _test_various_calls(force_libffi=False) + +def test_various_calls_libffi(): + _test_various_calls(force_libffi=True) From noreply at buildbot.pypy.org Tue Nov 12 18:29:27 2013 From: noreply at buildbot.pypy.org (bivab) Date: Tue, 12 Nov 2013 18:29:27 +0100 (CET) Subject: [pypy-commit] buildbot default: allow the BUILDJITLINUXARMHF_RARING builder to be forced Message-ID: <20131112172927.908D21C0F88@cobra.cs.uni-duesseldorf.de> Author: David Schneider Branch: Changeset: r889:2c32efb0fb5d Date: 2013-11-12 18:26 +0100 http://bitbucket.org/pypy/buildbot/changeset/2c32efb0fb5d/ Log: allow the BUILDJITLINUXARMHF_RARING builder to be forced diff --git a/bot2/pypybuildbot/arm_master.py b/bot2/pypybuildbot/arm_master.py --- a/bot2/pypybuildbot/arm_master.py +++ b/bot2/pypybuildbot/arm_master.py @@ -136,6 +136,7 @@ BUILDJITLINUXARM, BUILDLINUXARMHF_RASPBIAN, BUILDJITLINUXARMHF_RASPBIAN, + BUILDJITLINUXARMHF_RARING, ] schedulers = [ From noreply at buildbot.pypy.org Tue Nov 12 19:13:38 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Tue, 12 Nov 2013 19:13:38 +0100 (CET) Subject: [pypy-commit] pypy default: fix this test a bit more strictly Message-ID: <20131112181338.23D221C0206@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r67984:b1014f8d559c Date: 2013-11-12 10:12 -0800 http://bitbucket.org/pypy/pypy/changeset/b1014f8d559c/ Log: fix this test a bit more strictly diff --git a/pypy/module/test_lib_pypy/test_grp_extra.py b/pypy/module/test_lib_pypy/test_grp_extra.py --- a/pypy/module/test_lib_pypy/test_grp_extra.py +++ b/pypy/module/test_lib_pypy/test_grp_extra.py @@ -11,14 +11,18 @@ def test_basic(self): raises(KeyError, self.grp.getgrnam, "dEkLofcG") - try: - g = self.grp.getgrnam("root") - except KeyError: - return # no 'root' group on OS/X? - assert g.gr_gid == 0 - assert g.gr_mem == ['root'] or g.gr_mem == [] - assert g.gr_name == 'root' - assert isinstance(g.gr_passwd, str) # usually just 'x', don't hope :-) + for name in ["root", "wheel"]: + try: + g = self.grp.getgrnam(name) + except KeyError: + continue + assert g.gr_gid == 0 + assert g.gr_mem == ['root'] or g.gr_mem == [] + assert g.gr_name == name + assert isinstance(g.gr_passwd, str) # usually just 'x', don't hope :-) + break + else: + raise def test_extra(self): grp = self.grp From noreply at buildbot.pypy.org Tue Nov 12 19:19:58 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Tue, 12 Nov 2013 19:19:58 +0100 (CET) Subject: [pypy-commit] pypy default: update usemodules so it doesn't skip untranslated Message-ID: <20131112181958.7C3AA1C036B@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r67985:9c9132e23801 Date: 2013-11-12 13:19 -0500 http://bitbucket.org/pypy/pypy/changeset/9c9132e23801/ Log: update usemodules so it doesn't skip untranslated diff --git a/pypy/module/test_lib_pypy/test_grp_extra.py b/pypy/module/test_lib_pypy/test_grp_extra.py --- a/pypy/module/test_lib_pypy/test_grp_extra.py +++ b/pypy/module/test_lib_pypy/test_grp_extra.py @@ -2,8 +2,7 @@ class AppTestGrp: - - spaceconfig = dict(usemodules=('_ffi', '_rawffi', 'itertools')) + spaceconfig = dict(usemodules=('binascii', '_ffi', '_rawffi', 'itertools')) def setup_class(cls): cls.w_grp = import_lib_pypy(cls.space, 'grp', From noreply at buildbot.pypy.org Tue Nov 12 19:23:09 2013 From: noreply at buildbot.pypy.org (bivab) Date: Tue, 12 Nov 2013 19:23:09 +0100 (CET) Subject: [pypy-commit] pypy armhf-singlefloat: enable singlefloats on ARMHF Message-ID: <20131112182309.9FAA01C036B@cobra.cs.uni-duesseldorf.de> Author: David Schneider Branch: armhf-singlefloat Changeset: r67986:063d0dcda94f Date: 2013-11-12 09:46 -0600 http://bitbucket.org/pypy/pypy/changeset/063d0dcda94f/ Log: enable singlefloats on ARMHF diff --git a/rpython/jit/backend/arm/runner.py b/rpython/jit/backend/arm/runner.py --- a/rpython/jit/backend/arm/runner.py +++ b/rpython/jit/backend/arm/runner.py @@ -22,7 +22,7 @@ supports_floats = True supports_longlong = False # XXX requires an implementation of # read_timestamp that works in user mode - supports_singlefloats = not detect_hardfloat() + supports_singlefloats = True from rpython.jit.backend.arm.arch import JITFRAME_FIXED_SIZE all_reg_indexes = range(len(all_regs)) From noreply at buildbot.pypy.org Tue Nov 12 19:23:11 2013 From: noreply at buildbot.pypy.org (bivab) Date: Tue, 12 Nov 2013 19:23:11 +0100 (CET) Subject: [pypy-commit] pypy armhf-singlefloat: unused Message-ID: <20131112182311.38EEC1C036B@cobra.cs.uni-duesseldorf.de> Author: David Schneider Branch: armhf-singlefloat Changeset: r67987:41afcae95cfd Date: 2013-11-12 09:49 -0600 http://bitbucket.org/pypy/pypy/changeset/41afcae95cfd/ Log: unused diff --git a/rpython/jit/backend/arm/locations.py b/rpython/jit/backend/arm/locations.py --- a/rpython/jit/backend/arm/locations.py +++ b/rpython/jit/backend/arm/locations.py @@ -55,10 +55,6 @@ type = FLOAT width = 2 * WORD - def get_single_precision_regs(self): - return [VFPRegisterLocation(i) for i in - [self.value * 2, self.value * 2 + 1]] - def __repr__(self): return 'vfp%d' % self.value From noreply at buildbot.pypy.org Tue Nov 12 19:23:12 2013 From: noreply at buildbot.pypy.org (bivab) Date: Tue, 12 Nov 2013 19:23:12 +0100 (CET) Subject: [pypy-commit] pypy armhf-singlefloat: add single precission VFP register locations Message-ID: <20131112182312.973151C036B@cobra.cs.uni-duesseldorf.de> Author: David Schneider Branch: armhf-singlefloat Changeset: r67988:93c961fe1b79 Date: 2013-11-12 09:52 -0600 http://bitbucket.org/pypy/pypy/changeset/93c961fe1b79/ Log: add single precission VFP register locations diff --git a/rpython/jit/backend/arm/locations.py b/rpython/jit/backend/arm/locations.py --- a/rpython/jit/backend/arm/locations.py +++ b/rpython/jit/backend/arm/locations.py @@ -56,7 +56,7 @@ width = 2 * WORD def __repr__(self): - return 'vfp%d' % self.value + return 'vfp(d%d)' % self.value def is_core_reg(self): return False @@ -70,6 +70,14 @@ def is_float(self): return True +class SVFPRegisterLocation(VFPRegisterLocation): + """Single Precission VFP Register""" + _immutable_ = True + width = WORD + type = 'S' + + def __repr__(self): + return 'vfp(s%d)' % self.value class ImmLocation(AssemblerLocation): _immutable_ = True diff --git a/rpython/jit/backend/arm/registers.py b/rpython/jit/backend/arm/registers.py --- a/rpython/jit/backend/arm/registers.py +++ b/rpython/jit/backend/arm/registers.py @@ -1,8 +1,10 @@ from rpython.jit.backend.arm.locations import VFPRegisterLocation +from rpython.jit.backend.arm.locations import SVFPRegisterLocation from rpython.jit.backend.arm.locations import RegisterLocation registers = [RegisterLocation(i) for i in range(16)] vfpregisters = [VFPRegisterLocation(i) for i in range(16)] +svfpregisters = [SVFPRegisterLocation(i) for i in range(32)] [r0, r1, r2, r3, r4, r5, r6, r7, r8, r9, r10, r11, r12, r13, r14, r15] = registers @@ -10,6 +12,10 @@ [d0, d1, d2, d3, d4, d5, d6, d7, d8, d9, d10, d11, d12, d13, d14, d15] = vfpregisters +# single precission VFP registers, 32-bit +for i in range(32): + globals()['s%d' % i] = svfpregisters[i] + # aliases for registers fp = r11 ip = r12 @@ -27,6 +33,7 @@ callee_restored_registers = callee_resp + [pc] vfp_argument_regs = caller_vfp_resp = [d0, d1, d2, d3, d4, d5, d6, d7] +svfp_argument_regs = [globals()['s%i' % i] for i in range(16)] callee_vfp_resp = [d8, d9, d10, d11, d12, d13, d14, d15] callee_saved_vfp_registers = callee_vfp_resp From noreply at buildbot.pypy.org Tue Nov 12 19:23:14 2013 From: noreply at buildbot.pypy.org (bivab) Date: Tue, 12 Nov 2013 19:23:14 +0100 (CET) Subject: [pypy-commit] pypy armhf-singlefloat: add operations to move between core and single precision VFP registers Message-ID: <20131112182314.052781C0F88@cobra.cs.uni-duesseldorf.de> Author: David Schneider Branch: armhf-singlefloat Changeset: r67989:3c95419f55a7 Date: 2013-11-12 12:06 -0600 http://bitbucket.org/pypy/pypy/changeset/3c95419f55a7/ Log: add operations to move between core and single precision VFP registers diff --git a/rpython/jit/backend/arm/codebuilder.py b/rpython/jit/backend/arm/codebuilder.py --- a/rpython/jit/backend/arm/codebuilder.py +++ b/rpython/jit/backend/arm/codebuilder.py @@ -178,6 +178,30 @@ | (dm & 0xF)) self.write32(instr) + def VMOV_sc(self, dest, src): + """move a single precision vfp register[src] to a core reg[dest]""" + self._VMOV_32bit(src, dest, to_arm_register=1) + + def VMOV_cs(self, dest, src): + """move a core register[src] to a single precision vfp + register[dest]""" + self._VMOV_32bit(dest, src, to_arm_register=0) + + def _VMOV_32bit(self, float_reg, core_reg, to_arm_register, cond=cond.AL): + """This instruction transfers the contents of a single-precision VFP + register to an ARM core register, or the contents of an ARM core + register to a single-precision VFP register. + """ + instr = (cond << 28 + | 0xE << 24 + | to_arm_register << 20 + | ((float_reg >> 1) & 0xF) << 16 + | core_reg << 12 + | 0xA << 8 + | (float_reg & 0x1) << 7 + | 1 << 4) + self.write32(instr) + def VMOV_cc(self, dd, dm, cond=cond.AL): sz = 1 # for 64-bit mode instr = (cond << 28 From noreply at buildbot.pypy.org Tue Nov 12 19:23:15 2013 From: noreply at buildbot.pypy.org (bivab) Date: Tue, 12 Nov 2013 19:23:15 +0100 (CET) Subject: [pypy-commit] pypy armhf-singlefloat: handle singlefloats in callbuilder Message-ID: <20131112182315.76D6E1C036B@cobra.cs.uni-duesseldorf.de> Author: David Schneider Branch: armhf-singlefloat Changeset: r67990:7bc08fc09a9e Date: 2013-11-12 12:07 -0600 http://bitbucket.org/pypy/pypy/changeset/7bc08fc09a9e/ Log: handle singlefloats in callbuilder diff --git a/rpython/jit/backend/arm/callbuilder.py b/rpython/jit/backend/arm/callbuilder.py --- a/rpython/jit/backend/arm/callbuilder.py +++ b/rpython/jit/backend/arm/callbuilder.py @@ -227,20 +227,81 @@ class HardFloatCallBuilder(ARMCallbuilder): + next_arg_vfp = 0 + next_arg_svfp = 0 + + def get_next_vfp(self, tp): + assert tp in 'fS' + if self.next_arg_vfp == -1: + return None + if tp == 'S': + i = self.next_arg_svfp + next_vfp = (i >> 1) + 1 + if not (i + 1) & 1: # i is even + self.next_arg_vfp = max(self.next_arg_vfp, next_vfp) + self.next_arg_svfp = self.next_arg_vfp << 1 + else: + self.next_arg_svfp += 1 + self.next_arg_vfp = next_vfp + lst = r.svfp_argument_regs + else: # 64bit double + i = self.next_arg_vfp + self.next_arg_vfp += 1 + if self.next_arg_svfp >> 1 == i: + self.next_arg_svfp = self.next_arg_vfp << 1 + lst = r.vfp_argument_regs + try: + return lst[i] + except IndexError: + self.next_arg_vfp = self.next_arg_svfp = -1 + return None + def prepare_arguments(self): non_float_locs = [] non_float_regs = [] float_locs = [] float_regs = [] stack_args = [] + singlefloats = None arglocs = self.arglocs argtypes = self.argtypes count = 0 # stack alignment counter on_stack = 0 - for arg in arglocs: - if arg.type != FLOAT: + for i in range(len(arglocs)): + argtype = INT + if i < len(argtypes) and argtypes[i] == 'S': + argtype = argtypes[i] + arg = arglocs[i] + if arg.is_float(): + argtype = FLOAT + reg = self.get_next_vfp(argtype) + if reg: + assert len(float_regs) < len(r.vfp_argument_regs) + float_locs.append(arg) + assert reg not in float_regs + float_regs.append(reg) + else: # float argument that needs to go on the stack + if count % 2 != 0: + stack_args.append(None) + count = 0 + on_stack += 1 + stack_args.append(arg) + on_stack += 2 + elif argtype == 'S': + # Singlefloat argument + if singlefloats is None: + singlefloats = [] + tgt = self.get_next_vfp(argtype) + if tgt: + singlefloats.append((arg, tgt)) + else: # Singlefloat argument that needs to go on the stack + # treated the same as a regular core register argument + count += 1 + on_stack += 1 + stack_args.append(arg) + else: if len(non_float_regs) < len(r.argument_regs): reg = r.argument_regs[len(non_float_regs)] non_float_locs.append(arg) @@ -249,18 +310,6 @@ count += 1 on_stack += 1 stack_args.append(arg) - else: - if len(float_regs) < len(r.vfp_argument_regs): - reg = r.vfp_argument_regs[len(float_regs)] - float_locs.append(arg) - float_regs.append(reg) - else: # float argument that needs to go on the stack - if count % 2 != 0: - stack_args.append(None) - count = 0 - on_stack += 1 - stack_args.append(arg) - on_stack += 2 # align the stack if count % 2 != 0: stack_args.append(None) @@ -275,13 +324,28 @@ non_float_locs.append(self.fnloc) non_float_regs.append(r.r4) self.fnloc = r.r4 + # remap values stored in vfp registers + remap_frame_layout(self.asm, float_locs, float_regs, r.vfp_ip) + if singlefloats: + for src, dest in singlefloats: + if src.is_float(): + assert 0, 'unsupported case' + if src.is_stack(): + # use special VLDR for 32bit + self.asm.regalloc_mov(src, r.ip) + src = r.ip + if src.is_imm(): + self.mc.gen_load_int(r.ip.value, src.value) + src = r.ip + if src.is_core_reg(): + self.mc.VMOV_cs(dest.value, src.value) # remap values stored in core registers remap_frame_layout(self.asm, non_float_locs, non_float_regs, r.ip) - # remap values stored in vfp registers - remap_frame_layout(self.asm, float_locs, float_regs, r.vfp_ip) def load_result(self): resloc = self.resloc + if self.restype == 'S': + self.mc.VMOV_sc(resloc.value, r.s0.value) # ensure the result is wellformed and stored in the correct location if resloc is not None and resloc.is_core_reg(): self._ensure_result_bit_extension(resloc, From noreply at buildbot.pypy.org Tue Nov 12 21:32:32 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Tue, 12 Nov 2013 21:32:32 +0100 (CET) Subject: [pypy-commit] pypy default: clean up some numpypy references Message-ID: <20131112203232.D25BE1C13FC@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r67991:e85c7e78f011 Date: 2013-11-12 15:25 -0500 http://bitbucket.org/pypy/pypy/changeset/e85c7e78f011/ Log: clean up some numpypy references diff --git a/pypy/module/micronumpy/interp_dtype.py b/pypy/module/micronumpy/interp_dtype.py --- a/pypy/module/micronumpy/interp_dtype.py +++ b/pypy/module/micronumpy/interp_dtype.py @@ -401,7 +401,7 @@ raise operationerrfmt(space.w_TypeError, msg, w_dtype) W_Dtype.typedef = TypeDef("dtype", - __module__ = "numpypy", + __module__ = "numpy", __new__ = interp2app(descr__new__), __str__= interp2app(W_Dtype.descr_str), diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py --- a/pypy/module/micronumpy/interp_numarray.py +++ b/pypy/module/micronumpy/interp_numarray.py @@ -338,11 +338,11 @@ Returns an array containing the same data with a new shape. - Refer to `numpypy.reshape` for full documentation. + Refer to `numpy.reshape` for full documentation. See Also -------- - numpypy.reshape : equivalent function + numpy.reshape : equivalent function """ args_w, kw_w = __args__.unpack() order = NPY_CORDER @@ -1123,9 +1123,8 @@ return res """, filename=__file__).interphook('ptp') -W_NDimArray.typedef = TypeDef( - "ndarray", - __module__ = "numpypy", +W_NDimArray.typedef = TypeDef("ndarray", + __module__ = "numpy", __new__ = interp2app(descr_new_array), __len__ = interp2app(W_NDimArray.descr_len), @@ -1391,8 +1390,8 @@ return box -W_FlatIterator.typedef = TypeDef( - 'flatiter', +W_FlatIterator.typedef = TypeDef("flatiter", + __module__ = "numpy", __iter__ = interp2app(W_FlatIterator.descr_iter), __getitem__ = interp2app(W_FlatIterator.descr_getitem), __setitem__ = interp2app(W_FlatIterator.descr_setitem), diff --git a/pypy/module/micronumpy/interp_ufuncs.py b/pypy/module/micronumpy/interp_ufuncs.py --- a/pypy/module/micronumpy/interp_ufuncs.py +++ b/pypy/module/micronumpy/interp_ufuncs.py @@ -413,7 +413,7 @@ W_Ufunc.typedef = TypeDef("ufunc", - __module__ = "numpypy", + __module__ = "numpy", __call__ = interp2app(W_Ufunc.descr_call), __repr__ = interp2app(W_Ufunc.descr_repr), diff --git a/pypy/module/micronumpy/test/test_dtypes.py b/pypy/module/micronumpy/test/test_dtypes.py --- a/pypy/module/micronumpy/test/test_dtypes.py +++ b/pypy/module/micronumpy/test/test_dtypes.py @@ -26,6 +26,7 @@ assert d.kind == 'b' assert dtype(d) is d assert dtype('bool') is d + assert repr(type(d)) == "" assert dtype('int8').num == 1 assert dtype('int8').name == 'int8' diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py --- a/pypy/module/micronumpy/test/test_numarray.py +++ b/pypy/module/micronumpy/test/test_numarray.py @@ -249,9 +249,11 @@ return CustomIntObject(value) def test_ndarray(self): - from numpypy import ndarray, array, dtype + from numpy import ndarray, array, dtype, flatiter assert type(ndarray) is type + assert repr(ndarray) == "" + assert repr(flatiter) == "" assert type(array) is not type a = ndarray((2, 3)) assert a.shape == (2, 3) diff --git a/pypy/module/micronumpy/test/test_ufuncs.py b/pypy/module/micronumpy/test/test_ufuncs.py --- a/pypy/module/micronumpy/test/test_ufuncs.py +++ b/pypy/module/micronumpy/test/test_ufuncs.py @@ -81,7 +81,7 @@ assert isinstance(add, ufunc) assert repr(add) == "" - assert repr(ufunc) == "" or repr(ufunc) == "" + assert repr(ufunc) == "" def test_ufunc_attrs(self): from numpypy import add, multiply, sin From noreply at buildbot.pypy.org Tue Nov 12 21:50:13 2013 From: noreply at buildbot.pypy.org (bivab) Date: Tue, 12 Nov 2013 21:50:13 +0100 (CET) Subject: [pypy-commit] pypy armhf-singlefloat: use single precision operations for casts to and from float Message-ID: <20131112205013.C78371C1413@cobra.cs.uni-duesseldorf.de> Author: David Schneider Branch: armhf-singlefloat Changeset: r67992:d8b1b0c57e72 Date: 2013-11-12 14:41 -0600 http://bitbucket.org/pypy/pypy/changeset/d8b1b0c57e72/ Log: use single precision operations for casts to and from float diff --git a/rpython/jit/backend/arm/codebuilder.py b/rpython/jit/backend/arm/codebuilder.py --- a/rpython/jit/backend/arm/codebuilder.py +++ b/rpython/jit/backend/arm/codebuilder.py @@ -222,8 +222,16 @@ self._VCVT(target, source, cond, 0, 1) def _VCVT(self, target, source, cond, opc2, sz): - D = 0 - M = 0 + # A8.6.295 + to_integer = (opc2 >> 2) & 1 + if to_integer: + D = target & 1 + target >>= 1 + M = (source >> 4) & 1 + else: + M = source & 1 + source >>= 1 + D = (target >> 4) & 1 op = 1 instr = (cond << 28 | 0xEB8 << 16 @@ -240,8 +248,8 @@ def _VCVT_single_double(self, target, source, cond, sz): # double_to_single = (sz == '1'); - D = 0 - M = 0 + D = target & 1 if sz else (target >> 4) & 1 + M = (source >> 4) & 1 if sz else source & 1 instr = (cond << 28 | 0xEB7 << 16 | 0xAC << 4 diff --git a/rpython/jit/backend/arm/opassembler.py b/rpython/jit/backend/arm/opassembler.py --- a/rpython/jit/backend/arm/opassembler.py +++ b/rpython/jit/backend/arm/opassembler.py @@ -1102,17 +1102,16 @@ arg, res = arglocs assert arg.is_vfp_reg() assert res.is_core_reg() - self.mc.VCVT_float_to_int(r.vfp_ip.value, arg.value) - self.mc.VMOV_rc(res.value, r.ip.value, r.vfp_ip.value) + self.mc.VCVT_float_to_int(r.svfp_ip.value, arg.value) + self.mc.VMOV_sc(res.value, r.svfp_ip.value) return fcond def emit_op_cast_int_to_float(self, op, arglocs, regalloc, fcond): arg, res = arglocs assert res.is_vfp_reg() assert arg.is_core_reg() - self.mc.MOV_ri(r.ip.value, 0) - self.mc.VMOV_cr(res.value, arg.value, r.ip.value) - self.mc.VCVT_int_to_float(res.value, res.value) + self.mc.VMOV_cs(r.svfp_ip.value, arg.value) + self.mc.VCVT_int_to_float(res.value, r.svfp_ip.value) return fcond emit_op_llong_add = gen_emit_float_op('llong_add', 'VADD_i64') @@ -1147,15 +1146,14 @@ arg, res = arglocs assert arg.is_vfp_reg() assert res.is_core_reg() - self.mc.VCVT_f64_f32(r.vfp_ip.value, arg.value) - self.mc.VMOV_rc(res.value, r.ip.value, r.vfp_ip.value) + self.mc.VCVT_f64_f32(r.svfp_ip.value, arg.value) + self.mc.VMOV_sc(res.value, r.svfp_ip.value) return fcond def emit_op_cast_singlefloat_to_float(self, op, arglocs, regalloc, fcond): arg, res = arglocs assert res.is_vfp_reg() assert arg.is_core_reg() - self.mc.MOV_ri(r.ip.value, 0) - self.mc.VMOV_cr(res.value, arg.value, r.ip.value) - self.mc.VCVT_f32_f64(res.value, res.value) + self.mc.VMOV_cs(r.svfp_ip.value, arg.value) + self.mc.VCVT_f32_f64(res.value, r.svfp_ip.value) return fcond diff --git a/rpython/jit/backend/arm/registers.py b/rpython/jit/backend/arm/registers.py --- a/rpython/jit/backend/arm/registers.py +++ b/rpython/jit/backend/arm/registers.py @@ -23,6 +23,7 @@ lr = r14 pc = r15 vfp_ip = d15 +svfp_ip = s31 all_regs = [r0, r1, r2, r3, r4, r5, r6, r7, r8, r9, r10] all_vfp_regs = vfpregisters[:-1] From noreply at buildbot.pypy.org Tue Nov 12 22:43:17 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Tue, 12 Nov 2013 22:43:17 +0100 (CET) Subject: [pypy-commit] pypy default: add basic ndarray.flags object Message-ID: <20131112214317.81BBB1C0206@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r67993:49ca29878cd3 Date: 2013-11-12 16:24 -0500 http://bitbucket.org/pypy/pypy/changeset/49ca29878cd3/ Log: add basic ndarray.flags object diff --git a/pypy/module/micronumpy/interp_flagsobj.py b/pypy/module/micronumpy/interp_flagsobj.py new file mode 100644 --- /dev/null +++ b/pypy/module/micronumpy/interp_flagsobj.py @@ -0,0 +1,45 @@ +from pypy.interpreter.baseobjspace import W_Root +from pypy.interpreter.typedef import TypeDef, GetSetProperty +from pypy.interpreter.gateway import interp2app +from pypy.interpreter.error import OperationError + + +class W_FlagsObject(W_Root): + def __init__(self, arr): + self.arr = arr + + def descr_get_contiguous(self, space): + return space.w_True + + def descr_get_fortran(self, space): + return space.w_False + + def descr_get_writeable(self, space): + return space.w_True + + def descr_getitem(self, space, w_item): + key = space.str_w(w_item) + if key == "C" or key == "CONTIGUOUS" or key == "C_CONTIGUOUS": + return self.descr_get_contiguous(space) + if key == "F" or key == "FORTRAN" or key == "F_CONTIGUOUS": + return self.descr_get_fortran(space) + if key == "W" or key == "WRITEABLE": + return self.descr_get_writeable(space) + raise OperationError(space.w_KeyError, space.wrap( + "Unknown flag")) + + def descr_setitem(self, space, w_item, w_value): + raise OperationError(space.w_KeyError, space.wrap( + "Unknown flag")) + +W_FlagsObject.typedef = TypeDef("flagsobj", + __module__ = "numpy", + __getitem__ = interp2app(W_FlagsObject.descr_getitem), + __setitem__ = interp2app(W_FlagsObject.descr_setitem), + + contiguous = GetSetProperty(W_FlagsObject.descr_get_contiguous), + c_contiguous = GetSetProperty(W_FlagsObject.descr_get_contiguous), + f_contiguous = GetSetProperty(W_FlagsObject.descr_get_fortran), + fortran = GetSetProperty(W_FlagsObject.descr_get_fortran), + writeable = GetSetProperty(W_FlagsObject.descr_get_writeable), +) diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py --- a/pypy/module/micronumpy/interp_numarray.py +++ b/pypy/module/micronumpy/interp_numarray.py @@ -9,6 +9,7 @@ from pypy.module.micronumpy.strides import find_shape_and_elems,\ get_shape_from_iterable, to_coords, shape_agreement, \ shape_agreement_multiple +from pypy.module.micronumpy.interp_flagsobj import W_FlagsObject from pypy.module.micronumpy.interp_flatiter import W_FlatIterator from pypy.module.micronumpy.appbridge import get_appbridge_cache from pypy.module.micronumpy import loop @@ -610,13 +611,11 @@ raise OperationError(space.w_NotImplementedError, space.wrap( "dumps not implemented yet")) + w_flags = None def descr_get_flags(self, space): - raise OperationError(space.w_NotImplementedError, space.wrap( - "getting flags not implemented yet")) - - def descr_set_flags(self, space, w_args): - raise OperationError(space.w_NotImplementedError, space.wrap( - "setting flags not implemented yet")) + if self.w_flags is None: + self.w_flags = W_FlagsObject(self) + return self.w_flags @unwrap_spec(offset=int) def descr_getfield(self, space, w_dtype, offset): @@ -1203,6 +1202,7 @@ size = GetSetProperty(W_NDimArray.descr_get_size), itemsize = GetSetProperty(W_NDimArray.descr_get_itemsize), nbytes = GetSetProperty(W_NDimArray.descr_get_nbytes), + flags = GetSetProperty(W_NDimArray.descr_get_flags), fill = interp2app(W_NDimArray.descr_fill), tostring = interp2app(W_NDimArray.descr_tostring), diff --git a/pypy/module/micronumpy/test/test_flagsobj.py b/pypy/module/micronumpy/test/test_flagsobj.py new file mode 100644 --- /dev/null +++ b/pypy/module/micronumpy/test/test_flagsobj.py @@ -0,0 +1,17 @@ +from pypy.module.micronumpy.test.test_base import BaseNumpyAppTest + + +class AppTestFlagsObj(BaseNumpyAppTest): + def test_repr(self): + import numpy as np + a = np.array([1,2,3]) + assert repr(type(a.flags)) == "" + + def test_flags(self): + import numpy as np + a = np.array([1,2,3]) + assert a.flags.c_contiguous == True + assert a.flags['W'] == True + raises(KeyError, "a.flags['blah']") + raises(KeyError, "a.flags['C_CONTIGUOUS'] = False") + raises((TypeError, AttributeError), "a.flags.c_contiguous = False") From noreply at buildbot.pypy.org Tue Nov 12 22:43:18 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Tue, 12 Nov 2013 22:43:18 +0100 (CET) Subject: [pypy-commit] pypy default: provide flags for scalars also Message-ID: <20131112214318.9FFCC1C0206@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r67994:592629fa85f2 Date: 2013-11-12 16:29 -0500 http://bitbucket.org/pypy/pypy/changeset/592629fa85f2/ Log: provide flags for scalars also diff --git a/pypy/module/micronumpy/interp_boxes.py b/pypy/module/micronumpy/interp_boxes.py --- a/pypy/module/micronumpy/interp_boxes.py +++ b/pypy/module/micronumpy/interp_boxes.py @@ -11,6 +11,7 @@ from rpython.rtyper.lltypesystem import rffi from rpython.tool.sourcetools import func_with_new_name from pypy.module.micronumpy.arrayimpl.voidbox import VoidBoxStorage +from pypy.module.micronumpy.interp_flagsobj import W_FlagsObject from pypy.interpreter.mixedmodule import MixedModule from rpython.rtyper.lltypesystem import lltype from rpython.rlib.rstring import StringBuilder @@ -113,7 +114,7 @@ class W_GenericBox(W_Root): - _attrs_ = [] + _attrs_ = ['w_flags'] def descr__new__(space, w_subtype, __args__): raise operationerrfmt(space.w_TypeError, @@ -292,6 +293,12 @@ def descr_copy(self, space): return self.convert_to(self.get_dtype(space)) + w_flags = None + def descr_get_flags(self, space): + if self.w_flags is None: + self.w_flags = W_FlagsObject(self) + return self.w_flags + class W_BoolBox(W_GenericBox, PrimitiveBox): descr__new__, _get_dtype, descr_reduce = new_dtype_getter("bool") @@ -550,6 +557,7 @@ strides = GetSetProperty(W_GenericBox.descr_get_shape), ndim = GetSetProperty(W_GenericBox.descr_get_ndim), T = GetSetProperty(W_GenericBox.descr_self), + flags = GetSetProperty(W_GenericBox.descr_get_flags), ) W_BoolBox.typedef = TypeDef("bool_", W_GenericBox.typedef, diff --git a/pypy/module/micronumpy/test/test_flagsobj.py b/pypy/module/micronumpy/test/test_flagsobj.py --- a/pypy/module/micronumpy/test/test_flagsobj.py +++ b/pypy/module/micronumpy/test/test_flagsobj.py @@ -7,7 +7,7 @@ a = np.array([1,2,3]) assert repr(type(a.flags)) == "" - def test_flags(self): + def test_array_flags(self): import numpy as np a = np.array([1,2,3]) assert a.flags.c_contiguous == True @@ -15,3 +15,8 @@ raises(KeyError, "a.flags['blah']") raises(KeyError, "a.flags['C_CONTIGUOUS'] = False") raises((TypeError, AttributeError), "a.flags.c_contiguous = False") + + def test_scalar_flags(self): + import numpy as np + a = np.int32(2) + assert a.flags.c_contiguous == True From noreply at buildbot.pypy.org Tue Nov 12 23:44:54 2013 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 12 Nov 2013 23:44:54 +0100 (CET) Subject: [pypy-commit] pypy default: Write 'ConstPtr(null)' when we know the constant is null. Message-ID: <20131112224454.E79F11C00F8@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r67995:4d9da705ae8f Date: 2013-11-12 23:44 +0100 http://bitbucket.org/pypy/pypy/changeset/4d9da705ae8f/ Log: Write 'ConstPtr(null)' when we know the constant is null. May break some tests in pypyjit/test_pypy_c. diff --git a/rpython/jit/metainterp/logger.py b/rpython/jit/metainterp/logger.py --- a/rpython/jit/metainterp/logger.py +++ b/rpython/jit/metainterp/logger.py @@ -103,7 +103,9 @@ elif isinstance(arg, BoxInt): return 'i' + str(mv) elif isinstance(arg, self.ts.ConstRef): - return 'ConstPtr(ptr' + str(mv) + ')' + if arg.value: + return 'ConstPtr(ptr' + str(mv) + ')' + return 'ConstPtr(null)' elif isinstance(arg, self.ts.BoxRef): return 'p' + str(mv) elif isinstance(arg, ConstFloat): From noreply at buildbot.pypy.org Tue Nov 12 23:46:07 2013 From: noreply at buildbot.pypy.org (mattip) Date: Tue, 12 Nov 2013 23:46:07 +0100 (CET) Subject: [pypy-commit] pypy windows-packaging: add dlls for Tkinter Message-ID: <20131112224607.730331C00F8@cobra.cs.uni-duesseldorf.de> Author: Matti Picus Branch: windows-packaging Changeset: r67996:9b968361f205 Date: 2013-11-13 00:41 +0200 http://bitbucket.org/pypy/pypy/changeset/9b968361f205/ Log: add dlls for Tkinter diff --git a/lib_pypy/_tkinter/tklib.py b/lib_pypy/_tkinter/tklib.py --- a/lib_pypy/_tkinter/tklib.py +++ b/lib_pypy/_tkinter/tklib.py @@ -112,6 +112,10 @@ incdirs = ['/usr/local/include/tcl8.5', '/usr/local/include/tk8.5', '/usr/X11R6/include'] linklibs = ['tk85', 'tcl85'] libdirs = ['/usr/local/lib', '/usr/X11R6/lib'] +elif sys.platform == 'win32': + incdirs = [] + linklibs = ['tcl85', 'tk85'] + libdirs = [] else: incdirs=['/usr/include/tcl'] linklibs=['tcl', 'tk'] diff --git a/pypy/doc/windows.rst b/pypy/doc/windows.rst --- a/pypy/doc/windows.rst +++ b/pypy/doc/windows.rst @@ -157,6 +157,21 @@ ms\do_ms.bat nmake -f ms\nt.mak install +TkInter module support +~~~~~~~~~~~~~~~~~~~~~~ + +Download tcl from http:://http://www.tcl.tk/software/tcltk/download.html +Extract tcl85. cd into the ``win`` directory and compile:: + + nmake -nologo -f makefile.vc release OPTS=symbols,threads + +Download tcl from http:://http://www.tcl.tk/software/tcltk/download.html +Extract tcl85. cd into the ``win`` directory and compile:: + + set TCLDIR= + nmake -nologo -f makefile.vc release OPTS=symbols,threads + + Using the mingw compiler ------------------------ diff --git a/pypy/tool/release/package.py b/pypy/tool/release/package.py --- a/pypy/tool/release/package.py +++ b/pypy/tool/release/package.py @@ -67,18 +67,22 @@ raise PyPyCNotFound( 'Bogus path: %r does not exist (see docstring for more info)' % (os.path.dirname(str(pypy_c)),)) + win_extras = ['libpypy-c.dll', 'libexpat.dll', 'sqlite3.dll', + 'libeay32.dll', 'ssleay32.dll'] subprocess.check_call([str(pypy_c), '-c', 'import _sqlite3']) if not sys.platform == 'win32': subprocess.check_call([str(pypy_c), '-c', 'import _curses']) subprocess.check_call([str(pypy_c), '-c', 'import syslog']) - if not withouttk: - try: - subprocess.check_call([str(pypy_c), '-c', 'import _tkinter']) - except subprocess.CalledProcessError: - print >>sys.stderr, """Building Tk bindings failed. + if not withouttk: + try: + subprocess.check_call([str(pypy_c), '-c', 'import _tkinter']) + except subprocess.CalledProcessError: + print >>sys.stderr, """Building Tk bindings failed. You can either install Tk development headers package or add --without-tk option to skip packaging binary CFFI extension.""" - sys.exit(1) + sys.exit(1) + #Can the dependencies be found from cffi somehow? + win_extras += ['tk85t.dll', 'tk85.dll', 'tcl85t.dll', 'tcl85.dll'] if sys.platform == 'win32' and not rename_pypy_c.lower().endswith('.exe'): rename_pypy_c += '.exe' binaries = [(pypy_c, rename_pypy_c)] @@ -101,9 +105,7 @@ # Can't rename a DLL: it is always called 'libpypy-c.dll' - for extra in ['libpypy-c.dll', - 'libexpat.dll', 'sqlite3.dll', - 'libeay32.dll', 'ssleay32.dll']: + for extra in win_extras: p = pypy_c.dirpath().join(extra) if not p.check(): p = py.path.local.sysfind(extra) From noreply at buildbot.pypy.org Wed Nov 13 00:07:37 2013 From: noreply at buildbot.pypy.org (pjenvey) Date: Wed, 13 Nov 2013 00:07:37 +0100 (CET) Subject: [pypy-commit] extradoc extradoc: py3k update #12 Message-ID: <20131112230737.20B981C00F8@cobra.cs.uni-duesseldorf.de> Author: Philip Jenvey Branch: extradoc Changeset: r5109:ea4e00abbf96 Date: 2013-11-12 15:06 -0800 http://bitbucket.org/pypy/extradoc/changeset/ea4e00abbf96/ Log: py3k update #12 diff --git a/blog/draft/py3k-status-update-12.rst b/blog/draft/py3k-status-update-12.rst new file mode 100644 --- /dev/null +++ b/blog/draft/py3k-status-update-12.rst @@ -0,0 +1,45 @@ +Py3k status update #12 +---------------------- + +This is the 12th status update about our work on the `py3k branch`_, which we +can work on thanks to all of the people who donated_ to the `py3k proposal`_. + +Here's an update on the recent progress: + +* Thank you to everyone who has provided initial feedback on the PyPy3 2.1 beta + 1 release. We've gotten a number of bug reports, most of which have been + fixed. + +* As usual, we're continually keeping up with changes from the default + branch. Oftentimes these merges come at a cost (conflicts and or + reintegration of py3k changes) but occasionally we get goodies for free, such + as the `recent JIT optimizations`_ and `incremental garbage collection`_. + +* We've been focusing on re-optimizing Python 2 int sized (machine sized) + integers: + +We have a couple of known, notable speed regressions in the PyPy3 beta release +vs regular PyPy. The major one being with Python 2.x int sized (or machine +sized) integers. + +Python 3 drops the distinction between int and long types. CPython 3.x +accomplishes this by removing the old int type entirely and renaming the long +type to int. Initially, we've done the same for PyPy3 for the sake of +simplicity and getting everything working. + +However PyPy's JIT is capable of heavily optimizing these machine sized integer +operations, so this came with a regression in performance in this area. + +We're now in the process of solving this. Part of this work also involves some +house cleaning on these numeric types which will also benefit the default +branch. + +cheers, +Phil + +.. _donated: http://morepypy.blogspot.com/2012/01/py3k-and-numpy-first-stage-thanks-to.html +.. _`py3k proposal`: http://pypy.org/py3donate.html +.. _`py3k branch`: https://bitbucket.org/pypy/pypy/commits/all/tip/branch%28%22py3k%22%29 + +.. _`recent JIT optimizations`: http://morepypy.blogspot.com/2013/10/making-coveragepy-faster-under-pypy.html +.. _`incremental garbage collection`: http://morepypy.blogspot.com/2013/10/incremental-garbage-collector-in-pypy.html From noreply at buildbot.pypy.org Wed Nov 13 01:12:37 2013 From: noreply at buildbot.pypy.org (mattip) Date: Wed, 13 Nov 2013 01:12:37 +0100 (CET) Subject: [pypy-commit] pypy default: document sqlite3.dll version for windows Message-ID: <20131113001237.988221C00F8@cobra.cs.uni-duesseldorf.de> Author: Matti Picus Branch: Changeset: r67997:df0777fb0d9f Date: 2013-11-13 02:10 +0200 http://bitbucket.org/pypy/pypy/changeset/df0777fb0d9f/ Log: document sqlite3.dll version for windows diff --git a/pypy/doc/windows.rst b/pypy/doc/windows.rst --- a/pypy/doc/windows.rst +++ b/pypy/doc/windows.rst @@ -118,14 +118,10 @@ The sqlite3 database library ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Download http://www.sqlite.org/2013/sqlite-amalgamation-3071601.zip and extract -it into a directory under the base directory. Also get -http://www.sqlite.org/2013/sqlite-dll-win32-x86-3071601.zip and extract the dll -into the bin directory, and the sqlite3.def into the sources directory. -Now build the import library so cffi can use the header and dll:: +PyPy uses cffi to interact with sqlite3.dll. Only the dll is needed, the cffi +wrapper is compiled when the module is imported for the first time. +The sqlite3.dll should be version 3.6.21 for CPython2.7 compatablility. - lib /DEF:sqlite3.def" /OUT:sqlite3.lib" - copy sqlite3.lib path\to\libs The expat XML parser From noreply at buildbot.pypy.org Wed Nov 13 01:56:02 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Wed, 13 Nov 2013 01:56:02 +0100 (CET) Subject: [pypy-commit] pypy default: clean up some formatting Message-ID: <20131113005602.EB8FC1C0F88@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r67998:bda39936972c Date: 2013-11-12 19:30 -0500 http://bitbucket.org/pypy/pypy/changeset/bda39936972c/ Log: clean up some formatting diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py --- a/pypy/module/micronumpy/interp_numarray.py +++ b/pypy/module/micronumpy/interp_numarray.py @@ -98,11 +98,11 @@ def getitem_filter(self, space, arr): if len(arr.get_shape()) > 1 and arr.get_shape() != self.get_shape(): - raise OperationError(space.w_ValueError, - space.wrap("boolean index array should have 1 dimension")) + raise OperationError(space.w_ValueError, space.wrap( + "boolean index array should have 1 dimension")) if arr.get_size() > self.get_size(): - raise OperationError(space.w_ValueError, - space.wrap("index out of range for array")) + raise OperationError(space.w_ValueError, space.wrap( + "index out of range for array")) size = loop.count_all_true(arr) if len(arr.get_shape()) == 1: res_shape = [size] + self.get_shape()[1:] @@ -113,17 +113,18 @@ def setitem_filter(self, space, idx, val): if len(idx.get_shape()) > 1 and idx.get_shape() != self.get_shape(): - raise OperationError(space.w_ValueError, - space.wrap("boolean index array should have 1 dimension")) + raise OperationError(space.w_ValueError, space.wrap( + "boolean index array should have 1 dimension")) if idx.get_size() > self.get_size(): - raise OperationError(space.w_ValueError, - space.wrap("index out of range for array")) + raise OperationError(space.w_ValueError, space.wrap( + "index out of range for array")) size = loop.count_all_true(idx) if size > val.get_size() and val.get_size() != 1: - raise OperationError(space.w_ValueError, space.wrap("NumPy boolean array indexing assignment " - "cannot assign %d input values to " - "the %d output values where the mask is true" % - (val.get_size(), size))) + raise OperationError(space.w_ValueError, space.wrap( + "NumPy boolean array indexing assignment " + "cannot assign %d input values to " + "the %d output values where the mask is true" % + (val.get_size(), size))) loop.setitem_filter(self, idx, val, size) def _prepare_array_index(self, space, w_index): From noreply at buildbot.pypy.org Wed Nov 13 01:56:04 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Wed, 13 Nov 2013 01:56:04 +0100 (CET) Subject: [pypy-commit] pypy default: fix missing coerce in setitem_filter Message-ID: <20131113005604.588011C0F88@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r67999:63c04f0fba9f Date: 2013-11-12 19:42 -0500 http://bitbucket.org/pypy/pypy/changeset/63c04f0fba9f/ Log: fix missing coerce in setitem_filter diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py --- a/pypy/module/micronumpy/interp_numarray.py +++ b/pypy/module/micronumpy/interp_numarray.py @@ -125,7 +125,7 @@ "cannot assign %d input values to " "the %d output values where the mask is true" % (val.get_size(), size))) - loop.setitem_filter(self, idx, val, size) + loop.setitem_filter(space, self, idx, val, size) def _prepare_array_index(self, space, w_index): if isinstance(w_index, W_NDimArray): diff --git a/pypy/module/micronumpy/loop.py b/pypy/module/micronumpy/loop.py --- a/pypy/module/micronumpy/loop.py +++ b/pypy/module/micronumpy/loop.py @@ -398,7 +398,7 @@ 'index_dtype'], reds = 'auto') -def setitem_filter(arr, index, value, size): +def setitem_filter(space, arr, index, value, size): arr_iter = arr.create_iter() shapelen = len(arr.get_shape()) if shapelen > 1 and len(index.get_shape()) < 2: @@ -414,7 +414,7 @@ arr_dtype=arr_dtype, ) if index_iter.getitem_bool(): - arr_iter.setitem(value_iter.getitem()) + arr_iter.setitem(arr_dtype.coerce(space, value_iter.getitem())) value_iter.next() arr_iter.next() index_iter.next() diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py --- a/pypy/module/micronumpy/test/test_numarray.py +++ b/pypy/module/micronumpy/test/test_numarray.py @@ -511,6 +511,13 @@ for i in xrange(5): assert a[i] == i + def test_setitem_array(self): + import numpy as np + a = np.array((-1., 0, 1))/0. + b = np.array([False, False, True], dtype=bool) + a[b] = 100 + assert a[2] == 100 + def test_setitem_obj_index(self): from numpypy import arange a = arange(10) From noreply at buildbot.pypy.org Wed Nov 13 02:28:37 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Wed, 13 Nov 2013 02:28:37 +0100 (CET) Subject: [pypy-commit] pypy default: raise coerce conversion errors at app-level Message-ID: <20131113012837.D07581C0206@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r68000:d50d27c01818 Date: 2013-11-12 20:20 -0500 http://bitbucket.org/pypy/pypy/changeset/d50d27c01818/ Log: raise coerce conversion errors at app-level diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py --- a/pypy/module/micronumpy/interp_numarray.py +++ b/pypy/module/micronumpy/interp_numarray.py @@ -198,8 +198,7 @@ prefix) def descr_getitem(self, space, w_idx): - if (isinstance(w_idx, W_NDimArray) and - w_idx.get_dtype().is_bool_type()): + if isinstance(w_idx, W_NDimArray) and w_idx.get_dtype().is_bool_type(): return self.getitem_filter(space, w_idx) try: return self.implementation.descr_getitem(space, self, w_idx) @@ -213,9 +212,11 @@ self.implementation.setitem_index(space, index_list, w_value) def descr_setitem(self, space, w_idx, w_value): - if (isinstance(w_idx, W_NDimArray) and - w_idx.get_dtype().is_bool_type()): - self.setitem_filter(space, w_idx, convert_to_array(space, w_value)) + if isinstance(w_idx, W_NDimArray) and w_idx.get_dtype().is_bool_type(): + try: + self.setitem_filter(space, w_idx, convert_to_array(space, w_value)) + except ValueError, e: + raise OperationError(space.w_ValueError, space.wrap(str(e))) return try: self.implementation.descr_setitem(space, self, w_idx, w_value) diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py --- a/pypy/module/micronumpy/test/test_numarray.py +++ b/pypy/module/micronumpy/test/test_numarray.py @@ -2454,6 +2454,23 @@ assert exc.value[0].find('cannot assign') >= 0 assert (a == [[0, 1], [2, 3], [4, 5]]).all() + def test_nonarray_assignment(self): + import numpypy as np + a = np.arange(10) + b = np.ones(10, dtype=bool) + r = np.arange(10) + def assign(a, b, c): + a[b] = c + raises(ValueError, assign, a, b, np.nan) + #raises(ValueError, assign, a, r, np.nan) # XXX + import sys + if '__pypy__' not in sys.builtin_module_names: + a[b] = np.array(np.nan) + #a[r] = np.array(np.nan) + else: + raises(ValueError, assign, a, b, np.array(np.nan)) + #raises(ValueError, assign, a, r, np.array(np.nan)) + def test_copy_kwarg(self): from numpypy import array x = array([1, 2, 3]) From noreply at buildbot.pypy.org Wed Nov 13 04:38:09 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Wed, 13 Nov 2013 04:38:09 +0100 (CET) Subject: [pypy-commit] pypy default: support ndarray.reshape(()) to reshape to scalar Message-ID: <20131113033809.ACFF11C07D2@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r68001:b89a1dc6a9e1 Date: 2013-11-12 22:32 -0500 http://bitbucket.org/pypy/pypy/changeset/b89a1dc6a9e1/ Log: support ndarray.reshape(()) to reshape to scalar diff --git a/pypy/module/micronumpy/arrayimpl/concrete.py b/pypy/module/micronumpy/arrayimpl/concrete.py --- a/pypy/module/micronumpy/arrayimpl/concrete.py +++ b/pypy/module/micronumpy/arrayimpl/concrete.py @@ -73,6 +73,8 @@ return SliceArray(self.start, new_strides, new_backstrides, new_shape, self, orig_array) else: + if self.get_size() == 1 and len(new_shape) == 0: + return scalar.Scalar(self.dtype, self.getitem(0)) return None def get_view(self, orig_array, dtype, new_shape): diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py --- a/pypy/module/micronumpy/test/test_numarray.py +++ b/pypy/module/micronumpy/test/test_numarray.py @@ -710,7 +710,14 @@ def test_reshape(self): from numpypy import array, zeros + for a in [array(1), array([1])]: + for s in [(), (1,)]: + b = a.reshape(s) + assert b.shape == s + assert (b == [1]).all() a = array(range(12)) + exc = raises(ValueError, "b = a.reshape(())") + assert str(exc.value) == "total size of new array must be unchanged" exc = raises(ValueError, "b = a.reshape((3, 10))") assert str(exc.value) == "total size of new array must be unchanged" b = a.reshape((3, 4)) From noreply at buildbot.pypy.org Wed Nov 13 09:16:11 2013 From: noreply at buildbot.pypy.org (fijal) Date: Wed, 13 Nov 2013 09:16:11 +0100 (CET) Subject: [pypy-commit] pypy release-2.2.x: start light, reflow the paras Message-ID: <20131113081611.A12A11C0134@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: release-2.2.x Changeset: r68002:725bbfa926bd Date: 2013-11-13 10:15 +0200 http://bitbucket.org/pypy/pypy/changeset/725bbfa926bd/ Log: start light, reflow the paras diff --git a/pypy/doc/release-2.2.0.rst b/pypy/doc/release-2.2.0.rst --- a/pypy/doc/release-2.2.0.rst +++ b/pypy/doc/release-2.2.0.rst @@ -45,23 +45,27 @@ Highlights ========== -* Our Garbage Collector is now "incremental". It should avoid almost all pauses due - to a major collection taking place. Previously, it would pause the program (rarely) - to walk all live objects, which could take arbitrarily long if your process is using - a whole lot of RAM. Now the same work is done in steps. This should make PyPy - more responsive, e.g. in games. There are still other pauses, from the GC and the JIT, - but they should be on the order of 5 milliseconds each. +* Our Garbage Collector is now "incremental". It should avoid almost + all pauses due to a major collection taking place. Previously, it + would pause the program (rarely) to walk all live objects, which + could take arbitrarily long if your process is using a whole lot of + RAM. Now the same work is done in steps. This should make PyPy + more responsive, e.g. in games. There are still other pauses, from + the GC and the JIT, but they should be on the order of 5 + milliseconds each. -* The JIT counters for hot code were never reset, which meant that a process running - for long enough would eventually JIT-compile more and more rarely executed code. - Not only is it useless to compile such code, but as more compiled code means more - memory used, this gives the impression of a memory leak. This has been tentatively +* The JIT counters for hot code were never reset, which meant that a + process running for long enough would eventually JIT-compile more + and more rarely executed code. Not only is it useless to compile + such code, but as more compiled code means more memory used, this + gives the impression of a memory leak. This has been tentatively fixed by decreasing the counters from time to time. -* NumPy has been split: now PyPy only contains the core module, called ``_numpypy``. - The ``numpy`` module itself has been moved to ``https://bitbucket.org/pypy/numpy``. - You need to install it separately in a virtualenv with - ``pip install git+https://bitbucket.org/pypy/numpy.git``. +* NumPy has been split: now PyPy only contains the core module, called + ``_numpypy``. The ``numpy`` module itself has been moved to + ``https://bitbucket.org/pypy/numpy``. You need to install it + separately in a virtualenv with ``pip install + git+https://bitbucket.org/pypy/numpy.git``. * improvements to non-inlined calls @@ -71,17 +75,19 @@ * improvements in buffer copying -* tk is supported (XXX was already in pypy 2.1 it seems?? maybe not correctly packaged?) +* tk is supported (XXX was already in pypy 2.1 it seems?? maybe not + correctly packaged?) -* We finally wrote all the missing ``os.xxx()`` functions. There are a lot of strange - ones that nobody ever heard about, except those who really need them. +* We finally wrote all the missing ``os.xxx()`` functions. There are + a lot of strange ones that nobody ever heard about, except those who + really need them. * numpy C API - the core module is included in PyPy 2.2, but you must now install - an external fork of numpy from https://bitbucket.org/pypy/numpy + the core module is included in PyPy 2.2, but you must now install an + external fork of numpy from https://bitbucket.org/pypy/numpy -removed in favor of an external numpy fork -at https://bitbucket.org/pypy/numpy +removed in favor of an external numpy fork at +https://bitbucket.org/pypy/numpy From noreply at buildbot.pypy.org Wed Nov 13 10:49:38 2013 From: noreply at buildbot.pypy.org (fijal) Date: Wed, 13 Nov 2013 10:49:38 +0100 (CET) Subject: [pypy-commit] pypy release-2.2.x: work on the announcement Message-ID: <20131113094938.5CD011C13B9@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: release-2.2.x Changeset: r68003:b2133aa0bf11 Date: 2013-11-13 11:48 +0200 http://bitbucket.org/pypy/pypy/changeset/b2133aa0bf11/ Log: work on the announcement diff --git a/pypy/doc/release-2.2.0.rst b/pypy/doc/release-2.2.0.rst --- a/pypy/doc/release-2.2.0.rst +++ b/pypy/doc/release-2.2.0.rst @@ -67,27 +67,20 @@ separately in a virtualenv with ``pip install git+https://bitbucket.org/pypy/numpy.git``. -* improvements to non-inlined calls +* non-inlined calls have less overhead -* sys.set_trace is now JITted (think coverage) +* Things that use ``sys.set_trace`` are now JITted (like coverage) -* faster json +* JSON encoding is faster -* improvements in buffer copying - -* tk is supported (XXX was already in pypy 2.1 it seems?? maybe not - correctly packaged?) +* various buffer copying methods experience speedups (like list-of-ints to + ``int[]`` buffer from cffi) * We finally wrote all the missing ``os.xxx()`` functions. There are a lot of strange ones that nobody ever heard about, except those who really need them. -* numpy C API +* numpy has a rudimentary C API that cooperates with ``cpyext`` - - the core module is included in PyPy 2.2, but you must now install an - external fork of numpy from https://bitbucket.org/pypy/numpy - - -removed in favor of an external numpy fork at -https://bitbucket.org/pypy/numpy +Cheers, +Armin Rigo and Maciej Fijalkowski From noreply at buildbot.pypy.org Wed Nov 13 11:35:54 2013 From: noreply at buildbot.pypy.org (bivab) Date: Wed, 13 Nov 2013 11:35:54 +0100 (CET) Subject: [pypy-commit] pypy armhf-singlefloat: merge default Message-ID: <20131113103554.A22621C13E5@cobra.cs.uni-duesseldorf.de> Author: David Schneider Branch: armhf-singlefloat Changeset: r68004:c0363615e405 Date: 2013-11-13 11:34 +0100 http://bitbucket.org/pypy/pypy/changeset/c0363615e405/ Log: merge default diff --git a/pypy/doc/windows.rst b/pypy/doc/windows.rst --- a/pypy/doc/windows.rst +++ b/pypy/doc/windows.rst @@ -118,14 +118,10 @@ The sqlite3 database library ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Download http://www.sqlite.org/2013/sqlite-amalgamation-3071601.zip and extract -it into a directory under the base directory. Also get -http://www.sqlite.org/2013/sqlite-dll-win32-x86-3071601.zip and extract the dll -into the bin directory, and the sqlite3.def into the sources directory. -Now build the import library so cffi can use the header and dll:: +PyPy uses cffi to interact with sqlite3.dll. Only the dll is needed, the cffi +wrapper is compiled when the module is imported for the first time. +The sqlite3.dll should be version 3.6.21 for CPython2.7 compatablility. - lib /DEF:sqlite3.def" /OUT:sqlite3.lib" - copy sqlite3.lib path\to\libs The expat XML parser diff --git a/pypy/module/micronumpy/arrayimpl/concrete.py b/pypy/module/micronumpy/arrayimpl/concrete.py --- a/pypy/module/micronumpy/arrayimpl/concrete.py +++ b/pypy/module/micronumpy/arrayimpl/concrete.py @@ -73,6 +73,8 @@ return SliceArray(self.start, new_strides, new_backstrides, new_shape, self, orig_array) else: + if self.get_size() == 1 and len(new_shape) == 0: + return scalar.Scalar(self.dtype, self.getitem(0)) return None def get_view(self, orig_array, dtype, new_shape): diff --git a/pypy/module/micronumpy/interp_boxes.py b/pypy/module/micronumpy/interp_boxes.py --- a/pypy/module/micronumpy/interp_boxes.py +++ b/pypy/module/micronumpy/interp_boxes.py @@ -11,6 +11,7 @@ from rpython.rtyper.lltypesystem import rffi from rpython.tool.sourcetools import func_with_new_name from pypy.module.micronumpy.arrayimpl.voidbox import VoidBoxStorage +from pypy.module.micronumpy.interp_flagsobj import W_FlagsObject from pypy.interpreter.mixedmodule import MixedModule from rpython.rtyper.lltypesystem import lltype from rpython.rlib.rstring import StringBuilder @@ -113,7 +114,7 @@ class W_GenericBox(W_Root): - _attrs_ = [] + _attrs_ = ['w_flags'] def descr__new__(space, w_subtype, __args__): raise operationerrfmt(space.w_TypeError, @@ -292,6 +293,12 @@ def descr_copy(self, space): return self.convert_to(self.get_dtype(space)) + w_flags = None + def descr_get_flags(self, space): + if self.w_flags is None: + self.w_flags = W_FlagsObject(self) + return self.w_flags + class W_BoolBox(W_GenericBox, PrimitiveBox): descr__new__, _get_dtype, descr_reduce = new_dtype_getter("bool") @@ -550,6 +557,7 @@ strides = GetSetProperty(W_GenericBox.descr_get_shape), ndim = GetSetProperty(W_GenericBox.descr_get_ndim), T = GetSetProperty(W_GenericBox.descr_self), + flags = GetSetProperty(W_GenericBox.descr_get_flags), ) W_BoolBox.typedef = TypeDef("bool_", W_GenericBox.typedef, diff --git a/pypy/module/micronumpy/interp_dtype.py b/pypy/module/micronumpy/interp_dtype.py --- a/pypy/module/micronumpy/interp_dtype.py +++ b/pypy/module/micronumpy/interp_dtype.py @@ -401,7 +401,7 @@ raise operationerrfmt(space.w_TypeError, msg, w_dtype) W_Dtype.typedef = TypeDef("dtype", - __module__ = "numpypy", + __module__ = "numpy", __new__ = interp2app(descr__new__), __str__= interp2app(W_Dtype.descr_str), diff --git a/pypy/module/micronumpy/interp_flagsobj.py b/pypy/module/micronumpy/interp_flagsobj.py new file mode 100644 --- /dev/null +++ b/pypy/module/micronumpy/interp_flagsobj.py @@ -0,0 +1,45 @@ +from pypy.interpreter.baseobjspace import W_Root +from pypy.interpreter.typedef import TypeDef, GetSetProperty +from pypy.interpreter.gateway import interp2app +from pypy.interpreter.error import OperationError + + +class W_FlagsObject(W_Root): + def __init__(self, arr): + self.arr = arr + + def descr_get_contiguous(self, space): + return space.w_True + + def descr_get_fortran(self, space): + return space.w_False + + def descr_get_writeable(self, space): + return space.w_True + + def descr_getitem(self, space, w_item): + key = space.str_w(w_item) + if key == "C" or key == "CONTIGUOUS" or key == "C_CONTIGUOUS": + return self.descr_get_contiguous(space) + if key == "F" or key == "FORTRAN" or key == "F_CONTIGUOUS": + return self.descr_get_fortran(space) + if key == "W" or key == "WRITEABLE": + return self.descr_get_writeable(space) + raise OperationError(space.w_KeyError, space.wrap( + "Unknown flag")) + + def descr_setitem(self, space, w_item, w_value): + raise OperationError(space.w_KeyError, space.wrap( + "Unknown flag")) + +W_FlagsObject.typedef = TypeDef("flagsobj", + __module__ = "numpy", + __getitem__ = interp2app(W_FlagsObject.descr_getitem), + __setitem__ = interp2app(W_FlagsObject.descr_setitem), + + contiguous = GetSetProperty(W_FlagsObject.descr_get_contiguous), + c_contiguous = GetSetProperty(W_FlagsObject.descr_get_contiguous), + f_contiguous = GetSetProperty(W_FlagsObject.descr_get_fortran), + fortran = GetSetProperty(W_FlagsObject.descr_get_fortran), + writeable = GetSetProperty(W_FlagsObject.descr_get_writeable), +) diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py --- a/pypy/module/micronumpy/interp_numarray.py +++ b/pypy/module/micronumpy/interp_numarray.py @@ -9,6 +9,7 @@ from pypy.module.micronumpy.strides import find_shape_and_elems,\ get_shape_from_iterable, to_coords, shape_agreement, \ shape_agreement_multiple +from pypy.module.micronumpy.interp_flagsobj import W_FlagsObject from pypy.module.micronumpy.interp_flatiter import W_FlatIterator from pypy.module.micronumpy.appbridge import get_appbridge_cache from pypy.module.micronumpy import loop @@ -97,11 +98,11 @@ def getitem_filter(self, space, arr): if len(arr.get_shape()) > 1 and arr.get_shape() != self.get_shape(): - raise OperationError(space.w_ValueError, - space.wrap("boolean index array should have 1 dimension")) + raise OperationError(space.w_ValueError, space.wrap( + "boolean index array should have 1 dimension")) if arr.get_size() > self.get_size(): - raise OperationError(space.w_ValueError, - space.wrap("index out of range for array")) + raise OperationError(space.w_ValueError, space.wrap( + "index out of range for array")) size = loop.count_all_true(arr) if len(arr.get_shape()) == 1: res_shape = [size] + self.get_shape()[1:] @@ -112,18 +113,19 @@ def setitem_filter(self, space, idx, val): if len(idx.get_shape()) > 1 and idx.get_shape() != self.get_shape(): - raise OperationError(space.w_ValueError, - space.wrap("boolean index array should have 1 dimension")) + raise OperationError(space.w_ValueError, space.wrap( + "boolean index array should have 1 dimension")) if idx.get_size() > self.get_size(): - raise OperationError(space.w_ValueError, - space.wrap("index out of range for array")) + raise OperationError(space.w_ValueError, space.wrap( + "index out of range for array")) size = loop.count_all_true(idx) if size > val.get_size() and val.get_size() != 1: - raise OperationError(space.w_ValueError, space.wrap("NumPy boolean array indexing assignment " - "cannot assign %d input values to " - "the %d output values where the mask is true" % - (val.get_size(), size))) - loop.setitem_filter(self, idx, val, size) + raise OperationError(space.w_ValueError, space.wrap( + "NumPy boolean array indexing assignment " + "cannot assign %d input values to " + "the %d output values where the mask is true" % + (val.get_size(), size))) + loop.setitem_filter(space, self, idx, val, size) def _prepare_array_index(self, space, w_index): if isinstance(w_index, W_NDimArray): @@ -196,8 +198,7 @@ prefix) def descr_getitem(self, space, w_idx): - if (isinstance(w_idx, W_NDimArray) and - w_idx.get_dtype().is_bool_type()): + if isinstance(w_idx, W_NDimArray) and w_idx.get_dtype().is_bool_type(): return self.getitem_filter(space, w_idx) try: return self.implementation.descr_getitem(space, self, w_idx) @@ -211,9 +212,11 @@ self.implementation.setitem_index(space, index_list, w_value) def descr_setitem(self, space, w_idx, w_value): - if (isinstance(w_idx, W_NDimArray) and - w_idx.get_dtype().is_bool_type()): - self.setitem_filter(space, w_idx, convert_to_array(space, w_value)) + if isinstance(w_idx, W_NDimArray) and w_idx.get_dtype().is_bool_type(): + try: + self.setitem_filter(space, w_idx, convert_to_array(space, w_value)) + except ValueError, e: + raise OperationError(space.w_ValueError, space.wrap(str(e))) return try: self.implementation.descr_setitem(space, self, w_idx, w_value) @@ -338,11 +341,11 @@ Returns an array containing the same data with a new shape. - Refer to `numpypy.reshape` for full documentation. + Refer to `numpy.reshape` for full documentation. See Also -------- - numpypy.reshape : equivalent function + numpy.reshape : equivalent function """ args_w, kw_w = __args__.unpack() order = NPY_CORDER @@ -610,13 +613,11 @@ raise OperationError(space.w_NotImplementedError, space.wrap( "dumps not implemented yet")) + w_flags = None def descr_get_flags(self, space): - raise OperationError(space.w_NotImplementedError, space.wrap( - "getting flags not implemented yet")) - - def descr_set_flags(self, space, w_args): - raise OperationError(space.w_NotImplementedError, space.wrap( - "setting flags not implemented yet")) + if self.w_flags is None: + self.w_flags = W_FlagsObject(self) + return self.w_flags @unwrap_spec(offset=int) def descr_getfield(self, space, w_dtype, offset): @@ -1123,9 +1124,8 @@ return res """, filename=__file__).interphook('ptp') -W_NDimArray.typedef = TypeDef( - "ndarray", - __module__ = "numpypy", +W_NDimArray.typedef = TypeDef("ndarray", + __module__ = "numpy", __new__ = interp2app(descr_new_array), __len__ = interp2app(W_NDimArray.descr_len), @@ -1204,6 +1204,7 @@ size = GetSetProperty(W_NDimArray.descr_get_size), itemsize = GetSetProperty(W_NDimArray.descr_get_itemsize), nbytes = GetSetProperty(W_NDimArray.descr_get_nbytes), + flags = GetSetProperty(W_NDimArray.descr_get_flags), fill = interp2app(W_NDimArray.descr_fill), tostring = interp2app(W_NDimArray.descr_tostring), @@ -1391,8 +1392,8 @@ return box -W_FlatIterator.typedef = TypeDef( - 'flatiter', +W_FlatIterator.typedef = TypeDef("flatiter", + __module__ = "numpy", __iter__ = interp2app(W_FlatIterator.descr_iter), __getitem__ = interp2app(W_FlatIterator.descr_getitem), __setitem__ = interp2app(W_FlatIterator.descr_setitem), diff --git a/pypy/module/micronumpy/interp_ufuncs.py b/pypy/module/micronumpy/interp_ufuncs.py --- a/pypy/module/micronumpy/interp_ufuncs.py +++ b/pypy/module/micronumpy/interp_ufuncs.py @@ -413,7 +413,7 @@ W_Ufunc.typedef = TypeDef("ufunc", - __module__ = "numpypy", + __module__ = "numpy", __call__ = interp2app(W_Ufunc.descr_call), __repr__ = interp2app(W_Ufunc.descr_repr), diff --git a/pypy/module/micronumpy/loop.py b/pypy/module/micronumpy/loop.py --- a/pypy/module/micronumpy/loop.py +++ b/pypy/module/micronumpy/loop.py @@ -398,7 +398,7 @@ 'index_dtype'], reds = 'auto') -def setitem_filter(arr, index, value, size): +def setitem_filter(space, arr, index, value, size): arr_iter = arr.create_iter() shapelen = len(arr.get_shape()) if shapelen > 1 and len(index.get_shape()) < 2: @@ -414,7 +414,7 @@ arr_dtype=arr_dtype, ) if index_iter.getitem_bool(): - arr_iter.setitem(value_iter.getitem()) + arr_iter.setitem(arr_dtype.coerce(space, value_iter.getitem())) value_iter.next() arr_iter.next() index_iter.next() diff --git a/pypy/module/micronumpy/test/test_dtypes.py b/pypy/module/micronumpy/test/test_dtypes.py --- a/pypy/module/micronumpy/test/test_dtypes.py +++ b/pypy/module/micronumpy/test/test_dtypes.py @@ -26,6 +26,7 @@ assert d.kind == 'b' assert dtype(d) is d assert dtype('bool') is d + assert repr(type(d)) == "" assert dtype('int8').num == 1 assert dtype('int8').name == 'int8' diff --git a/pypy/module/micronumpy/test/test_flagsobj.py b/pypy/module/micronumpy/test/test_flagsobj.py new file mode 100644 --- /dev/null +++ b/pypy/module/micronumpy/test/test_flagsobj.py @@ -0,0 +1,22 @@ +from pypy.module.micronumpy.test.test_base import BaseNumpyAppTest + + +class AppTestFlagsObj(BaseNumpyAppTest): + def test_repr(self): + import numpy as np + a = np.array([1,2,3]) + assert repr(type(a.flags)) == "" + + def test_array_flags(self): + import numpy as np + a = np.array([1,2,3]) + assert a.flags.c_contiguous == True + assert a.flags['W'] == True + raises(KeyError, "a.flags['blah']") + raises(KeyError, "a.flags['C_CONTIGUOUS'] = False") + raises((TypeError, AttributeError), "a.flags.c_contiguous = False") + + def test_scalar_flags(self): + import numpy as np + a = np.int32(2) + assert a.flags.c_contiguous == True diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py --- a/pypy/module/micronumpy/test/test_numarray.py +++ b/pypy/module/micronumpy/test/test_numarray.py @@ -249,9 +249,11 @@ return CustomIntObject(value) def test_ndarray(self): - from numpypy import ndarray, array, dtype + from numpy import ndarray, array, dtype, flatiter assert type(ndarray) is type + assert repr(ndarray) == "" + assert repr(flatiter) == "" assert type(array) is not type a = ndarray((2, 3)) assert a.shape == (2, 3) @@ -509,6 +511,13 @@ for i in xrange(5): assert a[i] == i + def test_setitem_array(self): + import numpy as np + a = np.array((-1., 0, 1))/0. + b = np.array([False, False, True], dtype=bool) + a[b] = 100 + assert a[2] == 100 + def test_setitem_obj_index(self): from numpypy import arange a = arange(10) @@ -701,7 +710,14 @@ def test_reshape(self): from numpypy import array, zeros + for a in [array(1), array([1])]: + for s in [(), (1,)]: + b = a.reshape(s) + assert b.shape == s + assert (b == [1]).all() a = array(range(12)) + exc = raises(ValueError, "b = a.reshape(())") + assert str(exc.value) == "total size of new array must be unchanged" exc = raises(ValueError, "b = a.reshape((3, 10))") assert str(exc.value) == "total size of new array must be unchanged" b = a.reshape((3, 4)) @@ -2445,6 +2461,23 @@ assert exc.value[0].find('cannot assign') >= 0 assert (a == [[0, 1], [2, 3], [4, 5]]).all() + def test_nonarray_assignment(self): + import numpypy as np + a = np.arange(10) + b = np.ones(10, dtype=bool) + r = np.arange(10) + def assign(a, b, c): + a[b] = c + raises(ValueError, assign, a, b, np.nan) + #raises(ValueError, assign, a, r, np.nan) # XXX + import sys + if '__pypy__' not in sys.builtin_module_names: + a[b] = np.array(np.nan) + #a[r] = np.array(np.nan) + else: + raises(ValueError, assign, a, b, np.array(np.nan)) + #raises(ValueError, assign, a, r, np.array(np.nan)) + def test_copy_kwarg(self): from numpypy import array x = array([1, 2, 3]) diff --git a/pypy/module/micronumpy/test/test_ufuncs.py b/pypy/module/micronumpy/test/test_ufuncs.py --- a/pypy/module/micronumpy/test/test_ufuncs.py +++ b/pypy/module/micronumpy/test/test_ufuncs.py @@ -81,7 +81,7 @@ assert isinstance(add, ufunc) assert repr(add) == "" - assert repr(ufunc) == "" or repr(ufunc) == "" + assert repr(ufunc) == "" def test_ufunc_attrs(self): from numpypy import add, multiply, sin diff --git a/pypy/module/test_lib_pypy/test_grp_extra.py b/pypy/module/test_lib_pypy/test_grp_extra.py --- a/pypy/module/test_lib_pypy/test_grp_extra.py +++ b/pypy/module/test_lib_pypy/test_grp_extra.py @@ -2,8 +2,7 @@ class AppTestGrp: - - spaceconfig = dict(usemodules=('_ffi', '_rawffi', 'itertools')) + spaceconfig = dict(usemodules=('binascii', '_ffi', '_rawffi', 'itertools')) def setup_class(cls): cls.w_grp = import_lib_pypy(cls.space, 'grp', @@ -11,14 +10,18 @@ def test_basic(self): raises(KeyError, self.grp.getgrnam, "dEkLofcG") - try: - g = self.grp.getgrnam("root") - except KeyError: - return # no 'root' group on OS/X? - assert g.gr_gid == 0 - assert g.gr_mem == ['root'] or g.gr_mem == [] - assert g.gr_name == 'root' - assert isinstance(g.gr_passwd, str) # usually just 'x', don't hope :-) + for name in ["root", "wheel"]: + try: + g = self.grp.getgrnam(name) + except KeyError: + continue + assert g.gr_gid == 0 + assert g.gr_mem == ['root'] or g.gr_mem == [] + assert g.gr_name == name + assert isinstance(g.gr_passwd, str) # usually just 'x', don't hope :-) + break + else: + raise def test_extra(self): grp = self.grp diff --git a/rpython/jit/metainterp/logger.py b/rpython/jit/metainterp/logger.py --- a/rpython/jit/metainterp/logger.py +++ b/rpython/jit/metainterp/logger.py @@ -103,7 +103,9 @@ elif isinstance(arg, BoxInt): return 'i' + str(mv) elif isinstance(arg, self.ts.ConstRef): - return 'ConstPtr(ptr' + str(mv) + ')' + if arg.value: + return 'ConstPtr(ptr' + str(mv) + ')' + return 'ConstPtr(null)' elif isinstance(arg, self.ts.BoxRef): return 'p' + str(mv) elif isinstance(arg, ConstFloat): From noreply at buildbot.pypy.org Wed Nov 13 12:17:38 2013 From: noreply at buildbot.pypy.org (Raemi) Date: Wed, 13 Nov 2013 12:17:38 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c4: Add missing canmallocgcs. Ops that don't do malloc but can contain a GC safe- Message-ID: <20131113111738.862381C025A@cobra.cs.uni-duesseldorf.de> Author: Remi Meier Branch: stmgc-c4 Changeset: r68005:e5e15daf8958 Date: 2013-11-13 12:16 +0100 http://bitbucket.org/pypy/pypy/changeset/e5e15daf8958/ Log: Add missing canmallocgcs. Ops that don't do malloc but can contain a GC safe- point (e.g. BecomeInevitable) need to have canmallocgc=True... diff --git a/TODO b/TODO --- a/TODO +++ b/TODO @@ -1,3 +1,14 @@ +------------------------------------------------------------ + +POSSIBLE BUG: +investigate if another thread can force a jitframe. Thus, +making a transaction break *after* a guard_not_forced +would be wrong, as the force will only be visible after +the break. (The GIL doesn't get released inbetween +the GUARD and the next call that is allowed to, so no +problems there).. +Solution, move transaction breaks right before guard_not_forced, maybe + ------------------------------------------------------------ should stm_thread_local_obj always be read & writeable? would diff --git a/rpython/memory/gctransform/stmframework.py b/rpython/memory/gctransform/stmframework.py --- a/rpython/memory/gctransform/stmframework.py +++ b/rpython/memory/gctransform/stmframework.py @@ -97,6 +97,7 @@ self.default(hop) self.pop_roots(hop, livevars) + # sync with lloperation.py gct_stm_become_inevitable = _gct_with_roots_pushed gct_stm_set_transaction_length = _gct_with_roots_pushed gct_stm_stop_all_other_threads = _gct_with_roots_pushed diff --git a/rpython/rtyper/lltypesystem/lloperation.py b/rpython/rtyper/lltypesystem/lloperation.py --- a/rpython/rtyper/lltypesystem/lloperation.py +++ b/rpython/rtyper/lltypesystem/lloperation.py @@ -410,6 +410,9 @@ # direct_calls and maybe several casts, but it looks less heavy-weight # to keep them as operations until the genc stage) + # NOTE: use canmallocgc for all operations that can contain a collection. + # that includes all that do 'BecomeInevitable' or otherwise contain + # possible GC safe-points! (also sync with stmframework.py) 'stm_initialize': LLOp(), 'stm_finalize': LLOp(), 'stm_barrier': LLOp(sideeffects=False), @@ -429,7 +432,7 @@ 'stm_commit_transaction': LLOp(), 'stm_begin_inevitable_transaction': LLOp(), 'stm_should_break_transaction': LLOp(sideeffects=False), - 'stm_set_transaction_length': LLOp(), + 'stm_set_transaction_length': LLOp(canmallocgc=True), 'stm_change_atomic': LLOp(), 'stm_get_atomic': LLOp(sideeffects=False), 'stm_perform_transaction':LLOp(canmallocgc=True), @@ -447,7 +450,7 @@ 'stm_abort_info_push': LLOp(), 'stm_abort_info_pop': LLOp(), - 'stm_inspect_abort_info': LLOp(sideeffects=False), + 'stm_inspect_abort_info': LLOp(sideeffects=False, canmallocgc=True), 'stm_get_adr_of_private_rev_num':LLOp(), 'stm_get_adr_of_read_barrier_cache':LLOp(), diff --git a/rpython/translator/stm/inevitable.py b/rpython/translator/stm/inevitable.py --- a/rpython/translator/stm/inevitable.py +++ b/rpython/translator/stm/inevitable.py @@ -21,6 +21,7 @@ 'stm_threadlocalref_get', 'stm_threadlocalref_set', 'stm_threadlocalref_count', 'stm_threadlocalref_addr', 'jit_assembler_call', 'gc_writebarrier', + 'shrink_array', ]) ALWAYS_ALLOW_OPERATIONS |= set(lloperation.enum_tryfold_ops()) From noreply at buildbot.pypy.org Wed Nov 13 16:36:41 2013 From: noreply at buildbot.pypy.org (ltratt) Date: Wed, 13 Nov 2013 16:36:41 +0100 (CET) Subject: [pypy-commit] pypy more_strategies: Add IntegerListAscending strategy. Message-ID: <20131113153641.8F0DA1C13E5@cobra.cs.uni-duesseldorf.de> Author: Laurence Tratt Branch: more_strategies Changeset: r68006:ef5630ce70e3 Date: 2013-11-13 15:26 +0000 http://bitbucket.org/pypy/pypy/changeset/ef5630ce70e3/ Log: Add IntegerListAscending strategy. diff --git a/pypy/objspace/std/listobject.py b/pypy/objspace/std/listobject.py --- a/pypy/objspace/std/listobject.py +++ b/pypy/objspace/std/listobject.py @@ -896,7 +896,7 @@ def switch_to_correct_strategy(self, w_list, w_item): if type(w_item) is W_IntObject: - strategy = self.space.fromcache(IntegerListStrategy) + strategy = self.space.fromcache(IntegerListAscendingStrategy) elif type(w_item) is W_StringObject: strategy = self.space.fromcache(StringListStrategy) elif type(w_item) is W_UnicodeObject: @@ -1010,7 +1010,11 @@ def switch_to_integer_strategy(self, w_list): items = self._getitems_range(w_list, False) - strategy = w_list.strategy = self.space.fromcache(IntegerListStrategy) + start, step, length = self.unerase(w_list.lstorage) + if step > 0: + strategy = w_list.strategy = self.space.fromcache(IntegerListAscendingStrategy) + else: + strategy = w_list.strategy = self.space.fromcache(IntegerListStrategy) w_list.lstorage = strategy.erase(items) def wrap(self, intval): @@ -1518,6 +1522,25 @@ def unwrap(self, w_int): return self.space.int_w(w_int) + def init_from_list_w(self, w_list, list_w): + # While unpacking integer elements, also determine whether they're + # pre-sorted. + assert len(list_w) > 0 + asc = True + l = [0] * len(list_w) + lst = l[0] = self.unwrap(list_w[0]) + for i in range(1, len(list_w)): + item_w = list_w[i] + it = self.unwrap(item_w) + if asc and it < lst: + asc = False + l[i] = it + lst = it + w_list.lstorage = self.erase(l) + if asc: + # The list was already sorted into ascending order. + w_list.strategy = self.space.fromcache(IntegerListAscendingStrategy) + erase, unerase = rerased.new_erasing_pair("integer") erase = staticmethod(erase) unerase = staticmethod(unerase) @@ -1526,7 +1549,8 @@ return type(w_obj) is W_IntObject def list_is_correct_type(self, w_list): - return w_list.strategy is self.space.fromcache(IntegerListStrategy) + return w_list.strategy is self.space.fromcache(IntegerListStrategy) \ + or w_list.strategy is self.space.fromcache(IntegerListAscendingStrategy) def sort(self, w_list, reverse): l = self.unerase(w_list.lstorage) @@ -1534,6 +1558,8 @@ sorter.sort() if reverse: l.reverse() + else: + w_list.strategy = self.space.fromcache(IntegerListAscendingStrategy) def getitems_int(self, w_list): return self.unerase(w_list.lstorage) @@ -1611,6 +1637,94 @@ self.space, storage, self) return self._base_setslice(w_list, start, step, slicelength, w_other) +class IntegerListAscendingStrategy(IntegerListStrategy): + def sort(self, w_list, reverse): + if reverse: + self.unerase(w_list.lstorage).reverse() + w_list.strategy = self.space.fromcache(IntegerListStrategy) + + def append(self, w_list, w_item): + if type(w_item) is W_IntObject: + l = self.unerase(w_list.lstorage) + length = len(l) + item = self.unwrap(w_item) + if length == 0 or l[length - 1] <= item: + l.append(item) + return + w_list.strategy = self.space.fromcache(IntegerListStrategy) + IntegerListStrategy.append(self, w_list, w_item) + + def insert(self, w_list, index, w_item): + if type(w_item) is W_IntObject: + l = self.unerase(w_list.lstorage) + length = len(l) + item = self.unwrap(w_item) + if length == 0 or \ + ((index == 0 or l[index - 1] <= item) and (index == length or l[index] >= item)): + l.insert(index, item) + return + w_list.strategy = self.space.fromcache(IntegerListStrategy) + IntegerListStrategy.insert(self, w_list, index, w_item) + + def _extend_from_list(self, w_list, w_item): + if type(w_item) is W_ListObject and \ + w_item.strategy is self.space.fromcache(IntegerListAscendingStrategy): + self_l = self.unerase(w_list.lstorage) + other_l = self.unerase(w_item.lstorage) + if len(self_l) == 0 or len(other_l) == 0 or self_l[len(self_l) - 1] <= other_l[0]: + self_l.extend(other_l) + return + w_list.strategy = self.space.fromcache(IntegerListStrategy) + IntegerListStrategy._extend_from_list(self,w_list, w_item) + + def setitem(self, w_list, index, w_item): + if type(w_item) is W_IntObject: + item = self.unwrap(w_item) + l = self.unerase(w_list.lstorage) + length = len(l) + assert len(l) > 0 + if (index == 0 or l[index - 1] <= item) \ + and (index == length - 1 or l[index + 1] >= item): + l[index] = item + return + w_list.strategy = self.space.fromcache(IntegerListStrategy) + IntegerListStrategy.setitem(self, w_list, index, w_item) + + def setslice(self, w_list, start, step, slicelength, w_other): + # XXX could be supported if really desired + w_list.strategy = self.space.fromcache(IntegerListStrategy) + IntegerListStrategy.setslice(self, w_list, start, step, slicelength, w_other) + + def inplace_mul(self, w_list, times): + l = self.unerase(w_list.lstorage) + length = len(l) + if length == 0: + return + if l[0] != l[length - 1]: + w_list.strategy = self.space.fromcache(IntegerListStrategy) + IntegerListStrategy.inplace_mul(self, w_list, times) + + def reverse(self, w_list): + self.unerase(w_list.lstorage).reverse() + w_list.strategy = self.space.fromcache(IntegerListStrategy) + + def _safe_find(self, w_list, obj, start, stop): + if w_list.length() < 16: + return IntegerListStrategy._safe_find(self, w_list, obj, start, stop) + l = self.unerase(w_list.lstorage) + start -= 1 + stop += 1 + while stop - start > 1: + p = (start + stop) / 2 + if l[p] < obj: + start = p + else: + stop = p + if stop == len(l) or l[stop] != obj: + raise ValueError + return stop + + class FloatListStrategy(ListStrategy): import_from_mixin(AbstractUnwrappedStrategy) @@ -1640,7 +1754,7 @@ elif w_objt is W_IntObject or w_objt is W_LongObject: return self._safe_find(w_list, w_obj.float_w(self.space), start, stop) elif w_objt is W_StringObject or w_objt is W_UnicodeObject \ - or self.space.type(w_obj).compares_by_identity(): + or self.space.type(w_obj).compares_by_identity(): raise ValueError return ListStrategy.find(self, w_list, w_obj, start, stop) diff --git a/pypy/objspace/std/test/test_listobject.py b/pypy/objspace/std/test/test_listobject.py --- a/pypy/objspace/std/test/test_listobject.py +++ b/pypy/objspace/std/test/test_listobject.py @@ -526,6 +526,12 @@ assert not l.__contains__(-20) assert not l.__contains__(-21) + l = list(range(1000)) + assert l.index(123) == 123 + del l[123] + raises(ValueError, "l.index(123)") + assert l.index(124) == 123 + def test_call_list(self): assert list('') == [] assert list('abc') == ['a', 'b', 'c'] @@ -577,6 +583,10 @@ assert m == [5,2,3] assert l == [1,2,3] + l = [1,2,3] + l.extend([3,4]) + assert l == [1, 2, 3, 3, 4] + def test_extend_tuple(self): l = l0 = [1] l.extend((2,)) diff --git a/pypy/objspace/std/test/test_liststrategies.py b/pypy/objspace/std/test/test_liststrategies.py --- a/pypy/objspace/std/test/test_liststrategies.py +++ b/pypy/objspace/std/test/test_liststrategies.py @@ -1,5 +1,8 @@ import sys -from pypy.objspace.std.listobject import W_ListObject, EmptyListStrategy, ObjectListStrategy, IntegerListStrategy, FloatListStrategy, StringListStrategy, RangeListStrategy, make_range_list, UnicodeListStrategy +from pypy.objspace.std.listobject import W_ListObject, EmptyListStrategy, \ + ObjectListStrategy, IntegerListStrategy, IntegerListAscendingStrategy, \ + FloatListStrategy, StringListStrategy, RangeListStrategy, make_range_list, \ + UnicodeListStrategy from pypy.objspace.std import listobject from pypy.objspace.std.test.test_listobject import TestW_ListObject @@ -342,7 +345,6 @@ r = make_range_list(space, 1,3,7) empty.extend(r) assert isinstance(empty.strategy, RangeListStrategy) - print empty.getitem(6) assert space.is_true(space.eq(empty.getitem(1), w(4))) empty = W_ListObject(space, []) @@ -480,7 +482,8 @@ l1 = make_range_list(self.space, 0, 1, 100) l2 = W_ListObject(self.space, [self.space.wrap(1), self.space.wrap(2), self.space.wrap(3)]) l3 = self.space.add(l2, l1) - assert l3.strategy is l2.strategy + assert isinstance(l2.strategy, IntegerListAscendingStrategy) + assert isinstance(l3.strategy, IntegerListStrategy) def test_mul(self): l1 = W_ListObject(self.space, [self.space.wrap(1), self.space.wrap(2), self.space.wrap(3)]) @@ -659,6 +662,80 @@ list_copy[0] = 42 assert list_orig == [1, 2, 3] + def test_integerascending(self): + space = self.space + w_l = W_ListObject(space, [space.wrap(1), space.wrap(3)]) + assert isinstance(w_l.strategy, IntegerListAscendingStrategy) + w_l.append(space.wrap(5)) + assert isinstance(w_l.strategy, IntegerListAscendingStrategy) + + w_l.insert(0, space.wrap(0)) + assert isinstance(w_l.strategy, IntegerListAscendingStrategy) + w_l.insert(4, space.wrap(6)) + assert isinstance(w_l.strategy, IntegerListAscendingStrategy) + assert space.listview_int(w_l) == [0, 1, 3, 5 ,6] + + w_l = W_ListObject(space, []) + w_l.insert(0, space.wrap(1)) + assert isinstance(w_l.strategy, IntegerListAscendingStrategy) + + w_l = W_ListObject(space, [space.wrap(3), space.wrap(2), space.wrap(4), space.wrap(1)]) + assert isinstance(w_l.strategy, IntegerListStrategy) + l2 = [1, 2, 3, 4] + space.call_method(w_l, "sort") + assert isinstance(w_l.strategy, IntegerListAscendingStrategy) + assert space.listview_int(w_l) == l2 + space.call_method(w_l, "sort") + assert space.listview_int(w_l) == l2 + w_l.append(space.wrap(5)) + assert isinstance(w_l.strategy, IntegerListAscendingStrategy) + w_l.append(space.wrap(0)) + assert isinstance(w_l.strategy, IntegerListStrategy) + + w_l = W_ListObject(space, []) + space.call_method(w_l, "extend", W_ListObject(space, [space.wrap(1), space.wrap(2)])) + assert space.listview_int(w_l) == [1, 2] + assert isinstance(w_l.strategy, IntegerListAscendingStrategy) + + space.call_method(w_l, "extend", W_ListObject(space, [space.wrap(4)])) + assert space.listview_int(w_l) == [1, 2, 4] + assert isinstance(w_l.strategy, IntegerListAscendingStrategy) + + space.call_method(w_l, "pop") + space.call_method(w_l, "pop") + space.call_method(w_l, "pop") + assert space.listview_int(w_l) == [] + assert isinstance(w_l.strategy, IntegerListAscendingStrategy) + space.call_method(w_l, "extend", W_ListObject(space, [space.wrap(4)])) + assert space.listview_int(w_l) == [4] + assert isinstance(w_l.strategy, IntegerListAscendingStrategy) + + space.call_method(w_l, "extend", W_ListObject(space, [space.wrap(0)])) + assert space.listview_int(w_l) == [4, 0] + assert isinstance(w_l.strategy, IntegerListStrategy) + + w_l = W_ListObject(space, [space.wrap(1), space.wrap(3), space.wrap(5)]) + assert isinstance(w_l.strategy, IntegerListAscendingStrategy) + w_l.setitem(0, space.wrap(0)) + w_l.setitem(1, space.wrap(4)) + w_l.setitem(2, space.wrap(6)) + assert isinstance(w_l.strategy, IntegerListAscendingStrategy) + w_l.setitem(1, space.wrap(7)) + assert isinstance(w_l.strategy, IntegerListStrategy) + + w_l = W_ListObject(space, [space.wrap(1), space.wrap(1)]) + w_l.inplace_mul(2) + assert isinstance(w_l.strategy, IntegerListAscendingStrategy) + w_l.append(space.wrap(2)) + w_l.inplace_mul(2) + assert isinstance(w_l.strategy, IntegerListStrategy) + + w_l = W_ListObject(space, [space.wrap(1), space.wrap(2)]) + assert isinstance(w_l.strategy, IntegerListAscendingStrategy) + w_l.sort(True) + assert isinstance(w_l.strategy, IntegerListStrategy) + assert space.listview_int(w_l) == [2, 1] + class TestW_ListStrategiesDisabled: spaceconfig = {"objspace.std.withliststrategies": False} From noreply at buildbot.pypy.org Thu Nov 14 02:04:59 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Thu, 14 Nov 2013 02:04:59 +0100 (CET) Subject: [pypy-commit] pypy default: allow (unsupported) out arg for scalar.round Message-ID: <20131114010459.7F7AB1C0134@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r68007:bb54a2f34c40 Date: 2013-11-13 19:33 -0500 http://bitbucket.org/pypy/pypy/changeset/bb54a2f34c40/ Log: allow (unsupported) out arg for scalar.round diff --git a/pypy/module/micronumpy/interp_boxes.py b/pypy/module/micronumpy/interp_boxes.py --- a/pypy/module/micronumpy/interp_boxes.py +++ b/pypy/module/micronumpy/interp_boxes.py @@ -255,7 +255,10 @@ return convert_to_array(space, w_values) @unwrap_spec(decimals=int) - def descr_round(self, space, decimals=0): + def descr_round(self, space, decimals=0, w_out=None): + if not space.is_none(w_out): + raise OperationError(space.w_NotImplementedError, space.wrap( + "out not supported")) v = self.convert_to(self.get_dtype(space)) return self.get_dtype(space).itemtype.round(v, decimals) diff --git a/pypy/module/micronumpy/test/test_scalar.py b/pypy/module/micronumpy/test/test_scalar.py --- a/pypy/module/micronumpy/test/test_scalar.py +++ b/pypy/module/micronumpy/test/test_scalar.py @@ -53,6 +53,7 @@ assert f.round() == 13. assert f.round(decimals=-1) == 10. assert f.round(decimals=1) == 13.4 + assert f.round(decimals=1, out=None) == 13.4 assert b.round() == 1.0 assert b.round(decimals=5) is b From noreply at buildbot.pypy.org Thu Nov 14 02:05:01 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Thu, 14 Nov 2013 02:05:01 +0100 (CET) Subject: [pypy-commit] pypy default: backout d50d27c01818, doesn't work as intended when translated Message-ID: <20131114010501.037361C025A@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r68008:277365bc6cda Date: 2013-11-13 20:03 -0500 http://bitbucket.org/pypy/pypy/changeset/277365bc6cda/ Log: backout d50d27c01818, doesn't work as intended when translated diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py --- a/pypy/module/micronumpy/interp_numarray.py +++ b/pypy/module/micronumpy/interp_numarray.py @@ -213,10 +213,7 @@ def descr_setitem(self, space, w_idx, w_value): if isinstance(w_idx, W_NDimArray) and w_idx.get_dtype().is_bool_type(): - try: - self.setitem_filter(space, w_idx, convert_to_array(space, w_value)) - except ValueError, e: - raise OperationError(space.w_ValueError, space.wrap(str(e))) + self.setitem_filter(space, w_idx, convert_to_array(space, w_value)) return try: self.implementation.descr_setitem(space, self, w_idx, w_value) diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py --- a/pypy/module/micronumpy/test/test_numarray.py +++ b/pypy/module/micronumpy/test/test_numarray.py @@ -2461,23 +2461,6 @@ assert exc.value[0].find('cannot assign') >= 0 assert (a == [[0, 1], [2, 3], [4, 5]]).all() - def test_nonarray_assignment(self): - import numpypy as np - a = np.arange(10) - b = np.ones(10, dtype=bool) - r = np.arange(10) - def assign(a, b, c): - a[b] = c - raises(ValueError, assign, a, b, np.nan) - #raises(ValueError, assign, a, r, np.nan) # XXX - import sys - if '__pypy__' not in sys.builtin_module_names: - a[b] = np.array(np.nan) - #a[r] = np.array(np.nan) - else: - raises(ValueError, assign, a, b, np.array(np.nan)) - #raises(ValueError, assign, a, r, np.array(np.nan)) - def test_copy_kwarg(self): from numpypy import array x = array([1, 2, 3]) From noreply at buildbot.pypy.org Thu Nov 14 02:29:20 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Thu, 14 Nov 2013 02:29:20 +0100 (CET) Subject: [pypy-commit] pypy default: support eq/ne for array.flags Message-ID: <20131114012920.957251C13E5@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r68009:b9b04c08e3d3 Date: 2013-11-13 20:28 -0500 http://bitbucket.org/pypy/pypy/changeset/b9b04c08e3d3/ Log: support eq/ne for array.flags diff --git a/pypy/module/micronumpy/interp_flagsobj.py b/pypy/module/micronumpy/interp_flagsobj.py --- a/pypy/module/micronumpy/interp_flagsobj.py +++ b/pypy/module/micronumpy/interp_flagsobj.py @@ -7,6 +7,7 @@ class W_FlagsObject(W_Root): def __init__(self, arr): self.arr = arr + self.flags = 0 def descr_get_contiguous(self, space): return space.w_True @@ -32,10 +33,23 @@ raise OperationError(space.w_KeyError, space.wrap( "Unknown flag")) + def eq(self, space, w_other): + if not isinstance(w_other, W_FlagsObject): + return False + return self.flags == w_other.flags + + def descr_eq(self, space, w_other): + return space.wrap(self.eq(space, w_other)) + + def descr_ne(self, space, w_other): + return space.wrap(not self.eq(space, w_other)) + W_FlagsObject.typedef = TypeDef("flagsobj", __module__ = "numpy", __getitem__ = interp2app(W_FlagsObject.descr_getitem), __setitem__ = interp2app(W_FlagsObject.descr_setitem), + __eq__ = interp2app(W_FlagsObject.descr_eq), + __ne__ = interp2app(W_FlagsObject.descr_ne), contiguous = GetSetProperty(W_FlagsObject.descr_get_contiguous), c_contiguous = GetSetProperty(W_FlagsObject.descr_get_contiguous), diff --git a/pypy/module/micronumpy/test/test_flagsobj.py b/pypy/module/micronumpy/test/test_flagsobj.py --- a/pypy/module/micronumpy/test/test_flagsobj.py +++ b/pypy/module/micronumpy/test/test_flagsobj.py @@ -20,3 +20,10 @@ import numpy as np a = np.int32(2) assert a.flags.c_contiguous == True + + def test_compare(self): + import numpy as np + a = np.array([1,2,3]) + b = np.array([4,5,6,7]) + assert a.flags == b.flags + assert not a.flags != b.flags From noreply at buildbot.pypy.org Thu Nov 14 03:36:05 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Thu, 14 Nov 2013 03:36:05 +0100 (CET) Subject: [pypy-commit] pypy default: add size attribute for scalars Message-ID: <20131114023605.C98581C036B@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r68010:e14d70573095 Date: 2013-11-13 20:52 -0500 http://bitbucket.org/pypy/pypy/changeset/e14d70573095/ Log: add size attribute for scalars diff --git a/pypy/module/micronumpy/interp_boxes.py b/pypy/module/micronumpy/interp_boxes.py --- a/pypy/module/micronumpy/interp_boxes.py +++ b/pypy/module/micronumpy/interp_boxes.py @@ -284,6 +284,9 @@ def descr_get_dtype(self, space): return self.get_dtype(space) + def descr_get_size(self, space): + return space.wrap(1) + def descr_get_itemsize(self, space): return self.get_dtype(space).descr_get_itemsize(space) @@ -554,6 +557,7 @@ copy = interp2app(W_GenericBox.descr_copy), dtype = GetSetProperty(W_GenericBox.descr_get_dtype), + size = GetSetProperty(W_GenericBox.descr_get_size), itemsize = GetSetProperty(W_GenericBox.descr_get_itemsize), nbytes = GetSetProperty(W_GenericBox.descr_get_itemsize), shape = GetSetProperty(W_GenericBox.descr_get_shape), diff --git a/pypy/module/micronumpy/test/test_scalar.py b/pypy/module/micronumpy/test/test_scalar.py --- a/pypy/module/micronumpy/test/test_scalar.py +++ b/pypy/module/micronumpy/test/test_scalar.py @@ -85,6 +85,7 @@ import numpypy as np value = np.dtype('int64').type(12345) assert value.dtype == np.dtype('int64') + assert value.size == 1 assert value.itemsize == 8 assert value.nbytes == 8 assert value.shape == () From noreply at buildbot.pypy.org Thu Nov 14 03:36:07 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Thu, 14 Nov 2013 03:36:07 +0100 (CET) Subject: [pypy-commit] pypy default: support axis argument for array.squeeze Message-ID: <20131114023607.19BDE1C036B@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r68011:f85795564612 Date: 2013-11-13 21:33 -0500 http://bitbucket.org/pypy/pypy/changeset/f85795564612/ Log: support axis argument for array.squeeze diff --git a/pypy/module/micronumpy/conversion_utils.py b/pypy/module/micronumpy/conversion_utils.py --- a/pypy/module/micronumpy/conversion_utils.py +++ b/pypy/module/micronumpy/conversion_utils.py @@ -40,3 +40,24 @@ else: raise OperationError(space.w_TypeError, space.wrap( "order not understood")) + +def multi_axis_converter(space, w_axis, ndim): + if space.is_none(w_axis): + return [True] * ndim + out = [False] * ndim + if not space.isinstance_w(w_axis, space.w_tuple): + w_axis = space.newtuple([w_axis]) + for w_item in space.fixedview(w_axis): + item = space.int_w(w_item) + axis = item + if axis < 0: + axis += ndim + if axis < 0 or axis >= ndim: + raise OperationError(space.w_ValueError, space.wrap( + "'axis' entry %d is out of bounds [-%d, %d)" % + (item, ndim, ndim))) + if out[axis]: + raise OperationError(space.w_ValueError, space.wrap( + "duplicate value in 'axis'")) + out[axis] = True + return out diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py --- a/pypy/module/micronumpy/interp_numarray.py +++ b/pypy/module/micronumpy/interp_numarray.py @@ -19,7 +19,7 @@ from rpython.rlib import jit from rpython.rlib.rstring import StringBuilder from pypy.module.micronumpy.arrayimpl.base import BaseArrayImplementation -from pypy.module.micronumpy.conversion_utils import order_converter +from pypy.module.micronumpy.conversion_utils import order_converter, multi_axis_converter from pypy.module.micronumpy.constants import * def _find_shape(space, w_size, dtype): @@ -692,11 +692,20 @@ return self.implementation.sort(space, w_axis, w_order) def descr_squeeze(self, space, w_axis=None): + cur_shape = self.get_shape() if not space.is_none(w_axis): - raise OperationError(space.w_NotImplementedError, space.wrap( - "axis unsupported for squeeze")) - cur_shape = self.get_shape() - new_shape = [s for s in cur_shape if s != 1] + axes = multi_axis_converter(space, w_axis, len(cur_shape)) + new_shape = [] + for i in range(len(cur_shape)): + if axes[i]: + if cur_shape[i] != 1: + raise OperationError(space.w_ValueError, space.wrap( + "cannot select an axis to squeeze out " \ + "which has size greater than one")) + else: + new_shape.append(cur_shape[i]) + else: + new_shape = [s for s in cur_shape if s != 1] if len(cur_shape) == len(new_shape): return self return wrap_impl(space, space.type(self), self, diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py --- a/pypy/module/micronumpy/test/test_numarray.py +++ b/pypy/module/micronumpy/test/test_numarray.py @@ -1746,6 +1746,18 @@ assert (b == a).all() b[1] = -1 assert a[0][1] == -1 + a = np.arange(9).reshape((3, 1, 3, 1)) + b = a.squeeze(1) + assert b.shape == (3, 3, 1) + b = a.squeeze((1,)) + assert b.shape == (3, 3, 1) + b = a.squeeze((1, -1)) + assert b.shape == (3, 3) + exc = raises(ValueError, a.squeeze, 5) + assert exc.value.message == "'axis' entry 5 is out of bounds [-4, 4)" + exc = raises(ValueError, a.squeeze, 0) + assert exc.value.message == "cannot select an axis to squeeze out " \ + "which has size greater than one" def test_swapaxes(self): from numpypy import array From noreply at buildbot.pypy.org Thu Nov 14 03:37:34 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Thu, 14 Nov 2013 03:37:34 +0100 (CET) Subject: [pypy-commit] pypy default: test duplicate axes also Message-ID: <20131114023734.7351C1C036B@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r68012:c780cd5bf67f Date: 2013-11-13 21:36 -0500 http://bitbucket.org/pypy/pypy/changeset/c780cd5bf67f/ Log: test duplicate axes also diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py --- a/pypy/module/micronumpy/test/test_numarray.py +++ b/pypy/module/micronumpy/test/test_numarray.py @@ -1758,6 +1758,8 @@ exc = raises(ValueError, a.squeeze, 0) assert exc.value.message == "cannot select an axis to squeeze out " \ "which has size greater than one" + exc = raises(ValueError, a.squeeze, (1, 1)) + assert exc.value.message == "duplicate value in 'axis'" def test_swapaxes(self): from numpypy import array From noreply at buildbot.pypy.org Thu Nov 14 05:38:31 2013 From: noreply at buildbot.pypy.org (pjenvey) Date: Thu, 14 Nov 2013 05:38:31 +0100 (CET) Subject: [pypy-commit] pypy py3k: reapply frame/cell related py3k changes Message-ID: <20131114043831.9D9A81C13E5@cobra.cs.uni-duesseldorf.de> Author: Philip Jenvey Branch: py3k Changeset: r68014:0e3f24e124ac Date: 2013-11-13 20:36 -0800 http://bitbucket.org/pypy/pypy/changeset/0e3f24e124ac/ Log: reapply frame/cell related py3k changes diff --git a/pypy/interpreter/pyframe.py b/pypy/interpreter/pyframe.py --- a/pypy/interpreter/pyframe.py +++ b/pypy/interpreter/pyframe.py @@ -510,7 +510,7 @@ for i in range(min(len(varnames), self.getcode().co_nlocals)): name = varnames[i] w_value = self.locals_stack_w[i] - w_name = self.space.wrap(name) + w_name = self.space.wrap(name.decode('utf-8')) if w_value is not None: self.space.setitem(self.w_locals, w_name, w_value) else: @@ -547,7 +547,7 @@ new_fastlocals_w = [None] * numlocals for i in range(min(len(varnames), numlocals)): - w_name = self.space.wrap(varnames[i]) + w_name = self.space.wrap(varnames[i].decode('utf-8')) try: w_value = self.space.getitem(self.w_locals, w_name) except OperationError, e: @@ -576,6 +576,8 @@ """ Initialize cellvars from self.locals_stack_w. """ + if self.cells is None: + return args_to_copy = self.pycode._args_as_cellvars for i in range(len(args_to_copy)): argnum = args_to_copy[i] diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py --- a/pypy/interpreter/pyopcode.py +++ b/pypy/interpreter/pyopcode.py @@ -484,15 +484,7 @@ try: w_value = cell.get() except ValueError: - varname = self.getfreevarname(varindex) - if self.iscellvar(varindex): - message = "local variable '%s' referenced before assignment" % varname - w_exc_type = self.space.w_UnboundLocalError - else: - message = ("free variable '%s' referenced before assignment" - " in enclosing scope" % varname) - w_exc_type = self.space.w_NameError - raise OperationError(w_exc_type, self.space.wrap(message)) + self.raise_exc_unbound(varindex) else: self.pushvalue(w_value) @@ -502,6 +494,26 @@ cell = self.cells[varindex] cell.set(w_newvalue) + def DELETE_DEREF(self, varindex, next_instr): + cell = self.cells[varindex] + try: + cell.get() + except ValueError: + self.raise_exc_unbound(varindex) + else: + cell.set(None) + + def raise_exc_unbound(self, varindex): + varname = self.getfreevarname(varindex) + if self.iscellvar(varindex): + message = "local variable '%s' referenced before assignment"%varname + w_exc_type = self.space.w_UnboundLocalError + else: + message = ("free variable '%s' referenced before assignment" + " in enclosing scope"%varname) + w_exc_type = self.space.w_NameError + raise OperationError(w_exc_type, self.space.wrap(message)) + def LOAD_CLOSURE(self, varindex, next_instr): # nested scopes: access the cell object cell = self.cells[varindex] @@ -1200,16 +1212,11 @@ return self._make_function(oparg) @jit.unroll_safe - def MAKE_CLOSURE(self, numdefaults, next_instr): - w_codeobj = self.popvalue() - codeobj = self.space.interp_w(pycode.PyCode, w_codeobj) - w_freevarstuple = self.popvalue() + def MAKE_CLOSURE(self, oparg, next_instr): + w_freevarstuple = self.peekvalue(1) freevars = [self.space.interp_w(Cell, cell) for cell in self.space.fixedview(w_freevarstuple)] - defaultarguments = self.popvalues(numdefaults) - fn = function.Function(self.space, codeobj, self.w_globals, - defaultarguments, freevars) - self.pushvalue(self.space.wrap(fn)) + self._make_function(oparg, freevars) def BUILD_SLICE(self, numargs, next_instr): if numargs == 3: From noreply at buildbot.pypy.org Thu Nov 14 05:38:30 2013 From: noreply at buildbot.pypy.org (pjenvey) Date: Thu, 14 Nov 2013 05:38:30 +0100 (CET) Subject: [pypy-commit] pypy py3k: merge default Message-ID: <20131114043830.51EA61C13B9@cobra.cs.uni-duesseldorf.de> Author: Philip Jenvey Branch: py3k Changeset: r68013:f9c10dfe0d96 Date: 2013-11-13 19:47 -0800 http://bitbucket.org/pypy/pypy/changeset/f9c10dfe0d96/ Log: merge default diff too long, truncating to 2000 out of 23611 lines diff --git a/Makefile b/Makefile --- a/Makefile +++ b/Makefile @@ -1,7 +1,38 @@ all: pypy-c +PYPY_EXECUTABLE := $(shell which pypy) +URAM := $(shell python -c "import sys; print 4.5 if sys.maxint>1<<32 else 2.5") + +ifeq ($(PYPY_EXECUTABLE),) +RUNINTERP = python +else +RUNINTERP = $(PYPY_EXECUTABLE) +endif + pypy-c: - @echo "Building PyPy with JIT, it'll take about 40 minutes and 4G of RAM" - @sleep 3 - rpython/bin/rpython -Ojit pypy/goal/targetpypystandalone.py + @echo + @echo "====================================================================" +ifeq ($(PYPY_EXECUTABLE),) + @echo "Building a regular (jitting) version of PyPy, using CPython." + @echo "This takes around 2 hours and $(URAM) GB of RAM." + @echo "Note that pre-installing a PyPy binary would reduce this time" + @echo "and produce basically the same result." +else + @echo "Building a regular (jitting) version of PyPy, using" + @echo "$(PYPY_EXECUTABLE) to run the translation itself." + @echo "This takes up to 1 hour and $(URAM) GB of RAM." +endif + @echo + @echo "For more control (e.g. to use multiple CPU cores during part of" + @echo "the process) you need to run \`\`rpython/bin/rpython'' directly." + @echo "For more information see \`\`http://pypy.org/download.html''." + @echo "====================================================================" + @echo + @sleep 5 + $(RUNINTERP) rpython/bin/rpython -Ojit pypy/goal/targetpypystandalone.py + +# Note: the -jN option, or MAKEFLAGS=-jN, are not usable. They are +# replaced with an opaque --jobserver option by the time this Makefile +# runs. We cannot get their original value either: +# http://lists.gnu.org/archive/html/help-make/2010-08/msg00106.html diff --git a/README.rst b/README.rst --- a/README.rst +++ b/README.rst @@ -1,5 +1,5 @@ ===================================== -PyPy: Python in Python Implementation +PyPy: Python in Python Implementation ===================================== Welcome to PyPy! @@ -26,9 +26,11 @@ Building ======== -build with:: +build with: - rpython/bin/rpython -Ojit pypy/goal/targetpypystandalone.py +.. code-block:: console + + $ rpython/bin/rpython -Ojit pypy/goal/targetpypystandalone.py This ends up with ``pypy-c`` binary in the main pypy directory. We suggest to use virtualenv with the resulting pypy-c as the interpreter, you can diff --git a/lib-python/2.7/json/encoder.py b/lib-python/2.7/json/encoder.py --- a/lib-python/2.7/json/encoder.py +++ b/lib-python/2.7/json/encoder.py @@ -4,6 +4,21 @@ from __pypy__.builders import StringBuilder, UnicodeBuilder +class StringOrUnicodeBuilder(object): + def __init__(self): + self._builder = StringBuilder() + def append(self, string): + try: + self._builder.append(string) + except UnicodeEncodeError: + ub = UnicodeBuilder() + ub.append(self._builder.build()) + self._builder = ub + ub.append(string) + def build(self): + return self._builder.build() + + ESCAPE = re.compile(r'[\x00-\x1f\\"\b\f\n\r\t]') ESCAPE_ASCII = re.compile(r'([\\"]|[^\ -~])') HAS_UTF8 = re.compile(r'[\x80-\xff]') @@ -192,7 +207,7 @@ if self.ensure_ascii: builder = StringBuilder() else: - builder = UnicodeBuilder() + builder = StringOrUnicodeBuilder() self.__encode(o, markers, builder, 0) return builder.build() diff --git a/lib-python/2.7/string.py b/lib-python/2.7/string.py --- a/lib-python/2.7/string.py +++ b/lib-python/2.7/string.py @@ -66,16 +66,17 @@ must be of the same length. """ - if len(fromstr) != len(tostr): + n = len(fromstr) + if n != len(tostr): raise ValueError, "maketrans arguments must have same length" - global _idmapL - if not _idmapL: - _idmapL = list(_idmap) - L = _idmapL[:] - fromstr = map(ord, fromstr) - for i in range(len(fromstr)): - L[fromstr[i]] = tostr[i] - return ''.join(L) + # this function has been rewritten to suit PyPy better; it is + # almost 10x faster than the original. + buf = bytearray(256) + for i in range(256): + buf[i] = i + for i in range(n): + buf[ord(fromstr[i])] = tostr[i] + return str(buf) diff --git a/lib-python/2.7/weakref.py b/lib-python/2.7/weakref.py --- a/lib-python/2.7/weakref.py +++ b/lib-python/2.7/weakref.py @@ -48,7 +48,14 @@ def remove(wr, selfref=ref(self)): self = selfref() if self is not None: - del self.data[wr.key] + # Changed this for PyPy: made more resistent. The + # issue is that in some corner cases, self.data + # might already be changed or removed by the time + # this weakref's callback is called. If that is + # the case, we don't want to randomly kill an + # unrelated entry. + if self.data.get(wr.key) is wr: + del self.data[wr.key] self._remove = remove UserDict.UserDict.__init__(self, *args, **kw) @@ -160,22 +167,26 @@ try: o = self.data.pop(key)() except KeyError: + o = None + if o is None: if args: return args[0] - raise - if o is None: raise KeyError, key else: return o + # The logic above was fixed in PyPy def setdefault(self, key, default=None): try: - wr = self.data[key] + o = self.data[key]() except KeyError: + o = None + if o is None: self.data[key] = KeyedRef(default, self._remove, key) return default else: - return wr() + return o + # The logic above was fixed in PyPy def update(self, dict=None, **kwargs): d = self.data diff --git a/lib_pypy/cffi.egg-info b/lib_pypy/cffi.egg-info --- a/lib_pypy/cffi.egg-info +++ b/lib_pypy/cffi.egg-info @@ -1,6 +1,6 @@ Metadata-Version: 1.0 Name: cffi -Version: 0.7 +Version: 0.8 Summary: Foreign Function Interface for Python calling C code. Home-page: http://cffi.readthedocs.org Author: Armin Rigo, Maciej Fijalkowski diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py --- a/lib_pypy/cffi/__init__.py +++ b/lib_pypy/cffi/__init__.py @@ -4,5 +4,5 @@ from .api import FFI, CDefError, FFIError from .ffiplatform import VerificationError, VerificationMissing -__version__ = "0.7.2" -__version_info__ = (0, 7, 2) +__version__ = "0.8" +__version_info__ = (0, 8) diff --git a/lib_pypy/cffi/api.py b/lib_pypy/cffi/api.py --- a/lib_pypy/cffi/api.py +++ b/lib_pypy/cffi/api.py @@ -1,4 +1,5 @@ import types +from .lock import allocate_lock try: callable @@ -61,6 +62,7 @@ # rely on it! It's probably not going to work well.) self._backend = backend + self._lock = allocate_lock() self._parser = cparser.Parser() self._cached_btypes = {} self._parsed_types = types.ModuleType('parsed_types').__dict__ @@ -74,7 +76,8 @@ if name.startswith('RTLD_'): setattr(self, name, getattr(backend, name)) # - self.BVoidP = self._get_cached_btype(model.voidp_type) + with self._lock: + self.BVoidP = self._get_cached_btype(model.voidp_type) if isinstance(backend, types.ModuleType): # _cffi_backend: attach these constants to the class if not hasattr(FFI, 'NULL'): @@ -95,11 +98,12 @@ if not isinstance(csource, basestring): raise TypeError("cdef() argument must be a string") csource = csource.encode('ascii') - self._parser.parse(csource, override=override) - self._cdefsources.append(csource) - if override: - for cache in self._function_caches: - cache.clear() + with self._lock: + self._parser.parse(csource, override=override) + self._cdefsources.append(csource) + if override: + for cache in self._function_caches: + cache.clear() def dlopen(self, name, flags=0): """Load and return a dynamic library identified by 'name'. @@ -109,31 +113,47 @@ library we only look for the actual (untyped) symbols. """ assert isinstance(name, basestring) or name is None - lib, function_cache = _make_ffi_library(self, name, flags) - self._function_caches.append(function_cache) - self._libraries.append(lib) + with self._lock: + lib, function_cache = _make_ffi_library(self, name, flags) + self._function_caches.append(function_cache) + self._libraries.append(lib) return lib + def _typeof_locked(self, cdecl): + # call me with the lock! + key = cdecl + if key in self._parsed_types: + return self._parsed_types[key] + # + if not isinstance(cdecl, str): # unicode, on Python 2 + cdecl = cdecl.encode('ascii') + # + type = self._parser.parse_type(cdecl) + really_a_function_type = type.is_raw_function + if really_a_function_type: + type = type.as_function_pointer() + btype = self._get_cached_btype(type) + result = btype, really_a_function_type + self._parsed_types[key] = result + return result + def _typeof(self, cdecl, consider_function_as_funcptr=False): # string -> ctype object try: - btype, cfaf = self._parsed_types[cdecl] - if consider_function_as_funcptr and not cfaf: - raise KeyError + result = self._parsed_types[cdecl] except KeyError: - key = cdecl - if not isinstance(cdecl, str): # unicode, on Python 2 - cdecl = cdecl.encode('ascii') - cfaf = consider_function_as_funcptr - type = self._parser.parse_type(cdecl, - consider_function_as_funcptr=cfaf) - btype = self._get_cached_btype(type) - self._parsed_types[key] = btype, cfaf + with self._lock: + result = self._typeof_locked(cdecl) + # + btype, really_a_function_type = result + if really_a_function_type and not consider_function_as_funcptr: + raise CDefError("the type %r is a function type, not a " + "pointer-to-function type" % (cdecl,)) return btype def typeof(self, cdecl): """Parse the C type given as a string and return the - corresponding Python type: '>. + corresponding object. It can also be used on 'cdata' instance to get its C type. """ if isinstance(cdecl, basestring): @@ -144,6 +164,10 @@ res = _builtin_function_type(cdecl) if res is not None: return res + if (isinstance(cdecl, types.FunctionType) + and hasattr(cdecl, '_cffi_base_type')): + with self._lock: + return self._get_cached_btype(cdecl._cffi_base_type) raise TypeError(type(cdecl)) def sizeof(self, cdecl): @@ -280,14 +304,17 @@ data. Later, when this new cdata object is garbage-collected, 'destructor(old_cdata_object)' will be called. """ - try: - gc_weakrefs = self.gc_weakrefs - except AttributeError: - from .gc_weakref import GcWeakrefs - gc_weakrefs = self.gc_weakrefs = GcWeakrefs(self) - return gc_weakrefs.build(cdata, destructor) + with self._lock: + try: + gc_weakrefs = self.gc_weakrefs + except AttributeError: + from .gc_weakref import GcWeakrefs + gc_weakrefs = self.gc_weakrefs = GcWeakrefs(self) + return gc_weakrefs.build(cdata, destructor) def _get_cached_btype(self, type): + assert self._lock.acquire(False) is False + # call me with the lock! try: BType = self._cached_btypes[type] except KeyError: @@ -322,7 +349,8 @@ def _pointer_to(self, ctype): from . import model - return model.pointer_cache(self, ctype) + with self._lock: + return model.pointer_cache(self, ctype) def addressof(self, cdata, field=None): """Return the address of a . @@ -342,10 +370,12 @@ variables, which must anyway be accessed directly from the lib object returned by the original FFI instance. """ - self._parser.include(ffi_to_include._parser) - self._cdefsources.append('[') - self._cdefsources.extend(ffi_to_include._cdefsources) - self._cdefsources.append(']') + with ffi_to_include._lock: + with self._lock: + self._parser.include(ffi_to_include._parser) + self._cdefsources.append('[') + self._cdefsources.extend(ffi_to_include._cdefsources) + self._cdefsources.append(']') def new_handle(self, x): return self._backend.newp_handle(self.BVoidP, x) @@ -372,7 +402,7 @@ backendlib = backend.load_library(path, flags) copied_enums = [] # - def make_accessor(name): + def make_accessor_locked(name): key = 'function ' + name if key in ffi._parser._declarations: tp = ffi._parser._declarations[key] @@ -404,11 +434,17 @@ if enumname not in library.__dict__: library.__dict__[enumname] = enumval copied_enums.append(True) + if name in library.__dict__: + return # - if name in library.__dict__: # copied from an enum value just above, - return # or multithread's race condition raise AttributeError(name) # + def make_accessor(name): + with ffi._lock: + if name in library.__dict__ or name in FFILibrary.__dict__: + return # added by another thread while waiting for the lock + make_accessor_locked(name) + # class FFILibrary(object): def __getattr__(self, name): make_accessor(name) @@ -444,4 +480,5 @@ except (KeyError, AttributeError, TypeError): return None else: - return ffi._get_cached_btype(tp) + with ffi._lock: + return ffi._get_cached_btype(tp) diff --git a/lib_pypy/cffi/cparser.py b/lib_pypy/cffi/cparser.py --- a/lib_pypy/cffi/cparser.py +++ b/lib_pypy/cffi/cparser.py @@ -142,7 +142,7 @@ if 1 <= linenum <= len(csourcelines): line = csourcelines[linenum-1] if line: - msg = 'cannot parse "%s"\n%s' % (line, msg) + msg = 'cannot parse "%s"\n%s' % (line.strip(), msg) else: msg = 'parse error\n%s' % (msg,) raise api.CDefError(msg) @@ -217,19 +217,18 @@ # if decl.name: tp = self._get_type(node, partial_length_ok=True) - if self._is_constant_declaration(node): + if self._is_constant_globalvar(node): self._declare('constant ' + decl.name, tp) else: self._declare('variable ' + decl.name, tp) - def parse_type(self, cdecl, consider_function_as_funcptr=False): + def parse_type(self, cdecl): ast, macros = self._parse('void __dummy(\n%s\n);' % cdecl) assert not macros exprnode = ast.ext[-1].type.args.params[0] if isinstance(exprnode, pycparser.c_ast.ID): raise api.CDefError("unknown identifier '%s'" % (exprnode.name,)) - return self._get_type(exprnode.type, - consider_function_as_funcptr=consider_function_as_funcptr) + return self._get_type(exprnode.type) def _declare(self, name, obj): if name in self._declarations: @@ -249,28 +248,17 @@ return model.ConstPointerType(type) return model.PointerType(type) - def _get_type(self, typenode, convert_array_to_pointer=False, - name=None, partial_length_ok=False, - consider_function_as_funcptr=False): + def _get_type(self, typenode, name=None, partial_length_ok=False): # first, dereference typedefs, if we have it already parsed, we're good if (isinstance(typenode, pycparser.c_ast.TypeDecl) and isinstance(typenode.type, pycparser.c_ast.IdentifierType) and len(typenode.type.names) == 1 and ('typedef ' + typenode.type.names[0]) in self._declarations): type = self._declarations['typedef ' + typenode.type.names[0]] - if isinstance(type, model.ArrayType): - if convert_array_to_pointer: - return type.item - else: - if (consider_function_as_funcptr and - isinstance(type, model.RawFunctionType)): - return type.as_function_pointer() return type # if isinstance(typenode, pycparser.c_ast.ArrayDecl): # array type - if convert_array_to_pointer: - return self._get_type_pointer(self._get_type(typenode.type)) if typenode.dim is None: length = None else: @@ -331,10 +319,7 @@ # if isinstance(typenode, pycparser.c_ast.FuncDecl): # a function type - result = self._parse_function_type(typenode, name) - if consider_function_as_funcptr: - result = result.as_function_pointer() - return result + return self._parse_function_type(typenode, name) # # nested anonymous structs or unions end up here if isinstance(typenode, pycparser.c_ast.Struct): @@ -365,21 +350,24 @@ isinstance(params[0].type.type, pycparser.c_ast.IdentifierType) and list(params[0].type.type.names) == ['void']): del params[0] - args = [self._get_type(argdeclnode.type, - convert_array_to_pointer=True, - consider_function_as_funcptr=True) + args = [self._as_func_arg(self._get_type(argdeclnode.type)) for argdeclnode in params] result = self._get_type(typenode.type) return model.RawFunctionType(tuple(args), result, ellipsis) - def _is_constant_declaration(self, typenode, const=False): - if isinstance(typenode, pycparser.c_ast.ArrayDecl): - return self._is_constant_declaration(typenode.type) + def _as_func_arg(self, type): + if isinstance(type, model.ArrayType): + return model.PointerType(type.item) + elif isinstance(type, model.RawFunctionType): + return type.as_function_pointer() + else: + return type + + def _is_constant_globalvar(self, typenode): if isinstance(typenode, pycparser.c_ast.PtrDecl): - const = 'const' in typenode.quals - return self._is_constant_declaration(typenode.type, const) + return 'const' in typenode.quals if isinstance(typenode, pycparser.c_ast.TypeDecl): - return const or 'const' in typenode.quals + return 'const' in typenode.quals return False def _get_struct_union_enum_type(self, kind, type, name=None, nested=False): @@ -491,7 +479,7 @@ return tp def _make_partial(self, tp, nested): - if not isinstance(tp, model.StructType): + if not isinstance(tp, model.StructOrUnion): raise api.CDefError("%s cannot be partial" % (tp,)) if not tp.has_c_name() and not nested: raise NotImplementedError("%s is partial but has no C name" %(tp,)) @@ -511,7 +499,7 @@ if (isinstance(exprnode, pycparser.c_ast.ID) and exprnode.name == '__dotdotdotarray__'): self._partial_length = True - return None + return '...' # raise api.FFIError("unsupported expression: expected a " "simple numeric constant") diff --git a/lib_pypy/cffi/gc_weakref.py b/lib_pypy/cffi/gc_weakref.py --- a/lib_pypy/cffi/gc_weakref.py +++ b/lib_pypy/cffi/gc_weakref.py @@ -14,6 +14,6 @@ def build(self, cdata, destructor): # make a new cdata of the same type as the original one - new_cdata = self.ffi.cast(self.ffi.typeof(cdata), cdata) + new_cdata = self.ffi.cast(self.ffi._backend.typeof(cdata), cdata) self.data[ref(new_cdata, self.remove)] = destructor, cdata return new_cdata diff --git a/lib_pypy/cffi/lock.py b/lib_pypy/cffi/lock.py new file mode 100644 --- /dev/null +++ b/lib_pypy/cffi/lock.py @@ -0,0 +1,30 @@ +import sys + +if sys.version_info < (3,): + try: + from thread import allocate_lock + except ImportError: + from dummy_thread import allocate_lock +else: + try: + from _thread import allocate_lock + except ImportError: + from _dummy_thread import allocate_lock + + +##import sys +##l1 = allocate_lock + +##class allocate_lock(object): +## def __init__(self): +## self._real = l1() +## def __enter__(self): +## for i in range(4, 0, -1): +## print sys._getframe(i).f_code +## print +## return self._real.__enter__() +## def __exit__(self, *args): +## return self._real.__exit__(*args) +## def acquire(self, f): +## assert f is False +## return self._real.acquire(f) diff --git a/lib_pypy/cffi/model.py b/lib_pypy/cffi/model.py --- a/lib_pypy/cffi/model.py +++ b/lib_pypy/cffi/model.py @@ -1,7 +1,10 @@ import weakref +from .lock import allocate_lock + class BaseTypeByIdentity(object): is_array_type = False + is_raw_function = False def get_c_name(self, replace_with='', context='a C file'): result = self.c_name_with_marker @@ -146,6 +149,7 @@ # a function, but not a pointer-to-function. The backend has no # notion of such a type; it's used temporarily by parsing. _base_pattern = '(&)(%s)' + is_raw_function = True def build_backend_type(self, ffi, finishlist): from . import api @@ -192,10 +196,6 @@ _base_pattern = " const *&" _base_pattern_array = "(const *&)" - def build_backend_type(self, ffi, finishlist): - BPtr = PointerType(self.totype).get_cached_btype(ffi, finishlist) - return BPtr - const_voidp_type = ConstPointerType(void_type) @@ -216,10 +216,12 @@ self.item = item self.length = length # - if self.length is None: + if length is None: brackets = '&[]' + elif length == '...': + brackets = '&[/*...*/]' else: - brackets = '&[%d]' % self.length + brackets = '&[%d]' % length self.c_name_with_marker = ( self.item.c_name_with_marker.replace('&', brackets)) @@ -227,6 +229,10 @@ return ArrayType(self.item, newlength) def build_backend_type(self, ffi, finishlist): + if self.length == '...': + from . import api + raise api.CDefError("cannot render the type %r: unknown length" % + (self,)) self.item.get_cached_btype(ffi, finishlist) # force the item BType BPtrItem = PointerType(self.item).get_cached_btype(ffi, finishlist) return global_cache(self, ffi, 'new_array_type', BPtrItem, self.length) @@ -252,6 +258,7 @@ class StructOrUnion(StructOrUnionOrEnum): fixedlayout = None completed = False + partial = False def __init__(self, name, fldnames, fldtypes, fldbitsize): self.name = name @@ -303,20 +310,21 @@ return # not completing it: it's an opaque struct # self.completed = 1 - fldtypes = tuple(tp.get_cached_btype(ffi, finishlist) - for tp in self.fldtypes) # if self.fixedlayout is None: + fldtypes = [tp.get_cached_btype(ffi, finishlist) + for tp in self.fldtypes] lst = list(zip(self.fldnames, fldtypes, self.fldbitsize)) ffi._backend.complete_struct_or_union(BType, lst, self) # else: + fldtypes = [] fieldofs, fieldsize, totalsize, totalalignment = self.fixedlayout for i in range(len(self.fldnames)): fsize = fieldsize[i] ftype = self.fldtypes[i] # - if isinstance(ftype, ArrayType) and ftype.length is None: + if isinstance(ftype, ArrayType) and ftype.length == '...': # fix the length to match the total size BItemType = ftype.item.get_cached_btype(ffi, finishlist) nlen, nrest = divmod(fsize, ffi.sizeof(BItemType)) @@ -327,18 +335,20 @@ ftype = ftype.resolve_length(nlen) self.fldtypes = (self.fldtypes[:i] + (ftype,) + self.fldtypes[i+1:]) - BArrayType = ftype.get_cached_btype(ffi, finishlist) - fldtypes = (fldtypes[:i] + (BArrayType,) + - fldtypes[i+1:]) - continue # - bitemsize = ffi.sizeof(fldtypes[i]) - if bitemsize != fsize: - self._verification_error( - "field '%s.%s' is declared as %d bytes, but is " - "really %d bytes" % (self.name, - self.fldnames[i] or '{}', - bitemsize, fsize)) + BFieldType = ftype.get_cached_btype(ffi, finishlist) + if isinstance(ftype, ArrayType) and ftype.length is None: + assert fsize == 0 + else: + bitemsize = ffi.sizeof(BFieldType) + if bitemsize != fsize: + self._verification_error( + "field '%s.%s' is declared as %d bytes, but is " + "really %d bytes" % (self.name, + self.fldnames[i] or '{}', + bitemsize, fsize)) + fldtypes.append(BFieldType) + # lst = list(zip(self.fldnames, fldtypes, self.fldbitsize, fieldofs)) ffi._backend.complete_struct_or_union(BType, lst, self, totalsize, totalalignment) @@ -348,11 +358,6 @@ from .ffiplatform import VerificationError raise VerificationError(msg) - -class StructType(StructOrUnion): - kind = 'struct' - partial = False - def check_not_partial(self): if self.partial and self.fixedlayout is None: from . import ffiplatform @@ -361,19 +366,18 @@ def build_backend_type(self, ffi, finishlist): self.check_not_partial() finishlist.append(self) - - return global_cache(self, ffi, 'new_struct_type', + # + return global_cache(self, ffi, 'new_%s_type' % self.kind, self.get_official_name(), key=self) +class StructType(StructOrUnion): + kind = 'struct' + + class UnionType(StructOrUnion): kind = 'union' - def build_backend_type(self, ffi, finishlist): - finishlist.append(self) - return global_cache(self, ffi, 'new_union_type', - self.get_official_name(), key=self) - class EnumType(StructOrUnionOrEnum): kind = 'enum' @@ -387,6 +391,12 @@ self.baseinttype = baseinttype self.build_c_name_with_marker() + def force_the_name(self, forcename): + StructOrUnionOrEnum.force_the_name(self, forcename) + if self.forcename is None: + name = self.get_official_name() + self.forcename = '$' + name.replace(' ', '_') + def check_not_partial(self): if self.partial and not self.partial_resolved: from . import ffiplatform @@ -444,6 +454,9 @@ tp = StructType(structname, None, None, None) return NamedPointerType(tp, name) + +global_lock = allocate_lock() + def global_cache(srctype, ffi, funcname, *args, **kwds): key = kwds.pop('key', (funcname, args)) assert not kwds @@ -464,8 +477,17 @@ res = getattr(ffi._backend, funcname)(*args) except NotImplementedError as e: raise NotImplementedError("%r: %s" % (srctype, e)) - ffi._backend.__typecache[key] = res - return res + # note that setdefault() on WeakValueDictionary is not atomic + # and contains a rare bug (http://bugs.python.org/issue19542); + # we have to use a lock and do it ourselves + cache = ffi._backend.__typecache + with global_lock: + res1 = cache.get(key) + if res1 is None: + cache[key] = res + return res + else: + return res1 def pointer_cache(ffi, BType): return global_cache('?', ffi, 'new_pointer_type', BType) diff --git a/lib_pypy/cffi/vengine_cpy.py b/lib_pypy/cffi/vengine_cpy.py --- a/lib_pypy/cffi/vengine_cpy.py +++ b/lib_pypy/cffi/vengine_cpy.py @@ -15,7 +15,7 @@ def patch_extension_kwds(self, kwds): pass - def find_module(self, module_name, path, so_suffix): + def find_module(self, module_name, path, so_suffixes): try: f, filename, descr = imp.find_module(module_name, path) except ImportError: @@ -25,7 +25,7 @@ # Note that after a setuptools installation, there are both .py # and .so files with the same basename. The code here relies on # imp.find_module() locating the .so in priority. - if descr[0] != so_suffix: + if descr[0] not in so_suffixes: return None return filename @@ -160,7 +160,10 @@ def __dir__(self): return FFILibrary._cffi_dir + list(self.__dict__) library = FFILibrary() - module._cffi_setup(lst, ffiplatform.VerificationError, library) + if module._cffi_setup(lst, ffiplatform.VerificationError, library): + import warnings + warnings.warn("reimporting %r might overwrite older definitions" + % (self.verifier.get_module_name())) # # finally, call the loaded_cpy_xxx() functions. This will perform # the final adjustments, like copying the Python->C wrapper @@ -280,8 +283,8 @@ return '_cffi_from_c_pointer((char *)%s, _cffi_type(%d))' % ( var, self._gettypenum(tp)) elif isinstance(tp, model.ArrayType): - return '_cffi_from_c_deref((char *)%s, _cffi_type(%d))' % ( - var, self._gettypenum(tp)) + return '_cffi_from_c_pointer((char *)%s, _cffi_type(%d))' % ( + var, self._gettypenum(model.PointerType(tp.item))) elif isinstance(tp, model.StructType): if tp.fldnames is None: raise TypeError("'%s' is used as %s, but is opaque" % ( @@ -464,11 +467,14 @@ prnt(' static Py_ssize_t nums[] = {') prnt(' sizeof(%s),' % cname) prnt(' offsetof(struct _cffi_aligncheck, y),') - for fname, _, fbitsize in tp.enumfields(): + for fname, ftype, fbitsize in tp.enumfields(): if fbitsize >= 0: continue # xxx ignore fbitsize for now prnt(' offsetof(%s, %s),' % (cname, fname)) - prnt(' sizeof(((%s *)0)->%s),' % (cname, fname)) + if isinstance(ftype, model.ArrayType) and ftype.length is None: + prnt(' 0, /* %s */' % ftype._get_c_name()) + else: + prnt(' sizeof(((%s *)0)->%s),' % (cname, fname)) prnt(' -1') prnt(' };') prnt(' return _cffi_get_struct_layout(nums);') @@ -491,7 +497,7 @@ # function = getattr(module, layoutfuncname) layout = function() - if isinstance(tp, model.StructType) and tp.partial: + if isinstance(tp, model.StructOrUnion) and tp.partial: # use the function()'s sizes and offsets to guide the # layout of the struct totalsize = layout[0] @@ -528,9 +534,10 @@ continue # xxx ignore fbitsize for now check(layout[i], ffi.offsetof(BStruct, fname), "wrong offset for field %r" % (fname,)) - BField = ffi._get_cached_btype(ftype) - check(layout[i+1], ffi.sizeof(BField), - "wrong size for field %r" % (fname,)) + if layout[i+1] != 0: + BField = ffi._get_cached_btype(ftype) + check(layout[i+1], ffi.sizeof(BField), + "wrong size for field %r" % (fname,)) i += 2 assert i == len(layout) @@ -566,7 +573,7 @@ # constants, likely declared with '#define' def _generate_cpy_const(self, is_int, name, tp=None, category='const', - vartp=None, delayed=True): + vartp=None, delayed=True, size_too=False): prnt = self._prnt funcname = '_cffi_%s_%s' % (category, name) prnt('static int %s(PyObject *lib)' % funcname) @@ -597,6 +604,15 @@ '(unsigned long long)(%s));' % (name,)) prnt(' if (o == NULL)') prnt(' return -1;') + if size_too: + prnt(' {') + prnt(' PyObject *o1 = o;') + prnt(' o = Py_BuildValue("On", o1, (Py_ssize_t)sizeof(%s));' + % (name,)) + prnt(' Py_DECREF(o1);') + prnt(' if (o == NULL)') + prnt(' return -1;') + prnt(' }') prnt(' res = PyObject_SetAttrString(lib, "%s", o);' % name) prnt(' Py_DECREF(o);') prnt(' if (res < 0)') @@ -633,12 +649,23 @@ prnt('static int %s(PyObject *lib)' % funcname) prnt('{') for enumerator, enumvalue in zip(tp.enumerators, tp.enumvalues): - prnt(' if (%s != %d) {' % (enumerator, enumvalue)) + if enumvalue < 0: + prnt(' if ((%s) >= 0 || (long)(%s) != %dL) {' % ( + enumerator, enumerator, enumvalue)) + else: + prnt(' if ((%s) < 0 || (unsigned long)(%s) != %dUL) {' % ( + enumerator, enumerator, enumvalue)) + prnt(' char buf[64];') + prnt(' if ((%s) < 0)' % enumerator) + prnt(' snprintf(buf, 63, "%%ld", (long)(%s));' % enumerator) + prnt(' else') + prnt(' snprintf(buf, 63, "%%lu", (unsigned long)(%s));' % + enumerator) prnt(' PyErr_Format(_cffi_VerificationError,') - prnt(' "enum %s: %s has the real value %d, ' - 'not %d",') - prnt(' "%s", "%s", (int)%s, %d);' % ( - name, enumerator, enumerator, enumvalue)) + prnt(' "enum %s: %s has the real value %s, ' + 'not %s",') + prnt(' "%s", "%s", buf, "%d");' % ( + name, enumerator, enumvalue)) prnt(' return -1;') prnt(' }') prnt(' return %s;' % self._chained_list_constants[True]) @@ -677,15 +704,16 @@ def _generate_cpy_variable_collecttype(self, tp, name): if isinstance(tp, model.ArrayType): - self._do_collect_type(tp) + tp_ptr = model.PointerType(tp.item) else: tp_ptr = model.PointerType(tp) - self._do_collect_type(tp_ptr) + self._do_collect_type(tp_ptr) def _generate_cpy_variable_decl(self, tp, name): if isinstance(tp, model.ArrayType): tp_ptr = model.PointerType(tp.item) - self._generate_cpy_const(False, name, tp, vartp=tp_ptr) + self._generate_cpy_const(False, name, tp, vartp=tp_ptr, + size_too = (tp.length == '...')) else: tp_ptr = model.PointerType(tp) self._generate_cpy_const(False, name, tp_ptr, category='var') @@ -694,11 +722,29 @@ _loading_cpy_variable = _loaded_noop def _loaded_cpy_variable(self, tp, name, module, library): + value = getattr(library, name) if isinstance(tp, model.ArrayType): # int a[5] is "constant" in the - return # sense that "a=..." is forbidden + # sense that "a=..." is forbidden + if tp.length == '...': + assert isinstance(value, tuple) + (value, size) = value + BItemType = self.ffi._get_cached_btype(tp.item) + length, rest = divmod(size, self.ffi.sizeof(BItemType)) + if rest != 0: + raise ffiplatform.VerificationError( + "bad size: %r does not seem to be an array of %s" % + (name, tp.item)) + tp = tp.resolve_length(length) + # 'value' is a which we have to replace with + # a if the N is actually known + if tp.length is not None: + BArray = self.ffi._get_cached_btype(tp) + value = self.ffi.cast(BArray, value) + setattr(library, name, value) + return # remove ptr= from the library instance, and replace # it by a property on the class, which reads/writes into ptr[0]. - ptr = getattr(library, name) + ptr = value delattr(library, name) def getter(library): return ptr[0] @@ -711,12 +757,9 @@ def _generate_setup_custom(self): prnt = self._prnt - prnt('static PyObject *_cffi_setup_custom(PyObject *lib)') + prnt('static int _cffi_setup_custom(PyObject *lib)') prnt('{') - prnt(' if (%s < 0)' % self._chained_list_constants[True]) - prnt(' return NULL;') - prnt(' Py_INCREF(Py_None);') - prnt(' return Py_None;') + prnt(' return %s;' % self._chained_list_constants[True]) prnt('}') cffimod_header = r''' @@ -834,17 +877,20 @@ static void *_cffi_exports[_CFFI_NUM_EXPORTS]; static PyObject *_cffi_types, *_cffi_VerificationError; -static PyObject *_cffi_setup_custom(PyObject *lib); /* forward */ +static int _cffi_setup_custom(PyObject *lib); /* forward */ static PyObject *_cffi_setup(PyObject *self, PyObject *args) { PyObject *library; + int was_alive = (_cffi_types != NULL); if (!PyArg_ParseTuple(args, "OOO", &_cffi_types, &_cffi_VerificationError, &library)) return NULL; Py_INCREF(_cffi_types); Py_INCREF(_cffi_VerificationError); - return _cffi_setup_custom(library); + if (_cffi_setup_custom(library) < 0) + return NULL; + return PyBool_FromLong(was_alive); } static void _cffi_init(void) diff --git a/lib_pypy/cffi/vengine_gen.py b/lib_pypy/cffi/vengine_gen.py --- a/lib_pypy/cffi/vengine_gen.py +++ b/lib_pypy/cffi/vengine_gen.py @@ -20,15 +20,15 @@ # up in kwds['export_symbols']. kwds.setdefault('export_symbols', self.export_symbols) - def find_module(self, module_name, path, so_suffix): - basename = module_name + so_suffix - if path is None: - path = sys.path - for dirname in path: - filename = os.path.join(dirname, basename) - if os.path.isfile(filename): - return filename - return None + def find_module(self, module_name, path, so_suffixes): + for so_suffix in so_suffixes: + basename = module_name + so_suffix + if path is None: + path = sys.path + for dirname in path: + filename = os.path.join(dirname, basename) + if os.path.isfile(filename): + return filename def collect_types(self): pass # not needed in the generic engine @@ -173,6 +173,7 @@ newfunction = self._load_constant(False, tp, name, module) else: indirections = [] + base_tp = tp if any(isinstance(typ, model.StructOrUnion) for typ in tp.args): indirect_args = [] for i, typ in enumerate(tp.args): @@ -186,16 +187,18 @@ wrappername = '_cffi_f_%s' % name newfunction = module.load_function(BFunc, wrappername) for i, typ in indirections: - newfunction = self._make_struct_wrapper(newfunction, i, typ) + newfunction = self._make_struct_wrapper(newfunction, i, typ, + base_tp) setattr(library, name, newfunction) type(library)._cffi_dir.append(name) - def _make_struct_wrapper(self, oldfunc, i, tp): + def _make_struct_wrapper(self, oldfunc, i, tp, base_tp): backend = self.ffi._backend BType = self.ffi._get_cached_btype(tp) def newfunc(*args): args = args[:i] + (backend.newp(BType, args[i]),) + args[i+1:] return oldfunc(*args) + newfunc._cffi_base_type = base_tp return newfunc # ---------- @@ -252,11 +255,14 @@ prnt(' static ssize_t nums[] = {') prnt(' sizeof(%s),' % cname) prnt(' offsetof(struct _cffi_aligncheck, y),') - for fname, _, fbitsize in tp.enumfields(): + for fname, ftype, fbitsize in tp.enumfields(): if fbitsize >= 0: continue # xxx ignore fbitsize for now prnt(' offsetof(%s, %s),' % (cname, fname)) - prnt(' sizeof(((%s *)0)->%s),' % (cname, fname)) + if isinstance(ftype, model.ArrayType) and ftype.length is None: + prnt(' 0, /* %s */' % ftype._get_c_name()) + else: + prnt(' sizeof(((%s *)0)->%s),' % (cname, fname)) prnt(' -1') prnt(' };') prnt(' return nums[i];') @@ -270,7 +276,7 @@ return # nothing to do with opaque structs layoutfuncname = '_cffi_layout_%s_%s' % (prefix, name) # - BFunc = self.ffi.typeof("ssize_t(*)(ssize_t)") + BFunc = self.ffi._typeof_locked("ssize_t(*)(ssize_t)")[0] function = module.load_function(BFunc, layoutfuncname) layout = [] num = 0 @@ -279,7 +285,7 @@ if x < 0: break layout.append(x) num += 1 - if isinstance(tp, model.StructType) and tp.partial: + if isinstance(tp, model.StructOrUnion) and tp.partial: # use the function()'s sizes and offsets to guide the # layout of the struct totalsize = layout[0] @@ -316,9 +322,10 @@ continue # xxx ignore fbitsize for now check(layout[i], ffi.offsetof(BStruct, fname), "wrong offset for field %r" % (fname,)) - BField = ffi._get_cached_btype(ftype) - check(layout[i+1], ffi.sizeof(BField), - "wrong size for field %r" % (fname,)) + if layout[i+1] != 0: + BField = ffi._get_cached_btype(ftype) + check(layout[i+1], ffi.sizeof(BField), + "wrong size for field %r" % (fname,)) i += 2 assert i == len(layout) @@ -379,15 +386,17 @@ def _load_constant(self, is_int, tp, name, module): funcname = '_cffi_const_%s' % name if is_int: - BFunc = self.ffi.typeof("int(*)(long long*)") + BType = self.ffi._typeof_locked("long long*")[0] + BFunc = self.ffi._typeof_locked("int(*)(long long*)")[0] function = module.load_function(BFunc, funcname) - p = self.ffi.new("long long*") + p = self.ffi.new(BType) negative = function(p) value = int(p[0]) if value < 0 and not negative: - value += (1 << (8*self.ffi.sizeof("long long"))) + BLongLong = self.ffi._typeof_locked("long long")[0] + value += (1 << (8*self.ffi.sizeof(BLongLong))) else: - BFunc = self.ffi.typeof(tp.get_c_name('(*)(void)', name)) + BFunc = self.ffi._typeof_locked(tp.get_c_name('(*)(void)', name))[0] function = module.load_function(BFunc, funcname) value = function() return value @@ -413,11 +422,22 @@ prnt('int %s(char *out_error)' % funcname) prnt('{') for enumerator, enumvalue in zip(tp.enumerators, tp.enumvalues): - prnt(' if (%s != %d) {' % (enumerator, enumvalue)) + if enumvalue < 0: + prnt(' if ((%s) >= 0 || (long)(%s) != %dL) {' % ( + enumerator, enumerator, enumvalue)) + else: + prnt(' if ((%s) < 0 || (unsigned long)(%s) != %dUL) {' % ( + enumerator, enumerator, enumvalue)) + prnt(' char buf[64];') + prnt(' if ((%s) < 0)' % enumerator) + prnt(' snprintf(buf, 63, "%%ld", (long)(%s));' % enumerator) + prnt(' else') + prnt(' snprintf(buf, 63, "%%lu", (unsigned long)(%s));' % + enumerator) prnt(' snprintf(out_error, 255,' - '"%s has the real value %d, not %d",') - prnt(' "%s", (int)%s, %d);' % ( - enumerator, enumerator, enumvalue)) + ' "%s has the real value %s, not %s",') + prnt(' "%s", buf, "%d");' % ( + enumerator, enumvalue)) prnt(' return -1;') prnt(' }') prnt(' return 0;') @@ -431,10 +451,11 @@ tp.enumvalues = tuple(enumvalues) tp.partial_resolved = True else: - BFunc = self.ffi.typeof("int(*)(char*)") + BType = self.ffi._typeof_locked("char[]")[0] + BFunc = self.ffi._typeof_locked("int(*)(char*)")[0] funcname = '_cffi_e_%s_%s' % (prefix, name) function = module.load_function(BFunc, funcname) - p = self.ffi.new("char[]", 256) + p = self.ffi.new(BType, 256) if function(p) < 0: error = self.ffi.string(p) if sys.version_info >= (3,): @@ -465,6 +486,14 @@ def _generate_gen_variable_decl(self, tp, name): if isinstance(tp, model.ArrayType): + if tp.length == '...': + prnt = self._prnt + funcname = '_cffi_sizeof_%s' % (name,) + self.export_symbols.append(funcname) + prnt("size_t %s(void)" % funcname) + prnt("{") + prnt(" return sizeof(%s);" % (name,)) + prnt("}") tp_ptr = model.PointerType(tp.item) self._generate_gen_const(False, name, tp_ptr) else: @@ -476,6 +505,18 @@ def _loaded_gen_variable(self, tp, name, module, library): if isinstance(tp, model.ArrayType): # int a[5] is "constant" in the # sense that "a=..." is forbidden + if tp.length == '...': + funcname = '_cffi_sizeof_%s' % (name,) + BFunc = self.ffi._typeof_locked('size_t(*)(void)')[0] + function = module.load_function(BFunc, funcname) + size = function() + BItemType = self.ffi._get_cached_btype(tp.item) + length, rest = divmod(size, self.ffi.sizeof(BItemType)) + if rest != 0: + raise ffiplatform.VerificationError( + "bad size: %r does not seem to be an array of %s" % + (name, tp.item)) + tp = tp.resolve_length(length) tp_ptr = model.PointerType(tp.item) value = self._load_constant(False, tp_ptr, name, module) # 'value' is a which we have to replace with @@ -489,7 +530,7 @@ # remove ptr= from the library instance, and replace # it by a property on the class, which reads/writes into ptr[0]. funcname = '_cffi_var_%s' % name - BFunc = self.ffi.typeof(tp.get_c_name('*(*)(void)', name)) + BFunc = self.ffi._typeof_locked(tp.get_c_name('*(*)(void)', name))[0] function = module.load_function(BFunc, funcname) ptr = function() def getter(library): diff --git a/lib_pypy/cffi/verifier.py b/lib_pypy/cffi/verifier.py --- a/lib_pypy/cffi/verifier.py +++ b/lib_pypy/cffi/verifier.py @@ -31,7 +31,7 @@ k2 = k2.lstrip('0').rstrip('L') modulename = '_cffi_%s_%s%s%s' % (tag, self._vengine._class_key, k1, k2) - suffix = _get_so_suffix() + suffix = _get_so_suffixes()[0] self.tmpdir = tmpdir or _caller_dir_pycache() self.sourcefilename = os.path.join(self.tmpdir, modulename + '.c') self.modulefilename = os.path.join(self.tmpdir, modulename + suffix) @@ -42,18 +42,21 @@ def write_source(self, file=None): """Write the C source code. It is produced in 'self.sourcefilename', which can be tweaked beforehand.""" - if self._has_source and file is None: - raise ffiplatform.VerificationError("source code already written") - self._write_source(file) + with self.ffi._lock: + if self._has_source and file is None: + raise ffiplatform.VerificationError( + "source code already written") + self._write_source(file) def compile_module(self): """Write the C source code (if not done already) and compile it. This produces a dynamic link library in 'self.modulefilename'.""" - if self._has_module: - raise ffiplatform.VerificationError("module already compiled") - if not self._has_source: - self._write_source() - self._compile_module() + with self.ffi._lock: + if self._has_module: + raise ffiplatform.VerificationError("module already compiled") + if not self._has_source: + self._write_source() + self._compile_module() def load_library(self): """Get a C module from this Verifier instance. @@ -62,11 +65,14 @@ operations to the C module. If necessary, the C code is written and compiled first. """ - if not self._has_module: - self._locate_module() + with self.ffi._lock: if not self._has_module: - self.compile_module() - return self._load_library() + self._locate_module() + if not self._has_module: + if not self._has_source: + self._write_source() + self._compile_module() + return self._load_library() def get_module_name(self): basename = os.path.basename(self.modulefilename) @@ -81,7 +87,9 @@ def get_extension(self): if not self._has_source: - self._write_source() + with self.ffi._lock: + if not self._has_source: + self._write_source() sourcename = ffiplatform.maybe_relative_path(self.sourcefilename) modname = self.get_module_name() return ffiplatform.get_extension(sourcename, modname, **self.kwds) @@ -103,7 +111,7 @@ else: path = None filename = self._vengine.find_module(self.get_module_name(), path, - _get_so_suffix()) + _get_so_suffixes()) if filename is None: return self.modulefilename = filename @@ -193,7 +201,7 @@ if keep_so: suffix = '.c' # only remove .c files else: - suffix = _get_so_suffix().lower() + suffix = _get_so_suffixes()[0].lower() for fn in filelist: if fn.lower().startswith('_cffi_') and ( fn.lower().endswith(suffix) or fn.lower().endswith('.c')): @@ -213,15 +221,20 @@ except OSError: pass -def _get_so_suffix(): +def _get_so_suffixes(): + suffixes = [] for suffix, mode, type in imp.get_suffixes(): if type == imp.C_EXTENSION: - return suffix - # bah, no C_EXTENSION available. Occurs on pypy without cpyext - if sys.platform == 'win32': - return ".pyd" - else: - return ".so" + suffixes.append(suffix) + + if not suffixes: + # bah, no C_EXTENSION available. Occurs on pypy without cpyext + if sys.platform == 'win32': + suffixes = [".pyd"] + else: + suffixes = [".so"] + + return suffixes def _ensure_dir(filename): try: diff --git a/lib_pypy/numpypy/__init__.py b/lib_pypy/numpypy/__init__.py deleted file mode 100644 --- a/lib_pypy/numpypy/__init__.py +++ /dev/null @@ -1,17 +0,0 @@ -from . import core -from .core import * -from . import lib -from .lib import * - -from builtins import bool, int, int, float, complex, object, str, str - -from .core import round, abs, max, min - -__version__ = '1.7.0' - -__all__ = ['__version__'] -__all__ += core.__all__ -__all__ += lib.__all__ - -#import sys -#sys.modules.setdefault('numpy', sys.modules['numpypy']) diff --git a/lib_pypy/numpypy/core/__init__.py b/lib_pypy/numpypy/core/__init__.py deleted file mode 100644 --- a/lib_pypy/numpypy/core/__init__.py +++ /dev/null @@ -1,19 +0,0 @@ - - -from . import multiarray -from . import umath -from . import numeric -from .numeric import * -from . import fromnumeric -from .fromnumeric import * -from . import shape_base -from .shape_base import * - -from .fromnumeric import amax as max, amin as min, \ - round_ as round -from .numeric import absolute as abs - -__all__ = [] -__all__ += numeric.__all__ -__all__ += fromnumeric.__all__ -__all__ += shape_base.__all__ diff --git a/lib_pypy/numpypy/core/_methods.py b/lib_pypy/numpypy/core/_methods.py deleted file mode 100644 --- a/lib_pypy/numpypy/core/_methods.py +++ /dev/null @@ -1,109 +0,0 @@ -# Array methods which are called by the both the C-code for the method -# and the Python code for the NumPy-namespace function - -from . import multiarray as mu -from . import umath as um -from .numeric import asanyarray - -def _amax(a, axis=None, out=None, keepdims=False): - return um.maximum.reduce(a, axis=axis, - out=out, keepdims=keepdims) - -def _amin(a, axis=None, out=None, keepdims=False): - return um.minimum.reduce(a, axis=axis, - out=out, keepdims=keepdims) - -def _sum(a, axis=None, dtype=None, out=None, keepdims=False): - return um.add.reduce(a, axis=axis, dtype=dtype, - out=out, keepdims=keepdims) - -def _prod(a, axis=None, dtype=None, out=None, keepdims=False): - return um.multiply.reduce(a, axis=axis, dtype=dtype, - out=out, keepdims=keepdims) - -def _any(a, axis=None, dtype=None, out=None, keepdims=False): - return um.logical_or.reduce(a, axis=axis, dtype=dtype, out=out, - keepdims=keepdims) - -def _all(a, axis=None, dtype=None, out=None, keepdims=False): - return um.logical_and.reduce(a, axis=axis, dtype=dtype, out=out, - keepdims=keepdims) - -def _count_reduce_items(arr, axis): - if axis is None: - axis = tuple(range(arr.ndim)) - if not isinstance(axis, tuple): - axis = (axis,) - items = 1 - for ax in axis: - items *= arr.shape[ax] - return items - -def _mean(a, axis=None, dtype=None, out=None, keepdims=False): - arr = asanyarray(a) - - # Upgrade bool, unsigned int, and int to float64 - if dtype is None and arr.dtype.kind in ['b','u','i']: - ret = um.add.reduce(arr, axis=axis, dtype='f8', - out=out, keepdims=keepdims) - else: - ret = um.add.reduce(arr, axis=axis, dtype=dtype, - out=out, keepdims=keepdims) - rcount = _count_reduce_items(arr, axis) - if isinstance(ret, mu.ndarray): - ret = um.true_divide(ret, rcount, - out=ret, casting='unsafe', subok=False) - else: - ret = ret / float(rcount) - return ret - -def _var(a, axis=None, dtype=None, out=None, ddof=0, - keepdims=False): - arr = asanyarray(a) - - # First compute the mean, saving 'rcount' for reuse later - if dtype is None and arr.dtype.kind in ['b','u','i']: - arrmean = um.add.reduce(arr, axis=axis, dtype='f8', keepdims=True) - else: - arrmean = um.add.reduce(arr, axis=axis, dtype=dtype, keepdims=True) - rcount = _count_reduce_items(arr, axis) - if isinstance(arrmean, mu.ndarray): - arrmean = um.true_divide(arrmean, rcount, - out=arrmean, casting='unsafe', subok=False) - else: - arrmean = arrmean / float(rcount) - - # arr - arrmean - x = arr - arrmean - - # (arr - arrmean) ** 2 - if arr.dtype.kind == 'c': - x = um.multiply(x, um.conjugate(x), out=x).real - else: - x = um.multiply(x, x, out=x) - - # add.reduce((arr - arrmean) ** 2, axis) - ret = um.add.reduce(x, axis=axis, dtype=dtype, out=out, keepdims=keepdims) - - # add.reduce((arr - arrmean) ** 2, axis) / (n - ddof) - if not keepdims and isinstance(rcount, mu.ndarray): - rcount = rcount.squeeze(axis=axis) - rcount -= ddof - if isinstance(ret, mu.ndarray): - ret = um.true_divide(ret, rcount, - out=ret, casting='unsafe', subok=False) - else: - ret = ret / float(rcount) - - return ret - -def _std(a, axis=None, dtype=None, out=None, ddof=0, keepdims=False): - ret = _var(a, axis=axis, dtype=dtype, out=out, ddof=ddof, - keepdims=keepdims) - - if isinstance(ret, mu.ndarray): - ret = um.sqrt(ret, out=ret) - else: - ret = um.sqrt(ret) - - return ret diff --git a/lib_pypy/numpypy/core/arrayprint.py b/lib_pypy/numpypy/core/arrayprint.py deleted file mode 100644 --- a/lib_pypy/numpypy/core/arrayprint.py +++ /dev/null @@ -1,751 +0,0 @@ -"""Array printing function - -$Id: arrayprint.py,v 1.9 2005/09/13 13:58:44 teoliphant Exp $ -""" -__all__ = ["array2string", "set_printoptions", "get_printoptions"] -__docformat__ = 'restructuredtext' - -# -# Written by Konrad Hinsen -# last revision: 1996-3-13 -# modified by Jim Hugunin 1997-3-3 for repr's and str's (and other details) -# and by Perry Greenfield 2000-4-1 for numarray -# and by Travis Oliphant 2005-8-22 for numpy - -import sys -from . import numerictypes as _nt -from .umath import maximum, minimum, absolute, not_equal, isnan, isinf -#from multiarray import format_longfloat, datetime_as_string, datetime_data -from .fromnumeric import ravel - - -def product(x, y): return x*y - -_summaryEdgeItems = 3 # repr N leading and trailing items of each dimension -_summaryThreshold = 1000 # total items > triggers array summarization - -_float_output_precision = 8 -_float_output_suppress_small = False -_line_width = 75 -_nan_str = 'nan' -_inf_str = 'inf' -_formatter = None # formatting function for array elements - -if sys.version_info[0] >= 3: - from functools import reduce - -def set_printoptions(precision=None, threshold=None, edgeitems=None, - linewidth=None, suppress=None, - nanstr=None, infstr=None, - formatter=None): - """ - Set printing options. - - These options determine the way floating point numbers, arrays and - other NumPy objects are displayed. - - Parameters - ---------- - precision : int, optional - Number of digits of precision for floating point output (default 8). - threshold : int, optional - Total number of array elements which trigger summarization - rather than full repr (default 1000). - edgeitems : int, optional - Number of array items in summary at beginning and end of - each dimension (default 3). - linewidth : int, optional - The number of characters per line for the purpose of inserting - line breaks (default 75). - suppress : bool, optional - Whether or not suppress printing of small floating point values - using scientific notation (default False). - nanstr : str, optional - String representation of floating point not-a-number (default nan). - infstr : str, optional - String representation of floating point infinity (default inf). - formatter : dict of callables, optional - If not None, the keys should indicate the type(s) that the respective - formatting function applies to. Callables should return a string. - Types that are not specified (by their corresponding keys) are handled - by the default formatters. Individual types for which a formatter - can be set are:: - - - 'bool' - - 'int' - - 'timedelta' : a `numpy.timedelta64` - - 'datetime' : a `numpy.datetime64` - - 'float' - - 'longfloat' : 128-bit floats - - 'complexfloat' - - 'longcomplexfloat' : composed of two 128-bit floats - - 'numpy_str' : types `numpy.string_` and `numpy.unicode_` - - 'str' : all other strings - - Other keys that can be used to set a group of types at once are:: - - - 'all' : sets all types - - 'int_kind' : sets 'int' - - 'float_kind' : sets 'float' and 'longfloat' - - 'complex_kind' : sets 'complexfloat' and 'longcomplexfloat' - - 'str_kind' : sets 'str' and 'numpystr' - - See Also - -------- - get_printoptions, set_string_function, array2string - - Notes - ----- - `formatter` is always reset with a call to `set_printoptions`. - - Examples - -------- - Floating point precision can be set: - - >>> np.set_printoptions(precision=4) - >>> print np.array([1.123456789]) - [ 1.1235] - - Long arrays can be summarised: - - >>> np.set_printoptions(threshold=5) - >>> print np.arange(10) - [0 1 2 ..., 7 8 9] - - Small results can be suppressed: - - >>> eps = np.finfo(float).eps - >>> x = np.arange(4.) - >>> x**2 - (x + eps)**2 - array([ -4.9304e-32, -4.4409e-16, 0.0000e+00, 0.0000e+00]) - >>> np.set_printoptions(suppress=True) - >>> x**2 - (x + eps)**2 - array([-0., -0., 0., 0.]) - - A custom formatter can be used to display array elements as desired: - - >>> np.set_printoptions(formatter={'all':lambda x: 'int: '+str(-x)}) - >>> x = np.arange(3) - >>> x - array([int: 0, int: -1, int: -2]) - >>> np.set_printoptions() # formatter gets reset - >>> x - array([0, 1, 2]) - - To put back the default options, you can use: - - >>> np.set_printoptions(edgeitems=3,infstr='inf', - ... linewidth=75, nanstr='nan', precision=8, - ... suppress=False, threshold=1000, formatter=None) - """ - - global _summaryThreshold, _summaryEdgeItems, _float_output_precision, \ - _line_width, _float_output_suppress_small, _nan_str, _inf_str, \ - _formatter - if linewidth is not None: - _line_width = linewidth - if threshold is not None: - _summaryThreshold = threshold - if edgeitems is not None: - _summaryEdgeItems = edgeitems - if precision is not None: - _float_output_precision = precision - if suppress is not None: - _float_output_suppress_small = not not suppress - if nanstr is not None: - _nan_str = nanstr - if infstr is not None: - _inf_str = infstr - _formatter = formatter - -def get_printoptions(): - """ - Return the current print options. - - Returns - ------- - print_opts : dict - Dictionary of current print options with keys - - - precision : int - - threshold : int - - edgeitems : int - - linewidth : int - - suppress : bool - - nanstr : str - - infstr : str - - formatter : dict of callables - - For a full description of these options, see `set_printoptions`. - - See Also - -------- - set_printoptions, set_string_function - - """ - d = dict(precision=_float_output_precision, - threshold=_summaryThreshold, - edgeitems=_summaryEdgeItems, - linewidth=_line_width, - suppress=_float_output_suppress_small, - nanstr=_nan_str, - infstr=_inf_str, - formatter=_formatter) - return d - -def _leading_trailing(a): - from . import numeric as _nc - if a.ndim == 1: - if len(a) > 2*_summaryEdgeItems: - b = _nc.concatenate((a[:_summaryEdgeItems], - a[-_summaryEdgeItems:])) - else: - b = a - else: - if len(a) > 2*_summaryEdgeItems: - l = [_leading_trailing(a[i]) for i in range( - min(len(a), _summaryEdgeItems))] - l.extend([_leading_trailing(a[-i]) for i in range( - min(len(a), _summaryEdgeItems),0,-1)]) - else: - l = [_leading_trailing(a[i]) for i in range(0, len(a))] - b = _nc.concatenate(tuple(l)) - return b - -def _boolFormatter(x): - if x: - return ' True' - else: - return 'False' - - -def repr_format(x): - return repr(x) - -def _array2string(a, max_line_width, precision, suppress_small, separator=' ', - prefix="", formatter=None): - - if max_line_width is None: - max_line_width = _line_width - - if precision is None: - precision = _float_output_precision - - if suppress_small is None: - suppress_small = _float_output_suppress_small - - if formatter is None: - formatter = _formatter - - if a.size > _summaryThreshold: - summary_insert = "..., " - data = _leading_trailing(a) - else: - summary_insert = "" - data = ravel(a) - - formatdict = {'bool' : _boolFormatter, - 'int' : IntegerFormat(data), - 'float' : FloatFormat(data, precision, suppress_small), - 'longfloat' : FloatFormat(data, precision, suppress_small), - 'complexfloat' : ComplexFormat(data, precision, - suppress_small), - 'longcomplexfloat' : ComplexFormat(data, precision, - suppress_small), - 'datetime' : DatetimeFormat(data), - 'timedelta' : TimedeltaFormat(data), - 'numpystr' : repr_format, - 'str' : str} - - if formatter is not None: - fkeys = [k for k in list(formatter.keys()) if formatter[k] is not None] - if 'all' in fkeys: - for key in list(formatdict.keys()): - formatdict[key] = formatter['all'] - if 'int_kind' in fkeys: - for key in ['int']: - formatdict[key] = formatter['int_kind'] - if 'float_kind' in fkeys: - for key in ['float', 'longfloat']: - formatdict[key] = formatter['float_kind'] - if 'complex_kind' in fkeys: - for key in ['complexfloat', 'longcomplexfloat']: - formatdict[key] = formatter['complex_kind'] - if 'str_kind' in fkeys: - for key in ['numpystr', 'str']: - formatdict[key] = formatter['str_kind'] - for key in list(formatdict.keys()): - if key in fkeys: - formatdict[key] = formatter[key] - - try: - format_function = a._format - msg = "The `_format` attribute is deprecated in Numpy 2.0 and " \ - "will be removed in 2.1. Use the `formatter` kw instead." - import warnings - warnings.warn(msg, DeprecationWarning) - except AttributeError: - # find the right formatting function for the array - dtypeobj = a.dtype.type - if issubclass(dtypeobj, _nt.bool_): - format_function = formatdict['bool'] - elif issubclass(dtypeobj, _nt.integer): - #if issubclass(dtypeobj, _nt.timedelta64): - # format_function = formatdict['timedelta'] - #else: - format_function = formatdict['int'] - elif issubclass(dtypeobj, _nt.floating): - if hasattr(_nt, 'longfloat') and issubclass(dtypeobj, _nt.longfloat): - format_function = formatdict['longfloat'] - else: - format_function = formatdict['float'] - elif issubclass(dtypeobj, _nt.complexfloating): - if hasattr(_nt, 'clongfloat') and issubclass(dtypeobj, _nt.clongfloat): - format_function = formatdict['longcomplexfloat'] - else: - format_function = formatdict['complexfloat'] - elif issubclass(dtypeobj, (_nt.unicode_, _nt.string_)): - format_function = formatdict['numpystr'] - #elif issubclass(dtypeobj, _nt.datetime64): - # format_function = formatdict['datetime'] - else: - format_function = formatdict['str'] - - # skip over "[" - next_line_prefix = " " - # skip over array( - next_line_prefix += " "*len(prefix) - - lst = _formatArray(a, format_function, len(a.shape), max_line_width, - next_line_prefix, separator, - _summaryEdgeItems, summary_insert)[:-1] - return lst - -def _convert_arrays(obj): - from . import numeric as _nc - newtup = [] - for k in obj: - if isinstance(k, _nc.ndarray): - k = k.tolist() - elif isinstance(k, tuple): - k = _convert_arrays(k) - newtup.append(k) - return tuple(newtup) - - -def array2string(a, max_line_width=None, precision=None, - suppress_small=None, separator=' ', prefix="", - style=repr, formatter=None): - """ - Return a string representation of an array. - - Parameters - ---------- - a : ndarray - Input array. - max_line_width : int, optional - The maximum number of columns the string should span. Newline - characters splits the string appropriately after array elements. - precision : int, optional - Floating point precision. Default is the current printing - precision (usually 8), which can be altered using `set_printoptions`. - suppress_small : bool, optional - Represent very small numbers as zero. A number is "very small" if it - is smaller than the current printing precision. - separator : str, optional - Inserted between elements. - prefix : str, optional - An array is typically printed as:: - - 'prefix(' + array2string(a) + ')' - - The length of the prefix string is used to align the - output correctly. - style : function, optional - A function that accepts an ndarray and returns a string. Used only - when the shape of `a` is equal to ``()``, i.e. for 0-D arrays. - formatter : dict of callables, optional - If not None, the keys should indicate the type(s) that the respective - formatting function applies to. Callables should return a string. - Types that are not specified (by their corresponding keys) are handled - by the default formatters. Individual types for which a formatter - can be set are:: - - - 'bool' - - 'int' - - 'timedelta' : a `numpy.timedelta64` - - 'datetime' : a `numpy.datetime64` - - 'float' - - 'longfloat' : 128-bit floats - - 'complexfloat' - - 'longcomplexfloat' : composed of two 128-bit floats - - 'numpy_str' : types `numpy.string_` and `numpy.unicode_` - - 'str' : all other strings - - Other keys that can be used to set a group of types at once are:: - - - 'all' : sets all types - - 'int_kind' : sets 'int' - - 'float_kind' : sets 'float' and 'longfloat' - - 'complex_kind' : sets 'complexfloat' and 'longcomplexfloat' - - 'str_kind' : sets 'str' and 'numpystr' - - Returns - ------- - array_str : str - String representation of the array. - - Raises - ------ - TypeError : if a callable in `formatter` does not return a string. - - See Also - -------- - array_str, array_repr, set_printoptions, get_printoptions - - Notes - ----- - If a formatter is specified for a certain type, the `precision` keyword is - ignored for that type. - - Examples - -------- - >>> x = np.array([1e-16,1,2,3]) - >>> print np.array2string(x, precision=2, separator=',', - ... suppress_small=True) - [ 0., 1., 2., 3.] - - >>> x = np.arange(3.) - >>> np.array2string(x, formatter={'float_kind':lambda x: "%.2f" % x}) - '[0.00 1.00 2.00]' - - >>> x = np.arange(3) - >>> np.array2string(x, formatter={'int':lambda x: hex(x)}) - '[0x0L 0x1L 0x2L]' - - """ - - if a.shape == (): - x = a.item() - try: - lst = a._format(x) - msg = "The `_format` attribute is deprecated in Numpy " \ - "2.0 and will be removed in 2.1. Use the " \ - "`formatter` kw instead." - import warnings - warnings.warn(msg, DeprecationWarning) - except AttributeError: - if isinstance(x, tuple): - x = _convert_arrays(x) - lst = style(x) - elif reduce(product, a.shape) == 0: - # treat as a null array if any of shape elements == 0 - lst = "[]" - else: - lst = _array2string(a, max_line_width, precision, suppress_small, - separator, prefix, formatter=formatter) - return lst - -def _extendLine(s, line, word, max_line_len, next_line_prefix): - if len(line.rstrip()) + len(word.rstrip()) >= max_line_len: - s += line.rstrip() + "\n" - line = next_line_prefix - line += word - return s, line - - -def _formatArray(a, format_function, rank, max_line_len, - next_line_prefix, separator, edge_items, summary_insert): - """formatArray is designed for two modes of operation: - - 1. Full output - - 2. Summarized output - - """ - if rank == 0: - obj = a.item() - if isinstance(obj, tuple): - obj = _convert_arrays(obj) - return str(obj) - - if summary_insert and 2*edge_items < len(a): - leading_items, trailing_items, summary_insert1 = \ - edge_items, edge_items, summary_insert - else: - leading_items, trailing_items, summary_insert1 = 0, len(a), "" - - if rank == 1: - s = "" - line = next_line_prefix - for i in range(leading_items): - word = format_function(a[i]) + separator - s, line = _extendLine(s, line, word, max_line_len, next_line_prefix) - - if summary_insert1: - s, line = _extendLine(s, line, summary_insert1, max_line_len, next_line_prefix) From noreply at buildbot.pypy.org Thu Nov 14 09:17:03 2013 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 14 Nov 2013 09:17:03 +0100 (CET) Subject: [pypy-commit] pypy release-2.2.x: Updates Message-ID: <20131114081703.B30D01C0134@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: release-2.2.x Changeset: r68015:8eb5b5ac4bba Date: 2013-11-14 09:16 +0100 http://bitbucket.org/pypy/pypy/changeset/8eb5b5ac4bba/ Log: Updates diff --git a/pypy/doc/release-2.2.0.rst b/pypy/doc/release-2.2.0.rst --- a/pypy/doc/release-2.2.0.rst +++ b/pypy/doc/release-2.2.0.rst @@ -63,22 +63,25 @@ * NumPy has been split: now PyPy only contains the core module, called ``_numpypy``. The ``numpy`` module itself has been moved to - ``https://bitbucket.org/pypy/numpy``. You need to install it - separately in a virtualenv with ``pip install - git+https://bitbucket.org/pypy/numpy.git``. + ``https://bitbucket.org/pypy/numpy`` and ``numpypy`` disappeared. + You need to install NumPy separately with a virtualenv: + ``pip install git+https://bitbucket.org/pypy/numpy.git``; + or by directly doing + ``git clone https://bitbucket.org/pypy/numpy.git``, + ``cd numpy``, ``python setup.py install``. * non-inlined calls have less overhead * Things that use ``sys.set_trace`` are now JITted (like coverage) -* JSON encoding is faster +* JSON encoding used to be very fast, now decoding is as well * various buffer copying methods experience speedups (like list-of-ints to ``int[]`` buffer from cffi) -* We finally wrote all the missing ``os.xxx()`` functions. There are - a lot of strange ones that nobody ever heard about, except those who - really need them. +* We finally wrote (hopefully) all the missing ``os.xxx()`` functions, + including ``os.startfile()`` on Windows and a handful of rare ones + on Posix. * numpy has a rudimentary C API that cooperates with ``cpyext`` From noreply at buildbot.pypy.org Thu Nov 14 09:33:53 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Thu, 14 Nov 2013 09:33:53 +0100 (CET) Subject: [pypy-commit] pypy numpy-newbyteorder: merge default Message-ID: <20131114083353.336791C0134@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: numpy-newbyteorder Changeset: r68016:df3217200923 Date: 2013-11-13 23:54 -0500 http://bitbucket.org/pypy/pypy/changeset/df3217200923/ Log: merge default diff too long, truncating to 2000 out of 5219 lines diff --git a/lib-python/2.7/json/encoder.py b/lib-python/2.7/json/encoder.py --- a/lib-python/2.7/json/encoder.py +++ b/lib-python/2.7/json/encoder.py @@ -4,6 +4,21 @@ from __pypy__.builders import StringBuilder, UnicodeBuilder +class StringOrUnicodeBuilder(object): + def __init__(self): + self._builder = StringBuilder() + def append(self, string): + try: + self._builder.append(string) + except UnicodeEncodeError: + ub = UnicodeBuilder() + ub.append(self._builder.build()) + self._builder = ub + ub.append(string) + def build(self): + return self._builder.build() + + ESCAPE = re.compile(r'[\x00-\x1f\\"\b\f\n\r\t]') ESCAPE_ASCII = re.compile(r'([\\"]|[^\ -~])') HAS_UTF8 = re.compile(r'[\x80-\xff]') @@ -192,7 +207,7 @@ if self.ensure_ascii: builder = StringBuilder() else: - builder = UnicodeBuilder() + builder = StringOrUnicodeBuilder() self.__encode(o, markers, builder, 0) return builder.build() diff --git a/lib-python/2.7/weakref.py b/lib-python/2.7/weakref.py --- a/lib-python/2.7/weakref.py +++ b/lib-python/2.7/weakref.py @@ -48,7 +48,14 @@ def remove(wr, selfref=ref(self)): self = selfref() if self is not None: - del self.data[wr.key] + # Changed this for PyPy: made more resistent. The + # issue is that in some corner cases, self.data + # might already be changed or removed by the time + # this weakref's callback is called. If that is + # the case, we don't want to randomly kill an + # unrelated entry. + if self.data.get(wr.key) is wr: + del self.data[wr.key] self._remove = remove UserDict.UserDict.__init__(self, *args, **kw) @@ -160,22 +167,26 @@ try: o = self.data.pop(key)() except KeyError: + o = None + if o is None: if args: return args[0] - raise - if o is None: raise KeyError, key else: return o + # The logic above was fixed in PyPy def setdefault(self, key, default=None): try: - wr = self.data[key] + o = self.data[key]() except KeyError: + o = None + if o is None: self.data[key] = KeyedRef(default, self._remove, key) return default else: - return wr() + return o + # The logic above was fixed in PyPy def update(self, dict=None, **kwargs): d = self.data diff --git a/lib_pypy/cffi.egg-info b/lib_pypy/cffi.egg-info --- a/lib_pypy/cffi.egg-info +++ b/lib_pypy/cffi.egg-info @@ -1,6 +1,6 @@ Metadata-Version: 1.0 Name: cffi -Version: 0.7 +Version: 0.8 Summary: Foreign Function Interface for Python calling C code. Home-page: http://cffi.readthedocs.org Author: Armin Rigo, Maciej Fijalkowski diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py --- a/lib_pypy/cffi/__init__.py +++ b/lib_pypy/cffi/__init__.py @@ -4,5 +4,5 @@ from .api import FFI, CDefError, FFIError from .ffiplatform import VerificationError, VerificationMissing -__version__ = "0.7.2" -__version_info__ = (0, 7, 2) +__version__ = "0.8" +__version_info__ = (0, 8) diff --git a/lib_pypy/cffi/api.py b/lib_pypy/cffi/api.py --- a/lib_pypy/cffi/api.py +++ b/lib_pypy/cffi/api.py @@ -1,4 +1,5 @@ import types +from .lock import allocate_lock try: callable @@ -61,6 +62,7 @@ # rely on it! It's probably not going to work well.) self._backend = backend + self._lock = allocate_lock() self._parser = cparser.Parser() self._cached_btypes = {} self._parsed_types = types.ModuleType('parsed_types').__dict__ @@ -74,7 +76,8 @@ if name.startswith('RTLD_'): setattr(self, name, getattr(backend, name)) # - self.BVoidP = self._get_cached_btype(model.voidp_type) + with self._lock: + self.BVoidP = self._get_cached_btype(model.voidp_type) if isinstance(backend, types.ModuleType): # _cffi_backend: attach these constants to the class if not hasattr(FFI, 'NULL'): @@ -95,11 +98,12 @@ if not isinstance(csource, basestring): raise TypeError("cdef() argument must be a string") csource = csource.encode('ascii') - self._parser.parse(csource, override=override) - self._cdefsources.append(csource) - if override: - for cache in self._function_caches: - cache.clear() + with self._lock: + self._parser.parse(csource, override=override) + self._cdefsources.append(csource) + if override: + for cache in self._function_caches: + cache.clear() def dlopen(self, name, flags=0): """Load and return a dynamic library identified by 'name'. @@ -109,31 +113,47 @@ library we only look for the actual (untyped) symbols. """ assert isinstance(name, basestring) or name is None - lib, function_cache = _make_ffi_library(self, name, flags) - self._function_caches.append(function_cache) - self._libraries.append(lib) + with self._lock: + lib, function_cache = _make_ffi_library(self, name, flags) + self._function_caches.append(function_cache) + self._libraries.append(lib) return lib + def _typeof_locked(self, cdecl): + # call me with the lock! + key = cdecl + if key in self._parsed_types: + return self._parsed_types[key] + # + if not isinstance(cdecl, str): # unicode, on Python 2 + cdecl = cdecl.encode('ascii') + # + type = self._parser.parse_type(cdecl) + really_a_function_type = type.is_raw_function + if really_a_function_type: + type = type.as_function_pointer() + btype = self._get_cached_btype(type) + result = btype, really_a_function_type + self._parsed_types[key] = result + return result + def _typeof(self, cdecl, consider_function_as_funcptr=False): # string -> ctype object try: - btype, cfaf = self._parsed_types[cdecl] - if consider_function_as_funcptr and not cfaf: - raise KeyError + result = self._parsed_types[cdecl] except KeyError: - key = cdecl - if not isinstance(cdecl, str): # unicode, on Python 2 - cdecl = cdecl.encode('ascii') - cfaf = consider_function_as_funcptr - type = self._parser.parse_type(cdecl, - consider_function_as_funcptr=cfaf) - btype = self._get_cached_btype(type) - self._parsed_types[key] = btype, cfaf + with self._lock: + result = self._typeof_locked(cdecl) + # + btype, really_a_function_type = result + if really_a_function_type and not consider_function_as_funcptr: + raise CDefError("the type %r is a function type, not a " + "pointer-to-function type" % (cdecl,)) return btype def typeof(self, cdecl): """Parse the C type given as a string and return the - corresponding Python type: '>. + corresponding object. It can also be used on 'cdata' instance to get its C type. """ if isinstance(cdecl, basestring): @@ -144,6 +164,10 @@ res = _builtin_function_type(cdecl) if res is not None: return res + if (isinstance(cdecl, types.FunctionType) + and hasattr(cdecl, '_cffi_base_type')): + with self._lock: + return self._get_cached_btype(cdecl._cffi_base_type) raise TypeError(type(cdecl)) def sizeof(self, cdecl): @@ -280,14 +304,17 @@ data. Later, when this new cdata object is garbage-collected, 'destructor(old_cdata_object)' will be called. """ - try: - gc_weakrefs = self.gc_weakrefs - except AttributeError: - from .gc_weakref import GcWeakrefs - gc_weakrefs = self.gc_weakrefs = GcWeakrefs(self) - return gc_weakrefs.build(cdata, destructor) + with self._lock: + try: + gc_weakrefs = self.gc_weakrefs + except AttributeError: + from .gc_weakref import GcWeakrefs + gc_weakrefs = self.gc_weakrefs = GcWeakrefs(self) + return gc_weakrefs.build(cdata, destructor) def _get_cached_btype(self, type): + assert self._lock.acquire(False) is False + # call me with the lock! try: BType = self._cached_btypes[type] except KeyError: @@ -322,7 +349,8 @@ def _pointer_to(self, ctype): from . import model - return model.pointer_cache(self, ctype) + with self._lock: + return model.pointer_cache(self, ctype) def addressof(self, cdata, field=None): """Return the address of a . @@ -342,10 +370,12 @@ variables, which must anyway be accessed directly from the lib object returned by the original FFI instance. """ - self._parser.include(ffi_to_include._parser) - self._cdefsources.append('[') - self._cdefsources.extend(ffi_to_include._cdefsources) - self._cdefsources.append(']') + with ffi_to_include._lock: + with self._lock: + self._parser.include(ffi_to_include._parser) + self._cdefsources.append('[') + self._cdefsources.extend(ffi_to_include._cdefsources) + self._cdefsources.append(']') def new_handle(self, x): return self._backend.newp_handle(self.BVoidP, x) @@ -372,7 +402,7 @@ backendlib = backend.load_library(path, flags) copied_enums = [] # - def make_accessor(name): + def make_accessor_locked(name): key = 'function ' + name if key in ffi._parser._declarations: tp = ffi._parser._declarations[key] @@ -404,11 +434,17 @@ if enumname not in library.__dict__: library.__dict__[enumname] = enumval copied_enums.append(True) + if name in library.__dict__: + return # - if name in library.__dict__: # copied from an enum value just above, - return # or multithread's race condition raise AttributeError(name) # + def make_accessor(name): + with ffi._lock: + if name in library.__dict__ or name in FFILibrary.__dict__: + return # added by another thread while waiting for the lock + make_accessor_locked(name) + # class FFILibrary(object): def __getattr__(self, name): make_accessor(name) @@ -444,4 +480,5 @@ except (KeyError, AttributeError, TypeError): return None else: - return ffi._get_cached_btype(tp) + with ffi._lock: + return ffi._get_cached_btype(tp) diff --git a/lib_pypy/cffi/cparser.py b/lib_pypy/cffi/cparser.py --- a/lib_pypy/cffi/cparser.py +++ b/lib_pypy/cffi/cparser.py @@ -142,7 +142,7 @@ if 1 <= linenum <= len(csourcelines): line = csourcelines[linenum-1] if line: - msg = 'cannot parse "%s"\n%s' % (line, msg) + msg = 'cannot parse "%s"\n%s' % (line.strip(), msg) else: msg = 'parse error\n%s' % (msg,) raise api.CDefError(msg) @@ -217,19 +217,18 @@ # if decl.name: tp = self._get_type(node, partial_length_ok=True) - if self._is_constant_declaration(node): + if self._is_constant_globalvar(node): self._declare('constant ' + decl.name, tp) else: self._declare('variable ' + decl.name, tp) - def parse_type(self, cdecl, consider_function_as_funcptr=False): + def parse_type(self, cdecl): ast, macros = self._parse('void __dummy(\n%s\n);' % cdecl) assert not macros exprnode = ast.ext[-1].type.args.params[0] if isinstance(exprnode, pycparser.c_ast.ID): raise api.CDefError("unknown identifier '%s'" % (exprnode.name,)) - return self._get_type(exprnode.type, - consider_function_as_funcptr=consider_function_as_funcptr) + return self._get_type(exprnode.type) def _declare(self, name, obj): if name in self._declarations: @@ -249,28 +248,17 @@ return model.ConstPointerType(type) return model.PointerType(type) - def _get_type(self, typenode, convert_array_to_pointer=False, - name=None, partial_length_ok=False, - consider_function_as_funcptr=False): + def _get_type(self, typenode, name=None, partial_length_ok=False): # first, dereference typedefs, if we have it already parsed, we're good if (isinstance(typenode, pycparser.c_ast.TypeDecl) and isinstance(typenode.type, pycparser.c_ast.IdentifierType) and len(typenode.type.names) == 1 and ('typedef ' + typenode.type.names[0]) in self._declarations): type = self._declarations['typedef ' + typenode.type.names[0]] - if isinstance(type, model.ArrayType): - if convert_array_to_pointer: - return type.item - else: - if (consider_function_as_funcptr and - isinstance(type, model.RawFunctionType)): - return type.as_function_pointer() return type # if isinstance(typenode, pycparser.c_ast.ArrayDecl): # array type - if convert_array_to_pointer: - return self._get_type_pointer(self._get_type(typenode.type)) if typenode.dim is None: length = None else: @@ -331,10 +319,7 @@ # if isinstance(typenode, pycparser.c_ast.FuncDecl): # a function type - result = self._parse_function_type(typenode, name) - if consider_function_as_funcptr: - result = result.as_function_pointer() - return result + return self._parse_function_type(typenode, name) # # nested anonymous structs or unions end up here if isinstance(typenode, pycparser.c_ast.Struct): @@ -365,21 +350,24 @@ isinstance(params[0].type.type, pycparser.c_ast.IdentifierType) and list(params[0].type.type.names) == ['void']): del params[0] - args = [self._get_type(argdeclnode.type, - convert_array_to_pointer=True, - consider_function_as_funcptr=True) + args = [self._as_func_arg(self._get_type(argdeclnode.type)) for argdeclnode in params] result = self._get_type(typenode.type) return model.RawFunctionType(tuple(args), result, ellipsis) - def _is_constant_declaration(self, typenode, const=False): - if isinstance(typenode, pycparser.c_ast.ArrayDecl): - return self._is_constant_declaration(typenode.type) + def _as_func_arg(self, type): + if isinstance(type, model.ArrayType): + return model.PointerType(type.item) + elif isinstance(type, model.RawFunctionType): + return type.as_function_pointer() + else: + return type + + def _is_constant_globalvar(self, typenode): if isinstance(typenode, pycparser.c_ast.PtrDecl): - const = 'const' in typenode.quals - return self._is_constant_declaration(typenode.type, const) + return 'const' in typenode.quals if isinstance(typenode, pycparser.c_ast.TypeDecl): - return const or 'const' in typenode.quals + return 'const' in typenode.quals return False def _get_struct_union_enum_type(self, kind, type, name=None, nested=False): @@ -491,7 +479,7 @@ return tp def _make_partial(self, tp, nested): - if not isinstance(tp, model.StructType): + if not isinstance(tp, model.StructOrUnion): raise api.CDefError("%s cannot be partial" % (tp,)) if not tp.has_c_name() and not nested: raise NotImplementedError("%s is partial but has no C name" %(tp,)) @@ -511,7 +499,7 @@ if (isinstance(exprnode, pycparser.c_ast.ID) and exprnode.name == '__dotdotdotarray__'): self._partial_length = True - return None + return '...' # raise api.FFIError("unsupported expression: expected a " "simple numeric constant") diff --git a/lib_pypy/cffi/gc_weakref.py b/lib_pypy/cffi/gc_weakref.py --- a/lib_pypy/cffi/gc_weakref.py +++ b/lib_pypy/cffi/gc_weakref.py @@ -14,6 +14,6 @@ def build(self, cdata, destructor): # make a new cdata of the same type as the original one - new_cdata = self.ffi.cast(self.ffi.typeof(cdata), cdata) + new_cdata = self.ffi.cast(self.ffi._backend.typeof(cdata), cdata) self.data[ref(new_cdata, self.remove)] = destructor, cdata return new_cdata diff --git a/lib_pypy/cffi/lock.py b/lib_pypy/cffi/lock.py new file mode 100644 --- /dev/null +++ b/lib_pypy/cffi/lock.py @@ -0,0 +1,30 @@ +import sys + +if sys.version_info < (3,): + try: + from thread import allocate_lock + except ImportError: + from dummy_thread import allocate_lock +else: + try: + from _thread import allocate_lock + except ImportError: + from _dummy_thread import allocate_lock + + +##import sys +##l1 = allocate_lock + +##class allocate_lock(object): +## def __init__(self): +## self._real = l1() +## def __enter__(self): +## for i in range(4, 0, -1): +## print sys._getframe(i).f_code +## print +## return self._real.__enter__() +## def __exit__(self, *args): +## return self._real.__exit__(*args) +## def acquire(self, f): +## assert f is False +## return self._real.acquire(f) diff --git a/lib_pypy/cffi/model.py b/lib_pypy/cffi/model.py --- a/lib_pypy/cffi/model.py +++ b/lib_pypy/cffi/model.py @@ -1,7 +1,10 @@ import weakref +from .lock import allocate_lock + class BaseTypeByIdentity(object): is_array_type = False + is_raw_function = False def get_c_name(self, replace_with='', context='a C file'): result = self.c_name_with_marker @@ -146,6 +149,7 @@ # a function, but not a pointer-to-function. The backend has no # notion of such a type; it's used temporarily by parsing. _base_pattern = '(&)(%s)' + is_raw_function = True def build_backend_type(self, ffi, finishlist): from . import api @@ -192,10 +196,6 @@ _base_pattern = " const *&" _base_pattern_array = "(const *&)" - def build_backend_type(self, ffi, finishlist): - BPtr = PointerType(self.totype).get_cached_btype(ffi, finishlist) - return BPtr - const_voidp_type = ConstPointerType(void_type) @@ -216,10 +216,12 @@ self.item = item self.length = length # - if self.length is None: + if length is None: brackets = '&[]' + elif length == '...': + brackets = '&[/*...*/]' else: - brackets = '&[%d]' % self.length + brackets = '&[%d]' % length self.c_name_with_marker = ( self.item.c_name_with_marker.replace('&', brackets)) @@ -227,6 +229,10 @@ return ArrayType(self.item, newlength) def build_backend_type(self, ffi, finishlist): + if self.length == '...': + from . import api + raise api.CDefError("cannot render the type %r: unknown length" % + (self,)) self.item.get_cached_btype(ffi, finishlist) # force the item BType BPtrItem = PointerType(self.item).get_cached_btype(ffi, finishlist) return global_cache(self, ffi, 'new_array_type', BPtrItem, self.length) @@ -252,6 +258,7 @@ class StructOrUnion(StructOrUnionOrEnum): fixedlayout = None completed = False + partial = False def __init__(self, name, fldnames, fldtypes, fldbitsize): self.name = name @@ -303,20 +310,21 @@ return # not completing it: it's an opaque struct # self.completed = 1 - fldtypes = tuple(tp.get_cached_btype(ffi, finishlist) - for tp in self.fldtypes) # if self.fixedlayout is None: + fldtypes = [tp.get_cached_btype(ffi, finishlist) + for tp in self.fldtypes] lst = list(zip(self.fldnames, fldtypes, self.fldbitsize)) ffi._backend.complete_struct_or_union(BType, lst, self) # else: + fldtypes = [] fieldofs, fieldsize, totalsize, totalalignment = self.fixedlayout for i in range(len(self.fldnames)): fsize = fieldsize[i] ftype = self.fldtypes[i] # - if isinstance(ftype, ArrayType) and ftype.length is None: + if isinstance(ftype, ArrayType) and ftype.length == '...': # fix the length to match the total size BItemType = ftype.item.get_cached_btype(ffi, finishlist) nlen, nrest = divmod(fsize, ffi.sizeof(BItemType)) @@ -327,18 +335,20 @@ ftype = ftype.resolve_length(nlen) self.fldtypes = (self.fldtypes[:i] + (ftype,) + self.fldtypes[i+1:]) - BArrayType = ftype.get_cached_btype(ffi, finishlist) - fldtypes = (fldtypes[:i] + (BArrayType,) + - fldtypes[i+1:]) - continue # - bitemsize = ffi.sizeof(fldtypes[i]) - if bitemsize != fsize: - self._verification_error( - "field '%s.%s' is declared as %d bytes, but is " - "really %d bytes" % (self.name, - self.fldnames[i] or '{}', - bitemsize, fsize)) + BFieldType = ftype.get_cached_btype(ffi, finishlist) + if isinstance(ftype, ArrayType) and ftype.length is None: + assert fsize == 0 + else: + bitemsize = ffi.sizeof(BFieldType) + if bitemsize != fsize: + self._verification_error( + "field '%s.%s' is declared as %d bytes, but is " + "really %d bytes" % (self.name, + self.fldnames[i] or '{}', + bitemsize, fsize)) + fldtypes.append(BFieldType) + # lst = list(zip(self.fldnames, fldtypes, self.fldbitsize, fieldofs)) ffi._backend.complete_struct_or_union(BType, lst, self, totalsize, totalalignment) @@ -348,11 +358,6 @@ from .ffiplatform import VerificationError raise VerificationError(msg) - -class StructType(StructOrUnion): - kind = 'struct' - partial = False - def check_not_partial(self): if self.partial and self.fixedlayout is None: from . import ffiplatform @@ -361,19 +366,18 @@ def build_backend_type(self, ffi, finishlist): self.check_not_partial() finishlist.append(self) - - return global_cache(self, ffi, 'new_struct_type', + # + return global_cache(self, ffi, 'new_%s_type' % self.kind, self.get_official_name(), key=self) +class StructType(StructOrUnion): + kind = 'struct' + + class UnionType(StructOrUnion): kind = 'union' - def build_backend_type(self, ffi, finishlist): - finishlist.append(self) - return global_cache(self, ffi, 'new_union_type', - self.get_official_name(), key=self) - class EnumType(StructOrUnionOrEnum): kind = 'enum' @@ -387,6 +391,12 @@ self.baseinttype = baseinttype self.build_c_name_with_marker() + def force_the_name(self, forcename): + StructOrUnionOrEnum.force_the_name(self, forcename) + if self.forcename is None: + name = self.get_official_name() + self.forcename = '$' + name.replace(' ', '_') + def check_not_partial(self): if self.partial and not self.partial_resolved: from . import ffiplatform @@ -444,6 +454,9 @@ tp = StructType(structname, None, None, None) return NamedPointerType(tp, name) + +global_lock = allocate_lock() + def global_cache(srctype, ffi, funcname, *args, **kwds): key = kwds.pop('key', (funcname, args)) assert not kwds @@ -464,8 +477,17 @@ res = getattr(ffi._backend, funcname)(*args) except NotImplementedError as e: raise NotImplementedError("%r: %s" % (srctype, e)) - ffi._backend.__typecache[key] = res - return res + # note that setdefault() on WeakValueDictionary is not atomic + # and contains a rare bug (http://bugs.python.org/issue19542); + # we have to use a lock and do it ourselves + cache = ffi._backend.__typecache + with global_lock: + res1 = cache.get(key) + if res1 is None: + cache[key] = res + return res + else: + return res1 def pointer_cache(ffi, BType): return global_cache('?', ffi, 'new_pointer_type', BType) diff --git a/lib_pypy/cffi/vengine_cpy.py b/lib_pypy/cffi/vengine_cpy.py --- a/lib_pypy/cffi/vengine_cpy.py +++ b/lib_pypy/cffi/vengine_cpy.py @@ -15,7 +15,7 @@ def patch_extension_kwds(self, kwds): pass - def find_module(self, module_name, path, so_suffix): + def find_module(self, module_name, path, so_suffixes): try: f, filename, descr = imp.find_module(module_name, path) except ImportError: @@ -25,7 +25,7 @@ # Note that after a setuptools installation, there are both .py # and .so files with the same basename. The code here relies on # imp.find_module() locating the .so in priority. - if descr[0] != so_suffix: + if descr[0] not in so_suffixes: return None return filename @@ -160,7 +160,10 @@ def __dir__(self): return FFILibrary._cffi_dir + list(self.__dict__) library = FFILibrary() - module._cffi_setup(lst, ffiplatform.VerificationError, library) + if module._cffi_setup(lst, ffiplatform.VerificationError, library): + import warnings + warnings.warn("reimporting %r might overwrite older definitions" + % (self.verifier.get_module_name())) # # finally, call the loaded_cpy_xxx() functions. This will perform # the final adjustments, like copying the Python->C wrapper @@ -280,8 +283,8 @@ return '_cffi_from_c_pointer((char *)%s, _cffi_type(%d))' % ( var, self._gettypenum(tp)) elif isinstance(tp, model.ArrayType): - return '_cffi_from_c_deref((char *)%s, _cffi_type(%d))' % ( - var, self._gettypenum(tp)) + return '_cffi_from_c_pointer((char *)%s, _cffi_type(%d))' % ( + var, self._gettypenum(model.PointerType(tp.item))) elif isinstance(tp, model.StructType): if tp.fldnames is None: raise TypeError("'%s' is used as %s, but is opaque" % ( @@ -464,11 +467,14 @@ prnt(' static Py_ssize_t nums[] = {') prnt(' sizeof(%s),' % cname) prnt(' offsetof(struct _cffi_aligncheck, y),') - for fname, _, fbitsize in tp.enumfields(): + for fname, ftype, fbitsize in tp.enumfields(): if fbitsize >= 0: continue # xxx ignore fbitsize for now prnt(' offsetof(%s, %s),' % (cname, fname)) - prnt(' sizeof(((%s *)0)->%s),' % (cname, fname)) + if isinstance(ftype, model.ArrayType) and ftype.length is None: + prnt(' 0, /* %s */' % ftype._get_c_name()) + else: + prnt(' sizeof(((%s *)0)->%s),' % (cname, fname)) prnt(' -1') prnt(' };') prnt(' return _cffi_get_struct_layout(nums);') @@ -491,7 +497,7 @@ # function = getattr(module, layoutfuncname) layout = function() - if isinstance(tp, model.StructType) and tp.partial: + if isinstance(tp, model.StructOrUnion) and tp.partial: # use the function()'s sizes and offsets to guide the # layout of the struct totalsize = layout[0] @@ -528,9 +534,10 @@ continue # xxx ignore fbitsize for now check(layout[i], ffi.offsetof(BStruct, fname), "wrong offset for field %r" % (fname,)) - BField = ffi._get_cached_btype(ftype) - check(layout[i+1], ffi.sizeof(BField), - "wrong size for field %r" % (fname,)) + if layout[i+1] != 0: + BField = ffi._get_cached_btype(ftype) + check(layout[i+1], ffi.sizeof(BField), + "wrong size for field %r" % (fname,)) i += 2 assert i == len(layout) @@ -566,7 +573,7 @@ # constants, likely declared with '#define' def _generate_cpy_const(self, is_int, name, tp=None, category='const', - vartp=None, delayed=True): + vartp=None, delayed=True, size_too=False): prnt = self._prnt funcname = '_cffi_%s_%s' % (category, name) prnt('static int %s(PyObject *lib)' % funcname) @@ -597,6 +604,15 @@ '(unsigned long long)(%s));' % (name,)) prnt(' if (o == NULL)') prnt(' return -1;') + if size_too: + prnt(' {') + prnt(' PyObject *o1 = o;') + prnt(' o = Py_BuildValue("On", o1, (Py_ssize_t)sizeof(%s));' + % (name,)) + prnt(' Py_DECREF(o1);') + prnt(' if (o == NULL)') + prnt(' return -1;') + prnt(' }') prnt(' res = PyObject_SetAttrString(lib, "%s", o);' % name) prnt(' Py_DECREF(o);') prnt(' if (res < 0)') @@ -633,12 +649,23 @@ prnt('static int %s(PyObject *lib)' % funcname) prnt('{') for enumerator, enumvalue in zip(tp.enumerators, tp.enumvalues): - prnt(' if (%s != %d) {' % (enumerator, enumvalue)) + if enumvalue < 0: + prnt(' if ((%s) >= 0 || (long)(%s) != %dL) {' % ( + enumerator, enumerator, enumvalue)) + else: + prnt(' if ((%s) < 0 || (unsigned long)(%s) != %dUL) {' % ( + enumerator, enumerator, enumvalue)) + prnt(' char buf[64];') + prnt(' if ((%s) < 0)' % enumerator) + prnt(' snprintf(buf, 63, "%%ld", (long)(%s));' % enumerator) + prnt(' else') + prnt(' snprintf(buf, 63, "%%lu", (unsigned long)(%s));' % + enumerator) prnt(' PyErr_Format(_cffi_VerificationError,') - prnt(' "enum %s: %s has the real value %d, ' - 'not %d",') - prnt(' "%s", "%s", (int)%s, %d);' % ( - name, enumerator, enumerator, enumvalue)) + prnt(' "enum %s: %s has the real value %s, ' + 'not %s",') + prnt(' "%s", "%s", buf, "%d");' % ( + name, enumerator, enumvalue)) prnt(' return -1;') prnt(' }') prnt(' return %s;' % self._chained_list_constants[True]) @@ -677,15 +704,16 @@ def _generate_cpy_variable_collecttype(self, tp, name): if isinstance(tp, model.ArrayType): - self._do_collect_type(tp) + tp_ptr = model.PointerType(tp.item) else: tp_ptr = model.PointerType(tp) - self._do_collect_type(tp_ptr) + self._do_collect_type(tp_ptr) def _generate_cpy_variable_decl(self, tp, name): if isinstance(tp, model.ArrayType): tp_ptr = model.PointerType(tp.item) - self._generate_cpy_const(False, name, tp, vartp=tp_ptr) + self._generate_cpy_const(False, name, tp, vartp=tp_ptr, + size_too = (tp.length == '...')) else: tp_ptr = model.PointerType(tp) self._generate_cpy_const(False, name, tp_ptr, category='var') @@ -694,11 +722,29 @@ _loading_cpy_variable = _loaded_noop def _loaded_cpy_variable(self, tp, name, module, library): + value = getattr(library, name) if isinstance(tp, model.ArrayType): # int a[5] is "constant" in the - return # sense that "a=..." is forbidden + # sense that "a=..." is forbidden + if tp.length == '...': + assert isinstance(value, tuple) + (value, size) = value + BItemType = self.ffi._get_cached_btype(tp.item) + length, rest = divmod(size, self.ffi.sizeof(BItemType)) + if rest != 0: + raise ffiplatform.VerificationError( + "bad size: %r does not seem to be an array of %s" % + (name, tp.item)) + tp = tp.resolve_length(length) + # 'value' is a which we have to replace with + # a if the N is actually known + if tp.length is not None: + BArray = self.ffi._get_cached_btype(tp) + value = self.ffi.cast(BArray, value) + setattr(library, name, value) + return # remove ptr= from the library instance, and replace # it by a property on the class, which reads/writes into ptr[0]. - ptr = getattr(library, name) + ptr = value delattr(library, name) def getter(library): return ptr[0] @@ -711,12 +757,9 @@ def _generate_setup_custom(self): prnt = self._prnt - prnt('static PyObject *_cffi_setup_custom(PyObject *lib)') + prnt('static int _cffi_setup_custom(PyObject *lib)') prnt('{') - prnt(' if (%s < 0)' % self._chained_list_constants[True]) - prnt(' return NULL;') - prnt(' Py_INCREF(Py_None);') - prnt(' return Py_None;') + prnt(' return %s;' % self._chained_list_constants[True]) prnt('}') cffimod_header = r''' @@ -834,17 +877,20 @@ static void *_cffi_exports[_CFFI_NUM_EXPORTS]; static PyObject *_cffi_types, *_cffi_VerificationError; -static PyObject *_cffi_setup_custom(PyObject *lib); /* forward */ +static int _cffi_setup_custom(PyObject *lib); /* forward */ static PyObject *_cffi_setup(PyObject *self, PyObject *args) { PyObject *library; + int was_alive = (_cffi_types != NULL); if (!PyArg_ParseTuple(args, "OOO", &_cffi_types, &_cffi_VerificationError, &library)) return NULL; Py_INCREF(_cffi_types); Py_INCREF(_cffi_VerificationError); - return _cffi_setup_custom(library); + if (_cffi_setup_custom(library) < 0) + return NULL; + return PyBool_FromLong(was_alive); } static void _cffi_init(void) diff --git a/lib_pypy/cffi/vengine_gen.py b/lib_pypy/cffi/vengine_gen.py --- a/lib_pypy/cffi/vengine_gen.py +++ b/lib_pypy/cffi/vengine_gen.py @@ -20,15 +20,15 @@ # up in kwds['export_symbols']. kwds.setdefault('export_symbols', self.export_symbols) - def find_module(self, module_name, path, so_suffix): - basename = module_name + so_suffix - if path is None: - path = sys.path - for dirname in path: - filename = os.path.join(dirname, basename) - if os.path.isfile(filename): - return filename - return None + def find_module(self, module_name, path, so_suffixes): + for so_suffix in so_suffixes: + basename = module_name + so_suffix + if path is None: + path = sys.path + for dirname in path: + filename = os.path.join(dirname, basename) + if os.path.isfile(filename): + return filename def collect_types(self): pass # not needed in the generic engine @@ -173,6 +173,7 @@ newfunction = self._load_constant(False, tp, name, module) else: indirections = [] + base_tp = tp if any(isinstance(typ, model.StructOrUnion) for typ in tp.args): indirect_args = [] for i, typ in enumerate(tp.args): @@ -186,16 +187,18 @@ wrappername = '_cffi_f_%s' % name newfunction = module.load_function(BFunc, wrappername) for i, typ in indirections: - newfunction = self._make_struct_wrapper(newfunction, i, typ) + newfunction = self._make_struct_wrapper(newfunction, i, typ, + base_tp) setattr(library, name, newfunction) type(library)._cffi_dir.append(name) - def _make_struct_wrapper(self, oldfunc, i, tp): + def _make_struct_wrapper(self, oldfunc, i, tp, base_tp): backend = self.ffi._backend BType = self.ffi._get_cached_btype(tp) def newfunc(*args): args = args[:i] + (backend.newp(BType, args[i]),) + args[i+1:] return oldfunc(*args) + newfunc._cffi_base_type = base_tp return newfunc # ---------- @@ -252,11 +255,14 @@ prnt(' static ssize_t nums[] = {') prnt(' sizeof(%s),' % cname) prnt(' offsetof(struct _cffi_aligncheck, y),') - for fname, _, fbitsize in tp.enumfields(): + for fname, ftype, fbitsize in tp.enumfields(): if fbitsize >= 0: continue # xxx ignore fbitsize for now prnt(' offsetof(%s, %s),' % (cname, fname)) - prnt(' sizeof(((%s *)0)->%s),' % (cname, fname)) + if isinstance(ftype, model.ArrayType) and ftype.length is None: + prnt(' 0, /* %s */' % ftype._get_c_name()) + else: + prnt(' sizeof(((%s *)0)->%s),' % (cname, fname)) prnt(' -1') prnt(' };') prnt(' return nums[i];') @@ -270,7 +276,7 @@ return # nothing to do with opaque structs layoutfuncname = '_cffi_layout_%s_%s' % (prefix, name) # - BFunc = self.ffi.typeof("ssize_t(*)(ssize_t)") + BFunc = self.ffi._typeof_locked("ssize_t(*)(ssize_t)")[0] function = module.load_function(BFunc, layoutfuncname) layout = [] num = 0 @@ -279,7 +285,7 @@ if x < 0: break layout.append(x) num += 1 - if isinstance(tp, model.StructType) and tp.partial: + if isinstance(tp, model.StructOrUnion) and tp.partial: # use the function()'s sizes and offsets to guide the # layout of the struct totalsize = layout[0] @@ -316,9 +322,10 @@ continue # xxx ignore fbitsize for now check(layout[i], ffi.offsetof(BStruct, fname), "wrong offset for field %r" % (fname,)) - BField = ffi._get_cached_btype(ftype) - check(layout[i+1], ffi.sizeof(BField), - "wrong size for field %r" % (fname,)) + if layout[i+1] != 0: + BField = ffi._get_cached_btype(ftype) + check(layout[i+1], ffi.sizeof(BField), + "wrong size for field %r" % (fname,)) i += 2 assert i == len(layout) @@ -379,15 +386,17 @@ def _load_constant(self, is_int, tp, name, module): funcname = '_cffi_const_%s' % name if is_int: - BFunc = self.ffi.typeof("int(*)(long long*)") + BType = self.ffi._typeof_locked("long long*")[0] + BFunc = self.ffi._typeof_locked("int(*)(long long*)")[0] function = module.load_function(BFunc, funcname) - p = self.ffi.new("long long*") + p = self.ffi.new(BType) negative = function(p) value = int(p[0]) if value < 0 and not negative: - value += (1 << (8*self.ffi.sizeof("long long"))) + BLongLong = self.ffi._typeof_locked("long long")[0] + value += (1 << (8*self.ffi.sizeof(BLongLong))) else: - BFunc = self.ffi.typeof(tp.get_c_name('(*)(void)', name)) + BFunc = self.ffi._typeof_locked(tp.get_c_name('(*)(void)', name))[0] function = module.load_function(BFunc, funcname) value = function() return value @@ -413,11 +422,22 @@ prnt('int %s(char *out_error)' % funcname) prnt('{') for enumerator, enumvalue in zip(tp.enumerators, tp.enumvalues): - prnt(' if (%s != %d) {' % (enumerator, enumvalue)) + if enumvalue < 0: + prnt(' if ((%s) >= 0 || (long)(%s) != %dL) {' % ( + enumerator, enumerator, enumvalue)) + else: + prnt(' if ((%s) < 0 || (unsigned long)(%s) != %dUL) {' % ( + enumerator, enumerator, enumvalue)) + prnt(' char buf[64];') + prnt(' if ((%s) < 0)' % enumerator) + prnt(' snprintf(buf, 63, "%%ld", (long)(%s));' % enumerator) + prnt(' else') + prnt(' snprintf(buf, 63, "%%lu", (unsigned long)(%s));' % + enumerator) prnt(' snprintf(out_error, 255,' - '"%s has the real value %d, not %d",') - prnt(' "%s", (int)%s, %d);' % ( - enumerator, enumerator, enumvalue)) + ' "%s has the real value %s, not %s",') + prnt(' "%s", buf, "%d");' % ( + enumerator, enumvalue)) prnt(' return -1;') prnt(' }') prnt(' return 0;') @@ -431,10 +451,11 @@ tp.enumvalues = tuple(enumvalues) tp.partial_resolved = True else: - BFunc = self.ffi.typeof("int(*)(char*)") + BType = self.ffi._typeof_locked("char[]")[0] + BFunc = self.ffi._typeof_locked("int(*)(char*)")[0] funcname = '_cffi_e_%s_%s' % (prefix, name) function = module.load_function(BFunc, funcname) - p = self.ffi.new("char[]", 256) + p = self.ffi.new(BType, 256) if function(p) < 0: error = self.ffi.string(p) if sys.version_info >= (3,): @@ -465,6 +486,14 @@ def _generate_gen_variable_decl(self, tp, name): if isinstance(tp, model.ArrayType): + if tp.length == '...': + prnt = self._prnt + funcname = '_cffi_sizeof_%s' % (name,) + self.export_symbols.append(funcname) + prnt("size_t %s(void)" % funcname) + prnt("{") + prnt(" return sizeof(%s);" % (name,)) + prnt("}") tp_ptr = model.PointerType(tp.item) self._generate_gen_const(False, name, tp_ptr) else: @@ -476,6 +505,18 @@ def _loaded_gen_variable(self, tp, name, module, library): if isinstance(tp, model.ArrayType): # int a[5] is "constant" in the # sense that "a=..." is forbidden + if tp.length == '...': + funcname = '_cffi_sizeof_%s' % (name,) + BFunc = self.ffi._typeof_locked('size_t(*)(void)')[0] + function = module.load_function(BFunc, funcname) + size = function() + BItemType = self.ffi._get_cached_btype(tp.item) + length, rest = divmod(size, self.ffi.sizeof(BItemType)) + if rest != 0: + raise ffiplatform.VerificationError( + "bad size: %r does not seem to be an array of %s" % + (name, tp.item)) + tp = tp.resolve_length(length) tp_ptr = model.PointerType(tp.item) value = self._load_constant(False, tp_ptr, name, module) # 'value' is a which we have to replace with @@ -489,7 +530,7 @@ # remove ptr= from the library instance, and replace # it by a property on the class, which reads/writes into ptr[0]. funcname = '_cffi_var_%s' % name - BFunc = self.ffi.typeof(tp.get_c_name('*(*)(void)', name)) + BFunc = self.ffi._typeof_locked(tp.get_c_name('*(*)(void)', name))[0] function = module.load_function(BFunc, funcname) ptr = function() def getter(library): diff --git a/lib_pypy/cffi/verifier.py b/lib_pypy/cffi/verifier.py --- a/lib_pypy/cffi/verifier.py +++ b/lib_pypy/cffi/verifier.py @@ -31,7 +31,7 @@ k2 = k2.lstrip('0').rstrip('L') modulename = '_cffi_%s_%s%s%s' % (tag, self._vengine._class_key, k1, k2) - suffix = _get_so_suffix() + suffix = _get_so_suffixes()[0] self.tmpdir = tmpdir or _caller_dir_pycache() self.sourcefilename = os.path.join(self.tmpdir, modulename + '.c') self.modulefilename = os.path.join(self.tmpdir, modulename + suffix) @@ -42,18 +42,21 @@ def write_source(self, file=None): """Write the C source code. It is produced in 'self.sourcefilename', which can be tweaked beforehand.""" - if self._has_source and file is None: - raise ffiplatform.VerificationError("source code already written") - self._write_source(file) + with self.ffi._lock: + if self._has_source and file is None: + raise ffiplatform.VerificationError( + "source code already written") + self._write_source(file) def compile_module(self): """Write the C source code (if not done already) and compile it. This produces a dynamic link library in 'self.modulefilename'.""" - if self._has_module: - raise ffiplatform.VerificationError("module already compiled") - if not self._has_source: - self._write_source() - self._compile_module() + with self.ffi._lock: + if self._has_module: + raise ffiplatform.VerificationError("module already compiled") + if not self._has_source: + self._write_source() + self._compile_module() def load_library(self): """Get a C module from this Verifier instance. @@ -62,11 +65,14 @@ operations to the C module. If necessary, the C code is written and compiled first. """ - if not self._has_module: - self._locate_module() + with self.ffi._lock: if not self._has_module: - self.compile_module() - return self._load_library() + self._locate_module() + if not self._has_module: + if not self._has_source: + self._write_source() + self._compile_module() + return self._load_library() def get_module_name(self): basename = os.path.basename(self.modulefilename) @@ -81,7 +87,9 @@ def get_extension(self): if not self._has_source: - self._write_source() + with self.ffi._lock: + if not self._has_source: + self._write_source() sourcename = ffiplatform.maybe_relative_path(self.sourcefilename) modname = self.get_module_name() return ffiplatform.get_extension(sourcename, modname, **self.kwds) @@ -103,7 +111,7 @@ else: path = None filename = self._vengine.find_module(self.get_module_name(), path, - _get_so_suffix()) + _get_so_suffixes()) if filename is None: return self.modulefilename = filename @@ -193,7 +201,7 @@ if keep_so: suffix = '.c' # only remove .c files else: - suffix = _get_so_suffix().lower() + suffix = _get_so_suffixes()[0].lower() for fn in filelist: if fn.lower().startswith('_cffi_') and ( fn.lower().endswith(suffix) or fn.lower().endswith('.c')): @@ -213,15 +221,20 @@ except OSError: pass -def _get_so_suffix(): +def _get_so_suffixes(): + suffixes = [] for suffix, mode, type in imp.get_suffixes(): if type == imp.C_EXTENSION: - return suffix - # bah, no C_EXTENSION available. Occurs on pypy without cpyext - if sys.platform == 'win32': - return ".pyd" - else: - return ".so" + suffixes.append(suffix) + + if not suffixes: + # bah, no C_EXTENSION available. Occurs on pypy without cpyext + if sys.platform == 'win32': + suffixes = [".pyd"] + else: + suffixes = [".so"] + + return suffixes def _ensure_dir(filename): try: diff --git a/lib_pypy/pyrepl/reader.py b/lib_pypy/pyrepl/reader.py --- a/lib_pypy/pyrepl/reader.py +++ b/lib_pypy/pyrepl/reader.py @@ -19,11 +19,13 @@ # CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN # CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -import types +import types, re from pyrepl import unicodedata_ from pyrepl import commands from pyrepl import input +_r_csi_seq = re.compile(r"\033\[[ -@]*[A-~]") + def _make_unctrl_map(): uc_map = {} for c in map(unichr, range(256)): @@ -309,6 +311,10 @@ excluded from the length calculation. So also a copy of the prompt is returned with these control characters removed. """ + # The logic below also ignores the length of common escape + # sequences if they were not explicitly within \x01...\x02. + # They are CSI (or ANSI) sequences ( ESC [ ... LETTER ) + out_prompt = '' l = len(prompt) pos = 0 @@ -321,9 +327,13 @@ break # Found start and end brackets, subtract from string length l = l - (e-s+1) - out_prompt += prompt[pos:s] + prompt[s+1:e] + keep = prompt[pos:s] + l -= sum(map(len, _r_csi_seq.findall(keep))) + out_prompt += keep + prompt[s+1:e] pos = e+1 - out_prompt += prompt[pos:] + keep = prompt[pos:] + l -= sum(map(len, _r_csi_seq.findall(keep))) + out_prompt += keep return out_prompt, l def bow(self, p=None): diff --git a/pypy/doc/contributor.rst b/pypy/doc/contributor.rst --- a/pypy/doc/contributor.rst +++ b/pypy/doc/contributor.rst @@ -17,24 +17,24 @@ David Schneider Holger Krekel Christian Tismer + Matti Picus Hakan Ardo Benjamin Peterson - Matti Picus Philip Jenvey Anders Chrigstrom Brian Kearns + Manuel Jacob Eric van Riet Paap + Wim Lavrijsen Richard Emslie Alexander Schremmer - Wim Lavrijsen Dan Villiom Podlaski Christiansen - Manuel Jacob + Ronan Lamy Lukas Diekmann Sven Hager Anders Lehmann Aurelien Campeas Niklaus Haldimann - Ronan Lamy Camillo Bruni Laura Creighton Toon Verwaest @@ -45,8 +45,10 @@ David Edelsohn Anders Hammarquist Jakub Gustak + Romain Guillebert Guido Wesdorp Lawrence Oluyede + Remi Meier Bartosz Skowron Daniel Roberts Niko Matsakis @@ -54,18 +56,17 @@ Ludovic Aubry Alexander Hesse Jacob Hallen - Romain Guillebert Jason Creighton Alex Martelli Michal Bendowski Jan de Mooij + stian Michael Foord Stephan Diehl Stefan Schwarzer Valentino Volonghi Tomek Meka Patrick Maupin - stian Bob Ippolito Bruno Gola Jean-Paul Calderone @@ -74,29 +75,33 @@ Simon Burton Marius Gedminas John Witulski + Konstantin Lopuhin Greg Price Dario Bertini Mark Pearse Simon Cross - Konstantin Lopuhin Andreas Stührk Jean-Philippe St. Pierre Guido van Rossum Pavel Vinogradov + Paweł Piotr Przeradowski Paul deGrandis Ilya Osadchiy Adrian Kuhn Boris Feigin + Stefano Rivera tav + Taavi Burns Georg Brandl Bert Freudenberg Stian Andreassen - Stefano Rivera Wanja Saatkamp Gerald Klix Mike Blume - Taavi Burns Oscar Nierstrasz + Stefan H. Muller + Laurence Tratt + Rami Chowdhury David Malcolm Eugene Oden Henry Mason @@ -105,14 +110,15 @@ David Ripton Dusty Phillips Lukas Renggli + Edd Barrett Guenter Jantzen Tobias Oberstein - Remi Meier Ned Batchelder Amit Regmi Ben Young Nicolas Chauvat Andrew Durdin + Andrew Chambers Michael Schneider Nicholas Riley Jason Chu @@ -128,10 +134,13 @@ Olivier Dormond Jared Grubb Karl Bartel + Tobias Pape Brian Dorsey Victor Stinner + Andrews Medina Stuart Williams Jasper Schulz + Christian Hudon Toby Watson Antoine Pitrou Aaron Iles @@ -141,7 +150,6 @@ Neil Shepperd Mikael Schönenberg Elmo Mäntynen - Tobias Pape Jonathan David Riehl Stanislaw Halik Anders Qvist @@ -153,19 +161,15 @@ Alexander Sedov Corbin Simpson Christopher Pope - Laurence Tratt - Guillebert Romain Christian Tismer Dan Stromberg Stefano Parmesan - Christian Hudon Alexis Daboville Jens-Uwe Mager Carl Meyer Karl Ramm Pieter Zieschang Gabriel - Paweł Piotr Przeradowski Andrew Dalke Sylvain Thenault Nathan Taylor @@ -189,13 +193,15 @@ Martin Blais Lene Wagner Tomo Cocoa - Andrews Medina roberto at goyle + Yury V. Zaytsev + Anna Katrina Dominguez William Leslie Bobby Impollonia timo at eistee.fritz.box Andrew Thompson Yusei Tahara + Ben Darnell Roberto De Ioris Juan Francisco Cantero Hurtado Godefroid Chappelle @@ -209,7 +215,7 @@ Akira Li Gustavo Niemeyer Stephan Busemann - Anna Katrina Dominguez + Rafał Gałczyński Christian Muirhead James Lan shoma hosaka @@ -219,6 +225,7 @@ Dinu Gherman Chris Lambacher coolbutuseless at gmail.com + w31rd0 Jim Baker Rodrigo Araújo Armin Ronacher @@ -234,12 +241,12 @@ Even Wiik Thomassen jbs soareschen + Mike Bayer Flavio Percoco Kristoffer Kleine yasirs Michael Chermside Anna Ravencroft - Andrew Chambers Julien Phalip Dan Loewenherz diff --git a/pypy/doc/interpreter.rst b/pypy/doc/interpreter.rst --- a/pypy/doc/interpreter.rst +++ b/pypy/doc/interpreter.rst @@ -137,7 +137,8 @@ control flow of a function (such as ``while`` and ``try`` constructs) - a value stack where bytecode interpretation pulls object - from and puts results on. + from and puts results on. (``locals_stack_w`` is actually a single + list containing both the local scope and the value stack.) - a reference to the *globals* dictionary, containing module-level name-value bindings @@ -151,10 +152,7 @@ - the class ``PyFrame`` is defined in `pypy/interpreter/pyframe.py`_. -- the file `pypy/interpreter/pyopcode.py`_ add support for all Python opcode. - -- nested scope support is added to the ``PyFrame`` class in - `pypy/interpreter/nestedscope.py`_. +- the file `pypy/interpreter/pyopcode.py`_ add support for all Python opcodes. .. _Code: @@ -184,12 +182,6 @@ * ``co_name`` name of the code object (often the function name) * ``co_lnotab`` a helper table to compute the line-numbers corresponding to bytecodes -In PyPy, code objects also have the responsibility of creating their Frame_ objects -via the `'create_frame()`` method. With proper parser and compiler support this would -allow to create custom Frame objects extending the execution of functions -in various ways. The several Frame_ classes already utilize this flexibility -in order to implement Generators and Nested Scopes. - .. _Function: Function and Method classes diff --git a/pypy/doc/project-ideas.rst b/pypy/doc/project-ideas.rst --- a/pypy/doc/project-ideas.rst +++ b/pypy/doc/project-ideas.rst @@ -90,9 +90,6 @@ collectors can be written for specialized purposes, or even various experiments can be done for the general purpose. Examples: -* An incremental garbage collector that has specified maximal pause times, - crucial for games - * A garbage collector that compact memory better for mobile devices * A concurrent garbage collector (a lot of work) @@ -158,6 +155,8 @@ Embedding PyPy ---------------------------------------- +Note: there is a basic proof-of-concept for that as a `uwsgi pypy plugin`_ + Being able to embed PyPy, say with its own limited C API, would be useful. But here is the most interesting variant, straight from EuroPython live discussion :-) We can have a generic "libpypy.so" that @@ -166,6 +165,8 @@ exported. This would give us a one-size-fits-all generic .so file to be imported by any application that wants to load .so files :-) +.. _`uwsgi pypy plugin`: http://uwsgi-docs.readthedocs.org/en/latest/PyPy.html + Optimising cpyext (CPython C-API compatibility layer) ----------------------------------------------------- diff --git a/pypy/doc/tool/makecontributor.py b/pypy/doc/tool/makecontributor.py --- a/pypy/doc/tool/makecontributor.py +++ b/pypy/doc/tool/makecontributor.py @@ -1,3 +1,5 @@ +# NOTE: run this script with LANG=en_US.UTF-8 + import py import sys from collections import defaultdict @@ -132,7 +134,7 @@ if show_numbers: print '%5d %s' % (n, name) else: - print name + print ' ' + name if __name__ == '__main__': show_numbers = '-n' in sys.argv diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -91,6 +91,7 @@ .. branch: safe-win-mmap .. branch: boolean-indexing-cleanup .. branch: cpyext-best_base +.. branch: cpyext-int .. branch: fileops2 .. branch: nobold-backtrace diff --git a/pypy/doc/windows.rst b/pypy/doc/windows.rst --- a/pypy/doc/windows.rst +++ b/pypy/doc/windows.rst @@ -118,14 +118,10 @@ The sqlite3 database library ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Download http://www.sqlite.org/2013/sqlite-amalgamation-3071601.zip and extract -it into a directory under the base directory. Also get -http://www.sqlite.org/2013/sqlite-dll-win32-x86-3071601.zip and extract the dll -into the bin directory, and the sqlite3.def into the sources directory. -Now build the import library so cffi can use the header and dll:: +PyPy uses cffi to interact with sqlite3.dll. Only the dll is needed, the cffi +wrapper is compiled when the module is imported for the first time. +The sqlite3.dll should be version 3.6.21 for CPython2.7 compatablility. - lib /DEF:sqlite3.def" /OUT:sqlite3.lib" - copy sqlite3.lib path\to\libs The expat XML parser diff --git a/pypy/module/__pypy__/test/test_signal.py b/pypy/module/__pypy__/test/test_signal.py --- a/pypy/module/__pypy__/test/test_signal.py +++ b/pypy/module/__pypy__/test/test_signal.py @@ -56,7 +56,7 @@ interrupted = [] print('--- start ---') thread.start_new_thread(subthread, ()) - for j in range(10): + for j in range(30): if len(done): break print('.') time.sleep(0.25) diff --git a/pypy/module/_cffi_backend/__init__.py b/pypy/module/_cffi_backend/__init__.py --- a/pypy/module/_cffi_backend/__init__.py +++ b/pypy/module/_cffi_backend/__init__.py @@ -7,7 +7,7 @@ appleveldefs = { } interpleveldefs = { - '__version__': 'space.wrap("0.7")', + '__version__': 'space.wrap("0.8")', 'load_library': 'libraryobj.load_library', diff --git a/pypy/module/_cffi_backend/cdataobj.py b/pypy/module/_cffi_backend/cdataobj.py --- a/pypy/module/_cffi_backend/cdataobj.py +++ b/pypy/module/_cffi_backend/cdataobj.py @@ -19,9 +19,9 @@ _cdata = lltype.nullptr(rffi.CCHARP.TO) def __init__(self, space, cdata, ctype): - from pypy.module._cffi_backend import ctypeprim + from pypy.module._cffi_backend import ctypeobj assert lltype.typeOf(cdata) == rffi.CCHARP - assert isinstance(ctype, ctypeprim.W_CType) + assert isinstance(ctype, ctypeobj.W_CType) self.space = space self._cdata = cdata # don't forget keepalive_until_here! self.ctype = ctype @@ -211,7 +211,21 @@ keepalive_until_here(w_value) return # + # A fast path for [0:N] = "somestring". + from pypy.module._cffi_backend import ctypeprim space = self.space + if (space.isinstance_w(w_value, space.w_str) and + isinstance(ctitem, ctypeprim.W_CTypePrimitiveChar)): + from rpython.rtyper.annlowlevel import llstr + from rpython.rtyper.lltypesystem.rstr import copy_string_to_raw + value = space.str_w(w_value) + if len(value) != length: + raise operationerrfmt(space.w_ValueError, + "need a string of length %d, got %d", + length, len(value)) + copy_string_to_raw(llstr(value), cdata, 0, length) + return + # w_iter = space.iter(w_value) for i in range(length): try: @@ -245,19 +259,22 @@ space = self.space if isinstance(w_other, W_CData): from pypy.module._cffi_backend import ctypeptr, ctypearray + from pypy.module._cffi_backend import ctypevoid ct = w_other.ctype if isinstance(ct, ctypearray.W_CTypeArray): ct = ct.ctptr # if (ct is not self.ctype or not isinstance(ct, ctypeptr.W_CTypePointer) or - ct.ctitem.size <= 0): + (ct.ctitem.size <= 0 and not ct.is_void_ptr)): raise operationerrfmt(space.w_TypeError, "cannot subtract cdata '%s' and cdata '%s'", self.ctype.name, ct.name) # + itemsize = ct.ctitem.size + if itemsize <= 0: itemsize = 1 diff = (rffi.cast(lltype.Signed, self._cdata) - - rffi.cast(lltype.Signed, w_other._cdata)) // ct.ctitem.size + rffi.cast(lltype.Signed, w_other._cdata)) // itemsize return space.wrap(diff) # return self._add_or_sub(w_other, -1) @@ -441,6 +458,7 @@ __getitem__ = interp2app(W_CData.getitem), __setitem__ = interp2app(W_CData.setitem), __add__ = interp2app(W_CData.add), + __radd__ = interp2app(W_CData.add), __sub__ = interp2app(W_CData.sub), __getattr__ = interp2app(W_CData.getattr), __setattr__ = interp2app(W_CData.setattr), diff --git a/pypy/module/_cffi_backend/ctypearray.py b/pypy/module/_cffi_backend/ctypearray.py --- a/pypy/module/_cffi_backend/ctypearray.py +++ b/pypy/module/_cffi_backend/ctypearray.py @@ -34,19 +34,8 @@ datasize = self.size # if datasize < 0: - if (space.isinstance_w(w_init, space.w_list) or - space.isinstance_w(w_init, space.w_tuple)): - length = space.int_w(space.len(w_init)) - elif space.isinstance_w(w_init, space.w_basestring): - # from a string, we add the null terminator - length = space.int_w(space.len(w_init)) + 1 - else: - length = space.getindex_w(w_init, space.w_OverflowError) - if length < 0: - raise OperationError(space.w_ValueError, - space.wrap("negative array length")) - w_init = space.w_None - # + from pypy.module._cffi_backend import misc + w_init, length = misc.get_new_array_length(space, w_init) try: datasize = ovfcheck(length * self.ctitem.size) except OverflowError: diff --git a/pypy/module/_cffi_backend/ctypeptr.py b/pypy/module/_cffi_backend/ctypeptr.py --- a/pypy/module/_cffi_backend/ctypeptr.py +++ b/pypy/module/_cffi_backend/ctypeptr.py @@ -15,15 +15,12 @@ class W_CTypePtrOrArray(W_CType): - _attrs_ = ['ctitem', 'can_cast_anything', 'is_struct_ptr', - 'length'] - _immutable_fields_ = ['ctitem', 'can_cast_anything', 'is_struct_ptr', - 'length'] + _attrs_ = ['ctitem', 'can_cast_anything', 'length'] + _immutable_fields_ = ['ctitem', 'can_cast_anything', 'length'] length = -1 def __init__(self, space, size, extra, extra_position, ctitem, could_cast_anything=True): - from pypy.module._cffi_backend.ctypestruct import W_CTypeStructOrUnion name, name_position = ctitem.insert_name(extra, extra_position) W_CType.__init__(self, space, size, name, name_position) # this is the "underlying type": @@ -32,7 +29,6 @@ # - for functions, it is the return type self.ctitem = ctitem self.can_cast_anything = could_cast_anything and ctitem.cast_anything - self.is_struct_ptr = isinstance(ctitem, W_CTypeStructOrUnion) def is_char_ptr_or_array(self): return isinstance(self.ctitem, ctypeprim.W_CTypePrimitiveChar) @@ -195,6 +191,7 @@ W_CTypePtrBase.__init__(self, space, size, extra, 2, ctitem) def newp(self, w_init): + from pypy.module._cffi_backend.ctypestruct import W_CTypeStructOrUnion space = self.space ctitem = self.ctitem datasize = ctitem.size @@ -202,10 +199,15 @@ raise operationerrfmt(space.w_TypeError, "cannot instantiate ctype '%s' of unknown size", self.name) - if self.is_struct_ptr: + if isinstance(ctitem, W_CTypeStructOrUnion): # 'newp' on a struct-or-union pointer: in this case, we return # a W_CDataPtrToStruct object which has a strong reference # to a W_CDataNewOwning that really contains the structure. + # + if ctitem.with_var_array and not space.is_w(w_init, space.w_None): + datasize = ctitem.convert_struct_from_object( + lltype.nullptr(rffi.CCHARP.TO), w_init, datasize) + # cdatastruct = cdataobj.W_CDataNewOwning(space, datasize, ctitem) cdata = cdataobj.W_CDataPtrToStructOrUnion(space, cdatastruct._cdata, @@ -236,11 +238,15 @@ def add(self, cdata, i): space = self.space ctitem = self.ctitem + itemsize = ctitem.size if ctitem.size < 0: - raise operationerrfmt(space.w_TypeError, + if self.is_void_ptr: + itemsize = 1 + else: + raise operationerrfmt(space.w_TypeError, "ctype '%s' points to items of unknown size", self.name) - p = rffi.ptradd(cdata, i * self.ctitem.size) + p = rffi.ptradd(cdata, i * itemsize) return cdataobj.W_CData(space, p, self) def cast(self, w_ob): @@ -317,7 +323,8 @@ space = self.space ctype2 = cdata.ctype if (isinstance(ctype2, W_CTypeStructOrUnion) or - (isinstance(ctype2, W_CTypePtrOrArray) and ctype2.is_struct_ptr)): + (isinstance(ctype2, W_CTypePtrOrArray) and + isinstance(ctype2.ctitem, W_CTypeStructOrUnion))): ptrdata = rffi.ptradd(cdata._cdata, offset) return cdataobj.W_CData(space, ptrdata, self) else: diff --git a/pypy/module/_cffi_backend/ctypestruct.py b/pypy/module/_cffi_backend/ctypestruct.py --- a/pypy/module/_cffi_backend/ctypestruct.py +++ b/pypy/module/_cffi_backend/ctypestruct.py @@ -9,7 +9,8 @@ from rpython.rlib import jit from rpython.rlib.objectmodel import keepalive_until_here from rpython.rlib.rarithmetic import r_uint, r_ulonglong, r_longlong, intmask -from rpython.rtyper.lltypesystem import rffi +from rpython.rlib.rarithmetic import ovfcheck +from rpython.rtyper.lltypesystem import lltype, rffi from pypy.module._cffi_backend import cdataobj, ctypeprim, misc from pypy.module._cffi_backend.ctypeobj import W_CType @@ -17,12 +18,13 @@ class W_CTypeStructOrUnion(W_CType): _immutable_fields_ = ['alignment?', 'fields_list?', 'fields_dict?', - 'custom_field_pos?'] + 'custom_field_pos?', 'with_var_array?'] # fields added by complete_struct_or_union(): alignment = -1 fields_list = None fields_dict = None custom_field_pos = False + with_var_array = False def __init__(self, space, name): W_CType.__init__(self, space, -1, name, len(name)) @@ -90,12 +92,16 @@ pass def convert_from_object(self, cdata, w_ob): - space = self.space - if self._copy_from_same(cdata, w_ob): - return + if not self._copy_from_same(cdata, w_ob): + self.convert_struct_from_object(cdata, w_ob, optvarsize=-1) + @jit.look_inside_iff( + lambda self, cdata, w_ob, optvarsize: jit.isvirtual(w_ob) + ) + def convert_struct_from_object(self, cdata, w_ob, optvarsize): self._check_only_one_argument_for_union(w_ob) + space = self.space if (space.isinstance_w(w_ob, space.w_list) or space.isinstance_w(w_ob, space.w_tuple)): lst_w = space.listview(w_ob) @@ -104,7 +110,9 @@ "too many initializers for '%s' (got %d)", self.name, len(lst_w)) for i in range(len(lst_w)): - self.fields_list[i].write(cdata, lst_w[i]) + optvarsize = self.fields_list[i].write_v(cdata, lst_w[i], + optvarsize) + return optvarsize elif space.isinstance_w(w_ob, space.w_dict): lst_w = space.fixedview(w_ob) @@ -116,11 +124,16 @@ except KeyError: space.raise_key_error(w_key) assert 0 - cf.write(cdata, space.getitem(w_ob, w_key)) + optvarsize = cf.write_v(cdata, space.getitem(w_ob, w_key), + optvarsize) + return optvarsize else: - raise self._convert_error("list or tuple or dict or struct-cdata", - w_ob) + if optvarsize == -1: + msg = "list or tuple or dict or struct-cdata" + else: + msg = "list or tuple or dict" + raise self._convert_error(msg, w_ob) @jit.elidable def _getcfield_const(self, attr): @@ -192,6 +205,37 @@ else: self.ctype.convert_from_object(cdata, w_ob) + def write_v(self, cdata, w_ob, optvarsize): + # a special case for var-sized C99 arrays + from pypy.module._cffi_backend import ctypearray + ct = self.ctype + if isinstance(ct, ctypearray.W_CTypeArray) and ct.length < 0: + space = ct.space + w_ob, varsizelength = misc.get_new_array_length(space, w_ob) + if optvarsize != -1: + # in this mode, the only purpose of this function is to compute + # the real size of the structure from a var-sized C99 array + assert cdata == lltype.nullptr(rffi.CCHARP.TO) + itemsize = ct.ctitem.size + try: + varsize = ovfcheck(itemsize * varsizelength) + size = ovfcheck(self.offset + varsize) + except OverflowError: + raise OperationError(space.w_OverflowError, + space.wrap("array size would overflow a ssize_t")) + assert size >= 0 + return max(size, optvarsize) + # if 'value' was only an integer, get_new_array_length() returns + # w_ob = space.w_None. Detect if this was the case, + # and if so, stop here, leaving the content uninitialized + # (it should be zero-initialized from somewhere else). + if space.is_w(w_ob, space.w_None): + return optvarsize + # + if optvarsize == -1: + self.write(cdata, w_ob) + return optvarsize + def convert_bitfield_to_object(self, cdata): ctype = self.ctype space = ctype.space diff --git a/pypy/module/_cffi_backend/misc.py b/pypy/module/_cffi_backend/misc.py --- a/pypy/module/_cffi_backend/misc.py +++ b/pypy/module/_cffi_backend/misc.py @@ -278,6 +278,22 @@ # ____________________________________________________________ +def get_new_array_length(space, w_value): + if (space.isinstance_w(w_value, space.w_list) or + space.isinstance_w(w_value, space.w_tuple)): + return (w_value, space.int_w(space.len(w_value))) + elif space.isinstance_w(w_value, space.w_basestring): + # from a string, we add the null terminator + return (w_value, space.int_w(space.len(w_value)) + 1) + else: + explicitlength = space.getindex_w(w_value, space.w_OverflowError) + if explicitlength < 0: + raise OperationError(space.w_ValueError, + space.wrap("negative array length")) + return (space.w_None, explicitlength) + +# ____________________________________________________________ + @specialize.arg(0) def _raw_memcopy_tp(TPP, source, dest): # in its own function: LONGLONG may make the whole function jit-opaque diff --git a/pypy/module/_cffi_backend/newtype.py b/pypy/module/_cffi_backend/newtype.py --- a/pypy/module/_cffi_backend/newtype.py +++ b/pypy/module/_cffi_backend/newtype.py @@ -158,8 +158,10 @@ fields_list = [] fields_dict = {} custom_field_pos = False + with_var_array = False - for w_field in fields_w: + for i in range(len(fields_w)): + w_field = fields_w[i] field_w = space.fixedview(w_field) if not (2 <= len(field_w) <= 4): raise OperationError(space.w_TypeError, @@ -176,7 +178,11 @@ "duplicate field name '%s'", fname) # if ftype.size < 0: - raise operationerrfmt(space.w_TypeError, + if (isinstance(ftype, ctypearray.W_CTypeArray) and fbitsize < 0 + and (i == len(fields_w) - 1 or foffset != -1)): + with_var_array = True + else: + raise operationerrfmt(space.w_TypeError, "field '%s.%s' has ctype '%s' of unknown size", w_ctype.name, fname, ftype.name) # @@ -235,7 +241,8 @@ fields_list.append(fld) fields_dict[fname] = fld - boffset += ftype.size * 8 + if ftype.size >= 0: + boffset += ftype.size * 8 From noreply at buildbot.pypy.org Thu Nov 14 09:33:54 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Thu, 14 Nov 2013 09:33:54 +0100 (CET) Subject: [pypy-commit] pypy numpy-newbyteorder: move size from types to dtype Message-ID: <20131114083354.752D31C0134@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: numpy-newbyteorder Changeset: r68017:8c6fe040d6c7 Date: 2013-11-14 01:28 -0500 http://bitbucket.org/pypy/pypy/changeset/8c6fe040d6c7/ Log: move size from types to dtype diff --git a/pypy/module/micronumpy/arrayimpl/concrete.py b/pypy/module/micronumpy/arrayimpl/concrete.py --- a/pypy/module/micronumpy/arrayimpl/concrete.py +++ b/pypy/module/micronumpy/arrayimpl/concrete.py @@ -52,7 +52,7 @@ loop.setslice(space, shape, self, impl) def get_size(self): - return self.size // self.dtype.itemtype.get_element_size() + return self.size // self.dtype.get_size() def get_storage_size(self): return self.size @@ -399,7 +399,7 @@ self.storage = parent.storage self.order = parent.order self.dtype = dtype - self.size = support.product(shape) * self.dtype.itemtype.get_element_size() + self.size = support.product(shape) * self.dtype.get_size() self.start = start self.orig_arr = orig_arr diff --git a/pypy/module/micronumpy/interp_dtype.py b/pypy/module/micronumpy/interp_dtype.py --- a/pypy/module/micronumpy/interp_dtype.py +++ b/pypy/module/micronumpy/interp_dtype.py @@ -38,7 +38,7 @@ _immutable_fields_ = ["itemtype?", "num", "kind", "name?", "char", "w_box_type", "byteorder", "float_type"] def __init__(self, itemtype, num, kind, name, char, w_box_type, byteorder=NPY_NATIVE, - alternate_constructors=[], aliases=[], float_type=None, + size=1, alternate_constructors=[], aliases=[], float_type=None, fields=None, fieldnames=None, shape=[], subdtype=None): self.itemtype = itemtype self.num = num @@ -47,6 +47,7 @@ self.char = char self.w_box_type = w_box_type self.byteorder = byteorder + self.size = size self.alternate_constructors = alternate_constructors self.aliases = aliases self.float_type = float_type @@ -123,7 +124,7 @@ return self.byteorder in (NPY_NATIVE, NPY_NATBYTE) def get_size(self): - return self.itemtype.get_element_size() + return self.size * self.itemtype.get_element_size() def get_name(self): if self.char == 'S': @@ -137,7 +138,7 @@ return space.wrap("dtype('%s')" % self.get_name()) def descr_get_itemsize(self, space): - return space.wrap(self.itemtype.get_element_size()) + return space.wrap(self.get_size()) def descr_get_alignment(self, space): return space.wrap(self.itemtype.alignment) @@ -209,10 +210,11 @@ self.fields[space.str_w(key)] = offset, dtype ofs_and_items.append((offset, dtype.itemtype)) - size += dtype.itemtype.get_element_size() + size += dtype.get_size() - self.itemtype = types.RecordType(ofs_and_items, size) - self.name = "void" + str(8 * self.itemtype.get_element_size()) + self.itemtype = types.RecordType(ofs_and_items) + self.size = size + self.name = "void" + str(8 * self.get_size()) def descr_get_names(self, space): if self.fieldnames is None: @@ -264,7 +266,7 @@ w_class = space.type(self) kind = self.kind - elemsize = self.itemtype.get_element_size() + elemsize = self.get_size() builder_args = space.newtuple([space.wrap("%s%d" % (kind, elemsize)), space.wrap(0), space.wrap(1)]) version = space.wrap(3) @@ -319,7 +321,8 @@ elif newendian != NPY_IGNORE: endian = newendian itemtype = self.itemtype.__class__(endian in (NPY_NATIVE, NPY_NATBYTE)) - return W_Dtype(itemtype, self.num, self.kind, self.name, self.char, self.w_box_type, endian) + return W_Dtype(itemtype, self.num, self.kind, self.name, self.char, + self.w_box_type, endian, size=self.size) def dtype_from_list(space, w_lst): lst_w = space.listview(w_lst) @@ -343,12 +346,13 @@ assert isinstance(subdtype, W_Dtype) fields[fldname] = (offset, subdtype) ofs_and_items.append((offset, subdtype.itemtype)) - offset += subdtype.itemtype.get_element_size() * size + offset += subdtype.get_size() * size fieldnames.append(fldname) - itemtype = types.RecordType(ofs_and_items, offset) - return W_Dtype(itemtype, NPY_VOID, NPY_VOIDLTR, "void" + str(8 * itemtype.get_element_size()), - NPY_VOIDLTR, space.gettypefor(interp_boxes.W_VoidBox), fields=fields, - fieldnames=fieldnames) + itemtype = types.RecordType(ofs_and_items) + return W_Dtype(itemtype, NPY_VOID, NPY_VOIDLTR, + "void" + str(8 * offset * itemtype.get_element_size()), + NPY_VOIDLTR, space.gettypefor(interp_boxes.W_VoidBox), + fields=fields, fieldnames=fieldnames, size=offset) def dtype_from_dict(space, w_dict): raise OperationError(space.w_NotImplementedError, space.wrap( @@ -362,7 +366,8 @@ # w_align and w_copy are necessary for pickling cache = get_dtype_cache(space) - if w_shape is not None and (space.isinstance_w(w_shape, space.w_int) or space.len_w(w_shape) > 0): + if w_shape is not None and (space.isinstance_w(w_shape, space.w_int) or + space.len_w(w_shape) > 0): subdtype = descr__new__(space, w_subtype, w_dtype, w_align, w_copy) assert isinstance(subdtype, W_Dtype) size = 1 @@ -373,8 +378,11 @@ dim = space.int_w(w_dim) shape.append(dim) size *= dim - return W_Dtype(types.VoidType(subdtype.itemtype.get_element_size() * size), NPY_VOID, NPY_VOIDLTR, "void" + str(8 * subdtype.itemtype.get_element_size() * size), - NPY_VOIDLTR, space.gettypefor(interp_boxes.W_VoidBox), shape=shape, subdtype=subdtype) + return W_Dtype(types.VoidType(), NPY_VOID, NPY_VOIDLTR, + "void" + str(8 * subdtype.get_size() * size), + NPY_VOIDLTR, space.gettypefor(interp_boxes.W_VoidBox), + shape=shape, subdtype=subdtype, + size=subdtype.get_size() * size) if space.is_none(w_dtype): return cache.w_float64dtype @@ -464,17 +472,17 @@ size = 1 if char == NPY_STRINGLTR: - itemtype = types.StringType(size) + itemtype = types.StringType() basename = 'string' num = NPY_STRING w_box_type = space.gettypefor(interp_boxes.W_StringBox) elif char == NPY_VOIDLTR: - itemtype = types.VoidType(size) + itemtype = types.VoidType() basename = 'void' num = NPY_VOID w_box_type = space.gettypefor(interp_boxes.W_VoidBox) elif char == NPY_UNICODELTR: - itemtype = types.UnicodeType(size) + itemtype = types.UnicodeType() basename = 'unicode' num = NPY_UNICODE w_box_type = space.gettypefor(interp_boxes.W_UnicodeBox) @@ -482,27 +490,29 @@ assert False return W_Dtype(itemtype, num, char, - basename + str(8 * itemtype.get_element_size()), - char, w_box_type) + basename + str(8 * size * itemtype.get_element_size()), + char, w_box_type, size=size) def new_string_dtype(space, size): - itemtype = types.StringType(size) + itemtype = types.StringType() return W_Dtype( itemtype, + size=size, num=NPY_STRING, kind=NPY_STRINGLTR, - name='string' + str(8 * itemtype.get_element_size()), + name='string' + str(8 * size * itemtype.get_element_size()), char=NPY_STRINGLTR, w_box_type = space.gettypefor(interp_boxes.W_StringBox), ) def new_unicode_dtype(space, size): - itemtype = types.UnicodeType(size) + itemtype = types.UnicodeType() return W_Dtype( itemtype, + size=size, num=NPY_UNICODE, kind=NPY_UNICODELTR, - name='unicode' + str(8 * itemtype.get_element_size()), + name='unicode' + str(8 * size * itemtype.get_element_size()), char=NPY_UNICODELTR, w_box_type = space.gettypefor(interp_boxes.W_UnicodeBox), ) @@ -677,7 +687,8 @@ float_type = self.w_floatlongdtype, ) self.w_stringdtype = W_Dtype( - types.StringType(0), + types.StringType(), + size=0, num=NPY_STRING, kind=NPY_STRINGLTR, name='string', @@ -687,7 +698,8 @@ aliases=["str"], ) self.w_unicodedtype = W_Dtype( - types.UnicodeType(0), + types.UnicodeType(), + size=0, num=NPY_UNICODE, kind=NPY_UNICODELTR, name='unicode', @@ -696,7 +708,8 @@ alternate_constructors=[space.w_unicode], ) self.w_voiddtype = W_Dtype( - types.VoidType(0), + types.VoidType(), + size=0, num=NPY_VOID, kind=NPY_VOIDLTR, name='void', @@ -764,7 +777,7 @@ self.w_intpdtype, self.w_uintpdtype, ] self.float_dtypes_by_num_bytes = sorted( - (dtype.itemtype.get_element_size(), dtype) + (dtype.get_size(), dtype) for dtype in float_dtypes ) self.dtypes_by_num = {} @@ -774,7 +787,7 @@ for dtype in reversed(self.builtin_dtypes): self.dtypes_by_num[dtype.num] = dtype self.dtypes_by_name[dtype.name] = dtype - can_name = dtype.kind + str(dtype.itemtype.get_element_size()) + can_name = dtype.kind + str(dtype.get_size()) self.dtypes_by_name[can_name] = dtype self.dtypes_by_name[NPY_NATBYTE + can_name] = dtype self.dtypes_by_name[NPY_NATIVE + can_name] = dtype @@ -844,7 +857,7 @@ for k, v in typeinfo_partial.iteritems(): space.setitem(w_typeinfo, space.wrap(k), space.gettypefor(v)) for k, dtype in typeinfo_full.iteritems(): - itemsize = dtype.itemtype.get_element_size() + itemsize = dtype.get_size() items_w = [space.wrap(dtype.char), space.wrap(dtype.num), space.wrap(itemsize * 8), # in case of changing diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py --- a/pypy/module/micronumpy/interp_numarray.py +++ b/pypy/module/micronumpy/interp_numarray.py @@ -85,10 +85,10 @@ return space.wrap(len(self.get_shape())) def descr_get_itemsize(self, space): - return space.wrap(self.get_dtype().itemtype.get_element_size()) + return space.wrap(self.get_dtype().get_size()) def descr_get_nbytes(self, space): - return space.wrap(self.get_size() * self.get_dtype().itemtype.get_element_size()) + return space.wrap(self.get_size() * self.get_dtype().get_size()) def descr_fill(self, space, w_value): self.fill(self.get_dtype().coerce(space, w_value)) @@ -1340,7 +1340,7 @@ # not an array or incorrect dtype shape, elems_w = find_shape_and_elems(space, w_object, dtype) if dtype is None or ( - dtype.is_str_or_unicode() and dtype.itemtype.get_size() < 1): + dtype.is_str_or_unicode() and dtype.get_size() < 1): for w_elem in elems_w: dtype = interp_ufuncs.find_dtype_for_scalar(space, w_elem, dtype) @@ -1349,7 +1349,7 @@ if dtype is None: dtype = interp_dtype.get_dtype_cache(space).w_float64dtype - if dtype.is_str_or_unicode() and dtype.itemtype.get_size() < 1: + if dtype.is_str_or_unicode() and dtype.get_size() < 1: # promote S0 -> S1, U0 -> U1 dtype = interp_dtype.variable_dtype(space, dtype.char + '1') if ndmin > len(shape): diff --git a/pypy/module/micronumpy/interp_ufuncs.py b/pypy/module/micronumpy/interp_ufuncs.py --- a/pypy/module/micronumpy/interp_ufuncs.py +++ b/pypy/module/micronumpy/interp_ufuncs.py @@ -475,8 +475,7 @@ if dt2.is_record_type(): return dt2 if dt1.is_str_or_unicode(): - if dt2.itemtype.get_element_size() >= \ - dt1.itemtype.get_element_size(): + if dt2.get_size() >= dt1.get_size(): return dt2 return dt1 return dt2 @@ -556,7 +555,7 @@ return interp_dtype.variable_dtype(space, 'S%d' % space.len_w(w_obj)) elif current_guess.num == NPY_STRING: - if current_guess.itemtype.get_size() < space.len_w(w_obj): + if current_guess.get_size() < space.len_w(w_obj): return interp_dtype.variable_dtype(space, 'S%d' % space.len_w(w_obj)) return current_guess diff --git a/pypy/module/micronumpy/iter.py b/pypy/module/micronumpy/iter.py --- a/pypy/module/micronumpy/iter.py +++ b/pypy/module/micronumpy/iter.py @@ -164,7 +164,7 @@ self.array = array self.offset = 0 self.dtype = array.dtype - self.skip = self.dtype.itemtype.get_element_size() + self.skip = self.dtype.get_size() self.size = array.size def setitem(self, elem): diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py --- a/pypy/module/micronumpy/types.py +++ b/pypy/module/micronumpy/types.py @@ -1591,17 +1591,8 @@ ComponentBoxType = interp_boxes.W_FloatLongBox class BaseStringType(BaseType): - _immutable_fields = ['size'] - - def __init__(self, size=0): - BaseType.__init__(self) - self.size = size - def get_element_size(self): - return self.size * rffi.sizeof(self.T) - - def get_size(self): - return self.size + return rffi.sizeof(self.T) def str_unary_op(func): specialize.argtype(1)(func) @@ -1636,13 +1627,13 @@ def store(self, arr, i, offset, box): assert isinstance(box, interp_boxes.W_StringBox) - return self._store(arr.storage, i, offset, box) + size = min(arr.dtype.size, box.arr.size - box.ofs) + return self._store(arr.storage, i, offset, box, size) @jit.unroll_safe - def _store(self, storage, i, offset, box): + def _store(self, storage, i, offset, box, size): assert isinstance(box, interp_boxes.W_StringBox) - # XXX simplify to range(box.dtype.get_size()) ? - for k in range(min(self.size, box.arr.size-box.ofs)): + for k in range(size): storage[k + offset + i] = box.arr.storage[k + box.ofs] def read(self, arr, i, offset, dtype=None): @@ -1725,17 +1716,17 @@ else: w_arg = box.descr_str(space) arg = space.str_w(space.str(w_arg)) - arr = VoidBoxStorage(self.size, mydtype) + arr = VoidBoxStorage(mydtype.size, mydtype) i = 0 - for i in range(min(len(arg), self.size)): + for i in range(min(len(arg), mydtype.size)): arr.storage[i] = arg[i] - for j in range(i + 1, self.size): + for j in range(i + 1, mydtype.size): arr.storage[j] = '\x00' - return interp_boxes.W_StringBox(arr, 0, arr.dtype) + return interp_boxes.W_StringBox(arr, 0, arr.dtype) def fill(self, storage, width, box, start, stop, offset): for i in xrange(start, stop, width): - self._store(storage, i, offset, box) + self._store(storage, i, offset, box, width) class UnicodeType(BaseStringType): T = lltype.UniChar @@ -1772,14 +1763,14 @@ ofs += size def coerce(self, space, dtype, w_items): - arr = VoidBoxStorage(self.size, dtype) + arr = VoidBoxStorage(dtype.get_size(), dtype) self._coerce(space, arr, 0, dtype, w_items, dtype.shape) return interp_boxes.W_VoidBox(arr, 0, dtype) @jit.unroll_safe def store(self, arr, i, ofs, box): assert isinstance(box, interp_boxes.W_VoidBox) - for k in range(self.get_element_size()): + for k in range(box.arr.dtype.get_size()): arr.storage[k + ofs] = box.arr.storage[k + box.ofs] def readarray(self, arr, i, offset, dtype=None): @@ -1793,15 +1784,14 @@ class RecordType(BaseType): T = lltype.Char - _immutable_fields_ = ['offsets_and_fields', 'size'] + _immutable_fields_ = ['offsets_and_fields'] - def __init__(self, offsets_and_fields, size): + def __init__(self, offsets_and_fields): BaseType.__init__(self) self.offsets_and_fields = offsets_and_fields - self.size = size def get_element_size(self): - return self.size + return rffi.sizeof(self.T) def read(self, arr, i, offset, dtype=None): if dtype is None: @@ -1821,7 +1811,7 @@ raise OperationError(space.w_ValueError, space.wrap( "wrong length")) items_w = space.fixedview(w_item) - arr = VoidBoxStorage(self.size, dtype) + arr = VoidBoxStorage(dtype.get_size(), dtype) for i in range(len(items_w)): subdtype = dtype.fields[dtype.fieldnames[i]][1] ofs, itemtype = self.offsets_and_fields[i] @@ -1833,7 +1823,7 @@ @jit.unroll_safe def store(self, arr, i, ofs, box): assert isinstance(box, interp_boxes.W_VoidBox) - for k in range(self.get_element_size()): + for k in range(box.arr.dtype.get_size()): arr.storage[k + i] = box.arr.storage[k + box.ofs] @jit.unroll_safe From noreply at buildbot.pypy.org Thu Nov 14 09:33:55 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Thu, 14 Nov 2013 09:33:55 +0100 (CET) Subject: [pypy-commit] pypy numpy-newbyteorder: remove copy of offsets_and_fields on RecordType Message-ID: <20131114083355.98F8C1C0134@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: numpy-newbyteorder Changeset: r68018:fd7f87a0d522 Date: 2013-11-14 03:00 -0500 http://bitbucket.org/pypy/pypy/changeset/fd7f87a0d522/ Log: remove copy of offsets_and_fields on RecordType diff --git a/pypy/module/micronumpy/interp_dtype.py b/pypy/module/micronumpy/interp_dtype.py --- a/pypy/module/micronumpy/interp_dtype.py +++ b/pypy/module/micronumpy/interp_dtype.py @@ -198,7 +198,6 @@ self.fields = None else: self.fields = {} - ofs_and_items = [] size = 0 for key in space.listview(w_fields): value = space.getitem(w_fields, key) @@ -209,10 +208,9 @@ offset = space.int_w(space.getitem(value, space.wrap(1))) self.fields[space.str_w(key)] = offset, dtype - ofs_and_items.append((offset, dtype.itemtype)) size += dtype.get_size() - self.itemtype = types.RecordType(ofs_and_items) + self.itemtype = types.RecordType() self.size = size self.name = "void" + str(8 * self.get_size()) @@ -328,7 +326,6 @@ lst_w = space.listview(w_lst) fields = {} offset = 0 - ofs_and_items = [] fieldnames = [] for w_elem in lst_w: size = 1 @@ -345,10 +342,9 @@ raise OperationError(space.w_ValueError, space.wrap("two fields with the same name")) assert isinstance(subdtype, W_Dtype) fields[fldname] = (offset, subdtype) - ofs_and_items.append((offset, subdtype.itemtype)) offset += subdtype.get_size() * size fieldnames.append(fldname) - itemtype = types.RecordType(ofs_and_items) + itemtype = types.RecordType() return W_Dtype(itemtype, NPY_VOID, NPY_VOIDLTR, "void" + str(8 * offset * itemtype.get_element_size()), NPY_VOIDLTR, space.gettypefor(interp_boxes.W_VoidBox), diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py --- a/pypy/module/micronumpy/types.py +++ b/pypy/module/micronumpy/types.py @@ -1784,11 +1784,6 @@ class RecordType(BaseType): T = lltype.Char - _immutable_fields_ = ['offsets_and_fields'] - - def __init__(self, offsets_and_fields): - BaseType.__init__(self) - self.offsets_and_fields = offsets_and_fields def get_element_size(self): return rffi.sizeof(self.T) @@ -1807,14 +1802,14 @@ if not space.issequence_w(w_item): raise OperationError(space.w_TypeError, space.wrap( "expected sequence")) - if len(self.offsets_and_fields) != space.len_w(w_item): + if len(dtype.fields) != space.len_w(w_item): raise OperationError(space.w_ValueError, space.wrap( "wrong length")) items_w = space.fixedview(w_item) arr = VoidBoxStorage(dtype.get_size(), dtype) for i in range(len(items_w)): - subdtype = dtype.fields[dtype.fieldnames[i]][1] - ofs, itemtype = self.offsets_and_fields[i] + ofs, subdtype = dtype.fields[dtype.fieldnames[i]] + itemtype = subdtype.itemtype w_item = items_w[i] w_box = itemtype.coerce(space, subdtype, w_item) itemtype.store(arr, 0, ofs, w_box) @@ -1831,7 +1826,9 @@ assert isinstance(box, interp_boxes.W_VoidBox) pieces = ["("] first = True - for ofs, tp in self.offsets_and_fields: + for name in box.dtype.fieldnames: + ofs, subdtype = box.dtype.fields[name] + tp = subdtype.itemtype if first: first = False else: From noreply at buildbot.pypy.org Thu Nov 14 09:33:56 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Thu, 14 Nov 2013 09:33:56 +0100 (CET) Subject: [pypy-commit] pypy numpy-newbyteorder: provide ndarray.newbyteorder() Message-ID: <20131114083356.DAAF71C0134@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: numpy-newbyteorder Changeset: r68019:303889d4a9d6 Date: 2013-11-14 03:23 -0500 http://bitbucket.org/pypy/pypy/changeset/303889d4a9d6/ Log: provide ndarray.newbyteorder() diff --git a/pypy/module/micronumpy/conversion_utils.py b/pypy/module/micronumpy/conversion_utils.py --- a/pypy/module/micronumpy/conversion_utils.py +++ b/pypy/module/micronumpy/conversion_utils.py @@ -63,6 +63,7 @@ raise OperationError(space.w_TypeError, space.wrap( "order not understood")) + def multi_axis_converter(space, w_axis, ndim): if space.is_none(w_axis): return [True] * ndim diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py --- a/pypy/module/micronumpy/interp_numarray.py +++ b/pypy/module/micronumpy/interp_numarray.py @@ -625,10 +625,10 @@ raise OperationError(space.w_NotImplementedError, space.wrap( "itemset not implemented yet")) - @unwrap_spec(neworder=str) - def descr_newbyteorder(self, space, neworder): - raise OperationError(space.w_NotImplementedError, space.wrap( - "newbyteorder not implemented yet")) + @unwrap_spec(new_order=str) + def descr_newbyteorder(self, space, new_order=NPY_SWAP): + return self.descr_view(space, + self.get_dtype().descr_newbyteorder(space, new_order)) @unwrap_spec(w_axis=WrappedDefault(None), w_out=WrappedDefault(None)) @@ -1268,6 +1268,7 @@ diagonal = interp2app(W_NDimArray.descr_diagonal), trace = interp2app(W_NDimArray.descr_trace), view = interp2app(W_NDimArray.descr_view), + newbyteorder = interp2app(W_NDimArray.descr_newbyteorder), ctypes = GetSetProperty(W_NDimArray.descr_get_ctypes), # XXX unimplemented __array_interface__ = GetSetProperty(W_NDimArray.descr_array_iface), diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py --- a/pypy/module/micronumpy/test/test_numarray.py +++ b/pypy/module/micronumpy/test/test_numarray.py @@ -2928,6 +2928,15 @@ assert str(a.dtype) == '|S1' assert a == 'x' + def test_newbyteorder(self): + import numpy as np + a = np.array([1, 2], dtype=np.int16) + b = a.newbyteorder() + assert (b == [256, 512]).all() + c = b.byteswap() + assert (c == [1, 2]).all() + assert (a == [1, 2]).all() + def test_pickle(self): from numpypy import dtype, array from cPickle import loads, dumps From noreply at buildbot.pypy.org Thu Nov 14 09:33:58 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Thu, 14 Nov 2013 09:33:58 +0100 (CET) Subject: [pypy-commit] pypy numpy-newbyteorder: update immutable_fields on dtype Message-ID: <20131114083358.16CFC1C0134@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: numpy-newbyteorder Changeset: r68020:022135968606 Date: 2013-11-14 03:29 -0500 http://bitbucket.org/pypy/pypy/changeset/022135968606/ Log: update immutable_fields on dtype diff --git a/pypy/module/micronumpy/interp_dtype.py b/pypy/module/micronumpy/interp_dtype.py --- a/pypy/module/micronumpy/interp_dtype.py +++ b/pypy/module/micronumpy/interp_dtype.py @@ -35,7 +35,9 @@ return out class W_Dtype(W_Root): - _immutable_fields_ = ["itemtype?", "num", "kind", "name?", "char", "w_box_type", "byteorder", "float_type"] + _immutable_fields_ = ["itemtype?", "num", "kind", "name?", "char", + "w_box_type", "byteorder", "size?", "float_type", + "fields?", "fieldnames?", "shape", "subdtype", "base"] def __init__(self, itemtype, num, kind, name, char, w_box_type, byteorder=NPY_NATIVE, size=1, alternate_constructors=[], aliases=[], float_type=None, From noreply at buildbot.pypy.org Thu Nov 14 09:33:59 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Thu, 14 Nov 2013 09:33:59 +0100 (CET) Subject: [pypy-commit] pypy default: merge numpy-newbyteorder Message-ID: <20131114083359.53B6F1C0134@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r68021:a80ac0e94c7e Date: 2013-11-14 03:30 -0500 http://bitbucket.org/pypy/pypy/changeset/a80ac0e94c7e/ Log: merge numpy-newbyteorder diff --git a/pypy/module/micronumpy/arrayimpl/concrete.py b/pypy/module/micronumpy/arrayimpl/concrete.py --- a/pypy/module/micronumpy/arrayimpl/concrete.py +++ b/pypy/module/micronumpy/arrayimpl/concrete.py @@ -52,7 +52,7 @@ loop.setslice(space, shape, self, impl) def get_size(self): - return self.size // self.dtype.itemtype.get_element_size() + return self.size // self.dtype.get_size() def get_storage_size(self): return self.size @@ -399,7 +399,7 @@ self.storage = parent.storage self.order = parent.order self.dtype = dtype - self.size = support.product(shape) * self.dtype.itemtype.get_element_size() + self.size = support.product(shape) * self.dtype.get_size() self.start = start self.orig_arr = orig_arr diff --git a/pypy/module/micronumpy/conversion_utils.py b/pypy/module/micronumpy/conversion_utils.py --- a/pypy/module/micronumpy/conversion_utils.py +++ b/pypy/module/micronumpy/conversion_utils.py @@ -1,6 +1,27 @@ from pypy.interpreter.error import OperationError from pypy.module.micronumpy.constants import * + +def byteorder_converter(space, new_order): + endian = new_order[0] + if endian not in (NPY_BIG, NPY_LITTLE, NPY_NATIVE, NPY_IGNORE, NPY_SWAP): + ch = endian + if ch in ('b', 'B'): + endian = NPY_BIG + elif ch in ('l', 'L'): + endian = NPY_LITTLE + elif ch in ('n', 'N'): + endian = NPY_NATIVE + elif ch in ('i', 'I'): + endian = NPY_IGNORE + elif ch in ('s', 'S'): + endian = NPY_SWAP + else: + raise OperationError(space.w_ValueError, space.wrap( + "%s is an unrecognized byteorder" % new_order)) + return endian + + def clipmode_converter(space, w_mode): if space.is_none(w_mode): return NPY_RAISE @@ -19,6 +40,7 @@ raise OperationError(space.w_TypeError, space.wrap("clipmode not understood")) + def order_converter(space, w_order, default): if space.is_none(w_order): return default @@ -41,6 +63,7 @@ raise OperationError(space.w_TypeError, space.wrap( "order not understood")) + def multi_axis_converter(space, w_axis, ndim): if space.is_none(w_axis): return [True] * ndim diff --git a/pypy/module/micronumpy/interp_dtype.py b/pypy/module/micronumpy/interp_dtype.py --- a/pypy/module/micronumpy/interp_dtype.py +++ b/pypy/module/micronumpy/interp_dtype.py @@ -9,6 +9,7 @@ from rpython.rlib.rarithmetic import LONG_BIT, r_longlong, r_ulonglong from rpython.rtyper.lltypesystem import rffi from rpython.rlib import jit +from pypy.module.micronumpy.conversion_utils import byteorder_converter from pypy.module.micronumpy.constants import * @@ -34,10 +35,12 @@ return out class W_Dtype(W_Root): - _immutable_fields_ = ["itemtype?", "num", "kind", "name?", "char", "w_box_type", "byteorder", "float_type"] + _immutable_fields_ = ["itemtype?", "num", "kind", "name?", "char", + "w_box_type", "byteorder", "size?", "float_type", + "fields?", "fieldnames?", "shape", "subdtype", "base"] def __init__(self, itemtype, num, kind, name, char, w_box_type, byteorder=NPY_NATIVE, - alternate_constructors=[], aliases=[], float_type=None, + size=1, alternate_constructors=[], aliases=[], float_type=None, fields=None, fieldnames=None, shape=[], subdtype=None): self.itemtype = itemtype self.num = num @@ -46,6 +49,7 @@ self.char = char self.w_box_type = w_box_type self.byteorder = byteorder + self.size = size self.alternate_constructors = alternate_constructors self.aliases = aliases self.float_type = float_type @@ -122,7 +126,7 @@ return self.byteorder in (NPY_NATIVE, NPY_NATBYTE) def get_size(self): - return self.itemtype.get_element_size() + return self.size * self.itemtype.get_element_size() def get_name(self): if self.char == 'S': @@ -136,7 +140,7 @@ return space.wrap("dtype('%s')" % self.get_name()) def descr_get_itemsize(self, space): - return space.wrap(self.itemtype.get_element_size()) + return space.wrap(self.get_size()) def descr_get_alignment(self, space): return space.wrap(self.itemtype.alignment) @@ -196,7 +200,6 @@ self.fields = None else: self.fields = {} - ofs_and_items = [] size = 0 for key in space.listview(w_fields): value = space.getitem(w_fields, key) @@ -207,11 +210,11 @@ offset = space.int_w(space.getitem(value, space.wrap(1))) self.fields[space.str_w(key)] = offset, dtype - ofs_and_items.append((offset, dtype.itemtype)) - size += dtype.itemtype.get_element_size() + size += dtype.get_size() - self.itemtype = types.RecordType(ofs_and_items, size) - self.name = "void" + str(8 * self.itemtype.get_element_size()) + self.itemtype = types.RecordType() + self.size = size + self.name = "void" + str(8 * self.get_size()) def descr_get_names(self, space): if self.fieldnames is None: @@ -263,7 +266,7 @@ w_class = space.type(self) kind = self.kind - elemsize = self.itemtype.get_element_size() + elemsize = self.get_size() builder_args = space.newtuple([space.wrap("%s%d" % (kind, elemsize)), space.wrap(0), space.wrap(1)]) version = space.wrap(3) @@ -308,11 +311,23 @@ fields = space.getitem(w_data, space.wrap(4)) self.set_fields(space, fields) + @unwrap_spec(new_order=str) + def descr_newbyteorder(self, space, new_order=NPY_SWAP): + newendian = byteorder_converter(space, new_order) + endian = self.byteorder + if endian != NPY_IGNORE: + if newendian == NPY_SWAP: + endian = NPY_OPPBYTE if self.is_native() else NPY_NATBYTE + elif newendian != NPY_IGNORE: + endian = newendian + itemtype = self.itemtype.__class__(endian in (NPY_NATIVE, NPY_NATBYTE)) + return W_Dtype(itemtype, self.num, self.kind, self.name, self.char, + self.w_box_type, endian, size=self.size) + def dtype_from_list(space, w_lst): lst_w = space.listview(w_lst) fields = {} offset = 0 - ofs_and_items = [] fieldnames = [] for w_elem in lst_w: size = 1 @@ -329,13 +344,13 @@ raise OperationError(space.w_ValueError, space.wrap("two fields with the same name")) assert isinstance(subdtype, W_Dtype) fields[fldname] = (offset, subdtype) - ofs_and_items.append((offset, subdtype.itemtype)) - offset += subdtype.itemtype.get_element_size() * size + offset += subdtype.get_size() * size fieldnames.append(fldname) - itemtype = types.RecordType(ofs_and_items, offset) - return W_Dtype(itemtype, NPY_VOID, NPY_VOIDLTR, "void" + str(8 * itemtype.get_element_size()), - NPY_VOIDLTR, space.gettypefor(interp_boxes.W_VoidBox), fields=fields, - fieldnames=fieldnames) + itemtype = types.RecordType() + return W_Dtype(itemtype, NPY_VOID, NPY_VOIDLTR, + "void" + str(8 * offset * itemtype.get_element_size()), + NPY_VOIDLTR, space.gettypefor(interp_boxes.W_VoidBox), + fields=fields, fieldnames=fieldnames, size=offset) def dtype_from_dict(space, w_dict): raise OperationError(space.w_NotImplementedError, space.wrap( @@ -349,7 +364,8 @@ # w_align and w_copy are necessary for pickling cache = get_dtype_cache(space) - if w_shape is not None and (space.isinstance_w(w_shape, space.w_int) or space.len_w(w_shape) > 0): + if w_shape is not None and (space.isinstance_w(w_shape, space.w_int) or + space.len_w(w_shape) > 0): subdtype = descr__new__(space, w_subtype, w_dtype, w_align, w_copy) assert isinstance(subdtype, W_Dtype) size = 1 @@ -360,8 +376,11 @@ dim = space.int_w(w_dim) shape.append(dim) size *= dim - return W_Dtype(types.VoidType(subdtype.itemtype.get_element_size() * size), NPY_VOID, NPY_VOIDLTR, "void" + str(8 * subdtype.itemtype.get_element_size() * size), - NPY_VOIDLTR, space.gettypefor(interp_boxes.W_VoidBox), shape=shape, subdtype=subdtype) + return W_Dtype(types.VoidType(), NPY_VOID, NPY_VOIDLTR, + "void" + str(8 * subdtype.get_size() * size), + NPY_VOIDLTR, space.gettypefor(interp_boxes.W_VoidBox), + shape=shape, subdtype=subdtype, + size=subdtype.get_size() * size) if space.is_none(w_dtype): return cache.w_float64dtype @@ -413,6 +432,7 @@ __reduce__ = interp2app(W_Dtype.descr_reduce), __setstate__ = interp2app(W_Dtype.descr_setstate), + newbyteorder = interp2app(W_Dtype.descr_newbyteorder), type = interp_attrproperty_w("w_box_type", cls=W_Dtype), kind = interp_attrproperty("kind", cls=W_Dtype), @@ -450,17 +470,17 @@ size = 1 if char == NPY_STRINGLTR: - itemtype = types.StringType(size) + itemtype = types.StringType() basename = 'string' num = NPY_STRING w_box_type = space.gettypefor(interp_boxes.W_StringBox) elif char == NPY_VOIDLTR: - itemtype = types.VoidType(size) + itemtype = types.VoidType() basename = 'void' num = NPY_VOID w_box_type = space.gettypefor(interp_boxes.W_VoidBox) elif char == NPY_UNICODELTR: - itemtype = types.UnicodeType(size) + itemtype = types.UnicodeType() basename = 'unicode' num = NPY_UNICODE w_box_type = space.gettypefor(interp_boxes.W_UnicodeBox) @@ -468,27 +488,29 @@ assert False return W_Dtype(itemtype, num, char, - basename + str(8 * itemtype.get_element_size()), - char, w_box_type) + basename + str(8 * size * itemtype.get_element_size()), + char, w_box_type, size=size) def new_string_dtype(space, size): - itemtype = types.StringType(size) + itemtype = types.StringType() return W_Dtype( itemtype, + size=size, num=NPY_STRING, kind=NPY_STRINGLTR, - name='string' + str(8 * itemtype.get_element_size()), + name='string' + str(8 * size * itemtype.get_element_size()), char=NPY_STRINGLTR, w_box_type = space.gettypefor(interp_boxes.W_StringBox), ) def new_unicode_dtype(space, size): - itemtype = types.UnicodeType(size) + itemtype = types.UnicodeType() return W_Dtype( itemtype, + size=size, num=NPY_UNICODE, kind=NPY_UNICODELTR, - name='unicode' + str(8 * itemtype.get_element_size()), + name='unicode' + str(8 * size * itemtype.get_element_size()), char=NPY_UNICODELTR, w_box_type = space.gettypefor(interp_boxes.W_UnicodeBox), ) @@ -663,7 +685,8 @@ float_type = self.w_floatlongdtype, ) self.w_stringdtype = W_Dtype( - types.StringType(0), + types.StringType(), + size=0, num=NPY_STRING, kind=NPY_STRINGLTR, name='string', @@ -673,7 +696,8 @@ aliases=["str"], ) self.w_unicodedtype = W_Dtype( - types.UnicodeType(0), + types.UnicodeType(), + size=0, num=NPY_UNICODE, kind=NPY_UNICODELTR, name='unicode', @@ -682,7 +706,8 @@ alternate_constructors=[space.w_unicode], ) self.w_voiddtype = W_Dtype( - types.VoidType(0), + types.VoidType(), + size=0, num=NPY_VOID, kind=NPY_VOIDLTR, name='void', @@ -750,7 +775,7 @@ self.w_intpdtype, self.w_uintpdtype, ] self.float_dtypes_by_num_bytes = sorted( - (dtype.itemtype.get_element_size(), dtype) + (dtype.get_size(), dtype) for dtype in float_dtypes ) self.dtypes_by_num = {} @@ -760,7 +785,7 @@ for dtype in reversed(self.builtin_dtypes): self.dtypes_by_num[dtype.num] = dtype self.dtypes_by_name[dtype.name] = dtype - can_name = dtype.kind + str(dtype.itemtype.get_element_size()) + can_name = dtype.kind + str(dtype.get_size()) self.dtypes_by_name[can_name] = dtype self.dtypes_by_name[NPY_NATBYTE + can_name] = dtype self.dtypes_by_name[NPY_NATIVE + can_name] = dtype @@ -830,7 +855,7 @@ for k, v in typeinfo_partial.iteritems(): space.setitem(w_typeinfo, space.wrap(k), space.gettypefor(v)) for k, dtype in typeinfo_full.iteritems(): - itemsize = dtype.itemtype.get_element_size() + itemsize = dtype.get_size() items_w = [space.wrap(dtype.char), space.wrap(dtype.num), space.wrap(itemsize * 8), # in case of changing diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py --- a/pypy/module/micronumpy/interp_numarray.py +++ b/pypy/module/micronumpy/interp_numarray.py @@ -85,10 +85,10 @@ return space.wrap(len(self.get_shape())) def descr_get_itemsize(self, space): - return space.wrap(self.get_dtype().itemtype.get_element_size()) + return space.wrap(self.get_dtype().get_size()) def descr_get_nbytes(self, space): - return space.wrap(self.get_size() * self.get_dtype().itemtype.get_element_size()) + return space.wrap(self.get_size() * self.get_dtype().get_size()) def descr_fill(self, space, w_value): self.fill(self.get_dtype().coerce(space, w_value)) @@ -625,10 +625,10 @@ raise OperationError(space.w_NotImplementedError, space.wrap( "itemset not implemented yet")) - @unwrap_spec(neworder=str) - def descr_newbyteorder(self, space, neworder): - raise OperationError(space.w_NotImplementedError, space.wrap( - "newbyteorder not implemented yet")) + @unwrap_spec(new_order=str) + def descr_newbyteorder(self, space, new_order=NPY_SWAP): + return self.descr_view(space, + self.get_dtype().descr_newbyteorder(space, new_order)) @unwrap_spec(w_axis=WrappedDefault(None), w_out=WrappedDefault(None)) @@ -1268,6 +1268,7 @@ diagonal = interp2app(W_NDimArray.descr_diagonal), trace = interp2app(W_NDimArray.descr_trace), view = interp2app(W_NDimArray.descr_view), + newbyteorder = interp2app(W_NDimArray.descr_newbyteorder), ctypes = GetSetProperty(W_NDimArray.descr_get_ctypes), # XXX unimplemented __array_interface__ = GetSetProperty(W_NDimArray.descr_array_iface), @@ -1340,7 +1341,7 @@ # not an array or incorrect dtype shape, elems_w = find_shape_and_elems(space, w_object, dtype) if dtype is None or ( - dtype.is_str_or_unicode() and dtype.itemtype.get_size() < 1): + dtype.is_str_or_unicode() and dtype.get_size() < 1): for w_elem in elems_w: dtype = interp_ufuncs.find_dtype_for_scalar(space, w_elem, dtype) @@ -1349,7 +1350,7 @@ if dtype is None: dtype = interp_dtype.get_dtype_cache(space).w_float64dtype - if dtype.is_str_or_unicode() and dtype.itemtype.get_size() < 1: + if dtype.is_str_or_unicode() and dtype.get_size() < 1: # promote S0 -> S1, U0 -> U1 dtype = interp_dtype.variable_dtype(space, dtype.char + '1') if ndmin > len(shape): diff --git a/pypy/module/micronumpy/interp_ufuncs.py b/pypy/module/micronumpy/interp_ufuncs.py --- a/pypy/module/micronumpy/interp_ufuncs.py +++ b/pypy/module/micronumpy/interp_ufuncs.py @@ -475,8 +475,7 @@ if dt2.is_record_type(): return dt2 if dt1.is_str_or_unicode(): - if dt2.itemtype.get_element_size() >= \ - dt1.itemtype.get_element_size(): + if dt2.get_size() >= dt1.get_size(): return dt2 return dt1 return dt2 @@ -556,7 +555,7 @@ return interp_dtype.variable_dtype(space, 'S%d' % space.len_w(w_obj)) elif current_guess.num == NPY_STRING: - if current_guess.itemtype.get_size() < space.len_w(w_obj): + if current_guess.get_size() < space.len_w(w_obj): return interp_dtype.variable_dtype(space, 'S%d' % space.len_w(w_obj)) return current_guess diff --git a/pypy/module/micronumpy/iter.py b/pypy/module/micronumpy/iter.py --- a/pypy/module/micronumpy/iter.py +++ b/pypy/module/micronumpy/iter.py @@ -164,7 +164,7 @@ self.array = array self.offset = 0 self.dtype = array.dtype - self.skip = self.dtype.itemtype.get_element_size() + self.skip = self.dtype.get_size() self.size = array.size def setitem(self, elem): diff --git a/pypy/module/micronumpy/test/test_dtypes.py b/pypy/module/micronumpy/test/test_dtypes.py --- a/pypy/module/micronumpy/test/test_dtypes.py +++ b/pypy/module/micronumpy/test/test_dtypes.py @@ -288,6 +288,49 @@ assert a.dtype.__reduce__() == (dtype, ('i4', 0, 1), (3, '<', None, None, None, -1, -1, 0)) assert loads(dumps(a.dtype)) == a.dtype + def test_newbyteorder(self): + import numpypy as np + import sys + sys_is_le = sys.byteorder == 'little' + native_code = sys_is_le and '<' or '>' + swapped_code = sys_is_le and '>' or '<' + native_dt = np.dtype(native_code+'i2') + swapped_dt = np.dtype(swapped_code+'i2') + assert native_dt.newbyteorder('S') == swapped_dt + assert native_dt.newbyteorder() == swapped_dt + assert native_dt == swapped_dt.newbyteorder('S') + assert native_dt == swapped_dt.newbyteorder('=') + assert native_dt == swapped_dt.newbyteorder('N') + assert native_dt == native_dt.newbyteorder('|') + assert np.dtype('i2') == native_dt.newbyteorder('>') + assert np.dtype('>i2') == native_dt.newbyteorder('B') + + for t in [np.int_, np.float_]: + dt = np.dtype(t) + dt1 = dt.newbyteorder().newbyteorder() + dt2 = dt.newbyteorder("<") + dt3 = dt.newbyteorder(">") + assert dt.byteorder != dt1.byteorder + #assert hash(dt) == hash(dt1) + if dt == dt2: + assert dt.byteorder != dt2.byteorder + #assert hash(dt) == hash(dt2) + else: + assert dt.byteorder != dt3.byteorder + #assert hash(dt) == hash(dt3) + + exc = raises(ValueError, dt.newbyteorder, 'XX') + assert exc.value[0] == 'XX is an unrecognized byteorder' + + for t in [np.int_, np.float_]: + dt1 = np.dtype(t) + dt2 = dt1.newbyteorder() + s1 = np.array(123, dtype=dt1).tostring() + s2 = np.array(123, dtype=dt2).byteswap().tostring() + assert s1 == s2 + class AppTestTypes(BaseAppTestDtypes): def test_abstract_types(self): import numpypy as numpy diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py --- a/pypy/module/micronumpy/test/test_numarray.py +++ b/pypy/module/micronumpy/test/test_numarray.py @@ -2928,6 +2928,15 @@ assert str(a.dtype) == '|S1' assert a == 'x' + def test_newbyteorder(self): + import numpy as np + a = np.array([1, 2], dtype=np.int16) + b = a.newbyteorder() + assert (b == [256, 512]).all() + c = b.byteswap() + assert (c == [1, 2]).all() + assert (a == [1, 2]).all() + def test_pickle(self): from numpypy import dtype, array from cPickle import loads, dumps diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py --- a/pypy/module/micronumpy/types.py +++ b/pypy/module/micronumpy/types.py @@ -1591,17 +1591,8 @@ ComponentBoxType = interp_boxes.W_FloatLongBox class BaseStringType(BaseType): - _immutable_fields = ['size'] - - def __init__(self, size=0): - BaseType.__init__(self) - self.size = size - def get_element_size(self): - return self.size * rffi.sizeof(self.T) - - def get_size(self): - return self.size + return rffi.sizeof(self.T) def str_unary_op(func): specialize.argtype(1)(func) @@ -1636,13 +1627,13 @@ def store(self, arr, i, offset, box): assert isinstance(box, interp_boxes.W_StringBox) - return self._store(arr.storage, i, offset, box) + size = min(arr.dtype.size, box.arr.size - box.ofs) + return self._store(arr.storage, i, offset, box, size) @jit.unroll_safe - def _store(self, storage, i, offset, box): + def _store(self, storage, i, offset, box, size): assert isinstance(box, interp_boxes.W_StringBox) - # XXX simplify to range(box.dtype.get_size()) ? - for k in range(min(self.size, box.arr.size-box.ofs)): + for k in range(size): storage[k + offset + i] = box.arr.storage[k + box.ofs] def read(self, arr, i, offset, dtype=None): @@ -1725,17 +1716,17 @@ else: w_arg = box.descr_str(space) arg = space.str_w(space.str(w_arg)) - arr = VoidBoxStorage(self.size, mydtype) + arr = VoidBoxStorage(mydtype.size, mydtype) i = 0 - for i in range(min(len(arg), self.size)): + for i in range(min(len(arg), mydtype.size)): arr.storage[i] = arg[i] - for j in range(i + 1, self.size): + for j in range(i + 1, mydtype.size): arr.storage[j] = '\x00' - return interp_boxes.W_StringBox(arr, 0, arr.dtype) + return interp_boxes.W_StringBox(arr, 0, arr.dtype) def fill(self, storage, width, box, start, stop, offset): for i in xrange(start, stop, width): - self._store(storage, i, offset, box) + self._store(storage, i, offset, box, width) class UnicodeType(BaseStringType): T = lltype.UniChar @@ -1772,14 +1763,14 @@ ofs += size def coerce(self, space, dtype, w_items): - arr = VoidBoxStorage(self.size, dtype) + arr = VoidBoxStorage(dtype.get_size(), dtype) self._coerce(space, arr, 0, dtype, w_items, dtype.shape) return interp_boxes.W_VoidBox(arr, 0, dtype) @jit.unroll_safe def store(self, arr, i, ofs, box): assert isinstance(box, interp_boxes.W_VoidBox) - for k in range(self.get_element_size()): + for k in range(box.arr.dtype.get_size()): arr.storage[k + ofs] = box.arr.storage[k + box.ofs] def readarray(self, arr, i, offset, dtype=None): @@ -1793,15 +1784,9 @@ class RecordType(BaseType): T = lltype.Char - _immutable_fields_ = ['offsets_and_fields', 'size'] - - def __init__(self, offsets_and_fields, size): - BaseType.__init__(self) - self.offsets_and_fields = offsets_and_fields - self.size = size def get_element_size(self): - return self.size + return rffi.sizeof(self.T) def read(self, arr, i, offset, dtype=None): if dtype is None: @@ -1817,14 +1802,14 @@ if not space.issequence_w(w_item): raise OperationError(space.w_TypeError, space.wrap( "expected sequence")) - if len(self.offsets_and_fields) != space.len_w(w_item): + if len(dtype.fields) != space.len_w(w_item): raise OperationError(space.w_ValueError, space.wrap( "wrong length")) items_w = space.fixedview(w_item) - arr = VoidBoxStorage(self.size, dtype) + arr = VoidBoxStorage(dtype.get_size(), dtype) for i in range(len(items_w)): - subdtype = dtype.fields[dtype.fieldnames[i]][1] - ofs, itemtype = self.offsets_and_fields[i] + ofs, subdtype = dtype.fields[dtype.fieldnames[i]] + itemtype = subdtype.itemtype w_item = items_w[i] w_box = itemtype.coerce(space, subdtype, w_item) itemtype.store(arr, 0, ofs, w_box) @@ -1833,7 +1818,7 @@ @jit.unroll_safe def store(self, arr, i, ofs, box): assert isinstance(box, interp_boxes.W_VoidBox) - for k in range(self.get_element_size()): + for k in range(box.arr.dtype.get_size()): arr.storage[k + i] = box.arr.storage[k + box.ofs] @jit.unroll_safe @@ -1841,7 +1826,9 @@ assert isinstance(box, interp_boxes.W_VoidBox) pieces = ["("] first = True - for ofs, tp in self.offsets_and_fields: + for name in box.dtype.fieldnames: + ofs, subdtype = box.dtype.fields[name] + tp = subdtype.itemtype if first: first = False else: From noreply at buildbot.pypy.org Thu Nov 14 09:34:00 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Thu, 14 Nov 2013 09:34:00 +0100 (CET) Subject: [pypy-commit] pypy numpy-newbyteorder: close merged branch Message-ID: <20131114083400.69B0E1C0134@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: numpy-newbyteorder Changeset: r68022:cf222adc939f Date: 2013-11-14 03:32 -0500 http://bitbucket.org/pypy/pypy/changeset/cf222adc939f/ Log: close merged branch From noreply at buildbot.pypy.org Thu Nov 14 09:45:03 2013 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 14 Nov 2013 09:45:03 +0100 (CET) Subject: [pypy-commit] pypy default: hg merge release-2.2.x Message-ID: <20131114084503.6DC0A1C0134@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r68023:f997b57170d5 Date: 2013-11-14 09:42 +0100 http://bitbucket.org/pypy/pypy/changeset/f997b57170d5/ Log: hg merge release-2.2.x diff --git a/README.rst b/README.rst --- a/README.rst +++ b/README.rst @@ -33,7 +33,7 @@ $ rpython/bin/rpython -Ojit pypy/goal/targetpypystandalone.py This ends up with ``pypy-c`` binary in the main pypy directory. We suggest -to use virtualenv with the resulting pypy-c as the interpreter, you can +to use virtualenv with the resulting pypy-c as the interpreter; you can find more details about various installation schemes here: http://doc.pypy.org/en/latest/getting-started.html#installing-pypy diff --git a/lib_pypy/cffi/api.py b/lib_pypy/cffi/api.py --- a/lib_pypy/cffi/api.py +++ b/lib_pypy/cffi/api.py @@ -347,6 +347,9 @@ errno = property(_get_errno, _set_errno, None, "the value of 'errno' from/to the C calls") + def getwinerror(self, code=-1): + return self._backend.getwinerror(code) + def _pointer_to(self, ctype): from . import model with self._lock: diff --git a/pypy/doc/conf.py b/pypy/doc/conf.py --- a/pypy/doc/conf.py +++ b/pypy/doc/conf.py @@ -45,9 +45,9 @@ # built documents. # # The short X.Y version. -version = '2.1' +version = '2.2' # The full version, including alpha/beta/rc tags. -release = '2.1.0' +release = '2.2.0' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/pypy/doc/how-to-release.rst b/pypy/doc/how-to-release.rst --- a/pypy/doc/how-to-release.rst +++ b/pypy/doc/how-to-release.rst @@ -48,6 +48,6 @@ * send announcements to pypy-dev, python-list, python-announce, python-dev ... -* add a tag on jitviewer that corresponds to pypy release -* add a tag on codespeed that corresponds to pypy release +* add a tag on the pypy/jitviewer repo that corresponds to pypy release +* add a tag on the codespeed web site that corresponds to pypy release diff --git a/pypy/doc/index.rst b/pypy/doc/index.rst --- a/pypy/doc/index.rst +++ b/pypy/doc/index.rst @@ -40,7 +40,7 @@ * `FAQ`_: some frequently asked questions. -* `Release 2.1.0`_: the latest official release +* `Release 2.2.0`_: the latest official release * `PyPy Blog`_: news and status info about PyPy @@ -110,7 +110,7 @@ .. _`Getting Started`: getting-started.html .. _`Papers`: extradoc.html .. _`Videos`: video-index.html -.. _`Release 2.1.0`: http://pypy.org/download.html +.. _`Release 2.2.0`: http://pypy.org/download.html .. _`speed.pypy.org`: http://speed.pypy.org .. _`RPython toolchain`: translation.html .. _`potential project ideas`: project-ideas.html diff --git a/pypy/doc/release-2.2.0.rst b/pypy/doc/release-2.2.0.rst new file mode 100644 --- /dev/null +++ b/pypy/doc/release-2.2.0.rst @@ -0,0 +1,89 @@ +======================================= +PyPy 2.2 - Incrementalism +======================================= + +We're pleased to announce PyPy 2.2, which targets version 2.7.3 of the Python +language. This release main highlight is the introduction of the incremental +garbage collector, sponsored by the `Raspberry Pi Foundation`_. + +This release also contains several bugfixes and performance improvements. + +You can download the PyPy 2.2 release here: + + http://pypy.org/download.html + +We would like to thank our donors for the continued support of the PyPy +project. We showed quite a bit of progress on all three projects (see below) +and we're slowly running out of funds. +Please consider donating more so we can finish those projects! The three +projects are: + +* Py3k (supporting Python 3.x): the release PyPy3 2.2 is imminent. + +* STM (software transactional memory): a preview will be released very soon, + as soon as we fix a few bugs + +* NumPy: the work done is included in the PyPy 2.2 release. More details below. + +.. _`Raspberry Pi Foundation`: http://www.raspberrypi.org + +What is PyPy? +============= + +PyPy is a very compliant Python interpreter, almost a drop-in replacement for +CPython 2.7. It's fast (`pypy 2.2 and cpython 2.7.2`_ performance comparison) +due to its integrated tracing JIT compiler. + +This release supports x86 machines running Linux 32/64, Mac OS X 64, Windows +32, or ARM (ARMv6 or ARMv7, with VFPv3). + +Work on the native Windows 64 is still stalling, we would welcome a volunteer +to handle that. + +.. _`pypy 2.2 and cpython 2.7.2`: http://speed.pypy.org + +Highlights +========== + +* Our Garbage Collector is now "incremental". It should avoid almost + all pauses due to a major collection taking place. Previously, it + would pause the program (rarely) to walk all live objects, which + could take arbitrarily long if your process is using a whole lot of + RAM. Now the same work is done in steps. This should make PyPy + more responsive, e.g. in games. There are still other pauses, from + the GC and the JIT, but they should be on the order of 5 + milliseconds each. + +* The JIT counters for hot code were never reset, which meant that a + process running for long enough would eventually JIT-compile more + and more rarely executed code. Not only is it useless to compile + such code, but as more compiled code means more memory used, this + gives the impression of a memory leak. This has been tentatively + fixed by decreasing the counters from time to time. + +* NumPy has been split: now PyPy only contains the core module, called + ``_numpypy``. The ``numpy`` module itself has been moved to + ``https://bitbucket.org/pypy/numpy`` and ``numpypy`` disappeared. + You need to install NumPy separately with a virtualenv: + ``pip install git+https://bitbucket.org/pypy/numpy.git``; + or by directly doing + ``git clone https://bitbucket.org/pypy/numpy.git``, + ``cd numpy``, ``python setup.py install``. + +* non-inlined calls have less overhead + +* Things that use ``sys.set_trace`` are now JITted (like coverage) + +* JSON encoding used to be very fast, now decoding is as well + +* various buffer copying methods experience speedups (like list-of-ints to + ``int[]`` buffer from cffi) + +* We finally wrote (hopefully) all the missing ``os.xxx()`` functions, + including ``os.startfile()`` on Windows and a handful of rare ones + on Posix. + +* numpy has a rudimentary C API that cooperates with ``cpyext`` + +Cheers, +Armin Rigo and Maciej Fijalkowski diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-2.2.rst copy from pypy/doc/whatsnew-head.rst copy to pypy/doc/whatsnew-2.2.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-2.2.rst @@ -1,5 +1,5 @@ ====================== -What's new in PyPy 2.1 +What's new in PyPy 2.2 ====================== .. this is a revision shortly after release-2.1-beta diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -1,140 +1,7 @@ -====================== -What's new in PyPy 2.1 -====================== +======================= +What's new in PyPy 2.2+ +======================= -.. this is a revision shortly after release-2.1-beta -.. startrev: 4eb52818e7c0 +.. this is a revision shortly after release-2.2.x +.. startrev: 4cd1bc8b3111 -.. branch: sanitise_bytecode_dispatch -Make PyPy's bytecode dispatcher easy to read, and less reliant on RPython -magic. There is no functional change, though the removal of dead code leads -to many fewer tests to execute. - -.. branch: fastjson -Fast json decoder written in RPython, about 3-4x faster than the pure Python -decoder which comes with the stdlib - -.. branch: improve-str2charp -Improve the performance of I/O writing up to 15% by using memcpy instead of -copying char-by-char in str2charp and get_nonmovingbuffer - -.. branch: flowoperators -Simplify rpython/flowspace/ code by using more metaprogramming. Create -SpaceOperator class to gather static information about flow graph operations. - -.. branch: package-tk -Adapt package.py script to compile CFFI tk extension. Add a --without-tk switch -to optionally skip it. - -.. branch: distutils-cppldflags -Copy CPython's implementation of customize_compiler, dont call split on -environment variables, honour CFLAGS, CPPFLAGS, LDSHARED and LDFLAGS on Unices. - -.. branch: precise-instantiate -When an RPython class is instantiated via an indirect call (that is, which -class is being instantiated isn't known precisely) allow the optimizer to have -more precise information about which functions can be called. Needed for Topaz. - -.. branch: ssl_moving_write_buffer - -.. branch: pythoninspect-fix -Make PyPy respect PYTHONINSPECT variable set via os.putenv in the same process -to start interactive prompt when the script execution finishes. This adds -new __pypy__.os.real_getenv call that bypasses Python cache and looksup env -in the underlying OS. Translatorshell now works on PyPy. - -.. branch: add-statvfs -Added os.statvfs and os.fstatvfs - -.. branch: statvfs_tests -Added some addition tests for statvfs. - -.. branch: ndarray-subtype -Allow subclassing ndarray, i.e. matrix - -.. branch: ndarray-sort -Implement ndarray in-place sorting (for numeric types, no non-native byte order) - -.. branch: pypy-pyarray -Implement much of numpy's c api in cpyext, allows (slow) access to ndarray -from c - -.. branch: kill-ootype - -.. branch: fast-slowpath -Added an abstraction for functions with a fast and slow path in the JIT. This -speeds up list.append() and list.pop(). - -.. branch: curses_fixes - -.. branch: foldable-getarrayitem-indexerror -Constant-fold reading out of constant tuples in PyPy. - -.. branch: mro-reorder-numpypy-str -No longer delegate numpy string_ methods to space.StringObject, in numpy -this works by kind of by accident. Support for merging the refactor-str-types -branch - -.. branch: kill-typesystem -Remove the "type system" abstraction, now that there is only ever one kind of -type system used. - -.. branch: kill-gen-store-back-in -Kills gen_store_back_in_virtualizable - should improve non-inlined calls by -a bit - -.. branch: dotviewer-linewidth -.. branch: reflex-support -.. branch: numpypy-inplace-op -.. branch: rewritten-loop-logging -.. branch: no-release-gil -.. branch: safe-win-mmap -.. branch: boolean-indexing-cleanup -.. branch: cpyext-best_base -.. branch: cpyext-int -.. branch: fileops2 - -.. branch: nobold-backtrace -Work on improving UnionError messages and stack trace displays. - -.. branch: improve-errors-again -More improvements and refactorings of error messages. - -.. branch: improve-errors-again2 -Unbreak tests in rlib. - -.. branch: less-stringly-ops -Use subclasses of SpaceOperation instead of SpaceOperator objects. -Random cleanups in flowspace. - -.. branch: file-support-in-rpython -make open() and friends rpython - -.. branch: incremental-gc -Added the new incminimark GC which performs GC in incremental steps - -.. branch: fast_cffi_list_init -fastpath for cffi.new("long[]") - -.. branch: remove-eval-frame -remove a pointless abstraction - -.. branch: jit-settrace -Allow the jit to continue running when sys.settrace() is active, necessary to -make coverage.py fast - -.. branch: remove-numpypy -Remove lib_pypy/numpypy in favor of external numpy fork - -.. branch: jit-counter -Tweak the jit counters: decay them at minor collection (actually -only every 32 minor collection is enough). Should avoid the "memory -leaks" observed in long-running processes, actually created by the -jit compiling more and more rarely executed paths. - -.. branch: fix-trace-jit -Fixed the usage of sys.settrace() with the JIT. Also made it so using -sys.settrace() doesn't cause the GIL to be released on every single iteration. - -.. branch: rordereddict -Implement OrderedDict in RPython diff --git a/pypy/module/_cffi_backend/__init__.py b/pypy/module/_cffi_backend/__init__.py --- a/pypy/module/_cffi_backend/__init__.py +++ b/pypy/module/_cffi_backend/__init__.py @@ -1,3 +1,4 @@ +import sys from pypy.interpreter.mixedmodule import MixedModule from rpython.rlib import rdynload @@ -43,6 +44,8 @@ 'FFI_DEFAULT_ABI': 'ctypefunc._get_abi(space, "FFI_DEFAULT_ABI")', 'FFI_CDECL': 'ctypefunc._get_abi(space,"FFI_DEFAULT_ABI")',#win32 name } + if sys.platform == 'win32': + interpleveldefs['getwinerror'] = 'cerrno.getwinerror' for _name in ["RTLD_LAZY", "RTLD_NOW", "RTLD_GLOBAL", "RTLD_LOCAL", "RTLD_NODELETE", "RTLD_NOLOAD", "RTLD_DEEPBIND"]: diff --git a/pypy/module/_cffi_backend/cerrno.py b/pypy/module/_cffi_backend/cerrno.py --- a/pypy/module/_cffi_backend/cerrno.py +++ b/pypy/module/_cffi_backend/cerrno.py @@ -39,3 +39,14 @@ def set_errno(space, errno): ec = get_errno_container(space) ec._cffi_saved_errno = errno + +# ____________________________________________________________ + + at unwrap_spec(code=int) +def getwinerror(space, code=-1): + from rpython.rlib.rwin32 import FormatError + if code == -1: + ec = get_errno_container(space) + code = ec._cffi_saved_LastError + message = FormatError(code) + return space.newtuple([space.wrap(code), space.wrap(message)]) diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py --- a/pypy/module/_cffi_backend/test/_backend_test_c.py +++ b/pypy/module/_cffi_backend/test/_backend_test_c.py @@ -2687,6 +2687,16 @@ # res = GetLastError() assert res == 42 + # + SetLastError(2) + code, message = getwinerror() + assert code == 2 + assert message == "The system cannot find the file specified" + # + code, message = getwinerror(1155) + assert code == 1155 + assert message == ("No application is associated with the " + "specified file for this operation") def test_nonstandard_integer_types(): for typename in ['int8_t', 'uint8_t', 'int16_t', 'uint16_t', 'int32_t', diff --git a/pypy/module/posix/__init__.py b/pypy/module/posix/__init__.py --- a/pypy/module/posix/__init__.py +++ b/pypy/module/posix/__init__.py @@ -27,6 +27,7 @@ 'popen2': 'app_posix.popen2', 'popen3': 'app_posix.popen3', 'popen4': 'app_posix.popen4', + 'startfile': 'app_startfile.startfile', }) if hasattr(os, 'wait'): diff --git a/pypy/module/posix/app_startfile.py b/pypy/module/posix/app_startfile.py new file mode 100644 --- /dev/null +++ b/pypy/module/posix/app_startfile.py @@ -0,0 +1,44 @@ +# NOT_RPYTHON + +class CFFIWrapper(object): + def __init__(self): + import cffi + ffi = cffi.FFI() + ffi.cdef(""" + HINSTANCE ShellExecuteA(HWND, LPCSTR, LPCSTR, LPCSTR, LPCSTR, INT); + HINSTANCE ShellExecuteW(HWND, LPCWSTR, LPCWSTR, LPCWSTR, LPCWSTR, INT); + DWORD GetLastError(void); + """) + self.NULL = ffi.NULL + self.cast = ffi.cast + self.lib = ffi.dlopen("Shell32.dll") + self.SW_SHOWNORMAL = 1 + self.getwinerror = ffi.getwinerror + +_cffi_wrapper = None + + +def startfile(filepath, operation=None): + global _cffi_wrapper + if _cffi_wrapper is None: + _cffi_wrapper = CFFIWrapper() + w = _cffi_wrapper + # + if operation is None: + operation = w.NULL + if isinstance(filepath, str): + if isinstance(operation, unicode): + operation = operation.encode("ascii") + rc = w.lib.ShellExecuteA(w.NULL, operation, filepath, + w.NULL, w.NULL, w.SW_SHOWNORMAL) + elif isinstance(filepath, unicode): + if isinstance(operation, str): + operation = operation.decode("ascii") + rc = w.lib.ShellExecuteW(w.NULL, operation, filepath, + w.NULL, w.NULL, w.SW_SHOWNORMAL) + else: + raise TypeError("argument 1 must be str or unicode") + rc = int(w.cast("uintptr_t", rc)) + if rc <= 32: + code, msg = w.getwinerror() + raise WindowsError(code, msg, filepath) diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py --- a/pypy/module/posix/test/test_posix2.py +++ b/pypy/module/posix/test/test_posix2.py @@ -52,6 +52,7 @@ def setup_class(cls): cls.space = space + cls.w_runappdirect = space.wrap(cls.runappdirect) cls.w_posix = space.appexec([], GET_POSIX) cls.w_path = space.wrap(str(path)) cls.w_path2 = space.wrap(str(path2)) @@ -1108,6 +1109,28 @@ assert False, "urandom() always returns the same string" # Or very unlucky + if hasattr(os, 'startfile'): + def test_startfile(self): + if not self.runappdirect: + skip("should not try to import cffi at app-level") + startfile = self.posix.startfile + for t1 in [str, unicode]: + for t2 in [str, unicode]: + e = raises(WindowsError, startfile, t1("\\"), t2("close")) + assert e.value.args[0] == 1155 + assert e.value.args[1] == ( + "No application is associated with the " + "specified file for this operation") + if len(e.value.args) > 2: + assert e.value.args[2] == t1("\\") + # + e = raises(WindowsError, startfile, "\\foo\\bar\\baz") + assert e.value.args[0] == 2 + assert e.value.args[1] == ( + "The system cannot find the file specified") + if len(e.value.args) > 2: + assert e.value.args[2] == "\\foo\\bar\\baz" + class AppTestEnvironment(object): def setup_class(cls): diff --git a/pypy/module/test_lib_pypy/cffi_tests/test_ffi_backend.py b/pypy/module/test_lib_pypy/cffi_tests/test_ffi_backend.py --- a/pypy/module/test_lib_pypy/cffi_tests/test_ffi_backend.py +++ b/pypy/module/test_lib_pypy/cffi_tests/test_ffi_backend.py @@ -195,3 +195,20 @@ assert p.a[0] == 200 assert p.a[1] == 300 assert p.a[2] == 400 + + @pytest.mark.skipif("sys.platform != 'win32'") + def test_getwinerror(self): + ffi = FFI() + code, message = ffi.getwinerror(1155) + assert code == 1155 + assert message == ("No application is associated with the " + "specified file for this operation") + ffi.cdef("void SetLastError(int);") + lib = ffi.dlopen("Kernel32.dll") + lib.SetLastError(2) + code, message = ffi.getwinerror() + assert code == 2 + assert message == "The system cannot find the file specified" + code, message = ffi.getwinerror(-1) + assert code == 2 + assert message == "The system cannot find the file specified" diff --git a/pypy/module/test_lib_pypy/cffi_tests/test_verify.py b/pypy/module/test_lib_pypy/cffi_tests/test_verify.py --- a/pypy/module/test_lib_pypy/cffi_tests/test_verify.py +++ b/pypy/module/test_lib_pypy/cffi_tests/test_verify.py @@ -712,7 +712,7 @@ "#define BAZ ...\n") lib = ffi.verify("#define FOO 42\n" "#define BAR (-44)\n" - "#define BAZ 0xffffffffffffffffLL\n") + "#define BAZ 0xffffffffffffffffULL\n") assert lib.FOO == 42 assert lib.BAR == -44 assert lib.BAZ == 0xffffffffffffffff @@ -1602,6 +1602,8 @@ (maxulong, -1, ''), (-1, 0xffffffff, 'U'), (-1, maxulong, 'UL')]: + if c2c and sys.platform == 'win32': + continue # enums may always be signed with MSVC ffi = FFI() ffi.cdef("enum foo_e { AA=%s };" % c1) e = py.test.raises(VerificationError, ffi.verify, diff --git a/rpython/annotator/bookkeeper.py b/rpython/annotator/bookkeeper.py --- a/rpython/annotator/bookkeeper.py +++ b/rpython/annotator/bookkeeper.py @@ -5,7 +5,6 @@ from __future__ import absolute_import import sys, types, inspect, weakref -from collections import OrderedDict from rpython.flowspace.model import Constant from rpython.annotator.model import (SomeOrderedDict, @@ -371,7 +370,7 @@ for e in x: listdef.generalize(self.immutablevalue(e, False)) result = SomeList(listdef) - elif tp is dict or tp is r_dict or tp is OrderedDict: + elif tp is dict or tp is r_dict or tp is SomeOrderedDict.knowntype: if need_const: key = Constant(x) try: @@ -413,7 +412,7 @@ dictdef.generalize_key(self.immutablevalue(ek, False)) dictdef.generalize_value(self.immutablevalue(ev, False)) dictdef.seen_prebuilt_key(ek) - if tp is OrderedDict: + if tp is SomeOrderedDict.knowntype: result = SomeOrderedDict(dictdef) else: result = SomeDict(dictdef) diff --git a/rpython/annotator/builtin.py b/rpython/annotator/builtin.py --- a/rpython/annotator/builtin.py +++ b/rpython/annotator/builtin.py @@ -2,7 +2,6 @@ Built-in functions. """ import sys -from collections import OrderedDict from rpython.annotator.model import ( SomeInteger, SomeObject, SomeChar, SomeBool, SomeString, SomeTuple, s_Bool, @@ -364,7 +363,7 @@ BUILTIN_ANALYZERS[rpython.rlib.objectmodel.instantiate] = robjmodel_instantiate BUILTIN_ANALYZERS[rpython.rlib.objectmodel.r_dict] = robjmodel_r_dict BUILTIN_ANALYZERS[rpython.rlib.objectmodel.r_ordereddict] = robjmodel_r_ordereddict -BUILTIN_ANALYZERS[OrderedDict] = lambda : SomeOrderedDict(getbookkeeper().getdictdef()) +BUILTIN_ANALYZERS[SomeOrderedDict.knowntype] = lambda : SomeOrderedDict(getbookkeeper().getdictdef()) BUILTIN_ANALYZERS[rpython.rlib.objectmodel.hlinvoke] = robjmodel_hlinvoke BUILTIN_ANALYZERS[rpython.rlib.objectmodel.keepalive_until_here] = robjmodel_keepalive_until_here BUILTIN_ANALYZERS[rpython.rtyper.lltypesystem.llmemory.cast_ptr_to_adr] = llmemory_cast_ptr_to_adr diff --git a/rpython/annotator/model.py b/rpython/annotator/model.py --- a/rpython/annotator/model.py +++ b/rpython/annotator/model.py @@ -32,7 +32,6 @@ import inspect import weakref from types import BuiltinFunctionType, MethodType -from collections import OrderedDict import rpython from rpython.tool import descriptor @@ -357,7 +356,11 @@ return '{...%s...}' % (len(const),) class SomeOrderedDict(SomeDict): - knowntype = OrderedDict + try: + from collections import OrderedDict as knowntype + except ImportError: # Python 2.6 + class PseudoOrderedDict(dict): pass + knowntype = PseudoOrderedDict def method_copy(dct): return SomeOrderedDict(dct.dictdef) diff --git a/rpython/rlib/rwin32.py b/rpython/rlib/rwin32.py --- a/rpython/rlib/rwin32.py +++ b/rpython/rlib/rwin32.py @@ -216,7 +216,7 @@ def llimpl_FormatError(code): "Return a message corresponding to the given Windows error code." buf = lltype.malloc(rffi.CCHARPP.TO, 1, flavor='raw') - + buf[0] = lltype.nullptr(rffi.CCHARP.TO) try: msglen = FormatMessage(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM, @@ -225,17 +225,20 @@ DEFAULT_LANGUAGE, rffi.cast(rffi.CCHARP, buf), 0, None) + buflen = intmask(msglen) - if msglen <= 2: # includes the case msglen < 0 - return fake_FormatError(code) + # remove trailing cr/lf and dots + s_buf = buf[0] + while buflen > 0 and (s_buf[buflen - 1] <= ' ' or + s_buf[buflen - 1] == '.'): + buflen -= 1 - # FormatMessage always appends \r\n. - buflen = intmask(msglen - 2) - assert buflen > 0 - - result = rffi.charpsize2str(buf[0], buflen) + if buflen <= 0: + result = fake_FormatError(code) + else: + result = rffi.charpsize2str(s_buf, buflen) + finally: LocalFree(rffi.cast(rffi.VOIDP, buf[0])) - finally: lltype.free(buf, flavor='raw') return result diff --git a/rpython/rtyper/rbuiltin.py b/rpython/rtyper/rbuiltin.py --- a/rpython/rtyper/rbuiltin.py +++ b/rpython/rtyper/rbuiltin.py @@ -1,4 +1,3 @@ -from collections import OrderedDict from rpython.annotator import model as annmodel from rpython.flowspace.model import Constant @@ -750,7 +749,7 @@ BUILTIN_TYPER[isinstance] = rtype_builtin_isinstance BUILTIN_TYPER[hasattr] = rtype_builtin_hasattr BUILTIN_TYPER[objectmodel.r_dict] = rtype_r_dict -BUILTIN_TYPER[OrderedDict] = rtype_ordered_dict +BUILTIN_TYPER[annmodel.SomeOrderedDict.knowntype] = rtype_ordered_dict BUILTIN_TYPER[objectmodel.r_ordereddict] = rtype_ordered_dict # _________________________________________________________________ diff --git a/rpython/rtyper/test/test_rordereddict.py b/rpython/rtyper/test/test_rordereddict.py --- a/rpython/rtyper/test/test_rordereddict.py +++ b/rpython/rtyper/test/test_rordereddict.py @@ -1,6 +1,9 @@ import py -from collections import OrderedDict +try: + from collections import OrderedDict +except ImportError: # Python 2.6 + py.test.skip("requires collections.OrderedDict") from rpython.rtyper.lltypesystem import lltype, rffi from rpython.rtyper.lltypesystem import rordereddict, rstr from rpython.rlib.rarithmetic import intmask From noreply at buildbot.pypy.org Thu Nov 14 09:45:04 2013 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 14 Nov 2013 09:45:04 +0100 (CET) Subject: [pypy-commit] pypy default: update Message-ID: <20131114084504.B6BDE1C0134@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r68024:f606890f99ed Date: 2013-11-14 09:43 +0100 http://bitbucket.org/pypy/pypy/changeset/f606890f99ed/ Log: update diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -5,3 +5,4 @@ .. this is a revision shortly after release-2.2.x .. startrev: 4cd1bc8b3111 +.. branch: release-2.2.x From noreply at buildbot.pypy.org Thu Nov 14 09:45:06 2013 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 14 Nov 2013 09:45:06 +0100 (CET) Subject: [pypy-commit] pypy default: merge heads Message-ID: <20131114084506.0393D1C0134@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r68025:315906031082 Date: 2013-11-14 09:44 +0100 http://bitbucket.org/pypy/pypy/changeset/315906031082/ Log: merge heads diff --git a/pypy/module/micronumpy/arrayimpl/concrete.py b/pypy/module/micronumpy/arrayimpl/concrete.py --- a/pypy/module/micronumpy/arrayimpl/concrete.py +++ b/pypy/module/micronumpy/arrayimpl/concrete.py @@ -52,7 +52,7 @@ loop.setslice(space, shape, self, impl) def get_size(self): - return self.size // self.dtype.itemtype.get_element_size() + return self.size // self.dtype.get_size() def get_storage_size(self): return self.size @@ -399,7 +399,7 @@ self.storage = parent.storage self.order = parent.order self.dtype = dtype - self.size = support.product(shape) * self.dtype.itemtype.get_element_size() + self.size = support.product(shape) * self.dtype.get_size() self.start = start self.orig_arr = orig_arr diff --git a/pypy/module/micronumpy/conversion_utils.py b/pypy/module/micronumpy/conversion_utils.py --- a/pypy/module/micronumpy/conversion_utils.py +++ b/pypy/module/micronumpy/conversion_utils.py @@ -1,6 +1,27 @@ from pypy.interpreter.error import OperationError from pypy.module.micronumpy.constants import * + +def byteorder_converter(space, new_order): + endian = new_order[0] + if endian not in (NPY_BIG, NPY_LITTLE, NPY_NATIVE, NPY_IGNORE, NPY_SWAP): + ch = endian + if ch in ('b', 'B'): + endian = NPY_BIG + elif ch in ('l', 'L'): + endian = NPY_LITTLE + elif ch in ('n', 'N'): + endian = NPY_NATIVE + elif ch in ('i', 'I'): + endian = NPY_IGNORE + elif ch in ('s', 'S'): + endian = NPY_SWAP + else: + raise OperationError(space.w_ValueError, space.wrap( + "%s is an unrecognized byteorder" % new_order)) + return endian + + def clipmode_converter(space, w_mode): if space.is_none(w_mode): return NPY_RAISE @@ -19,6 +40,7 @@ raise OperationError(space.w_TypeError, space.wrap("clipmode not understood")) + def order_converter(space, w_order, default): if space.is_none(w_order): return default @@ -41,6 +63,7 @@ raise OperationError(space.w_TypeError, space.wrap( "order not understood")) + def multi_axis_converter(space, w_axis, ndim): if space.is_none(w_axis): return [True] * ndim diff --git a/pypy/module/micronumpy/interp_dtype.py b/pypy/module/micronumpy/interp_dtype.py --- a/pypy/module/micronumpy/interp_dtype.py +++ b/pypy/module/micronumpy/interp_dtype.py @@ -9,6 +9,7 @@ from rpython.rlib.rarithmetic import LONG_BIT, r_longlong, r_ulonglong from rpython.rtyper.lltypesystem import rffi from rpython.rlib import jit +from pypy.module.micronumpy.conversion_utils import byteorder_converter from pypy.module.micronumpy.constants import * @@ -34,10 +35,12 @@ return out class W_Dtype(W_Root): - _immutable_fields_ = ["itemtype?", "num", "kind", "name?", "char", "w_box_type", "byteorder", "float_type"] + _immutable_fields_ = ["itemtype?", "num", "kind", "name?", "char", + "w_box_type", "byteorder", "size?", "float_type", + "fields?", "fieldnames?", "shape", "subdtype", "base"] def __init__(self, itemtype, num, kind, name, char, w_box_type, byteorder=NPY_NATIVE, - alternate_constructors=[], aliases=[], float_type=None, + size=1, alternate_constructors=[], aliases=[], float_type=None, fields=None, fieldnames=None, shape=[], subdtype=None): self.itemtype = itemtype self.num = num @@ -46,6 +49,7 @@ self.char = char self.w_box_type = w_box_type self.byteorder = byteorder + self.size = size self.alternate_constructors = alternate_constructors self.aliases = aliases self.float_type = float_type @@ -122,7 +126,7 @@ return self.byteorder in (NPY_NATIVE, NPY_NATBYTE) def get_size(self): - return self.itemtype.get_element_size() + return self.size * self.itemtype.get_element_size() def get_name(self): if self.char == 'S': @@ -136,7 +140,7 @@ return space.wrap("dtype('%s')" % self.get_name()) def descr_get_itemsize(self, space): - return space.wrap(self.itemtype.get_element_size()) + return space.wrap(self.get_size()) def descr_get_alignment(self, space): return space.wrap(self.itemtype.alignment) @@ -196,7 +200,6 @@ self.fields = None else: self.fields = {} - ofs_and_items = [] size = 0 for key in space.listview(w_fields): value = space.getitem(w_fields, key) @@ -207,11 +210,11 @@ offset = space.int_w(space.getitem(value, space.wrap(1))) self.fields[space.str_w(key)] = offset, dtype - ofs_and_items.append((offset, dtype.itemtype)) - size += dtype.itemtype.get_element_size() + size += dtype.get_size() - self.itemtype = types.RecordType(ofs_and_items, size) - self.name = "void" + str(8 * self.itemtype.get_element_size()) + self.itemtype = types.RecordType() + self.size = size + self.name = "void" + str(8 * self.get_size()) def descr_get_names(self, space): if self.fieldnames is None: @@ -263,7 +266,7 @@ w_class = space.type(self) kind = self.kind - elemsize = self.itemtype.get_element_size() + elemsize = self.get_size() builder_args = space.newtuple([space.wrap("%s%d" % (kind, elemsize)), space.wrap(0), space.wrap(1)]) version = space.wrap(3) @@ -308,11 +311,23 @@ fields = space.getitem(w_data, space.wrap(4)) self.set_fields(space, fields) + @unwrap_spec(new_order=str) + def descr_newbyteorder(self, space, new_order=NPY_SWAP): + newendian = byteorder_converter(space, new_order) + endian = self.byteorder + if endian != NPY_IGNORE: + if newendian == NPY_SWAP: + endian = NPY_OPPBYTE if self.is_native() else NPY_NATBYTE + elif newendian != NPY_IGNORE: + endian = newendian + itemtype = self.itemtype.__class__(endian in (NPY_NATIVE, NPY_NATBYTE)) + return W_Dtype(itemtype, self.num, self.kind, self.name, self.char, + self.w_box_type, endian, size=self.size) + def dtype_from_list(space, w_lst): lst_w = space.listview(w_lst) fields = {} offset = 0 - ofs_and_items = [] fieldnames = [] for w_elem in lst_w: size = 1 @@ -329,13 +344,13 @@ raise OperationError(space.w_ValueError, space.wrap("two fields with the same name")) assert isinstance(subdtype, W_Dtype) fields[fldname] = (offset, subdtype) - ofs_and_items.append((offset, subdtype.itemtype)) - offset += subdtype.itemtype.get_element_size() * size + offset += subdtype.get_size() * size fieldnames.append(fldname) - itemtype = types.RecordType(ofs_and_items, offset) - return W_Dtype(itemtype, NPY_VOID, NPY_VOIDLTR, "void" + str(8 * itemtype.get_element_size()), - NPY_VOIDLTR, space.gettypefor(interp_boxes.W_VoidBox), fields=fields, - fieldnames=fieldnames) + itemtype = types.RecordType() + return W_Dtype(itemtype, NPY_VOID, NPY_VOIDLTR, + "void" + str(8 * offset * itemtype.get_element_size()), + NPY_VOIDLTR, space.gettypefor(interp_boxes.W_VoidBox), + fields=fields, fieldnames=fieldnames, size=offset) def dtype_from_dict(space, w_dict): raise OperationError(space.w_NotImplementedError, space.wrap( @@ -349,7 +364,8 @@ # w_align and w_copy are necessary for pickling cache = get_dtype_cache(space) - if w_shape is not None and (space.isinstance_w(w_shape, space.w_int) or space.len_w(w_shape) > 0): + if w_shape is not None and (space.isinstance_w(w_shape, space.w_int) or + space.len_w(w_shape) > 0): subdtype = descr__new__(space, w_subtype, w_dtype, w_align, w_copy) assert isinstance(subdtype, W_Dtype) size = 1 @@ -360,8 +376,11 @@ dim = space.int_w(w_dim) shape.append(dim) size *= dim - return W_Dtype(types.VoidType(subdtype.itemtype.get_element_size() * size), NPY_VOID, NPY_VOIDLTR, "void" + str(8 * subdtype.itemtype.get_element_size() * size), - NPY_VOIDLTR, space.gettypefor(interp_boxes.W_VoidBox), shape=shape, subdtype=subdtype) + return W_Dtype(types.VoidType(), NPY_VOID, NPY_VOIDLTR, + "void" + str(8 * subdtype.get_size() * size), + NPY_VOIDLTR, space.gettypefor(interp_boxes.W_VoidBox), + shape=shape, subdtype=subdtype, + size=subdtype.get_size() * size) if space.is_none(w_dtype): return cache.w_float64dtype @@ -413,6 +432,7 @@ __reduce__ = interp2app(W_Dtype.descr_reduce), __setstate__ = interp2app(W_Dtype.descr_setstate), + newbyteorder = interp2app(W_Dtype.descr_newbyteorder), type = interp_attrproperty_w("w_box_type", cls=W_Dtype), kind = interp_attrproperty("kind", cls=W_Dtype), @@ -450,17 +470,17 @@ size = 1 if char == NPY_STRINGLTR: - itemtype = types.StringType(size) + itemtype = types.StringType() basename = 'string' num = NPY_STRING w_box_type = space.gettypefor(interp_boxes.W_StringBox) elif char == NPY_VOIDLTR: - itemtype = types.VoidType(size) + itemtype = types.VoidType() basename = 'void' num = NPY_VOID w_box_type = space.gettypefor(interp_boxes.W_VoidBox) elif char == NPY_UNICODELTR: - itemtype = types.UnicodeType(size) + itemtype = types.UnicodeType() basename = 'unicode' num = NPY_UNICODE w_box_type = space.gettypefor(interp_boxes.W_UnicodeBox) @@ -468,27 +488,29 @@ assert False return W_Dtype(itemtype, num, char, - basename + str(8 * itemtype.get_element_size()), - char, w_box_type) + basename + str(8 * size * itemtype.get_element_size()), + char, w_box_type, size=size) def new_string_dtype(space, size): - itemtype = types.StringType(size) + itemtype = types.StringType() return W_Dtype( itemtype, + size=size, num=NPY_STRING, kind=NPY_STRINGLTR, - name='string' + str(8 * itemtype.get_element_size()), + name='string' + str(8 * size * itemtype.get_element_size()), char=NPY_STRINGLTR, w_box_type = space.gettypefor(interp_boxes.W_StringBox), ) def new_unicode_dtype(space, size): - itemtype = types.UnicodeType(size) + itemtype = types.UnicodeType() return W_Dtype( itemtype, + size=size, num=NPY_UNICODE, kind=NPY_UNICODELTR, - name='unicode' + str(8 * itemtype.get_element_size()), + name='unicode' + str(8 * size * itemtype.get_element_size()), char=NPY_UNICODELTR, w_box_type = space.gettypefor(interp_boxes.W_UnicodeBox), ) @@ -663,7 +685,8 @@ float_type = self.w_floatlongdtype, ) self.w_stringdtype = W_Dtype( - types.StringType(0), + types.StringType(), + size=0, num=NPY_STRING, kind=NPY_STRINGLTR, name='string', @@ -673,7 +696,8 @@ aliases=["str"], ) self.w_unicodedtype = W_Dtype( - types.UnicodeType(0), + types.UnicodeType(), + size=0, num=NPY_UNICODE, kind=NPY_UNICODELTR, name='unicode', @@ -682,7 +706,8 @@ alternate_constructors=[space.w_unicode], ) self.w_voiddtype = W_Dtype( - types.VoidType(0), + types.VoidType(), + size=0, num=NPY_VOID, kind=NPY_VOIDLTR, name='void', @@ -750,7 +775,7 @@ self.w_intpdtype, self.w_uintpdtype, ] self.float_dtypes_by_num_bytes = sorted( - (dtype.itemtype.get_element_size(), dtype) + (dtype.get_size(), dtype) for dtype in float_dtypes ) self.dtypes_by_num = {} @@ -760,7 +785,7 @@ for dtype in reversed(self.builtin_dtypes): self.dtypes_by_num[dtype.num] = dtype self.dtypes_by_name[dtype.name] = dtype - can_name = dtype.kind + str(dtype.itemtype.get_element_size()) + can_name = dtype.kind + str(dtype.get_size()) self.dtypes_by_name[can_name] = dtype self.dtypes_by_name[NPY_NATBYTE + can_name] = dtype self.dtypes_by_name[NPY_NATIVE + can_name] = dtype @@ -830,7 +855,7 @@ for k, v in typeinfo_partial.iteritems(): space.setitem(w_typeinfo, space.wrap(k), space.gettypefor(v)) for k, dtype in typeinfo_full.iteritems(): - itemsize = dtype.itemtype.get_element_size() + itemsize = dtype.get_size() items_w = [space.wrap(dtype.char), space.wrap(dtype.num), space.wrap(itemsize * 8), # in case of changing diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py --- a/pypy/module/micronumpy/interp_numarray.py +++ b/pypy/module/micronumpy/interp_numarray.py @@ -85,10 +85,10 @@ return space.wrap(len(self.get_shape())) def descr_get_itemsize(self, space): - return space.wrap(self.get_dtype().itemtype.get_element_size()) + return space.wrap(self.get_dtype().get_size()) def descr_get_nbytes(self, space): - return space.wrap(self.get_size() * self.get_dtype().itemtype.get_element_size()) + return space.wrap(self.get_size() * self.get_dtype().get_size()) def descr_fill(self, space, w_value): self.fill(self.get_dtype().coerce(space, w_value)) @@ -625,10 +625,10 @@ raise OperationError(space.w_NotImplementedError, space.wrap( "itemset not implemented yet")) - @unwrap_spec(neworder=str) - def descr_newbyteorder(self, space, neworder): - raise OperationError(space.w_NotImplementedError, space.wrap( - "newbyteorder not implemented yet")) + @unwrap_spec(new_order=str) + def descr_newbyteorder(self, space, new_order=NPY_SWAP): + return self.descr_view(space, + self.get_dtype().descr_newbyteorder(space, new_order)) @unwrap_spec(w_axis=WrappedDefault(None), w_out=WrappedDefault(None)) @@ -1268,6 +1268,7 @@ diagonal = interp2app(W_NDimArray.descr_diagonal), trace = interp2app(W_NDimArray.descr_trace), view = interp2app(W_NDimArray.descr_view), + newbyteorder = interp2app(W_NDimArray.descr_newbyteorder), ctypes = GetSetProperty(W_NDimArray.descr_get_ctypes), # XXX unimplemented __array_interface__ = GetSetProperty(W_NDimArray.descr_array_iface), @@ -1340,7 +1341,7 @@ # not an array or incorrect dtype shape, elems_w = find_shape_and_elems(space, w_object, dtype) if dtype is None or ( - dtype.is_str_or_unicode() and dtype.itemtype.get_size() < 1): + dtype.is_str_or_unicode() and dtype.get_size() < 1): for w_elem in elems_w: dtype = interp_ufuncs.find_dtype_for_scalar(space, w_elem, dtype) @@ -1349,7 +1350,7 @@ if dtype is None: dtype = interp_dtype.get_dtype_cache(space).w_float64dtype - if dtype.is_str_or_unicode() and dtype.itemtype.get_size() < 1: + if dtype.is_str_or_unicode() and dtype.get_size() < 1: # promote S0 -> S1, U0 -> U1 dtype = interp_dtype.variable_dtype(space, dtype.char + '1') if ndmin > len(shape): diff --git a/pypy/module/micronumpy/interp_ufuncs.py b/pypy/module/micronumpy/interp_ufuncs.py --- a/pypy/module/micronumpy/interp_ufuncs.py +++ b/pypy/module/micronumpy/interp_ufuncs.py @@ -475,8 +475,7 @@ if dt2.is_record_type(): return dt2 if dt1.is_str_or_unicode(): - if dt2.itemtype.get_element_size() >= \ - dt1.itemtype.get_element_size(): + if dt2.get_size() >= dt1.get_size(): return dt2 return dt1 return dt2 @@ -556,7 +555,7 @@ return interp_dtype.variable_dtype(space, 'S%d' % space.len_w(w_obj)) elif current_guess.num == NPY_STRING: - if current_guess.itemtype.get_size() < space.len_w(w_obj): + if current_guess.get_size() < space.len_w(w_obj): return interp_dtype.variable_dtype(space, 'S%d' % space.len_w(w_obj)) return current_guess diff --git a/pypy/module/micronumpy/iter.py b/pypy/module/micronumpy/iter.py --- a/pypy/module/micronumpy/iter.py +++ b/pypy/module/micronumpy/iter.py @@ -164,7 +164,7 @@ self.array = array self.offset = 0 self.dtype = array.dtype - self.skip = self.dtype.itemtype.get_element_size() + self.skip = self.dtype.get_size() self.size = array.size def setitem(self, elem): diff --git a/pypy/module/micronumpy/test/test_dtypes.py b/pypy/module/micronumpy/test/test_dtypes.py --- a/pypy/module/micronumpy/test/test_dtypes.py +++ b/pypy/module/micronumpy/test/test_dtypes.py @@ -288,6 +288,49 @@ assert a.dtype.__reduce__() == (dtype, ('i4', 0, 1), (3, '<', None, None, None, -1, -1, 0)) assert loads(dumps(a.dtype)) == a.dtype + def test_newbyteorder(self): + import numpypy as np + import sys + sys_is_le = sys.byteorder == 'little' + native_code = sys_is_le and '<' or '>' + swapped_code = sys_is_le and '>' or '<' + native_dt = np.dtype(native_code+'i2') + swapped_dt = np.dtype(swapped_code+'i2') + assert native_dt.newbyteorder('S') == swapped_dt + assert native_dt.newbyteorder() == swapped_dt + assert native_dt == swapped_dt.newbyteorder('S') + assert native_dt == swapped_dt.newbyteorder('=') + assert native_dt == swapped_dt.newbyteorder('N') + assert native_dt == native_dt.newbyteorder('|') + assert np.dtype('i2') == native_dt.newbyteorder('>') + assert np.dtype('>i2') == native_dt.newbyteorder('B') + + for t in [np.int_, np.float_]: + dt = np.dtype(t) + dt1 = dt.newbyteorder().newbyteorder() + dt2 = dt.newbyteorder("<") + dt3 = dt.newbyteorder(">") + assert dt.byteorder != dt1.byteorder + #assert hash(dt) == hash(dt1) + if dt == dt2: + assert dt.byteorder != dt2.byteorder + #assert hash(dt) == hash(dt2) + else: + assert dt.byteorder != dt3.byteorder + #assert hash(dt) == hash(dt3) + + exc = raises(ValueError, dt.newbyteorder, 'XX') + assert exc.value[0] == 'XX is an unrecognized byteorder' + + for t in [np.int_, np.float_]: + dt1 = np.dtype(t) + dt2 = dt1.newbyteorder() + s1 = np.array(123, dtype=dt1).tostring() + s2 = np.array(123, dtype=dt2).byteswap().tostring() + assert s1 == s2 + class AppTestTypes(BaseAppTestDtypes): def test_abstract_types(self): import numpypy as numpy diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py --- a/pypy/module/micronumpy/test/test_numarray.py +++ b/pypy/module/micronumpy/test/test_numarray.py @@ -2928,6 +2928,15 @@ assert str(a.dtype) == '|S1' assert a == 'x' + def test_newbyteorder(self): + import numpy as np + a = np.array([1, 2], dtype=np.int16) + b = a.newbyteorder() + assert (b == [256, 512]).all() + c = b.byteswap() + assert (c == [1, 2]).all() + assert (a == [1, 2]).all() + def test_pickle(self): from numpypy import dtype, array from cPickle import loads, dumps diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py --- a/pypy/module/micronumpy/types.py +++ b/pypy/module/micronumpy/types.py @@ -1591,17 +1591,8 @@ ComponentBoxType = interp_boxes.W_FloatLongBox class BaseStringType(BaseType): - _immutable_fields = ['size'] - - def __init__(self, size=0): - BaseType.__init__(self) - self.size = size - def get_element_size(self): - return self.size * rffi.sizeof(self.T) - - def get_size(self): - return self.size + return rffi.sizeof(self.T) def str_unary_op(func): specialize.argtype(1)(func) @@ -1636,13 +1627,13 @@ def store(self, arr, i, offset, box): assert isinstance(box, interp_boxes.W_StringBox) - return self._store(arr.storage, i, offset, box) + size = min(arr.dtype.size, box.arr.size - box.ofs) + return self._store(arr.storage, i, offset, box, size) @jit.unroll_safe - def _store(self, storage, i, offset, box): + def _store(self, storage, i, offset, box, size): assert isinstance(box, interp_boxes.W_StringBox) - # XXX simplify to range(box.dtype.get_size()) ? - for k in range(min(self.size, box.arr.size-box.ofs)): + for k in range(size): storage[k + offset + i] = box.arr.storage[k + box.ofs] def read(self, arr, i, offset, dtype=None): @@ -1725,17 +1716,17 @@ else: w_arg = box.descr_str(space) arg = space.str_w(space.str(w_arg)) - arr = VoidBoxStorage(self.size, mydtype) + arr = VoidBoxStorage(mydtype.size, mydtype) i = 0 - for i in range(min(len(arg), self.size)): + for i in range(min(len(arg), mydtype.size)): arr.storage[i] = arg[i] - for j in range(i + 1, self.size): + for j in range(i + 1, mydtype.size): arr.storage[j] = '\x00' - return interp_boxes.W_StringBox(arr, 0, arr.dtype) + return interp_boxes.W_StringBox(arr, 0, arr.dtype) def fill(self, storage, width, box, start, stop, offset): for i in xrange(start, stop, width): - self._store(storage, i, offset, box) + self._store(storage, i, offset, box, width) class UnicodeType(BaseStringType): T = lltype.UniChar @@ -1772,14 +1763,14 @@ ofs += size def coerce(self, space, dtype, w_items): - arr = VoidBoxStorage(self.size, dtype) + arr = VoidBoxStorage(dtype.get_size(), dtype) self._coerce(space, arr, 0, dtype, w_items, dtype.shape) return interp_boxes.W_VoidBox(arr, 0, dtype) @jit.unroll_safe def store(self, arr, i, ofs, box): assert isinstance(box, interp_boxes.W_VoidBox) - for k in range(self.get_element_size()): + for k in range(box.arr.dtype.get_size()): arr.storage[k + ofs] = box.arr.storage[k + box.ofs] def readarray(self, arr, i, offset, dtype=None): @@ -1793,15 +1784,9 @@ class RecordType(BaseType): T = lltype.Char - _immutable_fields_ = ['offsets_and_fields', 'size'] - - def __init__(self, offsets_and_fields, size): - BaseType.__init__(self) - self.offsets_and_fields = offsets_and_fields - self.size = size def get_element_size(self): - return self.size + return rffi.sizeof(self.T) def read(self, arr, i, offset, dtype=None): if dtype is None: @@ -1817,14 +1802,14 @@ if not space.issequence_w(w_item): raise OperationError(space.w_TypeError, space.wrap( "expected sequence")) - if len(self.offsets_and_fields) != space.len_w(w_item): + if len(dtype.fields) != space.len_w(w_item): raise OperationError(space.w_ValueError, space.wrap( "wrong length")) items_w = space.fixedview(w_item) - arr = VoidBoxStorage(self.size, dtype) + arr = VoidBoxStorage(dtype.get_size(), dtype) for i in range(len(items_w)): - subdtype = dtype.fields[dtype.fieldnames[i]][1] - ofs, itemtype = self.offsets_and_fields[i] + ofs, subdtype = dtype.fields[dtype.fieldnames[i]] + itemtype = subdtype.itemtype w_item = items_w[i] w_box = itemtype.coerce(space, subdtype, w_item) itemtype.store(arr, 0, ofs, w_box) @@ -1833,7 +1818,7 @@ @jit.unroll_safe def store(self, arr, i, ofs, box): assert isinstance(box, interp_boxes.W_VoidBox) - for k in range(self.get_element_size()): + for k in range(box.arr.dtype.get_size()): arr.storage[k + i] = box.arr.storage[k + box.ofs] @jit.unroll_safe @@ -1841,7 +1826,9 @@ assert isinstance(box, interp_boxes.W_VoidBox) pieces = ["("] first = True - for ofs, tp in self.offsets_and_fields: + for name in box.dtype.fieldnames: + ofs, subdtype = box.dtype.fields[name] + tp = subdtype.itemtype if first: first = False else: From noreply at buildbot.pypy.org Thu Nov 14 09:51:33 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Thu, 14 Nov 2013 09:51:33 +0100 (CET) Subject: [pypy-commit] pypy default: document merged branch Message-ID: <20131114085133.D911F1C0134@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r68026:c69c2692ce2e Date: 2013-11-14 03:50 -0500 http://bitbucket.org/pypy/pypy/changeset/c69c2692ce2e/ Log: document merged branch diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -6,3 +6,6 @@ .. startrev: 4cd1bc8b3111 .. branch: release-2.2.x + +.. branch: numpy-newbyteorder +Clean up numpy types, add newbyteorder functionality From noreply at buildbot.pypy.org Thu Nov 14 09:57:50 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Thu, 14 Nov 2013 09:57:50 +0100 (CET) Subject: [pypy-commit] pypy cleanup-numpypy-namespace: close merged branch Message-ID: <20131114085750.52B7F1C0134@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: cleanup-numpypy-namespace Changeset: r68027:9d9386541679 Date: 2013-11-14 03:54 -0500 http://bitbucket.org/pypy/pypy/changeset/9d9386541679/ Log: close merged branch From noreply at buildbot.pypy.org Thu Nov 14 09:57:51 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Thu, 14 Nov 2013 09:57:51 +0100 (CET) Subject: [pypy-commit] pypy numpypy-inplace-op: close merged branch Message-ID: <20131114085751.7E2381C0134@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: numpypy-inplace-op Changeset: r68028:3271bde53cb5 Date: 2013-11-14 03:55 -0500 http://bitbucket.org/pypy/pypy/changeset/3271bde53cb5/ Log: close merged branch From noreply at buildbot.pypy.org Thu Nov 14 09:57:52 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Thu, 14 Nov 2013 09:57:52 +0100 (CET) Subject: [pypy-commit] pypy numpypy_count_nonzero: close merged branch Message-ID: <20131114085752.B97771C0134@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: numpypy_count_nonzero Changeset: r68029:3112b0eadb86 Date: 2013-11-14 03:56 -0500 http://bitbucket.org/pypy/pypy/changeset/3112b0eadb86/ Log: close merged branch From noreply at buildbot.pypy.org Thu Nov 14 10:01:10 2013 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 14 Nov 2013 10:01:10 +0100 (CET) Subject: [pypy-commit] pypy release-2.2.x: Added tag release-2.2.0 for changeset 8eb5b5ac4bba Message-ID: <20131114090110.A059E1C0134@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: release-2.2.x Changeset: r68030:6c8420e89b80 Date: 2013-11-14 10:00 +0100 http://bitbucket.org/pypy/pypy/changeset/6c8420e89b80/ Log: Added tag release-2.2.0 for changeset 8eb5b5ac4bba diff --git a/.hgtags b/.hgtags --- a/.hgtags +++ b/.hgtags @@ -6,3 +6,4 @@ 9b623bc48b5950cf07184462a0e48f2c4df0d720 pypy-2.1-beta1-arm 9b623bc48b5950cf07184462a0e48f2c4df0d720 pypy-2.1-beta1-arm ab0dd631c22015ed88e583d9fdd4c43eebf0be21 pypy-2.1-beta1-arm +8eb5b5ac4bba366e7c7519c186bdfcf9c28c075d release-2.2.0 From noreply at buildbot.pypy.org Thu Nov 14 10:02:58 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Thu, 14 Nov 2013 10:02:58 +0100 (CET) Subject: [pypy-commit] pypy dtype-isnative: close merged branch Message-ID: <20131114090258.2FB691C0134@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: dtype-isnative Changeset: r68031:ebf84dd7f970 Date: 2013-11-14 04:00 -0500 http://bitbucket.org/pypy/pypy/changeset/ebf84dd7f970/ Log: close merged branch From noreply at buildbot.pypy.org Thu Nov 14 10:02:59 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Thu, 14 Nov 2013 10:02:59 +0100 (CET) Subject: [pypy-commit] pypy ndmin: close merged branch Message-ID: <20131114090259.64BED1C0134@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: ndmin Changeset: r68032:9d2bc73626af Date: 2013-11-14 04:01 -0500 http://bitbucket.org/pypy/pypy/changeset/9d2bc73626af/ Log: close merged branch From noreply at buildbot.pypy.org Thu Nov 14 10:03:00 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Thu, 14 Nov 2013 10:03:00 +0100 (CET) Subject: [pypy-commit] pypy array_equal: close merged branch Message-ID: <20131114090300.78ED51C0134@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: array_equal Changeset: r68033:7630ca70381b Date: 2013-11-14 04:01 -0500 http://bitbucket.org/pypy/pypy/changeset/7630ca70381b/ Log: close merged branch From noreply at buildbot.pypy.org Thu Nov 14 10:09:26 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Thu, 14 Nov 2013 10:09:26 +0100 (CET) Subject: [pypy-commit] pypy pickle-dumps: close merged branch Message-ID: <20131114090926.1B1D51C0134@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: pickle-dumps Changeset: r68034:3fec18025f0d Date: 2013-11-14 04:05 -0500 http://bitbucket.org/pypy/pypy/changeset/3fec18025f0d/ Log: close merged branch From noreply at buildbot.pypy.org Thu Nov 14 10:09:27 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Thu, 14 Nov 2013 10:09:27 +0100 (CET) Subject: [pypy-commit] pypy win32-fixes: close merged branch Message-ID: <20131114090927.47C001C0134@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: win32-fixes Changeset: r68035:45ecc4a42058 Date: 2013-11-14 04:06 -0500 http://bitbucket.org/pypy/pypy/changeset/45ecc4a42058/ Log: close merged branch From noreply at buildbot.pypy.org Thu Nov 14 10:09:28 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Thu, 14 Nov 2013 10:09:28 +0100 (CET) Subject: [pypy-commit] pypy stdlib-2.7.4-poll-fix: close merged branch Message-ID: <20131114090928.6EDF01C0134@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: stdlib-2.7.4-poll-fix Changeset: r68036:09e6c7429ebb Date: 2013-11-14 04:07 -0500 http://bitbucket.org/pypy/pypy/changeset/09e6c7429ebb/ Log: close merged branch From noreply at buildbot.pypy.org Thu Nov 14 10:09:29 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Thu, 14 Nov 2013 10:09:29 +0100 (CET) Subject: [pypy-commit] pypy stdlib-2.7.4-pwd-fix: close merged branch Message-ID: <20131114090929.ACBBD1C0134@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: stdlib-2.7.4-pwd-fix Changeset: r68037:048031fbb4f9 Date: 2013-11-14 04:07 -0500 http://bitbucket.org/pypy/pypy/changeset/048031fbb4f9/ Log: close merged branch From noreply at buildbot.pypy.org Thu Nov 14 10:09:30 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Thu, 14 Nov 2013 10:09:30 +0100 (CET) Subject: [pypy-commit] pypy stdlib-2.7.4-fixed-io: close merged branch Message-ID: <20131114090930.D56DB1C0134@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: stdlib-2.7.4-fixed-io Changeset: r68038:e6b76694838f Date: 2013-11-14 04:07 -0500 http://bitbucket.org/pypy/pypy/changeset/e6b76694838f/ Log: close merged branch From noreply at buildbot.pypy.org Thu Nov 14 10:09:31 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Thu, 14 Nov 2013 10:09:31 +0100 (CET) Subject: [pypy-commit] pypy stdlib-2.7.4-fixed-class: close merged branch Message-ID: <20131114090931.ECB291C0134@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: stdlib-2.7.4-fixed-class Changeset: r68039:3cbee0b9425c Date: 2013-11-14 04:07 -0500 http://bitbucket.org/pypy/pypy/changeset/3cbee0b9425c/ Log: close merged branch From noreply at buildbot.pypy.org Thu Nov 14 10:10:52 2013 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 14 Nov 2013 10:10:52 +0100 (CET) Subject: [pypy-commit] jitviewer default: Added tag pypy-2.2 for changeset 15e03325a227 Message-ID: <20131114091052.8D7641C0134@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r253:0d780832a697 Date: 2013-11-14 10:10 +0100 http://bitbucket.org/pypy/jitviewer/changeset/0d780832a697/ Log: Added tag pypy-2.2 for changeset 15e03325a227 diff --git a/.hgtags b/.hgtags --- a/.hgtags +++ b/.hgtags @@ -1,2 +1,3 @@ 24adc3403cd8fdcd9e3f76f31a8dc2c145471002 release-0.1 13e1f8c97ca7c47f807ea93f44392c3f48102675 pypy-1.9 +15e03325a227c4c7145a56e841b6a8a3c59730ed pypy-2.2 From noreply at buildbot.pypy.org Thu Nov 14 10:10:56 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Thu, 14 Nov 2013 10:10:56 +0100 (CET) Subject: [pypy-commit] pypy clean-up-remaining-pypy-rlib-refs: close merged branch Message-ID: <20131114091056.9B2421C0134@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: clean-up-remaining-pypy-rlib-refs Changeset: r68040:67779c333eea Date: 2013-11-14 04:10 -0500 http://bitbucket.org/pypy/pypy/changeset/67779c333eea/ Log: close merged branch From noreply at buildbot.pypy.org Thu Nov 14 10:10:57 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Thu, 14 Nov 2013 10:10:57 +0100 (CET) Subject: [pypy-commit] pypy coding-guide-update-rlib-refs: close merged branch Message-ID: <20131114091057.D53F51C0134@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: coding-guide-update-rlib-refs Changeset: r68041:4230db2e2e1e Date: 2013-11-14 04:10 -0500 http://bitbucket.org/pypy/pypy/changeset/4230db2e2e1e/ Log: close merged branch From noreply at buildbot.pypy.org Thu Nov 14 10:13:26 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Thu, 14 Nov 2013 10:13:26 +0100 (CET) Subject: [pypy-commit] pypy unbreak-freebsd: close merged branch Message-ID: <20131114091326.A2D9E1C0134@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: unbreak-freebsd Changeset: r68042:d28e53d9c806 Date: 2013-11-14 04:11 -0500 http://bitbucket.org/pypy/pypy/changeset/d28e53d9c806/ Log: close merged branch From noreply at buildbot.pypy.org Thu Nov 14 10:13:27 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Thu, 14 Nov 2013 10:13:27 +0100 (CET) Subject: [pypy-commit] pypy unquote-faster: close merged branch Message-ID: <20131114091327.E48FF1C0134@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: unquote-faster Changeset: r68043:c5943a5c41a9 Date: 2013-11-14 04:11 -0500 http://bitbucket.org/pypy/pypy/changeset/c5943a5c41a9/ Log: close merged branch From noreply at buildbot.pypy.org Thu Nov 14 10:13:29 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Thu, 14 Nov 2013 10:13:29 +0100 (CET) Subject: [pypy-commit] pypy urlparse-unquote-faster: close merged branch Message-ID: <20131114091329.301331C0134@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: urlparse-unquote-faster Changeset: r68044:06e7251aaed4 Date: 2013-11-14 04:12 -0500 http://bitbucket.org/pypy/pypy/changeset/06e7251aaed4/ Log: close merged branch From noreply at buildbot.pypy.org Thu Nov 14 10:19:01 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Thu, 14 Nov 2013 10:19:01 +0100 (CET) Subject: [pypy-commit] pypy nobold-backtrace: close merged branch Message-ID: <20131114091901.6D1191C0204@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: nobold-backtrace Changeset: r68045:be560078e413 Date: 2013-11-14 04:16 -0500 http://bitbucket.org/pypy/pypy/changeset/be560078e413/ Log: close merged branch From noreply at buildbot.pypy.org Thu Nov 14 10:19:04 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Thu, 14 Nov 2013 10:19:04 +0100 (CET) Subject: [pypy-commit] pypy remove-PYPY_NOT_MAIN_FILE: close merged branch Message-ID: <20131114091904.61DDD1C0204@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: remove-PYPY_NOT_MAIN_FILE Changeset: r68046:0a8ce6278f68 Date: 2013-11-14 04:17 -0500 http://bitbucket.org/pypy/pypy/changeset/0a8ce6278f68/ Log: close merged branch From noreply at buildbot.pypy.org Thu Nov 14 10:19:05 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Thu, 14 Nov 2013 10:19:05 +0100 (CET) Subject: [pypy-commit] pypy distutils-cppldflags: close merged branch Message-ID: <20131114091905.8D9451C0204@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: distutils-cppldflags Changeset: r68047:c28dc4ddd9be Date: 2013-11-14 04:17 -0500 http://bitbucket.org/pypy/pypy/changeset/c28dc4ddd9be/ Log: close merged branch From noreply at buildbot.pypy.org Thu Nov 14 10:19:06 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Thu, 14 Nov 2013 10:19:06 +0100 (CET) Subject: [pypy-commit] pypy improve-errors-again: close merged branch Message-ID: <20131114091906.A6C011C0204@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: improve-errors-again Changeset: r68048:5f5e401edccc Date: 2013-11-14 04:18 -0500 http://bitbucket.org/pypy/pypy/changeset/5f5e401edccc/ Log: close merged branch From noreply at buildbot.pypy.org Thu Nov 14 10:19:07 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Thu, 14 Nov 2013 10:19:07 +0100 (CET) Subject: [pypy-commit] pypy improve-errors-again2: close merged branch Message-ID: <20131114091907.CB2111C0204@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: improve-errors-again2 Changeset: r68049:fc8664971578 Date: 2013-11-14 04:18 -0500 http://bitbucket.org/pypy/pypy/changeset/fc8664971578/ Log: close merged branch From noreply at buildbot.pypy.org Thu Nov 14 10:19:55 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Thu, 14 Nov 2013 10:19:55 +0100 (CET) Subject: [pypy-commit] pypy popen2-removal: close merged branch Message-ID: <20131114091955.609621C0204@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: popen2-removal Changeset: r68050:6538f38f63da Date: 2013-11-14 04:19 -0500 http://bitbucket.org/pypy/pypy/changeset/6538f38f63da/ Log: close merged branch From noreply at buildbot.pypy.org Thu Nov 14 10:23:39 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Thu, 14 Nov 2013 10:23:39 +0100 (CET) Subject: [pypy-commit] pypy package-tk: close merged branch Message-ID: <20131114092339.C5B501C0204@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: package-tk Changeset: r68051:ca3a4cf272be Date: 2013-11-14 04:20 -0500 http://bitbucket.org/pypy/pypy/changeset/ca3a4cf272be/ Log: close merged branch From noreply at buildbot.pypy.org Thu Nov 14 10:23:43 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Thu, 14 Nov 2013 10:23:43 +0100 (CET) Subject: [pypy-commit] pypy enumerate-rstr: close merged branch Message-ID: <20131114092343.6821E1C0204@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: enumerate-rstr Changeset: r68052:cd07f4c70232 Date: 2013-11-14 04:21 -0500 http://bitbucket.org/pypy/pypy/changeset/cd07f4c70232/ Log: close merged branch From noreply at buildbot.pypy.org Thu Nov 14 10:23:44 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Thu, 14 Nov 2013 10:23:44 +0100 (CET) Subject: [pypy-commit] pypy ndarray-ptp: close merged branch Message-ID: <20131114092344.8C4F41C0204@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: ndarray-ptp Changeset: r68053:3a0298093f20 Date: 2013-11-14 04:21 -0500 http://bitbucket.org/pypy/pypy/changeset/3a0298093f20/ Log: close merged branch From noreply at buildbot.pypy.org Thu Nov 14 10:23:47 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Thu, 14 Nov 2013 10:23:47 +0100 (CET) Subject: [pypy-commit] pypy rlib-doc-rpython-refs: close merged branch Message-ID: <20131114092347.37BF71C0204@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: rlib-doc-rpython-refs Changeset: r68054:8412492e8e81 Date: 2013-11-14 04:22 -0500 http://bitbucket.org/pypy/pypy/changeset/8412492e8e81/ Log: close merged branch From noreply at buildbot.pypy.org Thu Nov 14 10:25:46 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Thu, 14 Nov 2013 10:25:46 +0100 (CET) Subject: [pypy-commit] pypy curses_fixes: close merged branch Message-ID: <20131114092546.56F3F1C0204@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: curses_fixes Changeset: r68055:8e4f7f8df8e1 Date: 2013-11-14 04:23 -0500 http://bitbucket.org/pypy/pypy/changeset/8e4f7f8df8e1/ Log: close merged branch From noreply at buildbot.pypy.org Thu Nov 14 10:25:47 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Thu, 14 Nov 2013 10:25:47 +0100 (CET) Subject: [pypy-commit] pypy ctypes-byref: close merged branch Message-ID: <20131114092547.7A7011C0204@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: ctypes-byref Changeset: r68056:dc20838c51f4 Date: 2013-11-14 04:24 -0500 http://bitbucket.org/pypy/pypy/changeset/dc20838c51f4/ Log: close merged branch From noreply at buildbot.pypy.org Thu Nov 14 10:25:48 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Thu, 14 Nov 2013 10:25:48 +0100 (CET) Subject: [pypy-commit] pypy statvfs_tests: close merged branch Message-ID: <20131114092548.91A2C1C0204@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: statvfs_tests Changeset: r68057:61ce3b8b16b1 Date: 2013-11-14 04:24 -0500 http://bitbucket.org/pypy/pypy/changeset/61ce3b8b16b1/ Log: close merged branch From noreply at buildbot.pypy.org Thu Nov 14 10:25:49 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Thu, 14 Nov 2013 10:25:49 +0100 (CET) Subject: [pypy-commit] pypy fix-jit-logs: close merged branch Message-ID: <20131114092549.C47C41C0204@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: fix-jit-logs Changeset: r68058:ea46bf3cc485 Date: 2013-11-14 04:24 -0500 http://bitbucket.org/pypy/pypy/changeset/ea46bf3cc485/ Log: close merged branch From noreply at buildbot.pypy.org Thu Nov 14 10:25:50 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Thu, 14 Nov 2013 10:25:50 +0100 (CET) Subject: [pypy-commit] pypy pycon2013-doc-fixes: close merged branch Message-ID: <20131114092550.DDCFD1C0204@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: pycon2013-doc-fixes Changeset: r68059:3f776b761e09 Date: 2013-11-14 04:25 -0500 http://bitbucket.org/pypy/pypy/changeset/3f776b761e09/ Log: close merged branch From noreply at buildbot.pypy.org Thu Nov 14 10:26:20 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Thu, 14 Nov 2013 10:26:20 +0100 (CET) Subject: [pypy-commit] pypy zlib-mem-pressure: close merged branch Message-ID: <20131114092620.AD3301C0204@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: zlib-mem-pressure Changeset: r68060:bfcb4d5e68e6 Date: 2013-11-14 04:25 -0500 http://bitbucket.org/pypy/pypy/changeset/bfcb4d5e68e6/ Log: close merged branch From noreply at buildbot.pypy.org Thu Nov 14 10:26:47 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Thu, 14 Nov 2013 10:26:47 +0100 (CET) Subject: [pypy-commit] pypy task-decorator: close merged branch Message-ID: <20131114092647.C5F571C0204@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: task-decorator Changeset: r68061:18d6db6fccf8 Date: 2013-11-14 04:26 -0500 http://bitbucket.org/pypy/pypy/changeset/18d6db6fccf8/ Log: close merged branch From noreply at buildbot.pypy.org Thu Nov 14 10:28:16 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Thu, 14 Nov 2013 10:28:16 +0100 (CET) Subject: [pypy-commit] pypy py3k-struct: close merged branch Message-ID: <20131114092816.87E3C1C0204@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: py3k-struct Changeset: r68062:d7746d32bf9d Date: 2013-11-14 04:27 -0500 http://bitbucket.org/pypy/pypy/changeset/d7746d32bf9d/ Log: close merged branch From noreply at buildbot.pypy.org Thu Nov 14 10:28:17 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Thu, 14 Nov 2013 10:28:17 +0100 (CET) Subject: [pypy-commit] pypy py3k-subprocess-new-session: close merged branch Message-ID: <20131114092817.B90181C0204@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: py3k-subprocess-new-session Changeset: r68063:3b448254587e Date: 2013-11-14 04:27 -0500 http://bitbucket.org/pypy/pypy/changeset/3b448254587e/ Log: close merged branch From noreply at buildbot.pypy.org Thu Nov 14 10:28:36 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Thu, 14 Nov 2013 10:28:36 +0100 (CET) Subject: [pypy-commit] pypy pythoninspect-fix: close merged branch Message-ID: <20131114092836.A61491C0204@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: pythoninspect-fix Changeset: r68064:56a9bd9b2769 Date: 2013-11-14 04:27 -0500 http://bitbucket.org/pypy/pypy/changeset/56a9bd9b2769/ Log: close merged branch From noreply at buildbot.pypy.org Thu Nov 14 10:31:57 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Thu, 14 Nov 2013 10:31:57 +0100 (CET) Subject: [pypy-commit] pypy py3k-list-compr-or: close merged branch Message-ID: <20131114093157.247491C13B7@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: py3k-list-compr-or Changeset: r68065:659f78e9b5b6 Date: 2013-11-14 04:31 -0500 http://bitbucket.org/pypy/pypy/changeset/659f78e9b5b6/ Log: close merged branch From noreply at buildbot.pypy.org Thu Nov 14 10:32:20 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Thu, 14 Nov 2013 10:32:20 +0100 (CET) Subject: [pypy-commit] pypy bridge-logging: close merged branch Message-ID: <20131114093220.A43E71C13B7@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: bridge-logging Changeset: r68066:83f0ba55bc1b Date: 2013-11-14 04:31 -0500 http://bitbucket.org/pypy/pypy/changeset/83f0ba55bc1b/ Log: close merged branch From noreply at buildbot.pypy.org Thu Nov 14 10:32:42 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Thu, 14 Nov 2013 10:32:42 +0100 (CET) Subject: [pypy-commit] pypy ssl_moving_write_buffer: close merged branch Message-ID: <20131114093242.5F5651C13B7@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: ssl_moving_write_buffer Changeset: r68067:800741417787 Date: 2013-11-14 04:32 -0500 http://bitbucket.org/pypy/pypy/changeset/800741417787/ Log: close merged branch From noreply at buildbot.pypy.org Thu Nov 14 10:34:31 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Thu, 14 Nov 2013 10:34:31 +0100 (CET) Subject: [pypy-commit] pypy improved_ebnfparse_error: close merged branch Message-ID: <20131114093431.03D221C1309@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: improved_ebnfparse_error Changeset: r68068:a1f52d6ae34e Date: 2013-11-14 04:32 -0500 http://bitbucket.org/pypy/pypy/changeset/a1f52d6ae34e/ Log: close merged branch From noreply at buildbot.pypy.org Thu Nov 14 10:34:32 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Thu, 14 Nov 2013 10:34:32 +0100 (CET) Subject: [pypy-commit] pypy cpyext-PyThreadState_New: close merged branch Message-ID: <20131114093432.2F9B41C1309@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: cpyext-PyThreadState_New Changeset: r68069:87a5f161be81 Date: 2013-11-14 04:33 -0500 http://bitbucket.org/pypy/pypy/changeset/87a5f161be81/ Log: close merged branch From noreply at buildbot.pypy.org Thu Nov 14 10:48:53 2013 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 14 Nov 2013 10:48:53 +0100 (CET) Subject: [pypy-commit] pypy default: Issue #1633 Message-ID: <20131114094853.45A8C1C0134@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r68070:8d1f6d47d417 Date: 2013-11-14 10:48 +0100 http://bitbucket.org/pypy/pypy/changeset/8d1f6d47d417/ Log: Issue #1633 Give a more sensible error message on int() or long() typeerrors. diff --git a/pypy/objspace/std/inttype.py b/pypy/objspace/std/inttype.py --- a/pypy/objspace/std/inttype.py +++ b/pypy/objspace/std/inttype.py @@ -119,9 +119,14 @@ if not ok: # otherwise, use the __int__() or the __trunc__() methods w_obj = w_value - if space.lookup(w_obj, '__int__') is None: + if space.lookup(w_obj, '__int__') is not None: + w_obj = space.int(w_obj) + elif space.lookup(w_obj, '__trunc__') is not None: w_obj = space.trunc(w_obj) - w_obj = space.int(w_obj) + else: + raise operationerrfmt(space.w_TypeError, + "int() argument must be a string or a number, not '%T'", + w_obj) # 'int(x)' should return what x.__int__() returned, which should # be an int or long or a subclass thereof. if space.is_w(w_inttype, space.w_int): diff --git a/pypy/objspace/std/longtype.py b/pypy/objspace/std/longtype.py --- a/pypy/objspace/std/longtype.py +++ b/pypy/objspace/std/longtype.py @@ -1,4 +1,4 @@ -from pypy.interpreter.error import OperationError +from pypy.interpreter.error import OperationError, operationerrfmt from pypy.interpreter import typedef from pypy.interpreter.gateway import interp2app, unwrap_spec, WrappedDefault,\ interpindirect2app @@ -39,13 +39,17 @@ if (space.lookup(w_obj, '__long__') is not None or space.lookup(w_obj, '__int__') is not None): w_obj = space.long(w_obj) - else: + elif space.lookup(w_obj, '__trunc__') is not None: w_obj = space.trunc(w_obj) # :-( blame CPython 2.7 if space.lookup(w_obj, '__long__') is not None: w_obj = space.long(w_obj) else: w_obj = space.int(w_obj) + else: + raise operationerrfmt(space.w_TypeError, + "long() argument must be a string or a number, not '%T'", + w_obj) bigint = space.bigint_w(w_obj) return newbigint(space, w_longtype, bigint) else: diff --git a/pypy/objspace/std/test/test_intobject.py b/pypy/objspace/std/test/test_intobject.py --- a/pypy/objspace/std/test/test_intobject.py +++ b/pypy/objspace/std/test/test_intobject.py @@ -498,6 +498,11 @@ b = A(5).real assert type(b) is int + def test_int_error_msg(self): + e = raises(TypeError, int, []) + assert str(e.value) == ( + "int() argument must be a string or a number, not 'list'") + class AppTestIntOptimizedAdd(AppTestInt): spaceconfig = {"objspace.std.optimized_int_add": True} diff --git a/pypy/objspace/std/test/test_longobject.py b/pypy/objspace/std/test/test_longobject.py --- a/pypy/objspace/std/test/test_longobject.py +++ b/pypy/objspace/std/test/test_longobject.py @@ -341,3 +341,8 @@ assert int(long(3)) == long(3) assert int(A(13)) == 42 + + def test_long_error_msg(self): + e = raises(TypeError, long, []) + assert str(e.value) == ( + "long() argument must be a string or a number, not 'list'") From noreply at buildbot.pypy.org Thu Nov 14 13:23:55 2013 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 14 Nov 2013 13:23:55 +0100 (CET) Subject: [pypy-commit] pypy default: Thanks Larry Message-ID: <20131114122355.C51381C12EC@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r68071:d13b9ffb9f1a Date: 2013-11-14 13:23 +0100 http://bitbucket.org/pypy/pypy/changeset/d13b9ffb9f1a/ Log: Thanks Larry diff --git a/pypy/doc/release-2.2.0.rst b/pypy/doc/release-2.2.0.rst --- a/pypy/doc/release-2.2.0.rst +++ b/pypy/doc/release-2.2.0.rst @@ -74,7 +74,7 @@ * Things that use ``sys.set_trace`` are now JITted (like coverage) -* JSON encoding used to be very fast, now decoding is as well +* JSON decoding is now very fast (JSON encoding was already very fast) * various buffer copying methods experience speedups (like list-of-ints to ``int[]`` buffer from cffi) From noreply at buildbot.pypy.org Thu Nov 14 13:29:08 2013 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 14 Nov 2013 13:29:08 +0100 (CET) Subject: [pypy-commit] cffi release-0.8: Release 0.8 Message-ID: <20131114122908.A78721C12EC@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: release-0.8 Changeset: r1417:2d6172481285 Date: 2013-11-14 13:28 +0100 http://bitbucket.org/cffi/cffi/changeset/2d6172481285/ Log: Release 0.8 From noreply at buildbot.pypy.org Thu Nov 14 13:30:21 2013 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 14 Nov 2013 13:30:21 +0100 (CET) Subject: [pypy-commit] cffi release-0.8: MD5/SHA1 Message-ID: <20131114123021.201541C12EC@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: release-0.8 Changeset: r1418:7f64d6952c56 Date: 2013-11-14 13:30 +0100 http://bitbucket.org/cffi/cffi/changeset/7f64d6952c56/ Log: MD5/SHA1 diff --git a/doc/source/index.rst b/doc/source/index.rst --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -92,9 +92,9 @@ - Or grab the most current version by following the instructions below. - - MD5: ... + - MD5: e61deb0515311bb42d5d58b9403bc923 - - SHA: ... + - SHA: 8332429193cb74d74f3347af180b448425d7d176 * Or get it from the `Bitbucket page`_: ``hg clone https://bitbucket.org/cffi/cffi`` From noreply at buildbot.pypy.org Thu Nov 14 13:44:26 2013 From: noreply at buildbot.pypy.org (bivab) Date: Thu, 14 Nov 2013 13:44:26 +0100 (CET) Subject: [pypy-commit] buildbot default: move NUMPY_64 to allegro64 and add it to the nightly-2-00 scheduler Message-ID: <20131114124426.592571C0134@cobra.cs.uni-duesseldorf.de> Author: David Schneider Branch: Changeset: r890:b7762c17e248 Date: 2013-11-14 13:43 +0100 http://bitbucket.org/pypy/buildbot/changeset/b7762c17e248/ Log: move NUMPY_64 to allegro64 and add it to the nightly-2-00 scheduler diff --git a/bot2/pypybuildbot/master.py b/bot2/pypybuildbot/master.py --- a/bot2/pypybuildbot/master.py +++ b/bot2/pypybuildbot/master.py @@ -225,6 +225,9 @@ Nightly("nightly-2-00", [ JITBENCH, # on tannit32, uses 1 core (in part exclusively) JITBENCH64, # on tannit64, uses 1 core (in part exclusively) + NUMPY_64, # on allegro64, uses 1 core + # XXX maybe use a trigger instead? + ], branch='default', hour=2, minute=0), Nightly("nightly-2-00-py3k", [ @@ -235,6 +238,7 @@ Nightly("nighly-ppc", [ JITONLYLINUXPPC64, # on gcc1 ], branch='ppc-jit-backend', hour=1, minute=0), + CustomForceScheduler('Force Scheduler', builderNames=[ PYPYBUILDBOT, @@ -437,11 +441,10 @@ 'category': 'openindiana32', }, {'name': NUMPY_64, - 'slavenames': ["tannit64"], + 'slavenames': ["allegro64"], 'builddir': NUMPY_64, 'factory': pypyNumpyCompatability, 'category': 'numpy', - 'locks': [TannitCPU.access('counting')], }, {'name': PYPYBUILDBOT, 'slavenames': ['cobra'], From noreply at buildbot.pypy.org Thu Nov 14 14:46:08 2013 From: noreply at buildbot.pypy.org (bivab) Date: Thu, 14 Nov 2013 14:46:08 +0100 (CET) Subject: [pypy-commit] buildbot default: move NUMPY_64 back to tannit64 Message-ID: <20131114134608.5600A1C0134@cobra.cs.uni-duesseldorf.de> Author: David Schneider Branch: Changeset: r891:4d6e4e3813f2 Date: 2013-11-14 14:45 +0100 http://bitbucket.org/pypy/buildbot/changeset/4d6e4e3813f2/ Log: move NUMPY_64 back to tannit64 diff --git a/bot2/pypybuildbot/master.py b/bot2/pypybuildbot/master.py --- a/bot2/pypybuildbot/master.py +++ b/bot2/pypybuildbot/master.py @@ -223,10 +223,10 @@ ], branch='default', hour=0, minute=0), Nightly("nightly-2-00", [ + NUMPY_64, # on tannit64, uses 1 core, takes about 15min. + # XXX maybe use a trigger instead? JITBENCH, # on tannit32, uses 1 core (in part exclusively) JITBENCH64, # on tannit64, uses 1 core (in part exclusively) - NUMPY_64, # on allegro64, uses 1 core - # XXX maybe use a trigger instead? ], branch='default', hour=2, minute=0), @@ -441,10 +441,11 @@ 'category': 'openindiana32', }, {'name': NUMPY_64, - 'slavenames': ["allegro64"], + 'slavenames': ["tannit64"], 'builddir': NUMPY_64, 'factory': pypyNumpyCompatability, 'category': 'numpy', + 'locks': [TannitCPU.access('counting')], }, {'name': PYPYBUILDBOT, 'slavenames': ['cobra'], From noreply at buildbot.pypy.org Thu Nov 14 19:02:42 2013 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 14 Nov 2013 19:02:42 +0100 (CET) Subject: [pypy-commit] pypy closed-branches: Merge closed head 7f1adb56a558 on branch bigint-with-int Message-ID: <20131114180242.8A3C21C1309@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: closed-branches Changeset: r68073:47504fdf3522 Date: 2013-11-14 19:00 +0100 http://bitbucket.org/pypy/pypy/changeset/47504fdf3522/ Log: Merge closed head 7f1adb56a558 on branch bigint-with-int From noreply at buildbot.pypy.org Thu Nov 14 19:02:41 2013 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 14 Nov 2013 19:02:41 +0100 (CET) Subject: [pypy-commit] pypy closed-branches: Merge closed head e57b90333d9a on branch default Message-ID: <20131114180241.6B2DA1C12EC@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: closed-branches Changeset: r68072:87b139c5154a Date: 2013-11-14 19:00 +0100 http://bitbucket.org/pypy/pypy/changeset/87b139c5154a/ Log: Merge closed head e57b90333d9a on branch default diff --git a/.hgsubstate b/.hgsubstate new file mode 100644 From noreply at buildbot.pypy.org Thu Nov 14 19:02:43 2013 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 14 Nov 2013 19:02:43 +0100 (CET) Subject: [pypy-commit] pypy closed-branches: Merge closed head 6a5f3429faad on branch optmodel-refactor Message-ID: <20131114180243.99E5D1C12EC@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: closed-branches Changeset: r68074:d42e9c73d526 Date: 2013-11-14 19:00 +0100 http://bitbucket.org/pypy/pypy/changeset/d42e9c73d526/ Log: Merge closed head 6a5f3429faad on branch optmodel-refactor From noreply at buildbot.pypy.org Thu Nov 14 19:02:44 2013 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 14 Nov 2013 19:02:44 +0100 (CET) Subject: [pypy-commit] pypy closed-branches: Merge closed head 31b7c9223f79 on branch rdict-experiments-2 Message-ID: <20131114180244.A68DE1C12EC@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: closed-branches Changeset: r68075:ff0c1dd2927c Date: 2013-11-14 19:00 +0100 http://bitbucket.org/pypy/pypy/changeset/ff0c1dd2927c/ Log: Merge closed head 31b7c9223f79 on branch rdict-experiments-2 From noreply at buildbot.pypy.org Thu Nov 14 19:02:45 2013 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 14 Nov 2013 19:02:45 +0100 (CET) Subject: [pypy-commit] pypy closed-branches: Merge closed head 9ad0c80d9cff on branch remove-numpypy Message-ID: <20131114180245.AABCF1C12EC@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: closed-branches Changeset: r68076:a76ac33d372a Date: 2013-11-14 19:00 +0100 http://bitbucket.org/pypy/pypy/changeset/a76ac33d372a/ Log: Merge closed head 9ad0c80d9cff on branch remove-numpypy From noreply at buildbot.pypy.org Thu Nov 14 19:02:46 2013 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 14 Nov 2013 19:02:46 +0100 (CET) Subject: [pypy-commit] pypy closed-branches: Merge closed head cf222adc939f on branch numpy-newbyteorder Message-ID: <20131114180246.BBEDC1C12EC@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: closed-branches Changeset: r68077:333251320e76 Date: 2013-11-14 19:00 +0100 http://bitbucket.org/pypy/pypy/changeset/333251320e76/ Log: Merge closed head cf222adc939f on branch numpy-newbyteorder From noreply at buildbot.pypy.org Thu Nov 14 19:02:47 2013 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 14 Nov 2013 19:02:47 +0100 (CET) Subject: [pypy-commit] pypy closed-branches: Merge closed head 9d9386541679 on branch cleanup-numpypy-namespace Message-ID: <20131114180247.C49971C12EC@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: closed-branches Changeset: r68078:cacf7ddae3aa Date: 2013-11-14 19:00 +0100 http://bitbucket.org/pypy/pypy/changeset/cacf7ddae3aa/ Log: Merge closed head 9d9386541679 on branch cleanup-numpypy-namespace From noreply at buildbot.pypy.org Thu Nov 14 19:02:48 2013 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 14 Nov 2013 19:02:48 +0100 (CET) Subject: [pypy-commit] pypy closed-branches: Merge closed head 3271bde53cb5 on branch numpypy-inplace-op Message-ID: <20131114180248.C620A1C12EC@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: closed-branches Changeset: r68079:2f71dbdd9470 Date: 2013-11-14 19:00 +0100 http://bitbucket.org/pypy/pypy/changeset/2f71dbdd9470/ Log: Merge closed head 3271bde53cb5 on branch numpypy-inplace-op From noreply at buildbot.pypy.org Thu Nov 14 19:02:49 2013 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 14 Nov 2013 19:02:49 +0100 (CET) Subject: [pypy-commit] pypy closed-branches: Merge closed head 3112b0eadb86 on branch numpypy_count_nonzero Message-ID: <20131114180249.CA4111C12EC@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: closed-branches Changeset: r68080:998704dde1e3 Date: 2013-11-14 19:00 +0100 http://bitbucket.org/pypy/pypy/changeset/998704dde1e3/ Log: Merge closed head 3112b0eadb86 on branch numpypy_count_nonzero From noreply at buildbot.pypy.org Thu Nov 14 19:02:51 2013 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 14 Nov 2013 19:02:51 +0100 (CET) Subject: [pypy-commit] pypy closed-branches: Merge closed head ebf84dd7f970 on branch dtype-isnative Message-ID: <20131114180251.6FA1E1C12EC@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: closed-branches Changeset: r68081:be7a78744f04 Date: 2013-11-14 19:00 +0100 http://bitbucket.org/pypy/pypy/changeset/be7a78744f04/ Log: Merge closed head ebf84dd7f970 on branch dtype-isnative From noreply at buildbot.pypy.org Thu Nov 14 19:02:52 2013 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 14 Nov 2013 19:02:52 +0100 (CET) Subject: [pypy-commit] pypy closed-branches: Merge closed head 9d2bc73626af on branch ndmin Message-ID: <20131114180252.7843B1C12EC@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: closed-branches Changeset: r68082:642ab92e26cf Date: 2013-11-14 19:00 +0100 http://bitbucket.org/pypy/pypy/changeset/642ab92e26cf/ Log: Merge closed head 9d2bc73626af on branch ndmin From noreply at buildbot.pypy.org Thu Nov 14 19:02:53 2013 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 14 Nov 2013 19:02:53 +0100 (CET) Subject: [pypy-commit] pypy closed-branches: Merge closed head 7630ca70381b on branch array_equal Message-ID: <20131114180253.745281C12EC@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: closed-branches Changeset: r68083:7a010056514c Date: 2013-11-14 19:00 +0100 http://bitbucket.org/pypy/pypy/changeset/7a010056514c/ Log: Merge closed head 7630ca70381b on branch array_equal From noreply at buildbot.pypy.org Thu Nov 14 19:02:54 2013 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 14 Nov 2013 19:02:54 +0100 (CET) Subject: [pypy-commit] pypy closed-branches: Merge closed head 3fec18025f0d on branch pickle-dumps Message-ID: <20131114180254.8AA301C12EC@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: closed-branches Changeset: r68084:b6d66231dba3 Date: 2013-11-14 19:00 +0100 http://bitbucket.org/pypy/pypy/changeset/b6d66231dba3/ Log: Merge closed head 3fec18025f0d on branch pickle-dumps From noreply at buildbot.pypy.org Thu Nov 14 19:02:55 2013 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 14 Nov 2013 19:02:55 +0100 (CET) Subject: [pypy-commit] pypy closed-branches: Merge closed head 45ecc4a42058 on branch win32-fixes Message-ID: <20131114180255.899C81C12EC@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: closed-branches Changeset: r68085:6d4fbb25fa85 Date: 2013-11-14 19:00 +0100 http://bitbucket.org/pypy/pypy/changeset/6d4fbb25fa85/ Log: Merge closed head 45ecc4a42058 on branch win32-fixes From noreply at buildbot.pypy.org Thu Nov 14 19:02:56 2013 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 14 Nov 2013 19:02:56 +0100 (CET) Subject: [pypy-commit] pypy closed-branches: Merge closed head 09e6c7429ebb on branch stdlib-2.7.4-poll-fix Message-ID: <20131114180256.91D2E1C12EC@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: closed-branches Changeset: r68086:aef60966ce17 Date: 2013-11-14 19:00 +0100 http://bitbucket.org/pypy/pypy/changeset/aef60966ce17/ Log: Merge closed head 09e6c7429ebb on branch stdlib-2.7.4-poll-fix From noreply at buildbot.pypy.org Thu Nov 14 19:02:57 2013 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 14 Nov 2013 19:02:57 +0100 (CET) Subject: [pypy-commit] pypy closed-branches: Merge closed head 048031fbb4f9 on branch stdlib-2.7.4-pwd-fix Message-ID: <20131114180257.97F2C1C12EC@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: closed-branches Changeset: r68087:57969d79b842 Date: 2013-11-14 19:00 +0100 http://bitbucket.org/pypy/pypy/changeset/57969d79b842/ Log: Merge closed head 048031fbb4f9 on branch stdlib-2.7.4-pwd-fix From noreply at buildbot.pypy.org Thu Nov 14 19:02:58 2013 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 14 Nov 2013 19:02:58 +0100 (CET) Subject: [pypy-commit] pypy closed-branches: Merge closed head e6b76694838f on branch stdlib-2.7.4-fixed-io Message-ID: <20131114180258.9AD711C12EC@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: closed-branches Changeset: r68088:d435baaea47f Date: 2013-11-14 19:00 +0100 http://bitbucket.org/pypy/pypy/changeset/d435baaea47f/ Log: Merge closed head e6b76694838f on branch stdlib-2.7.4-fixed-io From noreply at buildbot.pypy.org Thu Nov 14 19:02:59 2013 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 14 Nov 2013 19:02:59 +0100 (CET) Subject: [pypy-commit] pypy closed-branches: Merge closed head 3cbee0b9425c on branch stdlib-2.7.4-fixed-class Message-ID: <20131114180259.9A2801C12EC@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: closed-branches Changeset: r68089:8834bb5926f2 Date: 2013-11-14 19:00 +0100 http://bitbucket.org/pypy/pypy/changeset/8834bb5926f2/ Log: Merge closed head 3cbee0b9425c on branch stdlib-2.7.4-fixed-class From noreply at buildbot.pypy.org Thu Nov 14 19:03:00 2013 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 14 Nov 2013 19:03:00 +0100 (CET) Subject: [pypy-commit] pypy closed-branches: Merge closed head 67779c333eea on branch clean-up-remaining-pypy-rlib-refs Message-ID: <20131114180300.98BE11C12EC@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: closed-branches Changeset: r68090:6b3ebcd26d8a Date: 2013-11-14 19:00 +0100 http://bitbucket.org/pypy/pypy/changeset/6b3ebcd26d8a/ Log: Merge closed head 67779c333eea on branch clean-up-remaining-pypy- rlib-refs From noreply at buildbot.pypy.org Thu Nov 14 19:03:01 2013 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 14 Nov 2013 19:03:01 +0100 (CET) Subject: [pypy-commit] pypy closed-branches: Merge closed head 4230db2e2e1e on branch coding-guide-update-rlib-refs Message-ID: <20131114180301.A593B1C12EC@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: closed-branches Changeset: r68091:140a5b7e9376 Date: 2013-11-14 19:00 +0100 http://bitbucket.org/pypy/pypy/changeset/140a5b7e9376/ Log: Merge closed head 4230db2e2e1e on branch coding-guide-update-rlib- refs From noreply at buildbot.pypy.org Thu Nov 14 19:03:02 2013 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 14 Nov 2013 19:03:02 +0100 (CET) Subject: [pypy-commit] pypy closed-branches: Merge closed head d28e53d9c806 on branch unbreak-freebsd Message-ID: <20131114180302.A9B741C12EC@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: closed-branches Changeset: r68092:10757ec6ab02 Date: 2013-11-14 19:00 +0100 http://bitbucket.org/pypy/pypy/changeset/10757ec6ab02/ Log: Merge closed head d28e53d9c806 on branch unbreak-freebsd From noreply at buildbot.pypy.org Thu Nov 14 19:03:03 2013 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 14 Nov 2013 19:03:03 +0100 (CET) Subject: [pypy-commit] pypy closed-branches: Merge closed head c5943a5c41a9 on branch unquote-faster Message-ID: <20131114180303.A92141C12EC@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: closed-branches Changeset: r68093:5394ef2bda3d Date: 2013-11-14 19:00 +0100 http://bitbucket.org/pypy/pypy/changeset/5394ef2bda3d/ Log: Merge closed head c5943a5c41a9 on branch unquote-faster From noreply at buildbot.pypy.org Thu Nov 14 19:03:04 2013 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 14 Nov 2013 19:03:04 +0100 (CET) Subject: [pypy-commit] pypy closed-branches: Merge closed head 06e7251aaed4 on branch urlparse-unquote-faster Message-ID: <20131114180304.AB6281C12EC@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: closed-branches Changeset: r68094:2f23727661fa Date: 2013-11-14 19:00 +0100 http://bitbucket.org/pypy/pypy/changeset/2f23727661fa/ Log: Merge closed head 06e7251aaed4 on branch urlparse-unquote-faster From noreply at buildbot.pypy.org Thu Nov 14 19:03:05 2013 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 14 Nov 2013 19:03:05 +0100 (CET) Subject: [pypy-commit] pypy closed-branches: Merge closed head be560078e413 on branch nobold-backtrace Message-ID: <20131114180305.A93951C12EC@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: closed-branches Changeset: r68095:98531cd04c3a Date: 2013-11-14 19:00 +0100 http://bitbucket.org/pypy/pypy/changeset/98531cd04c3a/ Log: Merge closed head be560078e413 on branch nobold-backtrace From noreply at buildbot.pypy.org Thu Nov 14 19:03:06 2013 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 14 Nov 2013 19:03:06 +0100 (CET) Subject: [pypy-commit] pypy closed-branches: Merge closed head 0a8ce6278f68 on branch remove-PYPY_NOT_MAIN_FILE Message-ID: <20131114180306.B59931C12EC@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: closed-branches Changeset: r68096:e6a2260ea787 Date: 2013-11-14 19:00 +0100 http://bitbucket.org/pypy/pypy/changeset/e6a2260ea787/ Log: Merge closed head 0a8ce6278f68 on branch remove-PYPY_NOT_MAIN_FILE From noreply at buildbot.pypy.org Thu Nov 14 19:03:07 2013 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 14 Nov 2013 19:03:07 +0100 (CET) Subject: [pypy-commit] pypy closed-branches: Merge closed head c28dc4ddd9be on branch distutils-cppldflags Message-ID: <20131114180307.C0D0B1C12EC@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: closed-branches Changeset: r68097:47e35549c6d4 Date: 2013-11-14 19:00 +0100 http://bitbucket.org/pypy/pypy/changeset/47e35549c6d4/ Log: Merge closed head c28dc4ddd9be on branch distutils-cppldflags From noreply at buildbot.pypy.org Thu Nov 14 19:03:08 2013 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 14 Nov 2013 19:03:08 +0100 (CET) Subject: [pypy-commit] pypy closed-branches: Merge closed head 5f5e401edccc on branch improve-errors-again Message-ID: <20131114180308.C1F0A1C12EC@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: closed-branches Changeset: r68098:2051fc50063a Date: 2013-11-14 19:00 +0100 http://bitbucket.org/pypy/pypy/changeset/2051fc50063a/ Log: Merge closed head 5f5e401edccc on branch improve-errors-again From noreply at buildbot.pypy.org Thu Nov 14 19:03:09 2013 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 14 Nov 2013 19:03:09 +0100 (CET) Subject: [pypy-commit] pypy closed-branches: Merge closed head fc8664971578 on branch improve-errors-again2 Message-ID: <20131114180309.C12041C12EC@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: closed-branches Changeset: r68099:4edea7455192 Date: 2013-11-14 19:00 +0100 http://bitbucket.org/pypy/pypy/changeset/4edea7455192/ Log: Merge closed head fc8664971578 on branch improve-errors-again2 From noreply at buildbot.pypy.org Thu Nov 14 19:03:10 2013 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 14 Nov 2013 19:03:10 +0100 (CET) Subject: [pypy-commit] pypy closed-branches: Merge closed head 6538f38f63da on branch popen2-removal Message-ID: <20131114180310.C2F6F1C12EC@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: closed-branches Changeset: r68100:39ccd08e0b72 Date: 2013-11-14 19:00 +0100 http://bitbucket.org/pypy/pypy/changeset/39ccd08e0b72/ Log: Merge closed head 6538f38f63da on branch popen2-removal From noreply at buildbot.pypy.org Thu Nov 14 19:03:11 2013 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 14 Nov 2013 19:03:11 +0100 (CET) Subject: [pypy-commit] pypy closed-branches: Merge closed head ca3a4cf272be on branch package-tk Message-ID: <20131114180311.BDF8B1C12EC@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: closed-branches Changeset: r68101:9ac54ab0dbe0 Date: 2013-11-14 19:00 +0100 http://bitbucket.org/pypy/pypy/changeset/9ac54ab0dbe0/ Log: Merge closed head ca3a4cf272be on branch package-tk From noreply at buildbot.pypy.org Thu Nov 14 19:03:12 2013 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 14 Nov 2013 19:03:12 +0100 (CET) Subject: [pypy-commit] pypy closed-branches: Merge closed head cd07f4c70232 on branch enumerate-rstr Message-ID: <20131114180312.C6F061C12EC@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: closed-branches Changeset: r68102:4ed00f76a576 Date: 2013-11-14 19:00 +0100 http://bitbucket.org/pypy/pypy/changeset/4ed00f76a576/ Log: Merge closed head cd07f4c70232 on branch enumerate-rstr From noreply at buildbot.pypy.org Thu Nov 14 19:03:13 2013 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 14 Nov 2013 19:03:13 +0100 (CET) Subject: [pypy-commit] pypy closed-branches: Merge closed head 3a0298093f20 on branch ndarray-ptp Message-ID: <20131114180313.D311A1C12EC@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: closed-branches Changeset: r68103:63866b92a1ff Date: 2013-11-14 19:00 +0100 http://bitbucket.org/pypy/pypy/changeset/63866b92a1ff/ Log: Merge closed head 3a0298093f20 on branch ndarray-ptp From noreply at buildbot.pypy.org Thu Nov 14 19:03:14 2013 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 14 Nov 2013 19:03:14 +0100 (CET) Subject: [pypy-commit] pypy closed-branches: Merge closed head 8412492e8e81 on branch rlib-doc-rpython-refs Message-ID: <20131114180314.D5FF61C12EC@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: closed-branches Changeset: r68104:fab5aaa5a12b Date: 2013-11-14 19:00 +0100 http://bitbucket.org/pypy/pypy/changeset/fab5aaa5a12b/ Log: Merge closed head 8412492e8e81 on branch rlib-doc-rpython-refs From noreply at buildbot.pypy.org Thu Nov 14 19:03:15 2013 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 14 Nov 2013 19:03:15 +0100 (CET) Subject: [pypy-commit] pypy closed-branches: Merge closed head 8e4f7f8df8e1 on branch curses_fixes Message-ID: <20131114180315.DEAF71C12EC@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: closed-branches Changeset: r68105:fb503a734353 Date: 2013-11-14 19:00 +0100 http://bitbucket.org/pypy/pypy/changeset/fb503a734353/ Log: Merge closed head 8e4f7f8df8e1 on branch curses_fixes From noreply at buildbot.pypy.org Thu Nov 14 19:03:17 2013 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 14 Nov 2013 19:03:17 +0100 (CET) Subject: [pypy-commit] pypy closed-branches: Merge closed head dc20838c51f4 on branch ctypes-byref Message-ID: <20131114180317.1321B1C12EC@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: closed-branches Changeset: r68106:23c332c08fd1 Date: 2013-11-14 19:00 +0100 http://bitbucket.org/pypy/pypy/changeset/23c332c08fd1/ Log: Merge closed head dc20838c51f4 on branch ctypes-byref From noreply at buildbot.pypy.org Thu Nov 14 19:03:18 2013 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 14 Nov 2013 19:03:18 +0100 (CET) Subject: [pypy-commit] pypy closed-branches: Merge closed head 61ce3b8b16b1 on branch statvfs_tests Message-ID: <20131114180318.1AD971C12EC@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: closed-branches Changeset: r68107:2c59a6c70918 Date: 2013-11-14 19:00 +0100 http://bitbucket.org/pypy/pypy/changeset/2c59a6c70918/ Log: Merge closed head 61ce3b8b16b1 on branch statvfs_tests From noreply at buildbot.pypy.org Thu Nov 14 19:03:19 2013 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 14 Nov 2013 19:03:19 +0100 (CET) Subject: [pypy-commit] pypy closed-branches: Merge closed head ea46bf3cc485 on branch fix-jit-logs Message-ID: <20131114180319.2A4961C12EC@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: closed-branches Changeset: r68108:bc5a4505f3cf Date: 2013-11-14 19:00 +0100 http://bitbucket.org/pypy/pypy/changeset/bc5a4505f3cf/ Log: Merge closed head ea46bf3cc485 on branch fix-jit-logs From noreply at buildbot.pypy.org Thu Nov 14 19:03:20 2013 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 14 Nov 2013 19:03:20 +0100 (CET) Subject: [pypy-commit] pypy closed-branches: Merge closed head 3f776b761e09 on branch pycon2013-doc-fixes Message-ID: <20131114180320.2D8691C12EC@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: closed-branches Changeset: r68109:0cc3a7eb47cf Date: 2013-11-14 19:00 +0100 http://bitbucket.org/pypy/pypy/changeset/0cc3a7eb47cf/ Log: Merge closed head 3f776b761e09 on branch pycon2013-doc-fixes From noreply at buildbot.pypy.org Thu Nov 14 19:03:21 2013 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 14 Nov 2013 19:03:21 +0100 (CET) Subject: [pypy-commit] pypy default: Small updates Message-ID: <20131114180321.594031C12EC@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r68110:e5911b74577f Date: 2013-11-14 19:00 +0100 http://bitbucket.org/pypy/pypy/changeset/e5911b74577f/ Log: Small updates diff --git a/pypy/tool/clean_old_branches.py b/pypy/tool/clean_old_branches.py --- a/pypy/tool/clean_old_branches.py +++ b/pypy/tool/clean_old_branches.py @@ -20,8 +20,8 @@ for line in result: if len(line) != 3: raise ValueError("'result' contains: %r" % line) - result = [(head, branch) for (head, branch, extra) in result - if branch not in ['', 'closed-branches'] and 'close' in extra] + result = [(head, branch or 'default') for (head, branch, extra) in result + if branch != 'closed-branches' and 'close=1' in extra] return result @@ -38,7 +38,8 @@ for head, branch in closed_heads: print '\t', head, '\t', branch print -print 'The branches listed above will be merged to "closed-branches".' +print 'The %d branches listed above will be merged to "closed-branches".' % ( + len(closed_heads),) print 'You need to run this script in a clean working copy where you' print 'don''t mind all files being removed.' print From noreply at buildbot.pypy.org Thu Nov 14 19:03:22 2013 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 14 Nov 2013 19:03:22 +0100 (CET) Subject: [pypy-commit] pypy closed-branches: Merge closed head bfcb4d5e68e6 on branch zlib-mem-pressure Message-ID: <20131114180322.5B1001C12EC@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: closed-branches Changeset: r68111:9ab8a8397a95 Date: 2013-11-14 19:00 +0100 http://bitbucket.org/pypy/pypy/changeset/9ab8a8397a95/ Log: Merge closed head bfcb4d5e68e6 on branch zlib-mem-pressure From noreply at buildbot.pypy.org Thu Nov 14 19:03:23 2013 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 14 Nov 2013 19:03:23 +0100 (CET) Subject: [pypy-commit] pypy closed-branches: Merge closed head 18d6db6fccf8 on branch task-decorator Message-ID: <20131114180323.5E38B1C12EC@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: closed-branches Changeset: r68112:aed7cfe91a46 Date: 2013-11-14 19:01 +0100 http://bitbucket.org/pypy/pypy/changeset/aed7cfe91a46/ Log: Merge closed head 18d6db6fccf8 on branch task-decorator From noreply at buildbot.pypy.org Thu Nov 14 19:03:24 2013 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 14 Nov 2013 19:03:24 +0100 (CET) Subject: [pypy-commit] pypy closed-branches: Merge closed head d7746d32bf9d on branch py3k-struct Message-ID: <20131114180324.5A9C71C12EC@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: closed-branches Changeset: r68113:2ee19ed8f81f Date: 2013-11-14 19:01 +0100 http://bitbucket.org/pypy/pypy/changeset/2ee19ed8f81f/ Log: Merge closed head d7746d32bf9d on branch py3k-struct From noreply at buildbot.pypy.org Thu Nov 14 19:03:25 2013 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 14 Nov 2013 19:03:25 +0100 (CET) Subject: [pypy-commit] pypy closed-branches: Merge closed head 3b448254587e on branch py3k-subprocess-new-session Message-ID: <20131114180325.5CA951C12EC@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: closed-branches Changeset: r68114:6230319bfb3f Date: 2013-11-14 19:01 +0100 http://bitbucket.org/pypy/pypy/changeset/6230319bfb3f/ Log: Merge closed head 3b448254587e on branch py3k-subprocess-new-session From noreply at buildbot.pypy.org Thu Nov 14 19:03:26 2013 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 14 Nov 2013 19:03:26 +0100 (CET) Subject: [pypy-commit] pypy closed-branches: Merge closed head 56a9bd9b2769 on branch pythoninspect-fix Message-ID: <20131114180326.645CF1C12EC@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: closed-branches Changeset: r68115:88350d86996d Date: 2013-11-14 19:01 +0100 http://bitbucket.org/pypy/pypy/changeset/88350d86996d/ Log: Merge closed head 56a9bd9b2769 on branch pythoninspect-fix From noreply at buildbot.pypy.org Thu Nov 14 19:03:27 2013 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 14 Nov 2013 19:03:27 +0100 (CET) Subject: [pypy-commit] pypy closed-branches: Merge closed head 659f78e9b5b6 on branch py3k-list-compr-or Message-ID: <20131114180327.77E891C12EC@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: closed-branches Changeset: r68116:5fe418c3c217 Date: 2013-11-14 19:01 +0100 http://bitbucket.org/pypy/pypy/changeset/5fe418c3c217/ Log: Merge closed head 659f78e9b5b6 on branch py3k-list-compr-or From noreply at buildbot.pypy.org Thu Nov 14 19:03:28 2013 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 14 Nov 2013 19:03:28 +0100 (CET) Subject: [pypy-commit] pypy closed-branches: Merge closed head 83f0ba55bc1b on branch bridge-logging Message-ID: <20131114180328.C95AD1C12EC@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: closed-branches Changeset: r68117:7f78cebd672e Date: 2013-11-14 19:01 +0100 http://bitbucket.org/pypy/pypy/changeset/7f78cebd672e/ Log: Merge closed head 83f0ba55bc1b on branch bridge-logging From noreply at buildbot.pypy.org Thu Nov 14 19:03:29 2013 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 14 Nov 2013 19:03:29 +0100 (CET) Subject: [pypy-commit] pypy closed-branches: Merge closed head 800741417787 on branch ssl_moving_write_buffer Message-ID: <20131114180329.CB21F1C12EC@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: closed-branches Changeset: r68118:582320f11d88 Date: 2013-11-14 19:01 +0100 http://bitbucket.org/pypy/pypy/changeset/582320f11d88/ Log: Merge closed head 800741417787 on branch ssl_moving_write_buffer From noreply at buildbot.pypy.org Thu Nov 14 19:03:30 2013 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 14 Nov 2013 19:03:30 +0100 (CET) Subject: [pypy-commit] pypy closed-branches: Merge closed head a1f52d6ae34e on branch improved_ebnfparse_error Message-ID: <20131114180330.D1C8A1C12EC@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: closed-branches Changeset: r68119:b72f0aea3f18 Date: 2013-11-14 19:01 +0100 http://bitbucket.org/pypy/pypy/changeset/b72f0aea3f18/ Log: Merge closed head a1f52d6ae34e on branch improved_ebnfparse_error From noreply at buildbot.pypy.org Thu Nov 14 19:03:31 2013 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 14 Nov 2013 19:03:31 +0100 (CET) Subject: [pypy-commit] pypy closed-branches: Merge closed head 87a5f161be81 on branch cpyext-PyThreadState_New Message-ID: <20131114180331.D05BF1C12EC@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: closed-branches Changeset: r68120:c91c7524a1c9 Date: 2013-11-14 19:01 +0100 http://bitbucket.org/pypy/pypy/changeset/c91c7524a1c9/ Log: Merge closed head 87a5f161be81 on branch cpyext-PyThreadState_New From noreply at buildbot.pypy.org Thu Nov 14 19:03:32 2013 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 14 Nov 2013 19:03:32 +0100 (CET) Subject: [pypy-commit] pypy closed-branches: re-close this branch Message-ID: <20131114180332.D3CC61C12EC@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: closed-branches Changeset: r68121:58cc9b35e02f Date: 2013-11-14 19:01 +0100 http://bitbucket.org/pypy/pypy/changeset/58cc9b35e02f/ Log: re-close this branch From noreply at buildbot.pypy.org Thu Nov 14 21:21:41 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Thu, 14 Nov 2013 21:21:41 +0100 (CET) Subject: [pypy-commit] pypy default: rcomplex.c_pow does this Message-ID: <20131114202141.260A51C025A@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r68122:1e79df0ae309 Date: 2013-11-14 15:15 -0500 http://bitbucket.org/pypy/pypy/changeset/1e79df0ae309/ Log: rcomplex.c_pow does this diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py --- a/pypy/module/micronumpy/types.py +++ b/pypy/module/micronumpy/types.py @@ -1260,10 +1260,6 @@ @complex_binary_op def pow(self, v1, v2): - if v1[1] == 0 and v2[1] == 0 and v1[0] > 0: - return math.pow(v1[0], v2[0]), 0 - #if not rfloat.isfinite(v1[0]) or not rfloat.isfinite(v1[1]): - # return rfloat.NAN, rfloat.NAN try: return rcomplex.c_pow(v1, v2) except ZeroDivisionError: From noreply at buildbot.pypy.org Thu Nov 14 22:09:50 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Thu, 14 Nov 2013 22:09:50 +0100 (CET) Subject: [pypy-commit] pypy default: simplify by using newbyteorder here Message-ID: <20131114210950.A42191C025A@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r68123:9c0fabe8214b Date: 2013-11-14 16:09 -0500 http://bitbucket.org/pypy/pypy/changeset/9c0fabe8214b/ Log: simplify by using newbyteorder here diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py --- a/pypy/module/micronumpy/interp_numarray.py +++ b/pypy/module/micronumpy/interp_numarray.py @@ -519,13 +519,7 @@ # by converting nonnative byte order. if self.is_scalar(): return space.wrap(0) - if not self.get_dtype().is_flexible_type(): - s = self.get_dtype().name - if not self.get_dtype().is_native(): - s = s[1:] - dtype = interp_dtype.get_dtype_cache(space).dtypes_by_name[s] - else: - dtype = self.get_dtype() + dtype = self.get_dtype().descr_newbyteorder(space, NPY_NATIVE) contig = self.implementation.astype(space, dtype) return contig.argsort(space, w_axis) From noreply at buildbot.pypy.org Thu Nov 14 23:20:27 2013 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 14 Nov 2013 23:20:27 +0100 (CET) Subject: [pypy-commit] pypy default: Rewrite the introduction, linking to SECCOMP and to Victor Stinner's Message-ID: <20131114222027.AB4BE1C025A@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r68124:471b407d8d5b Date: 2013-11-14 23:19 +0100 http://bitbucket.org/pypy/pypy/changeset/471b407d8d5b/ Log: Rewrite the introduction, linking to SECCOMP and to Victor Stinner's mail on python-dev about pysandbox. diff --git a/pypy/doc/sandbox.rst b/pypy/doc/sandbox.rst --- a/pypy/doc/sandbox.rst +++ b/pypy/doc/sandbox.rst @@ -4,26 +4,36 @@ Introduction ------------ -It is possible to compile a version of pypy-c that runs -fully "virtualized", i.e. where an external process controls all -input/output. Such a pypy-c is a secure sandbox: it is safe to run -any untrusted Python code with it. The Python code cannot see or -modify any local file except via interaction with the external -process. It is also impossible to do any other I/O or consume more -than some amount of RAM or CPU time or real time. This works with no -OS support at all - just ANSI C code generated in a careful way. It's -the kind of thing you could embed in a browser plug-in, for example -(it would be safe even if it wasn't run as a separate process, -actually). +PyPy offers sandboxing at a level similar to OS-level sandboxing (e.g. +SECCOMP_ on Linux), but implemented in a fully portable way. To use it, +a (regular, trusted) program launches a subprocess that is a special +sandboxed version of PyPy. This subprocess can run arbitrary untrusted +Python code, but all its input/output is serialized to a stdin/stdout +pipe instead of being directly performed. The outer process reads the +pipe and decides which commands are allowed or not (sandboxing), or even +reinterprets them differently (virtualization). A potential attacker +can have arbitrary code run in the subprocess, but cannot actually do +any input/output not controlled by the outer process. Additional +barriers are put to limit the amount of RAM and CPU time used. -For comparison, trying to plug CPython into a special virtualizing C -library is not only OS-specific, but unsafe, because one of the known -ways to segfault CPython could be used by an attacker to trick CPython -into issuing malicious system calls directly. The C code generated by +Note that this is very different from sandboxing at the Python language +level, i.e. placing restrictions on what kind of Python code the +attacker is allowed to run (why? read about pysandbox_). + +.. _SECCOMP: http://code.google.com/p/seccompsandbox/wiki/overview +.. _pysandbox: https://mail.python.org/pipermail/python-dev/2013-November/130132.html + +Another point of comparison: if we were instead to try to plug CPython +into a special virtualizing C library, we would get a result +that is not only OS-specific, but unsafe, because CPython can be +segfaulted (in many ways, all of them really, really obscure). +Given enough efforts, an attacker can turn almost any +segfault into a vulnerability. The C code generated by PyPy is not segfaultable, as long as our code generators are correct - -that's a lower number of lines of code to trust. For the paranoid, in -this case we also generate systematic run-time checks against buffer -overflows. +that's a lower number of lines of code to trust. For the paranoid, +PyPy translated with sandboxing also contains systematic run-time +checks (against buffer overflows for example) +that are normally only present in debugging versions. .. warning:: From noreply at buildbot.pypy.org Fri Nov 15 01:58:38 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Fri, 15 Nov 2013 01:58:38 +0100 (CET) Subject: [pypy-commit] pypy default: put getitem/setitem/fill on array, not dtype Message-ID: <20131115005838.3764E1C030D@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r68125:bdb4b5232832 Date: 2013-11-14 19:44 -0500 http://bitbucket.org/pypy/pypy/changeset/bdb4b5232832/ Log: put getitem/setitem/fill on array, not dtype diff --git a/pypy/module/micronumpy/arrayimpl/concrete.py b/pypy/module/micronumpy/arrayimpl/concrete.py --- a/pypy/module/micronumpy/arrayimpl/concrete.py +++ b/pypy/module/micronumpy/arrayimpl/concrete.py @@ -36,10 +36,13 @@ return backstrides def getitem(self, index): - return self.dtype.getitem(self, index) + return self.dtype.itemtype.read(self, index, 0) + + def getitem_bool(self, index): + return self.dtype.itemtype.read_bool(self, index, 0) def setitem(self, index, value): - self.dtype.setitem(self, index, value) + self.dtype.itemtype.store(self, index, 0, value) def setslice(self, space, arr): impl = arr.implementation @@ -268,7 +271,7 @@ def create_dot_iter(self, shape, skip): r = calculate_dot_strides(self.get_strides(), self.get_backstrides(), shape, skip) - return iter.MultiDimViewIterator(self, self.dtype, self.start, r[0], r[1], shape) + return iter.MultiDimViewIterator(self, self.start, r[0], r[1], shape) def swapaxes(self, space, orig_arr, axis1, axis2): shape = self.get_shape()[:] @@ -331,21 +334,24 @@ support.product(shape) > support.product(self.get_shape()): r = calculate_broadcast_strides(self.get_strides(), self.get_backstrides(), - self.get_shape(), shape, backward_broadcast) - return iter.MultiDimViewIterator(self, self.dtype, self.start, r[0], r[1], shape) - + self.get_shape(), shape, + backward_broadcast) + return iter.MultiDimViewIterator(self, self.start, + r[0], r[1], shape) if not require_index: return iter.ConcreteArrayIterator(self) - else: - if len(self.get_shape()) == 1: - return iter.OneDimViewIterator(self, self.dtype, self.start, - self.get_strides(), self.get_shape()) - else: - return iter.MultiDimViewIterator(self, self.dtype, self.start, - self.get_strides(), self.get_backstrides(), self.get_shape()) + if len(self.get_shape()) == 1: + return iter.OneDimViewIterator(self, self.start, + self.get_strides(), + self.get_shape()) + return iter.MultiDimViewIterator(self, self.start, + self.get_strides(), + self.get_backstrides(), + self.get_shape()) def fill(self, box): - self.dtype.fill(self.storage, box, 0, self.size) + self.dtype.itemtype.fill(self.storage, self.dtype.get_size(), + box, 0, self.size, 0) def set_shape(self, space, orig_array, new_shape): strides, backstrides = support.calc_strides(new_shape, self.dtype, @@ -416,14 +422,16 @@ self.get_backstrides(), self.get_shape(), shape, backward_broadcast) - return iter.MultiDimViewIterator(self.parent, self.dtype, - self.start, r[0], r[1], shape) + return iter.MultiDimViewIterator(self, self.start, + r[0], r[1], shape) if len(self.get_shape()) == 1: - return iter.OneDimViewIterator(self.parent, self.dtype, self.start, - self.get_strides(), self.get_shape()) - return iter.MultiDimViewIterator(self.parent, self.dtype, self.start, - self.get_strides(), - self.get_backstrides(), self.get_shape()) + return iter.OneDimViewIterator(self, self.start, + self.get_strides(), + self.get_shape()) + return iter.MultiDimViewIterator(self, self.start, + self.get_strides(), + self.get_backstrides(), + self.get_shape()) def set_shape(self, space, orig_array, new_shape): if len(self.get_shape()) < 2 or self.size == 0: diff --git a/pypy/module/micronumpy/interp_dtype.py b/pypy/module/micronumpy/interp_dtype.py --- a/pypy/module/micronumpy/interp_dtype.py +++ b/pypy/module/micronumpy/interp_dtype.py @@ -81,19 +81,6 @@ def coerce(self, space, w_item): return self.itemtype.coerce(space, self, w_item) - def getitem(self, arr, i): - item = self.itemtype.read(arr, i, 0) - return item - - def getitem_bool(self, arr, i): - return self.itemtype.read_bool(arr, i, 0) - - def setitem(self, arr, i, box): - self.itemtype.store(arr, i, 0, box) - - def fill(self, storage, box, start, stop): - self.itemtype.fill(storage, self.get_size(), box, start, stop, 0) - def is_int_type(self): return (self.kind == NPY_SIGNEDLTR or self.kind == NPY_UNSIGNEDLTR or self.kind == NPY_GENBOOLLTR) diff --git a/pypy/module/micronumpy/iter.py b/pypy/module/micronumpy/iter.py --- a/pypy/module/micronumpy/iter.py +++ b/pypy/module/micronumpy/iter.py @@ -159,23 +159,21 @@ return [space.wrap(self.indexes[i]) for i in range(shapelen)] class ConcreteArrayIterator(base.BaseArrayIterator): - _immutable_fields_ = ['dtype', 'skip', 'size'] + _immutable_fields_ = ['array', 'skip', 'size'] def __init__(self, array): self.array = array self.offset = 0 - self.dtype = array.dtype - self.skip = self.dtype.get_size() + self.skip = array.dtype.get_size() self.size = array.size def setitem(self, elem): - self.dtype.setitem(self.array, self.offset, elem) + self.array.setitem(self.offset, elem) def getitem(self): - item = self.dtype.getitem(self.array, self.offset) - return item + return self.array.getitem(self.offset) def getitem_bool(self): - return self.dtype.getitem_bool(self.array, self.offset) + return self.array.getitem_bool(self.offset) def next(self): self.offset += self.skip @@ -190,12 +188,8 @@ self.offset %= self.size class OneDimViewIterator(ConcreteArrayIterator): - ''' The view iterator dtype can be different from the - array.dtype, this is what makes it a View - ''' - def __init__(self, array, dtype, start, strides, shape): + def __init__(self, array, start, strides, shape): self.array = array - self.dtype = dtype self.offset = start self.skip = strides[0] self.index = 0 @@ -219,13 +213,9 @@ return self.index class MultiDimViewIterator(ConcreteArrayIterator): - ''' The view iterator dtype can be different from the - array.dtype, this is what makes it a View - ''' - def __init__(self, array, dtype, start, strides, backstrides, shape): + def __init__(self, array, start, strides, backstrides, shape): self.indexes = [0] * len(shape) self.array = array - self.dtype = dtype self.shape = shape self.offset = start self.shapelen = len(shape) @@ -295,14 +285,12 @@ self.offset = array.start self.dim = dim self.array = array - self.dtype = array.dtype def setitem(self, elem): - self.dtype.setitem(self.array, self.offset, elem) + self.array.setitem(self.offset, elem) def getitem(self): - item = self.dtype.getitem(self.array, self.offset) - return item + return self.array.getitem(self.offset) @jit.unroll_safe def next(self): diff --git a/pypy/module/micronumpy/test/test_iter.py b/pypy/module/micronumpy/test/test_iter.py --- a/pypy/module/micronumpy/test/test_iter.py +++ b/pypy/module/micronumpy/test/test_iter.py @@ -13,7 +13,7 @@ strides = [5, 1] backstrides = [x * (y - 1) for x,y in zip(strides, shape)] assert backstrides == [10, 4] - i = MultiDimViewIterator(MockArray, None, start, strides, backstrides, shape) + i = MultiDimViewIterator(MockArray, start, strides, backstrides, shape) i.next() i.next() i.next() @@ -31,7 +31,7 @@ strides = [1, 3] backstrides = [x * (y - 1) for x,y in zip(strides, shape)] assert backstrides == [2, 12] - i = MultiDimViewIterator(MockArray, None, start, strides, backstrides, shape) + i = MultiDimViewIterator(MockArray, start, strides, backstrides, shape) i.next() i.next() i.next() @@ -52,7 +52,7 @@ strides = [5, 1] backstrides = [x * (y - 1) for x,y in zip(strides, shape)] assert backstrides == [10, 4] - i = MultiDimViewIterator(MockArray, None, start, strides, backstrides, shape) + i = MultiDimViewIterator(MockArray, start, strides, backstrides, shape) i.next_skip_x(2) i.next_skip_x(2) i.next_skip_x(2) @@ -75,7 +75,7 @@ strides = [1, 3] backstrides = [x * (y - 1) for x,y in zip(strides, shape)] assert backstrides == [2, 12] - i = MultiDimViewIterator(MockArray, None, start, strides, backstrides, shape) + i = MultiDimViewIterator(MockArray, start, strides, backstrides, shape) i.next_skip_x(2) i.next_skip_x(2) i.next_skip_x(2) From noreply at buildbot.pypy.org Fri Nov 15 02:13:02 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Fri, 15 Nov 2013 02:13:02 +0100 (CET) Subject: [pypy-commit] pypy default: test/fix int/trunc behavior Message-ID: <20131115011302.78FDE1C030D@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r68126:6f5e3c868dc6 Date: 2013-11-14 20:12 -0500 http://bitbucket.org/pypy/pypy/changeset/6f5e3c868dc6/ Log: test/fix int/trunc behavior diff --git a/pypy/objspace/std/inttype.py b/pypy/objspace/std/inttype.py --- a/pypy/objspace/std/inttype.py +++ b/pypy/objspace/std/inttype.py @@ -119,14 +119,14 @@ if not ok: # otherwise, use the __int__() or the __trunc__() methods w_obj = w_value - if space.lookup(w_obj, '__int__') is not None: - w_obj = space.int(w_obj) - elif space.lookup(w_obj, '__trunc__') is not None: - w_obj = space.trunc(w_obj) - else: - raise operationerrfmt(space.w_TypeError, - "int() argument must be a string or a number, not '%T'", - w_obj) + if space.lookup(w_obj, '__int__') is None: + if space.lookup(w_obj, '__trunc__') is not None: + w_obj = space.trunc(w_obj) + else: + raise operationerrfmt(space.w_TypeError, + "int() argument must be a string or a number, not '%T'", + w_obj) + w_obj = space.int(w_obj) # 'int(x)' should return what x.__int__() returned, which should # be an int or long or a subclass thereof. if space.is_w(w_inttype, space.w_int): diff --git a/pypy/objspace/std/test/test_intobject.py b/pypy/objspace/std/test/test_intobject.py --- a/pypy/objspace/std/test/test_intobject.py +++ b/pypy/objspace/std/test/test_intobject.py @@ -7,7 +7,6 @@ class TestW_IntObject: - def _longshiftresult(self, x): """ calculate an overflowing shift """ n = 1 @@ -40,7 +39,7 @@ space = self.space assert isinstance(space.bigint_w(space.wrap(42)), rbigint) assert space.bigint_w(space.wrap(42)).eq(rbigint.fromint(42)) - + def test_repr(self): x = 1 f1 = iobj.W_IntObject(x) @@ -71,7 +70,7 @@ method = getattr(iobj, '%s__Int_Int' % op) myres = method(self.space, wx, wy) assert self.space.unwrap(myres) == res - + def test_add(self): x = 1 y = 2 @@ -283,8 +282,8 @@ result = iobj.hex__Int(self.space, f1) assert self.space.unwrap(result) == hex(x) + class AppTestInt: - def test_conjugate(self): assert (1).conjugate() == 1 assert (-1).conjugate() == -1 @@ -326,7 +325,7 @@ assert "42" == str(42) assert "42" == repr(42) raises(ValueError, int, '0x2A') - + def test_int_two_param(self): assert 42 == int('0x2A', 0) assert 42 == int('2A', 16) @@ -431,28 +430,28 @@ def test_special_int(self): class a(object): - def __int__(self): - self.ar = True + def __int__(self): + self.ar = True return None inst = a() - raises(TypeError, int, inst) + raises(TypeError, int, inst) assert inst.ar == True - class b(object): + class b(object): pass raises((AttributeError,TypeError), int, b()) def test_special_long(self): class a(object): - def __long__(self): - self.ar = True + def __long__(self): + self.ar = True return None inst = a() - raises(TypeError, long, inst) - assert inst.ar == True + raises(TypeError, long, inst) + assert inst.ar == True - class b(object): - pass + class b(object): + pass raises((AttributeError,TypeError), long, b()) def test_just_trunc(self): @@ -470,6 +469,15 @@ pass assert int(myotherint(21)) == 21 + def test_trunc_returns_non_int(self): + class Integral(object): + def __int__(self): + return 42 + class TruncReturnsNonInt(object): + def __trunc__(self): + return Integral() + assert int(TruncReturnsNonInt()) == 42 + def test_getnewargs(self): assert 0 .__getnewargs__() == (0,) diff --git a/pypy/objspace/std/test/test_longobject.py b/pypy/objspace/std/test/test_longobject.py --- a/pypy/objspace/std/test/test_longobject.py +++ b/pypy/objspace/std/test/test_longobject.py @@ -41,7 +41,6 @@ class AppTestLong: - def test_trunc(self): import math assert math.trunc(1L) == 1L @@ -311,7 +310,6 @@ assert (-1<<40).bit_length() == 41 assert ((2**31)-1).bit_length() == 31 - def test_negative_zero(self): x = eval("-0L") assert x == 0L From noreply at buildbot.pypy.org Fri Nov 15 02:28:17 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Fri, 15 Nov 2013 02:28:17 +0100 (CET) Subject: [pypy-commit] pypy default: cleanup Message-ID: <20131115012817.EDB6F1C030D@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r68127:08c50e623f5f Date: 2013-11-14 20:25 -0500 http://bitbucket.org/pypy/pypy/changeset/08c50e623f5f/ Log: cleanup diff --git a/pypy/module/micronumpy/interp_dtype.py b/pypy/module/micronumpy/interp_dtype.py --- a/pypy/module/micronumpy/interp_dtype.py +++ b/pypy/module/micronumpy/interp_dtype.py @@ -3,7 +3,7 @@ from pypy.interpreter.error import OperationError, operationerrfmt from pypy.interpreter.gateway import interp2app, unwrap_spec from pypy.interpreter.typedef import (TypeDef, GetSetProperty, - interp_attrproperty, interp_attrproperty_w) + interp_attrproperty, interp_attrproperty_w) from pypy.module.micronumpy import types, interp_boxes, base from rpython.rlib.objectmodel import specialize from rpython.rlib.rarithmetic import LONG_BIT, r_longlong, r_ulonglong @@ -16,8 +16,9 @@ def decode_w_dtype(space, w_dtype): if space.is_none(w_dtype): return None - return space.interp_w(W_Dtype, - space.call_function(space.gettypefor(W_Dtype), w_dtype)) + return space.interp_w( + W_Dtype, space.call_function(space.gettypefor(W_Dtype), w_dtype)) + @jit.unroll_safe def dtype_agreement(space, w_arr_list, shape, out=None): @@ -34,6 +35,7 @@ out = base.W_NDimArray.from_shape(space, shape, dtype) return out + class W_Dtype(W_Root): _immutable_fields_ = ["itemtype?", "num", "kind", "name?", "char", "w_box_type", "byteorder", "size?", "float_type", @@ -92,7 +94,7 @@ return self.kind == NPY_COMPLEXLTR def is_float_type(self): - return (self.kind == NPY_FLOATINGLTR or self.float_type is not None) + return self.kind == NPY_FLOATINGLTR or self.kind == NPY_COMPLEXLTR def is_bool_type(self): return self.kind == NPY_GENBOOLLTR @@ -311,6 +313,7 @@ return W_Dtype(itemtype, self.num, self.kind, self.name, self.char, self.w_box_type, endian, size=self.size) + def dtype_from_list(space, w_lst): lst_w = space.listview(w_lst) fields = {} @@ -322,7 +325,7 @@ if space.len_w(w_elem) == 3: w_fldname, w_flddesc, w_shape = space.fixedview(w_elem) if not base.issequence_w(space, w_shape): - w_shape = space.newtuple([w_shape,]) + w_shape = space.newtuple([w_shape]) else: w_fldname, w_flddesc = space.fixedview(w_elem, 2) subdtype = descr__new__(space, space.gettypefor(W_Dtype), w_flddesc, w_shape=w_shape) @@ -339,14 +342,17 @@ NPY_VOIDLTR, space.gettypefor(interp_boxes.W_VoidBox), fields=fields, fieldnames=fieldnames, size=offset) + def dtype_from_dict(space, w_dict): raise OperationError(space.w_NotImplementedError, space.wrap( "dtype from dict")) + def dtype_from_spec(space, name): raise OperationError(space.w_NotImplementedError, space.wrap( "dtype from spec")) + def descr__new__(space, w_subtype, w_dtype, w_align=None, w_copy=None, w_shape=None): # w_align and w_copy are necessary for pickling cache = get_dtype_cache(space) @@ -384,7 +390,7 @@ if name[0] in 'VSUc' or name[0] in '<>=' and name[1] in 'VSUc': return variable_dtype(space, name) raise OperationError(space.w_TypeError, space.wrap( - "data type %s not understood" % name)) + "data type %s not understood" % name)) elif space.isinstance_w(w_dtype, space.w_list): return dtype_from_list(space, w_dtype) elif space.isinstance_w(w_dtype, space.w_tuple): @@ -478,6 +484,7 @@ basename + str(8 * size * itemtype.get_element_size()), char, w_box_type, size=size) + def new_string_dtype(space, size): itemtype = types.StringType() return W_Dtype( @@ -490,6 +497,7 @@ w_box_type = space.gettypefor(interp_boxes.W_StringBox), ) + def new_unicode_dtype(space, size): itemtype = types.UnicodeType() return W_Dtype( @@ -558,7 +566,7 @@ name="int32", char=NPY_INTLTR, w_box_type=space.gettypefor(interp_boxes.W_Int32Box), - ) + ) self.w_uint32dtype = W_Dtype( types.UInt32(), num=NPY_UINT, @@ -577,7 +585,7 @@ alternate_constructors=[space.w_int, space.gettypefor(interp_boxes.W_IntegerBox), space.gettypefor(interp_boxes.W_SignedIntegerBox), - ], + ], aliases=['int'], ) self.w_ulongdtype = W_Dtype( @@ -587,8 +595,8 @@ name="uint%d" % LONG_BIT, char=NPY_ULONGLTR, w_box_type=space.gettypefor(interp_boxes.W_ULongBox), - alternate_constructors=[ space.gettypefor(interp_boxes.W_UnsignedIntegerBox), - ], + alternate_constructors=[space.gettypefor(interp_boxes.W_UnsignedIntegerBox), + ], aliases=['uint'], ) self.w_int64dtype = W_Dtype( @@ -628,7 +636,7 @@ w_box_type = space.gettypefor(interp_boxes.W_Float64Box), alternate_constructors=[space.w_float, space.gettypefor(interp_boxes.W_NumberBox), - ], + ], aliases=["float", "double"], ) self.w_floatlongdtype = W_Dtype( @@ -779,18 +787,18 @@ new_name = NPY_OPPBYTE + can_name itemtype = type(dtype.itemtype)(False) self.dtypes_by_name[new_name] = W_Dtype( - itemtype, - dtype.num, dtype.kind, new_name, dtype.char, dtype.w_box_type, - byteorder=NPY_OPPBYTE, float_type=dtype.float_type) + itemtype, dtype.num, dtype.kind, new_name, dtype.char, + dtype.w_box_type, byteorder=NPY_OPPBYTE, + float_type=dtype.float_type) if dtype.kind != dtype.char: can_name = dtype.char self.dtypes_by_name[NPY_NATBYTE + can_name] = dtype self.dtypes_by_name[NPY_NATIVE + can_name] = dtype new_name = NPY_OPPBYTE + can_name self.dtypes_by_name[new_name] = W_Dtype( - itemtype, - dtype.num, dtype.kind, new_name, dtype.char, dtype.w_box_type, - byteorder=NPY_OPPBYTE, float_type=dtype.float_type) + itemtype, dtype.num, dtype.kind, new_name, dtype.char, + dtype.w_box_type, byteorder=NPY_OPPBYTE, + float_type=dtype.float_type) for alias in dtype.aliases: self.dtypes_by_name[alias] = dtype @@ -845,9 +853,11 @@ itemsize = dtype.get_size() items_w = [space.wrap(dtype.char), space.wrap(dtype.num), - space.wrap(itemsize * 8), # in case of changing + space.wrap(itemsize * 8), # in case of changing # number of bits per byte in the future - space.wrap(itemsize / (2 if dtype.kind == NPY_COMPLEXLTR else 1) or 1)] + space.wrap(itemsize / + (2 if dtype.kind == NPY_COMPLEXLTR else 1) + or 1)] if dtype.is_int_type(): if dtype.kind == NPY_GENBOOLLTR: w_maxobj = space.wrap(1) @@ -864,5 +874,6 @@ space.setitem(w_typeinfo, space.wrap(k), space.newtuple(items_w)) self.w_typeinfo = w_typeinfo + def get_dtype_cache(space): return space.fromcache(DtypeCache) From noreply at buildbot.pypy.org Fri Nov 15 07:05:42 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Fri, 15 Nov 2013 07:05:42 +0100 (CET) Subject: [pypy-commit] pypy numpy-andrew-tests: close obsolete branch Message-ID: <20131115060542.A707E1D2330@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: numpy-andrew-tests Changeset: r68128:ef46b0734b9c Date: 2013-11-14 20:34 -0500 http://bitbucket.org/pypy/pypy/changeset/ef46b0734b9c/ Log: close obsolete branch From noreply at buildbot.pypy.org Fri Nov 15 07:05:43 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Fri, 15 Nov 2013 07:05:43 +0100 (CET) Subject: [pypy-commit] pypy default: provide dtype.hasobject Message-ID: <20131115060543.CEC9D1D2332@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r68129:bf64e1985de3 Date: 2013-11-15 01:05 -0500 http://bitbucket.org/pypy/pypy/changeset/bf64e1985de3/ Log: provide dtype.hasobject diff --git a/pypy/module/micronumpy/interp_dtype.py b/pypy/module/micronumpy/interp_dtype.py --- a/pypy/module/micronumpy/interp_dtype.py +++ b/pypy/module/micronumpy/interp_dtype.py @@ -224,6 +224,9 @@ raise break + def descr_get_hasobject(self, space): + return space.w_False + def descr_getitem(self, space, w_item): if self.fields is None: raise OperationError(space.w_KeyError, space.wrap( @@ -443,6 +446,7 @@ isnative = GetSetProperty(W_Dtype.descr_get_isnative), fields = GetSetProperty(W_Dtype.descr_get_fields), names = GetSetProperty(W_Dtype.descr_get_names), + hasobject = GetSetProperty(W_Dtype.descr_get_hasobject), ) W_Dtype.typedef.acceptable_as_base_class = False diff --git a/pypy/module/micronumpy/test/test_dtypes.py b/pypy/module/micronumpy/test/test_dtypes.py --- a/pypy/module/micronumpy/test/test_dtypes.py +++ b/pypy/module/micronumpy/test/test_dtypes.py @@ -32,6 +32,7 @@ assert dtype('int8').name == 'int8' assert dtype(int).fields is None assert dtype(int).names is None + assert dtype(int).hasobject is False assert dtype(None) is dtype(float) From noreply at buildbot.pypy.org Fri Nov 15 07:43:21 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Fri, 15 Nov 2013 07:43:21 +0100 (CET) Subject: [pypy-commit] pypy default: fix/enable this dtype test Message-ID: <20131115064321.67F2F1C010D@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r68130:df81eae287fa Date: 2013-11-15 01:19 -0500 http://bitbucket.org/pypy/pypy/changeset/df81eae287fa/ Log: fix/enable this dtype test diff --git a/pypy/module/micronumpy/interp_dtype.py b/pypy/module/micronumpy/interp_dtype.py --- a/pypy/module/micronumpy/interp_dtype.py +++ b/pypy/module/micronumpy/interp_dtype.py @@ -143,7 +143,7 @@ if basic == NPY_UNICODELTR: size >>= 2 endian = NPY_NATBYTE - elif size <= 1: + elif size // (self.size or 1) <= 1: endian = NPY_IGNORE else: endian = self.byteorder diff --git a/pypy/module/micronumpy/test/test_dtypes.py b/pypy/module/micronumpy/test/test_dtypes.py --- a/pypy/module/micronumpy/test/test_dtypes.py +++ b/pypy/module/micronumpy/test/test_dtypes.py @@ -766,8 +766,8 @@ # strange assert dtype('string').str == '|S0' assert dtype('unicode').str == byteorder + 'U0' - #assert dtype(('string', 7)).str == '|S7' - #assert dtype(('unicode', 7)).str == ' Author: Brian Kearns Branch: Changeset: r68131:21243450f85a Date: 2013-11-15 01:22 -0500 http://bitbucket.org/pypy/pypy/changeset/21243450f85a/ Log: support dtype('|S#') diff --git a/pypy/module/micronumpy/interp_dtype.py b/pypy/module/micronumpy/interp_dtype.py --- a/pypy/module/micronumpy/interp_dtype.py +++ b/pypy/module/micronumpy/interp_dtype.py @@ -390,7 +390,7 @@ return cache.dtypes_by_name[name] except KeyError: pass - if name[0] in 'VSUc' or name[0] in '<>=' and name[1] in 'VSUc': + if name[0] in 'VSUc' or name[0] in '<>=|' and name[1] in 'VSUc': return variable_dtype(space, name) raise OperationError(space.w_TypeError, space.wrap( "data type %s not understood" % name)) @@ -452,7 +452,7 @@ def variable_dtype(space, name): - if name[0] in '<>=': + if name[0] in '<>=|': name = name[1:] char = name[0] if len(name) == 1: diff --git a/pypy/module/micronumpy/test/test_dtypes.py b/pypy/module/micronumpy/test/test_dtypes.py --- a/pypy/module/micronumpy/test/test_dtypes.py +++ b/pypy/module/micronumpy/test/test_dtypes.py @@ -828,13 +828,14 @@ from numpypy import dtype, str_ raises(TypeError, "dtype('Sx')") - d = dtype('S8') - assert d.itemsize == 8 - assert dtype(str) == dtype('S') - assert d.kind == 'S' - assert d.type is str_ - assert d.name == "string64" - assert d.num == 18 + for t in ['S8', '|S8', '=S8']: + d = dtype(t) + assert d.itemsize == 8 + assert dtype(str) == dtype('S') + assert d.kind == 'S' + assert d.type is str_ + assert d.name == "string64" + assert d.num == 18 for i in [1, 2, 3]: d = dtype('c%d' % i) assert d.itemsize == 1 From noreply at buildbot.pypy.org Fri Nov 15 07:43:23 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Fri, 15 Nov 2013 07:43:23 +0100 (CET) Subject: [pypy-commit] pypy default: fix an array scalar indexing case Message-ID: <20131115064323.D81141C010D@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r68132:b6ff9f5a0ee0 Date: 2013-11-15 01:39 -0500 http://bitbucket.org/pypy/pypy/changeset/b6ff9f5a0ee0/ Log: fix an array scalar indexing case diff --git a/pypy/module/micronumpy/arrayimpl/scalar.py b/pypy/module/micronumpy/arrayimpl/scalar.py --- a/pypy/module/micronumpy/arrayimpl/scalar.py +++ b/pypy/module/micronumpy/arrayimpl/scalar.py @@ -123,6 +123,9 @@ ) def descr_getitem(self, space, _, w_idx): + if space.isinstance_w(w_idx, space.w_tuple): + if space.len_w(w_idx) == 0: + return self.get_scalar_value() raise OperationError(space.w_IndexError, space.wrap("scalars cannot be indexed")) diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py --- a/pypy/module/micronumpy/test/test_numarray.py +++ b/pypy/module/micronumpy/test/test_numarray.py @@ -661,13 +661,16 @@ assert (b[newaxis] == [[2, 3, 4]]).all() def test_scalar(self): - from numpypy import array, dtype + from numpypy import array, dtype, int64 a = array(3) raises(IndexError, "a[0]") raises(IndexError, "a[0] = 5") assert a.size == 1 assert a.shape == () assert a.dtype is dtype(int) + b = a[()] + assert type(b) is int64 + assert b == 3 def test_len(self): from numpypy import array From noreply at buildbot.pypy.org Fri Nov 15 08:52:14 2013 From: noreply at buildbot.pypy.org (arigo) Date: Fri, 15 Nov 2013 08:52:14 +0100 (CET) Subject: [pypy-commit] pypy default: An easy fix for a large part of test_os on Windows Message-ID: <20131115075214.E6DDF1C3014@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r68133:aa90b6cfcff5 Date: 2013-11-15 08:51 +0100 http://bitbucket.org/pypy/pypy/changeset/aa90b6cfcff5/ Log: An easy fix for a large part of test_os on Windows diff --git a/lib-python/2.7/test/test_os.py b/lib-python/2.7/test/test_os.py --- a/lib-python/2.7/test/test_os.py +++ b/lib-python/2.7/test/test_os.py @@ -75,7 +75,7 @@ self.assertFalse(os.path.exists(name), "file already exists for temporary file") # make sure we can create the file - open(name, "w") + open(name, "w").close() self.files.append(name) def test_tempnam(self): From noreply at buildbot.pypy.org Fri Nov 15 09:42:24 2013 From: noreply at buildbot.pypy.org (arigo) Date: Fri, 15 Nov 2013 09:42:24 +0100 (CET) Subject: [pypy-commit] pypy default: Windows: attempt to fix stat() for long ints. Message-ID: <20131115084224.930D11C13FC@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r68134:203930ef1f4f Date: 2013-11-15 09:41 +0100 http://bitbucket.org/pypy/pypy/changeset/203930ef1f4f/ Log: Windows: attempt to fix stat() for long ints. diff --git a/rpython/rtyper/module/ll_os_stat.py b/rpython/rtyper/module/ll_os_stat.py --- a/rpython/rtyper/module/ll_os_stat.py +++ b/rpython/rtyper/module/ll_os_stat.py @@ -466,16 +466,14 @@ def attribute_data_to_stat(info): st_mode = attributes_to_mode(info.c_dwFileAttributes) st_size = make_longlong(info.c_nFileSizeHigh, info.c_nFileSizeLow) - ctime, ctime_ns = FILE_TIME_to_time_t_nsec(info.c_ftCreationTime) - mtime, mtime_ns = FILE_TIME_to_time_t_nsec(info.c_ftLastWriteTime) - atime, atime_ns = FILE_TIME_to_time_t_nsec(info.c_ftLastAccessTime) + ctime = FILE_TIME_to_time_t_float(info.c_ftCreationTime) + mtime = FILE_TIME_to_time_t_float(info.c_ftLastWriteTime) + atime = FILE_TIME_to_time_t_float(info.c_ftLastAccessTime) result = (st_mode, 0, 0, 0, 0, 0, st_size, - float(atime) + atime_ns * 1e-9, - float(mtime) + mtime_ns * 1e-9, - float(ctime) + ctime_ns * 1e-9) + atime, mtime, ctime) return make_stat_result(result) @@ -483,9 +481,9 @@ # similar to the one above st_mode = attributes_to_mode(info.c_dwFileAttributes) st_size = make_longlong(info.c_nFileSizeHigh, info.c_nFileSizeLow) - ctime, ctime_ns = FILE_TIME_to_time_t_nsec(info.c_ftCreationTime) - mtime, mtime_ns = FILE_TIME_to_time_t_nsec(info.c_ftLastWriteTime) - atime, atime_ns = FILE_TIME_to_time_t_nsec(info.c_ftLastAccessTime) + ctime = FILE_TIME_to_time_t_float(info.c_ftCreationTime) + mtime = FILE_TIME_to_time_t_float(info.c_ftLastWriteTime) + atime = FILE_TIME_to_time_t_float(info.c_ftLastAccessTime) # specific to fstat() st_ino = make_longlong(info.c_nFileIndexHigh, info.c_nFileIndexLow) @@ -494,9 +492,7 @@ result = (st_mode, st_ino, 0, st_nlink, 0, 0, st_size, - atime + atime_ns * 1e-9, - mtime + mtime_ns * 1e-9, - ctime + ctime_ns * 1e-9) + atime, mtime, ctime) return make_stat_result(result) @@ -579,12 +575,10 @@ # Seconds between 1.1.1601 and 1.1.1970 secs_between_epochs = rffi.r_longlong(11644473600) -def FILE_TIME_to_time_t_nsec(filetime): +def FILE_TIME_to_time_t_float(filetime): ft = make_longlong(filetime.c_dwHighDateTime, filetime.c_dwLowDateTime) # FILETIME is in units of 100 nsec - nsec = (ft % 10000000) * 100 - time = (ft / 10000000) - secs_between_epochs - return intmask(time), intmask(nsec) + return float(ft) * (1.0 / 10000000.0) - secs_between_epochs def time_t_to_FILE_TIME(time, filetime): ft = rffi.r_longlong((time + secs_between_epochs) * 10000000) diff --git a/rpython/rtyper/module/test/test_ll_os.py b/rpython/rtyper/module/test/test_ll_os.py --- a/rpython/rtyper/module/test/test_ll_os.py +++ b/rpython/rtyper/module/test/test_ll_os.py @@ -76,6 +76,11 @@ t1 = 1159195039.25 compile(f, (str, float))(str(fname), t1) assert t1 == os.stat(str(fname)).st_mtime + if sys.version_info < (2, 7): + py.test.skip('requires Python 2.7') + t1 = 5000000000.0 + compile(f, (str, float))(str(fname), t1) + assert t1 == os.stat(str(fname)).st_mtime def test__getfullpathname(): if os.name != 'nt': diff --git a/rpython/rtyper/module/test/test_ll_os_stat.py b/rpython/rtyper/module/test/test_ll_os_stat.py --- a/rpython/rtyper/module/test/test_ll_os_stat.py +++ b/rpython/rtyper/module/test/test_ll_os_stat.py @@ -1,4 +1,5 @@ from rpython.rtyper.module import ll_os_stat, ll_os +from rpython.tool.udir import udir import sys, os import py @@ -33,3 +34,13 @@ fstat = ll_os_stat.make_win32_stat_impl('fstat', ll_os.StringTraits()) stat = fstat(0) # stdout assert stat.st_mode != 0 + + def test_stat_large_number(self): + if sys.version_info < (2, 7): + py.test.skip('requires Python 2.7') + fname = udir.join('test_stat_large_number.txt') + fname.ensure() + t1 = 5000000000.0 + os.utime(str(fname), (t1, t1)) + stat = ll_os_stat.make_win32_stat_impl('stat', ll_os.StringTraits()) + assert stat(str(fname)).st_mtime == t1 From noreply at buildbot.pypy.org Sun Nov 17 22:54:40 2013 From: noreply at buildbot.pypy.org (mattip) Date: Sun, 17 Nov 2013 22:54:40 +0100 (CET) Subject: [pypy-commit] buildbot add-header-to-nightly: fix regression to test status in directory listing Message-ID: <20131117215440.E0C311C0173@cobra.cs.uni-duesseldorf.de> Author: Matti Picus Branch: add-header-to-nightly Changeset: r895:75c380a6cecc Date: 2013-11-17 23:53 +0200 http://bitbucket.org/pypy/buildbot/changeset/75c380a6cecc/ Log: fix regression to test status in directory listing diff --git a/bot2/pypybuildbot/pypylist.py b/bot2/pypybuildbot/pypylist.py --- a/bot2/pypybuildbot/pypylist.py +++ b/bot2/pypybuildbot/pypylist.py @@ -156,6 +156,10 @@ '''template based, uses master/templates/directory.html ''' + def render(self, request): + self.status = request.site.buildbot_service.getStatus() + return DirectoryLister.render(self, request) + def _getFilesAndDirectories(self, directory): dirs, files = DirectoryLister._getFilesAndDirectories(self, directory) rowClasses = itertools.cycle(['odd', 'even']) From noreply at buildbot.pypy.org Sun Nov 17 22:54:42 2013 From: noreply at buildbot.pypy.org (mattip) Date: Sun, 17 Nov 2013 22:54:42 +0100 (CET) Subject: [pypy-commit] buildbot add-header-to-nightly: merge default into branch Message-ID: <20131117215442.169281C0225@cobra.cs.uni-duesseldorf.de> Author: Matti Picus Branch: add-header-to-nightly Changeset: r896:d69069fecc21 Date: 2013-11-17 23:54 +0200 http://bitbucket.org/pypy/buildbot/changeset/d69069fecc21/ Log: merge default into branch diff --git a/bot2/pypybuildbot/arm_master.py b/bot2/pypybuildbot/arm_master.py --- a/bot2/pypybuildbot/arm_master.py +++ b/bot2/pypybuildbot/arm_master.py @@ -136,6 +136,7 @@ BUILDJITLINUXARM, BUILDLINUXARMHF_RASPBIAN, BUILDJITLINUXARMHF_RASPBIAN, + BUILDJITLINUXARMHF_RARING, ] schedulers = [ diff --git a/bot2/pypybuildbot/builds.py b/bot2/pypybuildbot/builds.py --- a/bot2/pypybuildbot/builds.py +++ b/bot2/pypybuildbot/builds.py @@ -857,6 +857,11 @@ #env={"PYTHONPATH": ['download']}, # shouldn't be needed, but what if it is set externally? )) if host == 'tannit': + self.addStep(ShellCmd( + description="install jinja2", + command=['install/bin/pip', 'install', 'jinja2'], + workdir='./', + haltOnFailure=True,)) pypy_c_rel = 'install/bin/python' self.addStep(ShellCmd( description="measure numpy compatibility", diff --git a/bot2/pypybuildbot/master.py b/bot2/pypybuildbot/master.py --- a/bot2/pypybuildbot/master.py +++ b/bot2/pypybuildbot/master.py @@ -214,18 +214,21 @@ # other platforms #MACOSX32, # on minime JITWIN32, # on aurora - JITFREEBSD764, # on headless - JITFREEBSD864, # on ananke - JITFREEBSD964, # on exarkun's freebsd + #JITFREEBSD764, # on headless + #JITFREEBSD864, # on ananke + JITFREEBSD964, # on tavendo JITMACOSX64, # on xerxes # buildbot selftest PYPYBUILDBOT # on cobra ], branch='default', hour=0, minute=0), Nightly("nightly-2-00", [ + NUMPY_64, # on tannit64, uses 1 core, takes about 15min. + # XXX maybe use a trigger instead? JITBENCH, # on tannit32, uses 1 core (in part exclusively) JITBENCH64, # on tannit64, uses 1 core (in part exclusively) - ], branch='default', hour=2, minute=0), + + ], branch=None, hour=2, minute=0), Nightly("nightly-2-00-py3k", [ LINUX64, # on allegro64, uses all cores @@ -235,6 +238,7 @@ Nightly("nighly-ppc", [ JITONLYLINUXPPC64, # on gcc1 ], branch='ppc-jit-backend', hour=1, minute=0), + CustomForceScheduler('Force Scheduler', builderNames=[ PYPYBUILDBOT, @@ -411,7 +415,7 @@ "category": 'freebsd64' }, {"name" : JITFREEBSD964, - "slavenames": ['hybridlogic'], + "slavenames": ['hybridlogic', 'tavendo-freebsd-9.2-amd64'], 'builddir' : JITFREEBSD964, 'factory' : pypyJITTranslatedTestFactoryFreeBSD, "category": 'freebsd64' From noreply at buildbot.pypy.org Mon Nov 18 03:21:26 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Mon, 18 Nov 2013 03:21:26 +0100 (CET) Subject: [pypy-commit] pypy default: close files in this test Message-ID: <20131118022126.7426E1C0175@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r68197:00c1596ab741 Date: 2013-11-17 21:20 -0500 http://bitbucket.org/pypy/pypy/changeset/00c1596ab741/ Log: close files in this test diff --git a/lib-python/2.7/test/test_old_mailbox.py b/lib-python/2.7/test/test_old_mailbox.py --- a/lib-python/2.7/test/test_old_mailbox.py +++ b/lib-python/2.7/test/test_old_mailbox.py @@ -73,7 +73,9 @@ self.createMessage("cur") self.mbox = mailbox.Maildir(test_support.TESTFN) self.assertTrue(len(self.mbox) == 1) - self.assertTrue(self.mbox.next() is not None) + msg = self.mbox.next() + self.assertTrue(msg is not None) + msg.fp.close() self.assertTrue(self.mbox.next() is None) self.assertTrue(self.mbox.next() is None) @@ -81,7 +83,9 @@ self.createMessage("new") self.mbox = mailbox.Maildir(test_support.TESTFN) self.assertTrue(len(self.mbox) == 1) - self.assertTrue(self.mbox.next() is not None) + msg = self.mbox.next() + self.assertTrue(msg is not None) + msg.fp.close() self.assertTrue(self.mbox.next() is None) self.assertTrue(self.mbox.next() is None) @@ -90,8 +94,12 @@ self.createMessage("new") self.mbox = mailbox.Maildir(test_support.TESTFN) self.assertTrue(len(self.mbox) == 2) - self.assertTrue(self.mbox.next() is not None) - self.assertTrue(self.mbox.next() is not None) + msg = self.mbox.next() + self.assertTrue(msg is not None) + msg.fp.close() + msg = self.mbox.next() + self.assertTrue(msg is not None) + msg.fp.close() self.assertTrue(self.mbox.next() is None) self.assertTrue(self.mbox.next() is None) From noreply at buildbot.pypy.org Mon Nov 18 04:22:54 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Mon, 18 Nov 2013 04:22:54 +0100 (CET) Subject: [pypy-commit] pypy default: fix this test for the bsd builder Message-ID: <20131118032254.610E11C0175@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r68198:edd6b3a3df4d Date: 2013-11-17 22:22 -0500 http://bitbucket.org/pypy/pypy/changeset/edd6b3a3df4d/ Log: fix this test for the bsd builder diff --git a/pypy/module/test_lib_pypy/test_grp_extra.py b/pypy/module/test_lib_pypy/test_grp_extra.py --- a/pypy/module/test_lib_pypy/test_grp_extra.py +++ b/pypy/module/test_lib_pypy/test_grp_extra.py @@ -16,7 +16,7 @@ except KeyError: continue assert g.gr_gid == 0 - assert g.gr_mem == ['root'] or g.gr_mem == [] + assert 'root' in g.gr_mem or g.gr_mem == [] assert g.gr_name == name assert isinstance(g.gr_passwd, str) # usually just 'x', don't hope :-) break From noreply at buildbot.pypy.org Mon Nov 18 09:28:49 2013 From: noreply at buildbot.pypy.org (Raemi) Date: Mon, 18 Nov 2013 09:28:49 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c4: after a transaction break, we need to check for invalidation again Message-ID: <20131118082849.2C00F1C07B6@cobra.cs.uni-duesseldorf.de> Author: Remi Meier Branch: stmgc-c4 Changeset: r68199:4fb6c8277d6c Date: 2013-11-18 09:28 +0100 http://bitbucket.org/pypy/pypy/changeset/4fb6c8277d6c/ Log: after a transaction break, we need to check for invalidation again diff --git a/rpython/jit/metainterp/optimizeopt/heap.py b/rpython/jit/metainterp/optimizeopt/heap.py --- a/rpython/jit/metainterp/optimizeopt/heap.py +++ b/rpython/jit/metainterp/optimizeopt/heap.py @@ -262,12 +262,14 @@ opnum == rop.COPYSTRCONTENT or # no effect on GC struct/array opnum == rop.COPYUNICODECONTENT): # no effect on GC struct/array return + if (opnum == rop.STM_TRANSACTION_BREAK or + opnum == rop.CALL_ASSEMBLER): + self._seen_guard_not_invalidated = False if (opnum == rop.CALL or opnum == rop.CALL_PURE or opnum == rop.COND_CALL or opnum == rop.CALL_MAY_FORCE or - opnum == rop.CALL_RELEASE_GIL or - opnum == rop.CALL_ASSEMBLER): + opnum == rop.CALL_RELEASE_GIL): if opnum == rop.CALL_ASSEMBLER: self._seen_guard_not_invalidated = False else: From noreply at buildbot.pypy.org Mon Nov 18 09:29:18 2013 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 18 Nov 2013 09:29:18 +0100 (CET) Subject: [pypy-commit] pypy default: Hack around a FreeBSD issue Message-ID: <20131118082918.B8F131C07B6@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r68200:afb227c80804 Date: 2013-11-18 09:28 +0100 http://bitbucket.org/pypy/pypy/changeset/afb227c80804/ Log: Hack around a FreeBSD issue diff --git a/rpython/rlib/rdynload.py b/rpython/rlib/rdynload.py --- a/rpython/rlib/rdynload.py +++ b/rpython/rlib/rdynload.py @@ -4,6 +4,7 @@ from rpython.rtyper.tool import rffi_platform from rpython.rtyper.lltypesystem import rffi from rpython.rlib.rarithmetic import r_uint +from rpython.rlib.objectmodel import we_are_translated from rpython.translator.tool.cbuild import ExternalCompilationInfo from rpython.translator.platform import platform @@ -79,6 +80,38 @@ RTLD_NOW = cConfig.RTLD_NOW RTLD_LAZY = cConfig.RTLD_LAZY + _t_opened = {} + + def t_dlopen(name): + # for direct execution: can't use the regular way on FreeBSD :-( + # http://factor-language.blogspot.de/2009/02/note-about-libdl-functions-on-netbsd.html + import ctypes + if name: + name = rffi.charp2str(name) + else: + name = None + try: + res = ctypes.cdll.LoadLibrary(name) + except OSError, e: + raise DLOpenError(str(e)) + h = rffi.cast(rffi.VOIDP, res._handle) + _t_opened[rffi.cast(rffi.LONG, h)] = res + return h + + def t_dlclose(handle): + _t_opened.pop(rffi.cast(rffi.LONG, handle)) + return rffi.cast(rffi.INT, 0) + + def t_dldym(handle, name): + import ctypes + lib = _t_opened[rffi.cast(rffi.LONG, handle)] + try: + symbol = lib[name] + except AttributeError: + raise KeyError(name) + res = ctypes.cast(symbol, ctypes.c_void_p) + return rffi.cast(rffi.VOIDP, res.value or 0) + def dlerror(): # XXX this would never work on top of ll2ctypes, because # ctypes are calling dlerror itself, unsure if I can do much in this @@ -91,6 +124,8 @@ def dlopen(name, mode=-1): """ Wrapper around C-level dlopen """ + if not we_are_translated(): + return t_dlopen(name) if mode == -1: if RTLD_LOCAL is not None: mode = RTLD_LOCAL @@ -104,11 +139,16 @@ raise DLOpenError(err) return res - dlclose = c_dlclose + def dlclose(handle): + if not we_are_translated(): + return t_dlclose(handle) + return c_dlclose(handle) def dlsym(libhandle, name): """ Wrapper around C-level dlsym """ + if not we_are_translated(): + return t_dldym(libhandle, name) res = c_dlsym(libhandle, name) if not res: raise KeyError(name) diff --git a/rpython/rlib/test/test_rdynload.py b/rpython/rlib/test/test_rdynload.py --- a/rpython/rlib/test/test_rdynload.py +++ b/rpython/rlib/test/test_rdynload.py @@ -21,3 +21,4 @@ lltype.Signed)), dlsym(lib, 'abs')) assert 1 == handle(1) assert 1 == handle(-1) + dlclose(lib) From noreply at buildbot.pypy.org Mon Nov 18 09:37:50 2013 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 18 Nov 2013 09:37:50 +0100 (CET) Subject: [pypy-commit] pypy default: Fix the test. Message-ID: <20131118083750.794501C07B6@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r68201:c4dd095a23ea Date: 2013-11-18 09:37 +0100 http://bitbucket.org/pypy/pypy/changeset/c4dd095a23ea/ Log: Fix the test. diff --git a/rpython/jit/backend/x86/test/test_ztranslation_basic.py b/rpython/jit/backend/x86/test/test_ztranslation_basic.py --- a/rpython/jit/backend/x86/test/test_ztranslation_basic.py +++ b/rpython/jit/backend/x86/test/test_ztranslation_basic.py @@ -1,11 +1,11 @@ from rpython.jit.backend.llsupport.test.ztranslation_test import TranslationTest -from rpython.translator.translator import TranslationContext -from rpython.config.translationoption import DEFL_GC +from rpython.jit.backend.x86.arch import WORD class TestTranslationX86(TranslationTest): def _check_cbuilder(self, cbuilder): # We assume here that we have sse2. If not, the CPUClass # needs to be changed to CPU386_NO_SSE2, but well. - assert '-msse2' in cbuilder.eci.compile_extra - assert '-mfpmath=sse' in cbuilder.eci.compile_extra + if WORD == 4: + assert '-msse2' in cbuilder.eci.compile_extra + assert '-mfpmath=sse' in cbuilder.eci.compile_extra From noreply at buildbot.pypy.org Mon Nov 18 11:20:01 2013 From: noreply at buildbot.pypy.org (Raemi) Date: Mon, 18 Nov 2013 11:20:01 +0100 (CET) Subject: [pypy-commit] stmgc default: fix some debug assert Message-ID: <20131118102001.ADBAC1C00EC@cobra.cs.uni-duesseldorf.de> Author: Remi Meier Branch: Changeset: r547:3d0358c80701 Date: 2013-11-08 15:28 +0100 http://bitbucket.org/pypy/stmgc/changeset/3d0358c80701/ Log: fix some debug assert diff --git a/c4/nursery.c b/c4/nursery.c --- a/c4/nursery.c +++ b/c4/nursery.c @@ -84,7 +84,7 @@ } #ifdef _GC_DEBUG if (P != NULL) { - assert(P->h_tid != 0); + assert((P->h_tid & STM_USER_TID_MASK) == (tid & STM_USER_TID_MASK)); assert_cleared(((char *)P) + sizeof(revision_t), size - sizeof(revision_t)); } diff --git a/c4/stmgc.h b/c4/stmgc.h --- a/c4/stmgc.h +++ b/c4/stmgc.h @@ -38,7 +38,7 @@ /* allocates a public reference to the object that will not be freed until stm_unregister_integer_address is - called on the result */ + called on the result (push roots!) */ intptr_t stm_allocate_public_integer_address(gcptr); void stm_unregister_integer_address(intptr_t); From noreply at buildbot.pypy.org Mon Nov 18 11:20:03 2013 From: noreply at buildbot.pypy.org (Raemi) Date: Mon, 18 Nov 2013 11:20:03 +0100 (CET) Subject: [pypy-commit] stmgc default: fix public ints (usage of public h_originals is not always right, they Message-ID: <20131118102003.1706D1C00EC@cobra.cs.uni-duesseldorf.de> Author: Remi Meier Branch: Changeset: r548:68677625f2be Date: 2013-11-18 11:19 +0100 http://bitbucket.org/pypy/stmgc/changeset/68677625f2be/ Log: fix public ints (usage of public h_originals is not always right, they need to be PREBUILT_ORIGINALs to be sure...) diff --git a/c4/demo_random.c b/c4/demo_random.c --- a/c4/demo_random.c +++ b/c4/demo_random.c @@ -278,8 +278,7 @@ gcptr obj = (gcptr)ip; assert(obj->h_tid & GCFLAG_PUBLIC); assert((obj->h_tid & GCFLAG_SMALLSTUB) - || (obj->h_original == 0 - || obj->h_tid & GCFLAG_PREBUILT_ORIGINAL)); + || (obj->h_tid & GCFLAG_PREBUILT_ORIGINAL)); check(obj); if (obj->h_revision & 2) check((gcptr)(obj->h_revision - 2)); @@ -304,7 +303,9 @@ if (td.num_public_ints == 0) return; + push_roots(); stm_unregister_integer_address(td.public_ints[--td.num_public_ints]); + pop_roots(); } gcptr read_barrier(gcptr p) diff --git a/c4/et.c b/c4/et.c --- a/c4/et.c +++ b/c4/et.c @@ -138,6 +138,7 @@ d->count_reads++; assert(IMPLIES(!(P->h_tid & GCFLAG_OLD), stmgc_is_in_nursery(d, P))); + assert(G->h_revision != 0); restart_all: if (P->h_tid & GCFLAG_PRIVATE_FROM_PROTECTED) @@ -355,7 +356,7 @@ assert(P->h_tid & GCFLAG_PUBLIC); assert(IMPLIES(!(P->h_tid & GCFLAG_OLD), stmgc_is_in_nursery(thread_descriptor, P))); - + assert(P->h_revision != 0); revision_t v = ACCESS_ONCE(P->h_revision); assert(IS_POINTER(v)); /* "is a pointer", "has a more recent revision" */ @@ -660,6 +661,7 @@ gcptr stm_RepeatWriteBarrier(gcptr P) { + assert(P->h_revision != 0); assert(IMPLIES(!(P->h_tid & GCFLAG_OLD), stmgc_is_in_nursery(thread_descriptor, P))); @@ -673,6 +675,7 @@ gcptr stm_WriteBarrier(gcptr P) { + assert(P->h_revision != 0); assert(!(P->h_tid & GCFLAG_IMMUTABLE)); assert((P->h_tid & GCFLAG_STUB) || stmgc_size(P) > sizeof(struct stm_stub_s) - WORD); diff --git a/c4/extra.c b/c4/extra.c --- a/c4/extra.c +++ b/c4/extra.c @@ -65,7 +65,7 @@ intptr_t stm_allocate_public_integer_address(gcptr obj) -{ +{ /* push roots! */ struct tx_descriptor *d = thread_descriptor; gcptr stub; intptr_t result; @@ -75,6 +75,12 @@ During major collections, we visit them and update their references. */ + /* stm_register_integer_address needs to run in inevitable + transaction */ + stm_push_root(obj); + stm_become_inevitable("stm_allocate_public_integer_address"); + obj = stm_pop_root(); + /* we don't want to deal with young objs */ if (!(obj->h_tid & GCFLAG_OLD)) { stm_push_root(obj); @@ -93,9 +99,11 @@ orig = (gcptr)obj->h_original; } - if (orig->h_tid & GCFLAG_PUBLIC) { - /* the original is public, so we can take that as a non-movable - object to register */ + if ((orig->h_tid & (GCFLAG_PUBLIC | GCFLAG_PREBUILT_ORIGINAL)) + == (GCFLAG_PUBLIC | GCFLAG_PREBUILT_ORIGINAL)) { + /* public is not enough as public stubs may get replaced + by the protected object they point to, if they are in the + same thread (I think...) */ result = (intptr_t)orig; } else { @@ -115,9 +123,11 @@ result = (intptr_t)stub; } spinlock_release(d->public_descriptor->collection_lock); + + dprintf(("allocate_public_int_adr(%p): %p", obj, (void*)result)); + stm_register_integer_address(result); - dprintf(("allocate_public_int_adr(%p): %p", obj, (void*)result)); return result; } diff --git a/c4/gcpage.c b/c4/gcpage.c --- a/c4/gcpage.c +++ b/c4/gcpage.c @@ -218,10 +218,11 @@ /***** registering of small stubs as integer addresses *****/ void stm_register_integer_address(intptr_t adr) -{ +{ /* needs to be inevitable! */ wlog_t *found; gcptr obj = (gcptr)adr; /* current limitations for 'adr': smallstub or h_original */ + assert(stm_active == 2); assert((obj->h_tid & GCFLAG_SMALLSTUB) || (obj->h_original == 0 || obj->h_tid & GCFLAG_PREBUILT_ORIGINAL)); assert(obj->h_tid & GCFLAG_PUBLIC); @@ -241,7 +242,7 @@ } void stm_unregister_integer_address(intptr_t adr) -{ +{ /* push roots! */ wlog_t *found; gcptr obj = (gcptr)adr; @@ -249,6 +250,11 @@ || (obj->h_original == 0 || obj->h_tid & GCFLAG_PREBUILT_ORIGINAL)); assert(obj->h_tid & GCFLAG_PUBLIC); + /* become inevitable because we would have to re-register them + on abort, but make sure only to re-register if not registered + in the same aborted transaction (XXX) */ + stm_become_inevitable("stm_unregister_integer_address()"); + stmgcpage_acquire_global_lock(); /* find and decrement refcount */ @@ -527,12 +533,18 @@ G2L_LOOP_FORWARD(registered_objs, item) { gcptr R = item->addr; assert(R->h_tid & GCFLAG_PUBLIC); - - if ((R->h_original == 0) || (R->h_tid & GCFLAG_PREBUILT_ORIGINAL)) { - /* the obj is an original and will therefore survive: */ - gcptr V = stmgcpage_visit(R); - assert(V == R); + + if (R->h_tid & GCFLAG_PREBUILT_ORIGINAL) { + /* already done by mark_prebuilt_roots */ + assert((R->h_tid & (GCFLAG_MARKED|GCFLAG_VISITED|GCFLAG_PUBLIC)) + == (GCFLAG_MARKED|GCFLAG_VISITED|GCFLAG_PUBLIC)); + continue; } + /* else if (R->h_original == 0) { */ + /* /\* the obj is an original and will therefore survive: *\/ */ + /* gcptr V = visit_public(R, NULL); */ + /* assert(V == R); */ + /* } */ else { assert(R->h_tid & GCFLAG_SMALLSTUB); /* only case for now */ /* make sure R stays valid: */ diff --git a/c4/stmgc.h b/c4/stmgc.h --- a/c4/stmgc.h +++ b/c4/stmgc.h @@ -40,7 +40,7 @@ not be freed until stm_unregister_integer_address is called on the result (push roots!) */ intptr_t stm_allocate_public_integer_address(gcptr); -void stm_unregister_integer_address(intptr_t); +void stm_unregister_integer_address(intptr_t); /* push roots too! */ /* returns a never changing hash for the object */ @@ -193,6 +193,7 @@ void stm_call_on_abort(void *key, void callback(void *)); /* only user currently is stm_allocate_public_integer_address() */ +/* needs to be in an inevitable transaction! */ void stm_register_integer_address(intptr_t); /* enter single-threaded mode. Used e.g. when patching assembler From noreply at buildbot.pypy.org Mon Nov 18 11:21:01 2013 From: noreply at buildbot.pypy.org (Raemi) Date: Mon, 18 Nov 2013 11:21:01 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c4: import stmgc Message-ID: <20131118102101.D4CB31C026D@cobra.cs.uni-duesseldorf.de> Author: Remi Meier Branch: stmgc-c4 Changeset: r68202:88a0f0cc46a5 Date: 2013-11-18 11:20 +0100 http://bitbucket.org/pypy/pypy/changeset/88a0f0cc46a5/ Log: import stmgc diff --git a/rpython/translator/stm/src_stm/et.c b/rpython/translator/stm/src_stm/et.c --- a/rpython/translator/stm/src_stm/et.c +++ b/rpython/translator/stm/src_stm/et.c @@ -139,6 +139,7 @@ d->count_reads++; assert(IMPLIES(!(P->h_tid & GCFLAG_OLD), stmgc_is_in_nursery(d, P))); + assert(G->h_revision != 0); restart_all: if (P->h_tid & GCFLAG_PRIVATE_FROM_PROTECTED) @@ -356,7 +357,7 @@ assert(P->h_tid & GCFLAG_PUBLIC); assert(IMPLIES(!(P->h_tid & GCFLAG_OLD), stmgc_is_in_nursery(thread_descriptor, P))); - + assert(P->h_revision != 0); revision_t v = ACCESS_ONCE(P->h_revision); assert(IS_POINTER(v)); /* "is a pointer", "has a more recent revision" */ @@ -661,6 +662,7 @@ gcptr stm_RepeatWriteBarrier(gcptr P) { + assert(P->h_revision != 0); assert(IMPLIES(!(P->h_tid & GCFLAG_OLD), stmgc_is_in_nursery(thread_descriptor, P))); @@ -674,6 +676,7 @@ gcptr stm_WriteBarrier(gcptr P) { + assert(P->h_revision != 0); assert(!(P->h_tid & GCFLAG_IMMUTABLE)); assert((P->h_tid & GCFLAG_STUB) || stmgc_size(P) > sizeof(struct stm_stub_s) - WORD); diff --git a/rpython/translator/stm/src_stm/extra.c b/rpython/translator/stm/src_stm/extra.c --- a/rpython/translator/stm/src_stm/extra.c +++ b/rpython/translator/stm/src_stm/extra.c @@ -66,7 +66,7 @@ intptr_t stm_allocate_public_integer_address(gcptr obj) -{ +{ /* push roots! */ struct tx_descriptor *d = thread_descriptor; gcptr stub; intptr_t result; @@ -76,6 +76,12 @@ During major collections, we visit them and update their references. */ + /* stm_register_integer_address needs to run in inevitable + transaction */ + stm_push_root(obj); + stm_become_inevitable("stm_allocate_public_integer_address"); + obj = stm_pop_root(); + /* we don't want to deal with young objs */ if (!(obj->h_tid & GCFLAG_OLD)) { stm_push_root(obj); @@ -94,9 +100,11 @@ orig = (gcptr)obj->h_original; } - if (orig->h_tid & GCFLAG_PUBLIC) { - /* the original is public, so we can take that as a non-movable - object to register */ + if ((orig->h_tid & (GCFLAG_PUBLIC | GCFLAG_PREBUILT_ORIGINAL)) + == (GCFLAG_PUBLIC | GCFLAG_PREBUILT_ORIGINAL)) { + /* public is not enough as public stubs may get replaced + by the protected object they point to, if they are in the + same thread (I think...) */ result = (intptr_t)orig; } else { @@ -116,9 +124,11 @@ result = (intptr_t)stub; } spinlock_release(d->public_descriptor->collection_lock); + + dprintf(("allocate_public_int_adr(%p): %p", obj, (void*)result)); + stm_register_integer_address(result); - dprintf(("allocate_public_int_adr(%p): %p", obj, (void*)result)); return result; } diff --git a/rpython/translator/stm/src_stm/gcpage.c b/rpython/translator/stm/src_stm/gcpage.c --- a/rpython/translator/stm/src_stm/gcpage.c +++ b/rpython/translator/stm/src_stm/gcpage.c @@ -219,10 +219,11 @@ /***** registering of small stubs as integer addresses *****/ void stm_register_integer_address(intptr_t adr) -{ +{ /* needs to be inevitable! */ wlog_t *found; gcptr obj = (gcptr)adr; /* current limitations for 'adr': smallstub or h_original */ + assert(stm_active == 2); assert((obj->h_tid & GCFLAG_SMALLSTUB) || (obj->h_original == 0 || obj->h_tid & GCFLAG_PREBUILT_ORIGINAL)); assert(obj->h_tid & GCFLAG_PUBLIC); @@ -242,7 +243,7 @@ } void stm_unregister_integer_address(intptr_t adr) -{ +{ /* push roots! */ wlog_t *found; gcptr obj = (gcptr)adr; @@ -250,6 +251,11 @@ || (obj->h_original == 0 || obj->h_tid & GCFLAG_PREBUILT_ORIGINAL)); assert(obj->h_tid & GCFLAG_PUBLIC); + /* become inevitable because we would have to re-register them + on abort, but make sure only to re-register if not registered + in the same aborted transaction (XXX) */ + stm_become_inevitable("stm_unregister_integer_address()"); + stmgcpage_acquire_global_lock(); /* find and decrement refcount */ @@ -528,12 +534,18 @@ G2L_LOOP_FORWARD(registered_objs, item) { gcptr R = item->addr; assert(R->h_tid & GCFLAG_PUBLIC); - - if ((R->h_original == 0) || (R->h_tid & GCFLAG_PREBUILT_ORIGINAL)) { - /* the obj is an original and will therefore survive: */ - gcptr V = stmgcpage_visit(R); - assert(V == R); + + if (R->h_tid & GCFLAG_PREBUILT_ORIGINAL) { + /* already done by mark_prebuilt_roots */ + assert((R->h_tid & (GCFLAG_MARKED|GCFLAG_VISITED|GCFLAG_PUBLIC)) + == (GCFLAG_MARKED|GCFLAG_VISITED|GCFLAG_PUBLIC)); + continue; } + /* else if (R->h_original == 0) { */ + /* /\* the obj is an original and will therefore survive: *\/ */ + /* gcptr V = visit_public(R, NULL); */ + /* assert(V == R); */ + /* } */ else { assert(R->h_tid & GCFLAG_SMALLSTUB); /* only case for now */ /* make sure R stays valid: */ diff --git a/rpython/translator/stm/src_stm/nursery.c b/rpython/translator/stm/src_stm/nursery.c --- a/rpython/translator/stm/src_stm/nursery.c +++ b/rpython/translator/stm/src_stm/nursery.c @@ -85,7 +85,7 @@ } #ifdef _GC_DEBUG if (P != NULL) { - assert(P->h_tid != 0); + assert((P->h_tid & STM_USER_TID_MASK) == (tid & STM_USER_TID_MASK)); assert_cleared(((char *)P) + sizeof(revision_t), size - sizeof(revision_t)); } diff --git a/rpython/translator/stm/src_stm/revision b/rpython/translator/stm/src_stm/revision --- a/rpython/translator/stm/src_stm/revision +++ b/rpython/translator/stm/src_stm/revision @@ -1,1 +1,1 @@ -14ac008e70a5 +68677625f2be diff --git a/rpython/translator/stm/src_stm/stmgc.h b/rpython/translator/stm/src_stm/stmgc.h --- a/rpython/translator/stm/src_stm/stmgc.h +++ b/rpython/translator/stm/src_stm/stmgc.h @@ -39,9 +39,9 @@ /* allocates a public reference to the object that will not be freed until stm_unregister_integer_address is - called on the result */ + called on the result (push roots!) */ intptr_t stm_allocate_public_integer_address(gcptr); -void stm_unregister_integer_address(intptr_t); +void stm_unregister_integer_address(intptr_t); /* push roots too! */ /* returns a never changing hash for the object */ @@ -194,6 +194,7 @@ void stm_call_on_abort(void *key, void callback(void *)); /* only user currently is stm_allocate_public_integer_address() */ +/* needs to be in an inevitable transaction! */ void stm_register_integer_address(intptr_t); /* enter single-threaded mode. Used e.g. when patching assembler From noreply at buildbot.pypy.org Mon Nov 18 12:05:31 2013 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 18 Nov 2013 12:05:31 +0100 (CET) Subject: [pypy-commit] cffi default: Add a note to avoid people getting confused by the error message. Message-ID: <20131118110531.AFFD71C07B6@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r1421:dd14c6808158 Date: 2013-11-18 12:05 +0100 http://bitbucket.org/cffi/cffi/changeset/dd14c6808158/ Log: Add a note to avoid people getting confused by the error message. diff --git a/setup.py b/setup.py --- a/setup.py +++ b/setup.py @@ -48,6 +48,7 @@ try: compiler.compile(['c/check__thread.c']) except distutils.errors.CompileError: + print >> sys.stderr, "the above error message can be safely ignored;" print >> sys.stderr, "will not use '__thread' in the C code" else: define_macros.append(('USE__THREAD', None)) From noreply at buildbot.pypy.org Mon Nov 18 12:06:54 2013 From: noreply at buildbot.pypy.org (mattip) Date: Mon, 18 Nov 2013 12:06:54 +0100 (CET) Subject: [pypy-commit] pypy windows-packaging: merge with default into branch Message-ID: <20131118110654.731D11C07B6@cobra.cs.uni-duesseldorf.de> Author: Matti Picus Branch: windows-packaging Changeset: r68203:7d65e31ab264 Date: 2013-11-18 13:04 +0200 http://bitbucket.org/pypy/pypy/changeset/7d65e31ab264/ Log: merge with default into branch diff --git a/lib-python/2.7/test/test_old_mailbox.py b/lib-python/2.7/test/test_old_mailbox.py --- a/lib-python/2.7/test/test_old_mailbox.py +++ b/lib-python/2.7/test/test_old_mailbox.py @@ -73,7 +73,9 @@ self.createMessage("cur") self.mbox = mailbox.Maildir(test_support.TESTFN) self.assertTrue(len(self.mbox) == 1) - self.assertTrue(self.mbox.next() is not None) + msg = self.mbox.next() + self.assertTrue(msg is not None) + msg.fp.close() self.assertTrue(self.mbox.next() is None) self.assertTrue(self.mbox.next() is None) @@ -81,7 +83,9 @@ self.createMessage("new") self.mbox = mailbox.Maildir(test_support.TESTFN) self.assertTrue(len(self.mbox) == 1) - self.assertTrue(self.mbox.next() is not None) + msg = self.mbox.next() + self.assertTrue(msg is not None) + msg.fp.close() self.assertTrue(self.mbox.next() is None) self.assertTrue(self.mbox.next() is None) @@ -90,8 +94,12 @@ self.createMessage("new") self.mbox = mailbox.Maildir(test_support.TESTFN) self.assertTrue(len(self.mbox) == 2) - self.assertTrue(self.mbox.next() is not None) - self.assertTrue(self.mbox.next() is not None) + msg = self.mbox.next() + self.assertTrue(msg is not None) + msg.fp.close() + msg = self.mbox.next() + self.assertTrue(msg is not None) + msg.fp.close() self.assertTrue(self.mbox.next() is None) self.assertTrue(self.mbox.next() is None) diff --git a/lib_pypy/_sqlite3.py b/lib_pypy/_sqlite3.py --- a/lib_pypy/_sqlite3.py +++ b/lib_pypy/_sqlite3.py @@ -268,10 +268,18 @@ if _has_load_extension(): _ffi.cdef("int sqlite3_enable_load_extension(sqlite3 *db, int onoff);") -_lib = _ffi.verify(""" -#include -""", libraries=['sqlite3'] -) +if sys.platform.startswith('freebsd'): + _lib = _ffi.verify(""" + #include + """, libraries=['sqlite3'], + include_dirs=['/usr/local/include'], + library_dirs=['/usr/local/lib'] + ) +else: + _lib = _ffi.verify(""" + #include + """, libraries=['sqlite3'] + ) exported_sqlite_symbols = [ 'SQLITE_ALTER_TABLE', diff --git a/pypy/interpreter/test/test_app_main.py b/pypy/interpreter/test/test_app_main.py --- a/pypy/interpreter/test/test_app_main.py +++ b/pypy/interpreter/test/test_app_main.py @@ -942,7 +942,8 @@ self.w_tmp_dir = self.space.wrap(tmp_dir) - foo_py = prefix.join('foo.py').write("pass") + foo_py = prefix.join('foo.py') + foo_py.write("pass") self.w_foo_py = self.space.wrap(str(foo_py)) def test_setup_bootstrap_path(self): diff --git a/pypy/module/termios/test/test_termios.py b/pypy/module/termios/test/test_termios.py --- a/pypy/module/termios/test/test_termios.py +++ b/pypy/module/termios/test/test_termios.py @@ -7,6 +7,9 @@ if os.name != 'posix': py.test.skip('termios module only available on unix') +if sys.platform.startswith('freebsd'): + raise Exception('XXX seems to hangs on FreeBSD9') + class TestTermios(object): def setup_class(cls): try: diff --git a/pypy/module/test_lib_pypy/cffi_tests/test_verify.py b/pypy/module/test_lib_pypy/cffi_tests/test_verify.py --- a/pypy/module/test_lib_pypy/cffi_tests/test_verify.py +++ b/pypy/module/test_lib_pypy/cffi_tests/test_verify.py @@ -1638,7 +1638,11 @@ #include #define alloca _alloca #else - #include + # ifdef __FreeBSD__ + # include + # else + # include + # endif #endif static int (*python_callback)(int how_many, int *values); static int c_callback(int how_many, ...) { diff --git a/pypy/module/test_lib_pypy/pyrepl/__init__.py b/pypy/module/test_lib_pypy/pyrepl/__init__.py --- a/pypy/module/test_lib_pypy/pyrepl/__init__.py +++ b/pypy/module/test_lib_pypy/pyrepl/__init__.py @@ -1,3 +1,6 @@ import sys import lib_pypy.pyrepl sys.modules['pyrepl'] = sys.modules['lib_pypy.pyrepl'] + +if sys.platform.startswith('freebsd'): + raise Exception('XXX seems to hangs on FreeBSD9') diff --git a/pypy/module/test_lib_pypy/test_grp_extra.py b/pypy/module/test_lib_pypy/test_grp_extra.py --- a/pypy/module/test_lib_pypy/test_grp_extra.py +++ b/pypy/module/test_lib_pypy/test_grp_extra.py @@ -16,7 +16,7 @@ except KeyError: continue assert g.gr_gid == 0 - assert g.gr_mem == ['root'] or g.gr_mem == [] + assert 'root' in g.gr_mem or g.gr_mem == [] assert g.gr_name == name assert isinstance(g.gr_passwd, str) # usually just 'x', don't hope :-) break diff --git a/rpython/jit/backend/x86/support.py b/rpython/jit/backend/x86/support.py --- a/rpython/jit/backend/x86/support.py +++ b/rpython/jit/backend/x86/support.py @@ -5,12 +5,13 @@ if WORD == 4: extra = ['-DPYPY_X86_CHECK_SSE2'] + if sys.platform != 'win32': + extra += ['-msse2', '-mfpmath=sse'] else: - extra = [] + extra = [] # the -m options above are always on by default on x86-64 if sys.platform != 'win32': - extra = ['-msse2', '-mfpmath=sse', - '-DPYPY_CPU_HAS_STANDARD_PRECISION'] + extra + extra = ['-DPYPY_CPU_HAS_STANDARD_PRECISION'] + extra ensure_sse2_floats = rffi.llexternal_use_eci(ExternalCompilationInfo( compile_extra = extra, diff --git a/rpython/jit/backend/x86/test/test_ztranslation_basic.py b/rpython/jit/backend/x86/test/test_ztranslation_basic.py --- a/rpython/jit/backend/x86/test/test_ztranslation_basic.py +++ b/rpython/jit/backend/x86/test/test_ztranslation_basic.py @@ -1,11 +1,11 @@ from rpython.jit.backend.llsupport.test.ztranslation_test import TranslationTest -from rpython.translator.translator import TranslationContext -from rpython.config.translationoption import DEFL_GC +from rpython.jit.backend.x86.arch import WORD class TestTranslationX86(TranslationTest): def _check_cbuilder(self, cbuilder): # We assume here that we have sse2. If not, the CPUClass # needs to be changed to CPU386_NO_SSE2, but well. - assert '-msse2' in cbuilder.eci.compile_extra - assert '-mfpmath=sse' in cbuilder.eci.compile_extra + if WORD == 4: + assert '-msse2' in cbuilder.eci.compile_extra + assert '-mfpmath=sse' in cbuilder.eci.compile_extra diff --git a/rpython/rlib/rdynload.py b/rpython/rlib/rdynload.py --- a/rpython/rlib/rdynload.py +++ b/rpython/rlib/rdynload.py @@ -4,6 +4,7 @@ from rpython.rtyper.tool import rffi_platform from rpython.rtyper.lltypesystem import rffi from rpython.rlib.rarithmetic import r_uint +from rpython.rlib.objectmodel import we_are_translated from rpython.translator.tool.cbuild import ExternalCompilationInfo from rpython.translator.platform import platform @@ -79,6 +80,38 @@ RTLD_NOW = cConfig.RTLD_NOW RTLD_LAZY = cConfig.RTLD_LAZY + _t_opened = {} + + def t_dlopen(name): + # for direct execution: can't use the regular way on FreeBSD :-( + # http://factor-language.blogspot.de/2009/02/note-about-libdl-functions-on-netbsd.html + import ctypes + if name: + name = rffi.charp2str(name) + else: + name = None + try: + res = ctypes.cdll.LoadLibrary(name) + except OSError, e: + raise DLOpenError(str(e)) + h = rffi.cast(rffi.VOIDP, res._handle) + _t_opened[rffi.cast(rffi.LONG, h)] = res + return h + + def t_dlclose(handle): + _t_opened.pop(rffi.cast(rffi.LONG, handle)) + return rffi.cast(rffi.INT, 0) + + def t_dldym(handle, name): + import ctypes + lib = _t_opened[rffi.cast(rffi.LONG, handle)] + try: + symbol = lib[name] + except AttributeError: + raise KeyError(name) + res = ctypes.cast(symbol, ctypes.c_void_p) + return rffi.cast(rffi.VOIDP, res.value or 0) + def dlerror(): # XXX this would never work on top of ll2ctypes, because # ctypes are calling dlerror itself, unsure if I can do much in this @@ -91,6 +124,8 @@ def dlopen(name, mode=-1): """ Wrapper around C-level dlopen """ + if not we_are_translated(): + return t_dlopen(name) if mode == -1: if RTLD_LOCAL is not None: mode = RTLD_LOCAL @@ -104,11 +139,16 @@ raise DLOpenError(err) return res - dlclose = c_dlclose + def dlclose(handle): + if not we_are_translated(): + return t_dlclose(handle) + return c_dlclose(handle) def dlsym(libhandle, name): """ Wrapper around C-level dlsym """ + if not we_are_translated(): + return t_dldym(libhandle, name) res = c_dlsym(libhandle, name) if not res: raise KeyError(name) diff --git a/rpython/rlib/test/test_rdynload.py b/rpython/rlib/test/test_rdynload.py --- a/rpython/rlib/test/test_rdynload.py +++ b/rpython/rlib/test/test_rdynload.py @@ -21,3 +21,4 @@ lltype.Signed)), dlsym(lib, 'abs')) assert 1 == handle(1) assert 1 == handle(-1) + dlclose(lib) From noreply at buildbot.pypy.org Mon Nov 18 12:06:55 2013 From: noreply at buildbot.pypy.org (mattip) Date: Mon, 18 Nov 2013 12:06:55 +0100 (CET) Subject: [pypy-commit] pypy windows-packaging: document branch Message-ID: <20131118110655.A74D81C07B6@cobra.cs.uni-duesseldorf.de> Author: Matti Picus Branch: windows-packaging Changeset: r68204:d1aeb871790a Date: 2013-11-18 13:06 +0200 http://bitbucket.org/pypy/pypy/changeset/d1aeb871790a/ Log: document branch diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -9,3 +9,7 @@ .. branch: numpy-newbyteorder Clean up numpy types, add newbyteorder functionality + +.. branch windows-packaging +Package tk/tcl runtime with win32 + From noreply at buildbot.pypy.org Mon Nov 18 12:06:56 2013 From: noreply at buildbot.pypy.org (mattip) Date: Mon, 18 Nov 2013 12:06:56 +0100 (CET) Subject: [pypy-commit] pypy windows-packaging: close branch to be merged Message-ID: <20131118110656.BFA841C07B6@cobra.cs.uni-duesseldorf.de> Author: Matti Picus Branch: windows-packaging Changeset: r68205:2d1eb388152e Date: 2013-11-18 13:06 +0200 http://bitbucket.org/pypy/pypy/changeset/2d1eb388152e/ Log: close branch to be merged From noreply at buildbot.pypy.org Mon Nov 18 12:06:58 2013 From: noreply at buildbot.pypy.org (mattip) Date: Mon, 18 Nov 2013 12:06:58 +0100 (CET) Subject: [pypy-commit] pypy default: merge windows-packaging, which add the tcl/tk runtime to win32 for cpython compatability Message-ID: <20131118110658.0DC141C07B6@cobra.cs.uni-duesseldorf.de> Author: Matti Picus Branch: Changeset: r68206:4cc0b451e81f Date: 2013-11-18 13:07 +0200 http://bitbucket.org/pypy/pypy/changeset/4cc0b451e81f/ Log: merge windows-packaging, which add the tcl/tk runtime to win32 for cpython compatability diff --git a/lib_pypy/_tkinter/tklib.py b/lib_pypy/_tkinter/tklib.py --- a/lib_pypy/_tkinter/tklib.py +++ b/lib_pypy/_tkinter/tklib.py @@ -112,6 +112,10 @@ incdirs = ['/usr/local/include/tcl8.5', '/usr/local/include/tk8.5', '/usr/X11R6/include'] linklibs = ['tk85', 'tcl85'] libdirs = ['/usr/local/lib', '/usr/X11R6/lib'] +elif sys.platform == 'win32': + incdirs = [] + linklibs = ['tcl85', 'tk85'] + libdirs = [] else: incdirs=['/usr/include/tcl'] linklibs=['tcl', 'tk'] diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -9,3 +9,7 @@ .. branch: numpy-newbyteorder Clean up numpy types, add newbyteorder functionality + +.. branch windows-packaging +Package tk/tcl runtime with win32 + diff --git a/pypy/doc/windows.rst b/pypy/doc/windows.rst --- a/pypy/doc/windows.rst +++ b/pypy/doc/windows.rst @@ -73,11 +73,11 @@ https://bitbucket.org/pypy/pypy/downloads/local.zip Then expand it into the base directory (base_dir) and modify your environment to reflect this:: - set PATH=\bin;%PATH% - set INCLUDE=\include;%INCLUDE% - set LIB=\lib;%LIB% + set PATH=\bin;\tcltk\bin;%PATH% + set INCLUDE=\include;\tcltk\include;%INCLUDE% + set LIB=\lib;\tcltk\lib;%LIB% -Now you should be good to go. Read on for more information. +Now you should be good to go. Read on for more information. The Boehm garbage collector ~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -109,11 +109,10 @@ The bz2 compression library ~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Download http://bzip.org/1.0.5/bzip2-1.0.5.tar.gz and extract it in -the base directory. Then compile:: - - cd bzip2-1.0.5 + svn export http://svn.python.org/projects/external/bzip2-1.0.6 + cd bzip2-1.0.6 nmake -f makefile.msc + copy bzip.dll \bzip.dll The sqlite3 database library ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -122,8 +121,6 @@ wrapper is compiled when the module is imported for the first time. The sqlite3.dll should be version 3.6.21 for CPython2.7 compatablility. - - The expat XML parser ~~~~~~~~~~~~~~~~~~~~ @@ -146,13 +143,33 @@ use the one distributed by ActiveState, or the one from cygwin. In both case the perl interpreter must be found on the PATH. -Get http://www.openssl.org/source/openssl-0.9.8k.tar.gz and extract it -in the base directory. Then compile:: - + svn export http://svn.python.org/projects/external/openssl-0.9.8y + cd openssl-0.9.8y perl Configure VC-WIN32 ms\do_ms.bat nmake -f ms\nt.mak install +TkInter module support +~~~~~~~~~~~~~~~~~~~~~~ + +Note that much of this is taken from the cpython build process. +Tkinter is imported via cffi, so the module is optional. To recreate the tcltk +directory found for the release script, create the dlls, libs, headers and +runtime by running:: + + svn export http://svn.python.org/projects/external/tcl-8.5.2.1 tcl85 + svn export http://svn.python.org/projects/external/tk-8.5.2.0 tk85 + cd tcl85\win + nmake -f makefile.vc COMPILERFLAGS=-DWINVER=0x0500 DEBUG=0 INSTALLDIR=..\..\tcltk clean all + nmake -f makefile.vc DEBUG=0 INSTALLDIR=..\..\tcltk install + cd ..\..\tk85\win + nmake -f makefile.vc COMPILERFLAGS=-DWINVER=0x0500 OPTS=noxp DEBUG=1 INSTALLDIR=..\..\tcltk TCLDIR=..\..\tcl85 clean all + nmake -f makefile.vc COMPILERFLAGS=-DWINVER=0x0500 OPTS=noxp DEBUG=1 INSTALLDIR=..\..\tcltk TCLDIR=..\..\tcl85 install + +Now you should have a tcktk\bin, tcltk\lib, and tcltk\include directory ready +for use. The release packaging script will pick up the tcltk runtime in the lib +directory and put it in the archive. + Using the mingw compiler ------------------------ diff --git a/pypy/tool/release/package.py b/pypy/tool/release/package.py --- a/pypy/tool/release/package.py +++ b/pypy/tool/release/package.py @@ -67,18 +67,22 @@ raise PyPyCNotFound( 'Bogus path: %r does not exist (see docstring for more info)' % (os.path.dirname(str(pypy_c)),)) + win_extras = ['libpypy-c.dll', 'libexpat.dll', 'sqlite3.dll', + 'libeay32.dll', 'ssleay32.dll'] subprocess.check_call([str(pypy_c), '-c', 'import _sqlite3']) if not sys.platform == 'win32': subprocess.check_call([str(pypy_c), '-c', 'import _curses']) subprocess.check_call([str(pypy_c), '-c', 'import syslog']) - if not withouttk: - try: - subprocess.check_call([str(pypy_c), '-c', 'import _tkinter']) - except subprocess.CalledProcessError: - print >>sys.stderr, """Building Tk bindings failed. + if not withouttk: + try: + subprocess.check_call([str(pypy_c), '-c', 'import _tkinter']) + except subprocess.CalledProcessError: + print >>sys.stderr, """Building Tk bindings failed. You can either install Tk development headers package or add --without-tk option to skip packaging binary CFFI extension.""" - sys.exit(1) + sys.exit(1) + #Can the dependencies be found from cffi somehow? + win_extras += ['tcl85.dll', 'tk85.dll'] if sys.platform == 'win32' and not rename_pypy_c.lower().endswith('.exe'): rename_pypy_c += '.exe' binaries = [(pypy_c, rename_pypy_c)] @@ -101,9 +105,7 @@ # Can't rename a DLL: it is always called 'libpypy-c.dll' - for extra in ['libpypy-c.dll', - 'libexpat.dll', 'sqlite3.dll', - 'libeay32.dll', 'ssleay32.dll']: + for extra in win_extras: p = pypy_c.dirpath().join(extra) if not p.check(): p = py.path.local.sysfind(extra) @@ -122,6 +124,19 @@ # XXX users will complain that they cannot compile cpyext # modules for windows, has the lib moved or are there no # exported functions in the dll so no import library is created? + if not withouttk: + try: + p = pypy_c.dirpath().join('tcl85.dll') + if not p.check(): + p = py.path.local.sysfind('tcl85.dll') + tktcldir = p.dirpath().join('..').join('lib') + shutil.copytree(str(tktcldir), str(pypydir.join('tcl'))) + except WindowsError: + print >>sys.stderr, """Packaging Tk runtime failed. +tk85.dll and tcl85.dll found, expecting to find runtime in ..\\lib +directory next to the dlls, as per build instructions.""" + import traceback;traceback.print_exc() + sys.exit(1) # Careful: to copy lib_pypy, copying just the hg-tracked files # would not be enough: there are also ctypes_config_cache/_*_cache.py. From noreply at buildbot.pypy.org Mon Nov 18 12:14:39 2013 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 18 Nov 2013 12:14:39 +0100 (CET) Subject: [pypy-commit] pypy release-2.2.x: Merge the non-doc parts of the 'windows-packaging' branch here. Message-ID: <20131118111439.D0B391C026D@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: release-2.2.x Changeset: r68207:1567dba349e6 Date: 2013-11-18 12:14 +0100 http://bitbucket.org/pypy/pypy/changeset/1567dba349e6/ Log: Merge the non-doc parts of the 'windows-packaging' branch here. diff --git a/lib_pypy/_tkinter/tklib.py b/lib_pypy/_tkinter/tklib.py --- a/lib_pypy/_tkinter/tklib.py +++ b/lib_pypy/_tkinter/tklib.py @@ -112,6 +112,10 @@ incdirs = ['/usr/local/include/tcl8.5', '/usr/local/include/tk8.5', '/usr/X11R6/include'] linklibs = ['tk85', 'tcl85'] libdirs = ['/usr/local/lib', '/usr/X11R6/lib'] +elif sys.platform == 'win32': + incdirs = [] + linklibs = ['tcl85', 'tk85'] + libdirs = [] else: incdirs=['/usr/include/tcl'] linklibs=['tcl', 'tk'] diff --git a/pypy/tool/release/package.py b/pypy/tool/release/package.py --- a/pypy/tool/release/package.py +++ b/pypy/tool/release/package.py @@ -67,18 +67,22 @@ raise PyPyCNotFound( 'Bogus path: %r does not exist (see docstring for more info)' % (os.path.dirname(str(pypy_c)),)) + win_extras = ['libpypy-c.dll', 'libexpat.dll', 'sqlite3.dll', + 'libeay32.dll', 'ssleay32.dll'] subprocess.check_call([str(pypy_c), '-c', 'import _sqlite3']) if not sys.platform == 'win32': subprocess.check_call([str(pypy_c), '-c', 'import _curses']) subprocess.check_call([str(pypy_c), '-c', 'import syslog']) - if not withouttk: - try: - subprocess.check_call([str(pypy_c), '-c', 'import _tkinter']) - except subprocess.CalledProcessError: - print >>sys.stderr, """Building Tk bindings failed. + if not withouttk: + try: + subprocess.check_call([str(pypy_c), '-c', 'import _tkinter']) + except subprocess.CalledProcessError: + print >>sys.stderr, """Building Tk bindings failed. You can either install Tk development headers package or add --without-tk option to skip packaging binary CFFI extension.""" - sys.exit(1) + sys.exit(1) + #Can the dependencies be found from cffi somehow? + win_extras += ['tcl85.dll', 'tk85.dll'] if sys.platform == 'win32' and not rename_pypy_c.lower().endswith('.exe'): rename_pypy_c += '.exe' binaries = [(pypy_c, rename_pypy_c)] @@ -101,9 +105,7 @@ # Can't rename a DLL: it is always called 'libpypy-c.dll' - for extra in ['libpypy-c.dll', - 'libexpat.dll', 'sqlite3.dll', - 'libeay32.dll', 'ssleay32.dll']: + for extra in win_extras: p = pypy_c.dirpath().join(extra) if not p.check(): p = py.path.local.sysfind(extra) @@ -122,6 +124,19 @@ # XXX users will complain that they cannot compile cpyext # modules for windows, has the lib moved or are there no # exported functions in the dll so no import library is created? + if not withouttk: + try: + p = pypy_c.dirpath().join('tcl85.dll') + if not p.check(): + p = py.path.local.sysfind('tcl85.dll') + tktcldir = p.dirpath().join('..').join('lib') + shutil.copytree(str(tktcldir), str(pypydir.join('tcl'))) + except WindowsError: + print >>sys.stderr, """Packaging Tk runtime failed. +tk85.dll and tcl85.dll found, expecting to find runtime in ..\\lib +directory next to the dlls, as per build instructions.""" + import traceback;traceback.print_exc() + sys.exit(1) # Careful: to copy lib_pypy, copying just the hg-tracked files # would not be enough: there are also ctypes_config_cache/_*_cache.py. From noreply at buildbot.pypy.org Mon Nov 18 13:42:08 2013 From: noreply at buildbot.pypy.org (fijal) Date: Mon, 18 Nov 2013 13:42:08 +0100 (CET) Subject: [pypy-commit] pypy release-2.2.x: make it possible to disable suprocess with an env var Message-ID: <20131118124208.73AEE1C02C5@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: release-2.2.x Changeset: r68208:1f5ff8d1ee6e Date: 2013-11-18 13:41 +0100 http://bitbucket.org/pypy/pypy/changeset/1f5ff8d1ee6e/ Log: make it possible to disable suprocess with an env var diff --git a/rpython/tool/runsubprocess.py b/rpython/tool/runsubprocess.py --- a/rpython/tool/runsubprocess.py +++ b/rpython/tool/runsubprocess.py @@ -49,7 +49,7 @@ sys.stdout.flush() -if sys.platform != 'win32' and hasattr(os, 'fork'): +if sys.platform != 'win32' and hasattr(os, 'fork') and not os.getenv("PYPY_DONT_RUN_SUBPROCESS", None): # do this at import-time, when the process is still tiny _source = os.path.dirname(os.path.abspath(__file__)) _source = os.path.join(_source, 'runsubprocess.py') # and not e.g. '.pyc' From noreply at buildbot.pypy.org Mon Nov 18 14:26:12 2013 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 18 Nov 2013 14:26:12 +0100 (CET) Subject: [pypy-commit] cffi default: Bah. "print >> sys.stderr, ..." is syntactically valid on Python 3, Message-ID: <20131118132612.B82B41C00EC@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r1422:c1c4583b0bd1 Date: 2013-11-18 14:26 +0100 http://bitbucket.org/cffi/cffi/changeset/c1c4583b0bd1/ Log: Bah. "print >> sys.stderr, ..." is syntactically valid on Python 3, but does nonsense. diff --git a/setup.py b/setup.py --- a/setup.py +++ b/setup.py @@ -48,8 +48,8 @@ try: compiler.compile(['c/check__thread.c']) except distutils.errors.CompileError: - print >> sys.stderr, "the above error message can be safely ignored;" - print >> sys.stderr, "will not use '__thread' in the C code" + sys.stderr.write("the above error message can be safely ignored;\n") + sys.stderr.write("will not use '__thread' in the C code\n") else: define_macros.append(('USE__THREAD', None)) try: From noreply at buildbot.pypy.org Mon Nov 18 15:13:40 2013 From: noreply at buildbot.pypy.org (oberstet) Date: Mon, 18 Nov 2013 15:13:40 +0100 (CET) Subject: [pypy-commit] pypy default: allow disable stripping/tk via env var Message-ID: <20131118141340.933931C00EC@cobra.cs.uni-duesseldorf.de> Author: Tobias Oberstein Branch: Changeset: r68209:679ca2fdefb6 Date: 2013-11-18 14:04 +0100 http://bitbucket.org/pypy/pypy/changeset/679ca2fdefb6/ Log: allow disable stripping/tk via env var diff --git a/pypy/tool/release/package.py b/pypy/tool/release/package.py --- a/pypy/tool/release/package.py +++ b/pypy/tool/release/package.py @@ -217,5 +217,11 @@ else: print_usage() + if os.environ.has_key("PYPY_PACKAGE_NOSTRIP"): + kw['nostrip'] = True + + if os.environ.has_key("PYPY_PACKAGE_WITHOUTTK"): + kw['withouttk'] = True + args = args[i:] package(*args, **kw) From noreply at buildbot.pypy.org Mon Nov 18 15:13:41 2013 From: noreply at buildbot.pypy.org (oberstet) Date: Mon, 18 Nov 2013 15:13:41 +0100 (CET) Subject: [pypy-commit] pypy default: allow disable strip/tk via env vars Message-ID: <20131118141341.C90851C026D@cobra.cs.uni-duesseldorf.de> Author: Tobias Oberstein Branch: Changeset: r68210:76bc9f2cd215 Date: 2013-11-18 14:10 +0100 http://bitbucket.org/pypy/pypy/changeset/76bc9f2cd215/ Log: allow disable strip/tk via env vars diff --git a/pypy/tool/release/package.py b/pypy/tool/release/package.py --- a/pypy/tool/release/package.py +++ b/pypy/tool/release/package.py @@ -232,5 +232,11 @@ else: print_usage() + if os.environ.has_key("PYPY_PACKAGE_NOSTRIP"): + kw['nostrip'] = True + + if os.environ.has_key("PYPY_PACKAGE_WITHOUTTK"): + kw['withouttk'] = True + args = args[i:] package(*args, **kw) From noreply at buildbot.pypy.org Mon Nov 18 15:13:42 2013 From: noreply at buildbot.pypy.org (oberstet) Date: Mon, 18 Nov 2013 15:13:42 +0100 (CET) Subject: [pypy-commit] pypy default: allow disable strip/tk via env vars Message-ID: <20131118141342.DDF1F1C00EC@cobra.cs.uni-duesseldorf.de> Author: Tobias Oberstein Branch: Changeset: r68211:0924956bd436 Date: 2013-11-18 14:11 +0100 http://bitbucket.org/pypy/pypy/changeset/0924956bd436/ Log: allow disable strip/tk via env vars From noreply at buildbot.pypy.org Mon Nov 18 15:13:43 2013 From: noreply at buildbot.pypy.org (oberstet) Date: Mon, 18 Nov 2013 15:13:43 +0100 (CET) Subject: [pypy-commit] pypy default: FreeBSD 9.2 / Tcl/Tk 8.6 paths include and lib paths Message-ID: <20131118141343.EF0841C00EC@cobra.cs.uni-duesseldorf.de> Author: Tobias Oberstein Branch: Changeset: r68212:68fe795f0c67 Date: 2013-11-18 14:18 +0100 http://bitbucket.org/pypy/pypy/changeset/68fe795f0c67/ Log: FreeBSD 9.2 / Tcl/Tk 8.6 paths include and lib paths diff --git a/lib_pypy/_tkinter/tklib.py b/lib_pypy/_tkinter/tklib.py --- a/lib_pypy/_tkinter/tklib.py +++ b/lib_pypy/_tkinter/tklib.py @@ -112,6 +112,10 @@ incdirs = ['/usr/local/include/tcl8.5', '/usr/local/include/tk8.5', '/usr/X11R6/include'] linklibs = ['tk85', 'tcl85'] libdirs = ['/usr/local/lib', '/usr/X11R6/lib'] +elif sys.platform.startswith("freebsd"): + incdirs = ['/usr/local/include/tcl8.6', '/usr/local/include/tk8.6', '/usr/local/include/X11'] + linklibs = ['tk86', 'tcl86'] + libdirs = ['/usr/local/lib'] elif sys.platform == 'win32': incdirs = [] linklibs = ['tcl85', 'tk85'] From noreply at buildbot.pypy.org Mon Nov 18 15:13:45 2013 From: noreply at buildbot.pypy.org (oberstet) Date: Mon, 18 Nov 2013 15:13:45 +0100 (CET) Subject: [pypy-commit] pypy default: skip test that assume glibc specific, non-standard-C behavior Message-ID: <20131118141345.2FBA41C00EC@cobra.cs.uni-duesseldorf.de> Author: Tobias Oberstein Branch: Changeset: r68213:dc08f917cd4f Date: 2013-11-18 14:43 +0100 http://bitbucket.org/pypy/pypy/changeset/dc08f917cd4f/ Log: skip test that assume glibc specific, non-standard-C behavior diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py --- a/pypy/module/_cffi_backend/test/_backend_test_c.py +++ b/pypy/module/_cffi_backend/test/_backend_test_c.py @@ -1086,7 +1086,9 @@ assert strlenaddr == cast(BVoidP, strlen) def test_read_variable(): - if sys.platform == 'win32' or sys.platform == 'darwin': + ## FIXME: this test assumes glibc specific behavior, it's not compliant with C standard + ## https://bugs.pypy.org/issue1643 + if sys.platform == 'win32' or sys.platform == 'darwin' or sys.platform.startswith('freebsd'): py.test.skip("untested") BVoidP = new_pointer_type(new_void_type()) ll = find_and_load_library('c') @@ -1094,7 +1096,9 @@ assert stderr == cast(BVoidP, _testfunc(8)) def test_read_variable_as_unknown_length_array(): - if sys.platform == 'win32' or sys.platform == 'darwin': + ## FIXME: this test assumes glibc specific behavior, it's not compliant with C standard + ## https://bugs.pypy.org/issue1643 + if sys.platform == 'win32' or sys.platform == 'darwin' or sys.platform.startswith('freebsd'): py.test.skip("untested") BCharP = new_pointer_type(new_primitive_type("char")) BArray = new_array_type(BCharP, None) @@ -1104,7 +1108,9 @@ # ^^ and not 'char[]', which is basically not allowed and would crash def test_write_variable(): - if sys.platform == 'win32' or sys.platform == 'darwin': + ## FIXME: this test assumes glibc specific behavior, it's not compliant with C standard + ## https://bugs.pypy.org/issue1643 + if sys.platform == 'win32' or sys.platform == 'darwin' or sys.platform.startswith('freebsd'): py.test.skip("untested") BVoidP = new_pointer_type(new_void_type()) ll = find_and_load_library('c') From noreply at buildbot.pypy.org Mon Nov 18 15:14:50 2013 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 18 Nov 2013 15:14:50 +0100 (CET) Subject: [pypy-commit] cffi default: Skip these tests on FreeBSD too Message-ID: <20131118141450.2A4821C00EC@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r1423:8b8f7428bd31 Date: 2013-11-18 15:14 +0100 http://bitbucket.org/cffi/cffi/changeset/8b8f7428bd31/ Log: Skip these tests on FreeBSD too diff --git a/c/test_c.py b/c/test_c.py --- a/c/test_c.py +++ b/c/test_c.py @@ -1097,7 +1097,9 @@ assert strlenaddr == cast(BVoidP, strlen) def test_read_variable(): - if sys.platform == 'win32' or sys.platform == 'darwin': + ## FIXME: this test assumes glibc specific behavior, it's not compliant with C standard + ## https://bugs.pypy.org/issue1643 + if sys.platform == 'win32' or sys.platform == 'darwin' or sys.platform.startswith('freebsd'): py.test.skip("untested") BVoidP = new_pointer_type(new_void_type()) ll = find_and_load_library('c') @@ -1105,7 +1107,9 @@ assert stderr == cast(BVoidP, _testfunc(8)) def test_read_variable_as_unknown_length_array(): - if sys.platform == 'win32' or sys.platform == 'darwin': + ## FIXME: this test assumes glibc specific behavior, it's not compliant with C standard + ## https://bugs.pypy.org/issue1643 + if sys.platform == 'win32' or sys.platform == 'darwin' or sys.platform.startswith('freebsd'): py.test.skip("untested") BCharP = new_pointer_type(new_primitive_type("char")) BArray = new_array_type(BCharP, None) @@ -1115,7 +1119,9 @@ # ^^ and not 'char[]', which is basically not allowed and would crash def test_write_variable(): - if sys.platform == 'win32' or sys.platform == 'darwin': + ## FIXME: this test assumes glibc specific behavior, it's not compliant with C standard + ## https://bugs.pypy.org/issue1643 + if sys.platform == 'win32' or sys.platform == 'darwin' or sys.platform.startswith('freebsd'): py.test.skip("untested") BVoidP = new_pointer_type(new_void_type()) ll = find_and_load_library('c') From noreply at buildbot.pypy.org Mon Nov 18 17:43:11 2013 From: noreply at buildbot.pypy.org (rguillebert) Date: Mon, 18 Nov 2013 17:43:11 +0100 (CET) Subject: [pypy-commit] pypy numpypy-array_prepare_-array_wrap: Revert Ufunc{1, 2}.call Message-ID: <20131118164311.0725F1C026D@cobra.cs.uni-duesseldorf.de> Author: Romain Guillebert Branch: numpypy-array_prepare_-array_wrap Changeset: r68214:d18c4175125e Date: 2013-11-18 17:39 +0100 http://bitbucket.org/pypy/pypy/changeset/d18c4175125e/ Log: Revert Ufunc{1,2}.call diff --git a/pypy/module/micronumpy/interp_ufuncs.py b/pypy/module/micronumpy/interp_ufuncs.py --- a/pypy/module/micronumpy/interp_ufuncs.py +++ b/pypy/module/micronumpy/interp_ufuncs.py @@ -343,17 +343,15 @@ w_obj.get_scalar_value().convert_to(calc_dtype)) if out is None: return w_val - if isinstance(out, W_NDimArray): - if out.is_scalar(): - out.set_scalar_value(w_val) - else: - out.fill(res_dtype.coerce(space, w_val)) - return self.call_prepare(space, out, w_obj, w_val) + if out.is_scalar(): + out.set_scalar_value(w_val) + else: + out.fill(res_dtype.coerce(space, w_val)) + return out shape = shape_agreement(space, w_obj.get_shape(), out, broadcast_down=False) - w_result = loop.call1(space, shape, self.func, calc_dtype, res_dtype, + return loop.call1(space, shape, self.func, calc_dtype, res_dtype, w_obj, out) - return self.call_prepare(space, out, w_obj, w_result) class W_Ufunc2(W_Ufunc): @@ -423,11 +421,11 @@ promote_bools=self.promote_bools) if space.is_none(w_out): out = None - #elif not isinstance(w_out, W_NDimArray): - # raise OperationError(space.w_TypeError, space.wrap( - # 'output must be an array')) + elif not isinstance(w_out, W_NDimArray): + raise OperationError(space.w_TypeError, space.wrap( + 'output must be an array')) else: - out = convert_to_array(space, w_out) + out = w_out calc_dtype = out.get_dtype() if self.comparison_func: res_dtype = interp_dtype.get_dtype_cache(space).w_booldtype @@ -443,15 +441,14 @@ out.set_scalar_value(arr) else: out.fill(arr) - arr = out - # XXX handle array_priority - return self.call_prepare(space, out, w_lhs, arr) + else: + out = arr + return out new_shape = shape_agreement(space, w_lhs.get_shape(), w_rhs) new_shape = shape_agreement(space, new_shape, out, broadcast_down=False) - w_result = loop.call2(space, new_shape, self.func, calc_dtype, + return loop.call2(space, new_shape, self.func, calc_dtype, res_dtype, w_lhs, w_rhs, out) - # XXX handle array_priority - return self.call_prepare(space, out, w_lhs, w_result) + W_Ufunc.typedef = TypeDef("ufunc", __module__ = "numpypy", From noreply at buildbot.pypy.org Mon Nov 18 17:43:12 2013 From: noreply at buildbot.pypy.org (rguillebert) Date: Mon, 18 Nov 2013 17:43:12 +0100 (CET) Subject: [pypy-commit] pypy numpypy-array_prepare_-array_wrap: Move call_prepare to loop.py Message-ID: <20131118164312.272211C026D@cobra.cs.uni-duesseldorf.de> Author: Romain Guillebert Branch: numpypy-array_prepare_-array_wrap Changeset: r68215:145ff824d697 Date: 2013-11-18 17:42 +0100 http://bitbucket.org/pypy/pypy/changeset/145ff824d697/ Log: Move call_prepare to loop.py diff --git a/pypy/module/micronumpy/interp_ufuncs.py b/pypy/module/micronumpy/interp_ufuncs.py --- a/pypy/module/micronumpy/interp_ufuncs.py +++ b/pypy/module/micronumpy/interp_ufuncs.py @@ -256,38 +256,6 @@ return out return res - def call_prepare(self, space, w_out, w_obj, w_result): - if isinstance(w_out, W_NDimArray): - w_array = space.lookup(w_out, "__array_prepare__") - w_caller = w_out - else: - w_array = space.lookup(w_obj, "__array_prepare__") - w_caller = w_obj - if w_array: - w_retVal = space.get_and_call_function(w_array, w_caller, w_result, None) - if not isinstance(w_retVal, W_NDimArray) and \ - not isinstance(w_retVal, interp_boxes.Box): - raise OperationError(space.w_ValueError, - space.wrap( "__array_prepare__ must return an " - "ndarray or subclass thereof")) - if isinstance(w_result, interp_boxes.Box) or \ - w_result.is_scalar(): - if not isinstance(w_retVal, interp_boxes.Box) and not w_retVal.is_scalar(): - raise OperationError(space.w_TypeError, - space.wrap( "__array_prepare__ must return an " - "ndarray or subclass thereof which is " - "otherwise identical to its input")) - elif w_result.get_shape() != w_retVal.get_shape() or \ - w_result.implementation.get_strides() != \ - w_retVal.implementation.get_strides(): - raise OperationError(space.w_TypeError, - space.wrap( "__array_prepare__ must return an " - "ndarray or subclass thereof which is " - "otherwise identical to its input")) - return w_retVal - return w_result - - class W_Ufunc1(W_Ufunc): _immutable_fields_ = ["func", "bool_result"] argcount = 1 diff --git a/pypy/module/micronumpy/loop.py b/pypy/module/micronumpy/loop.py --- a/pypy/module/micronumpy/loop.py +++ b/pypy/module/micronumpy/loop.py @@ -19,6 +19,37 @@ reds = ['shape', 'w_lhs', 'w_rhs', 'out', 'left_iter', 'right_iter', 'out_iter']) +def call_prepare(self, space, w_out, w_obj, w_result): + if isinstance(w_out, W_NDimArray): + w_array = space.lookup(w_out, "__array_prepare__") + w_caller = w_out + else: + w_array = space.lookup(w_obj, "__array_prepare__") + w_caller = w_obj + if w_array: + w_retVal = space.get_and_call_function(w_array, w_caller, w_result, None) + if not isinstance(w_retVal, W_NDimArray) and \ + not isinstance(w_retVal, interp_boxes.Box): + raise OperationError(space.w_ValueError, + space.wrap( "__array_prepare__ must return an " + "ndarray or subclass thereof")) + if isinstance(w_result, interp_boxes.Box) or \ + w_result.is_scalar(): + if not isinstance(w_retVal, interp_boxes.Box) and not w_retVal.is_scalar(): + raise OperationError(space.w_TypeError, + space.wrap( "__array_prepare__ must return an " + "ndarray or subclass thereof which is " + "otherwise identical to its input")) + elif w_result.get_shape() != w_retVal.get_shape() or \ + w_result.implementation.get_strides() != \ + w_retVal.implementation.get_strides(): + raise OperationError(space.w_TypeError, + space.wrap( "__array_prepare__ must return an " + "ndarray or subclass thereof which is " + "otherwise identical to its input")) + return w_retVal + return w_result + def call2(space, shape, func, calc_dtype, res_dtype, w_lhs, w_rhs, out): # handle array_priority # w_lhs and w_rhs could be of different ndarray subtypes. Numpy does: From noreply at buildbot.pypy.org Mon Nov 18 18:07:54 2013 From: noreply at buildbot.pypy.org (oberstet) Date: Mon, 18 Nov 2013 18:07:54 +0100 (CET) Subject: [pypy-commit] pypy default: C header include paths .. once more Message-ID: <20131118170754.1F5D01C036B@cobra.cs.uni-duesseldorf.de> Author: Tobias Oberstein Branch: Changeset: r68216:b6c763cd5357 Date: 2013-11-18 17:08 +0100 http://bitbucket.org/pypy/pypy/changeset/b6c763cd5357/ Log: C header include paths .. once more diff --git a/lib_pypy/_tkinter/tklib.py b/lib_pypy/_tkinter/tklib.py --- a/lib_pypy/_tkinter/tklib.py +++ b/lib_pypy/_tkinter/tklib.py @@ -113,7 +113,7 @@ linklibs = ['tk85', 'tcl85'] libdirs = ['/usr/local/lib', '/usr/X11R6/lib'] elif sys.platform.startswith("freebsd"): - incdirs = ['/usr/local/include/tcl8.6', '/usr/local/include/tk8.6', '/usr/local/include/X11'] + incdirs = ['/usr/local/include/tcl8.6', '/usr/local/include/tk8.6', '/usr/local/include/X11', '/usr/local/include'] linklibs = ['tk86', 'tcl86'] libdirs = ['/usr/local/lib'] elif sys.platform == 'win32': From noreply at buildbot.pypy.org Mon Nov 18 18:31:59 2013 From: noreply at buildbot.pypy.org (rguillebert) Date: Mon, 18 Nov 2013 18:31:59 +0100 (CET) Subject: [pypy-commit] pypy numpypy-array_prepare_-array_wrap: Remove useless code Message-ID: <20131118173159.1AAB91C026D@cobra.cs.uni-duesseldorf.de> Author: Romain Guillebert Branch: numpypy-array_prepare_-array_wrap Changeset: r68217:5848d1c58c12 Date: 2013-11-18 18:30 +0100 http://bitbucket.org/pypy/pypy/changeset/5848d1c58c12/ Log: Remove useless code diff --git a/pypy/module/micronumpy/interp_ufuncs.py b/pypy/module/micronumpy/interp_ufuncs.py --- a/pypy/module/micronumpy/interp_ufuncs.py +++ b/pypy/module/micronumpy/interp_ufuncs.py @@ -173,7 +173,6 @@ shapelen = len(obj_shape) axis = unwrap_axis_arg(space, shapelen, w_axis) assert axis >= 0 - size = obj.get_size() dtype = interp_dtype.decode_w_dtype(space, dtype) if dtype is None: if self.comparison_func: From noreply at buildbot.pypy.org Mon Nov 18 18:32:00 2013 From: noreply at buildbot.pypy.org (rguillebert) Date: Mon, 18 Nov 2013 18:32:00 +0100 (CET) Subject: [pypy-commit] pypy numpypy-array_prepare_-array_wrap: Implement __array_prepare__ for non-scalar Message-ID: <20131118173200.4433B1C02D8@cobra.cs.uni-duesseldorf.de> Author: Romain Guillebert Branch: numpypy-array_prepare_-array_wrap Changeset: r68218:25afd81e613b Date: 2013-11-18 18:31 +0100 http://bitbucket.org/pypy/pypy/changeset/25afd81e613b/ Log: Implement __array_prepare__ for non-scalar diff --git a/pypy/module/micronumpy/loop.py b/pypy/module/micronumpy/loop.py --- a/pypy/module/micronumpy/loop.py +++ b/pypy/module/micronumpy/loop.py @@ -12,22 +12,12 @@ from pypy.module.micronumpy.iter import PureShapeIterator from pypy.module.micronumpy import constants from pypy.module.micronumpy.support import int_w +from pypy.module.micronumpy import interp_boxes -call2_driver = jit.JitDriver(name='numpy_call2', - greens = ['shapelen', 'func', 'calc_dtype', - 'res_dtype'], - reds = ['shape', 'w_lhs', 'w_rhs', 'out', - 'left_iter', 'right_iter', 'out_iter']) - -def call_prepare(self, space, w_out, w_obj, w_result): - if isinstance(w_out, W_NDimArray): - w_array = space.lookup(w_out, "__array_prepare__") - w_caller = w_out - else: - w_array = space.lookup(w_obj, "__array_prepare__") - w_caller = w_obj +def call_prepare(space, w_obj, w_result): + w_array = space.lookup(w_obj, "__array_prepare__") if w_array: - w_retVal = space.get_and_call_function(w_array, w_caller, w_result, None) + w_retVal = space.get_and_call_function(w_array, w_obj, w_result, None) if not isinstance(w_retVal, W_NDimArray) and \ not isinstance(w_retVal, interp_boxes.Box): raise OperationError(space.w_ValueError, @@ -50,6 +40,11 @@ return w_retVal return w_result +call2_driver = jit.JitDriver(name='numpy_call2', + greens = ['shapelen', 'func', 'calc_dtype', + 'res_dtype'], + reds = ['shape', 'w_lhs', 'w_rhs', 'out', + 'left_iter', 'right_iter', 'out_iter']) def call2(space, shape, func, calc_dtype, res_dtype, w_lhs, w_rhs, out): # handle array_priority # w_lhs and w_rhs could be of different ndarray subtypes. Numpy does: @@ -78,6 +73,10 @@ if out is None: out = W_NDimArray.from_shape(space, shape, res_dtype, w_instance=lhs_for_subtype) + out = call_prepare(space, w_lhs, out) + else: + out = call_prepare(space, out, out) + left_iter = w_lhs.create_iter(shape) right_iter = w_rhs.create_iter(shape) out_iter = out.create_iter(shape) @@ -107,6 +106,9 @@ def call1(space, shape, func, calc_dtype, res_dtype, w_obj, out): if out is None: out = W_NDimArray.from_shape(space, shape, res_dtype, w_instance=w_obj) + out = call_prepare(space, w_obj, out) + else: + out = call_prepare(space, out, out) obj_iter = w_obj.create_iter(shape) out_iter = out.create_iter(shape) shapelen = len(shape) diff --git a/pypy/module/micronumpy/test/test_subtype.py b/pypy/module/micronumpy/test/test_subtype.py --- a/pypy/module/micronumpy/test/test_subtype.py +++ b/pypy/module/micronumpy/test/test_subtype.py @@ -260,7 +260,7 @@ assert type(x) == ndarray assert a.called_wrap - def test___array_prepare__2arg(self): + def test___array_prepare__2arg_scalar(self): from numpypy import ndarray, array, add, ones class with_prepare(ndarray): def __array_prepare__(self, arr, context): @@ -287,7 +287,7 @@ assert x.called_prepare raises(TypeError, add, a, b, out=c) - def test___array_prepare__1arg(self): + def test___array_prepare__1arg_scalar(self): from numpypy import ndarray, array, log, ones class with_prepare(ndarray): def __array_prepare__(self, arr, context): @@ -316,6 +316,61 @@ assert x.called_prepare raises(TypeError, log, a, out=c) + def test___array_prepare__2arg_array(self): + from numpypy import ndarray, array, add, ones + class with_prepare(ndarray): + def __array_prepare__(self, arr, context): + retVal = array(arr).view(type=with_prepare) + retVal.called_prepare = True + return retVal + class with_prepare_fail(ndarray): + called_prepare = False + def __array_prepare__(self, arr, context): + return array(arr[0]).view(type=with_prepare) + a = array([1]) + b = array([1]).view(type=with_prepare) + x = add(a, a, out=b) + assert x == 2 + assert type(x) == with_prepare + assert x.called_prepare + b.called_prepare = False + a = ones((3, 2)).view(type=with_prepare) + b = ones((3, 2)) + c = ones((3, 2)).view(type=with_prepare_fail) + x = add(a, b, out=a) + assert (x == 2).all() + assert type(x) == with_prepare + assert x.called_prepare + raises(TypeError, add, a, b, out=c) + + def test___array_prepare__1arg_array(self): + from numpypy import ndarray, array, log, ones + class with_prepare(ndarray): + def __array_prepare__(self, arr, context): + retVal = array(arr).view(type=with_prepare) + retVal.called_prepare = True + return retVal + class with_prepare_fail(ndarray): + def __array_prepare__(self, arr, context): + return array(arr[0]).view(type=with_prepare) + a = array([1]) + b = array([1]).view(type=with_prepare) + print 'xxxxxxxxxxxxxxxxxxxxxxxxxxxxx' + x = log(a, out=b) + print 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' + assert x == 0 + print 'xxxxxxxxxxxxxxxxxxxxxxxxxxxxx' + assert type(x) == with_prepare + assert x.called_prepare + x.called_prepare = False + a = ones((3, 2)).view(type=with_prepare) + b = ones((3, 2)) + c = ones((3, 2)).view(type=with_prepare_fail) + x = log(a) + assert (x == 0).all() + assert type(x) == with_prepare + assert x.called_prepare + raises(TypeError, log, a, out=c) def test___array_prepare__reduce(self): from numpypy import ndarray, array, sum, ones, add From noreply at buildbot.pypy.org Mon Nov 18 20:42:14 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Mon, 18 Nov 2013 20:42:14 +0100 (CET) Subject: [pypy-commit] pypy default: fix indexing using array scalars Message-ID: <20131118194214.206E71C00EC@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r68219:f5dbd427526c Date: 2013-11-18 14:41 -0500 http://bitbucket.org/pypy/pypy/changeset/f5dbd427526c/ Log: fix indexing using array scalars diff --git a/pypy/module/micronumpy/arrayimpl/concrete.py b/pypy/module/micronumpy/arrayimpl/concrete.py --- a/pypy/module/micronumpy/arrayimpl/concrete.py +++ b/pypy/module/micronumpy/arrayimpl/concrete.py @@ -211,7 +211,15 @@ "field named %s not found" % idx)) return RecordChunk(idx) if (space.isinstance_w(w_idx, space.w_int) or - space.isinstance_w(w_idx, space.w_slice)): + space.isinstance_w(w_idx, space.w_slice)): + return Chunks([Chunk(*space.decode_index4(w_idx, self.get_shape()[0]))]) + elif isinstance(w_idx, W_NDimArray) and \ + isinstance(w_idx.implementation, scalar.Scalar): + w_idx = w_idx.get_scalar_value().item(space) + if not space.isinstance_w(w_idx, space.w_int) and \ + not space.isinstance_w(w_idx, space.w_bool): + raise OperationError(space.w_IndexError, space.wrap( + "arrays used as indices must be of integer (or boolean) type")) return Chunks([Chunk(*space.decode_index4(w_idx, self.get_shape()[0]))]) elif space.is_w(w_idx, space.w_None): return Chunks([NewAxisChunk()]) diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py --- a/pypy/module/micronumpy/interp_numarray.py +++ b/pypy/module/micronumpy/interp_numarray.py @@ -198,7 +198,8 @@ prefix) def descr_getitem(self, space, w_idx): - if isinstance(w_idx, W_NDimArray) and w_idx.get_dtype().is_bool_type(): + if isinstance(w_idx, W_NDimArray) and w_idx.get_dtype().is_bool_type() \ + and len(w_idx.get_shape()) > 0: return self.getitem_filter(space, w_idx) try: return self.implementation.descr_getitem(space, self, w_idx) diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py --- a/pypy/module/micronumpy/test/test_numarray.py +++ b/pypy/module/micronumpy/test/test_numarray.py @@ -1862,12 +1862,16 @@ raises(IndexError, "arange(10)[array([10])] = 3") raises(IndexError, "arange(10)[[-11]] = 3") - def test_bool_single_index(self): + def test_array_scalar_index(self): import numpypy as np a = np.array([[1, 2, 3], [4, 5, 6], [7, 8, 9]]) - a[np.array(True)]; skip("broken") # check for crash but skip rest of test until correct + assert (a[np.array(0)] == a[0]).all() + assert (a[np.array(1)] == a[1]).all() + exc = raises(IndexError, "a[np.array(1.1)]") + assert exc.value.message == 'arrays used as indices must be of ' \ + 'integer (or boolean) type' assert (a[np.array(True)] == a[1]).all() assert (a[np.array(False)] == a[0]).all() From noreply at buildbot.pypy.org Mon Nov 18 20:49:13 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Mon, 18 Nov 2013 20:49:13 +0100 (CET) Subject: [pypy-commit] pypy default: fix setitem using array scalars also Message-ID: <20131118194913.65FA21C00EC@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r68220:c16f3a1dbc73 Date: 2013-11-18 14:48 -0500 http://bitbucket.org/pypy/pypy/changeset/c16f3a1dbc73/ Log: fix setitem using array scalars also diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py --- a/pypy/module/micronumpy/interp_numarray.py +++ b/pypy/module/micronumpy/interp_numarray.py @@ -213,7 +213,8 @@ self.implementation.setitem_index(space, index_list, w_value) def descr_setitem(self, space, w_idx, w_value): - if isinstance(w_idx, W_NDimArray) and w_idx.get_dtype().is_bool_type(): + if isinstance(w_idx, W_NDimArray) and w_idx.get_dtype().is_bool_type() \ + and len(w_idx.get_shape()) > 0: self.setitem_filter(space, w_idx, convert_to_array(space, w_value)) return try: diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py --- a/pypy/module/micronumpy/test/test_numarray.py +++ b/pypy/module/micronumpy/test/test_numarray.py @@ -1869,11 +1869,19 @@ [7, 8, 9]]) assert (a[np.array(0)] == a[0]).all() assert (a[np.array(1)] == a[1]).all() + assert (a[np.array(True)] == a[1]).all() + assert (a[np.array(False)] == a[0]).all() exc = raises(IndexError, "a[np.array(1.1)]") assert exc.value.message == 'arrays used as indices must be of ' \ 'integer (or boolean) type' - assert (a[np.array(True)] == a[1]).all() - assert (a[np.array(False)] == a[0]).all() + + a[np.array(1)] = a[2] + assert a[1][1] == 8 + a[np.array(True)] = a[0] + assert a[1][1] == 2 + exc = raises(IndexError, "a[np.array(1.1)] = a[2]") + assert exc.value.message == 'arrays used as indices must be of ' \ + 'integer (or boolean) type' def test_bool_array_index(self): from numpypy import arange, array From noreply at buildbot.pypy.org Mon Nov 18 21:04:43 2013 From: noreply at buildbot.pypy.org (mattip) Date: Mon, 18 Nov 2013 21:04:43 +0100 (CET) Subject: [pypy-commit] pypy voidtype_strformat: add failing test, almost implement Message-ID: <20131118200443.1A9DE1C00EC@cobra.cs.uni-duesseldorf.de> Author: Matti Picus Branch: voidtype_strformat Changeset: r68221:889adc434d6e Date: 2013-11-18 22:03 +0200 http://bitbucket.org/pypy/pypy/changeset/889adc434d6e/ Log: add failing test, almost implement diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py --- a/pypy/module/micronumpy/test/test_numarray.py +++ b/pypy/module/micronumpy/test/test_numarray.py @@ -3099,6 +3099,17 @@ assert len(list(a[0])) == 2 + def test_3d_record(self): + from numpypy import dtype, array + dt = dtype([('name', 'S4'), ('x', float), ('y', float), + ('block', int, (2, 2, 3))]) + a = array([('aaaa', 1.0, 8.0, [[[1, 2, 3], [4, 5, 6]], + [[7, 8, 9], [10, 11, 12]]])], + dtype=dt) + s = str(a) + assert s.endswith("[('aaaa', 1.0, 8.0, [[[1, 2, 3], [4, 5, 6]], [[7, 8, 9], [10, 11, 12]]])]") + + def test_issue_1589(self): import numpypy as numpy c = numpy.array([[(1, 2, 'a'), (3, 4, 'b')], [(5, 6, 'c'), (7, 8, 'd')]], diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py --- a/pypy/module/micronumpy/types.py +++ b/pypy/module/micronumpy/types.py @@ -1789,6 +1789,25 @@ dtype.subdtype) return W_NDimArray(implementation) + def str_format(self, val): + # only called with the results of readarray() + from pypy.module.micronumpy.base import W_NDimArray + assert isinstance(val, W_NDimArray) + i = val.create_iter() + first = True + dtype = val.get_dtype() + s = StringBuilder() + s.append('[') + while not i.done(): + if first: + first = False + else: + s.append(', ') + s.append(dtype.itemtype.str_format(i.getitem())) + i.next() + s.append(']') + return s.build() + class RecordType(FlexibleType): T = lltype.Char @@ -1848,7 +1867,11 @@ first = False else: pieces.append(", ") - pieces.append(tp.str_format(tp.read(box.arr, box.ofs, ofs))) + if isinstance(tp, VoidType): + val = tp.readarray(box.arr, box.ofs, ofs, subdtype) + else: + val = tp.read(box.arr, box.ofs, ofs, subdtype) + pieces.append(tp.str_format(val)) pieces.append(")") return "".join(pieces) From noreply at buildbot.pypy.org Mon Nov 18 22:52:20 2013 From: noreply at buildbot.pypy.org (rlamy) Date: Mon, 18 Nov 2013 22:52:20 +0100 (CET) Subject: [pypy-commit] pypy less-stringly-ops: cleanup ArgErr* Message-ID: <20131118215220.CA1DA1C026D@cobra.cs.uni-duesseldorf.de> Author: Ronan Lamy Branch: less-stringly-ops Changeset: r68222:ad17710e31c4 Date: 2013-10-30 06:25 +0000 http://bitbucket.org/pypy/pypy/changeset/ad17710e31c4/ Log: cleanup ArgErr* diff --git a/rpython/annotator/argument.py b/rpython/annotator/argument.py --- a/rpython/annotator/argument.py +++ b/rpython/annotator/argument.py @@ -105,10 +105,6 @@ # escape num_remainingkwds = len(keywords) for i, name in enumerate(keywords): - # If name was not encoded as a string, it could be None. In that - # case, it's definitely not going to be in the signature. - if name is None: - continue j = signature.find_argname(name) # if j == -1 nothing happens if j < input_argcount: @@ -122,8 +118,8 @@ if num_remainingkwds: if co_argcount == 0: raise ArgErrCount(num_args, num_kwds, signature, defaults_w, 0) - raise ArgErrUnknownKwds(self.space, num_remainingkwds, keywords, - kwds_mapping, self.keyword_names_w) + raise ArgErrUnknownKwds(num_remainingkwds, keywords, + kwds_mapping) # check for missing arguments and fill them from the kwds, # or with defaults, if available @@ -298,31 +294,12 @@ class ArgErrUnknownKwds(ArgErr): - def __init__(self, space, num_remainingkwds, keywords, kwds_mapping, - keyword_names_w): + def __init__(self, num_remainingkwds, keywords, kwds_mapping): name = '' self.num_kwds = num_remainingkwds if num_remainingkwds == 1: - for i in range(len(keywords)): - if i not in kwds_mapping: - name = keywords[i] - if name is None: - # We'll assume it's unicode. Encode it. - # Careful, I *think* it should not be possible to - # get an IndexError here but you never know. - try: - if keyword_names_w is None: - raise IndexError - # note: negative-based indexing from the end - w_name = keyword_names_w[i - len(keywords)] - except IndexError: - name = '?' - else: - w_enc = space.wrap(space.sys.defaultencoding) - w_err = space.wrap("replace") - w_name = space.call_method(w_name, "encode", w_enc, - w_err) - name = space.str_w(w_name) + for name in keywords: + if name not in kwds_mapping: break self.kwd_name = name From noreply at buildbot.pypy.org Mon Nov 18 22:52:22 2013 From: noreply at buildbot.pypy.org (rlamy) Date: Mon, 18 Nov 2013 22:52:22 +0100 (CET) Subject: [pypy-commit] pypy less-stringly-ops: make ArgsFT.keywords always be a list Message-ID: <20131118215222.1B1BD1C026D@cobra.cs.uni-duesseldorf.de> Author: Ronan Lamy Branch: less-stringly-ops Changeset: r68223:9c0b00eeb262 Date: 2013-10-31 03:42 +0000 http://bitbucket.org/pypy/pypy/changeset/9c0b00eeb262/ Log: make ArgsFT.keywords always be a list diff --git a/rpython/annotator/argument.py b/rpython/annotator/argument.py --- a/rpython/annotator/argument.py +++ b/rpython/annotator/argument.py @@ -11,9 +11,8 @@ assert w_starstararg is None assert isinstance(args_w, list) self.arguments_w = args_w - self.keywords = keywords - self.keywords_w = keywords_w - self.keyword_names_w = None + self.keywords = keywords or [] + self.keywords_w = keywords_w or [] def __repr__(self): """ NOT_RPYTHON """ @@ -71,7 +70,7 @@ args_w = self.positional_args num_args = len(args_w) - keywords = self.keywords or [] + keywords = self.keywords num_kwds = len(keywords) # put as many positional input arguments into place as available @@ -145,7 +144,7 @@ def unpack(self): "Return a ([w1,w2...], {'kw':w3...}) pair." - kwds_w = dict(zip(self.keywords, self.keywords_w)) if self.keywords else {} + kwds_w = dict(zip(self.keywords, self.keywords_w)) return self.positional_args, kwds_w def match_signature(self, signature, defaults_w): @@ -177,9 +176,8 @@ assert len(data_w) >= need_cnt args_w = data_w[:need_cnt] _kwds_w = dict(zip(argnames[need_cnt:], data_w[need_cnt:])) - keywords = self.keywords or [] - keywords_w = [_kwds_w[key] for key in keywords] - return ArgumentsForTranslation(args_w, keywords, keywords_w) + keywords_w = [_kwds_w[key] for key in self.keywords] + return ArgumentsForTranslation(args_w, self.keywords, keywords_w) @classmethod def fromshape(cls, (shape_cnt, shape_keys, shape_star, shape_stst), data_w): @@ -210,15 +208,11 @@ return (shape_cnt, shape_keys, shape_star, shape_stst), data_w def _rawshape(self, nextra=0): - shape_cnt = len(self.arguments_w) + nextra # Number of positional args - if self.keywords: - shape_keys = self.keywords[:] # List of keywords (strings) - shape_keys.sort() - else: - shape_keys = [] + shape_cnt = len(self.arguments_w) + nextra # Number of positional args + shape_keys = tuple(sorted(self.keywords)) shape_star = self.w_stararg is not None # Flag: presence of *arg - shape_stst = self.w_starstararg is not None # Flag: presence of **kwds - return shape_cnt, tuple(shape_keys), shape_star, shape_stst # shape_keys are sorted + shape_stst = self.w_starstararg is not None # Flag: presence of **kwds + return shape_cnt, shape_keys, shape_star, shape_stst def rawshape(args, nextra=0): diff --git a/rpython/annotator/test/test_argument.py b/rpython/annotator/test/test_argument.py --- a/rpython/annotator/test/test_argument.py +++ b/rpython/annotator/test/test_argument.py @@ -23,8 +23,8 @@ args1 = args.prepend("thingy") assert args1 is not args assert args1.arguments_w == ["thingy", "0"] - assert args1.keywords is args.keywords - assert args1.keywords_w is args.keywords_w + assert args1.keywords == args.keywords + assert args1.keywords_w == args.keywords_w def test_fixedunpacked(self): args = MockArgs([], ["k"], [1]) From noreply at buildbot.pypy.org Mon Nov 18 22:52:23 2013 From: noreply at buildbot.pypy.org (rlamy) Date: Mon, 18 Nov 2013 22:52:23 +0100 (CET) Subject: [pypy-commit] pypy less-stringly-ops: Turn ArgsFT.keywords into a dict Message-ID: <20131118215223.50F171C026D@cobra.cs.uni-duesseldorf.de> Author: Ronan Lamy Branch: less-stringly-ops Changeset: r68224:ec91da17a267 Date: 2013-10-31 18:40 +0000 http://bitbucket.org/pypy/pypy/changeset/ec91da17a267/ Log: Turn ArgsFT.keywords into a dict diff --git a/rpython/annotator/argument.py b/rpython/annotator/argument.py --- a/rpython/annotator/argument.py +++ b/rpython/annotator/argument.py @@ -5,14 +5,13 @@ class ArgumentsForTranslation(object): w_starstararg = None - def __init__(self, args_w, keywords=None, keywords_w=None, + def __init__(self, args_w, keywords=None, w_stararg=None, w_starstararg=None): self.w_stararg = w_stararg assert w_starstararg is None assert isinstance(args_w, list) self.arguments_w = args_w - self.keywords = keywords or [] - self.keywords_w = keywords_w or [] + self.keywords = keywords or {} def __repr__(self): """ NOT_RPYTHON """ @@ -20,8 +19,7 @@ if not self.keywords: return '%s(%s)' % (name, self.arguments_w,) else: - return '%s(%s, %s, %s)' % (name, self.arguments_w, - self.keywords, self.keywords_w) + return '%s(%s, %s)' % (name, self.arguments_w, self.keywords) @property def positional_args(self): @@ -52,12 +50,12 @@ def prepend(self, w_firstarg): # used often "Return a new Arguments with a new argument inserted first." return ArgumentsForTranslation([w_firstarg] + self.arguments_w, - self.keywords, self.keywords_w, self.w_stararg, + self.keywords, self.w_stararg, self.w_starstararg) def copy(self): return ArgumentsForTranslation(self.arguments_w, self.keywords, - self.keywords_w, self.w_stararg, self.w_starstararg) + self.w_stararg, self.w_starstararg) def _match_signature(self, scope_w, signature, defaults_w=None): """Parse args and kwargs according to the signature of a code object, @@ -92,18 +90,17 @@ # handle keyword arguments num_remainingkwds = 0 - keywords_w = self.keywords_w kwds_mapping = None if num_kwds: # kwds_mapping maps target indexes in the scope (minus input_argcount) - # to positions in the keywords_w list - kwds_mapping = [-1] * (co_argcount - input_argcount) + # to keyword names + kwds_mapping = [] # match the keywords given at the call site to the argument names # the called function takes # this function must not take a scope_w, to make the scope not # escape num_remainingkwds = len(keywords) - for i, name in enumerate(keywords): + for name in keywords: j = signature.find_argname(name) # if j == -1 nothing happens if j < input_argcount: @@ -111,7 +108,7 @@ if j >= 0: raise ArgErrMultipleValues(name) else: - kwds_mapping[j - input_argcount] = i # map to the right index + kwds_mapping.append(name) num_remainingkwds -= 1 if num_remainingkwds: @@ -126,14 +123,11 @@ if input_argcount < co_argcount: def_first = co_argcount - (0 if defaults_w is None else len(defaults_w)) j = 0 - kwds_index = -1 for i in range(input_argcount, co_argcount): - if kwds_mapping is not None: - kwds_index = kwds_mapping[j] - j += 1 - if kwds_index >= 0: - scope_w[i] = keywords_w[kwds_index] - continue + name = signature.argnames[i] + if name in keywords: + scope_w[i] = keywords[name] + continue defnum = i - def_first if defnum >= 0: scope_w[i] = defaults_w[defnum] @@ -144,8 +138,7 @@ def unpack(self): "Return a ([w1,w2...], {'kw':w3...}) pair." - kwds_w = dict(zip(self.keywords, self.keywords_w)) - return self.positional_args, kwds_w + return self.positional_args, self.keywords def match_signature(self, signature, defaults_w): """Parse args and kwargs according to the signature of a code object, @@ -169,7 +162,7 @@ args_w = data_w[:cnt] + stararg_w assert len(args_w) == need_cnt assert not self.keywords - return ArgumentsForTranslation(args_w, [], []) + return ArgumentsForTranslation(args_w, {}) else: data_w = data_w[:-1] assert len(data_w) == cnt @@ -177,7 +170,7 @@ args_w = data_w[:need_cnt] _kwds_w = dict(zip(argnames[need_cnt:], data_w[need_cnt:])) keywords_w = [_kwds_w[key] for key in self.keywords] - return ArgumentsForTranslation(args_w, self.keywords, keywords_w) + return ArgumentsForTranslation(args_w, dict(zip(self.keywords, keywords_w))) @classmethod def fromshape(cls, (shape_cnt, shape_keys, shape_star, shape_stst), data_w): @@ -193,14 +186,13 @@ p += 1 else: w_starstar = None - return cls(args_w, list(shape_keys), data_w[shape_cnt:end_keys], + return cls(args_w, dict(zip(shape_keys, data_w[shape_cnt:end_keys])), w_star, w_starstar) def flatten(self): """ Argument <-> list of w_objects together with "shape" information """ shape_cnt, shape_keys, shape_star, shape_stst = self._rawshape() - data_w = self.arguments_w + [self.keywords_w[self.keywords.index(key)] - for key in shape_keys] + data_w = self.arguments_w + [self.keywords[key] for key in shape_keys] if shape_star: data_w.append(self.w_stararg) if shape_stst: diff --git a/rpython/annotator/test/test_argument.py b/rpython/annotator/test/test_argument.py --- a/rpython/annotator/test/test_argument.py +++ b/rpython/annotator/test/test_argument.py @@ -11,11 +11,6 @@ return list(it) -def make_arguments_for_translation(args_w, keywords_w={}, w_stararg=None, - w_starstararg=None): - return MockArgs(args_w, keywords_w.keys(), keywords_w.values(), - w_stararg, w_starstararg) - class TestArgumentsForTranslation(object): def test_prepend(self): @@ -24,10 +19,9 @@ assert args1 is not args assert args1.arguments_w == ["thingy", "0"] assert args1.keywords == args.keywords - assert args1.keywords_w == args.keywords_w def test_fixedunpacked(self): - args = MockArgs([], ["k"], [1]) + args = MockArgs([], {"k": 1}) py.test.raises(ValueError, args.fixedunpack, 1) args = MockArgs(["a", "b"]) @@ -39,85 +33,84 @@ assert args.fixedunpack(2) == ['a', 'b'] def test_unmatch_signature(self): - args = make_arguments_for_translation([1, 2, 3]) + args = MockArgs([1, 2, 3]) sig = Signature(['a', 'b', 'c'], None, None) data = args.match_signature(sig, []) new_args = args.unmatch_signature(sig, data) assert args.unpack() == new_args.unpack() - args = make_arguments_for_translation([1]) + args = MockArgs([1]) sig = Signature(['a', 'b', 'c'], None, None) data = args.match_signature(sig, [2, 3]) new_args = args.unmatch_signature(sig, data) assert args.unpack() == new_args.unpack() - args = make_arguments_for_translation([1, 2, 3, 4, 5]) + args = MockArgs([1, 2, 3, 4, 5]) sig = Signature(['a', 'b', 'c'], 'r', None) data = args.match_signature(sig, []) new_args = args.unmatch_signature(sig, data) assert args.unpack() == new_args.unpack() - args = make_arguments_for_translation([1], {'c': 3, 'b': 2}) + args = MockArgs([1], {'c': 3, 'b': 2}) sig = Signature(['a', 'b', 'c'], None, None) data = args.match_signature(sig, []) new_args = args.unmatch_signature(sig, data) assert args.unpack() == new_args.unpack() - args = make_arguments_for_translation([1], {'c': 5}) + args = MockArgs([1], {'c': 5}) sig = Signature(['a', 'b', 'c'], None, None) data = args.match_signature(sig, [2, 3]) new_args = args.unmatch_signature(sig, data) assert args.unpack() == new_args.unpack() def test_rawshape(self): - args = make_arguments_for_translation([1, 2, 3]) + args = MockArgs([1, 2, 3]) assert rawshape(args) == (3, (), False, False) - args = make_arguments_for_translation([1]) + args = MockArgs([1]) assert rawshape(args, 2) == (3, (), False, False) - args = make_arguments_for_translation([1, 2, 3, 4, 5]) + args = MockArgs([1, 2, 3, 4, 5]) assert rawshape(args) == (5, (), False, False) - args = make_arguments_for_translation([1], {'c': 3, 'b': 2}) + args = MockArgs([1], {'c': 3, 'b': 2}) assert rawshape(args) == (1, ('b', 'c'), False, False) - args = make_arguments_for_translation([1], {'c': 5}) + args = MockArgs([1], {'c': 5}) assert rawshape(args) == (1, ('c', ), False, False) - args = make_arguments_for_translation([1], {'c': 5, 'd': 7}) + args = MockArgs([1], {'c': 5, 'd': 7}) assert rawshape(args) == (1, ('c', 'd'), False, False) - args = make_arguments_for_translation([1, 2, 3, 4, 5], {'e': 5, 'd': 7}) + args = MockArgs([1, 2, 3, 4, 5], {'e': 5, 'd': 7}) assert rawshape(args) == (5, ('d', 'e'), False, False) def test_flatten(self): - args = make_arguments_for_translation([1, 2, 3]) + args = MockArgs([1, 2, 3]) assert args.flatten() == ((3, (), False, False), [1, 2, 3]) - args = make_arguments_for_translation([1]) + args = MockArgs([1]) assert args.flatten() == ((1, (), False, False), [1]) - args = make_arguments_for_translation([1, 2, 3, 4, 5]) + args = MockArgs([1, 2, 3, 4, 5]) assert args.flatten() == ((5, (), False, False), [1, 2, 3, 4, 5]) - args = make_arguments_for_translation([1], {'c': 3, 'b': 2}) + args = MockArgs([1], {'c': 3, 'b': 2}) assert args.flatten() == ((1, ('b', 'c'), False, False), [1, 2, 3]) - args = make_arguments_for_translation([1], {'c': 5}) + args = MockArgs([1], {'c': 5}) assert args.flatten() == ((1, ('c', ), False, False), [1, 5]) - args = make_arguments_for_translation([1], {'c': 5, 'd': 7}) + args = MockArgs([1], {'c': 5, 'd': 7}) assert args.flatten() == ((1, ('c', 'd'), False, False), [1, 5, 7]) - args = make_arguments_for_translation([1, 2, 3, 4, 5], {'e': 5, 'd': 7}) + args = MockArgs([1, 2, 3, 4, 5], {'e': 5, 'd': 7}) assert args.flatten() == ((5, ('d', 'e'), False, False), [1, 2, 3, 4, 5, 7, 5]) def test_stararg_flowspace_variable(self): var = object() shape = ((2, ('g', ), True, False), [1, 2, 9, var]) - args = make_arguments_for_translation([1, 2], {'g': 9}, - w_stararg=var) + args = MockArgs([1, 2], {'g': 9}, w_stararg=var) assert args.flatten() == shape args = MockArgs.fromshape(*shape) diff --git a/rpython/rtyper/rbuiltin.py b/rpython/rtyper/rbuiltin.py --- a/rpython/rtyper/rbuiltin.py +++ b/rpython/rtyper/rbuiltin.py @@ -74,10 +74,8 @@ raise TyperError("kwds args not supported") # prefix keyword arguments with 'i_' kwds_i = {} - for i, key in enumerate(keywords): - index = arguments.keywords_w[i] - kwds_i['i_' + key] = index - + for key in keywords: + kwds_i['i_' + key] = keywords[key] return hop, kwds_i From noreply at buildbot.pypy.org Mon Nov 18 22:52:24 2013 From: noreply at buildbot.pypy.org (rlamy) Date: Mon, 18 Nov 2013 22:52:24 +0100 (CET) Subject: [pypy-commit] pypy less-stringly-ops: kill all (broken by design) support for **-unpacking Message-ID: <20131118215224.767A01C026D@cobra.cs.uni-duesseldorf.de> Author: Ronan Lamy Branch: less-stringly-ops Changeset: r68225:cda622311685 Date: 2013-11-18 03:49 +0000 http://bitbucket.org/pypy/pypy/changeset/cda622311685/ Log: kill all (broken by design) support for **-unpacking diff --git a/rpython/annotator/argument.py b/rpython/annotator/argument.py --- a/rpython/annotator/argument.py +++ b/rpython/annotator/argument.py @@ -4,11 +4,8 @@ from rpython.annotator.model import SomeTuple class ArgumentsForTranslation(object): - w_starstararg = None - def __init__(self, args_w, keywords=None, - w_stararg=None, w_starstararg=None): + def __init__(self, args_w, keywords=None, w_stararg=None): self.w_stararg = w_stararg - assert w_starstararg is None assert isinstance(args_w, list) self.arguments_w = args_w self.keywords = keywords or {} @@ -50,12 +47,11 @@ def prepend(self, w_firstarg): # used often "Return a new Arguments with a new argument inserted first." return ArgumentsForTranslation([w_firstarg] + self.arguments_w, - self.keywords, self.w_stararg, - self.w_starstararg) + self.keywords, self.w_stararg) def copy(self): return ArgumentsForTranslation(self.arguments_w, self.keywords, - self.w_stararg, self.w_starstararg) + self.w_stararg) def _match_signature(self, scope_w, signature, defaults_w=None): """Parse args and kwargs according to the signature of a code object, @@ -173,7 +169,7 @@ return ArgumentsForTranslation(args_w, dict(zip(self.keywords, keywords_w))) @classmethod - def fromshape(cls, (shape_cnt, shape_keys, shape_star, shape_stst), data_w): + def fromshape(cls, (shape_cnt, shape_keys, shape_star), data_w): args_w = data_w[:shape_cnt] p = end_keys = shape_cnt + len(shape_keys) if shape_star: @@ -181,30 +177,22 @@ p += 1 else: w_star = None - if shape_stst: - w_starstar = data_w[p] - p += 1 - else: - w_starstar = None return cls(args_w, dict(zip(shape_keys, data_w[shape_cnt:end_keys])), - w_star, w_starstar) + w_star) def flatten(self): """ Argument <-> list of w_objects together with "shape" information """ - shape_cnt, shape_keys, shape_star, shape_stst = self._rawshape() + shape_cnt, shape_keys, shape_star = self._rawshape() data_w = self.arguments_w + [self.keywords[key] for key in shape_keys] if shape_star: data_w.append(self.w_stararg) - if shape_stst: - data_w.append(self.w_starstararg) - return (shape_cnt, shape_keys, shape_star, shape_stst), data_w + return (shape_cnt, shape_keys, shape_star), data_w def _rawshape(self, nextra=0): shape_cnt = len(self.arguments_w) + nextra # Number of positional args shape_keys = tuple(sorted(self.keywords)) shape_star = self.w_stararg is not None # Flag: presence of *arg - shape_stst = self.w_starstararg is not None # Flag: presence of **kwds - return shape_cnt, shape_keys, shape_star, shape_stst + return shape_cnt, shape_keys, shape_star def rawshape(args, nextra=0): diff --git a/rpython/annotator/builtin.py b/rpython/annotator/builtin.py --- a/rpython/annotator/builtin.py +++ b/rpython/annotator/builtin.py @@ -307,7 +307,7 @@ r_func, nimplicitarg = s_repr.const.get_r_implfunc() nbargs = len(args_s) + nimplicitarg - s_sigs = r_func.get_s_signatures((nbargs, (), False, False)) + s_sigs = r_func.get_s_signatures((nbargs, (), False)) if len(s_sigs) != 1: raise TyperError("cannot hlinvoke callable %r with not uniform" "annotations: %r" % (s_repr.const, diff --git a/rpython/annotator/test/test_annrpython.py b/rpython/annotator/test/test_annrpython.py --- a/rpython/annotator/test/test_annrpython.py +++ b/rpython/annotator/test/test_annrpython.py @@ -1065,8 +1065,9 @@ gf2 = graphof(a, f2) gf3 = graphof(a, f3) - assert fam1.calltables == {(2, (), False, False): [{fdesc1: gf1}], (1, (), False, False): [{fdesc1: gf1}]} - assert fam2.calltables == {(1, (), False, False): [{fdesc2: gf2, fdesc3: gf3}]} + assert fam1.calltables == {(2, (), False): [{fdesc1: gf1}], + (1, (), False): [{fdesc1: gf1}]} + assert fam2.calltables == {(1, (), False): [{fdesc2: gf2, fdesc3: gf3}]} def test_pbc_call_ins(self): class A(object): @@ -1117,14 +1118,14 @@ gfA_m = graphof(a, A.m.im_func) gfC_m = graphof(a, C.m.im_func) - assert famB_n.calltables == {(1, (), False, False): [{mdescB_n.funcdesc: gfB_n}] } - assert famA_m.calltables == {(1, (), False, False): [{mdescA_m.funcdesc: gfA_m, mdescC_m.funcdesc: gfC_m }] } + assert famB_n.calltables == {(1, (), False): [{mdescB_n.funcdesc: gfB_n}] } + assert famA_m.calltables == {(1, (), False): [{mdescA_m.funcdesc: gfA_m, mdescC_m.funcdesc: gfC_m }] } mdescCinit = getmdesc(C().__init__) famCinit = mdescCinit.getcallfamily() gfCinit = graphof(a, C.__init__.im_func) - assert famCinit.calltables == {(1, (), False, False): [{mdescCinit.funcdesc: gfCinit}] } + assert famCinit.calltables == {(1, (), False): [{mdescCinit.funcdesc: gfCinit}] } def test_isinstance_usigned(self): def f(x): @@ -2053,7 +2054,7 @@ someint = annmodel.SomeInteger() - assert (fdesc.get_s_signatures((2,(),False,False)) + assert (fdesc.get_s_signatures((2, (), False)) == [([someint,someint],someint)]) def test_emulated_pbc_call_callback(self): @@ -4080,7 +4081,7 @@ with py.test.raises(annmodel.UnionError) as exc: a.build_types(f, [int]) - assert ("RPython cannot unify instances with no common base class" + assert ("RPython cannot unify instances with no common base class" in exc.value.msg) def test_unionerror_iters(self): @@ -4096,7 +4097,7 @@ with py.test.raises(annmodel.UnionError) as exc: a.build_types(f, [int]) - assert ("RPython cannot unify incompatible iterator variants" in + assert ("RPython cannot unify incompatible iterator variants" in exc.value.msg) def test_variable_getattr(self): diff --git a/rpython/annotator/test/test_argument.py b/rpython/annotator/test/test_argument.py --- a/rpython/annotator/test/test_argument.py +++ b/rpython/annotator/test/test_argument.py @@ -65,51 +65,51 @@ def test_rawshape(self): args = MockArgs([1, 2, 3]) - assert rawshape(args) == (3, (), False, False) + assert rawshape(args) == (3, (), False) args = MockArgs([1]) - assert rawshape(args, 2) == (3, (), False, False) + assert rawshape(args, 2) == (3, (), False) args = MockArgs([1, 2, 3, 4, 5]) - assert rawshape(args) == (5, (), False, False) + assert rawshape(args) == (5, (), False) args = MockArgs([1], {'c': 3, 'b': 2}) - assert rawshape(args) == (1, ('b', 'c'), False, False) + assert rawshape(args) == (1, ('b', 'c'), False) args = MockArgs([1], {'c': 5}) - assert rawshape(args) == (1, ('c', ), False, False) + assert rawshape(args) == (1, ('c', ), False) args = MockArgs([1], {'c': 5, 'd': 7}) - assert rawshape(args) == (1, ('c', 'd'), False, False) + assert rawshape(args) == (1, ('c', 'd'), False) args = MockArgs([1, 2, 3, 4, 5], {'e': 5, 'd': 7}) - assert rawshape(args) == (5, ('d', 'e'), False, False) + assert rawshape(args) == (5, ('d', 'e'), False) def test_flatten(self): args = MockArgs([1, 2, 3]) - assert args.flatten() == ((3, (), False, False), [1, 2, 3]) + assert args.flatten() == ((3, (), False), [1, 2, 3]) args = MockArgs([1]) - assert args.flatten() == ((1, (), False, False), [1]) + assert args.flatten() == ((1, (), False), [1]) args = MockArgs([1, 2, 3, 4, 5]) - assert args.flatten() == ((5, (), False, False), [1, 2, 3, 4, 5]) + assert args.flatten() == ((5, (), False), [1, 2, 3, 4, 5]) args = MockArgs([1], {'c': 3, 'b': 2}) - assert args.flatten() == ((1, ('b', 'c'), False, False), [1, 2, 3]) + assert args.flatten() == ((1, ('b', 'c'), False), [1, 2, 3]) args = MockArgs([1], {'c': 5}) - assert args.flatten() == ((1, ('c', ), False, False), [1, 5]) + assert args.flatten() == ((1, ('c', ), False), [1, 5]) args = MockArgs([1], {'c': 5, 'd': 7}) - assert args.flatten() == ((1, ('c', 'd'), False, False), [1, 5, 7]) + assert args.flatten() == ((1, ('c', 'd'), False), [1, 5, 7]) args = MockArgs([1, 2, 3, 4, 5], {'e': 5, 'd': 7}) - assert args.flatten() == ((5, ('d', 'e'), False, False), [1, 2, 3, 4, 5, 7, 5]) + assert args.flatten() == ((5, ('d', 'e'), False), [1, 2, 3, 4, 5, 7, 5]) def test_stararg_flowspace_variable(self): var = object() - shape = ((2, ('g', ), True, False), [1, 2, 9, var]) + shape = ((2, ('g', ), True), [1, 2, 9, var]) args = MockArgs([1, 2], {'g': 9}, w_stararg=var) assert args.flatten() == shape @@ -117,30 +117,30 @@ assert args.flatten() == shape def test_fromshape(self): - shape = ((3, (), False, False), [1, 2, 3]) + shape = ((3, (), False), [1, 2, 3]) args = MockArgs.fromshape(*shape) assert args.flatten() == shape - shape = ((1, (), False, False), [1]) + shape = ((1, (), False), [1]) args = MockArgs.fromshape(*shape) assert args.flatten() == shape - shape = ((5, (), False, False), [1, 2, 3, 4, 5]) + shape = ((5, (), False), [1, 2, 3, 4, 5]) args = MockArgs.fromshape(*shape) assert args.flatten() == shape - shape = ((1, ('b', 'c'), False, False), [1, 2, 3]) + shape = ((1, ('b', 'c'), False), [1, 2, 3]) args = MockArgs.fromshape(*shape) assert args.flatten() == shape - shape = ((1, ('c', ), False, False), [1, 5]) + shape = ((1, ('c', ), False), [1, 5]) args = MockArgs.fromshape(*shape) assert args.flatten() == shape - shape = ((1, ('c', 'd'), False, False), [1, 5, 7]) + shape = ((1, ('c', 'd'), False), [1, 5, 7]) args = MockArgs.fromshape(*shape) assert args.flatten() == shape - shape = ((5, ('d', 'e'), False, False), [1, 2, 3, 4, 5, 7, 5]) + shape = ((5, ('d', 'e'), False), [1, 2, 3, 4, 5, 7, 5]) args = MockArgs.fromshape(*shape) assert args.flatten() == shape diff --git a/rpython/flowspace/argument.py b/rpython/flowspace/argument.py --- a/rpython/flowspace/argument.py +++ b/rpython/flowspace/argument.py @@ -77,10 +77,8 @@ """Represents the arguments passed into a function call, i.e. the `a, b, *c, **d` part in `return func(a, b, *c, **d)`. """ - def __init__(self, args_w, keywords=None, w_stararg=None, - w_starstararg=None): + def __init__(self, args_w, keywords=None, w_stararg=None): self.w_stararg = w_stararg - assert w_starstararg is None, "No **-unpacking in RPython" assert isinstance(args_w, list) self.arguments_w = args_w self.keywords = keywords or {} @@ -90,11 +88,10 @@ shape_cnt = len(self.arguments_w) # Number of positional args shape_keys = tuple(sorted(self.keywords)) shape_star = self.w_stararg is not None # Flag: presence of *arg - shape_stst = False # Flag: presence of **kwds data_w = self.arguments_w + [self.keywords[key] for key in shape_keys] if shape_star: data_w.append(self.w_stararg) - return (shape_cnt, shape_keys, shape_star, shape_stst), data_w + return (shape_cnt, shape_keys, shape_star), data_w def as_list(self): assert not self.keywords diff --git a/rpython/flowspace/flowcontext.py b/rpython/flowspace/flowcontext.py --- a/rpython/flowspace/flowcontext.py +++ b/rpython/flowspace/flowcontext.py @@ -926,7 +926,7 @@ key = w_key.value keywords[key] = w_value arguments = self.popvalues(n_arguments) - args = CallSpec(arguments, keywords, w_star, w_starstar) + args = CallSpec(arguments, keywords, w_star) w_function = self.popvalue() w_result = self.space.call(w_function, args) self.pushvalue(w_result) diff --git a/rpython/flowspace/test/test_objspace.py b/rpython/flowspace/test/test_objspace.py --- a/rpython/flowspace/test/test_objspace.py +++ b/rpython/flowspace/test/test_objspace.py @@ -748,8 +748,7 @@ for block in graph.iterblocks(): for op in block.operations: assert op.opname == "call_args" - assert op.args == map(Constant, - [g, (0, ('x',), False, False), 2]) + assert op.args == map(Constant, [g, (0, ('x',), False), 2]) def test_catch_importerror_1(self): def f(): diff --git a/rpython/rtyper/normalizecalls.py b/rpython/rtyper/normalizecalls.py --- a/rpython/rtyper/normalizecalls.py +++ b/rpython/rtyper/normalizecalls.py @@ -92,9 +92,8 @@ else: return False # nothing to do, all signatures already match - shape_cnt, shape_keys, shape_star, shape_stst = shape + shape_cnt, shape_keys, shape_star = shape assert not shape_star, "XXX not implemented" - assert not shape_stst, "XXX not implemented" # for the first 'shape_cnt' arguments we need to generalize to # a common type diff --git a/rpython/rtyper/rbuiltin.py b/rpython/rtyper/rbuiltin.py --- a/rpython/rtyper/rbuiltin.py +++ b/rpython/rtyper/rbuiltin.py @@ -49,8 +49,6 @@ arguments = ArgumentsForTranslation.fromshape( hop.args_s[1].const, # shape range(hop.nb_args-2)) - if arguments.w_starstararg is not None: - raise TyperError("**kwds call not implemented") if arguments.w_stararg is not None: # expand the *arg in-place -- it must be a tuple from rpython.rtyper.rtuple import TupleRepr @@ -303,7 +301,7 @@ s_callable = r_callable.get_s_callable() nbargs = len(hop.args_s) - 1 + nimplicitarg - s_sigs = r_func.get_s_signatures((nbargs, (), False, False)) + s_sigs = r_func.get_s_signatures((nbargs, (), False)) if len(s_sigs) != 1: raise TyperError("cannot hlinvoke callable %r with not uniform" "annotations: %r" % (r_callable, diff --git a/rpython/rtyper/rpbc.py b/rpython/rtyper/rpbc.py --- a/rpython/rtyper/rpbc.py +++ b/rpython/rtyper/rpbc.py @@ -268,8 +268,8 @@ # lowleveltype wouldn't be Void otherwise funcdesc, = self.s_pbc.descriptions tables = [] # find the simple call in the calltable - for key, table in self.callfamily.calltables.items(): - if not key[1] and not key[2] and not key[3]: + for shape, table in self.callfamily.calltables.items(): + if not shape[1] and not shape[2]: tables.append(table) if len(tables) != 1: raise TyperError("cannot pass a function with various call shapes") diff --git a/rpython/rtyper/test/test_normalizecalls.py b/rpython/rtyper/test/test_normalizecalls.py --- a/rpython/rtyper/test/test_normalizecalls.py +++ b/rpython/rtyper/test/test_normalizecalls.py @@ -185,7 +185,7 @@ .+Sub1.fn .+Sub2.fn are called with inconsistent numbers of arguments -sometimes with 2 arguments, sometimes with 1 +sometimes with \d arguments, sometimes with \d the callers of these functions are: .+otherfunc .+dummyfn""" @@ -224,7 +224,7 @@ from rpython.rtyper import annlowlevel # annotate, normalize and rtype fn after the fact - annhelper = annlowlevel.MixLevelHelperAnnotator(typer) + annhelper = annlowlevel.MixLevelHelperAnnotator(typer) graph = annhelper.getgraph(fn, [a.typeannotation(argtype) for argtype in argtypes], s_result) annhelper.finish() @@ -239,7 +239,7 @@ assert res == 100 res = llinterp.eval_graph(graphof(t, prefn), [2]) assert res == 201 - + return t def test_mix_after_recursion(self): @@ -248,7 +248,7 @@ return 2*prefn(n-1) else: return 1 - + t = TranslationContext() a = t.buildannotator() a.build_types(prefn, [int]) @@ -260,10 +260,10 @@ return 1 from rpython.rtyper import annlowlevel - annhelper = annlowlevel.MixLevelHelperAnnotator(typer) + annhelper = annlowlevel.MixLevelHelperAnnotator(typer) graph = annhelper.getgraph(f, [], annmodel.SomeInteger()) annhelper.finish() - + def test_add_more_subclasses(self): from rpython.rtyper import rclass from rpython.rtyper.lltypesystem.rclass import ll_issubclass From noreply at buildbot.pypy.org Mon Nov 18 23:01:06 2013 From: noreply at buildbot.pypy.org (bivab) Date: Mon, 18 Nov 2013 23:01:06 +0100 (CET) Subject: [pypy-commit] buildbot add-header-to-nightly: close branch Message-ID: <20131118220106.9C1FE1C026D@cobra.cs.uni-duesseldorf.de> Author: David Schneider Branch: add-header-to-nightly Changeset: r897:76938234d843 Date: 2013-11-18 23:00 +0100 http://bitbucket.org/pypy/buildbot/changeset/76938234d843/ Log: close branch From noreply at buildbot.pypy.org Mon Nov 18 23:01:07 2013 From: noreply at buildbot.pypy.org (bivab) Date: Mon, 18 Nov 2013 23:01:07 +0100 (CET) Subject: [pypy-commit] buildbot default: merge add-header-to-nightly Message-ID: <20131118220107.E0D511C02D8@cobra.cs.uni-duesseldorf.de> Author: David Schneider Branch: Changeset: r898:3f8d7754ec20 Date: 2013-11-18 23:00 +0100 http://bitbucket.org/pypy/buildbot/changeset/3f8d7754ec20/ Log: merge add-header-to-nightly diff --git a/bot2/pypybuildbot/pypylist.py b/bot2/pypybuildbot/pypylist.py --- a/bot2/pypybuildbot/pypylist.py +++ b/bot2/pypybuildbot/pypylist.py @@ -5,7 +5,8 @@ import cgi import urllib import sys -from twisted.web.static import File, DirectoryLister +from twisted.web.static import File, formatFileSize +from buildbot.status.web.base import DirectoryLister class PyPyTarball(object): @@ -142,98 +143,39 @@ names = File.listNames(self) if is_pypy_dir(names): names = self.sortBuildNames(names) - Listener = PyPyDirectoryLister else: names = self.sortDirectoryNames(File.listEntities(self)) - Listener = DirectoryLister + Listener = PyPyDirectoryLister return Listener(self.path, names, self.contentTypes, self.contentEncodings, self.defaultType) -class NumpyStatusList(File): - pass - class PyPyDirectoryLister(DirectoryLister): - template = """ - -%(header)s - - - - -

%(header)s

- - - - - - - - - - - - -%(tableContent)s - -

Filename	Size	Date	own tests	applevel tests

- - - -""" - - linePattern = """ - %(text)s - %(size)s - %(date)s - %(own_summary)s - %(app_summary)s - -""" + '''template based, uses master/templates/directory.html + ''' def render(self, request): self.status = request.site.buildbot_service.getStatus() return DirectoryLister.render(self, request) - def _buildTableContent(self, elements): - tableContent = [] + def _getFilesAndDirectories(self, directory): + dirs, files = DirectoryLister._getFilesAndDirectories(self, directory) rowClasses = itertools.cycle(['odd', 'even']) - for element, rowClass in zip(elements, rowClasses): - element["class"] = rowClass - self._add_test_results(element, rowClass) - tableContent.append(self.linePattern % element) - return tableContent + for f, rowClass in zip(files, rowClasses): + f["class"] = rowClass + self._add_test_results(f, rowClass) + for d in dirs: + dirname = urllib.unquote(d['href']) + dd = py.path.local(self.path).join(dirname) + date = datetime.date.fromtimestamp(dd.mtime()) + d['date'] = date.isoformat() + # Assume dir is non-recursive + size = sum([f.size() for f in dd.listdir() if f.isfile()]) + d['size'] = formatFileSize(size) + + return dirs, files def _add_test_results(self, element, rowClass): filename = urllib.unquote(element['href']) @@ -292,3 +234,6 @@ else: return rowClass + '-failed' +class NumpyStatusList(PyPyList): + pass + diff --git a/master/templates/directory.html b/master/templates/directory.html new file mode 100644 --- /dev/null +++ b/master/templates/directory.html @@ -0,0 +1,94 @@ +{% extends "layout.html" %} +{% block morehead %} + + +{% endblock %} + +{% block content %} + +

Directory listing for {{ path }}

+ +{% set row_class = cycler('odd', 'even') %} + +{% set has_tests = files|join('', attribute='own_summary')|length > 0 or + files|join('', attribute='app_summary')|length > 0 %} + + + +{% if files|length > 0 %} + + + + +{% if has_tests %} + + +{% endif %} + +{% else %} + + + + +{% if has_tests %} + + +{% endif %} + +{% endif %} + +{% for d in directories %} + + + + +{% if has_tests %} + + +{% endif %} + +{% endfor %} + +{% for f in files %} + + + + +{% if has_tests %} + + +{% endif %} + +{% endfor %} +

Filename	Size	Date	own tests	applevel tests
Directory	Size	Date
{{ d.text }}	{{ d.size}}	{{ d.date}}
{{ f.text }}	{{ f.size }}	{{ f.date }}	{{ f.own_summary }}	{{ f.app_summary }}

+ +{% endblock %} diff --git a/master/templates/layout.html b/master/templates/layout.html --- a/master/templates/layout.html +++ b/master/templates/layout.html @@ -23,19 +23,19 @@ {% block header -%}

Home - - - Speed - Summary (trunk) - Summary - Nightly builds + - Speed + - Numpy compatability + - Summary (trunk) + - Summary + - Nightly builds - Waterfall + - Waterfall - Builders + - Builders diff --git a/requirements.txt b/requirements.txt --- a/requirements.txt +++ b/requirements.txt @@ -10,7 +10,7 @@ buildbot-slave==0.8.6p1 decorator==3.4.0 mock==1.0.1 -py==1.4.9 +py==1.4.18 pytest==2.2.4 python-dateutil==1.5 sqlalchemy-migrate==0.7.2 From noreply at buildbot.pypy.org Tue Nov 19 00:02:06 2013 From: noreply at buildbot.pypy.org (pjenvey) Date: Tue, 19 Nov 2013 00:02:06 +0100 (CET) Subject: [pypy-commit] pypy py3k: merge default Message-ID: <20131118230206.146F21C02D8@cobra.cs.uni-duesseldorf.de> Author: Philip Jenvey Branch: py3k Changeset: r68226:526680d19846 Date: 2013-11-18 15:01 -0800 http://bitbucket.org/pypy/pypy/changeset/526680d19846/ Log: merge default diff --git a/lib-python/2.7/test/test_old_mailbox.py b/lib-python/2.7/test/test_old_mailbox.py --- a/lib-python/2.7/test/test_old_mailbox.py +++ b/lib-python/2.7/test/test_old_mailbox.py @@ -73,7 +73,9 @@ self.createMessage("cur") self.mbox = mailbox.Maildir(test_support.TESTFN) self.assertTrue(len(self.mbox) == 1) - self.assertTrue(self.mbox.next() is not None) + msg = self.mbox.next() + self.assertTrue(msg is not None) + msg.fp.close() self.assertTrue(self.mbox.next() is None) self.assertTrue(self.mbox.next() is None) @@ -81,7 +83,9 @@ self.createMessage("new") self.mbox = mailbox.Maildir(test_support.TESTFN) self.assertTrue(len(self.mbox) == 1) - self.assertTrue(self.mbox.next() is not None) + msg = self.mbox.next() + self.assertTrue(msg is not None) + msg.fp.close() self.assertTrue(self.mbox.next() is None) self.assertTrue(self.mbox.next() is None) @@ -90,8 +94,12 @@ self.createMessage("new") self.mbox = mailbox.Maildir(test_support.TESTFN) self.assertTrue(len(self.mbox) == 2) - self.assertTrue(self.mbox.next() is not None) - self.assertTrue(self.mbox.next() is not None) + msg = self.mbox.next() + self.assertTrue(msg is not None) + msg.fp.close() + msg = self.mbox.next() + self.assertTrue(msg is not None) + msg.fp.close() self.assertTrue(self.mbox.next() is None) self.assertTrue(self.mbox.next() is None) diff --git a/lib-python/2.7/traceback.py b/lib-python/2.7/traceback.py --- a/lib-python/2.7/traceback.py +++ b/lib-python/2.7/traceback.py @@ -107,7 +107,7 @@ return list -def print_exception(etype, value, tb, limit=None, file=None): +def print_exception(etype, value, tb, limit=None, file=None, _encoding=None): """Print exception up to 'limit' stack trace entries from 'tb' to 'file'. This differs from print_tb() in the following ways: (1) if @@ -123,7 +123,7 @@ if tb: _print(file, 'Traceback (most recent call last):') print_tb(tb, limit, file) - lines = format_exception_only(etype, value) + lines = format_exception_only(etype, value, _encoding) for line in lines: _print(file, line, '') @@ -144,7 +144,7 @@ list = list + format_exception_only(etype, value) return list -def format_exception_only(etype, value): +def format_exception_only(etype, value, _encoding=None): """Format the exception part of a traceback. The arguments are the exception type and value such as given by @@ -170,12 +170,12 @@ if (isinstance(etype, BaseException) or isinstance(etype, types.InstanceType) or etype is None or type(etype) is str): - return [_format_final_exc_line(etype, value)] + return [_format_final_exc_line(etype, value, _encoding)] stype = etype.__name__ if not issubclass(etype, SyntaxError): - return [_format_final_exc_line(stype, value)] + return [_format_final_exc_line(stype, value, _encoding)] # It was a syntax error; show exactly where the problem was found. lines = [] @@ -196,26 +196,26 @@ lines.append(' %s^\n' % ''.join(caretspace)) value = msg - lines.append(_format_final_exc_line(stype, value)) + lines.append(_format_final_exc_line(stype, value, _encoding)) return lines -def _format_final_exc_line(etype, value): +def _format_final_exc_line(etype, value, _encoding=None): """Return a list of a single line -- normal case for format_exception_only""" - valuestr = _some_str(value) + valuestr = _some_str(value, _encoding) if value is None or not valuestr: line = "%s\n" % etype else: line = "%s: %s\n" % (etype, valuestr) return line -def _some_str(value): +def _some_str(value, _encoding=None): try: return str(value) except Exception: pass try: value = unicode(value) - return value.encode("ascii", "backslashreplace") + return value.encode(_encoding or "ascii", "backslashreplace") except Exception: pass return '' % type(value).__name__ diff --git a/lib_pypy/_sqlite3.py b/lib_pypy/_sqlite3.py --- a/lib_pypy/_sqlite3.py +++ b/lib_pypy/_sqlite3.py @@ -268,10 +268,18 @@ if _has_load_extension(): _ffi.cdef("int sqlite3_enable_load_extension(sqlite3 *db, int onoff);") -_lib = _ffi.verify(""" -#include -""", libraries=['sqlite3'] -) +if sys.platform.startswith('freebsd'): + _lib = _ffi.verify(""" + #include + """, libraries=['sqlite3'], + include_dirs=['/usr/local/include'], + library_dirs=['/usr/local/lib'] + ) +else: + _lib = _ffi.verify(""" + #include + """, libraries=['sqlite3'] + ) exported_sqlite_symbols = [ 'SQLITE_ALTER_TABLE', diff --git a/lib_pypy/_tkinter/tklib.py b/lib_pypy/_tkinter/tklib.py --- a/lib_pypy/_tkinter/tklib.py +++ b/lib_pypy/_tkinter/tklib.py @@ -112,6 +112,14 @@ incdirs = ['/usr/local/include/tcl8.5', '/usr/local/include/tk8.5', '/usr/X11R6/include'] linklibs = ['tk85', 'tcl85'] libdirs = ['/usr/local/lib', '/usr/X11R6/lib'] +elif sys.platform.startswith("freebsd"): + incdirs = ['/usr/local/include/tcl8.6', '/usr/local/include/tk8.6', '/usr/local/include/X11', '/usr/local/include'] + linklibs = ['tk86', 'tcl86'] + libdirs = ['/usr/local/lib'] +elif sys.platform == 'win32': + incdirs = [] + linklibs = ['tcl85', 'tk85'] + libdirs = [] else: incdirs=['/usr/include/tcl'] linklibs=['tcl', 'tk'] diff --git a/pypy/doc/release-2.2.0.rst b/pypy/doc/release-2.2.0.rst --- a/pypy/doc/release-2.2.0.rst +++ b/pypy/doc/release-2.2.0.rst @@ -66,9 +66,9 @@ ``https://bitbucket.org/pypy/numpy`` and ``numpypy`` disappeared. You need to install NumPy separately with a virtualenv: ``pip install git+https://bitbucket.org/pypy/numpy.git``; - or by directly doing - ``git clone https://bitbucket.org/pypy/numpy.git``, - ``cd numpy``, ``python setup.py install``. + or directly: + ``git clone https://bitbucket.org/pypy/numpy.git``; + ``cd numpy``; ``pypy setup.py install``. * non-inlined calls have less overhead diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -9,3 +9,7 @@ .. branch: numpy-newbyteorder Clean up numpy types, add newbyteorder functionality + +.. branch windows-packaging +Package tk/tcl runtime with win32 + diff --git a/pypy/doc/windows.rst b/pypy/doc/windows.rst --- a/pypy/doc/windows.rst +++ b/pypy/doc/windows.rst @@ -73,11 +73,11 @@ https://bitbucket.org/pypy/pypy/downloads/local.zip Then expand it into the base directory (base_dir) and modify your environment to reflect this:: - set PATH=\bin;%PATH% - set INCLUDE=\include;%INCLUDE% - set LIB=\lib;%LIB% + set PATH=\bin;\tcltk\bin;%PATH% + set INCLUDE=\include;\tcltk\include;%INCLUDE% + set LIB=\lib;\tcltk\lib;%LIB% -Now you should be good to go. Read on for more information. +Now you should be good to go. Read on for more information. The Boehm garbage collector ~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -109,11 +109,10 @@ The bz2 compression library ~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Download http://bzip.org/1.0.5/bzip2-1.0.5.tar.gz and extract it in -the base directory. Then compile:: - - cd bzip2-1.0.5 + svn export http://svn.python.org/projects/external/bzip2-1.0.6 + cd bzip2-1.0.6 nmake -f makefile.msc + copy bzip.dll \bzip.dll The sqlite3 database library ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -122,8 +121,6 @@ wrapper is compiled when the module is imported for the first time. The sqlite3.dll should be version 3.6.21 for CPython2.7 compatablility. - - The expat XML parser ~~~~~~~~~~~~~~~~~~~~ @@ -146,13 +143,33 @@ use the one distributed by ActiveState, or the one from cygwin. In both case the perl interpreter must be found on the PATH. -Get http://www.openssl.org/source/openssl-0.9.8k.tar.gz and extract it -in the base directory. Then compile:: - + svn export http://svn.python.org/projects/external/openssl-0.9.8y + cd openssl-0.9.8y perl Configure VC-WIN32 ms\do_ms.bat nmake -f ms\nt.mak install +TkInter module support +~~~~~~~~~~~~~~~~~~~~~~ + +Note that much of this is taken from the cpython build process. +Tkinter is imported via cffi, so the module is optional. To recreate the tcltk +directory found for the release script, create the dlls, libs, headers and +runtime by running:: + + svn export http://svn.python.org/projects/external/tcl-8.5.2.1 tcl85 + svn export http://svn.python.org/projects/external/tk-8.5.2.0 tk85 + cd tcl85\win + nmake -f makefile.vc COMPILERFLAGS=-DWINVER=0x0500 DEBUG=0 INSTALLDIR=..\..\tcltk clean all + nmake -f makefile.vc DEBUG=0 INSTALLDIR=..\..\tcltk install + cd ..\..\tk85\win + nmake -f makefile.vc COMPILERFLAGS=-DWINVER=0x0500 OPTS=noxp DEBUG=1 INSTALLDIR=..\..\tcltk TCLDIR=..\..\tcl85 clean all + nmake -f makefile.vc COMPILERFLAGS=-DWINVER=0x0500 OPTS=noxp DEBUG=1 INSTALLDIR=..\..\tcltk TCLDIR=..\..\tcl85 install + +Now you should have a tcktk\bin, tcltk\lib, and tcltk\include directory ready +for use. The release packaging script will pick up the tcltk runtime in the lib +directory and put it in the archive. + Using the mingw compiler ------------------------ diff --git a/pypy/interpreter/test/test_app_main.py b/pypy/interpreter/test/test_app_main.py --- a/pypy/interpreter/test/test_app_main.py +++ b/pypy/interpreter/test/test_app_main.py @@ -1061,7 +1061,8 @@ self.w_tmp_dir = self.space.wrap(tmp_dir) - foo_py = prefix.join('foo.py').write("pass") + foo_py = prefix.join('foo.py') + foo_py.write("pass") self.w_foo_py = self.space.wrap(str(foo_py)) def test_setup_bootstrap_path(self): diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py --- a/pypy/module/_cffi_backend/test/_backend_test_c.py +++ b/pypy/module/_cffi_backend/test/_backend_test_c.py @@ -1086,7 +1086,9 @@ assert strlenaddr == cast(BVoidP, strlen) def test_read_variable(): - if sys.platform == 'win32' or sys.platform == 'darwin': + ## FIXME: this test assumes glibc specific behavior, it's not compliant with C standard + ## https://bugs.pypy.org/issue1643 + if sys.platform == 'win32' or sys.platform == 'darwin' or sys.platform.startswith('freebsd'): py.test.skip("untested") BVoidP = new_pointer_type(new_void_type()) ll = find_and_load_library('c') @@ -1094,7 +1096,9 @@ assert stderr == cast(BVoidP, _testfunc(8)) def test_read_variable_as_unknown_length_array(): - if sys.platform == 'win32' or sys.platform == 'darwin': + ## FIXME: this test assumes glibc specific behavior, it's not compliant with C standard + ## https://bugs.pypy.org/issue1643 + if sys.platform == 'win32' or sys.platform == 'darwin' or sys.platform.startswith('freebsd'): py.test.skip("untested") BCharP = new_pointer_type(new_primitive_type("char")) BArray = new_array_type(BCharP, None) @@ -1104,7 +1108,9 @@ # ^^ and not 'char[]', which is basically not allowed and would crash def test_write_variable(): - if sys.platform == 'win32' or sys.platform == 'darwin': + ## FIXME: this test assumes glibc specific behavior, it's not compliant with C standard + ## https://bugs.pypy.org/issue1643 + if sys.platform == 'win32' or sys.platform == 'darwin' or sys.platform.startswith('freebsd'): py.test.skip("untested") BVoidP = new_pointer_type(new_void_type()) ll = find_and_load_library('c') diff --git a/pypy/module/binascii/interp_crc32.py b/pypy/module/binascii/interp_crc32.py --- a/pypy/module/binascii/interp_crc32.py +++ b/pypy/module/binascii/interp_crc32.py @@ -1,17 +1,9 @@ from pypy.interpreter.gateway import unwrap_spec -from rpython.rlib.rarithmetic import r_uint, intmask -from rpython.rtyper.lltypesystem import rffi -from rpython.rlib.rzipfile import crc_32_tab +from rpython.rlib.rarithmetic import r_uint +from rpython.rlib import rzipfile @unwrap_spec(data='bufferstr', oldcrc='truncatedint_w') def crc32(space, data, oldcrc=0): "Compute the CRC-32 incrementally." - - crc = r_uint(rffi.cast(rffi.UINT, ~oldcrc)) # signed => 32-bit unsigned - - # in the following loop, we have always 0 <= crc < 2**32 - for c in data: - crc = crc_32_tab[(crc & 0xff) ^ ord(c)] ^ (crc >> 8) - - return space.wrap(rffi.cast(rffi.UINT, ~crc)) - + crc = rzipfile.crc32(data, r_uint(oldcrc)) + return space.wrap(crc) diff --git a/pypy/module/marshal/test/test_marshalimpl.py b/pypy/module/marshal/test/test_marshalimpl.py --- a/pypy/module/marshal/test/test_marshalimpl.py +++ b/pypy/module/marshal/test/test_marshalimpl.py @@ -73,7 +73,7 @@ # u = interp_marshal.StringUnmarshaller(space, space.wrapbytes(expected)) w_long = u.load_w_obj() - assert space.eq_w(w_long, w_obj) is True + assert space.eq_w(w_long, w_obj) for sign in [1L, -1L]: for i in range(100): diff --git a/pypy/module/micronumpy/arrayimpl/concrete.py b/pypy/module/micronumpy/arrayimpl/concrete.py --- a/pypy/module/micronumpy/arrayimpl/concrete.py +++ b/pypy/module/micronumpy/arrayimpl/concrete.py @@ -80,7 +80,7 @@ return scalar.Scalar(self.dtype, self.getitem(0)) return None - def get_view(self, orig_array, dtype, new_shape): + def get_view(self, space, orig_array, dtype, new_shape): strides, backstrides = support.calc_strides(new_shape, dtype, self.order) return SliceArray(self.start, strides, backstrides, new_shape, @@ -211,7 +211,15 @@ "field named %s not found" % idx)) return RecordChunk(idx) if (space.isinstance_w(w_idx, space.w_int) or - space.isinstance_w(w_idx, space.w_slice)): + space.isinstance_w(w_idx, space.w_slice)): + return Chunks([Chunk(*space.decode_index4(w_idx, self.get_shape()[0]))]) + elif isinstance(w_idx, W_NDimArray) and \ + isinstance(w_idx.implementation, scalar.Scalar): + w_idx = w_idx.get_scalar_value().item(space) + if not space.isinstance_w(w_idx, space.w_int) and \ + not space.isinstance_w(w_idx, space.w_bool): + raise OperationError(space.w_IndexError, space.wrap( + "arrays used as indices must be of integer (or boolean) type")) return Chunks([Chunk(*space.decode_index4(w_idx, self.get_shape()[0]))]) elif space.is_w(w_idx, space.w_None): return Chunks([NewAxisChunk()]) diff --git a/pypy/module/micronumpy/arrayimpl/scalar.py b/pypy/module/micronumpy/arrayimpl/scalar.py --- a/pypy/module/micronumpy/arrayimpl/scalar.py +++ b/pypy/module/micronumpy/arrayimpl/scalar.py @@ -68,9 +68,15 @@ def transpose(self, _): return self - def get_view(self, orig_array, dtype, new_shape): + def get_view(self, space, orig_array, dtype, new_shape): scalar = Scalar(dtype) - scalar.value = self.value.convert_to(dtype) + if dtype.is_str_or_unicode(): + scalar.value = dtype.coerce(space, space.wrap(self.value.raw_str())) + elif dtype.is_record_type(): + raise OperationError(space.w_NotImplementedError, space.wrap( + "viewing scalar as record not implemented")) + else: + scalar.value = dtype.itemtype.runpack_str(space, self.value.raw_str()) return scalar def get_real(self, orig_array): @@ -127,19 +133,20 @@ if space.len_w(w_idx) == 0: return self.get_scalar_value() raise OperationError(space.w_IndexError, - space.wrap("scalars cannot be indexed")) + space.wrap("0-d arrays can't be indexed")) def getitem_index(self, space, idx): raise OperationError(space.w_IndexError, - space.wrap("scalars cannot be indexed")) + space.wrap("0-d arrays can't be indexed")) def descr_setitem(self, space, _, w_idx, w_val): raise OperationError(space.w_IndexError, - space.wrap("scalars cannot be indexed")) + space.wrap("0-d arrays can't be indexed")) def setitem_index(self, space, idx, w_val): raise OperationError(space.w_IndexError, - space.wrap("scalars cannot be indexed")) + space.wrap("0-d arrays can't be indexed")) + def set_shape(self, space, orig_array, new_shape): if not new_shape: return self diff --git a/pypy/module/micronumpy/interp_arrayops.py b/pypy/module/micronumpy/interp_arrayops.py --- a/pypy/module/micronumpy/interp_arrayops.py +++ b/pypy/module/micronumpy/interp_arrayops.py @@ -106,16 +106,26 @@ args_w = [convert_to_array(space, w_arg) for w_arg in args_w] dtype = args_w[0].get_dtype() shape = args_w[0].get_shape()[:] - _axis = axis + ndim = len(shape) + orig_axis = axis if axis < 0: - _axis = len(shape) + axis + axis = ndim + axis + if ndim == 1 and axis != 0: + axis = 0 + if axis < 0 or axis >= ndim: + raise operationerrfmt(space.w_IndexError, + "axis %d out of bounds [0, %d)", orig_axis, ndim) for arr in args_w[1:]: + if len(arr.get_shape()) != ndim: + raise OperationError(space.w_ValueError, space.wrap( + "all the input arrays must have same number of dimensions")) for i, axis_size in enumerate(arr.get_shape()): - if len(arr.get_shape()) != len(shape) or (i != _axis and axis_size != shape[i]): + if i == axis: + shape[i] += axis_size + elif axis_size != shape[i]: raise OperationError(space.w_ValueError, space.wrap( - "all the input arrays must have same number of dimensions")) - elif i == _axis: - shape[i] += axis_size + "all the input array dimensions except for the " + "concatenation axis must match exactly")) a_dt = arr.get_dtype() if dtype.is_record_type() and a_dt.is_record_type(): # Record types must match @@ -129,19 +139,17 @@ space.wrap("invalid type promotion")) dtype = interp_ufuncs.find_binop_result_dtype(space, dtype, arr.get_dtype()) - if _axis < 0 or len(arr.get_shape()) <= _axis: - raise operationerrfmt(space.w_IndexError, "axis %d out of bounds [0, %d)", axis, len(shape)) # concatenate does not handle ndarray subtypes, it always returns a ndarray res = W_NDimArray.from_shape(space, shape, dtype, 'C') chunks = [Chunk(0, i, 1, i) for i in shape] axis_start = 0 for arr in args_w: - if arr.get_shape()[_axis] == 0: + if arr.get_shape()[axis] == 0: continue - chunks[_axis] = Chunk(axis_start, axis_start + arr.get_shape()[_axis], 1, - arr.get_shape()[_axis]) + chunks[axis] = Chunk(axis_start, axis_start + arr.get_shape()[axis], 1, + arr.get_shape()[axis]) Chunks(chunks).apply(space, res).implementation.setslice(space, arr) - axis_start += arr.get_shape()[_axis] + axis_start += arr.get_shape()[axis] return res @unwrap_spec(repeats=int) diff --git a/pypy/module/micronumpy/interp_boxes.py b/pypy/module/micronumpy/interp_boxes.py --- a/pypy/module/micronumpy/interp_boxes.py +++ b/pypy/module/micronumpy/interp_boxes.py @@ -267,11 +267,19 @@ from pypy.module.micronumpy.interp_dtype import W_Dtype dtype = space.interp_w(W_Dtype, space.call_function(space.gettypefor(W_Dtype), w_dtype)) + if dtype.get_size() == 0: + raise OperationError(space.w_TypeError, space.wrap( + "data-type must not be 0-sized")) if dtype.get_size() != self.get_dtype(space).get_size(): raise OperationError(space.w_ValueError, space.wrap( "new type not compatible with array.")) - raise OperationError(space.w_NotImplementedError, space.wrap( - "view not implelemnted yet")) + if dtype.is_str_or_unicode(): + return dtype.coerce(space, space.wrap(self.raw_str())) + elif dtype.is_record_type(): + raise OperationError(space.w_NotImplementedError, space.wrap( + "viewing scalar as record not implemented")) + else: + return dtype.itemtype.runpack_str(space, self.raw_str()) def descr_self(self, space): return self @@ -408,6 +416,9 @@ def get_dtype(self, space): return self.arr.dtype + def raw_str(self): + return self.arr.dtype.itemtype.to_str(self) + class W_VoidBox(W_FlexibleBox): def descr_getitem(self, space, w_item): if (space.isinstance_w(w_item, space.w_unicode) or diff --git a/pypy/module/micronumpy/interp_dtype.py b/pypy/module/micronumpy/interp_dtype.py --- a/pypy/module/micronumpy/interp_dtype.py +++ b/pypy/module/micronumpy/interp_dtype.py @@ -855,24 +855,21 @@ for k, v in typeinfo_partial.iteritems(): space.setitem(w_typeinfo, space.wrap(k), space.gettypefor(v)) for k, dtype in typeinfo_full.iteritems(): - itemsize = dtype.get_size() + itembits = dtype.get_size() * 8 items_w = [space.wrap(dtype.char), space.wrap(dtype.num), - space.wrap(itemsize * 8), # in case of changing - # number of bits per byte in the future - space.wrap(itemsize / - (2 if dtype.kind == NPY_COMPLEXLTR else 1) - or 1)] + space.wrap(itembits), + space.wrap(dtype.itemtype.get_element_size())] if dtype.is_int_type(): if dtype.kind == NPY_GENBOOLLTR: w_maxobj = space.wrap(1) w_minobj = space.wrap(0) elif dtype.is_signed(): - w_maxobj = space.wrap(r_longlong((1 << (itemsize*8 - 1)) + w_maxobj = space.wrap(r_longlong((1 << (itembits - 1)) - 1)) - w_minobj = space.wrap(r_longlong(-1) << (itemsize*8 - 1)) + w_minobj = space.wrap(r_longlong(-1) << (itembits - 1)) else: - w_maxobj = space.wrap(r_ulonglong(1 << (itemsize*8)) - 1) + w_maxobj = space.wrap(r_ulonglong(1 << itembits) - 1) w_minobj = space.wrap(0) items_w = items_w + [w_maxobj, w_minobj] items_w = items_w + [dtype.w_box_type] diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py --- a/pypy/module/micronumpy/interp_numarray.py +++ b/pypy/module/micronumpy/interp_numarray.py @@ -198,7 +198,8 @@ prefix) def descr_getitem(self, space, w_idx): - if isinstance(w_idx, W_NDimArray) and w_idx.get_dtype().is_bool_type(): + if isinstance(w_idx, W_NDimArray) and w_idx.get_dtype().is_bool_type() \ + and len(w_idx.get_shape()) > 0: return self.getitem_filter(space, w_idx) try: return self.implementation.descr_getitem(space, self, w_idx) @@ -212,7 +213,8 @@ self.implementation.setitem_index(space, index_list, w_value) def descr_setitem(self, space, w_idx, w_value): - if isinstance(w_idx, W_NDimArray) and w_idx.get_dtype().is_bool_type(): + if isinstance(w_idx, W_NDimArray) and w_idx.get_dtype().is_bool_type() \ + and len(w_idx.get_shape()) > 0: self.setitem_filter(space, w_idx, convert_to_array(space, w_value)) return try: @@ -704,7 +706,7 @@ return self return wrap_impl(space, space.type(self), self, self.implementation.get_view( - self, self.get_dtype(), new_shape)) + space, self, self.get_dtype(), new_shape)) def descr_strides(self, space): raise OperationError(space.w_NotImplementedError, space.wrap( @@ -733,11 +735,14 @@ impl = self.implementation new_shape = self.get_shape()[:] dims = len(new_shape) + if new_itemsize == 0: + raise OperationError(space.w_TypeError, space.wrap( + "data-type must not be 0-sized")) if dims == 0: # Cannot resize scalars if old_itemsize != new_itemsize: raise OperationError(space.w_ValueError, space.wrap( - "new type not compatible with array shape")) + "new type not compatible with array.")) else: if dims == 1 or impl.get_strides()[0] < impl.get_strides()[-1]: # Column-major, resize first dimension @@ -751,7 +756,7 @@ raise OperationError(space.w_ValueError, space.wrap( "new type not compatible with array.")) new_shape[-1] = new_shape[-1] * old_itemsize / new_itemsize - v = impl.get_view(self, dtype, new_shape) + v = impl.get_view(space, self, dtype, new_shape) w_ret = wrap_impl(space, w_type, self, v) return w_ret diff --git a/pypy/module/micronumpy/loop.py b/pypy/module/micronumpy/loop.py --- a/pypy/module/micronumpy/loop.py +++ b/pypy/module/micronumpy/loop.py @@ -490,7 +490,7 @@ if dtype.is_str_or_unicode(): val = dtype.coerce(space, space.wrap(sub)) else: - val = dtype.itemtype.runpack_str(sub) + val = dtype.itemtype.runpack_str(space, sub) ai.setitem(val) ai.next() i += 1 diff --git a/pypy/module/micronumpy/test/dummy_module.py b/pypy/module/micronumpy/test/dummy_module.py --- a/pypy/module/micronumpy/test/dummy_module.py +++ b/pypy/module/micronumpy/test/dummy_module.py @@ -26,7 +26,7 @@ del types types = ['Generic', 'Number', 'Integer', 'SignedInteger', 'UnsignedInteger', - 'Inexact', 'Floating', 'ComplexFloating', 'Character'] + 'Inexact', 'Floating', 'ComplexFloating', 'Flexible', 'Character'] for t in types: globals()[t.lower()] = typeinfo[t] diff --git a/pypy/module/micronumpy/test/test_dtypes.py b/pypy/module/micronumpy/test/test_dtypes.py --- a/pypy/module/micronumpy/test/test_dtypes.py +++ b/pypy/module/micronumpy/test/test_dtypes.py @@ -18,6 +18,22 @@ class AppTestDtypes(BaseAppTestDtypes): spaceconfig = dict(usemodules=["micronumpy", "struct", "binascii"]) + def test_typeinfo(self): + import numpy as np + try: + from numpy.core.multiarray import typeinfo + except ImportError: + # running on dummy module + from numpypy import typeinfo + assert typeinfo['Number'] == np.number + assert typeinfo['LONGLONG'] == ('q', 9, 64, 8, 9223372036854775807L, + -9223372036854775808L, np.longlong) + assert typeinfo['VOID'] == ('V', 20, 0, 1, np.void) + assert typeinfo['BOOL'] == ('?', 0, 8, 1, 1, 0, np.bool_) + assert typeinfo['CFLOAT'] == ('F', 14, 64, 8, np.complex64) + assert typeinfo['CDOUBLE'] == ('D', 15, 128, 16, np.complex128) + assert typeinfo['HALF'] == ('e', 23, 16, 2, np.float16) + def test_dtype_basic(self): from numpypy import dtype @@ -816,12 +832,19 @@ assert (x == array(42)).all() class AppTestStrUnicodeDtypes(BaseNumpyAppTest): - def test_str_unicode(self): - skip('numpypy differs from numpy') + def test_mro(self): from numpypy import str_, unicode_, character, flexible, generic - - assert str_.mro() == [str_, str, character, flexible, generic, object] - assert unicode_.mro() == [unicode_, str, character, flexible, generic, object] + import sys + if '__pypy__' in sys.builtin_module_names: + assert str_.mro() == [str_, character, flexible, generic, + str, object] + assert unicode_.mro() == [unicode_, character, flexible, generic, + unicode, object] + else: + assert str_.mro() == [str_, str, character, flexible, + generic, object] + assert unicode_.mro() == [unicode_, unicode, character, + flexible, generic, object] def test_str_dtype(self): from numpypy import dtype, str_ @@ -862,12 +885,12 @@ def test_unicode_boxes(self): from numpypy import unicode_ - try: + import sys + if '__pypy__' in sys.builtin_module_names: + exc = raises(NotImplementedError, unicode_, 3) + assert exc.value.message.find('not supported yet') >= 0 + else: u = unicode_(3) - except NotImplementedError as e: - if e.message.find('not supported yet') >= 0: - skip('unicode box not implemented') - else: assert isinstance(u, str) def test_character_dtype(self): @@ -998,35 +1021,15 @@ assert a[0] == 1 assert (a + a)[1] == 4 -class AppTestPyPyOnly(BaseNumpyAppTest): - def setup_class(cls): - if option.runappdirect and '__pypy__' not in sys.builtin_module_names: - py.test.skip("pypy only test") - BaseNumpyAppTest.setup_class.im_func(cls) - - def test_typeinfo(self): - from numpypy import void, number, int64, bool_, complex64, complex128, float16 - try: - from numpy.core.multiarray import typeinfo - except ImportError: - # running on dummy module - from numpypy import typeinfo - assert typeinfo['Number'] == number - assert typeinfo['LONGLONG'] == ('q', 9, 64, 8, 9223372036854775807, -9223372036854775808, int64) - assert typeinfo['VOID'] == ('V', 20, 0, 1, void) - assert typeinfo['BOOL'] == ('?', 0, 8, 1, 1, 0, bool_) - assert typeinfo['CFLOAT'] == ('F', 14, 64, 4, complex64) - assert typeinfo['CDOUBLE'] == ('D', 15, 128, 8, complex128) - assert typeinfo['HALF'] == ('e', 23, 16, 2, float16) - class AppTestObjectDtypes(BaseNumpyAppTest): def test_scalar_from_object(self): from numpypy import array + import sys class Polynomial(object): pass - try: + if '__pypy__' in sys.builtin_module_names: + exc = raises(NotImplementedError, array, Polynomial()) + assert exc.value.message.find('unable to create dtype from objects') >= 0 + else: a = array(Polynomial()) assert a.shape == () - except NotImplementedError, e: - if e.message.find('unable to create dtype from objects')>=0: - skip('creating ojbect dtype not supported yet') diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py --- a/pypy/module/micronumpy/test/test_numarray.py +++ b/pypy/module/micronumpy/test/test_numarray.py @@ -661,15 +661,17 @@ assert (b[newaxis] == [[2, 3, 4]]).all() def test_scalar(self): - from numpypy import array, dtype, int64 + from numpypy import array, dtype, int_ a = array(3) - raises(IndexError, "a[0]") - raises(IndexError, "a[0] = 5") + exc = raises(IndexError, "a[0]") + assert exc.value[0] == "0-d arrays can't be indexed" + exc = raises(IndexError, "a[0] = 5") + assert exc.value[0] == "0-d arrays can't be indexed" assert a.size == 1 assert a.shape == () assert a.dtype is dtype(int) b = a[()] - assert type(b) is int64 + assert type(b) is int_ assert b == 3 def test_len(self): @@ -1520,7 +1522,36 @@ assert arange(4, dtype='>c8').real.max() == 3.0 assert arange(4, dtype=' #define alloca _alloca #else - #include + # ifdef __FreeBSD__ + # include + # else + # include + # endif #endif static int (*python_callback)(int how_many, int *values); static int c_callback(int how_many, ...) { diff --git a/pypy/module/test_lib_pypy/pyrepl/__init__.py b/pypy/module/test_lib_pypy/pyrepl/__init__.py --- a/pypy/module/test_lib_pypy/pyrepl/__init__.py +++ b/pypy/module/test_lib_pypy/pyrepl/__init__.py @@ -1,3 +1,6 @@ import sys import lib_pypy.pyrepl sys.modules['pyrepl'] = sys.modules['lib_pypy.pyrepl'] + +if sys.platform.startswith('freebsd'): + raise Exception('XXX seems to hangs on FreeBSD9') diff --git a/pypy/module/test_lib_pypy/test_grp_extra.py b/pypy/module/test_lib_pypy/test_grp_extra.py --- a/pypy/module/test_lib_pypy/test_grp_extra.py +++ b/pypy/module/test_lib_pypy/test_grp_extra.py @@ -17,7 +17,7 @@ except KeyError: continue assert g.gr_gid == 0 - assert g.gr_mem == ['root'] or g.gr_mem == [] + assert 'root' in g.gr_mem or g.gr_mem == [] assert g.gr_name == name assert isinstance(g.gr_passwd, str) # usually just 'x', don't hope :-) break diff --git a/pypy/tool/release/package.py b/pypy/tool/release/package.py --- a/pypy/tool/release/package.py +++ b/pypy/tool/release/package.py @@ -67,18 +67,22 @@ raise PyPyCNotFound( 'Bogus path: %r does not exist (see docstring for more info)' % (os.path.dirname(str(pypy_c)),)) + win_extras = ['libpypy-c.dll', 'libexpat.dll', 'sqlite3.dll', + 'libeay32.dll', 'ssleay32.dll'] subprocess.check_call([str(pypy_c), '-c', 'import _sqlite3']) if not sys.platform == 'win32': subprocess.check_call([str(pypy_c), '-c', 'import _curses']) subprocess.check_call([str(pypy_c), '-c', 'import syslog']) - if not withouttk: - try: - subprocess.check_call([str(pypy_c), '-c', 'import _tkinter']) - except subprocess.CalledProcessError: - print >>sys.stderr, """Building Tk bindings failed. + if not withouttk: + try: + subprocess.check_call([str(pypy_c), '-c', 'import _tkinter']) + except subprocess.CalledProcessError: + print >>sys.stderr, """Building Tk bindings failed. You can either install Tk development headers package or add --without-tk option to skip packaging binary CFFI extension.""" - sys.exit(1) + sys.exit(1) + #Can the dependencies be found from cffi somehow? + win_extras += ['tcl85.dll', 'tk85.dll'] if sys.platform == 'win32' and not rename_pypy_c.lower().endswith('.exe'): rename_pypy_c += '.exe' binaries = [(pypy_c, rename_pypy_c)] @@ -101,9 +105,7 @@ # Can't rename a DLL: it is always called 'libpypy-c.dll' - for extra in ['libpypy-c.dll', - 'libexpat.dll', 'sqlite3.dll', - 'libeay32.dll', 'ssleay32.dll']: + for extra in win_extras: p = pypy_c.dirpath().join(extra) if not p.check(): p = py.path.local.sysfind(extra) @@ -122,6 +124,19 @@ # XXX users will complain that they cannot compile cpyext # modules for windows, has the lib moved or are there no # exported functions in the dll so no import library is created? + if not withouttk: + try: + p = pypy_c.dirpath().join('tcl85.dll') + if not p.check(): + p = py.path.local.sysfind('tcl85.dll') + tktcldir = p.dirpath().join('..').join('lib') + shutil.copytree(str(tktcldir), str(pypydir.join('tcl'))) + except WindowsError: + print >>sys.stderr, """Packaging Tk runtime failed. +tk85.dll and tcl85.dll found, expecting to find runtime in ..\\lib +directory next to the dlls, as per build instructions.""" + import traceback;traceback.print_exc() + sys.exit(1) # Careful: to copy lib_pypy, copying just the hg-tracked files # would not be enough: there are also ctypes_config_cache/_*_cache.py. @@ -217,5 +232,11 @@ else: print_usage() + if os.environ.has_key("PYPY_PACKAGE_NOSTRIP"): + kw['nostrip'] = True + + if os.environ.has_key("PYPY_PACKAGE_WITHOUTTK"): + kw['withouttk'] = True + args = args[i:] package(*args, **kw) diff --git a/rpython/annotator/test/test_annrpython.py b/rpython/annotator/test/test_annrpython.py --- a/rpython/annotator/test/test_annrpython.py +++ b/rpython/annotator/test/test_annrpython.py @@ -3463,7 +3463,7 @@ py.test.raises(annmodel.AnnotatorError, "a.build_types(f, [int, int])") a.build_types(f, [annmodel.SomeInteger(nonneg=True), annmodel.SomeInteger(nonneg=True)]) - + def test_setslice(self): def f(): @@ -4140,6 +4140,14 @@ a.build_types(f, [str]) assert ("Cannot prove that the object is callable" in exc.value.msg) + def test_str_format_error(self): + def f(s, x): + return s.format(x) + a = self.RPythonAnnotator() + with py.test.raises(annmodel.AnnotatorError) as exc: + a.build_types(f, [str, str]) + assert ("format() is not RPython" in exc.value.msg) + def g(n): return [0, 1, 2, n] diff --git a/rpython/annotator/unaryop.py b/rpython/annotator/unaryop.py --- a/rpython/annotator/unaryop.py +++ b/rpython/annotator/unaryop.py @@ -523,6 +523,9 @@ return SomeObject.op_contains(str, s_element) op_contains.can_only_throw = [] + def method_format(self, *args): + raise AnnotatorError("Method format() is not RPython") + class __extend__(SomeByteArray): def getslice(ba, s_start, s_stop): diff --git a/rpython/conftest.py b/rpython/conftest.py --- a/rpython/conftest.py +++ b/rpython/conftest.py @@ -24,20 +24,20 @@ def pytest_configure(config): global option option = config.option - -def _set_platform(opt, opt_str, value, parser): from rpython.config.translationoption import PLATFORMS from rpython.translator.platform import set_platform - if value not in PLATFORMS: - raise ValueError("%s not in %s" % (value, PLATFORMS)) - set_platform(value, None) + platform = config.option.platform + if platform not in PLATFORMS: + raise ValueError("%s not in %s" % (platform, PLATFORMS)) + set_platform(platform, None) + def pytest_addoption(parser): group = parser.getgroup("rpython options") group.addoption('--view', action="store_true", dest="view", default=False, help="view translation tests' flow graphs with Pygame") - group.addoption('-P', '--platform', action="callback", type="string", - default="host", callback=_set_platform, + group.addoption('-P', '--platform', action="store", dest="platform", + type="string", default="host", help="set up tests to use specified platform as compile/run target") group = parser.getgroup("JIT options") group.addoption('--viewloops', action="store_true", @@ -60,7 +60,7 @@ class LeakFinder: """Track memory allocations during test execution. - + So far, only used by the function lltype.malloc(flavor='raw'). """ def pytest_runtest_setup(self, __multicall__, item): diff --git a/rpython/jit/backend/x86/support.py b/rpython/jit/backend/x86/support.py --- a/rpython/jit/backend/x86/support.py +++ b/rpython/jit/backend/x86/support.py @@ -5,12 +5,13 @@ if WORD == 4: extra = ['-DPYPY_X86_CHECK_SSE2'] + if sys.platform != 'win32': + extra += ['-msse2', '-mfpmath=sse'] else: - extra = [] + extra = [] # the -m options above are always on by default on x86-64 if sys.platform != 'win32': - extra = ['-msse2', '-mfpmath=sse', - '-DPYPY_CPU_HAS_STANDARD_PRECISION'] + extra + extra = ['-DPYPY_CPU_HAS_STANDARD_PRECISION'] + extra ensure_sse2_floats = rffi.llexternal_use_eci(ExternalCompilationInfo( compile_extra = extra, diff --git a/rpython/jit/backend/x86/test/test_ztranslation_basic.py b/rpython/jit/backend/x86/test/test_ztranslation_basic.py --- a/rpython/jit/backend/x86/test/test_ztranslation_basic.py +++ b/rpython/jit/backend/x86/test/test_ztranslation_basic.py @@ -1,11 +1,11 @@ from rpython.jit.backend.llsupport.test.ztranslation_test import TranslationTest -from rpython.translator.translator import TranslationContext -from rpython.config.translationoption import DEFL_GC +from rpython.jit.backend.x86.arch import WORD class TestTranslationX86(TranslationTest): def _check_cbuilder(self, cbuilder): # We assume here that we have sse2. If not, the CPUClass # needs to be changed to CPU386_NO_SSE2, but well. - assert '-msse2' in cbuilder.eci.compile_extra - assert '-mfpmath=sse' in cbuilder.eci.compile_extra + if WORD == 4: + assert '-msse2' in cbuilder.eci.compile_extra + assert '-mfpmath=sse' in cbuilder.eci.compile_extra diff --git a/rpython/rlib/rdynload.py b/rpython/rlib/rdynload.py --- a/rpython/rlib/rdynload.py +++ b/rpython/rlib/rdynload.py @@ -4,6 +4,7 @@ from rpython.rtyper.tool import rffi_platform from rpython.rtyper.lltypesystem import rffi from rpython.rlib.rarithmetic import r_uint +from rpython.rlib.objectmodel import we_are_translated from rpython.translator.tool.cbuild import ExternalCompilationInfo from rpython.translator.platform import platform @@ -79,6 +80,38 @@ RTLD_NOW = cConfig.RTLD_NOW RTLD_LAZY = cConfig.RTLD_LAZY + _t_opened = {} + + def t_dlopen(name): + # for direct execution: can't use the regular way on FreeBSD :-( + # http://factor-language.blogspot.de/2009/02/note-about-libdl-functions-on-netbsd.html + import ctypes + if name: + name = rffi.charp2str(name) + else: + name = None + try: + res = ctypes.cdll.LoadLibrary(name) + except OSError, e: + raise DLOpenError(str(e)) + h = rffi.cast(rffi.VOIDP, res._handle) + _t_opened[rffi.cast(rffi.LONG, h)] = res + return h + + def t_dlclose(handle): + _t_opened.pop(rffi.cast(rffi.LONG, handle)) + return rffi.cast(rffi.INT, 0) + + def t_dldym(handle, name): + import ctypes + lib = _t_opened[rffi.cast(rffi.LONG, handle)] + try: + symbol = lib[name] + except AttributeError: + raise KeyError(name) + res = ctypes.cast(symbol, ctypes.c_void_p) + return rffi.cast(rffi.VOIDP, res.value or 0) + def dlerror(): # XXX this would never work on top of ll2ctypes, because # ctypes are calling dlerror itself, unsure if I can do much in this @@ -91,6 +124,8 @@ def dlopen(name, mode=-1): """ Wrapper around C-level dlopen """ + if not we_are_translated(): + return t_dlopen(name) if mode == -1: if RTLD_LOCAL is not None: mode = RTLD_LOCAL @@ -104,11 +139,16 @@ raise DLOpenError(err) return res - dlclose = c_dlclose + def dlclose(handle): + if not we_are_translated(): + return t_dlclose(handle) + return c_dlclose(handle) def dlsym(libhandle, name): """ Wrapper around C-level dlsym """ + if not we_are_translated(): + return t_dldym(libhandle, name) res = c_dlsym(libhandle, name) if not res: raise KeyError(name) diff --git a/rpython/rlib/rzipfile.py b/rpython/rlib/rzipfile.py --- a/rpython/rlib/rzipfile.py +++ b/rpython/rlib/rzipfile.py @@ -67,20 +67,12 @@ ] crc_32_tab = map(r_uint, crc_32_tab) -# XXX hack to get crc32 to work - -rcrc_32_tab = [r_uint(i) for i in crc_32_tab] - -def crc32(s, crc=0): - result = 0 - crc = ~r_uint(crc) & r_uint(0xffffffffL) +def crc32(s, crc=r_uint(0)): + crc = ~crc & r_uint(0xffffffffL) for c in s: - crc = rcrc_32_tab[(crc ^ r_uint(ord(c))) & 0xffL] ^ (crc >> 8) + crc = crc_32_tab[(crc ^ r_uint(ord(c))) & 0xffL] ^ (crc >> 8) #/* Note: (crc >> 8) MUST zero fill on left - - result = crc ^ r_uint(0xffffffffL) - - return result + return crc ^ r_uint(0xffffffffL) # parts copied from zipfile library implementation diff --git a/rpython/rlib/test/test_rdynload.py b/rpython/rlib/test/test_rdynload.py --- a/rpython/rlib/test/test_rdynload.py +++ b/rpython/rlib/test/test_rdynload.py @@ -21,3 +21,4 @@ lltype.Signed)), dlsym(lib, 'abs')) assert 1 == handle(1) assert 1 == handle(-1) + dlclose(lib) From noreply at buildbot.pypy.org Tue Nov 19 01:48:03 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Tue, 19 Nov 2013 01:48:03 +0100 (CET) Subject: [pypy-commit] pypy voidtype_strformat: cleanup Message-ID: <20131119004803.A5F111C026D@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: voidtype_strformat Changeset: r68227:03bd65055e04 Date: 2013-11-18 19:32 -0500 http://bitbucket.org/pypy/pypy/changeset/03bd65055e04/ Log: cleanup diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py --- a/pypy/module/micronumpy/test/test_numarray.py +++ b/pypy/module/micronumpy/test/test_numarray.py @@ -3107,8 +3107,8 @@ [[7, 8, 9], [10, 11, 12]]])], dtype=dt) s = str(a) - assert s.endswith("[('aaaa', 1.0, 8.0, [[[1, 2, 3], [4, 5, 6]], [[7, 8, 9], [10, 11, 12]]])]") - + assert s.endswith("[('aaaa', 1.0, 8.0, [[[1, 2, 3], [4, 5, 6]], " \ + "[[7, 8, 9], [10, 11, 12]]])]") def test_issue_1589(self): import numpypy as numpy From noreply at buildbot.pypy.org Tue Nov 19 01:48:04 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Tue, 19 Nov 2013 01:48:04 +0100 (CET) Subject: [pypy-commit] pypy default: cleanup Message-ID: <20131119004804.DEEA71C026D@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r68228:911cd20bb189 Date: 2013-11-18 19:35 -0500 http://bitbucket.org/pypy/pypy/changeset/911cd20bb189/ Log: cleanup diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py --- a/pypy/module/micronumpy/test/test_numarray.py +++ b/pypy/module/micronumpy/test/test_numarray.py @@ -68,7 +68,8 @@ assert s.start == 1 assert s.strides == [2, 10, 50] assert s.backstrides == [6, 40, 100] - s = create_slice(self.space, a, [Chunk(1, 5, 3, 2), Chunk(1, 2, 1, 1), Chunk(1, 0, 0, 1)]) + s = create_slice(self.space, a, [Chunk(1, 5, 3, 2), Chunk(1, 2, 1, 1), + Chunk(1, 0, 0, 1)]) assert s.shape == [2, 1] assert s.strides == [3, 10] assert s.backstrides == [3, 0] @@ -2052,7 +2053,8 @@ a = array([1, 2], dtype="int64") data = a.__reduce__() - assert data[2][4] == '\x01\x00\x00\x00\x00\x00\x00\x00\x02\x00\x00\x00\x00\x00\x00\x00' + assert data[2][4] == '\x01\x00\x00\x00\x00\x00\x00\x00' \ + '\x02\x00\x00\x00\x00\x00\x00\x00' pickled_data = dumps(a) assert (loads(pickled_data) == a).all() @@ -2800,9 +2802,11 @@ assert k[0] == dtype('float16').type(5.) dt = array([5],dtype='longfloat').dtype if dt.itemsize == 12: - m = fromstring('\x00\x00\x00\x00\x00\x00\x00\xa0\x01@\x00\x00', dtype='float96') + m = fromstring('\x00\x00\x00\x00\x00\x00\x00\xa0\x01@\x00\x00', + dtype='float96') elif dt.itemsize == 16: - m = fromstring('\x00\x00\x00\x00\x00\x00\x00\xa0\x01@\x00\x00\x00\x00\x00\x00', dtype='float128') + m = fromstring('\x00\x00\x00\x00\x00\x00\x00\xa0\x01@\x00\x00' \ + '\x00\x00\x00\x00', dtype='float128') elif dt.itemsize == 8: skip('longfloat is float64') else: @@ -3027,7 +3031,8 @@ from numpypy import dtype, array, zeros d = dtype([("x", "int", 3), ("y", "float", 5)]) - a = array([([1, 2, 3], [0.5, 1.5, 2.5, 3.5, 4.5]), ([4, 5, 6], [5.5, 6.5, 7.5, 8.5, 9.5])], dtype=d) + a = array([([1, 2, 3], [0.5, 1.5, 2.5, 3.5, 4.5]), + ([4, 5, 6], [5.5, 6.5, 7.5, 8.5, 9.5])], dtype=d) for v in ['x', u'x', 0, -2]: assert (a[0][v] == [1, 2, 3]).all() @@ -3037,7 +3042,8 @@ assert (a[1][v] == [5.5, 6.5, 7.5, 8.5, 9.5]).all() for v in [-3, 2]: exc = raises(IndexError, "a[0][%d]" % v) - assert exc.value.message == "invalid index (%d)" % (v + 2 if v < 0 else v) + assert exc.value.message == "invalid index (%d)" % \ + (v + 2 if v < 0 else v) exc = raises(IndexError, "a[0]['z']") assert exc.value.message == "invalid index" exc = raises(IndexError, "a[0][None]") @@ -3107,7 +3113,8 @@ from numpypy import dtype, array d = dtype([("x", "int", 3), ("y", "float", 5)]) - a = array([([1, 2, 3], [0.5, 1.5, 2.5, 3.5, 4.5]), ([4, 5, 6], [5.5, 6.5, 7.5, 8.5, 9.5])], dtype=d) + a = array([([1, 2, 3], [0.5, 1.5, 2.5, 3.5, 4.5]), + ([4, 5, 6], [5.5, 6.5, 7.5, 8.5, 9.5])], dtype=d) assert len(list(a[0])) == 2 From noreply at buildbot.pypy.org Tue Nov 19 01:48:06 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Tue, 19 Nov 2013 01:48:06 +0100 (CET) Subject: [pypy-commit] pypy default: support order argument for array.tostring Message-ID: <20131119004806.0F0E91C026D@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r68229:5898d0122baa Date: 2013-11-18 19:47 -0500 http://bitbucket.org/pypy/pypy/changeset/5898d0122baa/ Log: support order argument for array.tostring diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py --- a/pypy/module/micronumpy/interp_numarray.py +++ b/pypy/module/micronumpy/interp_numarray.py @@ -93,7 +93,11 @@ def descr_fill(self, space, w_value): self.fill(self.get_dtype().coerce(space, w_value)) - def descr_tostring(self, space): + def descr_tostring(self, space, w_order=None): + order = order_converter(space, w_order, NPY_CORDER) + if order == NPY_FORTRANORDER: + raise OperationError(space.w_NotImplementedError, space.wrap( + "unsupported value for order")) return space.wrap(loop.tostring(space, self)) def getitem_filter(self, space, arr): diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py --- a/pypy/module/micronumpy/test/test_numarray.py +++ b/pypy/module/micronumpy/test/test_numarray.py @@ -2829,6 +2829,15 @@ assert array([1, 2, 3], 'i2')[::2].tostring() == '\x00\x01\x00\x03' assert array(0, dtype='i2').tostring() == '\x00\x00' + a = array([[1, 2], [3, 4]], dtype='i1') + for order in (None, False, 'C', 'K', 'a'): + assert a.tostring(order) == '\x01\x02\x03\x04' + import sys + for order in (True, 'F'): + if '__pypy__' in sys.builtin_module_names: + raises(NotImplementedError, a.tostring, order) + else: + assert a.tostring(order) == '\x01\x03\x02\x04' class AppTestRepr(BaseNumpyAppTest): From noreply at buildbot.pypy.org Tue Nov 19 02:11:29 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Tue, 19 Nov 2013 02:11:29 +0100 (CET) Subject: [pypy-commit] pypy default: provide dtype.descr for simple dtypes Message-ID: <20131119011129.949E31C00EC@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r68230:38abc15e315d Date: 2013-11-18 20:10 -0500 http://bitbucket.org/pypy/pypy/changeset/38abc15e315d/ Log: provide dtype.descr for simple dtypes diff --git a/pypy/module/micronumpy/interp_dtype.py b/pypy/module/micronumpy/interp_dtype.py --- a/pypy/module/micronumpy/interp_dtype.py +++ b/pypy/module/micronumpy/interp_dtype.py @@ -151,6 +151,14 @@ endian = NPY_NATBYTE return space.wrap("%s%s%s" % (endian, basic, size)) + def descr_get_descr(self, space): + if not self.is_record_type(): + return space.newlist([space.newtuple([space.wrap(""), + self.descr_get_str(space)])]) + else: + raise OperationError(space.w_NotImplementedError, space.wrap( + "descr not implemented for record types")) + def descr_get_base(self, space): return space.wrap(self.base) @@ -447,6 +455,7 @@ fields = GetSetProperty(W_Dtype.descr_get_fields), names = GetSetProperty(W_Dtype.descr_get_names), hasobject = GetSetProperty(W_Dtype.descr_get_hasobject), + descr = GetSetProperty(W_Dtype.descr_get_descr), ) W_Dtype.typedef.acceptable_as_base_class = False diff --git a/pypy/module/micronumpy/test/test_dtypes.py b/pypy/module/micronumpy/test/test_dtypes.py --- a/pypy/module/micronumpy/test/test_dtypes.py +++ b/pypy/module/micronumpy/test/test_dtypes.py @@ -832,6 +832,17 @@ assert x.dtype == int8 assert (x == array(42)).all() + def test_descr(self): + import numpy as np + assert np.dtype(' Author: Ronan Lamy Branch: less-stringly-ops Changeset: r68231:5f4e3a547a2c Date: 2013-11-18 22:07 +0000 http://bitbucket.org/pypy/pypy/changeset/5f4e3a547a2c/ Log: make ArgumentsForTranslation a subclass of CallSpec diff --git a/rpython/annotator/argument.py b/rpython/annotator/argument.py --- a/rpython/annotator/argument.py +++ b/rpython/annotator/argument.py @@ -2,22 +2,9 @@ Arguments objects. """ from rpython.annotator.model import SomeTuple +from rpython.flowspace.argument import CallSpec -class ArgumentsForTranslation(object): - def __init__(self, args_w, keywords=None, w_stararg=None): - self.w_stararg = w_stararg - assert isinstance(args_w, list) - self.arguments_w = args_w - self.keywords = keywords or {} - - def __repr__(self): - """ NOT_RPYTHON """ - name = self.__class__.__name__ - if not self.keywords: - return '%s(%s)' % (name, self.arguments_w,) - else: - return '%s(%s, %s)' % (name, self.arguments_w, self.keywords) - +class ArgumentsForTranslation(CallSpec): @property def positional_args(self): if self.w_stararg is not None: diff --git a/rpython/flowspace/argument.py b/rpython/flowspace/argument.py --- a/rpython/flowspace/argument.py +++ b/rpython/flowspace/argument.py @@ -83,6 +83,14 @@ self.arguments_w = args_w self.keywords = keywords or {} + def __repr__(self): + """ NOT_RPYTHON """ + name = self.__class__.__name__ + if not self.keywords: + return '%s(%s)' % (name, self.arguments_w,) + else: + return '%s(%s, %s)' % (name, self.arguments_w, self.keywords) + def flatten(self): """ Argument <-> list of w_objects together with "shape" information """ shape_cnt = len(self.arguments_w) # Number of positional args From noreply at buildbot.pypy.org Tue Nov 19 02:43:58 2013 From: noreply at buildbot.pypy.org (rlamy) Date: Tue, 19 Nov 2013 02:43:58 +0100 (CET) Subject: [pypy-commit] pypy less-stringly-ops: hg merge default Message-ID: <20131119014358.8B0981C026D@cobra.cs.uni-duesseldorf.de> Author: Ronan Lamy Branch: less-stringly-ops Changeset: r68232:3aeee725a045 Date: 2013-11-19 01:41 +0000 http://bitbucket.org/pypy/pypy/changeset/3aeee725a045/ Log: hg merge default diff too long, truncating to 2000 out of 38174 lines diff --git a/Makefile b/Makefile --- a/Makefile +++ b/Makefile @@ -1,7 +1,38 @@ all: pypy-c +PYPY_EXECUTABLE := $(shell which pypy) +URAM := $(shell python -c "import sys; print 4.5 if sys.maxint>1<<32 else 2.5") + +ifeq ($(PYPY_EXECUTABLE),) +RUNINTERP = python +else +RUNINTERP = $(PYPY_EXECUTABLE) +endif + pypy-c: - @echo "Building PyPy with JIT, it'll take about 40 minutes and 4G of RAM" - @sleep 3 - rpython/bin/rpython -Ojit pypy/goal/targetpypystandalone.py + @echo + @echo "====================================================================" +ifeq ($(PYPY_EXECUTABLE),) + @echo "Building a regular (jitting) version of PyPy, using CPython." + @echo "This takes around 2 hours and $(URAM) GB of RAM." + @echo "Note that pre-installing a PyPy binary would reduce this time" + @echo "and produce basically the same result." +else + @echo "Building a regular (jitting) version of PyPy, using" + @echo "$(PYPY_EXECUTABLE) to run the translation itself." + @echo "This takes up to 1 hour and $(URAM) GB of RAM." +endif + @echo + @echo "For more control (e.g. to use multiple CPU cores during part of" + @echo "the process) you need to run \`\`rpython/bin/rpython'' directly." + @echo "For more information see \`\`http://pypy.org/download.html''." + @echo "====================================================================" + @echo + @sleep 5 + $(RUNINTERP) rpython/bin/rpython -Ojit pypy/goal/targetpypystandalone.py + +# Note: the -jN option, or MAKEFLAGS=-jN, are not usable. They are +# replaced with an opaque --jobserver option by the time this Makefile +# runs. We cannot get their original value either: +# http://lists.gnu.org/archive/html/help-make/2010-08/msg00106.html diff --git a/README.rst b/README.rst --- a/README.rst +++ b/README.rst @@ -1,5 +1,5 @@ ===================================== -PyPy: Python in Python Implementation +PyPy: Python in Python Implementation ===================================== Welcome to PyPy! @@ -26,12 +26,14 @@ Building ======== -build with:: +build with: - rpython/bin/rpython -Ojit pypy/goal/targetpypystandalone.py +.. code-block:: console + + $ rpython/bin/rpython -Ojit pypy/goal/targetpypystandalone.py This ends up with ``pypy-c`` binary in the main pypy directory. We suggest -to use virtualenv with the resulting pypy-c as the interpreter, you can +to use virtualenv with the resulting pypy-c as the interpreter; you can find more details about various installation schemes here: http://doc.pypy.org/en/latest/getting-started.html#installing-pypy diff --git a/lib-python/2.7/json/encoder.py b/lib-python/2.7/json/encoder.py --- a/lib-python/2.7/json/encoder.py +++ b/lib-python/2.7/json/encoder.py @@ -4,6 +4,21 @@ from __pypy__.builders import StringBuilder, UnicodeBuilder +class StringOrUnicodeBuilder(object): + def __init__(self): + self._builder = StringBuilder() + def append(self, string): + try: + self._builder.append(string) + except UnicodeEncodeError: + ub = UnicodeBuilder() + ub.append(self._builder.build()) + self._builder = ub + ub.append(string) + def build(self): + return self._builder.build() + + ESCAPE = re.compile(r'[\x00-\x1f\\"\b\f\n\r\t]') ESCAPE_ASCII = re.compile(r'([\\"]|[^\ -~])') HAS_UTF8 = re.compile(r'[\x80-\xff]') @@ -192,7 +207,7 @@ if self.ensure_ascii: builder = StringBuilder() else: - builder = UnicodeBuilder() + builder = StringOrUnicodeBuilder() self.__encode(o, markers, builder, 0) return builder.build() diff --git a/lib-python/2.7/string.py b/lib-python/2.7/string.py --- a/lib-python/2.7/string.py +++ b/lib-python/2.7/string.py @@ -66,16 +66,17 @@ must be of the same length. """ - if len(fromstr) != len(tostr): + n = len(fromstr) + if n != len(tostr): raise ValueError, "maketrans arguments must have same length" - global _idmapL - if not _idmapL: - _idmapL = list(_idmap) - L = _idmapL[:] - fromstr = map(ord, fromstr) - for i in range(len(fromstr)): - L[fromstr[i]] = tostr[i] - return ''.join(L) + # this function has been rewritten to suit PyPy better; it is + # almost 10x faster than the original. + buf = bytearray(256) + for i in range(256): + buf[i] = i + for i in range(n): + buf[ord(fromstr[i])] = tostr[i] + return str(buf) diff --git a/lib-python/2.7/test/keycert.pem b/lib-python/2.7/test/keycert.pem --- a/lib-python/2.7/test/keycert.pem +++ b/lib-python/2.7/test/keycert.pem @@ -1,32 +1,31 @@ ------BEGIN RSA PRIVATE KEY----- -MIICXwIBAAKBgQC8ddrhm+LutBvjYcQlnH21PPIseJ1JVG2HMmN2CmZk2YukO+9L -opdJhTvbGfEj0DQs1IE8M+kTUyOmuKfVrFMKwtVeCJphrAnhoz7TYOuLBSqt7lVH -fhi/VwovESJlaBOp+WMnfhcduPEYHYx/6cnVapIkZnLt30zu2um+DzA9jQIDAQAB -AoGBAK0FZpaKj6WnJZN0RqhhK+ggtBWwBnc0U/ozgKz2j1s3fsShYeiGtW6CK5nU -D1dZ5wzhbGThI7LiOXDvRucc9n7vUgi0alqPQ/PFodPxAN/eEYkmXQ7W2k7zwsDA -IUK0KUhktQbLu8qF/m8qM86ba9y9/9YkXuQbZ3COl5ahTZrhAkEA301P08RKv3KM -oXnGU2UHTuJ1MAD2hOrPxjD4/wxA/39EWG9bZczbJyggB4RHu0I3NOSFjAm3HQm0 -ANOu5QK9owJBANgOeLfNNcF4pp+UikRFqxk5hULqRAWzVxVrWe85FlPm0VVmHbb/ -loif7mqjU8o1jTd/LM7RD9f2usZyE2psaw8CQQCNLhkpX3KO5kKJmS9N7JMZSc4j -oog58yeYO8BBqKKzpug0LXuQultYv2K4veaIO04iL9VLe5z9S/Q1jaCHBBuXAkEA -z8gjGoi1AOp6PBBLZNsncCvcV/0aC+1se4HxTNo2+duKSDnbq+ljqOM+E7odU+Nq -ewvIWOG//e8fssd0mq3HywJBAJ8l/c8GVmrpFTx8r/nZ2Pyyjt3dH1widooDXYSV -q6Gbf41Llo5sYAtmxdndTLASuHKecacTgZVhy0FryZpLKrU= ------END RSA PRIVATE KEY----- +-----BEGIN PRIVATE KEY----- +MIICdwIBADANBgkqhkiG9w0BAQEFAASCAmEwggJdAgEAAoGBANtb0+YrKuxevGpm +LrjaUhZSgz6zFAmuGFmKmUbdjmfv9zSmmdsQIksK++jK0Be9LeZy20j6ahOfuVa0 +ufEmPoP7Fy4hXegKZR9cCWcIe/A6H2xWF1IIJLRTLaU8ol/I7T+um5HD5AwAwNPP +USNU0Eegmvp+xxWu3NX2m1Veot85AgMBAAECgYA3ZdZ673X0oexFlq7AAmrutkHt +CL7LvwrpOiaBjhyTxTeSNWzvtQBkIU8DOI0bIazA4UreAFffwtvEuPmonDb3F+Iq +SMAu42XcGyVZEl+gHlTPU9XRX7nTOXVt+MlRRRxL6t9GkGfUAXI3XxJDXW3c0vBK +UL9xqD8cORXOfE06rQJBAP8mEX1ERkR64Ptsoe4281vjTlNfIbs7NMPkUnrn9N/Y +BLhjNIfQ3HFZG8BTMLfX7kCS9D593DW5tV4Z9BP/c6cCQQDcFzCcVArNh2JSywOQ +ZfTfRbJg/Z5Lt9Fkngv1meeGNPgIMLN8Sg679pAOOWmzdMO3V706rNPzSVMME7E5 +oPIfAkEA8pDddarP5tCvTTgUpmTFbakm0KoTZm2+FzHcnA4jRh+XNTjTOv98Y6Ik +eO5d1ZnKXseWvkZncQgxfdnMqqpj5wJAcNq/RVne1DbYlwWchT2Si65MYmmJ8t+F +0mcsULqjOnEMwf5e+ptq5LzwbyrHZYq5FNk7ocufPv/ZQrcSSC+cFwJBAKvOJByS +x56qyGeZLOQlWS2JS3KJo59XuLFGqcbgN9Om9xFa41Yb4N9NvplFivsvZdw3m1Q/ +SPIXQuT8RMPDVNQ= +-----END PRIVATE KEY----- -----BEGIN CERTIFICATE----- -MIICpzCCAhCgAwIBAgIJAP+qStv1cIGNMA0GCSqGSIb3DQEBBQUAMIGJMQswCQYD -VQQGEwJVUzERMA8GA1UECBMIRGVsYXdhcmUxEzARBgNVBAcTCldpbG1pbmd0b24x -IzAhBgNVBAoTGlB5dGhvbiBTb2Z0d2FyZSBGb3VuZGF0aW9uMQwwCgYDVQQLEwNT -U0wxHzAdBgNVBAMTFnNvbWVtYWNoaW5lLnB5dGhvbi5vcmcwHhcNMDcwODI3MTY1 -NDUwWhcNMTMwMjE2MTY1NDUwWjCBiTELMAkGA1UEBhMCVVMxETAPBgNVBAgTCERl -bGF3YXJlMRMwEQYDVQQHEwpXaWxtaW5ndG9uMSMwIQYDVQQKExpQeXRob24gU29m -dHdhcmUgRm91bmRhdGlvbjEMMAoGA1UECxMDU1NMMR8wHQYDVQQDExZzb21lbWFj -aGluZS5weXRob24ub3JnMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQC8ddrh -m+LutBvjYcQlnH21PPIseJ1JVG2HMmN2CmZk2YukO+9LopdJhTvbGfEj0DQs1IE8 -M+kTUyOmuKfVrFMKwtVeCJphrAnhoz7TYOuLBSqt7lVHfhi/VwovESJlaBOp+WMn -fhcduPEYHYx/6cnVapIkZnLt30zu2um+DzA9jQIDAQABoxUwEzARBglghkgBhvhC -AQEEBAMCBkAwDQYJKoZIhvcNAQEFBQADgYEAF4Q5BVqmCOLv1n8je/Jw9K669VXb -08hyGzQhkemEBYQd6fzQ9A/1ZzHkJKb1P6yreOLSEh4KcxYPyrLRC1ll8nr5OlCx -CMhKkTnR6qBsdNV0XtdU2+N25hqW+Ma4ZeqsN/iiJVCGNOZGnvQuvCAGWF8+J/f/ -iHkC6gGdBJhogs4= +MIICVDCCAb2gAwIBAgIJANfHOBkZr8JOMA0GCSqGSIb3DQEBBQUAMF8xCzAJBgNV +BAYTAlhZMRcwFQYDVQQHEw5DYXN0bGUgQW50aHJheDEjMCEGA1UEChMaUHl0aG9u +IFNvZnR3YXJlIEZvdW5kYXRpb24xEjAQBgNVBAMTCWxvY2FsaG9zdDAeFw0xMDEw +MDgyMzAxNTZaFw0yMDEwMDUyMzAxNTZaMF8xCzAJBgNVBAYTAlhZMRcwFQYDVQQH +Ew5DYXN0bGUgQW50aHJheDEjMCEGA1UEChMaUHl0aG9uIFNvZnR3YXJlIEZvdW5k +YXRpb24xEjAQBgNVBAMTCWxvY2FsaG9zdDCBnzANBgkqhkiG9w0BAQEFAAOBjQAw +gYkCgYEA21vT5isq7F68amYuuNpSFlKDPrMUCa4YWYqZRt2OZ+/3NKaZ2xAiSwr7 +6MrQF70t5nLbSPpqE5+5VrS58SY+g/sXLiFd6AplH1wJZwh78DofbFYXUggktFMt +pTyiX8jtP66bkcPkDADA089RI1TQR6Ca+n7HFa7c1fabVV6i3zkCAwEAAaMYMBYw +FAYDVR0RBA0wC4IJbG9jYWxob3N0MA0GCSqGSIb3DQEBBQUAA4GBAHPctQBEQ4wd +BJ6+JcpIraopLn8BGhbjNWj40mmRqWB/NAWF6M5ne7KpGAu7tLeG4hb1zLaldK8G +lxy2GPSRF6LFS48dpEj2HbMv2nvv6xxalDMJ9+DicWgAKTQ6bcX2j3GUkCR0g/T1 +CRlNBAAlvhKzO7Clpf9l0YKBEfraJByX -----END CERTIFICATE----- diff --git a/lib-python/2.7/test/sha256.pem b/lib-python/2.7/test/sha256.pem --- a/lib-python/2.7/test/sha256.pem +++ b/lib-python/2.7/test/sha256.pem @@ -1,129 +1,128 @@ # Certificate chain for https://sha256.tbs-internet.com - 0 s:/C=FR/postalCode=14000/ST=Calvados/L=CAEN/street=22 rue de Bretagne/O=TBS INTERNET/OU=0002 440443810/OU=sha-256 production/CN=sha256.tbs-internet.com - i:/C=FR/ST=Calvados/L=Caen/O=TBS INTERNET/OU=Terms and Conditions: http://www.tbs-internet.com/CA/repository/OU=TBS INTERNET CA/CN=TBS X509 CA SGC + 0 s:/C=FR/postalCode=14000/ST=Calvados/L=CAEN/street=22 rue de Bretagne/O=TBS INTERNET/OU=0002 440443810/OU=Certificats TBS X509/CN=ecom.tbs-x509.com + i:/C=FR/ST=Calvados/L=Caen/O=TBS INTERNET/OU=Terms and Conditions: http://www.tbs-internet.com/CA/repository/OU=TBS INTERNET CA/CN=TBS X509 CA business -----BEGIN CERTIFICATE----- -MIIGXTCCBUWgAwIBAgIRAMmag+ygSAdxZsbyzYjhuW0wDQYJKoZIhvcNAQELBQAw -gcQxCzAJBgNVBAYTAkZSMREwDwYDVQQIEwhDYWx2YWRvczENMAsGA1UEBxMEQ2Fl +MIIGTjCCBTagAwIBAgIQOh3d9dNDPq1cSdJmEiMpqDANBgkqhkiG9w0BAQUFADCB +yTELMAkGA1UEBhMCRlIxETAPBgNVBAgTCENhbHZhZG9zMQ0wCwYDVQQHEwRDYWVu +MRUwEwYDVQQKEwxUQlMgSU5URVJORVQxSDBGBgNVBAsTP1Rlcm1zIGFuZCBDb25k +aXRpb25zOiBodHRwOi8vd3d3LnRicy1pbnRlcm5ldC5jb20vQ0EvcmVwb3NpdG9y +eTEYMBYGA1UECxMPVEJTIElOVEVSTkVUIENBMR0wGwYDVQQDExRUQlMgWDUwOSBD +QSBidXNpbmVzczAeFw0xMTAxMjUwMDAwMDBaFw0xMzAyMDUyMzU5NTlaMIHHMQsw +CQYDVQQGEwJGUjEOMAwGA1UEERMFMTQwMDAxETAPBgNVBAgTCENhbHZhZG9zMQ0w +CwYDVQQHEwRDQUVOMRswGQYDVQQJExIyMiBydWUgZGUgQnJldGFnbmUxFTATBgNV +BAoTDFRCUyBJTlRFUk5FVDEXMBUGA1UECxMOMDAwMiA0NDA0NDM4MTAxHTAbBgNV +BAsTFENlcnRpZmljYXRzIFRCUyBYNTA5MRowGAYDVQQDExFlY29tLnRicy14NTA5 +LmNvbTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAKRrlHUnJ++1lpcg +jtYco7cdmRe+EEfTmwPfCdfV3G1QfsTSvY6FfMpm/83pqHfT+4ANwr18wD9ZrAEN +G16mf9VdCGK12+TP7DmqeZyGIqlFFoahQnmb8EarvE43/1UeQ2CV9XmzwZvpqeli +LfXsFonawrY3H6ZnMwS64St61Z+9gdyuZ/RbsoZBbT5KUjDEG844QRU4OT1IGeEI +eY5NM5RNIh6ZNhVtqeeCxMS7afONkHQrOco73RdSTRck/Hj96Ofl3MHNHryr+AMK +DGFk1kLCZGpPdXtkxXvaDeQoiYDlil26CWc+YK6xyDPMdsWvoG14ZLyCpzMXA7/7 +4YAQRH0CAwEAAaOCAjAwggIsMB8GA1UdIwQYMBaAFBoJBMz5CY+7HqDO1KQUf0vV +I1jNMB0GA1UdDgQWBBQgOU8HsWzbmD4WZP5Wtdw7jca2WDAOBgNVHQ8BAf8EBAMC +BaAwDAYDVR0TAQH/BAIwADAdBgNVHSUEFjAUBggrBgEFBQcDAQYIKwYBBQUHAwIw +TAYDVR0gBEUwQzBBBgsrBgEEAYDlNwIBATAyMDAGCCsGAQUFBwIBFiRodHRwczov +L3d3dy50YnMtaW50ZXJuZXQuY29tL0NBL0NQUzEwdwYDVR0fBHAwbjA3oDWgM4Yx +aHR0cDovL2NybC50YnMtaW50ZXJuZXQuY29tL1RCU1g1MDlDQWJ1c2luZXNzLmNy +bDAzoDGgL4YtaHR0cDovL2NybC50YnMteDUwOS5jb20vVEJTWDUwOUNBYnVzaW5l +c3MuY3JsMIGwBggrBgEFBQcBAQSBozCBoDA9BggrBgEFBQcwAoYxaHR0cDovL2Ny +dC50YnMtaW50ZXJuZXQuY29tL1RCU1g1MDlDQWJ1c2luZXNzLmNydDA5BggrBgEF +BQcwAoYtaHR0cDovL2NydC50YnMteDUwOS5jb20vVEJTWDUwOUNBYnVzaW5lc3Mu +Y3J0MCQGCCsGAQUFBzABhhhodHRwOi8vb2NzcC50YnMteDUwOS5jb20wMwYDVR0R +BCwwKoIRZWNvbS50YnMteDUwOS5jb22CFXd3dy5lY29tLnRicy14NTA5LmNvbTAN +BgkqhkiG9w0BAQUFAAOCAQEArT4NHfbY87bGAw8lPV4DmHlmuDuVp/y7ltO3Ynse +3Rz8RxW2AzuO0Oy2F0Cu4yWKtMyEyMXyHqWtae7ElRbdTu5w5GwVBLJHClCzC8S9 +SpgMMQTx3Rgn8vjkHuU9VZQlulZyiPK7yunjc7c310S9FRZ7XxOwf8Nnx4WnB+No +WrfApzhhQl31w+RyrNxZe58hCfDDHmevRvwLjQ785ZoQXJDj2j3qAD4aI2yB8lB5 +oaE1jlCJzC7Kmz/Y9jzfmv/zAs1LQTm9ktevv4BTUFaGjv9jxnQ1xnS862ZiouLW +zZYIlYPf4F6JjXGiIQgQRglILUfq3ftJd9/ok9W9ZF8h8w== +-----END CERTIFICATE----- + 1 s:/C=FR/ST=Calvados/L=Caen/O=TBS INTERNET/OU=Terms and Conditions: http://www.tbs-internet.com/CA/repository/OU=TBS INTERNET CA/CN=TBS X509 CA business + i:/C=SE/O=AddTrust AB/OU=AddTrust External TTP Network/CN=AddTrust External CA Root +-----BEGIN CERTIFICATE----- +MIIFPzCCBCegAwIBAgIQDlBz/++iRSmLDeVRHT/hADANBgkqhkiG9w0BAQUFADBv +MQswCQYDVQQGEwJTRTEUMBIGA1UEChMLQWRkVHJ1c3QgQUIxJjAkBgNVBAsTHUFk +ZFRydXN0IEV4dGVybmFsIFRUUCBOZXR3b3JrMSIwIAYDVQQDExlBZGRUcnVzdCBF +eHRlcm5hbCBDQSBSb290MB4XDTA1MTIwMTAwMDAwMFoXDTE5MDcwOTE4MTkyMlow +gckxCzAJBgNVBAYTAkZSMREwDwYDVQQIEwhDYWx2YWRvczENMAsGA1UEBxMEQ2Fl bjEVMBMGA1UEChMMVEJTIElOVEVSTkVUMUgwRgYDVQQLEz9UZXJtcyBhbmQgQ29u ZGl0aW9uczogaHR0cDovL3d3dy50YnMtaW50ZXJuZXQuY29tL0NBL3JlcG9zaXRv -cnkxGDAWBgNVBAsTD1RCUyBJTlRFUk5FVCBDQTEYMBYGA1UEAxMPVEJTIFg1MDkg -Q0EgU0dDMB4XDTEwMDIxODAwMDAwMFoXDTEyMDIxOTIzNTk1OVowgcsxCzAJBgNV -BAYTAkZSMQ4wDAYDVQQREwUxNDAwMDERMA8GA1UECBMIQ2FsdmFkb3MxDTALBgNV -BAcTBENBRU4xGzAZBgNVBAkTEjIyIHJ1ZSBkZSBCcmV0YWduZTEVMBMGA1UEChMM -VEJTIElOVEVSTkVUMRcwFQYDVQQLEw4wMDAyIDQ0MDQ0MzgxMDEbMBkGA1UECxMS -c2hhLTI1NiBwcm9kdWN0aW9uMSAwHgYDVQQDExdzaGEyNTYudGJzLWludGVybmV0 -LmNvbTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAKbuM8VT7f0nntwu -N3F7v9KIBlhKNAxqCrziOXU5iqUt8HrQB3DtHbdmII+CpVUlwlmepsx6G+srEZ9a -MIGAy0nxi5aLb7watkyIdPjJTMvTUBQ/+RPWzt5JtYbbY9BlJ+yci0dctP74f4NU -ISLtlrEjUbf2gTohLrcE01TfmOF6PDEbB5PKDi38cB3NzKfizWfrOaJW6Q1C1qOJ -y4/4jkUREX1UFUIxzx7v62VfjXSGlcjGpBX1fvtABQOSLeE0a6gciDZs1REqroFf -5eXtqYphpTa14Z83ITXMfgg5Nze1VtMnzI9Qx4blYBw4dgQVEuIsYr7FDBOITDzc -VEVXZx0CAwEAAaOCAj8wggI7MB8GA1UdIwQYMBaAFAdEdoWTKLx/bXjSCuv6TEvf -2YIfMB0GA1UdDgQWBBSJKI/AYVI9RQNY0QPIqc8ej2QivTAOBgNVHQ8BAf8EBAMC -BaAwDAYDVR0TAQH/BAIwADA0BgNVHSUELTArBggrBgEFBQcDAQYIKwYBBQUHAwIG -CisGAQQBgjcKAwMGCWCGSAGG+EIEATBMBgNVHSAERTBDMEEGCysGAQQBgOU3AgQB -MDIwMAYIKwYBBQUHAgEWJGh0dHBzOi8vd3d3LnRicy1pbnRlcm5ldC5jb20vQ0Ev -Q1BTNDBtBgNVHR8EZjBkMDKgMKAuhixodHRwOi8vY3JsLnRicy1pbnRlcm5ldC5j -b20vVEJTWDUwOUNBU0dDLmNybDAuoCygKoYoaHR0cDovL2NybC50YnMteDUwOS5j -b20vVEJTWDUwOUNBU0dDLmNybDCBpgYIKwYBBQUHAQEEgZkwgZYwOAYIKwYBBQUH -MAKGLGh0dHA6Ly9jcnQudGJzLWludGVybmV0LmNvbS9UQlNYNTA5Q0FTR0MuY3J0 -MDQGCCsGAQUFBzAChihodHRwOi8vY3J0LnRicy14NTA5LmNvbS9UQlNYNTA5Q0FT -R0MuY3J0MCQGCCsGAQUFBzABhhhodHRwOi8vb2NzcC50YnMteDUwOS5jb20wPwYD -VR0RBDgwNoIXc2hhMjU2LnRicy1pbnRlcm5ldC5jb22CG3d3dy5zaGEyNTYudGJz -LWludGVybmV0LmNvbTANBgkqhkiG9w0BAQsFAAOCAQEAA5NL0D4QSqhErhlkdPmz -XtiMvdGL+ZehM4coTRIpasM/Agt36Rc0NzCvnQwKE+wkngg1Gy2qe7Q0E/ziqBtB -fZYzdVgu1zdiL4kTaf+wFKYAFGsFbyeEmXysy+CMwaNoF2vpSjCU1UD56bEnTX/W -fxVZYxtBQUpnu2wOsm8cDZuZRv9XrYgAhGj9Tt6F0aVHSDGn59uwShG1+BVF/uju -SCyPTTjL1oc7YElJUzR/x4mQJYvtQI8gDIDAGEOs7v3R/gKa5EMfbUQUI4C84UbI -Yz09Jdnws/MkC/Hm1BZEqk89u7Hvfv+oHqEb0XaUo0TDfsxE0M1sMdnLb91QNQBm -UQ== ------END CERTIFICATE----- - 1 s:/C=FR/ST=Calvados/L=Caen/O=TBS INTERNET/OU=Terms and Conditions: http://www.tbs-internet.com/CA/repository/OU=TBS INTERNET CA/CN=TBS X509 CA SGC - i:/C=SE/O=AddTrust AB/OU=AddTrust External TTP Network/CN=AddTrust External CA Root ------BEGIN CERTIFICATE----- -MIIFVjCCBD6gAwIBAgIQXpDZ0ETJMV02WTx3GTnhhTANBgkqhkiG9w0BAQUFADBv -MQswCQYDVQQGEwJTRTEUMBIGA1UEChMLQWRkVHJ1c3QgQUIxJjAkBgNVBAsTHUFk -ZFRydXN0IEV4dGVybmFsIFRUUCBOZXR3b3JrMSIwIAYDVQQDExlBZGRUcnVzdCBF -eHRlcm5hbCBDQSBSb290MB4XDTA1MTIwMTAwMDAwMFoXDTE5MDYyNDE5MDYzMFow -gcQxCzAJBgNVBAYTAkZSMREwDwYDVQQIEwhDYWx2YWRvczENMAsGA1UEBxMEQ2Fl -bjEVMBMGA1UEChMMVEJTIElOVEVSTkVUMUgwRgYDVQQLEz9UZXJtcyBhbmQgQ29u -ZGl0aW9uczogaHR0cDovL3d3dy50YnMtaW50ZXJuZXQuY29tL0NBL3JlcG9zaXRv -cnkxGDAWBgNVBAsTD1RCUyBJTlRFUk5FVCBDQTEYMBYGA1UEAxMPVEJTIFg1MDkg -Q0EgU0dDMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAsgOkO3f7wzN6 -rOjg45tR5vjBfzK7qmV9IBxb/QW9EEXxG+E7FNhZqQLtwGBKoSsHTnQqV75wWMk0 -9tinWvftBkSpj5sTi/8cbzJfUvTSVYh3Qxv6AVVjMMH/ruLjE6y+4PoaPs8WoYAQ -ts5R4Z1g8c/WnTepLst2x0/Wv7GmuoQi+gXvHU6YrBiu7XkeYhzc95QdviWSJRDk -owhb5K43qhcvjRmBfO/paGlCliDGZp8mHwrI21mwobWpVjTxZRwYO3bd4+TGcI4G -Ie5wmHwE8F7SK1tgSqbBacKjDa93j7txKkfz/Yd2n7TGqOXiHPsJpG655vrKtnXk -9vs1zoDeJQIDAQABo4IBljCCAZIwHQYDVR0OBBYEFAdEdoWTKLx/bXjSCuv6TEvf -2YIfMA4GA1UdDwEB/wQEAwIBBjASBgNVHRMBAf8ECDAGAQH/AgEAMCAGA1UdJQQZ -MBcGCisGAQQBgjcKAwMGCWCGSAGG+EIEATAYBgNVHSAEETAPMA0GCysGAQQBgOU3 -AgQBMHsGA1UdHwR0MHIwOKA2oDSGMmh0dHA6Ly9jcmwuY29tb2RvY2EuY29tL0Fk -ZFRydXN0RXh0ZXJuYWxDQVJvb3QuY3JsMDagNKAyhjBodHRwOi8vY3JsLmNvbW9k -by5uZXQvQWRkVHJ1c3RFeHRlcm5hbENBUm9vdC5jcmwwgYAGCCsGAQUFBwEBBHQw -cjA4BggrBgEFBQcwAoYsaHR0cDovL2NydC5jb21vZG9jYS5jb20vQWRkVHJ1c3RV -VE5TR0NDQS5jcnQwNgYIKwYBBQUHMAKGKmh0dHA6Ly9jcnQuY29tb2RvLm5ldC9B -ZGRUcnVzdFVUTlNHQ0NBLmNydDARBglghkgBhvhCAQEEBAMCAgQwDQYJKoZIhvcN -AQEFBQADggEBAK2zEzs+jcIrVK9oDkdDZNvhuBYTdCfpxfFs+OAujW0bIfJAy232 -euVsnJm6u/+OrqKudD2tad2BbejLLXhMZViaCmK7D9nrXHx4te5EP8rL19SUVqLY -1pTnv5dhNgEgvA7n5lIzDSYs7yRLsr7HJsYPr6SeYSuZizyX1SNz7ooJ32/F3X98 -RB0Mlc/E0OyOrkQ9/y5IrnpnaSora8CnUrV5XNOg+kyCz9edCyx4D5wXYcwZPVWz -8aDqquESrezPyjtfi4WRO4s/VD3HLZvOxzMrWAVYCDG9FxaOhF0QGuuG1F7F3GKV -v6prNyCl016kRl2j1UT+a7gLd8fA25A4C9E= +cnkxGDAWBgNVBAsTD1RCUyBJTlRFUk5FVCBDQTEdMBsGA1UEAxMUVEJTIFg1MDkg +Q0EgYnVzaW5lc3MwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDB1PAU +qudCcz3tmyGcf+u6EkZqonKKHrV4gZYbvVkIRojmmlhfi/jwvpHvo8bqSt/9Rj5S +jhCDW0pcbI+IPPtD1Jy+CHNSfnMqVDy6CKQ3p5maTzCMG6ZT+XjnvcND5v+FtaiB +xk1iCX6uvt0jeUtdZvYbyytsSDE6c3Y5//wRxOF8tM1JxibwO3pyER26jbbN2gQz +m/EkdGjLdJ4svPk23WDAvQ6G0/z2LcAaJB+XLfqRwfQpHQvfKa1uTi8PivC8qtip +rmNQMMPMjxSK2azX8cKjjTDJiUKaCb4VHlJDWKEsCFRpgJAoAuX8f7Yfs1M4esGo +sWb3PGspK3O22uIlAgMBAAGjggF6MIIBdjAdBgNVHQ4EFgQUGgkEzPkJj7seoM7U +pBR/S9UjWM0wDgYDVR0PAQH/BAQDAgEGMBIGA1UdEwEB/wQIMAYBAf8CAQAwGAYD +VR0gBBEwDzANBgsrBgEEAYDlNwIBATB7BgNVHR8EdDByMDigNqA0hjJodHRwOi8v +Y3JsLmNvbW9kb2NhLmNvbS9BZGRUcnVzdEV4dGVybmFsQ0FSb290LmNybDA2oDSg +MoYwaHR0cDovL2NybC5jb21vZG8ubmV0L0FkZFRydXN0RXh0ZXJuYWxDQVJvb3Qu +Y3JsMIGGBggrBgEFBQcBAQR6MHgwOwYIKwYBBQUHMAKGL2h0dHA6Ly9jcnQuY29t +b2RvY2EuY29tL0FkZFRydXN0VVROU2VydmVyQ0EuY3J0MDkGCCsGAQUFBzAChi1o +dHRwOi8vY3J0LmNvbW9kby5uZXQvQWRkVHJ1c3RVVE5TZXJ2ZXJDQS5jcnQwEQYJ +YIZIAYb4QgEBBAQDAgIEMA0GCSqGSIb3DQEBBQUAA4IBAQA7mqrMgk/MrE6QnbNA +h4nRCn2ti4bg4w2C3lB6bSvRPnYwuNw9Jb8vuKkNFzRDxNJXqVDZdfFW5CVQJuyd +nfAx83+wk+spzvFaE1KhFYfN9G9pQfXUfvDRoIcJgPEKUXL1wRiOG+IjU3VVI8pg +IgqHkr7ylln5i5zCiFAPuIJmYUSFg/gxH5xkCNcjJqqrHrHatJr6Qrrke93joupw +oU1njfAcZtYp6fbiK6u2b1pJqwkVBE8RsfLnPhRj+SFbpvjv8Od7o/ieJhFIYQNU +k2jX2u8qZnAiNw93LZW9lpYjtuvMXq8QQppENNja5b53q7UwI+lU7ZGjZ7quuESp +J6/5 -----END CERTIFICATE----- 2 s:/C=SE/O=AddTrust AB/OU=AddTrust External TTP Network/CN=AddTrust External CA Root - i:/C=US/ST=UT/L=Salt Lake City/O=The USERTRUST Network/OU=http://www.usertrust.com/CN=UTN - DATACorp SGC + i:/C=US/ST=UT/L=Salt Lake City/O=The USERTRUST Network/OU=http://www.usertrust.com/CN=UTN-USERFirst-Hardware -----BEGIN CERTIFICATE----- -MIIEZjCCA06gAwIBAgIQUSYKkxzif5zDpV954HKugjANBgkqhkiG9w0BAQUFADCB -kzELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAlVUMRcwFQYDVQQHEw5TYWx0IExha2Ug +MIIETzCCAzegAwIBAgIQHM5EYpUZep1jUvnyI6m2mDANBgkqhkiG9w0BAQUFADCB +lzELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAlVUMRcwFQYDVQQHEw5TYWx0IExha2Ug Q2l0eTEeMBwGA1UEChMVVGhlIFVTRVJUUlVTVCBOZXR3b3JrMSEwHwYDVQQLExho -dHRwOi8vd3d3LnVzZXJ0cnVzdC5jb20xGzAZBgNVBAMTElVUTiAtIERBVEFDb3Jw -IFNHQzAeFw0wNTA2MDcwODA5MTBaFw0xOTA2MjQxOTA2MzBaMG8xCzAJBgNVBAYT -AlNFMRQwEgYDVQQKEwtBZGRUcnVzdCBBQjEmMCQGA1UECxMdQWRkVHJ1c3QgRXh0 -ZXJuYWwgVFRQIE5ldHdvcmsxIjAgBgNVBAMTGUFkZFRydXN0IEV4dGVybmFsIENB -IFJvb3QwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQC39xoz5vIABC05 -4E5b7R+8bA/Ntfojts7emxEzl6QpTH2Tn71KvJPtAxrjj8/lbVBa1pcplFqAsEl6 -2y6V/bjKvzc4LR4+kUGtcFbH8E8/6DKedMrIkFTpxl8PeJ2aQDwOrGGqXhSPnoeh -alDc15pOrwWzpnGUnHGzUGAKxxOdOAeGAqjpqGkmGJCrTLBPI6s6T4TY386f4Wlv -u9dC12tE5Met7m1BX3JacQg3s3llpFmglDf3AC8NwpJy2tA4ctsUqEXEXSp9t7TW -xO6szRNEt8kr3UMAJfphuWlqWCMRt6czj1Z1WfXNKddGtworZbbTQm8Vsrh7++/p -XVPVNFonAgMBAAGjgdgwgdUwHwYDVR0jBBgwFoAUUzLRs89/+uDxoF2FTpLSnkUd -tE8wHQYDVR0OBBYEFK29mHo0tCb3+sQmVO8DveAky1QaMA4GA1UdDwEB/wQEAwIB -BjAPBgNVHRMBAf8EBTADAQH/MBEGCWCGSAGG+EIBAQQEAwIBAjAgBgNVHSUEGTAX -BgorBgEEAYI3CgMDBglghkgBhvhCBAEwPQYDVR0fBDYwNDAyoDCgLoYsaHR0cDov -L2NybC51c2VydHJ1c3QuY29tL1VUTi1EQVRBQ29ycFNHQy5jcmwwDQYJKoZIhvcN -AQEFBQADggEBAMbuUxdoFLJRIh6QWA2U/b3xcOWGLcM2MY9USEbnLQg3vGwKYOEO -rVE04BKT6b64q7gmtOmWPSiPrmQH/uAB7MXjkesYoPF1ftsK5p+R26+udd8jkWjd -FwBaS/9kbHDrARrQkNnHptZt9hPk/7XJ0h4qy7ElQyZ42TCbTg0evmnv3+r+LbPM -+bDdtRTKkdSytaX7ARmjR3mfnYyVhzT4HziS2jamEfpr62vp3EV4FTkG101B5CHI -3C+H0be/SGB1pWLLJN47YaApIKa+xWycxOkKaSLvkTr6Jq/RW0GnOuL4OAdCq8Fb -+M5tug8EPzI0rNwEKNdwMBQmBsTkm5jVz3g= +dHRwOi8vd3d3LnVzZXJ0cnVzdC5jb20xHzAdBgNVBAMTFlVUTi1VU0VSRmlyc3Qt +SGFyZHdhcmUwHhcNMDUwNjA3MDgwOTEwWhcNMTkwNzA5MTgxOTIyWjBvMQswCQYD +VQQGEwJTRTEUMBIGA1UEChMLQWRkVHJ1c3QgQUIxJjAkBgNVBAsTHUFkZFRydXN0 +IEV4dGVybmFsIFRUUCBOZXR3b3JrMSIwIAYDVQQDExlBZGRUcnVzdCBFeHRlcm5h +bCBDQSBSb290MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAt/caM+by +AAQtOeBOW+0fvGwPzbX6I7bO3psRM5ekKUx9k5+9SryT7QMa44/P5W1QWtaXKZRa +gLBJetsulf24yr83OC0ePpFBrXBWx/BPP+gynnTKyJBU6cZfD3idmkA8Dqxhql4U +j56HoWpQ3NeaTq8Fs6ZxlJxxs1BgCscTnTgHhgKo6ahpJhiQq0ywTyOrOk+E2N/O +n+Fpb7vXQtdrROTHre5tQV9yWnEIN7N5ZaRZoJQ39wAvDcKSctrQOHLbFKhFxF0q +fbe01sTurM0TRLfJK91DACX6YblpalgjEbenM49WdVn1zSnXRrcKK2W200JvFbK4 +e/vv6V1T1TRaJwIDAQABo4G9MIG6MB8GA1UdIwQYMBaAFKFyXyYbKJhDlV0HN9WF +lp1L0sNFMB0GA1UdDgQWBBStvZh6NLQm9/rEJlTvA73gJMtUGjAOBgNVHQ8BAf8E +BAMCAQYwDwYDVR0TAQH/BAUwAwEB/zARBglghkgBhvhCAQEEBAMCAQIwRAYDVR0f +BD0wOzA5oDegNYYzaHR0cDovL2NybC51c2VydHJ1c3QuY29tL1VUTi1VU0VSRmly +c3QtSGFyZHdhcmUuY3JsMA0GCSqGSIb3DQEBBQUAA4IBAQByQhANOs4kClrwF8BW +onvUOGCSjRK52zYZgDXYNjDtmr5rJ6NyPFDNn+JxkLpjYetIFMTbSRe679Bt8m7a +gIAoQYFQtxMuyLnJegB2aEbQiIxh/tC21UcFF7ktdnDoTlA6w3pLuvunaI84Of3o +2YBrhzkTbCfaYk5JRlTpudW9DkUkHBsyx3nknPKnplkIGaK0jgn8E0n+SFabYaHk +I9LroYT/+JtLefh9lgBdAgVv0UPbzoGfuDsrk/Zh+UrgbLFpHoVnElhzbkh64Z0X +OGaJunQc68cCZu5HTn/aK7fBGMcVflRCXLVEQpU9PIAdGA8Ynvg684t8GMaKsRl1 +jIGZ -----END CERTIFICATE----- - 3 s:/C=US/ST=UT/L=Salt Lake City/O=The USERTRUST Network/OU=http://www.usertrust.com/CN=UTN - DATACorp SGC - i:/C=US/ST=UT/L=Salt Lake City/O=The USERTRUST Network/OU=http://www.usertrust.com/CN=UTN - DATACorp SGC + 3 s:/C=US/ST=UT/L=Salt Lake City/O=The USERTRUST Network/OU=http://www.usertrust.com/CN=UTN-USERFirst-Hardware + i:/C=US/ST=UT/L=Salt Lake City/O=The USERTRUST Network/OU=http://www.usertrust.com/CN=UTN-USERFirst-Hardware -----BEGIN CERTIFICATE----- -MIIEXjCCA0agAwIBAgIQRL4Mi1AAIbQR0ypoBqmtaTANBgkqhkiG9w0BAQUFADCB -kzELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAlVUMRcwFQYDVQQHEw5TYWx0IExha2Ug +MIIEdDCCA1ygAwIBAgIQRL4Mi1AAJLQR0zYq/mUK/TANBgkqhkiG9w0BAQUFADCB +lzELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAlVUMRcwFQYDVQQHEw5TYWx0IExha2Ug Q2l0eTEeMBwGA1UEChMVVGhlIFVTRVJUUlVTVCBOZXR3b3JrMSEwHwYDVQQLExho -dHRwOi8vd3d3LnVzZXJ0cnVzdC5jb20xGzAZBgNVBAMTElVUTiAtIERBVEFDb3Jw -IFNHQzAeFw05OTA2MjQxODU3MjFaFw0xOTA2MjQxOTA2MzBaMIGTMQswCQYDVQQG -EwJVUzELMAkGA1UECBMCVVQxFzAVBgNVBAcTDlNhbHQgTGFrZSBDaXR5MR4wHAYD -VQQKExVUaGUgVVNFUlRSVVNUIE5ldHdvcmsxITAfBgNVBAsTGGh0dHA6Ly93d3cu -dXNlcnRydXN0LmNvbTEbMBkGA1UEAxMSVVROIC0gREFUQUNvcnAgU0dDMIIBIjAN -BgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA3+5YEKIrblXEjr8uRgnn4AgPLit6 -E5Qbvfa2gI5lBZMAHryv4g+OGQ0SR+ysraP6LnD43m77VkIVni5c7yPeIbkFdicZ -D0/Ww5y0vpQZY/KmEQrrU0icvvIpOxboGqBMpsn0GFlowHDyUwDAXlCCpVZvNvlK -4ESGoE1O1kduSUrLZ9emxAW5jh70/P/N5zbgnAVssjMiFdC04MwXwLLA9P4yPykq -lXvY8qdOD1R8oQ2AswkDwf9c3V6aPryuvEeKaq5xyh+xKrhfQgUL7EYw0XILyulW -bfXv33i+Ybqypa4ETLyorGkVl73v67SMvzX41MPRKA5cOp9wGDMgd8SirwIDAQAB -o4GrMIGoMAsGA1UdDwQEAwIBxjAPBgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBRT -MtGzz3/64PGgXYVOktKeRR20TzA9BgNVHR8ENjA0MDKgMKAuhixodHRwOi8vY3Js -LnVzZXJ0cnVzdC5jb20vVVROLURBVEFDb3JwU0dDLmNybDAqBgNVHSUEIzAhBggr -BgEFBQcDAQYKKwYBBAGCNwoDAwYJYIZIAYb4QgQBMA0GCSqGSIb3DQEBBQUAA4IB -AQAnNZcAiosovcYzMB4p/OL31ZjUQLtgyr+rFywJNn9Q+kHcrpY6CiM+iVnJowft -Gzet/Hy+UUla3joKVAgWRcKZsYfNjGjgaQPpxE6YsjuMFrMOoAyYUJuTqXAJyCyj -j98C5OBxOvG0I3KgqgHf35g+FFCgMSa9KOlaMCZ1+XtgHI3zzVAmbQQnmt/VDUVH -KWss5nbZqSl9Mt3JNjy9rjXxEZ4du5A/EkdOjtd+D2JzHVImOBwYSf0wdJrE5SIv -2MCN7ZF6TACPcn9d2t0bi0Vr591pl6jFVkwPDPafepE39peC4N1xaf92P2BNPM/3 -mfnGV/TJVTl4uix5yaaIK/QI +dHRwOi8vd3d3LnVzZXJ0cnVzdC5jb20xHzAdBgNVBAMTFlVUTi1VU0VSRmlyc3Qt +SGFyZHdhcmUwHhcNOTkwNzA5MTgxMDQyWhcNMTkwNzA5MTgxOTIyWjCBlzELMAkG +A1UEBhMCVVMxCzAJBgNVBAgTAlVUMRcwFQYDVQQHEw5TYWx0IExha2UgQ2l0eTEe +MBwGA1UEChMVVGhlIFVTRVJUUlVTVCBOZXR3b3JrMSEwHwYDVQQLExhodHRwOi8v +d3d3LnVzZXJ0cnVzdC5jb20xHzAdBgNVBAMTFlVUTi1VU0VSRmlyc3QtSGFyZHdh +cmUwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCx98M4P7Sof885glFn +0G2f0v9Y8+efK+wNiVSZuTiZFvfgIXlIwrthdBKWHTxqctU8EGc6Oe0rE81m65UJ +M6Rsl7HoxuzBdXmcRl6Nq9Bq/bkqVRcQVLMZ8Jr28bFdtqdt++BxF2uiiPsA3/4a +MXcMmgF6sTLjKwEHOG7DpV4jvEWbe1DByTCP2+UretNb+zNAHqDVmBe8i4fDidNd +oI6yqqr2jmmIBsX6iSHzCJ1pLgkzmykNRg+MzEk0sGlRvfkGzWitZky8PqxhvQqI +DsjfPe58BEydCl5rkdbux+0ojatNh4lz0G6k0B4WixThdkQDf2Os5M1JnMWS9Ksy +oUhbAgMBAAGjgbkwgbYwCwYDVR0PBAQDAgHGMA8GA1UdEwEB/wQFMAMBAf8wHQYD +VR0OBBYEFKFyXyYbKJhDlV0HN9WFlp1L0sNFMEQGA1UdHwQ9MDswOaA3oDWGM2h0 +dHA6Ly9jcmwudXNlcnRydXN0LmNvbS9VVE4tVVNFUkZpcnN0LUhhcmR3YXJlLmNy +bDAxBgNVHSUEKjAoBggrBgEFBQcDAQYIKwYBBQUHAwUGCCsGAQUFBwMGBggrBgEF +BQcDBzANBgkqhkiG9w0BAQUFAAOCAQEARxkP3nTGmZev/K0oXnWO6y1n7k57K9cM +//bey1WiCuFMVGWTYGufEpytXoMs61quwOQt9ABjHbjAbPLPSbtNk28Gpgoiskli +CE7/yMgUsogWXecB5BKV5UU0s4tpvc+0hY91UZ59Ojg6FEgSxvunOxqNDYJAB+gE +CJChicsZUN/KHAG8HQQZexB2lzvukJDKxA4fFm517zP4029bHpbj4HR3dHuKom4t +3XbWOTCC8KucUvIqx69JXn7HaOWCgchqJ/kniCrVWFCVH/A7HFe7fRQ5YiuayZSS +KqMiDP+JJn1fIytH1xUdqWqeUQ0qUZ6B+dQ7XnASfxAynB67nfhmqA== -----END CERTIFICATE----- diff --git a/lib-python/2.7/test/test_mailbox.py b/lib-python/2.7/test/test_mailbox.py --- a/lib-python/2.7/test/test_mailbox.py +++ b/lib-python/2.7/test/test_mailbox.py @@ -38,14 +38,9 @@ def _delete_recursively(self, target): # Delete a file or delete a directory recursively if os.path.isdir(target): - for path, dirs, files in os.walk(target, topdown=False): - for name in files: - os.remove(os.path.join(path, name)) - for name in dirs: - os.rmdir(os.path.join(path, name)) - os.rmdir(target) + test_support.rmtree(target) elif os.path.exists(target): - os.remove(target) + test_support.unlink(target) class TestMailbox(TestBase): @@ -137,6 +132,7 @@ msg = self._box.get(key1) self.assertEqual(msg['from'], 'foo') self.assertEqual(msg.fp.read(), '1') + msg.fp.close() def test_getitem(self): # Retrieve message using __getitem__() @@ -169,10 +165,14 @@ # Get file representations of messages key0 = self._box.add(self._template % 0) key1 = self._box.add(_sample_message) - self.assertEqual(self._box.get_file(key0).read().replace(os.linesep, '\n'), + msg0 = self._box.get_file(key0) + self.assertEqual(msg0.read().replace(os.linesep, '\n'), self._template % 0) - self.assertEqual(self._box.get_file(key1).read().replace(os.linesep, '\n'), + msg1 = self._box.get_file(key1) + self.assertEqual(msg1.read().replace(os.linesep, '\n'), _sample_message) + msg0.close() + msg1.close() def test_get_file_can_be_closed_twice(self): # Issue 11700 @@ -407,6 +407,7 @@ self._box.add(contents[0]) self._box.add(contents[1]) self._box.add(contents[2]) + oldbox = self._box method() if should_call_close: self._box.close() @@ -415,6 +416,7 @@ self.assertEqual(len(keys), 3) for key in keys: self.assertIn(self._box.get_string(key), contents) + oldbox.close() def test_dump_message(self): # Write message representations to disk @@ -1835,6 +1837,10 @@ def setUp(self): # create a new maildir mailbox to work with: self._dir = test_support.TESTFN + if os.path.isdir(self._dir): + test_support.rmtree(self._dir) + if os.path.isfile(self._dir): + test_support.unlink(self._dir) os.mkdir(self._dir) os.mkdir(os.path.join(self._dir, "cur")) os.mkdir(os.path.join(self._dir, "tmp")) @@ -1844,10 +1850,10 @@ def tearDown(self): map(os.unlink, self._msgfiles) - os.rmdir(os.path.join(self._dir, "cur")) - os.rmdir(os.path.join(self._dir, "tmp")) - os.rmdir(os.path.join(self._dir, "new")) - os.rmdir(self._dir) + test_support.rmdir(os.path.join(self._dir, "cur")) + test_support.rmdir(os.path.join(self._dir, "tmp")) + test_support.rmdir(os.path.join(self._dir, "new")) + test_support.rmdir(self._dir) def createMessage(self, dir, mbox=False): t = int(time.time() % 1000000) @@ -1883,7 +1889,9 @@ self.createMessage("cur") self.mbox = mailbox.Maildir(test_support.TESTFN) #self.assertTrue(len(self.mbox.boxes) == 1) - self.assertIsNot(self.mbox.next(), None) + msg = self.mbox.next() + self.assertIsNot(msg, None) + msg.fp.close() self.assertIs(self.mbox.next(), None) self.assertIs(self.mbox.next(), None) @@ -1891,7 +1899,9 @@ self.createMessage("new") self.mbox = mailbox.Maildir(test_support.TESTFN) #self.assertTrue(len(self.mbox.boxes) == 1) - self.assertIsNot(self.mbox.next(), None) + msg = self.mbox.next() + self.assertIsNot(msg, None) + msg.fp.close() self.assertIs(self.mbox.next(), None) self.assertIs(self.mbox.next(), None) @@ -1900,8 +1910,12 @@ self.createMessage("new") self.mbox = mailbox.Maildir(test_support.TESTFN) #self.assertTrue(len(self.mbox.boxes) == 2) - self.assertIsNot(self.mbox.next(), None) - self.assertIsNot(self.mbox.next(), None) + msg = self.mbox.next() + self.assertIsNot(msg, None) + msg.fp.close() + msg = self.mbox.next() + self.assertIsNot(msg, None) + msg.fp.close() self.assertIs(self.mbox.next(), None) self.assertIs(self.mbox.next(), None) @@ -1910,11 +1924,13 @@ import email.parser fname = self.createMessage("cur", True) n = 0 - for msg in mailbox.PortableUnixMailbox(open(fname), + fid = open(fname) + for msg in mailbox.PortableUnixMailbox(fid, email.parser.Parser().parse): n += 1 self.assertEqual(msg["subject"], "Simple Test") self.assertEqual(len(str(msg)), len(FROM_)+len(DUMMY_MESSAGE)) + fid.close() self.assertEqual(n, 1) ## End: classes from the original module (for backward compatibility). diff --git a/lib-python/2.7/test/test_mmap.py b/lib-python/2.7/test/test_mmap.py --- a/lib-python/2.7/test/test_mmap.py +++ b/lib-python/2.7/test/test_mmap.py @@ -11,7 +11,7 @@ def setUp(self): if os.path.exists(TESTFN): - os.unlink(TESTFN) + unlink(TESTFN) def tearDown(self): try: diff --git a/lib-python/2.7/test/test_old_mailbox.py b/lib-python/2.7/test/test_old_mailbox.py --- a/lib-python/2.7/test/test_old_mailbox.py +++ b/lib-python/2.7/test/test_old_mailbox.py @@ -73,7 +73,9 @@ self.createMessage("cur") self.mbox = mailbox.Maildir(test_support.TESTFN) self.assertTrue(len(self.mbox) == 1) - self.assertTrue(self.mbox.next() is not None) + msg = self.mbox.next() + self.assertTrue(msg is not None) + msg.fp.close() self.assertTrue(self.mbox.next() is None) self.assertTrue(self.mbox.next() is None) @@ -81,7 +83,9 @@ self.createMessage("new") self.mbox = mailbox.Maildir(test_support.TESTFN) self.assertTrue(len(self.mbox) == 1) - self.assertTrue(self.mbox.next() is not None) + msg = self.mbox.next() + self.assertTrue(msg is not None) + msg.fp.close() self.assertTrue(self.mbox.next() is None) self.assertTrue(self.mbox.next() is None) @@ -90,8 +94,12 @@ self.createMessage("new") self.mbox = mailbox.Maildir(test_support.TESTFN) self.assertTrue(len(self.mbox) == 2) - self.assertTrue(self.mbox.next() is not None) - self.assertTrue(self.mbox.next() is not None) + msg = self.mbox.next() + self.assertTrue(msg is not None) + msg.fp.close() + msg = self.mbox.next() + self.assertTrue(msg is not None) + msg.fp.close() self.assertTrue(self.mbox.next() is None) self.assertTrue(self.mbox.next() is None) diff --git a/lib-python/2.7/test/test_os.py b/lib-python/2.7/test/test_os.py --- a/lib-python/2.7/test/test_os.py +++ b/lib-python/2.7/test/test_os.py @@ -75,7 +75,7 @@ self.assertFalse(os.path.exists(name), "file already exists for temporary file") # make sure we can create the file - open(name, "w") + open(name, "w").close() self.files.append(name) def test_tempnam(self): diff --git a/lib-python/2.7/test/test_ssl.py b/lib-python/2.7/test/test_ssl.py --- a/lib-python/2.7/test/test_ssl.py +++ b/lib-python/2.7/test/test_ssl.py @@ -111,13 +111,12 @@ if test_support.verbose: sys.stdout.write("\n" + pprint.pformat(p) + "\n") self.assertEqual(p['subject'], - ((('countryName', u'US'),), - (('stateOrProvinceName', u'Delaware'),), - (('localityName', u'Wilmington'),), - (('organizationName', u'Python Software Foundation'),), - (('organizationalUnitName', u'SSL'),), - (('commonName', u'somemachine.python.org'),)), + ((('countryName', 'XY'),), + (('localityName', 'Castle Anthrax'),), + (('organizationName', 'Python Software Foundation'),), + (('commonName', 'localhost'),)) ) + self.assertEqual(p['subjectAltName'], (('DNS', 'localhost'),)) # Issue #13034: the subjectAltName in some certificates # (notably projects.developer.nokia.com:443) wasn't parsed p = ssl._ssl._test_decode_cert(NOKIACERT) diff --git a/lib-python/2.7/test/test_support.py b/lib-python/2.7/test/test_support.py --- a/lib-python/2.7/test/test_support.py +++ b/lib-python/2.7/test/test_support.py @@ -179,15 +179,79 @@ except KeyError: pass +if sys.platform.startswith("win"): + def _waitfor(func, pathname, waitall=False): + # Peform the operation + func(pathname) + # Now setup the wait loop + if waitall: + dirname = pathname + else: + dirname, name = os.path.split(pathname) + dirname = dirname or '.' + # Check for `pathname` to be removed from the filesystem. + # The exponential backoff of the timeout amounts to a total + # of ~1 second after which the deletion is probably an error + # anyway. + # Testing on a i7 at 4.3GHz shows that usually only 1 iteration is + # required when contention occurs. + timeout = 0.001 + while timeout < 1.0: + # Note we are only testing for the existance of the file(s) in + # the contents of the directory regardless of any security or + # access rights. If we have made it this far, we have sufficient + # permissions to do that much using Python's equivalent of the + # Windows API FindFirstFile. + # Other Windows APIs can fail or give incorrect results when + # dealing with files that are pending deletion. + L = os.listdir(dirname) + if not (L if waitall else name in L): + return + # Increase the timeout and try again + time.sleep(timeout) + timeout *= 2 + warnings.warn('tests may fail, delete still pending for ' + pathname, + RuntimeWarning, stacklevel=4) + + def _unlink(filename): + _waitfor(os.unlink, filename) + + def _rmdir(dirname): + _waitfor(os.rmdir, dirname) + + def _rmtree(path): + def _rmtree_inner(path): + for name in os.listdir(path): + fullname = os.path.join(path, name) + if os.path.isdir(fullname): + _waitfor(_rmtree_inner, fullname, waitall=True) + os.rmdir(fullname) + else: + os.unlink(fullname) + _waitfor(_rmtree_inner, path, waitall=True) + _waitfor(os.rmdir, path) +else: + _unlink = os.unlink + _rmdir = os.rmdir + _rmtree = shutil.rmtree + def unlink(filename): try: - os.unlink(filename) + _unlink(filename) except OSError: pass +def rmdir(dirname): + try: + _rmdir(dirname) + except OSError as error: + # The directory need not exist. + if error.errno != errno.ENOENT: + raise + def rmtree(path): try: - shutil.rmtree(path) + _rmtree(path) except OSError, e: # Unix returns ENOENT, Windows returns ESRCH. if e.errno not in (errno.ENOENT, errno.ESRCH): diff --git a/lib-python/2.7/test/test_tarfile.py b/lib-python/2.7/test/test_tarfile.py --- a/lib-python/2.7/test/test_tarfile.py +++ b/lib-python/2.7/test/test_tarfile.py @@ -300,26 +300,21 @@ def test_extract_hardlink(self): # Test hardlink extraction (e.g. bug #857297). - tar = tarfile.open(tarname, errorlevel=1, encoding="iso8859-1") + with tarfile.open(tarname, errorlevel=1, encoding="iso8859-1") as tar: + tar.extract("ustar/regtype", TEMPDIR) + self.addCleanup(os.remove, os.path.join(TEMPDIR, "ustar/regtype")) - tar.extract("ustar/regtype", TEMPDIR) - try: tar.extract("ustar/lnktype", TEMPDIR) - except EnvironmentError, e: - if e.errno == errno.ENOENT: - self.fail("hardlink not extracted properly") + self.addCleanup(os.remove, os.path.join(TEMPDIR, "ustar/lnktype")) + with open(os.path.join(TEMPDIR, "ustar/lnktype"), "rb") as f: + data = f.read() + self.assertEqual(md5sum(data), md5_regtype) - data = open(os.path.join(TEMPDIR, "ustar/lnktype"), "rb").read() - self.assertEqual(md5sum(data), md5_regtype) - - try: tar.extract("ustar/symtype", TEMPDIR) - except EnvironmentError, e: - if e.errno == errno.ENOENT: - self.fail("symlink not extracted properly") - - data = open(os.path.join(TEMPDIR, "ustar/symtype"), "rb").read() - self.assertEqual(md5sum(data), md5_regtype) + self.addCleanup(os.remove, os.path.join(TEMPDIR, "ustar/symtype")) + with open(os.path.join(TEMPDIR, "ustar/symtype"), "rb") as f: + data = f.read() + self.assertEqual(md5sum(data), md5_regtype) def test_extractall(self): # Test if extractall() correctly restores directory permissions @@ -340,7 +335,7 @@ # constructor in case of an error. For the test we rely on # the fact that opening an empty file raises a ReadError. empty = os.path.join(TEMPDIR, "empty") - open(empty, "wb").write("") + open(empty, "wb").close() try: tar = object.__new__(tarfile.TarFile) @@ -351,7 +346,7 @@ else: self.fail("ReadError not raised") finally: - os.remove(empty) + test_support.unlink(empty) class StreamReadTest(CommonReadTest): @@ -1327,7 +1322,7 @@ def setUp(self): self.tarname = tmpname if os.path.exists(self.tarname): - os.remove(self.tarname) + test_support.unlink(self.tarname) def _add_testfile(self, fileobj=None): tar = tarfile.open(self.tarname, "a", fileobj=fileobj) diff --git a/lib-python/2.7/traceback.py b/lib-python/2.7/traceback.py --- a/lib-python/2.7/traceback.py +++ b/lib-python/2.7/traceback.py @@ -107,7 +107,7 @@ return list -def print_exception(etype, value, tb, limit=None, file=None): +def print_exception(etype, value, tb, limit=None, file=None, _encoding=None): """Print exception up to 'limit' stack trace entries from 'tb' to 'file'. This differs from print_tb() in the following ways: (1) if @@ -123,7 +123,7 @@ if tb: _print(file, 'Traceback (most recent call last):') print_tb(tb, limit, file) - lines = format_exception_only(etype, value) + lines = format_exception_only(etype, value, _encoding) for line in lines: _print(file, line, '') @@ -144,7 +144,7 @@ list = list + format_exception_only(etype, value) return list -def format_exception_only(etype, value): +def format_exception_only(etype, value, _encoding=None): """Format the exception part of a traceback. The arguments are the exception type and value such as given by @@ -170,12 +170,12 @@ if (isinstance(etype, BaseException) or isinstance(etype, types.InstanceType) or etype is None or type(etype) is str): - return [_format_final_exc_line(etype, value)] + return [_format_final_exc_line(etype, value, _encoding)] stype = etype.__name__ if not issubclass(etype, SyntaxError): - return [_format_final_exc_line(stype, value)] + return [_format_final_exc_line(stype, value, _encoding)] # It was a syntax error; show exactly where the problem was found. lines = [] @@ -196,26 +196,26 @@ lines.append(' %s^\n' % ''.join(caretspace)) value = msg - lines.append(_format_final_exc_line(stype, value)) + lines.append(_format_final_exc_line(stype, value, _encoding)) return lines -def _format_final_exc_line(etype, value): +def _format_final_exc_line(etype, value, _encoding=None): """Return a list of a single line -- normal case for format_exception_only""" - valuestr = _some_str(value) + valuestr = _some_str(value, _encoding) if value is None or not valuestr: line = "%s\n" % etype else: line = "%s: %s\n" % (etype, valuestr) return line -def _some_str(value): +def _some_str(value, _encoding=None): try: return str(value) except Exception: pass try: value = unicode(value) - return value.encode("ascii", "backslashreplace") + return value.encode(_encoding or "ascii", "backslashreplace") except Exception: pass return '' % type(value).__name__ diff --git a/lib-python/2.7/weakref.py b/lib-python/2.7/weakref.py --- a/lib-python/2.7/weakref.py +++ b/lib-python/2.7/weakref.py @@ -48,7 +48,14 @@ def remove(wr, selfref=ref(self)): self = selfref() if self is not None: - del self.data[wr.key] + # Changed this for PyPy: made more resistent. The + # issue is that in some corner cases, self.data + # might already be changed or removed by the time + # this weakref's callback is called. If that is + # the case, we don't want to randomly kill an + # unrelated entry. + if self.data.get(wr.key) is wr: + del self.data[wr.key] self._remove = remove UserDict.UserDict.__init__(self, *args, **kw) @@ -160,22 +167,26 @@ try: o = self.data.pop(key)() except KeyError: + o = None + if o is None: if args: return args[0] - raise - if o is None: raise KeyError, key else: return o + # The logic above was fixed in PyPy def setdefault(self, key, default=None): try: - wr = self.data[key] + o = self.data[key]() except KeyError: + o = None + if o is None: self.data[key] = KeyedRef(default, self._remove, key) return default else: - return wr() + return o + # The logic above was fixed in PyPy def update(self, dict=None, **kwargs): d = self.data diff --git a/lib_pypy/_sqlite3.py b/lib_pypy/_sqlite3.py --- a/lib_pypy/_sqlite3.py +++ b/lib_pypy/_sqlite3.py @@ -268,10 +268,18 @@ if _has_load_extension(): _ffi.cdef("int sqlite3_enable_load_extension(sqlite3 *db, int onoff);") -_lib = _ffi.verify(""" -#include -""", libraries=['sqlite3'] -) +if sys.platform.startswith('freebsd'): + _lib = _ffi.verify(""" + #include + """, libraries=['sqlite3'], + include_dirs=['/usr/local/include'], + library_dirs=['/usr/local/lib'] + ) +else: + _lib = _ffi.verify(""" + #include + """, libraries=['sqlite3'] + ) exported_sqlite_symbols = [ 'SQLITE_ALTER_TABLE', @@ -363,9 +371,11 @@ pass -def connect(database, **kwargs): - factory = kwargs.get("factory", Connection) - return factory(database, **kwargs) +def connect(database, timeout=5.0, detect_types=0, isolation_level="", + check_same_thread=True, factory=None, cached_statements=100): + factory = Connection if not factory else factory + return factory(database, timeout, detect_types, isolation_level, + check_same_thread, factory, cached_statements) def _unicode_text_factory(x): diff --git a/lib_pypy/_tkinter/tclobj.py b/lib_pypy/_tkinter/tclobj.py --- a/lib_pypy/_tkinter/tclobj.py +++ b/lib_pypy/_tkinter/tclobj.py @@ -28,9 +28,11 @@ return result elif value.typePtr == typeCache.BooleanType: - return result + return bool(value.internalRep.longValue) elif value.typePtr == typeCache.ByteArrayType: - return result + size = tkffi.new('int*') + data = tklib.Tcl_GetByteArrayFromObj(value, size) + return tkffi.buffer(data, size[0])[:] elif value.typePtr == typeCache.DoubleType: return value.internalRep.doubleValue elif value.typePtr == typeCache.IntType: @@ -50,7 +52,7 @@ result.append(FromObj(app, tcl_elem[0])) return tuple(result) elif value.typePtr == typeCache.ProcBodyType: - return result + pass # fall through and return tcl object. elif value.typePtr == typeCache.StringType: buf = tklib.Tcl_GetUnicode(value) length = tklib.Tcl_GetCharLength(value) diff --git a/lib_pypy/_tkinter/tklib.py b/lib_pypy/_tkinter/tklib.py --- a/lib_pypy/_tkinter/tklib.py +++ b/lib_pypy/_tkinter/tklib.py @@ -72,6 +72,7 @@ int Tcl_GetBoolean(Tcl_Interp* interp, const char* src, int* boolPtr); char *Tcl_GetString(Tcl_Obj* objPtr); char *Tcl_GetStringFromObj(Tcl_Obj* objPtr, int* lengthPtr); +unsigned char *Tcl_GetByteArrayFromObj(Tcl_Obj* objPtr, int* lengthPtr); Tcl_UniChar *Tcl_GetUnicode(Tcl_Obj* objPtr); int Tcl_GetCharLength(Tcl_Obj* objPtr); @@ -111,6 +112,14 @@ incdirs = ['/usr/local/include/tcl8.5', '/usr/local/include/tk8.5', '/usr/X11R6/include'] linklibs = ['tk85', 'tcl85'] libdirs = ['/usr/local/lib', '/usr/X11R6/lib'] +elif sys.platform.startswith("freebsd"): + incdirs = ['/usr/local/include/tcl8.6', '/usr/local/include/tk8.6', '/usr/local/include/X11', '/usr/local/include'] + linklibs = ['tk86', 'tcl86'] + libdirs = ['/usr/local/lib'] +elif sys.platform == 'win32': + incdirs = [] + linklibs = ['tcl85', 'tk85'] + libdirs = [] else: incdirs=['/usr/include/tcl'] linklibs=['tcl', 'tk'] diff --git a/lib_pypy/cffi.egg-info b/lib_pypy/cffi.egg-info --- a/lib_pypy/cffi.egg-info +++ b/lib_pypy/cffi.egg-info @@ -1,6 +1,6 @@ Metadata-Version: 1.0 Name: cffi -Version: 0.7 +Version: 0.8 Summary: Foreign Function Interface for Python calling C code. Home-page: http://cffi.readthedocs.org Author: Armin Rigo, Maciej Fijalkowski diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py --- a/lib_pypy/cffi/__init__.py +++ b/lib_pypy/cffi/__init__.py @@ -4,5 +4,5 @@ from .api import FFI, CDefError, FFIError from .ffiplatform import VerificationError, VerificationMissing -__version__ = "0.7.2" -__version_info__ = (0, 7, 2) +__version__ = "0.8" +__version_info__ = (0, 8) diff --git a/lib_pypy/cffi/api.py b/lib_pypy/cffi/api.py --- a/lib_pypy/cffi/api.py +++ b/lib_pypy/cffi/api.py @@ -1,4 +1,5 @@ import types +from .lock import allocate_lock try: callable @@ -61,6 +62,7 @@ # rely on it! It's probably not going to work well.) self._backend = backend + self._lock = allocate_lock() self._parser = cparser.Parser() self._cached_btypes = {} self._parsed_types = types.ModuleType('parsed_types').__dict__ @@ -74,7 +76,8 @@ if name.startswith('RTLD_'): setattr(self, name, getattr(backend, name)) # - self.BVoidP = self._get_cached_btype(model.voidp_type) + with self._lock: + self.BVoidP = self._get_cached_btype(model.voidp_type) if isinstance(backend, types.ModuleType): # _cffi_backend: attach these constants to the class if not hasattr(FFI, 'NULL'): @@ -95,11 +98,12 @@ if not isinstance(csource, basestring): raise TypeError("cdef() argument must be a string") csource = csource.encode('ascii') - self._parser.parse(csource, override=override) - self._cdefsources.append(csource) - if override: - for cache in self._function_caches: - cache.clear() + with self._lock: + self._parser.parse(csource, override=override) + self._cdefsources.append(csource) + if override: + for cache in self._function_caches: + cache.clear() def dlopen(self, name, flags=0): """Load and return a dynamic library identified by 'name'. @@ -109,31 +113,47 @@ library we only look for the actual (untyped) symbols. """ assert isinstance(name, basestring) or name is None - lib, function_cache = _make_ffi_library(self, name, flags) - self._function_caches.append(function_cache) - self._libraries.append(lib) + with self._lock: + lib, function_cache = _make_ffi_library(self, name, flags) + self._function_caches.append(function_cache) + self._libraries.append(lib) return lib + def _typeof_locked(self, cdecl): + # call me with the lock! + key = cdecl + if key in self._parsed_types: + return self._parsed_types[key] + # + if not isinstance(cdecl, str): # unicode, on Python 2 + cdecl = cdecl.encode('ascii') + # + type = self._parser.parse_type(cdecl) + really_a_function_type = type.is_raw_function + if really_a_function_type: + type = type.as_function_pointer() + btype = self._get_cached_btype(type) + result = btype, really_a_function_type + self._parsed_types[key] = result + return result + def _typeof(self, cdecl, consider_function_as_funcptr=False): # string -> ctype object try: - btype, cfaf = self._parsed_types[cdecl] - if consider_function_as_funcptr and not cfaf: - raise KeyError + result = self._parsed_types[cdecl] except KeyError: - key = cdecl - if not isinstance(cdecl, str): # unicode, on Python 2 - cdecl = cdecl.encode('ascii') - cfaf = consider_function_as_funcptr - type = self._parser.parse_type(cdecl, - consider_function_as_funcptr=cfaf) - btype = self._get_cached_btype(type) - self._parsed_types[key] = btype, cfaf + with self._lock: + result = self._typeof_locked(cdecl) + # + btype, really_a_function_type = result + if really_a_function_type and not consider_function_as_funcptr: + raise CDefError("the type %r is a function type, not a " + "pointer-to-function type" % (cdecl,)) return btype def typeof(self, cdecl): """Parse the C type given as a string and return the - corresponding Python type: '>. + corresponding object. It can also be used on 'cdata' instance to get its C type. """ if isinstance(cdecl, basestring): @@ -144,6 +164,10 @@ res = _builtin_function_type(cdecl) if res is not None: return res + if (isinstance(cdecl, types.FunctionType) + and hasattr(cdecl, '_cffi_base_type')): + with self._lock: + return self._get_cached_btype(cdecl._cffi_base_type) raise TypeError(type(cdecl)) def sizeof(self, cdecl): @@ -280,14 +304,17 @@ data. Later, when this new cdata object is garbage-collected, 'destructor(old_cdata_object)' will be called. """ - try: - gc_weakrefs = self.gc_weakrefs - except AttributeError: - from .gc_weakref import GcWeakrefs - gc_weakrefs = self.gc_weakrefs = GcWeakrefs(self) - return gc_weakrefs.build(cdata, destructor) + with self._lock: + try: + gc_weakrefs = self.gc_weakrefs + except AttributeError: + from .gc_weakref import GcWeakrefs + gc_weakrefs = self.gc_weakrefs = GcWeakrefs(self) + return gc_weakrefs.build(cdata, destructor) def _get_cached_btype(self, type): + assert self._lock.acquire(False) is False + # call me with the lock! try: BType = self._cached_btypes[type] except KeyError: @@ -320,9 +347,13 @@ errno = property(_get_errno, _set_errno, None, "the value of 'errno' from/to the C calls") + def getwinerror(self, code=-1): + return self._backend.getwinerror(code) + def _pointer_to(self, ctype): from . import model - return model.pointer_cache(self, ctype) + with self._lock: + return model.pointer_cache(self, ctype) def addressof(self, cdata, field=None): """Return the address of a . @@ -342,10 +373,12 @@ variables, which must anyway be accessed directly from the lib object returned by the original FFI instance. """ - self._parser.include(ffi_to_include._parser) - self._cdefsources.append('[') - self._cdefsources.extend(ffi_to_include._cdefsources) - self._cdefsources.append(']') + with ffi_to_include._lock: + with self._lock: + self._parser.include(ffi_to_include._parser) + self._cdefsources.append('[') + self._cdefsources.extend(ffi_to_include._cdefsources) + self._cdefsources.append(']') def new_handle(self, x): return self._backend.newp_handle(self.BVoidP, x) @@ -372,7 +405,7 @@ backendlib = backend.load_library(path, flags) copied_enums = [] # - def make_accessor(name): + def make_accessor_locked(name): key = 'function ' + name if key in ffi._parser._declarations: tp = ffi._parser._declarations[key] @@ -404,11 +437,17 @@ if enumname not in library.__dict__: library.__dict__[enumname] = enumval copied_enums.append(True) + if name in library.__dict__: + return # - if name in library.__dict__: # copied from an enum value just above, - return # or multithread's race condition raise AttributeError(name) # + def make_accessor(name): + with ffi._lock: + if name in library.__dict__ or name in FFILibrary.__dict__: + return # added by another thread while waiting for the lock + make_accessor_locked(name) + # class FFILibrary(object): def __getattr__(self, name): make_accessor(name) @@ -444,4 +483,5 @@ except (KeyError, AttributeError, TypeError): return None else: - return ffi._get_cached_btype(tp) + with ffi._lock: + return ffi._get_cached_btype(tp) diff --git a/lib_pypy/cffi/cparser.py b/lib_pypy/cffi/cparser.py --- a/lib_pypy/cffi/cparser.py +++ b/lib_pypy/cffi/cparser.py @@ -142,7 +142,7 @@ if 1 <= linenum <= len(csourcelines): line = csourcelines[linenum-1] if line: - msg = 'cannot parse "%s"\n%s' % (line, msg) + msg = 'cannot parse "%s"\n%s' % (line.strip(), msg) else: msg = 'parse error\n%s' % (msg,) raise api.CDefError(msg) @@ -217,19 +217,18 @@ # if decl.name: tp = self._get_type(node, partial_length_ok=True) - if self._is_constant_declaration(node): + if self._is_constant_globalvar(node): self._declare('constant ' + decl.name, tp) else: self._declare('variable ' + decl.name, tp) - def parse_type(self, cdecl, consider_function_as_funcptr=False): + def parse_type(self, cdecl): ast, macros = self._parse('void __dummy(\n%s\n);' % cdecl) assert not macros exprnode = ast.ext[-1].type.args.params[0] if isinstance(exprnode, pycparser.c_ast.ID): raise api.CDefError("unknown identifier '%s'" % (exprnode.name,)) - return self._get_type(exprnode.type, - consider_function_as_funcptr=consider_function_as_funcptr) + return self._get_type(exprnode.type) def _declare(self, name, obj): if name in self._declarations: @@ -249,28 +248,17 @@ return model.ConstPointerType(type) return model.PointerType(type) - def _get_type(self, typenode, convert_array_to_pointer=False, - name=None, partial_length_ok=False, - consider_function_as_funcptr=False): + def _get_type(self, typenode, name=None, partial_length_ok=False): # first, dereference typedefs, if we have it already parsed, we're good if (isinstance(typenode, pycparser.c_ast.TypeDecl) and isinstance(typenode.type, pycparser.c_ast.IdentifierType) and len(typenode.type.names) == 1 and ('typedef ' + typenode.type.names[0]) in self._declarations): type = self._declarations['typedef ' + typenode.type.names[0]] - if isinstance(type, model.ArrayType): - if convert_array_to_pointer: - return type.item - else: - if (consider_function_as_funcptr and - isinstance(type, model.RawFunctionType)): - return type.as_function_pointer() return type # if isinstance(typenode, pycparser.c_ast.ArrayDecl): # array type - if convert_array_to_pointer: - return self._get_type_pointer(self._get_type(typenode.type)) if typenode.dim is None: length = None else: @@ -331,10 +319,7 @@ # if isinstance(typenode, pycparser.c_ast.FuncDecl): # a function type - result = self._parse_function_type(typenode, name) - if consider_function_as_funcptr: - result = result.as_function_pointer() - return result + return self._parse_function_type(typenode, name) # # nested anonymous structs or unions end up here if isinstance(typenode, pycparser.c_ast.Struct): @@ -365,21 +350,24 @@ isinstance(params[0].type.type, pycparser.c_ast.IdentifierType) and list(params[0].type.type.names) == ['void']): del params[0] - args = [self._get_type(argdeclnode.type, - convert_array_to_pointer=True, - consider_function_as_funcptr=True) + args = [self._as_func_arg(self._get_type(argdeclnode.type)) for argdeclnode in params] result = self._get_type(typenode.type) return model.RawFunctionType(tuple(args), result, ellipsis) - def _is_constant_declaration(self, typenode, const=False): - if isinstance(typenode, pycparser.c_ast.ArrayDecl): - return self._is_constant_declaration(typenode.type) + def _as_func_arg(self, type): + if isinstance(type, model.ArrayType): + return model.PointerType(type.item) + elif isinstance(type, model.RawFunctionType): + return type.as_function_pointer() + else: + return type + + def _is_constant_globalvar(self, typenode): if isinstance(typenode, pycparser.c_ast.PtrDecl): - const = 'const' in typenode.quals - return self._is_constant_declaration(typenode.type, const) + return 'const' in typenode.quals if isinstance(typenode, pycparser.c_ast.TypeDecl): - return const or 'const' in typenode.quals + return 'const' in typenode.quals return False def _get_struct_union_enum_type(self, kind, type, name=None, nested=False): @@ -491,7 +479,7 @@ return tp def _make_partial(self, tp, nested): - if not isinstance(tp, model.StructType): + if not isinstance(tp, model.StructOrUnion): raise api.CDefError("%s cannot be partial" % (tp,)) if not tp.has_c_name() and not nested: raise NotImplementedError("%s is partial but has no C name" %(tp,)) @@ -511,7 +499,7 @@ if (isinstance(exprnode, pycparser.c_ast.ID) and exprnode.name == '__dotdotdotarray__'): self._partial_length = True - return None + return '...' # raise api.FFIError("unsupported expression: expected a " "simple numeric constant") diff --git a/lib_pypy/cffi/gc_weakref.py b/lib_pypy/cffi/gc_weakref.py --- a/lib_pypy/cffi/gc_weakref.py +++ b/lib_pypy/cffi/gc_weakref.py @@ -14,6 +14,6 @@ def build(self, cdata, destructor): # make a new cdata of the same type as the original one - new_cdata = self.ffi.cast(self.ffi.typeof(cdata), cdata) + new_cdata = self.ffi.cast(self.ffi._backend.typeof(cdata), cdata) self.data[ref(new_cdata, self.remove)] = destructor, cdata return new_cdata diff --git a/lib_pypy/cffi/lock.py b/lib_pypy/cffi/lock.py new file mode 100644 --- /dev/null +++ b/lib_pypy/cffi/lock.py @@ -0,0 +1,30 @@ +import sys + +if sys.version_info < (3,): + try: + from thread import allocate_lock + except ImportError: + from dummy_thread import allocate_lock +else: + try: + from _thread import allocate_lock + except ImportError: + from _dummy_thread import allocate_lock + + +##import sys +##l1 = allocate_lock + +##class allocate_lock(object): +## def __init__(self): +## self._real = l1() +## def __enter__(self): +## for i in range(4, 0, -1): +## print sys._getframe(i).f_code +## print +## return self._real.__enter__() +## def __exit__(self, *args): +## return self._real.__exit__(*args) +## def acquire(self, f): +## assert f is False +## return self._real.acquire(f) diff --git a/lib_pypy/cffi/model.py b/lib_pypy/cffi/model.py --- a/lib_pypy/cffi/model.py +++ b/lib_pypy/cffi/model.py @@ -1,7 +1,10 @@ import weakref +from .lock import allocate_lock + class BaseTypeByIdentity(object): is_array_type = False + is_raw_function = False def get_c_name(self, replace_with='', context='a C file'): result = self.c_name_with_marker @@ -146,6 +149,7 @@ # a function, but not a pointer-to-function. The backend has no # notion of such a type; it's used temporarily by parsing. _base_pattern = '(&)(%s)' + is_raw_function = True def build_backend_type(self, ffi, finishlist): from . import api @@ -192,10 +196,6 @@ _base_pattern = " const *&" _base_pattern_array = "(const *&)" - def build_backend_type(self, ffi, finishlist): - BPtr = PointerType(self.totype).get_cached_btype(ffi, finishlist) - return BPtr - const_voidp_type = ConstPointerType(void_type) @@ -216,10 +216,12 @@ self.item = item self.length = length # - if self.length is None: + if length is None: brackets = '&[]' + elif length == '...': + brackets = '&[/*...*/]' else: - brackets = '&[%d]' % self.length + brackets = '&[%d]' % length self.c_name_with_marker = ( self.item.c_name_with_marker.replace('&', brackets)) @@ -227,6 +229,10 @@ return ArrayType(self.item, newlength) def build_backend_type(self, ffi, finishlist): + if self.length == '...': + from . import api + raise api.CDefError("cannot render the type %r: unknown length" % + (self,)) self.item.get_cached_btype(ffi, finishlist) # force the item BType BPtrItem = PointerType(self.item).get_cached_btype(ffi, finishlist) return global_cache(self, ffi, 'new_array_type', BPtrItem, self.length) @@ -252,6 +258,7 @@ class StructOrUnion(StructOrUnionOrEnum): fixedlayout = None completed = False + partial = False def __init__(self, name, fldnames, fldtypes, fldbitsize): self.name = name @@ -303,20 +310,21 @@ return # not completing it: it's an opaque struct # self.completed = 1 - fldtypes = tuple(tp.get_cached_btype(ffi, finishlist) - for tp in self.fldtypes) # if self.fixedlayout is None: + fldtypes = [tp.get_cached_btype(ffi, finishlist) + for tp in self.fldtypes] lst = list(zip(self.fldnames, fldtypes, self.fldbitsize)) ffi._backend.complete_struct_or_union(BType, lst, self) # else: + fldtypes = [] fieldofs, fieldsize, totalsize, totalalignment = self.fixedlayout for i in range(len(self.fldnames)): fsize = fieldsize[i] ftype = self.fldtypes[i] # - if isinstance(ftype, ArrayType) and ftype.length is None: + if isinstance(ftype, ArrayType) and ftype.length == '...': # fix the length to match the total size BItemType = ftype.item.get_cached_btype(ffi, finishlist) nlen, nrest = divmod(fsize, ffi.sizeof(BItemType)) @@ -327,18 +335,20 @@ ftype = ftype.resolve_length(nlen) self.fldtypes = (self.fldtypes[:i] + (ftype,) + self.fldtypes[i+1:]) - BArrayType = ftype.get_cached_btype(ffi, finishlist) - fldtypes = (fldtypes[:i] + (BArrayType,) + - fldtypes[i+1:]) - continue # - bitemsize = ffi.sizeof(fldtypes[i]) - if bitemsize != fsize: - self._verification_error( - "field '%s.%s' is declared as %d bytes, but is " - "really %d bytes" % (self.name, - self.fldnames[i] or '{}', - bitemsize, fsize)) + BFieldType = ftype.get_cached_btype(ffi, finishlist) + if isinstance(ftype, ArrayType) and ftype.length is None: + assert fsize == 0 + else: + bitemsize = ffi.sizeof(BFieldType) + if bitemsize != fsize: + self._verification_error( + "field '%s.%s' is declared as %d bytes, but is " + "really %d bytes" % (self.name, + self.fldnames[i] or '{}', + bitemsize, fsize)) + fldtypes.append(BFieldType) + # lst = list(zip(self.fldnames, fldtypes, self.fldbitsize, fieldofs)) ffi._backend.complete_struct_or_union(BType, lst, self, totalsize, totalalignment) @@ -348,11 +358,6 @@ from .ffiplatform import VerificationError raise VerificationError(msg) - -class StructType(StructOrUnion): - kind = 'struct' - partial = False - def check_not_partial(self): if self.partial and self.fixedlayout is None: from . import ffiplatform @@ -361,19 +366,18 @@ def build_backend_type(self, ffi, finishlist): self.check_not_partial() finishlist.append(self) - - return global_cache(self, ffi, 'new_struct_type', + # + return global_cache(self, ffi, 'new_%s_type' % self.kind, self.get_official_name(), key=self) +class StructType(StructOrUnion): + kind = 'struct' + + class UnionType(StructOrUnion): kind = 'union' - def build_backend_type(self, ffi, finishlist): - finishlist.append(self) - return global_cache(self, ffi, 'new_union_type', - self.get_official_name(), key=self) - class EnumType(StructOrUnionOrEnum): kind = 'enum' @@ -387,6 +391,12 @@ self.baseinttype = baseinttype self.build_c_name_with_marker() + def force_the_name(self, forcename): + StructOrUnionOrEnum.force_the_name(self, forcename) + if self.forcename is None: + name = self.get_official_name() + self.forcename = '$' + name.replace(' ', '_') + def check_not_partial(self): if self.partial and not self.partial_resolved: from . import ffiplatform @@ -444,6 +454,9 @@ tp = StructType(structname, None, None, None) return NamedPointerType(tp, name) + +global_lock = allocate_lock() + def global_cache(srctype, ffi, funcname, *args, **kwds): key = kwds.pop('key', (funcname, args)) assert not kwds @@ -464,8 +477,17 @@ res = getattr(ffi._backend, funcname)(*args) except NotImplementedError as e: raise NotImplementedError("%r: %s" % (srctype, e)) - ffi._backend.__typecache[key] = res - return res + # note that setdefault() on WeakValueDictionary is not atomic + # and contains a rare bug (http://bugs.python.org/issue19542); + # we have to use a lock and do it ourselves + cache = ffi._backend.__typecache + with global_lock: + res1 = cache.get(key) + if res1 is None: + cache[key] = res + return res + else: + return res1 def pointer_cache(ffi, BType): return global_cache('?', ffi, 'new_pointer_type', BType) diff --git a/lib_pypy/cffi/vengine_cpy.py b/lib_pypy/cffi/vengine_cpy.py --- a/lib_pypy/cffi/vengine_cpy.py +++ b/lib_pypy/cffi/vengine_cpy.py @@ -15,7 +15,7 @@ def patch_extension_kwds(self, kwds): pass - def find_module(self, module_name, path, so_suffix): + def find_module(self, module_name, path, so_suffixes): try: f, filename, descr = imp.find_module(module_name, path) except ImportError: @@ -25,7 +25,7 @@ # Note that after a setuptools installation, there are both .py # and .so files with the same basename. The code here relies on # imp.find_module() locating the .so in priority. - if descr[0] != so_suffix: + if descr[0] not in so_suffixes: return None return filename @@ -160,7 +160,10 @@ def __dir__(self): return FFILibrary._cffi_dir + list(self.__dict__) library = FFILibrary() - module._cffi_setup(lst, ffiplatform.VerificationError, library) + if module._cffi_setup(lst, ffiplatform.VerificationError, library): + import warnings + warnings.warn("reimporting %r might overwrite older definitions" + % (self.verifier.get_module_name())) # # finally, call the loaded_cpy_xxx() functions. This will perform # the final adjustments, like copying the Python->C wrapper @@ -280,8 +283,8 @@ return '_cffi_from_c_pointer((char *)%s, _cffi_type(%d))' % ( var, self._gettypenum(tp)) elif isinstance(tp, model.ArrayType): - return '_cffi_from_c_deref((char *)%s, _cffi_type(%d))' % ( - var, self._gettypenum(tp)) + return '_cffi_from_c_pointer((char *)%s, _cffi_type(%d))' % ( + var, self._gettypenum(model.PointerType(tp.item))) elif isinstance(tp, model.StructType): if tp.fldnames is None: raise TypeError("'%s' is used as %s, but is opaque" % ( @@ -464,11 +467,14 @@ prnt(' static Py_ssize_t nums[] = {') prnt(' sizeof(%s),' % cname) prnt(' offsetof(struct _cffi_aligncheck, y),') - for fname, _, fbitsize in tp.enumfields(): + for fname, ftype, fbitsize in tp.enumfields(): if fbitsize >= 0: continue # xxx ignore fbitsize for now prnt(' offsetof(%s, %s),' % (cname, fname)) - prnt(' sizeof(((%s *)0)->%s),' % (cname, fname)) + if isinstance(ftype, model.ArrayType) and ftype.length is None: + prnt(' 0, /* %s */' % ftype._get_c_name()) + else: + prnt(' sizeof(((%s *)0)->%s),' % (cname, fname)) prnt(' -1') prnt(' };') prnt(' return _cffi_get_struct_layout(nums);') @@ -491,7 +497,7 @@ # function = getattr(module, layoutfuncname) layout = function() - if isinstance(tp, model.StructType) and tp.partial: + if isinstance(tp, model.StructOrUnion) and tp.partial: # use the function()'s sizes and offsets to guide the # layout of the struct totalsize = layout[0] @@ -528,9 +534,10 @@ continue # xxx ignore fbitsize for now check(layout[i], ffi.offsetof(BStruct, fname), "wrong offset for field %r" % (fname,)) - BField = ffi._get_cached_btype(ftype) - check(layout[i+1], ffi.sizeof(BField), - "wrong size for field %r" % (fname,)) + if layout[i+1] != 0: + BField = ffi._get_cached_btype(ftype) + check(layout[i+1], ffi.sizeof(BField), + "wrong size for field %r" % (fname,)) i += 2 assert i == len(layout) @@ -566,7 +573,7 @@ # constants, likely declared with '#define' def _generate_cpy_const(self, is_int, name, tp=None, category='const', - vartp=None, delayed=True): + vartp=None, delayed=True, size_too=False): prnt = self._prnt funcname = '_cffi_%s_%s' % (category, name) prnt('static int %s(PyObject *lib)' % funcname) @@ -597,6 +604,15 @@ '(unsigned long long)(%s));' % (name,)) prnt(' if (o == NULL)') prnt(' return -1;') + if size_too: + prnt(' {') + prnt(' PyObject *o1 = o;') + prnt(' o = Py_BuildValue("On", o1, (Py_ssize_t)sizeof(%s));' + % (name,)) + prnt(' Py_DECREF(o1);') + prnt(' if (o == NULL)') + prnt(' return -1;') + prnt(' }') prnt(' res = PyObject_SetAttrString(lib, "%s", o);' % name) prnt(' Py_DECREF(o);') prnt(' if (res < 0)') @@ -633,12 +649,23 @@ prnt('static int %s(PyObject *lib)' % funcname) prnt('{') for enumerator, enumvalue in zip(tp.enumerators, tp.enumvalues): - prnt(' if (%s != %d) {' % (enumerator, enumvalue)) + if enumvalue < 0: + prnt(' if ((%s) >= 0 || (long)(%s) != %dL) {' % ( + enumerator, enumerator, enumvalue)) + else: + prnt(' if ((%s) < 0 || (unsigned long)(%s) != %dUL) {' % ( + enumerator, enumerator, enumvalue)) + prnt(' char buf[64];') + prnt(' if ((%s) < 0)' % enumerator) + prnt(' snprintf(buf, 63, "%%ld", (long)(%s));' % enumerator) + prnt(' else') + prnt(' snprintf(buf, 63, "%%lu", (unsigned long)(%s));' % + enumerator) prnt(' PyErr_Format(_cffi_VerificationError,') - prnt(' "enum %s: %s has the real value %d, ' - 'not %d",') - prnt(' "%s", "%s", (int)%s, %d);' % ( - name, enumerator, enumerator, enumvalue)) + prnt(' "enum %s: %s has the real value %s, ' + 'not %s",') + prnt(' "%s", "%s", buf, "%d");' % ( + name, enumerator, enumvalue)) prnt(' return -1;') prnt(' }') prnt(' return %s;' % self._chained_list_constants[True]) @@ -677,15 +704,16 @@ def _generate_cpy_variable_collecttype(self, tp, name): if isinstance(tp, model.ArrayType): - self._do_collect_type(tp) + tp_ptr = model.PointerType(tp.item) else: tp_ptr = model.PointerType(tp) - self._do_collect_type(tp_ptr) + self._do_collect_type(tp_ptr) def _generate_cpy_variable_decl(self, tp, name): if isinstance(tp, model.ArrayType): tp_ptr = model.PointerType(tp.item) - self._generate_cpy_const(False, name, tp, vartp=tp_ptr) + self._generate_cpy_const(False, name, tp, vartp=tp_ptr, + size_too = (tp.length == '...')) else: tp_ptr = model.PointerType(tp) self._generate_cpy_const(False, name, tp_ptr, category='var') @@ -694,11 +722,29 @@ _loading_cpy_variable = _loaded_noop def _loaded_cpy_variable(self, tp, name, module, library): + value = getattr(library, name) if isinstance(tp, model.ArrayType): # int a[5] is "constant" in the - return # sense that "a=..." is forbidden + # sense that "a=..." is forbidden + if tp.length == '...': + assert isinstance(value, tuple) + (value, size) = value + BItemType = self.ffi._get_cached_btype(tp.item) + length, rest = divmod(size, self.ffi.sizeof(BItemType)) + if rest != 0: + raise ffiplatform.VerificationError( + "bad size: %r does not seem to be an array of %s" % + (name, tp.item)) + tp = tp.resolve_length(length) + # 'value' is a which we have to replace with + # a if the N is actually known + if tp.length is not None: + BArray = self.ffi._get_cached_btype(tp) + value = self.ffi.cast(BArray, value) + setattr(library, name, value) + return # remove ptr= from the library instance, and replace # it by a property on the class, which reads/writes into ptr[0]. - ptr = getattr(library, name) + ptr = value delattr(library, name) def getter(library): return ptr[0] @@ -711,12 +757,9 @@ def _generate_setup_custom(self): prnt = self._prnt - prnt('static PyObject *_cffi_setup_custom(PyObject *lib)') + prnt('static int _cffi_setup_custom(PyObject *lib)') prnt('{') - prnt(' if (%s < 0)' % self._chained_list_constants[True]) - prnt(' return NULL;') - prnt(' Py_INCREF(Py_None);') - prnt(' return Py_None;') + prnt(' return %s;' % self._chained_list_constants[True]) prnt('}') cffimod_header = r''' @@ -834,17 +877,20 @@ static void *_cffi_exports[_CFFI_NUM_EXPORTS]; static PyObject *_cffi_types, *_cffi_VerificationError; -static PyObject *_cffi_setup_custom(PyObject *lib); /* forward */ +static int _cffi_setup_custom(PyObject *lib); /* forward */ static PyObject *_cffi_setup(PyObject *self, PyObject *args) { PyObject *library; + int was_alive = (_cffi_types != NULL); if (!PyArg_ParseTuple(args, "OOO", &_cffi_types, &_cffi_VerificationError, &library)) return NULL; Py_INCREF(_cffi_types); Py_INCREF(_cffi_VerificationError); - return _cffi_setup_custom(library); + if (_cffi_setup_custom(library) < 0) + return NULL; + return PyBool_FromLong(was_alive); } static void _cffi_init(void) diff --git a/lib_pypy/cffi/vengine_gen.py b/lib_pypy/cffi/vengine_gen.py --- a/lib_pypy/cffi/vengine_gen.py +++ b/lib_pypy/cffi/vengine_gen.py @@ -20,15 +20,15 @@ # up in kwds['export_symbols']. kwds.setdefault('export_symbols', self.export_symbols) - def find_module(self, module_name, path, so_suffix): - basename = module_name + so_suffix - if path is None: - path = sys.path - for dirname in path: - filename = os.path.join(dirname, basename) - if os.path.isfile(filename): - return filename - return None + def find_module(self, module_name, path, so_suffixes): + for so_suffix in so_suffixes: + basename = module_name + so_suffix + if path is None: + path = sys.path + for dirname in path: + filename = os.path.join(dirname, basename) + if os.path.isfile(filename): + return filename def collect_types(self): pass # not needed in the generic engine @@ -173,6 +173,7 @@ newfunction = self._load_constant(False, tp, name, module) else: indirections = [] + base_tp = tp if any(isinstance(typ, model.StructOrUnion) for typ in tp.args): indirect_args = [] for i, typ in enumerate(tp.args): @@ -186,16 +187,18 @@ wrappername = '_cffi_f_%s' % name newfunction = module.load_function(BFunc, wrappername) for i, typ in indirections: - newfunction = self._make_struct_wrapper(newfunction, i, typ) + newfunction = self._make_struct_wrapper(newfunction, i, typ, + base_tp) setattr(library, name, newfunction) type(library)._cffi_dir.append(name) - def _make_struct_wrapper(self, oldfunc, i, tp): + def _make_struct_wrapper(self, oldfunc, i, tp, base_tp): backend = self.ffi._backend BType = self.ffi._get_cached_btype(tp) def newfunc(*args): args = args[:i] + (backend.newp(BType, args[i]),) + args[i+1:] return oldfunc(*args) + newfunc._cffi_base_type = base_tp return newfunc # ---------- @@ -252,11 +255,14 @@ prnt(' static ssize_t nums[] = {') prnt(' sizeof(%s),' % cname) prnt(' offsetof(struct _cffi_aligncheck, y),') - for fname, _, fbitsize in tp.enumfields(): + for fname, ftype, fbitsize in tp.enumfields(): if fbitsize >= 0: continue # xxx ignore fbitsize for now prnt(' offsetof(%s, %s),' % (cname, fname)) - prnt(' sizeof(((%s *)0)->%s),' % (cname, fname)) + if isinstance(ftype, model.ArrayType) and ftype.length is None: + prnt(' 0, /* %s */' % ftype._get_c_name()) + else: + prnt(' sizeof(((%s *)0)->%s),' % (cname, fname)) prnt(' -1') prnt(' };') prnt(' return nums[i];') @@ -270,7 +276,7 @@ return # nothing to do with opaque structs layoutfuncname = '_cffi_layout_%s_%s' % (prefix, name) # - BFunc = self.ffi.typeof("ssize_t(*)(ssize_t)") + BFunc = self.ffi._typeof_locked("ssize_t(*)(ssize_t)")[0] function = module.load_function(BFunc, layoutfuncname) layout = [] num = 0 @@ -279,7 +285,7 @@ if x < 0: break layout.append(x) num += 1 - if isinstance(tp, model.StructType) and tp.partial: + if isinstance(tp, model.StructOrUnion) and tp.partial: # use the function()'s sizes and offsets to guide the # layout of the struct totalsize = layout[0] @@ -316,9 +322,10 @@ continue # xxx ignore fbitsize for now check(layout[i], ffi.offsetof(BStruct, fname), "wrong offset for field %r" % (fname,)) - BField = ffi._get_cached_btype(ftype) - check(layout[i+1], ffi.sizeof(BField), - "wrong size for field %r" % (fname,)) + if layout[i+1] != 0: + BField = ffi._get_cached_btype(ftype) + check(layout[i+1], ffi.sizeof(BField), + "wrong size for field %r" % (fname,)) i += 2 assert i == len(layout) @@ -379,15 +386,17 @@ def _load_constant(self, is_int, tp, name, module): funcname = '_cffi_const_%s' % name if is_int: From noreply at buildbot.pypy.org Tue Nov 19 06:47:36 2013 From: noreply at buildbot.pypy.org (mattip) Date: Tue, 19 Nov 2013 06:47:36 +0100 (CET) Subject: [pypy-commit] pypy default: whoops Message-ID: <20131119054736.B07691C0225@cobra.cs.uni-duesseldorf.de> Author: Matti Picus Branch: Changeset: r68233:f265ce8a2524 Date: 2013-11-19 07:48 +0200 http://bitbucket.org/pypy/pypy/changeset/f265ce8a2524/ Log: whoops diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -10,6 +10,6 @@ .. branch: numpy-newbyteorder Clean up numpy types, add newbyteorder functionality -.. branch windows-packaging +.. branch: windows-packaging Package tk/tcl runtime with win32 From noreply at buildbot.pypy.org Tue Nov 19 09:31:44 2013 From: noreply at buildbot.pypy.org (bivab) Date: Tue, 19 Nov 2013 09:31:44 +0100 (CET) Subject: [pypy-commit] pypy armhf-singlefloat: merge default Message-ID: <20131119083144.EED111C0330@cobra.cs.uni-duesseldorf.de> Author: David Schneider Branch: armhf-singlefloat Changeset: r68234:9f4093666c5d Date: 2013-11-19 09:28 +0100 http://bitbucket.org/pypy/pypy/changeset/9f4093666c5d/ Log: merge default diff too long, truncating to 2000 out of 4681 lines diff --git a/README.rst b/README.rst --- a/README.rst +++ b/README.rst @@ -33,7 +33,7 @@ $ rpython/bin/rpython -Ojit pypy/goal/targetpypystandalone.py This ends up with ``pypy-c`` binary in the main pypy directory. We suggest -to use virtualenv with the resulting pypy-c as the interpreter, you can +to use virtualenv with the resulting pypy-c as the interpreter; you can find more details about various installation schemes here: http://doc.pypy.org/en/latest/getting-started.html#installing-pypy diff --git a/lib-python/2.7/test/test_mailbox.py b/lib-python/2.7/test/test_mailbox.py --- a/lib-python/2.7/test/test_mailbox.py +++ b/lib-python/2.7/test/test_mailbox.py @@ -38,14 +38,9 @@ def _delete_recursively(self, target): # Delete a file or delete a directory recursively if os.path.isdir(target): - for path, dirs, files in os.walk(target, topdown=False): - for name in files: - os.remove(os.path.join(path, name)) - for name in dirs: - os.rmdir(os.path.join(path, name)) - os.rmdir(target) + test_support.rmtree(target) elif os.path.exists(target): - os.remove(target) + test_support.unlink(target) class TestMailbox(TestBase): @@ -137,6 +132,7 @@ msg = self._box.get(key1) self.assertEqual(msg['from'], 'foo') self.assertEqual(msg.fp.read(), '1') + msg.fp.close() def test_getitem(self): # Retrieve message using __getitem__() @@ -169,10 +165,14 @@ # Get file representations of messages key0 = self._box.add(self._template % 0) key1 = self._box.add(_sample_message) - self.assertEqual(self._box.get_file(key0).read().replace(os.linesep, '\n'), + msg0 = self._box.get_file(key0) + self.assertEqual(msg0.read().replace(os.linesep, '\n'), self._template % 0) - self.assertEqual(self._box.get_file(key1).read().replace(os.linesep, '\n'), + msg1 = self._box.get_file(key1) + self.assertEqual(msg1.read().replace(os.linesep, '\n'), _sample_message) + msg0.close() + msg1.close() def test_get_file_can_be_closed_twice(self): # Issue 11700 @@ -407,6 +407,7 @@ self._box.add(contents[0]) self._box.add(contents[1]) self._box.add(contents[2]) + oldbox = self._box method() if should_call_close: self._box.close() @@ -415,6 +416,7 @@ self.assertEqual(len(keys), 3) for key in keys: self.assertIn(self._box.get_string(key), contents) + oldbox.close() def test_dump_message(self): # Write message representations to disk @@ -1835,6 +1837,10 @@ def setUp(self): # create a new maildir mailbox to work with: self._dir = test_support.TESTFN + if os.path.isdir(self._dir): + test_support.rmtree(self._dir) + if os.path.isfile(self._dir): + test_support.unlink(self._dir) os.mkdir(self._dir) os.mkdir(os.path.join(self._dir, "cur")) os.mkdir(os.path.join(self._dir, "tmp")) @@ -1844,10 +1850,10 @@ def tearDown(self): map(os.unlink, self._msgfiles) - os.rmdir(os.path.join(self._dir, "cur")) - os.rmdir(os.path.join(self._dir, "tmp")) - os.rmdir(os.path.join(self._dir, "new")) - os.rmdir(self._dir) + test_support.rmdir(os.path.join(self._dir, "cur")) + test_support.rmdir(os.path.join(self._dir, "tmp")) + test_support.rmdir(os.path.join(self._dir, "new")) + test_support.rmdir(self._dir) def createMessage(self, dir, mbox=False): t = int(time.time() % 1000000) @@ -1883,7 +1889,9 @@ self.createMessage("cur") self.mbox = mailbox.Maildir(test_support.TESTFN) #self.assertTrue(len(self.mbox.boxes) == 1) - self.assertIsNot(self.mbox.next(), None) + msg = self.mbox.next() + self.assertIsNot(msg, None) + msg.fp.close() self.assertIs(self.mbox.next(), None) self.assertIs(self.mbox.next(), None) @@ -1891,7 +1899,9 @@ self.createMessage("new") self.mbox = mailbox.Maildir(test_support.TESTFN) #self.assertTrue(len(self.mbox.boxes) == 1) - self.assertIsNot(self.mbox.next(), None) + msg = self.mbox.next() + self.assertIsNot(msg, None) + msg.fp.close() self.assertIs(self.mbox.next(), None) self.assertIs(self.mbox.next(), None) @@ -1900,8 +1910,12 @@ self.createMessage("new") self.mbox = mailbox.Maildir(test_support.TESTFN) #self.assertTrue(len(self.mbox.boxes) == 2) - self.assertIsNot(self.mbox.next(), None) - self.assertIsNot(self.mbox.next(), None) + msg = self.mbox.next() + self.assertIsNot(msg, None) + msg.fp.close() + msg = self.mbox.next() + self.assertIsNot(msg, None) + msg.fp.close() self.assertIs(self.mbox.next(), None) self.assertIs(self.mbox.next(), None) @@ -1910,11 +1924,13 @@ import email.parser fname = self.createMessage("cur", True) n = 0 - for msg in mailbox.PortableUnixMailbox(open(fname), + fid = open(fname) + for msg in mailbox.PortableUnixMailbox(fid, email.parser.Parser().parse): n += 1 self.assertEqual(msg["subject"], "Simple Test") self.assertEqual(len(str(msg)), len(FROM_)+len(DUMMY_MESSAGE)) + fid.close() self.assertEqual(n, 1) ## End: classes from the original module (for backward compatibility). diff --git a/lib-python/2.7/test/test_mmap.py b/lib-python/2.7/test/test_mmap.py --- a/lib-python/2.7/test/test_mmap.py +++ b/lib-python/2.7/test/test_mmap.py @@ -11,7 +11,7 @@ def setUp(self): if os.path.exists(TESTFN): - os.unlink(TESTFN) + unlink(TESTFN) def tearDown(self): try: diff --git a/lib-python/2.7/test/test_old_mailbox.py b/lib-python/2.7/test/test_old_mailbox.py --- a/lib-python/2.7/test/test_old_mailbox.py +++ b/lib-python/2.7/test/test_old_mailbox.py @@ -73,7 +73,9 @@ self.createMessage("cur") self.mbox = mailbox.Maildir(test_support.TESTFN) self.assertTrue(len(self.mbox) == 1) - self.assertTrue(self.mbox.next() is not None) + msg = self.mbox.next() + self.assertTrue(msg is not None) + msg.fp.close() self.assertTrue(self.mbox.next() is None) self.assertTrue(self.mbox.next() is None) @@ -81,7 +83,9 @@ self.createMessage("new") self.mbox = mailbox.Maildir(test_support.TESTFN) self.assertTrue(len(self.mbox) == 1) - self.assertTrue(self.mbox.next() is not None) + msg = self.mbox.next() + self.assertTrue(msg is not None) + msg.fp.close() self.assertTrue(self.mbox.next() is None) self.assertTrue(self.mbox.next() is None) @@ -90,8 +94,12 @@ self.createMessage("new") self.mbox = mailbox.Maildir(test_support.TESTFN) self.assertTrue(len(self.mbox) == 2) - self.assertTrue(self.mbox.next() is not None) - self.assertTrue(self.mbox.next() is not None) + msg = self.mbox.next() + self.assertTrue(msg is not None) + msg.fp.close() + msg = self.mbox.next() + self.assertTrue(msg is not None) + msg.fp.close() self.assertTrue(self.mbox.next() is None) self.assertTrue(self.mbox.next() is None) diff --git a/lib-python/2.7/test/test_os.py b/lib-python/2.7/test/test_os.py --- a/lib-python/2.7/test/test_os.py +++ b/lib-python/2.7/test/test_os.py @@ -75,7 +75,7 @@ self.assertFalse(os.path.exists(name), "file already exists for temporary file") # make sure we can create the file - open(name, "w") + open(name, "w").close() self.files.append(name) def test_tempnam(self): diff --git a/lib-python/2.7/test/test_support.py b/lib-python/2.7/test/test_support.py --- a/lib-python/2.7/test/test_support.py +++ b/lib-python/2.7/test/test_support.py @@ -179,15 +179,79 @@ except KeyError: pass +if sys.platform.startswith("win"): + def _waitfor(func, pathname, waitall=False): + # Peform the operation + func(pathname) + # Now setup the wait loop + if waitall: + dirname = pathname + else: + dirname, name = os.path.split(pathname) + dirname = dirname or '.' + # Check for `pathname` to be removed from the filesystem. + # The exponential backoff of the timeout amounts to a total + # of ~1 second after which the deletion is probably an error + # anyway. + # Testing on a i7 at 4.3GHz shows that usually only 1 iteration is + # required when contention occurs. + timeout = 0.001 + while timeout < 1.0: + # Note we are only testing for the existance of the file(s) in + # the contents of the directory regardless of any security or + # access rights. If we have made it this far, we have sufficient + # permissions to do that much using Python's equivalent of the + # Windows API FindFirstFile. + # Other Windows APIs can fail or give incorrect results when + # dealing with files that are pending deletion. + L = os.listdir(dirname) + if not (L if waitall else name in L): + return + # Increase the timeout and try again + time.sleep(timeout) + timeout *= 2 + warnings.warn('tests may fail, delete still pending for ' + pathname, + RuntimeWarning, stacklevel=4) + + def _unlink(filename): + _waitfor(os.unlink, filename) + + def _rmdir(dirname): + _waitfor(os.rmdir, dirname) + + def _rmtree(path): + def _rmtree_inner(path): + for name in os.listdir(path): + fullname = os.path.join(path, name) + if os.path.isdir(fullname): + _waitfor(_rmtree_inner, fullname, waitall=True) + os.rmdir(fullname) + else: + os.unlink(fullname) + _waitfor(_rmtree_inner, path, waitall=True) + _waitfor(os.rmdir, path) +else: + _unlink = os.unlink + _rmdir = os.rmdir + _rmtree = shutil.rmtree + def unlink(filename): try: - os.unlink(filename) + _unlink(filename) except OSError: pass +def rmdir(dirname): + try: + _rmdir(dirname) + except OSError as error: + # The directory need not exist. + if error.errno != errno.ENOENT: + raise + def rmtree(path): try: - shutil.rmtree(path) + _rmtree(path) except OSError, e: # Unix returns ENOENT, Windows returns ESRCH. if e.errno not in (errno.ENOENT, errno.ESRCH): diff --git a/lib-python/2.7/test/test_tarfile.py b/lib-python/2.7/test/test_tarfile.py --- a/lib-python/2.7/test/test_tarfile.py +++ b/lib-python/2.7/test/test_tarfile.py @@ -300,26 +300,21 @@ def test_extract_hardlink(self): # Test hardlink extraction (e.g. bug #857297). - tar = tarfile.open(tarname, errorlevel=1, encoding="iso8859-1") + with tarfile.open(tarname, errorlevel=1, encoding="iso8859-1") as tar: + tar.extract("ustar/regtype", TEMPDIR) + self.addCleanup(os.remove, os.path.join(TEMPDIR, "ustar/regtype")) - tar.extract("ustar/regtype", TEMPDIR) - try: tar.extract("ustar/lnktype", TEMPDIR) - except EnvironmentError, e: - if e.errno == errno.ENOENT: - self.fail("hardlink not extracted properly") + self.addCleanup(os.remove, os.path.join(TEMPDIR, "ustar/lnktype")) + with open(os.path.join(TEMPDIR, "ustar/lnktype"), "rb") as f: + data = f.read() + self.assertEqual(md5sum(data), md5_regtype) - data = open(os.path.join(TEMPDIR, "ustar/lnktype"), "rb").read() - self.assertEqual(md5sum(data), md5_regtype) - - try: tar.extract("ustar/symtype", TEMPDIR) - except EnvironmentError, e: - if e.errno == errno.ENOENT: - self.fail("symlink not extracted properly") - - data = open(os.path.join(TEMPDIR, "ustar/symtype"), "rb").read() - self.assertEqual(md5sum(data), md5_regtype) + self.addCleanup(os.remove, os.path.join(TEMPDIR, "ustar/symtype")) + with open(os.path.join(TEMPDIR, "ustar/symtype"), "rb") as f: + data = f.read() + self.assertEqual(md5sum(data), md5_regtype) def test_extractall(self): # Test if extractall() correctly restores directory permissions @@ -340,7 +335,7 @@ # constructor in case of an error. For the test we rely on # the fact that opening an empty file raises a ReadError. empty = os.path.join(TEMPDIR, "empty") - open(empty, "wb").write("") + open(empty, "wb").close() try: tar = object.__new__(tarfile.TarFile) @@ -351,7 +346,7 @@ else: self.fail("ReadError not raised") finally: - os.remove(empty) + test_support.unlink(empty) class StreamReadTest(CommonReadTest): @@ -1327,7 +1322,7 @@ def setUp(self): self.tarname = tmpname if os.path.exists(self.tarname): - os.remove(self.tarname) + test_support.unlink(self.tarname) def _add_testfile(self, fileobj=None): tar = tarfile.open(self.tarname, "a", fileobj=fileobj) diff --git a/lib-python/2.7/traceback.py b/lib-python/2.7/traceback.py --- a/lib-python/2.7/traceback.py +++ b/lib-python/2.7/traceback.py @@ -107,7 +107,7 @@ return list -def print_exception(etype, value, tb, limit=None, file=None): +def print_exception(etype, value, tb, limit=None, file=None, _encoding=None): """Print exception up to 'limit' stack trace entries from 'tb' to 'file'. This differs from print_tb() in the following ways: (1) if @@ -123,7 +123,7 @@ if tb: _print(file, 'Traceback (most recent call last):') print_tb(tb, limit, file) - lines = format_exception_only(etype, value) + lines = format_exception_only(etype, value, _encoding) for line in lines: _print(file, line, '') @@ -144,7 +144,7 @@ list = list + format_exception_only(etype, value) return list -def format_exception_only(etype, value): +def format_exception_only(etype, value, _encoding=None): """Format the exception part of a traceback. The arguments are the exception type and value such as given by @@ -170,12 +170,12 @@ if (isinstance(etype, BaseException) or isinstance(etype, types.InstanceType) or etype is None or type(etype) is str): - return [_format_final_exc_line(etype, value)] + return [_format_final_exc_line(etype, value, _encoding)] stype = etype.__name__ if not issubclass(etype, SyntaxError): - return [_format_final_exc_line(stype, value)] + return [_format_final_exc_line(stype, value, _encoding)] # It was a syntax error; show exactly where the problem was found. lines = [] @@ -196,26 +196,26 @@ lines.append(' %s^\n' % ''.join(caretspace)) value = msg - lines.append(_format_final_exc_line(stype, value)) + lines.append(_format_final_exc_line(stype, value, _encoding)) return lines -def _format_final_exc_line(etype, value): +def _format_final_exc_line(etype, value, _encoding=None): """Return a list of a single line -- normal case for format_exception_only""" - valuestr = _some_str(value) + valuestr = _some_str(value, _encoding) if value is None or not valuestr: line = "%s\n" % etype else: line = "%s: %s\n" % (etype, valuestr) return line -def _some_str(value): +def _some_str(value, _encoding=None): try: return str(value) except Exception: pass try: value = unicode(value) - return value.encode("ascii", "backslashreplace") + return value.encode(_encoding or "ascii", "backslashreplace") except Exception: pass return '' % type(value).__name__ diff --git a/lib_pypy/_sqlite3.py b/lib_pypy/_sqlite3.py --- a/lib_pypy/_sqlite3.py +++ b/lib_pypy/_sqlite3.py @@ -268,10 +268,18 @@ if _has_load_extension(): _ffi.cdef("int sqlite3_enable_load_extension(sqlite3 *db, int onoff);") -_lib = _ffi.verify(""" -#include -""", libraries=['sqlite3'] -) +if sys.platform.startswith('freebsd'): + _lib = _ffi.verify(""" + #include + """, libraries=['sqlite3'], + include_dirs=['/usr/local/include'], + library_dirs=['/usr/local/lib'] + ) +else: + _lib = _ffi.verify(""" + #include + """, libraries=['sqlite3'] + ) exported_sqlite_symbols = [ 'SQLITE_ALTER_TABLE', diff --git a/lib_pypy/_tkinter/tklib.py b/lib_pypy/_tkinter/tklib.py --- a/lib_pypy/_tkinter/tklib.py +++ b/lib_pypy/_tkinter/tklib.py @@ -112,6 +112,14 @@ incdirs = ['/usr/local/include/tcl8.5', '/usr/local/include/tk8.5', '/usr/X11R6/include'] linklibs = ['tk85', 'tcl85'] libdirs = ['/usr/local/lib', '/usr/X11R6/lib'] +elif sys.platform.startswith("freebsd"): + incdirs = ['/usr/local/include/tcl8.6', '/usr/local/include/tk8.6', '/usr/local/include/X11', '/usr/local/include'] + linklibs = ['tk86', 'tcl86'] + libdirs = ['/usr/local/lib'] +elif sys.platform == 'win32': + incdirs = [] + linklibs = ['tcl85', 'tk85'] + libdirs = [] else: incdirs=['/usr/include/tcl'] linklibs=['tcl', 'tk'] diff --git a/lib_pypy/cffi/api.py b/lib_pypy/cffi/api.py --- a/lib_pypy/cffi/api.py +++ b/lib_pypy/cffi/api.py @@ -347,6 +347,9 @@ errno = property(_get_errno, _set_errno, None, "the value of 'errno' from/to the C calls") + def getwinerror(self, code=-1): + return self._backend.getwinerror(code) + def _pointer_to(self, ctype): from . import model with self._lock: diff --git a/pypy/doc/conf.py b/pypy/doc/conf.py --- a/pypy/doc/conf.py +++ b/pypy/doc/conf.py @@ -45,9 +45,9 @@ # built documents. # # The short X.Y version. -version = '2.1' +version = '2.2' # The full version, including alpha/beta/rc tags. -release = '2.1.0' +release = '2.2.0' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/pypy/doc/how-to-release.rst b/pypy/doc/how-to-release.rst --- a/pypy/doc/how-to-release.rst +++ b/pypy/doc/how-to-release.rst @@ -48,6 +48,6 @@ * send announcements to pypy-dev, python-list, python-announce, python-dev ... -* add a tag on jitviewer that corresponds to pypy release -* add a tag on codespeed that corresponds to pypy release +* add a tag on the pypy/jitviewer repo that corresponds to pypy release +* add a tag on the codespeed web site that corresponds to pypy release diff --git a/pypy/doc/index.rst b/pypy/doc/index.rst --- a/pypy/doc/index.rst +++ b/pypy/doc/index.rst @@ -40,7 +40,7 @@ * `FAQ`_: some frequently asked questions. -* `Release 2.1.0`_: the latest official release +* `Release 2.2.0`_: the latest official release * `PyPy Blog`_: news and status info about PyPy @@ -110,7 +110,7 @@ .. _`Getting Started`: getting-started.html .. _`Papers`: extradoc.html .. _`Videos`: video-index.html -.. _`Release 2.1.0`: http://pypy.org/download.html +.. _`Release 2.2.0`: http://pypy.org/download.html .. _`speed.pypy.org`: http://speed.pypy.org .. _`RPython toolchain`: translation.html .. _`potential project ideas`: project-ideas.html diff --git a/pypy/doc/release-2.2.0.rst b/pypy/doc/release-2.2.0.rst new file mode 100644 --- /dev/null +++ b/pypy/doc/release-2.2.0.rst @@ -0,0 +1,89 @@ +======================================= +PyPy 2.2 - Incrementalism +======================================= + +We're pleased to announce PyPy 2.2, which targets version 2.7.3 of the Python +language. This release main highlight is the introduction of the incremental +garbage collector, sponsored by the `Raspberry Pi Foundation`_. + +This release also contains several bugfixes and performance improvements. + +You can download the PyPy 2.2 release here: + + http://pypy.org/download.html + +We would like to thank our donors for the continued support of the PyPy +project. We showed quite a bit of progress on all three projects (see below) +and we're slowly running out of funds. +Please consider donating more so we can finish those projects! The three +projects are: + +* Py3k (supporting Python 3.x): the release PyPy3 2.2 is imminent. + +* STM (software transactional memory): a preview will be released very soon, + as soon as we fix a few bugs + +* NumPy: the work done is included in the PyPy 2.2 release. More details below. + +.. _`Raspberry Pi Foundation`: http://www.raspberrypi.org + +What is PyPy? +============= + +PyPy is a very compliant Python interpreter, almost a drop-in replacement for +CPython 2.7. It's fast (`pypy 2.2 and cpython 2.7.2`_ performance comparison) +due to its integrated tracing JIT compiler. + +This release supports x86 machines running Linux 32/64, Mac OS X 64, Windows +32, or ARM (ARMv6 or ARMv7, with VFPv3). + +Work on the native Windows 64 is still stalling, we would welcome a volunteer +to handle that. + +.. _`pypy 2.2 and cpython 2.7.2`: http://speed.pypy.org + +Highlights +========== + +* Our Garbage Collector is now "incremental". It should avoid almost + all pauses due to a major collection taking place. Previously, it + would pause the program (rarely) to walk all live objects, which + could take arbitrarily long if your process is using a whole lot of + RAM. Now the same work is done in steps. This should make PyPy + more responsive, e.g. in games. There are still other pauses, from + the GC and the JIT, but they should be on the order of 5 + milliseconds each. + +* The JIT counters for hot code were never reset, which meant that a + process running for long enough would eventually JIT-compile more + and more rarely executed code. Not only is it useless to compile + such code, but as more compiled code means more memory used, this + gives the impression of a memory leak. This has been tentatively + fixed by decreasing the counters from time to time. + +* NumPy has been split: now PyPy only contains the core module, called + ``_numpypy``. The ``numpy`` module itself has been moved to + ``https://bitbucket.org/pypy/numpy`` and ``numpypy`` disappeared. + You need to install NumPy separately with a virtualenv: + ``pip install git+https://bitbucket.org/pypy/numpy.git``; + or directly: + ``git clone https://bitbucket.org/pypy/numpy.git``; + ``cd numpy``; ``pypy setup.py install``. + +* non-inlined calls have less overhead + +* Things that use ``sys.set_trace`` are now JITted (like coverage) + +* JSON decoding is now very fast (JSON encoding was already very fast) + +* various buffer copying methods experience speedups (like list-of-ints to + ``int[]`` buffer from cffi) + +* We finally wrote (hopefully) all the missing ``os.xxx()`` functions, + including ``os.startfile()`` on Windows and a handful of rare ones + on Posix. + +* numpy has a rudimentary C API that cooperates with ``cpyext`` + +Cheers, +Armin Rigo and Maciej Fijalkowski diff --git a/pypy/doc/sandbox.rst b/pypy/doc/sandbox.rst --- a/pypy/doc/sandbox.rst +++ b/pypy/doc/sandbox.rst @@ -4,26 +4,36 @@ Introduction ------------ -It is possible to compile a version of pypy-c that runs -fully "virtualized", i.e. where an external process controls all -input/output. Such a pypy-c is a secure sandbox: it is safe to run -any untrusted Python code with it. The Python code cannot see or -modify any local file except via interaction with the external -process. It is also impossible to do any other I/O or consume more -than some amount of RAM or CPU time or real time. This works with no -OS support at all - just ANSI C code generated in a careful way. It's -the kind of thing you could embed in a browser plug-in, for example -(it would be safe even if it wasn't run as a separate process, -actually). +PyPy offers sandboxing at a level similar to OS-level sandboxing (e.g. +SECCOMP_ on Linux), but implemented in a fully portable way. To use it, +a (regular, trusted) program launches a subprocess that is a special +sandboxed version of PyPy. This subprocess can run arbitrary untrusted +Python code, but all its input/output is serialized to a stdin/stdout +pipe instead of being directly performed. The outer process reads the +pipe and decides which commands are allowed or not (sandboxing), or even +reinterprets them differently (virtualization). A potential attacker +can have arbitrary code run in the subprocess, but cannot actually do +any input/output not controlled by the outer process. Additional +barriers are put to limit the amount of RAM and CPU time used. -For comparison, trying to plug CPython into a special virtualizing C -library is not only OS-specific, but unsafe, because one of the known -ways to segfault CPython could be used by an attacker to trick CPython -into issuing malicious system calls directly. The C code generated by +Note that this is very different from sandboxing at the Python language +level, i.e. placing restrictions on what kind of Python code the +attacker is allowed to run (why? read about pysandbox_). + +.. _SECCOMP: http://code.google.com/p/seccompsandbox/wiki/overview +.. _pysandbox: https://mail.python.org/pipermail/python-dev/2013-November/130132.html + +Another point of comparison: if we were instead to try to plug CPython +into a special virtualizing C library, we would get a result +that is not only OS-specific, but unsafe, because CPython can be +segfaulted (in many ways, all of them really, really obscure). +Given enough efforts, an attacker can turn almost any +segfault into a vulnerability. The C code generated by PyPy is not segfaultable, as long as our code generators are correct - -that's a lower number of lines of code to trust. For the paranoid, in -this case we also generate systematic run-time checks against buffer -overflows. +that's a lower number of lines of code to trust. For the paranoid, +PyPy translated with sandboxing also contains systematic run-time +checks (against buffer overflows for example) +that are normally only present in debugging versions. .. warning:: diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-2.2.rst copy from pypy/doc/whatsnew-head.rst copy to pypy/doc/whatsnew-2.2.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-2.2.rst @@ -1,5 +1,5 @@ ====================== -What's new in PyPy 2.1 +What's new in PyPy 2.2 ====================== .. this is a revision shortly after release-2.1-beta diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -1,140 +1,15 @@ -====================== -What's new in PyPy 2.1 -====================== +======================= +What's new in PyPy 2.2+ +======================= -.. this is a revision shortly after release-2.1-beta -.. startrev: 4eb52818e7c0 +.. this is a revision shortly after release-2.2.x +.. startrev: 4cd1bc8b3111 -.. branch: sanitise_bytecode_dispatch -Make PyPy's bytecode dispatcher easy to read, and less reliant on RPython -magic. There is no functional change, though the removal of dead code leads -to many fewer tests to execute. +.. branch: release-2.2.x -.. branch: fastjson -Fast json decoder written in RPython, about 3-4x faster than the pure Python -decoder which comes with the stdlib +.. branch: numpy-newbyteorder +Clean up numpy types, add newbyteorder functionality -.. branch: improve-str2charp -Improve the performance of I/O writing up to 15% by using memcpy instead of -copying char-by-char in str2charp and get_nonmovingbuffer +.. branch: windows-packaging +Package tk/tcl runtime with win32 -.. branch: flowoperators -Simplify rpython/flowspace/ code by using more metaprogramming. Create -SpaceOperator class to gather static information about flow graph operations. - -.. branch: package-tk -Adapt package.py script to compile CFFI tk extension. Add a --without-tk switch -to optionally skip it. - -.. branch: distutils-cppldflags -Copy CPython's implementation of customize_compiler, dont call split on -environment variables, honour CFLAGS, CPPFLAGS, LDSHARED and LDFLAGS on Unices. - -.. branch: precise-instantiate -When an RPython class is instantiated via an indirect call (that is, which -class is being instantiated isn't known precisely) allow the optimizer to have -more precise information about which functions can be called. Needed for Topaz. - -.. branch: ssl_moving_write_buffer - -.. branch: pythoninspect-fix -Make PyPy respect PYTHONINSPECT variable set via os.putenv in the same process -to start interactive prompt when the script execution finishes. This adds -new __pypy__.os.real_getenv call that bypasses Python cache and looksup env -in the underlying OS. Translatorshell now works on PyPy. - -.. branch: add-statvfs -Added os.statvfs and os.fstatvfs - -.. branch: statvfs_tests -Added some addition tests for statvfs. - -.. branch: ndarray-subtype -Allow subclassing ndarray, i.e. matrix - -.. branch: ndarray-sort -Implement ndarray in-place sorting (for numeric types, no non-native byte order) - -.. branch: pypy-pyarray -Implement much of numpy's c api in cpyext, allows (slow) access to ndarray -from c - -.. branch: kill-ootype - -.. branch: fast-slowpath -Added an abstraction for functions with a fast and slow path in the JIT. This -speeds up list.append() and list.pop(). - -.. branch: curses_fixes - -.. branch: foldable-getarrayitem-indexerror -Constant-fold reading out of constant tuples in PyPy. - -.. branch: mro-reorder-numpypy-str -No longer delegate numpy string_ methods to space.StringObject, in numpy -this works by kind of by accident. Support for merging the refactor-str-types -branch - -.. branch: kill-typesystem -Remove the "type system" abstraction, now that there is only ever one kind of -type system used. - -.. branch: kill-gen-store-back-in -Kills gen_store_back_in_virtualizable - should improve non-inlined calls by -a bit - -.. branch: dotviewer-linewidth -.. branch: reflex-support -.. branch: numpypy-inplace-op -.. branch: rewritten-loop-logging -.. branch: no-release-gil -.. branch: safe-win-mmap -.. branch: boolean-indexing-cleanup -.. branch: cpyext-best_base -.. branch: cpyext-int -.. branch: fileops2 - -.. branch: nobold-backtrace -Work on improving UnionError messages and stack trace displays. - -.. branch: improve-errors-again -More improvements and refactorings of error messages. - -.. branch: improve-errors-again2 -Unbreak tests in rlib. - -.. branch: less-stringly-ops -Use subclasses of SpaceOperation instead of SpaceOperator objects. -Random cleanups in flowspace. - -.. branch: file-support-in-rpython -make open() and friends rpython - -.. branch: incremental-gc -Added the new incminimark GC which performs GC in incremental steps - -.. branch: fast_cffi_list_init -fastpath for cffi.new("long[]") - -.. branch: remove-eval-frame -remove a pointless abstraction - -.. branch: jit-settrace -Allow the jit to continue running when sys.settrace() is active, necessary to -make coverage.py fast - -.. branch: remove-numpypy -Remove lib_pypy/numpypy in favor of external numpy fork - -.. branch: jit-counter -Tweak the jit counters: decay them at minor collection (actually -only every 32 minor collection is enough). Should avoid the "memory -leaks" observed in long-running processes, actually created by the -jit compiling more and more rarely executed paths. - -.. branch: fix-trace-jit -Fixed the usage of sys.settrace() with the JIT. Also made it so using -sys.settrace() doesn't cause the GIL to be released on every single iteration. - -.. branch: rordereddict -Implement OrderedDict in RPython diff --git a/pypy/doc/windows.rst b/pypy/doc/windows.rst --- a/pypy/doc/windows.rst +++ b/pypy/doc/windows.rst @@ -73,11 +73,11 @@ https://bitbucket.org/pypy/pypy/downloads/local.zip Then expand it into the base directory (base_dir) and modify your environment to reflect this:: - set PATH=\bin;%PATH% - set INCLUDE=\include;%INCLUDE% - set LIB=\lib;%LIB% + set PATH=\bin;\tcltk\bin;%PATH% + set INCLUDE=\include;\tcltk\include;%INCLUDE% + set LIB=\lib;\tcltk\lib;%LIB% -Now you should be good to go. Read on for more information. +Now you should be good to go. Read on for more information. The Boehm garbage collector ~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -109,11 +109,10 @@ The bz2 compression library ~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Download http://bzip.org/1.0.5/bzip2-1.0.5.tar.gz and extract it in -the base directory. Then compile:: - - cd bzip2-1.0.5 + svn export http://svn.python.org/projects/external/bzip2-1.0.6 + cd bzip2-1.0.6 nmake -f makefile.msc + copy bzip.dll \bzip.dll The sqlite3 database library ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -122,8 +121,6 @@ wrapper is compiled when the module is imported for the first time. The sqlite3.dll should be version 3.6.21 for CPython2.7 compatablility. - - The expat XML parser ~~~~~~~~~~~~~~~~~~~~ @@ -146,13 +143,33 @@ use the one distributed by ActiveState, or the one from cygwin. In both case the perl interpreter must be found on the PATH. -Get http://www.openssl.org/source/openssl-0.9.8k.tar.gz and extract it -in the base directory. Then compile:: - + svn export http://svn.python.org/projects/external/openssl-0.9.8y + cd openssl-0.9.8y perl Configure VC-WIN32 ms\do_ms.bat nmake -f ms\nt.mak install +TkInter module support +~~~~~~~~~~~~~~~~~~~~~~ + +Note that much of this is taken from the cpython build process. +Tkinter is imported via cffi, so the module is optional. To recreate the tcltk +directory found for the release script, create the dlls, libs, headers and +runtime by running:: + + svn export http://svn.python.org/projects/external/tcl-8.5.2.1 tcl85 + svn export http://svn.python.org/projects/external/tk-8.5.2.0 tk85 + cd tcl85\win + nmake -f makefile.vc COMPILERFLAGS=-DWINVER=0x0500 DEBUG=0 INSTALLDIR=..\..\tcltk clean all + nmake -f makefile.vc DEBUG=0 INSTALLDIR=..\..\tcltk install + cd ..\..\tk85\win + nmake -f makefile.vc COMPILERFLAGS=-DWINVER=0x0500 OPTS=noxp DEBUG=1 INSTALLDIR=..\..\tcltk TCLDIR=..\..\tcl85 clean all + nmake -f makefile.vc COMPILERFLAGS=-DWINVER=0x0500 OPTS=noxp DEBUG=1 INSTALLDIR=..\..\tcltk TCLDIR=..\..\tcl85 install + +Now you should have a tcktk\bin, tcltk\lib, and tcltk\include directory ready +for use. The release packaging script will pick up the tcltk runtime in the lib +directory and put it in the archive. + Using the mingw compiler ------------------------ diff --git a/pypy/interpreter/test/test_app_main.py b/pypy/interpreter/test/test_app_main.py --- a/pypy/interpreter/test/test_app_main.py +++ b/pypy/interpreter/test/test_app_main.py @@ -942,7 +942,8 @@ self.w_tmp_dir = self.space.wrap(tmp_dir) - foo_py = prefix.join('foo.py').write("pass") + foo_py = prefix.join('foo.py') + foo_py.write("pass") self.w_foo_py = self.space.wrap(str(foo_py)) def test_setup_bootstrap_path(self): diff --git a/pypy/module/_cffi_backend/__init__.py b/pypy/module/_cffi_backend/__init__.py --- a/pypy/module/_cffi_backend/__init__.py +++ b/pypy/module/_cffi_backend/__init__.py @@ -1,3 +1,4 @@ +import sys from pypy.interpreter.mixedmodule import MixedModule from rpython.rlib import rdynload @@ -43,6 +44,8 @@ 'FFI_DEFAULT_ABI': 'ctypefunc._get_abi(space, "FFI_DEFAULT_ABI")', 'FFI_CDECL': 'ctypefunc._get_abi(space,"FFI_DEFAULT_ABI")',#win32 name } + if sys.platform == 'win32': + interpleveldefs['getwinerror'] = 'cerrno.getwinerror' for _name in ["RTLD_LAZY", "RTLD_NOW", "RTLD_GLOBAL", "RTLD_LOCAL", "RTLD_NODELETE", "RTLD_NOLOAD", "RTLD_DEEPBIND"]: diff --git a/pypy/module/_cffi_backend/cerrno.py b/pypy/module/_cffi_backend/cerrno.py --- a/pypy/module/_cffi_backend/cerrno.py +++ b/pypy/module/_cffi_backend/cerrno.py @@ -39,3 +39,14 @@ def set_errno(space, errno): ec = get_errno_container(space) ec._cffi_saved_errno = errno + +# ____________________________________________________________ + + at unwrap_spec(code=int) +def getwinerror(space, code=-1): + from rpython.rlib.rwin32 import FormatError + if code == -1: + ec = get_errno_container(space) + code = ec._cffi_saved_LastError + message = FormatError(code) + return space.newtuple([space.wrap(code), space.wrap(message)]) diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py --- a/pypy/module/_cffi_backend/test/_backend_test_c.py +++ b/pypy/module/_cffi_backend/test/_backend_test_c.py @@ -1086,7 +1086,9 @@ assert strlenaddr == cast(BVoidP, strlen) def test_read_variable(): - if sys.platform == 'win32' or sys.platform == 'darwin': + ## FIXME: this test assumes glibc specific behavior, it's not compliant with C standard + ## https://bugs.pypy.org/issue1643 + if sys.platform == 'win32' or sys.platform == 'darwin' or sys.platform.startswith('freebsd'): py.test.skip("untested") BVoidP = new_pointer_type(new_void_type()) ll = find_and_load_library('c') @@ -1094,7 +1096,9 @@ assert stderr == cast(BVoidP, _testfunc(8)) def test_read_variable_as_unknown_length_array(): - if sys.platform == 'win32' or sys.platform == 'darwin': + ## FIXME: this test assumes glibc specific behavior, it's not compliant with C standard + ## https://bugs.pypy.org/issue1643 + if sys.platform == 'win32' or sys.platform == 'darwin' or sys.platform.startswith('freebsd'): py.test.skip("untested") BCharP = new_pointer_type(new_primitive_type("char")) BArray = new_array_type(BCharP, None) @@ -1104,7 +1108,9 @@ # ^^ and not 'char[]', which is basically not allowed and would crash def test_write_variable(): - if sys.platform == 'win32' or sys.platform == 'darwin': + ## FIXME: this test assumes glibc specific behavior, it's not compliant with C standard + ## https://bugs.pypy.org/issue1643 + if sys.platform == 'win32' or sys.platform == 'darwin' or sys.platform.startswith('freebsd'): py.test.skip("untested") BVoidP = new_pointer_type(new_void_type()) ll = find_and_load_library('c') @@ -2687,6 +2693,16 @@ # res = GetLastError() assert res == 42 + # + SetLastError(2) + code, message = getwinerror() + assert code == 2 + assert message == "The system cannot find the file specified" + # + code, message = getwinerror(1155) + assert code == 1155 + assert message == ("No application is associated with the " + "specified file for this operation") def test_nonstandard_integer_types(): for typename in ['int8_t', 'uint8_t', 'int16_t', 'uint16_t', 'int32_t', diff --git a/pypy/module/_pypyjson/interp_decoder.py b/pypy/module/_pypyjson/interp_decoder.py --- a/pypy/module/_pypyjson/interp_decoder.py +++ b/pypy/module/_pypyjson/interp_decoder.py @@ -2,7 +2,7 @@ import math from rpython.rlib.rstring import StringBuilder from rpython.rlib.objectmodel import specialize -from rpython.rlib import rfloat +from rpython.rlib import rfloat, runicode from rpython.rtyper.lltypesystem import lltype, rffi from pypy.interpreter.error import OperationError, operationerrfmt from pypy.interpreter.gateway import unwrap_spec @@ -373,7 +373,7 @@ return # help the annotator to know that we'll never go beyond # this point # - uchr = unichr(val) + uchr = runicode.code_to_unichr(val) # may be a surrogate pair again utf8_ch = unicodehelper.encode_utf8(self.space, uchr) builder.append(utf8_ch) return i diff --git a/pypy/module/_pypyjson/test/test__pypyjson.py b/pypy/module/_pypyjson/test/test__pypyjson.py --- a/pypy/module/_pypyjson/test/test__pypyjson.py +++ b/pypy/module/_pypyjson/test/test__pypyjson.py @@ -1,5 +1,4 @@ # -*- encoding: utf-8 -*- -import py, sys from pypy.module._pypyjson.interp_decoder import JSONDecoder def test_skip_whitespace(): @@ -16,9 +15,6 @@ class AppTest(object): spaceconfig = {"objspace.usemodules._pypyjson": True} - def setup_class(cls): - cls.w_run_on_16bit = cls.space.wrap(sys.maxunicode == 65535) - def test_raise_on_unicode(self): import _pypyjson raises(TypeError, _pypyjson.loads, u"42") @@ -183,8 +179,6 @@ raises(ValueError, """_pypyjson.loads('["extra comma",]')""") def test_unicode_surrogate_pair(self): - if self.run_on_16bit: - skip("XXX fix me or mark definitely skipped") import _pypyjson expected = u'z\U0001d120x' res = _pypyjson.loads('"z\\ud834\\udd20x"') diff --git a/pypy/module/array/interp_array.py b/pypy/module/array/interp_array.py --- a/pypy/module/array/interp_array.py +++ b/pypy/module/array/interp_array.py @@ -10,9 +10,11 @@ from pypy.interpreter.baseobjspace import W_Root from pypy.interpreter.buffer import RWBuffer -from pypy.interpreter.error import OperationError -from pypy.interpreter.gateway import interp2app, unwrap_spec, interpindirect2app -from pypy.interpreter.typedef import GetSetProperty, make_weakref_descr, TypeDef +from pypy.interpreter.error import OperationError, operationerrfmt +from pypy.interpreter.gateway import ( + interp2app, interpindirect2app, unwrap_spec) +from pypy.interpreter.typedef import ( + GetSetProperty, TypeDef, make_weakref_descr) from pypy.module._file.interp_file import W_File from pypy.objspace.std.floatobject import W_FloatObject @@ -60,12 +62,12 @@ def descr_typecode(space, self): return space.wrap(self.typecode) -arr_eq_driver = jit.JitDriver(name='array_eq_driver', greens = ['comp_func'], reds = 'auto') +arr_eq_driver = jit.JitDriver(name='array_eq_driver', greens=['comp_func'], + reds='auto') EQ, NE, LT, LE, GT, GE = range(6) def compare_arrays(space, arr1, arr2, comp_op): - if (not isinstance(arr1, W_ArrayBase) or - not isinstance(arr2, W_ArrayBase)): + if not (isinstance(arr1, W_ArrayBase) and isinstance(arr2, W_ArrayBase)): return space.w_NotImplemented if comp_op == EQ and arr1.len != arr2.len: return space.w_False @@ -236,9 +238,12 @@ raise OperationError(self.space.w_ValueError, self.space.wrap(msg)) oldlen = self.len new = len(s) / self.itemsize + if not new: + return self.setlen(oldlen + new) cbuf = self._charbuf_start() - copy_string_to_raw(llstr(s), rffi.ptradd(cbuf, oldlen * self.itemsize), 0, len(s)) + copy_string_to_raw(llstr(s), rffi.ptradd(cbuf, oldlen * self.itemsize), + 0, len(s)) self._charbuf_stop() @unwrap_spec(w_f=W_File, n=int) @@ -268,8 +273,8 @@ def descr_tofile(self, space, w_f): """ tofile(f) - Write all items (as machine values) to the file object f. Also called as - write. + Write all items (as machine values) to the file object f. Also + called as write. """ w_s = self.descr_tostring(space) space.call_method(w_f, 'write', w_s) @@ -351,8 +356,8 @@ def descr_byteswap(self, space): """ byteswap() - Byteswap all items of the array. If the items in the array are not 1, 2, - 4, or 8 bytes in size, RuntimeError is raised. + Byteswap all items of the array. If the items in the array are + not 1, 2, 4, or 8 bytes in size, RuntimeError is raised. """ if self.itemsize not in [1, 2, 4, 8]: msg = "byteswap not supported for this array" @@ -434,7 +439,8 @@ return self.delitem(space, start, stop) def descr_delslice(self, space, w_start, w_stop): - self.descr_delitem(space, space.newslice(w_start, w_stop, space.w_None)) + self.descr_delitem(space, space.newslice(w_start, w_stop, + space.w_None)) def descr_add(self, space, w_other): raise NotImplementedError @@ -478,7 +484,7 @@ W_ArrayBase.typedef = TypeDef( 'array', __new__ = interp2app(w_array), - __module__ = 'array', + __module__ = 'array', __len__ = interp2app(W_ArrayBase.descr_len), __eq__ = interp2app(W_ArrayBase.descr_eq), @@ -534,7 +540,8 @@ class TypeCode(object): - def __init__(self, itemtype, unwrap, canoverflow=False, signed=False, method='__int__'): + def __init__(self, itemtype, unwrap, canoverflow=False, signed=False, + method='__int__'): self.itemtype = itemtype self.bytes = rffi.sizeof(itemtype) self.arraytype = lltype.Array(itemtype, hints={'nolength': True}) @@ -547,7 +554,7 @@ if self.canoverflow: assert self.bytes <= rffi.sizeof(rffi.ULONG) if self.bytes == rffi.sizeof(rffi.ULONG) and not signed and \ - self.unwrap == 'int_w': + self.unwrap == 'int_w': # Treat this type as a ULONG self.unwrap = 'bigint_w' self.canoverflow = False @@ -619,14 +626,15 @@ try: item = unwrap(w_item) except OperationError, e: - if isinstance(w_item, W_FloatObject): # Odd special case from cpython + if isinstance(w_item, W_FloatObject): + # Odd special case from cpython raise if mytype.method != '' and e.match(space, space.w_TypeError): try: item = unwrap(space.call_method(w_item, mytype.method)) except OperationError: msg = 'array item must be ' + mytype.unwrap[:-2] - raise OperationError(space.w_TypeError, space.wrap(msg)) + raise operationerrfmt(space.w_TypeError, msg) else: raise if mytype.unwrap == 'bigint_w': @@ -681,14 +689,13 @@ some = 0 self.allocated = size + some if zero: - new_buffer = lltype.malloc(mytype.arraytype, - self.allocated, flavor='raw', - add_memory_pressure=True, - zero=True) + new_buffer = lltype.malloc( + mytype.arraytype, self.allocated, flavor='raw', + add_memory_pressure=True, zero=True) else: - new_buffer = lltype.malloc(mytype.arraytype, - self.allocated, flavor='raw', - add_memory_pressure=True) + new_buffer = lltype.malloc( + mytype.arraytype, self.allocated, flavor='raw', + add_memory_pressure=True) for i in range(min(size, self.len)): new_buffer[i] = self.buffer[i] else: @@ -882,9 +889,9 @@ if i >= j: return None oldbuffer = self.buffer - self.buffer = lltype.malloc(mytype.arraytype, - max(self.len - (j - i), 0), flavor='raw', - add_memory_pressure=True) + self.buffer = lltype.malloc( + mytype.arraytype, max(self.len - (j - i), 0), flavor='raw', + add_memory_pressure=True) if i: rffi.c_memcpy( rffi.cast(rffi.VOIDP, self.buffer), diff --git a/pypy/module/array/test/test_array.py b/pypy/module/array/test/test_array.py --- a/pypy/module/array/test/test_array.py +++ b/pypy/module/array/test/test_array.py @@ -171,6 +171,9 @@ a = self.array('c') a.fromstring('Hi!') assert a[0] == 'H' and a[1] == 'i' and a[2] == '!' and len(a) == 3 + a = self.array('c') + a.fromstring('') + assert not len(a) for t in 'bBhHiIlLfd': a = self.array(t) diff --git a/pypy/module/binascii/interp_crc32.py b/pypy/module/binascii/interp_crc32.py --- a/pypy/module/binascii/interp_crc32.py +++ b/pypy/module/binascii/interp_crc32.py @@ -1,17 +1,12 @@ from pypy.interpreter.gateway import unwrap_spec +from rpython.rtyper.lltypesystem import rffi from rpython.rlib.rarithmetic import r_uint, intmask -from rpython.rtyper.lltypesystem import rffi -from rpython.rlib.rzipfile import crc_32_tab +from rpython.rlib import rzipfile @unwrap_spec(data='bufferstr', oldcrc='truncatedint_w') def crc32(space, data, oldcrc=0): "Compute the CRC-32 incrementally." - crc = r_uint(rffi.cast(rffi.UINT, ~oldcrc)) # signed => 32-bit unsigned - - # in the following loop, we have always 0 <= crc < 2**32 - for c in data: - crc = crc_32_tab[(crc & 0xff) ^ ord(c)] ^ (crc >> 8) - - crc = ~intmask(rffi.cast(rffi.INT, crc)) # unsigned => 32-bit signed - return space.wrap(crc) + crc = rzipfile.crc32(data, r_uint(oldcrc)) + crc = rffi.cast(rffi.INT, crc) # unsigned => 32-bit signed + return space.wrap(intmask(crc)) diff --git a/pypy/module/cpyext/include/patchlevel.h b/pypy/module/cpyext/include/patchlevel.h --- a/pypy/module/cpyext/include/patchlevel.h +++ b/pypy/module/cpyext/include/patchlevel.h @@ -29,7 +29,7 @@ #define PY_VERSION "2.7.3" /* PyPy version as a string */ -#define PYPY_VERSION "2.2.1-alpha0" +#define PYPY_VERSION "2.3.0-alpha0" /* Subversion Revision number of this file (not of the repository). * Empty since Mercurial migration. */ diff --git a/pypy/module/marshal/test/test_marshalimpl.py b/pypy/module/marshal/test/test_marshalimpl.py --- a/pypy/module/marshal/test/test_marshalimpl.py +++ b/pypy/module/marshal/test/test_marshalimpl.py @@ -80,7 +80,7 @@ # u = interp_marshal.StringUnmarshaller(space, space.wrap(expected)) w_long = u.load_w_obj() - assert space.eq_w(w_long, w_obj) is True + assert space.eq_w(w_long, w_obj) for sign in [1L, -1L]: for i in range(100): diff --git a/pypy/module/micronumpy/arrayimpl/concrete.py b/pypy/module/micronumpy/arrayimpl/concrete.py --- a/pypy/module/micronumpy/arrayimpl/concrete.py +++ b/pypy/module/micronumpy/arrayimpl/concrete.py @@ -36,10 +36,13 @@ return backstrides def getitem(self, index): - return self.dtype.getitem(self, index) + return self.dtype.itemtype.read(self, index, 0) + + def getitem_bool(self, index): + return self.dtype.itemtype.read_bool(self, index, 0) def setitem(self, index, value): - self.dtype.setitem(self, index, value) + self.dtype.itemtype.store(self, index, 0, value) def setslice(self, space, arr): impl = arr.implementation @@ -52,7 +55,7 @@ loop.setslice(space, shape, self, impl) def get_size(self): - return self.size // self.dtype.itemtype.get_element_size() + return self.size // self.dtype.get_size() def get_storage_size(self): return self.size @@ -77,7 +80,7 @@ return scalar.Scalar(self.dtype, self.getitem(0)) return None - def get_view(self, orig_array, dtype, new_shape): + def get_view(self, space, orig_array, dtype, new_shape): strides, backstrides = support.calc_strides(new_shape, dtype, self.order) return SliceArray(self.start, strides, backstrides, new_shape, @@ -208,7 +211,15 @@ "field named %s not found" % idx)) return RecordChunk(idx) if (space.isinstance_w(w_idx, space.w_int) or - space.isinstance_w(w_idx, space.w_slice)): + space.isinstance_w(w_idx, space.w_slice)): + return Chunks([Chunk(*space.decode_index4(w_idx, self.get_shape()[0]))]) + elif isinstance(w_idx, W_NDimArray) and \ + isinstance(w_idx.implementation, scalar.Scalar): + w_idx = w_idx.get_scalar_value().item(space) + if not space.isinstance_w(w_idx, space.w_int) and \ + not space.isinstance_w(w_idx, space.w_bool): + raise OperationError(space.w_IndexError, space.wrap( + "arrays used as indices must be of integer (or boolean) type")) return Chunks([Chunk(*space.decode_index4(w_idx, self.get_shape()[0]))]) elif space.is_w(w_idx, space.w_None): return Chunks([NewAxisChunk()]) @@ -268,7 +279,7 @@ def create_dot_iter(self, shape, skip): r = calculate_dot_strides(self.get_strides(), self.get_backstrides(), shape, skip) - return iter.MultiDimViewIterator(self, self.dtype, self.start, r[0], r[1], shape) + return iter.MultiDimViewIterator(self, self.start, r[0], r[1], shape) def swapaxes(self, space, orig_arr, axis1, axis2): shape = self.get_shape()[:] @@ -331,21 +342,24 @@ support.product(shape) > support.product(self.get_shape()): r = calculate_broadcast_strides(self.get_strides(), self.get_backstrides(), - self.get_shape(), shape, backward_broadcast) - return iter.MultiDimViewIterator(self, self.dtype, self.start, r[0], r[1], shape) - + self.get_shape(), shape, + backward_broadcast) + return iter.MultiDimViewIterator(self, self.start, + r[0], r[1], shape) if not require_index: return iter.ConcreteArrayIterator(self) - else: - if len(self.get_shape()) == 1: - return iter.OneDimViewIterator(self, self.dtype, self.start, - self.get_strides(), self.get_shape()) - else: - return iter.MultiDimViewIterator(self, self.dtype, self.start, - self.get_strides(), self.get_backstrides(), self.get_shape()) + if len(self.get_shape()) == 1: + return iter.OneDimViewIterator(self, self.start, + self.get_strides(), + self.get_shape()) + return iter.MultiDimViewIterator(self, self.start, + self.get_strides(), + self.get_backstrides(), + self.get_shape()) def fill(self, box): - self.dtype.fill(self.storage, box, 0, self.size) + self.dtype.itemtype.fill(self.storage, self.dtype.get_size(), + box, 0, self.size, 0) def set_shape(self, space, orig_array, new_shape): strides, backstrides = support.calc_strides(new_shape, self.dtype, @@ -399,7 +413,7 @@ self.storage = parent.storage self.order = parent.order self.dtype = dtype - self.size = support.product(shape) * self.dtype.itemtype.get_element_size() + self.size = support.product(shape) * self.dtype.get_size() self.start = start self.orig_arr = orig_arr @@ -416,14 +430,16 @@ self.get_backstrides(), self.get_shape(), shape, backward_broadcast) - return iter.MultiDimViewIterator(self.parent, self.dtype, - self.start, r[0], r[1], shape) + return iter.MultiDimViewIterator(self, self.start, + r[0], r[1], shape) if len(self.get_shape()) == 1: - return iter.OneDimViewIterator(self.parent, self.dtype, self.start, - self.get_strides(), self.get_shape()) - return iter.MultiDimViewIterator(self.parent, self.dtype, self.start, - self.get_strides(), - self.get_backstrides(), self.get_shape()) + return iter.OneDimViewIterator(self, self.start, + self.get_strides(), + self.get_shape()) + return iter.MultiDimViewIterator(self, self.start, + self.get_strides(), + self.get_backstrides(), + self.get_shape()) def set_shape(self, space, orig_array, new_shape): if len(self.get_shape()) < 2 or self.size == 0: diff --git a/pypy/module/micronumpy/arrayimpl/scalar.py b/pypy/module/micronumpy/arrayimpl/scalar.py --- a/pypy/module/micronumpy/arrayimpl/scalar.py +++ b/pypy/module/micronumpy/arrayimpl/scalar.py @@ -68,9 +68,15 @@ def transpose(self, _): return self - def get_view(self, orig_array, dtype, new_shape): + def get_view(self, space, orig_array, dtype, new_shape): scalar = Scalar(dtype) - scalar.value = self.value.convert_to(dtype) + if dtype.is_str_or_unicode(): + scalar.value = dtype.coerce(space, space.wrap(self.value.raw_str())) + elif dtype.is_record_type(): + raise OperationError(space.w_NotImplementedError, space.wrap( + "viewing scalar as record not implemented")) + else: + scalar.value = dtype.itemtype.runpack_str(space, self.value.raw_str()) return scalar def get_real(self, orig_array): @@ -123,20 +129,24 @@ ) def descr_getitem(self, space, _, w_idx): + if space.isinstance_w(w_idx, space.w_tuple): + if space.len_w(w_idx) == 0: + return self.get_scalar_value() raise OperationError(space.w_IndexError, - space.wrap("scalars cannot be indexed")) + space.wrap("0-d arrays can't be indexed")) def getitem_index(self, space, idx): raise OperationError(space.w_IndexError, - space.wrap("scalars cannot be indexed")) + space.wrap("0-d arrays can't be indexed")) def descr_setitem(self, space, _, w_idx, w_val): raise OperationError(space.w_IndexError, - space.wrap("scalars cannot be indexed")) + space.wrap("0-d arrays can't be indexed")) def setitem_index(self, space, idx, w_val): raise OperationError(space.w_IndexError, - space.wrap("scalars cannot be indexed")) + space.wrap("0-d arrays can't be indexed")) + def set_shape(self, space, orig_array, new_shape): if not new_shape: return self diff --git a/pypy/module/micronumpy/conversion_utils.py b/pypy/module/micronumpy/conversion_utils.py --- a/pypy/module/micronumpy/conversion_utils.py +++ b/pypy/module/micronumpy/conversion_utils.py @@ -1,6 +1,27 @@ from pypy.interpreter.error import OperationError from pypy.module.micronumpy.constants import * + +def byteorder_converter(space, new_order): + endian = new_order[0] + if endian not in (NPY_BIG, NPY_LITTLE, NPY_NATIVE, NPY_IGNORE, NPY_SWAP): + ch = endian + if ch in ('b', 'B'): + endian = NPY_BIG + elif ch in ('l', 'L'): + endian = NPY_LITTLE + elif ch in ('n', 'N'): + endian = NPY_NATIVE + elif ch in ('i', 'I'): + endian = NPY_IGNORE + elif ch in ('s', 'S'): + endian = NPY_SWAP + else: + raise OperationError(space.w_ValueError, space.wrap( + "%s is an unrecognized byteorder" % new_order)) + return endian + + def clipmode_converter(space, w_mode): if space.is_none(w_mode): return NPY_RAISE @@ -19,6 +40,7 @@ raise OperationError(space.w_TypeError, space.wrap("clipmode not understood")) + def order_converter(space, w_order, default): if space.is_none(w_order): return default @@ -40,3 +62,25 @@ else: raise OperationError(space.w_TypeError, space.wrap( "order not understood")) + + +def multi_axis_converter(space, w_axis, ndim): + if space.is_none(w_axis): + return [True] * ndim + out = [False] * ndim + if not space.isinstance_w(w_axis, space.w_tuple): + w_axis = space.newtuple([w_axis]) + for w_item in space.fixedview(w_axis): + item = space.int_w(w_item) + axis = item + if axis < 0: + axis += ndim + if axis < 0 or axis >= ndim: + raise OperationError(space.w_ValueError, space.wrap( + "'axis' entry %d is out of bounds [-%d, %d)" % + (item, ndim, ndim))) + if out[axis]: + raise OperationError(space.w_ValueError, space.wrap( + "duplicate value in 'axis'")) + out[axis] = True + return out diff --git a/pypy/module/micronumpy/interp_arrayops.py b/pypy/module/micronumpy/interp_arrayops.py --- a/pypy/module/micronumpy/interp_arrayops.py +++ b/pypy/module/micronumpy/interp_arrayops.py @@ -106,16 +106,26 @@ args_w = [convert_to_array(space, w_arg) for w_arg in args_w] dtype = args_w[0].get_dtype() shape = args_w[0].get_shape()[:] - _axis = axis + ndim = len(shape) + orig_axis = axis if axis < 0: - _axis = len(shape) + axis + axis = ndim + axis + if ndim == 1 and axis != 0: + axis = 0 + if axis < 0 or axis >= ndim: + raise operationerrfmt(space.w_IndexError, + "axis %d out of bounds [0, %d)", orig_axis, ndim) for arr in args_w[1:]: + if len(arr.get_shape()) != ndim: + raise OperationError(space.w_ValueError, space.wrap( + "all the input arrays must have same number of dimensions")) for i, axis_size in enumerate(arr.get_shape()): - if len(arr.get_shape()) != len(shape) or (i != _axis and axis_size != shape[i]): + if i == axis: + shape[i] += axis_size + elif axis_size != shape[i]: raise OperationError(space.w_ValueError, space.wrap( - "all the input arrays must have same number of dimensions")) - elif i == _axis: - shape[i] += axis_size + "all the input array dimensions except for the " + "concatenation axis must match exactly")) a_dt = arr.get_dtype() if dtype.is_record_type() and a_dt.is_record_type(): # Record types must match @@ -129,19 +139,17 @@ space.wrap("invalid type promotion")) dtype = interp_ufuncs.find_binop_result_dtype(space, dtype, arr.get_dtype()) - if _axis < 0 or len(arr.get_shape()) <= _axis: - raise operationerrfmt(space.w_IndexError, "axis %d out of bounds [0, %d)", axis, len(shape)) # concatenate does not handle ndarray subtypes, it always returns a ndarray res = W_NDimArray.from_shape(space, shape, dtype, 'C') chunks = [Chunk(0, i, 1, i) for i in shape] axis_start = 0 for arr in args_w: - if arr.get_shape()[_axis] == 0: + if arr.get_shape()[axis] == 0: continue - chunks[_axis] = Chunk(axis_start, axis_start + arr.get_shape()[_axis], 1, - arr.get_shape()[_axis]) + chunks[axis] = Chunk(axis_start, axis_start + arr.get_shape()[axis], 1, + arr.get_shape()[axis]) Chunks(chunks).apply(space, res).implementation.setslice(space, arr) - axis_start += arr.get_shape()[_axis] + axis_start += arr.get_shape()[axis] return res @unwrap_spec(repeats=int) diff --git a/pypy/module/micronumpy/interp_boxes.py b/pypy/module/micronumpy/interp_boxes.py --- a/pypy/module/micronumpy/interp_boxes.py +++ b/pypy/module/micronumpy/interp_boxes.py @@ -255,7 +255,10 @@ return convert_to_array(space, w_values) @unwrap_spec(decimals=int) - def descr_round(self, space, decimals=0): + def descr_round(self, space, decimals=0, w_out=None): + if not space.is_none(w_out): + raise OperationError(space.w_NotImplementedError, space.wrap( + "out not supported")) v = self.convert_to(self.get_dtype(space)) return self.get_dtype(space).itemtype.round(v, decimals) @@ -269,11 +272,19 @@ from pypy.module.micronumpy.interp_dtype import W_Dtype dtype = space.interp_w(W_Dtype, space.call_function(space.gettypefor(W_Dtype), w_dtype)) + if dtype.get_size() == 0: + raise OperationError(space.w_TypeError, space.wrap( + "data-type must not be 0-sized")) if dtype.get_size() != self.get_dtype(space).get_size(): raise OperationError(space.w_ValueError, space.wrap( "new type not compatible with array.")) - raise OperationError(space.w_NotImplementedError, space.wrap( - "view not implelemnted yet")) + if dtype.is_str_or_unicode(): + return dtype.coerce(space, space.wrap(self.raw_str())) + elif dtype.is_record_type(): + raise OperationError(space.w_NotImplementedError, space.wrap( + "viewing scalar as record not implemented")) + else: + return dtype.itemtype.runpack_str(space, self.raw_str()) def descr_self(self, space): return self @@ -281,6 +292,9 @@ def descr_get_dtype(self, space): return self.get_dtype(space) + def descr_get_size(self, space): + return space.wrap(1) + def descr_get_itemsize(self, space): return self.get_dtype(space).descr_get_itemsize(space) @@ -407,6 +421,9 @@ def get_dtype(self, space): return self.arr.dtype + def raw_str(self): + return self.arr.dtype.itemtype.to_str(self) + class W_VoidBox(W_FlexibleBox): def descr_getitem(self, space, w_item): if space.isinstance_w(w_item, space.w_basestring): @@ -551,6 +568,7 @@ copy = interp2app(W_GenericBox.descr_copy), dtype = GetSetProperty(W_GenericBox.descr_get_dtype), + size = GetSetProperty(W_GenericBox.descr_get_size), itemsize = GetSetProperty(W_GenericBox.descr_get_itemsize), nbytes = GetSetProperty(W_GenericBox.descr_get_itemsize), shape = GetSetProperty(W_GenericBox.descr_get_shape), diff --git a/pypy/module/micronumpy/interp_dtype.py b/pypy/module/micronumpy/interp_dtype.py --- a/pypy/module/micronumpy/interp_dtype.py +++ b/pypy/module/micronumpy/interp_dtype.py @@ -3,20 +3,22 @@ from pypy.interpreter.error import OperationError, operationerrfmt from pypy.interpreter.gateway import interp2app, unwrap_spec from pypy.interpreter.typedef import (TypeDef, GetSetProperty, - interp_attrproperty, interp_attrproperty_w) + interp_attrproperty, interp_attrproperty_w) from pypy.module.micronumpy import types, interp_boxes, base from rpython.rlib.objectmodel import specialize from rpython.rlib.rarithmetic import LONG_BIT, r_longlong, r_ulonglong from rpython.rtyper.lltypesystem import rffi from rpython.rlib import jit +from pypy.module.micronumpy.conversion_utils import byteorder_converter from pypy.module.micronumpy.constants import * def decode_w_dtype(space, w_dtype): if space.is_none(w_dtype): return None - return space.interp_w(W_Dtype, - space.call_function(space.gettypefor(W_Dtype), w_dtype)) + return space.interp_w( + W_Dtype, space.call_function(space.gettypefor(W_Dtype), w_dtype)) + @jit.unroll_safe def dtype_agreement(space, w_arr_list, shape, out=None): @@ -33,11 +35,14 @@ out = base.W_NDimArray.from_shape(space, shape, dtype) return out + class W_Dtype(W_Root): - _immutable_fields_ = ["itemtype?", "num", "kind", "name?", "char", "w_box_type", "byteorder", "float_type"] + _immutable_fields_ = ["itemtype?", "num", "kind", "name?", "char", + "w_box_type", "byteorder", "size?", "float_type", + "fields?", "fieldnames?", "shape", "subdtype", "base"] def __init__(self, itemtype, num, kind, name, char, w_box_type, byteorder=NPY_NATIVE, - alternate_constructors=[], aliases=[], float_type=None, + size=1, alternate_constructors=[], aliases=[], float_type=None, fields=None, fieldnames=None, shape=[], subdtype=None): self.itemtype = itemtype self.num = num @@ -46,6 +51,7 @@ self.char = char self.w_box_type = w_box_type self.byteorder = byteorder + self.size = size self.alternate_constructors = alternate_constructors self.aliases = aliases self.float_type = float_type @@ -77,19 +83,6 @@ def coerce(self, space, w_item): return self.itemtype.coerce(space, self, w_item) - def getitem(self, arr, i): - item = self.itemtype.read(arr, i, 0) - return item - - def getitem_bool(self, arr, i): - return self.itemtype.read_bool(arr, i, 0) - - def setitem(self, arr, i, box): - self.itemtype.store(arr, i, 0, box) - - def fill(self, storage, box, start, stop): - self.itemtype.fill(storage, self.get_size(), box, start, stop, 0) - def is_int_type(self): return (self.kind == NPY_SIGNEDLTR or self.kind == NPY_UNSIGNEDLTR or self.kind == NPY_GENBOOLLTR) @@ -101,7 +94,7 @@ return self.kind == NPY_COMPLEXLTR def is_float_type(self): - return (self.kind == NPY_FLOATINGLTR or self.float_type is not None) + return self.kind == NPY_FLOATINGLTR or self.kind == NPY_COMPLEXLTR def is_bool_type(self): return self.kind == NPY_GENBOOLLTR @@ -122,7 +115,7 @@ return self.byteorder in (NPY_NATIVE, NPY_NATBYTE) def get_size(self): - return self.itemtype.get_element_size() + return self.size * self.itemtype.get_element_size() def get_name(self): if self.char == 'S': @@ -136,7 +129,7 @@ return space.wrap("dtype('%s')" % self.get_name()) def descr_get_itemsize(self, space): - return space.wrap(self.itemtype.get_element_size()) + return space.wrap(self.get_size()) def descr_get_alignment(self, space): return space.wrap(self.itemtype.alignment) @@ -150,7 +143,7 @@ if basic == NPY_UNICODELTR: size >>= 2 endian = NPY_NATBYTE - elif size <= 1: + elif size // (self.size or 1) <= 1: endian = NPY_IGNORE else: endian = self.byteorder @@ -158,6 +151,14 @@ endian = NPY_NATBYTE return space.wrap("%s%s%s" % (endian, basic, size)) + def descr_get_descr(self, space): + if not self.is_record_type(): + return space.newlist([space.newtuple([space.wrap(""), + self.descr_get_str(space)])]) + else: + raise OperationError(space.w_NotImplementedError, space.wrap( + "descr not implemented for record types")) + def descr_get_base(self, space): return space.wrap(self.base) @@ -196,7 +197,6 @@ self.fields = None else: self.fields = {} - ofs_and_items = [] size = 0 for key in space.listview(w_fields): value = space.getitem(w_fields, key) @@ -207,11 +207,11 @@ offset = space.int_w(space.getitem(value, space.wrap(1))) self.fields[space.str_w(key)] = offset, dtype - ofs_and_items.append((offset, dtype.itemtype)) - size += dtype.itemtype.get_element_size() + size += dtype.get_size() - self.itemtype = types.RecordType(ofs_and_items, size) - self.name = "void" + str(8 * self.itemtype.get_element_size()) + self.itemtype = types.RecordType() + self.size = size + self.name = "void" + str(8 * self.get_size()) def descr_get_names(self, space): if self.fieldnames is None: @@ -232,6 +232,9 @@ raise break + def descr_get_hasobject(self, space): + return space.w_False + def descr_getitem(self, space, w_item): if self.fields is None: raise OperationError(space.w_KeyError, space.wrap( @@ -263,7 +266,7 @@ w_class = space.type(self) kind = self.kind - elemsize = self.itemtype.get_element_size() + elemsize = self.get_size() builder_args = space.newtuple([space.wrap("%s%d" % (kind, elemsize)), space.wrap(0), space.wrap(1)]) version = space.wrap(3) @@ -308,11 +311,24 @@ fields = space.getitem(w_data, space.wrap(4)) self.set_fields(space, fields) + @unwrap_spec(new_order=str) + def descr_newbyteorder(self, space, new_order=NPY_SWAP): + newendian = byteorder_converter(space, new_order) + endian = self.byteorder + if endian != NPY_IGNORE: + if newendian == NPY_SWAP: + endian = NPY_OPPBYTE if self.is_native() else NPY_NATBYTE + elif newendian != NPY_IGNORE: + endian = newendian + itemtype = self.itemtype.__class__(endian in (NPY_NATIVE, NPY_NATBYTE)) + return W_Dtype(itemtype, self.num, self.kind, self.name, self.char, + self.w_box_type, endian, size=self.size) + + def dtype_from_list(space, w_lst): lst_w = space.listview(w_lst) fields = {} offset = 0 - ofs_and_items = [] fieldnames = [] for w_elem in lst_w: size = 1 @@ -320,7 +336,7 @@ if space.len_w(w_elem) == 3: w_fldname, w_flddesc, w_shape = space.fixedview(w_elem) if not base.issequence_w(space, w_shape): - w_shape = space.newtuple([w_shape,]) + w_shape = space.newtuple([w_shape]) else: w_fldname, w_flddesc = space.fixedview(w_elem, 2) subdtype = descr__new__(space, space.gettypefor(W_Dtype), w_flddesc, w_shape=w_shape) @@ -329,27 +345,31 @@ raise OperationError(space.w_ValueError, space.wrap("two fields with the same name")) assert isinstance(subdtype, W_Dtype) fields[fldname] = (offset, subdtype) - ofs_and_items.append((offset, subdtype.itemtype)) - offset += subdtype.itemtype.get_element_size() * size + offset += subdtype.get_size() * size fieldnames.append(fldname) - itemtype = types.RecordType(ofs_and_items, offset) - return W_Dtype(itemtype, NPY_VOID, NPY_VOIDLTR, "void" + str(8 * itemtype.get_element_size()), - NPY_VOIDLTR, space.gettypefor(interp_boxes.W_VoidBox), fields=fields, - fieldnames=fieldnames) + itemtype = types.RecordType() + return W_Dtype(itemtype, NPY_VOID, NPY_VOIDLTR, + "void" + str(8 * offset * itemtype.get_element_size()), + NPY_VOIDLTR, space.gettypefor(interp_boxes.W_VoidBox), + fields=fields, fieldnames=fieldnames, size=offset) + def dtype_from_dict(space, w_dict): raise OperationError(space.w_NotImplementedError, space.wrap( "dtype from dict")) + def dtype_from_spec(space, name): raise OperationError(space.w_NotImplementedError, space.wrap( "dtype from spec")) + def descr__new__(space, w_subtype, w_dtype, w_align=None, w_copy=None, w_shape=None): # w_align and w_copy are necessary for pickling cache = get_dtype_cache(space) - if w_shape is not None and (space.isinstance_w(w_shape, space.w_int) or space.len_w(w_shape) > 0): + if w_shape is not None and (space.isinstance_w(w_shape, space.w_int) or + space.len_w(w_shape) > 0): subdtype = descr__new__(space, w_subtype, w_dtype, w_align, w_copy) assert isinstance(subdtype, W_Dtype) size = 1 @@ -360,8 +380,11 @@ dim = space.int_w(w_dim) shape.append(dim) size *= dim - return W_Dtype(types.VoidType(subdtype.itemtype.get_element_size() * size), NPY_VOID, NPY_VOIDLTR, "void" + str(8 * subdtype.itemtype.get_element_size() * size), - NPY_VOIDLTR, space.gettypefor(interp_boxes.W_VoidBox), shape=shape, subdtype=subdtype) + return W_Dtype(types.VoidType(), NPY_VOID, NPY_VOIDLTR, + "void" + str(8 * subdtype.get_size() * size), + NPY_VOIDLTR, space.gettypefor(interp_boxes.W_VoidBox), + shape=shape, subdtype=subdtype, + size=subdtype.get_size() * size) if space.is_none(w_dtype): return cache.w_float64dtype @@ -375,10 +398,10 @@ return cache.dtypes_by_name[name] except KeyError: pass - if name[0] in 'VSUc' or name[0] in '<>=' and name[1] in 'VSUc': + if name[0] in 'VSUc' or name[0] in '<>=|' and name[1] in 'VSUc': return variable_dtype(space, name) raise OperationError(space.w_TypeError, space.wrap( - "data type %s not understood" % name)) + "data type %s not understood" % name)) elif space.isinstance_w(w_dtype, space.w_list): return dtype_from_list(space, w_dtype) elif space.isinstance_w(w_dtype, space.w_tuple): @@ -413,6 +436,7 @@ __reduce__ = interp2app(W_Dtype.descr_reduce), __setstate__ = interp2app(W_Dtype.descr_setstate), + newbyteorder = interp2app(W_Dtype.descr_newbyteorder), type = interp_attrproperty_w("w_box_type", cls=W_Dtype), kind = interp_attrproperty("kind", cls=W_Dtype), @@ -430,12 +454,14 @@ isnative = GetSetProperty(W_Dtype.descr_get_isnative), fields = GetSetProperty(W_Dtype.descr_get_fields), names = GetSetProperty(W_Dtype.descr_get_names), + hasobject = GetSetProperty(W_Dtype.descr_get_hasobject), + descr = GetSetProperty(W_Dtype.descr_get_descr), ) W_Dtype.typedef.acceptable_as_base_class = False def variable_dtype(space, name): - if name[0] in '<>=': + if name[0] in '<>=|': name = name[1:] char = name[0] if len(name) == 1: @@ -450,17 +476,17 @@ size = 1 if char == NPY_STRINGLTR: - itemtype = types.StringType(size) + itemtype = types.StringType() basename = 'string' num = NPY_STRING w_box_type = space.gettypefor(interp_boxes.W_StringBox) elif char == NPY_VOIDLTR: - itemtype = types.VoidType(size) + itemtype = types.VoidType() basename = 'void' num = NPY_VOID From noreply at buildbot.pypy.org Tue Nov 19 09:31:46 2013 From: noreply at buildbot.pypy.org (bivab) Date: Tue, 19 Nov 2013 09:31:46 +0100 (CET) Subject: [pypy-commit] pypy armhf-singlefloat: document and close about to be merged branch Message-ID: <20131119083146.244311C0330@cobra.cs.uni-duesseldorf.de> Author: David Schneider Branch: armhf-singlefloat Changeset: r68235:83952b768a05 Date: 2013-11-19 09:30 +0100 http://bitbucket.org/pypy/pypy/changeset/83952b768a05/ Log: document and close about to be merged branch diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -13,3 +13,5 @@ .. branch: windows-packaging Package tk/tcl runtime with win32 +.. branch: armhf-singlefloat +JIT support for singlefloats on ARM using the hardfloat ABI From noreply at buildbot.pypy.org Tue Nov 19 09:31:47 2013 From: noreply at buildbot.pypy.org (bivab) Date: Tue, 19 Nov 2013 09:31:47 +0100 (CET) Subject: [pypy-commit] pypy default: merge armhf-singlefloat Message-ID: <20131119083147.631281C0330@cobra.cs.uni-duesseldorf.de> Author: David Schneider Branch: Changeset: r68236:cc499f0c7d91 Date: 2013-11-19 09:30 +0100 http://bitbucket.org/pypy/pypy/changeset/cc499f0c7d91/ Log: merge armhf-singlefloat diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -13,3 +13,5 @@ .. branch: windows-packaging Package tk/tcl runtime with win32 +.. branch: armhf-singlefloat +JIT support for singlefloats on ARM using the hardfloat ABI diff --git a/rpython/jit/backend/arm/callbuilder.py b/rpython/jit/backend/arm/callbuilder.py --- a/rpython/jit/backend/arm/callbuilder.py +++ b/rpython/jit/backend/arm/callbuilder.py @@ -227,20 +227,81 @@ class HardFloatCallBuilder(ARMCallbuilder): + next_arg_vfp = 0 + next_arg_svfp = 0 + + def get_next_vfp(self, tp): + assert tp in 'fS' + if self.next_arg_vfp == -1: + return None + if tp == 'S': + i = self.next_arg_svfp + next_vfp = (i >> 1) + 1 + if not (i + 1) & 1: # i is even + self.next_arg_vfp = max(self.next_arg_vfp, next_vfp) + self.next_arg_svfp = self.next_arg_vfp << 1 + else: + self.next_arg_svfp += 1 + self.next_arg_vfp = next_vfp + lst = r.svfp_argument_regs + else: # 64bit double + i = self.next_arg_vfp + self.next_arg_vfp += 1 + if self.next_arg_svfp >> 1 == i: + self.next_arg_svfp = self.next_arg_vfp << 1 + lst = r.vfp_argument_regs + try: + return lst[i] + except IndexError: + self.next_arg_vfp = self.next_arg_svfp = -1 + return None + def prepare_arguments(self): non_float_locs = [] non_float_regs = [] float_locs = [] float_regs = [] stack_args = [] + singlefloats = None arglocs = self.arglocs argtypes = self.argtypes count = 0 # stack alignment counter on_stack = 0 - for arg in arglocs: - if arg.type != FLOAT: + for i in range(len(arglocs)): + argtype = INT + if i < len(argtypes) and argtypes[i] == 'S': + argtype = argtypes[i] + arg = arglocs[i] + if arg.is_float(): + argtype = FLOAT + reg = self.get_next_vfp(argtype) + if reg: + assert len(float_regs) < len(r.vfp_argument_regs) + float_locs.append(arg) + assert reg not in float_regs + float_regs.append(reg) + else: # float argument that needs to go on the stack + if count % 2 != 0: + stack_args.append(None) + count = 0 + on_stack += 1 + stack_args.append(arg) + on_stack += 2 + elif argtype == 'S': + # Singlefloat argument + if singlefloats is None: + singlefloats = [] + tgt = self.get_next_vfp(argtype) + if tgt: + singlefloats.append((arg, tgt)) + else: # Singlefloat argument that needs to go on the stack + # treated the same as a regular core register argument + count += 1 + on_stack += 1 + stack_args.append(arg) + else: if len(non_float_regs) < len(r.argument_regs): reg = r.argument_regs[len(non_float_regs)] non_float_locs.append(arg) @@ -249,18 +310,6 @@ count += 1 on_stack += 1 stack_args.append(arg) - else: - if len(float_regs) < len(r.vfp_argument_regs): - reg = r.vfp_argument_regs[len(float_regs)] - float_locs.append(arg) - float_regs.append(reg) - else: # float argument that needs to go on the stack - if count % 2 != 0: - stack_args.append(None) - count = 0 - on_stack += 1 - stack_args.append(arg) - on_stack += 2 # align the stack if count % 2 != 0: stack_args.append(None) @@ -275,13 +324,28 @@ non_float_locs.append(self.fnloc) non_float_regs.append(r.r4) self.fnloc = r.r4 + # remap values stored in vfp registers + remap_frame_layout(self.asm, float_locs, float_regs, r.vfp_ip) + if singlefloats: + for src, dest in singlefloats: + if src.is_float(): + assert 0, 'unsupported case' + if src.is_stack(): + # use special VLDR for 32bit + self.asm.regalloc_mov(src, r.ip) + src = r.ip + if src.is_imm(): + self.mc.gen_load_int(r.ip.value, src.value) + src = r.ip + if src.is_core_reg(): + self.mc.VMOV_cs(dest.value, src.value) # remap values stored in core registers remap_frame_layout(self.asm, non_float_locs, non_float_regs, r.ip) - # remap values stored in vfp registers - remap_frame_layout(self.asm, float_locs, float_regs, r.vfp_ip) def load_result(self): resloc = self.resloc + if self.restype == 'S': + self.mc.VMOV_sc(resloc.value, r.s0.value) # ensure the result is wellformed and stored in the correct location if resloc is not None and resloc.is_core_reg(): self._ensure_result_bit_extension(resloc, diff --git a/rpython/jit/backend/arm/codebuilder.py b/rpython/jit/backend/arm/codebuilder.py --- a/rpython/jit/backend/arm/codebuilder.py +++ b/rpython/jit/backend/arm/codebuilder.py @@ -178,6 +178,30 @@ | (dm & 0xF)) self.write32(instr) + def VMOV_sc(self, dest, src): + """move a single precision vfp register[src] to a core reg[dest]""" + self._VMOV_32bit(src, dest, to_arm_register=1) + + def VMOV_cs(self, dest, src): + """move a core register[src] to a single precision vfp + register[dest]""" + self._VMOV_32bit(dest, src, to_arm_register=0) + + def _VMOV_32bit(self, float_reg, core_reg, to_arm_register, cond=cond.AL): + """This instruction transfers the contents of a single-precision VFP + register to an ARM core register, or the contents of an ARM core + register to a single-precision VFP register. + """ + instr = (cond << 28 + | 0xE << 24 + | to_arm_register << 20 + | ((float_reg >> 1) & 0xF) << 16 + | core_reg << 12 + | 0xA << 8 + | (float_reg & 0x1) << 7 + | 1 << 4) + self.write32(instr) + def VMOV_cc(self, dd, dm, cond=cond.AL): sz = 1 # for 64-bit mode instr = (cond << 28 @@ -198,8 +222,16 @@ self._VCVT(target, source, cond, 0, 1) def _VCVT(self, target, source, cond, opc2, sz): - D = 0 - M = 0 + # A8.6.295 + to_integer = (opc2 >> 2) & 1 + if to_integer: + D = target & 1 + target >>= 1 + M = (source >> 4) & 1 + else: + M = source & 1 + source >>= 1 + D = (target >> 4) & 1 op = 1 instr = (cond << 28 | 0xEB8 << 16 @@ -216,8 +248,8 @@ def _VCVT_single_double(self, target, source, cond, sz): # double_to_single = (sz == '1'); - D = 0 - M = 0 + D = target & 1 if sz else (target >> 4) & 1 + M = (source >> 4) & 1 if sz else source & 1 instr = (cond << 28 | 0xEB7 << 16 | 0xAC << 4 diff --git a/rpython/jit/backend/arm/locations.py b/rpython/jit/backend/arm/locations.py --- a/rpython/jit/backend/arm/locations.py +++ b/rpython/jit/backend/arm/locations.py @@ -55,12 +55,8 @@ type = FLOAT width = 2 * WORD - def get_single_precision_regs(self): - return [VFPRegisterLocation(i) for i in - [self.value * 2, self.value * 2 + 1]] - def __repr__(self): - return 'vfp%d' % self.value + return 'vfp(d%d)' % self.value def is_core_reg(self): return False @@ -74,6 +70,14 @@ def is_float(self): return True +class SVFPRegisterLocation(VFPRegisterLocation): + """Single Precission VFP Register""" + _immutable_ = True + width = WORD + type = 'S' + + def __repr__(self): + return 'vfp(s%d)' % self.value class ImmLocation(AssemblerLocation): _immutable_ = True diff --git a/rpython/jit/backend/arm/opassembler.py b/rpython/jit/backend/arm/opassembler.py --- a/rpython/jit/backend/arm/opassembler.py +++ b/rpython/jit/backend/arm/opassembler.py @@ -1102,17 +1102,16 @@ arg, res = arglocs assert arg.is_vfp_reg() assert res.is_core_reg() - self.mc.VCVT_float_to_int(r.vfp_ip.value, arg.value) - self.mc.VMOV_rc(res.value, r.ip.value, r.vfp_ip.value) + self.mc.VCVT_float_to_int(r.svfp_ip.value, arg.value) + self.mc.VMOV_sc(res.value, r.svfp_ip.value) return fcond def emit_op_cast_int_to_float(self, op, arglocs, regalloc, fcond): arg, res = arglocs assert res.is_vfp_reg() assert arg.is_core_reg() - self.mc.MOV_ri(r.ip.value, 0) - self.mc.VMOV_cr(res.value, arg.value, r.ip.value) - self.mc.VCVT_int_to_float(res.value, res.value) + self.mc.VMOV_cs(r.svfp_ip.value, arg.value) + self.mc.VCVT_int_to_float(res.value, r.svfp_ip.value) return fcond emit_op_llong_add = gen_emit_float_op('llong_add', 'VADD_i64') @@ -1147,15 +1146,14 @@ arg, res = arglocs assert arg.is_vfp_reg() assert res.is_core_reg() - self.mc.VCVT_f64_f32(r.vfp_ip.value, arg.value) - self.mc.VMOV_rc(res.value, r.ip.value, r.vfp_ip.value) + self.mc.VCVT_f64_f32(r.svfp_ip.value, arg.value) + self.mc.VMOV_sc(res.value, r.svfp_ip.value) return fcond def emit_op_cast_singlefloat_to_float(self, op, arglocs, regalloc, fcond): arg, res = arglocs assert res.is_vfp_reg() assert arg.is_core_reg() - self.mc.MOV_ri(r.ip.value, 0) - self.mc.VMOV_cr(res.value, arg.value, r.ip.value) - self.mc.VCVT_f32_f64(res.value, res.value) + self.mc.VMOV_cs(r.svfp_ip.value, arg.value) + self.mc.VCVT_f32_f64(res.value, r.svfp_ip.value) return fcond diff --git a/rpython/jit/backend/arm/registers.py b/rpython/jit/backend/arm/registers.py --- a/rpython/jit/backend/arm/registers.py +++ b/rpython/jit/backend/arm/registers.py @@ -1,8 +1,10 @@ from rpython.jit.backend.arm.locations import VFPRegisterLocation +from rpython.jit.backend.arm.locations import SVFPRegisterLocation from rpython.jit.backend.arm.locations import RegisterLocation registers = [RegisterLocation(i) for i in range(16)] vfpregisters = [VFPRegisterLocation(i) for i in range(16)] +svfpregisters = [SVFPRegisterLocation(i) for i in range(32)] [r0, r1, r2, r3, r4, r5, r6, r7, r8, r9, r10, r11, r12, r13, r14, r15] = registers @@ -10,6 +12,10 @@ [d0, d1, d2, d3, d4, d5, d6, d7, d8, d9, d10, d11, d12, d13, d14, d15] = vfpregisters +# single precission VFP registers, 32-bit +for i in range(32): + globals()['s%d' % i] = svfpregisters[i] + # aliases for registers fp = r11 ip = r12 @@ -17,6 +23,7 @@ lr = r14 pc = r15 vfp_ip = d15 +svfp_ip = s31 all_regs = [r0, r1, r2, r3, r4, r5, r6, r7, r8, r9, r10] all_vfp_regs = vfpregisters[:-1] @@ -27,6 +34,7 @@ callee_restored_registers = callee_resp + [pc] vfp_argument_regs = caller_vfp_resp = [d0, d1, d2, d3, d4, d5, d6, d7] +svfp_argument_regs = [globals()['s%i' % i] for i in range(16)] callee_vfp_resp = [d8, d9, d10, d11, d12, d13, d14, d15] callee_saved_vfp_registers = callee_vfp_resp diff --git a/rpython/jit/backend/arm/runner.py b/rpython/jit/backend/arm/runner.py --- a/rpython/jit/backend/arm/runner.py +++ b/rpython/jit/backend/arm/runner.py @@ -22,7 +22,7 @@ supports_floats = True supports_longlong = False # XXX requires an implementation of # read_timestamp that works in user mode - supports_singlefloats = not detect_hardfloat() + supports_singlefloats = True from rpython.jit.backend.arm.arch import JITFRAME_FIXED_SIZE all_reg_indexes = range(len(all_regs)) From noreply at buildbot.pypy.org Tue Nov 19 09:56:33 2013 From: noreply at buildbot.pypy.org (oberstet) Date: Tue, 19 Nov 2013 09:56:33 +0100 (CET) Subject: [pypy-commit] pypy default: deactivate test_multiprocessing until #1644 is resolved - buildslave hangs Message-ID: <20131119085633.8E36F1C0330@cobra.cs.uni-duesseldorf.de> Author: Tobias Oberstein Branch: Changeset: r68237:4cb276ba3e4a Date: 2013-11-18 21:54 +0100 http://bitbucket.org/pypy/pypy/changeset/4cb276ba3e4a/ Log: deactivate test_multiprocessing until #1644 is resolved - buildslave hangs diff --git a/lib-python/2.7/test/test_multiprocessing.py b/lib-python/2.7/test/test_multiprocessing.py --- a/lib-python/2.7/test/test_multiprocessing.py +++ b/lib-python/2.7/test/test_multiprocessing.py @@ -1,5 +1,10 @@ #!/usr/bin/env python +## FIXME: remove when https://bugs.pypy.org/issue1644 is resolved +import sys +if sys.platform.startswith('freebsd'): + raise Exception("This test hangs on FreeBSD. Test deactivated for now until https://bugs.pypy.org/issue1644 get resolved") + # # Unit tests for the multiprocessing package # From noreply at buildbot.pypy.org Tue Nov 19 09:56:34 2013 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 19 Nov 2013 09:56:34 +0100 (CET) Subject: [pypy-commit] pypy default: Merged in oberstet/pypy (pull request #200) Message-ID: <20131119085634.CAB121C0330@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r68238:4369d6c2378e Date: 2013-11-19 09:56 +0100 http://bitbucket.org/pypy/pypy/changeset/4369d6c2378e/ Log: Merged in oberstet/pypy (pull request #200) deactivate test_multiprocessing until #1644 is resolved - buildslave hangs diff --git a/lib-python/2.7/test/test_multiprocessing.py b/lib-python/2.7/test/test_multiprocessing.py --- a/lib-python/2.7/test/test_multiprocessing.py +++ b/lib-python/2.7/test/test_multiprocessing.py @@ -1,5 +1,10 @@ #!/usr/bin/env python +## FIXME: remove when https://bugs.pypy.org/issue1644 is resolved +import sys +if sys.platform.startswith('freebsd'): + raise Exception("This test hangs on FreeBSD. Test deactivated for now until https://bugs.pypy.org/issue1644 get resolved") + # # Unit tests for the multiprocessing package # From noreply at buildbot.pypy.org Tue Nov 19 14:51:10 2013 From: noreply at buildbot.pypy.org (antocuni) Date: Tue, 19 Nov 2013 14:51:10 +0100 (CET) Subject: [pypy-commit] pypy ndarray-buffer: a branch where to support the buffer argument to ndarray() Message-ID: <20131119135110.B20421C237F@cobra.cs.uni-duesseldorf.de> Author: Antonio Cuni Branch: ndarray-buffer Changeset: r68239:c1f8cadec802 Date: 2013-11-19 11:39 +0100 http://bitbucket.org/pypy/pypy/changeset/c1f8cadec802/ Log: a branch where to support the buffer argument to ndarray() From noreply at buildbot.pypy.org Tue Nov 19 14:51:12 2013 From: noreply at buildbot.pypy.org (antocuni) Date: Tue, 19 Nov 2013 14:51:12 +0100 (CET) Subject: [pypy-commit] pypy ndarray-buffer: first passing test: only buffers which implement get_raw_address are supported of course, because the others can be movable Message-ID: <20131119135112.17E961C237F@cobra.cs.uni-duesseldorf.de> Author: Antonio Cuni Branch: ndarray-buffer Changeset: r68240:b9d813cc6c0a Date: 2013-11-19 12:21 +0100 http://bitbucket.org/pypy/pypy/changeset/b9d813cc6c0a/ Log: first passing test: only buffers which implement get_raw_address are supported of course, because the others can be movable diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py --- a/pypy/module/micronumpy/interp_numarray.py +++ b/pypy/module/micronumpy/interp_numarray.py @@ -1,3 +1,5 @@ +from rpython.rtyper.lltypesystem import rffi +from rpython.rlib.rawstorage import RAW_STORAGE_PTR from pypy.interpreter.error import operationerrfmt, OperationError from pypy.interpreter.typedef import TypeDef, GetSetProperty, make_weakref_descr from pypy.interpreter.gateway import interp2app, unwrap_spec, applevel, \ @@ -1065,13 +1067,27 @@ offset=0, w_strides=None, order='C'): from pypy.module.micronumpy.arrayimpl.concrete import ConcreteArray from pypy.module.micronumpy.support import calc_strides - if (offset != 0 or not space.is_none(w_strides) or - not space.is_none(w_buffer)): + if (offset != 0 or not space.is_none(w_strides)): raise OperationError(space.w_NotImplementedError, space.wrap("unsupported param")) + dtype = space.interp_w(interp_dtype.W_Dtype, space.call_function(space.gettypefor(interp_dtype.W_Dtype), w_dtype)) shape = _find_shape(space, w_shape, dtype) + + if not space.is_none(w_buffer): + buf = space.buffer_w(w_buffer) + try: + raw_ptr = buf.get_raw_address() + except ValueError: + raise OperationError(space.w_TypeError, space.wrap( + "Only raw buffers are supported")) + if not shape: + raise OperationError(space.w_TypeError, space.wrap( + "numpy scalars from buffers not supported yet")) + storage = rffi.cast(RAW_STORAGE_PTR, raw_ptr) + return W_NDimArray.from_shape_and_storage(space, shape, storage, dtype) + if not shape: return W_NDimArray.new_scalar(space, dtype) if space.is_w(w_subtype, space.gettypefor(W_NDimArray)): @@ -1091,8 +1107,6 @@ Create an array from an existing buffer, given its address as int. PyPy-only implementation detail. """ - from rpython.rtyper.lltypesystem import rffi - from rpython.rlib.rawstorage import RAW_STORAGE_PTR storage = rffi.cast(RAW_STORAGE_PTR, addr) dtype = space.interp_w(interp_dtype.W_Dtype, space.call_function(space.gettypefor(interp_dtype.W_Dtype), diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py --- a/pypy/module/micronumpy/test/test_numarray.py +++ b/pypy/module/micronumpy/test/test_numarray.py @@ -218,7 +218,7 @@ assert get(1, 1) == 3 class AppTestNumArray(BaseNumpyAppTest): - spaceconfig = dict(usemodules=["micronumpy", "struct", "binascii"]) + spaceconfig = dict(usemodules=["micronumpy", "struct", "binascii", "array"]) def w_CustomIndexObject(self, index): class CustomIndexObject(object): def __init__(self, index): @@ -2087,6 +2087,17 @@ a = np.ndarray([1], dtype=bool) assert a[0] == True + def test_ndarray_from_buffer(self): + import numpypy as np + import array + buf = array.array('c', ['\x00']*2*3) + a = np.ndarray((3,), buffer=buf, dtype='i2') + a[0] = ord('b') + a[1] = ord('a') + a[2] = ord('r') + assert list(buf) == ['b', '\x00', 'a', '\x00', 'r', '\x00'] + + class AppTestMultiDim(BaseNumpyAppTest): def test_init(self): import numpypy From noreply at buildbot.pypy.org Tue Nov 19 14:51:13 2013 From: noreply at buildbot.pypy.org (antocuni) Date: Tue, 19 Nov 2013 14:51:13 +0100 (CET) Subject: [pypy-commit] pypy ndarray-buffer: add a new implementation of arrays which keeps alive the original buffer Message-ID: <20131119135113.73BEC1C237F@cobra.cs.uni-duesseldorf.de> Author: Antonio Cuni Branch: ndarray-buffer Changeset: r68241:193128b62e58 Date: 2013-11-19 14:50 +0100 http://bitbucket.org/pypy/pypy/changeset/193128b62e58/ Log: add a new implementation of arrays which keeps alive the original buffer diff --git a/pypy/module/micronumpy/arrayimpl/concrete.py b/pypy/module/micronumpy/arrayimpl/concrete.py --- a/pypy/module/micronumpy/arrayimpl/concrete.py +++ b/pypy/module/micronumpy/arrayimpl/concrete.py @@ -392,7 +392,15 @@ def __del__(self): free_raw_storage(self.storage, track_allocation=False) +class ConcreteArrayWithBase(ConcreteArrayNotOwning): + def __init__(self, shape, dtype, order, strides, backstrides, storage, orig_base): + ConcreteArrayNotOwning.__init__(self, shape, dtype, order, + strides, backstrides, storage) + self.orig_base = orig_base + def base(self): + return self.orig_base + class NonWritableArray(ConcreteArray): def descr_setitem(self, space, orig_array, w_index, w_value): raise OperationError(space.w_ValueError, space.wrap( diff --git a/pypy/module/micronumpy/base.py b/pypy/module/micronumpy/base.py --- a/pypy/module/micronumpy/base.py +++ b/pypy/module/micronumpy/base.py @@ -49,11 +49,18 @@ return W_NDimArray(impl) @staticmethod - def from_shape_and_storage(space, shape, storage, dtype, order='C', owning=False, w_subtype=None): + def from_shape_and_storage(space, shape, storage, dtype, order='C', owning=False, + w_subtype=None, w_base=None): from pypy.module.micronumpy.arrayimpl import concrete assert shape strides, backstrides = calc_strides(shape, dtype, order) - if owning: + if w_base is not None: + if owning: + raise OperationError(space.w_ValueError, + space.wrap("Cannot have owning=True when specifying a buffer")) + impl = concrete.ConcreteArrayWithBase(shape, dtype, order, strides, + backstrides, storage, w_base) + elif owning: # Will free storage when GCd impl = concrete.ConcreteArray(shape, dtype, order, strides, backstrides, storage=storage) diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py --- a/pypy/module/micronumpy/interp_numarray.py +++ b/pypy/module/micronumpy/interp_numarray.py @@ -1086,7 +1086,8 @@ raise OperationError(space.w_TypeError, space.wrap( "numpy scalars from buffers not supported yet")) storage = rffi.cast(RAW_STORAGE_PTR, raw_ptr) - return W_NDimArray.from_shape_and_storage(space, shape, storage, dtype) + return W_NDimArray.from_shape_and_storage(space, shape, storage, dtype, + w_base=w_buffer) if not shape: return W_NDimArray.new_scalar(space, dtype) diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py --- a/pypy/module/micronumpy/test/test_numarray.py +++ b/pypy/module/micronumpy/test/test_numarray.py @@ -2096,7 +2096,8 @@ a[1] = ord('a') a[2] = ord('r') assert list(buf) == ['b', '\x00', 'a', '\x00', 'r', '\x00'] - + assert a.base is buf + class AppTestMultiDim(BaseNumpyAppTest): def test_init(self): From noreply at buildbot.pypy.org Tue Nov 19 15:34:30 2013 From: noreply at buildbot.pypy.org (antocuni) Date: Tue, 19 Nov 2013 15:34:30 +0100 (CET) Subject: [pypy-commit] pypy ndarray-buffer: implement the offset param Message-ID: <20131119143430.BF5191C23D2@cobra.cs.uni-duesseldorf.de> Author: Antonio Cuni Branch: ndarray-buffer Changeset: r68242:2187c37bb8b0 Date: 2013-11-19 15:17 +0100 http://bitbucket.org/pypy/pypy/changeset/2187c37bb8b0/ Log: implement the offset param diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py --- a/pypy/module/micronumpy/interp_numarray.py +++ b/pypy/module/micronumpy/interp_numarray.py @@ -1067,15 +1067,15 @@ offset=0, w_strides=None, order='C'): from pypy.module.micronumpy.arrayimpl.concrete import ConcreteArray from pypy.module.micronumpy.support import calc_strides - if (offset != 0 or not space.is_none(w_strides)): - raise OperationError(space.w_NotImplementedError, - space.wrap("unsupported param")) - dtype = space.interp_w(interp_dtype.W_Dtype, space.call_function(space.gettypefor(interp_dtype.W_Dtype), w_dtype)) shape = _find_shape(space, w_shape, dtype) if not space.is_none(w_buffer): + if (not space.is_none(w_strides)): + raise OperationError(space.w_NotImplementedError, + space.wrap("unsupported param")) + buf = space.buffer_w(w_buffer) try: raw_ptr = buf.get_raw_address() @@ -1086,6 +1086,7 @@ raise OperationError(space.w_TypeError, space.wrap( "numpy scalars from buffers not supported yet")) storage = rffi.cast(RAW_STORAGE_PTR, raw_ptr) + storage = rffi.ptradd(storage, offset) return W_NDimArray.from_shape_and_storage(space, shape, storage, dtype, w_base=w_buffer) diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py --- a/pypy/module/micronumpy/test/test_numarray.py +++ b/pypy/module/micronumpy/test/test_numarray.py @@ -2098,6 +2098,17 @@ assert list(buf) == ['b', '\x00', 'a', '\x00', 'r', '\x00'] assert a.base is buf + def test_ndarray_from_buffer_and_offset(self): + import numpypy as np + import array + buf = array.array('c', ['\x00']*7) + buf[0] = 'X' + a = np.ndarray((3,), buffer=buf, offset=1, dtype='i2') + a[0] = ord('b') + a[1] = ord('a') + a[2] = ord('r') + assert list(buf) == ['X', 'b', '\x00', 'a', '\x00', 'r', '\x00'] + class AppTestMultiDim(BaseNumpyAppTest): def test_init(self): From noreply at buildbot.pypy.org Tue Nov 19 15:34:32 2013 From: noreply at buildbot.pypy.org (antocuni) Date: Tue, 19 Nov 2013 15:34:32 +0100 (CET) Subject: [pypy-commit] pypy ndarray-buffer: check the size of the buffer Message-ID: <20131119143432.0A2B01C2435@cobra.cs.uni-duesseldorf.de> Author: Antonio Cuni Branch: ndarray-buffer Changeset: r68243:8d2b54062cfc Date: 2013-11-19 15:33 +0100 http://bitbucket.org/pypy/pypy/changeset/8d2b54062cfc/ Log: check the size of the buffer diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py --- a/pypy/module/micronumpy/interp_numarray.py +++ b/pypy/module/micronumpy/interp_numarray.py @@ -22,6 +22,7 @@ from rpython.rlib.rstring import StringBuilder from pypy.module.micronumpy.arrayimpl.base import BaseArrayImplementation from pypy.module.micronumpy.conversion_utils import order_converter, multi_axis_converter +from pypy.module.micronumpy import support from pypy.module.micronumpy.constants import * def _find_shape(space, w_size, dtype): @@ -1085,6 +1086,10 @@ if not shape: raise OperationError(space.w_TypeError, space.wrap( "numpy scalars from buffers not supported yet")) + totalsize = support.product(shape) * dtype.get_size() + if totalsize+offset > buf.getlength(): + raise OperationError(space.w_TypeError, space.wrap( + "buffer is too small for requested array")) storage = rffi.cast(RAW_STORAGE_PTR, raw_ptr) storage = rffi.ptradd(storage, offset) return W_NDimArray.from_shape_and_storage(space, shape, storage, dtype, diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py --- a/pypy/module/micronumpy/test/test_numarray.py +++ b/pypy/module/micronumpy/test/test_numarray.py @@ -2109,6 +2109,16 @@ a[2] = ord('r') assert list(buf) == ['X', 'b', '\x00', 'a', '\x00', 'r', '\x00'] + def test_ndarray_from_buffer_out_of_bounds(self): + import numpypy as np + import array + buf = array.array('c', ['\x00']*2*10) # 20 bytes + info = raises(TypeError, "np.ndarray((11,), buffer=buf, dtype='i2')") + assert str(info.value).startswith('buffer is too small') + info = raises(TypeError, "np.ndarray((5,), buffer=buf, offset=15, dtype='i2')") + assert str(info.value).startswith('buffer is too small') + + class AppTestMultiDim(BaseNumpyAppTest): def test_init(self): From noreply at buildbot.pypy.org Tue Nov 19 16:34:17 2013 From: noreply at buildbot.pypy.org (antocuni) Date: Tue, 19 Nov 2013 16:34:17 +0100 (CET) Subject: [pypy-commit] pypy ndarray-buffer: test and fix Message-ID: <20131119153417.78A7B1C2FFE@cobra.cs.uni-duesseldorf.de> Author: Antonio Cuni Branch: ndarray-buffer Changeset: r68244:8593deb29c94 Date: 2013-11-19 16:32 +0100 http://bitbucket.org/pypy/pypy/changeset/8593deb29c94/ Log: test and fix diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py --- a/pypy/module/micronumpy/interp_numarray.py +++ b/pypy/module/micronumpy/interp_numarray.py @@ -1093,6 +1093,7 @@ storage = rffi.cast(RAW_STORAGE_PTR, raw_ptr) storage = rffi.ptradd(storage, offset) return W_NDimArray.from_shape_and_storage(space, shape, storage, dtype, + w_subtype=w_subtype, w_base=w_buffer) if not shape: diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py --- a/pypy/module/micronumpy/test/test_numarray.py +++ b/pypy/module/micronumpy/test/test_numarray.py @@ -2098,6 +2098,15 @@ assert list(buf) == ['b', '\x00', 'a', '\x00', 'r', '\x00'] assert a.base is buf + def test_ndarray_subclass_from_buffer(self): + import numpypy as np + import array + buf = array.array('c', ['\x00']*2*3) + class X(np.ndarray): + pass + a = X((3,), buffer=buf, dtype='i2') + assert type(a) is X + def test_ndarray_from_buffer_and_offset(self): import numpypy as np import array From noreply at buildbot.pypy.org Tue Nov 19 17:15:55 2013 From: noreply at buildbot.pypy.org (Raemi) Date: Tue, 19 Nov 2013 17:15:55 +0100 (CET) Subject: [pypy-commit] stmgc default: document the real reason why we can't use just any h_original and Message-ID: <20131119161555.CC0E31C147C@cobra.cs.uni-duesseldorf.de> Author: Remi Meier Branch: Changeset: r549:b89b61f0df98 Date: 2013-11-19 17:15 +0100 http://bitbucket.org/pypy/stmgc/changeset/b89b61f0df98/ Log: document the real reason why we can't use just any h_original and prevent things like stub->stub->stub on public addresses. diff --git a/c4/doc-objects.txt b/c4/doc-objects.txt --- a/c4/doc-objects.txt +++ b/c4/doc-objects.txt @@ -62,6 +62,7 @@ - prebuilt object, never modified 1 - other public object, never modified GT - outdated ptr to a more recent public copy +- stolen protected, made public some PRN Public stubs (have also a ref to one thread): - from stealing ptr (maybe to priv/prot) | 2 @@ -353,3 +354,6 @@ it is a predefined HASH value for this object. This is used by `stm_hash` which otherwise returns a hashed version of the ID of the object. + +DONT ASSUME THE H_ORIGINAL TO BE INITIALIZED. IT MAY HAVE E.G. +H_REVISION SET TO AN OLD, FREED OBJECT. diff --git a/c4/et.c b/c4/et.c --- a/c4/et.c +++ b/c4/et.c @@ -183,7 +183,8 @@ if (v & 2) goto follow_stub; - /* we update P_prev->h_revision as a shortcut */ + /* we update P_prev->h_revision as a shortcut + P_prev->P->v => P_prev->v */ /* XXX check if this really gives a worse performance than only doing this write occasionally based on a counter in d */ P_prev->h_revision = v; @@ -576,6 +577,7 @@ B = stmgc_duplicate_old(P); B->h_tid |= GCFLAG_BACKUP_COPY; B->h_tid &= ~GCFLAG_HAS_ID; + if (!(P->h_original) && (P->h_tid & GCFLAG_OLD)) { /* if P is old, it must be the original if P is young, it will create a shadow original later diff --git a/c4/extra.c b/c4/extra.c --- a/c4/extra.c +++ b/c4/extra.c @@ -89,21 +89,30 @@ } assert(obj->h_tid & GCFLAG_OLD); + if (stm_is_registered(obj)) { + /* prevents stub->stub->stub->... */ + /* only increment refcount: */ + stm_register_integer_address((intptr_t)obj); + return (intptr_t)obj; + } + spinlock_acquire(d->public_descriptor->collection_lock, 'P'); /* it must have a h_original */ gcptr orig; - if (obj->h_original == 0 || obj->h_tid & GCFLAG_PREBUILT_ORIGINAL) { + if ((!obj->h_original) || (obj->h_tid & GCFLAG_PREBUILT_ORIGINAL)) { orig = obj; } else { orig = (gcptr)obj->h_original; } - if ((orig->h_tid & (GCFLAG_PUBLIC | GCFLAG_PREBUILT_ORIGINAL)) - == (GCFLAG_PUBLIC | GCFLAG_PREBUILT_ORIGINAL)) { - /* public is not enough as public stubs may get replaced - by the protected object they point to, if they are in the - same thread (I think...) */ + if ((orig->h_tid & GCFLAG_PUBLIC) + && (obj->h_tid & GCFLAG_PREBUILT_ORIGINAL)) { + /* we can't just use *any* public original because their + h_revision is not kept up-to-date during major collections. + Meaning it can point to some long gone object. + Prebuilt originals, however, always get visited in major + collections. */ result = (intptr_t)orig; } else { @@ -233,10 +242,11 @@ /* must create shadow original object XXX: or use backup, if exists */ gcptr O = (gcptr)stmgcpage_malloc(stmgc_size(p)); - memcpy(O, p, stmgc_size(p)); /* at least major collections - depend on some content of id_copy. - remove after fixing that XXX */ + memcpy(O, p, sizeof(struct stm_object_s)); + O->h_tid |= GCFLAG_OLD; + assert(O->h_original == 0); + assert(O->h_revision = -1); /* debugging */ p->h_original = (revision_t)O; p->h_tid |= GCFLAG_HAS_ID; @@ -245,6 +255,7 @@ gcptr B = (gcptr)p->h_revision; /* not stolen already: */ assert(!(B->h_tid & GCFLAG_PUBLIC)); + assert(!B->h_original); B->h_original = (revision_t)O; } diff --git a/c4/gcpage.c b/c4/gcpage.c --- a/c4/gcpage.c +++ b/c4/gcpage.c @@ -37,6 +37,13 @@ } } +static void check_consistent(gcptr obj) { + if ((obj->h_revision & 3) == 2) + assert(stm_pointer_equal(obj, (gcptr)(obj->h_revision-2))); + else if ((obj->h_revision & 1) == 0) + assert(stm_pointer_equal(obj, (gcptr)(obj->h_revision))); +} + /***** Support code *****/ @@ -217,6 +224,22 @@ /***** registering of small stubs as integer addresses *****/ +_Bool stm_is_registered(gcptr obj) +{ + wlog_t *found; + _Bool res = 0; + + stmgcpage_acquire_global_lock(); + /* find and increment refcount; or insert */ + G2L_FIND(registered_objs, obj, found, goto finish); + found->val = (gcptr)(((revision_t)found->val) + 1); + goto finish; + res = 1; + finish: + stmgcpage_release_global_lock(); + return res; +} + void stm_register_integer_address(intptr_t adr) { /* needs to be inevitable! */ wlog_t *found; @@ -253,6 +276,7 @@ /* become inevitable because we would have to re-register them on abort, but make sure only to re-register if not registered in the same aborted transaction (XXX) */ + /* (obj will not move) */ stm_become_inevitable("stm_unregister_integer_address()"); stmgcpage_acquire_global_lock(); @@ -279,6 +303,10 @@ static gcptr copy_over_original(gcptr obj, gcptr id_copy) { + /* no obj->h_revision = obj->h_original = id_copy */ + assert(!((obj->h_revision <= ((revision_t)id_copy + 2)) && + (obj->h_revision >= ((revision_t)id_copy)))); + assert(obj != id_copy); assert(id_copy == (gcptr)obj->h_original); assert(!(id_copy->h_revision & 1)); /* not head-revision itself */ @@ -540,11 +568,6 @@ == (GCFLAG_MARKED|GCFLAG_VISITED|GCFLAG_PUBLIC)); continue; } - /* else if (R->h_original == 0) { */ - /* /\* the obj is an original and will therefore survive: *\/ */ - /* gcptr V = visit_public(R, NULL); */ - /* assert(V == R); */ - /* } */ else { assert(R->h_tid & GCFLAG_SMALLSTUB); /* only case for now */ /* make sure R stays valid: */ diff --git a/c4/nursery.c b/c4/nursery.c --- a/c4/nursery.c +++ b/c4/nursery.c @@ -186,6 +186,9 @@ /* priv_from_prot's backup->h_originals already point to id_obj */ + assert(IMPLIES(obj->h_tid & GCFLAG_PRIVATE_FROM_PROTECTED, + ((gcptr)obj->h_revision)->h_original + == obj->h_original)); } else { /* make a copy of it outside */ diff --git a/c4/steal.c b/c4/steal.c --- a/c4/steal.c +++ b/c4/steal.c @@ -83,6 +83,7 @@ obj->h_original = (revision_t)stub; stub->h_original = 0; /* stub_malloc does not set to 0... */ if (obj->h_tid & GCFLAG_PRIVATE_FROM_PROTECTED) { + assert(!(((gcptr)obj->h_revision)->h_original)); ((gcptr)obj->h_revision)->h_original = (revision_t)stub; } } diff --git a/c4/stmgc.h b/c4/stmgc.h --- a/c4/stmgc.h +++ b/c4/stmgc.h @@ -41,7 +41,7 @@ called on the result (push roots!) */ intptr_t stm_allocate_public_integer_address(gcptr); void stm_unregister_integer_address(intptr_t); /* push roots too! */ - +_Bool stm_is_registered(gcptr); /* returns a never changing hash for the object */ revision_t stm_hash(gcptr); From noreply at buildbot.pypy.org Tue Nov 19 17:17:40 2013 From: noreply at buildbot.pypy.org (Raemi) Date: Tue, 19 Nov 2013 17:17:40 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c4: import stmgc Message-ID: <20131119161740.63AC01C147C@cobra.cs.uni-duesseldorf.de> Author: Remi Meier Branch: stmgc-c4 Changeset: r68245:6c38f0067571 Date: 2013-11-19 17:16 +0100 http://bitbucket.org/pypy/pypy/changeset/6c38f0067571/ Log: import stmgc diff --git a/rpython/translator/stm/src_stm/et.c b/rpython/translator/stm/src_stm/et.c --- a/rpython/translator/stm/src_stm/et.c +++ b/rpython/translator/stm/src_stm/et.c @@ -184,7 +184,8 @@ if (v & 2) goto follow_stub; - /* we update P_prev->h_revision as a shortcut */ + /* we update P_prev->h_revision as a shortcut + P_prev->P->v => P_prev->v */ /* XXX check if this really gives a worse performance than only doing this write occasionally based on a counter in d */ P_prev->h_revision = v; @@ -577,6 +578,7 @@ B = stmgc_duplicate_old(P); B->h_tid |= GCFLAG_BACKUP_COPY; B->h_tid &= ~GCFLAG_HAS_ID; + if (!(P->h_original) && (P->h_tid & GCFLAG_OLD)) { /* if P is old, it must be the original if P is young, it will create a shadow original later diff --git a/rpython/translator/stm/src_stm/extra.c b/rpython/translator/stm/src_stm/extra.c --- a/rpython/translator/stm/src_stm/extra.c +++ b/rpython/translator/stm/src_stm/extra.c @@ -90,21 +90,30 @@ } assert(obj->h_tid & GCFLAG_OLD); + if (stm_is_registered(obj)) { + /* prevents stub->stub->stub->... */ + /* only increment refcount: */ + stm_register_integer_address((intptr_t)obj); + return (intptr_t)obj; + } + spinlock_acquire(d->public_descriptor->collection_lock, 'P'); /* it must have a h_original */ gcptr orig; - if (obj->h_original == 0 || obj->h_tid & GCFLAG_PREBUILT_ORIGINAL) { + if ((!obj->h_original) || (obj->h_tid & GCFLAG_PREBUILT_ORIGINAL)) { orig = obj; } else { orig = (gcptr)obj->h_original; } - if ((orig->h_tid & (GCFLAG_PUBLIC | GCFLAG_PREBUILT_ORIGINAL)) - == (GCFLAG_PUBLIC | GCFLAG_PREBUILT_ORIGINAL)) { - /* public is not enough as public stubs may get replaced - by the protected object they point to, if they are in the - same thread (I think...) */ + if ((orig->h_tid & GCFLAG_PUBLIC) + && (obj->h_tid & GCFLAG_PREBUILT_ORIGINAL)) { + /* we can't just use *any* public original because their + h_revision is not kept up-to-date during major collections. + Meaning it can point to some long gone object. + Prebuilt originals, however, always get visited in major + collections. */ result = (intptr_t)orig; } else { @@ -234,10 +243,11 @@ /* must create shadow original object XXX: or use backup, if exists */ gcptr O = (gcptr)stmgcpage_malloc(stmgc_size(p)); - memcpy(O, p, stmgc_size(p)); /* at least major collections - depend on some content of id_copy. - remove after fixing that XXX */ + memcpy(O, p, sizeof(struct stm_object_s)); + O->h_tid |= GCFLAG_OLD; + assert(O->h_original == 0); + assert(O->h_revision = -1); /* debugging */ p->h_original = (revision_t)O; p->h_tid |= GCFLAG_HAS_ID; @@ -246,6 +256,7 @@ gcptr B = (gcptr)p->h_revision; /* not stolen already: */ assert(!(B->h_tid & GCFLAG_PUBLIC)); + assert(!B->h_original); B->h_original = (revision_t)O; } diff --git a/rpython/translator/stm/src_stm/gcpage.c b/rpython/translator/stm/src_stm/gcpage.c --- a/rpython/translator/stm/src_stm/gcpage.c +++ b/rpython/translator/stm/src_stm/gcpage.c @@ -38,6 +38,13 @@ } } +static void check_consistent(gcptr obj) { + if ((obj->h_revision & 3) == 2) + assert(stm_pointer_equal(obj, (gcptr)(obj->h_revision-2))); + else if ((obj->h_revision & 1) == 0) + assert(stm_pointer_equal(obj, (gcptr)(obj->h_revision))); +} + /***** Support code *****/ @@ -218,6 +225,22 @@ /***** registering of small stubs as integer addresses *****/ +_Bool stm_is_registered(gcptr obj) +{ + wlog_t *found; + _Bool res = 0; + + stmgcpage_acquire_global_lock(); + /* find and increment refcount; or insert */ + G2L_FIND(registered_objs, obj, found, goto finish); + found->val = (gcptr)(((revision_t)found->val) + 1); + goto finish; + res = 1; + finish: + stmgcpage_release_global_lock(); + return res; +} + void stm_register_integer_address(intptr_t adr) { /* needs to be inevitable! */ wlog_t *found; @@ -254,6 +277,7 @@ /* become inevitable because we would have to re-register them on abort, but make sure only to re-register if not registered in the same aborted transaction (XXX) */ + /* (obj will not move) */ stm_become_inevitable("stm_unregister_integer_address()"); stmgcpage_acquire_global_lock(); @@ -280,6 +304,10 @@ static gcptr copy_over_original(gcptr obj, gcptr id_copy) { + /* no obj->h_revision = obj->h_original = id_copy */ + assert(!((obj->h_revision <= ((revision_t)id_copy + 2)) && + (obj->h_revision >= ((revision_t)id_copy)))); + assert(obj != id_copy); assert(id_copy == (gcptr)obj->h_original); assert(!(id_copy->h_revision & 1)); /* not head-revision itself */ @@ -541,11 +569,6 @@ == (GCFLAG_MARKED|GCFLAG_VISITED|GCFLAG_PUBLIC)); continue; } - /* else if (R->h_original == 0) { */ - /* /\* the obj is an original and will therefore survive: *\/ */ - /* gcptr V = visit_public(R, NULL); */ - /* assert(V == R); */ - /* } */ else { assert(R->h_tid & GCFLAG_SMALLSTUB); /* only case for now */ /* make sure R stays valid: */ diff --git a/rpython/translator/stm/src_stm/nursery.c b/rpython/translator/stm/src_stm/nursery.c --- a/rpython/translator/stm/src_stm/nursery.c +++ b/rpython/translator/stm/src_stm/nursery.c @@ -187,6 +187,9 @@ /* priv_from_prot's backup->h_originals already point to id_obj */ + assert(IMPLIES(obj->h_tid & GCFLAG_PRIVATE_FROM_PROTECTED, + ((gcptr)obj->h_revision)->h_original + == obj->h_original)); } else { /* make a copy of it outside */ diff --git a/rpython/translator/stm/src_stm/revision b/rpython/translator/stm/src_stm/revision --- a/rpython/translator/stm/src_stm/revision +++ b/rpython/translator/stm/src_stm/revision @@ -1,1 +1,1 @@ -68677625f2be +b89b61f0df98 diff --git a/rpython/translator/stm/src_stm/steal.c b/rpython/translator/stm/src_stm/steal.c --- a/rpython/translator/stm/src_stm/steal.c +++ b/rpython/translator/stm/src_stm/steal.c @@ -84,6 +84,7 @@ obj->h_original = (revision_t)stub; stub->h_original = 0; /* stub_malloc does not set to 0... */ if (obj->h_tid & GCFLAG_PRIVATE_FROM_PROTECTED) { + assert(!(((gcptr)obj->h_revision)->h_original)); ((gcptr)obj->h_revision)->h_original = (revision_t)stub; } } diff --git a/rpython/translator/stm/src_stm/stmgc.h b/rpython/translator/stm/src_stm/stmgc.h --- a/rpython/translator/stm/src_stm/stmgc.h +++ b/rpython/translator/stm/src_stm/stmgc.h @@ -42,7 +42,7 @@ called on the result (push roots!) */ intptr_t stm_allocate_public_integer_address(gcptr); void stm_unregister_integer_address(intptr_t); /* push roots too! */ - +_Bool stm_is_registered(gcptr); /* returns a never changing hash for the object */ revision_t stm_hash(gcptr); From noreply at buildbot.pypy.org Tue Nov 19 19:34:55 2013 From: noreply at buildbot.pypy.org (rguillebert) Date: Tue, 19 Nov 2013 19:34:55 +0100 (CET) Subject: [pypy-commit] pypy numpypy-array_prepare_-array_wrap: Merge default Message-ID: <20131119183455.24BAF1C147C@cobra.cs.uni-duesseldorf.de> Author: Romain Guillebert Branch: numpypy-array_prepare_-array_wrap Changeset: r68246:4d92d4a9ffe3 Date: 2013-11-19 19:33 +0100 http://bitbucket.org/pypy/pypy/changeset/4d92d4a9ffe3/ Log: Merge default diff too long, truncating to 2000 out of 28531 lines diff --git a/Makefile b/Makefile --- a/Makefile +++ b/Makefile @@ -1,7 +1,38 @@ all: pypy-c +PYPY_EXECUTABLE := $(shell which pypy) +URAM := $(shell python -c "import sys; print 4.5 if sys.maxint>1<<32 else 2.5") + +ifeq ($(PYPY_EXECUTABLE),) +RUNINTERP = python +else +RUNINTERP = $(PYPY_EXECUTABLE) +endif + pypy-c: - @echo "Building PyPy with JIT, it'll take about 40 minutes and 4G of RAM" - @sleep 3 - rpython/bin/rpython -Ojit pypy/goal/targetpypystandalone.py + @echo + @echo "====================================================================" +ifeq ($(PYPY_EXECUTABLE),) + @echo "Building a regular (jitting) version of PyPy, using CPython." + @echo "This takes around 2 hours and $(URAM) GB of RAM." + @echo "Note that pre-installing a PyPy binary would reduce this time" + @echo "and produce basically the same result." +else + @echo "Building a regular (jitting) version of PyPy, using" + @echo "$(PYPY_EXECUTABLE) to run the translation itself." + @echo "This takes up to 1 hour and $(URAM) GB of RAM." +endif + @echo + @echo "For more control (e.g. to use multiple CPU cores during part of" + @echo "the process) you need to run \`\`rpython/bin/rpython'' directly." + @echo "For more information see \`\`http://pypy.org/download.html''." + @echo "====================================================================" + @echo + @sleep 5 + $(RUNINTERP) rpython/bin/rpython -Ojit pypy/goal/targetpypystandalone.py + +# Note: the -jN option, or MAKEFLAGS=-jN, are not usable. They are +# replaced with an opaque --jobserver option by the time this Makefile +# runs. We cannot get their original value either: +# http://lists.gnu.org/archive/html/help-make/2010-08/msg00106.html diff --git a/README.rst b/README.rst --- a/README.rst +++ b/README.rst @@ -1,5 +1,5 @@ ===================================== -PyPy: Python in Python Implementation +PyPy: Python in Python Implementation ===================================== Welcome to PyPy! @@ -26,12 +26,14 @@ Building ======== -build with:: +build with: - rpython/bin/rpython -Ojit pypy/goal/targetpypystandalone.py +.. code-block:: console + + $ rpython/bin/rpython -Ojit pypy/goal/targetpypystandalone.py This ends up with ``pypy-c`` binary in the main pypy directory. We suggest -to use virtualenv with the resulting pypy-c as the interpreter, you can +to use virtualenv with the resulting pypy-c as the interpreter; you can find more details about various installation schemes here: http://doc.pypy.org/en/latest/getting-started.html#installing-pypy diff --git a/lib-python/2.7/json/encoder.py b/lib-python/2.7/json/encoder.py --- a/lib-python/2.7/json/encoder.py +++ b/lib-python/2.7/json/encoder.py @@ -4,6 +4,21 @@ from __pypy__.builders import StringBuilder, UnicodeBuilder +class StringOrUnicodeBuilder(object): + def __init__(self): + self._builder = StringBuilder() + def append(self, string): + try: + self._builder.append(string) + except UnicodeEncodeError: + ub = UnicodeBuilder() + ub.append(self._builder.build()) + self._builder = ub + ub.append(string) + def build(self): + return self._builder.build() + + ESCAPE = re.compile(r'[\x00-\x1f\\"\b\f\n\r\t]') ESCAPE_ASCII = re.compile(r'([\\"]|[^\ -~])') HAS_UTF8 = re.compile(r'[\x80-\xff]') @@ -192,7 +207,7 @@ if self.ensure_ascii: builder = StringBuilder() else: - builder = UnicodeBuilder() + builder = StringOrUnicodeBuilder() self.__encode(o, markers, builder, 0) return builder.build() diff --git a/lib-python/2.7/string.py b/lib-python/2.7/string.py --- a/lib-python/2.7/string.py +++ b/lib-python/2.7/string.py @@ -66,16 +66,17 @@ must be of the same length. """ - if len(fromstr) != len(tostr): + n = len(fromstr) + if n != len(tostr): raise ValueError, "maketrans arguments must have same length" - global _idmapL - if not _idmapL: - _idmapL = list(_idmap) - L = _idmapL[:] - fromstr = map(ord, fromstr) - for i in range(len(fromstr)): - L[fromstr[i]] = tostr[i] - return ''.join(L) + # this function has been rewritten to suit PyPy better; it is + # almost 10x faster than the original. + buf = bytearray(256) + for i in range(256): + buf[i] = i + for i in range(n): + buf[ord(fromstr[i])] = tostr[i] + return str(buf) diff --git a/lib-python/2.7/test/test_mailbox.py b/lib-python/2.7/test/test_mailbox.py --- a/lib-python/2.7/test/test_mailbox.py +++ b/lib-python/2.7/test/test_mailbox.py @@ -38,14 +38,9 @@ def _delete_recursively(self, target): # Delete a file or delete a directory recursively if os.path.isdir(target): - for path, dirs, files in os.walk(target, topdown=False): - for name in files: - os.remove(os.path.join(path, name)) - for name in dirs: - os.rmdir(os.path.join(path, name)) - os.rmdir(target) + test_support.rmtree(target) elif os.path.exists(target): - os.remove(target) + test_support.unlink(target) class TestMailbox(TestBase): @@ -137,6 +132,7 @@ msg = self._box.get(key1) self.assertEqual(msg['from'], 'foo') self.assertEqual(msg.fp.read(), '1') + msg.fp.close() def test_getitem(self): # Retrieve message using __getitem__() @@ -169,10 +165,14 @@ # Get file representations of messages key0 = self._box.add(self._template % 0) key1 = self._box.add(_sample_message) - self.assertEqual(self._box.get_file(key0).read().replace(os.linesep, '\n'), + msg0 = self._box.get_file(key0) + self.assertEqual(msg0.read().replace(os.linesep, '\n'), self._template % 0) - self.assertEqual(self._box.get_file(key1).read().replace(os.linesep, '\n'), + msg1 = self._box.get_file(key1) + self.assertEqual(msg1.read().replace(os.linesep, '\n'), _sample_message) + msg0.close() + msg1.close() def test_get_file_can_be_closed_twice(self): # Issue 11700 @@ -407,6 +407,7 @@ self._box.add(contents[0]) self._box.add(contents[1]) self._box.add(contents[2]) + oldbox = self._box method() if should_call_close: self._box.close() @@ -415,6 +416,7 @@ self.assertEqual(len(keys), 3) for key in keys: self.assertIn(self._box.get_string(key), contents) + oldbox.close() def test_dump_message(self): # Write message representations to disk @@ -1835,6 +1837,10 @@ def setUp(self): # create a new maildir mailbox to work with: self._dir = test_support.TESTFN + if os.path.isdir(self._dir): + test_support.rmtree(self._dir) + if os.path.isfile(self._dir): + test_support.unlink(self._dir) os.mkdir(self._dir) os.mkdir(os.path.join(self._dir, "cur")) os.mkdir(os.path.join(self._dir, "tmp")) @@ -1844,10 +1850,10 @@ def tearDown(self): map(os.unlink, self._msgfiles) - os.rmdir(os.path.join(self._dir, "cur")) - os.rmdir(os.path.join(self._dir, "tmp")) - os.rmdir(os.path.join(self._dir, "new")) - os.rmdir(self._dir) + test_support.rmdir(os.path.join(self._dir, "cur")) + test_support.rmdir(os.path.join(self._dir, "tmp")) + test_support.rmdir(os.path.join(self._dir, "new")) + test_support.rmdir(self._dir) def createMessage(self, dir, mbox=False): t = int(time.time() % 1000000) @@ -1883,7 +1889,9 @@ self.createMessage("cur") self.mbox = mailbox.Maildir(test_support.TESTFN) #self.assertTrue(len(self.mbox.boxes) == 1) - self.assertIsNot(self.mbox.next(), None) + msg = self.mbox.next() + self.assertIsNot(msg, None) + msg.fp.close() self.assertIs(self.mbox.next(), None) self.assertIs(self.mbox.next(), None) @@ -1891,7 +1899,9 @@ self.createMessage("new") self.mbox = mailbox.Maildir(test_support.TESTFN) #self.assertTrue(len(self.mbox.boxes) == 1) - self.assertIsNot(self.mbox.next(), None) + msg = self.mbox.next() + self.assertIsNot(msg, None) + msg.fp.close() self.assertIs(self.mbox.next(), None) self.assertIs(self.mbox.next(), None) @@ -1900,8 +1910,12 @@ self.createMessage("new") self.mbox = mailbox.Maildir(test_support.TESTFN) #self.assertTrue(len(self.mbox.boxes) == 2) - self.assertIsNot(self.mbox.next(), None) - self.assertIsNot(self.mbox.next(), None) + msg = self.mbox.next() + self.assertIsNot(msg, None) + msg.fp.close() + msg = self.mbox.next() + self.assertIsNot(msg, None) + msg.fp.close() self.assertIs(self.mbox.next(), None) self.assertIs(self.mbox.next(), None) @@ -1910,11 +1924,13 @@ import email.parser fname = self.createMessage("cur", True) n = 0 - for msg in mailbox.PortableUnixMailbox(open(fname), + fid = open(fname) + for msg in mailbox.PortableUnixMailbox(fid, email.parser.Parser().parse): n += 1 self.assertEqual(msg["subject"], "Simple Test") self.assertEqual(len(str(msg)), len(FROM_)+len(DUMMY_MESSAGE)) + fid.close() self.assertEqual(n, 1) ## End: classes from the original module (for backward compatibility). diff --git a/lib-python/2.7/test/test_mmap.py b/lib-python/2.7/test/test_mmap.py --- a/lib-python/2.7/test/test_mmap.py +++ b/lib-python/2.7/test/test_mmap.py @@ -11,7 +11,7 @@ def setUp(self): if os.path.exists(TESTFN): - os.unlink(TESTFN) + unlink(TESTFN) def tearDown(self): try: diff --git a/lib-python/2.7/test/test_multiprocessing.py b/lib-python/2.7/test/test_multiprocessing.py --- a/lib-python/2.7/test/test_multiprocessing.py +++ b/lib-python/2.7/test/test_multiprocessing.py @@ -1,5 +1,10 @@ #!/usr/bin/env python +## FIXME: remove when https://bugs.pypy.org/issue1644 is resolved +import sys +if sys.platform.startswith('freebsd'): + raise Exception("This test hangs on FreeBSD. Test deactivated for now until https://bugs.pypy.org/issue1644 get resolved") + # # Unit tests for the multiprocessing package # diff --git a/lib-python/2.7/test/test_old_mailbox.py b/lib-python/2.7/test/test_old_mailbox.py --- a/lib-python/2.7/test/test_old_mailbox.py +++ b/lib-python/2.7/test/test_old_mailbox.py @@ -73,7 +73,9 @@ self.createMessage("cur") self.mbox = mailbox.Maildir(test_support.TESTFN) self.assertTrue(len(self.mbox) == 1) - self.assertTrue(self.mbox.next() is not None) + msg = self.mbox.next() + self.assertTrue(msg is not None) + msg.fp.close() self.assertTrue(self.mbox.next() is None) self.assertTrue(self.mbox.next() is None) @@ -81,7 +83,9 @@ self.createMessage("new") self.mbox = mailbox.Maildir(test_support.TESTFN) self.assertTrue(len(self.mbox) == 1) - self.assertTrue(self.mbox.next() is not None) + msg = self.mbox.next() + self.assertTrue(msg is not None) + msg.fp.close() self.assertTrue(self.mbox.next() is None) self.assertTrue(self.mbox.next() is None) @@ -90,8 +94,12 @@ self.createMessage("new") self.mbox = mailbox.Maildir(test_support.TESTFN) self.assertTrue(len(self.mbox) == 2) - self.assertTrue(self.mbox.next() is not None) - self.assertTrue(self.mbox.next() is not None) + msg = self.mbox.next() + self.assertTrue(msg is not None) + msg.fp.close() + msg = self.mbox.next() + self.assertTrue(msg is not None) + msg.fp.close() self.assertTrue(self.mbox.next() is None) self.assertTrue(self.mbox.next() is None) diff --git a/lib-python/2.7/test/test_os.py b/lib-python/2.7/test/test_os.py --- a/lib-python/2.7/test/test_os.py +++ b/lib-python/2.7/test/test_os.py @@ -75,7 +75,7 @@ self.assertFalse(os.path.exists(name), "file already exists for temporary file") # make sure we can create the file - open(name, "w") + open(name, "w").close() self.files.append(name) def test_tempnam(self): diff --git a/lib-python/2.7/test/test_support.py b/lib-python/2.7/test/test_support.py --- a/lib-python/2.7/test/test_support.py +++ b/lib-python/2.7/test/test_support.py @@ -179,15 +179,79 @@ except KeyError: pass +if sys.platform.startswith("win"): + def _waitfor(func, pathname, waitall=False): + # Peform the operation + func(pathname) + # Now setup the wait loop + if waitall: + dirname = pathname + else: + dirname, name = os.path.split(pathname) + dirname = dirname or '.' + # Check for `pathname` to be removed from the filesystem. + # The exponential backoff of the timeout amounts to a total + # of ~1 second after which the deletion is probably an error + # anyway. + # Testing on a i7 at 4.3GHz shows that usually only 1 iteration is + # required when contention occurs. + timeout = 0.001 + while timeout < 1.0: + # Note we are only testing for the existance of the file(s) in + # the contents of the directory regardless of any security or + # access rights. If we have made it this far, we have sufficient + # permissions to do that much using Python's equivalent of the + # Windows API FindFirstFile. + # Other Windows APIs can fail or give incorrect results when + # dealing with files that are pending deletion. + L = os.listdir(dirname) + if not (L if waitall else name in L): + return + # Increase the timeout and try again + time.sleep(timeout) + timeout *= 2 + warnings.warn('tests may fail, delete still pending for ' + pathname, + RuntimeWarning, stacklevel=4) + + def _unlink(filename): + _waitfor(os.unlink, filename) + + def _rmdir(dirname): + _waitfor(os.rmdir, dirname) + + def _rmtree(path): + def _rmtree_inner(path): + for name in os.listdir(path): + fullname = os.path.join(path, name) + if os.path.isdir(fullname): + _waitfor(_rmtree_inner, fullname, waitall=True) + os.rmdir(fullname) + else: + os.unlink(fullname) + _waitfor(_rmtree_inner, path, waitall=True) + _waitfor(os.rmdir, path) +else: + _unlink = os.unlink + _rmdir = os.rmdir + _rmtree = shutil.rmtree + def unlink(filename): try: - os.unlink(filename) + _unlink(filename) except OSError: pass +def rmdir(dirname): + try: + _rmdir(dirname) + except OSError as error: + # The directory need not exist. + if error.errno != errno.ENOENT: + raise + def rmtree(path): try: - shutil.rmtree(path) + _rmtree(path) except OSError, e: # Unix returns ENOENT, Windows returns ESRCH. if e.errno not in (errno.ENOENT, errno.ESRCH): diff --git a/lib-python/2.7/test/test_tarfile.py b/lib-python/2.7/test/test_tarfile.py --- a/lib-python/2.7/test/test_tarfile.py +++ b/lib-python/2.7/test/test_tarfile.py @@ -300,26 +300,21 @@ def test_extract_hardlink(self): # Test hardlink extraction (e.g. bug #857297). - tar = tarfile.open(tarname, errorlevel=1, encoding="iso8859-1") + with tarfile.open(tarname, errorlevel=1, encoding="iso8859-1") as tar: + tar.extract("ustar/regtype", TEMPDIR) + self.addCleanup(os.remove, os.path.join(TEMPDIR, "ustar/regtype")) - tar.extract("ustar/regtype", TEMPDIR) - try: tar.extract("ustar/lnktype", TEMPDIR) - except EnvironmentError, e: - if e.errno == errno.ENOENT: - self.fail("hardlink not extracted properly") + self.addCleanup(os.remove, os.path.join(TEMPDIR, "ustar/lnktype")) + with open(os.path.join(TEMPDIR, "ustar/lnktype"), "rb") as f: + data = f.read() + self.assertEqual(md5sum(data), md5_regtype) - data = open(os.path.join(TEMPDIR, "ustar/lnktype"), "rb").read() - self.assertEqual(md5sum(data), md5_regtype) - - try: tar.extract("ustar/symtype", TEMPDIR) - except EnvironmentError, e: - if e.errno == errno.ENOENT: - self.fail("symlink not extracted properly") - - data = open(os.path.join(TEMPDIR, "ustar/symtype"), "rb").read() - self.assertEqual(md5sum(data), md5_regtype) + self.addCleanup(os.remove, os.path.join(TEMPDIR, "ustar/symtype")) + with open(os.path.join(TEMPDIR, "ustar/symtype"), "rb") as f: + data = f.read() + self.assertEqual(md5sum(data), md5_regtype) def test_extractall(self): # Test if extractall() correctly restores directory permissions @@ -340,7 +335,7 @@ # constructor in case of an error. For the test we rely on # the fact that opening an empty file raises a ReadError. empty = os.path.join(TEMPDIR, "empty") - open(empty, "wb").write("") + open(empty, "wb").close() try: tar = object.__new__(tarfile.TarFile) @@ -351,7 +346,7 @@ else: self.fail("ReadError not raised") finally: - os.remove(empty) + test_support.unlink(empty) class StreamReadTest(CommonReadTest): @@ -1327,7 +1322,7 @@ def setUp(self): self.tarname = tmpname if os.path.exists(self.tarname): - os.remove(self.tarname) + test_support.unlink(self.tarname) def _add_testfile(self, fileobj=None): tar = tarfile.open(self.tarname, "a", fileobj=fileobj) diff --git a/lib-python/2.7/traceback.py b/lib-python/2.7/traceback.py --- a/lib-python/2.7/traceback.py +++ b/lib-python/2.7/traceback.py @@ -107,7 +107,7 @@ return list -def print_exception(etype, value, tb, limit=None, file=None): +def print_exception(etype, value, tb, limit=None, file=None, _encoding=None): """Print exception up to 'limit' stack trace entries from 'tb' to 'file'. This differs from print_tb() in the following ways: (1) if @@ -123,7 +123,7 @@ if tb: _print(file, 'Traceback (most recent call last):') print_tb(tb, limit, file) - lines = format_exception_only(etype, value) + lines = format_exception_only(etype, value, _encoding) for line in lines: _print(file, line, '') @@ -144,7 +144,7 @@ list = list + format_exception_only(etype, value) return list -def format_exception_only(etype, value): +def format_exception_only(etype, value, _encoding=None): """Format the exception part of a traceback. The arguments are the exception type and value such as given by @@ -170,12 +170,12 @@ if (isinstance(etype, BaseException) or isinstance(etype, types.InstanceType) or etype is None or type(etype) is str): - return [_format_final_exc_line(etype, value)] + return [_format_final_exc_line(etype, value, _encoding)] stype = etype.__name__ if not issubclass(etype, SyntaxError): - return [_format_final_exc_line(stype, value)] + return [_format_final_exc_line(stype, value, _encoding)] # It was a syntax error; show exactly where the problem was found. lines = [] @@ -196,26 +196,26 @@ lines.append(' %s^\n' % ''.join(caretspace)) value = msg - lines.append(_format_final_exc_line(stype, value)) + lines.append(_format_final_exc_line(stype, value, _encoding)) return lines -def _format_final_exc_line(etype, value): +def _format_final_exc_line(etype, value, _encoding=None): """Return a list of a single line -- normal case for format_exception_only""" - valuestr = _some_str(value) + valuestr = _some_str(value, _encoding) if value is None or not valuestr: line = "%s\n" % etype else: line = "%s: %s\n" % (etype, valuestr) return line -def _some_str(value): +def _some_str(value, _encoding=None): try: return str(value) except Exception: pass try: value = unicode(value) - return value.encode("ascii", "backslashreplace") + return value.encode(_encoding or "ascii", "backslashreplace") except Exception: pass return '' % type(value).__name__ diff --git a/lib-python/2.7/weakref.py b/lib-python/2.7/weakref.py --- a/lib-python/2.7/weakref.py +++ b/lib-python/2.7/weakref.py @@ -48,7 +48,14 @@ def remove(wr, selfref=ref(self)): self = selfref() if self is not None: - del self.data[wr.key] + # Changed this for PyPy: made more resistent. The + # issue is that in some corner cases, self.data + # might already be changed or removed by the time + # this weakref's callback is called. If that is + # the case, we don't want to randomly kill an + # unrelated entry. + if self.data.get(wr.key) is wr: + del self.data[wr.key] self._remove = remove UserDict.UserDict.__init__(self, *args, **kw) @@ -160,22 +167,26 @@ try: o = self.data.pop(key)() except KeyError: + o = None + if o is None: if args: return args[0] - raise - if o is None: raise KeyError, key else: return o + # The logic above was fixed in PyPy def setdefault(self, key, default=None): try: - wr = self.data[key] + o = self.data[key]() except KeyError: + o = None + if o is None: self.data[key] = KeyedRef(default, self._remove, key) return default else: - return wr() + return o + # The logic above was fixed in PyPy def update(self, dict=None, **kwargs): d = self.data diff --git a/lib_pypy/_sqlite3.py b/lib_pypy/_sqlite3.py --- a/lib_pypy/_sqlite3.py +++ b/lib_pypy/_sqlite3.py @@ -268,10 +268,18 @@ if _has_load_extension(): _ffi.cdef("int sqlite3_enable_load_extension(sqlite3 *db, int onoff);") -_lib = _ffi.verify(""" -#include -""", libraries=['sqlite3'] -) +if sys.platform.startswith('freebsd'): + _lib = _ffi.verify(""" + #include + """, libraries=['sqlite3'], + include_dirs=['/usr/local/include'], + library_dirs=['/usr/local/lib'] + ) +else: + _lib = _ffi.verify(""" + #include + """, libraries=['sqlite3'] + ) exported_sqlite_symbols = [ 'SQLITE_ALTER_TABLE', diff --git a/lib_pypy/_tkinter/tklib.py b/lib_pypy/_tkinter/tklib.py --- a/lib_pypy/_tkinter/tklib.py +++ b/lib_pypy/_tkinter/tklib.py @@ -112,6 +112,14 @@ incdirs = ['/usr/local/include/tcl8.5', '/usr/local/include/tk8.5', '/usr/X11R6/include'] linklibs = ['tk85', 'tcl85'] libdirs = ['/usr/local/lib', '/usr/X11R6/lib'] +elif sys.platform.startswith("freebsd"): + incdirs = ['/usr/local/include/tcl8.6', '/usr/local/include/tk8.6', '/usr/local/include/X11', '/usr/local/include'] + linklibs = ['tk86', 'tcl86'] + libdirs = ['/usr/local/lib'] +elif sys.platform == 'win32': + incdirs = [] + linklibs = ['tcl85', 'tk85'] + libdirs = [] else: incdirs=['/usr/include/tcl'] linklibs=['tcl', 'tk'] diff --git a/lib_pypy/cffi.egg-info b/lib_pypy/cffi.egg-info --- a/lib_pypy/cffi.egg-info +++ b/lib_pypy/cffi.egg-info @@ -1,6 +1,6 @@ Metadata-Version: 1.0 Name: cffi -Version: 0.7 +Version: 0.8 Summary: Foreign Function Interface for Python calling C code. Home-page: http://cffi.readthedocs.org Author: Armin Rigo, Maciej Fijalkowski diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py --- a/lib_pypy/cffi/__init__.py +++ b/lib_pypy/cffi/__init__.py @@ -4,5 +4,5 @@ from .api import FFI, CDefError, FFIError from .ffiplatform import VerificationError, VerificationMissing -__version__ = "0.7.2" -__version_info__ = (0, 7, 2) +__version__ = "0.8" +__version_info__ = (0, 8) diff --git a/lib_pypy/cffi/api.py b/lib_pypy/cffi/api.py --- a/lib_pypy/cffi/api.py +++ b/lib_pypy/cffi/api.py @@ -1,4 +1,5 @@ import types +from .lock import allocate_lock try: callable @@ -61,6 +62,7 @@ # rely on it! It's probably not going to work well.) self._backend = backend + self._lock = allocate_lock() self._parser = cparser.Parser() self._cached_btypes = {} self._parsed_types = types.ModuleType('parsed_types').__dict__ @@ -74,7 +76,8 @@ if name.startswith('RTLD_'): setattr(self, name, getattr(backend, name)) # - self.BVoidP = self._get_cached_btype(model.voidp_type) + with self._lock: + self.BVoidP = self._get_cached_btype(model.voidp_type) if isinstance(backend, types.ModuleType): # _cffi_backend: attach these constants to the class if not hasattr(FFI, 'NULL'): @@ -95,11 +98,12 @@ if not isinstance(csource, basestring): raise TypeError("cdef() argument must be a string") csource = csource.encode('ascii') - self._parser.parse(csource, override=override) - self._cdefsources.append(csource) - if override: - for cache in self._function_caches: - cache.clear() + with self._lock: + self._parser.parse(csource, override=override) + self._cdefsources.append(csource) + if override: + for cache in self._function_caches: + cache.clear() def dlopen(self, name, flags=0): """Load and return a dynamic library identified by 'name'. @@ -109,31 +113,47 @@ library we only look for the actual (untyped) symbols. """ assert isinstance(name, basestring) or name is None - lib, function_cache = _make_ffi_library(self, name, flags) - self._function_caches.append(function_cache) - self._libraries.append(lib) + with self._lock: + lib, function_cache = _make_ffi_library(self, name, flags) + self._function_caches.append(function_cache) + self._libraries.append(lib) return lib + def _typeof_locked(self, cdecl): + # call me with the lock! + key = cdecl + if key in self._parsed_types: + return self._parsed_types[key] + # + if not isinstance(cdecl, str): # unicode, on Python 2 + cdecl = cdecl.encode('ascii') + # + type = self._parser.parse_type(cdecl) + really_a_function_type = type.is_raw_function + if really_a_function_type: + type = type.as_function_pointer() + btype = self._get_cached_btype(type) + result = btype, really_a_function_type + self._parsed_types[key] = result + return result + def _typeof(self, cdecl, consider_function_as_funcptr=False): # string -> ctype object try: - btype, cfaf = self._parsed_types[cdecl] - if consider_function_as_funcptr and not cfaf: - raise KeyError + result = self._parsed_types[cdecl] except KeyError: - key = cdecl - if not isinstance(cdecl, str): # unicode, on Python 2 - cdecl = cdecl.encode('ascii') - cfaf = consider_function_as_funcptr - type = self._parser.parse_type(cdecl, - consider_function_as_funcptr=cfaf) - btype = self._get_cached_btype(type) - self._parsed_types[key] = btype, cfaf + with self._lock: + result = self._typeof_locked(cdecl) + # + btype, really_a_function_type = result + if really_a_function_type and not consider_function_as_funcptr: + raise CDefError("the type %r is a function type, not a " + "pointer-to-function type" % (cdecl,)) return btype def typeof(self, cdecl): """Parse the C type given as a string and return the - corresponding Python type: '>. + corresponding object. It can also be used on 'cdata' instance to get its C type. """ if isinstance(cdecl, basestring): @@ -144,6 +164,10 @@ res = _builtin_function_type(cdecl) if res is not None: return res + if (isinstance(cdecl, types.FunctionType) + and hasattr(cdecl, '_cffi_base_type')): + with self._lock: + return self._get_cached_btype(cdecl._cffi_base_type) raise TypeError(type(cdecl)) def sizeof(self, cdecl): @@ -280,14 +304,17 @@ data. Later, when this new cdata object is garbage-collected, 'destructor(old_cdata_object)' will be called. """ - try: - gc_weakrefs = self.gc_weakrefs - except AttributeError: - from .gc_weakref import GcWeakrefs - gc_weakrefs = self.gc_weakrefs = GcWeakrefs(self) - return gc_weakrefs.build(cdata, destructor) + with self._lock: + try: + gc_weakrefs = self.gc_weakrefs + except AttributeError: + from .gc_weakref import GcWeakrefs + gc_weakrefs = self.gc_weakrefs = GcWeakrefs(self) + return gc_weakrefs.build(cdata, destructor) def _get_cached_btype(self, type): + assert self._lock.acquire(False) is False + # call me with the lock! try: BType = self._cached_btypes[type] except KeyError: @@ -320,9 +347,13 @@ errno = property(_get_errno, _set_errno, None, "the value of 'errno' from/to the C calls") + def getwinerror(self, code=-1): + return self._backend.getwinerror(code) + def _pointer_to(self, ctype): from . import model - return model.pointer_cache(self, ctype) + with self._lock: + return model.pointer_cache(self, ctype) def addressof(self, cdata, field=None): """Return the address of a . @@ -342,10 +373,12 @@ variables, which must anyway be accessed directly from the lib object returned by the original FFI instance. """ - self._parser.include(ffi_to_include._parser) - self._cdefsources.append('[') - self._cdefsources.extend(ffi_to_include._cdefsources) - self._cdefsources.append(']') + with ffi_to_include._lock: + with self._lock: + self._parser.include(ffi_to_include._parser) + self._cdefsources.append('[') + self._cdefsources.extend(ffi_to_include._cdefsources) + self._cdefsources.append(']') def new_handle(self, x): return self._backend.newp_handle(self.BVoidP, x) @@ -372,7 +405,7 @@ backendlib = backend.load_library(path, flags) copied_enums = [] # - def make_accessor(name): + def make_accessor_locked(name): key = 'function ' + name if key in ffi._parser._declarations: tp = ffi._parser._declarations[key] @@ -404,11 +437,17 @@ if enumname not in library.__dict__: library.__dict__[enumname] = enumval copied_enums.append(True) + if name in library.__dict__: + return # - if name in library.__dict__: # copied from an enum value just above, - return # or multithread's race condition raise AttributeError(name) # + def make_accessor(name): + with ffi._lock: + if name in library.__dict__ or name in FFILibrary.__dict__: + return # added by another thread while waiting for the lock + make_accessor_locked(name) + # class FFILibrary(object): def __getattr__(self, name): make_accessor(name) @@ -444,4 +483,5 @@ except (KeyError, AttributeError, TypeError): return None else: - return ffi._get_cached_btype(tp) + with ffi._lock: + return ffi._get_cached_btype(tp) diff --git a/lib_pypy/cffi/cparser.py b/lib_pypy/cffi/cparser.py --- a/lib_pypy/cffi/cparser.py +++ b/lib_pypy/cffi/cparser.py @@ -142,7 +142,7 @@ if 1 <= linenum <= len(csourcelines): line = csourcelines[linenum-1] if line: - msg = 'cannot parse "%s"\n%s' % (line, msg) + msg = 'cannot parse "%s"\n%s' % (line.strip(), msg) else: msg = 'parse error\n%s' % (msg,) raise api.CDefError(msg) @@ -217,19 +217,18 @@ # if decl.name: tp = self._get_type(node, partial_length_ok=True) - if self._is_constant_declaration(node): + if self._is_constant_globalvar(node): self._declare('constant ' + decl.name, tp) else: self._declare('variable ' + decl.name, tp) - def parse_type(self, cdecl, consider_function_as_funcptr=False): + def parse_type(self, cdecl): ast, macros = self._parse('void __dummy(\n%s\n);' % cdecl) assert not macros exprnode = ast.ext[-1].type.args.params[0] if isinstance(exprnode, pycparser.c_ast.ID): raise api.CDefError("unknown identifier '%s'" % (exprnode.name,)) - return self._get_type(exprnode.type, - consider_function_as_funcptr=consider_function_as_funcptr) + return self._get_type(exprnode.type) def _declare(self, name, obj): if name in self._declarations: @@ -249,28 +248,17 @@ return model.ConstPointerType(type) return model.PointerType(type) - def _get_type(self, typenode, convert_array_to_pointer=False, - name=None, partial_length_ok=False, - consider_function_as_funcptr=False): + def _get_type(self, typenode, name=None, partial_length_ok=False): # first, dereference typedefs, if we have it already parsed, we're good if (isinstance(typenode, pycparser.c_ast.TypeDecl) and isinstance(typenode.type, pycparser.c_ast.IdentifierType) and len(typenode.type.names) == 1 and ('typedef ' + typenode.type.names[0]) in self._declarations): type = self._declarations['typedef ' + typenode.type.names[0]] - if isinstance(type, model.ArrayType): - if convert_array_to_pointer: - return type.item - else: - if (consider_function_as_funcptr and - isinstance(type, model.RawFunctionType)): - return type.as_function_pointer() return type # if isinstance(typenode, pycparser.c_ast.ArrayDecl): # array type - if convert_array_to_pointer: - return self._get_type_pointer(self._get_type(typenode.type)) if typenode.dim is None: length = None else: @@ -331,10 +319,7 @@ # if isinstance(typenode, pycparser.c_ast.FuncDecl): # a function type - result = self._parse_function_type(typenode, name) - if consider_function_as_funcptr: - result = result.as_function_pointer() - return result + return self._parse_function_type(typenode, name) # # nested anonymous structs or unions end up here if isinstance(typenode, pycparser.c_ast.Struct): @@ -365,21 +350,24 @@ isinstance(params[0].type.type, pycparser.c_ast.IdentifierType) and list(params[0].type.type.names) == ['void']): del params[0] - args = [self._get_type(argdeclnode.type, - convert_array_to_pointer=True, - consider_function_as_funcptr=True) + args = [self._as_func_arg(self._get_type(argdeclnode.type)) for argdeclnode in params] result = self._get_type(typenode.type) return model.RawFunctionType(tuple(args), result, ellipsis) - def _is_constant_declaration(self, typenode, const=False): - if isinstance(typenode, pycparser.c_ast.ArrayDecl): - return self._is_constant_declaration(typenode.type) + def _as_func_arg(self, type): + if isinstance(type, model.ArrayType): + return model.PointerType(type.item) + elif isinstance(type, model.RawFunctionType): + return type.as_function_pointer() + else: + return type + + def _is_constant_globalvar(self, typenode): if isinstance(typenode, pycparser.c_ast.PtrDecl): - const = 'const' in typenode.quals - return self._is_constant_declaration(typenode.type, const) + return 'const' in typenode.quals if isinstance(typenode, pycparser.c_ast.TypeDecl): - return const or 'const' in typenode.quals + return 'const' in typenode.quals return False def _get_struct_union_enum_type(self, kind, type, name=None, nested=False): @@ -491,7 +479,7 @@ return tp def _make_partial(self, tp, nested): - if not isinstance(tp, model.StructType): + if not isinstance(tp, model.StructOrUnion): raise api.CDefError("%s cannot be partial" % (tp,)) if not tp.has_c_name() and not nested: raise NotImplementedError("%s is partial but has no C name" %(tp,)) @@ -511,7 +499,7 @@ if (isinstance(exprnode, pycparser.c_ast.ID) and exprnode.name == '__dotdotdotarray__'): self._partial_length = True - return None + return '...' # raise api.FFIError("unsupported expression: expected a " "simple numeric constant") diff --git a/lib_pypy/cffi/gc_weakref.py b/lib_pypy/cffi/gc_weakref.py --- a/lib_pypy/cffi/gc_weakref.py +++ b/lib_pypy/cffi/gc_weakref.py @@ -14,6 +14,6 @@ def build(self, cdata, destructor): # make a new cdata of the same type as the original one - new_cdata = self.ffi.cast(self.ffi.typeof(cdata), cdata) + new_cdata = self.ffi.cast(self.ffi._backend.typeof(cdata), cdata) self.data[ref(new_cdata, self.remove)] = destructor, cdata return new_cdata diff --git a/lib_pypy/cffi/lock.py b/lib_pypy/cffi/lock.py new file mode 100644 --- /dev/null +++ b/lib_pypy/cffi/lock.py @@ -0,0 +1,30 @@ +import sys + +if sys.version_info < (3,): + try: + from thread import allocate_lock + except ImportError: + from dummy_thread import allocate_lock +else: + try: + from _thread import allocate_lock + except ImportError: + from _dummy_thread import allocate_lock + + +##import sys +##l1 = allocate_lock + +##class allocate_lock(object): +## def __init__(self): +## self._real = l1() +## def __enter__(self): +## for i in range(4, 0, -1): +## print sys._getframe(i).f_code +## print +## return self._real.__enter__() +## def __exit__(self, *args): +## return self._real.__exit__(*args) +## def acquire(self, f): +## assert f is False +## return self._real.acquire(f) diff --git a/lib_pypy/cffi/model.py b/lib_pypy/cffi/model.py --- a/lib_pypy/cffi/model.py +++ b/lib_pypy/cffi/model.py @@ -1,7 +1,10 @@ import weakref +from .lock import allocate_lock + class BaseTypeByIdentity(object): is_array_type = False + is_raw_function = False def get_c_name(self, replace_with='', context='a C file'): result = self.c_name_with_marker @@ -146,6 +149,7 @@ # a function, but not a pointer-to-function. The backend has no # notion of such a type; it's used temporarily by parsing. _base_pattern = '(&)(%s)' + is_raw_function = True def build_backend_type(self, ffi, finishlist): from . import api @@ -192,10 +196,6 @@ _base_pattern = " const *&" _base_pattern_array = "(const *&)" - def build_backend_type(self, ffi, finishlist): - BPtr = PointerType(self.totype).get_cached_btype(ffi, finishlist) - return BPtr - const_voidp_type = ConstPointerType(void_type) @@ -216,10 +216,12 @@ self.item = item self.length = length # - if self.length is None: + if length is None: brackets = '&[]' + elif length == '...': + brackets = '&[/*...*/]' else: - brackets = '&[%d]' % self.length + brackets = '&[%d]' % length self.c_name_with_marker = ( self.item.c_name_with_marker.replace('&', brackets)) @@ -227,6 +229,10 @@ return ArrayType(self.item, newlength) def build_backend_type(self, ffi, finishlist): + if self.length == '...': + from . import api + raise api.CDefError("cannot render the type %r: unknown length" % + (self,)) self.item.get_cached_btype(ffi, finishlist) # force the item BType BPtrItem = PointerType(self.item).get_cached_btype(ffi, finishlist) return global_cache(self, ffi, 'new_array_type', BPtrItem, self.length) @@ -252,6 +258,7 @@ class StructOrUnion(StructOrUnionOrEnum): fixedlayout = None completed = False + partial = False def __init__(self, name, fldnames, fldtypes, fldbitsize): self.name = name @@ -303,20 +310,21 @@ return # not completing it: it's an opaque struct # self.completed = 1 - fldtypes = tuple(tp.get_cached_btype(ffi, finishlist) - for tp in self.fldtypes) # if self.fixedlayout is None: + fldtypes = [tp.get_cached_btype(ffi, finishlist) + for tp in self.fldtypes] lst = list(zip(self.fldnames, fldtypes, self.fldbitsize)) ffi._backend.complete_struct_or_union(BType, lst, self) # else: + fldtypes = [] fieldofs, fieldsize, totalsize, totalalignment = self.fixedlayout for i in range(len(self.fldnames)): fsize = fieldsize[i] ftype = self.fldtypes[i] # - if isinstance(ftype, ArrayType) and ftype.length is None: + if isinstance(ftype, ArrayType) and ftype.length == '...': # fix the length to match the total size BItemType = ftype.item.get_cached_btype(ffi, finishlist) nlen, nrest = divmod(fsize, ffi.sizeof(BItemType)) @@ -327,18 +335,20 @@ ftype = ftype.resolve_length(nlen) self.fldtypes = (self.fldtypes[:i] + (ftype,) + self.fldtypes[i+1:]) - BArrayType = ftype.get_cached_btype(ffi, finishlist) - fldtypes = (fldtypes[:i] + (BArrayType,) + - fldtypes[i+1:]) - continue # - bitemsize = ffi.sizeof(fldtypes[i]) - if bitemsize != fsize: - self._verification_error( - "field '%s.%s' is declared as %d bytes, but is " - "really %d bytes" % (self.name, - self.fldnames[i] or '{}', - bitemsize, fsize)) + BFieldType = ftype.get_cached_btype(ffi, finishlist) + if isinstance(ftype, ArrayType) and ftype.length is None: + assert fsize == 0 + else: + bitemsize = ffi.sizeof(BFieldType) + if bitemsize != fsize: + self._verification_error( + "field '%s.%s' is declared as %d bytes, but is " + "really %d bytes" % (self.name, + self.fldnames[i] or '{}', + bitemsize, fsize)) + fldtypes.append(BFieldType) + # lst = list(zip(self.fldnames, fldtypes, self.fldbitsize, fieldofs)) ffi._backend.complete_struct_or_union(BType, lst, self, totalsize, totalalignment) @@ -348,11 +358,6 @@ from .ffiplatform import VerificationError raise VerificationError(msg) - -class StructType(StructOrUnion): - kind = 'struct' - partial = False - def check_not_partial(self): if self.partial and self.fixedlayout is None: from . import ffiplatform @@ -361,19 +366,18 @@ def build_backend_type(self, ffi, finishlist): self.check_not_partial() finishlist.append(self) - - return global_cache(self, ffi, 'new_struct_type', + # + return global_cache(self, ffi, 'new_%s_type' % self.kind, self.get_official_name(), key=self) +class StructType(StructOrUnion): + kind = 'struct' + + class UnionType(StructOrUnion): kind = 'union' - def build_backend_type(self, ffi, finishlist): - finishlist.append(self) - return global_cache(self, ffi, 'new_union_type', - self.get_official_name(), key=self) - class EnumType(StructOrUnionOrEnum): kind = 'enum' @@ -387,6 +391,12 @@ self.baseinttype = baseinttype self.build_c_name_with_marker() + def force_the_name(self, forcename): + StructOrUnionOrEnum.force_the_name(self, forcename) + if self.forcename is None: + name = self.get_official_name() + self.forcename = '$' + name.replace(' ', '_') + def check_not_partial(self): if self.partial and not self.partial_resolved: from . import ffiplatform @@ -444,6 +454,9 @@ tp = StructType(structname, None, None, None) return NamedPointerType(tp, name) + +global_lock = allocate_lock() + def global_cache(srctype, ffi, funcname, *args, **kwds): key = kwds.pop('key', (funcname, args)) assert not kwds @@ -464,8 +477,17 @@ res = getattr(ffi._backend, funcname)(*args) except NotImplementedError as e: raise NotImplementedError("%r: %s" % (srctype, e)) - ffi._backend.__typecache[key] = res - return res + # note that setdefault() on WeakValueDictionary is not atomic + # and contains a rare bug (http://bugs.python.org/issue19542); + # we have to use a lock and do it ourselves + cache = ffi._backend.__typecache + with global_lock: + res1 = cache.get(key) + if res1 is None: + cache[key] = res + return res + else: + return res1 def pointer_cache(ffi, BType): return global_cache('?', ffi, 'new_pointer_type', BType) diff --git a/lib_pypy/cffi/vengine_cpy.py b/lib_pypy/cffi/vengine_cpy.py --- a/lib_pypy/cffi/vengine_cpy.py +++ b/lib_pypy/cffi/vengine_cpy.py @@ -15,7 +15,7 @@ def patch_extension_kwds(self, kwds): pass - def find_module(self, module_name, path, so_suffix): + def find_module(self, module_name, path, so_suffixes): try: f, filename, descr = imp.find_module(module_name, path) except ImportError: @@ -25,7 +25,7 @@ # Note that after a setuptools installation, there are both .py # and .so files with the same basename. The code here relies on # imp.find_module() locating the .so in priority. - if descr[0] != so_suffix: + if descr[0] not in so_suffixes: return None return filename @@ -160,7 +160,10 @@ def __dir__(self): return FFILibrary._cffi_dir + list(self.__dict__) library = FFILibrary() - module._cffi_setup(lst, ffiplatform.VerificationError, library) + if module._cffi_setup(lst, ffiplatform.VerificationError, library): + import warnings + warnings.warn("reimporting %r might overwrite older definitions" + % (self.verifier.get_module_name())) # # finally, call the loaded_cpy_xxx() functions. This will perform # the final adjustments, like copying the Python->C wrapper @@ -280,8 +283,8 @@ return '_cffi_from_c_pointer((char *)%s, _cffi_type(%d))' % ( var, self._gettypenum(tp)) elif isinstance(tp, model.ArrayType): - return '_cffi_from_c_deref((char *)%s, _cffi_type(%d))' % ( - var, self._gettypenum(tp)) + return '_cffi_from_c_pointer((char *)%s, _cffi_type(%d))' % ( + var, self._gettypenum(model.PointerType(tp.item))) elif isinstance(tp, model.StructType): if tp.fldnames is None: raise TypeError("'%s' is used as %s, but is opaque" % ( @@ -464,11 +467,14 @@ prnt(' static Py_ssize_t nums[] = {') prnt(' sizeof(%s),' % cname) prnt(' offsetof(struct _cffi_aligncheck, y),') - for fname, _, fbitsize in tp.enumfields(): + for fname, ftype, fbitsize in tp.enumfields(): if fbitsize >= 0: continue # xxx ignore fbitsize for now prnt(' offsetof(%s, %s),' % (cname, fname)) - prnt(' sizeof(((%s *)0)->%s),' % (cname, fname)) + if isinstance(ftype, model.ArrayType) and ftype.length is None: + prnt(' 0, /* %s */' % ftype._get_c_name()) + else: + prnt(' sizeof(((%s *)0)->%s),' % (cname, fname)) prnt(' -1') prnt(' };') prnt(' return _cffi_get_struct_layout(nums);') @@ -491,7 +497,7 @@ # function = getattr(module, layoutfuncname) layout = function() - if isinstance(tp, model.StructType) and tp.partial: + if isinstance(tp, model.StructOrUnion) and tp.partial: # use the function()'s sizes and offsets to guide the # layout of the struct totalsize = layout[0] @@ -528,9 +534,10 @@ continue # xxx ignore fbitsize for now check(layout[i], ffi.offsetof(BStruct, fname), "wrong offset for field %r" % (fname,)) - BField = ffi._get_cached_btype(ftype) - check(layout[i+1], ffi.sizeof(BField), - "wrong size for field %r" % (fname,)) + if layout[i+1] != 0: + BField = ffi._get_cached_btype(ftype) + check(layout[i+1], ffi.sizeof(BField), + "wrong size for field %r" % (fname,)) i += 2 assert i == len(layout) @@ -566,7 +573,7 @@ # constants, likely declared with '#define' def _generate_cpy_const(self, is_int, name, tp=None, category='const', - vartp=None, delayed=True): + vartp=None, delayed=True, size_too=False): prnt = self._prnt funcname = '_cffi_%s_%s' % (category, name) prnt('static int %s(PyObject *lib)' % funcname) @@ -597,6 +604,15 @@ '(unsigned long long)(%s));' % (name,)) prnt(' if (o == NULL)') prnt(' return -1;') + if size_too: + prnt(' {') + prnt(' PyObject *o1 = o;') + prnt(' o = Py_BuildValue("On", o1, (Py_ssize_t)sizeof(%s));' + % (name,)) + prnt(' Py_DECREF(o1);') + prnt(' if (o == NULL)') + prnt(' return -1;') + prnt(' }') prnt(' res = PyObject_SetAttrString(lib, "%s", o);' % name) prnt(' Py_DECREF(o);') prnt(' if (res < 0)') @@ -633,12 +649,23 @@ prnt('static int %s(PyObject *lib)' % funcname) prnt('{') for enumerator, enumvalue in zip(tp.enumerators, tp.enumvalues): - prnt(' if (%s != %d) {' % (enumerator, enumvalue)) + if enumvalue < 0: + prnt(' if ((%s) >= 0 || (long)(%s) != %dL) {' % ( + enumerator, enumerator, enumvalue)) + else: + prnt(' if ((%s) < 0 || (unsigned long)(%s) != %dUL) {' % ( + enumerator, enumerator, enumvalue)) + prnt(' char buf[64];') + prnt(' if ((%s) < 0)' % enumerator) + prnt(' snprintf(buf, 63, "%%ld", (long)(%s));' % enumerator) + prnt(' else') + prnt(' snprintf(buf, 63, "%%lu", (unsigned long)(%s));' % + enumerator) prnt(' PyErr_Format(_cffi_VerificationError,') - prnt(' "enum %s: %s has the real value %d, ' - 'not %d",') - prnt(' "%s", "%s", (int)%s, %d);' % ( - name, enumerator, enumerator, enumvalue)) + prnt(' "enum %s: %s has the real value %s, ' + 'not %s",') + prnt(' "%s", "%s", buf, "%d");' % ( + name, enumerator, enumvalue)) prnt(' return -1;') prnt(' }') prnt(' return %s;' % self._chained_list_constants[True]) @@ -677,15 +704,16 @@ def _generate_cpy_variable_collecttype(self, tp, name): if isinstance(tp, model.ArrayType): - self._do_collect_type(tp) + tp_ptr = model.PointerType(tp.item) else: tp_ptr = model.PointerType(tp) - self._do_collect_type(tp_ptr) + self._do_collect_type(tp_ptr) def _generate_cpy_variable_decl(self, tp, name): if isinstance(tp, model.ArrayType): tp_ptr = model.PointerType(tp.item) - self._generate_cpy_const(False, name, tp, vartp=tp_ptr) + self._generate_cpy_const(False, name, tp, vartp=tp_ptr, + size_too = (tp.length == '...')) else: tp_ptr = model.PointerType(tp) self._generate_cpy_const(False, name, tp_ptr, category='var') @@ -694,11 +722,29 @@ _loading_cpy_variable = _loaded_noop def _loaded_cpy_variable(self, tp, name, module, library): + value = getattr(library, name) if isinstance(tp, model.ArrayType): # int a[5] is "constant" in the - return # sense that "a=..." is forbidden + # sense that "a=..." is forbidden + if tp.length == '...': + assert isinstance(value, tuple) + (value, size) = value + BItemType = self.ffi._get_cached_btype(tp.item) + length, rest = divmod(size, self.ffi.sizeof(BItemType)) + if rest != 0: + raise ffiplatform.VerificationError( + "bad size: %r does not seem to be an array of %s" % + (name, tp.item)) + tp = tp.resolve_length(length) + # 'value' is a which we have to replace with + # a if the N is actually known + if tp.length is not None: + BArray = self.ffi._get_cached_btype(tp) + value = self.ffi.cast(BArray, value) + setattr(library, name, value) + return # remove ptr= from the library instance, and replace # it by a property on the class, which reads/writes into ptr[0]. - ptr = getattr(library, name) + ptr = value delattr(library, name) def getter(library): return ptr[0] @@ -711,12 +757,9 @@ def _generate_setup_custom(self): prnt = self._prnt - prnt('static PyObject *_cffi_setup_custom(PyObject *lib)') + prnt('static int _cffi_setup_custom(PyObject *lib)') prnt('{') - prnt(' if (%s < 0)' % self._chained_list_constants[True]) - prnt(' return NULL;') - prnt(' Py_INCREF(Py_None);') - prnt(' return Py_None;') + prnt(' return %s;' % self._chained_list_constants[True]) prnt('}') cffimod_header = r''' @@ -834,17 +877,20 @@ static void *_cffi_exports[_CFFI_NUM_EXPORTS]; static PyObject *_cffi_types, *_cffi_VerificationError; -static PyObject *_cffi_setup_custom(PyObject *lib); /* forward */ +static int _cffi_setup_custom(PyObject *lib); /* forward */ static PyObject *_cffi_setup(PyObject *self, PyObject *args) { PyObject *library; + int was_alive = (_cffi_types != NULL); if (!PyArg_ParseTuple(args, "OOO", &_cffi_types, &_cffi_VerificationError, &library)) return NULL; Py_INCREF(_cffi_types); Py_INCREF(_cffi_VerificationError); - return _cffi_setup_custom(library); + if (_cffi_setup_custom(library) < 0) + return NULL; + return PyBool_FromLong(was_alive); } static void _cffi_init(void) diff --git a/lib_pypy/cffi/vengine_gen.py b/lib_pypy/cffi/vengine_gen.py --- a/lib_pypy/cffi/vengine_gen.py +++ b/lib_pypy/cffi/vengine_gen.py @@ -20,15 +20,15 @@ # up in kwds['export_symbols']. kwds.setdefault('export_symbols', self.export_symbols) - def find_module(self, module_name, path, so_suffix): - basename = module_name + so_suffix - if path is None: - path = sys.path - for dirname in path: - filename = os.path.join(dirname, basename) - if os.path.isfile(filename): - return filename - return None + def find_module(self, module_name, path, so_suffixes): + for so_suffix in so_suffixes: + basename = module_name + so_suffix + if path is None: + path = sys.path + for dirname in path: + filename = os.path.join(dirname, basename) + if os.path.isfile(filename): + return filename def collect_types(self): pass # not needed in the generic engine @@ -173,6 +173,7 @@ newfunction = self._load_constant(False, tp, name, module) else: indirections = [] + base_tp = tp if any(isinstance(typ, model.StructOrUnion) for typ in tp.args): indirect_args = [] for i, typ in enumerate(tp.args): @@ -186,16 +187,18 @@ wrappername = '_cffi_f_%s' % name newfunction = module.load_function(BFunc, wrappername) for i, typ in indirections: - newfunction = self._make_struct_wrapper(newfunction, i, typ) + newfunction = self._make_struct_wrapper(newfunction, i, typ, + base_tp) setattr(library, name, newfunction) type(library)._cffi_dir.append(name) - def _make_struct_wrapper(self, oldfunc, i, tp): + def _make_struct_wrapper(self, oldfunc, i, tp, base_tp): backend = self.ffi._backend BType = self.ffi._get_cached_btype(tp) def newfunc(*args): args = args[:i] + (backend.newp(BType, args[i]),) + args[i+1:] return oldfunc(*args) + newfunc._cffi_base_type = base_tp return newfunc # ---------- @@ -252,11 +255,14 @@ prnt(' static ssize_t nums[] = {') prnt(' sizeof(%s),' % cname) prnt(' offsetof(struct _cffi_aligncheck, y),') - for fname, _, fbitsize in tp.enumfields(): + for fname, ftype, fbitsize in tp.enumfields(): if fbitsize >= 0: continue # xxx ignore fbitsize for now prnt(' offsetof(%s, %s),' % (cname, fname)) - prnt(' sizeof(((%s *)0)->%s),' % (cname, fname)) + if isinstance(ftype, model.ArrayType) and ftype.length is None: + prnt(' 0, /* %s */' % ftype._get_c_name()) + else: + prnt(' sizeof(((%s *)0)->%s),' % (cname, fname)) prnt(' -1') prnt(' };') prnt(' return nums[i];') @@ -270,7 +276,7 @@ return # nothing to do with opaque structs layoutfuncname = '_cffi_layout_%s_%s' % (prefix, name) # - BFunc = self.ffi.typeof("ssize_t(*)(ssize_t)") + BFunc = self.ffi._typeof_locked("ssize_t(*)(ssize_t)")[0] function = module.load_function(BFunc, layoutfuncname) layout = [] num = 0 @@ -279,7 +285,7 @@ if x < 0: break layout.append(x) num += 1 - if isinstance(tp, model.StructType) and tp.partial: + if isinstance(tp, model.StructOrUnion) and tp.partial: # use the function()'s sizes and offsets to guide the # layout of the struct totalsize = layout[0] @@ -316,9 +322,10 @@ continue # xxx ignore fbitsize for now check(layout[i], ffi.offsetof(BStruct, fname), "wrong offset for field %r" % (fname,)) - BField = ffi._get_cached_btype(ftype) - check(layout[i+1], ffi.sizeof(BField), - "wrong size for field %r" % (fname,)) + if layout[i+1] != 0: + BField = ffi._get_cached_btype(ftype) + check(layout[i+1], ffi.sizeof(BField), + "wrong size for field %r" % (fname,)) i += 2 assert i == len(layout) @@ -379,15 +386,17 @@ def _load_constant(self, is_int, tp, name, module): funcname = '_cffi_const_%s' % name if is_int: - BFunc = self.ffi.typeof("int(*)(long long*)") + BType = self.ffi._typeof_locked("long long*")[0] + BFunc = self.ffi._typeof_locked("int(*)(long long*)")[0] function = module.load_function(BFunc, funcname) - p = self.ffi.new("long long*") + p = self.ffi.new(BType) negative = function(p) value = int(p[0]) if value < 0 and not negative: - value += (1 << (8*self.ffi.sizeof("long long"))) + BLongLong = self.ffi._typeof_locked("long long")[0] + value += (1 << (8*self.ffi.sizeof(BLongLong))) else: - BFunc = self.ffi.typeof(tp.get_c_name('(*)(void)', name)) + BFunc = self.ffi._typeof_locked(tp.get_c_name('(*)(void)', name))[0] function = module.load_function(BFunc, funcname) value = function() return value @@ -413,11 +422,22 @@ prnt('int %s(char *out_error)' % funcname) prnt('{') for enumerator, enumvalue in zip(tp.enumerators, tp.enumvalues): - prnt(' if (%s != %d) {' % (enumerator, enumvalue)) + if enumvalue < 0: + prnt(' if ((%s) >= 0 || (long)(%s) != %dL) {' % ( + enumerator, enumerator, enumvalue)) + else: + prnt(' if ((%s) < 0 || (unsigned long)(%s) != %dUL) {' % ( + enumerator, enumerator, enumvalue)) + prnt(' char buf[64];') + prnt(' if ((%s) < 0)' % enumerator) + prnt(' snprintf(buf, 63, "%%ld", (long)(%s));' % enumerator) + prnt(' else') + prnt(' snprintf(buf, 63, "%%lu", (unsigned long)(%s));' % + enumerator) prnt(' snprintf(out_error, 255,' - '"%s has the real value %d, not %d",') - prnt(' "%s", (int)%s, %d);' % ( - enumerator, enumerator, enumvalue)) + ' "%s has the real value %s, not %s",') + prnt(' "%s", buf, "%d");' % ( + enumerator, enumvalue)) prnt(' return -1;') prnt(' }') prnt(' return 0;') @@ -431,10 +451,11 @@ tp.enumvalues = tuple(enumvalues) tp.partial_resolved = True else: - BFunc = self.ffi.typeof("int(*)(char*)") + BType = self.ffi._typeof_locked("char[]")[0] + BFunc = self.ffi._typeof_locked("int(*)(char*)")[0] funcname = '_cffi_e_%s_%s' % (prefix, name) function = module.load_function(BFunc, funcname) - p = self.ffi.new("char[]", 256) + p = self.ffi.new(BType, 256) if function(p) < 0: error = self.ffi.string(p) if sys.version_info >= (3,): @@ -465,6 +486,14 @@ def _generate_gen_variable_decl(self, tp, name): if isinstance(tp, model.ArrayType): + if tp.length == '...': + prnt = self._prnt + funcname = '_cffi_sizeof_%s' % (name,) + self.export_symbols.append(funcname) + prnt("size_t %s(void)" % funcname) + prnt("{") + prnt(" return sizeof(%s);" % (name,)) + prnt("}") tp_ptr = model.PointerType(tp.item) self._generate_gen_const(False, name, tp_ptr) else: @@ -476,6 +505,18 @@ def _loaded_gen_variable(self, tp, name, module, library): if isinstance(tp, model.ArrayType): # int a[5] is "constant" in the # sense that "a=..." is forbidden + if tp.length == '...': + funcname = '_cffi_sizeof_%s' % (name,) + BFunc = self.ffi._typeof_locked('size_t(*)(void)')[0] + function = module.load_function(BFunc, funcname) + size = function() + BItemType = self.ffi._get_cached_btype(tp.item) + length, rest = divmod(size, self.ffi.sizeof(BItemType)) + if rest != 0: + raise ffiplatform.VerificationError( + "bad size: %r does not seem to be an array of %s" % + (name, tp.item)) + tp = tp.resolve_length(length) tp_ptr = model.PointerType(tp.item) value = self._load_constant(False, tp_ptr, name, module) # 'value' is a which we have to replace with @@ -489,7 +530,7 @@ # remove ptr= from the library instance, and replace # it by a property on the class, which reads/writes into ptr[0]. funcname = '_cffi_var_%s' % name - BFunc = self.ffi.typeof(tp.get_c_name('*(*)(void)', name)) + BFunc = self.ffi._typeof_locked(tp.get_c_name('*(*)(void)', name))[0] function = module.load_function(BFunc, funcname) ptr = function() def getter(library): diff --git a/lib_pypy/cffi/verifier.py b/lib_pypy/cffi/verifier.py --- a/lib_pypy/cffi/verifier.py +++ b/lib_pypy/cffi/verifier.py @@ -31,7 +31,7 @@ k2 = k2.lstrip('0').rstrip('L') modulename = '_cffi_%s_%s%s%s' % (tag, self._vengine._class_key, k1, k2) - suffix = _get_so_suffix() + suffix = _get_so_suffixes()[0] self.tmpdir = tmpdir or _caller_dir_pycache() self.sourcefilename = os.path.join(self.tmpdir, modulename + '.c') self.modulefilename = os.path.join(self.tmpdir, modulename + suffix) @@ -42,18 +42,21 @@ def write_source(self, file=None): """Write the C source code. It is produced in 'self.sourcefilename', which can be tweaked beforehand.""" - if self._has_source and file is None: - raise ffiplatform.VerificationError("source code already written") - self._write_source(file) + with self.ffi._lock: + if self._has_source and file is None: + raise ffiplatform.VerificationError( + "source code already written") + self._write_source(file) def compile_module(self): """Write the C source code (if not done already) and compile it. This produces a dynamic link library in 'self.modulefilename'.""" - if self._has_module: - raise ffiplatform.VerificationError("module already compiled") - if not self._has_source: - self._write_source() - self._compile_module() + with self.ffi._lock: + if self._has_module: + raise ffiplatform.VerificationError("module already compiled") + if not self._has_source: + self._write_source() + self._compile_module() def load_library(self): """Get a C module from this Verifier instance. @@ -62,11 +65,14 @@ operations to the C module. If necessary, the C code is written and compiled first. """ - if not self._has_module: - self._locate_module() + with self.ffi._lock: if not self._has_module: - self.compile_module() - return self._load_library() + self._locate_module() + if not self._has_module: + if not self._has_source: + self._write_source() + self._compile_module() + return self._load_library() def get_module_name(self): basename = os.path.basename(self.modulefilename) @@ -81,7 +87,9 @@ def get_extension(self): if not self._has_source: - self._write_source() + with self.ffi._lock: + if not self._has_source: + self._write_source() sourcename = ffiplatform.maybe_relative_path(self.sourcefilename) modname = self.get_module_name() return ffiplatform.get_extension(sourcename, modname, **self.kwds) @@ -103,7 +111,7 @@ else: path = None filename = self._vengine.find_module(self.get_module_name(), path, - _get_so_suffix()) + _get_so_suffixes()) if filename is None: return self.modulefilename = filename @@ -193,7 +201,7 @@ if keep_so: suffix = '.c' # only remove .c files else: - suffix = _get_so_suffix().lower() + suffix = _get_so_suffixes()[0].lower() for fn in filelist: if fn.lower().startswith('_cffi_') and ( fn.lower().endswith(suffix) or fn.lower().endswith('.c')): @@ -213,15 +221,20 @@ except OSError: pass -def _get_so_suffix(): +def _get_so_suffixes(): + suffixes = [] for suffix, mode, type in imp.get_suffixes(): if type == imp.C_EXTENSION: - return suffix - # bah, no C_EXTENSION available. Occurs on pypy without cpyext - if sys.platform == 'win32': - return ".pyd" - else: - return ".so" + suffixes.append(suffix) + + if not suffixes: + # bah, no C_EXTENSION available. Occurs on pypy without cpyext + if sys.platform == 'win32': + suffixes = [".pyd"] + else: + suffixes = [".so"] + + return suffixes def _ensure_dir(filename): try: diff --git a/lib_pypy/numpypy/__init__.py b/lib_pypy/numpypy/__init__.py deleted file mode 100644 --- a/lib_pypy/numpypy/__init__.py +++ /dev/null @@ -1,17 +0,0 @@ -from . import core -from .core import * -from . import lib -from .lib import * - -from __builtin__ import bool, int, long, float, complex, object, unicode, str - -from .core import round, abs, max, min - -__version__ = '1.7.0' - -__all__ = ['__version__'] -__all__ += core.__all__ -__all__ += lib.__all__ - -#import sys -#sys.modules.setdefault('numpy', sys.modules['numpypy']) diff --git a/lib_pypy/numpypy/core/__init__.py b/lib_pypy/numpypy/core/__init__.py deleted file mode 100644 --- a/lib_pypy/numpypy/core/__init__.py +++ /dev/null @@ -1,19 +0,0 @@ -from __future__ import division, absolute_import, print_function - -from . import multiarray -from . import umath -from . import numeric -from .numeric import * -from . import fromnumeric -from .fromnumeric import * -from . import shape_base -from .shape_base import * - -from .fromnumeric import amax as max, amin as min, \ - round_ as round -from .numeric import absolute as abs - -__all__ = [] -__all__ += numeric.__all__ -__all__ += fromnumeric.__all__ -__all__ += shape_base.__all__ diff --git a/lib_pypy/numpypy/core/_methods.py b/lib_pypy/numpypy/core/_methods.py deleted file mode 100644 --- a/lib_pypy/numpypy/core/_methods.py +++ /dev/null @@ -1,109 +0,0 @@ -# Array methods which are called by the both the C-code for the method -# and the Python code for the NumPy-namespace function - -import multiarray as mu -import umath as um -from numeric import asanyarray - -def _amax(a, axis=None, out=None, keepdims=False): - return um.maximum.reduce(a, axis=axis, - out=out, keepdims=keepdims) - -def _amin(a, axis=None, out=None, keepdims=False): - return um.minimum.reduce(a, axis=axis, - out=out, keepdims=keepdims) - -def _sum(a, axis=None, dtype=None, out=None, keepdims=False): - return um.add.reduce(a, axis=axis, dtype=dtype, - out=out, keepdims=keepdims) - -def _prod(a, axis=None, dtype=None, out=None, keepdims=False): - return um.multiply.reduce(a, axis=axis, dtype=dtype, - out=out, keepdims=keepdims) - -def _any(a, axis=None, dtype=None, out=None, keepdims=False): - return um.logical_or.reduce(a, axis=axis, dtype=dtype, out=out, - keepdims=keepdims) - -def _all(a, axis=None, dtype=None, out=None, keepdims=False): - return um.logical_and.reduce(a, axis=axis, dtype=dtype, out=out, - keepdims=keepdims) - -def _count_reduce_items(arr, axis): - if axis is None: - axis = tuple(xrange(arr.ndim)) - if not isinstance(axis, tuple): - axis = (axis,) - items = 1 - for ax in axis: - items *= arr.shape[ax] - return items - -def _mean(a, axis=None, dtype=None, out=None, keepdims=False): - arr = asanyarray(a) - - # Upgrade bool, unsigned int, and int to float64 - if dtype is None and arr.dtype.kind in ['b','u','i']: - ret = um.add.reduce(arr, axis=axis, dtype='f8', - out=out, keepdims=keepdims) - else: - ret = um.add.reduce(arr, axis=axis, dtype=dtype, - out=out, keepdims=keepdims) - rcount = _count_reduce_items(arr, axis) - if isinstance(ret, mu.ndarray): - ret = um.true_divide(ret, rcount, - out=ret, casting='unsafe', subok=False) - else: - ret = ret / float(rcount) - return ret - -def _var(a, axis=None, dtype=None, out=None, ddof=0, - keepdims=False): - arr = asanyarray(a) - - # First compute the mean, saving 'rcount' for reuse later - if dtype is None and arr.dtype.kind in ['b','u','i']: - arrmean = um.add.reduce(arr, axis=axis, dtype='f8', keepdims=True) - else: - arrmean = um.add.reduce(arr, axis=axis, dtype=dtype, keepdims=True) - rcount = _count_reduce_items(arr, axis) - if isinstance(arrmean, mu.ndarray): - arrmean = um.true_divide(arrmean, rcount, - out=arrmean, casting='unsafe', subok=False) - else: - arrmean = arrmean / float(rcount) - - # arr - arrmean - x = arr - arrmean - - # (arr - arrmean) ** 2 - if arr.dtype.kind == 'c': - x = um.multiply(x, um.conjugate(x), out=x).real - else: - x = um.multiply(x, x, out=x) - - # add.reduce((arr - arrmean) ** 2, axis) - ret = um.add.reduce(x, axis=axis, dtype=dtype, out=out, keepdims=keepdims) - - # add.reduce((arr - arrmean) ** 2, axis) / (n - ddof) - if not keepdims and isinstance(rcount, mu.ndarray): - rcount = rcount.squeeze(axis=axis) - rcount -= ddof - if isinstance(ret, mu.ndarray): - ret = um.true_divide(ret, rcount, - out=ret, casting='unsafe', subok=False) - else: - ret = ret / float(rcount) - - return ret - -def _std(a, axis=None, dtype=None, out=None, ddof=0, keepdims=False): From noreply at buildbot.pypy.org Tue Nov 19 20:10:54 2013 From: noreply at buildbot.pypy.org (rguillebert) Date: Tue, 19 Nov 2013 20:10:54 +0100 (CET) Subject: [pypy-commit] pypy numpypy-array_prepare_-array_wrap: Forgot to solve this merge conflict Message-ID: <20131119191054.9AA051C0225@cobra.cs.uni-duesseldorf.de> Author: Romain Guillebert Branch: numpypy-array_prepare_-array_wrap Changeset: r68247:0559ff165dad Date: 2013-11-19 20:10 +0100 http://bitbucket.org/pypy/pypy/changeset/0559ff165dad/ Log: Forgot to solve this merge conflict diff --git a/pypy/module/micronumpy/interp_boxes.py b/pypy/module/micronumpy/interp_boxes.py --- a/pypy/module/micronumpy/interp_boxes.py +++ b/pypy/module/micronumpy/interp_boxes.py @@ -313,9 +313,6 @@ self.w_flags = W_FlagsObject(self) return self.w_flags - def descr_dtype(self, space): - return self._get_dtype(space) - class W_BoolBox(W_GenericBox, PrimitiveBox): descr__new__, _get_dtype, descr_reduce = new_dtype_getter("bool") @@ -567,9 +564,6 @@ conjugate = interp2app(W_GenericBox.descr_conjugate), astype = interp2app(W_GenericBox.descr_astype), view = interp2app(W_GenericBox.descr_view), -<<<<<<< local - dtype = GetSetProperty(W_GenericBox.descr_dtype) -======= squeeze = interp2app(W_GenericBox.descr_self), copy = interp2app(W_GenericBox.descr_copy), @@ -582,7 +576,6 @@ ndim = GetSetProperty(W_GenericBox.descr_get_ndim), T = GetSetProperty(W_GenericBox.descr_self), flags = GetSetProperty(W_GenericBox.descr_get_flags), ->>>>>>> other ) W_BoolBox.typedef = TypeDef("bool_", W_GenericBox.typedef, From noreply at buildbot.pypy.org Tue Nov 19 21:09:07 2013 From: noreply at buildbot.pypy.org (rguillebert) Date: Tue, 19 Nov 2013 21:09:07 +0100 (CET) Subject: [pypy-commit] pypy numpypy-array_prepare_-array_wrap: sum doesn't exist in numpypy Message-ID: <20131119200907.816361C147C@cobra.cs.uni-duesseldorf.de> Author: Romain Guillebert Branch: numpypy-array_prepare_-array_wrap Changeset: r68248:b5c80215f7e5 Date: 2013-11-19 21:08 +0100 http://bitbucket.org/pypy/pypy/changeset/b5c80215f7e5/ Log: sum doesn't exist in numpypy diff --git a/pypy/module/micronumpy/test/test_subtype.py b/pypy/module/micronumpy/test/test_subtype.py --- a/pypy/module/micronumpy/test/test_subtype.py +++ b/pypy/module/micronumpy/test/test_subtype.py @@ -374,7 +374,7 @@ raises(TypeError, log, a, out=c) def test___array_prepare__reduce(self): - from numpypy import ndarray, array, sum, ones, add + from numpypy import ndarray, array, ones, add class with_prepare(ndarray): def __array_prepare__(self, arr, context): x = array(arr).view(type=with_prepare) @@ -382,7 +382,7 @@ print 'called_prepare',arr return x a = ones(2).view(type=with_prepare) - x = sum(a) + x = a.sum() assert type(x) == with_prepare assert x.shape == () # reduce functions do not call prepare, is this a numpy 'feature'? @@ -391,6 +391,6 @@ assert type(x) == with_prepare assert not getattr(x, 'called_prepare',False) a = ones((2,3)).view(type=with_prepare) - x = sum(a, axis=0) + x = a.sum(axis=0) assert type(x) == with_prepare assert not getattr(x, 'called_prepare',False) From noreply at buildbot.pypy.org Wed Nov 20 01:47:51 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Wed, 20 Nov 2013 01:47:51 +0100 (CET) Subject: [pypy-commit] pypy default: backout afb227c, breaks everything else and doesn't fix freebsd Message-ID: <20131120004751.D41301C1DBD@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r68249:55b6a37713d9 Date: 2013-11-19 19:46 -0500 http://bitbucket.org/pypy/pypy/changeset/55b6a37713d9/ Log: backout afb227c, breaks everything else and doesn't fix freebsd diff --git a/rpython/rlib/rdynload.py b/rpython/rlib/rdynload.py --- a/rpython/rlib/rdynload.py +++ b/rpython/rlib/rdynload.py @@ -4,7 +4,6 @@ from rpython.rtyper.tool import rffi_platform from rpython.rtyper.lltypesystem import rffi from rpython.rlib.rarithmetic import r_uint -from rpython.rlib.objectmodel import we_are_translated from rpython.translator.tool.cbuild import ExternalCompilationInfo from rpython.translator.platform import platform @@ -80,38 +79,6 @@ RTLD_NOW = cConfig.RTLD_NOW RTLD_LAZY = cConfig.RTLD_LAZY - _t_opened = {} - - def t_dlopen(name): - # for direct execution: can't use the regular way on FreeBSD :-( - # http://factor-language.blogspot.de/2009/02/note-about-libdl-functions-on-netbsd.html - import ctypes - if name: - name = rffi.charp2str(name) - else: - name = None - try: - res = ctypes.cdll.LoadLibrary(name) - except OSError, e: - raise DLOpenError(str(e)) - h = rffi.cast(rffi.VOIDP, res._handle) - _t_opened[rffi.cast(rffi.LONG, h)] = res - return h - - def t_dlclose(handle): - _t_opened.pop(rffi.cast(rffi.LONG, handle)) - return rffi.cast(rffi.INT, 0) - - def t_dldym(handle, name): - import ctypes - lib = _t_opened[rffi.cast(rffi.LONG, handle)] - try: - symbol = lib[name] - except AttributeError: - raise KeyError(name) - res = ctypes.cast(symbol, ctypes.c_void_p) - return rffi.cast(rffi.VOIDP, res.value or 0) - def dlerror(): # XXX this would never work on top of ll2ctypes, because # ctypes are calling dlerror itself, unsure if I can do much in this @@ -124,8 +91,6 @@ def dlopen(name, mode=-1): """ Wrapper around C-level dlopen """ - if not we_are_translated(): - return t_dlopen(name) if mode == -1: if RTLD_LOCAL is not None: mode = RTLD_LOCAL @@ -139,16 +104,11 @@ raise DLOpenError(err) return res - def dlclose(handle): - if not we_are_translated(): - return t_dlclose(handle) - return c_dlclose(handle) + dlclose = c_dlclose def dlsym(libhandle, name): """ Wrapper around C-level dlsym """ - if not we_are_translated(): - return t_dldym(libhandle, name) res = c_dlsym(libhandle, name) if not res: raise KeyError(name) diff --git a/rpython/rlib/test/test_rdynload.py b/rpython/rlib/test/test_rdynload.py --- a/rpython/rlib/test/test_rdynload.py +++ b/rpython/rlib/test/test_rdynload.py @@ -21,4 +21,3 @@ lltype.Signed)), dlsym(lib, 'abs')) assert 1 == handle(1) assert 1 == handle(-1) - dlclose(lib) From noreply at buildbot.pypy.org Wed Nov 20 02:04:57 2013 From: noreply at buildbot.pypy.org (Manuel Jacob) Date: Wed, 20 Nov 2013 02:04:57 +0100 (CET) Subject: [pypy-commit] pypy llvm-translation-backend: hg merge default Message-ID: <20131120010457.CE6931C237F@cobra.cs.uni-duesseldorf.de> Author: Manuel Jacob Branch: llvm-translation-backend Changeset: r68250:fb005e046dc4 Date: 2013-11-19 23:41 +0100 http://bitbucket.org/pypy/pypy/changeset/fb005e046dc4/ Log: hg merge default diff too long, truncating to 2000 out of 47956 lines diff --git a/Makefile b/Makefile --- a/Makefile +++ b/Makefile @@ -1,7 +1,38 @@ all: pypy-c +PYPY_EXECUTABLE := $(shell which pypy) +URAM := $(shell python -c "import sys; print 4.5 if sys.maxint>1<<32 else 2.5") + +ifeq ($(PYPY_EXECUTABLE),) +RUNINTERP = python +else +RUNINTERP = $(PYPY_EXECUTABLE) +endif + pypy-c: - @echo "Building PyPy with JIT, it'll take about 40 minutes and 4G of RAM" - @sleep 3 - rpython/bin/rpython -Ojit pypy/goal/targetpypystandalone.py + @echo + @echo "====================================================================" +ifeq ($(PYPY_EXECUTABLE),) + @echo "Building a regular (jitting) version of PyPy, using CPython." + @echo "This takes around 2 hours and $(URAM) GB of RAM." + @echo "Note that pre-installing a PyPy binary would reduce this time" + @echo "and produce basically the same result." +else + @echo "Building a regular (jitting) version of PyPy, using" + @echo "$(PYPY_EXECUTABLE) to run the translation itself." + @echo "This takes up to 1 hour and $(URAM) GB of RAM." +endif + @echo + @echo "For more control (e.g. to use multiple CPU cores during part of" + @echo "the process) you need to run \`\`rpython/bin/rpython'' directly." + @echo "For more information see \`\`http://pypy.org/download.html''." + @echo "====================================================================" + @echo + @sleep 5 + $(RUNINTERP) rpython/bin/rpython -Ojit pypy/goal/targetpypystandalone.py + +# Note: the -jN option, or MAKEFLAGS=-jN, are not usable. They are +# replaced with an opaque --jobserver option by the time this Makefile +# runs. We cannot get their original value either: +# http://lists.gnu.org/archive/html/help-make/2010-08/msg00106.html diff --git a/README.rst b/README.rst --- a/README.rst +++ b/README.rst @@ -1,5 +1,5 @@ ===================================== -PyPy: Python in Python Implementation +PyPy: Python in Python Implementation ===================================== Welcome to PyPy! @@ -26,12 +26,14 @@ Building ======== -build with:: +build with: - rpython/bin/rpython -Ojit pypy/goal/targetpypystandalone.py +.. code-block:: console + + $ rpython/bin/rpython -Ojit pypy/goal/targetpypystandalone.py This ends up with ``pypy-c`` binary in the main pypy directory. We suggest -to use virtualenv with the resulting pypy-c as the interpreter, you can +to use virtualenv with the resulting pypy-c as the interpreter; you can find more details about various installation schemes here: http://doc.pypy.org/en/latest/getting-started.html#installing-pypy diff --git a/lib-python/2.7/argparse.py b/lib-python/2.7/argparse.py --- a/lib-python/2.7/argparse.py +++ b/lib-python/2.7/argparse.py @@ -1780,7 +1780,19 @@ # error if this argument is not allowed with other previously # seen arguments, assuming that actions that use the default # value don't really count as "present" - if argument_values is not action.default: + + # XXX PyPy bug-to-bug compatibility: "is" on primitive types + # is not consistent in CPython. We'll assume it is close + # enough for ints (which is true only for "small ints"), but + # for floats and longs and complexes we'll go for the option + # of forcing "is" to say False, like it usually does on + # CPython. A fix is pending on CPython trunk + # (http://bugs.python.org/issue18943) but that might change + # the details of the semantics and so not be applied to 2.7. + # See the line AA below. + + if (argument_values is not action.default or + type(argument_values) in (float, long, complex)): # AA seen_non_default_actions.add(action) for conflict_action in action_conflicts.get(action, []): if conflict_action in seen_non_default_actions: diff --git a/lib-python/2.7/json/encoder.py b/lib-python/2.7/json/encoder.py --- a/lib-python/2.7/json/encoder.py +++ b/lib-python/2.7/json/encoder.py @@ -4,6 +4,21 @@ from __pypy__.builders import StringBuilder, UnicodeBuilder +class StringOrUnicodeBuilder(object): + def __init__(self): + self._builder = StringBuilder() + def append(self, string): + try: + self._builder.append(string) + except UnicodeEncodeError: + ub = UnicodeBuilder() + ub.append(self._builder.build()) + self._builder = ub + ub.append(string) + def build(self): + return self._builder.build() + + ESCAPE = re.compile(r'[\x00-\x1f\\"\b\f\n\r\t]') ESCAPE_ASCII = re.compile(r'([\\"]|[^\ -~])') HAS_UTF8 = re.compile(r'[\x80-\xff]') @@ -192,7 +207,7 @@ if self.ensure_ascii: builder = StringBuilder() else: - builder = UnicodeBuilder() + builder = StringOrUnicodeBuilder() self.__encode(o, markers, builder, 0) return builder.build() diff --git a/lib-python/2.7/string.py b/lib-python/2.7/string.py --- a/lib-python/2.7/string.py +++ b/lib-python/2.7/string.py @@ -66,16 +66,17 @@ must be of the same length. """ - if len(fromstr) != len(tostr): + n = len(fromstr) + if n != len(tostr): raise ValueError, "maketrans arguments must have same length" - global _idmapL - if not _idmapL: - _idmapL = list(_idmap) - L = _idmapL[:] - fromstr = map(ord, fromstr) - for i in range(len(fromstr)): - L[fromstr[i]] = tostr[i] - return ''.join(L) + # this function has been rewritten to suit PyPy better; it is + # almost 10x faster than the original. + buf = bytearray(256) + for i in range(256): + buf[i] = i + for i in range(n): + buf[ord(fromstr[i])] = tostr[i] + return str(buf) diff --git a/lib-python/2.7/test/keycert.pem b/lib-python/2.7/test/keycert.pem --- a/lib-python/2.7/test/keycert.pem +++ b/lib-python/2.7/test/keycert.pem @@ -1,32 +1,31 @@ ------BEGIN RSA PRIVATE KEY----- -MIICXwIBAAKBgQC8ddrhm+LutBvjYcQlnH21PPIseJ1JVG2HMmN2CmZk2YukO+9L -opdJhTvbGfEj0DQs1IE8M+kTUyOmuKfVrFMKwtVeCJphrAnhoz7TYOuLBSqt7lVH -fhi/VwovESJlaBOp+WMnfhcduPEYHYx/6cnVapIkZnLt30zu2um+DzA9jQIDAQAB -AoGBAK0FZpaKj6WnJZN0RqhhK+ggtBWwBnc0U/ozgKz2j1s3fsShYeiGtW6CK5nU -D1dZ5wzhbGThI7LiOXDvRucc9n7vUgi0alqPQ/PFodPxAN/eEYkmXQ7W2k7zwsDA -IUK0KUhktQbLu8qF/m8qM86ba9y9/9YkXuQbZ3COl5ahTZrhAkEA301P08RKv3KM -oXnGU2UHTuJ1MAD2hOrPxjD4/wxA/39EWG9bZczbJyggB4RHu0I3NOSFjAm3HQm0 -ANOu5QK9owJBANgOeLfNNcF4pp+UikRFqxk5hULqRAWzVxVrWe85FlPm0VVmHbb/ -loif7mqjU8o1jTd/LM7RD9f2usZyE2psaw8CQQCNLhkpX3KO5kKJmS9N7JMZSc4j -oog58yeYO8BBqKKzpug0LXuQultYv2K4veaIO04iL9VLe5z9S/Q1jaCHBBuXAkEA -z8gjGoi1AOp6PBBLZNsncCvcV/0aC+1se4HxTNo2+duKSDnbq+ljqOM+E7odU+Nq -ewvIWOG//e8fssd0mq3HywJBAJ8l/c8GVmrpFTx8r/nZ2Pyyjt3dH1widooDXYSV -q6Gbf41Llo5sYAtmxdndTLASuHKecacTgZVhy0FryZpLKrU= ------END RSA PRIVATE KEY----- +-----BEGIN PRIVATE KEY----- +MIICdwIBADANBgkqhkiG9w0BAQEFAASCAmEwggJdAgEAAoGBANtb0+YrKuxevGpm +LrjaUhZSgz6zFAmuGFmKmUbdjmfv9zSmmdsQIksK++jK0Be9LeZy20j6ahOfuVa0 +ufEmPoP7Fy4hXegKZR9cCWcIe/A6H2xWF1IIJLRTLaU8ol/I7T+um5HD5AwAwNPP +USNU0Eegmvp+xxWu3NX2m1Veot85AgMBAAECgYA3ZdZ673X0oexFlq7AAmrutkHt +CL7LvwrpOiaBjhyTxTeSNWzvtQBkIU8DOI0bIazA4UreAFffwtvEuPmonDb3F+Iq +SMAu42XcGyVZEl+gHlTPU9XRX7nTOXVt+MlRRRxL6t9GkGfUAXI3XxJDXW3c0vBK +UL9xqD8cORXOfE06rQJBAP8mEX1ERkR64Ptsoe4281vjTlNfIbs7NMPkUnrn9N/Y +BLhjNIfQ3HFZG8BTMLfX7kCS9D593DW5tV4Z9BP/c6cCQQDcFzCcVArNh2JSywOQ +ZfTfRbJg/Z5Lt9Fkngv1meeGNPgIMLN8Sg679pAOOWmzdMO3V706rNPzSVMME7E5 +oPIfAkEA8pDddarP5tCvTTgUpmTFbakm0KoTZm2+FzHcnA4jRh+XNTjTOv98Y6Ik +eO5d1ZnKXseWvkZncQgxfdnMqqpj5wJAcNq/RVne1DbYlwWchT2Si65MYmmJ8t+F +0mcsULqjOnEMwf5e+ptq5LzwbyrHZYq5FNk7ocufPv/ZQrcSSC+cFwJBAKvOJByS +x56qyGeZLOQlWS2JS3KJo59XuLFGqcbgN9Om9xFa41Yb4N9NvplFivsvZdw3m1Q/ +SPIXQuT8RMPDVNQ= +-----END PRIVATE KEY----- -----BEGIN CERTIFICATE----- -MIICpzCCAhCgAwIBAgIJAP+qStv1cIGNMA0GCSqGSIb3DQEBBQUAMIGJMQswCQYD -VQQGEwJVUzERMA8GA1UECBMIRGVsYXdhcmUxEzARBgNVBAcTCldpbG1pbmd0b24x -IzAhBgNVBAoTGlB5dGhvbiBTb2Z0d2FyZSBGb3VuZGF0aW9uMQwwCgYDVQQLEwNT -U0wxHzAdBgNVBAMTFnNvbWVtYWNoaW5lLnB5dGhvbi5vcmcwHhcNMDcwODI3MTY1 -NDUwWhcNMTMwMjE2MTY1NDUwWjCBiTELMAkGA1UEBhMCVVMxETAPBgNVBAgTCERl -bGF3YXJlMRMwEQYDVQQHEwpXaWxtaW5ndG9uMSMwIQYDVQQKExpQeXRob24gU29m -dHdhcmUgRm91bmRhdGlvbjEMMAoGA1UECxMDU1NMMR8wHQYDVQQDExZzb21lbWFj -aGluZS5weXRob24ub3JnMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQC8ddrh -m+LutBvjYcQlnH21PPIseJ1JVG2HMmN2CmZk2YukO+9LopdJhTvbGfEj0DQs1IE8 -M+kTUyOmuKfVrFMKwtVeCJphrAnhoz7TYOuLBSqt7lVHfhi/VwovESJlaBOp+WMn -fhcduPEYHYx/6cnVapIkZnLt30zu2um+DzA9jQIDAQABoxUwEzARBglghkgBhvhC -AQEEBAMCBkAwDQYJKoZIhvcNAQEFBQADgYEAF4Q5BVqmCOLv1n8je/Jw9K669VXb -08hyGzQhkemEBYQd6fzQ9A/1ZzHkJKb1P6yreOLSEh4KcxYPyrLRC1ll8nr5OlCx -CMhKkTnR6qBsdNV0XtdU2+N25hqW+Ma4ZeqsN/iiJVCGNOZGnvQuvCAGWF8+J/f/ -iHkC6gGdBJhogs4= +MIICVDCCAb2gAwIBAgIJANfHOBkZr8JOMA0GCSqGSIb3DQEBBQUAMF8xCzAJBgNV +BAYTAlhZMRcwFQYDVQQHEw5DYXN0bGUgQW50aHJheDEjMCEGA1UEChMaUHl0aG9u +IFNvZnR3YXJlIEZvdW5kYXRpb24xEjAQBgNVBAMTCWxvY2FsaG9zdDAeFw0xMDEw +MDgyMzAxNTZaFw0yMDEwMDUyMzAxNTZaMF8xCzAJBgNVBAYTAlhZMRcwFQYDVQQH +Ew5DYXN0bGUgQW50aHJheDEjMCEGA1UEChMaUHl0aG9uIFNvZnR3YXJlIEZvdW5k +YXRpb24xEjAQBgNVBAMTCWxvY2FsaG9zdDCBnzANBgkqhkiG9w0BAQEFAAOBjQAw +gYkCgYEA21vT5isq7F68amYuuNpSFlKDPrMUCa4YWYqZRt2OZ+/3NKaZ2xAiSwr7 +6MrQF70t5nLbSPpqE5+5VrS58SY+g/sXLiFd6AplH1wJZwh78DofbFYXUggktFMt +pTyiX8jtP66bkcPkDADA089RI1TQR6Ca+n7HFa7c1fabVV6i3zkCAwEAAaMYMBYw +FAYDVR0RBA0wC4IJbG9jYWxob3N0MA0GCSqGSIb3DQEBBQUAA4GBAHPctQBEQ4wd +BJ6+JcpIraopLn8BGhbjNWj40mmRqWB/NAWF6M5ne7KpGAu7tLeG4hb1zLaldK8G +lxy2GPSRF6LFS48dpEj2HbMv2nvv6xxalDMJ9+DicWgAKTQ6bcX2j3GUkCR0g/T1 +CRlNBAAlvhKzO7Clpf9l0YKBEfraJByX -----END CERTIFICATE----- diff --git a/lib-python/2.7/test/sha256.pem b/lib-python/2.7/test/sha256.pem --- a/lib-python/2.7/test/sha256.pem +++ b/lib-python/2.7/test/sha256.pem @@ -1,129 +1,128 @@ # Certificate chain for https://sha256.tbs-internet.com - 0 s:/C=FR/postalCode=14000/ST=Calvados/L=CAEN/street=22 rue de Bretagne/O=TBS INTERNET/OU=0002 440443810/OU=sha-256 production/CN=sha256.tbs-internet.com - i:/C=FR/ST=Calvados/L=Caen/O=TBS INTERNET/OU=Terms and Conditions: http://www.tbs-internet.com/CA/repository/OU=TBS INTERNET CA/CN=TBS X509 CA SGC + 0 s:/C=FR/postalCode=14000/ST=Calvados/L=CAEN/street=22 rue de Bretagne/O=TBS INTERNET/OU=0002 440443810/OU=Certificats TBS X509/CN=ecom.tbs-x509.com + i:/C=FR/ST=Calvados/L=Caen/O=TBS INTERNET/OU=Terms and Conditions: http://www.tbs-internet.com/CA/repository/OU=TBS INTERNET CA/CN=TBS X509 CA business -----BEGIN CERTIFICATE----- -MIIGXTCCBUWgAwIBAgIRAMmag+ygSAdxZsbyzYjhuW0wDQYJKoZIhvcNAQELBQAw -gcQxCzAJBgNVBAYTAkZSMREwDwYDVQQIEwhDYWx2YWRvczENMAsGA1UEBxMEQ2Fl +MIIGTjCCBTagAwIBAgIQOh3d9dNDPq1cSdJmEiMpqDANBgkqhkiG9w0BAQUFADCB +yTELMAkGA1UEBhMCRlIxETAPBgNVBAgTCENhbHZhZG9zMQ0wCwYDVQQHEwRDYWVu +MRUwEwYDVQQKEwxUQlMgSU5URVJORVQxSDBGBgNVBAsTP1Rlcm1zIGFuZCBDb25k +aXRpb25zOiBodHRwOi8vd3d3LnRicy1pbnRlcm5ldC5jb20vQ0EvcmVwb3NpdG9y +eTEYMBYGA1UECxMPVEJTIElOVEVSTkVUIENBMR0wGwYDVQQDExRUQlMgWDUwOSBD +QSBidXNpbmVzczAeFw0xMTAxMjUwMDAwMDBaFw0xMzAyMDUyMzU5NTlaMIHHMQsw +CQYDVQQGEwJGUjEOMAwGA1UEERMFMTQwMDAxETAPBgNVBAgTCENhbHZhZG9zMQ0w +CwYDVQQHEwRDQUVOMRswGQYDVQQJExIyMiBydWUgZGUgQnJldGFnbmUxFTATBgNV +BAoTDFRCUyBJTlRFUk5FVDEXMBUGA1UECxMOMDAwMiA0NDA0NDM4MTAxHTAbBgNV +BAsTFENlcnRpZmljYXRzIFRCUyBYNTA5MRowGAYDVQQDExFlY29tLnRicy14NTA5 +LmNvbTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAKRrlHUnJ++1lpcg +jtYco7cdmRe+EEfTmwPfCdfV3G1QfsTSvY6FfMpm/83pqHfT+4ANwr18wD9ZrAEN +G16mf9VdCGK12+TP7DmqeZyGIqlFFoahQnmb8EarvE43/1UeQ2CV9XmzwZvpqeli +LfXsFonawrY3H6ZnMwS64St61Z+9gdyuZ/RbsoZBbT5KUjDEG844QRU4OT1IGeEI +eY5NM5RNIh6ZNhVtqeeCxMS7afONkHQrOco73RdSTRck/Hj96Ofl3MHNHryr+AMK +DGFk1kLCZGpPdXtkxXvaDeQoiYDlil26CWc+YK6xyDPMdsWvoG14ZLyCpzMXA7/7 +4YAQRH0CAwEAAaOCAjAwggIsMB8GA1UdIwQYMBaAFBoJBMz5CY+7HqDO1KQUf0vV +I1jNMB0GA1UdDgQWBBQgOU8HsWzbmD4WZP5Wtdw7jca2WDAOBgNVHQ8BAf8EBAMC +BaAwDAYDVR0TAQH/BAIwADAdBgNVHSUEFjAUBggrBgEFBQcDAQYIKwYBBQUHAwIw +TAYDVR0gBEUwQzBBBgsrBgEEAYDlNwIBATAyMDAGCCsGAQUFBwIBFiRodHRwczov +L3d3dy50YnMtaW50ZXJuZXQuY29tL0NBL0NQUzEwdwYDVR0fBHAwbjA3oDWgM4Yx +aHR0cDovL2NybC50YnMtaW50ZXJuZXQuY29tL1RCU1g1MDlDQWJ1c2luZXNzLmNy +bDAzoDGgL4YtaHR0cDovL2NybC50YnMteDUwOS5jb20vVEJTWDUwOUNBYnVzaW5l +c3MuY3JsMIGwBggrBgEFBQcBAQSBozCBoDA9BggrBgEFBQcwAoYxaHR0cDovL2Ny +dC50YnMtaW50ZXJuZXQuY29tL1RCU1g1MDlDQWJ1c2luZXNzLmNydDA5BggrBgEF +BQcwAoYtaHR0cDovL2NydC50YnMteDUwOS5jb20vVEJTWDUwOUNBYnVzaW5lc3Mu +Y3J0MCQGCCsGAQUFBzABhhhodHRwOi8vb2NzcC50YnMteDUwOS5jb20wMwYDVR0R +BCwwKoIRZWNvbS50YnMteDUwOS5jb22CFXd3dy5lY29tLnRicy14NTA5LmNvbTAN +BgkqhkiG9w0BAQUFAAOCAQEArT4NHfbY87bGAw8lPV4DmHlmuDuVp/y7ltO3Ynse +3Rz8RxW2AzuO0Oy2F0Cu4yWKtMyEyMXyHqWtae7ElRbdTu5w5GwVBLJHClCzC8S9 +SpgMMQTx3Rgn8vjkHuU9VZQlulZyiPK7yunjc7c310S9FRZ7XxOwf8Nnx4WnB+No +WrfApzhhQl31w+RyrNxZe58hCfDDHmevRvwLjQ785ZoQXJDj2j3qAD4aI2yB8lB5 +oaE1jlCJzC7Kmz/Y9jzfmv/zAs1LQTm9ktevv4BTUFaGjv9jxnQ1xnS862ZiouLW +zZYIlYPf4F6JjXGiIQgQRglILUfq3ftJd9/ok9W9ZF8h8w== +-----END CERTIFICATE----- + 1 s:/C=FR/ST=Calvados/L=Caen/O=TBS INTERNET/OU=Terms and Conditions: http://www.tbs-internet.com/CA/repository/OU=TBS INTERNET CA/CN=TBS X509 CA business + i:/C=SE/O=AddTrust AB/OU=AddTrust External TTP Network/CN=AddTrust External CA Root +-----BEGIN CERTIFICATE----- +MIIFPzCCBCegAwIBAgIQDlBz/++iRSmLDeVRHT/hADANBgkqhkiG9w0BAQUFADBv +MQswCQYDVQQGEwJTRTEUMBIGA1UEChMLQWRkVHJ1c3QgQUIxJjAkBgNVBAsTHUFk +ZFRydXN0IEV4dGVybmFsIFRUUCBOZXR3b3JrMSIwIAYDVQQDExlBZGRUcnVzdCBF +eHRlcm5hbCBDQSBSb290MB4XDTA1MTIwMTAwMDAwMFoXDTE5MDcwOTE4MTkyMlow +gckxCzAJBgNVBAYTAkZSMREwDwYDVQQIEwhDYWx2YWRvczENMAsGA1UEBxMEQ2Fl bjEVMBMGA1UEChMMVEJTIElOVEVSTkVUMUgwRgYDVQQLEz9UZXJtcyBhbmQgQ29u ZGl0aW9uczogaHR0cDovL3d3dy50YnMtaW50ZXJuZXQuY29tL0NBL3JlcG9zaXRv -cnkxGDAWBgNVBAsTD1RCUyBJTlRFUk5FVCBDQTEYMBYGA1UEAxMPVEJTIFg1MDkg -Q0EgU0dDMB4XDTEwMDIxODAwMDAwMFoXDTEyMDIxOTIzNTk1OVowgcsxCzAJBgNV -BAYTAkZSMQ4wDAYDVQQREwUxNDAwMDERMA8GA1UECBMIQ2FsdmFkb3MxDTALBgNV -BAcTBENBRU4xGzAZBgNVBAkTEjIyIHJ1ZSBkZSBCcmV0YWduZTEVMBMGA1UEChMM -VEJTIElOVEVSTkVUMRcwFQYDVQQLEw4wMDAyIDQ0MDQ0MzgxMDEbMBkGA1UECxMS -c2hhLTI1NiBwcm9kdWN0aW9uMSAwHgYDVQQDExdzaGEyNTYudGJzLWludGVybmV0 -LmNvbTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAKbuM8VT7f0nntwu -N3F7v9KIBlhKNAxqCrziOXU5iqUt8HrQB3DtHbdmII+CpVUlwlmepsx6G+srEZ9a -MIGAy0nxi5aLb7watkyIdPjJTMvTUBQ/+RPWzt5JtYbbY9BlJ+yci0dctP74f4NU -ISLtlrEjUbf2gTohLrcE01TfmOF6PDEbB5PKDi38cB3NzKfizWfrOaJW6Q1C1qOJ -y4/4jkUREX1UFUIxzx7v62VfjXSGlcjGpBX1fvtABQOSLeE0a6gciDZs1REqroFf -5eXtqYphpTa14Z83ITXMfgg5Nze1VtMnzI9Qx4blYBw4dgQVEuIsYr7FDBOITDzc -VEVXZx0CAwEAAaOCAj8wggI7MB8GA1UdIwQYMBaAFAdEdoWTKLx/bXjSCuv6TEvf -2YIfMB0GA1UdDgQWBBSJKI/AYVI9RQNY0QPIqc8ej2QivTAOBgNVHQ8BAf8EBAMC -BaAwDAYDVR0TAQH/BAIwADA0BgNVHSUELTArBggrBgEFBQcDAQYIKwYBBQUHAwIG -CisGAQQBgjcKAwMGCWCGSAGG+EIEATBMBgNVHSAERTBDMEEGCysGAQQBgOU3AgQB -MDIwMAYIKwYBBQUHAgEWJGh0dHBzOi8vd3d3LnRicy1pbnRlcm5ldC5jb20vQ0Ev -Q1BTNDBtBgNVHR8EZjBkMDKgMKAuhixodHRwOi8vY3JsLnRicy1pbnRlcm5ldC5j -b20vVEJTWDUwOUNBU0dDLmNybDAuoCygKoYoaHR0cDovL2NybC50YnMteDUwOS5j -b20vVEJTWDUwOUNBU0dDLmNybDCBpgYIKwYBBQUHAQEEgZkwgZYwOAYIKwYBBQUH -MAKGLGh0dHA6Ly9jcnQudGJzLWludGVybmV0LmNvbS9UQlNYNTA5Q0FTR0MuY3J0 -MDQGCCsGAQUFBzAChihodHRwOi8vY3J0LnRicy14NTA5LmNvbS9UQlNYNTA5Q0FT -R0MuY3J0MCQGCCsGAQUFBzABhhhodHRwOi8vb2NzcC50YnMteDUwOS5jb20wPwYD -VR0RBDgwNoIXc2hhMjU2LnRicy1pbnRlcm5ldC5jb22CG3d3dy5zaGEyNTYudGJz -LWludGVybmV0LmNvbTANBgkqhkiG9w0BAQsFAAOCAQEAA5NL0D4QSqhErhlkdPmz -XtiMvdGL+ZehM4coTRIpasM/Agt36Rc0NzCvnQwKE+wkngg1Gy2qe7Q0E/ziqBtB -fZYzdVgu1zdiL4kTaf+wFKYAFGsFbyeEmXysy+CMwaNoF2vpSjCU1UD56bEnTX/W -fxVZYxtBQUpnu2wOsm8cDZuZRv9XrYgAhGj9Tt6F0aVHSDGn59uwShG1+BVF/uju -SCyPTTjL1oc7YElJUzR/x4mQJYvtQI8gDIDAGEOs7v3R/gKa5EMfbUQUI4C84UbI -Yz09Jdnws/MkC/Hm1BZEqk89u7Hvfv+oHqEb0XaUo0TDfsxE0M1sMdnLb91QNQBm -UQ== ------END CERTIFICATE----- - 1 s:/C=FR/ST=Calvados/L=Caen/O=TBS INTERNET/OU=Terms and Conditions: http://www.tbs-internet.com/CA/repository/OU=TBS INTERNET CA/CN=TBS X509 CA SGC - i:/C=SE/O=AddTrust AB/OU=AddTrust External TTP Network/CN=AddTrust External CA Root ------BEGIN CERTIFICATE----- -MIIFVjCCBD6gAwIBAgIQXpDZ0ETJMV02WTx3GTnhhTANBgkqhkiG9w0BAQUFADBv -MQswCQYDVQQGEwJTRTEUMBIGA1UEChMLQWRkVHJ1c3QgQUIxJjAkBgNVBAsTHUFk -ZFRydXN0IEV4dGVybmFsIFRUUCBOZXR3b3JrMSIwIAYDVQQDExlBZGRUcnVzdCBF -eHRlcm5hbCBDQSBSb290MB4XDTA1MTIwMTAwMDAwMFoXDTE5MDYyNDE5MDYzMFow -gcQxCzAJBgNVBAYTAkZSMREwDwYDVQQIEwhDYWx2YWRvczENMAsGA1UEBxMEQ2Fl -bjEVMBMGA1UEChMMVEJTIElOVEVSTkVUMUgwRgYDVQQLEz9UZXJtcyBhbmQgQ29u -ZGl0aW9uczogaHR0cDovL3d3dy50YnMtaW50ZXJuZXQuY29tL0NBL3JlcG9zaXRv -cnkxGDAWBgNVBAsTD1RCUyBJTlRFUk5FVCBDQTEYMBYGA1UEAxMPVEJTIFg1MDkg -Q0EgU0dDMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAsgOkO3f7wzN6 -rOjg45tR5vjBfzK7qmV9IBxb/QW9EEXxG+E7FNhZqQLtwGBKoSsHTnQqV75wWMk0 -9tinWvftBkSpj5sTi/8cbzJfUvTSVYh3Qxv6AVVjMMH/ruLjE6y+4PoaPs8WoYAQ -ts5R4Z1g8c/WnTepLst2x0/Wv7GmuoQi+gXvHU6YrBiu7XkeYhzc95QdviWSJRDk -owhb5K43qhcvjRmBfO/paGlCliDGZp8mHwrI21mwobWpVjTxZRwYO3bd4+TGcI4G -Ie5wmHwE8F7SK1tgSqbBacKjDa93j7txKkfz/Yd2n7TGqOXiHPsJpG655vrKtnXk -9vs1zoDeJQIDAQABo4IBljCCAZIwHQYDVR0OBBYEFAdEdoWTKLx/bXjSCuv6TEvf -2YIfMA4GA1UdDwEB/wQEAwIBBjASBgNVHRMBAf8ECDAGAQH/AgEAMCAGA1UdJQQZ -MBcGCisGAQQBgjcKAwMGCWCGSAGG+EIEATAYBgNVHSAEETAPMA0GCysGAQQBgOU3 -AgQBMHsGA1UdHwR0MHIwOKA2oDSGMmh0dHA6Ly9jcmwuY29tb2RvY2EuY29tL0Fk -ZFRydXN0RXh0ZXJuYWxDQVJvb3QuY3JsMDagNKAyhjBodHRwOi8vY3JsLmNvbW9k -by5uZXQvQWRkVHJ1c3RFeHRlcm5hbENBUm9vdC5jcmwwgYAGCCsGAQUFBwEBBHQw -cjA4BggrBgEFBQcwAoYsaHR0cDovL2NydC5jb21vZG9jYS5jb20vQWRkVHJ1c3RV -VE5TR0NDQS5jcnQwNgYIKwYBBQUHMAKGKmh0dHA6Ly9jcnQuY29tb2RvLm5ldC9B -ZGRUcnVzdFVUTlNHQ0NBLmNydDARBglghkgBhvhCAQEEBAMCAgQwDQYJKoZIhvcN -AQEFBQADggEBAK2zEzs+jcIrVK9oDkdDZNvhuBYTdCfpxfFs+OAujW0bIfJAy232 -euVsnJm6u/+OrqKudD2tad2BbejLLXhMZViaCmK7D9nrXHx4te5EP8rL19SUVqLY -1pTnv5dhNgEgvA7n5lIzDSYs7yRLsr7HJsYPr6SeYSuZizyX1SNz7ooJ32/F3X98 -RB0Mlc/E0OyOrkQ9/y5IrnpnaSora8CnUrV5XNOg+kyCz9edCyx4D5wXYcwZPVWz -8aDqquESrezPyjtfi4WRO4s/VD3HLZvOxzMrWAVYCDG9FxaOhF0QGuuG1F7F3GKV -v6prNyCl016kRl2j1UT+a7gLd8fA25A4C9E= +cnkxGDAWBgNVBAsTD1RCUyBJTlRFUk5FVCBDQTEdMBsGA1UEAxMUVEJTIFg1MDkg +Q0EgYnVzaW5lc3MwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDB1PAU +qudCcz3tmyGcf+u6EkZqonKKHrV4gZYbvVkIRojmmlhfi/jwvpHvo8bqSt/9Rj5S +jhCDW0pcbI+IPPtD1Jy+CHNSfnMqVDy6CKQ3p5maTzCMG6ZT+XjnvcND5v+FtaiB +xk1iCX6uvt0jeUtdZvYbyytsSDE6c3Y5//wRxOF8tM1JxibwO3pyER26jbbN2gQz +m/EkdGjLdJ4svPk23WDAvQ6G0/z2LcAaJB+XLfqRwfQpHQvfKa1uTi8PivC8qtip +rmNQMMPMjxSK2azX8cKjjTDJiUKaCb4VHlJDWKEsCFRpgJAoAuX8f7Yfs1M4esGo +sWb3PGspK3O22uIlAgMBAAGjggF6MIIBdjAdBgNVHQ4EFgQUGgkEzPkJj7seoM7U +pBR/S9UjWM0wDgYDVR0PAQH/BAQDAgEGMBIGA1UdEwEB/wQIMAYBAf8CAQAwGAYD +VR0gBBEwDzANBgsrBgEEAYDlNwIBATB7BgNVHR8EdDByMDigNqA0hjJodHRwOi8v +Y3JsLmNvbW9kb2NhLmNvbS9BZGRUcnVzdEV4dGVybmFsQ0FSb290LmNybDA2oDSg +MoYwaHR0cDovL2NybC5jb21vZG8ubmV0L0FkZFRydXN0RXh0ZXJuYWxDQVJvb3Qu +Y3JsMIGGBggrBgEFBQcBAQR6MHgwOwYIKwYBBQUHMAKGL2h0dHA6Ly9jcnQuY29t +b2RvY2EuY29tL0FkZFRydXN0VVROU2VydmVyQ0EuY3J0MDkGCCsGAQUFBzAChi1o +dHRwOi8vY3J0LmNvbW9kby5uZXQvQWRkVHJ1c3RVVE5TZXJ2ZXJDQS5jcnQwEQYJ +YIZIAYb4QgEBBAQDAgIEMA0GCSqGSIb3DQEBBQUAA4IBAQA7mqrMgk/MrE6QnbNA +h4nRCn2ti4bg4w2C3lB6bSvRPnYwuNw9Jb8vuKkNFzRDxNJXqVDZdfFW5CVQJuyd +nfAx83+wk+spzvFaE1KhFYfN9G9pQfXUfvDRoIcJgPEKUXL1wRiOG+IjU3VVI8pg +IgqHkr7ylln5i5zCiFAPuIJmYUSFg/gxH5xkCNcjJqqrHrHatJr6Qrrke93joupw +oU1njfAcZtYp6fbiK6u2b1pJqwkVBE8RsfLnPhRj+SFbpvjv8Od7o/ieJhFIYQNU +k2jX2u8qZnAiNw93LZW9lpYjtuvMXq8QQppENNja5b53q7UwI+lU7ZGjZ7quuESp +J6/5 -----END CERTIFICATE----- 2 s:/C=SE/O=AddTrust AB/OU=AddTrust External TTP Network/CN=AddTrust External CA Root - i:/C=US/ST=UT/L=Salt Lake City/O=The USERTRUST Network/OU=http://www.usertrust.com/CN=UTN - DATACorp SGC + i:/C=US/ST=UT/L=Salt Lake City/O=The USERTRUST Network/OU=http://www.usertrust.com/CN=UTN-USERFirst-Hardware -----BEGIN CERTIFICATE----- -MIIEZjCCA06gAwIBAgIQUSYKkxzif5zDpV954HKugjANBgkqhkiG9w0BAQUFADCB -kzELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAlVUMRcwFQYDVQQHEw5TYWx0IExha2Ug +MIIETzCCAzegAwIBAgIQHM5EYpUZep1jUvnyI6m2mDANBgkqhkiG9w0BAQUFADCB +lzELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAlVUMRcwFQYDVQQHEw5TYWx0IExha2Ug Q2l0eTEeMBwGA1UEChMVVGhlIFVTRVJUUlVTVCBOZXR3b3JrMSEwHwYDVQQLExho -dHRwOi8vd3d3LnVzZXJ0cnVzdC5jb20xGzAZBgNVBAMTElVUTiAtIERBVEFDb3Jw -IFNHQzAeFw0wNTA2MDcwODA5MTBaFw0xOTA2MjQxOTA2MzBaMG8xCzAJBgNVBAYT -AlNFMRQwEgYDVQQKEwtBZGRUcnVzdCBBQjEmMCQGA1UECxMdQWRkVHJ1c3QgRXh0 -ZXJuYWwgVFRQIE5ldHdvcmsxIjAgBgNVBAMTGUFkZFRydXN0IEV4dGVybmFsIENB -IFJvb3QwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQC39xoz5vIABC05 -4E5b7R+8bA/Ntfojts7emxEzl6QpTH2Tn71KvJPtAxrjj8/lbVBa1pcplFqAsEl6 -2y6V/bjKvzc4LR4+kUGtcFbH8E8/6DKedMrIkFTpxl8PeJ2aQDwOrGGqXhSPnoeh -alDc15pOrwWzpnGUnHGzUGAKxxOdOAeGAqjpqGkmGJCrTLBPI6s6T4TY386f4Wlv -u9dC12tE5Met7m1BX3JacQg3s3llpFmglDf3AC8NwpJy2tA4ctsUqEXEXSp9t7TW -xO6szRNEt8kr3UMAJfphuWlqWCMRt6czj1Z1WfXNKddGtworZbbTQm8Vsrh7++/p -XVPVNFonAgMBAAGjgdgwgdUwHwYDVR0jBBgwFoAUUzLRs89/+uDxoF2FTpLSnkUd -tE8wHQYDVR0OBBYEFK29mHo0tCb3+sQmVO8DveAky1QaMA4GA1UdDwEB/wQEAwIB -BjAPBgNVHRMBAf8EBTADAQH/MBEGCWCGSAGG+EIBAQQEAwIBAjAgBgNVHSUEGTAX -BgorBgEEAYI3CgMDBglghkgBhvhCBAEwPQYDVR0fBDYwNDAyoDCgLoYsaHR0cDov -L2NybC51c2VydHJ1c3QuY29tL1VUTi1EQVRBQ29ycFNHQy5jcmwwDQYJKoZIhvcN -AQEFBQADggEBAMbuUxdoFLJRIh6QWA2U/b3xcOWGLcM2MY9USEbnLQg3vGwKYOEO -rVE04BKT6b64q7gmtOmWPSiPrmQH/uAB7MXjkesYoPF1ftsK5p+R26+udd8jkWjd -FwBaS/9kbHDrARrQkNnHptZt9hPk/7XJ0h4qy7ElQyZ42TCbTg0evmnv3+r+LbPM -+bDdtRTKkdSytaX7ARmjR3mfnYyVhzT4HziS2jamEfpr62vp3EV4FTkG101B5CHI -3C+H0be/SGB1pWLLJN47YaApIKa+xWycxOkKaSLvkTr6Jq/RW0GnOuL4OAdCq8Fb -+M5tug8EPzI0rNwEKNdwMBQmBsTkm5jVz3g= +dHRwOi8vd3d3LnVzZXJ0cnVzdC5jb20xHzAdBgNVBAMTFlVUTi1VU0VSRmlyc3Qt +SGFyZHdhcmUwHhcNMDUwNjA3MDgwOTEwWhcNMTkwNzA5MTgxOTIyWjBvMQswCQYD +VQQGEwJTRTEUMBIGA1UEChMLQWRkVHJ1c3QgQUIxJjAkBgNVBAsTHUFkZFRydXN0 +IEV4dGVybmFsIFRUUCBOZXR3b3JrMSIwIAYDVQQDExlBZGRUcnVzdCBFeHRlcm5h +bCBDQSBSb290MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAt/caM+by +AAQtOeBOW+0fvGwPzbX6I7bO3psRM5ekKUx9k5+9SryT7QMa44/P5W1QWtaXKZRa +gLBJetsulf24yr83OC0ePpFBrXBWx/BPP+gynnTKyJBU6cZfD3idmkA8Dqxhql4U +j56HoWpQ3NeaTq8Fs6ZxlJxxs1BgCscTnTgHhgKo6ahpJhiQq0ywTyOrOk+E2N/O +n+Fpb7vXQtdrROTHre5tQV9yWnEIN7N5ZaRZoJQ39wAvDcKSctrQOHLbFKhFxF0q +fbe01sTurM0TRLfJK91DACX6YblpalgjEbenM49WdVn1zSnXRrcKK2W200JvFbK4 +e/vv6V1T1TRaJwIDAQABo4G9MIG6MB8GA1UdIwQYMBaAFKFyXyYbKJhDlV0HN9WF +lp1L0sNFMB0GA1UdDgQWBBStvZh6NLQm9/rEJlTvA73gJMtUGjAOBgNVHQ8BAf8E +BAMCAQYwDwYDVR0TAQH/BAUwAwEB/zARBglghkgBhvhCAQEEBAMCAQIwRAYDVR0f +BD0wOzA5oDegNYYzaHR0cDovL2NybC51c2VydHJ1c3QuY29tL1VUTi1VU0VSRmly +c3QtSGFyZHdhcmUuY3JsMA0GCSqGSIb3DQEBBQUAA4IBAQByQhANOs4kClrwF8BW +onvUOGCSjRK52zYZgDXYNjDtmr5rJ6NyPFDNn+JxkLpjYetIFMTbSRe679Bt8m7a +gIAoQYFQtxMuyLnJegB2aEbQiIxh/tC21UcFF7ktdnDoTlA6w3pLuvunaI84Of3o +2YBrhzkTbCfaYk5JRlTpudW9DkUkHBsyx3nknPKnplkIGaK0jgn8E0n+SFabYaHk +I9LroYT/+JtLefh9lgBdAgVv0UPbzoGfuDsrk/Zh+UrgbLFpHoVnElhzbkh64Z0X +OGaJunQc68cCZu5HTn/aK7fBGMcVflRCXLVEQpU9PIAdGA8Ynvg684t8GMaKsRl1 +jIGZ -----END CERTIFICATE----- - 3 s:/C=US/ST=UT/L=Salt Lake City/O=The USERTRUST Network/OU=http://www.usertrust.com/CN=UTN - DATACorp SGC - i:/C=US/ST=UT/L=Salt Lake City/O=The USERTRUST Network/OU=http://www.usertrust.com/CN=UTN - DATACorp SGC + 3 s:/C=US/ST=UT/L=Salt Lake City/O=The USERTRUST Network/OU=http://www.usertrust.com/CN=UTN-USERFirst-Hardware + i:/C=US/ST=UT/L=Salt Lake City/O=The USERTRUST Network/OU=http://www.usertrust.com/CN=UTN-USERFirst-Hardware -----BEGIN CERTIFICATE----- -MIIEXjCCA0agAwIBAgIQRL4Mi1AAIbQR0ypoBqmtaTANBgkqhkiG9w0BAQUFADCB -kzELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAlVUMRcwFQYDVQQHEw5TYWx0IExha2Ug +MIIEdDCCA1ygAwIBAgIQRL4Mi1AAJLQR0zYq/mUK/TANBgkqhkiG9w0BAQUFADCB +lzELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAlVUMRcwFQYDVQQHEw5TYWx0IExha2Ug Q2l0eTEeMBwGA1UEChMVVGhlIFVTRVJUUlVTVCBOZXR3b3JrMSEwHwYDVQQLExho -dHRwOi8vd3d3LnVzZXJ0cnVzdC5jb20xGzAZBgNVBAMTElVUTiAtIERBVEFDb3Jw -IFNHQzAeFw05OTA2MjQxODU3MjFaFw0xOTA2MjQxOTA2MzBaMIGTMQswCQYDVQQG -EwJVUzELMAkGA1UECBMCVVQxFzAVBgNVBAcTDlNhbHQgTGFrZSBDaXR5MR4wHAYD -VQQKExVUaGUgVVNFUlRSVVNUIE5ldHdvcmsxITAfBgNVBAsTGGh0dHA6Ly93d3cu -dXNlcnRydXN0LmNvbTEbMBkGA1UEAxMSVVROIC0gREFUQUNvcnAgU0dDMIIBIjAN -BgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA3+5YEKIrblXEjr8uRgnn4AgPLit6 -E5Qbvfa2gI5lBZMAHryv4g+OGQ0SR+ysraP6LnD43m77VkIVni5c7yPeIbkFdicZ -D0/Ww5y0vpQZY/KmEQrrU0icvvIpOxboGqBMpsn0GFlowHDyUwDAXlCCpVZvNvlK -4ESGoE1O1kduSUrLZ9emxAW5jh70/P/N5zbgnAVssjMiFdC04MwXwLLA9P4yPykq -lXvY8qdOD1R8oQ2AswkDwf9c3V6aPryuvEeKaq5xyh+xKrhfQgUL7EYw0XILyulW -bfXv33i+Ybqypa4ETLyorGkVl73v67SMvzX41MPRKA5cOp9wGDMgd8SirwIDAQAB -o4GrMIGoMAsGA1UdDwQEAwIBxjAPBgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBRT -MtGzz3/64PGgXYVOktKeRR20TzA9BgNVHR8ENjA0MDKgMKAuhixodHRwOi8vY3Js -LnVzZXJ0cnVzdC5jb20vVVROLURBVEFDb3JwU0dDLmNybDAqBgNVHSUEIzAhBggr -BgEFBQcDAQYKKwYBBAGCNwoDAwYJYIZIAYb4QgQBMA0GCSqGSIb3DQEBBQUAA4IB -AQAnNZcAiosovcYzMB4p/OL31ZjUQLtgyr+rFywJNn9Q+kHcrpY6CiM+iVnJowft -Gzet/Hy+UUla3joKVAgWRcKZsYfNjGjgaQPpxE6YsjuMFrMOoAyYUJuTqXAJyCyj -j98C5OBxOvG0I3KgqgHf35g+FFCgMSa9KOlaMCZ1+XtgHI3zzVAmbQQnmt/VDUVH -KWss5nbZqSl9Mt3JNjy9rjXxEZ4du5A/EkdOjtd+D2JzHVImOBwYSf0wdJrE5SIv -2MCN7ZF6TACPcn9d2t0bi0Vr591pl6jFVkwPDPafepE39peC4N1xaf92P2BNPM/3 -mfnGV/TJVTl4uix5yaaIK/QI +dHRwOi8vd3d3LnVzZXJ0cnVzdC5jb20xHzAdBgNVBAMTFlVUTi1VU0VSRmlyc3Qt +SGFyZHdhcmUwHhcNOTkwNzA5MTgxMDQyWhcNMTkwNzA5MTgxOTIyWjCBlzELMAkG +A1UEBhMCVVMxCzAJBgNVBAgTAlVUMRcwFQYDVQQHEw5TYWx0IExha2UgQ2l0eTEe +MBwGA1UEChMVVGhlIFVTRVJUUlVTVCBOZXR3b3JrMSEwHwYDVQQLExhodHRwOi8v +d3d3LnVzZXJ0cnVzdC5jb20xHzAdBgNVBAMTFlVUTi1VU0VSRmlyc3QtSGFyZHdh +cmUwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCx98M4P7Sof885glFn +0G2f0v9Y8+efK+wNiVSZuTiZFvfgIXlIwrthdBKWHTxqctU8EGc6Oe0rE81m65UJ +M6Rsl7HoxuzBdXmcRl6Nq9Bq/bkqVRcQVLMZ8Jr28bFdtqdt++BxF2uiiPsA3/4a +MXcMmgF6sTLjKwEHOG7DpV4jvEWbe1DByTCP2+UretNb+zNAHqDVmBe8i4fDidNd +oI6yqqr2jmmIBsX6iSHzCJ1pLgkzmykNRg+MzEk0sGlRvfkGzWitZky8PqxhvQqI +DsjfPe58BEydCl5rkdbux+0ojatNh4lz0G6k0B4WixThdkQDf2Os5M1JnMWS9Ksy +oUhbAgMBAAGjgbkwgbYwCwYDVR0PBAQDAgHGMA8GA1UdEwEB/wQFMAMBAf8wHQYD +VR0OBBYEFKFyXyYbKJhDlV0HN9WFlp1L0sNFMEQGA1UdHwQ9MDswOaA3oDWGM2h0 +dHA6Ly9jcmwudXNlcnRydXN0LmNvbS9VVE4tVVNFUkZpcnN0LUhhcmR3YXJlLmNy +bDAxBgNVHSUEKjAoBggrBgEFBQcDAQYIKwYBBQUHAwUGCCsGAQUFBwMGBggrBgEF +BQcDBzANBgkqhkiG9w0BAQUFAAOCAQEARxkP3nTGmZev/K0oXnWO6y1n7k57K9cM +//bey1WiCuFMVGWTYGufEpytXoMs61quwOQt9ABjHbjAbPLPSbtNk28Gpgoiskli +CE7/yMgUsogWXecB5BKV5UU0s4tpvc+0hY91UZ59Ojg6FEgSxvunOxqNDYJAB+gE +CJChicsZUN/KHAG8HQQZexB2lzvukJDKxA4fFm517zP4029bHpbj4HR3dHuKom4t +3XbWOTCC8KucUvIqx69JXn7HaOWCgchqJ/kniCrVWFCVH/A7HFe7fRQ5YiuayZSS +KqMiDP+JJn1fIytH1xUdqWqeUQ0qUZ6B+dQ7XnASfxAynB67nfhmqA== -----END CERTIFICATE----- diff --git a/lib-python/2.7/test/test_mailbox.py b/lib-python/2.7/test/test_mailbox.py --- a/lib-python/2.7/test/test_mailbox.py +++ b/lib-python/2.7/test/test_mailbox.py @@ -38,14 +38,9 @@ def _delete_recursively(self, target): # Delete a file or delete a directory recursively if os.path.isdir(target): - for path, dirs, files in os.walk(target, topdown=False): - for name in files: - os.remove(os.path.join(path, name)) - for name in dirs: - os.rmdir(os.path.join(path, name)) - os.rmdir(target) + test_support.rmtree(target) elif os.path.exists(target): - os.remove(target) + test_support.unlink(target) class TestMailbox(TestBase): @@ -137,6 +132,7 @@ msg = self._box.get(key1) self.assertEqual(msg['from'], 'foo') self.assertEqual(msg.fp.read(), '1') + msg.fp.close() def test_getitem(self): # Retrieve message using __getitem__() @@ -169,10 +165,14 @@ # Get file representations of messages key0 = self._box.add(self._template % 0) key1 = self._box.add(_sample_message) - self.assertEqual(self._box.get_file(key0).read().replace(os.linesep, '\n'), + msg0 = self._box.get_file(key0) + self.assertEqual(msg0.read().replace(os.linesep, '\n'), self._template % 0) - self.assertEqual(self._box.get_file(key1).read().replace(os.linesep, '\n'), + msg1 = self._box.get_file(key1) + self.assertEqual(msg1.read().replace(os.linesep, '\n'), _sample_message) + msg0.close() + msg1.close() def test_get_file_can_be_closed_twice(self): # Issue 11700 @@ -407,6 +407,7 @@ self._box.add(contents[0]) self._box.add(contents[1]) self._box.add(contents[2]) + oldbox = self._box method() if should_call_close: self._box.close() @@ -415,6 +416,7 @@ self.assertEqual(len(keys), 3) for key in keys: self.assertIn(self._box.get_string(key), contents) + oldbox.close() def test_dump_message(self): # Write message representations to disk @@ -1835,6 +1837,10 @@ def setUp(self): # create a new maildir mailbox to work with: self._dir = test_support.TESTFN + if os.path.isdir(self._dir): + test_support.rmtree(self._dir) + if os.path.isfile(self._dir): + test_support.unlink(self._dir) os.mkdir(self._dir) os.mkdir(os.path.join(self._dir, "cur")) os.mkdir(os.path.join(self._dir, "tmp")) @@ -1844,10 +1850,10 @@ def tearDown(self): map(os.unlink, self._msgfiles) - os.rmdir(os.path.join(self._dir, "cur")) - os.rmdir(os.path.join(self._dir, "tmp")) - os.rmdir(os.path.join(self._dir, "new")) - os.rmdir(self._dir) + test_support.rmdir(os.path.join(self._dir, "cur")) + test_support.rmdir(os.path.join(self._dir, "tmp")) + test_support.rmdir(os.path.join(self._dir, "new")) + test_support.rmdir(self._dir) def createMessage(self, dir, mbox=False): t = int(time.time() % 1000000) @@ -1883,7 +1889,9 @@ self.createMessage("cur") self.mbox = mailbox.Maildir(test_support.TESTFN) #self.assertTrue(len(self.mbox.boxes) == 1) - self.assertIsNot(self.mbox.next(), None) + msg = self.mbox.next() + self.assertIsNot(msg, None) + msg.fp.close() self.assertIs(self.mbox.next(), None) self.assertIs(self.mbox.next(), None) @@ -1891,7 +1899,9 @@ self.createMessage("new") self.mbox = mailbox.Maildir(test_support.TESTFN) #self.assertTrue(len(self.mbox.boxes) == 1) - self.assertIsNot(self.mbox.next(), None) + msg = self.mbox.next() + self.assertIsNot(msg, None) + msg.fp.close() self.assertIs(self.mbox.next(), None) self.assertIs(self.mbox.next(), None) @@ -1900,8 +1910,12 @@ self.createMessage("new") self.mbox = mailbox.Maildir(test_support.TESTFN) #self.assertTrue(len(self.mbox.boxes) == 2) - self.assertIsNot(self.mbox.next(), None) - self.assertIsNot(self.mbox.next(), None) + msg = self.mbox.next() + self.assertIsNot(msg, None) + msg.fp.close() + msg = self.mbox.next() + self.assertIsNot(msg, None) + msg.fp.close() self.assertIs(self.mbox.next(), None) self.assertIs(self.mbox.next(), None) @@ -1910,11 +1924,13 @@ import email.parser fname = self.createMessage("cur", True) n = 0 - for msg in mailbox.PortableUnixMailbox(open(fname), + fid = open(fname) + for msg in mailbox.PortableUnixMailbox(fid, email.parser.Parser().parse): n += 1 self.assertEqual(msg["subject"], "Simple Test") self.assertEqual(len(str(msg)), len(FROM_)+len(DUMMY_MESSAGE)) + fid.close() self.assertEqual(n, 1) ## End: classes from the original module (for backward compatibility). diff --git a/lib-python/2.7/test/test_mmap.py b/lib-python/2.7/test/test_mmap.py --- a/lib-python/2.7/test/test_mmap.py +++ b/lib-python/2.7/test/test_mmap.py @@ -11,7 +11,7 @@ def setUp(self): if os.path.exists(TESTFN): - os.unlink(TESTFN) + unlink(TESTFN) def tearDown(self): try: diff --git a/lib-python/2.7/test/test_multiprocessing.py b/lib-python/2.7/test/test_multiprocessing.py --- a/lib-python/2.7/test/test_multiprocessing.py +++ b/lib-python/2.7/test/test_multiprocessing.py @@ -1,5 +1,10 @@ #!/usr/bin/env python +## FIXME: remove when https://bugs.pypy.org/issue1644 is resolved +import sys +if sys.platform.startswith('freebsd'): + raise Exception("This test hangs on FreeBSD. Test deactivated for now until https://bugs.pypy.org/issue1644 get resolved") + # # Unit tests for the multiprocessing package # diff --git a/lib-python/2.7/test/test_old_mailbox.py b/lib-python/2.7/test/test_old_mailbox.py --- a/lib-python/2.7/test/test_old_mailbox.py +++ b/lib-python/2.7/test/test_old_mailbox.py @@ -73,7 +73,9 @@ self.createMessage("cur") self.mbox = mailbox.Maildir(test_support.TESTFN) self.assertTrue(len(self.mbox) == 1) - self.assertTrue(self.mbox.next() is not None) + msg = self.mbox.next() + self.assertTrue(msg is not None) + msg.fp.close() self.assertTrue(self.mbox.next() is None) self.assertTrue(self.mbox.next() is None) @@ -81,7 +83,9 @@ self.createMessage("new") self.mbox = mailbox.Maildir(test_support.TESTFN) self.assertTrue(len(self.mbox) == 1) - self.assertTrue(self.mbox.next() is not None) + msg = self.mbox.next() + self.assertTrue(msg is not None) + msg.fp.close() self.assertTrue(self.mbox.next() is None) self.assertTrue(self.mbox.next() is None) @@ -90,8 +94,12 @@ self.createMessage("new") self.mbox = mailbox.Maildir(test_support.TESTFN) self.assertTrue(len(self.mbox) == 2) - self.assertTrue(self.mbox.next() is not None) - self.assertTrue(self.mbox.next() is not None) + msg = self.mbox.next() + self.assertTrue(msg is not None) + msg.fp.close() + msg = self.mbox.next() + self.assertTrue(msg is not None) + msg.fp.close() self.assertTrue(self.mbox.next() is None) self.assertTrue(self.mbox.next() is None) diff --git a/lib-python/2.7/test/test_os.py b/lib-python/2.7/test/test_os.py --- a/lib-python/2.7/test/test_os.py +++ b/lib-python/2.7/test/test_os.py @@ -75,7 +75,7 @@ self.assertFalse(os.path.exists(name), "file already exists for temporary file") # make sure we can create the file - open(name, "w") + open(name, "w").close() self.files.append(name) def test_tempnam(self): diff --git a/lib-python/2.7/test/test_ssl.py b/lib-python/2.7/test/test_ssl.py --- a/lib-python/2.7/test/test_ssl.py +++ b/lib-python/2.7/test/test_ssl.py @@ -111,13 +111,12 @@ if test_support.verbose: sys.stdout.write("\n" + pprint.pformat(p) + "\n") self.assertEqual(p['subject'], - ((('countryName', u'US'),), - (('stateOrProvinceName', u'Delaware'),), - (('localityName', u'Wilmington'),), - (('organizationName', u'Python Software Foundation'),), - (('organizationalUnitName', u'SSL'),), - (('commonName', u'somemachine.python.org'),)), + ((('countryName', 'XY'),), + (('localityName', 'Castle Anthrax'),), + (('organizationName', 'Python Software Foundation'),), + (('commonName', 'localhost'),)) ) + self.assertEqual(p['subjectAltName'], (('DNS', 'localhost'),)) # Issue #13034: the subjectAltName in some certificates # (notably projects.developer.nokia.com:443) wasn't parsed p = ssl._ssl._test_decode_cert(NOKIACERT) diff --git a/lib-python/2.7/test/test_support.py b/lib-python/2.7/test/test_support.py --- a/lib-python/2.7/test/test_support.py +++ b/lib-python/2.7/test/test_support.py @@ -179,15 +179,79 @@ except KeyError: pass +if sys.platform.startswith("win"): + def _waitfor(func, pathname, waitall=False): + # Peform the operation + func(pathname) + # Now setup the wait loop + if waitall: + dirname = pathname + else: + dirname, name = os.path.split(pathname) + dirname = dirname or '.' + # Check for `pathname` to be removed from the filesystem. + # The exponential backoff of the timeout amounts to a total + # of ~1 second after which the deletion is probably an error + # anyway. + # Testing on a i7 at 4.3GHz shows that usually only 1 iteration is + # required when contention occurs. + timeout = 0.001 + while timeout < 1.0: + # Note we are only testing for the existance of the file(s) in + # the contents of the directory regardless of any security or + # access rights. If we have made it this far, we have sufficient + # permissions to do that much using Python's equivalent of the + # Windows API FindFirstFile. + # Other Windows APIs can fail or give incorrect results when + # dealing with files that are pending deletion. + L = os.listdir(dirname) + if not (L if waitall else name in L): + return + # Increase the timeout and try again + time.sleep(timeout) + timeout *= 2 + warnings.warn('tests may fail, delete still pending for ' + pathname, + RuntimeWarning, stacklevel=4) + + def _unlink(filename): + _waitfor(os.unlink, filename) + + def _rmdir(dirname): + _waitfor(os.rmdir, dirname) + + def _rmtree(path): + def _rmtree_inner(path): + for name in os.listdir(path): + fullname = os.path.join(path, name) + if os.path.isdir(fullname): + _waitfor(_rmtree_inner, fullname, waitall=True) + os.rmdir(fullname) + else: + os.unlink(fullname) + _waitfor(_rmtree_inner, path, waitall=True) + _waitfor(os.rmdir, path) +else: + _unlink = os.unlink + _rmdir = os.rmdir + _rmtree = shutil.rmtree + def unlink(filename): try: - os.unlink(filename) + _unlink(filename) except OSError: pass +def rmdir(dirname): + try: + _rmdir(dirname) + except OSError as error: + # The directory need not exist. + if error.errno != errno.ENOENT: + raise + def rmtree(path): try: - shutil.rmtree(path) + _rmtree(path) except OSError, e: # Unix returns ENOENT, Windows returns ESRCH. if e.errno not in (errno.ENOENT, errno.ESRCH): diff --git a/lib-python/2.7/test/test_tarfile.py b/lib-python/2.7/test/test_tarfile.py --- a/lib-python/2.7/test/test_tarfile.py +++ b/lib-python/2.7/test/test_tarfile.py @@ -300,26 +300,21 @@ def test_extract_hardlink(self): # Test hardlink extraction (e.g. bug #857297). - tar = tarfile.open(tarname, errorlevel=1, encoding="iso8859-1") + with tarfile.open(tarname, errorlevel=1, encoding="iso8859-1") as tar: + tar.extract("ustar/regtype", TEMPDIR) + self.addCleanup(os.remove, os.path.join(TEMPDIR, "ustar/regtype")) - tar.extract("ustar/regtype", TEMPDIR) - try: tar.extract("ustar/lnktype", TEMPDIR) - except EnvironmentError, e: - if e.errno == errno.ENOENT: - self.fail("hardlink not extracted properly") + self.addCleanup(os.remove, os.path.join(TEMPDIR, "ustar/lnktype")) + with open(os.path.join(TEMPDIR, "ustar/lnktype"), "rb") as f: + data = f.read() + self.assertEqual(md5sum(data), md5_regtype) - data = open(os.path.join(TEMPDIR, "ustar/lnktype"), "rb").read() - self.assertEqual(md5sum(data), md5_regtype) - - try: tar.extract("ustar/symtype", TEMPDIR) - except EnvironmentError, e: - if e.errno == errno.ENOENT: - self.fail("symlink not extracted properly") - - data = open(os.path.join(TEMPDIR, "ustar/symtype"), "rb").read() - self.assertEqual(md5sum(data), md5_regtype) + self.addCleanup(os.remove, os.path.join(TEMPDIR, "ustar/symtype")) + with open(os.path.join(TEMPDIR, "ustar/symtype"), "rb") as f: + data = f.read() + self.assertEqual(md5sum(data), md5_regtype) def test_extractall(self): # Test if extractall() correctly restores directory permissions @@ -340,7 +335,7 @@ # constructor in case of an error. For the test we rely on # the fact that opening an empty file raises a ReadError. empty = os.path.join(TEMPDIR, "empty") - open(empty, "wb").write("") + open(empty, "wb").close() try: tar = object.__new__(tarfile.TarFile) @@ -351,7 +346,7 @@ else: self.fail("ReadError not raised") finally: - os.remove(empty) + test_support.unlink(empty) class StreamReadTest(CommonReadTest): @@ -1327,7 +1322,7 @@ def setUp(self): self.tarname = tmpname if os.path.exists(self.tarname): - os.remove(self.tarname) + test_support.unlink(self.tarname) def _add_testfile(self, fileobj=None): tar = tarfile.open(self.tarname, "a", fileobj=fileobj) diff --git a/lib-python/2.7/traceback.py b/lib-python/2.7/traceback.py --- a/lib-python/2.7/traceback.py +++ b/lib-python/2.7/traceback.py @@ -107,7 +107,7 @@ return list -def print_exception(etype, value, tb, limit=None, file=None): +def print_exception(etype, value, tb, limit=None, file=None, _encoding=None): """Print exception up to 'limit' stack trace entries from 'tb' to 'file'. This differs from print_tb() in the following ways: (1) if @@ -123,7 +123,7 @@ if tb: _print(file, 'Traceback (most recent call last):') print_tb(tb, limit, file) - lines = format_exception_only(etype, value) + lines = format_exception_only(etype, value, _encoding) for line in lines: _print(file, line, '') @@ -144,7 +144,7 @@ list = list + format_exception_only(etype, value) return list -def format_exception_only(etype, value): +def format_exception_only(etype, value, _encoding=None): """Format the exception part of a traceback. The arguments are the exception type and value such as given by @@ -170,12 +170,12 @@ if (isinstance(etype, BaseException) or isinstance(etype, types.InstanceType) or etype is None or type(etype) is str): - return [_format_final_exc_line(etype, value)] + return [_format_final_exc_line(etype, value, _encoding)] stype = etype.__name__ if not issubclass(etype, SyntaxError): - return [_format_final_exc_line(stype, value)] + return [_format_final_exc_line(stype, value, _encoding)] # It was a syntax error; show exactly where the problem was found. lines = [] @@ -196,26 +196,26 @@ lines.append(' %s^\n' % ''.join(caretspace)) value = msg - lines.append(_format_final_exc_line(stype, value)) + lines.append(_format_final_exc_line(stype, value, _encoding)) return lines -def _format_final_exc_line(etype, value): +def _format_final_exc_line(etype, value, _encoding=None): """Return a list of a single line -- normal case for format_exception_only""" - valuestr = _some_str(value) + valuestr = _some_str(value, _encoding) if value is None or not valuestr: line = "%s\n" % etype else: line = "%s: %s\n" % (etype, valuestr) return line -def _some_str(value): +def _some_str(value, _encoding=None): try: return str(value) except Exception: pass try: value = unicode(value) - return value.encode("ascii", "backslashreplace") + return value.encode(_encoding or "ascii", "backslashreplace") except Exception: pass return '' % type(value).__name__ diff --git a/lib-python/2.7/uuid.py b/lib-python/2.7/uuid.py --- a/lib-python/2.7/uuid.py +++ b/lib-python/2.7/uuid.py @@ -127,26 +127,39 @@ overriding the given 'hex', 'bytes', 'bytes_le', 'fields', or 'int'. """ - if [hex, bytes, bytes_le, fields, int].count(None) != 4: - raise TypeError('need one of hex, bytes, bytes_le, fields, or int') if hex is not None: + if (bytes is not None or bytes_le is not None or + fields is not None or int is not None): + raise TypeError('if the hex argument is given, bytes,' + ' bytes_le, fields, and int need to be None') hex = hex.replace('urn:', '').replace('uuid:', '') hex = hex.strip('{}').replace('-', '') if len(hex) != 32: raise ValueError('badly formed hexadecimal UUID string') int = long(hex, 16) - if bytes_le is not None: + elif bytes_le is not None: + if bytes is not None or fields is not None or int is not None: + raise TypeError('if the bytes_le argument is given, bytes,' + ' fields, and int need to be None') if len(bytes_le) != 16: raise ValueError('bytes_le is not a 16-char string') bytes = (bytes_le[3] + bytes_le[2] + bytes_le[1] + bytes_le[0] + bytes_le[5] + bytes_le[4] + bytes_le[7] + bytes_le[6] + bytes_le[8:]) - if bytes is not None: + int = (struct.unpack('>Q', bytes[:8])[0] << 64 | + struct.unpack('>Q', bytes[8:])[0]) + elif bytes is not None: + if fields is not None or int is not None: + raise TypeError('if the bytes argument is given, fields ' + 'and int need to be None') if len(bytes) != 16: raise ValueError('bytes is not a 16-char string') int = (struct.unpack('>Q', bytes[:8])[0] << 64 | struct.unpack('>Q', bytes[8:])[0]) - if fields is not None: + elif fields is not None: + if int is not None: + raise TypeError('if the fields argument is given, int needs' + ' to be None') if len(fields) != 6: raise ValueError('fields is not a 6-tuple') (time_low, time_mid, time_hi_version, @@ -166,9 +179,12 @@ clock_seq = (clock_seq_hi_variant << 8L) | clock_seq_low int = ((time_low << 96L) | (time_mid << 80L) | (time_hi_version << 64L) | (clock_seq << 48L) | node) - if int is not None: + elif int is not None: if not 0 <= int < 1<<128L: raise ValueError('int is out of range (need a 128-bit value)') + else: + raise TypeError('one of hex, bytes, bytes_le, fields,' + ' or int need to be not None') if version is not None: if not 1 <= version <= 5: raise ValueError('illegal version number') @@ -178,7 +194,7 @@ # Set the version number. int &= ~(0xf000 << 64L) int |= version << 76L - self.__dict__['int'] = int + object.__setattr__(self, 'int', int) def __cmp__(self, other): if isinstance(other, UUID): diff --git a/lib-python/2.7/weakref.py b/lib-python/2.7/weakref.py --- a/lib-python/2.7/weakref.py +++ b/lib-python/2.7/weakref.py @@ -48,7 +48,14 @@ def remove(wr, selfref=ref(self)): self = selfref() if self is not None: - del self.data[wr.key] + # Changed this for PyPy: made more resistent. The + # issue is that in some corner cases, self.data + # might already be changed or removed by the time + # this weakref's callback is called. If that is + # the case, we don't want to randomly kill an + # unrelated entry. + if self.data.get(wr.key) is wr: + del self.data[wr.key] self._remove = remove UserDict.UserDict.__init__(self, *args, **kw) @@ -160,22 +167,26 @@ try: o = self.data.pop(key)() except KeyError: + o = None + if o is None: if args: return args[0] - raise - if o is None: raise KeyError, key else: return o + # The logic above was fixed in PyPy def setdefault(self, key, default=None): try: - wr = self.data[key] + o = self.data[key]() except KeyError: + o = None + if o is None: self.data[key] = KeyedRef(default, self._remove, key) return default else: - return wr() + return o + # The logic above was fixed in PyPy def update(self, dict=None, **kwargs): d = self.data diff --git a/lib_pypy/_curses.py b/lib_pypy/_curses.py --- a/lib_pypy/_curses.py +++ b/lib_pypy/_curses.py @@ -1,6 +1,9 @@ """Reimplementation of the standard extension module '_curses' using cffi.""" import sys +if sys.platform == 'win32': + #This module does not exist in windows + raise ImportError('No module named _curses') from functools import wraps from cffi import FFI diff --git a/lib_pypy/_sqlite3.py b/lib_pypy/_sqlite3.py --- a/lib_pypy/_sqlite3.py +++ b/lib_pypy/_sqlite3.py @@ -268,10 +268,18 @@ if _has_load_extension(): _ffi.cdef("int sqlite3_enable_load_extension(sqlite3 *db, int onoff);") -_lib = _ffi.verify(""" -#include -""", libraries=['sqlite3'] -) +if sys.platform.startswith('freebsd'): + _lib = _ffi.verify(""" + #include + """, libraries=['sqlite3'], + include_dirs=['/usr/local/include'], + library_dirs=['/usr/local/lib'] + ) +else: + _lib = _ffi.verify(""" + #include + """, libraries=['sqlite3'] + ) exported_sqlite_symbols = [ 'SQLITE_ALTER_TABLE', @@ -363,9 +371,11 @@ pass -def connect(database, **kwargs): - factory = kwargs.get("factory", Connection) - return factory(database, **kwargs) +def connect(database, timeout=5.0, detect_types=0, isolation_level="", + check_same_thread=True, factory=None, cached_statements=100): + factory = Connection if not factory else factory + return factory(database, timeout, detect_types, isolation_level, + check_same_thread, factory, cached_statements) def _unicode_text_factory(x): diff --git a/lib_pypy/_tkinter/__init__.py b/lib_pypy/_tkinter/__init__.py --- a/lib_pypy/_tkinter/__init__.py +++ b/lib_pypy/_tkinter/__init__.py @@ -22,6 +22,7 @@ READABLE = tklib.TCL_READABLE WRITABLE = tklib.TCL_WRITABLE EXCEPTION = tklib.TCL_EXCEPTION +DONT_WAIT = tklib.TCL_DONT_WAIT def create(screenName=None, baseName=None, className=None, interactive=False, wantobjects=False, wantTk=True, diff --git a/lib_pypy/_tkinter/app.py b/lib_pypy/_tkinter/app.py --- a/lib_pypy/_tkinter/app.py +++ b/lib_pypy/_tkinter/app.py @@ -4,7 +4,23 @@ from . import TclError from .tclobj import TclObject, FromObj, AsObj, TypeCache +import contextlib import sys +import threading +import time + + +class _DummyLock(object): + "A lock-like object that does not do anything" + def acquire(self): + pass + def release(self): + pass + def __enter__(self): + pass + def __exit__(self, *exc): + pass + def varname_converter(input): if isinstance(input, TclObject): @@ -37,17 +53,18 @@ def PythonCmd(clientData, interp, argc, argv): self = tkffi.from_handle(clientData) assert self.app.interp == interp - try: - args = [tkffi.string(arg) for arg in argv[1:argc]] - result = self.func(*args) - obj = AsObj(result) - tklib.Tcl_SetObjResult(interp, obj) - except: - self.app.errorInCmd = True - self.app.exc_info = sys.exc_info() - return tklib.TCL_ERROR - else: - return tklib.TCL_OK + with self.app._tcl_lock_released(): + try: + args = [tkffi.string(arg) for arg in argv[1:argc]] + result = self.func(*args) + obj = AsObj(result) + tklib.Tcl_SetObjResult(interp, obj) + except: + self.app.errorInCmd = True + self.app.exc_info = sys.exc_info() + return tklib.TCL_ERROR + else: + return tklib.TCL_OK @tkffi.callback("Tcl_CmdDeleteProc") def PythonCmdDelete(clientData): @@ -58,6 +75,8 @@ class TkApp(object): + _busywaitinterval = 0.02 # 20ms. + def __new__(cls, screenName, baseName, className, interactive, wantobjects, wantTk, sync, use): if not wantobjects: @@ -73,6 +92,12 @@ self.quitMainLoop = False self.errorInCmd = False + if not self.threaded: + # TCL is not thread-safe, calls needs to be serialized. + self._tcl_lock = threading.Lock() + else: + self._tcl_lock = _DummyLock() + self._typeCache = TypeCache() self._commands = {} @@ -133,6 +158,13 @@ if self.threaded and self.thread_id != tklib.Tcl_GetCurrentThread(): raise RuntimeError("Calling Tcl from different appartment") + @contextlib.contextmanager + def _tcl_lock_released(self): + "Context manager to temporarily release the tcl lock." + self._tcl_lock.release() + yield + self._tcl_lock.acquire() + def loadtk(self): # We want to guard against calling Tk_Init() multiple times err = tklib.Tcl_Eval(self.interp, "info exists tk_version") @@ -159,22 +191,25 @@ flags=tklib.TCL_LEAVE_ERR_MSG if global_only: flags |= tklib.TCL_GLOBAL_ONLY - res = tklib.Tcl_GetVar2Ex(self.interp, name1, name2, flags) - if not res: - self.raiseTclError() - assert self._wantobjects - return FromObj(self, res) + with self._tcl_lock: + res = tklib.Tcl_GetVar2Ex(self.interp, name1, name2, flags) + if not res: + self.raiseTclError() + assert self._wantobjects + return FromObj(self, res) def _setvar(self, name1, value, global_only=False): name1 = varname_converter(name1) + # XXX Acquire tcl lock??? newval = AsObj(value) flags=tklib.TCL_LEAVE_ERR_MSG if global_only: flags |= tklib.TCL_GLOBAL_ONLY - res = tklib.Tcl_SetVar2Ex(self.interp, name1, tkffi.NULL, - newval, flags) - if not res: - self.raiseTclError() + with self._tcl_lock: + res = tklib.Tcl_SetVar2Ex(self.interp, name1, tkffi.NULL, + newval, flags) + if not res: + self.raiseTclError() def _unsetvar(self, name1, name2=None, global_only=False): name1 = varname_converter(name1) @@ -183,9 +218,10 @@ flags=tklib.TCL_LEAVE_ERR_MSG if global_only: flags |= tklib.TCL_GLOBAL_ONLY - res = tklib.Tcl_UnsetVar2(self.interp, name1, name2, flags) - if res == tklib.TCL_ERROR: - self.raiseTclError() + with self._tcl_lock: + res = tklib.Tcl_UnsetVar2(self.interp, name1, name2, flags) + if res == tklib.TCL_ERROR: + self.raiseTclError() def getvar(self, name1, name2=None): return self._var_invoke(self._getvar, name1, name2) @@ -219,9 +255,10 @@ if self.threaded and self.thread_id != tklib.Tcl_GetCurrentThread(): raise NotImplementedError("Call from another thread") - res = tklib.Tcl_CreateCommand( - self.interp, cmdName, _CommandData.PythonCmd, - clientData, _CommandData.PythonCmdDelete) + with self._tcl_lock: + res = tklib.Tcl_CreateCommand( + self.interp, cmdName, _CommandData.PythonCmd, + clientData, _CommandData.PythonCmdDelete) if not res: raise TclError("can't create Tcl command") @@ -229,7 +266,8 @@ if self.threaded and self.thread_id != tklib.Tcl_GetCurrentThread(): raise NotImplementedError("Call from another thread") - res = tklib.Tcl_DeleteCommand(self.interp, cmdName) + with self._tcl_lock: + res = tklib.Tcl_DeleteCommand(self.interp, cmdName) if res == -1: raise TclError("can't delete Tcl command") @@ -256,11 +294,12 @@ tklib.Tcl_IncrRefCount(obj) objects[i] = obj - res = tklib.Tcl_EvalObjv(self.interp, argc, objects, flags) - if res == tklib.TCL_ERROR: - self.raiseTclError() - else: - result = self._callResult() + with self._tcl_lock: + res = tklib.Tcl_EvalObjv(self.interp, argc, objects, flags) + if res == tklib.TCL_ERROR: + self.raiseTclError() + else: + result = self._callResult() finally: for obj in objects: if obj: @@ -280,17 +319,19 @@ def eval(self, script): self._check_tcl_appartment() - res = tklib.Tcl_Eval(self.interp, script) - if res == tklib.TCL_ERROR: - self.raiseTclError() - return tkffi.string(tklib.Tcl_GetStringResult(self.interp)) + with self._tcl_lock: + res = tklib.Tcl_Eval(self.interp, script) + if res == tklib.TCL_ERROR: + self.raiseTclError() + return tkffi.string(tklib.Tcl_GetStringResult(self.interp)) def evalfile(self, filename): self._check_tcl_appartment() - res = tklib.Tcl_EvalFile(self.interp, filename) - if res == tklib.TCL_ERROR: - self.raiseTclError() - return tkffi.string(tklib.Tcl_GetStringResult(self.interp)) + with self._tcl_lock: + res = tklib.Tcl_EvalFile(self.interp, filename) + if res == tklib.TCL_ERROR: + self.raiseTclError() + return tkffi.string(tklib.Tcl_GetStringResult(self.interp)) def split(self, arg): if isinstance(arg, tuple): @@ -375,7 +416,10 @@ if self.threaded: result = tklib.Tcl_DoOneEvent(0) else: - raise NotImplementedError("TCL configured without threads") + with self._tcl_lock: + result = tklib.Tcl_DoOneEvent(tklib.TCL_DONT_WAIT) + if result == 0: + time.sleep(self._busywaitinterval) if result < 0: break diff --git a/lib_pypy/_tkinter/tclobj.py b/lib_pypy/_tkinter/tclobj.py --- a/lib_pypy/_tkinter/tclobj.py +++ b/lib_pypy/_tkinter/tclobj.py @@ -28,9 +28,11 @@ return result elif value.typePtr == typeCache.BooleanType: - return result + return bool(value.internalRep.longValue) elif value.typePtr == typeCache.ByteArrayType: - return result + size = tkffi.new('int*') + data = tklib.Tcl_GetByteArrayFromObj(value, size) + return tkffi.buffer(data, size[0])[:] elif value.typePtr == typeCache.DoubleType: return value.internalRep.doubleValue elif value.typePtr == typeCache.IntType: @@ -50,7 +52,7 @@ result.append(FromObj(app, tcl_elem[0])) return tuple(result) elif value.typePtr == typeCache.ProcBodyType: - return result + pass # fall through and return tcl object. elif value.typePtr == typeCache.StringType: buf = tklib.Tcl_GetUnicode(value) length = tklib.Tcl_GetCharLength(value) diff --git a/lib_pypy/_tkinter/tklib.py b/lib_pypy/_tkinter/tklib.py --- a/lib_pypy/_tkinter/tklib.py +++ b/lib_pypy/_tkinter/tklib.py @@ -1,6 +1,7 @@ # C bindings with libtcl and libtk. from cffi import FFI +import sys tkffi = FFI() @@ -18,6 +19,8 @@ #define TCL_EVAL_DIRECT ... #define TCL_EVAL_GLOBAL ... +#define TCL_DONT_WAIT ... + typedef unsigned short Tcl_UniChar; typedef ... Tcl_Interp; typedef ...* Tcl_ThreadId; @@ -69,6 +72,7 @@ int Tcl_GetBoolean(Tcl_Interp* interp, const char* src, int* boolPtr); char *Tcl_GetString(Tcl_Obj* objPtr); char *Tcl_GetStringFromObj(Tcl_Obj* objPtr, int* lengthPtr); +unsigned char *Tcl_GetByteArrayFromObj(Tcl_Obj* objPtr, int* lengthPtr); Tcl_UniChar *Tcl_GetUnicode(Tcl_Obj* objPtr); int Tcl_GetCharLength(Tcl_Obj* objPtr); @@ -102,6 +106,25 @@ int Tk_GetNumMainWindows(); """) +# XXX find a better way to detect paths +# XXX pick up CPPFLAGS and LDFLAGS and add to these paths? +if sys.platform.startswith("openbsd"): + incdirs = ['/usr/local/include/tcl8.5', '/usr/local/include/tk8.5', '/usr/X11R6/include'] + linklibs = ['tk85', 'tcl85'] + libdirs = ['/usr/local/lib', '/usr/X11R6/lib'] +elif sys.platform.startswith("freebsd"): + incdirs = ['/usr/local/include/tcl8.6', '/usr/local/include/tk8.6', '/usr/local/include/X11', '/usr/local/include'] + linklibs = ['tk86', 'tcl86'] + libdirs = ['/usr/local/lib'] +elif sys.platform == 'win32': + incdirs = [] + linklibs = ['tcl85', 'tk85'] + libdirs = [] +else: + incdirs=['/usr/include/tcl'] + linklibs=['tcl', 'tk'] + libdirs = [] + tklib = tkffi.verify(""" #include #include @@ -109,6 +132,7 @@ char *get_tk_version() { return TK_VERSION; } char *get_tcl_version() { return TCL_VERSION; } """, -include_dirs=['/usr/include/tcl'], -libraries=['tcl', 'tk'], +include_dirs=incdirs, +libraries=linklibs, +library_dirs = libdirs ) diff --git a/lib_pypy/cffi.egg-info b/lib_pypy/cffi.egg-info --- a/lib_pypy/cffi.egg-info +++ b/lib_pypy/cffi.egg-info @@ -1,6 +1,6 @@ Metadata-Version: 1.0 Name: cffi -Version: 0.7 +Version: 0.8 Summary: Foreign Function Interface for Python calling C code. Home-page: http://cffi.readthedocs.org Author: Armin Rigo, Maciej Fijalkowski diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py --- a/lib_pypy/cffi/__init__.py +++ b/lib_pypy/cffi/__init__.py @@ -4,5 +4,5 @@ from .api import FFI, CDefError, FFIError from .ffiplatform import VerificationError, VerificationMissing -__version__ = "0.7" -__version_info__ = (0, 7) +__version__ = "0.8" +__version_info__ = (0, 8) diff --git a/lib_pypy/cffi/api.py b/lib_pypy/cffi/api.py --- a/lib_pypy/cffi/api.py +++ b/lib_pypy/cffi/api.py @@ -1,4 +1,5 @@ import types +from .lock import allocate_lock try: callable @@ -54,12 +55,14 @@ # _cffi_backend.so compiled. import _cffi_backend as backend from . import __version__ - assert backend.__version__ == __version__ + assert (backend.__version__ == __version__ or + backend.__version__ == __version__[:3]) # (If you insist you can also try to pass the option # 'backend=backend_ctypes.CTypesBackend()', but don't # rely on it! It's probably not going to work well.) self._backend = backend + self._lock = allocate_lock() self._parser = cparser.Parser() self._cached_btypes = {} self._parsed_types = types.ModuleType('parsed_types').__dict__ @@ -73,7 +76,8 @@ if name.startswith('RTLD_'): setattr(self, name, getattr(backend, name)) # - self.BVoidP = self._get_cached_btype(model.voidp_type) + with self._lock: + self.BVoidP = self._get_cached_btype(model.voidp_type) if isinstance(backend, types.ModuleType): # _cffi_backend: attach these constants to the class if not hasattr(FFI, 'NULL'): @@ -94,11 +98,12 @@ if not isinstance(csource, basestring): raise TypeError("cdef() argument must be a string") csource = csource.encode('ascii') - self._parser.parse(csource, override=override) - self._cdefsources.append(csource) - if override: - for cache in self._function_caches: - cache.clear() + with self._lock: + self._parser.parse(csource, override=override) + self._cdefsources.append(csource) + if override: + for cache in self._function_caches: + cache.clear() def dlopen(self, name, flags=0): """Load and return a dynamic library identified by 'name'. @@ -108,31 +113,47 @@ library we only look for the actual (untyped) symbols. """ assert isinstance(name, basestring) or name is None - lib, function_cache = _make_ffi_library(self, name, flags) - self._function_caches.append(function_cache) - self._libraries.append(lib) + with self._lock: + lib, function_cache = _make_ffi_library(self, name, flags) + self._function_caches.append(function_cache) + self._libraries.append(lib) return lib + def _typeof_locked(self, cdecl): + # call me with the lock! + key = cdecl + if key in self._parsed_types: + return self._parsed_types[key] + # + if not isinstance(cdecl, str): # unicode, on Python 2 + cdecl = cdecl.encode('ascii') + # + type = self._parser.parse_type(cdecl) + really_a_function_type = type.is_raw_function + if really_a_function_type: + type = type.as_function_pointer() + btype = self._get_cached_btype(type) + result = btype, really_a_function_type + self._parsed_types[key] = result + return result + def _typeof(self, cdecl, consider_function_as_funcptr=False): # string -> ctype object try: - btype, cfaf = self._parsed_types[cdecl] - if consider_function_as_funcptr and not cfaf: - raise KeyError + result = self._parsed_types[cdecl] except KeyError: - key = cdecl - if not isinstance(cdecl, str): # unicode, on Python 2 - cdecl = cdecl.encode('ascii') - cfaf = consider_function_as_funcptr - type = self._parser.parse_type(cdecl, - consider_function_as_funcptr=cfaf) - btype = self._get_cached_btype(type) - self._parsed_types[key] = btype, cfaf + with self._lock: + result = self._typeof_locked(cdecl) + # + btype, really_a_function_type = result + if really_a_function_type and not consider_function_as_funcptr: + raise CDefError("the type %r is a function type, not a " + "pointer-to-function type" % (cdecl,)) return btype def typeof(self, cdecl): """Parse the C type given as a string and return the - corresponding Python type: '>. + corresponding object. It can also be used on 'cdata' instance to get its C type. """ if isinstance(cdecl, basestring): @@ -143,6 +164,10 @@ res = _builtin_function_type(cdecl) if res is not None: return res + if (isinstance(cdecl, types.FunctionType) + and hasattr(cdecl, '_cffi_base_type')): + with self._lock: + return self._get_cached_btype(cdecl._cffi_base_type) raise TypeError(type(cdecl)) def sizeof(self, cdecl): @@ -279,14 +304,17 @@ data. Later, when this new cdata object is garbage-collected, 'destructor(old_cdata_object)' will be called. """ - try: - gc_weakrefs = self.gc_weakrefs - except AttributeError: - from .gc_weakref import GcWeakrefs - gc_weakrefs = self.gc_weakrefs = GcWeakrefs(self) - return gc_weakrefs.build(cdata, destructor) + with self._lock: + try: + gc_weakrefs = self.gc_weakrefs + except AttributeError: + from .gc_weakref import GcWeakrefs + gc_weakrefs = self.gc_weakrefs = GcWeakrefs(self) + return gc_weakrefs.build(cdata, destructor) def _get_cached_btype(self, type): + assert self._lock.acquire(False) is False + # call me with the lock! try: BType = self._cached_btypes[type] except KeyError: @@ -319,9 +347,13 @@ errno = property(_get_errno, _set_errno, None, "the value of 'errno' from/to the C calls") + def getwinerror(self, code=-1): + return self._backend.getwinerror(code) + def _pointer_to(self, ctype): from . import model - return model.pointer_cache(self, ctype) + with self._lock: + return model.pointer_cache(self, ctype) def addressof(self, cdata, field=None): """Return the address of a

%(header)s

%%(header)s

%%(header)s

%%(header)s

%%(header)s

Directory listing for {{ path }}

%(header)s

Directory listing for {{ path }}

%(header)s

Directory listing for {{ path }}